xref: /xsrc/external/mit/xf86-video-intel-old/dist/src/i965_render.c

/*
 * Copyright © 2006,2008 Intel Corporation
 * Copyright © 2007 Red Hat, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 * Authors:
 *    Wang Zhenyu <zhenyu.z.wang@intel.com>
 *    Eric Anholt <eric@anholt.net>
 *    Carl Worth <cworth@redhat.com>
 *    Keith Packard <keithp@keithp.com>
 *
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <assert.h>
#include "xf86.h"
#include "i830.h"
#include "i915_reg.h"

/* bring in brw structs */
#include "brw_defines.h"
#include "brw_structs.h"

/* 24 = 4 vertices/composite * 3 texcoords/vertex * 2 floats/texcoord
 *
 * This is an upper-bound based on the case of a non-affine
 * transformation and with a mask, but useful for sizing all cases for
 * simplicity.
 */
#define VERTEX_FLOATS_PER_COMPOSITE	24
#define VERTEX_BUFFER_SIZE		(256 * VERTEX_FLOATS_PER_COMPOSITE)

struct blendinfo {
    Bool dst_alpha;
    Bool src_alpha;
    uint32_t src_blend;
    uint32_t dst_blend;
};

struct formatinfo {
    int fmt;
    uint32_t card_fmt;
};

// refer vol2, 3d rasterization 3.8.1

/* defined in brw_defines.h */
static struct blendinfo i965_blend_op[] = {
    /* Clear */
    {0, 0, BRW_BLENDFACTOR_ZERO,          BRW_BLENDFACTOR_ZERO},
    /* Src */
    {0, 0, BRW_BLENDFACTOR_ONE,           BRW_BLENDFACTOR_ZERO},
    /* Dst */
    {0, 0, BRW_BLENDFACTOR_ZERO,          BRW_BLENDFACTOR_ONE},
    /* Over */
    {0, 1, BRW_BLENDFACTOR_ONE,           BRW_BLENDFACTOR_INV_SRC_ALPHA},
    /* OverReverse */
    {1, 0, BRW_BLENDFACTOR_INV_DST_ALPHA, BRW_BLENDFACTOR_ONE},
    /* In */
    {1, 0, BRW_BLENDFACTOR_DST_ALPHA,     BRW_BLENDFACTOR_ZERO},
    /* InReverse */
    {0, 1, BRW_BLENDFACTOR_ZERO,          BRW_BLENDFACTOR_SRC_ALPHA},
    /* Out */
    {1, 0, BRW_BLENDFACTOR_INV_DST_ALPHA, BRW_BLENDFACTOR_ZERO},
    /* OutReverse */
    {0, 1, BRW_BLENDFACTOR_ZERO,          BRW_BLENDFACTOR_INV_SRC_ALPHA},
    /* Atop */
    {1, 1, BRW_BLENDFACTOR_DST_ALPHA,     BRW_BLENDFACTOR_INV_SRC_ALPHA},
    /* AtopReverse */
    {1, 1, BRW_BLENDFACTOR_INV_DST_ALPHA, BRW_BLENDFACTOR_SRC_ALPHA},
    /* Xor */
    {1, 1, BRW_BLENDFACTOR_INV_DST_ALPHA, BRW_BLENDFACTOR_INV_SRC_ALPHA},
    /* Add */
    {0, 0, BRW_BLENDFACTOR_ONE,           BRW_BLENDFACTOR_ONE},
};
/**
 * Highest-valued BLENDFACTOR used in i965_blend_op.
 *
 * This leaves out BRW_BLENDFACTOR_INV_DST_COLOR,
 * BRW_BLENDFACTOR_INV_CONST_{COLOR,ALPHA},
 * BRW_BLENDFACTOR_INV_SRC1_{COLOR,ALPHA}
 */
#define BRW_BLENDFACTOR_COUNT (BRW_BLENDFACTOR_INV_DST_ALPHA + 1)

/* FIXME: surface format defined in brw_defines.h, shared Sampling engine
 * 1.7.2
 */
static struct formatinfo i965_tex_formats[] = {
    {PICT_a8r8g8b8, BRW_SURFACEFORMAT_B8G8R8A8_UNORM },
    {PICT_x8r8g8b8, BRW_SURFACEFORMAT_B8G8R8X8_UNORM },
    {PICT_a8b8g8r8, BRW_SURFACEFORMAT_R8G8B8A8_UNORM },
    {PICT_x8b8g8r8, BRW_SURFACEFORMAT_R8G8B8X8_UNORM },
    {PICT_r5g6b5,   BRW_SURFACEFORMAT_B5G6R5_UNORM   },
    {PICT_a1r5g5b5, BRW_SURFACEFORMAT_B5G5R5A1_UNORM },
    {PICT_a8,       BRW_SURFACEFORMAT_A8_UNORM	 },
};

static void i965_get_blend_cntl(int op, PicturePtr pMask, uint32_t dst_format,
				uint32_t *sblend, uint32_t *dblend)
{

    *sblend = i965_blend_op[op].src_blend;
    *dblend = i965_blend_op[op].dst_blend;

    /* If there's no dst alpha channel, adjust the blend op so that we'll treat
     * it as always 1.
     */
    if (PICT_FORMAT_A(dst_format) == 0 && i965_blend_op[op].dst_alpha) {
        if (*sblend == BRW_BLENDFACTOR_DST_ALPHA)
            *sblend = BRW_BLENDFACTOR_ONE;
        else if (*sblend == BRW_BLENDFACTOR_INV_DST_ALPHA)
            *sblend = BRW_BLENDFACTOR_ZERO;
    }

    /* If the source alpha is being used, then we should only be in a case where
     * the source blend factor is 0, and the source blend value is the mask
     * channels multiplied by the source picture's alpha.
     */
    if (pMask && pMask->componentAlpha && PICT_FORMAT_RGB(pMask->format)
            && i965_blend_op[op].src_alpha) {
        if (*dblend == BRW_BLENDFACTOR_SRC_ALPHA) {
	    *dblend = BRW_BLENDFACTOR_SRC_COLOR;
        } else if (*dblend == BRW_BLENDFACTOR_INV_SRC_ALPHA) {
	    *dblend = BRW_BLENDFACTOR_INV_SRC_COLOR;
        }
    }

}

static Bool i965_get_dest_format(PicturePtr pDstPicture, uint32_t *dst_format)
{
    ScrnInfoPtr pScrn = xf86Screens[pDstPicture->pDrawable->pScreen->myNum];

    switch (pDstPicture->format) {
    case PICT_a8r8g8b8:
    case PICT_x8r8g8b8:
        *dst_format = BRW_SURFACEFORMAT_B8G8R8A8_UNORM;
        break;
    case PICT_r5g6b5:
        *dst_format = BRW_SURFACEFORMAT_B5G6R5_UNORM;
        break;
    case PICT_a1r5g5b5:
    	*dst_format = BRW_SURFACEFORMAT_B5G5R5A1_UNORM;
	break;
    case PICT_x1r5g5b5:
        *dst_format = BRW_SURFACEFORMAT_B5G5R5X1_UNORM;
        break;
    case PICT_a8:
        *dst_format = BRW_SURFACEFORMAT_A8_UNORM;
        break;
    case PICT_a4r4g4b4:
    case PICT_x4r4g4b4:
	*dst_format = BRW_SURFACEFORMAT_B4G4R4A4_UNORM;
	break;
    default:
        I830FALLBACK("Unsupported dest format 0x%x\n",
		     (int)pDstPicture->format);
    }

    return TRUE;
}

static Bool i965_check_composite_texture(ScrnInfoPtr pScrn, PicturePtr pPict, int unit)
{
    if (pPict->repeatType > RepeatReflect)
	I830FALLBACK("extended repeat (%d) not supported\n",
		     pPict->repeatType);

    if (pPict->filter != PictFilterNearest &&
        pPict->filter != PictFilterBilinear)
    {
        I830FALLBACK("Unsupported filter 0x%x\n", pPict->filter);
    }

    if (pPict->pDrawable)
    {
	int w, h, i;

	w = pPict->pDrawable->width;
	h = pPict->pDrawable->height;
	if ((w > 8192) || (h > 8192))
	    I830FALLBACK("Picture w/h too large (%dx%d)\n", w, h);

	for (i = 0; i < sizeof(i965_tex_formats) / sizeof(i965_tex_formats[0]);
	     i++)
	{
	    if (i965_tex_formats[i].fmt == pPict->format)
		break;
	}
	if (i == sizeof(i965_tex_formats) / sizeof(i965_tex_formats[0]))
	    I830FALLBACK("Unsupported picture format 0x%x\n",
			 (int)pPict->format);
    }

    return TRUE;
}

Bool
i965_check_composite(int op, PicturePtr pSrcPicture, PicturePtr pMaskPicture,
		     PicturePtr pDstPicture)
{
    ScrnInfoPtr pScrn = xf86Screens[pDstPicture->pDrawable->pScreen->myNum];
    uint32_t tmp1;

    /* Check for unsupported compositing operations. */
    if (op >= sizeof(i965_blend_op) / sizeof(i965_blend_op[0]))
        I830FALLBACK("Unsupported Composite op 0x%x\n", op);

    if (pMaskPicture && pMaskPicture->componentAlpha &&
            PICT_FORMAT_RGB(pMaskPicture->format)) {
        /* Check if it's component alpha that relies on a source alpha and on
         * the source value.  We can only get one of those into the single
         * source value that we get to blend with.
         */
        if (i965_blend_op[op].src_alpha &&
            (i965_blend_op[op].src_blend != BRW_BLENDFACTOR_ZERO))
	{
	    I830FALLBACK("Component alpha not supported with source "
			 "alpha and source value blending.\n");
	}
    }

    if (!i965_check_composite_texture(pScrn, pSrcPicture, 0))
        I830FALLBACK("Check Src picture texture\n");
    if (pMaskPicture != NULL && !i965_check_composite_texture(pScrn, pMaskPicture, 1))
        I830FALLBACK("Check Mask picture texture\n");

    if (!i965_get_dest_format(pDstPicture, &tmp1))
	I830FALLBACK("Get Color buffer format\n");

    return TRUE;

}

#define BRW_GRF_BLOCKS(nreg)    ((nreg + 15) / 16 - 1)

/* Set up a default static partitioning of the URB, which is supposed to
 * allow anything we would want to do, at potentially lower performance.
 */
#define URB_CS_ENTRY_SIZE     0
#define URB_CS_ENTRIES	      0

#define URB_VS_ENTRY_SIZE     1	  // each 512-bit row
#define URB_VS_ENTRIES	      8	  // we needs at least 8 entries

#define URB_GS_ENTRY_SIZE     0
#define URB_GS_ENTRIES	      0

#define URB_CLIP_ENTRY_SIZE   0
#define URB_CLIP_ENTRIES      0

#define URB_SF_ENTRY_SIZE     2
#define URB_SF_ENTRIES	      1

static const uint32_t sip_kernel_static[][4] = {
/*    wait (1) a0<1>UW a145<0,1,0>UW { align1 +  } */
    { 0x00000030, 0x20000108, 0x00001220, 0x00000000 },
/*    nop (4) g0<1>UD { align1 +  } */
    { 0x0040007e, 0x20000c21, 0x00690000, 0x00000000 },
/*    nop (4) g0<1>UD { align1 +  } */
    { 0x0040007e, 0x20000c21, 0x00690000, 0x00000000 },
/*    nop (4) g0<1>UD { align1 +  } */
    { 0x0040007e, 0x20000c21, 0x00690000, 0x00000000 },
/*    nop (4) g0<1>UD { align1 +  } */
    { 0x0040007e, 0x20000c21, 0x00690000, 0x00000000 },
/*    nop (4) g0<1>UD { align1 +  } */
    { 0x0040007e, 0x20000c21, 0x00690000, 0x00000000 },
/*    nop (4) g0<1>UD { align1 +  } */
    { 0x0040007e, 0x20000c21, 0x00690000, 0x00000000 },
/*    nop (4) g0<1>UD { align1 +  } */
    { 0x0040007e, 0x20000c21, 0x00690000, 0x00000000 },
/*    nop (4) g0<1>UD { align1 +  } */
    { 0x0040007e, 0x20000c21, 0x00690000, 0x00000000 },
/*    nop (4) g0<1>UD { align1 +  } */
    { 0x0040007e, 0x20000c21, 0x00690000, 0x00000000 },
};

/*
 * this program computes dA/dx and dA/dy for the texture coordinates along
 * with the base texture coordinate. It was extracted from the Mesa driver
 */

#define SF_KERNEL_NUM_GRF  16
#define SF_MAX_THREADS	   2

static const uint32_t sf_kernel_static[][4] = {
#include "exa_sf.g4b"
};

static const uint32_t sf_kernel_mask_static[][4] = {
#include "exa_sf_mask.g4b"
};

/* ps kernels */
#define PS_KERNEL_NUM_GRF   32
#define PS_MAX_THREADS	    48

static const uint32_t ps_kernel_nomask_affine_static [][4] = {
#include "exa_wm_xy.g4b"
#include "exa_wm_src_affine.g4b"
#include "exa_wm_src_sample_argb.g4b"
#include "exa_wm_write.g4b"
};

static const uint32_t ps_kernel_nomask_projective_static [][4] = {
#include "exa_wm_xy.g4b"
#include "exa_wm_src_projective.g4b"
#include "exa_wm_src_sample_argb.g4b"
#include "exa_wm_write.g4b"
};

static const uint32_t ps_kernel_maskca_affine_static [][4] = {
#include "exa_wm_xy.g4b"
#include "exa_wm_src_affine.g4b"
#include "exa_wm_src_sample_argb.g4b"
#include "exa_wm_mask_affine.g4b"
#include "exa_wm_mask_sample_argb.g4b"
#include "exa_wm_ca.g4b"
#include "exa_wm_write.g4b"
};

static const uint32_t ps_kernel_maskca_projective_static [][4] = {
#include "exa_wm_xy.g4b"
#include "exa_wm_src_projective.g4b"
#include "exa_wm_src_sample_argb.g4b"
#include "exa_wm_mask_projective.g4b"
#include "exa_wm_mask_sample_argb.g4b"
#include "exa_wm_ca.g4b"
#include "exa_wm_write.g4b"
};

static const uint32_t ps_kernel_maskca_srcalpha_affine_static [][4] = {
#include "exa_wm_xy.g4b"
#include "exa_wm_src_affine.g4b"
#include "exa_wm_src_sample_a.g4b"
#include "exa_wm_mask_affine.g4b"
#include "exa_wm_mask_sample_argb.g4b"
#include "exa_wm_ca_srcalpha.g4b"
#include "exa_wm_write.g4b"
};

static const uint32_t ps_kernel_maskca_srcalpha_projective_static [][4] = {
#include "exa_wm_xy.g4b"
#include "exa_wm_src_projective.g4b"
#include "exa_wm_src_sample_a.g4b"
#include "exa_wm_mask_projective.g4b"
#include "exa_wm_mask_sample_argb.g4b"
#include "exa_wm_ca_srcalpha.g4b"
#include "exa_wm_write.g4b"
};

static const uint32_t ps_kernel_masknoca_affine_static [][4] = {
#include "exa_wm_xy.g4b"
#include "exa_wm_src_affine.g4b"
#include "exa_wm_src_sample_argb.g4b"
#include "exa_wm_mask_affine.g4b"
#include "exa_wm_mask_sample_a.g4b"
#include "exa_wm_noca.g4b"
#include "exa_wm_write.g4b"
};

static const uint32_t ps_kernel_masknoca_projective_static [][4] = {
#include "exa_wm_xy.g4b"
#include "exa_wm_src_projective.g4b"
#include "exa_wm_src_sample_argb.g4b"
#include "exa_wm_mask_projective.g4b"
#include "exa_wm_mask_sample_a.g4b"
#include "exa_wm_noca.g4b"
#include "exa_wm_write.g4b"
};

/* new programs for IGDNG */
static const uint32_t sf_kernel_static_gen5[][4] = {
#include "exa_sf.g4b.gen5"
};

static const uint32_t sf_kernel_mask_static_gen5[][4] = {
#include "exa_sf_mask.g4b.gen5"
};

static const uint32_t ps_kernel_nomask_affine_static_gen5 [][4] = {
#include "exa_wm_xy.g4b.gen5"
#include "exa_wm_src_affine.g4b.gen5"
#include "exa_wm_src_sample_argb.g4b.gen5"
#include "exa_wm_write.g4b.gen5"
};

static const uint32_t ps_kernel_nomask_projective_static_gen5 [][4] = {
#include "exa_wm_xy.g4b.gen5"
#include "exa_wm_src_projective.g4b.gen5"
#include "exa_wm_src_sample_argb.g4b.gen5"
#include "exa_wm_write.g4b.gen5"
};

static const uint32_t ps_kernel_maskca_affine_static_gen5 [][4] = {
#include "exa_wm_xy.g4b.gen5"
#include "exa_wm_src_affine.g4b.gen5"
#include "exa_wm_src_sample_argb.g4b.gen5"
#include "exa_wm_mask_affine.g4b.gen5"
#include "exa_wm_mask_sample_argb.g4b.gen5"
#include "exa_wm_ca.g4b.gen5"
#include "exa_wm_write.g4b.gen5"
};

static const uint32_t ps_kernel_maskca_projective_static_gen5 [][4] = {
#include "exa_wm_xy.g4b.gen5"
#include "exa_wm_src_projective.g4b.gen5"
#include "exa_wm_src_sample_argb.g4b.gen5"
#include "exa_wm_mask_projective.g4b.gen5"
#include "exa_wm_mask_sample_argb.g4b.gen5"
#include "exa_wm_ca.g4b.gen5"
#include "exa_wm_write.g4b.gen5"
};

static const uint32_t ps_kernel_maskca_srcalpha_affine_static_gen5 [][4] = {
#include "exa_wm_xy.g4b.gen5"
#include "exa_wm_src_affine.g4b.gen5"
#include "exa_wm_src_sample_a.g4b.gen5"
#include "exa_wm_mask_affine.g4b.gen5"
#include "exa_wm_mask_sample_argb.g4b.gen5"
#include "exa_wm_ca_srcalpha.g4b.gen5"
#include "exa_wm_write.g4b.gen5"
};

static const uint32_t ps_kernel_maskca_srcalpha_projective_static_gen5 [][4] = {
#include "exa_wm_xy.g4b.gen5"
#include "exa_wm_src_projective.g4b.gen5"
#include "exa_wm_src_sample_a.g4b.gen5"
#include "exa_wm_mask_projective.g4b.gen5"
#include "exa_wm_mask_sample_argb.g4b.gen5"
#include "exa_wm_ca_srcalpha.g4b.gen5"
#include "exa_wm_write.g4b.gen5"
};

static const uint32_t ps_kernel_masknoca_affine_static_gen5 [][4] = {
#include "exa_wm_xy.g4b.gen5"
#include "exa_wm_src_affine.g4b.gen5"
#include "exa_wm_src_sample_argb.g4b.gen5"
#include "exa_wm_mask_affine.g4b.gen5"
#include "exa_wm_mask_sample_a.g4b.gen5"
#include "exa_wm_noca.g4b.gen5"
#include "exa_wm_write.g4b.gen5"
};

static const uint32_t ps_kernel_masknoca_projective_static_gen5 [][4] = {
#include "exa_wm_xy.g4b.gen5"
#include "exa_wm_src_projective.g4b.gen5"
#include "exa_wm_src_sample_argb.g4b.gen5"
#include "exa_wm_mask_projective.g4b.gen5"
#include "exa_wm_mask_sample_a.g4b.gen5"
#include "exa_wm_noca.g4b.gen5"
#include "exa_wm_write.g4b.gen5"
};

#define WM_STATE_DECL(kernel) \
    struct brw_wm_unit_state wm_state_ ## kernel[SAMPLER_STATE_FILTER_COUNT] \
						[SAMPLER_STATE_EXTEND_COUNT] \
						[SAMPLER_STATE_FILTER_COUNT] \
						[SAMPLER_STATE_EXTEND_COUNT]

/* Many of the fields in the state structure must be aligned to a
 * 64-byte boundary, (or a 32-byte boundary, but 64 is good enough for
 * those too).
 */
#define PAD64_MULTI(previous, idx, factor) char previous ## _pad ## idx [(64 - (sizeof(struct previous) * (factor)) % 64) % 64]
#define PAD64(previous, idx) PAD64_MULTI(previous, idx, 1)

typedef enum {
    SAMPLER_STATE_FILTER_NEAREST,
    SAMPLER_STATE_FILTER_BILINEAR,
    SAMPLER_STATE_FILTER_COUNT
} sampler_state_filter_t;

typedef enum {
    SAMPLER_STATE_EXTEND_NONE,
    SAMPLER_STATE_EXTEND_REPEAT,
    SAMPLER_STATE_EXTEND_PAD,
    SAMPLER_STATE_EXTEND_REFLECT,
    SAMPLER_STATE_EXTEND_COUNT
} sampler_state_extend_t;

typedef enum {
    WM_KERNEL_NOMASK_AFFINE,
    WM_KERNEL_NOMASK_PROJECTIVE,
    WM_KERNEL_MASKCA_AFFINE,
    WM_KERNEL_MASKCA_PROJECTIVE,
    WM_KERNEL_MASKCA_SRCALPHA_AFFINE,
    WM_KERNEL_MASKCA_SRCALPHA_PROJECTIVE,
    WM_KERNEL_MASKNOCA_AFFINE,
    WM_KERNEL_MASKNOCA_PROJECTIVE,
    WM_KERNEL_COUNT
} wm_kernel_t;

#define KERNEL(kernel_enum, kernel, masked) \
    [kernel_enum] = {__UNCONST(&kernel), sizeof(kernel), masked}
struct wm_kernel_info {
    void *data;
    unsigned int size;
    Bool has_mask;
};

static struct wm_kernel_info wm_kernels[] = {
    KERNEL(WM_KERNEL_NOMASK_AFFINE,
	   ps_kernel_nomask_affine_static, FALSE),
    KERNEL(WM_KERNEL_NOMASK_PROJECTIVE,
	   ps_kernel_nomask_projective_static, FALSE),
    KERNEL(WM_KERNEL_MASKCA_AFFINE,
	   ps_kernel_maskca_affine_static, TRUE),
    KERNEL(WM_KERNEL_MASKCA_PROJECTIVE,
	   ps_kernel_maskca_projective_static, TRUE),
    KERNEL(WM_KERNEL_MASKCA_SRCALPHA_AFFINE,
	   ps_kernel_maskca_srcalpha_affine_static, TRUE),
    KERNEL(WM_KERNEL_MASKCA_SRCALPHA_PROJECTIVE,
	   ps_kernel_maskca_srcalpha_projective_static, TRUE),
    KERNEL(WM_KERNEL_MASKNOCA_AFFINE,
	   ps_kernel_masknoca_affine_static, TRUE),
    KERNEL(WM_KERNEL_MASKNOCA_PROJECTIVE,
	   ps_kernel_masknoca_projective_static, TRUE),
};

static struct wm_kernel_info wm_kernels_gen5[] = {
    KERNEL(WM_KERNEL_NOMASK_AFFINE,
	   ps_kernel_nomask_affine_static_gen5, FALSE),
    KERNEL(WM_KERNEL_NOMASK_PROJECTIVE,
	   ps_kernel_nomask_projective_static_gen5, FALSE),
    KERNEL(WM_KERNEL_MASKCA_AFFINE,
	   ps_kernel_maskca_affine_static_gen5, TRUE),
    KERNEL(WM_KERNEL_MASKCA_PROJECTIVE,
	   ps_kernel_maskca_projective_static_gen5, TRUE),
    KERNEL(WM_KERNEL_MASKCA_SRCALPHA_AFFINE,
	   ps_kernel_maskca_srcalpha_affine_static_gen5, TRUE),
    KERNEL(WM_KERNEL_MASKCA_SRCALPHA_PROJECTIVE,
	   ps_kernel_maskca_srcalpha_projective_static_gen5, TRUE),
    KERNEL(WM_KERNEL_MASKNOCA_AFFINE,
	   ps_kernel_masknoca_affine_static_gen5, TRUE),
    KERNEL(WM_KERNEL_MASKNOCA_PROJECTIVE,
	   ps_kernel_masknoca_projective_static_gen5, TRUE),
};
#undef KERNEL

typedef struct _brw_cc_unit_state_padded {
    struct brw_cc_unit_state state;
    char pad[64 - sizeof (struct brw_cc_unit_state)];
} brw_cc_unit_state_padded;

typedef struct brw_surface_state_padded {
    struct brw_surface_state state;
    char pad[32 - sizeof (struct brw_surface_state)];
} brw_surface_state_padded;

struct gen4_cc_unit_state {
    /* Index by [src_blend][dst_blend] */
    brw_cc_unit_state_padded cc_state[BRW_BLENDFACTOR_COUNT]
				     [BRW_BLENDFACTOR_COUNT];
};

typedef float gen4_vertex_buffer[VERTEX_BUFFER_SIZE];

typedef struct gen4_composite_op {
    int		op;
    PicturePtr	source_picture;
    PicturePtr	mask_picture;
    PicturePtr	dest_picture;
    PixmapPtr	source;
    PixmapPtr	mask;
    PixmapPtr	dest;
    drm_intel_bo *binding_table_bo;
    sampler_state_filter_t src_filter;
    sampler_state_filter_t mask_filter;
    sampler_state_extend_t src_extend;
    sampler_state_extend_t mask_extend;
    Bool is_affine;
    wm_kernel_t wm_kernel;
} gen4_composite_op;

/** Private data for gen4 render accel implementation. */
struct gen4_render_state {
    drm_intel_bo *vs_state_bo;
    drm_intel_bo *sf_state_bo;
    drm_intel_bo *sf_mask_state_bo;
    drm_intel_bo *cc_state_bo;
    drm_intel_bo *wm_state_bo[WM_KERNEL_COUNT]
			     [SAMPLER_STATE_FILTER_COUNT]
			     [SAMPLER_STATE_EXTEND_COUNT]
			     [SAMPLER_STATE_FILTER_COUNT]
			     [SAMPLER_STATE_EXTEND_COUNT];
    drm_intel_bo *wm_kernel_bo[WM_KERNEL_COUNT];

    drm_intel_bo *sip_kernel_bo;
    dri_bo* vertex_buffer_bo;

    gen4_composite_op composite_op;

    int vb_offset;
    int vertex_size;

    Bool needs_state_emit;
};

/**
 * Sets up the SF state pointing at an SF kernel.
 *
 * The SF kernel does coord interp: for each attribute,
 * calculate dA/dx and dA/dy.  Hand these interpolation coefficients
 * back to SF which then hands pixels off to WM.
 */
static drm_intel_bo *
gen4_create_sf_state(ScrnInfoPtr scrn, drm_intel_bo *kernel_bo)
{
    I830Ptr pI830 = I830PTR(scrn);
    struct brw_sf_unit_state *sf_state;
    drm_intel_bo *sf_state_bo;

    sf_state_bo = drm_intel_bo_alloc(pI830->bufmgr, "gen4 SF state",
				     sizeof(*sf_state), 4096);
    drm_intel_bo_map(sf_state_bo, TRUE);
    sf_state = sf_state_bo->virtual;

    memset(sf_state, 0, sizeof(*sf_state));
    sf_state->thread0.grf_reg_count = BRW_GRF_BLOCKS(SF_KERNEL_NUM_GRF);
    sf_state->thread0.kernel_start_pointer =
	intel_emit_reloc(sf_state_bo,
			 offsetof(struct brw_sf_unit_state, thread0),
			 kernel_bo, sf_state->thread0.grf_reg_count << 1,
			 I915_GEM_DOMAIN_INSTRUCTION, 0) >> 6;
    sf_state->sf1.single_program_flow = 1;
    sf_state->sf1.binding_table_entry_count = 0;
    sf_state->sf1.thread_priority = 0;
    sf_state->sf1.floating_point_mode = 0; /* Mesa does this */
    sf_state->sf1.illegal_op_exception_enable = 1;
    sf_state->sf1.mask_stack_exception_enable = 1;
    sf_state->sf1.sw_exception_enable = 1;
    sf_state->thread2.per_thread_scratch_space = 0;
    /* scratch space is not used in our kernel */
    sf_state->thread2.scratch_space_base_pointer = 0;
    sf_state->thread3.const_urb_entry_read_length = 0; /* no const URBs */
    sf_state->thread3.const_urb_entry_read_offset = 0; /* no const URBs */
    sf_state->thread3.urb_entry_read_length = 1; /* 1 URB per vertex */
    /* don't smash vertex header, read start from dw8 */
    sf_state->thread3.urb_entry_read_offset = 1;
    sf_state->thread3.dispatch_grf_start_reg = 3;
    sf_state->thread4.max_threads = SF_MAX_THREADS - 1;
    sf_state->thread4.urb_entry_allocation_size = URB_SF_ENTRY_SIZE - 1;
    sf_state->thread4.nr_urb_entries = URB_SF_ENTRIES;
    sf_state->thread4.stats_enable = 1;
    sf_state->sf5.viewport_transform = FALSE; /* skip viewport */
    sf_state->sf6.cull_mode = BRW_CULLMODE_NONE;
    sf_state->sf6.scissor = 0;
    sf_state->sf7.trifan_pv = 2;
    sf_state->sf6.dest_org_vbias = 0x8;
    sf_state->sf6.dest_org_hbias = 0x8;

    drm_intel_bo_unmap(sf_state_bo);

    return sf_state_bo;
}

static drm_intel_bo *
sampler_border_color_create(ScrnInfoPtr scrn)
{
    struct brw_sampler_legacy_border_color sampler_border_color;

    /* Set up the sampler border color (always transparent black) */
    memset(&sampler_border_color, 0, sizeof(sampler_border_color));
    sampler_border_color.color[0] = 0; /* R */
    sampler_border_color.color[1] = 0; /* G */
    sampler_border_color.color[2] = 0; /* B */
    sampler_border_color.color[3] = 0; /* A */

    return intel_bo_alloc_for_data(scrn,
				   &sampler_border_color,
				   sizeof(sampler_border_color),
				   "gen4 render sampler border color");
}

static void
sampler_state_init (drm_intel_bo *sampler_state_bo,
		    struct brw_sampler_state *sampler_state,
		    sampler_state_filter_t filter,
		    sampler_state_extend_t extend,
		    drm_intel_bo *border_color_bo)
{
    uint32_t sampler_state_offset;

    sampler_state_offset = (char *)sampler_state -
	(char *)sampler_state_bo->virtual;

    /* PS kernel use this sampler */
    memset(sampler_state, 0, sizeof(*sampler_state));

    sampler_state->ss0.lod_preclamp = 1; /* GL mode */

    /* We use the legacy mode to get the semantics specified by
     * the Render extension. */
    sampler_state->ss0.border_color_mode = BRW_BORDER_COLOR_MODE_LEGACY;

    switch(filter) {
    default:
    case SAMPLER_STATE_FILTER_NEAREST:
	sampler_state->ss0.min_filter = BRW_MAPFILTER_NEAREST;
	sampler_state->ss0.mag_filter = BRW_MAPFILTER_NEAREST;
	break;
    case SAMPLER_STATE_FILTER_BILINEAR:
	sampler_state->ss0.min_filter = BRW_MAPFILTER_LINEAR;
	sampler_state->ss0.mag_filter = BRW_MAPFILTER_LINEAR;
	break;
    }

    switch (extend) {
    default:
    case SAMPLER_STATE_EXTEND_NONE:
	sampler_state->ss1.r_wrap_mode = BRW_TEXCOORDMODE_CLAMP_BORDER;
	sampler_state->ss1.s_wrap_mode = BRW_TEXCOORDMODE_CLAMP_BORDER;
	sampler_state->ss1.t_wrap_mode = BRW_TEXCOORDMODE_CLAMP_BORDER;
	break;
    case SAMPLER_STATE_EXTEND_REPEAT:
	sampler_state->ss1.r_wrap_mode = BRW_TEXCOORDMODE_WRAP;
	sampler_state->ss1.s_wrap_mode = BRW_TEXCOORDMODE_WRAP;
	sampler_state->ss1.t_wrap_mode = BRW_TEXCOORDMODE_WRAP;
	break;
    case SAMPLER_STATE_EXTEND_PAD:
	sampler_state->ss1.r_wrap_mode = BRW_TEXCOORDMODE_CLAMP;
	sampler_state->ss1.s_wrap_mode = BRW_TEXCOORDMODE_CLAMP;
	sampler_state->ss1.t_wrap_mode = BRW_TEXCOORDMODE_CLAMP;
	break;
    case SAMPLER_STATE_EXTEND_REFLECT:
	sampler_state->ss1.r_wrap_mode = BRW_TEXCOORDMODE_MIRROR;
	sampler_state->ss1.s_wrap_mode = BRW_TEXCOORDMODE_MIRROR;
	sampler_state->ss1.t_wrap_mode = BRW_TEXCOORDMODE_MIRROR;
	break;
    }

    sampler_state->ss2.border_color_pointer =
	intel_emit_reloc(sampler_state_bo, sampler_state_offset +
			 offsetof(struct brw_sampler_state, ss2),
			 border_color_bo, 0,
			 I915_GEM_DOMAIN_SAMPLER, 0) >> 5;

    sampler_state->ss3.chroma_key_enable = 0; /* disable chromakey */
}

static drm_intel_bo *
gen4_create_sampler_state(ScrnInfoPtr scrn,
			  sampler_state_filter_t src_filter,
			  sampler_state_extend_t src_extend,
			  sampler_state_filter_t mask_filter,
			  sampler_state_extend_t mask_extend,
			  drm_intel_bo *border_color_bo)
{
    I830Ptr pI830 = I830PTR(scrn);
    drm_intel_bo *sampler_state_bo;
    struct brw_sampler_state *sampler_state;

    sampler_state_bo = drm_intel_bo_alloc(pI830->bufmgr, "gen4 sampler state",
					  sizeof(struct brw_sampler_state) * 2,
					  4096);
    drm_intel_bo_map(sampler_state_bo, TRUE);
    sampler_state = sampler_state_bo->virtual;

    sampler_state_init(sampler_state_bo,
		       &sampler_state[0],
		       src_filter, src_extend,
		       border_color_bo);
    sampler_state_init(sampler_state_bo,
		       &sampler_state[1],
		       mask_filter, mask_extend,
		       border_color_bo);

    drm_intel_bo_unmap(sampler_state_bo);

    return sampler_state_bo;
}

static void
cc_state_init (drm_intel_bo *cc_state_bo,
	       uint32_t cc_state_offset,
	       int src_blend,
	       int dst_blend,
	       drm_intel_bo *cc_vp_bo)
{
    struct brw_cc_unit_state *cc_state;

    cc_state = (struct brw_cc_unit_state *)((char *)cc_state_bo->virtual +
					    cc_state_offset);

    memset(cc_state, 0, sizeof(*cc_state));
    cc_state->cc0.stencil_enable = 0;   /* disable stencil */
    cc_state->cc2.depth_test = 0;       /* disable depth test */
    cc_state->cc2.logicop_enable = 0;   /* disable logic op */
    cc_state->cc3.ia_blend_enable = 0;  /* blend alpha same as colors */
    cc_state->cc3.blend_enable = 1;     /* enable color blend */
    cc_state->cc3.alpha_test = 0;       /* disable alpha test */

    cc_state->cc4.cc_viewport_state_offset =
	intel_emit_reloc(cc_state_bo, cc_state_offset +
			 offsetof(struct brw_cc_unit_state, cc4),
			 cc_vp_bo, 0,
			 I915_GEM_DOMAIN_INSTRUCTION, 0) >> 5;

    cc_state->cc5.dither_enable = 0;    /* disable dither */
    cc_state->cc5.logicop_func = 0xc;   /* COPY */
    cc_state->cc5.statistics_enable = 1;
    cc_state->cc5.ia_blend_function = BRW_BLENDFUNCTION_ADD;

    /* Fill in alpha blend factors same as color, for the future. */
    cc_state->cc5.ia_src_blend_factor = src_blend;
    cc_state->cc5.ia_dest_blend_factor = dst_blend;

    cc_state->cc6.blend_function = BRW_BLENDFUNCTION_ADD;
    cc_state->cc6.clamp_post_alpha_blend = 1;
    cc_state->cc6.clamp_pre_alpha_blend = 1;
    cc_state->cc6.clamp_range = 0;  /* clamp range [0,1] */

    cc_state->cc6.src_blend_factor = src_blend;
    cc_state->cc6.dest_blend_factor = dst_blend;
}

static drm_intel_bo *
gen4_create_wm_state(ScrnInfoPtr scrn,
		     Bool has_mask, drm_intel_bo *kernel_bo,
		     drm_intel_bo *sampler_bo)
{
    I830Ptr pI830 = I830PTR(scrn);
    struct brw_wm_unit_state *wm_state;
    drm_intel_bo *wm_state_bo;

    wm_state_bo = drm_intel_bo_alloc(pI830->bufmgr, "gen4 WM state",
				     sizeof(*wm_state), 4096);
    drm_intel_bo_map(wm_state_bo, TRUE);
    wm_state = wm_state_bo->virtual;

    memset(wm_state, 0, sizeof (*wm_state));
    wm_state->thread0.grf_reg_count = BRW_GRF_BLOCKS(PS_KERNEL_NUM_GRF);
    wm_state->thread0.kernel_start_pointer =
	intel_emit_reloc(wm_state_bo,
			 offsetof(struct brw_wm_unit_state, thread0),
                         kernel_bo, wm_state->thread0.grf_reg_count << 1,
                         I915_GEM_DOMAIN_INSTRUCTION, 0) >> 6;

    wm_state->thread1.single_program_flow = 0;

    /* scratch space is not used in our kernel */
    wm_state->thread2.scratch_space_base_pointer = 0;
    wm_state->thread2.per_thread_scratch_space = 0;

    wm_state->thread3.const_urb_entry_read_length = 0;
    wm_state->thread3.const_urb_entry_read_offset = 0;

    wm_state->thread3.urb_entry_read_offset = 0;
    /* wm kernel use urb from 3, see wm_program in compiler module */
    wm_state->thread3.dispatch_grf_start_reg = 3; /* must match kernel */

    wm_state->wm4.stats_enable = 1;  /* statistic */

    if (IS_IGDNG(pI830))
        wm_state->wm4.sampler_count = 0; /* hardware requirement */
    else
        wm_state->wm4.sampler_count = 1; /* 1-4 samplers used */

    wm_state->wm4.sampler_state_pointer =
	intel_emit_reloc(wm_state_bo, offsetof(struct brw_wm_unit_state, wm4),
			 sampler_bo,
			 wm_state->wm4.stats_enable +
			 (wm_state->wm4.sampler_count << 2),
			 I915_GEM_DOMAIN_INSTRUCTION, 0) >> 5;
    wm_state->wm5.max_threads = PS_MAX_THREADS - 1;
    wm_state->wm5.transposed_urb_read = 0;
    wm_state->wm5.thread_dispatch_enable = 1;
    /* just use 16-pixel dispatch (4 subspans), don't need to change kernel
     * start point
     */
    wm_state->wm5.enable_16_pix = 1;
    wm_state->wm5.enable_8_pix = 0;
    wm_state->wm5.early_depth_test = 1;

    /* Each pair of attributes (src/mask coords) is two URB entries */
    if (has_mask) {
	wm_state->thread1.binding_table_entry_count = 3; /* 2 tex and fb */
	wm_state->thread3.urb_entry_read_length = 4;
    } else {
	wm_state->thread1.binding_table_entry_count = 2; /* 1 tex and fb */
	wm_state->thread3.urb_entry_read_length = 2;
    }

    /* binding table entry count is only used for prefetching, and it has to
     * be set 0 for IGDNG
     */
    if (IS_IGDNG(pI830))
        wm_state->thread1.binding_table_entry_count = 0;

    drm_intel_bo_unmap(wm_state_bo);

    return wm_state_bo;
}

static drm_intel_bo *
gen4_create_cc_viewport(ScrnInfoPtr scrn)
{
    I830Ptr pI830 = I830PTR(scrn);
    drm_intel_bo *bo;
    struct brw_cc_viewport cc_viewport;

    cc_viewport.min_depth = -1.e35;
    cc_viewport.max_depth = 1.e35;

    bo = drm_intel_bo_alloc(pI830->bufmgr, "gen4 render unit state",
			    sizeof(cc_viewport), 4096);
    drm_intel_bo_subdata(bo, 0, sizeof(cc_viewport), &cc_viewport);

    return bo;
}

static drm_intel_bo *
gen4_create_vs_unit_state(ScrnInfoPtr scrn)
{
    I830Ptr pI830 = I830PTR(scrn);
    struct brw_vs_unit_state vs_state;
    memset(&vs_state, 0, sizeof(vs_state));

    /* Set up the vertex shader to be disabled (passthrough) */
    if (IS_IGDNG(pI830))
        vs_state.thread4.nr_urb_entries = URB_VS_ENTRIES >> 2; /* hardware requirement */
    else
        vs_state.thread4.nr_urb_entries = URB_VS_ENTRIES;
    vs_state.thread4.urb_entry_allocation_size = URB_VS_ENTRY_SIZE - 1;
    vs_state.vs6.vs_enable = 0;
    vs_state.vs6.vert_cache_disable = 1;

    return intel_bo_alloc_for_data(scrn, &vs_state, sizeof(vs_state),
				   "gen4 render VS state");
}

/**
 * Set up all combinations of cc state: each blendfactor for source and
 * dest.
 */
static drm_intel_bo *
gen4_create_cc_unit_state(ScrnInfoPtr scrn)
{
    I830Ptr pI830 = I830PTR(scrn);
    struct gen4_cc_unit_state *cc_state;
    drm_intel_bo *cc_state_bo, *cc_vp_bo;
    int i, j;

    cc_vp_bo = gen4_create_cc_viewport(scrn);

    cc_state_bo = drm_intel_bo_alloc(pI830->bufmgr, "gen4 CC state",
				     sizeof(*cc_state), 4096);
    drm_intel_bo_map(cc_state_bo, TRUE);
    cc_state = cc_state_bo->virtual;
    for (i = 0; i < BRW_BLENDFACTOR_COUNT; i++) {
	for (j = 0; j < BRW_BLENDFACTOR_COUNT; j++) {
	    cc_state_init(cc_state_bo,
			  offsetof(struct gen4_cc_unit_state,
				   cc_state[i][j].state),
			  i, j, cc_vp_bo);
	}
    }
    drm_intel_bo_unmap(cc_state_bo);

    drm_intel_bo_unreference(cc_vp_bo);

    return cc_state_bo;
}

static uint32_t
i965_get_card_format(PicturePtr pPict)
{
    int i;

    for (i = 0; i < sizeof(i965_tex_formats) / sizeof(i965_tex_formats[0]);
	 i++)
    {
	if (i965_tex_formats[i].fmt == pPict->format)
	    break;
    }
    assert(i != sizeof(i965_tex_formats) / sizeof(i965_tex_formats[0]));

    return i965_tex_formats[i].card_fmt;
}

static sampler_state_filter_t
sampler_state_filter_from_picture (int filter)
{
    switch (filter) {
    case PictFilterNearest:
	return SAMPLER_STATE_FILTER_NEAREST;
    case PictFilterBilinear:
	return SAMPLER_STATE_FILTER_BILINEAR;
    default:
	return -1;
    }
}

static sampler_state_extend_t
sampler_state_extend_from_picture (int repeat_type)
{
    switch (repeat_type) {
    case RepeatNone:
	return SAMPLER_STATE_EXTEND_NONE;
    case RepeatNormal:
	return SAMPLER_STATE_EXTEND_REPEAT;
    case RepeatPad:
	return SAMPLER_STATE_EXTEND_PAD;
    case RepeatReflect:
	return SAMPLER_STATE_EXTEND_REFLECT;
    default:
	return -1;
    }
}

/**
 * Sets up the common fields for a surface state buffer for the given
 * picture in the given surface state buffer.
 */
static void
i965_set_picture_surface_state(dri_bo *ss_bo, int ss_index,
			       PicturePtr pPicture, PixmapPtr pPixmap,
			       Bool is_dst)
{
    struct brw_surface_state_padded *ss;
    struct brw_surface_state local_ss;
    dri_bo *pixmap_bo = i830_get_pixmap_bo(pPixmap);

    ss = (struct brw_surface_state_padded *)ss_bo->virtual + ss_index;

    /* Since ss is a pointer to WC memory, do all of our bit operations
     * into a local temporary first.
     */
    memset(&local_ss, 0, sizeof(local_ss));
    local_ss.ss0.surface_type = BRW_SURFACE_2D;
    if (is_dst) {
	uint32_t dst_format = 0;
	Bool ret = TRUE;

	ret = i965_get_dest_format(pPicture, &dst_format);
	assert(ret == TRUE);
	local_ss.ss0.surface_format = dst_format;
    } else {
	local_ss.ss0.surface_format = i965_get_card_format(pPicture);
    }

    local_ss.ss0.data_return_format = BRW_SURFACERETURNFORMAT_FLOAT32;
    local_ss.ss0.writedisable_alpha = 0;
    local_ss.ss0.writedisable_red = 0;
    local_ss.ss0.writedisable_green = 0;
    local_ss.ss0.writedisable_blue = 0;
    local_ss.ss0.color_blend = 1;
    local_ss.ss0.vert_line_stride = 0;
    local_ss.ss0.vert_line_stride_ofs = 0;
    local_ss.ss0.mipmap_layout_mode = 0;
    local_ss.ss0.render_cache_read_mode = 0;
    if (pixmap_bo != NULL)
	local_ss.ss1.base_addr = pixmap_bo->offset;
    else
	local_ss.ss1.base_addr = intel_get_pixmap_offset(pPixmap);

    local_ss.ss2.mip_count = 0;
    local_ss.ss2.render_target_rotation = 0;
    local_ss.ss2.height = pPixmap->drawable.height - 1;
    local_ss.ss2.width = pPixmap->drawable.width - 1;
    local_ss.ss3.pitch = intel_get_pixmap_pitch(pPixmap) - 1;
    local_ss.ss3.tile_walk = 0; /* Tiled X */
    local_ss.ss3.tiled_surface = i830_pixmap_tiled(pPixmap) ? 1 : 0;

    memcpy(ss, &local_ss, sizeof(local_ss));

    if (pixmap_bo != NULL) {
	uint32_t write_domain, read_domains;

	if (is_dst) {
	    write_domain = I915_GEM_DOMAIN_RENDER;
	    read_domains = I915_GEM_DOMAIN_RENDER;
	} else {
	    write_domain = 0;
	    read_domains = I915_GEM_DOMAIN_SAMPLER;
	}
	dri_bo_emit_reloc(ss_bo, read_domains, write_domain,
			  0,
			  ss_index * sizeof(*ss) +
			  offsetof(struct brw_surface_state, ss1),
			  pixmap_bo);
    }
}

static void
i965_emit_composite_state(ScrnInfoPtr pScrn)
{
    I830Ptr pI830 = I830PTR(pScrn);
    struct gen4_render_state *render_state= pI830->gen4_render_state;
    gen4_composite_op *composite_op = &render_state->composite_op;
    int op = composite_op->op;
    PicturePtr pMaskPicture = composite_op->mask_picture;
    PicturePtr pDstPicture = composite_op->dest_picture;
    PixmapPtr pMask = composite_op->mask;
    PixmapPtr pDst = composite_op->dest;
    sampler_state_filter_t src_filter = composite_op->src_filter;
    sampler_state_filter_t mask_filter = composite_op->mask_filter;
    sampler_state_extend_t src_extend = composite_op->src_extend;
    sampler_state_extend_t mask_extend = composite_op->mask_extend;
    Bool is_affine = composite_op->is_affine;
    int urb_vs_start, urb_vs_size;
    int urb_gs_start, urb_gs_size;
    int urb_clip_start, urb_clip_size;
    int urb_sf_start, urb_sf_size;
    int urb_cs_start, urb_cs_size;
    uint32_t src_blend, dst_blend;
    dri_bo *binding_table_bo = composite_op->binding_table_bo;

    render_state->needs_state_emit = FALSE;

    IntelEmitInvarientState(pScrn);
    pI830->last_3d = LAST_3D_RENDER;

    urb_vs_start = 0;
    urb_vs_size = URB_VS_ENTRIES * URB_VS_ENTRY_SIZE;
    urb_gs_start = urb_vs_start + urb_vs_size;
    urb_gs_size = URB_GS_ENTRIES * URB_GS_ENTRY_SIZE;
    urb_clip_start = urb_gs_start + urb_gs_size;
    urb_clip_size = URB_CLIP_ENTRIES * URB_CLIP_ENTRY_SIZE;
    urb_sf_start = urb_clip_start + urb_clip_size;
    urb_sf_size = URB_SF_ENTRIES * URB_SF_ENTRY_SIZE;
    urb_cs_start = urb_sf_start + urb_sf_size;
    urb_cs_size = URB_CS_ENTRIES * URB_CS_ENTRY_SIZE;

    i965_get_blend_cntl(op, pMaskPicture, pDstPicture->format,
			&src_blend, &dst_blend);

    /* Begin the long sequence of commands needed to set up the 3D
     * rendering pipe
     */
    {
	BEGIN_BATCH(2);
	OUT_BATCH(MI_FLUSH |
		  MI_STATE_INSTRUCTION_CACHE_FLUSH |
		  BRW_MI_GLOBAL_SNAPSHOT_RESET);
	OUT_BATCH(MI_NOOP);
	ADVANCE_BATCH();
    }
    {
        if (IS_IGDNG(pI830))
            BEGIN_BATCH(14);
        else
            BEGIN_BATCH(12);

        /* Match Mesa driver setup */
	if (IS_G4X(pI830) || IS_IGDNG(pI830))
	    OUT_BATCH(NEW_PIPELINE_SELECT | PIPELINE_SELECT_3D);
	else
	    OUT_BATCH(BRW_PIPELINE_SELECT | PIPELINE_SELECT_3D);

	OUT_BATCH(BRW_CS_URB_STATE | 0);
	OUT_BATCH((0 << 4) |  /* URB Entry Allocation Size */
		  (0 << 0));  /* Number of URB Entries */

	/* Zero out the two base address registers so all offsets are
	 * absolute.
	 */
        if (IS_IGDNG(pI830)) {
            OUT_BATCH(BRW_STATE_BASE_ADDRESS | 6);
            OUT_BATCH(0 | BASE_ADDRESS_MODIFY);  /* Generate state base address */
            OUT_BATCH(0 | BASE_ADDRESS_MODIFY);  /* Surface state base address */
            OUT_BATCH(0 | BASE_ADDRESS_MODIFY);  /* media base addr, don't care */
            OUT_BATCH(0 | BASE_ADDRESS_MODIFY);  /* Instruction base address */
            /* general state max addr, disabled */
            OUT_BATCH(0x10000000 | BASE_ADDRESS_MODIFY);
            /* media object state max addr, disabled */
            OUT_BATCH(0x10000000 | BASE_ADDRESS_MODIFY);
            /* Instruction max addr, disabled */
            OUT_BATCH(0x10000000 | BASE_ADDRESS_MODIFY);
        } else {
            OUT_BATCH(BRW_STATE_BASE_ADDRESS | 4);
            OUT_BATCH(0 | BASE_ADDRESS_MODIFY);  /* Generate state base address */
            OUT_BATCH(0 | BASE_ADDRESS_MODIFY);  /* Surface state base address */
            OUT_BATCH(0 | BASE_ADDRESS_MODIFY);  /* media base addr, don't care */
            /* general state max addr, disabled */
            OUT_BATCH(0x10000000 | BASE_ADDRESS_MODIFY);
            /* media object state max addr, disabled */
            OUT_BATCH(0x10000000 | BASE_ADDRESS_MODIFY);
        }
	/* Set system instruction pointer */
	OUT_BATCH(BRW_STATE_SIP | 0);
	OUT_RELOC(render_state->sip_kernel_bo,
		  I915_GEM_DOMAIN_INSTRUCTION, 0, 0);
	OUT_BATCH(MI_NOOP);
	ADVANCE_BATCH();
    }
    {
	int pipe_ctrl;
	BEGIN_BATCH(26);
	/* Pipe control */

	if (IS_IGDNG(pI830))
            pipe_ctrl = BRW_PIPE_CONTROL_NOWRITE;
	else
            pipe_ctrl = BRW_PIPE_CONTROL_NOWRITE | BRW_PIPE_CONTROL_IS_FLUSH;

	OUT_BATCH(BRW_PIPE_CONTROL | pipe_ctrl | 2);
	OUT_BATCH(0);			       /* Destination address */
	OUT_BATCH(0);			       /* Immediate data low DW */
	OUT_BATCH(0);			       /* Immediate data high DW */

	/* Binding table pointers */
	OUT_BATCH(BRW_3DSTATE_BINDING_TABLE_POINTERS | 4);
	OUT_BATCH(0); /* vs */
	OUT_BATCH(0); /* gs */
	OUT_BATCH(0); /* clip */
	OUT_BATCH(0); /* sf */
	/* Only the PS uses the binding table */
	OUT_RELOC(binding_table_bo, I915_GEM_DOMAIN_SAMPLER, 0, 0);

	/* The drawing rectangle clipping is always on.  Set it to values that
	 * shouldn't do any clipping.
	 */
	OUT_BATCH(BRW_3DSTATE_DRAWING_RECTANGLE | 2);
	OUT_BATCH(0x00000000);	/* ymin, xmin */
	OUT_BATCH(DRAW_YMAX(pDst->drawable.height - 1) |
		  DRAW_XMAX(pDst->drawable.width - 1)); /* ymax, xmax */
	OUT_BATCH(0x00000000);	/* yorigin, xorigin */

	/* skip the depth buffer */
	/* skip the polygon stipple */
	/* skip the polygon stipple offset */
	/* skip the line stipple */

	/* Set the pointers to the 3d pipeline state */
	OUT_BATCH(BRW_3DSTATE_PIPELINED_POINTERS | 5);
	OUT_RELOC(render_state->vs_state_bo, I915_GEM_DOMAIN_INSTRUCTION, 0, 0);
	OUT_BATCH(BRW_GS_DISABLE);   /* disable GS, resulting in passthrough */
	OUT_BATCH(BRW_CLIP_DISABLE); /* disable CLIP, resulting in passthrough */
	if (pMask) {
	    OUT_RELOC(render_state->sf_mask_state_bo,
		      I915_GEM_DOMAIN_INSTRUCTION, 0, 0);
	} else {
	    OUT_RELOC(render_state->sf_state_bo,
		      I915_GEM_DOMAIN_INSTRUCTION, 0, 0);
	}

	OUT_RELOC(render_state->wm_state_bo[composite_op->wm_kernel]
		  [src_filter][src_extend]
		  [mask_filter][mask_extend],
		  I915_GEM_DOMAIN_INSTRUCTION, 0, 0);

	OUT_RELOC(render_state->cc_state_bo,
		  I915_GEM_DOMAIN_INSTRUCTION, 0,
		  offsetof(struct gen4_cc_unit_state,
			   cc_state[src_blend][dst_blend]));

	/* URB fence */
	OUT_BATCH(BRW_URB_FENCE |
		  UF0_CS_REALLOC |
		  UF0_SF_REALLOC |
		  UF0_CLIP_REALLOC |
		  UF0_GS_REALLOC |
		  UF0_VS_REALLOC |
		  1);
	OUT_BATCH(((urb_clip_start + urb_clip_size) << UF1_CLIP_FENCE_SHIFT) |
		  ((urb_gs_start + urb_gs_size) << UF1_GS_FENCE_SHIFT) |
		  ((urb_vs_start + urb_vs_size) << UF1_VS_FENCE_SHIFT));
	OUT_BATCH(((urb_cs_start + urb_cs_size) << UF2_CS_FENCE_SHIFT) |
		  ((urb_sf_start + urb_sf_size) << UF2_SF_FENCE_SHIFT));

	/* Constant buffer state */
	OUT_BATCH(BRW_CS_URB_STATE | 0);
	OUT_BATCH(((URB_CS_ENTRY_SIZE - 1) << 4) |
		  (URB_CS_ENTRIES << 0));
	ADVANCE_BATCH();
    }
    {
	/*
	 * number of extra parameters per vertex
	 */
        int nelem = pMask ? 2: 1;
	/*
	 * size of extra parameters:
	 *  3 for homogenous (xyzw)
	 *  2 for cartesian (xy)
	 */
	int selem = is_affine ? 2 : 3;
	uint32_t    w_component;
	uint32_t    src_format;

	render_state->vertex_size = 4 * (2 + nelem * selem);

	if (is_affine)
	{
	    src_format = BRW_SURFACEFORMAT_R32G32_FLOAT;
	    w_component = BRW_VFCOMPONENT_STORE_1_FLT;
	}
	else
	{
	    src_format = BRW_SURFACEFORMAT_R32G32B32_FLOAT;
	    w_component = BRW_VFCOMPONENT_STORE_SRC;
	}

        if (IS_IGDNG(pI830)) {
            BEGIN_BATCH(pMask?9:7);
	    /*
	     * The reason to add this extra vertex element in the header is that
	     * IGDNG has different vertex header definition and origin method to
	     * set destination element offset doesn't exist anymore, which means
	     * hardware requires a predefined vertex element layout.
	     *
	     * haihao proposed this approach to fill the first vertex element, so
	     * origin layout for Gen4 doesn't need to change, and origin shader
	     * programs behavior is also kept.
	     *
	     * I think this is not bad. - zhenyu
	     */

	    OUT_BATCH(BRW_3DSTATE_VERTEX_ELEMENTS | ((2 * (2 + nelem)) - 1));
	    OUT_BATCH((0 << VE0_VERTEX_BUFFER_INDEX_SHIFT) |
                      VE0_VALID |
                      (BRW_SURFACEFORMAT_R32G32_FLOAT << VE0_FORMAT_SHIFT) |
                      (0 << VE0_OFFSET_SHIFT));

	    OUT_BATCH((BRW_VFCOMPONENT_STORE_0 << VE1_VFCOMPONENT_0_SHIFT) |
                      (BRW_VFCOMPONENT_STORE_0 << VE1_VFCOMPONENT_1_SHIFT) |
                      (BRW_VFCOMPONENT_STORE_0 << VE1_VFCOMPONENT_2_SHIFT) |
                      (BRW_VFCOMPONENT_STORE_0 << VE1_VFCOMPONENT_3_SHIFT));
        } else {
            BEGIN_BATCH(pMask?7:5);
            /* Set up our vertex elements, sourced from the single vertex buffer.
             * that will be set up later.
             */
            OUT_BATCH(BRW_3DSTATE_VERTEX_ELEMENTS | ((2 * (1 + nelem)) - 1));
        }

	/* x,y */
	OUT_BATCH((0 << VE0_VERTEX_BUFFER_INDEX_SHIFT) |
		  VE0_VALID |
		  (BRW_SURFACEFORMAT_R32G32_FLOAT << VE0_FORMAT_SHIFT) |
		  (0                            << VE0_OFFSET_SHIFT));

        if (IS_IGDNG(pI830))
            OUT_BATCH((BRW_VFCOMPONENT_STORE_SRC	<< VE1_VFCOMPONENT_0_SHIFT) |
                      (BRW_VFCOMPONENT_STORE_SRC	<< VE1_VFCOMPONENT_1_SHIFT) |
                      (BRW_VFCOMPONENT_STORE_1_FLT	<< VE1_VFCOMPONENT_2_SHIFT) |
                      (BRW_VFCOMPONENT_STORE_1_FLT	<< VE1_VFCOMPONENT_3_SHIFT));
        else
            OUT_BATCH((BRW_VFCOMPONENT_STORE_SRC	<< VE1_VFCOMPONENT_0_SHIFT) |
                      (BRW_VFCOMPONENT_STORE_SRC	<< VE1_VFCOMPONENT_1_SHIFT) |
                      (BRW_VFCOMPONENT_STORE_1_FLT	<< VE1_VFCOMPONENT_2_SHIFT) |
                      (BRW_VFCOMPONENT_STORE_1_FLT	<< VE1_VFCOMPONENT_3_SHIFT) |
                      (4				<< VE1_DESTINATION_ELEMENT_OFFSET_SHIFT));
	/* u0, v0, w0 */
	OUT_BATCH((0				<< VE0_VERTEX_BUFFER_INDEX_SHIFT) |
		  VE0_VALID					     |
		  (src_format			<< VE0_FORMAT_SHIFT) |
		  ((2 * 4)                      << VE0_OFFSET_SHIFT)); /* offset vb in bytes */

        if (IS_IGDNG(pI830))
            OUT_BATCH((BRW_VFCOMPONENT_STORE_SRC	<< VE1_VFCOMPONENT_0_SHIFT) |
                      (BRW_VFCOMPONENT_STORE_SRC	<< VE1_VFCOMPONENT_1_SHIFT) |
                      (w_component			<< VE1_VFCOMPONENT_2_SHIFT) |
                      (BRW_VFCOMPONENT_STORE_1_FLT	<< VE1_VFCOMPONENT_3_SHIFT));
        else
            OUT_BATCH((BRW_VFCOMPONENT_STORE_SRC	<< VE1_VFCOMPONENT_0_SHIFT) |
                      (BRW_VFCOMPONENT_STORE_SRC	<< VE1_VFCOMPONENT_1_SHIFT) |
                      (w_component			<< VE1_VFCOMPONENT_2_SHIFT) |
                      (BRW_VFCOMPONENT_STORE_1_FLT	<< VE1_VFCOMPONENT_3_SHIFT) |
                      ((4 + 4)			<< VE1_DESTINATION_ELEMENT_OFFSET_SHIFT)); /* VUE offset in dwords */
	/* u1, v1, w1 */
   	if (pMask) {
	    OUT_BATCH((0			    << VE0_VERTEX_BUFFER_INDEX_SHIFT) |
		      VE0_VALID							    |
		      (src_format		    << VE0_FORMAT_SHIFT) |
		      (((2 + selem) * 4)            << VE0_OFFSET_SHIFT));  /* vb offset in bytes */

            if (IS_IGDNG(pI830))
                OUT_BATCH((BRW_VFCOMPONENT_STORE_SRC    << VE1_VFCOMPONENT_0_SHIFT) |
                          (BRW_VFCOMPONENT_STORE_SRC    << VE1_VFCOMPONENT_1_SHIFT) |
                          (w_component		    << VE1_VFCOMPONENT_2_SHIFT) |
                          (BRW_VFCOMPONENT_STORE_1_FLT  << VE1_VFCOMPONENT_3_SHIFT));
            else
                OUT_BATCH((BRW_VFCOMPONENT_STORE_SRC    << VE1_VFCOMPONENT_0_SHIFT) |
                          (BRW_VFCOMPONENT_STORE_SRC    << VE1_VFCOMPONENT_1_SHIFT) |
                          (w_component		    << VE1_VFCOMPONENT_2_SHIFT) |
                          (BRW_VFCOMPONENT_STORE_1_FLT  << VE1_VFCOMPONENT_3_SHIFT) |
                          ((4 + 4 + 4)		    << VE1_DESTINATION_ELEMENT_OFFSET_SHIFT)); /* VUE offset in dwords */
   	}

	ADVANCE_BATCH();
    }
}

/**
 * Returns whether the current set of composite state plus vertex buffer is
 * expected to fit in the aperture.
 */
static Bool
i965_composite_check_aperture(ScrnInfoPtr pScrn)
{
    I830Ptr pI830 = I830PTR(pScrn);
    struct gen4_render_state *render_state= pI830->gen4_render_state;
    gen4_composite_op *composite_op = &render_state->composite_op;
    drm_intel_bo *bo_table[] = {
	pI830->batch_bo,
	composite_op->binding_table_bo,
	render_state->vertex_buffer_bo,
	render_state->vs_state_bo,
	render_state->sf_state_bo,
	render_state->sf_mask_state_bo,
	render_state->wm_state_bo[composite_op->wm_kernel]
				 [composite_op->src_filter]
				 [composite_op->src_extend]
				 [composite_op->mask_filter]
				 [composite_op->mask_extend],
	render_state->cc_state_bo,
	render_state->sip_kernel_bo,
    };

    return drm_intel_bufmgr_check_aperture_space(bo_table,
						 ARRAY_SIZE(bo_table)) == 0;
}

Bool
i965_prepare_composite(int op, PicturePtr pSrcPicture,
		       PicturePtr pMaskPicture, PicturePtr pDstPicture,
		       PixmapPtr pSrc, PixmapPtr pMask, PixmapPtr pDst)
{
    ScrnInfoPtr pScrn = xf86Screens[pDstPicture->pDrawable->pScreen->myNum];
    I830Ptr pI830 = I830PTR(pScrn);
    struct gen4_render_state *render_state= pI830->gen4_render_state;
    gen4_composite_op *composite_op = &render_state->composite_op;
    uint32_t *binding_table;
    drm_intel_bo *binding_table_bo, *surface_state_bo;

    if (composite_op->src_filter < 0)
	I830FALLBACK("Bad src filter 0x%x\n", pSrcPicture->filter);
    composite_op->src_extend =
	sampler_state_extend_from_picture(pSrcPicture->repeatType);
    if (composite_op->src_extend < 0)
	I830FALLBACK("Bad src repeat 0x%x\n", pSrcPicture->repeatType);

    if (pMaskPicture) {
	composite_op->mask_filter =
	    sampler_state_filter_from_picture(pMaskPicture->filter);
	if (composite_op->mask_filter < 0)
	    I830FALLBACK("Bad mask filter 0x%x\n", pMaskPicture->filter);
	composite_op->mask_extend =
	    sampler_state_extend_from_picture(pMaskPicture->repeatType);
	if (composite_op->mask_extend < 0)
	    I830FALLBACK("Bad mask repeat 0x%x\n", pMaskPicture->repeatType);
    } else {
	composite_op->mask_filter = SAMPLER_STATE_FILTER_NEAREST;
	composite_op->mask_extend = SAMPLER_STATE_EXTEND_NONE;
    }

    /* Set up the surface states. */
    surface_state_bo = dri_bo_alloc(pI830->bufmgr, "surface_state",
				    3 * sizeof (brw_surface_state_padded),
				    4096);
    if (dri_bo_map(surface_state_bo, 1) != 0) {
	dri_bo_unreference(surface_state_bo);
	return FALSE;
    }
    /* Set up the state buffer for the destination surface */
    i965_set_picture_surface_state(surface_state_bo, 0,
				   pDstPicture, pDst, TRUE);
    /* Set up the source surface state buffer */
    i965_set_picture_surface_state(surface_state_bo, 1,
				   pSrcPicture, pSrc, FALSE);
    if (pMask) {
	/* Set up the mask surface state buffer */
	i965_set_picture_surface_state(surface_state_bo, 2,
				       pMaskPicture, pMask,
				       FALSE);
    }
    dri_bo_unmap(surface_state_bo);

    /* Set up the binding table of surface indices to surface state. */
    binding_table_bo = dri_bo_alloc(pI830->bufmgr, "binding_table",
				    3 * sizeof(uint32_t), 4096);
    if (dri_bo_map (binding_table_bo, 1) != 0) {
	dri_bo_unreference(binding_table_bo);
	dri_bo_unreference(surface_state_bo);
	return FALSE;
    }

    binding_table = binding_table_bo->virtual;
    binding_table[0] = intel_emit_reloc(binding_table_bo,
					0 * sizeof(uint32_t),
					surface_state_bo,
					0 * sizeof(brw_surface_state_padded),
					I915_GEM_DOMAIN_INSTRUCTION, 0);

    binding_table[1] = intel_emit_reloc(binding_table_bo,
					1 * sizeof(uint32_t),
					surface_state_bo,
					1 * sizeof(brw_surface_state_padded),
					I915_GEM_DOMAIN_INSTRUCTION, 0);

    if (pMask) {
	binding_table[2] = intel_emit_reloc(binding_table_bo,
					    2 * sizeof(uint32_t),
					    surface_state_bo,
					    2 * sizeof(brw_surface_state_padded),
					    I915_GEM_DOMAIN_INSTRUCTION, 0);
    } else {
	binding_table[2] = 0;
    }
    dri_bo_unmap(binding_table_bo);
    /* All refs to surface_state are now contained in binding_table_bo. */
    drm_intel_bo_unreference(surface_state_bo);

    composite_op->op = op;
    composite_op->source_picture = pSrcPicture;
    composite_op->mask_picture = pMaskPicture;
    composite_op->dest_picture = pDstPicture;
    composite_op->source = pSrc;
    composite_op->mask = pMask;
    composite_op->dest = pDst;
    drm_intel_bo_unreference(composite_op->binding_table_bo);
    composite_op->binding_table_bo = binding_table_bo;
    composite_op->src_filter =
	sampler_state_filter_from_picture(pSrcPicture->filter);

    pI830->scale_units[0][0] = pSrc->drawable.width;
    pI830->scale_units[0][1] = pSrc->drawable.height;

    pI830->transform[0] = pSrcPicture->transform;
    composite_op->is_affine =
	i830_transform_is_affine(pI830->transform[0]);

    if (!pMask) {
	pI830->transform[1] = NULL;
	pI830->scale_units[1][0] = -1;
	pI830->scale_units[1][1] = -1;
    } else {
	pI830->transform[1] = pMaskPicture->transform;
	pI830->scale_units[1][0] = pMask->drawable.width;
	pI830->scale_units[1][1] = pMask->drawable.height;
	composite_op->is_affine |=
	    i830_transform_is_affine(pI830->transform[1]);
    }


    if (pMask) {
	if (pMaskPicture->componentAlpha &&
	    PICT_FORMAT_RGB(pMaskPicture->format))
	{
	    if (i965_blend_op[op].src_alpha) {
		if (composite_op->is_affine)
		    composite_op->wm_kernel = WM_KERNEL_MASKCA_SRCALPHA_AFFINE;
		else
		    composite_op->wm_kernel = WM_KERNEL_MASKCA_SRCALPHA_PROJECTIVE;
	    } else {
		if (composite_op->is_affine)
		    composite_op->wm_kernel = WM_KERNEL_MASKCA_AFFINE;
		else
		    composite_op->wm_kernel = WM_KERNEL_MASKCA_PROJECTIVE;
	    }
	} else {
	    if (composite_op->is_affine)
		composite_op->wm_kernel = WM_KERNEL_MASKNOCA_AFFINE;
	    else
		composite_op->wm_kernel = WM_KERNEL_MASKNOCA_PROJECTIVE;
	}
    } else {
	if (composite_op->is_affine)
	    composite_op->wm_kernel = WM_KERNEL_NOMASK_AFFINE;
	else
	    composite_op->wm_kernel = WM_KERNEL_NOMASK_PROJECTIVE;
    }

    if (!i965_composite_check_aperture(pScrn)) {
	intel_batch_flush(pScrn, FALSE);
	if (!i965_composite_check_aperture(pScrn))
	    I830FALLBACK("Couldn't fit render operation in aperture\n");
    }

    render_state->needs_state_emit = TRUE;

    return TRUE;
}

static drm_intel_bo *
i965_get_vb_space(ScrnInfoPtr pScrn)
{
    I830Ptr pI830 = I830PTR(pScrn);
    struct gen4_render_state *render_state = pI830->gen4_render_state;

    /* If the vertex buffer is too full, then we free the old and a new one
     * gets made.
     */
    if (render_state->vb_offset + VERTEX_FLOATS_PER_COMPOSITE >
	VERTEX_BUFFER_SIZE) {
	drm_intel_bo_unreference(render_state->vertex_buffer_bo);
	render_state->vertex_buffer_bo = NULL;
    }

    /* Alloc a new vertex buffer if necessary. */
    if (render_state->vertex_buffer_bo == NULL) {
	render_state->vertex_buffer_bo = drm_intel_bo_alloc(pI830->bufmgr, "vb",
							    sizeof(gen4_vertex_buffer),
							    4096);
	render_state->vb_offset = 0;
    }

    drm_intel_bo_reference(render_state->vertex_buffer_bo);
    return render_state->vertex_buffer_bo;
}

void
i965_composite(PixmapPtr pDst, int srcX, int srcY, int maskX, int maskY,
	       int dstX, int dstY, int w, int h)
{
    ScrnInfoPtr pScrn = xf86Screens[pDst->drawable.pScreen->myNum];
    I830Ptr pI830 = I830PTR(pScrn);
    struct gen4_render_state *render_state = pI830->gen4_render_state;
    Bool has_mask;
    float src_x[3], src_y[3], src_w[3], mask_x[3], mask_y[3], mask_w[3];
    int i;
    drm_intel_bo *vb_bo;
    float vb[18];
    Bool is_affine = render_state->composite_op.is_affine;

    if (is_affine)
    {
	if (!i830_get_transformed_coordinates(srcX, srcY,
					      pI830->transform[0],
					      &src_x[0], &src_y[0]))
	    return;
	if (!i830_get_transformed_coordinates(srcX, srcY + h,
					      pI830->transform[0],
					      &src_x[1], &src_y[1]))
	    return;
	if (!i830_get_transformed_coordinates(srcX + w, srcY + h,
					      pI830->transform[0],
					      &src_x[2], &src_y[2]))
	    return;
    }
    else
    {
	if (!i830_get_transformed_coordinates_3d(srcX, srcY,
						 pI830->transform[0],
						 &src_x[0], &src_y[0],
						 &src_w[0]))
	    return;
	if (!i830_get_transformed_coordinates_3d(srcX, srcY + h,
						 pI830->transform[0],
						 &src_x[1], &src_y[1],
						 &src_w[1]))
	    return;
	if (!i830_get_transformed_coordinates_3d(srcX + w, srcY + h,
						 pI830->transform[0],
						 &src_x[2], &src_y[2],
						 &src_w[2]))
	    return;
    }

    if (pI830->scale_units[1][0] == -1 || pI830->scale_units[1][1] == -1) {
	has_mask = FALSE;
    } else {
	has_mask = TRUE;
	if (is_affine) {
	    if (!i830_get_transformed_coordinates(maskX, maskY,
						  pI830->transform[1],
						  &mask_x[0], &mask_y[0]))
		return;
	    if (!i830_get_transformed_coordinates(maskX, maskY + h,
						  pI830->transform[1],
						  &mask_x[1], &mask_y[1]))
		return;
	    if (!i830_get_transformed_coordinates(maskX + w, maskY + h,
						  pI830->transform[1],
						  &mask_x[2], &mask_y[2]))
		return;
	} else {
	    if (!i830_get_transformed_coordinates_3d(maskX, maskY,
						     pI830->transform[1],
						     &mask_x[0], &mask_y[0],
						     &mask_w[0]))
		return;
	    if (!i830_get_transformed_coordinates_3d(maskX, maskY + h,
						     pI830->transform[1],
						     &mask_x[1], &mask_y[1],
						     &mask_w[1]))
		return;
	    if (!i830_get_transformed_coordinates_3d(maskX + w, maskY + h,
						     pI830->transform[1],
						     &mask_x[2], &mask_y[2],
						     &mask_w[2]))
		return;
	}
    }

    vb_bo = i965_get_vb_space(pScrn);
    if (vb_bo == NULL)
	return;
    i = 0;
    /* rect (x2,y2) */
    vb[i++] = (float)(dstX + w);
    vb[i++] = (float)(dstY + h);
    vb[i++] = src_x[2] / pI830->scale_units[0][0];
    vb[i++] = src_y[2] / pI830->scale_units[0][1];
    if (!is_affine)
	vb[i++] = src_w[2];
    if (has_mask) {
        vb[i++] = mask_x[2] / pI830->scale_units[1][0];
        vb[i++] = mask_y[2] / pI830->scale_units[1][1];
	if (!is_affine)
	    vb[i++] = mask_w[2];
    }

    /* rect (x1,y2) */
    vb[i++] = (float)dstX;
    vb[i++] = (float)(dstY + h);
    vb[i++] = src_x[1] / pI830->scale_units[0][0];
    vb[i++] = src_y[1] / pI830->scale_units[0][1];
    if (!is_affine)
	vb[i++] = src_w[1];
    if (has_mask) {
        vb[i++] = mask_x[1] / pI830->scale_units[1][0];
        vb[i++] = mask_y[1] / pI830->scale_units[1][1];
	if (!is_affine)
	    vb[i++] = mask_w[1];
    }

    /* rect (x1,y1) */
    vb[i++] = (float)dstX;
    vb[i++] = (float)dstY;
    vb[i++] = src_x[0] / pI830->scale_units[0][0];
    vb[i++] = src_y[0] / pI830->scale_units[0][1];
    if (!is_affine)
	vb[i++] = src_w[0];
    if (has_mask) {
        vb[i++] = mask_x[0] / pI830->scale_units[1][0];
        vb[i++] = mask_y[0] / pI830->scale_units[1][1];
	if (!is_affine)
	    vb[i++] = mask_w[0];
    }
    assert (i <= VERTEX_BUFFER_SIZE);
    drm_intel_bo_subdata(vb_bo, render_state->vb_offset * 4, i * 4, vb);

    if (!i965_composite_check_aperture(pScrn))
	intel_batch_flush(pScrn, FALSE);

    intel_batch_start_atomic(pScrn, 200);
    if (render_state->needs_state_emit)
	i965_emit_composite_state(pScrn);

    BEGIN_BATCH(12);
    OUT_BATCH(MI_FLUSH);
    /* Set up the pointer to our (single) vertex buffer */
    OUT_BATCH(BRW_3DSTATE_VERTEX_BUFFERS | 3);
    OUT_BATCH((0 << VB0_BUFFER_INDEX_SHIFT) |
	      VB0_VERTEXDATA |
	      (render_state->vertex_size << VB0_BUFFER_PITCH_SHIFT));
    OUT_RELOC(vb_bo, I915_GEM_DOMAIN_VERTEX, 0, render_state->vb_offset * 4);

    if (IS_IGDNG(pI830))
        OUT_RELOC(vb_bo, I915_GEM_DOMAIN_VERTEX, 0, render_state->vb_offset * 4 + i * 4);
    else
        OUT_BATCH(3);

    OUT_BATCH(0); // ignore for VERTEXDATA, but still there

    OUT_BATCH(BRW_3DPRIMITIVE |
	      BRW_3DPRIMITIVE_VERTEX_SEQUENTIAL |
	      (_3DPRIM_RECTLIST << BRW_3DPRIMITIVE_TOPOLOGY_SHIFT) |
	      (0 << 9) |  /* CTG - indirect vertex count */
	      4);
    OUT_BATCH(3);  /* vertex count per instance */
    OUT_BATCH(0); /* start vertex offset */
    OUT_BATCH(1); /* single instance */
    OUT_BATCH(0); /* start instance location */
    OUT_BATCH(0); /* index buffer offset, ignored */
    ADVANCE_BATCH();

    render_state->vb_offset += i;
    drm_intel_bo_unreference(vb_bo);

    intel_batch_end_atomic(pScrn);

    i830_debug_sync(pScrn);
}

void
i965_batch_flush_notify(ScrnInfoPtr pScrn)
{
    I830Ptr pI830 = I830PTR(pScrn);
    struct gen4_render_state *render_state = pI830->gen4_render_state;

    /* Once a batch is emitted, we never want to map again any buffer
     * object being referenced by that batch, (which would be very
     * expensive). */
    if (render_state->vertex_buffer_bo) {
	dri_bo_unreference (render_state->vertex_buffer_bo);
	render_state->vertex_buffer_bo = NULL;
    }

    render_state->needs_state_emit = TRUE;
}

/**
 * Called at EnterVT so we can set up our offsets into the state buffer.
 */
void
gen4_render_state_init(ScrnInfoPtr pScrn)
{
    I830Ptr pI830 = I830PTR(pScrn);
    struct gen4_render_state *render_state;
    int i, j, k, l, m;
    drm_intel_bo *sf_kernel_bo, *sf_kernel_mask_bo;
    drm_intel_bo *border_color_bo;

    if (pI830->gen4_render_state == NULL)
	pI830->gen4_render_state = calloc(sizeof(*render_state), 1);

    render_state = pI830->gen4_render_state;
    render_state->vb_offset = 0;

    render_state->vs_state_bo = gen4_create_vs_unit_state(pScrn);

    /* Set up the two SF states (one for blending with a mask, one without) */
    if (IS_IGDNG(pI830)) {
	sf_kernel_bo = intel_bo_alloc_for_data(pScrn,
					       sf_kernel_static_gen5,
					       sizeof(sf_kernel_static_gen5),
					       "sf kernel gen5");
	sf_kernel_mask_bo = intel_bo_alloc_for_data(pScrn,
						    sf_kernel_mask_static_gen5,
						    sizeof(sf_kernel_mask_static_gen5),
						    "sf mask kernel");
    } else {
	sf_kernel_bo = intel_bo_alloc_for_data(pScrn,
					       sf_kernel_static,
					       sizeof(sf_kernel_static),
					       "sf kernel");
	sf_kernel_mask_bo = intel_bo_alloc_for_data(pScrn,
						    sf_kernel_mask_static,
						    sizeof(sf_kernel_mask_static),
						    "sf mask kernel");
    }
    render_state->sf_state_bo = gen4_create_sf_state(pScrn, sf_kernel_bo);
    render_state->sf_mask_state_bo = gen4_create_sf_state(pScrn,
							  sf_kernel_mask_bo);
    drm_intel_bo_unreference(sf_kernel_bo);
    drm_intel_bo_unreference(sf_kernel_mask_bo);

    for (m = 0; m < WM_KERNEL_COUNT; m++) {
	if (IS_IGDNG(pI830))
	    render_state->wm_kernel_bo[m] =
		intel_bo_alloc_for_data(pScrn,
				        wm_kernels_gen5[m].data, wm_kernels_gen5[m].size,
				        "WM kernel gen5");
	else
	    render_state->wm_kernel_bo[m] =
		intel_bo_alloc_for_data(pScrn,
				        wm_kernels[m].data, wm_kernels[m].size,
				        "WM kernel");
    }

    /* Set up the WM states: each filter/extend type for source and mask, per
     * kernel.
     */
    border_color_bo = sampler_border_color_create(pScrn);
    for (i = 0; i < SAMPLER_STATE_FILTER_COUNT; i++) {
	for (j = 0; j < SAMPLER_STATE_EXTEND_COUNT; j++) {
	    for (k = 0; k < SAMPLER_STATE_FILTER_COUNT; k++) {
		for (l = 0; l < SAMPLER_STATE_EXTEND_COUNT; l++) {
		    drm_intel_bo *sampler_state_bo;

		    sampler_state_bo =
			gen4_create_sampler_state(pScrn,
						  i, j,
						  k, l,
						  border_color_bo);

		    for (m = 0; m < WM_KERNEL_COUNT; m++) {
			if (IS_IGDNG(pI830))
			    render_state->wm_state_bo[m][i][j][k][l] =
				gen4_create_wm_state(pScrn,
						     wm_kernels_gen5[m].has_mask,
						     render_state->wm_kernel_bo[m],
						     sampler_state_bo);
			else
			    render_state->wm_state_bo[m][i][j][k][l] =
				gen4_create_wm_state(pScrn,
						     wm_kernels[m].has_mask,
						     render_state->wm_kernel_bo[m],
						     sampler_state_bo);
		    }
		    drm_intel_bo_unreference(sampler_state_bo);
		}
	    }
	}
    }
    drm_intel_bo_unreference(border_color_bo);

    render_state->cc_state_bo = gen4_create_cc_unit_state(pScrn);
    render_state->sip_kernel_bo = intel_bo_alloc_for_data(pScrn,
							  sip_kernel_static,
							  sizeof(sip_kernel_static),
							  "sip kernel");
}

/**
 * Called at LeaveVT.
 */
void
gen4_render_state_cleanup(ScrnInfoPtr pScrn)
{
    I830Ptr pI830 = I830PTR(pScrn);
    struct gen4_render_state *render_state= pI830->gen4_render_state;
    int i, j, k, l, m;
    gen4_composite_op *composite_op = &render_state->composite_op;

    drm_intel_bo_unreference(composite_op->binding_table_bo);
    drm_intel_bo_unreference(render_state->vertex_buffer_bo);

    drm_intel_bo_unreference(render_state->vs_state_bo);
    drm_intel_bo_unreference(render_state->sf_state_bo);
    drm_intel_bo_unreference(render_state->sf_mask_state_bo);

    for (i = 0; i < WM_KERNEL_COUNT; i++)
	drm_intel_bo_unreference(render_state->wm_kernel_bo[i]);

    for (i = 0; i < SAMPLER_STATE_FILTER_COUNT; i++)
	for (j = 0; j < SAMPLER_STATE_EXTEND_COUNT; j++)
	    for (k = 0; k < SAMPLER_STATE_FILTER_COUNT; k++)
		for (l = 0; l < SAMPLER_STATE_EXTEND_COUNT; l++)
		    for (m = 0; m < WM_KERNEL_COUNT; m++)
			drm_intel_bo_unreference(render_state->wm_state_bo[m][i][j][k][l]);

    drm_intel_bo_unreference(render_state->cc_state_bo);
    drm_intel_bo_unreference(render_state->sip_kernel_bo);

    free(pI830->gen4_render_state);
    pI830->gen4_render_state = NULL;
}