1#include <sys/types.h>
2#include <sys/stat.h>
3#include <fcntl.h>
4#include <errno.h>
5
6#include "xorg-server.h"
7#include "nv_include.h"
8#ifdef DRI2
9#include "dri2.h"
10#else
11#error "This driver requires a DRI2-enabled X server"
12#endif
13#ifdef DRI3
14#include "dri3.h"
15#endif
16#include "xf86drmMode.h"
17
18struct nouveau_dri2_buffer {
19	DRI2BufferRec base;
20	PixmapPtr ppix;
21};
22
23static inline struct nouveau_dri2_buffer *
24nouveau_dri2_buffer(DRI2BufferPtr buf)
25{
26	return (struct nouveau_dri2_buffer *)buf;
27}
28
29static PixmapPtr get_drawable_pixmap(DrawablePtr drawable)
30{
31	if (drawable->type == DRAWABLE_PIXMAP)
32		return (PixmapPtr)drawable;
33	else
34		return (*drawable->pScreen->GetWindowPixmap)((WindowPtr)drawable);
35}
36
37static DRI2BufferPtr
38nouveau_dri2_create_buffer2(ScreenPtr pScreen, DrawablePtr pDraw, unsigned int attachment,
39			   unsigned int format)
40{
41	NVPtr pNv = NVPTR(xf86ScreenToScrn(pScreen));
42	struct nouveau_dri2_buffer *nvbuf;
43	struct nouveau_pixmap *nvpix;
44	PixmapPtr ppix = NULL;
45
46	nvbuf = calloc(1, sizeof(*nvbuf));
47	if (!nvbuf)
48		return NULL;
49
50	if (attachment == DRI2BufferFrontLeft) {
51		ppix = get_drawable_pixmap(pDraw);
52		if (pScreen != ppix->drawable.pScreen)
53			ppix = NULL;
54
55		if (pDraw->type == DRAWABLE_WINDOW) {
56#if DRI2INFOREC_VERSION >= 6
57			/* Set initial swap limit on drawable. */
58			DRI2SwapLimit(pDraw, pNv->swap_limit);
59#endif
60		}
61		if (ppix)
62			ppix->refcnt++;
63	} else {
64		int bpp;
65		unsigned int usage_hint = 0;
66
67		if (pNv->Architecture >= NV_ARCH_10)
68			usage_hint |= NOUVEAU_CREATE_PIXMAP_TILED;
69
70		/* 'format' is just depth (or 0, or maybe it depends on the caller) */
71		bpp = round_up_pow2(format ? format : pDraw->depth);
72
73		if (attachment == DRI2BufferDepth ||
74		    attachment == DRI2BufferDepthStencil)
75			usage_hint |= NOUVEAU_CREATE_PIXMAP_ZETA;
76		else
77			usage_hint |= NOUVEAU_CREATE_PIXMAP_SCANOUT;
78
79		ppix = pScreen->CreatePixmap(pScreen, pDraw->width,
80					     pDraw->height, bpp,
81					     usage_hint);
82	}
83
84	if (ppix) {
85		pNv->exa_force_cp = TRUE;
86		exaMoveInPixmap(ppix);
87		pNv->exa_force_cp = FALSE;
88
89		nvbuf->base.pitch = ppix->devKind;
90		nvbuf->base.cpp = ppix->drawable.bitsPerPixel / 8;
91	}
92
93	nvbuf->base.attachment = attachment;
94	nvbuf->base.driverPrivate = nvbuf;
95	nvbuf->base.format = format;
96	nvbuf->base.flags = 0;
97	nvbuf->ppix = ppix;
98
99	if (ppix) {
100		nvpix = nouveau_pixmap(ppix);
101		if (!nvpix || !nvpix->bo ||
102		    nouveau_bo_name_get(nvpix->bo, &nvbuf->base.name)) {
103			dixDestroyPixmap(nvbuf->ppix, 0);
104			free(nvbuf);
105			return NULL;
106		}
107	}
108	return &nvbuf->base;
109}
110
111static DRI2BufferPtr
112nouveau_dri2_create_buffer(DrawablePtr pDraw, unsigned int attachment,
113			   unsigned int format)
114{
115	return nouveau_dri2_create_buffer2(pDraw->pScreen, pDraw,
116					   attachment, format);
117}
118
119static void
120nouveau_dri2_destroy_buffer2(ScreenPtr pScreen, DrawablePtr pDraw, DRI2BufferPtr buf)
121{
122	struct nouveau_dri2_buffer *nvbuf;
123
124	nvbuf = nouveau_dri2_buffer(buf);
125	if (!nvbuf)
126		return;
127
128	if (nvbuf->ppix)
129	    dixDestroyPixmap(nvbuf->ppix, 0);
130	free(nvbuf);
131}
132
133static void
134nouveau_dri2_destroy_buffer(DrawablePtr pDraw, DRI2BufferPtr buf)
135{
136	nouveau_dri2_destroy_buffer2(pDraw->pScreen, pDraw, buf);
137}
138
139static void
140nouveau_dri2_copy_region2(ScreenPtr pScreen, DrawablePtr pDraw, RegionPtr pRegion,
141			 DRI2BufferPtr pDstBuffer, DRI2BufferPtr pSrcBuffer)
142{
143	struct nouveau_dri2_buffer *src = nouveau_dri2_buffer(pSrcBuffer);
144	struct nouveau_dri2_buffer *dst = nouveau_dri2_buffer(pDstBuffer);
145	NVPtr pNv = NVPTR(xf86ScreenToScrn(pScreen));
146	RegionPtr pCopyClip;
147	GCPtr pGC;
148	DrawablePtr src_draw, dst_draw;
149	Bool translate = FALSE;
150	int off_x = 0, off_y = 0;
151
152	src_draw = &src->ppix->drawable;
153	dst_draw = &dst->ppix->drawable;
154#if 0
155	ErrorF("attachments src %d, dst %d, drawable %p %p pDraw %p\n",
156	       src->base.attachment, dst->base.attachment,
157	       src_draw, dst_draw, pDraw);
158#endif
159	if (src->base.attachment == DRI2BufferFrontLeft)
160		src_draw = pDraw;
161	if (dst->base.attachment == DRI2BufferFrontLeft) {
162#ifdef NOUVEAU_PIXMAP_SHARING
163		if (pDraw->pScreen != pScreen) {
164			dst_draw = DRI2UpdatePrime(pDraw, pDstBuffer);
165			if (!dst_draw)
166				return;
167		}
168		else
169#endif
170			dst_draw = pDraw;
171		if (dst_draw != pDraw)
172			translate = TRUE;
173	}
174
175	if (translate && pDraw->type == DRAWABLE_WINDOW) {
176#ifdef COMPOSITE
177		PixmapPtr pPix = get_drawable_pixmap(pDraw);
178		off_x = -pPix->screen_x;
179		off_y = -pPix->screen_y;
180#endif
181		off_x += pDraw->x;
182		off_y += pDraw->y;
183	}
184
185	pGC = GetScratchGC(pDraw->depth, pScreen);
186	pCopyClip = REGION_CREATE(pScreen, NULL, 0);
187	REGION_COPY(pScreen, pCopyClip, pRegion);
188
189	if (translate) {
190		REGION_TRANSLATE(pScreen, pCopyClip, off_x, off_y);
191	}
192	pGC->funcs->ChangeClip(pGC, CT_REGION, pCopyClip, 0);
193	ValidateGC(dst_draw, pGC);
194
195	/* If this is a full buffer swap or frontbuffer flush, throttle on
196	 * the previous one.
197	 */
198	if (dst->base.attachment == DRI2BufferFrontLeft &&
199	    REGION_NUM_RECTS(pRegion) == 1) {
200		BoxPtr extents = REGION_EXTENTS(pScreen, pRegion);
201		if (extents->x1 == 0 && extents->y1 == 0 &&
202		    extents->x2 == pDraw->width &&
203		    extents->y2 == pDraw->height) {
204			PixmapPtr fpix = get_drawable_pixmap(dst_draw);
205			struct nouveau_bo *bo = nouveau_pixmap_bo(fpix);
206			if (bo)
207				nouveau_bo_wait(bo, NOUVEAU_BO_RD, pNv->client);
208		}
209	}
210
211	pGC->ops->CopyArea(src_draw, dst_draw, pGC, 0, 0,
212			   pDraw->width, pDraw->height, off_x, off_y);
213
214	FreeScratchGC(pGC);
215}
216
217static void
218nouveau_dri2_copy_region(DrawablePtr pDraw, RegionPtr pRegion,
219			 DRI2BufferPtr pDstBuffer, DRI2BufferPtr pSrcBuffer)
220{
221    return nouveau_dri2_copy_region2(pDraw->pScreen, pDraw, pRegion,
222				     pDstBuffer, pSrcBuffer);
223}
224
225struct nouveau_dri2_vblank_state {
226	enum {
227		SWAP,
228		BLIT,
229		WAIT
230	} action;
231
232	ClientPtr client;
233	XID draw;
234
235	DRI2BufferPtr dst;
236	DRI2BufferPtr src;
237	DRI2SwapEventPtr func;
238	void *data;
239	unsigned int frame;
240};
241
242struct dri2_vblank {
243	struct nouveau_dri2_vblank_state *s;
244};
245
246static uint64_t dri2_sequence;
247
248static Bool
249update_front(DrawablePtr draw, DRI2BufferPtr front)
250{
251	ScrnInfoPtr scrn = xf86ScreenToScrn(draw->pScreen);
252	NVPtr pNv = NVPTR(scrn);
253	struct nouveau_dri2_buffer *nvbuf = nouveau_dri2_buffer(front);
254	struct nouveau_bo *pixmap_bo;
255
256	PixmapPtr pixmap;
257	int r;
258
259	if (draw->type == DRAWABLE_PIXMAP)
260		pixmap = (PixmapPtr)draw;
261	else
262		pixmap = (*draw->pScreen->GetWindowPixmap)((WindowPtr)draw);
263
264	pixmap->refcnt++;
265
266	pNv->exa_force_cp = TRUE;
267	exaMoveInPixmap(pixmap);
268	pNv->exa_force_cp = FALSE;
269	pixmap_bo = nouveau_pixmap_bo(pixmap);
270
271	if (!pixmap_bo)
272		r = -1;
273	else
274		r = nouveau_bo_name_get(pixmap_bo, &front->name);
275
276	if (r) {
277		dixDestroyPixmap(pixmap, 0);
278		return FALSE;
279	}
280
281	if (nvbuf->ppix)
282		dixDestroyPixmap(nvbuf->ppix, 0);
283
284	front->pitch = pixmap->devKind;
285	front->cpp = pixmap->drawable.bitsPerPixel / 8;
286	nvbuf->ppix = pixmap;
287
288	return TRUE;
289}
290
291static Bool
292can_exchange(DrawablePtr draw, PixmapPtr dst_pix, PixmapPtr src_pix)
293{
294	ScrnInfoPtr scrn = xf86ScreenToScrn(draw->pScreen);
295	xf86CrtcConfigPtr xf86_config = XF86_CRTC_CONFIG_PTR(scrn);
296	NVPtr pNv = NVPTR(scrn);
297	int i, active_crtc_count = 0;
298
299	if (!xf86_config->num_crtc)
300		return FALSE;
301
302	for (i = 0; i < xf86_config->num_crtc; i++) {
303		xf86CrtcPtr crtc = xf86_config->crtc[i];
304		if (xf86_crtc_on(crtc)) {
305			if (crtc->rotatedData)
306				return FALSE;
307
308			active_crtc_count++;
309		}
310	}
311
312	return ((DRI2CanFlip(draw) && pNv->has_pageflip)) &&
313		dst_pix->drawable.width == src_pix->drawable.width &&
314		dst_pix->drawable.height == src_pix->drawable.height &&
315		dst_pix->drawable.bitsPerPixel == src_pix->drawable.bitsPerPixel &&
316		dst_pix->devKind == src_pix->devKind &&
317		active_crtc_count;
318}
319
320static Bool
321can_sync_to_vblank(DrawablePtr draw)
322{
323	ScrnInfoPtr scrn = xf86ScreenToScrn(draw->pScreen);
324	NVPtr pNv = NVPTR(scrn);
325
326	return pNv->glx_vblank &&
327		nv_window_belongs_to_crtc(scrn, draw->x, draw->y,
328					  draw->width, draw->height);
329}
330
331#if DRI2INFOREC_VERSION >= 6
332static Bool
333nouveau_dri2_swap_limit_validate(DrawablePtr draw, int swap_limit)
334{
335	ScrnInfoPtr scrn = xf86ScreenToScrn(draw->pScreen);
336	NVPtr pNv = NVPTR(scrn);
337
338	if ((swap_limit < 1 ) || (swap_limit > pNv->max_swap_limit))
339		return FALSE;
340
341	return TRUE;
342}
343#endif
344
345/* Shall we intentionally violate the OML_sync_control spec to
346 * get some sort of triple-buffering behaviour on a pre 1.12.0
347 * x-server?
348 */
349static Bool violate_oml(DrawablePtr draw)
350{
351	ScrnInfoPtr scrn = xf86ScreenToScrn(draw->pScreen);
352	NVPtr pNv = NVPTR(scrn);
353
354	return (DRI2INFOREC_VERSION < 6) && (pNv->swap_limit > 1);
355}
356
357typedef struct {
358    int fd;
359    unsigned old_fb_id;
360    int flip_count;
361    void *event_data;
362    unsigned int fe_msc;
363    uint64_t fe_ust;
364} dri2_flipdata_rec, *dri2_flipdata_ptr;
365
366typedef struct {
367    dri2_flipdata_ptr flipdata;
368    Bool dispatch_me;
369} dri2_flipevtcarrier_rec, *dri2_flipevtcarrier_ptr;
370
371static void
372nouveau_dri2_flip_event_handler(unsigned int frame, unsigned int tv_sec,
373				unsigned int tv_usec, void *event_data)
374{
375	struct nouveau_dri2_vblank_state *flip = event_data;
376	DrawablePtr draw;
377	ScreenPtr screen;
378	ScrnInfoPtr scrn;
379	int status;
380
381	status = dixLookupDrawable(&draw, flip->draw, serverClient,
382				   M_ANY, DixWriteAccess);
383	if (status != Success) {
384		free(flip);
385		return;
386	}
387
388	screen = draw->pScreen;
389	scrn = xf86ScreenToScrn(screen);
390
391	/* We assume our flips arrive in order, so we don't check the frame */
392	switch (flip->action) {
393	case SWAP:
394		/* Check for too small vblank count of pageflip completion,
395		 * taking wraparound into account. This usually means some
396		 * defective kms pageflip completion, causing wrong (msc, ust)
397		 * return values and possible visual corruption.
398		 * Skip test for frame == 0, as this is a valid constant value
399		 * reported by all Linux kernels at least up to Linux 3.0.
400		 */
401		if ((frame != 0) &&
402		    (frame < flip->frame) && (flip->frame - frame < 5)) {
403			xf86DrvMsg(scrn->scrnIndex, X_WARNING,
404				   "%s: Pageflip has impossible msc %d < target_msc %d\n",
405				   __func__, frame, flip->frame);
406			/* All-Zero values signal failure of (msc, ust)
407			 * timestamping to client.
408			 */
409			frame = tv_sec = tv_usec = 0;
410		}
411
412		DRI2SwapComplete(flip->client, draw, frame, tv_sec, tv_usec,
413				 DRI2_FLIP_COMPLETE, flip->func,
414				 flip->data);
415		break;
416	default:
417		xf86DrvMsg(scrn->scrnIndex, X_WARNING,
418			   "%s: unknown vblank event received\n", __func__);
419		/* Unknown type */
420		break;
421	}
422
423	free(flip);
424}
425
426static void
427nouveau_dri2_flip_handler(void *priv, uint64_t name, uint64_t ust, uint32_t msc)
428{
429	dri2_flipevtcarrier_ptr flipcarrier = priv;
430	dri2_flipdata_ptr flipdata = flipcarrier->flipdata;
431
432	/* Is this the event whose info shall be delivered to higher level? */
433	if (flipcarrier->dispatch_me) {
434		/* Yes: Cache msc, ust for later delivery. */
435		flipdata->fe_msc = msc;
436		flipdata->fe_ust = ust;
437	}
438
439	/* Last crtc completed flip? */
440	flipdata->flip_count--;
441	if (flipdata->flip_count > 0)
442		return;
443
444	/* Release framebuffer */
445	drmModeRmFB(flipdata->fd, flipdata->old_fb_id);
446
447	if (flipdata->event_data == NULL) {
448		free(flipdata);
449		return;
450	}
451
452	/* Deliver cached msc, ust from reference crtc to flip event handler */
453	nouveau_dri2_flip_event_handler(flipdata->fe_msc,
454					flipdata->fe_ust / 1000000,
455					flipdata->fe_ust % 1000000,
456					flipdata->event_data);
457	free(flipdata);
458}
459
460static Bool
461dri2_page_flip(DrawablePtr draw, PixmapPtr back, void *priv,
462			   xf86CrtcPtr ref_crtc)
463{
464	ScrnInfoPtr scrn = xf86ScreenToScrn(draw->pScreen);
465	xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(scrn);
466	NVPtr pNv = NVPTR(scrn);
467	uint32_t next_fb;
468	int emitted = 0;
469	int ret, i;
470	dri2_flipdata_ptr flipdata;
471	dri2_flipevtcarrier_ptr flipcarrier;
472
473	ret = drmModeAddFB(pNv->dev->fd, scrn->virtualX, scrn->virtualY,
474			   scrn->depth, scrn->bitsPerPixel,
475			   scrn->displayWidth * scrn->bitsPerPixel / 8,
476			   nouveau_pixmap(back)->bo->handle, &next_fb);
477	if (ret) {
478		xf86DrvMsg(scrn->scrnIndex, X_WARNING,
479			   "add fb failed: %s\n", strerror(errno));
480		return FALSE;
481	}
482
483	flipdata = calloc(1, sizeof(dri2_flipdata_rec));
484	if (!flipdata) {
485		xf86DrvMsg(scrn->scrnIndex, X_WARNING,
486		"flip queue: data alloc failed.\n");
487		goto error_undo;
488	}
489
490	flipdata->event_data = priv;
491	flipdata->fd = pNv->dev->fd;
492
493	for (i = 0; i < config->num_crtc; i++) {
494		int head = drmmode_crtc(config->crtc[i]);
495		void *token;
496
497		if (!xf86_crtc_on(config->crtc[i]))
498			continue;
499
500		flipdata->flip_count++;
501
502		flipcarrier = drmmode_event_queue(scrn, ++dri2_sequence,
503						  sizeof(*flipcarrier),
504						  nouveau_dri2_flip_handler,
505						  &token);
506		if (!flipcarrier) {
507			xf86DrvMsg(scrn->scrnIndex, X_WARNING,
508				   "flip queue: carrier alloc failed.\n");
509			if (emitted == 0)
510				free(flipdata);
511			goto error_undo;
512		}
513
514		/* Only the reference crtc will finally deliver its page flip
515		 * completion event. All other crtc's events will be discarded.
516		 */
517		flipcarrier->dispatch_me = (config->crtc[i] == ref_crtc);
518		flipcarrier->flipdata = flipdata;
519
520		ret = drmModePageFlip(pNv->dev->fd, head, next_fb,
521				      DRM_MODE_PAGE_FLIP_EVENT, token);
522		if (ret) {
523			xf86DrvMsg(scrn->scrnIndex, X_WARNING,
524				   "flip queue failed: %s\n", strerror(errno));
525			drmmode_event_abort(scrn, dri2_sequence--, false);
526			if (emitted == 0)
527				free(flipdata);
528			goto error_undo;
529		}
530
531		emitted++;
532	}
533
534	/* Will release old fb after all crtc's completed flip. */
535	drmmode_swap(scrn, next_fb, &flipdata->old_fb_id);
536	return TRUE;
537
538error_undo:
539	drmModeRmFB(pNv->dev->fd, next_fb);
540	return FALSE;
541}
542
543static void
544nouveau_dri2_finish_swap(DrawablePtr draw, unsigned int frame,
545			 unsigned int tv_sec, unsigned int tv_usec,
546			 struct nouveau_dri2_vblank_state *s);
547
548static void
549nouveau_dri2_vblank_handler(void *priv, uint64_t name, uint64_t ust, uint32_t frame)
550{
551	struct dri2_vblank *event = priv;
552	struct nouveau_dri2_vblank_state *s = event->s;
553	uint32_t tv_sec  = ust / 1000000;
554	uint32_t tv_usec = ust % 1000000;
555	DrawablePtr draw;
556	int ret;
557
558	ret = dixLookupDrawable(&draw, s->draw, serverClient,
559				M_ANY, DixWriteAccess);
560	if (ret) {
561		free(s);
562		return;
563	}
564
565	switch (s->action) {
566	case SWAP:
567		nouveau_dri2_finish_swap(draw, frame, tv_sec, tv_usec, s);
568#if DRI2INFOREC_VERSION >= 6
569		/* Restore real swap limit on drawable, now that it is safe. */
570		ScrnInfoPtr scrn = xf86ScreenToScrn(draw->pScreen);
571		DRI2SwapLimit(draw, NVPTR(scrn)->swap_limit);
572#endif
573		break;
574
575	case WAIT:
576		DRI2WaitMSCComplete(s->client, draw, frame, tv_sec, tv_usec);
577		free(s);
578		break;
579
580	case BLIT:
581		DRI2SwapComplete(s->client, draw, frame, tv_sec, tv_usec,
582				 DRI2_BLIT_COMPLETE, s->func, s->data);
583		free(s);
584		break;
585	}
586}
587
588static int
589nouveau_wait_vblank(DrawablePtr draw, int type, CARD64 msc,
590		    CARD64 *pmsc, CARD64 *pust, void *data)
591{
592	ScrnInfoPtr scrn = xf86ScreenToScrn(draw->pScreen);
593	NVPtr pNv = NVPTR(scrn);
594	xf86CrtcPtr crtc;
595	drmVBlank vbl;
596	struct dri2_vblank *event = NULL;
597	void *token = NULL;
598	int ret;
599	int head;
600
601	/* Select crtc which shows the largest part of the drawable */
602	crtc = nouveau_pick_best_crtc(scrn,
603				      draw->x, draw->y, draw->width, draw->height);
604
605	if (!crtc) {
606		xf86DrvMsg(scrn->scrnIndex, X_WARNING,
607				   "Wait for VBlank failed: No valid crtc for drawable.\n");
608		return -EINVAL;
609	}
610
611	if (type & DRM_VBLANK_EVENT) {
612		event = drmmode_event_queue(scrn, ++dri2_sequence,
613					    sizeof(*event),
614					    nouveau_dri2_vblank_handler,
615					    &token);
616		if (!event)
617			return -ENOMEM;
618
619		event->s = data;
620	}
621
622	/* Map xf86CrtcPtr to drmWaitVBlank compatible display head index. */
623	head = drmmode_head(crtc);
624
625	if (head == 1)
626		type |= DRM_VBLANK_SECONDARY;
627	else if (head > 1)
628#ifdef DRM_VBLANK_HIGH_CRTC_SHIFT
629		type |= (head << DRM_VBLANK_HIGH_CRTC_SHIFT) &
630				DRM_VBLANK_HIGH_CRTC_MASK;
631#else
632	xf86DrvMsg(scrn->scrnIndex, X_WARNING,
633			   "Wait for VBlank failed: Called for CRTC %d > 1, but "
634			   "DRM_VBLANK_HIGH_CRTC_SHIFT not defined at build time.\n",
635			   head);
636#endif
637
638	vbl.request.type = type;
639	vbl.request.sequence = msc;
640	vbl.request.signal = (unsigned long)token;
641
642	ret = drmWaitVBlank(pNv->dev->fd, &vbl);
643	if (ret) {
644		xf86DrvMsg(scrn->scrnIndex, X_WARNING,
645			   "Wait for VBlank failed: %s\n", strerror(errno));
646		if (event)
647			drmmode_event_abort(scrn, dri2_sequence--, false);
648		return ret;
649	}
650
651	if (pmsc)
652		*pmsc = vbl.reply.sequence;
653	if (pust)
654		*pust = (CARD64)vbl.reply.tval_sec * 1000000 +
655			vbl.reply.tval_usec;
656	return 0;
657}
658
659static void
660nouveau_dri2_finish_swap(DrawablePtr draw, unsigned int frame,
661			 unsigned int tv_sec, unsigned int tv_usec,
662			 struct nouveau_dri2_vblank_state *s)
663{
664	ScrnInfoPtr scrn = xf86ScreenToScrn(draw->pScreen);
665	NVPtr pNv = NVPTR(scrn);
666	PixmapPtr dst_pix;
667	PixmapPtr src_pix = nouveau_dri2_buffer(s->src)->ppix;
668	struct nouveau_bo *dst_bo;
669	struct nouveau_bo *src_bo = nouveau_pixmap_bo(src_pix);
670	struct nouveau_pushbuf *push = pNv->pushbuf;
671	RegionRec reg;
672	int type, ret;
673	Bool front_updated, will_exchange;
674	xf86CrtcPtr ref_crtc;
675
676	REGION_INIT(0, &reg, (&(BoxRec){ 0, 0, draw->width, draw->height }), 0);
677	REGION_TRANSLATE(0, &reg, draw->x, draw->y);
678
679	/* Main crtc for this drawable shall finally deliver pageflip event. */
680	ref_crtc = nouveau_pick_best_crtc(scrn, draw->x, draw->y,
681					  draw->width, draw->height);
682
683	/* Update frontbuffer pixmap and name: Could have changed due to
684	 * window (un)redirection as part of compositing.
685	 */
686	front_updated = update_front(draw, s->dst);
687
688	/* Assign frontbuffer pixmap, after update in update_front() */
689	dst_pix = nouveau_dri2_buffer(s->dst)->ppix;
690	dst_bo = nouveau_pixmap_bo(dst_pix);
691
692	/* Throttle on the previous frame before swapping */
693	nouveau_bo_wait(dst_bo, NOUVEAU_BO_RD, push->client);
694
695	/* Swap by buffer exchange possible? */
696	will_exchange = front_updated && can_exchange(draw, dst_pix, src_pix);
697
698	/* Only emit a wait for vblank pushbuf here if this is a copy-swap, or
699	 * if it is a kms pageflip-swap on an old kernel. Pure exchange swaps
700	 * don't need sync to vblank. kms pageflip-swaps on Linux 3.13+ are
701	 * synced to vblank in the kms driver, so we must not sync here, or
702	 * framerate will be cut in half!
703	 */
704	if (can_sync_to_vblank(draw) &&
705		(!will_exchange ||
706		(!pNv->has_async_pageflip && nouveau_exa_pixmap_is_onscreen(dst_pix)))) {
707		/* Reference the back buffer to sync it to vblank */
708		nouveau_pushbuf_refn(push, &(struct nouveau_pushbuf_refn) {
709					   src_bo,
710					   NOUVEAU_BO_VRAM | NOUVEAU_BO_RD
711				     }, 1);
712
713		if (pNv->Architecture >= NV_FERMI)
714			NVC0SyncToVBlank(dst_pix, REGION_EXTENTS(0, &reg));
715		else
716		if (pNv->Architecture >= NV_TESLA)
717			NV50SyncToVBlank(dst_pix, REGION_EXTENTS(0, &reg));
718		else
719			NV11SyncToVBlank(dst_pix, REGION_EXTENTS(0, &reg));
720
721		nouveau_pushbuf_kick(push, push->channel);
722	}
723
724	if (will_exchange) {
725		type = DRI2_EXCHANGE_COMPLETE;
726		DamageRegionAppend(draw, &reg);
727
728		if (nouveau_exa_pixmap_is_onscreen(dst_pix)) {
729			type = DRI2_FLIP_COMPLETE;
730			ret = dri2_page_flip(draw, src_pix, violate_oml(draw) ?
731					     NULL : s, ref_crtc);
732			if (!ret)
733				goto out;
734		}
735
736		SWAP(s->dst->name, s->src->name);
737		SWAP(nouveau_pixmap(dst_pix)->bo, nouveau_pixmap(src_pix)->bo);
738
739		DamageRegionProcessPending(draw);
740
741		/* If it is a page flip, finish it in the flip event handler. */
742		if ((type == DRI2_FLIP_COMPLETE) && !violate_oml(draw))
743			return;
744	} else {
745		type = DRI2_BLIT_COMPLETE;
746
747		/* Reference the front buffer to let throttling work
748		 * on occluded drawables. */
749		nouveau_pushbuf_refn(push, &(struct nouveau_pushbuf_refn) {
750					   dst_bo,
751					   NOUVEAU_BO_VRAM | NOUVEAU_BO_RD
752				     }, 1);
753
754		REGION_TRANSLATE(0, &reg, -draw->x, -draw->y);
755		nouveau_dri2_copy_region(draw, &reg, s->dst, s->src);
756
757		if (can_sync_to_vblank(draw) && !violate_oml(draw)) {
758			/* Request a vblank event one vblank from now, the most
759			 * likely (optimistic?) time a direct framebuffer blit
760			 * will complete or a desktop compositor will update its
761			 * screen. This defers DRI2SwapComplete() to the earliest
762			 * likely time of real swap completion.
763			 */
764			s->action = BLIT;
765			ret = nouveau_wait_vblank(draw, DRM_VBLANK_EVENT |
766						  DRM_VBLANK_RELATIVE, 1,
767						  NULL, NULL, s);
768			/* Done, if success. Otherwise use fallback below. */
769			if (!ret)
770				return;
771		}
772	}
773
774	/* Special triple-buffering hack for old pre 1.12.0 x-servers used? */
775	if (violate_oml(draw)) {
776		/* Signal to client that swap completion timestamps and counts
777		 * are invalid - they violate the specification.
778		 */
779		frame = tv_sec = tv_usec = 0;
780	}
781
782	/*
783	 * Tell the X server buffers are already swapped even if they're
784	 * not, to prevent it from blocking the client on the next
785	 * GetBuffers request (and let the client do triple-buffering).
786	 *
787	 * XXX - The DRI2SwapLimit() API allowed us to move this to
788	 *	 the flip handler with no FPS hit for page flipped swaps.
789	 *       It is still needed as a fallback for some copy swaps as
790	 *       we lack a method to detect true swap completion for
791	 *       DRI2_BLIT_COMPLETE.
792	 *
793	 *       It is also used if triple-buffering is requested on
794	 *       old x-servers which don't support the DRI2SwapLimit()
795	 *       function.
796	 */
797	DRI2SwapComplete(s->client, draw, frame, tv_sec, tv_usec,
798			 type, s->func, s->data);
799out:
800	free(s);
801}
802
803static Bool
804nouveau_dri2_schedule_swap(ClientPtr client, DrawablePtr draw,
805			   DRI2BufferPtr dst, DRI2BufferPtr src,
806			   CARD64 *target_msc, CARD64 divisor, CARD64 remainder,
807			   DRI2SwapEventPtr func, void *data)
808{
809	struct nouveau_dri2_vblank_state *s;
810	CARD64 current_msc, expect_msc;
811	CARD64 current_ust;
812	int ret;
813
814	/* Initialize a swap structure */
815	s = malloc(sizeof(*s));
816	if (!s)
817		return FALSE;
818
819	*s = (struct nouveau_dri2_vblank_state)
820		{ SWAP, client, draw->id, dst, src, func, data, 0 };
821
822	if (can_sync_to_vblank(draw)) {
823		/* Get current sequence and vblank time*/
824		ret = nouveau_wait_vblank(draw, DRM_VBLANK_RELATIVE, 0,
825					  &current_msc, &current_ust, NULL);
826		if (ret)
827			goto fail;
828
829		/* Truncate to match kernel interfaces; means occasional overflow
830		 * misses, but that's generally not a big deal.
831		 */
832		*target_msc &= 0xffffffff;
833		divisor &= 0xffffffff;
834		remainder &= 0xffffffff;
835
836		/* Calculate a swap target if we don't have one */
837		if (current_msc >= *target_msc && divisor)
838			*target_msc = current_msc + divisor
839				- (current_msc - remainder) % divisor;
840
841		/* Avoid underflow of unsigned value below */
842		if (*target_msc == 0)
843			*target_msc = 1;
844
845		/* Swap at next possible vblank requested? */
846		if (current_msc >= *target_msc - 1) {
847			/* Special case: Need to swap at next vblank.
848			 * Schedule swap immediately, bypassing the kernel
849			 * vblank event mechanism to avoid a dangerous race
850			 * between the client and the x-server vblank event
851			 * dispatch in the main x-server dispatch loop when
852			 * the swap_limit is set to 2 for triple-buffering.
853			 *
854			 * This also optimizes for the common case of swap
855			 * at next vblank, avoiding vblank dispatch delay.
856			 */
857			s->frame = 1 + ((unsigned int) current_msc & 0xffffffff);
858			*target_msc = 1 + current_msc;
859			nouveau_dri2_finish_swap(draw, current_msc,
860						 (unsigned int) (current_ust / 1000000),
861						 (unsigned int) (current_ust % 1000000),
862						 s);
863			return TRUE;
864		}
865
866		/* This is a swap in the future, ie. the vblank event will
867		 * only get dispatched at least 2 vblanks into the future.
868		 */
869
870#if DRI2INFOREC_VERSION >= 6
871		/* On XOrg 1.12+ we need to temporarily lower the swaplimit to 1,
872		 * so that DRI2GetBuffersWithFormat() requests from the client get
873		 * deferred in the x-server until the vblank event has been
874		 * dispatched to us and nouveau_dri2_finish_swap() is done. If
875		 * we wouldn't do this, DRI2GetBuffersWithFormat() would operate
876		 * on wrong (pre-swap) buffers, and cause a segfault later on in
877		 * nouveau_dri2_finish_swap(). Our vblank event handler will restore
878		 * the old swaplimit immediately after nouveau_dri2_finish_swap()
879		 * is done, so we still get 1 video refresh cycle worth of triple-
880		 * buffering, because the client can start rendering again 1 cycle
881		 * before the pending swap is completed.
882		 *
883		 * The same race would happen for the "swap at next vblank" case,
884		 * but the special case "swap immediately" code above prevents this.
885		 */
886		DRI2SwapLimit(draw, 1);
887#endif
888
889		/* Request a vblank event one frame before the target */
890		ret = nouveau_wait_vblank(draw, DRM_VBLANK_ABSOLUTE |
891					  DRM_VBLANK_EVENT,
892					  max(current_msc, *target_msc - 1),
893					  &expect_msc, NULL, s);
894		if (ret)
895			goto fail;
896		s->frame = 1 + ((unsigned int) expect_msc & 0xffffffff);
897		*target_msc = 1 + expect_msc;
898	} else {
899		/* We can't/don't want to sync to vblank, just swap. */
900		nouveau_dri2_finish_swap(draw, 0, 0, 0, s);
901	}
902
903	return TRUE;
904
905fail:
906	free(s);
907	return FALSE;
908}
909
910static Bool
911nouveau_dri2_schedule_wait(ClientPtr client, DrawablePtr draw,
912			   CARD64 target_msc, CARD64 divisor, CARD64 remainder)
913{
914	struct nouveau_dri2_vblank_state *s;
915	CARD64 current_msc;
916	int ret;
917
918	/* Truncate to match kernel interfaces; means occasional overflow
919	 * misses, but that's generally not a big deal.
920	 */
921	target_msc &= 0xffffffff;
922	divisor &= 0xffffffff;
923	remainder &= 0xffffffff;
924
925	if (!can_sync_to_vblank(draw)) {
926		DRI2WaitMSCComplete(client, draw, target_msc, 0, 0);
927		return TRUE;
928	}
929
930	/* Initialize a vblank structure */
931	s = malloc(sizeof(*s));
932	if (!s)
933		return FALSE;
934
935	*s = (struct nouveau_dri2_vblank_state) { WAIT, client, draw->id };
936
937	/* Get current sequence */
938	ret = nouveau_wait_vblank(draw, DRM_VBLANK_RELATIVE, 0,
939				  &current_msc, NULL, NULL);
940	if (ret)
941		goto fail;
942
943	/* Calculate a wait target if we don't have one */
944	if (current_msc >= target_msc && divisor)
945		target_msc = current_msc + divisor
946			- (current_msc - remainder) % divisor;
947
948	/* Request a vblank event */
949	ret = nouveau_wait_vblank(draw, DRM_VBLANK_ABSOLUTE |
950				  DRM_VBLANK_EVENT,
951				  max(current_msc, target_msc),
952				  NULL, NULL, s);
953	if (ret)
954		goto fail;
955
956	DRI2BlockClient(client, draw);
957	return TRUE;
958fail:
959	free(s);
960	return FALSE;
961}
962
963static Bool
964nouveau_dri2_get_msc(DrawablePtr draw, CARD64 *ust, CARD64 *msc)
965{
966	int ret;
967
968	if (!can_sync_to_vblank(draw)) {
969		*ust = 0;
970		*msc = 0;
971		return TRUE;
972	}
973
974	/* Get current sequence */
975	ret = nouveau_wait_vblank(draw, DRM_VBLANK_RELATIVE, 0, msc, ust, NULL);
976	if (ret)
977		return FALSE;
978
979	return TRUE;
980}
981
982Bool
983nouveau_dri2_init(ScreenPtr pScreen)
984{
985	ScrnInfoPtr pScrn = xf86ScreenToScrn(pScreen);
986	NVPtr pNv = NVPTR(pScrn);
987	DRI2InfoRec dri2 = { 0 };
988	const char *drivernames[2][2] = {
989		{ "nouveau", "nouveau" },
990		{ "nouveau_vieux", "nouveau_vieux" }
991	};
992
993	if (pNv->AccelMethod != EXA)
994		return FALSE;
995
996	if (pNv->Architecture >= NV_ARCH_30)
997		dri2.driverNames = drivernames[0];
998	else
999		dri2.driverNames = drivernames[1];
1000	dri2.numDrivers = 2;
1001	dri2.driverName = dri2.driverNames[0];
1002
1003	dri2.fd = pNv->dev->fd;
1004	dri2.deviceName = pNv->drm_device_name;
1005
1006	dri2.version = DRI2INFOREC_VERSION;
1007	dri2.CreateBuffer = nouveau_dri2_create_buffer;
1008	dri2.DestroyBuffer = nouveau_dri2_destroy_buffer;
1009	dri2.CopyRegion = nouveau_dri2_copy_region;
1010	dri2.ScheduleSwap = nouveau_dri2_schedule_swap;
1011	dri2.ScheduleWaitMSC = nouveau_dri2_schedule_wait;
1012	dri2.GetMSC = nouveau_dri2_get_msc;
1013
1014#if DRI2INFOREC_VERSION >= 6
1015	dri2.SwapLimitValidate = nouveau_dri2_swap_limit_validate;
1016#endif
1017
1018#if DRI2INFOREC_VERSION >= 7
1019	dri2.version = 7;
1020	dri2.GetParam = NULL;
1021#endif
1022
1023#if DRI2INFOREC_VERSION >= 9
1024	dri2.version = 9;
1025	dri2.CreateBuffer2 = nouveau_dri2_create_buffer2;
1026	dri2.DestroyBuffer2 = nouveau_dri2_destroy_buffer2;
1027	dri2.CopyRegion2 = nouveau_dri2_copy_region2;
1028#endif
1029	return DRI2ScreenInit(pScreen, &dri2);
1030}
1031
1032void
1033nouveau_dri2_fini(ScreenPtr pScreen)
1034{
1035	ScrnInfoPtr pScrn = xf86ScreenToScrn(pScreen);
1036	NVPtr pNv = NVPTR(pScrn);
1037	if (pNv->AccelMethod == EXA)
1038		DRI2CloseScreen(pScreen);
1039}
1040
1041#ifdef DRI3
1042static int is_render_node(int fd)
1043{
1044	struct stat st;
1045	if (fstat(fd, &st))
1046		return 0;
1047
1048	if (!S_ISCHR(st.st_mode))
1049		return 0;
1050
1051	return st.st_rdev & 0x80;
1052  }
1053
1054static int
1055nouveau_dri3_open(ScreenPtr screen, RRProviderPtr provider, int *out)
1056{
1057	ScrnInfoPtr pScrn = xf86ScreenToScrn(screen);
1058	NVPtr pNv = NVPTR(pScrn);
1059	int fd = -1;
1060
1061#ifdef O_CLOEXEC
1062	fd = open(pNv->render_node, O_RDWR | O_CLOEXEC);
1063#endif
1064	if (fd < 0)
1065		fd = open(pNv->render_node, O_RDWR);
1066	if (fd < 0)
1067		return -BadAlloc;
1068
1069	if (!is_render_node(fd)) {
1070		drm_magic_t magic;
1071
1072		if (drmGetMagic(fd, &magic) || drmAuthMagic(pNv->dev->fd, magic)) {
1073			close(fd);
1074			return -BadMatch;
1075		}
1076	}
1077
1078	*out = fd;
1079	return Success;
1080}
1081
1082static PixmapPtr nouveau_dri3_pixmap_from_fd(ScreenPtr screen, int fd, CARD16 width, CARD16 height, CARD16 stride, CARD8 depth, CARD8 bpp)
1083{
1084	ScrnInfoPtr pScrn = xf86ScreenToScrn(screen);
1085	NVPtr pNv = NVPTR(pScrn);
1086	PixmapPtr pixmap;
1087	struct nouveau_bo *bo = NULL;
1088	struct nouveau_pixmap *nvpix;
1089
1090	if (depth < 8 || depth > 32)
1091		return NULL;
1092
1093	pixmap = screen->CreatePixmap(screen, 0, 0, depth, 0);
1094	if (!pixmap)
1095		return NULL;
1096
1097	if (pixmap->drawable.bitsPerPixel % 8)
1098		goto free_pixmap;
1099
1100	if (!screen->ModifyPixmapHeader(pixmap, width, height, 0, 0, stride, NULL))
1101		goto free_pixmap;
1102
1103	if (nouveau_bo_prime_handle_ref(pNv->dev, fd, &bo))
1104		goto free_pixmap;
1105
1106	nvpix = nouveau_pixmap(pixmap);
1107	nouveau_bo_ref(NULL, &nvpix->bo);
1108	nvpix->bo = bo;
1109	nvpix->shared = (bo->flags & NOUVEAU_BO_APER) == NOUVEAU_BO_GART;
1110	return pixmap;
1111
1112free_pixmap:
1113	dixDestroyPixmap(pixmap, 0);
1114	return NULL;
1115}
1116
1117static int nouveau_dri3_fd_from_pixmap(ScreenPtr screen, PixmapPtr pixmap, CARD16 *stride, CARD32 *size)
1118{
1119	struct nouveau_bo *bo = nouveau_pixmap_bo(pixmap);
1120	int fd;
1121
1122	if (!bo || nouveau_bo_set_prime(bo, &fd) < 0)
1123		return -EINVAL;
1124
1125	*stride = pixmap->devKind;
1126	*size = bo->size;
1127	return fd;
1128}
1129
1130static dri3_screen_info_rec nouveau_dri3_screen_info = {
1131        .version = DRI3_SCREEN_INFO_VERSION,
1132
1133        .open = nouveau_dri3_open,
1134        .pixmap_from_fd = nouveau_dri3_pixmap_from_fd,
1135        .fd_from_pixmap = nouveau_dri3_fd_from_pixmap
1136};
1137#endif
1138
1139Bool
1140nouveau_dri3_screen_init(ScreenPtr screen)
1141{
1142#ifdef DRI3
1143	ScrnInfoPtr pScrn = xf86ScreenToScrn(screen);
1144	NVPtr pNv = NVPTR(pScrn);
1145	char *buf;
1146
1147	if (is_render_node(pNv->dev->fd))
1148		return TRUE;
1149
1150	buf = drmGetRenderDeviceNameFromFd(pNv->dev->fd);
1151	if (buf) {
1152		pNv->render_node = buf;
1153		if (dri3_screen_init(screen, &nouveau_dri3_screen_info)) {
1154			xf86DrvMsg(pScrn->scrnIndex, X_INFO,
1155				   "DRI3 on EXA enabled\n");
1156			return TRUE;
1157		}
1158		else {
1159			xf86DrvMsg(pScrn->scrnIndex, X_WARNING,
1160				   "DRI3 on EXA initialization failed\n");
1161			return FALSE;
1162		}
1163	} else
1164		free(buf);
1165#endif
1166
1167        return TRUE;
1168}
1169