msm_gem_submit.c 14 KB
Newer Older
Rob Clark's avatar
Rob Clark committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Copyright (C) 2013 Red Hat
 * Author: Rob Clark <robdclark@gmail.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published by
 * the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program.  If not, see <http://www.gnu.org/licenses/>.
 */

18 19
#include <linux/sync_file.h>

Rob Clark's avatar
Rob Clark committed
20 21 22
#include "msm_drv.h"
#include "msm_gpu.h"
#include "msm_gem.h"
23
#include "msm_gpu_trace.h"
Rob Clark's avatar
Rob Clark committed
24 25 26 27 28 29

/*
 * Cmdstream submission:
 */

/* make sure these don't conflict w/ MSM_SUBMIT_BO_x */
30
#define BO_VALID    0x8000   /* is current addr in cmdstream correct/valid? */
Rob Clark's avatar
Rob Clark committed
31 32 33 34
#define BO_LOCKED   0x4000
#define BO_PINNED   0x2000

static struct msm_gem_submit *submit_create(struct drm_device *dev,
35 36
		struct msm_gpu *gpu, struct msm_gpu_submitqueue *queue,
		uint32_t nr_bos, uint32_t nr_cmds)
Rob Clark's avatar
Rob Clark committed
37 38
{
	struct msm_gem_submit *submit;
39 40
	uint64_t sz = sizeof(*submit) + ((u64)nr_bos * sizeof(submit->bos[0])) +
		((u64)nr_cmds * sizeof(submit->cmd[0]));
41 42 43

	if (sz > SIZE_MAX)
		return NULL;
Rob Clark's avatar
Rob Clark committed
44

45
	submit = kmalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
46 47
	if (!submit)
		return NULL;
Rob Clark's avatar
Rob Clark committed
48

49 50
	submit->dev = dev;
	submit->gpu = gpu;
51
	submit->fence = NULL;
52
	submit->cmd = (void *)&submit->bos[nr_bos];
53
	submit->queue = queue;
54
	submit->ring = gpu->rb[queue->prio];
Rob Clark's avatar
Rob Clark committed
55

56 57 58 59
	/* initially, until copy_from_user() and bo lookup succeeds: */
	submit->nr_bos = 0;
	submit->nr_cmds = 0;

60
	INIT_LIST_HEAD(&submit->node);
61 62
	INIT_LIST_HEAD(&submit->bo_list);
	ww_acquire_init(&submit->ticket, &reservation_ww_class);
Rob Clark's avatar
Rob Clark committed
63 64 65 66

	return submit;
}

67 68
void msm_gem_submit_free(struct msm_gem_submit *submit)
{
69
	dma_fence_put(submit->fence);
70
	list_del(&submit->node);
71
	put_pid(submit->pid);
72 73
	msm_submitqueue_put(submit->queue);

74 75 76
	kfree(submit);
}

77 78 79 80 81 82 83 84
static inline unsigned long __must_check
copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
{
	if (access_ok(VERIFY_READ, from, n))
		return __copy_from_user_inatomic(to, from, n);
	return -EFAULT;
}

Rob Clark's avatar
Rob Clark committed
85 86 87 88 89 90 91
static int submit_lookup_objects(struct msm_gem_submit *submit,
		struct drm_msm_gem_submit *args, struct drm_file *file)
{
	unsigned i;
	int ret = 0;

	spin_lock(&file->table_lock);
92
	pagefault_disable();
Rob Clark's avatar
Rob Clark committed
93 94 95 96 97 98

	for (i = 0; i < args->nr_bos; i++) {
		struct drm_msm_gem_submit_bo submit_bo;
		struct drm_gem_object *obj;
		struct msm_gem_object *msm_obj;
		void __user *userptr =
99
			u64_to_user_ptr(args->bos + (i * sizeof(submit_bo)));
Rob Clark's avatar
Rob Clark committed
100

101 102 103 104 105
		/* make sure we don't have garbage flags, in case we hit
		 * error path before flags is initialized:
		 */
		submit->bos[i].flags = 0;

106
		if (copy_from_user_inatomic(&submit_bo, userptr, sizeof(submit_bo))) {
107 108
			pagefault_enable();
			spin_unlock(&file->table_lock);
109 110
			if (copy_from_user(&submit_bo, userptr, sizeof(submit_bo))) {
				ret = -EFAULT;
111
				goto out;
112
			}
113 114
			spin_lock(&file->table_lock);
			pagefault_disable();
Rob Clark's avatar
Rob Clark committed
115 116
		}

117 118
		if ((submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) ||
			!(submit_bo.flags & MSM_SUBMIT_BO_FLAGS)) {
119
			DRM_ERROR("invalid flags: %x\n", submit_bo.flags);
Rob Clark's avatar
Rob Clark committed
120 121 122 123 124 125 126 127 128 129 130 131 132
			ret = -EINVAL;
			goto out_unlock;
		}

		submit->bos[i].flags = submit_bo.flags;
		/* in validate_objects() we figure out if this is true: */
		submit->bos[i].iova  = submit_bo.presumed;

		/* normally use drm_gem_object_lookup(), but for bulk lookup
		 * all under single table_lock just hit object_idr directly:
		 */
		obj = idr_find(&file->object_idr, submit_bo.handle);
		if (!obj) {
133
			DRM_ERROR("invalid handle %u at index %u\n", submit_bo.handle, i);
Rob Clark's avatar
Rob Clark committed
134 135 136 137 138 139 140
			ret = -EINVAL;
			goto out_unlock;
		}

		msm_obj = to_msm_bo(obj);

		if (!list_empty(&msm_obj->submit_entry)) {
141
			DRM_ERROR("handle %u at index %u already on submit list\n",
Rob Clark's avatar
Rob Clark committed
142 143 144 145 146
					submit_bo.handle, i);
			ret = -EINVAL;
			goto out_unlock;
		}

147
		drm_gem_object_get(obj);
Rob Clark's avatar
Rob Clark committed
148 149 150 151 152 153 154

		submit->bos[i].obj = msm_obj;

		list_add_tail(&msm_obj->submit_entry, &submit->bo_list);
	}

out_unlock:
155
	pagefault_enable();
Rob Clark's avatar
Rob Clark committed
156 157
	spin_unlock(&file->table_lock);

158 159 160
out:
	submit->nr_bos = i;

Rob Clark's avatar
Rob Clark committed
161 162 163
	return ret;
}

164 165
static void submit_unlock_unpin_bo(struct msm_gem_submit *submit,
		int i, bool backoff)
Rob Clark's avatar
Rob Clark committed
166 167 168 169
{
	struct msm_gem_object *msm_obj = submit->bos[i].obj;

	if (submit->bos[i].flags & BO_PINNED)
170
		msm_gem_put_iova(&msm_obj->base, submit->gpu->aspace);
Rob Clark's avatar
Rob Clark committed
171 172 173 174

	if (submit->bos[i].flags & BO_LOCKED)
		ww_mutex_unlock(&msm_obj->resv->lock);

175
	if (backoff && !(submit->bos[i].flags & BO_VALID))
Rob Clark's avatar
Rob Clark committed
176 177 178 179 180 181
		submit->bos[i].iova = 0;

	submit->bos[i].flags &= ~(BO_LOCKED | BO_PINNED);
}

/* This is where we make sure all the bo's are reserved and pin'd: */
182
static int submit_lock_objects(struct msm_gem_submit *submit)
Rob Clark's avatar
Rob Clark committed
183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209
{
	int contended, slow_locked = -1, i, ret = 0;

retry:
	for (i = 0; i < submit->nr_bos; i++) {
		struct msm_gem_object *msm_obj = submit->bos[i].obj;

		if (slow_locked == i)
			slow_locked = -1;

		contended = i;

		if (!(submit->bos[i].flags & BO_LOCKED)) {
			ret = ww_mutex_lock_interruptible(&msm_obj->resv->lock,
					&submit->ticket);
			if (ret)
				goto fail;
			submit->bos[i].flags |= BO_LOCKED;
		}
	}

	ww_acquire_done(&submit->ticket);

	return 0;

fail:
	for (; i >= 0; i--)
210
		submit_unlock_unpin_bo(submit, i, true);
Rob Clark's avatar
Rob Clark committed
211 212

	if (slow_locked > 0)
213
		submit_unlock_unpin_bo(submit, slow_locked, true);
Rob Clark's avatar
Rob Clark committed
214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229

	if (ret == -EDEADLK) {
		struct msm_gem_object *msm_obj = submit->bos[contended].obj;
		/* we lost out in a seqno race, lock and retry.. */
		ret = ww_mutex_lock_slow_interruptible(&msm_obj->resv->lock,
				&submit->ticket);
		if (!ret) {
			submit->bos[contended].flags |= BO_LOCKED;
			slow_locked = contended;
			goto retry;
		}
	}

	return ret;
}

230
static int submit_fence_sync(struct msm_gem_submit *submit, bool no_implicit)
231 232 233 234 235 236 237
{
	int i, ret = 0;

	for (i = 0; i < submit->nr_bos; i++) {
		struct msm_gem_object *msm_obj = submit->bos[i].obj;
		bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE;

238 239 240 241 242 243
		if (!write) {
			/* NOTE: _reserve_shared() must happen before
			 * _add_shared_fence(), which makes this a slightly
			 * strange place to call it.  OTOH this is a
			 * convenient can-fail point to hook it in.
			 */
244 245
			ret = reservation_object_reserve_shared(msm_obj->resv,
								1);
246 247 248 249 250 251 252
			if (ret)
				return ret;
		}

		if (no_implicit)
			continue;

253 254
		ret = msm_gem_sync_object(&msm_obj->base, submit->ring->fctx,
			write);
255 256 257 258 259 260 261
		if (ret)
			break;
	}

	return ret;
}

262 263 264 265 266 267 268 269
static int submit_pin_objects(struct msm_gem_submit *submit)
{
	int i, ret = 0;

	submit->valid = true;

	for (i = 0; i < submit->nr_bos; i++) {
		struct msm_gem_object *msm_obj = submit->bos[i].obj;
Rob Clark's avatar
Rob Clark committed
270
		uint64_t iova;
271 272

		/* if locking succeeded, pin bo: */
273
		ret = msm_gem_get_iova(&msm_obj->base,
274
				submit->gpu->aspace, &iova);
275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293

		if (ret)
			break;

		submit->bos[i].flags |= BO_PINNED;

		if (iova == submit->bos[i].iova) {
			submit->bos[i].flags |= BO_VALID;
		} else {
			submit->bos[i].iova = iova;
			/* iova changed, so address in cmdstream is not valid: */
			submit->bos[i].flags &= ~BO_VALID;
			submit->valid = false;
		}
	}

	return ret;
}

Rob Clark's avatar
Rob Clark committed
294
static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
Rob Clark's avatar
Rob Clark committed
295
		struct msm_gem_object **obj, uint64_t *iova, bool *valid)
Rob Clark's avatar
Rob Clark committed
296 297
{
	if (idx >= submit->nr_bos) {
298 299 300
		DRM_ERROR("invalid buffer index: %u (out of %u)\n",
				idx, submit->nr_bos);
		return -EINVAL;
Rob Clark's avatar
Rob Clark committed
301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318
	}

	if (obj)
		*obj = submit->bos[idx].obj;
	if (iova)
		*iova = submit->bos[idx].iova;
	if (valid)
		*valid = !!(submit->bos[idx].flags & BO_VALID);

	return 0;
}

/* process the reloc's and patch up the cmdstream as needed: */
static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *obj,
		uint32_t offset, uint32_t nr_relocs, uint64_t relocs)
{
	uint32_t i, last_offset = 0;
	uint32_t *ptr;
319
	int ret = 0;
Rob Clark's avatar
Rob Clark committed
320 321

	if (offset % 4) {
322
		DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset);
Rob Clark's avatar
Rob Clark committed
323 324 325 326 327 328
		return -EINVAL;
	}

	/* For now, just map the entire thing.  Eventually we probably
	 * to do it page-by-page, w/ kmap() if not vmap()d..
	 */
329
	ptr = msm_gem_get_vaddr(&obj->base);
Rob Clark's avatar
Rob Clark committed
330 331 332 333 334 335 336 337 338 339

	if (IS_ERR(ptr)) {
		ret = PTR_ERR(ptr);
		DBG("failed to map: %d", ret);
		return ret;
	}

	for (i = 0; i < nr_relocs; i++) {
		struct drm_msm_gem_submit_reloc submit_reloc;
		void __user *userptr =
340
			u64_to_user_ptr(relocs + (i * sizeof(submit_reloc)));
Rob Clark's avatar
Rob Clark committed
341 342
		uint32_t off;
		uint64_t iova;
Rob Clark's avatar
Rob Clark committed
343 344
		bool valid;

345 346
		if (copy_from_user(&submit_reloc, userptr, sizeof(submit_reloc))) {
			ret = -EFAULT;
347
			goto out;
348
		}
Rob Clark's avatar
Rob Clark committed
349 350

		if (submit_reloc.submit_offset % 4) {
351
			DRM_ERROR("non-aligned reloc offset: %u\n",
Rob Clark's avatar
Rob Clark committed
352
					submit_reloc.submit_offset);
353 354
			ret = -EINVAL;
			goto out;
Rob Clark's avatar
Rob Clark committed
355 356 357 358 359 360 361
		}

		/* offset in dwords: */
		off = submit_reloc.submit_offset / 4;

		if ((off >= (obj->base.size / 4)) ||
				(off < last_offset)) {
362
			DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
363 364
			ret = -EINVAL;
			goto out;
Rob Clark's avatar
Rob Clark committed
365 366 367 368
		}

		ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);
		if (ret)
369
			goto out;
Rob Clark's avatar
Rob Clark committed
370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385

		if (valid)
			continue;

		iova += submit_reloc.reloc_offset;

		if (submit_reloc.shift < 0)
			iova >>= -submit_reloc.shift;
		else
			iova <<= submit_reloc.shift;

		ptr[off] = iova | submit_reloc.or;

		last_offset = off;
	}

386
out:
387
	msm_gem_put_vaddr(&obj->base);
388

389
	return ret;
Rob Clark's avatar
Rob Clark committed
390 391
}

392
static void submit_cleanup(struct msm_gem_submit *submit)
Rob Clark's avatar
Rob Clark committed
393 394 395 396 397
{
	unsigned i;

	for (i = 0; i < submit->nr_bos; i++) {
		struct msm_gem_object *msm_obj = submit->bos[i].obj;
398
		submit_unlock_unpin_bo(submit, i, false);
Rob Clark's avatar
Rob Clark committed
399
		list_del_init(&msm_obj->submit_entry);
400
		drm_gem_object_put(&msm_obj->base);
Rob Clark's avatar
Rob Clark committed
401 402 403 404 405 406 407 408
	}

	ww_acquire_fini(&submit->ticket);
}

int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
		struct drm_file *file)
{
409
	static atomic_t ident = ATOMIC_INIT(0);
Rob Clark's avatar
Rob Clark committed
410 411 412 413
	struct msm_drm_private *priv = dev->dev_private;
	struct drm_msm_gem_submit *args = data;
	struct msm_file_private *ctx = file->driver_priv;
	struct msm_gem_submit *submit;
414
	struct msm_gpu *gpu = priv->gpu;
415
	struct dma_fence *in_fence = NULL;
416
	struct sync_file *sync_file = NULL;
417
	struct msm_gpu_submitqueue *queue;
418
	struct msm_ringbuffer *ring;
419
	int out_fence_fd = -1;
420
	struct pid *pid = get_pid(task_pid(current));
Rob Clark's avatar
Rob Clark committed
421
	unsigned i;
422
	int ret, submitid;
423 424 425
	if (!gpu)
		return -ENXIO;

Rob Clark's avatar
Rob Clark committed
426 427 428
	/* for now, we just have 3d pipe.. eventually this would need to
	 * be more clever to dispatch to appropriate gpu module:
	 */
429 430 431 432
	if (MSM_PIPE_ID(args->flags) != MSM_PIPE_3D0)
		return -EINVAL;

	if (MSM_PIPE_FLAGS(args->flags) & ~MSM_SUBMIT_FLAGS)
Rob Clark's avatar
Rob Clark committed
433 434
		return -EINVAL;

435 436 437 438 439 440
	if (args->flags & MSM_SUBMIT_SUDO) {
		if (!IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) ||
		    !capable(CAP_SYS_RAWIO))
			return -EINVAL;
	}

441 442 443 444
	queue = msm_submitqueue_get(ctx, args->queueid);
	if (!queue)
		return -ENOENT;

445 446 447
	/* Get a unique identifier for the submission for logging purposes */
	submitid = atomic_inc_return(&ident) - 1;

448
	ring = gpu->rb[queue->prio];
449 450
	trace_msm_gpu_submit(pid_nr(pid), ring->id, submitid,
		args->nr_bos, args->nr_cmds);
451

452 453 454 455 456 457
	if (args->flags & MSM_SUBMIT_FENCE_FD_IN) {
		in_fence = sync_file_get_fence(args->fence_fd);

		if (!in_fence)
			return -EINVAL;

458 459 460
		/*
		 * Wait if the fence is from a foreign context, or if the fence
		 * array contains any fence from a foreign context.
461
		 */
462
		if (!dma_fence_match_context(in_fence, ring->fctx->context)) {
463 464 465 466 467 468
			ret = dma_fence_wait(in_fence, true);
			if (ret)
				return ret;
		}
	}

469 470 471
	ret = mutex_lock_interruptible(&dev->struct_mutex);
	if (ret)
		return ret;
472

473 474 475 476 477 478 479 480
	if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
		out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
		if (out_fence_fd < 0) {
			ret = out_fence_fd;
			goto out_unlock;
		}
	}

481
	submit = submit_create(dev, gpu, queue, args->nr_bos, args->nr_cmds);
482 483 484 485
	if (!submit) {
		ret = -ENOMEM;
		goto out_unlock;
	}
Rob Clark's avatar
Rob Clark committed
486

487 488 489
	submit->pid = pid;
	submit->ident = submitid;

490 491 492
	if (args->flags & MSM_SUBMIT_SUDO)
		submit->in_rb = true;

Rob Clark's avatar
Rob Clark committed
493 494 495 496
	ret = submit_lookup_objects(submit, args, file);
	if (ret)
		goto out;

497 498 499 500
	ret = submit_lock_objects(submit);
	if (ret)
		goto out;

501 502 503
	ret = submit_fence_sync(submit, !!(args->flags & MSM_SUBMIT_NO_IMPLICIT));
	if (ret)
		goto out;
504

505
	ret = submit_pin_objects(submit);
Rob Clark's avatar
Rob Clark committed
506 507 508 509 510 511
	if (ret)
		goto out;

	for (i = 0; i < args->nr_cmds; i++) {
		struct drm_msm_gem_submit_cmd submit_cmd;
		void __user *userptr =
512
			u64_to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
Rob Clark's avatar
Rob Clark committed
513
		struct msm_gem_object *msm_obj;
Rob Clark's avatar
Rob Clark committed
514
		uint64_t iova;
Rob Clark's avatar
Rob Clark committed
515 516 517 518 519 520 521

		ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
		if (ret) {
			ret = -EFAULT;
			goto out;
		}

Rob Clark's avatar
Rob Clark committed
522 523 524 525 526 527 528 529 530 531 532 533
		/* validate input from userspace: */
		switch (submit_cmd.type) {
		case MSM_SUBMIT_CMD_BUF:
		case MSM_SUBMIT_CMD_IB_TARGET_BUF:
		case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
			break;
		default:
			DRM_ERROR("invalid type: %08x\n", submit_cmd.type);
			ret = -EINVAL;
			goto out;
		}

Rob Clark's avatar
Rob Clark committed
534 535 536 537 538 539
		ret = submit_bo(submit, submit_cmd.submit_idx,
				&msm_obj, &iova, NULL);
		if (ret)
			goto out;

		if (submit_cmd.size % 4) {
540
			DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
Rob Clark's avatar
Rob Clark committed
541 542 543 544 545
					submit_cmd.size);
			ret = -EINVAL;
			goto out;
		}

546 547 548
		if (!submit_cmd.size ||
			((submit_cmd.size + submit_cmd.submit_offset) >
				msm_obj->base.size)) {
549
			DRM_ERROR("invalid cmdstream size: %u\n", submit_cmd.size);
Rob Clark's avatar
Rob Clark committed
550 551 552 553 554 555 556
			ret = -EINVAL;
			goto out;
		}

		submit->cmd[i].type = submit_cmd.type;
		submit->cmd[i].size = submit_cmd.size / 4;
		submit->cmd[i].iova = iova + submit_cmd.submit_offset;
Rob Clark's avatar
Rob Clark committed
557
		submit->cmd[i].idx  = submit_cmd.submit_idx;
Rob Clark's avatar
Rob Clark committed
558 559 560 561 562 563 564 565 566 567 568 569

		if (submit->valid)
			continue;

		ret = submit_reloc(submit, msm_obj, submit_cmd.submit_offset,
				submit_cmd.nr_relocs, submit_cmd.relocs);
		if (ret)
			goto out;
	}

	submit->nr_cmds = i;

570
	submit->fence = msm_fence_alloc(ring->fctx);
571 572 573 574 575 576
	if (IS_ERR(submit->fence)) {
		ret = PTR_ERR(submit->fence);
		submit->fence = NULL;
		goto out;
	}

577 578 579 580 581 582 583 584
	if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
		sync_file = sync_file_create(submit->fence);
		if (!sync_file) {
			ret = -ENOMEM;
			goto out;
		}
	}

585
	msm_gpu_submit(gpu, submit, ctx);
Rob Clark's avatar
Rob Clark committed
586

587
	args->fence = submit->fence->seqno;
Rob Clark's avatar
Rob Clark committed
588

589 590 591 592 593
	if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
		fd_install(out_fence_fd, sync_file->file);
		args->fence_fd = out_fence_fd;
	}

Rob Clark's avatar
Rob Clark committed
594
out:
595
	if (in_fence)
596
		dma_fence_put(in_fence);
597 598 599
	submit_cleanup(submit);
	if (ret)
		msm_gem_submit_free(submit);
600
out_unlock:
601 602
	if (ret && (out_fence_fd >= 0))
		put_unused_fd(out_fence_fd);
603
	mutex_unlock(&dev->struct_mutex);
Rob Clark's avatar
Rob Clark committed
604 605
	return ret;
}