vc4_drv.h 31.6 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0-only */
2 3 4
/*
 * Copyright (C) 2015 Broadcom
 */
5 6
#ifndef _VC4_DRV_H_
#define _VC4_DRV_H_
7

Sam Ravnborg's avatar
Sam Ravnborg committed
8
#include <linux/delay.h>
9
#include <linux/of.h>
Sam Ravnborg's avatar
Sam Ravnborg committed
10 11 12 13 14 15
#include <linux/refcount.h>
#include <linux/uaccess.h>

#include <drm/drm_atomic.h>
#include <drm/drm_debugfs.h>
#include <drm/drm_device.h>
16
#include <drm/drm_encoder.h>
17
#include <drm/drm_gem_dma_helper.h>
18
#include <drm/drm_managed.h>
Sam Ravnborg's avatar
Sam Ravnborg committed
19 20
#include <drm/drm_mm.h>
#include <drm/drm_modeset_lock.h>
21

22 23
#include <kunit/test-bug.h>

24 25
#include "uapi/drm/vc4_drm.h"

Sam Ravnborg's avatar
Sam Ravnborg committed
26 27 28
struct drm_device;
struct drm_gem_object;

29 30 31
extern const struct drm_driver vc4_drm_driver;
extern const struct drm_driver vc5_drm_driver;

32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49
/* Don't forget to update vc4_bo.c: bo_type_names[] when adding to
 * this.
 */
enum vc4_kernel_bo_type {
	/* Any kernel allocation (gem_create_object hook) before it
	 * gets another type set.
	 */
	VC4_BO_TYPE_KERNEL,
	VC4_BO_TYPE_V3D,
	VC4_BO_TYPE_V3D_SHADER,
	VC4_BO_TYPE_DUMB,
	VC4_BO_TYPE_BIN,
	VC4_BO_TYPE_RCL,
	VC4_BO_TYPE_BCL,
	VC4_BO_TYPE_KERNEL_CACHE,
	VC4_BO_TYPE_COUNT
};

50 51 52 53 54 55 56
/* Performance monitor object. The perform lifetime is controlled by userspace
 * using perfmon related ioctls. A perfmon can be attached to a submit_cl
 * request, and when this is the case, HW perf counters will be activated just
 * before the submit_cl is submitted to the GPU and disabled when the job is
 * done. This way, only events related to a specific job will be counted.
 */
struct vc4_perfmon {
57 58
	struct vc4_dev *dev;

59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
	/* Tracks the number of users of the perfmon, when this counter reaches
	 * zero the perfmon is destroyed.
	 */
	refcount_t refcnt;

	/* Number of counters activated in this perfmon instance
	 * (should be less than DRM_VC4_MAX_PERF_COUNTERS).
	 */
	u8 ncounters;

	/* Events counted by the HW perf counters. */
	u8 events[DRM_VC4_MAX_PERF_COUNTERS];

	/* Storage for counter values. Counters are incremented by the HW
	 * perf counter values every time the perfmon is attached to a GPU job.
	 * This way, perfmon users don't have to retrieve the results after
	 * each job if they want to track events covering several submissions.
	 * Note that counter values can't be reset, but you can fake a reset by
	 * destroying the perfmon and creating a new one.
	 */
79
	u64 counters[];
80 81
};

82
struct vc4_dev {
83
	struct drm_device base;
84
	struct device *dev;
85

86 87
	bool is_vc5;

88 89
	unsigned int irq;

90
	struct vc4_hvs *hvs;
91
	struct vc4_v3d *v3d;
92

93 94
	struct vc4_hang_state *hang_state;

Eric Anholt's avatar
Eric Anholt committed
95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
	/* The kernel-space BO cache.  Tracks buffers that have been
	 * unreferenced by all other users (refcounts of 0!) but not
	 * yet freed, so we can do cheap allocations.
	 */
	struct vc4_bo_cache {
		/* Array of list heads for entries in the BO cache,
		 * based on number of pages, so we can do O(1) lookups
		 * in the cache when allocating.
		 */
		struct list_head *size_list;
		uint32_t size_list_size;

		/* List of all BOs in the cache, ordered by age, so we
		 * can do O(1) lookups when trying to free old
		 * buffers.
		 */
		struct list_head time_list;
		struct work_struct time_work;
		struct timer_list time_timer;
	} bo_cache;

116 117 118
	u32 num_labels;
	struct vc4_label {
		const char *name;
Eric Anholt's avatar
Eric Anholt committed
119 120
		u32 num_allocated;
		u32 size_allocated;
121
	} *bo_labels;
Eric Anholt's avatar
Eric Anholt committed
122

123
	/* Protects bo_cache and bo_labels. */
Eric Anholt's avatar
Eric Anholt committed
124
	struct mutex bo_lock;
125

126 127 128 129 130 131 132 133 134 135 136 137 138
	/* Purgeable BO pool. All BOs in this pool can have their memory
	 * reclaimed if the driver is unable to allocate new BOs. We also
	 * keep stats related to the purge mechanism here.
	 */
	struct {
		struct list_head list;
		unsigned int num;
		size_t size;
		unsigned int purged_num;
		size_t purged_size;
		struct mutex lock;
	} purgeable;

139 140
	uint64_t dma_fence_context;

141
	/* Sequence number for the last job queued in bin_job_list.
142 143 144 145 146 147 148 149 150
	 * Starts at 0 (no jobs emitted).
	 */
	uint64_t emit_seqno;

	/* Sequence number for the last completed job on the GPU.
	 * Starts at 0 (no jobs completed).
	 */
	uint64_t finished_seqno;

151 152 153
	/* List of all struct vc4_exec_info for jobs to be executed in
	 * the binner.  The first job in the list is the one currently
	 * programmed into ct0ca for execution.
154
	 */
155 156 157 158 159 160 161 162 163
	struct list_head bin_job_list;

	/* List of all struct vc4_exec_info for jobs that have
	 * completed binning and are ready for rendering.  The first
	 * job in the list is the one currently programmed into ct1ca
	 * for execution.
	 */
	struct list_head render_job_list;

164 165 166 167 168 169 170 171 172 173 174
	/* List of the finished vc4_exec_infos waiting to be freed by
	 * job_done_work.
	 */
	struct list_head job_done_list;
	/* Spinlock used to synchronize the job_list and seqno
	 * accesses between the IRQ handler and GEM ioctls.
	 */
	spinlock_t job_lock;
	wait_queue_head_t job_wait_queue;
	struct work_struct job_done_work;

175 176 177 178 179
	/* Used to track the active perfmon if any. Access to this field is
	 * protected by job_lock.
	 */
	struct vc4_perfmon *active_perfmon;

180 181 182 183 184
	/* List of struct vc4_seqno_cb for callbacks to be made from a
	 * workqueue when the given seqno is passed.
	 */
	struct list_head seqno_cb_list;

185 186 187
	/* The memory used for storing binner tile alloc, tile state,
	 * and overflow memory allocations.  This is freed when V3D
	 * powers down.
188
	 */
189 190 191 192 193 194 195 196 197 198 199 200 201
	struct vc4_bo *bin_bo;

	/* Size of blocks allocated within bin_bo. */
	uint32_t bin_alloc_size;

	/* Bitmask of the bin_alloc_size chunks in bin_bo that are
	 * used.
	 */
	uint32_t bin_alloc_used;

	/* Bitmask of the current bin_alloc used for overflow memory. */
	uint32_t bin_alloc_overflow;

202 203 204 205 206 207 208
	/* Incremented when an underrun error happened after an atomic commit.
	 * This is particularly useful to detect when a specific modeset is too
	 * demanding in term of memory or HVS bandwidth which is hard to guess
	 * at atomic check time.
	 */
	atomic_t underrun;

209 210
	struct work_struct overflow_mem_work;

211 212
	int power_refcount;

213 214 215
	/* Set to true when the load tracker is active. */
	bool load_tracker_enabled;

216 217 218
	/* Mutex controlling the power refcount. */
	struct mutex power_lock;

219 220 221 222 223
	struct {
		struct timer_list timer;
		struct work_struct reset_work;
	} hangcheck;

Stefan Schake's avatar
Stefan Schake committed
224 225
	struct drm_modeset_lock ctm_state_lock;
	struct drm_private_obj ctm_manager;
226
	struct drm_private_obj hvs_channels;
227
	struct drm_private_obj load_tracker;
228

229 230 231 232
	/* Mutex for binner bo allocation. */
	struct mutex bin_bo_lock;
	/* Reference count for our binner bo. */
	struct kref bin_bo_kref;
233 234 235
};

static inline struct vc4_dev *
236
to_vc4_dev(const struct drm_device *dev)
237
{
238
	return container_of(dev, struct vc4_dev, base);
239 240 241
}

struct vc4_bo {
242
	struct drm_gem_dma_object base;
Eric Anholt's avatar
Eric Anholt committed
243

244
	/* seqno of the last job to render using this BO. */
245 246
	uint64_t seqno;

247 248 249 250 251 252 253
	/* seqno of the last job to use the RCL to write to this BO.
	 *
	 * Note that this doesn't include binner overflow memory
	 * writes.
	 */
	uint64_t write_seqno;

254 255
	bool t_format;

Eric Anholt's avatar
Eric Anholt committed
256 257 258 259 260 261 262 263 264 265
	/* List entry for the BO's position in either
	 * vc4_exec_info->unref_list or vc4_dev->bo_cache.time_list
	 */
	struct list_head unref_head;

	/* Time in jiffies when the BO was put in vc4->bo_cache. */
	unsigned long free_time;

	/* List entry for the BO's position in vc4_dev->bo_cache.size_list */
	struct list_head size_head;
266 267 268 269 270

	/* Struct for shader validation state, if created by
	 * DRM_IOCTL_VC4_CREATE_SHADER_BO.
	 */
	struct vc4_validated_shader_info *validated_shader;
271

272 273 274 275
	/* One of enum vc4_kernel_bo_type, or VC4_BO_TYPE_COUNT + i
	 * for user-allocated labels.
	 */
	int label;
276 277 278 279 280 281 282 283 284 285

	/* Count the number of active users. This is needed to determine
	 * whether we can move the BO to the purgeable list or not (when the BO
	 * is used by the GPU or the display engine we can't purge it).
	 */
	refcount_t usecnt;

	/* Store purgeable/purged state here */
	u32 madv;
	struct mutex madv_lock;
286 287 288
};

static inline struct vc4_bo *
289
to_vc4_bo(const struct drm_gem_object *bo)
290
{
291
	return container_of(to_drm_gem_dma_obj(bo), struct vc4_bo, base);
292 293
}

294 295 296 297 298 299 300 301
struct vc4_fence {
	struct dma_fence base;
	struct drm_device *dev;
	/* vc4 seqno for signaled() test */
	uint64_t seqno;
};

static inline struct vc4_fence *
302
to_vc4_fence(const struct dma_fence *fence)
303
{
304
	return container_of(fence, struct vc4_fence, base);
305 306
}

307 308 309 310 311 312
struct vc4_seqno_cb {
	struct work_struct work;
	uint64_t seqno;
	void (*func)(struct vc4_seqno_cb *cb);
};

313
struct vc4_v3d {
Eric Anholt's avatar
Eric Anholt committed
314
	struct vc4_dev *vc4;
315 316
	struct platform_device *pdev;
	void __iomem *regs;
317
	struct clk *clk;
318
	struct debugfs_regset32 regset;
319 320
};

321
struct vc4_hvs {
322
	struct vc4_dev *vc4;
323 324
	struct platform_device *pdev;
	void __iomem *regs;
325 326
	u32 __iomem *dlist;

327 328
	struct clk *core_clk;

329 330
	unsigned long max_core_rate;

331 332 333 334
	/* Memory manager for CRTCs to allocate space in the display
	 * list.  Units are dwords.
	 */
	struct drm_mm dlist_mm;
335 336
	/* Memory manager for the LBM memory used by HVS scaling. */
	struct drm_mm lbm_mm;
337
	spinlock_t mm_lock;
338 339

	struct drm_mm_node mitchell_netravali_filter;
340

341
	struct debugfs_regset32 regset;
342 343 344 345 346 347 348 349

	/*
	 * Even if HDMI0 on the RPi4 can output modes requiring a pixel
	 * rate higher than 297MHz, it needs some adjustments in the
	 * config.txt file to be able to do so and thus won't always be
	 * available.
	 */
	bool vc5_hdmi_enable_hdmi_20;
350 351 352 353 354 355

	/*
	 * 4096x2160@60 requires a core overclock to work, so register
	 * whether that is sufficient.
	 */
	bool vc5_hdmi_enable_4096by2160;
356 357
};

358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380
#define HVS_NUM_CHANNELS 3

struct vc4_hvs_state {
	struct drm_private_state base;
	unsigned long core_clock_rate;

	struct {
		unsigned in_use: 1;
		unsigned long fifo_load;
		struct drm_crtc_commit *pending_commit;
	} fifo_state[HVS_NUM_CHANNELS];
};

static inline struct vc4_hvs_state *
to_vc4_hvs_state(const struct drm_private_state *priv)
{
	return container_of(priv, struct vc4_hvs_state, base);
}

struct vc4_hvs_state *vc4_hvs_get_global_state(struct drm_atomic_state *state);
struct vc4_hvs_state *vc4_hvs_get_old_global_state(const struct drm_atomic_state *state);
struct vc4_hvs_state *vc4_hvs_get_new_global_state(const struct drm_atomic_state *state);

381 382 383 384 385
struct vc4_plane {
	struct drm_plane base;
};

static inline struct vc4_plane *
386
to_vc4_plane(const struct drm_plane *plane)
387
{
388
	return container_of(plane, struct vc4_plane, base);
389 390
}

391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411
enum vc4_scaling_mode {
	VC4_SCALING_NONE,
	VC4_SCALING_TPZ,
	VC4_SCALING_PPF,
};

struct vc4_plane_state {
	struct drm_plane_state base;
	/* System memory copy of the display list for this element, computed
	 * at atomic_check time.
	 */
	u32 *dlist;
	u32 dlist_size; /* Number of dwords allocated for the display list */
	u32 dlist_count; /* Number of used dwords in the display list. */

	/* Offset in the dlist to various words, for pageflip or
	 * cursor updates.
	 */
	u32 pos0_offset;
	u32 pos2_offset;
	u32 ptr0_offset;
412
	u32 lbm_offset;
413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443

	/* Offset where the plane's dlist was last stored in the
	 * hardware at vc4_crtc_atomic_flush() time.
	 */
	u32 __iomem *hw_dlist;

	/* Clipped coordinates of the plane on the display. */
	int crtc_x, crtc_y, crtc_w, crtc_h;
	/* Clipped area being scanned from in the FB. */
	u32 src_x, src_y;

	u32 src_w[2], src_h[2];

	/* Scaling selection for the RGB/Y plane and the Cb/Cr planes. */
	enum vc4_scaling_mode x_scaling[2], y_scaling[2];
	bool is_unity;
	bool is_yuv;

	/* Offset to start scanning out from the start of the plane's
	 * BO.
	 */
	u32 offsets[3];

	/* Our allocation in LBM for temporary storage during scaling. */
	struct drm_mm_node lbm;

	/* Set when the plane has per-pixel alpha content or does not cover
	 * the entire screen. This is a hint to the CRTC that it might need
	 * to enable background color fill.
	 */
	bool needs_bg_fill;
444 445 446 447 448

	/* Mark the dlist as initialized. Useful to avoid initializing it twice
	 * when async update is not possible.
	 */
	bool dlist_initialized;
449 450 451 452 453 454 455 456 457 458

	/* Load of this plane on the HVS block. The load is expressed in HVS
	 * cycles/sec.
	 */
	u64 hvs_load;

	/* Memory bandwidth needed for this plane. This is expressed in
	 * bytes/sec.
	 */
	u64 membus_load;
459 460 461
};

static inline struct vc4_plane_state *
462
to_vc4_plane_state(const struct drm_plane_state *state)
463
{
464
	return container_of(state, struct vc4_plane_state, base);
465 466
}

467
enum vc4_encoder_type {
468
	VC4_ENCODER_TYPE_NONE,
469
	VC4_ENCODER_TYPE_HDMI0,
470
	VC4_ENCODER_TYPE_HDMI1,
471 472 473 474 475
	VC4_ENCODER_TYPE_VEC,
	VC4_ENCODER_TYPE_DSI0,
	VC4_ENCODER_TYPE_DSI1,
	VC4_ENCODER_TYPE_SMI,
	VC4_ENCODER_TYPE_DPI,
476
	VC4_ENCODER_TYPE_TXP,
477 478 479 480 481 482
};

struct vc4_encoder {
	struct drm_encoder base;
	enum vc4_encoder_type type;
	u32 clock_select;
483

484 485 486
	void (*pre_crtc_configure)(struct drm_encoder *encoder, struct drm_atomic_state *state);
	void (*pre_crtc_enable)(struct drm_encoder *encoder, struct drm_atomic_state *state);
	void (*post_crtc_enable)(struct drm_encoder *encoder, struct drm_atomic_state *state);
487

488 489
	void (*post_crtc_disable)(struct drm_encoder *encoder, struct drm_atomic_state *state);
	void (*post_crtc_powerdown)(struct drm_encoder *encoder, struct drm_atomic_state *state);
490 491 492
};

static inline struct vc4_encoder *
493
to_vc4_encoder(const struct drm_encoder *encoder)
494 495 496 497
{
	return container_of(encoder, struct vc4_encoder, base);
}

498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
static inline
struct drm_encoder *vc4_find_encoder_by_type(struct drm_device *drm,
					     enum vc4_encoder_type type)
{
	struct drm_encoder *encoder;

	drm_for_each_encoder(encoder, drm) {
		struct vc4_encoder *vc4_encoder = to_vc4_encoder(encoder);

		if (vc4_encoder->type == type)
			return encoder;
	}

	return NULL;
}

514
struct vc4_crtc_data {
515 516
	const char *name;

517 518
	const char *debugfs_name;

519 520 521
	/* Bitmask of channels (FIFOs) of the HVS that the output can source from */
	unsigned int hvs_available_channels;

522 523
	/* Which output of the HVS this pixelvalve sources from. */
	int hvs_output;
524 525
};

526 527
extern const struct vc4_crtc_data vc4_txp_crtc_data;

528 529
struct vc4_pv_data {
	struct vc4_crtc_data	base;
530

531 532 533
	/* Depth of the PixelValve FIFO in bytes */
	unsigned int fifo_depth;

534 535 536
	/* Number of pixels output per clock period */
	u8 pixels_per_clock;

537 538 539
	enum vc4_encoder_type encoder_types[4];
};

540 541 542 543 544 545 546 547 548
extern const struct vc4_pv_data bcm2835_pv0_data;
extern const struct vc4_pv_data bcm2835_pv1_data;
extern const struct vc4_pv_data bcm2835_pv2_data;
extern const struct vc4_pv_data bcm2711_pv0_data;
extern const struct vc4_pv_data bcm2711_pv1_data;
extern const struct vc4_pv_data bcm2711_pv2_data;
extern const struct vc4_pv_data bcm2711_pv3_data;
extern const struct vc4_pv_data bcm2711_pv4_data;

549 550
struct vc4_crtc {
	struct drm_crtc base;
551
	struct platform_device *pdev;
552 553 554 555 556 557 558 559 560 561 562
	const struct vc4_crtc_data *data;
	void __iomem *regs;

	/* Timestamp at start of vblank irq - unaffected by lock delays. */
	ktime_t t_vblank;

	u8 lut_r[256];
	u8 lut_g[256];
	u8 lut_b[256];

	struct drm_pending_vblank_event *event;
563 564

	struct debugfs_regset32 regset;
565 566 567 568 569

	/**
	 * @feeds_txp: True if the CRTC feeds our writeback controller.
	 */
	bool feeds_txp;
570 571 572 573 574 575 576 577 578 579 580 581 582 583

	/**
	 * @irq_lock: Spinlock protecting the resources shared between
	 * the atomic code and our vblank handler.
	 */
	spinlock_t irq_lock;

	/**
	 * @current_dlist: Start offset of the display list currently
	 * set in the HVS for that CRTC. Protected by @irq_lock, and
	 * copied in vc4_hvs_update_dlist() for the CRTC interrupt
	 * handler to have access to that value.
	 */
	unsigned int current_dlist;
584 585 586 587 588 589 590 591

	/**
	 * @current_hvs_channel: HVS channel currently assigned to the
	 * CRTC. Protected by @irq_lock, and copied in
	 * vc4_hvs_atomic_begin() for the CRTC interrupt handler to have
	 * access to that value.
	 */
	unsigned int current_hvs_channel;
592 593 594
};

static inline struct vc4_crtc *
595
to_vc4_crtc(const struct drm_crtc *crtc)
596
{
597
	return container_of(crtc, struct vc4_crtc, base);
598 599
}

600 601 602 603 604 605 606 607 608 609 610 611 612 613
static inline const struct vc4_crtc_data *
vc4_crtc_to_vc4_crtc_data(const struct vc4_crtc *crtc)
{
	return crtc->data;
}

static inline const struct vc4_pv_data *
vc4_crtc_to_vc4_pv_data(const struct vc4_crtc *crtc)
{
	const struct vc4_crtc_data *data = vc4_crtc_to_vc4_crtc_data(crtc);

	return container_of(data, struct vc4_pv_data, base);
}

614
struct drm_encoder *vc4_get_crtc_encoder(struct drm_crtc *crtc,
615
					 struct drm_crtc_state *state);
616

617 618 619 620 621
struct vc4_crtc_state {
	struct drm_crtc_state base;
	/* Dlist area for this CRTC configuration. */
	struct drm_mm_node mm;
	bool txp_armed;
622
	unsigned int assigned_channel;
623 624 625 626 627 628 629

	struct {
		unsigned int left;
		unsigned int right;
		unsigned int top;
		unsigned int bottom;
	} margins;
630

631 632
	unsigned long hvs_load;

633 634
	/* Transitional state below, only valid during atomic commits */
	bool update_muxing;
635 636
};

637 638
#define VC4_HVS_CHANNEL_DISABLED ((unsigned int)-1)

639
static inline struct vc4_crtc_state *
640
to_vc4_crtc_state(const struct drm_crtc_state *crtc_state)
641
{
642
	return container_of(crtc_state, struct vc4_crtc_state, base);
643 644
}

645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667
#define V3D_READ(offset)								\
	({										\
		kunit_fail_current_test("Accessing a register in a unit test!\n");	\
		readl(vc4->v3d->regs + (offset));						\
	})

#define V3D_WRITE(offset, val)								\
	do {										\
		kunit_fail_current_test("Accessing a register in a unit test!\n");	\
		writel(val, vc4->v3d->regs + (offset));					\
	} while (0)

#define HVS_READ(offset)								\
	({										\
		kunit_fail_current_test("Accessing a register in a unit test!\n");	\
		readl(hvs->regs + (offset));						\
	})

#define HVS_WRITE(offset, val)								\
	do {										\
		kunit_fail_current_test("Accessing a register in a unit test!\n");	\
		writel(val, hvs->regs + (offset));					\
	} while (0)
668

669 670
#define VC4_REG32(reg) { .name = #reg, .offset = reg }

671
struct vc4_exec_info {
672 673
	struct vc4_dev *dev;

674 675 676
	/* Sequence number for this bin/render job. */
	uint64_t seqno;

677 678 679
	/* Latest write_seqno of any BO that binning depends on. */
	uint64_t bin_dep_seqno;

680 681
	struct dma_fence *fence;

682 683 684 685 686
	/* Last current addresses the hardware was processing when the
	 * hangcheck timer checked on us.
	 */
	uint32_t last_ct0ca, last_ct1ca;

687 688 689 690 691 692
	/* Kernel-space copy of the ioctl arguments */
	struct drm_vc4_submit_cl *args;

	/* This is the array of BOs that were looked up at the start of exec.
	 * Command validation will use indices into this array.
	 */
693
	struct drm_gem_object **bo;
694 695
	uint32_t bo_count;

696 697 698 699
	/* List of BOs that are being written by the RCL.  Other than
	 * the binner temporary storage, this is all the BOs written
	 * by the job.
	 */
700
	struct drm_gem_dma_object *rcl_write_bo[4];
701 702
	uint32_t rcl_write_bo_count;

703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718
	/* Pointers for our position in vc4->job_list */
	struct list_head head;

	/* List of other BOs used in the job that need to be released
	 * once the job is complete.
	 */
	struct list_head unref_list;

	/* Current unvalidated indices into @bo loaded by the non-hardware
	 * VC4_PACKET_GEM_HANDLES.
	 */
	uint32_t bo_index[2];

	/* This is the BO where we store the validated command lists, shader
	 * records, and uniforms.
	 */
719
	struct drm_gem_dma_object *exec_bo;
720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744

	/**
	 * This tracks the per-shader-record state (packet 64) that
	 * determines the length of the shader record and the offset
	 * it's expected to be found at.  It gets read in from the
	 * command lists.
	 */
	struct vc4_shader_state {
		uint32_t addr;
		/* Maximum vertex index referenced by any primitive using this
		 * shader state.
		 */
		uint32_t max_index;
	} *shader_state;

	/** How many shader states the user declared they were using. */
	uint32_t shader_state_size;
	/** How many shader state records the validator has seen. */
	uint32_t shader_state_count;

	bool found_tile_binning_mode_config_packet;
	bool found_start_tile_binning_packet;
	bool found_increment_semaphore_packet;
	bool found_flush;
	uint8_t bin_tiles_x, bin_tiles_y;
745 746 747
	/* Physical address of the start of the tile alloc array
	 * (where each tile's binned CL will start)
	 */
748
	uint32_t tile_alloc_offset;
749 750
	/* Bitmask of which binner slots are freed when this job completes. */
	uint32_t bin_slots;
751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778

	/**
	 * Computed addresses pointing into exec_bo where we start the
	 * bin thread (ct0) and render thread (ct1).
	 */
	uint32_t ct0ca, ct0ea;
	uint32_t ct1ca, ct1ea;

	/* Pointer to the unvalidated bin CL (if present). */
	void *bin_u;

	/* Pointers to the shader recs.  These paddr gets incremented as CL
	 * packets are relocated in validate_gl_shader_state, and the vaddrs
	 * (u and v) get incremented and size decremented as the shader recs
	 * themselves are validated.
	 */
	void *shader_rec_u;
	void *shader_rec_v;
	uint32_t shader_rec_p;
	uint32_t shader_rec_size;

	/* Pointers to the uniform data.  These pointers are incremented, and
	 * size decremented, as each batch of uniforms is uploaded.
	 */
	void *uniforms_u;
	void *uniforms_v;
	uint32_t uniforms_p;
	uint32_t uniforms_size;
779 780 781 782 783

	/* Pointer to a performance monitor object if the user requested it,
	 * NULL otherwise.
	 */
	struct vc4_perfmon *perfmon;
784 785 786 787 788

	/* Whether the exec has taken a reference to the binner BO, which should
	 * happen with a VC4_PACKET_TILE_BINNING_MODE_CONFIG packet.
	 */
	bool bin_bo_used;
789 790 791 792 793 794
};

/* Per-open file private data. Any driver-specific resource that has to be
 * released when the DRM file is closed should be placed here.
 */
struct vc4_file {
795 796
	struct vc4_dev *dev;

797 798 799 800
	struct {
		struct idr idr;
		struct mutex lock;
	} perfmon;
801 802

	bool bin_bo_used;
803 804 805
};

static inline struct vc4_exec_info *
806 807
vc4_first_bin_job(struct vc4_dev *vc4)
{
808 809
	return list_first_entry_or_null(&vc4->bin_job_list,
					struct vc4_exec_info, head);
810 811 812 813
}

static inline struct vc4_exec_info *
vc4_first_render_job(struct vc4_dev *vc4)
814
{
815 816
	return list_first_entry_or_null(&vc4->render_job_list,
					struct vc4_exec_info, head);
817 818 819 820 821 822 823 824 825
}

static inline struct vc4_exec_info *
vc4_last_render_job(struct vc4_dev *vc4)
{
	if (list_empty(&vc4->render_job_list))
		return NULL;
	return list_last_entry(&vc4->render_job_list,
			       struct vc4_exec_info, head);
826 827
}

828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861
/**
 * struct vc4_texture_sample_info - saves the offsets into the UBO for texture
 * setup parameters.
 *
 * This will be used at draw time to relocate the reference to the texture
 * contents in p0, and validate that the offset combined with
 * width/height/stride/etc. from p1 and p2/p3 doesn't sample outside the BO.
 * Note that the hardware treats unprovided config parameters as 0, so not all
 * of them need to be set up for every texure sample, and we'll store ~0 as
 * the offset to mark the unused ones.
 *
 * See the VC4 3D architecture guide page 41 ("Texture and Memory Lookup Unit
 * Setup") for definitions of the texture parameters.
 */
struct vc4_texture_sample_info {
	bool is_direct;
	uint32_t p_offset[4];
};

/**
 * struct vc4_validated_shader_info - information about validated shaders that
 * needs to be used from command list validation.
 *
 * For a given shader, each time a shader state record references it, we need
 * to verify that the shader doesn't read more uniforms than the shader state
 * record's uniform BO pointer can provide, and we need to apply relocations
 * and validate the shader state record's uniforms that define the texture
 * samples.
 */
struct vc4_validated_shader_info {
	uint32_t uniforms_size;
	uint32_t uniforms_src_size;
	uint32_t num_texture_samples;
	struct vc4_texture_sample_info *texture_samples;
862 863 864

	uint32_t num_uniform_addr_offsets;
	uint32_t *uniform_addr_offsets;
865 866

	bool is_threaded;
867 868
};

869
/**
870
 * __wait_for - magic wait macro
871
 *
872 873 874 875
 * Macro to help avoid open coding check/wait/timeout patterns. Note that it's
 * important that we check the condition again after having timed out, since the
 * timeout could be due to preemption or similar and we've never had a chance to
 * check the condition before the timeout.
876
 */
877 878 879 880 881 882 883 884 885 886 887 888
#define __wait_for(OP, COND, US, Wmin, Wmax) ({ \
	const ktime_t end__ = ktime_add_ns(ktime_get_raw(), 1000ll * (US)); \
	long wait__ = (Wmin); /* recommended min for usleep is 10 us */	\
	int ret__;							\
	might_sleep();							\
	for (;;) {							\
		const bool expired__ = ktime_after(ktime_get_raw(), end__); \
		OP;							\
		/* Guarantee COND check prior to timeout */		\
		barrier();						\
		if (COND) {						\
			ret__ = 0;					\
889 890
			break;						\
		}							\
891 892 893
		if (expired__) {					\
			ret__ = -ETIMEDOUT;				\
			break;						\
894
		}							\
895 896 897
		usleep_range(wait__, wait__ * 2);			\
		if (wait__ < (Wmax))					\
			wait__ <<= 1;					\
898 899 900 901
	}								\
	ret__;								\
})

902 903 904
#define _wait_for(COND, US, Wmin, Wmax)	__wait_for(, (COND), (US), (Wmin), \
						   (Wmax))
#define wait_for(COND, MS)		_wait_for((COND), (MS) * 1000, 10, 1000)
905 906

/* vc4_bo.c */
Eric Anholt's avatar
Eric Anholt committed
907 908
struct drm_gem_object *vc4_create_object(struct drm_device *dev, size_t size);
struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t size,
909
			     bool from_cache, enum vc4_kernel_bo_type type);
910 911 912
int vc4_bo_dumb_create(struct drm_file *file_priv,
		       struct drm_device *dev,
		       struct drm_mode_create_dumb *args);
913 914
int vc4_create_bo_ioctl(struct drm_device *dev, void *data,
			struct drm_file *file_priv);
915 916
int vc4_create_shader_bo_ioctl(struct drm_device *dev, void *data,
			       struct drm_file *file_priv);
917 918
int vc4_mmap_bo_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file_priv);
919 920 921 922
int vc4_set_tiling_ioctl(struct drm_device *dev, void *data,
			 struct drm_file *file_priv);
int vc4_get_tiling_ioctl(struct drm_device *dev, void *data,
			 struct drm_file *file_priv);
923 924
int vc4_get_hang_state_ioctl(struct drm_device *dev, void *data,
			     struct drm_file *file_priv);
925 926 927
int vc4_label_bo_ioctl(struct drm_device *dev, void *data,
		       struct drm_file *file_priv);
int vc4_bo_cache_init(struct drm_device *dev);
928 929 930 931
int vc4_bo_inc_usecnt(struct vc4_bo *bo);
void vc4_bo_dec_usecnt(struct vc4_bo *bo);
void vc4_bo_add_to_purgeable_pool(struct vc4_bo *bo);
void vc4_bo_remove_from_purgeable_pool(struct vc4_bo *bo);
932
int vc4_bo_debugfs_init(struct drm_minor *minor);
933 934 935

/* vc4_crtc.c */
extern struct platform_driver vc4_crtc_driver;
936
int vc4_crtc_disable_at_boot(struct drm_crtc *crtc);
937 938 939 940 941 942
int __vc4_crtc_init(struct drm_device *drm, struct platform_device *pdev,
		    struct vc4_crtc *vc4_crtc, const struct vc4_crtc_data *data,
		    struct drm_plane *primary_plane,
		    const struct drm_crtc_funcs *crtc_funcs,
		    const struct drm_crtc_helper_funcs *crtc_helper_funcs,
		    bool feeds_txp);
943 944
int vc4_crtc_init(struct drm_device *drm, struct platform_device *pdev,
		  struct vc4_crtc *vc4_crtc, const struct vc4_crtc_data *data,
945
		  const struct drm_crtc_funcs *crtc_funcs,
946 947
		  const struct drm_crtc_helper_funcs *crtc_helper_funcs,
		  bool feeds_txp);
948 949 950 951 952
int vc4_page_flip(struct drm_crtc *crtc,
		  struct drm_framebuffer *fb,
		  struct drm_pending_vblank_event *event,
		  uint32_t flags,
		  struct drm_modeset_acquire_ctx *ctx);
953 954
int vc4_crtc_atomic_check(struct drm_crtc *crtc,
			  struct drm_atomic_state *state);
955 956 957 958
struct drm_crtc_state *vc4_crtc_duplicate_state(struct drm_crtc *crtc);
void vc4_crtc_destroy_state(struct drm_crtc *crtc,
			    struct drm_crtc_state *state);
void vc4_crtc_reset(struct drm_crtc *crtc);
959
void vc4_crtc_handle_vblank(struct vc4_crtc *crtc);
960
void vc4_crtc_send_vblank(struct drm_crtc *crtc);
961
int vc4_crtc_late_register(struct drm_crtc *crtc);
962
void vc4_crtc_get_margins(struct drm_crtc_state *state,
963
			  unsigned int *left, unsigned int *right,
964
			  unsigned int *top, unsigned int *bottom);
965 966

/* vc4_debugfs.c */
967
void vc4_debugfs_init(struct drm_minor *minor);
968
#ifdef CONFIG_DEBUG_FS
969 970 971
void vc4_debugfs_add_regset32(struct drm_device *drm,
			      const char *filename,
			      struct debugfs_regset32 *regset);
972 973
#else

974 975 976 977
static inline void vc4_debugfs_add_regset32(struct drm_device *drm,
					    const char *filename,
					    struct debugfs_regset32 *regset)
{}
978
#endif
979 980 981

/* vc4_drv.c */
void __iomem *vc4_ioremap_regs(struct platform_device *dev, int index);
982
int vc4_dumb_fixup_args(struct drm_mode_create_dumb *args);
983

Eric Anholt's avatar
Eric Anholt committed
984 985 986
/* vc4_dpi.c */
extern struct platform_driver vc4_dpi_driver;

Eric Anholt's avatar
Eric Anholt committed
987 988 989
/* vc4_dsi.c */
extern struct platform_driver vc4_dsi_driver;

990 991 992
/* vc4_fence.c */
extern const struct dma_fence_ops vc4_fence_ops;

993
/* vc4_gem.c */
994
int vc4_gem_init(struct drm_device *dev);
995 996 997 998 999 1000
int vc4_submit_cl_ioctl(struct drm_device *dev, void *data,
			struct drm_file *file_priv);
int vc4_wait_seqno_ioctl(struct drm_device *dev, void *data,
			 struct drm_file *file_priv);
int vc4_wait_bo_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file_priv);
1001 1002 1003
void vc4_submit_next_bin_job(struct drm_device *dev);
void vc4_submit_next_render_job(struct drm_device *dev);
void vc4_move_job_to_render(struct drm_device *dev, struct vc4_exec_info *exec);
1004 1005 1006
int vc4_wait_for_seqno(struct drm_device *dev, uint64_t seqno,
		       uint64_t timeout_ns, bool interruptible);
void vc4_job_handle_completed(struct vc4_dev *vc4);
1007 1008 1009
int vc4_queue_seqno_cb(struct drm_device *dev,
		       struct vc4_seqno_cb *cb, uint64_t seqno,
		       void (*func)(struct vc4_seqno_cb *cb));
1010 1011
int vc4_gem_madvise_ioctl(struct drm_device *dev, void *data,
			  struct drm_file *file_priv);
1012

1013 1014 1015
/* vc4_hdmi.c */
extern struct platform_driver vc4_hdmi_driver;

1016
/* vc4_vec.c */
1017 1018
extern struct platform_driver vc4_vec_driver;

1019 1020 1021
/* vc4_txp.c */
extern struct platform_driver vc4_txp_driver;

1022
/* vc4_irq.c */
1023 1024 1025
void vc4_irq_enable(struct drm_device *dev);
void vc4_irq_disable(struct drm_device *dev);
int vc4_irq_install(struct drm_device *dev, int irq);
1026 1027 1028
void vc4_irq_uninstall(struct drm_device *dev);
void vc4_irq_reset(struct drm_device *dev);

1029 1030
/* vc4_hvs.c */
extern struct platform_driver vc4_hvs_driver;
1031
struct vc4_hvs *__vc4_hvs_alloc(struct vc4_dev *vc4, struct platform_device *pdev);
1032 1033 1034
void vc4_hvs_stop_channel(struct vc4_hvs *hvs, unsigned int output);
int vc4_hvs_get_fifo_from_output(struct vc4_hvs *hvs, unsigned int output);
u8 vc4_hvs_get_fifo_frame_count(struct vc4_hvs *hvs, unsigned int fifo);
1035
int vc4_hvs_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state);
1036
void vc4_hvs_atomic_begin(struct drm_crtc *crtc, struct drm_atomic_state *state);
1037 1038 1039
void vc4_hvs_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_state *state);
void vc4_hvs_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_state *state);
void vc4_hvs_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_state *state);
1040 1041 1042
void vc4_hvs_dump_state(struct vc4_hvs *hvs);
void vc4_hvs_unmask_underrun(struct vc4_hvs *hvs, int channel);
void vc4_hvs_mask_underrun(struct vc4_hvs *hvs, int channel);
1043
int vc4_hvs_debugfs_init(struct drm_minor *minor);
1044 1045 1046 1047 1048 1049

/* vc4_kms.c */
int vc4_kms_load(struct drm_device *dev);

/* vc4_plane.c */
struct drm_plane *vc4_plane_init(struct drm_device *dev,
1050 1051
				 enum drm_plane_type type,
				 uint32_t possible_crtcs);
1052
int vc4_plane_create_additional_planes(struct drm_device *dev);
1053
u32 vc4_plane_write_dlist(struct drm_plane *plane, u32 __iomem *dlist);
1054
u32 vc4_plane_dlist_size(const struct drm_plane_state *state);
1055 1056
void vc4_plane_async_set_fb(struct drm_plane *plane,
			    struct drm_framebuffer *fb);
1057

1058 1059
/* vc4_v3d.c */
extern struct platform_driver vc4_v3d_driver;
1060
extern const struct of_device_id vc4_v3d_dt_match[];
1061
int vc4_v3d_get_bin_slot(struct vc4_dev *vc4);
1062 1063
int vc4_v3d_bin_bo_get(struct vc4_dev *vc4, bool *used);
void vc4_v3d_bin_bo_put(struct vc4_dev *vc4);
1064 1065
int vc4_v3d_pm_get(struct vc4_dev *vc4);
void vc4_v3d_pm_put(struct vc4_dev *vc4);
1066
int vc4_v3d_debugfs_init(struct drm_minor *minor);
1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077

/* vc4_validate.c */
int
vc4_validate_bin_cl(struct drm_device *dev,
		    void *validated,
		    void *unvalidated,
		    struct vc4_exec_info *exec);

int
vc4_validate_shader_recs(struct drm_device *dev, struct vc4_exec_info *exec);

1078
struct drm_gem_dma_object *vc4_use_bo(struct vc4_exec_info *exec,
1079 1080 1081 1082 1083
				      uint32_t hindex);

int vc4_get_rcl(struct drm_device *dev, struct vc4_exec_info *exec);

bool vc4_check_tex_size(struct vc4_exec_info *exec,
1084
			struct drm_gem_dma_object *fbo,
1085 1086
			uint32_t offset, uint8_t tiling_format,
			uint32_t width, uint32_t height, uint8_t cpp);
1087

1088 1089
/* vc4_validate_shader.c */
struct vc4_validated_shader_info *
1090
vc4_validate_shader(struct drm_gem_dma_object *shader_obj);
1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106

/* vc4_perfmon.c */
void vc4_perfmon_get(struct vc4_perfmon *perfmon);
void vc4_perfmon_put(struct vc4_perfmon *perfmon);
void vc4_perfmon_start(struct vc4_dev *vc4, struct vc4_perfmon *perfmon);
void vc4_perfmon_stop(struct vc4_dev *vc4, struct vc4_perfmon *perfmon,
		      bool capture);
struct vc4_perfmon *vc4_perfmon_find(struct vc4_file *vc4file, int id);
void vc4_perfmon_open_file(struct vc4_file *vc4file);
void vc4_perfmon_close_file(struct vc4_file *vc4file);
int vc4_perfmon_create_ioctl(struct drm_device *dev, void *data,
			     struct drm_file *file_priv);
int vc4_perfmon_destroy_ioctl(struct drm_device *dev, void *data,
			      struct drm_file *file_priv);
int vc4_perfmon_get_values_ioctl(struct drm_device *dev, void *data,
				 struct drm_file *file_priv);
1107 1108

#endif /* _VC4_DRV_H_ */