media: intel/ipu6: support line-based metadata capture support

Some camera sensor can output the embedded data in specific data type.
This patch adds the support for metadata capture in IPU6 ISYS driver.
Signed-off-by: Hongju Wang <hongju.wang@intel.com>
Signed-off-by: Bingbu Cao <bingbu.cao@intel.com>
Co-developed-by: Sakari Ailus <sakari.ailus@linux.intel.com>
Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>

drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c

@@ -42,6 +42,11 @@ static const u32 csi2_supported_codes[] = {
 	MEDIA_BUS_FMT_SGBRG8_1X8,
 	MEDIA_BUS_FMT_SGRBG8_1X8,
 	MEDIA_BUS_FMT_SRGGB8_1X8,
+	MEDIA_BUS_FMT_META_8,
+	MEDIA_BUS_FMT_META_10,
+	MEDIA_BUS_FMT_META_12,
+	MEDIA_BUS_FMT_META_16,
+	MEDIA_BUS_FMT_META_24,
 	0
 };
 

drivers/media/pci/intel/ipu6/ipu6-isys-queue.c

@@ -28,7 +28,7 @@ static int queue_setup(struct vb2_queue *q, unsigned int *num_buffers,
 	struct ipu6_isys_queue *aq = vb2_queue_to_isys_queue(q);
 	struct ipu6_isys_video *av = ipu6_isys_queue_to_video(aq);
 	struct device *dev = &av->isys->adev->auxdev.dev;
-	u32 size = av->pix_fmt.sizeimage;
+	u32 size = ipu6_isys_get_data_size(av);
 
 	/* num_planes == 0: we're being called through VIDIOC_REQBUFS */
 	if (!*num_planes) {
@@ -49,17 +49,17 @@ static int ipu6_isys_buf_prepare(struct vb2_buffer *vb)
 	struct ipu6_isys_queue *aq = vb2_queue_to_isys_queue(vb->vb2_queue);
 	struct ipu6_isys_video *av = ipu6_isys_queue_to_video(aq);
 	struct device *dev = &av->isys->adev->auxdev.dev;
+	u32 bytesperline = ipu6_isys_get_bytes_per_line(av);
+	u32 height = ipu6_isys_get_frame_height(av);
+	u32 size = ipu6_isys_get_data_size(av);
 
 	dev_dbg(dev, "buffer: %s: configured size %u, buffer size %lu\n",
-		av->vdev.name, av->pix_fmt.sizeimage,
-		vb2_plane_size(vb, 0));
+		av->vdev.name, size, vb2_plane_size(vb, 0));
 
-	if (av->pix_fmt.sizeimage > vb2_plane_size(vb, 0))
+	if (size > vb2_plane_size(vb, 0))
 		return -EINVAL;
 
-	vb2_set_plane_payload(vb, 0, av->pix_fmt.bytesperline *
-			      av->pix_fmt.height);
-	vb->planes[0].data_offset = 0;
+	vb2_set_plane_payload(vb, 0, bytesperline * height);
 
 	return 0;
 }
@@ -328,15 +328,12 @@ static void buf_queue(struct vb2_buffer *vb)
 	struct isys_fw_msgs *msg;
 	unsigned long flags;
 	dma_addr_t dma;
-	unsigned int i;
 	int ret;
 
 	dev_dbg(dev, "queue buffer %u for %s\n", vb->index, av->vdev.name);
 
-	for (i = 0; i < vb->num_planes; i++) {
-		dma = vb2_dma_contig_plane_dma_addr(vb, i);
-		dev_dbg(dev, "iova: plane %u iova %pad\n", i, &dma);
-	}
+	dma = vb2_dma_contig_plane_dma_addr(vb, 0);
+	dev_dbg(dev, "iova: iova %pad\n", &dma);
 
 	spin_lock_irqsave(&aq->lock, flags);
 	list_add(&ib->head, &aq->incoming);
@@ -409,7 +406,7 @@ static int ipu6_isys_link_fmt_validate(struct ipu6_isys_queue *aq)
 	struct media_pad *remote_pad =
 		media_pad_remote_pad_first(av->vdev.entity.pads);
 	struct v4l2_subdev *sd;
-	u32 r_stream;
+	u32 r_stream, code;
 	int ret;
 
 	if (!remote_pad)
@@ -427,17 +424,19 @@ static int ipu6_isys_link_fmt_validate(struct ipu6_isys_queue *aq)
 		return ret;
 	}
 
-	if (format.width != av->pix_fmt.width ||
-	    format.height != av->pix_fmt.height) {
-		dev_dbg(dev, "wrong width or height %ux%u (%ux%u expected)\n",
-			av->pix_fmt.width, av->pix_fmt.height,
-			format.width, format.height);
+	if (format.width != ipu6_isys_get_frame_width(av) ||
+	    format.height != ipu6_isys_get_frame_height(av)) {
+		dev_err(dev, "wrong width or height %ux%u (%ux%u expected)\n",
+			ipu6_isys_get_frame_width(av),
+			ipu6_isys_get_frame_height(av), format.width,
+			format.height);
 		return -EINVAL;
 	}
 
-	if (format.code != av->pfmt->code) {
+	code = ipu6_isys_get_isys_format(ipu6_isys_get_format(av), 0)->code;
+	if (format.code != code) {
 		dev_dbg(dev, "wrong mbus code 0x%8.8x (0x%8.8x expected)\n",
-			av->pfmt->code, format.code);
+			code, format.code);
 		return -EINVAL;
 	}
 
@@ -509,14 +508,16 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 	struct ipu6_isys_queue *aq = vb2_queue_to_isys_queue(q);
 	struct ipu6_isys_video *av = ipu6_isys_queue_to_video(aq);
 	struct device *dev = &av->isys->adev->auxdev.dev;
+	const struct ipu6_isys_pixelformat *pfmt =
+		ipu6_isys_get_isys_format(ipu6_isys_get_format(av), 0);
 	struct ipu6_isys_buffer_list __bl, *bl = NULL;
 	struct ipu6_isys_stream *stream;
 	struct media_entity *source_entity = NULL;
 	int nr_queues, ret;
 
 	dev_dbg(dev, "stream: %s: width %u, height %u, css pixelformat %u\n",
-		av->vdev.name, av->pix_fmt.width, av->pix_fmt.height,
-		av->pfmt->css_pixelformat);
+		av->vdev.name, ipu6_isys_get_frame_width(av),
+		ipu6_isys_get_frame_height(av), pfmt->css_pixelformat);
 
 	ret = ipu6_isys_setup_video(av, &source_entity, &nr_queues);
 	if (ret < 0) {

drivers/media/pci/intel/ipu6/ipu6-isys-subdev.c

@@ -20,25 +20,30 @@ unsigned int ipu6_isys_mbus_code_to_bpp(u32 code)
 {
 	switch (code) {
 	case MEDIA_BUS_FMT_RGB888_1X24:
+	case MEDIA_BUS_FMT_META_24:
 		return 24;
 	case MEDIA_BUS_FMT_RGB565_1X16:
 	case MEDIA_BUS_FMT_UYVY8_1X16:
 	case MEDIA_BUS_FMT_YUYV8_1X16:
+	case MEDIA_BUS_FMT_META_16:
 		return 16;
 	case MEDIA_BUS_FMT_SBGGR12_1X12:
 	case MEDIA_BUS_FMT_SGBRG12_1X12:
 	case MEDIA_BUS_FMT_SGRBG12_1X12:
 	case MEDIA_BUS_FMT_SRGGB12_1X12:
+	case MEDIA_BUS_FMT_META_12:
 		return 12;
 	case MEDIA_BUS_FMT_SBGGR10_1X10:
 	case MEDIA_BUS_FMT_SGBRG10_1X10:
 	case MEDIA_BUS_FMT_SGRBG10_1X10:
 	case MEDIA_BUS_FMT_SRGGB10_1X10:
+	case MEDIA_BUS_FMT_META_10:
 		return 10;
 	case MEDIA_BUS_FMT_SBGGR8_1X8:
 	case MEDIA_BUS_FMT_SGBRG8_1X8:
 	case MEDIA_BUS_FMT_SGRBG8_1X8:
 	case MEDIA_BUS_FMT_SRGGB8_1X8:
+	case MEDIA_BUS_FMT_META_8:
 		return 8;
 	default:
 		WARN_ON(1);

drivers/media/pci/intel/ipu6/ipu6-isys-video.h

@@ -29,6 +29,7 @@ struct ipu6_isys_pixelformat {
 	u32 bpp_packed;
 	u32 code;
 	u32 css_pixelformat;
+	bool is_meta;
 };
 
 struct sequence_info {
@@ -91,7 +92,7 @@ struct ipu6_isys_video {
 	struct media_pad pad;
 	struct video_device vdev;
 	struct v4l2_pix_format pix_fmt;
-	const struct ipu6_isys_pixelformat *pfmt;
+	struct v4l2_meta_format meta_fmt;
 	struct ipu6_isys *isys;
 	struct ipu6_isys_csi2 *csi2;
 	struct ipu6_isys_stream *stream;
@@ -108,6 +109,8 @@ struct ipu6_isys_video {
 extern const struct ipu6_isys_pixelformat ipu6_isys_pfmts[];
 extern const struct ipu6_isys_pixelformat ipu6_isys_pfmts_packed[];
 
+const struct ipu6_isys_pixelformat *
+ipu6_isys_get_isys_format(u32 pixelformat, u32 code);
 int ipu6_isys_video_prepare_stream(struct ipu6_isys_video *av,
 				   struct media_entity *source_entity,
 				   int nr_queues);
@@ -129,4 +132,10 @@ void ipu6_isys_configure_stream_watermark(struct ipu6_isys_video *av,
 					  bool state);
 void ipu6_isys_update_stream_watermark(struct ipu6_isys_video *av, bool state);
 
+u32 ipu6_isys_get_format(struct ipu6_isys_video *av);
+u32 ipu6_isys_get_data_size(struct ipu6_isys_video *av);
+u32 ipu6_isys_get_bytes_per_line(struct ipu6_isys_video *av);
+u32 ipu6_isys_get_frame_width(struct ipu6_isys_video *av);
+u32 ipu6_isys_get_frame_height(struct ipu6_isys_video *av);
+
 #endif /* IPU6_ISYS_VIDEO_H */

drivers/media/pci/intel/ipu6/ipu6-isys.c

@@ -249,7 +249,6 @@ static int isys_register_video_devices(struct ipu6_isys *isys)
 			av->isys = isys;
 			av->aq.vbq.buf_struct_size =
 				sizeof(struct ipu6_isys_video_buffer);
-			av->pfmt = &ipu6_isys_pfmts[0];
 
 			ret = ipu6_isys_video_init(av);
 			if (ret)