[PATCH] USB: gadgetfs AIO support

This patch adds AIO support to gadgetfs, letting user mode programs use
"libaio" to manage USB I/O concurrency with the same API as they may
already be using for disk files.  In particular, it's practical to
stream isochronous data to/from userspace using this API, by keeping
an endpoint's I/O queue from emptying.

Each AIO "iocb" in userspace corresponds directly to one "usb_request"
(and one kiocb) in the kernel.  Their lifecycles, including cancelation,
overlap completely.  That's much of why the patch is so small (surprised
the heck out of me!); that, and using copy_{to,from}_user() rather than
trying fancy dma mapping tricks to attain zerocopy nirvana.

The kernel AIO module forgot to export kick_iocb(), so this adds the
missing declaration ... needed when using gadgetfs as a module.
Signed-off-by: David Brownell <dbrownell@users.sourceforge.net>
Signed-off-by: Greg Kroah-Hartman <greg@kroah.com>

drivers/usb/gadget/inode.c

@@ -20,7 +20,7 @@
  */
 
 
-#define	DEBUG 1			/* data to help fault diagnosis */
+// #define	DEBUG 			/* data to help fault diagnosis */
 // #define	VERBOSE		/* extra debug messages (success too) */
 
 #include <linux/init.h>
@@ -44,7 +44,8 @@
  * The gadgetfs API maps each endpoint to a file descriptor so that you
  * can use standard synchronous read/write calls for I/O.  There's some
  * O_NONBLOCK and O_ASYNC/FASYNC style i/o support.  Example usermode
- * drivers show how this works in practice.
+ * drivers show how this works in practice.  You can also use AIO to
+ * eliminate I/O gaps between requests, to help when streaming data.
  *
  * Key parts that must be USB-specific are protocols defining how the
  * read/write operations relate to the hardware state machines.  There
@@ -70,7 +71,7 @@
  */
 
 #define	DRIVER_DESC	"USB Gadget filesystem"
-#define	DRIVER_VERSION	"20 Aug 2003"
+#define	DRIVER_VERSION	"18 Nov 2003"
 
 static const char driver_desc [] = DRIVER_DESC;
 static const char shortname [] = "gadgetfs";
@@ -539,6 +540,177 @@ static int ep_ioctl (struct inode *inode, struct file *fd,
 	return status;
 }
 
+/*----------------------------------------------------------------------*/
+
+/* ASYNCHRONOUS ENDPOINT I/O OPERATIONS (bulk/intr/iso) */
+
+struct kiocb_priv {
+	struct usb_request	*req;
+	struct ep_data		*epdata;
+	void			*buf;
+	char __user		*ubuf;
+	unsigned		actual;
+};
+
+static int ep_aio_cancel(struct kiocb *iocb, struct io_event *e)
+{
+	struct kiocb_priv	*priv = (void *) &iocb->private;
+	struct ep_data		*epdata;
+	int			value;
+
+	local_irq_disable();
+	epdata = priv->epdata;
+	// spin_lock(&epdata->dev->lock);
+	kiocbSetCancelled(iocb);
+	if (likely(epdata && epdata->ep && priv->req))
+		value = usb_ep_dequeue (epdata->ep, priv->req);
+	else
+		value = -EINVAL;
+	// spin_unlock(&epdata->dev->lock);
+	local_irq_enable();
+
+	aio_put_req(iocb);
+	return value;
+}
+
+static long ep_aio_read_retry(struct kiocb *iocb)
+{
+	struct kiocb_priv	*priv = (void *) &iocb->private;
+	int			status = priv->actual;
+
+	/* we "retry" to get the right mm context for this: */
+	status = copy_to_user(priv->ubuf, priv->buf, priv->actual);
+	if (unlikely(0 != status))
+		status = -EFAULT;
+	else
+		status = priv->actual;
+	kfree(priv->buf);
+	aio_put_req(iocb);
+	return status;
+}
+
+static void ep_aio_complete(struct usb_ep *ep, struct usb_request *req)
+{
+	struct kiocb		*iocb = req->context;
+	struct kiocb_priv	*priv = (void *) &iocb->private;
+	struct ep_data		*epdata = priv->epdata;
+
+	/* lock against disconnect (and ideally, cancel) */
+	spin_lock(&epdata->dev->lock);
+	priv->req = 0;
+	priv->epdata = 0;
+	if (NULL == iocb->ki_retry
+			|| unlikely(0 == req->actual)
+			|| unlikely(kiocbIsCancelled(iocb))) {
+		kfree(req->buf);
+		/* aio_complete() reports bytes-transferred _and_ faults */
+		if (unlikely(kiocbIsCancelled(iocb)))
+			aio_put_req(iocb);
+		else
+			aio_complete(iocb,
+				req->actual ? req->actual : req->status,
+				req->status);
+	} else {
+		/* retry() won't report both; so we hide some faults */
+		if (unlikely(0 != req->status))
+			DBG(epdata->dev, "%s fault %d len %d\n",
+				ep->name, req->status, req->actual);
+
+		priv->buf = req->buf;
+		priv->actual = req->actual;
+		kick_iocb(iocb);
+	}
+	spin_unlock(&epdata->dev->lock);
+
+	usb_ep_free_request(ep, req);
+	put_ep(epdata);
+}
+
+static ssize_t
+ep_aio_rwtail(struct kiocb *iocb, char *buf, size_t len, struct ep_data *epdata)
+{
+	struct kiocb_priv	*priv = (void *) &iocb->private;
+	struct usb_request	*req;
+	ssize_t			value;
+
+	value = get_ready_ep(iocb->ki_filp->f_flags, epdata);
+	if (unlikely(value < 0)) {
+		kfree(buf);
+		return value;
+	}
+
+	iocb->ki_cancel = ep_aio_cancel;
+	get_ep(epdata);
+	priv->epdata = epdata;
+	priv->actual = 0;
+
+	/* each kiocb is coupled to one usb_request, but we can't
+	 * allocate or submit those if the host disconnected.
+	 */
+	spin_lock_irq(&epdata->dev->lock);
+	if (likely(epdata->ep)) {
+		req = usb_ep_alloc_request(epdata->ep, GFP_ATOMIC);
+		if (likely(req)) {
+			priv->req = req;
+			req->buf = buf;
+			req->length = len;
+			req->complete = ep_aio_complete;
+			req->context = iocb;
+			value = usb_ep_queue(epdata->ep, req, GFP_ATOMIC);
+			if (unlikely(0 != value))
+				usb_ep_free_request(epdata->ep, req);
+		} else
+			value = -EAGAIN;
+	} else
+		value = -ENODEV;
+	spin_unlock_irq(&epdata->dev->lock);
+
+	up(&epdata->lock);
+
+	if (unlikely(value))
+		put_ep(epdata);
+	else
+		value = -EIOCBQUEUED;
+	return value;
+}
+
+static ssize_t
+ep_aio_read(struct kiocb *iocb, char __user *ubuf, size_t len, loff_t o)
+{
+	struct kiocb_priv	*priv = (void *) &iocb->private;
+	struct ep_data		*epdata = iocb->ki_filp->private_data;
+	char			*buf;
+
+	if (unlikely(epdata->desc.bEndpointAddress & USB_DIR_IN))
+		return -EINVAL;
+	buf = kmalloc(len, GFP_KERNEL);
+	if (unlikely(!buf))
+		return -ENOMEM;
+	iocb->ki_retry = ep_aio_read_retry;
+	priv->ubuf = ubuf;
+	return ep_aio_rwtail(iocb, buf, len, epdata);
+}
+
+static ssize_t
+ep_aio_write(struct kiocb *iocb, const char __user *ubuf, size_t len, loff_t o)
+{
+	struct ep_data		*epdata = iocb->ki_filp->private_data;
+	char			*buf;
+
+	if (unlikely(!(epdata->desc.bEndpointAddress & USB_DIR_IN)))
+		return -EINVAL;
+	buf = kmalloc(len, GFP_KERNEL);
+	if (unlikely(!buf))
+		return -ENOMEM;
+	if (unlikely(copy_from_user(buf, ubuf, len) != 0)) {
+		kfree(buf);
+		return -EFAULT;
+	}
+	return ep_aio_rwtail(iocb, buf, len, epdata);
+}
+
+/*----------------------------------------------------------------------*/
+
 /* used after endpoint configuration */
 static struct file_operations ep_io_operations = {
 	.owner =	THIS_MODULE,
@@ -547,8 +719,8 @@ static struct file_operations ep_io_operations = {
 	.ioctl =	ep_ioctl,
 	.release =	ep_release,
 
-	// .aio_read =	ep_aio_read,
-	// .aio_write =	ep_aio_write,
+	.aio_read =	ep_aio_read,
+	.aio_write =	ep_aio_write,
 };
 
 /* ENDPOINT INITIALIZATION

fs/aio.c

@@ -618,6 +618,7 @@ void fastcall kick_iocb(struct kiocb *iocb)
 		queue_work(aio_wq, &ctx->wq);
 	}
 }
+EXPORT_SYMBOL(kick_iocb);
 
 /* aio_complete
  *	Called when the io request on the given iocb is complete.