Commit 531fc7be authored by Benjamin LaHaise's avatar Benjamin LaHaise

Merge toomuch.toronto.redhat.com:/md0/linus-2.5

into toomuch.toronto.redhat.com:/md0/aio-2.5
parents e38e1e7f 040ffec6
This diff is collapsed.
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
# #
export-objs := open.o dcache.o buffer.o bio.o inode.o dquot.o mpage.o aio.o \ export-objs := open.o dcache.o buffer.o bio.o inode.o dquot.o mpage.o aio.o \
fcntl.o fcntl.o read_write.o
obj-y := open.o read_write.o devices.o file_table.o buffer.o \ obj-y := open.o read_write.o devices.o file_table.o buffer.o \
bio.o super.o block_dev.o char_dev.o stat.o exec.o pipe.o \ bio.o super.o block_dev.o char_dev.o stat.o exec.o pipe.o \
......
...@@ -35,6 +35,7 @@ ...@@ -35,6 +35,7 @@
#include <asm/kmap_types.h> #include <asm/kmap_types.h>
#include <asm/uaccess.h> #include <asm/uaccess.h>
#include <asm/mmu_context.h>
#if DEBUG > 1 #if DEBUG > 1
#define dprintk printk #define dprintk printk
...@@ -59,6 +60,8 @@ static DECLARE_WORK(fput_work, aio_fput_routine, NULL); ...@@ -59,6 +60,8 @@ static DECLARE_WORK(fput_work, aio_fput_routine, NULL);
static spinlock_t fput_lock = SPIN_LOCK_UNLOCKED; static spinlock_t fput_lock = SPIN_LOCK_UNLOCKED;
LIST_HEAD(fput_head); LIST_HEAD(fput_head);
static void aio_kick_handler(void *);
/* aio_setup /* aio_setup
* Creates the slab caches used by the aio routines, panic on * Creates the slab caches used by the aio routines, panic on
* failure as this is done early during the boot sequence. * failure as this is done early during the boot sequence.
...@@ -175,31 +178,28 @@ static int aio_setup_ring(struct kioctx *ctx) ...@@ -175,31 +178,28 @@ static int aio_setup_ring(struct kioctx *ctx)
return 0; return 0;
} }
/* aio_ring_event: returns a pointer to the event at the given index from /* aio_ring_event: returns a pointer to the event at the given index from
* kmap_atomic(, km). Release the pointer with put_aio_ring_event(); * kmap_atomic(, km). Release the pointer with put_aio_ring_event();
*/ */
static inline struct io_event *aio_ring_event(struct aio_ring_info *info, int nr, enum km_type km)
{
struct io_event *events;
#define AIO_EVENTS_PER_PAGE (PAGE_SIZE / sizeof(struct io_event)) #define AIO_EVENTS_PER_PAGE (PAGE_SIZE / sizeof(struct io_event))
#define AIO_EVENTS_FIRST_PAGE ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event)) #define AIO_EVENTS_FIRST_PAGE ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event))
#define AIO_EVENTS_OFFSET (AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE)
if (nr < AIO_EVENTS_FIRST_PAGE) {
struct aio_ring *ring; #define aio_ring_event(info, nr, km) ({ \
ring = kmap_atomic(info->ring_pages[0], km); unsigned pos = (nr) + AIO_EVENTS_OFFSET; \
return &ring->io_events[nr]; struct io_event *__event; \
} __event = kmap_atomic( \
nr -= AIO_EVENTS_FIRST_PAGE; (info)->ring_pages[pos / AIO_EVENTS_PER_PAGE], km); \
__event += pos % AIO_EVENTS_PER_PAGE; \
events = kmap_atomic(info->ring_pages[1 + nr / AIO_EVENTS_PER_PAGE], km); __event; \
})
return events + (nr % AIO_EVENTS_PER_PAGE);
} #define put_aio_ring_event(event, km) do { \
struct io_event *__event = (event); \
static inline void put_aio_ring_event(struct io_event *event, enum km_type km) (void)__event; \
{ kunmap_atomic((void *)((unsigned long)__event & PAGE_MASK), km); \
kunmap_atomic((void *)((unsigned long)event & PAGE_MASK), km); } while(0)
}
/* ioctx_alloc /* ioctx_alloc
* Allocates and initializes an ioctx. Returns an ERR_PTR if it failed. * Allocates and initializes an ioctx. Returns an ERR_PTR if it failed.
...@@ -234,14 +234,18 @@ static struct kioctx *ioctx_alloc(unsigned nr_events) ...@@ -234,14 +234,18 @@ static struct kioctx *ioctx_alloc(unsigned nr_events)
init_waitqueue_head(&ctx->wait); init_waitqueue_head(&ctx->wait);
INIT_LIST_HEAD(&ctx->active_reqs); INIT_LIST_HEAD(&ctx->active_reqs);
INIT_LIST_HEAD(&ctx->run_list);
INIT_WORK(&ctx->wq, aio_kick_handler, ctx);
if (aio_setup_ring(ctx) < 0) if (aio_setup_ring(ctx) < 0)
goto out_freectx; goto out_freectx;
/* now link into global list. kludge. FIXME */ /* limit the number of system wide aios */
atomic_add(ctx->max_reqs, &aio_nr); /* undone by __put_ioctx */ atomic_add(ctx->max_reqs, &aio_nr); /* undone by __put_ioctx */
if (unlikely(atomic_read(&aio_nr) > aio_max_nr)) if (unlikely(atomic_read(&aio_nr) > aio_max_nr))
goto out_cleanup; goto out_cleanup;
/* now link into global list. kludge. FIXME */
write_lock(&mm->ioctx_list_lock); write_lock(&mm->ioctx_list_lock);
ctx->next = mm->ioctx_list; ctx->next = mm->ioctx_list;
mm->ioctx_list = ctx; mm->ioctx_list = ctx;
...@@ -383,17 +387,26 @@ static struct kiocb *__aio_get_req(struct kioctx *ctx) ...@@ -383,17 +387,26 @@ static struct kiocb *__aio_get_req(struct kioctx *ctx)
{ {
struct kiocb *req = NULL; struct kiocb *req = NULL;
struct aio_ring *ring; struct aio_ring *ring;
int okay = 0;
req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL); req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL);
if (unlikely(!req)) if (unlikely(!req))
return NULL; return NULL;
req->ki_flags = 1 << KIF_LOCKED;
req->ki_users = 1;
req->ki_key = 0;
req->ki_ctx = ctx;
req->ki_cancel = NULL;
req->ki_retry = NULL;
req->ki_user_obj = NULL;
/* Check if the completion queue has enough free space to /* Check if the completion queue has enough free space to
* accept an event from this io. * accept an event from this io.
*/ */
spin_lock_irq(&ctx->ctx_lock); spin_lock_irq(&ctx->ctx_lock);
ring = kmap_atomic(ctx->ring_info.ring_pages[0], KM_USER0); ring = kmap_atomic(ctx->ring_info.ring_pages[0], KM_USER0);
if (likely(ctx->reqs_active < aio_ring_avail(&ctx->ring_info, ring))) { if (ctx->reqs_active < aio_ring_avail(&ctx->ring_info, ring)) {
list_add(&req->ki_list, &ctx->active_reqs); list_add(&req->ki_list, &ctx->active_reqs);
get_ioctx(ctx); get_ioctx(ctx);
ctx->reqs_active++; ctx->reqs_active++;
...@@ -402,11 +415,16 @@ static struct kiocb *__aio_get_req(struct kioctx *ctx) ...@@ -402,11 +415,16 @@ static struct kiocb *__aio_get_req(struct kioctx *ctx)
req->ki_users = 1; req->ki_users = 1;
} else { } else {
kmem_cache_free(kiocb_cachep, req); kmem_cache_free(kiocb_cachep, req);
req = NULL; okay = 1;
} }
kunmap_atomic(ring, KM_USER0); kunmap_atomic(ring, KM_USER0);
spin_unlock_irq(&ctx->ctx_lock); spin_unlock_irq(&ctx->ctx_lock);
if (!okay) {
kmem_cache_free(kiocb_cachep, req);
req = NULL;
}
return req; return req;
} }
...@@ -476,6 +494,7 @@ static inline int __aio_put_req(struct kioctx *ctx, struct kiocb *req) ...@@ -476,6 +494,7 @@ static inline int __aio_put_req(struct kioctx *ctx, struct kiocb *req)
return 0; return 0;
list_del(&req->ki_list); /* remove from active_reqs */ list_del(&req->ki_list); /* remove from active_reqs */
req->ki_cancel = NULL; req->ki_cancel = NULL;
req->ki_retry = NULL;
/* Must be done under the lock to serialise against cancellation. /* Must be done under the lock to serialise against cancellation.
* Call this aio_fput as it duplicates fput via the fput_work. * Call this aio_fput as it duplicates fput via the fput_work.
...@@ -527,6 +546,82 @@ static inline struct kioctx *lookup_ioctx(unsigned long ctx_id) ...@@ -527,6 +546,82 @@ static inline struct kioctx *lookup_ioctx(unsigned long ctx_id)
return ioctx; return ioctx;
} }
static void use_mm(struct mm_struct *mm)
{
struct mm_struct *active_mm = current->active_mm;
atomic_inc(&mm->mm_count);
current->mm = mm;
if (mm != active_mm) {
current->active_mm = mm;
activate_mm(active_mm, mm);
}
mmdrop(active_mm);
}
static void unuse_mm(struct mm_struct *mm)
{
current->mm = NULL;
/* active_mm is still 'mm' */
enter_lazy_tlb(mm, current, smp_processor_id());
}
/* Run on kevent's context. FIXME: needs to be per-cpu and warn if an
* operation blocks.
*/
static void aio_kick_handler(void *data)
{
struct kioctx *ctx = data;
use_mm(ctx->mm);
spin_lock_irq(&ctx->ctx_lock);
while (!list_empty(&ctx->run_list)) {
struct kiocb *iocb;
long ret;
iocb = list_entry(ctx->run_list.next, struct kiocb,
ki_run_list);
list_del(&iocb->ki_run_list);
iocb->ki_users ++;
spin_unlock_irq(&ctx->ctx_lock);
kiocbClearKicked(iocb);
ret = iocb->ki_retry(iocb);
if (-EIOCBQUEUED != ret) {
aio_complete(iocb, ret, 0);
iocb = NULL;
}
spin_lock_irq(&ctx->ctx_lock);
if (NULL != iocb)
__aio_put_req(ctx, iocb);
}
spin_unlock_irq(&ctx->ctx_lock);
unuse_mm(ctx->mm);
}
void kick_iocb(struct kiocb *iocb)
{
struct kioctx *ctx = iocb->ki_ctx;
/* sync iocbs are easy: they can only ever be executing from a
* single context. */
if (is_sync_kiocb(iocb)) {
kiocbSetKicked(iocb);
wake_up_process(iocb->ki_user_obj);
return;
}
if (!kiocbTryKick(iocb)) {
long flags;
spin_lock_irqsave(&ctx->ctx_lock, flags);
list_add_tail(&iocb->ki_run_list, &ctx->run_list);
spin_unlock_irqrestore(&ctx->ctx_lock, flags);
schedule_work(&ctx->wq);
}
}
/* aio_complete /* aio_complete
* Called when the io request on the given iocb is complete. * Called when the io request on the given iocb is complete.
* Returns true if this is the last user of the request. The * Returns true if this is the last user of the request. The
...@@ -548,7 +643,7 @@ int aio_complete(struct kiocb *iocb, long res, long res2) ...@@ -548,7 +643,7 @@ int aio_complete(struct kiocb *iocb, long res, long res2)
* case the usage count checks will have to move under ctx_lock * case the usage count checks will have to move under ctx_lock
* for all cases. * for all cases.
*/ */
if (ctx == &ctx->mm->default_kioctx) { if (is_sync_kiocb(iocb)) {
int ret; int ret;
iocb->ki_user_data = res; iocb->ki_user_data = res;
...@@ -560,7 +655,10 @@ int aio_complete(struct kiocb *iocb, long res, long res2) ...@@ -560,7 +655,10 @@ int aio_complete(struct kiocb *iocb, long res, long res2)
iocb->ki_users--; iocb->ki_users--;
ret = (0 == iocb->ki_users); ret = (0 == iocb->ki_users);
spin_unlock_irq(&ctx->ctx_lock); spin_unlock_irq(&ctx->ctx_lock);
return 0;
/* sync iocbs put the task here for us */
wake_up_process(iocb->ki_user_obj);
return ret;
} }
info = &ctx->ring_info; info = &ctx->ring_info;
...@@ -940,6 +1038,7 @@ static int io_submit_one(struct kioctx *ctx, struct iocb *user_iocb, ...@@ -940,6 +1038,7 @@ static int io_submit_one(struct kioctx *ctx, struct iocb *user_iocb,
req->ki_user_obj = user_iocb; req->ki_user_obj = user_iocb;
req->ki_user_data = iocb->aio_data; req->ki_user_data = iocb->aio_data;
req->ki_pos = iocb->aio_offset;
buf = (char *)(unsigned long)iocb->aio_buf; buf = (char *)(unsigned long)iocb->aio_buf;
...@@ -954,7 +1053,7 @@ static int io_submit_one(struct kioctx *ctx, struct iocb *user_iocb, ...@@ -954,7 +1053,7 @@ static int io_submit_one(struct kioctx *ctx, struct iocb *user_iocb,
ret = -EINVAL; ret = -EINVAL;
if (file->f_op->aio_read) if (file->f_op->aio_read)
ret = file->f_op->aio_read(req, buf, ret = file->f_op->aio_read(req, buf,
iocb->aio_nbytes, iocb->aio_offset); iocb->aio_nbytes, req->ki_pos);
break; break;
case IOCB_CMD_PWRITE: case IOCB_CMD_PWRITE:
ret = -EBADF; ret = -EBADF;
...@@ -966,7 +1065,7 @@ static int io_submit_one(struct kioctx *ctx, struct iocb *user_iocb, ...@@ -966,7 +1065,7 @@ static int io_submit_one(struct kioctx *ctx, struct iocb *user_iocb,
ret = -EINVAL; ret = -EINVAL;
if (file->f_op->aio_write) if (file->f_op->aio_write)
ret = file->f_op->aio_write(req, buf, ret = file->f_op->aio_write(req, buf,
iocb->aio_nbytes, iocb->aio_offset); iocb->aio_nbytes, req->ki_pos);
break; break;
case IOCB_CMD_FDSYNC: case IOCB_CMD_FDSYNC:
ret = -EINVAL; ret = -EINVAL;
...@@ -983,12 +1082,10 @@ static int io_submit_one(struct kioctx *ctx, struct iocb *user_iocb, ...@@ -983,12 +1082,10 @@ static int io_submit_one(struct kioctx *ctx, struct iocb *user_iocb,
ret = -EINVAL; ret = -EINVAL;
} }
if (likely(EIOCBQUEUED == ret)) if (likely(-EIOCBQUEUED == ret))
return 0; return 0;
if (ret >= 0) {
aio_complete(req, ret, 0); aio_complete(req, ret, 0);
return 0; return 0;
}
out_put_req: out_put_req:
aio_put_req(req); aio_put_req(req);
......
...@@ -131,7 +131,7 @@ asmlinkage long sys_uselib(const char * library) ...@@ -131,7 +131,7 @@ asmlinkage long sys_uselib(const char * library)
goto out; goto out;
error = -ENOEXEC; error = -ENOEXEC;
if(file->f_op && file->f_op->read) { if(file->f_op) {
struct linux_binfmt * fmt; struct linux_binfmt * fmt;
read_lock(&binfmt_lock); read_lock(&binfmt_lock);
...@@ -453,19 +453,16 @@ struct file *open_exec(const char *name) ...@@ -453,19 +453,16 @@ struct file *open_exec(const char *name)
} }
int kernel_read(struct file *file, unsigned long offset, int kernel_read(struct file *file, unsigned long offset,
char * addr, unsigned long count) char *addr, unsigned long count)
{ {
mm_segment_t old_fs; mm_segment_t old_fs;
loff_t pos = offset; loff_t pos = offset;
int result = -ENOSYS; int result;
if (!file->f_op->read)
goto fail;
old_fs = get_fs(); old_fs = get_fs();
set_fs(get_ds()); set_fs(get_ds());
result = file->f_op->read(file, addr, count, &pos); result = vfs_read(file, addr, count, &pos);
set_fs(old_fs); set_fs(old_fs);
fail:
return result; return result;
} }
......
...@@ -41,6 +41,8 @@ struct file_operations ext2_file_operations = { ...@@ -41,6 +41,8 @@ struct file_operations ext2_file_operations = {
.llseek = generic_file_llseek, .llseek = generic_file_llseek,
.read = generic_file_read, .read = generic_file_read,
.write = generic_file_write, .write = generic_file_write,
.aio_read = generic_file_aio_read,
.aio_write = generic_file_aio_write,
.ioctl = ext2_ioctl, .ioctl = ext2_ioctl,
.mmap = generic_file_mmap, .mmap = generic_file_mmap,
.open = generic_file_open, .open = generic_file_open,
......
...@@ -61,8 +61,9 @@ static int ext3_open_file (struct inode * inode, struct file * filp) ...@@ -61,8 +61,9 @@ static int ext3_open_file (struct inode * inode, struct file * filp)
*/ */
static ssize_t static ssize_t
ext3_file_write(struct file *file, const char *buf, size_t count, loff_t *ppos) ext3_file_write(struct kiocb *iocb, const char *buf, size_t count, loff_t pos)
{ {
struct file *file = iocb->ki_filp;
struct inode *inode = file->f_dentry->d_inode; struct inode *inode = file->f_dentry->d_inode;
/* /*
...@@ -75,13 +76,15 @@ ext3_file_write(struct file *file, const char *buf, size_t count, loff_t *ppos) ...@@ -75,13 +76,15 @@ ext3_file_write(struct file *file, const char *buf, size_t count, loff_t *ppos)
if (IS_SYNC(inode) || (file->f_flags & O_SYNC)) if (IS_SYNC(inode) || (file->f_flags & O_SYNC))
mark_inode_dirty(inode); mark_inode_dirty(inode);
return generic_file_write(file, buf, count, ppos); return generic_file_aio_write(iocb, buf, count, pos);
} }
struct file_operations ext3_file_operations = { struct file_operations ext3_file_operations = {
.llseek = generic_file_llseek, .llseek = generic_file_llseek,
.read = generic_file_read, .read = do_sync_read,
.write = ext3_file_write, .write = do_sync_write,
.aio_read = generic_file_aio_read,
.aio_write = ext3_file_write,
.readv = generic_file_readv, .readv = generic_file_readv,
.writev = generic_file_writev, .writev = generic_file_writev,
.ioctl = ext3_ioctl, .ioctl = ext3_ioctl,
......
...@@ -35,15 +35,17 @@ ...@@ -35,15 +35,17 @@
#define NFSDBG_FACILITY NFSDBG_FILE #define NFSDBG_FACILITY NFSDBG_FILE
static int nfs_file_mmap(struct file *, struct vm_area_struct *); static int nfs_file_mmap(struct file *, struct vm_area_struct *);
static ssize_t nfs_file_read(struct file *, char *, size_t, loff_t *); static ssize_t nfs_file_read(struct kiocb *, char *, size_t, loff_t);
static ssize_t nfs_file_write(struct file *, const char *, size_t, loff_t *); static ssize_t nfs_file_write(struct kiocb *, const char *, size_t, loff_t);
static int nfs_file_flush(struct file *); static int nfs_file_flush(struct file *);
static int nfs_fsync(struct file *, struct dentry *dentry, int datasync); static int nfs_fsync(struct file *, struct dentry *dentry, int datasync);
struct file_operations nfs_file_operations = { struct file_operations nfs_file_operations = {
.llseek = remote_llseek, .llseek = remote_llseek,
.read = nfs_file_read, .read = do_sync_read,
.write = nfs_file_write, .write = do_sync_write,
.aio_read = nfs_file_read,
.aio_write = nfs_file_write,
.mmap = nfs_file_mmap, .mmap = nfs_file_mmap,
.open = nfs_open, .open = nfs_open,
.flush = nfs_file_flush, .flush = nfs_file_flush,
...@@ -89,19 +91,19 @@ nfs_file_flush(struct file *file) ...@@ -89,19 +91,19 @@ nfs_file_flush(struct file *file)
} }
static ssize_t static ssize_t
nfs_file_read(struct file * file, char * buf, size_t count, loff_t *ppos) nfs_file_read(struct kiocb *iocb, char * buf, size_t count, loff_t pos)
{ {
struct dentry * dentry = file->f_dentry; struct dentry * dentry = iocb->ki_filp->f_dentry;
struct inode * inode = dentry->d_inode; struct inode * inode = dentry->d_inode;
ssize_t result; ssize_t result;
dfprintk(VFS, "nfs: read(%s/%s, %lu@%lu)\n", dfprintk(VFS, "nfs: read(%s/%s, %lu@%lu)\n",
dentry->d_parent->d_name.name, dentry->d_name.name, dentry->d_parent->d_name.name, dentry->d_name.name,
(unsigned long) count, (unsigned long) *ppos); (unsigned long) count, (unsigned long) pos);
result = nfs_revalidate_inode(NFS_SERVER(inode), inode); result = nfs_revalidate_inode(NFS_SERVER(inode), inode);
if (!result) if (!result)
result = generic_file_read(file, buf, count, ppos); result = generic_file_aio_read(iocb, buf, count, pos);
return result; return result;
} }
...@@ -209,15 +211,15 @@ struct address_space_operations nfs_file_aops = { ...@@ -209,15 +211,15 @@ struct address_space_operations nfs_file_aops = {
* Write to a file (through the page cache). * Write to a file (through the page cache).
*/ */
static ssize_t static ssize_t
nfs_file_write(struct file *file, const char *buf, size_t count, loff_t *ppos) nfs_file_write(struct kiocb *iocb, const char *buf, size_t count, loff_t pos)
{ {
struct dentry * dentry = file->f_dentry; struct dentry * dentry = iocb->ki_filp->f_dentry;
struct inode * inode = dentry->d_inode; struct inode * inode = dentry->d_inode;
ssize_t result; ssize_t result;
dfprintk(VFS, "nfs: write(%s/%s(%ld), %lu@%lu)\n", dfprintk(VFS, "nfs: write(%s/%s(%ld), %lu@%lu)\n",
dentry->d_parent->d_name.name, dentry->d_name.name, dentry->d_parent->d_name.name, dentry->d_name.name,
inode->i_ino, (unsigned long) count, (unsigned long) *ppos); inode->i_ino, (unsigned long) count, (unsigned long) pos);
result = -EBUSY; result = -EBUSY;
if (IS_SWAPFILE(inode)) if (IS_SWAPFILE(inode))
...@@ -230,7 +232,7 @@ nfs_file_write(struct file *file, const char *buf, size_t count, loff_t *ppos) ...@@ -230,7 +232,7 @@ nfs_file_write(struct file *file, const char *buf, size_t count, loff_t *ppos)
if (!count) if (!count)
goto out; goto out;
result = generic_file_write(file, buf, count, ppos); result = generic_file_aio_write(iocb, buf, count, pos);
out: out:
return result; return result;
......
...@@ -176,6 +176,20 @@ asmlinkage long sys_llseek(unsigned int fd, unsigned long offset_high, ...@@ -176,6 +176,20 @@ asmlinkage long sys_llseek(unsigned int fd, unsigned long offset_high,
} }
#endif #endif
ssize_t do_sync_read(struct file *filp, char *buf, size_t len, loff_t *ppos)
{
struct kiocb kiocb;
ssize_t ret;
init_sync_kiocb(&kiocb, filp);
kiocb.ki_pos = *ppos;
ret = filp->f_op->aio_read(&kiocb, buf, len, kiocb.ki_pos);
if (-EIOCBQUEUED == ret)
ret = wait_on_sync_kiocb(&kiocb);
*ppos = kiocb.ki_pos;
return ret;
}
ssize_t vfs_read(struct file *file, char *buf, size_t count, loff_t *pos) ssize_t vfs_read(struct file *file, char *buf, size_t count, loff_t *pos)
{ {
struct inode *inode = file->f_dentry->d_inode; struct inode *inode = file->f_dentry->d_inode;
...@@ -183,14 +197,17 @@ ssize_t vfs_read(struct file *file, char *buf, size_t count, loff_t *pos) ...@@ -183,14 +197,17 @@ ssize_t vfs_read(struct file *file, char *buf, size_t count, loff_t *pos)
if (!(file->f_mode & FMODE_READ)) if (!(file->f_mode & FMODE_READ))
return -EBADF; return -EBADF;
if (!file->f_op || !file->f_op->read) if (!file->f_op || (!file->f_op->read && !file->f_op->aio_read))
return -EINVAL; return -EINVAL;
ret = locks_verify_area(FLOCK_VERIFY_READ, inode, file, *pos, count); ret = locks_verify_area(FLOCK_VERIFY_READ, inode, file, *pos, count);
if (!ret) { if (!ret) {
ret = security_ops->file_permission (file, MAY_READ); ret = security_ops->file_permission (file, MAY_READ);
if (!ret) { if (!ret) {
if (file->f_op->read)
ret = file->f_op->read(file, buf, count, pos); ret = file->f_op->read(file, buf, count, pos);
else
ret = do_sync_read(file, buf, count, pos);
if (ret > 0) if (ret > 0)
dnotify_parent(file->f_dentry, DN_ACCESS); dnotify_parent(file->f_dentry, DN_ACCESS);
} }
...@@ -199,6 +216,20 @@ ssize_t vfs_read(struct file *file, char *buf, size_t count, loff_t *pos) ...@@ -199,6 +216,20 @@ ssize_t vfs_read(struct file *file, char *buf, size_t count, loff_t *pos)
return ret; return ret;
} }
ssize_t do_sync_write(struct file *filp, const char *buf, size_t len, loff_t *ppos)
{
struct kiocb kiocb;
ssize_t ret;
init_sync_kiocb(&kiocb, filp);
kiocb.ki_pos = *ppos;
ret = filp->f_op->aio_write(&kiocb, buf, len, kiocb.ki_pos);
if (-EIOCBQUEUED == ret)
ret = wait_on_sync_kiocb(&kiocb);
*ppos = kiocb.ki_pos;
return ret;
}
ssize_t vfs_write(struct file *file, const char *buf, size_t count, loff_t *pos) ssize_t vfs_write(struct file *file, const char *buf, size_t count, loff_t *pos)
{ {
struct inode *inode = file->f_dentry->d_inode; struct inode *inode = file->f_dentry->d_inode;
...@@ -206,14 +237,17 @@ ssize_t vfs_write(struct file *file, const char *buf, size_t count, loff_t *pos) ...@@ -206,14 +237,17 @@ ssize_t vfs_write(struct file *file, const char *buf, size_t count, loff_t *pos)
if (!(file->f_mode & FMODE_WRITE)) if (!(file->f_mode & FMODE_WRITE))
return -EBADF; return -EBADF;
if (!file->f_op || !file->f_op->write) if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write))
return -EINVAL; return -EINVAL;
ret = locks_verify_area(FLOCK_VERIFY_WRITE, inode, file, *pos, count); ret = locks_verify_area(FLOCK_VERIFY_WRITE, inode, file, *pos, count);
if (!ret) { if (!ret) {
ret = security_ops->file_permission (file, MAY_WRITE); ret = security_ops->file_permission (file, MAY_WRITE);
if (!ret) { if (!ret) {
if (file->f_op->write)
ret = file->f_op->write(file, buf, count, pos); ret = file->f_op->write(file, buf, count, pos);
else
ret = do_sync_write(file, buf, count, pos);
if (ret > 0) if (ret > 0)
dnotify_parent(file->f_dentry, DN_MODIFY); dnotify_parent(file->f_dentry, DN_MODIFY);
} }
...@@ -580,3 +614,6 @@ asmlinkage ssize_t sys_sendfile64(int out_fd, int in_fd, loff_t *offset, size_t ...@@ -580,3 +614,6 @@ asmlinkage ssize_t sys_sendfile64(int out_fd, int in_fd, loff_t *offset, size_t
return do_sendfile(out_fd, in_fd, NULL, count, 0); return do_sendfile(out_fd, in_fd, NULL, count, 0);
} }
EXPORT_SYMBOL(do_sync_read);
EXPORT_SYMBOL(do_sync_write);
...@@ -2,10 +2,11 @@ ...@@ -2,10 +2,11 @@
#define __LINUX__AIO_H #define __LINUX__AIO_H
#include <linux/list.h> #include <linux/list.h>
#include <asm/atomic.h> #include <linux/workqueue.h>
#include <linux/aio_abi.h> #include <linux/aio_abi.h>
#include <asm/atomic.h>
#define AIO_MAXSEGS 4 #define AIO_MAXSEGS 4
#define AIO_KIOGRP_NR_ATOMIC 8 #define AIO_KIOGRP_NR_ATOMIC 8
...@@ -22,28 +23,54 @@ struct kioctx; ...@@ -22,28 +23,54 @@ struct kioctx;
#define KIOCB_SYNC_KEY (~0U) #define KIOCB_SYNC_KEY (~0U)
#define KIOCB_PRIVATE_SIZE (16 * sizeof(long)) #define KIOCB_PRIVATE_SIZE (24 * sizeof(long))
/* ki_flags bits */
#define KIF_LOCKED 0
#define KIF_KICKED 1
#define KIF_CANCELLED 2
#define kiocbTryLock(iocb) test_and_set_bit(KIF_LOCKED, &(iocb)->ki_flags)
#define kiocbTryKick(iocb) test_and_set_bit(KIF_KICKED, &(iocb)->ki_flags)
#define kiocbSetLocked(iocb) set_bit(KIF_LOCKED, &(iocb)->ki_flags)
#define kiocbSetKicked(iocb) set_bit(KIF_KICKED, &(iocb)->ki_flags)
#define kiocbSetCancelled(iocb) set_bit(KIF_CANCELLED, &(iocb)->ki_flags)
#define kiocbClearLocked(iocb) set_bit(KIF_LOCKED, &(iocb)->ki_flags)
#define kiocbClearKicked(iocb) set_bit(KIF_KICKED, &(iocb)->ki_flags)
#define kiocbClearCancelled(iocb) set_bit(KIF_CANCELLED, &(iocb)->ki_flags)
#define kiocbIsLocked(iocb) test_bit(0, &(iocb)->ki_flags)
#define kiocbIsKicked(iocb) test_bit(1, &(iocb)->ki_flags)
#define kiocbIsCancelled(iocb) test_bit(2, &(iocb)->ki_flags)
struct kiocb { struct kiocb {
struct list_head ki_run_list;
long ki_flags;
int ki_users; int ki_users;
unsigned ki_key; /* id of this request */ unsigned ki_key; /* id of this request */
struct file *ki_filp; struct file *ki_filp;
struct kioctx *ki_ctx; /* may be NULL for sync ops */ struct kioctx *ki_ctx; /* may be NULL for sync ops */
int (*ki_cancel)(struct kiocb *, struct io_event *); int (*ki_cancel)(struct kiocb *, struct io_event *);
long (*ki_retry)(struct kiocb *);
struct list_head ki_list; struct list_head ki_list; /* the aio core uses this
* for cancellation */
void *ki_data; /* for use by the the file */
void *ki_user_obj; /* pointer to userland's iocb */ void *ki_user_obj; /* pointer to userland's iocb */
__u64 ki_user_data; /* user's data for completion */ __u64 ki_user_data; /* user's data for completion */
loff_t ki_pos;
long private[KIOCB_PRIVATE_SIZE/sizeof(long)]; char private[KIOCB_PRIVATE_SIZE];
}; };
#define is_sync_kiocb(iocb) ((iocb)->ki_key == KIOCB_SYNC_KEY)
#define init_sync_kiocb(x, filp) \ #define init_sync_kiocb(x, filp) \
do { \ do { \
struct task_struct *tsk = current; \ struct task_struct *tsk = current; \
(x)->ki_flags = 0; \
(x)->ki_users = 1; \ (x)->ki_users = 1; \
(x)->ki_key = KIOCB_SYNC_KEY; \ (x)->ki_key = KIOCB_SYNC_KEY; \
(x)->ki_filp = (filp); \ (x)->ki_filp = (filp); \
...@@ -101,10 +128,13 @@ struct kioctx { ...@@ -101,10 +128,13 @@ struct kioctx {
int reqs_active; int reqs_active;
struct list_head active_reqs; /* used for cancellation */ struct list_head active_reqs; /* used for cancellation */
struct list_head run_list; /* used for kicked reqs */
unsigned max_reqs; unsigned max_reqs;
struct aio_ring_info ring_info; struct aio_ring_info ring_info;
struct work_struct wq;
}; };
/* prototypes */ /* prototypes */
...@@ -112,6 +142,7 @@ extern unsigned aio_max_size; ...@@ -112,6 +142,7 @@ extern unsigned aio_max_size;
extern ssize_t FASTCALL(wait_on_sync_kiocb(struct kiocb *iocb)); extern ssize_t FASTCALL(wait_on_sync_kiocb(struct kiocb *iocb));
extern int FASTCALL(aio_put_req(struct kiocb *iocb)); extern int FASTCALL(aio_put_req(struct kiocb *iocb));
extern void FASTCALL(kick_iocb(struct kiocb *iocb));
extern int FASTCALL(aio_complete(struct kiocb *iocb, long res, long res2)); extern int FASTCALL(aio_complete(struct kiocb *iocb, long res, long res2));
extern void FASTCALL(__put_ioctx(struct kioctx *ctx)); extern void FASTCALL(__put_ioctx(struct kioctx *ctx));
struct mm_struct; struct mm_struct;
......
...@@ -746,7 +746,7 @@ struct file_operations { ...@@ -746,7 +746,7 @@ struct file_operations {
ssize_t (*read) (struct file *, char *, size_t, loff_t *); ssize_t (*read) (struct file *, char *, size_t, loff_t *);
ssize_t (*aio_read) (struct kiocb *, char *, size_t, loff_t); ssize_t (*aio_read) (struct kiocb *, char *, size_t, loff_t);
ssize_t (*write) (struct file *, const char *, size_t, loff_t *); ssize_t (*write) (struct file *, const char *, size_t, loff_t *);
ssize_t (*aio_write) (struct kiocb *, char *, size_t, loff_t); ssize_t (*aio_write) (struct kiocb *, const char *, size_t, loff_t);
int (*readdir) (struct file *, void *, filldir_t); int (*readdir) (struct file *, void *, filldir_t);
unsigned int (*poll) (struct file *, struct poll_table_struct *); unsigned int (*poll) (struct file *, struct poll_table_struct *);
int (*ioctl) (struct inode *, struct file *, unsigned int, unsigned long); int (*ioctl) (struct inode *, struct file *, unsigned int, unsigned long);
...@@ -1238,6 +1238,10 @@ extern int generic_file_mmap(struct file *, struct vm_area_struct *); ...@@ -1238,6 +1238,10 @@ extern int generic_file_mmap(struct file *, struct vm_area_struct *);
extern int file_read_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size); extern int file_read_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size);
extern ssize_t generic_file_read(struct file *, char *, size_t, loff_t *); extern ssize_t generic_file_read(struct file *, char *, size_t, loff_t *);
extern ssize_t generic_file_write(struct file *, const char *, size_t, loff_t *); extern ssize_t generic_file_write(struct file *, const char *, size_t, loff_t *);
extern ssize_t generic_file_aio_read(struct kiocb *, char *, size_t, loff_t);
extern ssize_t generic_file_aio_write(struct kiocb *, const char *, size_t, loff_t);
extern ssize_t do_sync_read(struct file *filp, char *buf, size_t len, loff_t *ppos);
extern ssize_t do_sync_write(struct file *filp, const char *buf, size_t len, loff_t *ppos);
ssize_t generic_file_write_nolock(struct file *file, const struct iovec *iov, ssize_t generic_file_write_nolock(struct file *file, const struct iovec *iov,
unsigned long nr_segs, loff_t *ppos); unsigned long nr_segs, loff_t *ppos);
extern ssize_t generic_file_sendfile(struct file *, struct file *, loff_t *, size_t); extern ssize_t generic_file_sendfile(struct file *, struct file *, loff_t *, size_t);
......
...@@ -201,6 +201,9 @@ struct sysinfo { ...@@ -201,6 +201,9 @@ struct sysinfo {
#define BUG_ON(condition) do { if (unlikely((condition)!=0)) BUG(); } while(0) #define BUG_ON(condition) do { if (unlikely((condition)!=0)) BUG(); } while(0)
extern void BUILD_BUG(void);
#define BUILD_BUG_ON(condition) do { if (condition) BUILD_BUG(); } while(0)
/* Trap pasters of __FUNCTION__ at compile-time */ /* Trap pasters of __FUNCTION__ at compile-time */
#if __GNUC__ > 2 || __GNUC_MINOR__ >= 95 #if __GNUC__ > 2 || __GNUC_MINOR__ >= 95
#define __FUNCTION__ (__func__) #define __FUNCTION__ (__func__)
......
...@@ -885,11 +885,12 @@ __generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, ...@@ -885,11 +885,12 @@ __generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
} }
ssize_t ssize_t
generic_file_aio_read(struct kiocb *iocb, char *buf, size_t count, loff_t *ppos) generic_file_aio_read(struct kiocb *iocb, char *buf, size_t count, loff_t pos)
{ {
struct iovec local_iov = { .iov_base = buf, .iov_len = count }; struct iovec local_iov = { .iov_base = buf, .iov_len = count };
return __generic_file_aio_read(iocb, &local_iov, 1, ppos); BUG_ON(iocb->ki_pos != pos);
return __generic_file_aio_read(iocb, &local_iov, 1, &iocb->ki_pos);
} }
ssize_t ssize_t
...@@ -1645,6 +1646,12 @@ generic_file_write_nolock(struct file *file, const struct iovec *iov, ...@@ -1645,6 +1646,12 @@ generic_file_write_nolock(struct file *file, const struct iovec *iov,
return err; return err;
} }
ssize_t generic_file_aio_write(struct kiocb *iocb, const char *buf,
size_t count, loff_t pos)
{
return generic_file_write(iocb->ki_filp, buf, count, &iocb->ki_pos);
}
ssize_t generic_file_write(struct file *file, const char *buf, ssize_t generic_file_write(struct file *file, const char *buf,
size_t count, loff_t *ppos) size_t count, loff_t *ppos)
{ {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment