Commit f7f7e7a0 authored by Yan, Zheng's avatar Yan, Zheng Committed by Ilya Dryomov

ceph: improve fscache revalidation

There are several issues in fscache revalidation code.
- In ceph_revalidate_work(), fscache_invalidate() is called when
  fscache_check_consistency() return 0. This is complete wrong
  because 0 means cache is valid.
- Handle_cap_grant() calls ceph_queue_revalidate() if client
  already has CAP_FILE_CACHE. This code is confusing. Client
  should revalidate the cache each time it got CAP_FILE_CACHE
  anew.
- In Handle_cap_grant(), fscache_invalidate() is called if MDS
  revokes CAP_FILE_CACHE. This is inconsistency with the case
  that inode get evicted. In the later case, the cache is not
  discarded. Client may use the cache when inode is reloaded.

This patch moves the fscache revalidation into ceph_get_caps().
Client revalidates the cache after it gets CAP_FILE_CACHE.
i_rdcache_gen should keep constance while CAP_FILE_CACHE is
used. If i_fscache_gen is not equal to i_rdcache_gen, client
needs to check cache's consistency.
Signed-off-by: default avatarYan, Zheng <zyan@redhat.com>
parent 46b59b2b
...@@ -69,15 +69,8 @@ int ceph_fscache_register_fs(struct ceph_fs_client* fsc) ...@@ -69,15 +69,8 @@ int ceph_fscache_register_fs(struct ceph_fs_client* fsc)
fsc->fscache = fscache_acquire_cookie(ceph_cache_netfs.primary_index, fsc->fscache = fscache_acquire_cookie(ceph_cache_netfs.primary_index,
&ceph_fscache_fsid_object_def, &ceph_fscache_fsid_object_def,
fsc, true); fsc, true);
if (!fsc->fscache)
if (fsc->fscache == NULL) {
pr_err("Unable to resgister fsid: %p fscache cookie", fsc); pr_err("Unable to resgister fsid: %p fscache cookie", fsc);
return 0;
}
fsc->revalidate_wq = alloc_workqueue("ceph-revalidate", 0, 1);
if (fsc->revalidate_wq == NULL)
return -ENOMEM;
return 0; return 0;
} }
...@@ -260,8 +253,7 @@ static void ceph_vfs_readpage_complete_unlock(struct page *page, void *data, int ...@@ -260,8 +253,7 @@ static void ceph_vfs_readpage_complete_unlock(struct page *page, void *data, int
static inline bool cache_valid(struct ceph_inode_info *ci) static inline bool cache_valid(struct ceph_inode_info *ci)
{ {
return ((ceph_caps_issued(ci) & CEPH_CAP_FILE_CACHE) && return ci->i_fscache_gen == ci->i_rdcache_gen;
(ci->i_fscache_gen == ci->i_rdcache_gen));
} }
...@@ -354,69 +346,27 @@ void ceph_invalidate_fscache_page(struct inode* inode, struct page *page) ...@@ -354,69 +346,27 @@ void ceph_invalidate_fscache_page(struct inode* inode, struct page *page)
void ceph_fscache_unregister_fs(struct ceph_fs_client* fsc) void ceph_fscache_unregister_fs(struct ceph_fs_client* fsc)
{ {
if (fsc->revalidate_wq)
destroy_workqueue(fsc->revalidate_wq);
fscache_relinquish_cookie(fsc->fscache, 0); fscache_relinquish_cookie(fsc->fscache, 0);
fsc->fscache = NULL; fsc->fscache = NULL;
} }
static void ceph_revalidate_work(struct work_struct *work) /*
{ * caller should hold CEPH_CAP_FILE_{RD,CACHE}
int issued; */
u32 orig_gen; void ceph_fscache_revalidate_cookie(struct ceph_inode_info *ci)
struct ceph_inode_info *ci = container_of(work, struct ceph_inode_info,
i_revalidate_work);
struct inode *inode = &ci->vfs_inode;
spin_lock(&ci->i_ceph_lock);
issued = __ceph_caps_issued(ci, NULL);
orig_gen = ci->i_rdcache_gen;
spin_unlock(&ci->i_ceph_lock);
if (!(issued & CEPH_CAP_FILE_CACHE)) {
dout("revalidate_work lost cache before validation %p\n",
inode);
goto out;
}
if (!fscache_check_consistency(ci->fscache))
fscache_invalidate(ci->fscache);
spin_lock(&ci->i_ceph_lock);
/* Update the new valid generation (backwards sanity check too) */
if (orig_gen > ci->i_fscache_gen) {
ci->i_fscache_gen = orig_gen;
}
spin_unlock(&ci->i_ceph_lock);
out:
iput(&ci->vfs_inode);
}
void ceph_queue_revalidate(struct inode *inode)
{ {
struct ceph_fs_client *fsc = ceph_sb_to_client(inode->i_sb); if (cache_valid(ci))
struct ceph_inode_info *ci = ceph_inode(inode);
if (fsc->revalidate_wq == NULL || ci->fscache == NULL)
return; return;
ihold(inode); /* resue i_truncate_mutex. There should be no pending
* truncate while the caller holds CEPH_CAP_FILE_RD */
if (queue_work(ceph_sb_to_client(inode->i_sb)->revalidate_wq, mutex_lock(&ci->i_truncate_mutex);
&ci->i_revalidate_work)) { if (!cache_valid(ci)) {
dout("ceph_queue_revalidate %p\n", inode); if (fscache_check_consistency(ci->fscache))
} else { fscache_invalidate(ci->fscache);
dout("ceph_queue_revalidate %p failed\n)", inode); spin_lock(&ci->i_ceph_lock);
iput(inode); ci->i_fscache_gen = ci->i_rdcache_gen;
spin_unlock(&ci->i_ceph_lock);
} }
} mutex_unlock(&ci->i_truncate_mutex);
void ceph_fscache_inode_init(struct ceph_inode_info *ci)
{
ci->fscache = NULL;
/* The first load is verifed cookie open time */
ci->i_fscache_gen = 1;
INIT_WORK(&ci->i_revalidate_work, ceph_revalidate_work);
} }
...@@ -34,10 +34,10 @@ void ceph_fscache_unregister(void); ...@@ -34,10 +34,10 @@ void ceph_fscache_unregister(void);
int ceph_fscache_register_fs(struct ceph_fs_client* fsc); int ceph_fscache_register_fs(struct ceph_fs_client* fsc);
void ceph_fscache_unregister_fs(struct ceph_fs_client* fsc); void ceph_fscache_unregister_fs(struct ceph_fs_client* fsc);
void ceph_fscache_inode_init(struct ceph_inode_info *ci);
void ceph_fscache_register_inode_cookie(struct inode *inode); void ceph_fscache_register_inode_cookie(struct inode *inode);
void ceph_fscache_unregister_inode_cookie(struct ceph_inode_info* ci); void ceph_fscache_unregister_inode_cookie(struct ceph_inode_info* ci);
void ceph_fscache_file_set_cookie(struct inode *inode, struct file *filp); void ceph_fscache_file_set_cookie(struct inode *inode, struct file *filp);
void ceph_fscache_revalidate_cookie(struct ceph_inode_info *ci);
int ceph_readpage_from_fscache(struct inode *inode, struct page *page); int ceph_readpage_from_fscache(struct inode *inode, struct page *page);
int ceph_readpages_from_fscache(struct inode *inode, int ceph_readpages_from_fscache(struct inode *inode,
...@@ -46,7 +46,12 @@ int ceph_readpages_from_fscache(struct inode *inode, ...@@ -46,7 +46,12 @@ int ceph_readpages_from_fscache(struct inode *inode,
unsigned *nr_pages); unsigned *nr_pages);
void ceph_readpage_to_fscache(struct inode *inode, struct page *page); void ceph_readpage_to_fscache(struct inode *inode, struct page *page);
void ceph_invalidate_fscache_page(struct inode* inode, struct page *page); void ceph_invalidate_fscache_page(struct inode* inode, struct page *page);
void ceph_queue_revalidate(struct inode *inode);
static inline void ceph_fscache_inode_init(struct ceph_inode_info *ci)
{
ci->fscache = NULL;
ci->i_fscache_gen = 0;
}
static inline void ceph_fscache_invalidate(struct inode *inode) static inline void ceph_fscache_invalidate(struct inode *inode)
{ {
...@@ -82,6 +87,11 @@ static inline void ceph_fscache_readpages_cancel(struct inode *inode, ...@@ -82,6 +87,11 @@ static inline void ceph_fscache_readpages_cancel(struct inode *inode,
return fscache_readpages_cancel(ci->fscache, pages); return fscache_readpages_cancel(ci->fscache, pages);
} }
static inline void ceph_disable_fscache_readpage(struct ceph_inode_info *ci)
{
ci->i_fscache_gen = ci->i_rdcache_gen - 1;
}
#else #else
static inline int ceph_fscache_register(void) static inline int ceph_fscache_register(void)
...@@ -119,6 +129,10 @@ static inline void ceph_fscache_file_set_cookie(struct inode *inode, ...@@ -119,6 +129,10 @@ static inline void ceph_fscache_file_set_cookie(struct inode *inode,
{ {
} }
static inline void ceph_fscache_revalidate_cookie(struct ceph_inode_info *ci)
{
}
static inline void ceph_fscache_uncache_page(struct inode *inode, static inline void ceph_fscache_uncache_page(struct inode *inode,
struct page *pages) struct page *pages)
{ {
...@@ -167,7 +181,7 @@ static inline void ceph_fscache_readpages_cancel(struct inode *inode, ...@@ -167,7 +181,7 @@ static inline void ceph_fscache_readpages_cancel(struct inode *inode,
{ {
} }
static inline void ceph_queue_revalidate(struct inode *inode) static inline void ceph_disable_fscache_readpage(struct ceph_inode_info *ci)
{ {
} }
......
...@@ -2393,6 +2393,9 @@ static int try_get_cap_refs(struct ceph_inode_info *ci, int need, int want, ...@@ -2393,6 +2393,9 @@ static int try_get_cap_refs(struct ceph_inode_info *ci, int need, int want,
snap_rwsem_locked = true; snap_rwsem_locked = true;
} }
*got = need | (have & want); *got = need | (have & want);
if ((need & CEPH_CAP_FILE_RD) &&
!(*got & CEPH_CAP_FILE_CACHE))
ceph_disable_fscache_readpage(ci);
__take_cap_refs(ci, *got, true); __take_cap_refs(ci, *got, true);
ret = 1; ret = 1;
} }
...@@ -2554,6 +2557,9 @@ int ceph_get_caps(struct ceph_inode_info *ci, int need, int want, ...@@ -2554,6 +2557,9 @@ int ceph_get_caps(struct ceph_inode_info *ci, int need, int want,
break; break;
} }
if ((_got & CEPH_CAP_FILE_RD) && (_got & CEPH_CAP_FILE_CACHE))
ceph_fscache_revalidate_cookie(ci);
*got = _got; *got = _got;
return 0; return 0;
} }
...@@ -2795,7 +2801,6 @@ static void handle_cap_grant(struct ceph_mds_client *mdsc, ...@@ -2795,7 +2801,6 @@ static void handle_cap_grant(struct ceph_mds_client *mdsc,
bool writeback = false; bool writeback = false;
bool queue_trunc = false; bool queue_trunc = false;
bool queue_invalidate = false; bool queue_invalidate = false;
bool queue_revalidate = false;
bool deleted_inode = false; bool deleted_inode = false;
bool fill_inline = false; bool fill_inline = false;
...@@ -2837,8 +2842,6 @@ static void handle_cap_grant(struct ceph_mds_client *mdsc, ...@@ -2837,8 +2842,6 @@ static void handle_cap_grant(struct ceph_mds_client *mdsc,
ci->i_rdcache_revoking = ci->i_rdcache_gen; ci->i_rdcache_revoking = ci->i_rdcache_gen;
} }
} }
ceph_fscache_invalidate(inode);
} }
/* side effects now are allowed */ /* side effects now are allowed */
...@@ -2880,11 +2883,6 @@ static void handle_cap_grant(struct ceph_mds_client *mdsc, ...@@ -2880,11 +2883,6 @@ static void handle_cap_grant(struct ceph_mds_client *mdsc,
} }
} }
/* Do we need to revalidate our fscache cookie. Don't bother on the
* first cache cap as we already validate at cookie creation time. */
if ((issued & CEPH_CAP_FILE_CACHE) && ci->i_rdcache_gen > 1)
queue_revalidate = true;
if (newcaps & CEPH_CAP_ANY_RD) { if (newcaps & CEPH_CAP_ANY_RD) {
/* ctime/mtime/atime? */ /* ctime/mtime/atime? */
ceph_decode_timespec(&mtime, &grant->mtime); ceph_decode_timespec(&mtime, &grant->mtime);
...@@ -2995,8 +2993,6 @@ static void handle_cap_grant(struct ceph_mds_client *mdsc, ...@@ -2995,8 +2993,6 @@ static void handle_cap_grant(struct ceph_mds_client *mdsc,
if (queue_trunc) if (queue_trunc)
ceph_queue_vmtruncate(inode); ceph_queue_vmtruncate(inode);
else if (queue_revalidate)
ceph_queue_revalidate(inode);
if (writeback) if (writeback)
/* /*
......
...@@ -103,7 +103,6 @@ struct ceph_fs_client { ...@@ -103,7 +103,6 @@ struct ceph_fs_client {
#ifdef CONFIG_CEPH_FSCACHE #ifdef CONFIG_CEPH_FSCACHE
struct fscache_cookie *fscache; struct fscache_cookie *fscache;
struct workqueue_struct *revalidate_wq;
#endif #endif
}; };
...@@ -360,8 +359,7 @@ struct ceph_inode_info { ...@@ -360,8 +359,7 @@ struct ceph_inode_info {
#ifdef CONFIG_CEPH_FSCACHE #ifdef CONFIG_CEPH_FSCACHE
struct fscache_cookie *fscache; struct fscache_cookie *fscache;
u32 i_fscache_gen; /* sequence, for delayed fscache validate */ u32 i_fscache_gen;
struct work_struct i_revalidate_work;
#endif #endif
struct inode vfs_inode; /* at end */ struct inode vfs_inode; /* at end */
}; };
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment