[PATCH 01/17] orangefs: implement xattr cache

From: Martin Brandenburg
Date: Mon Sep 17 2018 - 16:12:59 EST

Next message: Martin Brandenburg: "[PATCH 02/17] orangefs: do not invalidate attributes on inode create"
Previous message: Martin Brandenburg: "[PATCH 05/17] orangefs: hold i_lock during inode_getattr"
In reply to: Martin Brandenburg: "[PATCH 05/17] orangefs: hold i_lock during inode_getattr"
Next in thread: Martin Brandenburg: "[PATCH 02/17] orangefs: do not invalidate attributes on inode create"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

This uses the same timeout as the getattr cache. This substantially
increases performance when writing files with smaller buffer sizes.

When writing, the size is (often) changed, which causes a call to
notify_change which calls security_inode_need_killpriv which needs a
getxattr. Caching it reduces traffic to the server.

Signed-off-by: Martin Brandenburg <martin@xxxxxxxxxxxx>
---
fs/orangefs/inode.c | 1 +
fs/orangefs/orangefs-kernel.h | 10 ++++
fs/orangefs/super.c | 9 +++
fs/orangefs/xattr.c | 104 ++++++++++++++++++++++++++++++++++
4 files changed, 124 insertions(+)

diff --git a/fs/orangefs/inode.c b/fs/orangefs/inode.c
index 31932879b716..a7a8d3647ffe 100644
--- a/fs/orangefs/inode.c
+++ b/fs/orangefs/inode.c
@@ -367,6 +367,7 @@ static int orangefs_set_inode(struct inode *inode, void *data)
struct orangefs_object_kref *ref = (struct orangefs_object_kref *) data;
ORANGEFS_I(inode)->refn.fs_id = ref->fs_id;
ORANGEFS_I(inode)->refn.khandle = ref->khandle;
+ hash_init(ORANGEFS_I(inode)->xattr_cache);
return 0;
}

diff --git a/fs/orangefs/orangefs-kernel.h b/fs/orangefs/orangefs-kernel.h
index 17b24ad6b264..0c76b8899fd1 100644
--- a/fs/orangefs/orangefs-kernel.h
+++ b/fs/orangefs/orangefs-kernel.h
@@ -193,6 +193,8 @@ struct orangefs_inode_s {

unsigned long getattr_time;
u32 getattr_mask;
+
+ DECLARE_HASHTABLE(xattr_cache, 4);
};

/* per superblock private orangefs info */
@@ -217,6 +219,14 @@ struct orangefs_stats {
unsigned long writes;
};

+struct orangefs_cached_xattr {
+ struct hlist_node node;
+ char key[ORANGEFS_MAX_XATTR_NAMELEN];
+ char val[ORANGEFS_MAX_XATTR_VALUELEN];
+ ssize_t length;
+ unsigned long timeout;
+};
+
extern struct orangefs_stats orangefs_stats;

/*
diff --git a/fs/orangefs/super.c b/fs/orangefs/super.c
index dfaee90d30bd..4c36481208f5 100644
--- a/fs/orangefs/super.c
+++ b/fs/orangefs/super.c
@@ -128,6 +128,15 @@ static void orangefs_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
+ struct orangefs_cached_xattr *cx;
+ struct hlist_node *tmp;
+ int i;
+
+ hash_for_each_safe(orangefs_inode->xattr_cache, i, tmp, cx, node) {
+ hlist_del(&cx->node);
+ kfree(cx);
+ }
+
kmem_cache_free(orangefs_inode_cache, orangefs_inode);
}

diff --git a/fs/orangefs/xattr.c b/fs/orangefs/xattr.c
index 03bcb871544d..998c3563bcdd 100644
--- a/fs/orangefs/xattr.c
+++ b/fs/orangefs/xattr.c
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
/*
* (C) 2001 Clemson University and The University of Chicago
+ * Copyright 2018 Omnibond Systems, L.L.C.
*
* See COPYING in top-level directory.
*/
@@ -50,6 +51,35 @@ static inline int convert_to_internal_xattr_flags(int setxattr_flags)
return internal_flag;
}

+static unsigned int xattr_key(const char *key)
+{
+ unsigned int i = 0;
+ while (key)
+ i += *key++;
+ return i % 16;
+}
+
+static struct orangefs_cached_xattr *find_cached_xattr(struct inode *inode,
+ const char *key)
+{
+ struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
+ struct orangefs_cached_xattr *cx;
+ struct hlist_head *h;
+ struct hlist_node *tmp;
+ h = &orangefs_inode->xattr_cache[xattr_key(key)];
+ if (hlist_empty(h))
+ return NULL;
+ hlist_for_each_entry_safe(cx, tmp, h, node) {
+/* if (!time_before(jiffies, cx->timeout)) {
+ hlist_del(&cx->node);
+ kfree(cx);
+ continue;
+ }*/
+ if (!strcmp(cx->key, key))
+ return cx;
+ }
+ return NULL;
+}

/*
* Tries to get a specified key's attributes of a given
@@ -65,6 +95,7 @@ ssize_t orangefs_inode_getxattr(struct inode *inode, const char *name,
{
struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
struct orangefs_kernel_op_s *new_op = NULL;
+ struct orangefs_cached_xattr *cx;
ssize_t ret = -ENOMEM;
ssize_t length = 0;
int fsuid;
@@ -93,6 +124,27 @@ ssize_t orangefs_inode_getxattr(struct inode *inode, const char *name,

down_read(&orangefs_inode->xattr_sem);

+ cx = find_cached_xattr(inode, name);
+ if (cx && time_before(jiffies, cx->timeout)) {
+ if (cx->length == -1) {
+ ret = -ENODATA;
+ goto out_unlock;
+ } else {
+ if (size == 0) {
+ ret = cx->length;
+ goto out_unlock;
+ }
+ if (cx->length > size) {
+ ret = -ERANGE;
+ goto out_unlock;
+ }
+ memcpy(buffer, cx->val, cx->length);
+ memset(buffer + cx->length, 0, size - cx->length);
+ ret = cx->length;
+ goto out_unlock;
+ }
+ }
+
new_op = op_alloc(ORANGEFS_VFS_OP_GETXATTR);
if (!new_op)
goto out_unlock;
@@ -117,6 +169,15 @@ ssize_t orangefs_inode_getxattr(struct inode *inode, const char *name,
" does not exist!\n",
get_khandle_from_ino(inode),
(char *)new_op->upcall.req.getxattr.key);
+ cx = kmalloc(sizeof *cx, GFP_KERNEL);
+ if (cx) {
+ strcpy(cx->key, name);
+ cx->length = -1;
+ cx->timeout = jiffies +
+ orangefs_getattr_timeout_msecs*HZ/1000;
+ hash_add(orangefs_inode->xattr_cache, &cx->node,
+ xattr_key(cx->key));
+ }
}
goto out_release_op;
}
@@ -156,6 +217,23 @@ ssize_t orangefs_inode_getxattr(struct inode *inode, const char *name,

ret = length;

+ if (cx) {
+ strcpy(cx->key, name);
+ memcpy(cx->val, buffer, length);
+ cx->length = length;
+ cx->timeout = jiffies + HZ;
+ } else {
+ cx = kmalloc(sizeof *cx, GFP_KERNEL);
+ if (cx) {
+ strcpy(cx->key, name);
+ memcpy(cx->val, buffer, length);
+ cx->length = length;
+ cx->timeout = jiffies + HZ;
+ hash_add(orangefs_inode->xattr_cache, &cx->node,
+ xattr_key(cx->key));
+ }
+ }
+
out_release_op:
op_release(new_op);
out_unlock:
@@ -168,6 +246,9 @@ static int orangefs_inode_removexattr(struct inode *inode, const char *name,
{
struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
struct orangefs_kernel_op_s *new_op = NULL;
+ struct orangefs_cached_xattr *cx;
+ struct hlist_head *h;
+ struct hlist_node *tmp;
int ret = -ENOMEM;

if (strlen(name) >= ORANGEFS_MAX_XATTR_NAMELEN)
@@ -209,6 +290,16 @@ static int orangefs_inode_removexattr(struct inode *inode, const char *name,
"orangefs_inode_removexattr: returning %d\n", ret);

op_release(new_op);
+
+ h = &orangefs_inode->xattr_cache[xattr_key(name)];
+ hlist_for_each_entry_safe(cx, tmp, h, node) {
+ if (!strcmp(cx->key, name)) {
+ hlist_del(&cx->node);
+ kfree(cx);
+ break;
+ }
+ }
+
out_unlock:
up_write(&orangefs_inode->xattr_sem);
return ret;
@@ -226,6 +317,9 @@ int orangefs_inode_setxattr(struct inode *inode, const char *name,
struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
struct orangefs_kernel_op_s *new_op;
int internal_flag = 0;
+ struct orangefs_cached_xattr *cx;
+ struct hlist_head *h;
+ struct hlist_node *tmp;
int ret = -ENOMEM;

gossip_debug(GOSSIP_XATTR_DEBUG,
@@ -287,6 +381,16 @@ int orangefs_inode_setxattr(struct inode *inode, const char *name,

/* when request is serviced properly, free req op struct */
op_release(new_op);
+
+ h = &orangefs_inode->xattr_cache[xattr_key(name)];
+ hlist_for_each_entry_safe(cx, tmp, h, node) {
+ if (!strcmp(cx->key, name)) {
+ hlist_del(&cx->node);
+ kfree(cx);
+ break;
+ }
+ }
+
out_unlock:
up_write(&orangefs_inode->xattr_sem);
return ret;
--
2.19.0

Next message: Martin Brandenburg: "[PATCH 02/17] orangefs: do not invalidate attributes on inode create"
Previous message: Martin Brandenburg: "[PATCH 05/17] orangefs: hold i_lock during inode_getattr"
In reply to: Martin Brandenburg: "[PATCH 05/17] orangefs: hold i_lock during inode_getattr"
Next in thread: Martin Brandenburg: "[PATCH 02/17] orangefs: do not invalidate attributes on inode create"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]