aboutsummaryrefslogtreecommitdiffstats
path: root/fs/dcache.c
diff options
context:
space:
mode:
Diffstat (limited to 'fs/dcache.c')
-rw-r--r--fs/dcache.c56
1 files changed, 39 insertions, 17 deletions
diff --git a/fs/dcache.c b/fs/dcache.c
index 3fc74e471924..d40e9f1f797d 100644
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -2419,9 +2419,10 @@ EXPORT_SYMBOL(d_rehash);
static inline unsigned start_dir_add(struct inode *dir)
{
+ preempt_disable_rt();
for (;;) {
- unsigned n = dir->i_dir_seq;
- if (!(n & 1) && cmpxchg(&dir->i_dir_seq, n, n + 1) == n)
+ unsigned n = dir->__i_dir_seq;
+ if (!(n & 1) && cmpxchg(&dir->__i_dir_seq, n, n + 1) == n)
return n;
cpu_relax();
}
@@ -2429,26 +2430,30 @@ static inline unsigned start_dir_add(struct inode *dir)
static inline void end_dir_add(struct inode *dir, unsigned n)
{
- smp_store_release(&dir->i_dir_seq, n + 2);
+ smp_store_release(&dir->__i_dir_seq, n + 2);
+ preempt_enable_rt();
}
static void d_wait_lookup(struct dentry *dentry)
{
- if (d_in_lookup(dentry)) {
- DECLARE_WAITQUEUE(wait, current);
- add_wait_queue(dentry->d_wait, &wait);
- do {
- set_current_state(TASK_UNINTERRUPTIBLE);
- spin_unlock(&dentry->d_lock);
- schedule();
- spin_lock(&dentry->d_lock);
- } while (d_in_lookup(dentry));
- }
+ struct swait_queue __wait;
+
+ if (!d_in_lookup(dentry))
+ return;
+
+ INIT_LIST_HEAD(&__wait.task_list);
+ do {
+ prepare_to_swait_exclusive(dentry->d_wait, &__wait, TASK_UNINTERRUPTIBLE);
+ spin_unlock(&dentry->d_lock);
+ schedule();
+ spin_lock(&dentry->d_lock);
+ } while (d_in_lookup(dentry));
+ finish_swait(dentry->d_wait, &__wait);
}
struct dentry *d_alloc_parallel(struct dentry *parent,
const struct qstr *name,
- wait_queue_head_t *wq)
+ struct swait_queue_head *wq)
{
unsigned int hash = name->hash;
struct hlist_bl_head *b = in_lookup_hash(parent, hash);
@@ -2462,7 +2467,7 @@ struct dentry *d_alloc_parallel(struct dentry *parent,
retry:
rcu_read_lock();
- seq = smp_load_acquire(&parent->d_inode->i_dir_seq);
+ seq = smp_load_acquire(&parent->d_inode->__i_dir_seq);
r_seq = read_seqbegin(&rename_lock);
dentry = __d_lookup_rcu(parent, name, &d_seq);
if (unlikely(dentry)) {
@@ -2490,7 +2495,7 @@ retry:
}
hlist_bl_lock(b);
- if (unlikely(READ_ONCE(parent->d_inode->i_dir_seq) != seq)) {
+ if (unlikely(READ_ONCE(parent->d_inode->__i_dir_seq) != seq)) {
hlist_bl_unlock(b);
rcu_read_unlock();
goto retry;
@@ -2563,7 +2568,7 @@ void __d_lookup_done(struct dentry *dentry)
hlist_bl_lock(b);
dentry->d_flags &= ~DCACHE_PAR_LOOKUP;
__hlist_bl_del(&dentry->d_u.d_in_lookup_hash);
- wake_up_all(dentry->d_wait);
+ swake_up_all(dentry->d_wait);
dentry->d_wait = NULL;
hlist_bl_unlock(b);
INIT_HLIST_NODE(&dentry->d_u.d_alias);
@@ -2577,6 +2582,9 @@ static inline void __d_add(struct dentry *dentry, struct inode *inode)
{
struct inode *dir = NULL;
unsigned n;
+#ifdef CONFIG_PREEMPT_RT_FULL
+ preempt_disable();
+#endif
spin_lock(&dentry->d_lock);
if (unlikely(d_in_lookup(dentry))) {
dir = dentry->d_parent->d_inode;
@@ -2595,6 +2603,9 @@ static inline void __d_add(struct dentry *dentry, struct inode *inode)
if (dir)
end_dir_add(dir, n);
spin_unlock(&dentry->d_lock);
+#ifdef CONFIG_PREEMPT_RT_FULL
+ preempt_enable();
+#endif
if (inode)
spin_unlock(&inode->i_lock);
}
@@ -3077,6 +3088,8 @@ __setup("dhash_entries=", set_dhash_entries);
static void __init dcache_init_early(void)
{
+ unsigned int loop;
+
/* If hashes are distributed across NUMA nodes, defer
* hash allocation until vmalloc space is available.
*/
@@ -3093,11 +3106,16 @@ static void __init dcache_init_early(void)
NULL,
0,
0);
+
+ for (loop = 0; loop < (1U << d_hash_shift); loop++)
+ INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
+
d_hash_shift = 32 - d_hash_shift;
}
static void __init dcache_init(void)
{
+ unsigned int loop;
/*
* A constructor could be added for stable state like the lists,
* but it is probably not worth it because of the cache nature
@@ -3121,6 +3139,10 @@ static void __init dcache_init(void)
NULL,
0,
0);
+
+ for (loop = 0; loop < (1U << d_hash_shift); loop++)
+ INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
+
d_hash_shift = 32 - d_hash_shift;
}