Skip to content

Commit 8282947

Browse files
KAGA-KOKOIngo Molnar
KAGA-KOKO
authored and
Ingo Molnar
committed
locking/rwlock: Provide RT variant
Similar to rw_semaphores, on RT the rwlock substitution is not writer fair, because it's not feasible to have a writer inherit its priority to multiple readers. Readers blocked on a writer follow the normal rules of priority inheritance. Like RT spinlocks, RT rwlocks are state preserving across the slow lock operations (contended case). Signed-off-by: Thomas Gleixner <[email protected]> Signed-off-by: Peter Zijlstra (Intel) <[email protected]> Signed-off-by: Ingo Molnar <[email protected]> Link: https://lore.kernel.org/r/[email protected]
1 parent 0f383b6 commit 8282947

File tree

7 files changed

+323
-13
lines changed

7 files changed

+323
-13
lines changed

‎include/linux/rwlock_rt.h

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
// SPDX-License-Identifier: GPL-2.0-only
2+
#ifndef __LINUX_RWLOCK_RT_H
3+
#define __LINUX_RWLOCK_RT_H
4+
5+
#ifndef __LINUX_SPINLOCK_RT_H
6+
#error Do not #include directly. Use <linux/spinlock.h>.
7+
#endif
8+
9+
#ifdef CONFIG_DEBUG_LOCK_ALLOC
10+
extern void __rt_rwlock_init(rwlock_t *rwlock, const char *name,
11+
struct lock_class_key *key);
12+
#else
13+
static inline void __rt_rwlock_init(rwlock_t *rwlock, char *name,
14+
struct lock_class_key *key)
15+
{
16+
}
17+
#endif
18+
19+
#define rwlock_init(rwl) \
20+
do { \
21+
static struct lock_class_key __key; \
22+
\
23+
init_rwbase_rt(&(rwl)->rwbase); \
24+
__rt_rwlock_init(rwl, #rwl, &__key); \
25+
} while (0)
26+
27+
extern void rt_read_lock(rwlock_t *rwlock);
28+
extern int rt_read_trylock(rwlock_t *rwlock);
29+
extern void rt_read_unlock(rwlock_t *rwlock);
30+
extern void rt_write_lock(rwlock_t *rwlock);
31+
extern int rt_write_trylock(rwlock_t *rwlock);
32+
extern void rt_write_unlock(rwlock_t *rwlock);
33+
34+
static __always_inline void read_lock(rwlock_t *rwlock)
35+
{
36+
rt_read_lock(rwlock);
37+
}
38+
39+
static __always_inline void read_lock_bh(rwlock_t *rwlock)
40+
{
41+
local_bh_disable();
42+
rt_read_lock(rwlock);
43+
}
44+
45+
static __always_inline void read_lock_irq(rwlock_t *rwlock)
46+
{
47+
rt_read_lock(rwlock);
48+
}
49+
50+
#define read_lock_irqsave(lock, flags) \
51+
do { \
52+
typecheck(unsigned long, flags); \
53+
rt_read_lock(lock); \
54+
flags = 0; \
55+
} while (0)
56+
57+
#define read_trylock(lock) __cond_lock(lock, rt_read_trylock(lock))
58+
59+
static __always_inline void read_unlock(rwlock_t *rwlock)
60+
{
61+
rt_read_unlock(rwlock);
62+
}
63+
64+
static __always_inline void read_unlock_bh(rwlock_t *rwlock)
65+
{
66+
rt_read_unlock(rwlock);
67+
local_bh_enable();
68+
}
69+
70+
static __always_inline void read_unlock_irq(rwlock_t *rwlock)
71+
{
72+
rt_read_unlock(rwlock);
73+
}
74+
75+
static __always_inline void read_unlock_irqrestore(rwlock_t *rwlock,
76+
unsigned long flags)
77+
{
78+
rt_read_unlock(rwlock);
79+
}
80+
81+
static __always_inline void write_lock(rwlock_t *rwlock)
82+
{
83+
rt_write_lock(rwlock);
84+
}
85+
86+
static __always_inline void write_lock_bh(rwlock_t *rwlock)
87+
{
88+
local_bh_disable();
89+
rt_write_lock(rwlock);
90+
}
91+
92+
static __always_inline void write_lock_irq(rwlock_t *rwlock)
93+
{
94+
rt_write_lock(rwlock);
95+
}
96+
97+
#define write_lock_irqsave(lock, flags) \
98+
do { \
99+
typecheck(unsigned long, flags); \
100+
rt_write_lock(lock); \
101+
flags = 0; \
102+
} while (0)
103+
104+
#define write_trylock(lock) __cond_lock(lock, rt_write_trylock(lock))
105+
106+
#define write_trylock_irqsave(lock, flags) \
107+
({ \
108+
int __locked; \
109+
\
110+
typecheck(unsigned long, flags); \
111+
flags = 0; \
112+
__locked = write_trylock(lock); \
113+
__locked; \
114+
})
115+
116+
static __always_inline void write_unlock(rwlock_t *rwlock)
117+
{
118+
rt_write_unlock(rwlock);
119+
}
120+
121+
static __always_inline void write_unlock_bh(rwlock_t *rwlock)
122+
{
123+
rt_write_unlock(rwlock);
124+
local_bh_enable();
125+
}
126+
127+
static __always_inline void write_unlock_irq(rwlock_t *rwlock)
128+
{
129+
rt_write_unlock(rwlock);
130+
}
131+
132+
static __always_inline void write_unlock_irqrestore(rwlock_t *rwlock,
133+
unsigned long flags)
134+
{
135+
rt_write_unlock(rwlock);
136+
}
137+
138+
#define rwlock_is_contended(lock) (((void)(lock), 0))
139+
140+
#endif /* __LINUX_RWLOCK_RT_H */

‎include/linux/rwlock_types.h

Lines changed: 37 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,19 @@
55
# error "Do not include directly, include spinlock_types.h"
66
#endif
77

8+
#ifdef CONFIG_DEBUG_LOCK_ALLOC
9+
# define RW_DEP_MAP_INIT(lockname) \
10+
.dep_map = { \
11+
.name = #lockname, \
12+
.wait_type_inner = LD_WAIT_CONFIG, \
13+
}
14+
#else
15+
# define RW_DEP_MAP_INIT(lockname)
16+
#endif
17+
18+
#ifndef CONFIG_PREEMPT_RT
819
/*
9-
* include/linux/rwlock_types.h - generic rwlock type definitions
10-
* and initializers
20+
* generic rwlock type definitions and initializers
1121
*
1222
* portions Copyright 2005, Red Hat, Inc., Ingo Molnar
1323
* Released under the General Public License (GPL).
@@ -25,16 +35,6 @@ typedef struct {
2535

2636
#define RWLOCK_MAGIC 0xdeaf1eed
2737

28-
#ifdef CONFIG_DEBUG_LOCK_ALLOC
29-
# define RW_DEP_MAP_INIT(lockname) \
30-
.dep_map = { \
31-
.name = #lockname, \
32-
.wait_type_inner = LD_WAIT_CONFIG, \
33-
}
34-
#else
35-
# define RW_DEP_MAP_INIT(lockname)
36-
#endif
37-
3838
#ifdef CONFIG_DEBUG_SPINLOCK
3939
#define __RW_LOCK_UNLOCKED(lockname) \
4040
(rwlock_t) { .raw_lock = __ARCH_RW_LOCK_UNLOCKED, \
@@ -50,4 +50,29 @@ typedef struct {
5050

5151
#define DEFINE_RWLOCK(x) rwlock_t x = __RW_LOCK_UNLOCKED(x)
5252

53+
#else /* !CONFIG_PREEMPT_RT */
54+
55+
#include <linux/rwbase_rt.h>
56+
57+
typedef struct {
58+
struct rwbase_rt rwbase;
59+
atomic_t readers;
60+
#ifdef CONFIG_DEBUG_LOCK_ALLOC
61+
struct lockdep_map dep_map;
62+
#endif
63+
} rwlock_t;
64+
65+
#define __RWLOCK_RT_INITIALIZER(name) \
66+
{ \
67+
.rwbase = __RWBASE_INITIALIZER(name), \
68+
RW_DEP_MAP_INIT(name) \
69+
}
70+
71+
#define __RW_LOCK_UNLOCKED(name) __RWLOCK_RT_INITIALIZER(name)
72+
73+
#define DEFINE_RWLOCK(name) \
74+
rwlock_t name = __RW_LOCK_UNLOCKED(name)
75+
76+
#endif /* CONFIG_PREEMPT_RT */
77+
5378
#endif /* __LINUX_RWLOCK_TYPES_H */

‎include/linux/spinlock_rt.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,4 +146,6 @@ static inline int spin_is_locked(spinlock_t *lock)
146146

147147
#define assert_spin_locked(lock) BUG_ON(!spin_is_locked(lock))
148148

149+
#include <linux/rwlock_rt.h>
150+
149151
#endif

‎kernel/Kconfig.locks

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ config ARCH_USE_QUEUED_RWLOCKS
251251

252252
config QUEUED_RWLOCKS
253253
def_bool y if ARCH_USE_QUEUED_RWLOCKS
254-
depends on SMP
254+
depends on SMP && !PREEMPT_RT
255255

256256
config ARCH_HAS_MMIOWB
257257
bool

‎kernel/locking/spinlock.c

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,8 +124,11 @@ void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \
124124
* __[spin|read|write]_lock_bh()
125125
*/
126126
BUILD_LOCK_OPS(spin, raw_spinlock);
127+
128+
#ifndef CONFIG_PREEMPT_RT
127129
BUILD_LOCK_OPS(read, rwlock);
128130
BUILD_LOCK_OPS(write, rwlock);
131+
#endif
129132

130133
#endif
131134

@@ -209,6 +212,8 @@ void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock)
209212
EXPORT_SYMBOL(_raw_spin_unlock_bh);
210213
#endif
211214

215+
#ifndef CONFIG_PREEMPT_RT
216+
212217
#ifndef CONFIG_INLINE_READ_TRYLOCK
213218
int __lockfunc _raw_read_trylock(rwlock_t *lock)
214219
{
@@ -353,6 +358,8 @@ void __lockfunc _raw_write_unlock_bh(rwlock_t *lock)
353358
EXPORT_SYMBOL(_raw_write_unlock_bh);
354359
#endif
355360

361+
#endif /* !CONFIG_PREEMPT_RT */
362+
356363
#ifdef CONFIG_DEBUG_LOCK_ALLOC
357364

358365
void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass)

‎kernel/locking/spinlock_debug.c

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
3131

3232
EXPORT_SYMBOL(__raw_spin_lock_init);
3333

34+
#ifndef CONFIG_PREEMPT_RT
3435
void __rwlock_init(rwlock_t *lock, const char *name,
3536
struct lock_class_key *key)
3637
{
@@ -48,6 +49,7 @@ void __rwlock_init(rwlock_t *lock, const char *name,
4849
}
4950

5051
EXPORT_SYMBOL(__rwlock_init);
52+
#endif
5153

5254
static void spin_dump(raw_spinlock_t *lock, const char *msg)
5355
{
@@ -139,6 +141,7 @@ void do_raw_spin_unlock(raw_spinlock_t *lock)
139141
arch_spin_unlock(&lock->raw_lock);
140142
}
141143

144+
#ifndef CONFIG_PREEMPT_RT
142145
static void rwlock_bug(rwlock_t *lock, const char *msg)
143146
{
144147
if (!debug_locks_off())
@@ -228,3 +231,5 @@ void do_raw_write_unlock(rwlock_t *lock)
228231
debug_write_unlock(lock);
229232
arch_write_unlock(&lock->raw_lock);
230233
}
234+
235+
#endif /* !CONFIG_PREEMPT_RT */

0 commit comments

Comments
 (0)