From 270b5ec34970b33b20b04f397cefbea81a6702ba Mon Sep 17 00:00:00 2001 From: visa Date: Thu, 20 Apr 2017 13:33:00 +0000 Subject: [PATCH] Hook up rwlock(9) to witness(4). Loosely based on a diff from Christian Ludwig --- sys/kern/kern_rwlock.c | 100 ++++++++++++++++++++++++++++++++--------- sys/kern/kern_synch.c | 10 ++++- sys/sys/rwlock.h | 95 +++++++++++++++++++++++++++++++++------ 3 files changed, 168 insertions(+), 37 deletions(-) diff --git a/sys/kern/kern_rwlock.c b/sys/kern/kern_rwlock.c index bdaee0c1ae7..d202a01e16d 100644 --- a/sys/kern/kern_rwlock.c +++ b/sys/kern/kern_rwlock.c @@ -1,4 +1,4 @@ -/* $OpenBSD: kern_rwlock.c,v 1.27 2015/03/14 07:33:42 jsg Exp $ */ +/* $OpenBSD: kern_rwlock.c,v 1.28 2017/04/20 13:33:00 visa Exp $ */ /* * Copyright (c) 2002, 2003 Artur Grabowski @@ -23,6 +23,7 @@ #include #include #include +#include /* XXX - temporary measure until proc0 is properly aligned */ #define RW_PROC(p) (((long)p) & ~RWLOCK_MASK) @@ -87,31 +88,39 @@ static const struct rwlock_op { }; void -rw_enter_read(struct rwlock *rwl) +_rw_enter_read(struct rwlock *rwl LOCK_FL_VARS) { unsigned long owner = rwl->rwl_owner; if (__predict_false((owner & RWLOCK_WRLOCK) || rw_cas(&rwl->rwl_owner, owner, owner + RWLOCK_READ_INCR))) - rw_enter(rwl, RW_READ); - else + _rw_enter(rwl, RW_READ LOCK_FL_ARGS); + else { membar_enter(); + WITNESS_CHECKORDER(&rwl->rwl_lock_obj, LOP_NEWORDER, file, line, + NULL); + WITNESS_LOCK(&rwl->rwl_lock_obj, 0, file, line); + } } void -rw_enter_write(struct rwlock *rwl) +_rw_enter_write(struct rwlock *rwl LOCK_FL_VARS) { struct proc *p = curproc; if (__predict_false(rw_cas(&rwl->rwl_owner, 0, RW_PROC(p) | RWLOCK_WRLOCK))) - rw_enter(rwl, RW_WRITE); - else + _rw_enter(rwl, RW_WRITE LOCK_FL_ARGS); + else { membar_enter(); + WITNESS_CHECKORDER(&rwl->rwl_lock_obj, + LOP_EXCLUSIVE | LOP_NEWORDER, file, line, NULL); + WITNESS_LOCK(&rwl->rwl_lock_obj, LOP_EXCLUSIVE, file, line); + } } void -rw_exit_read(struct rwlock *rwl) +_rw_exit_read(struct rwlock *rwl LOCK_FL_VARS) { unsigned long owner = rwl->rwl_owner; @@ -120,11 +129,13 @@ rw_exit_read(struct rwlock *rwl) membar_exit(); if (__predict_false((owner & RWLOCK_WAIT) || rw_cas(&rwl->rwl_owner, owner, owner - RWLOCK_READ_INCR))) - rw_exit(rwl); + _rw_exit(rwl LOCK_FL_ARGS); + else + WITNESS_UNLOCK(&rwl->rwl_lock_obj, 0, file, line); } void -rw_exit_write(struct rwlock *rwl) +_rw_exit_write(struct rwlock *rwl LOCK_FL_VARS) { unsigned long owner = rwl->rwl_owner; @@ -133,7 +144,9 @@ rw_exit_write(struct rwlock *rwl) membar_exit(); if (__predict_false((owner & RWLOCK_WAIT) || rw_cas(&rwl->rwl_owner, owner, 0))) - rw_exit(rwl); + _rw_exit(rwl LOCK_FL_ARGS); + else + WITNESS_UNLOCK(&rwl->rwl_lock_obj, LOP_EXCLUSIVE, file, line); } #ifdef DIAGNOSTIC @@ -172,20 +185,48 @@ rw_enter_diag(struct rwlock *rwl, int flags) #define rw_enter_diag(r, f) #endif -void -rw_init(struct rwlock *rwl, const char *name) +static void +_rw_init_flags_witness(struct rwlock *rwl, const char *name, int lo_flags, + struct lock_type *type) { rwl->rwl_owner = 0; rwl->rwl_name = name; + +#ifdef WITNESS + rwl->rwl_lock_obj.lo_flags = lo_flags; + rwl->rwl_lock_obj.lo_name = name; + rwl->rwl_lock_obj.lo_type = type; + WITNESS_INIT(&rwl->rwl_lock_obj, type); +#else + (void)type; + (void)lo_flags; +#endif +} + +void +_rw_init_flags(struct rwlock *rwl, const char *name, int flags, + struct lock_type *type) +{ + _rw_init_flags_witness(rwl, name, RWLOCK_LO_FLAGS(flags), type); } int -rw_enter(struct rwlock *rwl, int flags) +_rw_enter(struct rwlock *rwl, int flags LOCK_FL_VARS) { const struct rwlock_op *op; struct sleep_state sls; unsigned long inc, o; int error; +#ifdef WITNESS + int lop_flags; + + lop_flags = LOP_NEWORDER; + if (flags & RW_WRITE) + lop_flags |= LOP_EXCLUSIVE; + if ((flags & RW_NOSLEEP) == 0 && (flags & RW_DOWNGRADE) == 0) + WITNESS_CHECKORDER(&rwl->rwl_lock_obj, lop_flags, file, line, + NULL); +#endif op = &rw_ops[(flags & RW_OPMASK) - 1]; @@ -227,11 +268,16 @@ retry: (RWLOCK_WRLOCK|RWLOCK_WAIT))) wakeup(rwl); + if (flags & RW_DOWNGRADE) + WITNESS_DOWNGRADE(&rwl->rwl_lock_obj, lop_flags, file, line); + else + WITNESS_LOCK(&rwl->rwl_lock_obj, lop_flags, file, line); + return (0); } void -rw_exit(struct rwlock *rwl) +_rw_exit(struct rwlock *rwl LOCK_FL_VARS) { unsigned long owner = rwl->rwl_owner; int wrlock = owner & RWLOCK_WRLOCK; @@ -242,6 +288,9 @@ rw_exit(struct rwlock *rwl) else rw_assert_rdlock(rwl); + WITNESS_UNLOCK(&rwl->rwl_lock_obj, wrlock ? LOP_EXCLUSIVE : 0, + file, line); + membar_exit(); do { owner = rwl->rwl_owner; @@ -298,14 +347,16 @@ rw_assert_unlocked(struct rwlock *rwl) /* recursive rwlocks; */ void -rrw_init(struct rrwlock *rrwl, char *name) +_rrw_init_flags(struct rrwlock *rrwl, char *name, int flags, + struct lock_type *type) { memset(rrwl, 0, sizeof(struct rrwlock)); - rw_init(&rrwl->rrwl_lock, name); + _rw_init_flags_witness(&rrwl->rrwl_lock, name, RRWLOCK_LO_FLAGS(flags), + type); } int -rrw_enter(struct rrwlock *rrwl, int flags) +_rrw_enter(struct rrwlock *rrwl, int flags LOCK_FL_VARS) { int rv; @@ -315,11 +366,13 @@ rrw_enter(struct rrwlock *rrwl, int flags) return (EDEADLK); else { rrwl->rrwl_wcnt++; + WITNESS_LOCK(&rrwl->rrwl_lock.rwl_lock_obj, + LOP_EXCLUSIVE, file, line); return (0); } } - rv = rw_enter(&rrwl->rrwl_lock, flags); + rv = _rw_enter(&rrwl->rrwl_lock, flags LOCK_FL_ARGS); if (rv == 0) rrwl->rrwl_wcnt = 1; @@ -327,18 +380,21 @@ rrw_enter(struct rrwlock *rrwl, int flags) } void -rrw_exit(struct rrwlock *rrwl) +_rrw_exit(struct rrwlock *rrwl LOCK_FL_VARS) { if (RWLOCK_OWNER(&rrwl->rrwl_lock) == (struct proc *)RW_PROC(curproc)) { KASSERT(rrwl->rrwl_wcnt > 0); rrwl->rrwl_wcnt--; - if (rrwl->rrwl_wcnt != 0) + if (rrwl->rrwl_wcnt != 0) { + WITNESS_UNLOCK(&rrwl->rrwl_lock.rwl_lock_obj, + LOP_EXCLUSIVE, file, line); return; + } } - rw_exit(&rrwl->rrwl_lock); + _rw_exit(&rrwl->rrwl_lock LOCK_FL_ARGS); } int diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index a2e7347d74b..79ff976529a 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -1,4 +1,4 @@ -/* $OpenBSD: kern_synch.c,v 1.138 2017/01/31 12:16:20 mpi Exp $ */ +/* $OpenBSD: kern_synch.c,v 1.139 2017/04/20 13:33:00 visa Exp $ */ /* $NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $ */ /* @@ -50,6 +50,7 @@ #include #include #include +#include #include #include @@ -236,6 +237,7 @@ rwsleep(const volatile void *ident, struct rwlock *wl, int priority, { struct sleep_state sls; int error, error1; + WITNESS_SAVE_DECL(lock_fl); KASSERT((priority & ~(PRIMASK | PCATCH | PNORELOCK)) == 0); rw_assert_wrlock(wl); @@ -244,14 +246,18 @@ rwsleep(const volatile void *ident, struct rwlock *wl, int priority, sleep_setup_timeout(&sls, timo); sleep_setup_signal(&sls, priority); + WITNESS_SAVE(&wl->rwl_lock_obj, lock_fl); + rw_exit_write(wl); sleep_finish(&sls, 1); error1 = sleep_finish_timeout(&sls); error = sleep_finish_signal(&sls); - if ((priority & PNORELOCK) == 0) + if ((priority & PNORELOCK) == 0) { rw_enter_write(wl); + WITNESS_RESTORE(&wl->rwl_lock_obj, lock_fl); + } /* Signal errors are higher priority than timeouts. */ if (error == 0 && error1 != 0) diff --git a/sys/sys/rwlock.h b/sys/sys/rwlock.h index 439d5adf792..06c6a031c39 100644 --- a/sys/sys/rwlock.h +++ b/sys/sys/rwlock.h @@ -1,4 +1,4 @@ -/* $OpenBSD: rwlock.h,v 1.20 2016/09/21 10:19:13 dlg Exp $ */ +/* $OpenBSD: rwlock.h,v 1.21 2017/04/20 13:33:00 visa Exp $ */ /* * Copyright (c) 2002 Artur Grabowski * @@ -54,14 +54,48 @@ #ifndef _SYS_RWLOCK_H #define _SYS_RWLOCK_H +#include + struct proc; struct rwlock { volatile unsigned long rwl_owner; const char *rwl_name; +#ifdef WITNESS + struct lock_object rwl_lock_obj; +#endif }; -#define RWLOCK_INITIALIZER(name) { 0, name } +#define RWLOCK_LO_FLAGS(flags) \ + ((ISSET(flags, RWL_DUPOK) ? LO_DUPOK : 0) | \ + (ISSET(flags, RWL_NOWITNESS) ? 0 : LO_WITNESS) | \ + (ISSET(flags, RWL_IS_VNODE) ? LO_IS_VNODE : 0) | \ + LO_INITIALIZED | LO_SLEEPABLE | LO_UPGRADABLE | \ + (LO_CLASS_RWLOCK << LO_CLASSSHIFT)) + +#define RRWLOCK_LO_FLAGS(flags) \ + ((ISSET(flags, RWL_DUPOK) ? LO_DUPOK : 0) | \ + (ISSET(flags, RWL_NOWITNESS) ? 0 : LO_WITNESS) | \ + (ISSET(flags, RWL_IS_VNODE) ? LO_IS_VNODE : 0) | \ + LO_INITIALIZED | LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE | \ + (LO_CLASS_RRWLOCK << LO_CLASSSHIFT)) + +#define RWLOCK_LO_INITIALIZER(name, flags) \ + { .lo_type = &(struct lock_type){ .lt_name = name }, \ + .lo_name = (name), \ + .lo_flags = RWLOCK_LO_FLAGS(flags) } + +#define RWL_DUPOK 0x01 +#define RWL_NOWITNESS 0x02 +#define RWL_IS_VNODE 0x04 + +#ifdef WITNESS +#define RWLOCK_INITIALIZER(name) \ + { 0, name, .rwl_lock_obj = RWLOCK_LO_INITIALIZER(name, 0) } +#else +#define RWLOCK_INITIALIZER(name) \ + { 0, name } +#endif #define RWLOCK_WAIT 0x01UL #define RWLOCK_WRWANT 0x02UL @@ -97,12 +131,29 @@ struct rrwlock { #ifdef _KERNEL -void rw_init(struct rwlock *, const char *); - -void rw_enter_read(struct rwlock *); -void rw_enter_write(struct rwlock *); -void rw_exit_read(struct rwlock *); -void rw_exit_write(struct rwlock *); +void _rw_init_flags(struct rwlock *, const char *, int, struct lock_type *); + +#ifdef WITNESS +#define rw_init_flags(rwl, name, flags) do { \ + static struct lock_type __lock_type = { .lt_name = #rwl }; \ + _rw_init_flags(rwl, name, flags, &__lock_type); \ +} while (0) +#define rw_init(rwl, name) rw_init_flags(rwl, name, 0) +#else /* WITNESS */ +#define rw_init_flags(rwl, name, flags) \ + _rw_init_flags(rwl, name, flags, NULL) +#define rw_init(rwl, name) _rw_init_flags(rwl, name, 0, NULL) +#endif /* WITNESS */ + +void _rw_enter_read(struct rwlock * LOCK_FL_VARS); +void _rw_enter_write(struct rwlock * LOCK_FL_VARS); +void _rw_exit_read(struct rwlock * LOCK_FL_VARS); +void _rw_exit_write(struct rwlock * LOCK_FL_VARS); + +#define rw_enter_read(rwl) _rw_enter_read(rwl LOCK_FILE_LINE) +#define rw_enter_write(rwl) _rw_enter_write(rwl LOCK_FILE_LINE) +#define rw_exit_read(rwl) _rw_exit_read(rwl LOCK_FILE_LINE) +#define rw_exit_write(rwl) _rw_exit_write(rwl LOCK_FILE_LINE) #ifdef DIAGNOSTIC void rw_assert_wrlock(struct rwlock *); @@ -114,15 +165,33 @@ void rw_assert_unlocked(struct rwlock *); #define rw_assert_unlocked(rwl) ((void)0) #endif -int rw_enter(struct rwlock *, int); -void rw_exit(struct rwlock *); +int _rw_enter(struct rwlock *, int LOCK_FL_VARS); +void _rw_exit(struct rwlock * LOCK_FL_VARS); int rw_status(struct rwlock *); -void rrw_init(struct rrwlock *, char *); -int rrw_enter(struct rrwlock *, int); -void rrw_exit(struct rrwlock *); +#define rw_enter(rwl, flags) _rw_enter(rwl, flags LOCK_FILE_LINE) +#define rw_exit(rwl) _rw_exit(rwl LOCK_FILE_LINE) + +void _rrw_init_flags(struct rrwlock *, char *, int, struct lock_type *); +int _rrw_enter(struct rrwlock *, int LOCK_FL_VARS); +void _rrw_exit(struct rrwlock * LOCK_FL_VARS); int rrw_status(struct rrwlock *); +#ifdef WITNESS +#define rrw_init_flags(rrwl, name, flags) do { \ + static struct lock_type __lock_type = { .lt_name = #rrwl }; \ + _rrw_init_flags(rrwl, name, flags, &__lock_type); \ +} while (0) +#define rrw_init(rrwl, name) rrw_init_flags(rrwl, name, 0) +#else /* WITNESS */ +#define rrw_init_flags(rrwl, name, flags) \ + _rrw_init_flags(rrwl, name, 0, NULL) +#define rrw_init(rrwl, name) _rrw_init_flags(rrwl, name, 0, NULL) +#endif /* WITNESS */ + +#define rrw_enter(rrwl, flags) _rrw_enter(rrwl, flags LOCK_FILE_LINE) +#define rrw_exit(rrwl) _rrw_exit(rrwl LOCK_FILE_LINE) + #endif /* _KERNEL */ #endif /* _SYS_RWLOCK_H */ -- 2.20.1