Hook up rwlock(9) to witness(4).

Loosely based on a diff from Christian Ludwig

diff --git a/sys/kern/kern_rwlock.c b/sys/kern/kern_rwlock.c
index bdaee0c..d202a01 100644 (file)
--- a/sys/kern/kern_rwlock.c
+++ b/sys/kern/kern_rwlock.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: kern_rwlock.c,v 1.27 2015/03/14 07:33:42 jsg Exp $    */
+/*     $OpenBSD: kern_rwlock.c,v 1.28 2017/04/20 13:33:00 visa Exp $   */
 
 /*
  * Copyright (c) 2002, 2003 Artur Grabowski <art@openbsd.org>
@@ -23,6 +23,7 @@
 #include <sys/rwlock.h>
 #include <sys/limits.h>
 #include <sys/atomic.h>
+#include <sys/witness.h>
 
 /* XXX - temporary measure until proc0 is properly aligned */
 #define RW_PROC(p) (((long)p) & ~RWLOCK_MASK)
@@ -87,31 +88,39 @@ static const struct rwlock_op {
 };
 
 void
-rw_enter_read(struct rwlock *rwl)
+_rw_enter_read(struct rwlock *rwl LOCK_FL_VARS)
 {
        unsigned long owner = rwl->rwl_owner;
 
        if (__predict_false((owner & RWLOCK_WRLOCK) ||
            rw_cas(&rwl->rwl_owner, owner, owner + RWLOCK_READ_INCR)))
-               rw_enter(rwl, RW_READ);
-       else
+               _rw_enter(rwl, RW_READ LOCK_FL_ARGS);
+       else {
                membar_enter();
+               WITNESS_CHECKORDER(&rwl->rwl_lock_obj, LOP_NEWORDER, file, line,
+                   NULL);
+               WITNESS_LOCK(&rwl->rwl_lock_obj, 0, file, line);
+       }
 }
 
 void
-rw_enter_write(struct rwlock *rwl)
+_rw_enter_write(struct rwlock *rwl LOCK_FL_VARS)
 {
        struct proc *p = curproc;
 
        if (__predict_false(rw_cas(&rwl->rwl_owner, 0,
            RW_PROC(p) | RWLOCK_WRLOCK)))
-               rw_enter(rwl, RW_WRITE);
-       else
+               _rw_enter(rwl, RW_WRITE LOCK_FL_ARGS);
+       else {
                membar_enter();
+               WITNESS_CHECKORDER(&rwl->rwl_lock_obj,
+                   LOP_EXCLUSIVE | LOP_NEWORDER, file, line, NULL);
+               WITNESS_LOCK(&rwl->rwl_lock_obj, LOP_EXCLUSIVE, file, line);
+       }
 }
 
 void
-rw_exit_read(struct rwlock *rwl)
+_rw_exit_read(struct rwlock *rwl LOCK_FL_VARS)
 {
        unsigned long owner = rwl->rwl_owner;
 
@@ -120,11 +129,13 @@ rw_exit_read(struct rwlock *rwl)
        membar_exit();
        if (__predict_false((owner & RWLOCK_WAIT) ||
            rw_cas(&rwl->rwl_owner, owner, owner - RWLOCK_READ_INCR)))
-               rw_exit(rwl);
+               _rw_exit(rwl LOCK_FL_ARGS);
+       else
+               WITNESS_UNLOCK(&rwl->rwl_lock_obj, 0, file, line);
 }
 
 void
-rw_exit_write(struct rwlock *rwl)
+_rw_exit_write(struct rwlock *rwl LOCK_FL_VARS)
 {
        unsigned long owner = rwl->rwl_owner;
 
@@ -133,7 +144,9 @@ rw_exit_write(struct rwlock *rwl)
        membar_exit();
        if (__predict_false((owner & RWLOCK_WAIT) ||
            rw_cas(&rwl->rwl_owner, owner, 0)))
-               rw_exit(rwl);
+               _rw_exit(rwl LOCK_FL_ARGS);
+       else
+               WITNESS_UNLOCK(&rwl->rwl_lock_obj, LOP_EXCLUSIVE, file, line);
 }
 
 #ifdef DIAGNOSTIC
@@ -172,20 +185,48 @@ rw_enter_diag(struct rwlock *rwl, int flags)
 #define rw_enter_diag(r, f)
 #endif
 
-void
-rw_init(struct rwlock *rwl, const char *name)
+static void
+_rw_init_flags_witness(struct rwlock *rwl, const char *name, int lo_flags,
+    struct lock_type *type)
 {
        rwl->rwl_owner = 0;
        rwl->rwl_name = name;
+
+#ifdef WITNESS
+       rwl->rwl_lock_obj.lo_flags = lo_flags;
+       rwl->rwl_lock_obj.lo_name = name;
+       rwl->rwl_lock_obj.lo_type = type;
+       WITNESS_INIT(&rwl->rwl_lock_obj, type);
+#else
+       (void)type;
+       (void)lo_flags;
+#endif
+}
+
+void
+_rw_init_flags(struct rwlock *rwl, const char *name, int flags,
+    struct lock_type *type)
+{
+       _rw_init_flags_witness(rwl, name, RWLOCK_LO_FLAGS(flags), type);
 }
 
 int
-rw_enter(struct rwlock *rwl, int flags)
+_rw_enter(struct rwlock *rwl, int flags LOCK_FL_VARS)
 {
        const struct rwlock_op *op;
        struct sleep_state sls;
        unsigned long inc, o;
        int error;
+#ifdef WITNESS
+       int lop_flags;
+
+       lop_flags = LOP_NEWORDER;
+       if (flags & RW_WRITE)
+               lop_flags |= LOP_EXCLUSIVE;
+       if ((flags & RW_NOSLEEP) == 0 && (flags & RW_DOWNGRADE) == 0)
+               WITNESS_CHECKORDER(&rwl->rwl_lock_obj, lop_flags, file, line,
+                   NULL);
+#endif
 
        op = &rw_ops[(flags & RW_OPMASK) - 1];
 
@@ -227,11 +268,16 @@ retry:
            (RWLOCK_WRLOCK|RWLOCK_WAIT)))
                wakeup(rwl);
 
+       if (flags & RW_DOWNGRADE)
+               WITNESS_DOWNGRADE(&rwl->rwl_lock_obj, lop_flags, file, line);
+       else
+               WITNESS_LOCK(&rwl->rwl_lock_obj, lop_flags, file, line);
+
        return (0);
 }
 
 void
-rw_exit(struct rwlock *rwl)
+_rw_exit(struct rwlock *rwl LOCK_FL_VARS)
 {
        unsigned long owner = rwl->rwl_owner;
        int wrlock = owner & RWLOCK_WRLOCK;
@@ -242,6 +288,9 @@ rw_exit(struct rwlock *rwl)
        else
                rw_assert_rdlock(rwl);
 
+       WITNESS_UNLOCK(&rwl->rwl_lock_obj, wrlock ? LOP_EXCLUSIVE : 0,
+           file, line);
+
        membar_exit();
        do {
                owner = rwl->rwl_owner;
@@ -298,14 +347,16 @@ rw_assert_unlocked(struct rwlock *rwl)
 
 /* recursive rwlocks; */
 void
-rrw_init(struct rrwlock *rrwl, char *name)
+_rrw_init_flags(struct rrwlock *rrwl, char *name, int flags,
+    struct lock_type *type)
 {
        memset(rrwl, 0, sizeof(struct rrwlock));
-       rw_init(&rrwl->rrwl_lock, name);
+       _rw_init_flags_witness(&rrwl->rrwl_lock, name, RRWLOCK_LO_FLAGS(flags),
+           type);
 }
 
 int
-rrw_enter(struct rrwlock *rrwl, int flags)
+_rrw_enter(struct rrwlock *rrwl, int flags LOCK_FL_VARS)
 {
        int     rv;
 
@@ -315,11 +366,13 @@ rrw_enter(struct rrwlock *rrwl, int flags)
                        return (EDEADLK);
                else {
                        rrwl->rrwl_wcnt++;
+                       WITNESS_LOCK(&rrwl->rrwl_lock.rwl_lock_obj,
+                           LOP_EXCLUSIVE, file, line);
                        return (0);
                }
        }
 
-       rv = rw_enter(&rrwl->rrwl_lock, flags);
+       rv = _rw_enter(&rrwl->rrwl_lock, flags LOCK_FL_ARGS);
        if (rv == 0)
                rrwl->rrwl_wcnt = 1;
 
@@ -327,18 +380,21 @@ rrw_enter(struct rrwlock *rrwl, int flags)
 }
 
 void
-rrw_exit(struct rrwlock *rrwl)
+_rrw_exit(struct rrwlock *rrwl LOCK_FL_VARS)
 {
 
        if (RWLOCK_OWNER(&rrwl->rrwl_lock) ==
            (struct proc *)RW_PROC(curproc)) {
                KASSERT(rrwl->rrwl_wcnt > 0);
                rrwl->rrwl_wcnt--;
-               if (rrwl->rrwl_wcnt != 0)
+               if (rrwl->rrwl_wcnt != 0) {
+                       WITNESS_UNLOCK(&rrwl->rrwl_lock.rwl_lock_obj,
+                           LOP_EXCLUSIVE, file, line);
                        return;
+               }
        }
 
-       rw_exit(&rrwl->rrwl_lock);
+       _rw_exit(&rrwl->rrwl_lock LOCK_FL_ARGS);
 }
 
 int

diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index a2e7347..79ff976 100644 (file)
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: kern_synch.c,v 1.138 2017/01/31 12:16:20 mpi Exp $    */
+/*     $OpenBSD: kern_synch.c,v 1.139 2017/04/20 13:33:00 visa Exp $   */
 /*     $NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $ */
 
 /*
@@ -50,6 +50,7 @@
 #include <sys/pool.h>
 #include <sys/refcnt.h>
 #include <sys/atomic.h>
+#include <sys/witness.h>
 #include <ddb/db_output.h>
 
 #include <machine/spinlock.h>
@@ -236,6 +237,7 @@ rwsleep(const volatile void *ident, struct rwlock *wl, int priority,
 {
        struct sleep_state sls;
        int error, error1;
+       WITNESS_SAVE_DECL(lock_fl);
 
        KASSERT((priority & ~(PRIMASK | PCATCH | PNORELOCK)) == 0);
        rw_assert_wrlock(wl);
@@ -244,14 +246,18 @@ rwsleep(const volatile void *ident, struct rwlock *wl, int priority,
        sleep_setup_timeout(&sls, timo);
        sleep_setup_signal(&sls, priority);
 
+       WITNESS_SAVE(&wl->rwl_lock_obj, lock_fl);
+
        rw_exit_write(wl);
 
        sleep_finish(&sls, 1);
        error1 = sleep_finish_timeout(&sls);
        error = sleep_finish_signal(&sls);
 
-       if ((priority & PNORELOCK) == 0)
+       if ((priority & PNORELOCK) == 0) {
                rw_enter_write(wl);
+               WITNESS_RESTORE(&wl->rwl_lock_obj, lock_fl);
+       }
 
        /* Signal errors are higher priority than timeouts. */
        if (error == 0 && error1 != 0)

diff --git a/sys/sys/rwlock.h b/sys/sys/rwlock.h
index 439d5ad..06c6a03 100644 (file)
--- a/sys/sys/rwlock.h
+++ b/sys/sys/rwlock.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: rwlock.h,v 1.20 2016/09/21 10:19:13 dlg Exp $ */
+/*     $OpenBSD: rwlock.h,v 1.21 2017/04/20 13:33:00 visa Exp $        */
 /*
  * Copyright (c) 2002 Artur Grabowski <art@openbsd.org>
  *
@@ -54,14 +54,48 @@
 #ifndef _SYS_RWLOCK_H
 #define _SYS_RWLOCK_H
 
+#include <sys/_lock.h>
+
 struct proc;
 
 struct rwlock {
        volatile unsigned long   rwl_owner;
        const char              *rwl_name;
+#ifdef WITNESS
+       struct lock_object       rwl_lock_obj;
+#endif
 };
 
-#define RWLOCK_INITIALIZER(name)       { 0, name }
+#define RWLOCK_LO_FLAGS(flags) \
+       ((ISSET(flags, RWL_DUPOK) ? LO_DUPOK : 0) |                     \
+        (ISSET(flags, RWL_NOWITNESS) ? 0 : LO_WITNESS) |               \
+        (ISSET(flags, RWL_IS_VNODE) ? LO_IS_VNODE : 0) |               \
+        LO_INITIALIZED | LO_SLEEPABLE | LO_UPGRADABLE |                \
+        (LO_CLASS_RWLOCK << LO_CLASSSHIFT))
+
+#define RRWLOCK_LO_FLAGS(flags) \
+       ((ISSET(flags, RWL_DUPOK) ? LO_DUPOK : 0) |                     \
+        (ISSET(flags, RWL_NOWITNESS) ? 0 : LO_WITNESS) |               \
+        (ISSET(flags, RWL_IS_VNODE) ? LO_IS_VNODE : 0) |               \
+        LO_INITIALIZED | LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE | \
+        (LO_CLASS_RRWLOCK << LO_CLASSSHIFT))
+
+#define RWLOCK_LO_INITIALIZER(name, flags) \
+       { .lo_type = &(struct lock_type){ .lt_name = name },            \
+         .lo_name = (name),                                            \
+         .lo_flags = RWLOCK_LO_FLAGS(flags) }
+
+#define RWL_DUPOK              0x01
+#define RWL_NOWITNESS          0x02
+#define RWL_IS_VNODE           0x04
+
+#ifdef WITNESS
+#define RWLOCK_INITIALIZER(name) \
+       { 0, name, .rwl_lock_obj = RWLOCK_LO_INITIALIZER(name, 0) }
+#else
+#define RWLOCK_INITIALIZER(name) \
+       { 0, name }
+#endif
 
 #define RWLOCK_WAIT            0x01UL
 #define RWLOCK_WRWANT          0x02UL
@@ -97,12 +131,29 @@ struct rrwlock {
 
 #ifdef _KERNEL
 
-void   rw_init(struct rwlock *, const char *);
-
-void   rw_enter_read(struct rwlock *);
-void   rw_enter_write(struct rwlock *);
-void   rw_exit_read(struct rwlock *);
-void   rw_exit_write(struct rwlock *);
+void   _rw_init_flags(struct rwlock *, const char *, int, struct lock_type *);
+
+#ifdef WITNESS
+#define rw_init_flags(rwl, name, flags) do {                           \
+       static struct lock_type __lock_type = { .lt_name = #rwl };      \
+       _rw_init_flags(rwl, name, flags, &__lock_type);                 \
+} while (0)
+#define rw_init(rwl, name)     rw_init_flags(rwl, name, 0)
+#else /* WITNESS */
+#define rw_init_flags(rwl, name, flags) \
+                               _rw_init_flags(rwl, name, flags, NULL)
+#define rw_init(rwl, name)     _rw_init_flags(rwl, name, 0, NULL)
+#endif /* WITNESS */
+
+void   _rw_enter_read(struct rwlock * LOCK_FL_VARS);
+void   _rw_enter_write(struct rwlock * LOCK_FL_VARS);
+void   _rw_exit_read(struct rwlock * LOCK_FL_VARS);
+void   _rw_exit_write(struct rwlock * LOCK_FL_VARS);
+
+#define rw_enter_read(rwl)     _rw_enter_read(rwl LOCK_FILE_LINE)
+#define rw_enter_write(rwl)    _rw_enter_write(rwl LOCK_FILE_LINE)
+#define rw_exit_read(rwl)      _rw_exit_read(rwl LOCK_FILE_LINE)
+#define rw_exit_write(rwl)     _rw_exit_write(rwl LOCK_FILE_LINE)
 
 #ifdef DIAGNOSTIC
 void   rw_assert_wrlock(struct rwlock *);
@@ -114,15 +165,33 @@ void      rw_assert_unlocked(struct rwlock *);
 #define rw_assert_unlocked(rwl)        ((void)0)
 #endif
 
-int    rw_enter(struct rwlock *, int);
-void   rw_exit(struct rwlock *);
+int    _rw_enter(struct rwlock *, int LOCK_FL_VARS);
+void   _rw_exit(struct rwlock * LOCK_FL_VARS);
 int    rw_status(struct rwlock *);
 
-void   rrw_init(struct rrwlock *, char *);
-int    rrw_enter(struct rrwlock *, int);
-void   rrw_exit(struct rrwlock *);
+#define rw_enter(rwl, flags)   _rw_enter(rwl, flags LOCK_FILE_LINE)
+#define rw_exit(rwl)           _rw_exit(rwl LOCK_FILE_LINE)
+
+void   _rrw_init_flags(struct rrwlock *, char *, int, struct lock_type *);
+int    _rrw_enter(struct rrwlock *, int LOCK_FL_VARS);
+void   _rrw_exit(struct rrwlock * LOCK_FL_VARS);
 int    rrw_status(struct rrwlock *);
 
+#ifdef WITNESS
+#define rrw_init_flags(rrwl, name, flags) do {                         \
+       static struct lock_type __lock_type = { .lt_name = #rrwl };     \
+       _rrw_init_flags(rrwl, name, flags, &__lock_type);               \
+} while (0)
+#define rrw_init(rrwl, name)   rrw_init_flags(rrwl, name, 0)
+#else /* WITNESS */
+#define rrw_init_flags(rrwl, name, flags) \
+                               _rrw_init_flags(rrwl, name, 0, NULL)
+#define rrw_init(rrwl, name)   _rrw_init_flags(rrwl, name, 0, NULL)
+#endif /* WITNESS */
+
+#define rrw_enter(rrwl, flags) _rrw_enter(rrwl, flags LOCK_FILE_LINE)
+#define rrw_exit(rrwl)         _rrw_exit(rrwl LOCK_FILE_LINE)
+
 #endif /* _KERNEL */
 
 #endif /* _SYS_RWLOCK_H */