From 25753472864103d052de1cff23b9cb1e66ef307c Mon Sep 17 00:00:00 2001 From: visa Date: Thu, 20 Apr 2017 13:20:17 +0000 Subject: [PATCH] Hook up mplock to witness(4) on amd64 and i386. --- sys/arch/amd64/amd64/lock_machdep.c | 43 ++++++++++++++++++++++------ sys/arch/amd64/include/mplock.h | 44 ++++++++++++++++++++++++----- sys/arch/i386/i386/lock_machdep.c | 43 ++++++++++++++++++++++------ sys/arch/i386/include/mplock.h | 44 ++++++++++++++++++++++++----- sys/kern/kern_lock.c | 30 ++++++++++++++++++-- sys/sys/systm.h | 6 ++-- 6 files changed, 175 insertions(+), 35 deletions(-) diff --git a/sys/arch/amd64/amd64/lock_machdep.c b/sys/arch/amd64/amd64/lock_machdep.c index f02efe0bb04..937ca7c5272 100644 --- a/sys/arch/amd64/amd64/lock_machdep.c +++ b/sys/arch/amd64/amd64/lock_machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: lock_machdep.c,v 1.12 2017/04/16 14:28:07 visa Exp $ */ +/* $OpenBSD: lock_machdep.c,v 1.13 2017/04/20 13:20:17 visa Exp $ */ /* * Copyright (c) 2007 Artur Grabowski @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include #include @@ -27,7 +29,7 @@ #include void -__mp_lock_init(struct __mp_lock *mpl) +___mp_lock_init(struct __mp_lock *mpl) { memset(mpl->mpl_cpus, 0, sizeof(mpl->mpl_cpus)); mpl->mpl_users = 0; @@ -65,10 +67,18 @@ __mp_lock_spin(struct __mp_lock *mpl, u_int me) } void -__mp_lock(struct __mp_lock *mpl) +___mp_lock(struct __mp_lock *mpl LOCK_FL_VARS) { struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[cpu_number()]; long rf = read_rflags(); +#ifdef WITNESS + int lock_held; + + lock_held = __mp_lock_held(mpl); + if (!lock_held) + WITNESS_CHECKORDER(&mpl->mpl_lock_obj, + LOP_EXCLUSIVE | LOP_NEWORDER, file, line, NULL); +#endif disable_intr(); if (cpu->mplc_depth++ == 0) @@ -76,10 +86,12 @@ __mp_lock(struct __mp_lock *mpl) write_rflags(rf); __mp_lock_spin(mpl, cpu->mplc_ticket); + + WITNESS_LOCK(&mpl->mpl_lock_obj, LOP_EXCLUSIVE, file, line); } void -__mp_unlock(struct __mp_lock *mpl) +___mp_unlock(struct __mp_lock *mpl LOCK_FL_VARS) { struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[cpu_number()]; long rf = read_rflags(); @@ -91,6 +103,8 @@ __mp_unlock(struct __mp_lock *mpl) } #endif + WITNESS_UNLOCK(&mpl->mpl_lock_obj, LOP_EXCLUSIVE, file, line); + disable_intr(); if (--cpu->mplc_depth == 0) mpl->mpl_ticket++; @@ -98,14 +112,21 @@ __mp_unlock(struct __mp_lock *mpl) } int -__mp_release_all(struct __mp_lock *mpl) +___mp_release_all(struct __mp_lock *mpl LOCK_FL_VARS) { struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[cpu_number()]; long rf = read_rflags(); int rv; +#ifdef WITNESS + int i; +#endif disable_intr(); rv = cpu->mplc_depth; +#ifdef WITNESS + for (i = 0; i < rv; i++) + WITNESS_UNLOCK(&mpl->mpl_lock_obj, LOP_EXCLUSIVE, file, line); +#endif cpu->mplc_depth = 0; mpl->mpl_ticket++; write_rflags(rf); @@ -114,10 +135,16 @@ __mp_release_all(struct __mp_lock *mpl) } int -__mp_release_all_but_one(struct __mp_lock *mpl) +___mp_release_all_but_one(struct __mp_lock *mpl LOCK_FL_VARS) { struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[cpu_number()]; int rv = cpu->mplc_depth - 1; +#ifdef WITNESS + int i; + + for (i = 0; i < rv; i++) + WITNESS_UNLOCK(&mpl->mpl_lock_obj, LOP_EXCLUSIVE, file, line); +#endif #ifdef MP_LOCKDEBUG if (!__mp_lock_held(mpl)) { @@ -132,10 +159,10 @@ __mp_release_all_but_one(struct __mp_lock *mpl) } void -__mp_acquire_count(struct __mp_lock *mpl, int count) +___mp_acquire_count(struct __mp_lock *mpl, int count LOCK_FL_VARS) { while (count--) - __mp_lock(mpl); + ___mp_lock(mpl LOCK_FL_ARGS); } int diff --git a/sys/arch/amd64/include/mplock.h b/sys/arch/amd64/include/mplock.h index 118fabe0a2a..2caf7439f32 100644 --- a/sys/arch/amd64/include/mplock.h +++ b/sys/arch/amd64/include/mplock.h @@ -1,4 +1,4 @@ -/* $OpenBSD: mplock.h,v 1.3 2014/03/14 02:08:57 dlg Exp $ */ +/* $OpenBSD: mplock.h,v 1.4 2017/04/20 13:20:17 visa Exp $ */ /* * Copyright (c) 2004 Niklas Hallqvist. All rights reserved. @@ -27,6 +27,8 @@ #ifndef _MACHINE_MPLOCK_H_ #define _MACHINE_MPLOCK_H_ +#include + struct __mp_lock_cpu { u_int mplc_ticket; u_int mplc_depth; @@ -36,18 +38,46 @@ struct __mp_lock { struct __mp_lock_cpu mpl_cpus[MAXCPUS]; volatile u_int mpl_ticket; u_int mpl_users; +#ifdef WITNESS + struct lock_object mpl_lock_obj; +#endif }; #ifndef _LOCORE -void __mp_lock_init(struct __mp_lock *); -void __mp_lock(struct __mp_lock *); -void __mp_unlock(struct __mp_lock *); -int __mp_release_all(struct __mp_lock *); -int __mp_release_all_but_one(struct __mp_lock *); -void __mp_acquire_count(struct __mp_lock *, int); +void ___mp_lock_init(struct __mp_lock *); +void ___mp_lock(struct __mp_lock * LOCK_FL_VARS); +void ___mp_unlock(struct __mp_lock * LOCK_FL_VARS); +int ___mp_release_all(struct __mp_lock * LOCK_FL_VARS); +int ___mp_release_all_but_one(struct __mp_lock * LOCK_FL_VARS); +void ___mp_acquire_count(struct __mp_lock *, int LOCK_FL_VARS); int __mp_lock_held(struct __mp_lock *); +#ifdef WITNESS + +void _mp_lock_init(struct __mp_lock *, struct lock_type *); + +#define __mp_lock_init(mpl) do { \ + static struct lock_type __lock_type = { .lt_name = #mpl }; \ + _mp_lock_init((mpl), &__lock_type); \ +} while (0) + +#else /* WITNESS */ + +#define __mp_lock_init ___mp_lock_init + +#endif /* WITNESS */ + +#define __mp_lock(mpl) ___mp_lock((mpl) LOCK_FILE_LINE) +#define __mp_unlock(mpl) ___mp_unlock((mpl) LOCK_FILE_LINE) + +#define __mp_release_all(mpl) \ + ___mp_release_all((mpl) LOCK_FILE_LINE) +#define __mp_release_all_but_one(mpl) \ + ___mp_release_all_but_one((mpl) LOCK_FILE_LINE) +#define __mp_acquire_count(mpl, count) \ + ___mp_acquire_count((mpl), (count) LOCK_FILE_LINE) + #endif #endif /* !_MACHINE_MPLOCK_H */ diff --git a/sys/arch/i386/i386/lock_machdep.c b/sys/arch/i386/i386/lock_machdep.c index 9ab26924484..4439df8f7a5 100644 --- a/sys/arch/i386/i386/lock_machdep.c +++ b/sys/arch/i386/i386/lock_machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: lock_machdep.c,v 1.21 2017/04/16 14:28:07 visa Exp $ */ +/* $OpenBSD: lock_machdep.c,v 1.22 2017/04/20 13:20:17 visa Exp $ */ /* $NetBSD: lock_machdep.c,v 1.1.2.3 2000/05/03 14:40:30 sommerfeld Exp $ */ /*- @@ -38,6 +38,8 @@ #include #include #include +#include +#include #include #include @@ -46,7 +48,7 @@ #ifdef MULTIPROCESSOR void -__mp_lock_init(struct __mp_lock *mpl) +___mp_lock_init(struct __mp_lock *mpl) { memset(mpl->mpl_cpus, 0, sizeof(mpl->mpl_cpus)); mpl->mpl_users = 0; @@ -84,10 +86,18 @@ __mp_lock_spin(struct __mp_lock *mpl, u_int me) } void -__mp_lock(struct __mp_lock *mpl) +___mp_lock(struct __mp_lock *mpl LOCK_FL_VARS) { struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[cpu_number()]; long ef = read_eflags(); +#ifdef WITNESS + int lock_held; + + lock_held = __mp_lock_held(mpl); + if (!lock_held) + WITNESS_CHECKORDER(&mpl->mpl_lock_obj, + LOP_EXCLUSIVE | LOP_NEWORDER, file, line, NULL); +#endif disable_intr(); if (cpu->mplc_depth++ == 0) @@ -95,10 +105,12 @@ __mp_lock(struct __mp_lock *mpl) write_eflags(ef); __mp_lock_spin(mpl, cpu->mplc_ticket); + + WITNESS_LOCK(&mpl->mpl_lock_obj, LOP_EXCLUSIVE, file, line); } void -__mp_unlock(struct __mp_lock *mpl) +___mp_unlock(struct __mp_lock *mpl LOCK_FL_VARS) { struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[cpu_number()]; int ef = read_eflags(); @@ -110,6 +122,8 @@ __mp_unlock(struct __mp_lock *mpl) } #endif + WITNESS_UNLOCK(&mpl->mpl_lock_obj, LOP_EXCLUSIVE, file, line); + disable_intr(); if (--cpu->mplc_depth == 0) mpl->mpl_ticket++; @@ -117,14 +131,21 @@ __mp_unlock(struct __mp_lock *mpl) } int -__mp_release_all(struct __mp_lock *mpl) +___mp_release_all(struct __mp_lock *mpl LOCK_FL_VARS) { struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[cpu_number()]; int ef = read_eflags(); int rv; +#ifdef WITNESS + int i; +#endif disable_intr(); rv = cpu->mplc_depth; +#ifdef WITNESS + for (i = 0; i < rv; i++) + WITNESS_UNLOCK(&mpl->mpl_lock_obj, LOP_EXCLUSIVE, file, line); +#endif cpu->mplc_depth = 0; mpl->mpl_ticket++; write_eflags(ef); @@ -133,10 +154,16 @@ __mp_release_all(struct __mp_lock *mpl) } int -__mp_release_all_but_one(struct __mp_lock *mpl) +___mp_release_all_but_one(struct __mp_lock *mpl LOCK_FL_VARS) { struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[cpu_number()]; int rv = cpu->mplc_depth - 1; +#ifdef WITNESS + int i; + + for (i = 0; i < rv; i++) + WITNESS_UNLOCK(&mpl->mpl_lock_obj, LOP_EXCLUSIVE, file, line); +#endif cpu->mplc_depth = 1; @@ -144,10 +171,10 @@ __mp_release_all_but_one(struct __mp_lock *mpl) } void -__mp_acquire_count(struct __mp_lock *mpl, int count) +___mp_acquire_count(struct __mp_lock *mpl, int count LOCK_FL_VARS) { while (count--) - __mp_lock(mpl); + ___mp_lock(mpl LOCK_FL_ARGS); } int diff --git a/sys/arch/i386/include/mplock.h b/sys/arch/i386/include/mplock.h index 3f7d81fba96..cd166dde690 100644 --- a/sys/arch/i386/include/mplock.h +++ b/sys/arch/i386/include/mplock.h @@ -17,6 +17,8 @@ #ifndef _MACHINE_MPLOCK_H_ #define _MACHINE_MPLOCK_H_ +#include + struct __mp_lock_cpu { u_int mplc_ticket; u_int mplc_depth; @@ -26,18 +28,46 @@ struct __mp_lock { struct __mp_lock_cpu mpl_cpus[MAXCPUS]; volatile u_int mpl_ticket; u_int mpl_users; +#ifdef WITNESS + struct lock_object mpl_lock_obj; +#endif }; #ifndef _LOCORE -void __mp_lock_init(struct __mp_lock *); -void __mp_lock(struct __mp_lock *); -void __mp_unlock(struct __mp_lock *); -int __mp_release_all(struct __mp_lock *); -int __mp_release_all_but_one(struct __mp_lock *); -void __mp_acquire_count(struct __mp_lock *, int); +void ___mp_lock_init(struct __mp_lock *); +void ___mp_lock(struct __mp_lock * LOCK_FL_VARS); +void ___mp_unlock(struct __mp_lock * LOCK_FL_VARS); +int ___mp_release_all(struct __mp_lock * LOCK_FL_VARS); +int ___mp_release_all_but_one(struct __mp_lock * LOCK_FL_VARS); +void ___mp_acquire_count(struct __mp_lock *, int LOCK_FL_VARS); int __mp_lock_held(struct __mp_lock *); -#endif +#ifdef WITNESS + +void _mp_lock_init(struct __mp_lock *, struct lock_type *); + +#define __mp_lock_init(mpl) do { \ + static struct lock_type __lock_type = { .lt_name = #mpl }; \ + _mp_lock_init((mpl), &__lock_type); \ +} while (0) + +#else /* WITNESS */ + +#define __mp_lock_init ___mp_lock_init + +#endif /* WITNESS */ + +#define __mp_lock(mpl) ___mp_lock((mpl) LOCK_FILE_LINE) +#define __mp_unlock(mpl) ___mp_unlock((mpl) LOCK_FILE_LINE) + +#define __mp_release_all(mpl) \ + ___mp_release_all((mpl) LOCK_FILE_LINE) +#define __mp_release_all_but_one(mpl) \ + ___mp_release_all_but_one((mpl) LOCK_FILE_LINE) +#define __mp_acquire_count(mpl, count) \ + ___mp_acquire_count((mpl), (count) LOCK_FILE_LINE) #endif + +#endif /* !_MACHINE_MPLOCK_H */ diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c index 7edc1b6511c..a5af0a5b830 100644 --- a/sys/kern/kern_lock.c +++ b/sys/kern/kern_lock.c @@ -1,4 +1,4 @@ -/* $OpenBSD: kern_lock.c,v 1.47 2016/06/19 11:54:33 natano Exp $ */ +/* $OpenBSD: kern_lock.c,v 1.48 2017/04/20 13:20:17 visa Exp $ */ /* * Copyright (c) 1995 @@ -37,8 +37,11 @@ #include #include +#include #include #include +#include +#include #ifdef MP_LOCKDEBUG /* CPU-dependent timing, this needs to be settable from ddb. */ @@ -65,10 +68,14 @@ _kernel_lock_init(void) */ void -_kernel_lock(void) +_kernel_lock(const char *file, int line) { SCHED_ASSERT_UNLOCKED(); +#ifdef WITNESS + ___mp_lock(&kernel_lock, file, line); +#else __mp_lock(&kernel_lock); +#endif } void @@ -82,4 +89,23 @@ _kernel_lock_held(void) { return (__mp_lock_held(&kernel_lock)); } + +#ifdef WITNESS +void +_mp_lock_init(struct __mp_lock *mpl, struct lock_type *type) +{ + mpl->mpl_lock_obj.lo_name = type->lt_name; + mpl->mpl_lock_obj.lo_type = type; + if (mpl == &kernel_lock) + mpl->mpl_lock_obj.lo_flags = LO_WITNESS | LO_INITIALIZED | + LO_SLEEPABLE | (LO_CLASS_KERNEL_LOCK << LO_CLASSSHIFT); + else if (mpl == &sched_lock) + mpl->mpl_lock_obj.lo_flags = LO_WITNESS | LO_INITIALIZED | + LO_RECURSABLE | (LO_CLASS_SCHED_LOCK << LO_CLASSSHIFT); + WITNESS_INIT(&mpl->mpl_lock_obj, type); + + ___mp_lock_init(mpl); +} +#endif /* WITNESS */ + #endif /* MULTIPROCESSOR */ diff --git a/sys/sys/systm.h b/sys/sys/systm.h index f382c37d72e..83a211260e4 100644 --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -1,4 +1,4 @@ -/* $OpenBSD: systm.h,v 1.125 2017/03/17 17:19:16 mpi Exp $ */ +/* $OpenBSD: systm.h,v 1.126 2017/04/20 13:20:17 visa Exp $ */ /* $NetBSD: systm.h,v 1.50 1996/06/09 04:55:09 briggs Exp $ */ /*- @@ -349,12 +349,12 @@ void user_config(void); #if defined(MULTIPROCESSOR) void _kernel_lock_init(void); -void _kernel_lock(void); +void _kernel_lock(const char *, int); void _kernel_unlock(void); int _kernel_lock_held(void); #define KERNEL_LOCK_INIT() _kernel_lock_init() -#define KERNEL_LOCK() _kernel_lock() +#define KERNEL_LOCK() _kernel_lock(__FILE__, __LINE__) #define KERNEL_UNLOCK() _kernel_unlock() #define KERNEL_ASSERT_LOCKED() KASSERT(_kernel_lock_held()) #define KERNEL_ASSERT_UNLOCKED() KASSERT(!_kernel_lock_held()) -- 2.20.1