From 7720a192de8b0bcc3d69ba0245b8d11c2379e11d Mon Sep 17 00:00:00 2001
From: claudio <claudio@openbsd.org>
Date: Wed, 24 Nov 2021 09:47:49 +0000
Subject: [PATCH] Add a few dt(4) TRACEPOINTS to SMR. Should help to better
 understand what goes on in SMR. OK mpi@

---
 sys/dev/dt/dt_prov_static.c | 20 +++++++++++++++++++-
 sys/kern/kern_smr.c         | 21 ++++++++++++++++-----
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/sys/dev/dt/dt_prov_static.c b/sys/dev/dt/dt_prov_static.c
index 9c052abf731..c1371c311a3 100644
--- a/sys/dev/dt/dt_prov_static.c
+++ b/sys/dev/dt/dt_prov_static.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: dt_prov_static.c,v 1.10 2021/09/03 16:45:45 jasper Exp $ */
+/*	$OpenBSD: dt_prov_static.c,v 1.11 2021/11/24 09:47:49 claudio Exp $ */
 
 /*
  * Copyright (c) 2019 Martin Pieuchot <mpi@openbsd.org>
@@ -75,6 +75,17 @@ DT_STATIC_PROBE4(vfs, cleaner, "long", "int", "long", "long");
 DT_STATIC_PROBE2(vmm, guest_enter, "void *", "void *");
 DT_STATIC_PROBE3(vmm, guest_exit, "void *", "void *", "uint64_t");
 
+/*
+ * SMR
+ */
+DT_STATIC_PROBE3(smr, call, "void *", "void *", "int");
+DT_STATIC_PROBE2(smr, called, "void *", "void *");
+DT_STATIC_PROBE1(smr, barrier_enter, "int");
+DT_STATIC_PROBE1(smr, barrier_exit, "int");
+DT_STATIC_PROBE0(smr, wakeup);
+DT_STATIC_PROBE2(smr, thread, "uint64_t", "uint64_t");
+
+
 /*
  * List of all static probes
  */
@@ -105,6 +116,13 @@ struct dt_probe *dtps_static[] = {
 	/* VMM */
 	&_DT_STATIC_P(vmm, guest_enter),
 	&_DT_STATIC_P(vmm, guest_exit),
+	/* SMR */
+	&_DT_STATIC_P(smr, call),
+	&_DT_STATIC_P(smr, called),
+	&_DT_STATIC_P(smr, barrier_enter),
+	&_DT_STATIC_P(smr, barrier_exit),
+	&_DT_STATIC_P(smr, wakeup),
+	&_DT_STATIC_P(smr, thread),
 };
 
 int
diff --git a/sys/kern/kern_smr.c b/sys/kern/kern_smr.c
index f95e18cae3a..dc0d2e94e97 100644
--- a/sys/kern/kern_smr.c
+++ b/sys/kern/kern_smr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_smr.c,v 1.12 2021/07/06 09:34:07 kettenis Exp $	*/
+/*	$OpenBSD: kern_smr.c,v 1.13 2021/11/24 09:47:49 claudio Exp $	*/
 
 /*
  * Copyright (c) 2019-2020 Visa Hankala
@@ -26,6 +26,7 @@
 #include <sys/smr.h>
 #include <sys/time.h>
 #include <sys/witness.h>
+#include <sys/tracepoint.h>
 
 #include <machine/cpu.h>
 
@@ -84,6 +85,8 @@ smr_thread(void *arg)
 	struct timeval elapsed, end, loglast, start;
 	struct smr_entry_list deferred;
 	struct smr_entry *smr;
+	size_t count;
+	uint64_t nsec;
 
 	KERNEL_ASSERT_LOCKED();
 	KERNEL_UNLOCK();
@@ -115,19 +118,23 @@ smr_thread(void *arg)
 		WITNESS_CHECKORDER(&smr_lock_obj, LOP_NEWORDER, NULL);
 		WITNESS_LOCK(&smr_lock_obj, 0);
 
+		count = 0;
 		while ((smr = SIMPLEQ_FIRST(&deferred)) != NULL) {
 			SIMPLEQ_REMOVE_HEAD(&deferred, smr_list);
+			TRACEPOINT(smr, called, smr->smr_func, smr->smr_arg);
 			smr->smr_func(smr->smr_arg);
+			count++;
 		}
 
 		WITNESS_UNLOCK(&smr_lock_obj, 0);
 
 		getmicrouptime(&end);
 		timersub(&end, &start, &elapsed);
-		if (elapsed.tv_sec >= 5 &&
+		nsec = TIMEVAL_TO_NSEC(&elapsed);
+		if (nsec / 1000000000ULL >= 2 &&
 		    ratecheck(&loglast, &smr_logintvl))
-			printf("smr: dispatch took %ld seconds\n",
-			    (long)elapsed.tv_sec);
+			printf("smr: dispatch took %lluus\n", nsec / 1000);
+		TRACEPOINT(smr, thread, nsec, count);
 	}
 }
 
@@ -163,6 +170,7 @@ smr_grace_wait(void)
 void
 smr_wakeup(void *arg)
 {
+	TRACEPOINT(smr, wakeup, NULL);
 	wakeup(&smr_ndeferred);
 }
 
@@ -207,7 +215,7 @@ smr_dispatch(struct schedstate_percpu *spc)
 	mtx_leave(&smr_lock);
 
 	if (expedite)
-		wakeup(&smr_ndeferred);
+		smr_wakeup(NULL);
 	else if (wake)
 		timeout_add_msec(&smr_wakeup_tmo, SMR_PAUSE);
 }
@@ -256,6 +264,7 @@ smr_call_impl(struct smr_entry *smr, void (*func)(void *), void *arg,
 	spc->spc_ndeferred++;
 	spc->spc_smrexpedite |= expedite;
 	splx(s);
+	TRACEPOINT(smr, call, func, arg, expedite);
 
 	/*
 	 * If this call was made from an interrupt context that
@@ -287,7 +296,9 @@ smr_barrier_impl(int expedite)
 
 	WITNESS_CHECKORDER(&smr_lock_obj, LOP_NEWORDER, NULL);
 
+	TRACEPOINT(smr, barrier_enter, expedite);
 	smr_init(&smr);
 	smr_call_impl(&smr, smr_barrier_func, &c, expedite);
 	cond_wait(&c, "smrbar");
+	TRACEPOINT(smr, barrier_exit, expedite);
 }
-- 
2.20.1