Use btrace(8) to debug reference counting. dt(4) provides a static

tracepoint for each type of refcnt we have.  As a start, add inpcb
and tdb refcnt.  When the counter changes, btrace may print the
actual object, the current counter, the change value and optionally
the stack trace.
discussed with visa@; OK mpi@

diff --git a/sys/dev/dt/dt_prov_static.c b/sys/dev/dt/dt_prov_static.c
index 144b354..d608373 100644 (file)
--- a/sys/dev/dt/dt_prov_static.c
+++ b/sys/dev/dt/dt_prov_static.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: dt_prov_static.c,v 1.13 2022/03/17 14:53:59 bluhm Exp $ */
+/*     $OpenBSD: dt_prov_static.c,v 1.14 2022/06/28 09:32:27 bluhm Exp $ */
 
 /*
  * Copyright (c) 2019 Martin Pieuchot <mpi@openbsd.org>
@@ -87,6 +87,12 @@ DT_STATIC_PROBE1(smr, barrier_exit, "int");
 DT_STATIC_PROBE0(smr, wakeup);
 DT_STATIC_PROBE2(smr, thread, "uint64_t", "uint64_t");
 
+/*
+ * reference counting
+ */
+DT_STATIC_PROBE0(refcnt, none);
+DT_STATIC_PROBE3(refcnt, inpcb, "void *", "int", "int");
+DT_STATIC_PROBE3(refcnt, tdb, "void *", "int", "int");
 
 /*
  * List of all static probes
@@ -127,15 +133,24 @@ struct dt_probe *const dtps_static[] = {
        &_DT_STATIC_P(smr, barrier_exit),
        &_DT_STATIC_P(smr, wakeup),
        &_DT_STATIC_P(smr, thread),
+       /* refcnt */
+       &_DT_STATIC_P(refcnt, none),
+       &_DT_STATIC_P(refcnt, inpcb),
+       &_DT_STATIC_P(refcnt, tdb),
 };
 
+struct dt_probe *const *dtps_index_refcnt;
+
 int
 dt_prov_static_init(void)
 {
        int i;
 
-       for (i = 0; i < nitems(dtps_static); i++)
+       for (i = 0; i < nitems(dtps_static); i++) {
+               if (dtps_static[i] == &_DT_STATIC_P(refcnt, none))
+                       dtps_index_refcnt = &dtps_static[i];
                dt_dev_register_probe(dtps_static[i]);
+       }
 
        return i;
 }

diff --git a/sys/dev/dt/dtvar.h b/sys/dev/dt/dtvar.h
index 5d5a38b..9f425df 100644 (file)
--- a/sys/dev/dt/dtvar.h
+++ b/sys/dev/dt/dtvar.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: dtvar.h,v 1.13 2022/02/27 10:14:01 bluhm Exp $ */
+/*     $OpenBSD: dtvar.h,v 1.14 2022/06/28 09:32:27 bluhm Exp $ */
 
 /*
  * Copyright (c) 2019 Martin Pieuchot <mpi@openbsd.org>
@@ -310,16 +310,35 @@ extern volatile uint32_t  dt_tracing;     /* currently tracing? */
 #define        DT_STATIC_PROBE5(func, name, arg0, arg1, arg2, arg3, arg4)      \
        _DT_STATIC_PROBEN(func, name, arg0, arg1, arg2, arg3, arg4, 5)
 
-#define        DT_STATIC_ENTER(func, name, args...) do {                       \
+#define DT_STATIC_ENTER(func, name, args...) do {                      \
        extern struct dt_probe _DT_STATIC_P(func, name);                \
        struct dt_probe *dtp = &_DT_STATIC_P(func, name);               \
-       struct dt_provider *dtpv = dtp->dtp_prov;                       \
                                                                        \
        if (__predict_false(dt_tracing) &&                              \
            __predict_false(dtp->dtp_recording)) {                      \
+               struct dt_provider *dtpv = dtp->dtp_prov;               \
+                                                                       \
                dtpv->dtpv_enter(dtpv, dtp, args);                      \
        }                                                               \
 } while (0)
 
+#define _DT_INDEX_P(func)              (dtps_index_##func)
+
+#define DT_INDEX_ENTER(func, index, args...) do {                      \
+       extern struct dt_probe **_DT_INDEX_P(func);                     \
+                                                                       \
+       if (__predict_false(dt_tracing) &&                              \
+           __predict_false(index > 0) &&                               \
+           __predict_true(_DT_INDEX_P(func) != NULL)) {                \
+               struct dt_probe *dtp = _DT_INDEX_P(func)[index];        \
+                                                                       \
+               if(__predict_false(dtp->dtp_recording)) {               \
+                       struct dt_provider *dtpv = dtp->dtp_prov;       \
+                                                                       \
+                       dtpv->dtpv_enter(dtpv, dtp, args);              \
+               }                                                       \
+       }                                                               \
+} while (0)
+
 #endif /* !_KERNEL */
 #endif /* !_DT_H_ */

diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index 917b604..be5dc46 100644 (file)
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: kern_synch.c,v 1.188 2022/06/12 10:36:04 visa Exp $   */
+/*     $OpenBSD: kern_synch.c,v 1.189 2022/06/28 09:32:27 bluhm Exp $  */
 /*     $NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $ */
 
 /*
@@ -783,7 +783,15 @@ sys___thrwakeup(struct proc *p, void *v, register_t *retval)
 void
 refcnt_init(struct refcnt *r)
 {
+       refcnt_init_trace(r, 0);
+}
+
+void
+refcnt_init_trace(struct refcnt *r, int idx)
+{
+       r->r_traceidx = idx;
        atomic_store_int(&r->r_refs, 1);
+       TRACEINDEX(refcnt, r->r_traceidx, r, 0, +1);
 }
 
 void
@@ -793,6 +801,7 @@ refcnt_take(struct refcnt *r)
 
        refs = atomic_inc_int_nv(&r->r_refs);
        KASSERT(refs != 0);
+       TRACEINDEX(refcnt, r->r_traceidx, r, refs - 1, +1);
        (void)refs;
 }
 
@@ -804,6 +813,7 @@ refcnt_rele(struct refcnt *r)
        membar_exit_before_atomic();
        refs = atomic_dec_int_nv(&r->r_refs);
        KASSERT(refs != ~0);
+       TRACEINDEX(refcnt, r->r_traceidx, r, refs + 1, -1);
        if (refs == 0) {
                membar_enter_after_atomic();
                return (1);
@@ -827,11 +837,13 @@ refcnt_finalize(struct refcnt *r, const char *wmesg)
        membar_exit_before_atomic();
        refs = atomic_dec_int_nv(&r->r_refs);
        KASSERT(refs != ~0);
+       TRACEINDEX(refcnt, r->r_traceidx, r, refs + 1, -1);
        while (refs) {
                sleep_setup(&sls, r, PWAIT, wmesg, 0);
                refs = atomic_load_int(&r->r_refs);
                sleep_finish(&sls, refs);
        }
+       TRACEINDEX(refcnt, r->r_traceidx, r, refs, 0);
        /* Order subsequent loads and stores after refs == 0 load. */
        membar_sync();
 }
@@ -842,6 +854,7 @@ refcnt_shared(struct refcnt *r)
        u_int refs;
 
        refs = atomic_load_int(&r->r_refs);
+       TRACEINDEX(refcnt, r->r_traceidx, r, refs, 0);
        return (refs > 1);
 }
 
@@ -851,6 +864,7 @@ refcnt_read(struct refcnt *r)
        u_int refs;
 
        refs = atomic_load_int(&r->r_refs);
+       TRACEINDEX(refcnt, r->r_traceidx, r, refs, 0);
        return (refs);
 }
 

diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
index 0998998..cf8667b 100644 (file)
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: in_pcb.c,v 1.267 2022/06/06 14:45:41 claudio Exp $    */
+/*     $OpenBSD: in_pcb.c,v 1.268 2022/06/28 09:32:27 bluhm Exp $      */
 /*     $NetBSD: in_pcb.c,v 1.25 1996/02/13 23:41:53 christos Exp $     */
 
 /*
@@ -235,7 +235,7 @@ in_pcballoc(struct socket *so, struct inpcbtable *table)
                return (ENOBUFS);
        inp->inp_table = table;
        inp->inp_socket = so;
-       refcnt_init(&inp->inp_refcnt);
+       refcnt_init_trace(&inp->inp_refcnt, DT_REFCNT_IDX_INPCB);
        inp->inp_seclevel[SL_AUTH] = IPSEC_AUTH_LEVEL_DEFAULT;
        inp->inp_seclevel[SL_ESP_TRANS] = IPSEC_ESP_TRANS_LEVEL_DEFAULT;
        inp->inp_seclevel[SL_ESP_NETWORK] = IPSEC_ESP_NETWORK_LEVEL_DEFAULT;

diff --git a/sys/netinet/ip_ipsp.c b/sys/netinet/ip_ipsp.c
index cf39622..3d7c9a0 100644 (file)
--- a/sys/netinet/ip_ipsp.c
+++ b/sys/netinet/ip_ipsp.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: ip_ipsp.c,v 1.270 2022/04/30 13:28:53 mvs Exp $       */
+/*     $OpenBSD: ip_ipsp.c,v 1.271 2022/06/28 09:32:27 bluhm Exp $     */
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr),
@@ -1048,7 +1048,7 @@ tdb_alloc(u_int rdomain)
 
        tdbp = pool_get(&tdb_pool, PR_WAITOK | PR_ZERO);
 
-       refcnt_init(&tdbp->tdb_refcnt);
+       refcnt_init_trace(&tdbp->tdb_refcnt, DT_REFCNT_IDX_TDB);
        mtx_init(&tdbp->tdb_mtx, IPL_SOFTNET);
        TAILQ_INIT(&tdbp->tdb_policy_head);
 

diff --git a/sys/sys/refcnt.h b/sys/sys/refcnt.h
index 7f7b4e8..bb6e968 100644 (file)
--- a/sys/sys/refcnt.h
+++ b/sys/sys/refcnt.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: refcnt.h,v 1.6 2022/03/16 14:13:01 visa Exp $ */
+/*     $OpenBSD: refcnt.h,v 1.7 2022/06/28 09:32:28 bluhm Exp $ */
 
 /*
  * Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
@@ -21,18 +21,21 @@
 
 /*
  * Locks used to protect struct members in this file:
+ *     I       immutable after creation
  *     a       atomic operations
  */
 
 struct refcnt {
        unsigned int    r_refs;         /* [a] reference counter */
+       int             r_traceidx;     /* [I] index for dt(4) tracing  */
 };
 
-#define REFCNT_INITIALIZER()           { .r_refs = 1 }
+#define REFCNT_INITIALIZER()           { .r_refs = 1, .r_traceidx = 0 }
 
 #ifdef _KERNEL
 
 void   refcnt_init(struct refcnt *);
+void   refcnt_init_trace(struct refcnt *, int id);
 void   refcnt_take(struct refcnt *);
 int    refcnt_rele(struct refcnt *);
 void   refcnt_rele_wake(struct refcnt *);
@@ -40,6 +43,9 @@ void  refcnt_finalize(struct refcnt *, const char *);
 int    refcnt_shared(struct refcnt *);
 unsigned int   refcnt_read(struct refcnt *);
 
+#define DT_REFCNT_IDX_INPCB    1
+#define DT_REFCNT_IDX_TDB      2
+
 #endif /* _KERNEL */
 
 #endif /* _SYS_REFCNT_H_ */

diff --git a/sys/sys/tracepoint.h b/sys/sys/tracepoint.h
index d9a674a..9395148 100644 (file)
--- a/sys/sys/tracepoint.h
+++ b/sys/sys/tracepoint.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: tracepoint.h,v 1.1 2020/01/21 16:16:23 mpi Exp $ */
+/*     $OpenBSD: tracepoint.h,v 1.2 2022/06/28 09:32:28 bluhm Exp $ */
 
 /*
  * Copyright (c) 2019 Martin Pieuchot <mpi@openbsd.org>
@@ -25,11 +25,13 @@
 #if NDT > 0
 #include <dev/dt/dtvar.h>
 
-#define        TRACEPOINT(func, name, args...) DT_STATIC_ENTER(func, name, args)
+#define TRACEPOINT(func, name, args...)        DT_STATIC_ENTER(func, name, args)
+#define TRACEINDEX(func, index, args...) DT_INDEX_ENTER(func, index, args)
 
 #else /* NDT > 0 */
 
-#define        TRACEPOINT(func, name, args...)
+#define TRACEPOINT(func, name, args...)
+#define TRACEINDEX(func, index, args...)
 
 #endif /* NDT > 0 */
 #endif /* _KERNEL */