add garbage collection of unused lists percpu cached items.

the cpu caches in pools amortise the cost of accessing global
structures by moving lists of items around instead of individual
items. excess lists of items are stored in the global pool struct,
but these idle lists never get returned back to the system for use
elsewhere.

this adds a timestamp to the global idle list, which is updated
when the idle list stops being empty. if the idle list hasn't been
empty for a while, it means the per cpu caches arent using the idle
entries and they can be recovered. timestamping the pages prevents
recovery of a lot of items that may be used again shortly. eg, rx
ring processing and replenishing from rate limited interrupts tends
to allocate and free items in large chunks, which the timestamping
smooths out.

gc'ed lists are returned to the pool pages, which in turn get gc'ed
back to uvm.

ok visa@

diff --git a/sys/kern/subr_pool.c b/sys/kern/subr_pool.c
index 975d0d5..62ac8cc 100644 (file)
--- a/sys/kern/subr_pool.c
+++ b/sys/kern/subr_pool.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: subr_pool.c,v 1.213 2017/06/16 01:33:20 dlg Exp $     */
+/*     $OpenBSD: subr_pool.c,v 1.214 2017/06/16 01:55:45 dlg Exp $     */
 /*     $NetBSD: subr_pool.c,v 1.61 2001/09/26 07:14:56 chs Exp $       */
 
 /*-
@@ -135,6 +135,7 @@ struct pool_cache {
 void   *pool_cache_get(struct pool *);
 void    pool_cache_put(struct pool *, void *);
 void    pool_cache_destroy(struct pool *);
+void    pool_cache_gc(struct pool *);
 #endif
 void    pool_cache_pool_info(struct pool *, struct kinfo_pool *);
 int     pool_cache_info(struct pool *, void *, size_t *);
@@ -1476,6 +1477,11 @@ pool_gc_pages(void *null)
        rw_enter_read(&pool_lock);
        s = splvm(); /* XXX go to splvm until all pools _setipl properly */
        SIMPLEQ_FOREACH(pp, &pool_head, pr_poollist) {
+#ifdef MULTIPROCESSOR
+               if (pp->pr_cache != NULL)
+                       pool_cache_gc(pp);
+#endif
+
                if (pp->pr_nidle <= pp->pr_minpages || /* guess */
                    !mtx_enter_try(&pp->pr_mtx)) /* try */
                        continue;
@@ -1642,8 +1648,10 @@ pool_cache_init(struct pool *pp)
        arc4random_buf(pp->pr_cache_magic, sizeof(pp->pr_cache_magic));
        TAILQ_INIT(&pp->pr_cache_lists);
        pp->pr_cache_nlist = 0;
+       pp->pr_cache_tick = ticks;
        pp->pr_cache_items = 8;
        pp->pr_cache_contention = 0;
+       pp->pr_cache_ngc = 0;
 
        CPUMEM_FOREACH(pc, &i, cm) {
                pc->pc_actv = NULL;
@@ -1659,6 +1667,8 @@ pool_cache_init(struct pool *pp)
                pc->pc_nout = 0;
        }
 
+       membar_producer();
+
        pp->pr_cache = cm;
 }
 
@@ -1740,6 +1750,9 @@ pool_cache_list_free(struct pool *pp, struct pool_cache *pc,
     struct pool_cache_item *ci)
 {
        pool_list_enter(pp);
+       if (TAILQ_EMPTY(&pp->pr_cache_lists))
+               pp->pr_cache_tick = ticks;
+
        TAILQ_INSERT_TAIL(&pp->pr_cache_lists, ci, ci_nextl);
        pp->pr_cache_nlist++;
 
@@ -1893,8 +1906,10 @@ pool_cache_destroy(struct pool *pp)
        struct cpumem_iter i;
        struct cpumem *cm;
 
+       rw_enter_write(&pool_lock); /* serialise with the gc */
        cm = pp->pr_cache;
        pp->pr_cache = NULL; /* make pool_put avoid the cache */
+       rw_exit_write(&pool_lock);
 
        CPUMEM_FOREACH(pc, &i, cm) {
                pool_cache_list_put(pp, pc->pc_actv);
@@ -1908,6 +1923,29 @@ pool_cache_destroy(struct pool *pp)
                pl = pool_cache_list_put(pp, pl);
 }
 
+void
+pool_cache_gc(struct pool *pp)
+{
+       if ((ticks - pp->pr_cache_tick) > (hz * pool_wait_gc) &&
+           !TAILQ_EMPTY(&pp->pr_cache_lists) &&
+           mtx_enter_try(&pp->pr_cache_mtx)) {
+               struct pool_cache_item *pl = NULL;
+
+               pl = TAILQ_FIRST(&pp->pr_cache_lists);
+               if (pl != NULL) {
+                       TAILQ_REMOVE(&pp->pr_cache_lists, pl, ci_nextl);
+                       pp->pr_cache_tick = ticks;
+                       pp->pr_cache_nlist--;
+
+                       pp->pr_cache_ngc++;
+               }
+
+               mtx_leave(&pp->pr_cache_mtx);
+
+               pool_cache_list_put(pp, pl);
+       }
+}
+
 void
 pool_cache_pool_info(struct pool *pp, struct kinfo_pool *pi)
 {
@@ -1955,7 +1993,7 @@ pool_cache_info(struct pool *pp, void *oldp, size_t *oldlenp)
        memset(&kpc, 0, sizeof(kpc)); /* don't leak padding */
 
        mtx_enter(&pp->pr_cache_mtx);
-       kpc.pr_ngc = 0; /* notyet */
+       kpc.pr_ngc = pp->pr_cache_ngc;
        kpc.pr_len = pp->pr_cache_items;
        kpc.pr_nlist = pp->pr_cache_nlist;
        kpc.pr_contention = pp->pr_cache_contention;

diff --git a/sys/sys/pool.h b/sys/sys/pool.h
index 1a5380e..82aa9ba 100644 (file)
--- a/sys/sys/pool.h
+++ b/sys/sys/pool.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: pool.h,v 1.70 2017/06/15 02:52:30 dlg Exp $   */
+/*     $OpenBSD: pool.h,v 1.71 2017/06/16 01:55:45 dlg Exp $   */
 /*     $NetBSD: pool.h,v 1.27 2001/06/06 22:00:17 rafal Exp $  */
 
 /*-
@@ -185,11 +185,13 @@ struct pool {
        unsigned long   pr_cache_magic[2];
        struct mutex    pr_cache_mtx;
        struct pool_cache_lists
-                       pr_cache_lists;
-       u_int           pr_cache_nlist; /* # of lists */
+                       pr_cache_lists; /* list of idle item lists */
+       u_int           pr_cache_nlist; /* # of idle lists */
        u_int           pr_cache_items; /* target list length */
        u_int           pr_cache_contention;
+       int             pr_cache_tick;  /* time idle list was empty */
        int             pr_cache_nout;
+       uint64_t        pr_cache_ngc;   /* # of times the gc released a list */
 
        u_int           pr_align;
        u_int           pr_maxcolors;   /* Cache coloring */