replace the page LISTS with page TAILQs. this will let me pull pages from

either end of the lists cheaply.

ok kettenis@ tedu@

diff --git a/sys/kern/subr_pool.c b/sys/kern/subr_pool.c
index 2c0698f..9b51c93 100644 (file)
--- a/sys/kern/subr_pool.c
+++ b/sys/kern/subr_pool.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: subr_pool.c,v 1.169 2014/12/04 03:12:05 dlg Exp $     */
+/*     $OpenBSD: subr_pool.c,v 1.170 2014/12/19 02:15:25 dlg Exp $     */
 /*     $NetBSD: subr_pool.c,v 1.61 2001/09/26 07:14:56 chs Exp $       */
 
 /*-
@@ -74,7 +74,7 @@ struct pool phpool;
 
 struct pool_item_header {
        /* Page headers */
-       LIST_ENTRY(pool_item_header)
+       TAILQ_ENTRY(pool_item_header)
                                ph_pagelist;    /* pool page list */
        XSIMPLEQ_HEAD(,pool_item) ph_itemlist;  /* chunk list for this page */
        RB_ENTRY(pool_item_header)
@@ -263,9 +263,9 @@ pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags,
         * Initialize the pool structure.
         */
        memset(pp, 0, sizeof(*pp));
-       LIST_INIT(&pp->pr_emptypages);
-       LIST_INIT(&pp->pr_fullpages);
-       LIST_INIT(&pp->pr_partpages);
+       TAILQ_INIT(&pp->pr_emptypages);
+       TAILQ_INIT(&pp->pr_fullpages);
+       TAILQ_INIT(&pp->pr_partpages);
        pp->pr_curpage = NULL;
        pp->pr_npages = 0;
        pp->pr_minitems = 0;
@@ -375,14 +375,14 @@ pool_destroy(struct pool *pp)
        rw_exit_write(&pool_lock);
 
        /* Remove all pages */
-       while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL) {
+       while ((ph = TAILQ_FIRST(&pp->pr_emptypages)) != NULL) {
                mtx_enter(&pp->pr_mtx);
                pool_p_remove(pp, ph);
                mtx_leave(&pp->pr_mtx);
                pool_p_free(pp, ph);
        }
-       KASSERT(LIST_EMPTY(&pp->pr_fullpages));
-       KASSERT(LIST_EMPTY(&pp->pr_partpages));
+       KASSERT(TAILQ_EMPTY(&pp->pr_fullpages));
+       KASSERT(TAILQ_EMPTY(&pp->pr_partpages));
 }
 
 void
@@ -591,8 +591,8 @@ pool_do_get(struct pool *pp, int flags, int *slowdown)
                 * This page was previously empty.  Move it to the list of
                 * partially-full pages.  This page is already curpage.
                 */
-               LIST_REMOVE(ph, ph_pagelist);
-               LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist);
+               TAILQ_REMOVE(&pp->pr_emptypages, ph, ph_pagelist);
+               TAILQ_INSERT_TAIL(&pp->pr_partpages, ph, ph_pagelist);
 
                pp->pr_nidle--;
        }
@@ -602,8 +602,8 @@ pool_do_get(struct pool *pp, int flags, int *slowdown)
                 * This page is now full.  Move it to the full list
                 * and select a new current page.
                 */
-               LIST_REMOVE(ph, ph_pagelist);
-               LIST_INSERT_HEAD(&pp->pr_fullpages, ph, ph_pagelist);
+               TAILQ_REMOVE(&pp->pr_partpages, ph, ph_pagelist);
+               TAILQ_INSERT_TAIL(&pp->pr_fullpages, ph, ph_pagelist);
                pool_update_curpage(pp);
        }
 
@@ -654,8 +654,8 @@ pool_put(struct pool *pp, void *v)
                 * The page was previously completely full, move it to the
                 * partially-full list.
                 */
-               LIST_REMOVE(ph, ph_pagelist);
-               LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist);
+               TAILQ_REMOVE(&pp->pr_fullpages, ph, ph_pagelist);
+               TAILQ_INSERT_TAIL(&pp->pr_partpages, ph, ph_pagelist);
        }
 
        if (ph->ph_nmissing == 0) {
@@ -664,8 +664,8 @@ pool_put(struct pool *pp, void *v)
                 */
                pp->pr_nidle++;
 
-               LIST_REMOVE(ph, ph_pagelist);
-               LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist);
+               TAILQ_REMOVE(&pp->pr_partpages, ph, ph_pagelist);
+               TAILQ_INSERT_TAIL(&pp->pr_emptypages, ph, ph_pagelist);
                pool_update_curpage(pp);
        }
 
@@ -674,7 +674,7 @@ pool_put(struct pool *pp, void *v)
 
        /* is it time to free a page? */
        if (pp->pr_nidle > pp->pr_maxpages &&
-           (freeph = LIST_FIRST(&pp->pr_emptypages)) != NULL)
+           (freeph = TAILQ_FIRST(&pp->pr_emptypages)) != NULL)
                pool_p_remove(pp, freeph);
        mtx_leave(&pp->pr_mtx);
 
@@ -692,7 +692,7 @@ pool_put(struct pool *pp, void *v)
 int
 pool_prime(struct pool *pp, int n)
 {
-       struct pool_pagelist pl = LIST_HEAD_INITIALIZER(pl);
+       struct pool_pagelist pl = TAILQ_HEAD_INITIALIZER(pl);
        struct pool_item_header *ph;
        int newpages;
 
@@ -705,12 +705,12 @@ pool_prime(struct pool *pp, int n)
                if (ph == NULL) /* or slowdown? */
                        break;
 
-               LIST_INSERT_HEAD(&pl, ph, ph_pagelist);
+               TAILQ_INSERT_TAIL(&pl, ph, ph_pagelist);
        }
 
        mtx_enter(&pp->pr_mtx);
-       while ((ph = LIST_FIRST(&pl)) != NULL) {
-               LIST_REMOVE(ph, ph_pagelist);
+       while ((ph = TAILQ_FIRST(&pl)) != NULL) {
+               TAILQ_REMOVE(&pl, ph, ph_pagelist);
                pool_p_insert(pp, ph);
        }
        mtx_leave(&pp->pr_mtx);
@@ -819,7 +819,7 @@ pool_p_insert(struct pool *pp, struct pool_item_header *ph)
        if (pp->pr_curpage == NULL)
                pp->pr_curpage = ph;
 
-       LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist);
+       TAILQ_INSERT_TAIL(&pp->pr_emptypages, ph, ph_pagelist);
        if (!POOL_INPGHDR(pp))
                RB_INSERT(phtree, &pp->pr_phtree, ph);
 
@@ -843,7 +843,7 @@ pool_p_remove(struct pool *pp, struct pool_item_header *ph)
 
        if (!POOL_INPGHDR(pp))
                RB_REMOVE(phtree, &pp->pr_phtree, ph);
-       LIST_REMOVE(ph, ph_pagelist);
+       TAILQ_REMOVE(&pp->pr_emptypages, ph, ph_pagelist);
 
        pool_update_curpage(pp);
 }
@@ -851,9 +851,9 @@ pool_p_remove(struct pool *pp, struct pool_item_header *ph)
 void
 pool_update_curpage(struct pool *pp)
 {
-       pp->pr_curpage = LIST_FIRST(&pp->pr_partpages);
+       pp->pr_curpage = TAILQ_FIRST(&pp->pr_partpages);
        if (pp->pr_curpage == NULL) {
-               pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages);
+               pp->pr_curpage = TAILQ_FIRST(&pp->pr_emptypages);
        }
 }
 
@@ -919,11 +919,11 @@ int
 pool_reclaim(struct pool *pp)
 {
        struct pool_item_header *ph, *phnext;
-       struct pool_pagelist pl = LIST_HEAD_INITIALIZER(pl);
+       struct pool_pagelist pl = TAILQ_HEAD_INITIALIZER(pl);
 
        mtx_enter(&pp->pr_mtx);
-       for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) {
-               phnext = LIST_NEXT(ph, ph_pagelist);
+       for (ph = TAILQ_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) {
+               phnext = TAILQ_NEXT(ph, ph_pagelist);
 
                /* Check our minimum page claim */
                if (pp->pr_npages <= pp->pr_minpages)
@@ -938,15 +938,15 @@ pool_reclaim(struct pool *pp)
                        break;
 
                pool_p_remove(pp, ph);
-               LIST_INSERT_HEAD(&pl, ph, ph_pagelist);
+               TAILQ_INSERT_TAIL(&pl, ph, ph_pagelist);
        }
        mtx_leave(&pp->pr_mtx);
 
-       if (LIST_EMPTY(&pl))
+       if (TAILQ_EMPTY(&pl))
                return (0);
 
-       while ((ph = LIST_FIRST(&pl)) != NULL) {
-               LIST_REMOVE(ph, ph_pagelist);
+       while ((ph = TAILQ_FIRST(&pl)) != NULL) {
+               TAILQ_REMOVE(&pl, ph, ph_pagelist);
                pool_p_free(pp, ph);
        }
 
@@ -990,7 +990,7 @@ pool_print_pagelist(struct pool_pagelist *pl,
        struct pool_item_header *ph;
        struct pool_item *pi;
 
-       LIST_FOREACH(ph, pl, ph_pagelist) {
+       TAILQ_FOREACH(ph, pl, ph_pagelist) {
                (*pr)("\t\tpage %p, nmissing %d\n",
                    ph->ph_page, ph->ph_nmissing);
                XSIMPLEQ_FOREACH(pi, &ph->ph_itemlist, pi_list) {
@@ -1033,13 +1033,13 @@ pool_print1(struct pool *pp, const char *modif,
        if (print_pagelist == 0)
                return;
 
-       if ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)
+       if ((ph = TAILQ_FIRST(&pp->pr_emptypages)) != NULL)
                (*pr)("\n\tempty page list:\n");
        pool_print_pagelist(&pp->pr_emptypages, pr);
-       if ((ph = LIST_FIRST(&pp->pr_fullpages)) != NULL)
+       if ((ph = TAILQ_FIRST(&pp->pr_fullpages)) != NULL)
                (*pr)("\n\tfull page list:\n");
        pool_print_pagelist(&pp->pr_fullpages, pr);
-       if ((ph = LIST_FIRST(&pp->pr_partpages)) != NULL)
+       if ((ph = TAILQ_FIRST(&pp->pr_partpages)) != NULL)
                (*pr)("\n\tpartial-page list:\n");
        pool_print_pagelist(&pp->pr_partpages, pr);
 
@@ -1203,11 +1203,11 @@ pool_chk(struct pool *pp)
        struct pool_item_header *ph;
        int r = 0;
 
-       LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist)
+       TAILQ_FOREACH(ph, &pp->pr_emptypages, ph_pagelist)
                r += pool_chk_page(pp, ph, pp->pr_itemsperpage);
-       LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist)
+       TAILQ_FOREACH(ph, &pp->pr_fullpages, ph_pagelist)
                r += pool_chk_page(pp, ph, 0);
-       LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist)
+       TAILQ_FOREACH(ph, &pp->pr_partpages, ph_pagelist)
                r += pool_chk_page(pp, ph, -1);
 
        return (r);
@@ -1226,7 +1226,7 @@ pool_walk(struct pool *pp, int full,
        caddr_t cp;
        int n;
 
-       LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) {
+       TAILQ_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) {
                cp = ph->ph_page;
                n = ph->ph_nmissing;
 
@@ -1236,7 +1236,7 @@ pool_walk(struct pool *pp, int full,
                }
        }
 
-       LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) {
+       TAILQ_FOREACH(ph, &pp->pr_partpages, ph_pagelist) {
                cp = ph->ph_page;
                n = ph->ph_nmissing;
 

diff --git a/sys/sys/pool.h b/sys/sys/pool.h
index 206634f..12fa6e1 100644 (file)
--- a/sys/sys/pool.h
+++ b/sys/sys/pool.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: pool.h,v 1.54 2014/11/01 23:58:07 tedu Exp $  */
+/*     $OpenBSD: pool.h,v 1.55 2014/12/19 02:15:25 dlg Exp $   */
 /*     $NetBSD: pool.h,v 1.27 2001/06/06 22:00:17 rafal Exp $  */
 
 /*-
@@ -82,7 +82,7 @@ struct pool_allocator {
        int pa_pagesz;
 };
 
-LIST_HEAD(pool_pagelist, pool_item_header);
+TAILQ_HEAD(pool_pagelist, pool_item_header);
 
 struct pool {
        struct mutex    pr_mtx;