step one in avoiding unneccesary init of chunk_info;

some cleanup; tested by sthen@ on a ports build

diff --git a/lib/libc/stdlib/malloc.c b/lib/libc/stdlib/malloc.c
index e86f941..ad0b187 100644 (file)
--- a/lib/libc/stdlib/malloc.c
+++ b/lib/libc/stdlib/malloc.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: malloc.c,v 1.236 2017/11/02 14:01:50 otto Exp $       */
+/*     $OpenBSD: malloc.c,v 1.237 2017/12/27 10:05:23 otto Exp $       */
 /*
  * Copyright (c) 2008, 2010, 2011, 2016 Otto Moerbeek <otto@drijf.net>
  * Copyright (c) 2012 Matthew Dempsky <matthew@openbsd.org>
@@ -123,12 +123,12 @@ struct dir_info {
                                        /* free pages cache */
        struct region_info free_regions[MALLOC_MAXCACHE];
                                        /* delayed free chunk slots */
+       u_int rotor;
        void *delayed_chunks[MALLOC_DELAYED_CHUNK_MASK + 1];
        size_t rbytesused;              /* random bytes used */
        char *func;                     /* current function */
        int mutex;
        u_char rbytes[32];              /* random bytes */
-       u_short chunk_start;
 #ifdef MALLOC_STATS
        size_t inserts;
        size_t insert_collisions;
@@ -166,14 +166,14 @@ struct dir_info {
 struct chunk_info {
        LIST_ENTRY(chunk_info) entries;
        void *page;                     /* pointer to the page */
-       u_int32_t canary;
+       u_short canary;
        u_short size;                   /* size of this page's chunks */
        u_short shift;                  /* how far to shift for this size */
        u_short free;                   /* how many free chunks */
        u_short total;                  /* how many chunks */
        u_short offset;                 /* requested size table offset */
-                                       /* which chunks are free */
-       u_short bits[1];
+       u_short rotor;                  /* randomization rotor */
+       u_short bits[1];                /* which chunks are free */
 };
 
 struct malloc_readonly {
@@ -416,7 +416,7 @@ map(struct dir_info *d, void *hint, size_t sz, int zero_fill)
 {
        size_t psz = sz >> MALLOC_PAGESHIFT;
        struct region_info *r, *big = NULL;
-       u_int i, offset;
+       u_int i;
        void *p;
 
        if (mopts.malloc_canary != (d->canary1 ^ (u_int32_t)(uintptr_t)d) ||
@@ -425,7 +425,7 @@ map(struct dir_info *d, void *hint, size_t sz, int zero_fill)
        if (sz != PAGEROUND(sz))
                wrterror(d, "map round");
 
-       if (!hint && psz > d->free_regions_size) {
+       if (hint == NULL && psz > d->free_regions_size) {
                _MALLOC_LEAVE(d);
                p = MMAP(sz);
                _MALLOC_ENTER(d);
@@ -434,11 +434,10 @@ map(struct dir_info *d, void *hint, size_t sz, int zero_fill)
                /* zero fill not needed */
                return p;
        }
-       offset = getrbyte(d);
        for (i = 0; i < mopts.malloc_cache; i++) {
-               r = &d->free_regions[(i + offset) & (mopts.malloc_cache - 1)];
+               r = &d->free_regions[(i + d->rotor) & (mopts.malloc_cache - 1)];
                if (r->p != NULL) {
-                       if (hint && r->p != hint)
+                       if (hint != NULL && r->p != hint)
                                continue;
                        if (r->size == psz) {
                                p = r->p;
@@ -452,6 +451,7 @@ map(struct dir_info *d, void *hint, size_t sz, int zero_fill)
                                else if (mopts.malloc_junk == 2 &&
                                    mopts.malloc_freeunmap)
                                        memset(p, SOME_FREEJUNK, sz);
+                               d->rotor += i + 1;
                                return p;
                        } else if (r->size > psz)
                                big = r;
@@ -471,7 +471,7 @@ map(struct dir_info *d, void *hint, size_t sz, int zero_fill)
                        memset(p, SOME_FREEJUNK, sz);
                return p;
        }
-       if (hint)
+       if (hint != NULL)
                return MAP_FAILED;
        if (d->free_regions_size > mopts.malloc_cache)
                wrterror(d, "malloc cache");
@@ -718,36 +718,35 @@ static struct chunk_info *
 alloc_chunk_info(struct dir_info *d, int bits)
 {
        struct chunk_info *p;
-       size_t size, count;
-
-       if (bits == 0)
-               count = MALLOC_PAGESIZE / MALLOC_MINSIZE;
-       else
-               count = MALLOC_PAGESIZE >> bits;
-
-       size = howmany(count, MALLOC_BITS);
-       size = sizeof(struct chunk_info) + (size - 1) * sizeof(u_short);
-       if (mopts.chunk_canaries)
-               size += count * sizeof(u_short);
-       size = ALIGN(size);
 
        if (LIST_EMPTY(&d->chunk_info_list[bits])) {
+               size_t size, count, i;
                char *q;
-               size_t i;
+
+               if (bits == 0)
+                       count = MALLOC_PAGESIZE / MALLOC_MINSIZE;
+               else
+                       count = MALLOC_PAGESIZE >> bits;
+
+               size = howmany(count, MALLOC_BITS);
+               size = sizeof(struct chunk_info) + (size - 1) * sizeof(u_short);
+               if (mopts.chunk_canaries)
+                       size += count * sizeof(u_short);
+               size = ALIGN(size);
 
                q = MMAP(MALLOC_PAGESIZE);
                if (q == MAP_FAILED)
                        return NULL;
                STATS_ADD(d->malloc_used, MALLOC_PAGESIZE);
                count = MALLOC_PAGESIZE / size;
-               for (i = 0; i < count; i++, q += size)
-                       LIST_INSERT_HEAD(&d->chunk_info_list[bits],
-                           (struct chunk_info *)q, entries);
+               for (i = 0; i < count; i++, q += size) {
+                       p = (struct chunk_info *)q;
+                       p->canary = (u_short)d->canary1;
+                       LIST_INSERT_HEAD(&d->chunk_info_list[bits], p, entries);
+               }
        }
        p = LIST_FIRST(&d->chunk_info_list[bits]);
        LIST_REMOVE(p, entries);
-       memset(p, 0, size);
-       p->canary = d->canary1;
        return p;
 }
 
@@ -896,6 +895,8 @@ omalloc_make_chunks(struct dir_info *d, int bits, int listnum)
        for (; (k - i) >= MALLOC_BITS; i += MALLOC_BITS)
                bp->bits[i / MALLOC_BITS] = (u_short)~0U;
 
+       if (i < k)
+               bp->bits[i / MALLOC_BITS] = 0;
        for (; i < k; i++)
                bp->bits[i / MALLOC_BITS] |= (u_short)1U << (i % MALLOC_BITS);
 
@@ -937,10 +938,12 @@ find_chunksize(size_t size)
 static void *
 malloc_bytes(struct dir_info *d, size_t size, void *f)
 {
-       int             i, j, listnum;
+       u_int           i, r;
+       int             j, listnum;
        size_t          k;
-       u_short         u, *lp;
+       u_short         u, b, *lp;
        struct chunk_info *bp;
+       void *p;
 
        if (mopts.malloc_canary != (d->canary1 ^ (u_int32_t)(uintptr_t)d) ||
            d->canary1 != ~d->canary2)
@@ -948,7 +951,8 @@ malloc_bytes(struct dir_info *d, size_t size, void *f)
 
        j = find_chunksize(size);
 
-       listnum = getrbyte(d) % MALLOC_CHUNK_LISTS;
+       r = getrbyte(d);
+       listnum = r % MALLOC_CHUNK_LISTS;
        /* If it's empty, make a page more of that size chunks */
        if ((bp = LIST_FIRST(&d->chunk_dir[j][listnum])) == NULL) {
                bp = omalloc_make_chunks(d, j, listnum);
@@ -956,35 +960,47 @@ malloc_bytes(struct dir_info *d, size_t size, void *f)
                        return NULL;
        }
 
-       if (bp->canary != d->canary1)
+       if (bp->canary != (u_short)d->canary1)
                wrterror(d, "chunk info corrupted");
 
-       i = d->chunk_start;
        if (bp->free > 1)
-               i += getrbyte(d);
-       if (i >= bp->total)
-               i &= bp->total - 1;
-       for (;;) {
-               for (;;) {
-                       lp = &bp->bits[i / MALLOC_BITS];
-                       if (!*lp) {
-                               i += MALLOC_BITS;
-                               i &= ~(MALLOC_BITS - 1);
-                               if (i >= bp->total)
-                                       i = 0;
-                       } else
-                               break;
-               }
+               bp->rotor += r;
+       i = bp->rotor++ & (bp->total - 1);
+
+       /* start somewhere in a short */
+       lp = &bp->bits[i / MALLOC_BITS];
+       if (*lp) {
+               b = *lp;
                k = i % MALLOC_BITS;
                u = 1 << k;
-               if (*lp & u)
-                       break;
-               if (++i >= bp->total)
+               while (k < MALLOC_BITS) {
+                       if (b & u)
+                               goto found;
+                       k++;
+                       u <<= 1;
+               } 
+       }
+       /* no bit halfway, go to next full short */
+       i /= MALLOC_BITS;
+       for (;;) {
+               if (++i >= bp->total / MALLOC_BITS)
                        i = 0;
+               lp = &bp->bits[i];
+               if (*lp) {
+                       b = *lp;
+                       k = 0;
+                       u = 1;
+                       for (;;) {
+                               if (b & u)
+                                       goto found;
+                               k++;
+                               u <<= 1;
+                       }
+               }
        }
-       d->chunk_start += i + 1;
+found:
 #ifdef MALLOC_STATS
-       if (i == 0) {
+       if (i == 0 && k == 0) {
                struct region_info *r = find(d, bp->page);
                r->f = f;
        }
@@ -1004,13 +1020,14 @@ malloc_bytes(struct dir_info *d, size_t size, void *f)
 
        k <<= bp->shift;
 
+       p = (char *)bp->page + k;
        if (bp->size > 0) {
                if (mopts.malloc_junk == 2)
-                       memset((char *)bp->page + k, SOME_JUNK, bp->size);
+                       memset(p, SOME_JUNK, bp->size);
                else if (mopts.chunk_canaries)
-                       fill_canary((char *)bp->page + k, size, bp->size);
+                       fill_canary(p, size, bp->size);
        }
-       return ((char *)bp->page + k);
+       return p;
 }
 
 static void
@@ -1045,13 +1062,11 @@ validate_canary(struct dir_info *d, u_char *ptr, size_t sz, size_t allocated)
 }
 
 static uint32_t
-find_chunknum(struct dir_info *d, struct region_info *r, void *ptr, int check)
+find_chunknum(struct dir_info *d, struct chunk_info *info, void *ptr, int check)
 {
-       struct chunk_info *info;
        uint32_t chunknum;
 
-       info = (struct chunk_info *)r->size;
-       if (info->canary != d->canary1)
+       if (info->canary != (u_short)d->canary1)
                wrterror(d, "chunk info corrupted");
 
        /* Find the chunk number on the page */
@@ -1081,7 +1096,7 @@ free_bytes(struct dir_info *d, struct region_info *r, void *ptr)
        int listnum;
 
        info = (struct chunk_info *)r->size;
-       chunknum = find_chunknum(d, r, ptr, 0);
+       chunknum = find_chunknum(d, info, ptr, 0);
 
        info->bits[chunknum / MALLOC_BITS] |= 1U << (chunknum % MALLOC_BITS);
        info->free++;
@@ -1326,7 +1341,7 @@ ofree(struct dir_info *argpool, void *p, int clear, int check, size_t argsz)
                                struct chunk_info *info =
                                    (struct chunk_info *)r->size;
                                uint32_t chunknum =
-                                   find_chunknum(pool, r, p, 0);
+                                   find_chunknum(pool, info, p, 0);
 
                                if (info->bits[info->offset + chunknum] <
                                    argsz)
@@ -1373,7 +1388,8 @@ ofree(struct dir_info *argpool, void *p, int clear, int check, size_t argsz)
                delete(pool, r);
        } else {
                /* Validate and optionally canary check */
-               find_chunknum(pool, r, p, mopts.chunk_canaries);
+               struct chunk_info *info = (struct chunk_info *)r->size;
+               find_chunknum(pool, info, p, mopts.chunk_canaries);
                if (!clear) {
                        void *tmp;
                        int i;
@@ -1520,8 +1536,8 @@ orealloc(struct dir_info *argpool, void *p, size_t newsz, void *f)
 
        REALSIZE(oldsz, r);
        if (mopts.chunk_canaries && oldsz <= MALLOC_MAXCHUNK) {
-               chunknum = find_chunknum(pool, r, p, 0);
                info = (struct chunk_info *)r->size;
+               chunknum = find_chunknum(pool, info, p, 0);
        }
 
        goldsz = oldsz;
@@ -1788,7 +1804,7 @@ orecallocarray(struct dir_info *argpool, void *p, size_t oldsize,
        if (sz <= MALLOC_MAXCHUNK) {
                if (mopts.chunk_canaries && sz > 0) {
                        struct chunk_info *info = (struct chunk_info *)r->size;
-                       uint32_t chunknum = find_chunknum(pool, r, p, 0);
+                       uint32_t chunknum = find_chunknum(pool, info, p, 0);
 
                        if (info->bits[info->offset + chunknum] != oldsize)
                                wrterror(pool, "recorded old size %hu != %zu",