From: claudio Date: Mon, 15 Mar 2021 08:56:31 +0000 (+0000) Subject: Allocate the repo structs individually linked by a SLIST instead of using X-Git-Url: http://artulab.com/gitweb/?a=commitdiff_plain;h=bc7143cd8860cc46921507e46065bac0dd19dac5;p=openbsd Allocate the repo structs individually linked by a SLIST instead of using an array that is reallocated during runtime. With this the entityq can move back into struct repo. OK tb@ --- diff --git a/usr.sbin/rpki-client/main.c b/usr.sbin/rpki-client/main.c index 0b59f4ebf4e..92d193924a7 100644 --- a/usr.sbin/rpki-client/main.c +++ b/usr.sbin/rpki-client/main.c @@ -1,4 +1,4 @@ -/* $OpenBSD: main.c,v 1.118 2021/03/11 11:57:45 claudio Exp $ */ +/* $OpenBSD: main.c,v 1.119 2021/03/15 08:56:31 claudio Exp $ */ /* * Copyright (c) 2019 Kristaps Dzonsons * @@ -53,10 +53,12 @@ */ #define REPO_MAX_URI 2 struct repo { + SLIST_ENTRY(repo) entry; char *repouri; /* CA repository base URI */ char *local; /* local path name */ char *temp; /* temporary file / dir */ char *uris[REPO_MAX_URI]; /* URIs to fetch from */ + struct entityq queue; /* files waiting for this repo */ size_t id; /* identifier (array index) */ int uriidx; /* which URI is fetched */ int loaded; /* whether loaded or not */ @@ -70,10 +72,8 @@ void suicide(int sig); /* * Table of all known repositories. */ -static struct repotab { - struct repo *repos; /* repositories */ - size_t reposz; /* number of repos */ -} rt; +SLIST_HEAD(, repo) repos = SLIST_HEAD_INITIALIZER(repos); +size_t repoid; /* * Database of all file path accessed during a run. @@ -211,15 +211,13 @@ entity_write_req(const struct entity *ent) * repo, then flush those into the parser process. */ static void -entityq_flush(struct entityq *q, const struct repo *repo) +entityq_flush(struct repo *repo) { struct entity *p, *np; - TAILQ_FOREACH_SAFE(p, q, entries, np) { - if (p->repo < 0 || repo->id != (size_t)p->repo) - continue; + TAILQ_FOREACH_SAFE(p, &repo->queue, entries, np) { entity_write_req(p); - TAILQ_REMOVE(q, p, entries); + TAILQ_REMOVE(&repo->queue, p, entries); entity_free(p); } } @@ -228,9 +226,8 @@ entityq_flush(struct entityq *q, const struct repo *repo) * Add the heap-allocated file to the queue for processing. */ static void -entityq_add(struct entityq *q, char *file, enum rtype type, - const struct repo *rp, const unsigned char *pkey, size_t pkeysz, - char *descr) +entityq_add(char *file, enum rtype type, struct repo *rp, + const unsigned char *pkey, size_t pkeysz, char *descr) { struct entity *p; @@ -244,7 +241,6 @@ entityq_add(struct entityq *q, char *file, enum rtype type, p->type = type; p->file = file; - p->repo = (rp != NULL) ? (ssize_t)rp->id : -1; p->has_pkey = pkey != NULL; if (p->has_pkey) { p->pkeysz = pkeysz; @@ -267,25 +263,35 @@ entityq_add(struct entityq *q, char *file, enum rtype type, entity_write_req(p); entity_free(p); } else - TAILQ_INSERT_TAIL(q, p, entries); + TAILQ_INSERT_TAIL(&rp->queue, p, entries); } /* - * Allocat a new repository be extending the repotable. + * Allocate and insert a new repository. */ static struct repo * repo_alloc(void) { struct repo *rp; - rt.repos = recallocarray(rt.repos, rt.reposz, rt.reposz + 1, - sizeof(struct repo)); - if (rt.repos == NULL) + if ((rp = calloc(1, sizeof(*rp))) == NULL) err(1, NULL); - rp = &rt.repos[rt.reposz++]; - rp->id = rt.reposz - 1; + rp->id = ++repoid; + TAILQ_INIT(&rp->queue); + SLIST_INSERT_HEAD(&repos, rp, entry); + + return rp; +} + +static struct repo * +repo_find(size_t id) +{ + struct repo *rp; + SLIST_FOREACH(rp, &repos, entry) + if (id == rp->id) + break; return rp; } @@ -396,7 +402,7 @@ repo_fetch(struct repo *rp) /* * Look up a trust anchor, queueing it for download if not found. */ -static const struct repo * +static struct repo * ta_lookup(const struct tal *tal) { struct repo *rp; @@ -407,11 +413,11 @@ ta_lookup(const struct tal *tal) err(1, NULL); /* Look up in repository table. (Lookup should actually fail here) */ - for (i = 0; i < rt.reposz; i++) { - if (strcmp(rt.repos[i].local, local) != 0) + SLIST_FOREACH(rp, &repos, entry) { + if (strcmp(rp->local, local) != 0) continue; free(local); - return &rt.repos[i]; + return rp; } rp = repo_alloc(); @@ -430,23 +436,22 @@ ta_lookup(const struct tal *tal) /* * Look up a repository, queueing it for discovery if not found. */ -static const struct repo * +static struct repo * repo_lookup(const char *uri) { char *local, *repo; struct repo *rp; - size_t i; if ((repo = rsync_base_uri(uri)) == NULL) return NULL; /* Look up in repository table. */ - for (i = 0; i < rt.reposz; i++) { - if (rt.repos[i].repouri == NULL || - strcmp(rt.repos[i].repouri, repo) != 0) + SLIST_FOREACH(rp, &repos, entry) { + if (rp->repouri == NULL || + strcmp(rp->repouri, repo) != 0) continue; free(repo); - return &rt.repos[i]; + return rp; } rp = repo_alloc(); @@ -500,8 +505,7 @@ repo_filename(const struct repo *repo, const char *uri) * These are always relative to the directory in which "mft" sits. */ static void -queue_add_from_mft(struct entityq *q, const char *mft, - const struct mftfile *file, enum rtype type) +queue_add_from_mft(const char *mft, const struct mftfile *file, enum rtype type) { char *cp, *nfile; @@ -517,7 +521,7 @@ queue_add_from_mft(struct entityq *q, const char *mft, * that the repository has already been loaded. */ - entityq_add(q, nfile, type, NULL, NULL, 0, NULL); + entityq_add(nfile, type, NULL, NULL, 0, NULL); } /* @@ -529,7 +533,7 @@ queue_add_from_mft(struct entityq *q, const char *mft, * check the suffix anyway). */ static void -queue_add_from_mft_set(struct entityq *q, const struct mft *mft) +queue_add_from_mft_set(const struct mft *mft) { size_t i, sz; const struct mftfile *f; @@ -540,7 +544,7 @@ queue_add_from_mft_set(struct entityq *q, const struct mft *mft) assert(sz > 4); if (strcasecmp(f->file + sz - 4, ".crl") != 0) continue; - queue_add_from_mft(q, mft->file, f, RTYPE_CRL); + queue_add_from_mft(mft->file, f, RTYPE_CRL); } for (i = 0; i < mft->filesz; i++) { @@ -550,11 +554,11 @@ queue_add_from_mft_set(struct entityq *q, const struct mft *mft) if (strcasecmp(f->file + sz - 4, ".crl") == 0) continue; else if (strcasecmp(f->file + sz - 4, ".cer") == 0) - queue_add_from_mft(q, mft->file, f, RTYPE_CER); + queue_add_from_mft(mft->file, f, RTYPE_CER); else if (strcasecmp(f->file + sz - 4, ".roa") == 0) - queue_add_from_mft(q, mft->file, f, RTYPE_ROA); + queue_add_from_mft(mft->file, f, RTYPE_ROA); else if (strcasecmp(f->file + sz - 4, ".gbr") == 0) - queue_add_from_mft(q, mft->file, f, RTYPE_GBR); + queue_add_from_mft(mft->file, f, RTYPE_GBR); else logx("%s: unsupported file type: %s", mft->file, f->file); @@ -565,7 +569,7 @@ queue_add_from_mft_set(struct entityq *q, const struct mft *mft) * Add a local TAL file (RFC 7730) to the queue of files to fetch. */ static void -queue_add_tal(struct entityq *q, const char *file) +queue_add_tal(const char *file) { char *nfile, *buf; @@ -586,7 +590,7 @@ queue_add_tal(struct entityq *q, const char *file) } /* Not in a repository, so directly add to queue. */ - entityq_add(q, nfile, RTYPE_TAL, NULL, NULL, 0, buf); + entityq_add(nfile, RTYPE_TAL, NULL, NULL, 0, buf); /* entityq_add makes a copy of buf */ free(buf); } @@ -595,10 +599,10 @@ queue_add_tal(struct entityq *q, const char *file) * Add URIs (CER) from a TAL file, RFC 8630. */ static void -queue_add_from_tal(struct entityq *q, const struct tal *tal) +queue_add_from_tal(const struct tal *tal) { - char *nfile; - const struct repo *repo; + char *nfile; + struct repo *repo; assert(tal->urisz); @@ -606,7 +610,7 @@ queue_add_from_tal(struct entityq *q, const struct tal *tal) repo = ta_lookup(tal); nfile = ta_filename(repo, 0); - entityq_add(q, nfile, RTYPE_CER, repo, tal->pkey, + entityq_add(nfile, RTYPE_CER, repo, tal->pkey, tal->pkeysz, tal->descr); } @@ -614,10 +618,10 @@ queue_add_from_tal(struct entityq *q, const struct tal *tal) * Add a manifest (MFT) found in an X509 certificate, RFC 6487. */ static void -queue_add_from_cert(struct entityq *q, const struct cert *cert) +queue_add_from_cert(const struct cert *cert) { - const struct repo *repo; - char *nfile; + struct repo *repo; + char *nfile; repo = repo_lookup(cert->mft); if (repo == NULL) /* bad repository URI */ @@ -625,7 +629,7 @@ queue_add_from_cert(struct entityq *q, const struct cert *cert) nfile = repo_filename(repo, cert->mft); - entityq_add(q, nfile, RTYPE_MFT, repo, NULL, 0, NULL); + entityq_add(nfile, RTYPE_MFT, repo, NULL, 0, NULL); } /* @@ -635,8 +639,7 @@ queue_add_from_cert(struct entityq *q, const struct cert *cert) * In all cases, we gather statistics. */ static void -entity_process(int proc, struct stats *st, struct entityq *q, - struct vrp_tree *tree) +entity_process(int proc, struct stats *st, struct vrp_tree *tree) { enum rtype type; struct tal *tal; @@ -657,7 +660,7 @@ entity_process(int proc, struct stats *st, struct entityq *q, case RTYPE_TAL: st->tals++; tal = tal_read(proc); - queue_add_from_tal(q, tal); + queue_add_from_tal(tal); tal_free(tal); break; case RTYPE_CER: @@ -675,7 +678,7 @@ entity_process(int proc, struct stats *st, struct entityq *q, * we're revoked and then we don't want to * process the MFT. */ - queue_add_from_cert(q, cert); + queue_add_from_cert(cert); } else st->certs_invalid++; cert_free(cert); @@ -690,7 +693,7 @@ entity_process(int proc, struct stats *st, struct entityq *q, mft = mft_read(proc); if (mft->stale) st->mfts_stale++; - queue_add_from_mft_set(q, mft); + queue_add_from_mft_set(mft); mft_free(mft); break; case RTYPE_CRL: @@ -771,12 +774,12 @@ repo_cleanup(void) { size_t i, delsz = 0; char *argv[2], **del = NULL; + struct repo *rp; FTS *fts; FTSENT *e; - for (i = 0; i < rt.reposz; i++) { - if (asprintf(&argv[0], "%s", rt.repos[i].local) == -1) - err(1, NULL); + SLIST_FOREACH(rp, &repos, entry) { + argv[0] = rp->local; argv[1] = NULL; if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) @@ -841,12 +844,12 @@ main(int argc, char *argv[]) { int rc = 1, c, st, proc, rsync, http, fl = SOCK_STREAM | SOCK_CLOEXEC; - size_t i, outsz = 0, talsz = 0; + size_t i, id, outsz = 0, talsz = 0; pid_t procpid, rsyncpid, httppid; int fd[2]; - struct entityq q; struct pollfd pfd[3]; struct roa **out = NULL; + struct repo *rp; char *rsync_prog = "openrsync"; char *bind_addr = NULL; const char *cachedir = NULL, *outputdir = NULL; @@ -959,8 +962,6 @@ main(int argc, char *argv[]) if (talsz == 0) err(1, "no TAL files found in %s", "/etc/rpki"); - TAILQ_INIT(&q); - /* change working directory to the cache directory */ if (fchdir(cachefd) == -1) err(1, "fchdir"); @@ -1080,7 +1081,7 @@ main(int argc, char *argv[]) */ for (i = 0; i < talsz; i++) - queue_add_tal(&q, tals[i]); + queue_add_tal(tals[i]); while (entity_queue > 0 && !killme) { pfd[0].events = POLLIN; @@ -1143,33 +1144,36 @@ main(int argc, char *argv[]) if ((pfd[0].revents & POLLIN)) { int ok; - io_simple_read(rsync, &i, sizeof(size_t)); + io_simple_read(rsync, &id, sizeof(id)); io_simple_read(rsync, &ok, sizeof(ok)); - assert(i < rt.reposz); + rp = repo_find(id); + if (rp == NULL) + errx(1, "unknown repository id: %zu", id); - assert(!rt.repos[i].loaded); + assert(!rp->loaded); if (ok) - logx("%s: loaded from network", - rt.repos[i].local); + logx("%s: loaded from network", rp->local); else logx("%s: load from network failed, " - "fallback to cache", rt.repos[i].local); - rt.repos[i].loaded = 1; + "fallback to cache", rp->local); + rp->loaded = 1; stats.repos++; - entityq_flush(&q, &rt.repos[i]); + entityq_flush(rp); } if ((pfd[2].revents & POLLIN)) { int ok; - io_simple_read(http, &i, sizeof(size_t)); + io_simple_read(http, &id, sizeof(id)); io_simple_read(http, &ok, sizeof(ok)); - assert(i < rt.reposz); + rp = repo_find(id); + if (rp == NULL) + errx(1, "unknown repository id: %zu", id); - assert(!rt.repos[i].loaded); - if (http_done(&rt.repos[i], ok)) { - rt.repos[i].loaded = 1; + assert(!rp->loaded); + if (http_done(rp, ok)) { + rp->loaded = 1; stats.repos++; - entityq_flush(&q, &rt.repos[i]); + entityq_flush(rp); } } @@ -1179,7 +1183,7 @@ main(int argc, char *argv[]) */ if ((pfd[1].revents & POLLIN)) { - entity_process(proc, &stats, &q, &v); + entity_process(proc, &stats, &v); } } @@ -1189,7 +1193,7 @@ main(int argc, char *argv[]) errx(1, "excessive runtime (%d seconds), giving up", timeout); } - assert(TAILQ_EMPTY(&q)); + assert(entity_queue == 0); logx("all files parsed: generating output"); rc = 0; @@ -1260,14 +1264,15 @@ main(int argc, char *argv[]) logx("VRP Entries: %zu (%zu unique)", stats.vrps, stats.uniqs); /* Memory cleanup. */ - for (i = 0; i < rt.reposz; i++) { - free(rt.repos[i].repouri); - free(rt.repos[i].local); - free(rt.repos[i].temp); - free(rt.repos[i].uris[0]); - free(rt.repos[i].uris[1]); + while ((rp = SLIST_FIRST(&repos)) != NULL) { + SLIST_REMOVE_HEAD(&repos, entry); + free(rp->repouri); + free(rp->local); + free(rp->temp); + free(rp->uris[0]); + free(rp->uris[1]); + free(rp); } - free(rt.repos); for (i = 0; i < outsz; i++) roa_free(out[i]);