From: claudio Date: Wed, 26 Apr 2023 16:32:41 +0000 (+0000) Subject: Improve accounting by tracking things by repo and tal. X-Git-Url: http://artulab.com/gitweb/?a=commitdiff_plain;h=1fc2657f8f4c83a1a4f32c473d00f26e7c46058d;p=openbsd Improve accounting by tracking things by repo and tal. This fixes some wrong accounting for repositories that are referenced from more than one TAL. It changes the ometric lable output a little bit since there are repository metrics that no longer include the 'name' label. OK tb@ --- diff --git a/usr.sbin/rpki-client/aspa.c b/usr.sbin/rpki-client/aspa.c index 79a98caea0a..9f56abd26ec 100644 --- a/usr.sbin/rpki-client/aspa.c +++ b/usr.sbin/rpki-client/aspa.c @@ -1,4 +1,4 @@ -/* $OpenBSD: aspa.c,v 1.16 2023/03/12 11:54:56 job Exp $ */ +/* $OpenBSD: aspa.c,v 1.17 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2022 Job Snijders * Copyright (c) 2022 Theo Buehler @@ -283,6 +283,7 @@ aspa_buffer(struct ibuf *b, const struct aspa *p) { io_simple_buffer(b, &p->valid, sizeof(p->valid)); io_simple_buffer(b, &p->custasid, sizeof(p->custasid)); + io_simple_buffer(b, &p->talid, sizeof(p->talid)); io_simple_buffer(b, &p->expires, sizeof(p->expires)); io_simple_buffer(b, &p->providersz, sizeof(size_t)); @@ -309,6 +310,7 @@ aspa_read(struct ibuf *b) io_read_buf(b, &p->valid, sizeof(p->valid)); io_read_buf(b, &p->custasid, sizeof(p->custasid)); + io_read_buf(b, &p->talid, sizeof(p->talid)); io_read_buf(b, &p->expires, sizeof(p->expires)); io_read_buf(b, &p->providersz, sizeof(size_t)); @@ -350,20 +352,32 @@ aspa_insert_vaps(struct vap_tree *tree, struct aspa *aspa, struct repo *rp) struct vap *v, *found; size_t i, j; - repo_stat_inc(rp, RTYPE_ASPA, STYPE_TOTAL); - if ((v = calloc(1, sizeof(*v))) == NULL) err(1, NULL); v->custasid = aspa->custasid; + v->talid = aspa->talid; + if (rp != NULL) + v->repoid = repo_id(rp); + else + v->repoid = 0; v->expires = aspa->expires; if ((found = RB_INSERT(vap_tree, tree, v)) != NULL) { - if (found->expires > v->expires) + if (found->expires > v->expires) { + /* decrement found */ + repo_stat_inc(repo_byid(found->repoid), found->talid, + RTYPE_ASPA, STYPE_DEC_UNIQUE); found->expires = v->expires; + found->talid = v->talid; + found->repoid = v->repoid; + repo_stat_inc(rp, v->talid, RTYPE_ASPA, STYPE_UNIQUE); + } free(v); v = found; } else - repo_stat_inc(rp, RTYPE_ASPA, STYPE_UNIQUE); + repo_stat_inc(rp, v->talid, RTYPE_ASPA, STYPE_UNIQUE); + + repo_stat_inc(rp, aspa->talid, RTYPE_ASPA, STYPE_TOTAL); v->providers = reallocarray(v->providers, v->providersz + aspa->providersz, sizeof(*v->providers)); @@ -379,14 +393,14 @@ aspa_insert_vaps(struct vap_tree *tree, struct aspa *aspa, struct repo *rp) if (j == v->providersz || aspa->providers[i].as < v->providers[j].as) { /* merge provider from aspa into v */ - repo_stat_inc(rp, RTYPE_ASPA, + repo_stat_inc(rp, v->talid, RTYPE_ASPA, STYPE_BOTH + aspa->providers[i].afi); insert_vap(v, j, &aspa->providers[i]); i++; } else if (aspa->providers[i].as == v->providers[j].as) { /* duplicate provider, merge afi */ if (v->providers[j].afi != aspa->providers[i].afi) { - repo_stat_inc(rp, RTYPE_ASPA, + repo_stat_inc(rp, v->talid, RTYPE_ASPA, STYPE_BOTH + aspa->providers[i].afi); v->providers[j].afi = 0; } diff --git a/usr.sbin/rpki-client/extern.h b/usr.sbin/rpki-client/extern.h index ea83b04dbde..d2cab063636 100644 --- a/usr.sbin/rpki-client/extern.h +++ b/usr.sbin/rpki-client/extern.h @@ -1,4 +1,4 @@ -/* $OpenBSD: extern.h,v 1.177 2023/04/13 17:04:02 job Exp $ */ +/* $OpenBSD: extern.h,v 1.178 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2019 Kristaps Dzonsons * @@ -217,6 +217,7 @@ struct mft { time_t expires; /* when the signature path expires */ size_t filesz; /* number of filenames */ unsigned int repoid; + int talid; int stale; /* if a stale manifest */ }; @@ -383,6 +384,8 @@ struct vap { struct aspa_provider *providers; size_t providersz; time_t expires; + int talid; + unsigned int repoid; }; /* @@ -397,12 +400,12 @@ RB_PROTOTYPE(vap_tree, vap, entry, vapcmp); struct vrp { RB_ENTRY(vrp) entry; struct ip_addr addr; - int talid; /* covered by which TAL */ - unsigned int repoid; uint32_t asid; enum afi afi; unsigned char maxlength; time_t expires; /* transitive expiry moment */ + int talid; /* covered by which TAL */ + unsigned int repoid; }; /* * Tree of VRP sorted by afi, addr, maxlength and asid @@ -539,7 +542,7 @@ RB_HEAD(filepath_tree, filepath); /* * Statistics collected during run-time. */ -struct repostats { +struct repotalstats { uint32_t certs; /* certificates */ uint32_t certs_fail; /* invalid certificate */ uint32_t mfts; /* total number of manifests */ @@ -562,6 +565,13 @@ struct repostats { uint32_t vaps_pas6; /* total number of IPv6 only providers */ uint32_t vrps; /* total number of Validated ROA Payloads */ uint32_t vrps_uniqs; /* number of unique vrps */ +}; + +struct repostats { + uint32_t del_files; /* number of files removed in cleanup */ + uint32_t extra_files; /* number of superfluous files */ + uint32_t del_extra_files;/* number of removed extra files */ + uint32_t del_dirs; /* number of dirs removed in cleanup */ struct timespec sync_time; /* time to sync repo */ }; @@ -574,11 +584,9 @@ struct stats { uint32_t http_fails; /* failed http repositories */ uint32_t rrdp_repos; /* synced rrdp repositories */ uint32_t rrdp_fails; /* failed rrdp repositories */ - uint32_t del_files; /* number of files removed in cleanup */ - uint32_t extra_files; /* number of superfluous files */ - uint32_t del_dirs; /* number of dirs removed in cleanup */ uint32_t skiplistentries; /* number of skiplist entries */ + struct repotalstats repo_tal_stats; struct repostats repo_stats; struct timespec elapsed_time; struct timespec user_time; @@ -595,7 +603,7 @@ extern int excludeaspa; extern const char *tals[]; extern const char *taldescs[]; extern unsigned int talrepocnt[]; -extern struct repostats talstats[]; +extern struct repotalstats talstats[]; extern int talsz; /* Routines for RPKI entities. */ @@ -765,7 +773,9 @@ struct repo *repo_byid(unsigned int); int repo_queued(struct repo *, struct entity *); void repo_cleanup(struct filepath_tree *, int); int repo_check_timeout(int); -void repo_stat_inc(struct repo *, enum rtype, enum stype); +void repo_stat_inc(struct repo *, int, enum rtype, enum stype); +void repo_tal_stats_collect(void (*)(const struct repo *, + const struct repotalstats *, void *), int, void *); void repo_stats_collect(void (*)(const struct repo *, const struct repostats *, void *), void *); void repo_free(void); diff --git a/usr.sbin/rpki-client/filemode.c b/usr.sbin/rpki-client/filemode.c index 66098099429..b2e729c337d 100644 --- a/usr.sbin/rpki-client/filemode.c +++ b/usr.sbin/rpki-client/filemode.c @@ -1,4 +1,4 @@ -/* $OpenBSD: filemode.c,v 1.29 2023/03/15 11:09:34 job Exp $ */ +/* $OpenBSD: filemode.c,v 1.30 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2019 Claudio Jeker * Copyright (c) 2019 Kristaps Dzonsons @@ -609,6 +609,7 @@ parse_file(struct entityq *q, struct msgbuf *msgq) b = io_new_buffer(); io_simple_buffer(b, &entp->type, sizeof(entp->type)); io_simple_buffer(b, &entp->repoid, sizeof(entp->repoid)); + io_simple_buffer(b, &entp->talid, sizeof(entp->talid)); io_str_buffer(b, entp->file); io_close_buffer(msgq, b); entity_free(entp); diff --git a/usr.sbin/rpki-client/main.c b/usr.sbin/rpki-client/main.c index bfa71ff65c0..516fcd7f514 100644 --- a/usr.sbin/rpki-client/main.c +++ b/usr.sbin/rpki-client/main.c @@ -1,4 +1,4 @@ -/* $OpenBSD: main.c,v 1.233 2023/04/13 17:04:02 job Exp $ */ +/* $OpenBSD: main.c,v 1.234 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2021 Claudio Jeker * Copyright (c) 2019 Kristaps Dzonsons @@ -51,7 +51,7 @@ const char *tals[TALSZ_MAX]; const char *taldescs[TALSZ_MAX]; unsigned int talrepocnt[TALSZ_MAX]; -struct repostats talstats[TALSZ_MAX]; +struct repotalstats talstats[TALSZ_MAX]; int talsz; size_t entity_queue; @@ -400,7 +400,7 @@ queue_add_from_mft(const struct mft *mft) if ((mftaki = strdup(mft->aki)) == NULL) err(1, NULL); entityq_add(npath, nfile, f->type, f->location, rp, NULL, 0, - -1, mftaki); + mft->talid, mftaki); } } @@ -527,8 +527,8 @@ queue_add_from_cert(const struct cert *cert) err(1, NULL); } - entityq_add(npath, nfile, RTYPE_MFT, DIR_UNKNOWN, repo, NULL, 0, -1, - NULL); + entityq_add(npath, nfile, RTYPE_MFT, DIR_UNKNOWN, repo, NULL, 0, + cert->talid, NULL); } /* @@ -550,6 +550,7 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, struct repo *rp; char *file; unsigned int id; + int talid; int c; /* @@ -560,6 +561,7 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, */ io_read_buf(b, &type, sizeof(type)); io_read_buf(b, &id, sizeof(id)); + io_read_buf(b, &talid, sizeof(talid)); io_read_str(b, &file); /* in filemode messages can be ignored, only the accounting matters */ @@ -572,7 +574,7 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, } rp = repo_byid(id); - repo_stat_inc(rp, type, STYPE_OK); + repo_stat_inc(rp, talid, type, STYPE_OK); switch (type) { case RTYPE_TAL: st->tals++; @@ -583,7 +585,7 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, case RTYPE_CER: io_read_buf(b, &c, sizeof(c)); if (c == 0) { - repo_stat_inc(rp, type, STYPE_FAIL); + repo_stat_inc(rp, talid, type, STYPE_FAIL); break; } cert = cert_read(b); @@ -593,7 +595,7 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, break; case CERT_PURPOSE_BGPSEC_ROUTER: cert_insert_brks(brktree, cert); - repo_stat_inc(rp, type, STYPE_BGPSEC); + repo_stat_inc(rp, talid, type, STYPE_BGPSEC); break; default: errx(1, "unexpected cert purpose received"); @@ -604,14 +606,14 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, case RTYPE_MFT: io_read_buf(b, &c, sizeof(c)); if (c == 0) { - repo_stat_inc(rp, type, STYPE_FAIL); + repo_stat_inc(rp, talid, type, STYPE_FAIL); break; } mft = mft_read(b); if (!mft->stale) queue_add_from_mft(mft); else - repo_stat_inc(rp, type, STYPE_STALE); + repo_stat_inc(rp, talid, type, STYPE_STALE); mft_free(mft); break; case RTYPE_CRL: @@ -621,14 +623,14 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, case RTYPE_ROA: io_read_buf(b, &c, sizeof(c)); if (c == 0) { - repo_stat_inc(rp, type, STYPE_FAIL); + repo_stat_inc(rp, talid, type, STYPE_FAIL); break; } roa = roa_read(b); if (roa->valid) roa_insert_vrps(tree, roa, rp); else - repo_stat_inc(rp, type, STYPE_INVALID); + repo_stat_inc(rp, talid, type, STYPE_INVALID); roa_free(roa); break; case RTYPE_GBR: @@ -636,14 +638,14 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, case RTYPE_ASPA: io_read_buf(b, &c, sizeof(c)); if (c == 0) { - repo_stat_inc(rp, type, STYPE_FAIL); + repo_stat_inc(rp, talid, type, STYPE_FAIL); break; } aspa = aspa_read(b); if (aspa->valid) aspa_insert_vaps(vaptree, aspa, rp); else - repo_stat_inc(rp, type, STYPE_INVALID); + repo_stat_inc(rp, talid, type, STYPE_INVALID); aspa_free(aspa); break; case RTYPE_TAK: @@ -716,12 +718,9 @@ rrdp_process(struct ibuf *b) } static void -sum_stats(const struct repo *rp, const struct repostats *in, void *arg) +sum_stats(const struct repo *rp, const struct repotalstats *in, void *arg) { - struct repostats *out = arg; - - if (rp != NULL) - sum_stats(NULL, in, &talstats[repo_talid(rp)]); + struct repotalstats *out = arg; out->mfts += in->mfts; out->mfts_fail += in->mfts_fail; @@ -745,7 +744,16 @@ sum_stats(const struct repo *rp, const struct repostats *in, void *arg) out->vaps_pas += in->vaps_pas; out->vaps_pas4 += in->vaps_pas4; out->vaps_pas6 += in->vaps_pas6; +} +static void +sum_repostats(const struct repo *rp, const struct repostats *in, void *arg) +{ + struct repostats *out = arg; + + out->del_files += in->del_files; + out->extra_files += in->extra_files; + out->del_dirs += in->del_dirs; timespecadd(&in->sync_time, &out->sync_time, &out->sync_time); } @@ -1396,7 +1404,11 @@ main(int argc, char *argv[]) if (fchdir(outdirfd) == -1) err(1, "fchdir output dir"); - repo_stats_collect(sum_stats, &stats.repo_stats); + for (i = 0; i < talsz; i++) { + repo_tal_stats_collect(sum_stats, i, &talstats[i]); + repo_tal_stats_collect(sum_stats, i, &stats.repo_tal_stats); + } + repo_stats_collect(sum_repostats, &stats.repo_stats); if (outputfiles(&vrps, &brks, &vaps, &stats)) rc = 1; @@ -1408,29 +1420,32 @@ main(int argc, char *argv[]) (long long)stats.system_time.tv_sec); printf("Skiplist entries: %u\n", stats.skiplistentries); printf("Route Origin Authorizations: %u (%u failed parse, %u " - "invalid)\n", stats.repo_stats.roas, stats.repo_stats.roas_fail, - stats.repo_stats.roas_invalid); + "invalid)\n", stats.repo_tal_stats.roas, + stats.repo_tal_stats.roas_fail, + stats.repo_tal_stats.roas_invalid); printf("AS Provider Attestations: %u (%u failed parse, %u " - "invalid)\n", stats.repo_stats.aspas, stats.repo_stats.aspas_fail, - stats.repo_stats.aspas_invalid); - printf("BGPsec Router Certificates: %u\n", stats.repo_stats.brks); + "invalid)\n", stats.repo_tal_stats.aspas, + stats.repo_tal_stats.aspas_fail, + stats.repo_tal_stats.aspas_invalid); + printf("BGPsec Router Certificates: %u\n", stats.repo_tal_stats.brks); printf("Certificates: %u (%u invalid)\n", - stats.repo_stats.certs, stats.repo_stats.certs_fail); + stats.repo_tal_stats.certs, stats.repo_tal_stats.certs_fail); printf("Trust Anchor Locators: %u (%u invalid)\n", stats.tals, talsz - stats.tals); printf("Manifests: %u (%u failed parse, %u stale)\n", - stats.repo_stats.mfts, stats.repo_stats.mfts_fail, - stats.repo_stats.mfts_stale); - printf("Certificate revocation lists: %u\n", stats.repo_stats.crls); - printf("Ghostbuster records: %u\n", stats.repo_stats.gbrs); - printf("Trust Anchor Keys: %u\n", stats.repo_stats.taks); + stats.repo_tal_stats.mfts, stats.repo_tal_stats.mfts_fail, + stats.repo_tal_stats.mfts_stale); + printf("Certificate revocation lists: %u\n", stats.repo_tal_stats.crls); + printf("Ghostbuster records: %u\n", stats.repo_tal_stats.gbrs); + printf("Trust Anchor Keys: %u\n", stats.repo_tal_stats.taks); printf("Repositories: %u\n", stats.repos); printf("Cleanup: removed %u files, %u directories, %u superfluous\n", - stats.del_files, stats.del_dirs, stats.extra_files); - printf("VRP Entries: %u (%u unique)\n", stats.repo_stats.vrps, - stats.repo_stats.vrps_uniqs); - printf("VAP Entries: %u (%u unique)\n", stats.repo_stats.vaps, - stats.repo_stats.vaps_uniqs); + stats.repo_stats.del_files, stats.repo_stats.del_dirs, + stats.repo_stats.extra_files); + printf("VRP Entries: %u (%u unique)\n", stats.repo_tal_stats.vrps, + stats.repo_tal_stats.vrps_uniqs); + printf("VAP Entries: %u (%u unique)\n", stats.repo_tal_stats.vaps, + stats.repo_tal_stats.vaps_uniqs); /* Memory cleanup. */ repo_free(); diff --git a/usr.sbin/rpki-client/mft.c b/usr.sbin/rpki-client/mft.c index 5173ec0ba1e..f702caf3b7a 100644 --- a/usr.sbin/rpki-client/mft.c +++ b/usr.sbin/rpki-client/mft.c @@ -1,4 +1,4 @@ -/* $OpenBSD: mft.c,v 1.90 2023/04/24 17:11:33 claudio Exp $ */ +/* $OpenBSD: mft.c,v 1.91 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2022 Theo Buehler * Copyright (c) 2019 Kristaps Dzonsons @@ -470,6 +470,7 @@ mft_buffer(struct ibuf *b, const struct mft *p) io_simple_buffer(b, &p->stale, sizeof(p->stale)); io_simple_buffer(b, &p->repoid, sizeof(p->repoid)); + io_simple_buffer(b, &p->talid, sizeof(p->talid)); io_str_buffer(b, p->path); io_str_buffer(b, p->aia); @@ -502,6 +503,7 @@ mft_read(struct ibuf *b) io_read_buf(b, &p->stale, sizeof(p->stale)); io_read_buf(b, &p->repoid, sizeof(p->repoid)); + io_read_buf(b, &p->talid, sizeof(p->talid)); io_read_str(b, &p->path); io_read_str(b, &p->aia); diff --git a/usr.sbin/rpki-client/output-json.c b/usr.sbin/rpki-client/output-json.c index cf752e7ed02..28375f096fd 100644 --- a/usr.sbin/rpki-client/output-json.c +++ b/usr.sbin/rpki-client/output-json.c @@ -1,4 +1,4 @@ -/* $OpenBSD: output-json.c,v 1.32 2023/04/20 15:05:44 job Exp $ */ +/* $OpenBSD: output-json.c,v 1.33 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2019 Claudio Jeker * @@ -58,16 +58,16 @@ outputheader_json(FILE *out, struct stats *st) "\t\t\"talfiles\": [\n", hn, tbuf, (long long)st->elapsed_time.tv_sec, (long long)st->user_time.tv_sec, (long long)st->system_time.tv_sec, - st->repo_stats.roas, - st->repo_stats.roas_fail, - st->repo_stats.roas_invalid, - st->repo_stats.aspas, - st->repo_stats.aspas_fail, - st->repo_stats.aspas_invalid, - st->repo_stats.brks, - st->repo_stats.certs, - st->repo_stats.certs_fail, - st->repo_stats.taks, + st->repo_tal_stats.roas, + st->repo_tal_stats.roas_fail, + st->repo_tal_stats.roas_invalid, + st->repo_tal_stats.aspas, + st->repo_tal_stats.aspas_fail, + st->repo_tal_stats.aspas_invalid, + st->repo_tal_stats.brks, + st->repo_tal_stats.certs, + st->repo_tal_stats.certs_fail, + st->repo_tal_stats.taks, st->tals, talsz - st->tals) < 0) return -1; @@ -94,19 +94,19 @@ outputheader_json(FILE *out, struct stats *st) "\t\t\"cachedir_superfluous_files\": %u,\n" "\t\t\"cachedir_del_dirs\": %u\n" "\t},\n\n", - st->repo_stats.mfts, - st->repo_stats.mfts_fail, - st->repo_stats.mfts_stale, - st->repo_stats.crls, - st->repo_stats.gbrs, + st->repo_tal_stats.mfts, + st->repo_tal_stats.mfts_fail, + st->repo_tal_stats.mfts_stale, + st->repo_tal_stats.crls, + st->repo_tal_stats.gbrs, st->repos, - st->repo_stats.vrps, - st->repo_stats.vrps_uniqs, - st->repo_stats.vaps, - st->repo_stats.vaps_uniqs, - st->del_files, - st->extra_files, - st->del_dirs) < 0) + st->repo_tal_stats.vrps, + st->repo_tal_stats.vrps_uniqs, + st->repo_tal_stats.vaps, + st->repo_tal_stats.vaps_uniqs, + st->repo_stats.del_files, + st->repo_stats.extra_files, + st->repo_stats.del_dirs) < 0) return -1; return 0; } diff --git a/usr.sbin/rpki-client/output-ometric.c b/usr.sbin/rpki-client/output-ometric.c index 412309d53a4..9ddd8960dd4 100644 --- a/usr.sbin/rpki-client/output-ometric.c +++ b/usr.sbin/rpki-client/output-ometric.c @@ -1,4 +1,4 @@ -/* $OpenBSD: output-ometric.c,v 1.2 2023/03/30 15:29:15 claudio Exp $ */ +/* $OpenBSD: output-ometric.c,v 1.3 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2022 Claudio Jeker * @@ -35,7 +35,7 @@ static const char * const repo_states[2] = { "failed", "synced" }; static const char * const repo_protos[3] = { "rrdp", "rsync", "https" }; static void -set_common_stats(const struct repostats *in, struct ometric *metric, +set_common_stats(const struct repotalstats *in, struct ometric *metric, struct olabels *ol) { ometric_set_int_with_labels(metric, in->certs, @@ -106,19 +106,44 @@ ta_stats(int id) } static void -repo_stats(const struct repo *rp, const struct repostats *in, void *arg) +repo_tal_stats(const struct repo *rp, const struct repotalstats *in, void *arg) { struct olabels *ol; const char *keys[4] = { "name", "carepo", "notify", NULL }; const char *values[4]; + int talid = *(int *)arg; - values[0] = taldescs[repo_talid(rp)]; + values[0] = taldescs[talid]; repo_fetch_uris(rp, &values[1], &values[2]); values[3] = NULL; ol = olabels_new(keys, values); set_common_stats(in, rpki_repo_obj, ol); + olabels_free(ol); +} + +static void +repo_stats(const struct repo *rp, const struct repostats *in, void *arg) +{ + struct olabels *ol; + const char *keys[3] = { "carepo", "notify", NULL }; + const char *values[3]; + + repo_fetch_uris(rp, &values[0], &values[1]); + values[2] = NULL; + + ol = olabels_new(keys, values); ometric_set_timespec(rpki_repo_duration, &in->sync_time, ol); + + ometric_set_int_with_labels(rpki_repo_obj, in->del_files, + OKV("type", "state"), OKV("files", "deleted"), ol); + ometric_set_int_with_labels(rpki_repo_obj, in->extra_files, + OKV("type", "state"), OKV("files", "extra"), ol); + ometric_set_int_with_labels(rpki_repo_obj, in->del_extra_files, + OKV("type", "state"), OKV("files", "deleted_extra"), ol); + ometric_set_int_with_labels(rpki_repo_obj, in->del_dirs, + OKV("type", "state"), OKV("dirs", "deleted"), ol); + ometric_set_state(rpki_repo_state, repo_states[repo_synced(rp)], ol); if (repo_synced(rp)) ometric_set_state(rpki_repo_proto, repo_proto(rp), ol); @@ -184,10 +209,12 @@ output_ometric(FILE *out, struct vrp_tree *vrps, struct brk_tree *brks, ometric_set_info(rpki_info, NULL, NULL, ol); olabels_free(ol); - repo_stats_collect(repo_stats, NULL); - for (i = 0; i < talsz; i++) + for (i = 0; i < talsz; i++) { + repo_tal_stats_collect(repo_tal_stats, i, &i); ta_stats(i); - set_common_stats(&st->repo_stats, rpki_obj, NULL); + } + repo_stats_collect(repo_stats, NULL); + set_common_stats(&st->repo_tal_stats, rpki_obj, NULL); ometric_set_int(rpki_repo, st->repos, NULL); ometric_set_int_with_labels(rpki_repo, st->rsync_repos, diff --git a/usr.sbin/rpki-client/output.c b/usr.sbin/rpki-client/output.c index 878b6b68cd7..659476d350a 100644 --- a/usr.sbin/rpki-client/output.c +++ b/usr.sbin/rpki-client/output.c @@ -1,4 +1,4 @@ -/* $OpenBSD: output.c,v 1.30 2023/04/19 12:58:16 jsg Exp $ */ +/* $OpenBSD: output.c,v 1.31 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2019 Theo de Raadt * @@ -218,9 +218,9 @@ outputheader(FILE *out, struct stats *st) "# Certificates: %u (%u invalid)\n", hn, tbuf, (long long)st->elapsed_time.tv_sec, (long long)st->user_time.tv_sec, (long long)st->system_time.tv_sec, - st->repo_stats.roas, st->repo_stats.roas_fail, - st->repo_stats.roas_invalid, st->repo_stats.brks, - st->repo_stats.certs, st->repo_stats.certs_fail) < 0) + st->repo_tal_stats.roas, st->repo_tal_stats.roas_fail, + st->repo_tal_stats.roas_invalid, st->repo_tal_stats.brks, + st->repo_tal_stats.certs, st->repo_tal_stats.certs_fail) < 0) return -1; if (fprintf(out, @@ -238,12 +238,12 @@ outputheader(FILE *out, struct stats *st) "# Ghostbuster records: %u\n" "# Repositories: %u\n" "# VRP Entries: %u (%u unique)\n", - st->repo_stats.mfts, st->repo_stats.mfts_fail, - st->repo_stats.mfts_stale, - st->repo_stats.crls, - st->repo_stats.gbrs, + st->repo_tal_stats.mfts, st->repo_tal_stats.mfts_fail, + st->repo_tal_stats.mfts_stale, + st->repo_tal_stats.crls, + st->repo_tal_stats.gbrs, st->repos, - st->repo_stats.vrps, st->repo_stats.vrps_uniqs) < 0) + st->repo_tal_stats.vrps, st->repo_tal_stats.vrps_uniqs) < 0) return -1; return 0; } diff --git a/usr.sbin/rpki-client/parser.c b/usr.sbin/rpki-client/parser.c index 58e95235b53..23eaf602e68 100644 --- a/usr.sbin/rpki-client/parser.c +++ b/usr.sbin/rpki-client/parser.c @@ -1,4 +1,4 @@ -/* $OpenBSD: parser.c,v 1.90 2023/04/13 17:04:02 job Exp $ */ +/* $OpenBSD: parser.c,v 1.91 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2019 Claudio Jeker * Copyright (c) 2019 Kristaps Dzonsons @@ -298,6 +298,8 @@ proc_parser_mft_pre(struct entity *entp, enum location loc, char **file, X509_free(x509); mft->repoid = entp->repoid; + mft->talid = a->cert->talid; + return mft; } @@ -635,6 +637,7 @@ parse_entity(struct entityq *q, struct msgbuf *msgq) b = io_new_buffer(); io_simple_buffer(b, &entp->type, sizeof(entp->type)); io_simple_buffer(b, &entp->repoid, sizeof(entp->repoid)); + io_simple_buffer(b, &entp->talid, sizeof(entp->talid)); file = NULL; f = NULL; @@ -689,6 +692,8 @@ parse_entity(struct entityq *q, struct msgbuf *msgq) io_simple_buffer(b2, &type, sizeof(type)); io_simple_buffer(b2, &entp->repoid, sizeof(entp->repoid)); + io_simple_buffer(b2, &entp->talid, + sizeof(entp->talid)); io_str_buffer(b2, crlfile); free(crlfile); diff --git a/usr.sbin/rpki-client/repo.c b/usr.sbin/rpki-client/repo.c index 08ab2a0d206..81b06b8d4dd 100644 --- a/usr.sbin/rpki-client/repo.c +++ b/usr.sbin/rpki-client/repo.c @@ -1,4 +1,4 @@ -/* $OpenBSD: repo.c,v 1.43 2023/03/30 15:29:15 claudio Exp $ */ +/* $OpenBSD: repo.c,v 1.44 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2021 Claudio Jeker * Copyright (c) 2019 Kristaps Dzonsons @@ -97,10 +97,12 @@ struct repo { const struct rsyncrepo *rsync; const struct tarepo *ta; struct entityq queue; /* files waiting for repo */ - struct repostats stats; + struct repotalstats stats[TALSZ_MAX]; + struct repostats repostats; struct timespec start_time; time_t alarm; /* sync timeout */ int talid; + int stats_used[TALSZ_MAX]; unsigned int id; /* identifier */ }; static SLIST_HEAD(, repo) repos = SLIST_HEAD_INITIALIZER(repos); @@ -302,7 +304,8 @@ repo_done(const void *vp, int ok) entityq_flush(&rp->queue, rp); clock_gettime(CLOCK_MONOTONIC, &flush_time); - timespecsub(&flush_time, &rp->start_time, &rp->stats.sync_time); + timespecsub(&flush_time, &rp->start_time, + &rp->repostats.sync_time); } } @@ -555,16 +558,18 @@ rrdp_free(void) * Check if a directory is an active rrdp repository. * Returns 1 if found else 0. */ -static int -rrdp_is_active(const char *dir) +static struct repo * +repo_rrdp_bypath(const char *dir) { - struct rrdprepo *rr; - - SLIST_FOREACH(rr, &rrdprepos, entry) - if (strcmp(dir, rr->basedir) == 0) - return rr->state != REPO_FAILED; + struct repo *rp; - return 0; + SLIST_FOREACH(rp, &repos, entry) { + if (rp->rrdp == NULL) + continue; + if (strcmp(dir, rp->rrdp->basedir) == 0) + return rp; + } + return NULL; } /* @@ -1322,48 +1327,49 @@ repo_check_timeout(int timeout) * Update stats object of repository depending on rtype and subtype. */ void -repo_stat_inc(struct repo *rp, enum rtype type, enum stype subtype) +repo_stat_inc(struct repo *rp, int talid, enum rtype type, enum stype subtype) { if (rp == NULL) return; + rp->stats_used[talid] = 1; switch (type) { case RTYPE_CER: if (subtype == STYPE_OK) - rp->stats.certs++; + rp->stats[talid].certs++; if (subtype == STYPE_FAIL) - rp->stats.certs_fail++; + rp->stats[talid].certs_fail++; if (subtype == STYPE_BGPSEC) { - rp->stats.certs--; - rp->stats.brks++; + rp->stats[talid].certs--; + rp->stats[talid].brks++; } break; case RTYPE_MFT: if (subtype == STYPE_OK) - rp->stats.mfts++; + rp->stats[talid].mfts++; if (subtype == STYPE_FAIL) - rp->stats.mfts_fail++; + rp->stats[talid].mfts_fail++; if (subtype == STYPE_STALE) - rp->stats.mfts_stale++; + rp->stats[talid].mfts_stale++; break; case RTYPE_ROA: switch (subtype) { case STYPE_OK: - rp->stats.roas++; + rp->stats[talid].roas++; break; case STYPE_FAIL: - rp->stats.roas_fail++; + rp->stats[talid].roas_fail++; break; case STYPE_INVALID: - rp->stats.roas_invalid++; + rp->stats[talid].roas_invalid++; break; case STYPE_TOTAL: - rp->stats.vrps++; + rp->stats[talid].vrps++; break; case STYPE_UNIQUE: - rp->stats.vrps_uniqs++; + rp->stats[talid].vrps_uniqs++; break; case STYPE_DEC_UNIQUE: - rp->stats.vrps_uniqs--; + rp->stats[talid].vrps_uniqs--; break; default: break; @@ -1372,41 +1378,44 @@ repo_stat_inc(struct repo *rp, enum rtype type, enum stype subtype) case RTYPE_ASPA: switch (subtype) { case STYPE_OK: - rp->stats.aspas++; + rp->stats[talid].aspas++; break; case STYPE_FAIL: - rp->stats.aspas_fail++; + rp->stats[talid].aspas_fail++; break; case STYPE_INVALID: - rp->stats.aspas_invalid++; + rp->stats[talid].aspas_invalid++; break; case STYPE_TOTAL: - rp->stats.vaps++; + rp->stats[talid].vaps++; break; case STYPE_UNIQUE: - rp->stats.vaps_uniqs++; + rp->stats[talid].vaps_uniqs++; + break; + case STYPE_DEC_UNIQUE: + rp->stats[talid].vaps_uniqs--; break; case STYPE_BOTH: - rp->stats.vaps_pas++; + rp->stats[talid].vaps_pas++; break; case STYPE_ONLY_IPV4: - rp->stats.vaps_pas4++; + rp->stats[talid].vaps_pas4++; break; case STYPE_ONLY_IPV6: - rp->stats.vaps_pas6++; + rp->stats[talid].vaps_pas6++; break; default: break; } break; case RTYPE_CRL: - rp->stats.crls++; + rp->stats[talid].crls++; break; case RTYPE_GBR: - rp->stats.gbrs++; + rp->stats[talid].gbrs++; break; case RTYPE_TAK: - rp->stats.taks++; + rp->stats[talid].taks++; break; default: break; @@ -1414,16 +1423,27 @@ repo_stat_inc(struct repo *rp, enum rtype type, enum stype subtype) } void -repo_stats_collect(void (*cb)(const struct repo *, const struct repostats *, - void *), void *arg) +repo_tal_stats_collect(void (*cb)(const struct repo *, + const struct repotalstats *, void *), int talid, void *arg) { struct repo *rp; SLIST_FOREACH(rp, &repos, entry) { - cb(rp, &rp->stats, arg); + if (rp->stats_used[talid]) + cb(rp, &rp->stats[talid], arg); } } +void +repo_stats_collect(void (*cb)(const struct repo *, const struct repostats *, + void *), void *arg) +{ + struct repo *rp; + + SLIST_FOREACH(rp, &repos, entry) + cb(rp, &rp->repostats, arg); +} + /* * Delayed delete of files from RRDP. Since RRDP has no security built-in * this code needs to check if this RRDP repository is actually allowed to @@ -1432,11 +1452,15 @@ repo_stats_collect(void (*cb)(const struct repo *, const struct repostats *, static void repo_cleanup_rrdp(struct filepath_tree *tree) { + struct repo *rp; struct rrdprepo *rr; struct filepath *fp, *nfp; char *fn; - SLIST_FOREACH(rr, &rrdprepos, entry) { + SLIST_FOREACH(rp, &repos, entry) { + if (rp->rrdp == NULL) + continue; + rr = (struct rrdprepo *)rp->rrdp; RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) { if (!rrdp_uri_valid(rr, fp->file)) { warnx("%s: external URI %s", rr->notifyuri, @@ -1453,7 +1477,7 @@ repo_cleanup_rrdp(struct filepath_tree *tree) } else { if (verbose > 1) logx("deleted %s", fn); - stats.del_files++; + rp->repostats.del_files++; } free(fn); @@ -1466,7 +1490,7 @@ repo_cleanup_rrdp(struct filepath_tree *tree) } else { if (verbose > 1) logx("deleted %s", fn); - stats.del_files++; + rp->repostats.del_files++; } } else warnx("%s: referenced file supposed to be " @@ -1524,18 +1548,16 @@ repo_move_valid(struct filepath_tree *tree) } } -#define BASE_DIR (void *)0x01 -#define RSYNC_DIR (void *)0x02 -#define RRDP_DIR (void *)0x03 +struct fts_state { + enum { BASE_DIR, RSYNC_DIR, RRDP_DIR } type; + struct repo *rp; +} fts_state; static const struct rrdprepo * repo_is_rrdp(struct repo *rp) { /* check for special pointers first these are not a repository */ - if (rp == NULL || rp == BASE_DIR || rp == RSYNC_DIR || rp == RRDP_DIR) - return NULL; - - if (rp->rrdp) + if (rp != NULL && rp->rrdp != NULL) return rp->rrdp->state == REPO_DONE ? rp->rrdp : NULL; return NULL; } @@ -1548,152 +1570,166 @@ skip_dotslash(char *in) return in; } -void -repo_cleanup(struct filepath_tree *tree, int cachefd) +static void +repo_cleanup_entry(FTSENT *e, struct filepath_tree *tree, int cachefd) { - char *argv[2] = { ".", NULL }; - FTS *fts; - FTSENT *e; const struct rrdprepo *rr; + char *path; - /* first move temp files which have been used to valid dir */ - repo_move_valid(tree); - /* then delete files requested by rrdp */ - repo_cleanup_rrdp(tree); - - if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) - err(1, "fts_open"); - errno = 0; - while ((e = fts_read(fts)) != NULL) { - char *path = skip_dotslash(e->fts_path); - switch (e->fts_info) { - case FTS_NSOK: - if (filepath_exists(tree, path)) { - e->fts_parent->fts_number++; + path = skip_dotslash(e->fts_path); + switch (e->fts_info) { + case FTS_NSOK: + if (filepath_exists(tree, path)) { + e->fts_parent->fts_number++; + break; + } + if (fts_state.type == RRDP_DIR && fts_state.rp != NULL) { + e->fts_parent->fts_number++; + /* handle rrdp .state files explicitly */ + if (e->fts_level == 3 && + strcmp(e->fts_name, ".state") == 0) break; + /* can't delete these extra files */ + fts_state.rp->repostats.extra_files++; + if (verbose > 1) + logx("superfluous %s", path); + break; + } + rr = repo_is_rrdp(fts_state.rp); + if (rr != NULL) { + struct stat st; + char *fn; + + if (asprintf(&fn, "%s/%s", rr->basedir, path) == -1) + err(1, NULL); + + /* + * If the file exists in the rrdp dir + * that file is newer and needs to be kept + * so unlink this file instead of moving + * it over the file in the rrdp dir. + */ + if (fstatat(cachefd, fn, &st, 0) == 0 && + S_ISREG(st.st_mode)) { + free(fn); + goto unlink; } - if (e->fts_parent->fts_pointer == RRDP_DIR) { - e->fts_parent->fts_number++; - /* handle rrdp .state files explicitly */ - if (e->fts_level == 3 && - strcmp(e->fts_name, ".state") == 0) - break; - /* can't delete these extra files */ - stats.extra_files++; - if (verbose > 1) - logx("superfluous %s", path); - break; + if (repo_mkpath(cachefd, fn) == 0) { + if (renameat(AT_FDCWD, e->fts_accpath, + cachefd, fn) == -1) + warn("rename %s to %s", path, fn); + else if (verbose > 1) + logx("moved %s", path); + fts_state.rp->repostats.extra_files++; } - if (e->fts_parent->fts_pointer == RSYNC_DIR) { + free(fn); + } else { + unlink: + if (unlink(e->fts_accpath) == -1) { + warn("unlink %s", path); + } else if (fts_state.type == RSYNC_DIR) { /* no need to keep rsync files */ if (verbose > 1) logx("superfluous %s", path); + if (fts_state.rp != NULL) + fts_state.rp->repostats.del_extra_files++; + else + stats.repo_stats.del_extra_files++; + } else { + if (verbose > 1) + logx("deleted %s", path); + if (fts_state.rp != NULL) + fts_state.rp->repostats.del_files++; + else + stats.repo_stats.del_files++; } - rr = repo_is_rrdp(e->fts_parent->fts_pointer); - if (rr != NULL) { - struct stat st; - char *fn; - - if (asprintf(&fn, "%s/%s", rr->basedir, - path) == -1) - err(1, NULL); - - /* - * If the file exists in the rrdp dir - * that file is newer and needs to be kept - * so unlink this file instead of moving - * it over the file in the rrdp dir. - */ - if (fstatat(cachefd, fn, &st, 0) == 0 && - S_ISREG(st.st_mode)) { - free(fn); - goto unlink; - } - if (repo_mkpath(cachefd, fn) == 0) { - if (renameat(AT_FDCWD, e->fts_accpath, - cachefd, fn) == -1) - warn("rename %s to %s", path, - fn); - else if (verbose > 1) - logx("moved %s", path); - stats.extra_files++; - } - free(fn); + } + break; + case FTS_D: + if (e->fts_level == FTS_ROOTLEVEL) + fts_state.type = BASE_DIR; + if (e->fts_level == 1) { + if (strcmp(".rsync", e->fts_name) == 0) { + fts_state.type = RSYNC_DIR; + fts_state.rp = NULL; + } else if (strcmp(".rrdp", e->fts_name) == 0) { + fts_state.type = RRDP_DIR; + fts_state.rp = NULL; } else { - unlink: - if (unlink(e->fts_accpath) == -1) { - warn("unlink %s", path); - } else { - if (verbose > 1) - logx("deleted %s", path); - stats.del_files++; - } + fts_state.type = BASE_DIR; + fts_state.rp = repo_bypath(path); } - break; - case FTS_D: - if (e->fts_level == 1) { - if (strcmp(".rsync", e->fts_name) == 0) - e->fts_pointer = RSYNC_DIR; - else if (strcmp(".rrdp", e->fts_name) == 0) - e->fts_pointer = RRDP_DIR; - else - e->fts_pointer = BASE_DIR; - } else - e->fts_pointer = e->fts_parent->fts_pointer; - + } + if (e->fts_level == 2) { + if (fts_state.type == RSYNC_DIR) + fts_state.rp = repo_bypath(path); /* * special handling for rrdp directories, * clear them if they are not used anymore but * only if rrdp is active. */ - if (e->fts_pointer == RRDP_DIR && e->fts_level == 2) { - if (!rrdp_is_active(path)) - e->fts_pointer = NULL; - } - if (e->fts_pointer == BASE_DIR && e->fts_level > 1) { - e->fts_pointer = repo_bypath(path); - if (e->fts_pointer == NULL) - e->fts_pointer = BASE_DIR; - } + if (fts_state.type == RRDP_DIR) + fts_state.rp = repo_rrdp_bypath(path); + } + break; + case FTS_DP: + if (e->fts_level == FTS_ROOTLEVEL) break; - case FTS_DP: - if (e->fts_level == FTS_ROOTLEVEL) + if (e->fts_level == 1) { + /* do not remove .rsync and .rrdp */ + fts_state.rp = NULL; + if (fts_state.type == RRDP_DIR || + fts_state.type == RSYNC_DIR) break; - if (e->fts_level == 1) - /* do not remove .rsync and .rrdp */ - if (e->fts_pointer == RRDP_DIR || - e->fts_pointer == RSYNC_DIR) - break; - - e->fts_parent->fts_number += e->fts_number; - - if (e->fts_number == 0) { - if (rmdir(e->fts_accpath) == -1) - warn("rmdir %s", path); - else - stats.del_dirs++; - } - break; - case FTS_SL: - case FTS_SLNONE: - warnx("symlink %s", path); - if (unlink(e->fts_accpath) == -1) - warn("unlink %s", path); - break; - case FTS_NS: - case FTS_ERR: - if (e->fts_errno == ENOENT && - e->fts_level == FTS_ROOTLEVEL) - continue; - warnx("fts_read %s: %s", path, - strerror(e->fts_errno)); - break; - default: - warnx("fts_read %s: unhandled[%x]", path, - e->fts_info); - break; } + e->fts_parent->fts_number += e->fts_number; + + if (e->fts_number == 0) { + if (rmdir(e->fts_accpath) == -1) + warn("rmdir %s", path); + if (fts_state.rp != NULL) + fts_state.rp->repostats.del_dirs++; + else + stats.repo_stats.del_dirs++; + } + break; + case FTS_SL: + case FTS_SLNONE: + warnx("symlink %s", path); + if (unlink(e->fts_accpath) == -1) + warn("unlink %s", path); + stats.repo_stats.del_extra_files++; + break; + case FTS_NS: + case FTS_ERR: + if (e->fts_errno == ENOENT && e->fts_level == FTS_ROOTLEVEL) + break; + warnx("fts_read %s: %s", path, strerror(e->fts_errno)); + break; + default: + warnx("fts_read %s: unhandled[%x]", path, e->fts_info); + break; + } +} + +void +repo_cleanup(struct filepath_tree *tree, int cachefd) +{ + char *argv[2] = { ".", NULL }; + FTS *fts; + FTSENT *e; + + /* first move temp files which have been used to valid dir */ + repo_move_valid(tree); + /* then delete files requested by rrdp */ + repo_cleanup_rrdp(tree); + + if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) + err(1, "fts_open"); + errno = 0; + while ((e = fts_read(fts)) != NULL) { + repo_cleanup_entry(e, tree, cachefd); errno = 0; } if (errno) diff --git a/usr.sbin/rpki-client/roa.c b/usr.sbin/rpki-client/roa.c index eba2a474458..705208cbd2d 100644 --- a/usr.sbin/rpki-client/roa.c +++ b/usr.sbin/rpki-client/roa.c @@ -1,4 +1,4 @@ -/* $OpenBSD: roa.c,v 1.65 2023/03/12 11:54:56 job Exp $ */ +/* $OpenBSD: roa.c,v 1.66 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2022 Theo Buehler * Copyright (c) 2019 Kristaps Dzonsons @@ -384,19 +384,20 @@ roa_insert_vrps(struct vrp_tree *tree, struct roa *roa, struct repo *rp) /* already exists */ if (found->expires < v->expires) { /* update found with preferred data */ - found->talid = v->talid; - found->expires = v->expires; /* adjust unique count */ repo_stat_inc(repo_byid(found->repoid), - RTYPE_ROA, STYPE_DEC_UNIQUE); + found->talid, RTYPE_ROA, STYPE_DEC_UNIQUE); + found->expires = v->expires; + found->talid = v->talid; found->repoid = v->repoid; - repo_stat_inc(rp, RTYPE_ROA, STYPE_UNIQUE); + repo_stat_inc(rp, v->talid, RTYPE_ROA, + STYPE_UNIQUE); } free(v); } else - repo_stat_inc(rp, RTYPE_ROA, STYPE_UNIQUE); + repo_stat_inc(rp, v->talid, RTYPE_ROA, STYPE_UNIQUE); - repo_stat_inc(rp, RTYPE_ROA, STYPE_TOTAL); + repo_stat_inc(rp, roa->talid, RTYPE_ROA, STYPE_TOTAL); } }