From 1fc2657f8f4c83a1a4f32c473d00f26e7c46058d Mon Sep 17 00:00:00 2001 From: claudio Date: Wed, 26 Apr 2023 16:32:41 +0000 Subject: [PATCH] Improve accounting by tracking things by repo and tal. This fixes some wrong accounting for repositories that are referenced from more than one TAL. It changes the ometric lable output a little bit since there are repository metrics that no longer include the 'name' label. OK tb@ --- usr.sbin/rpki-client/aspa.c | 28 +- usr.sbin/rpki-client/extern.h | 28 +- usr.sbin/rpki-client/filemode.c | 3 +- usr.sbin/rpki-client/main.c | 87 +++--- usr.sbin/rpki-client/mft.c | 4 +- usr.sbin/rpki-client/output-json.c | 46 +-- usr.sbin/rpki-client/output-ometric.c | 41 ++- usr.sbin/rpki-client/output.c | 18 +- usr.sbin/rpki-client/parser.c | 7 +- usr.sbin/rpki-client/repo.c | 384 ++++++++++++++------------ usr.sbin/rpki-client/roa.c | 15 +- 11 files changed, 386 insertions(+), 275 deletions(-) diff --git a/usr.sbin/rpki-client/aspa.c b/usr.sbin/rpki-client/aspa.c index 79a98caea0a..9f56abd26ec 100644 --- a/usr.sbin/rpki-client/aspa.c +++ b/usr.sbin/rpki-client/aspa.c @@ -1,4 +1,4 @@ -/* $OpenBSD: aspa.c,v 1.16 2023/03/12 11:54:56 job Exp $ */ +/* $OpenBSD: aspa.c,v 1.17 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2022 Job Snijders * Copyright (c) 2022 Theo Buehler @@ -283,6 +283,7 @@ aspa_buffer(struct ibuf *b, const struct aspa *p) { io_simple_buffer(b, &p->valid, sizeof(p->valid)); io_simple_buffer(b, &p->custasid, sizeof(p->custasid)); + io_simple_buffer(b, &p->talid, sizeof(p->talid)); io_simple_buffer(b, &p->expires, sizeof(p->expires)); io_simple_buffer(b, &p->providersz, sizeof(size_t)); @@ -309,6 +310,7 @@ aspa_read(struct ibuf *b) io_read_buf(b, &p->valid, sizeof(p->valid)); io_read_buf(b, &p->custasid, sizeof(p->custasid)); + io_read_buf(b, &p->talid, sizeof(p->talid)); io_read_buf(b, &p->expires, sizeof(p->expires)); io_read_buf(b, &p->providersz, sizeof(size_t)); @@ -350,20 +352,32 @@ aspa_insert_vaps(struct vap_tree *tree, struct aspa *aspa, struct repo *rp) struct vap *v, *found; size_t i, j; - repo_stat_inc(rp, RTYPE_ASPA, STYPE_TOTAL); - if ((v = calloc(1, sizeof(*v))) == NULL) err(1, NULL); v->custasid = aspa->custasid; + v->talid = aspa->talid; + if (rp != NULL) + v->repoid = repo_id(rp); + else + v->repoid = 0; v->expires = aspa->expires; if ((found = RB_INSERT(vap_tree, tree, v)) != NULL) { - if (found->expires > v->expires) + if (found->expires > v->expires) { + /* decrement found */ + repo_stat_inc(repo_byid(found->repoid), found->talid, + RTYPE_ASPA, STYPE_DEC_UNIQUE); found->expires = v->expires; + found->talid = v->talid; + found->repoid = v->repoid; + repo_stat_inc(rp, v->talid, RTYPE_ASPA, STYPE_UNIQUE); + } free(v); v = found; } else - repo_stat_inc(rp, RTYPE_ASPA, STYPE_UNIQUE); + repo_stat_inc(rp, v->talid, RTYPE_ASPA, STYPE_UNIQUE); + + repo_stat_inc(rp, aspa->talid, RTYPE_ASPA, STYPE_TOTAL); v->providers = reallocarray(v->providers, v->providersz + aspa->providersz, sizeof(*v->providers)); @@ -379,14 +393,14 @@ aspa_insert_vaps(struct vap_tree *tree, struct aspa *aspa, struct repo *rp) if (j == v->providersz || aspa->providers[i].as < v->providers[j].as) { /* merge provider from aspa into v */ - repo_stat_inc(rp, RTYPE_ASPA, + repo_stat_inc(rp, v->talid, RTYPE_ASPA, STYPE_BOTH + aspa->providers[i].afi); insert_vap(v, j, &aspa->providers[i]); i++; } else if (aspa->providers[i].as == v->providers[j].as) { /* duplicate provider, merge afi */ if (v->providers[j].afi != aspa->providers[i].afi) { - repo_stat_inc(rp, RTYPE_ASPA, + repo_stat_inc(rp, v->talid, RTYPE_ASPA, STYPE_BOTH + aspa->providers[i].afi); v->providers[j].afi = 0; } diff --git a/usr.sbin/rpki-client/extern.h b/usr.sbin/rpki-client/extern.h index ea83b04dbde..d2cab063636 100644 --- a/usr.sbin/rpki-client/extern.h +++ b/usr.sbin/rpki-client/extern.h @@ -1,4 +1,4 @@ -/* $OpenBSD: extern.h,v 1.177 2023/04/13 17:04:02 job Exp $ */ +/* $OpenBSD: extern.h,v 1.178 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2019 Kristaps Dzonsons * @@ -217,6 +217,7 @@ struct mft { time_t expires; /* when the signature path expires */ size_t filesz; /* number of filenames */ unsigned int repoid; + int talid; int stale; /* if a stale manifest */ }; @@ -383,6 +384,8 @@ struct vap { struct aspa_provider *providers; size_t providersz; time_t expires; + int talid; + unsigned int repoid; }; /* @@ -397,12 +400,12 @@ RB_PROTOTYPE(vap_tree, vap, entry, vapcmp); struct vrp { RB_ENTRY(vrp) entry; struct ip_addr addr; - int talid; /* covered by which TAL */ - unsigned int repoid; uint32_t asid; enum afi afi; unsigned char maxlength; time_t expires; /* transitive expiry moment */ + int talid; /* covered by which TAL */ + unsigned int repoid; }; /* * Tree of VRP sorted by afi, addr, maxlength and asid @@ -539,7 +542,7 @@ RB_HEAD(filepath_tree, filepath); /* * Statistics collected during run-time. */ -struct repostats { +struct repotalstats { uint32_t certs; /* certificates */ uint32_t certs_fail; /* invalid certificate */ uint32_t mfts; /* total number of manifests */ @@ -562,6 +565,13 @@ struct repostats { uint32_t vaps_pas6; /* total number of IPv6 only providers */ uint32_t vrps; /* total number of Validated ROA Payloads */ uint32_t vrps_uniqs; /* number of unique vrps */ +}; + +struct repostats { + uint32_t del_files; /* number of files removed in cleanup */ + uint32_t extra_files; /* number of superfluous files */ + uint32_t del_extra_files;/* number of removed extra files */ + uint32_t del_dirs; /* number of dirs removed in cleanup */ struct timespec sync_time; /* time to sync repo */ }; @@ -574,11 +584,9 @@ struct stats { uint32_t http_fails; /* failed http repositories */ uint32_t rrdp_repos; /* synced rrdp repositories */ uint32_t rrdp_fails; /* failed rrdp repositories */ - uint32_t del_files; /* number of files removed in cleanup */ - uint32_t extra_files; /* number of superfluous files */ - uint32_t del_dirs; /* number of dirs removed in cleanup */ uint32_t skiplistentries; /* number of skiplist entries */ + struct repotalstats repo_tal_stats; struct repostats repo_stats; struct timespec elapsed_time; struct timespec user_time; @@ -595,7 +603,7 @@ extern int excludeaspa; extern const char *tals[]; extern const char *taldescs[]; extern unsigned int talrepocnt[]; -extern struct repostats talstats[]; +extern struct repotalstats talstats[]; extern int talsz; /* Routines for RPKI entities. */ @@ -765,7 +773,9 @@ struct repo *repo_byid(unsigned int); int repo_queued(struct repo *, struct entity *); void repo_cleanup(struct filepath_tree *, int); int repo_check_timeout(int); -void repo_stat_inc(struct repo *, enum rtype, enum stype); +void repo_stat_inc(struct repo *, int, enum rtype, enum stype); +void repo_tal_stats_collect(void (*)(const struct repo *, + const struct repotalstats *, void *), int, void *); void repo_stats_collect(void (*)(const struct repo *, const struct repostats *, void *), void *); void repo_free(void); diff --git a/usr.sbin/rpki-client/filemode.c b/usr.sbin/rpki-client/filemode.c index 66098099429..b2e729c337d 100644 --- a/usr.sbin/rpki-client/filemode.c +++ b/usr.sbin/rpki-client/filemode.c @@ -1,4 +1,4 @@ -/* $OpenBSD: filemode.c,v 1.29 2023/03/15 11:09:34 job Exp $ */ +/* $OpenBSD: filemode.c,v 1.30 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2019 Claudio Jeker * Copyright (c) 2019 Kristaps Dzonsons @@ -609,6 +609,7 @@ parse_file(struct entityq *q, struct msgbuf *msgq) b = io_new_buffer(); io_simple_buffer(b, &entp->type, sizeof(entp->type)); io_simple_buffer(b, &entp->repoid, sizeof(entp->repoid)); + io_simple_buffer(b, &entp->talid, sizeof(entp->talid)); io_str_buffer(b, entp->file); io_close_buffer(msgq, b); entity_free(entp); diff --git a/usr.sbin/rpki-client/main.c b/usr.sbin/rpki-client/main.c index bfa71ff65c0..516fcd7f514 100644 --- a/usr.sbin/rpki-client/main.c +++ b/usr.sbin/rpki-client/main.c @@ -1,4 +1,4 @@ -/* $OpenBSD: main.c,v 1.233 2023/04/13 17:04:02 job Exp $ */ +/* $OpenBSD: main.c,v 1.234 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2021 Claudio Jeker * Copyright (c) 2019 Kristaps Dzonsons @@ -51,7 +51,7 @@ const char *tals[TALSZ_MAX]; const char *taldescs[TALSZ_MAX]; unsigned int talrepocnt[TALSZ_MAX]; -struct repostats talstats[TALSZ_MAX]; +struct repotalstats talstats[TALSZ_MAX]; int talsz; size_t entity_queue; @@ -400,7 +400,7 @@ queue_add_from_mft(const struct mft *mft) if ((mftaki = strdup(mft->aki)) == NULL) err(1, NULL); entityq_add(npath, nfile, f->type, f->location, rp, NULL, 0, - -1, mftaki); + mft->talid, mftaki); } } @@ -527,8 +527,8 @@ queue_add_from_cert(const struct cert *cert) err(1, NULL); } - entityq_add(npath, nfile, RTYPE_MFT, DIR_UNKNOWN, repo, NULL, 0, -1, - NULL); + entityq_add(npath, nfile, RTYPE_MFT, DIR_UNKNOWN, repo, NULL, 0, + cert->talid, NULL); } /* @@ -550,6 +550,7 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, struct repo *rp; char *file; unsigned int id; + int talid; int c; /* @@ -560,6 +561,7 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, */ io_read_buf(b, &type, sizeof(type)); io_read_buf(b, &id, sizeof(id)); + io_read_buf(b, &talid, sizeof(talid)); io_read_str(b, &file); /* in filemode messages can be ignored, only the accounting matters */ @@ -572,7 +574,7 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, } rp = repo_byid(id); - repo_stat_inc(rp, type, STYPE_OK); + repo_stat_inc(rp, talid, type, STYPE_OK); switch (type) { case RTYPE_TAL: st->tals++; @@ -583,7 +585,7 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, case RTYPE_CER: io_read_buf(b, &c, sizeof(c)); if (c == 0) { - repo_stat_inc(rp, type, STYPE_FAIL); + repo_stat_inc(rp, talid, type, STYPE_FAIL); break; } cert = cert_read(b); @@ -593,7 +595,7 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, break; case CERT_PURPOSE_BGPSEC_ROUTER: cert_insert_brks(brktree, cert); - repo_stat_inc(rp, type, STYPE_BGPSEC); + repo_stat_inc(rp, talid, type, STYPE_BGPSEC); break; default: errx(1, "unexpected cert purpose received"); @@ -604,14 +606,14 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, case RTYPE_MFT: io_read_buf(b, &c, sizeof(c)); if (c == 0) { - repo_stat_inc(rp, type, STYPE_FAIL); + repo_stat_inc(rp, talid, type, STYPE_FAIL); break; } mft = mft_read(b); if (!mft->stale) queue_add_from_mft(mft); else - repo_stat_inc(rp, type, STYPE_STALE); + repo_stat_inc(rp, talid, type, STYPE_STALE); mft_free(mft); break; case RTYPE_CRL: @@ -621,14 +623,14 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, case RTYPE_ROA: io_read_buf(b, &c, sizeof(c)); if (c == 0) { - repo_stat_inc(rp, type, STYPE_FAIL); + repo_stat_inc(rp, talid, type, STYPE_FAIL); break; } roa = roa_read(b); if (roa->valid) roa_insert_vrps(tree, roa, rp); else - repo_stat_inc(rp, type, STYPE_INVALID); + repo_stat_inc(rp, talid, type, STYPE_INVALID); roa_free(roa); break; case RTYPE_GBR: @@ -636,14 +638,14 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, case RTYPE_ASPA: io_read_buf(b, &c, sizeof(c)); if (c == 0) { - repo_stat_inc(rp, type, STYPE_FAIL); + repo_stat_inc(rp, talid, type, STYPE_FAIL); break; } aspa = aspa_read(b); if (aspa->valid) aspa_insert_vaps(vaptree, aspa, rp); else - repo_stat_inc(rp, type, STYPE_INVALID); + repo_stat_inc(rp, talid, type, STYPE_INVALID); aspa_free(aspa); break; case RTYPE_TAK: @@ -716,12 +718,9 @@ rrdp_process(struct ibuf *b) } static void -sum_stats(const struct repo *rp, const struct repostats *in, void *arg) +sum_stats(const struct repo *rp, const struct repotalstats *in, void *arg) { - struct repostats *out = arg; - - if (rp != NULL) - sum_stats(NULL, in, &talstats[repo_talid(rp)]); + struct repotalstats *out = arg; out->mfts += in->mfts; out->mfts_fail += in->mfts_fail; @@ -745,7 +744,16 @@ sum_stats(const struct repo *rp, const struct repostats *in, void *arg) out->vaps_pas += in->vaps_pas; out->vaps_pas4 += in->vaps_pas4; out->vaps_pas6 += in->vaps_pas6; +} +static void +sum_repostats(const struct repo *rp, const struct repostats *in, void *arg) +{ + struct repostats *out = arg; + + out->del_files += in->del_files; + out->extra_files += in->extra_files; + out->del_dirs += in->del_dirs; timespecadd(&in->sync_time, &out->sync_time, &out->sync_time); } @@ -1396,7 +1404,11 @@ main(int argc, char *argv[]) if (fchdir(outdirfd) == -1) err(1, "fchdir output dir"); - repo_stats_collect(sum_stats, &stats.repo_stats); + for (i = 0; i < talsz; i++) { + repo_tal_stats_collect(sum_stats, i, &talstats[i]); + repo_tal_stats_collect(sum_stats, i, &stats.repo_tal_stats); + } + repo_stats_collect(sum_repostats, &stats.repo_stats); if (outputfiles(&vrps, &brks, &vaps, &stats)) rc = 1; @@ -1408,29 +1420,32 @@ main(int argc, char *argv[]) (long long)stats.system_time.tv_sec); printf("Skiplist entries: %u\n", stats.skiplistentries); printf("Route Origin Authorizations: %u (%u failed parse, %u " - "invalid)\n", stats.repo_stats.roas, stats.repo_stats.roas_fail, - stats.repo_stats.roas_invalid); + "invalid)\n", stats.repo_tal_stats.roas, + stats.repo_tal_stats.roas_fail, + stats.repo_tal_stats.roas_invalid); printf("AS Provider Attestations: %u (%u failed parse, %u " - "invalid)\n", stats.repo_stats.aspas, stats.repo_stats.aspas_fail, - stats.repo_stats.aspas_invalid); - printf("BGPsec Router Certificates: %u\n", stats.repo_stats.brks); + "invalid)\n", stats.repo_tal_stats.aspas, + stats.repo_tal_stats.aspas_fail, + stats.repo_tal_stats.aspas_invalid); + printf("BGPsec Router Certificates: %u\n", stats.repo_tal_stats.brks); printf("Certificates: %u (%u invalid)\n", - stats.repo_stats.certs, stats.repo_stats.certs_fail); + stats.repo_tal_stats.certs, stats.repo_tal_stats.certs_fail); printf("Trust Anchor Locators: %u (%u invalid)\n", stats.tals, talsz - stats.tals); printf("Manifests: %u (%u failed parse, %u stale)\n", - stats.repo_stats.mfts, stats.repo_stats.mfts_fail, - stats.repo_stats.mfts_stale); - printf("Certificate revocation lists: %u\n", stats.repo_stats.crls); - printf("Ghostbuster records: %u\n", stats.repo_stats.gbrs); - printf("Trust Anchor Keys: %u\n", stats.repo_stats.taks); + stats.repo_tal_stats.mfts, stats.repo_tal_stats.mfts_fail, + stats.repo_tal_stats.mfts_stale); + printf("Certificate revocation lists: %u\n", stats.repo_tal_stats.crls); + printf("Ghostbuster records: %u\n", stats.repo_tal_stats.gbrs); + printf("Trust Anchor Keys: %u\n", stats.repo_tal_stats.taks); printf("Repositories: %u\n", stats.repos); printf("Cleanup: removed %u files, %u directories, %u superfluous\n", - stats.del_files, stats.del_dirs, stats.extra_files); - printf("VRP Entries: %u (%u unique)\n", stats.repo_stats.vrps, - stats.repo_stats.vrps_uniqs); - printf("VAP Entries: %u (%u unique)\n", stats.repo_stats.vaps, - stats.repo_stats.vaps_uniqs); + stats.repo_stats.del_files, stats.repo_stats.del_dirs, + stats.repo_stats.extra_files); + printf("VRP Entries: %u (%u unique)\n", stats.repo_tal_stats.vrps, + stats.repo_tal_stats.vrps_uniqs); + printf("VAP Entries: %u (%u unique)\n", stats.repo_tal_stats.vaps, + stats.repo_tal_stats.vaps_uniqs); /* Memory cleanup. */ repo_free(); diff --git a/usr.sbin/rpki-client/mft.c b/usr.sbin/rpki-client/mft.c index 5173ec0ba1e..f702caf3b7a 100644 --- a/usr.sbin/rpki-client/mft.c +++ b/usr.sbin/rpki-client/mft.c @@ -1,4 +1,4 @@ -/* $OpenBSD: mft.c,v 1.90 2023/04/24 17:11:33 claudio Exp $ */ +/* $OpenBSD: mft.c,v 1.91 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2022 Theo Buehler * Copyright (c) 2019 Kristaps Dzonsons @@ -470,6 +470,7 @@ mft_buffer(struct ibuf *b, const struct mft *p) io_simple_buffer(b, &p->stale, sizeof(p->stale)); io_simple_buffer(b, &p->repoid, sizeof(p->repoid)); + io_simple_buffer(b, &p->talid, sizeof(p->talid)); io_str_buffer(b, p->path); io_str_buffer(b, p->aia); @@ -502,6 +503,7 @@ mft_read(struct ibuf *b) io_read_buf(b, &p->stale, sizeof(p->stale)); io_read_buf(b, &p->repoid, sizeof(p->repoid)); + io_read_buf(b, &p->talid, sizeof(p->talid)); io_read_str(b, &p->path); io_read_str(b, &p->aia); diff --git a/usr.sbin/rpki-client/output-json.c b/usr.sbin/rpki-client/output-json.c index cf752e7ed02..28375f096fd 100644 --- a/usr.sbin/rpki-client/output-json.c +++ b/usr.sbin/rpki-client/output-json.c @@ -1,4 +1,4 @@ -/* $OpenBSD: output-json.c,v 1.32 2023/04/20 15:05:44 job Exp $ */ +/* $OpenBSD: output-json.c,v 1.33 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2019 Claudio Jeker * @@ -58,16 +58,16 @@ outputheader_json(FILE *out, struct stats *st) "\t\t\"talfiles\": [\n", hn, tbuf, (long long)st->elapsed_time.tv_sec, (long long)st->user_time.tv_sec, (long long)st->system_time.tv_sec, - st->repo_stats.roas, - st->repo_stats.roas_fail, - st->repo_stats.roas_invalid, - st->repo_stats.aspas, - st->repo_stats.aspas_fail, - st->repo_stats.aspas_invalid, - st->repo_stats.brks, - st->repo_stats.certs, - st->repo_stats.certs_fail, - st->repo_stats.taks, + st->repo_tal_stats.roas, + st->repo_tal_stats.roas_fail, + st->repo_tal_stats.roas_invalid, + st->repo_tal_stats.aspas, + st->repo_tal_stats.aspas_fail, + st->repo_tal_stats.aspas_invalid, + st->repo_tal_stats.brks, + st->repo_tal_stats.certs, + st->repo_tal_stats.certs_fail, + st->repo_tal_stats.taks, st->tals, talsz - st->tals) < 0) return -1; @@ -94,19 +94,19 @@ outputheader_json(FILE *out, struct stats *st) "\t\t\"cachedir_superfluous_files\": %u,\n" "\t\t\"cachedir_del_dirs\": %u\n" "\t},\n\n", - st->repo_stats.mfts, - st->repo_stats.mfts_fail, - st->repo_stats.mfts_stale, - st->repo_stats.crls, - st->repo_stats.gbrs, + st->repo_tal_stats.mfts, + st->repo_tal_stats.mfts_fail, + st->repo_tal_stats.mfts_stale, + st->repo_tal_stats.crls, + st->repo_tal_stats.gbrs, st->repos, - st->repo_stats.vrps, - st->repo_stats.vrps_uniqs, - st->repo_stats.vaps, - st->repo_stats.vaps_uniqs, - st->del_files, - st->extra_files, - st->del_dirs) < 0) + st->repo_tal_stats.vrps, + st->repo_tal_stats.vrps_uniqs, + st->repo_tal_stats.vaps, + st->repo_tal_stats.vaps_uniqs, + st->repo_stats.del_files, + st->repo_stats.extra_files, + st->repo_stats.del_dirs) < 0) return -1; return 0; } diff --git a/usr.sbin/rpki-client/output-ometric.c b/usr.sbin/rpki-client/output-ometric.c index 412309d53a4..9ddd8960dd4 100644 --- a/usr.sbin/rpki-client/output-ometric.c +++ b/usr.sbin/rpki-client/output-ometric.c @@ -1,4 +1,4 @@ -/* $OpenBSD: output-ometric.c,v 1.2 2023/03/30 15:29:15 claudio Exp $ */ +/* $OpenBSD: output-ometric.c,v 1.3 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2022 Claudio Jeker * @@ -35,7 +35,7 @@ static const char * const repo_states[2] = { "failed", "synced" }; static const char * const repo_protos[3] = { "rrdp", "rsync", "https" }; static void -set_common_stats(const struct repostats *in, struct ometric *metric, +set_common_stats(const struct repotalstats *in, struct ometric *metric, struct olabels *ol) { ometric_set_int_with_labels(metric, in->certs, @@ -106,19 +106,44 @@ ta_stats(int id) } static void -repo_stats(const struct repo *rp, const struct repostats *in, void *arg) +repo_tal_stats(const struct repo *rp, const struct repotalstats *in, void *arg) { struct olabels *ol; const char *keys[4] = { "name", "carepo", "notify", NULL }; const char *values[4]; + int talid = *(int *)arg; - values[0] = taldescs[repo_talid(rp)]; + values[0] = taldescs[talid]; repo_fetch_uris(rp, &values[1], &values[2]); values[3] = NULL; ol = olabels_new(keys, values); set_common_stats(in, rpki_repo_obj, ol); + olabels_free(ol); +} + +static void +repo_stats(const struct repo *rp, const struct repostats *in, void *arg) +{ + struct olabels *ol; + const char *keys[3] = { "carepo", "notify", NULL }; + const char *values[3]; + + repo_fetch_uris(rp, &values[0], &values[1]); + values[2] = NULL; + + ol = olabels_new(keys, values); ometric_set_timespec(rpki_repo_duration, &in->sync_time, ol); + + ometric_set_int_with_labels(rpki_repo_obj, in->del_files, + OKV("type", "state"), OKV("files", "deleted"), ol); + ometric_set_int_with_labels(rpki_repo_obj, in->extra_files, + OKV("type", "state"), OKV("files", "extra"), ol); + ometric_set_int_with_labels(rpki_repo_obj, in->del_extra_files, + OKV("type", "state"), OKV("files", "deleted_extra"), ol); + ometric_set_int_with_labels(rpki_repo_obj, in->del_dirs, + OKV("type", "state"), OKV("dirs", "deleted"), ol); + ometric_set_state(rpki_repo_state, repo_states[repo_synced(rp)], ol); if (repo_synced(rp)) ometric_set_state(rpki_repo_proto, repo_proto(rp), ol); @@ -184,10 +209,12 @@ output_ometric(FILE *out, struct vrp_tree *vrps, struct brk_tree *brks, ometric_set_info(rpki_info, NULL, NULL, ol); olabels_free(ol); - repo_stats_collect(repo_stats, NULL); - for (i = 0; i < talsz; i++) + for (i = 0; i < talsz; i++) { + repo_tal_stats_collect(repo_tal_stats, i, &i); ta_stats(i); - set_common_stats(&st->repo_stats, rpki_obj, NULL); + } + repo_stats_collect(repo_stats, NULL); + set_common_stats(&st->repo_tal_stats, rpki_obj, NULL); ometric_set_int(rpki_repo, st->repos, NULL); ometric_set_int_with_labels(rpki_repo, st->rsync_repos, diff --git a/usr.sbin/rpki-client/output.c b/usr.sbin/rpki-client/output.c index 878b6b68cd7..659476d350a 100644 --- a/usr.sbin/rpki-client/output.c +++ b/usr.sbin/rpki-client/output.c @@ -1,4 +1,4 @@ -/* $OpenBSD: output.c,v 1.30 2023/04/19 12:58:16 jsg Exp $ */ +/* $OpenBSD: output.c,v 1.31 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2019 Theo de Raadt * @@ -218,9 +218,9 @@ outputheader(FILE *out, struct stats *st) "# Certificates: %u (%u invalid)\n", hn, tbuf, (long long)st->elapsed_time.tv_sec, (long long)st->user_time.tv_sec, (long long)st->system_time.tv_sec, - st->repo_stats.roas, st->repo_stats.roas_fail, - st->repo_stats.roas_invalid, st->repo_stats.brks, - st->repo_stats.certs, st->repo_stats.certs_fail) < 0) + st->repo_tal_stats.roas, st->repo_tal_stats.roas_fail, + st->repo_tal_stats.roas_invalid, st->repo_tal_stats.brks, + st->repo_tal_stats.certs, st->repo_tal_stats.certs_fail) < 0) return -1; if (fprintf(out, @@ -238,12 +238,12 @@ outputheader(FILE *out, struct stats *st) "# Ghostbuster records: %u\n" "# Repositories: %u\n" "# VRP Entries: %u (%u unique)\n", - st->repo_stats.mfts, st->repo_stats.mfts_fail, - st->repo_stats.mfts_stale, - st->repo_stats.crls, - st->repo_stats.gbrs, + st->repo_tal_stats.mfts, st->repo_tal_stats.mfts_fail, + st->repo_tal_stats.mfts_stale, + st->repo_tal_stats.crls, + st->repo_tal_stats.gbrs, st->repos, - st->repo_stats.vrps, st->repo_stats.vrps_uniqs) < 0) + st->repo_tal_stats.vrps, st->repo_tal_stats.vrps_uniqs) < 0) return -1; return 0; } diff --git a/usr.sbin/rpki-client/parser.c b/usr.sbin/rpki-client/parser.c index 58e95235b53..23eaf602e68 100644 --- a/usr.sbin/rpki-client/parser.c +++ b/usr.sbin/rpki-client/parser.c @@ -1,4 +1,4 @@ -/* $OpenBSD: parser.c,v 1.90 2023/04/13 17:04:02 job Exp $ */ +/* $OpenBSD: parser.c,v 1.91 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2019 Claudio Jeker * Copyright (c) 2019 Kristaps Dzonsons @@ -298,6 +298,8 @@ proc_parser_mft_pre(struct entity *entp, enum location loc, char **file, X509_free(x509); mft->repoid = entp->repoid; + mft->talid = a->cert->talid; + return mft; } @@ -635,6 +637,7 @@ parse_entity(struct entityq *q, struct msgbuf *msgq) b = io_new_buffer(); io_simple_buffer(b, &entp->type, sizeof(entp->type)); io_simple_buffer(b, &entp->repoid, sizeof(entp->repoid)); + io_simple_buffer(b, &entp->talid, sizeof(entp->talid)); file = NULL; f = NULL; @@ -689,6 +692,8 @@ parse_entity(struct entityq *q, struct msgbuf *msgq) io_simple_buffer(b2, &type, sizeof(type)); io_simple_buffer(b2, &entp->repoid, sizeof(entp->repoid)); + io_simple_buffer(b2, &entp->talid, + sizeof(entp->talid)); io_str_buffer(b2, crlfile); free(crlfile); diff --git a/usr.sbin/rpki-client/repo.c b/usr.sbin/rpki-client/repo.c index 08ab2a0d206..81b06b8d4dd 100644 --- a/usr.sbin/rpki-client/repo.c +++ b/usr.sbin/rpki-client/repo.c @@ -1,4 +1,4 @@ -/* $OpenBSD: repo.c,v 1.43 2023/03/30 15:29:15 claudio Exp $ */ +/* $OpenBSD: repo.c,v 1.44 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2021 Claudio Jeker * Copyright (c) 2019 Kristaps Dzonsons @@ -97,10 +97,12 @@ struct repo { const struct rsyncrepo *rsync; const struct tarepo *ta; struct entityq queue; /* files waiting for repo */ - struct repostats stats; + struct repotalstats stats[TALSZ_MAX]; + struct repostats repostats; struct timespec start_time; time_t alarm; /* sync timeout */ int talid; + int stats_used[TALSZ_MAX]; unsigned int id; /* identifier */ }; static SLIST_HEAD(, repo) repos = SLIST_HEAD_INITIALIZER(repos); @@ -302,7 +304,8 @@ repo_done(const void *vp, int ok) entityq_flush(&rp->queue, rp); clock_gettime(CLOCK_MONOTONIC, &flush_time); - timespecsub(&flush_time, &rp->start_time, &rp->stats.sync_time); + timespecsub(&flush_time, &rp->start_time, + &rp->repostats.sync_time); } } @@ -555,16 +558,18 @@ rrdp_free(void) * Check if a directory is an active rrdp repository. * Returns 1 if found else 0. */ -static int -rrdp_is_active(const char *dir) +static struct repo * +repo_rrdp_bypath(const char *dir) { - struct rrdprepo *rr; - - SLIST_FOREACH(rr, &rrdprepos, entry) - if (strcmp(dir, rr->basedir) == 0) - return rr->state != REPO_FAILED; + struct repo *rp; - return 0; + SLIST_FOREACH(rp, &repos, entry) { + if (rp->rrdp == NULL) + continue; + if (strcmp(dir, rp->rrdp->basedir) == 0) + return rp; + } + return NULL; } /* @@ -1322,48 +1327,49 @@ repo_check_timeout(int timeout) * Update stats object of repository depending on rtype and subtype. */ void -repo_stat_inc(struct repo *rp, enum rtype type, enum stype subtype) +repo_stat_inc(struct repo *rp, int talid, enum rtype type, enum stype subtype) { if (rp == NULL) return; + rp->stats_used[talid] = 1; switch (type) { case RTYPE_CER: if (subtype == STYPE_OK) - rp->stats.certs++; + rp->stats[talid].certs++; if (subtype == STYPE_FAIL) - rp->stats.certs_fail++; + rp->stats[talid].certs_fail++; if (subtype == STYPE_BGPSEC) { - rp->stats.certs--; - rp->stats.brks++; + rp->stats[talid].certs--; + rp->stats[talid].brks++; } break; case RTYPE_MFT: if (subtype == STYPE_OK) - rp->stats.mfts++; + rp->stats[talid].mfts++; if (subtype == STYPE_FAIL) - rp->stats.mfts_fail++; + rp->stats[talid].mfts_fail++; if (subtype == STYPE_STALE) - rp->stats.mfts_stale++; + rp->stats[talid].mfts_stale++; break; case RTYPE_ROA: switch (subtype) { case STYPE_OK: - rp->stats.roas++; + rp->stats[talid].roas++; break; case STYPE_FAIL: - rp->stats.roas_fail++; + rp->stats[talid].roas_fail++; break; case STYPE_INVALID: - rp->stats.roas_invalid++; + rp->stats[talid].roas_invalid++; break; case STYPE_TOTAL: - rp->stats.vrps++; + rp->stats[talid].vrps++; break; case STYPE_UNIQUE: - rp->stats.vrps_uniqs++; + rp->stats[talid].vrps_uniqs++; break; case STYPE_DEC_UNIQUE: - rp->stats.vrps_uniqs--; + rp->stats[talid].vrps_uniqs--; break; default: break; @@ -1372,41 +1378,44 @@ repo_stat_inc(struct repo *rp, enum rtype type, enum stype subtype) case RTYPE_ASPA: switch (subtype) { case STYPE_OK: - rp->stats.aspas++; + rp->stats[talid].aspas++; break; case STYPE_FAIL: - rp->stats.aspas_fail++; + rp->stats[talid].aspas_fail++; break; case STYPE_INVALID: - rp->stats.aspas_invalid++; + rp->stats[talid].aspas_invalid++; break; case STYPE_TOTAL: - rp->stats.vaps++; + rp->stats[talid].vaps++; break; case STYPE_UNIQUE: - rp->stats.vaps_uniqs++; + rp->stats[talid].vaps_uniqs++; + break; + case STYPE_DEC_UNIQUE: + rp->stats[talid].vaps_uniqs--; break; case STYPE_BOTH: - rp->stats.vaps_pas++; + rp->stats[talid].vaps_pas++; break; case STYPE_ONLY_IPV4: - rp->stats.vaps_pas4++; + rp->stats[talid].vaps_pas4++; break; case STYPE_ONLY_IPV6: - rp->stats.vaps_pas6++; + rp->stats[talid].vaps_pas6++; break; default: break; } break; case RTYPE_CRL: - rp->stats.crls++; + rp->stats[talid].crls++; break; case RTYPE_GBR: - rp->stats.gbrs++; + rp->stats[talid].gbrs++; break; case RTYPE_TAK: - rp->stats.taks++; + rp->stats[talid].taks++; break; default: break; @@ -1414,16 +1423,27 @@ repo_stat_inc(struct repo *rp, enum rtype type, enum stype subtype) } void -repo_stats_collect(void (*cb)(const struct repo *, const struct repostats *, - void *), void *arg) +repo_tal_stats_collect(void (*cb)(const struct repo *, + const struct repotalstats *, void *), int talid, void *arg) { struct repo *rp; SLIST_FOREACH(rp, &repos, entry) { - cb(rp, &rp->stats, arg); + if (rp->stats_used[talid]) + cb(rp, &rp->stats[talid], arg); } } +void +repo_stats_collect(void (*cb)(const struct repo *, const struct repostats *, + void *), void *arg) +{ + struct repo *rp; + + SLIST_FOREACH(rp, &repos, entry) + cb(rp, &rp->repostats, arg); +} + /* * Delayed delete of files from RRDP. Since RRDP has no security built-in * this code needs to check if this RRDP repository is actually allowed to @@ -1432,11 +1452,15 @@ repo_stats_collect(void (*cb)(const struct repo *, const struct repostats *, static void repo_cleanup_rrdp(struct filepath_tree *tree) { + struct repo *rp; struct rrdprepo *rr; struct filepath *fp, *nfp; char *fn; - SLIST_FOREACH(rr, &rrdprepos, entry) { + SLIST_FOREACH(rp, &repos, entry) { + if (rp->rrdp == NULL) + continue; + rr = (struct rrdprepo *)rp->rrdp; RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) { if (!rrdp_uri_valid(rr, fp->file)) { warnx("%s: external URI %s", rr->notifyuri, @@ -1453,7 +1477,7 @@ repo_cleanup_rrdp(struct filepath_tree *tree) } else { if (verbose > 1) logx("deleted %s", fn); - stats.del_files++; + rp->repostats.del_files++; } free(fn); @@ -1466,7 +1490,7 @@ repo_cleanup_rrdp(struct filepath_tree *tree) } else { if (verbose > 1) logx("deleted %s", fn); - stats.del_files++; + rp->repostats.del_files++; } } else warnx("%s: referenced file supposed to be " @@ -1524,18 +1548,16 @@ repo_move_valid(struct filepath_tree *tree) } } -#define BASE_DIR (void *)0x01 -#define RSYNC_DIR (void *)0x02 -#define RRDP_DIR (void *)0x03 +struct fts_state { + enum { BASE_DIR, RSYNC_DIR, RRDP_DIR } type; + struct repo *rp; +} fts_state; static const struct rrdprepo * repo_is_rrdp(struct repo *rp) { /* check for special pointers first these are not a repository */ - if (rp == NULL || rp == BASE_DIR || rp == RSYNC_DIR || rp == RRDP_DIR) - return NULL; - - if (rp->rrdp) + if (rp != NULL && rp->rrdp != NULL) return rp->rrdp->state == REPO_DONE ? rp->rrdp : NULL; return NULL; } @@ -1548,152 +1570,166 @@ skip_dotslash(char *in) return in; } -void -repo_cleanup(struct filepath_tree *tree, int cachefd) +static void +repo_cleanup_entry(FTSENT *e, struct filepath_tree *tree, int cachefd) { - char *argv[2] = { ".", NULL }; - FTS *fts; - FTSENT *e; const struct rrdprepo *rr; + char *path; - /* first move temp files which have been used to valid dir */ - repo_move_valid(tree); - /* then delete files requested by rrdp */ - repo_cleanup_rrdp(tree); - - if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) - err(1, "fts_open"); - errno = 0; - while ((e = fts_read(fts)) != NULL) { - char *path = skip_dotslash(e->fts_path); - switch (e->fts_info) { - case FTS_NSOK: - if (filepath_exists(tree, path)) { - e->fts_parent->fts_number++; + path = skip_dotslash(e->fts_path); + switch (e->fts_info) { + case FTS_NSOK: + if (filepath_exists(tree, path)) { + e->fts_parent->fts_number++; + break; + } + if (fts_state.type == RRDP_DIR && fts_state.rp != NULL) { + e->fts_parent->fts_number++; + /* handle rrdp .state files explicitly */ + if (e->fts_level == 3 && + strcmp(e->fts_name, ".state") == 0) break; + /* can't delete these extra files */ + fts_state.rp->repostats.extra_files++; + if (verbose > 1) + logx("superfluous %s", path); + break; + } + rr = repo_is_rrdp(fts_state.rp); + if (rr != NULL) { + struct stat st; + char *fn; + + if (asprintf(&fn, "%s/%s", rr->basedir, path) == -1) + err(1, NULL); + + /* + * If the file exists in the rrdp dir + * that file is newer and needs to be kept + * so unlink this file instead of moving + * it over the file in the rrdp dir. + */ + if (fstatat(cachefd, fn, &st, 0) == 0 && + S_ISREG(st.st_mode)) { + free(fn); + goto unlink; } - if (e->fts_parent->fts_pointer == RRDP_DIR) { - e->fts_parent->fts_number++; - /* handle rrdp .state files explicitly */ - if (e->fts_level == 3 && - strcmp(e->fts_name, ".state") == 0) - break; - /* can't delete these extra files */ - stats.extra_files++; - if (verbose > 1) - logx("superfluous %s", path); - break; + if (repo_mkpath(cachefd, fn) == 0) { + if (renameat(AT_FDCWD, e->fts_accpath, + cachefd, fn) == -1) + warn("rename %s to %s", path, fn); + else if (verbose > 1) + logx("moved %s", path); + fts_state.rp->repostats.extra_files++; } - if (e->fts_parent->fts_pointer == RSYNC_DIR) { + free(fn); + } else { + unlink: + if (unlink(e->fts_accpath) == -1) { + warn("unlink %s", path); + } else if (fts_state.type == RSYNC_DIR) { /* no need to keep rsync files */ if (verbose > 1) logx("superfluous %s", path); + if (fts_state.rp != NULL) + fts_state.rp->repostats.del_extra_files++; + else + stats.repo_stats.del_extra_files++; + } else { + if (verbose > 1) + logx("deleted %s", path); + if (fts_state.rp != NULL) + fts_state.rp->repostats.del_files++; + else + stats.repo_stats.del_files++; } - rr = repo_is_rrdp(e->fts_parent->fts_pointer); - if (rr != NULL) { - struct stat st; - char *fn; - - if (asprintf(&fn, "%s/%s", rr->basedir, - path) == -1) - err(1, NULL); - - /* - * If the file exists in the rrdp dir - * that file is newer and needs to be kept - * so unlink this file instead of moving - * it over the file in the rrdp dir. - */ - if (fstatat(cachefd, fn, &st, 0) == 0 && - S_ISREG(st.st_mode)) { - free(fn); - goto unlink; - } - if (repo_mkpath(cachefd, fn) == 0) { - if (renameat(AT_FDCWD, e->fts_accpath, - cachefd, fn) == -1) - warn("rename %s to %s", path, - fn); - else if (verbose > 1) - logx("moved %s", path); - stats.extra_files++; - } - free(fn); + } + break; + case FTS_D: + if (e->fts_level == FTS_ROOTLEVEL) + fts_state.type = BASE_DIR; + if (e->fts_level == 1) { + if (strcmp(".rsync", e->fts_name) == 0) { + fts_state.type = RSYNC_DIR; + fts_state.rp = NULL; + } else if (strcmp(".rrdp", e->fts_name) == 0) { + fts_state.type = RRDP_DIR; + fts_state.rp = NULL; } else { - unlink: - if (unlink(e->fts_accpath) == -1) { - warn("unlink %s", path); - } else { - if (verbose > 1) - logx("deleted %s", path); - stats.del_files++; - } + fts_state.type = BASE_DIR; + fts_state.rp = repo_bypath(path); } - break; - case FTS_D: - if (e->fts_level == 1) { - if (strcmp(".rsync", e->fts_name) == 0) - e->fts_pointer = RSYNC_DIR; - else if (strcmp(".rrdp", e->fts_name) == 0) - e->fts_pointer = RRDP_DIR; - else - e->fts_pointer = BASE_DIR; - } else - e->fts_pointer = e->fts_parent->fts_pointer; - + } + if (e->fts_level == 2) { + if (fts_state.type == RSYNC_DIR) + fts_state.rp = repo_bypath(path); /* * special handling for rrdp directories, * clear them if they are not used anymore but * only if rrdp is active. */ - if (e->fts_pointer == RRDP_DIR && e->fts_level == 2) { - if (!rrdp_is_active(path)) - e->fts_pointer = NULL; - } - if (e->fts_pointer == BASE_DIR && e->fts_level > 1) { - e->fts_pointer = repo_bypath(path); - if (e->fts_pointer == NULL) - e->fts_pointer = BASE_DIR; - } + if (fts_state.type == RRDP_DIR) + fts_state.rp = repo_rrdp_bypath(path); + } + break; + case FTS_DP: + if (e->fts_level == FTS_ROOTLEVEL) break; - case FTS_DP: - if (e->fts_level == FTS_ROOTLEVEL) + if (e->fts_level == 1) { + /* do not remove .rsync and .rrdp */ + fts_state.rp = NULL; + if (fts_state.type == RRDP_DIR || + fts_state.type == RSYNC_DIR) break; - if (e->fts_level == 1) - /* do not remove .rsync and .rrdp */ - if (e->fts_pointer == RRDP_DIR || - e->fts_pointer == RSYNC_DIR) - break; - - e->fts_parent->fts_number += e->fts_number; - - if (e->fts_number == 0) { - if (rmdir(e->fts_accpath) == -1) - warn("rmdir %s", path); - else - stats.del_dirs++; - } - break; - case FTS_SL: - case FTS_SLNONE: - warnx("symlink %s", path); - if (unlink(e->fts_accpath) == -1) - warn("unlink %s", path); - break; - case FTS_NS: - case FTS_ERR: - if (e->fts_errno == ENOENT && - e->fts_level == FTS_ROOTLEVEL) - continue; - warnx("fts_read %s: %s", path, - strerror(e->fts_errno)); - break; - default: - warnx("fts_read %s: unhandled[%x]", path, - e->fts_info); - break; } + e->fts_parent->fts_number += e->fts_number; + + if (e->fts_number == 0) { + if (rmdir(e->fts_accpath) == -1) + warn("rmdir %s", path); + if (fts_state.rp != NULL) + fts_state.rp->repostats.del_dirs++; + else + stats.repo_stats.del_dirs++; + } + break; + case FTS_SL: + case FTS_SLNONE: + warnx("symlink %s", path); + if (unlink(e->fts_accpath) == -1) + warn("unlink %s", path); + stats.repo_stats.del_extra_files++; + break; + case FTS_NS: + case FTS_ERR: + if (e->fts_errno == ENOENT && e->fts_level == FTS_ROOTLEVEL) + break; + warnx("fts_read %s: %s", path, strerror(e->fts_errno)); + break; + default: + warnx("fts_read %s: unhandled[%x]", path, e->fts_info); + break; + } +} + +void +repo_cleanup(struct filepath_tree *tree, int cachefd) +{ + char *argv[2] = { ".", NULL }; + FTS *fts; + FTSENT *e; + + /* first move temp files which have been used to valid dir */ + repo_move_valid(tree); + /* then delete files requested by rrdp */ + repo_cleanup_rrdp(tree); + + if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) + err(1, "fts_open"); + errno = 0; + while ((e = fts_read(fts)) != NULL) { + repo_cleanup_entry(e, tree, cachefd); errno = 0; } if (errno) diff --git a/usr.sbin/rpki-client/roa.c b/usr.sbin/rpki-client/roa.c index eba2a474458..705208cbd2d 100644 --- a/usr.sbin/rpki-client/roa.c +++ b/usr.sbin/rpki-client/roa.c @@ -1,4 +1,4 @@ -/* $OpenBSD: roa.c,v 1.65 2023/03/12 11:54:56 job Exp $ */ +/* $OpenBSD: roa.c,v 1.66 2023/04/26 16:32:41 claudio Exp $ */ /* * Copyright (c) 2022 Theo Buehler * Copyright (c) 2019 Kristaps Dzonsons @@ -384,19 +384,20 @@ roa_insert_vrps(struct vrp_tree *tree, struct roa *roa, struct repo *rp) /* already exists */ if (found->expires < v->expires) { /* update found with preferred data */ - found->talid = v->talid; - found->expires = v->expires; /* adjust unique count */ repo_stat_inc(repo_byid(found->repoid), - RTYPE_ROA, STYPE_DEC_UNIQUE); + found->talid, RTYPE_ROA, STYPE_DEC_UNIQUE); + found->expires = v->expires; + found->talid = v->talid; found->repoid = v->repoid; - repo_stat_inc(rp, RTYPE_ROA, STYPE_UNIQUE); + repo_stat_inc(rp, v->talid, RTYPE_ROA, + STYPE_UNIQUE); } free(v); } else - repo_stat_inc(rp, RTYPE_ROA, STYPE_UNIQUE); + repo_stat_inc(rp, v->talid, RTYPE_ROA, STYPE_UNIQUE); - repo_stat_inc(rp, RTYPE_ROA, STYPE_TOTAL); + repo_stat_inc(rp, roa->talid, RTYPE_ROA, STYPE_TOTAL); } } -- 2.20.1