From: claudio Date: Thu, 15 Dec 2022 12:02:29 +0000 (+0000) Subject: Rework statistic collection to be per repository and add metric output option X-Git-Url: http://artulab.com/gitweb/?a=commitdiff_plain;h=4f5f25cbf9d7d654412dce563250c73403fab1fd;p=openbsd Rework statistic collection to be per repository and add metric output option Many statistic values are now accounted by repository via repo_stat_inc() At end of the run sum_stats() accumulates these stats per TAL and globally. The new output file metrics is written when the -m output flag is specified. The metrics file is written in OpenMetrics format (with a few tweaks to allow node_exporter to parse the file as well). The ometric code is a copy from bgpctl(8) and should be kept in sync. OK tb@ --- diff --git a/usr.sbin/rpki-client/Makefile b/usr.sbin/rpki-client/Makefile index 052ee49d035..153f1bf5b78 100644 --- a/usr.sbin/rpki-client/Makefile +++ b/usr.sbin/rpki-client/Makefile @@ -1,11 +1,12 @@ -# $OpenBSD: Makefile,v 1.28 2022/11/26 12:02:36 job Exp $ +# $OpenBSD: Makefile,v 1.29 2022/12/15 12:02:29 claudio Exp $ PROG= rpki-client SRCS= as.c aspa.c cert.c cms.c crl.c encoding.c filemode.c gbr.c geofeed.c \ - http.c io.c ip.c log.c main.c mft.c mkdir.c output.c output-bgpd.c \ - output-bird.c output-csv.c output-json.c parser.c print.c repo.c \ - roa.c rrdp.c rrdp_delta.c rrdp_notification.c rrdp_snapshot.c \ - rrdp_util.c rsc.c rsync.c tak.c tal.c validate.c x509.c + http.c io.c ip.c log.c main.c mft.c mkdir.c ometric.c output.c \ + output-bgpd.c output-bird.c output-csv.c output-json.c \ + output-ometric.c parser.c print.c repo.c roa.c rrdp.c rrdp_delta.c \ + rrdp_notification.c rrdp_snapshot.c rrdp_util.c rsc.c rsync.c tak.c \ + tal.c validate.c x509.c MAN= rpki-client.8 LDADD+= -lexpat -ltls -lssl -lcrypto -lutil diff --git a/usr.sbin/rpki-client/aspa.c b/usr.sbin/rpki-client/aspa.c index 79bafc03939..d41ef8bfc8a 100644 --- a/usr.sbin/rpki-client/aspa.c +++ b/usr.sbin/rpki-client/aspa.c @@ -1,4 +1,4 @@ -/* $OpenBSD: aspa.c,v 1.9 2022/11/29 20:41:32 job Exp $ */ +/* $OpenBSD: aspa.c,v 1.10 2022/12/15 12:02:29 claudio Exp $ */ /* * Copyright (c) 2022 Job Snijders * Copyright (c) 2022 Theo Buehler @@ -351,9 +351,9 @@ aspa_read(struct ibuf *b) * Ensure there are no duplicates in the 'providers' array. * Always compare 'expires': use the soonest expiration moment. */ -static void +static int insert_vap(struct vap_tree *tree, uint32_t cas, uint32_t pas, time_t expires, - enum afi afi) + enum afi afi, struct repo *rp) { struct vap *v, *found; size_t i; @@ -371,7 +371,8 @@ insert_vap(struct vap_tree *tree, uint32_t cas, uint32_t pas, time_t expires, v->providers[0] = pas; v->providersz = 1; - return; + repo_stat_inc(rp, RTYPE_ASPA, STYPE_UNIQUE); + return 1; } free(v); @@ -381,7 +382,7 @@ insert_vap(struct vap_tree *tree, uint32_t cas, uint32_t pas, time_t expires, for (i = 0; i < found->providersz; i++) { if (found->providers[i] == pas) - return; + return 0; } found->providers = reallocarray(found->providers, @@ -389,6 +390,7 @@ insert_vap(struct vap_tree *tree, uint32_t cas, uint32_t pas, time_t expires, if (found->providers == NULL) err(1, NULL); found->providers[found->providersz++] = pas; + return 1; } /* @@ -397,34 +399,36 @@ insert_vap(struct vap_tree *tree, uint32_t cas, uint32_t pas, time_t expires, * pre-'AFI explosion' deduplicated count. */ void -aspa_insert_vaps(struct vap_tree *tree, struct aspa *aspa, size_t *vaps, - size_t *uniqs) +aspa_insert_vaps(struct vap_tree *tree, struct aspa *aspa, struct repo *rp) { size_t i; uint32_t cas, pas; time_t expires; + int new; cas = aspa->custasid; expires = aspa->expires; - *uniqs += aspa->providersz; + repo_stat_inc(rp, RTYPE_ASPA, STYPE_TOTAL); for (i = 0; i < aspa->providersz; i++) { pas = aspa->providers[i].as; switch (aspa->providers[i].afi) { case AFI_IPV4: - insert_vap(tree, cas, pas, expires, AFI_IPV4); - (*vaps)++; + if (insert_vap(tree, cas, pas, expires, AFI_IPV4, rp)) + repo_stat_inc(rp, RTYPE_ASPA, STYPE_ONLY_IPV4); break; case AFI_IPV6: - insert_vap(tree, cas, pas, expires, AFI_IPV6); - (*vaps)++; + if (insert_vap(tree, cas, pas, expires, AFI_IPV6, rp)) + repo_stat_inc(rp, RTYPE_ASPA, STYPE_ONLY_IPV6); break; default: - insert_vap(tree, cas, pas, expires, AFI_IPV4); - insert_vap(tree, cas, pas, expires, AFI_IPV6); - *vaps += 2; + new = insert_vap(tree, cas, pas, expires, AFI_IPV4, rp); + new += insert_vap(tree, cas, pas, expires, AFI_IPV6, + rp); + if (new != 0) + repo_stat_inc(rp, RTYPE_ASPA, STYPE_BOTH); break; } } diff --git a/usr.sbin/rpki-client/extern.h b/usr.sbin/rpki-client/extern.h index 9143c457766..d6c3b93de79 100644 --- a/usr.sbin/rpki-client/extern.h +++ b/usr.sbin/rpki-client/extern.h @@ -1,4 +1,4 @@ -/* $OpenBSD: extern.h,v 1.163 2022/12/14 10:34:49 claudio Exp $ */ +/* $OpenBSD: extern.h,v 1.164 2022/12/15 12:02:29 claudio Exp $ */ /* * Copyright (c) 2019 Kristaps Dzonsons * @@ -376,6 +376,7 @@ struct vrp { RB_ENTRY(vrp) entry; struct ip_addr addr; int talid; /* covered by which TAL */ + unsigned int repoid; uint32_t asid; enum afi afi; unsigned char maxlength; @@ -493,6 +494,20 @@ struct entity { }; TAILQ_HEAD(entityq, entity); +enum stype { + STYPE_OK, + STYPE_FAIL, + STYPE_INVALID, + STYPE_STALE, + STYPE_BGPSEC, + STYPE_TOTAL, + STYPE_UNIQUE, + STYPE_DEC_UNIQUE, + STYPE_ONLY_IPV4, + STYPE_ONLY_IPV6, + STYPE_BOTH, +}; + struct repo; struct filepath; RB_HEAD(filepath_tree, filepath); @@ -501,41 +516,50 @@ RB_HEAD(filepath_tree, filepath); /* * Statistics collected during run-time. */ +struct repostats { + uint32_t certs; /* certificates */ + uint32_t certs_fail; /* invalid certificate */ + uint32_t mfts; /* total number of manifests */ + uint32_t mfts_fail; /* failing syntactic parse */ + uint32_t mfts_stale; /* stale manifests */ + uint32_t roas; /* route origin authorizations */ + uint32_t roas_fail; /* failing syntactic parse */ + uint32_t roas_invalid; /* invalid resources */ + uint32_t aspas; /* ASPA objects */ + uint32_t aspas_fail; /* ASPA objects failing syntactic parse */ + uint32_t aspas_invalid; /* ASPAs with invalid customerASID */ + uint32_t brks; /* number of BGPsec Router Key (BRK) certs */ + uint32_t crls; /* revocation lists */ + uint32_t gbrs; /* ghostbuster records */ + uint32_t taks; /* signed TAL objects */ + uint32_t vaps; /* total number of Validated ASPA Payloads */ + uint32_t vaps_uniqs; /* total number of unique VAPs */ + uint32_t vaps_pas; /* total number of providers */ + uint32_t vaps_pas4; /* total number of IPv4 only providers */ + uint32_t vaps_pas6; /* total number of IPv6 only providers */ + uint32_t vrps; /* total number of Validated ROA Payloads */ + uint32_t vrps_uniqs; /* number of unique vrps */ + struct timespec sync_time; /* time to sync repo */ +}; + struct stats { - size_t tals; /* total number of locators */ - size_t mfts; /* total number of manifests */ - size_t mfts_fail; /* failing syntactic parse */ - size_t mfts_stale; /* stale manifests */ - size_t certs; /* certificates */ - size_t certs_fail; /* invalid certificate */ - size_t roas; /* route origin authorizations */ - size_t roas_fail; /* failing syntactic parse */ - size_t roas_invalid; /* invalid resources */ - size_t repos; /* repositories */ - size_t rsync_repos; /* synced rsync repositories */ - size_t rsync_fails; /* failed rsync repositories */ - size_t http_repos; /* synced http repositories */ - size_t http_fails; /* failed http repositories */ - size_t rrdp_repos; /* synced rrdp repositories */ - size_t rrdp_fails; /* failed rrdp repositories */ - size_t crls; /* revocation lists */ - size_t gbrs; /* ghostbuster records */ - size_t taks; /* signed TAL objects */ - size_t aspas; /* ASPA objects */ - size_t aspas_fail; /* ASPA objects failing syntactic parse */ - size_t aspas_invalid; /* ASPAs with invalid customerASID */ - size_t vaps; /* total number of Validated ASPA Payloads */ - size_t vaps_uniqs; /* total number of unique VAPs */ - size_t vrps; /* total number of vrps */ - size_t uniqs; /* number of unique vrps */ - size_t del_files; /* number of files removed in cleanup */ - size_t extra_files; /* number of superfluous files */ - size_t del_dirs; /* number of directories removed in cleanup */ - size_t brks; /* number of BGPsec Router Key (BRK) certificates */ - size_t skiplistentries; /* number of skiplist entries */ - struct timespec elapsed_time; - struct timespec user_time; - struct timespec system_time; + uint32_t tals; /* total number of locators */ + uint32_t repos; /* repositories */ + uint32_t rsync_repos; /* synced rsync repositories */ + uint32_t rsync_fails; /* failed rsync repositories */ + uint32_t http_repos; /* synced http repositories */ + uint32_t http_fails; /* failed http repositories */ + uint32_t rrdp_repos; /* synced rrdp repositories */ + uint32_t rrdp_fails; /* failed rrdp repositories */ + uint32_t del_files; /* number of files removed in cleanup */ + uint32_t extra_files; /* number of superfluous files */ + uint32_t del_dirs; /* number of dirs removed in cleanup */ + uint32_t skiplistentries; /* number of skiplist entries */ + + struct repostats repo_stats; + struct timespec elapsed_time; + struct timespec user_time; + struct timespec system_time; }; struct ibuf; @@ -547,6 +571,7 @@ extern int filemode; extern const char *tals[]; extern const char *taldescs[]; extern unsigned int talrepocnt[]; +extern struct repostats talstats[]; extern int talsz; /* Routines for RPKI entities. */ @@ -580,8 +605,8 @@ void roa_free(struct roa *); struct roa *roa_parse(X509 **, const char *, const unsigned char *, size_t); struct roa *roa_read(struct ibuf *); -void roa_insert_vrps(struct vrp_tree *, struct roa *, size_t *, - size_t *); +void roa_insert_vrps(struct vrp_tree *, struct roa *, + struct repo *); void gbr_free(struct gbr *); struct gbr *gbr_parse(X509 **, const char *, const unsigned char *, @@ -602,8 +627,8 @@ struct tak *tak_read(struct ibuf *); void aspa_buffer(struct ibuf *, const struct aspa *); void aspa_free(struct aspa *); -void aspa_insert_vaps(struct vap_tree *, struct aspa *, size_t *, - size_t *); +void aspa_insert_vaps(struct vap_tree *, struct aspa *, + struct repo *); struct aspa *aspa_parse(X509 **, const char *, const unsigned char *, size_t); struct aspa *aspa_read(struct ibuf *); @@ -703,11 +728,19 @@ int rrdp_handle_file(unsigned int, enum publish_type, char *, char *repo_basedir(const struct repo *, int); unsigned int repo_id(const struct repo *); const char *repo_uri(const struct repo *); +void repo_fetch_uris(const struct repo *, const char **, + const char **); +int repo_synced(const struct repo *); +int repo_talid(const struct repo *); struct repo *ta_lookup(int, struct tal *); struct repo *repo_lookup(int, const char *, const char *); struct repo *repo_byid(unsigned int); int repo_queued(struct repo *, struct entity *); void repo_cleanup(struct filepath_tree *, int); +int repo_check_timeout(int); +void repo_stat_inc(struct repo *, enum rtype, enum stype); +void repo_stats_collect(void (*)(const struct repo *, + const struct repostats *, void *), void *); void repo_free(void); void rsync_finish(unsigned int, int); @@ -722,7 +755,6 @@ void rrdp_fetch(unsigned int, const char *, const char *, struct rrdp_session *); void rrdp_abort(unsigned int); void rrdp_http_done(unsigned int, enum http_result, const char *); -int repo_check_timeout(int); /* Logging (though really used for OpenSSL errors). */ @@ -797,6 +829,7 @@ extern int outformats; #define FORMAT_BIRD 0x02 #define FORMAT_CSV 0x04 #define FORMAT_JSON 0x08 +#define FORMAT_OMETRIC 0x10 int outputfiles(struct vrp_tree *v, struct brk_tree *b, struct vap_tree *, struct stats *); @@ -813,6 +846,8 @@ int output_csv(FILE *, struct vrp_tree *, struct brk_tree *, struct vap_tree *, struct stats *); int output_json(FILE *, struct vrp_tree *, struct brk_tree *, struct vap_tree *, struct stats *); +int output_ometric(FILE *, struct vrp_tree *, struct brk_tree *, + struct vap_tree *, struct stats *); void logx(const char *fmt, ...) __attribute__((format(printf, 1, 2))); diff --git a/usr.sbin/rpki-client/main.c b/usr.sbin/rpki-client/main.c index c78655b6fcb..d93aaeb1a6f 100644 --- a/usr.sbin/rpki-client/main.c +++ b/usr.sbin/rpki-client/main.c @@ -1,4 +1,4 @@ -/* $OpenBSD: main.c,v 1.228 2022/12/14 10:34:49 claudio Exp $ */ +/* $OpenBSD: main.c,v 1.229 2022/12/15 12:02:29 claudio Exp $ */ /* * Copyright (c) 2021 Claudio Jeker * Copyright (c) 2019 Kristaps Dzonsons @@ -51,6 +51,7 @@ const char *tals[TALSZ_MAX]; const char *taldescs[TALSZ_MAX]; unsigned int talrepocnt[TALSZ_MAX]; +struct repostats talstats[TALSZ_MAX]; int talsz; size_t entity_queue; @@ -535,7 +536,9 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, struct mft *mft; struct roa *roa; struct aspa *aspa; + struct repo *rp; char *file; + unsigned int id; int c; /* @@ -556,6 +559,9 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, goto done; } + io_read_buf(b, &id, sizeof(id)); + rp = repo_byid(id); + repo_stat_inc(rp, type, STYPE_OK); switch (type) { case RTYPE_TAL: st->tals++; @@ -564,10 +570,9 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, tal_free(tal); break; case RTYPE_CER: - st->certs++; io_read_buf(b, &c, sizeof(c)); if (c == 0) { - st->certs_fail++; + repo_stat_inc(rp, type, STYPE_FAIL); break; } cert = cert_read(b); @@ -577,65 +582,58 @@ entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, break; case CERT_PURPOSE_BGPSEC_ROUTER: cert_insert_brks(brktree, cert); - st->brks++; + repo_stat_inc(rp, type, STYPE_BGPSEC); break; default: - st->certs_fail++; + errx(1, "unexpected cert purpose received"); break; } cert_free(cert); break; case RTYPE_MFT: - st->mfts++; io_read_buf(b, &c, sizeof(c)); if (c == 0) { - st->mfts_fail++; + repo_stat_inc(rp, type, STYPE_FAIL); break; } mft = mft_read(b); if (!mft->stale) queue_add_from_mft(mft); else - st->mfts_stale++; + repo_stat_inc(rp, type, STYPE_STALE); mft_free(mft); break; case RTYPE_CRL: - st->crls++; break; case RTYPE_ROA: - st->roas++; io_read_buf(b, &c, sizeof(c)); if (c == 0) { - st->roas_fail++; + repo_stat_inc(rp, type, STYPE_FAIL); break; } roa = roa_read(b); if (roa->valid) - roa_insert_vrps(tree, roa, &st->vrps, &st->uniqs); + roa_insert_vrps(tree, roa, rp); else - st->roas_invalid++; + repo_stat_inc(rp, type, STYPE_INVALID); roa_free(roa); break; case RTYPE_GBR: - st->gbrs++; break; case RTYPE_ASPA: - st->aspas++; io_read_buf(b, &c, sizeof(c)); if (c == 0) { - st->aspas_fail++; + repo_stat_inc(rp, type, STYPE_FAIL); break; } aspa = aspa_read(b); if (aspa->valid) - aspa_insert_vaps(vaptree, aspa, &st->vaps, - &st->vaps_uniqs); + aspa_insert_vaps(vaptree, aspa, rp); else - st->aspas_invalid++; + repo_stat_inc(rp, type, STYPE_INVALID); aspa_free(aspa); break; case RTYPE_TAK: - st->taks++; break; case RTYPE_FILE: break; @@ -703,6 +701,40 @@ rrdp_process(struct ibuf *b) } } +static void +sum_stats(const struct repo *rp, const struct repostats *in, void *arg) +{ + struct repostats *out = arg; + + if (rp != NULL) + sum_stats(NULL, in, &talstats[repo_talid(rp)]); + + out->mfts += in->mfts; + out->mfts_fail += in->mfts_fail; + out->mfts_stale += in->mfts_stale; + out->certs += in->certs; + out->certs_fail += in->certs_fail; + out->roas += in->roas; + out->roas_fail += in->roas_fail; + out->roas_invalid += in->roas_invalid; + out->aspas += in->aspas; + out->aspas_fail += in->aspas_fail; + out->aspas_invalid += in->aspas_invalid; + out->brks += in->brks; + out->crls += in->crls; + out->gbrs += in->gbrs; + out->taks += in->taks; + out->vrps += in->vrps; + out->vrps_uniqs += in->vrps_uniqs; + out->vaps += in->vaps; + out->vaps_uniqs += in->vaps_uniqs; + out->vaps_pas += in->vaps_pas; + out->vaps_pas4 += in->vaps_pas4; + out->vaps_pas6 += in->vaps_pas6; + + timespecadd(&in->sync_time, &out->sync_time, &out->sync_time); +} + /* * Assign filenames ending in ".tal" in "/etc/rpki" into "tals", * returning the number of files found and filled-in. @@ -909,7 +941,7 @@ main(int argc, char *argv[]) "proc exec unveil", NULL) == -1) err(1, "pledge"); - while ((c = getopt(argc, argv, "b:Bcd:e:fH:jnorRs:S:t:T:vV")) != -1) + while ((c = getopt(argc, argv, "b:Bcd:e:fH:jmnorRs:S:t:T:vV")) != -1) switch (c) { case 'b': bind_addr = optarg; @@ -937,6 +969,9 @@ main(int argc, char *argv[]) case 'j': outformats |= FORMAT_JSON; break; + case 'm': + outformats |= FORMAT_OMETRIC; + break; case 'n': noop = 1; break; @@ -1344,6 +1379,8 @@ main(int argc, char *argv[]) if (fchdir(outdirfd) == -1) err(1, "fchdir output dir"); + repo_stats_collect(sum_stats, &stats.repo_stats); + if (outputfiles(&vrps, &brks, &vaps, &stats)) rc = 1; @@ -1352,26 +1389,31 @@ main(int argc, char *argv[]) (long long)stats.elapsed_time.tv_sec, (long long)stats.user_time.tv_sec, (long long)stats.system_time.tv_sec); - printf("Skiplist entries: %zu\n", stats.skiplistentries); - printf("Route Origin Authorizations: %zu (%zu failed parse, %zu " - "invalid)\n", stats.roas, stats.roas_fail, stats.roas_invalid); - printf("AS Provider Attestations: %zu (%zu failed parse, %zu " - "invalid)\n", stats.aspas, stats.aspas_fail, stats.aspas_invalid); - printf("BGPsec Router Certificates: %zu\n", stats.brks); - printf("Certificates: %zu (%zu invalid)\n", - stats.certs, stats.certs_fail); - printf("Trust Anchor Locators: %zu (%zu invalid)\n", + printf("Skiplist entries: %u\n", stats.skiplistentries); + printf("Route Origin Authorizations: %u (%u failed parse, %u " + "invalid)\n", stats.repo_stats.roas, stats.repo_stats.roas_fail, + stats.repo_stats.roas_invalid); + printf("AS Provider Attestations: %u (%u failed parse, %u " + "invalid)\n", stats.repo_stats.aspas, stats.repo_stats.aspas_fail, + stats.repo_stats.aspas_invalid); + printf("BGPsec Router Certificates: %u\n", stats.repo_stats.brks); + printf("Certificates: %u (%u invalid)\n", + stats.repo_stats.certs, stats.repo_stats.certs_fail); + printf("Trust Anchor Locators: %u (%u invalid)\n", stats.tals, talsz - stats.tals); - printf("Manifests: %zu (%zu failed parse, %zu stale)\n", - stats.mfts, stats.mfts_fail, stats.mfts_stale); - printf("Certificate revocation lists: %zu\n", stats.crls); - printf("Ghostbuster records: %zu\n", stats.gbrs); - printf("Trust Anchor Keys: %zu\n", stats.taks); - printf("Repositories: %zu\n", stats.repos); - printf("Cleanup: removed %zu files, %zu directories, %zu superfluous\n", + printf("Manifests: %u (%u failed parse, %u stale)\n", + stats.repo_stats.mfts, stats.repo_stats.mfts_fail, + stats.repo_stats.mfts_stale); + printf("Certificate revocation lists: %u\n", stats.repo_stats.crls); + printf("Ghostbuster records: %u\n", stats.repo_stats.gbrs); + printf("Trust Anchor Keys: %u\n", stats.repo_stats.taks); + printf("Repositories: %u\n", stats.repos); + printf("Cleanup: removed %u files, %u directories, %u superfluous\n", stats.del_files, stats.del_dirs, stats.extra_files); - printf("VRP Entries: %zu (%zu unique)\n", stats.vrps, stats.uniqs); - printf("VAP Entries: %zu (%zu unique)\n", stats.vaps, stats.vaps_uniqs); + printf("VRP Entries: %u (%u unique)\n", stats.repo_stats.vrps, + stats.repo_stats.vrps_uniqs); + printf("VAP Entries: %u (%u unique)\n", stats.repo_stats.vaps, + stats.repo_stats.vaps_uniqs); /* Memory cleanup. */ repo_free(); @@ -1380,7 +1422,7 @@ main(int argc, char *argv[]) usage: fprintf(stderr, - "usage: rpki-client [-BcjnoRrVv] [-b sourceaddr] [-d cachedir]" + "usage: rpki-client [-BcjmnoRrVv] [-b sourceaddr] [-d cachedir]" " [-e rsync_prog]\n" " [-H fqdn] [-S skiplist] [-s timeout] [-T table]" " [-t tal]\n" diff --git a/usr.sbin/rpki-client/ometric.c b/usr.sbin/rpki-client/ometric.c new file mode 100644 index 00000000000..801586d237b --- /dev/null +++ b/usr.sbin/rpki-client/ometric.c @@ -0,0 +1,527 @@ +/* $OpenBSD: ometric.c,v 1.1 2022/12/15 12:02:29 claudio Exp $ */ + +/* + * Copyright (c) 2022 Claudio Jeker + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "ometric.h" + +struct olabel { + STAILQ_ENTRY(olabel) entry; + const char *key; + char *value; +}; + +struct olabels { + STAILQ_HEAD(, olabel) labels; + struct olabels *next; + int refcnt; +}; + +enum ovalue_type { + OVT_INTEGER, + OVT_DOUBLE, + OVT_TIMESPEC, +}; + +struct ovalue { + STAILQ_ENTRY(ovalue) entry; + struct olabels *labels; + union { + unsigned long long i; + double f; + struct timespec ts; + } value; + enum ovalue_type valtype; +}; + +STAILQ_HEAD(ovalues, ovalue); + +struct ometric { + STAILQ_ENTRY(ometric) entry; + struct ovalues vals; + const char *name; + const char *help; + const char *const *stateset; + size_t setsize; + enum ometric_type type; +}; + +STAILQ_HEAD(, ometric) ometrics = STAILQ_HEAD_INITIALIZER(ometrics); + +static const char *suffixes[] = { "_total", "_created", "_count", + "_sum", "_bucket", "_gcount", "_gsum", "_info", +}; + +/* + * Return true if name has one of the above suffixes. + */ +static int +strsuffix(const char *name) +{ + const char *suffix; + size_t i; + + suffix = strrchr(name, '_'); + if (suffix == NULL) + return 0; + for (i = 0; i < sizeof(suffixes) / sizeof(suffixes[0]); i++) { + if (strcmp(suffix, suffixes[i]) == 0) + return 1; + } + return 0; +} + +static void +ometric_check(const char *name) +{ + struct ometric *om; + + if (strsuffix(name)) + errx(1, "reserved name suffix used: %s", name); + STAILQ_FOREACH(om, &ometrics, entry) + if (strcmp(name, om->name) == 0) + errx(1, "duplicate name: %s", name); +} + +/* + * Allocate and return new ometric. The name and help string need to remain + * valid until the ometric is freed. Normally constant strings should be used. + */ +struct ometric * +ometric_new(enum ometric_type type, const char *name, const char *help) +{ + struct ometric *om; + + ometric_check(name); + + if ((om = calloc(1, sizeof(*om))) == NULL) + err(1, NULL); + + om->name = name; + om->help = help; + om->type = type; + STAILQ_INIT(&om->vals); + + STAILQ_INSERT_TAIL(&ometrics, om, entry); + + return om; +} + +/* + * Same as above but for a stateset. The states is an array of constant strings + * with statecnt elements. The states, name and help pointers need to remain + * valid until the ometric is freed. + */ +struct ometric * +ometric_new_state(const char * const *states, size_t statecnt, const char *name, + const char *help) +{ + struct ometric *om; + + ometric_check(name); + + if ((om = calloc(1, sizeof(*om))) == NULL) + err(1, NULL); + + om->name = name; + om->help = help; + om->type = OMT_STATESET; + om->stateset = states; + om->setsize = statecnt; + STAILQ_INIT(&om->vals); + + STAILQ_INSERT_TAIL(&ometrics, om, entry); + + return om; +} + +void +ometric_free_all(void) +{ + struct ometric *om; + struct ovalue *ov; + + while ((om = STAILQ_FIRST(&ometrics)) != NULL) { + STAILQ_REMOVE_HEAD(&ometrics, entry); + while ((ov = STAILQ_FIRST(&om->vals)) != NULL) { + STAILQ_REMOVE_HEAD(&om->vals, entry); + olabels_free(ov->labels); + free(ov); + } + free(om); + } +} + +static struct olabels * +olabels_ref(struct olabels *ol) +{ + struct olabels *x = ol; + + while (x != NULL) { + x->refcnt++; + x = x->next; + } + + return ol; +} + +/* + * Create a new set of labels based on keys and values arrays. + * keys must end in a NULL element. values needs to hold as many elements + * but the elements can be NULL. values are copied for the olabel but + * keys needs to point to constant memory. + */ +struct olabels * +olabels_new(const char * const *keys, const char **values) +{ + struct olabels *ol; + struct olabel *l; + + if ((ol = malloc(sizeof(*ol))) == NULL) + err(1, NULL); + STAILQ_INIT(&ol->labels); + ol->refcnt = 1; + ol->next = NULL; + + while (*keys != NULL) { + if (*values && **values != '\0') { + if ((l = malloc(sizeof(*l))) == NULL) + err(1, NULL); + l->key = *keys; + if ((l->value = strdup(*values)) == NULL) + err(1, NULL); + STAILQ_INSERT_TAIL(&ol->labels, l, entry); + } + + keys++; + values++; + } + + return ol; +} + +/* + * Free olables once nothing uses them anymore. + */ +void +olabels_free(struct olabels *ol) +{ + struct olabels *next; + struct olabel *l; + + for ( ; ol != NULL; ol = next) { + next = ol->next; + + if (--ol->refcnt == 0) { + while ((l = STAILQ_FIRST(&ol->labels)) != NULL) { + STAILQ_REMOVE_HEAD(&ol->labels, entry); + free(l->value); + free(l); + } + free(ol); + } + } +} + +/* + * Add one extra label onto the label stack. Once no longer used the + * value needs to be freed with olabels_free(). + */ +static struct olabels * +olabels_add_extras(struct olabels *ol, const char **keys, const char **values) +{ + struct olabels *new; + + new = olabels_new(keys, values); + new->next = olabels_ref(ol); + + return new; +} + +/* + * Output function called last. + */ +static const char * +ometric_type(enum ometric_type type) +{ + switch (type) { + case OMT_GAUGE: + return "gauge"; + case OMT_COUNTER: + return "counter"; + case OMT_STATESET: + /* return "stateset"; node_exporter does not like this type */ + return "gauge"; + case OMT_HISTOGRAM: + return "histogram"; + case OMT_SUMMARY: + return "summary"; + case OMT_INFO: + /* return "info"; node_exporter does not like this type */ + return "gauge"; + default: + return "unknown"; + } +} + +static int +ometric_output_labels(FILE *out, const struct olabels *ol) +{ + struct olabel *l; + const char *comma = ""; + + if (ol == NULL) + return fprintf(out, " "); + + if (fprintf(out, "{") < 0) + return -1; + + while (ol != NULL) { + STAILQ_FOREACH(l, &ol->labels, entry) { + if (fprintf(out, "%s%s=\"%s\"", comma, l->key, + l->value) < 0) + return -1; + comma = ","; + } + ol = ol->next; + } + + return fprintf(out, "} "); +} + +static int +ometric_output_value(FILE *out, const struct ovalue *ov) +{ + switch (ov->valtype) { + case OVT_INTEGER: + return fprintf(out, "%llu", ov->value.i); + case OVT_DOUBLE: + return fprintf(out, "%g", ov->value.f); + case OVT_TIMESPEC: + return fprintf(out, "%lld.%09ld", + (long long)ov->value.ts.tv_sec, ov->value.ts.tv_nsec); + } + return -1; +} + +static int +ometric_output_name(FILE *out, const struct ometric *om) +{ + const char *suffix; + + switch (om->type) { + case OMT_COUNTER: + suffix = "_total"; + break; + case OMT_INFO: + suffix = "_info"; + break; + default: + suffix = ""; + break; + } + return fprintf(out, "%s%s", om->name, suffix); +} + +/* + * Output all metric values with TYPE and optional HELP strings. + */ +int +ometric_output_all(FILE *out) +{ + struct ometric *om; + struct ovalue *ov; + + STAILQ_FOREACH(om, &ometrics, entry) { + if (om->help) + if (fprintf(out, "# HELP %s %s\n", om->name, + om->help) < 0) + return -1; + + if (fprintf(out, "# TYPE %s %s\n", om->name, + ometric_type(om->type)) < 0) + return -1; + + STAILQ_FOREACH(ov, &om->vals, entry) { + if (ometric_output_name(out, om) < 0) + return -1; + if (ometric_output_labels(out, ov->labels) < 0) + return -1; + if (ometric_output_value(out, ov) < 0) + return -1; + if (fprintf(out, "\n") < 0) + return -1; + } + } + + if (fprintf(out, "# EOF\n") < 0) + return -1; + return 0; +} + +/* + * Value setters + */ +static void +ometric_set_int_value(struct ometric *om, uint64_t val, struct olabels *ol) +{ + struct ovalue *ov; + + if ((ov = malloc(sizeof(*ov))) == NULL) + err(1, NULL); + + ov->value.i = val; + ov->valtype = OVT_INTEGER; + ov->labels = olabels_ref(ol); + + STAILQ_INSERT_TAIL(&om->vals, ov, entry); +} + +/* + * Set an integer value with label ol. ol can be NULL. + */ +void +ometric_set_int(struct ometric *om, uint64_t val, struct olabels *ol) +{ + if (om->type != OMT_COUNTER && om->type != OMT_GAUGE) + errx(1, "%s incorrect ometric type", __func__); + + ometric_set_int_value(om, val, ol); +} + +/* + * Set a floating point value with label ol. ol can be NULL. + */ +void +ometric_set_float(struct ometric *om, double val, struct olabels *ol) +{ + struct ovalue *ov; + + if (om->type != OMT_COUNTER && om->type != OMT_GAUGE) + errx(1, "%s incorrect ometric type", __func__); + + if ((ov = malloc(sizeof(*ov))) == NULL) + err(1, NULL); + + ov->value.f = val; + ov->valtype = OVT_DOUBLE; + ov->labels = olabels_ref(ol); + + STAILQ_INSERT_TAIL(&om->vals, ov, entry); +} + +/* + * Set an timespec value with label ol. ol can be NULL. + */ +void +ometric_set_timespec(struct ometric *om, const struct timespec *ts, + struct olabels *ol) +{ + struct ovalue *ov; + + if (om->type != OMT_GAUGE) + errx(1, "%s incorrect ometric type", __func__); + + if ((ov = malloc(sizeof(*ov))) == NULL) + err(1, NULL); + + ov->value.ts = *ts; + ov->valtype = OVT_TIMESPEC; + ov->labels = olabels_ref(ol); + + STAILQ_INSERT_TAIL(&om->vals, ov, entry); +} + +/* + * Add an info value (which is the value 1 but with extra key-value pairs). + */ +void +ometric_set_info(struct ometric *om, const char **keys, const char **values, + struct olabels *ol) +{ + struct olabels *extra = NULL; + + if (om->type != OMT_INFO) + errx(1, "%s incorrect ometric type", __func__); + + if (keys != NULL) + extra = olabels_add_extras(ol, keys, values); + + ometric_set_int_value(om, 1, extra != NULL ? extra : ol); + olabels_free(extra); +} + +/* + * Set a stateset to one of its states. + */ +void +ometric_set_state(struct ometric *om, const char *state, struct olabels *ol) +{ + struct olabels *extra; + size_t i; + int val; + + if (om->type != OMT_STATESET) + errx(1, "%s incorrect ometric type", __func__); + + for (i = 0; i < om->setsize; i++) { + if (strcasecmp(state, om->stateset[i]) == 0) + val = 1; + else + val = 0; + + extra = olabels_add_extras(ol, OKV(om->name), + OKV(om->stateset[i])); + ometric_set_int_value(om, val, extra); + olabels_free(extra); + } +} + +/* + * Set a value with an extra label, the key should be a constant string while + * the value is copied into the extra label. + */ +void +ometric_set_int_with_labels(struct ometric *om, uint64_t val, + const char **keys, const char **values, struct olabels *ol) +{ + struct olabels *extra; + + extra = olabels_add_extras(ol, keys, values); + ometric_set_int(om, val, extra); + olabels_free(extra); +} + +void +ometric_set_timespec_with_labels(struct ometric *om, struct timespec *ts, + const char **keys, const char **values, struct olabels *ol) +{ + struct olabels *extra; + + extra = olabels_add_extras(ol, keys, values); + ometric_set_timespec(om, ts, extra); + olabels_free(extra); +} diff --git a/usr.sbin/rpki-client/ometric.h b/usr.sbin/rpki-client/ometric.h new file mode 100644 index 00000000000..f7b140b9b8c --- /dev/null +++ b/usr.sbin/rpki-client/ometric.h @@ -0,0 +1,53 @@ +/* $OpenBSD: ometric.h,v 1.1 2022/12/15 12:02:29 claudio Exp $ */ + +/* + * Copyright (c) 2022 Claudio Jeker + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +enum ometric_type { + OMT_UNKNOWN, + OMT_GAUGE, + OMT_COUNTER, + OMT_STATESET, + OMT_HISTOGRAM, + OMT_SUMMARY, + OMT_INFO, +}; + +struct ometric; +struct olabels; + +struct ometric *ometric_new(enum ometric_type, const char *, const char *); +struct ometric *ometric_new_state(const char * const *, size_t, const char *, + const char *); +void ometric_free_all(void); +struct olabels *olabels_new(const char * const *, const char **); +void olabels_free(struct olabels *); + +int ometric_output_all(FILE *); + +/* functions to set gauge and counter metrics */ +void ometric_set_int(struct ometric *, uint64_t, struct olabels *); +void ometric_set_float(struct ometric *, double, struct olabels *); +void ometric_set_timespec(struct ometric *, const struct timespec *, + struct olabels *); +void ometric_set_info(struct ometric *, const char **, const char **, + struct olabels *); +void ometric_set_state(struct ometric *, const char *, struct olabels *); +void ometric_set_int_with_labels(struct ometric *, uint64_t, const char **, + const char **, struct olabels *); +void ometric_set_timespec_with_labels(struct ometric *, struct timespec *, + const char **, const char **, struct olabels *); +#define OKV(...) (const char *[]){ __VA_ARGS__, NULL } diff --git a/usr.sbin/rpki-client/output-json.c b/usr.sbin/rpki-client/output-json.c index 51b0c839e9a..4945af19c9f 100644 --- a/usr.sbin/rpki-client/output-json.c +++ b/usr.sbin/rpki-client/output-json.c @@ -1,4 +1,4 @@ -/* $OpenBSD: output-json.c,v 1.29 2022/11/02 12:43:02 job Exp $ */ +/* $OpenBSD: output-json.c,v 1.30 2022/12/15 12:02:29 claudio Exp $ */ /* * Copyright (c) 2019 Claudio Jeker * @@ -43,24 +43,31 @@ outputheader_json(FILE *out, struct stats *st) "\t\t\"elapsedtime\": \"%lld\",\n" "\t\t\"usertime\": \"%lld\",\n" "\t\t\"systemtime\": \"%lld\",\n" - "\t\t\"roas\": %zu,\n" - "\t\t\"failedroas\": %zu,\n" - "\t\t\"invalidroas\": %zu,\n" - "\t\t\"aspas\": %zu,\n" - "\t\t\"failedaspas\": %zu,\n" - "\t\t\"invalidaspas\": %zu,\n" - "\t\t\"bgpsec_pubkeys\": %zu,\n" - "\t\t\"certificates\": %zu,\n" - "\t\t\"invalidcertificates\": %zu,\n" - "\t\t\"taks\": %zu,\n" - "\t\t\"tals\": %zu,\n" - "\t\t\"invalidtals\": %zu,\n" + "\t\t\"roas\": %u,\n" + "\t\t\"failedroas\": %u,\n" + "\t\t\"invalidroas\": %u,\n" + "\t\t\"aspas\": %u,\n" + "\t\t\"failedaspas\": %u,\n" + "\t\t\"invalidaspas\": %u,\n" + "\t\t\"bgpsec_pubkeys\": %u,\n" + "\t\t\"certificates\": %u,\n" + "\t\t\"invalidcertificates\": %u,\n" + "\t\t\"taks\": %u,\n" + "\t\t\"tals\": %u,\n" + "\t\t\"invalidtals\": %u,\n" "\t\t\"talfiles\": [\n", hn, tbuf, (long long)st->elapsed_time.tv_sec, (long long)st->user_time.tv_sec, (long long)st->system_time.tv_sec, - st->roas, st->roas_fail, st->roas_invalid, - st->aspas, st->aspas_fail, st->aspas_invalid, - st->brks, st->certs, st->certs_fail, st->taks, + st->repo_stats.roas, + st->repo_stats.roas_fail, + st->repo_stats.roas_invalid, + st->repo_stats.aspas, + st->repo_stats.aspas_fail, + st->repo_stats.aspas_invalid, + st->repo_stats.brks, + st->repo_stats.certs, + st->repo_stats.certs_fail, + st->repo_stats.taks, st->tals, talsz - st->tals) < 0) return -1; @@ -73,27 +80,33 @@ outputheader_json(FILE *out, struct stats *st) if (fprintf(out, "\t\t],\n" - "\t\t\"manifests\": %zu,\n" - "\t\t\"failedmanifests\": %zu,\n" - "\t\t\"stalemanifests\": %zu,\n" - "\t\t\"crls\": %zu,\n" - "\t\t\"gbrs\": %zu,\n" - "\t\t\"repositories\": %zu,\n" - "\t\t\"vrps\": %zu,\n" - "\t\t\"uniquevrps\": %zu,\n" - "\t\t\"vaps\": %zu,\n" - "\t\t\"uniquevaps\": %zu,\n" - "\t\t\"cachedir_del_files\": %zu,\n" - "\t\t\"cachedir_superfluous_files\": %zu,\n" - "\t\t\"cachedir_del_dirs\": %zu\n" + "\t\t\"manifests\": %u,\n" + "\t\t\"failedmanifests\": %u,\n" + "\t\t\"stalemanifests\": %u,\n" + "\t\t\"crls\": %u,\n" + "\t\t\"gbrs\": %u,\n" + "\t\t\"repositories\": %u,\n" + "\t\t\"vrps\": %u,\n" + "\t\t\"uniquevrps\": %u,\n" + "\t\t\"vaps\": %u,\n" + "\t\t\"uniquevaps\": %u,\n" + "\t\t\"cachedir_del_files\": %u,\n" + "\t\t\"cachedir_superfluous_files\": %u,\n" + "\t\t\"cachedir_del_dirs\": %u\n" "\t},\n\n", - st->mfts, st->mfts_fail, st->mfts_stale, - st->crls, - st->gbrs, + st->repo_stats.mfts, + st->repo_stats.mfts_fail, + st->repo_stats.mfts_stale, + st->repo_stats.crls, + st->repo_stats.gbrs, st->repos, - st->vrps, st->uniqs, - st->vaps, st->vaps_uniqs, - st->del_files, st->extra_files, st->del_dirs) < 0) + st->repo_stats.vrps, + st->repo_stats.vrps_uniqs, + st->repo_stats.vaps, + st->repo_stats.vaps_uniqs, + st->del_files, + st->extra_files, + st->del_dirs) < 0) return -1; return 0; } diff --git a/usr.sbin/rpki-client/output-ometric.c b/usr.sbin/rpki-client/output-ometric.c new file mode 100644 index 00000000000..df6ef097671 --- /dev/null +++ b/usr.sbin/rpki-client/output-ometric.c @@ -0,0 +1,212 @@ +/* $OpenBSD: output-ometric.c,v 1.1 2022/12/15 12:02:29 claudio Exp $ */ +/* + * Copyright (c) 2022 Claudio Jeker + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include + +#include "extern.h" +#include "ometric.h" +#include "version.h" + +struct ometric *rpki_info, *rpki_completion_time, *rpki_duration; +struct ometric *rpki_repo, *rpki_obj, *rpki_ta_obj; +struct ometric *rpki_repo_obj, *rpki_repo_duration, *rpki_repo_state; + +static const char * const repo_states[2] = { "failed", "synced" }; + +static void +set_common_stats(const struct repostats *in, struct ometric *metric, + struct olabels *ol) +{ + ometric_set_int_with_labels(metric, in->certs, + OKV("type", "state"), OKV("cert", "valid"), ol); + ometric_set_int_with_labels(metric, in->certs_fail, + OKV("type", "state"), OKV("cert", "failed parse"), ol); + + ometric_set_int_with_labels(metric, in->mfts, + OKV("type", "state"), OKV("manifest", "valid"), ol); + ometric_set_int_with_labels(metric, in->mfts_fail, + OKV("type", "state"), OKV("manifest", "failed parse"), ol); + ometric_set_int_with_labels(metric, in->mfts_stale, + OKV("type", "state"), OKV("manifest", "stale"), ol); + + ometric_set_int_with_labels(metric, in->roas, + OKV("type", "state"), OKV("roa", "valid"), ol); + ometric_set_int_with_labels(metric, in->roas_fail, + OKV("type", "state"), OKV("roa", "failed parse"), ol); + ometric_set_int_with_labels(metric, in->roas_invalid, + OKV("type", "state"), OKV("roa", "invalid"), ol); + + ometric_set_int_with_labels(metric, in->aspas, + OKV("type", "state"), OKV("aspa", "valid"), ol); + ometric_set_int_with_labels(metric, in->aspas_fail, + OKV("type", "state"), OKV("aspa", "failed parse"), ol); + ometric_set_int_with_labels(metric, in->aspas_invalid, + OKV("type", "state"), OKV("aspa", "invalid"), ol); + + ometric_set_int_with_labels(metric, in->brks, + OKV("type", "state"), OKV("router_key", "valid"), ol); + ometric_set_int_with_labels(metric, in->crls, + OKV("type", "state"), OKV("crl", "valid"), ol); + ometric_set_int_with_labels(metric, in->gbrs, + OKV("type", "state"), OKV("gbr", "valid"), ol); + ometric_set_int_with_labels(metric, in->taks, + OKV("type", "state"), OKV("tak", "valid"), ol); + + ometric_set_int_with_labels(metric, in->vrps, + OKV("type", "state"), OKV("vrp", "total"), ol); + ometric_set_int_with_labels(metric, in->vrps_uniqs, + OKV("type", "state"), OKV("vrp", "unique"), ol); + + ometric_set_int_with_labels(metric, in->vaps, + OKV("type", "state"), OKV("vap", "total"), ol); + ometric_set_int_with_labels(metric, in->vaps_uniqs, + OKV("type", "state"), OKV("vap", "unique"), ol); + ometric_set_int_with_labels(metric, in->vaps_pas, + OKV("type", "state"), OKV("vap providers", "both"), ol); + ometric_set_int_with_labels(metric, in->vaps_pas4, + OKV("type", "state"), OKV("vap providers", "IPv4 only"), ol); + ometric_set_int_with_labels(metric, in->vaps_pas6, + OKV("type", "state"), OKV("vap providers", "IPv6 only"), ol); +} + +static void +ta_stats(int id) +{ + struct olabels *ol; + const char *keys[2] = { "name", NULL }; + const char *values[2]; + + values[0] = taldescs[id]; + values[1] = NULL; + + ol = olabels_new(keys, values); + set_common_stats(&talstats[id], rpki_ta_obj, ol); + olabels_free(ol); +} + +static void +repo_stats(const struct repo *rp, const struct repostats *in, void *arg) +{ + struct olabels *ol; + const char *keys[4] = { "name", "carepo", "notify", NULL }; + const char *values[4]; + + values[0] = taldescs[repo_talid(rp)]; + repo_fetch_uris(rp, &values[1], &values[2]); + values[3] = NULL; + + ol = olabels_new(keys, values); + set_common_stats(in, rpki_repo_obj, ol); + ometric_set_timespec(rpki_repo_duration, &in->sync_time, ol); + ometric_set_state(rpki_repo_state, repo_states[repo_synced(rp)], ol); + olabels_free(ol); +} + +int +output_ometric(FILE *out, struct vrp_tree *vrps, struct brk_tree *brks, + struct vap_tree *vaps, struct stats *st) +{ + struct olabels *ol; + const char *keys[4] = { "nodename", "domainname", "release", NULL }; + const char *values[4]; + char hostname[HOST_NAME_MAX + 1]; + char *domainname; + struct timespec now_time; + int rv, i; + + rpki_info = ometric_new(OMT_INFO, "rpki_client", + "rpki-client information"); + rpki_completion_time = ometric_new(OMT_GAUGE, + "rpki_client_job_completion_time", + "end of this run as epoch timestamp"); + + rpki_repo = ometric_new(OMT_GAUGE, "rpki_client_repository", + "total number of repositories"); + rpki_obj = ometric_new(OMT_GAUGE, "rpki_client_objects", + "total number of objects"); + + rpki_duration = ometric_new(OMT_GAUGE, "rpki_client_duration", + "duration in seconds"); + + rpki_ta_obj = ometric_new(OMT_GAUGE, "rpki_client_ta_objects", + "total number of objects per TAL"); + rpki_repo_obj = ometric_new(OMT_GAUGE, "rpki_client_repository_objects", + "total number of objects per repository"); + rpki_repo_duration = ometric_new(OMT_GAUGE, + "rpki_client_repository_duration", + "duration used to sync this repository in seconds"); + rpki_repo_state = ometric_new_state(repo_states, + sizeof(repo_states) / sizeof(repo_states[0]), + "rpki_client_repository_state", + "repository state"); + + /* + * Dump statistics + */ + if (gethostname(hostname, sizeof(hostname))) + err(1, "gethostname"); + if ((domainname = strchr(hostname, '.'))) + *domainname++ = '\0'; + + values[0] = hostname; + values[1] = domainname; + values[2] = RPKI_VERSION; + values[3] = NULL; + + ol = olabels_new(keys, values); + ometric_set_info(rpki_info, NULL, NULL, ol); + olabels_free(ol); + + repo_stats_collect(repo_stats, NULL); + for (i = 0; i < talsz; i++) + ta_stats(i); + set_common_stats(&st->repo_stats, rpki_obj, NULL); + + ometric_set_int(rpki_repo, st->repos, NULL); + ometric_set_int_with_labels(rpki_repo, st->rsync_repos, + OKV("type", "state"), OKV("rsync", "synced"), NULL); + ometric_set_int_with_labels(rpki_repo, st->rsync_fails, + OKV("type", "state"), OKV("rsync", "failed"), NULL); + ometric_set_int_with_labels(rpki_repo, st->http_repos, + OKV("type", "state"), OKV("http", "synced"), NULL); + ometric_set_int_with_labels(rpki_repo, st->http_fails, + OKV("type", "state"), OKV("http", "failed"), NULL); + ometric_set_int_with_labels(rpki_repo, st->rrdp_repos, + OKV("type", "state"), OKV("rrdp", "synced"), NULL); + ometric_set_int_with_labels(rpki_repo, st->rrdp_fails, + OKV("type", "state"), OKV("rrdp", "failed"), NULL); + + ometric_set_timespec_with_labels(rpki_duration, &st->elapsed_time, + OKV("type"), OKV("elapsed"), NULL); + ometric_set_timespec_with_labels(rpki_duration, &st->user_time, + OKV("type"), OKV("user"), NULL); + ometric_set_timespec_with_labels(rpki_duration, &st->system_time, + OKV("type"), OKV("system"), NULL); + + clock_gettime(CLOCK_REALTIME, &now_time); + ometric_set_timespec(rpki_completion_time, &now_time, NULL); + + rv = ometric_output_all(out); + ometric_free_all(); + + return rv; +} diff --git a/usr.sbin/rpki-client/output.c b/usr.sbin/rpki-client/output.c index d2a3b13b6fd..79b425ec0a8 100644 --- a/usr.sbin/rpki-client/output.c +++ b/usr.sbin/rpki-client/output.c @@ -1,4 +1,4 @@ -/* $OpenBSD: output.c,v 1.28 2022/11/04 13:01:19 tb Exp $ */ +/* $OpenBSD: output.c,v 1.29 2022/12/15 12:02:29 claudio Exp $ */ /* * Copyright (c) 2019 Theo de Raadt * @@ -73,6 +73,7 @@ static const struct outputs { { FORMAT_BIRD, "bird", output_bird2 }, { FORMAT_CSV, "csv", output_csv }, { FORMAT_JSON, "json", output_json }, + { FORMAT_OMETRIC, "metrics", output_ometric }, { 0, NULL, NULL } }; @@ -213,17 +214,18 @@ outputheader(FILE *out, struct stats *st) if (fprintf(out, "# Generated on host %s at %s\n" "# Processing time %lld seconds (%llds user, %llds system)\n" - "# Route Origin Authorizations: %zu (%zu failed parse, %zu invalid)\n" - "# BGPsec Router Certificates: %zu\n" - "# Certificates: %zu (%zu invalid)\n", + "# Route Origin Authorizations: %u (%u failed parse, %u invalid)\n" + "# BGPsec Router Certificates: %u\n" + "# Certificates: %u (%u invalid)\n", hn, tbuf, (long long)st->elapsed_time.tv_sec, (long long)st->user_time.tv_sec, (long long)st->system_time.tv_sec, - st->roas, st->roas_fail, st->roas_invalid, - st->brks, st->certs, st->certs_fail) < 0) + st->repo_stats.roas, st->repo_stats.roas_fail, + st->repo_stats.roas_invalid, st->repo_stats.brks, + st->repo_stats.certs, st->repo_stats.certs_fail) < 0) return -1; if (fprintf(out, - "# Trust Anchor Locators: %zu (%zu invalid) [", st->tals, + "# Trust Anchor Locators: %u (%u invalid) [", st->tals, talsz - st->tals) < 0) return -1; for (i = 0; i < talsz; i++) @@ -232,16 +234,17 @@ outputheader(FILE *out, struct stats *st) if (fprintf(out, " ]\n" - "# Manifests: %zu (%zu failed parse, %zu stale)\n" - "# Certificate revocation lists: %zu\n" - "# Ghostbuster records: %zu\n" - "# Repositories: %zu\n" - "# VRP Entries: %zu (%zu unique)\n", - st->mfts, st->mfts_fail, st->mfts_stale, - st->crls, - st->gbrs, + "# Manifests: %u (%u failed parse, %u stale)\n" + "# Certificate revocation lists: %u\n" + "# Ghostbuster records: %u\n" + "# Repositories: %u\n" + "# VRP Entries: %u (%u unique)\n", + st->repo_stats.mfts, st->repo_stats.mfts_fail, + st->repo_stats.mfts_stale, + st->repo_stats.crls, + st->repo_stats.gbrs, st->repos, - st->vrps, st->uniqs) < 0) + st->repo_stats.vrps, st->repo_stats.vrps_uniqs) < 0) return -1; return 0; } diff --git a/usr.sbin/rpki-client/parser.c b/usr.sbin/rpki-client/parser.c index a32ebfcc22e..deb09e77afb 100644 --- a/usr.sbin/rpki-client/parser.c +++ b/usr.sbin/rpki-client/parser.c @@ -1,4 +1,4 @@ -/* $OpenBSD: parser.c,v 1.80 2022/11/29 20:26:22 job Exp $ */ +/* $OpenBSD: parser.c,v 1.81 2022/12/15 12:02:29 claudio Exp $ */ /* * Copyright (c) 2019 Claudio Jeker * Copyright (c) 2019 Kristaps Dzonsons @@ -636,6 +636,8 @@ parse_entity(struct entityq *q, struct msgbuf *msgq) switch (entp->type) { case RTYPE_TAL: io_str_buffer(b, entp->file); + io_simple_buffer(b, &entp->repoid, + sizeof(entp->repoid)); if ((tal = tal_parse(entp->file, entp->data, entp->datasz)) == NULL) errx(1, "%s: could not parse tal file", @@ -647,6 +649,8 @@ parse_entity(struct entityq *q, struct msgbuf *msgq) case RTYPE_CER: file = parse_load_file(entp, &f, &flen); io_str_buffer(b, file); + io_simple_buffer(b, &entp->repoid, + sizeof(entp->repoid)); if (entp->data != NULL) cert = proc_parser_root_cert(file, f, flen, entp->data, entp->datasz, @@ -673,10 +677,14 @@ parse_entity(struct entityq *q, struct msgbuf *msgq) file = parse_filepath(entp->repoid, entp->path, entp->file, entp->location); io_str_buffer(b, file); + io_simple_buffer(b, &entp->repoid, + sizeof(entp->repoid)); break; case RTYPE_MFT: file = proc_parser_mft(entp, &mft); io_str_buffer(b, file); + io_simple_buffer(b, &entp->repoid, + sizeof(entp->repoid)); c = (mft != NULL); io_simple_buffer(b, &c, sizeof(int)); if (mft != NULL) @@ -686,6 +694,8 @@ parse_entity(struct entityq *q, struct msgbuf *msgq) case RTYPE_ROA: file = parse_load_file(entp, &f, &flen); io_str_buffer(b, file); + io_simple_buffer(b, &entp->repoid, + sizeof(entp->repoid)); roa = proc_parser_roa(file, f, flen); c = (roa != NULL); io_simple_buffer(b, &c, sizeof(int)); @@ -696,11 +706,15 @@ parse_entity(struct entityq *q, struct msgbuf *msgq) case RTYPE_GBR: file = parse_load_file(entp, &f, &flen); io_str_buffer(b, file); + io_simple_buffer(b, &entp->repoid, + sizeof(entp->repoid)); proc_parser_gbr(file, f, flen); break; case RTYPE_ASPA: file = parse_load_file(entp, &f, &flen); io_str_buffer(b, file); + io_simple_buffer(b, &entp->repoid, + sizeof(entp->repoid)); aspa = proc_parser_aspa(file, f, flen); c = (aspa != NULL); io_simple_buffer(b, &c, sizeof(int)); @@ -711,6 +725,8 @@ parse_entity(struct entityq *q, struct msgbuf *msgq) case RTYPE_TAK: file = parse_load_file(entp, &f, &flen); io_str_buffer(b, file); + io_simple_buffer(b, &entp->repoid, + sizeof(entp->repoid)); proc_parser_tak(file, f, flen); break; default: diff --git a/usr.sbin/rpki-client/repo.c b/usr.sbin/rpki-client/repo.c index 84550a9e7e9..f4a21430f34 100644 --- a/usr.sbin/rpki-client/repo.c +++ b/usr.sbin/rpki-client/repo.c @@ -1,4 +1,4 @@ -/* $OpenBSD: repo.c,v 1.39 2022/09/02 21:56:45 claudio Exp $ */ +/* $OpenBSD: repo.c,v 1.40 2022/12/15 12:02:29 claudio Exp $ */ /* * Copyright (c) 2021 Claudio Jeker * Copyright (c) 2019 Kristaps Dzonsons @@ -97,6 +97,8 @@ struct repo { const struct rsyncrepo *rsync; const struct tarepo *ta; struct entityq queue; /* files waiting for repo */ + struct repostats stats; + struct timespec start_time; time_t alarm; /* sync timeout */ int talid; unsigned int id; /* identifier */ @@ -263,7 +265,7 @@ repo_mkpath(int fd, char *file) * Return the state of a repository. */ static enum repo_state -repo_state(struct repo *rp) +repo_state(const struct repo *rp) { if (rp->ta) return rp->ta->state; @@ -283,24 +285,24 @@ static void repo_done(const void *vp, int ok) { struct repo *rp; + struct timespec flush_time; SLIST_FOREACH(rp, &repos, entry) { - if (vp == rp->ta) - entityq_flush(&rp->queue, rp); - if (vp == rp->rsync) - entityq_flush(&rp->queue, rp); - if (vp == rp->rrdp) { - if (!ok && !nofetch) { - /* try to fall back to rsync */ - rp->rrdp = NULL; - rp->rsync = rsync_get(rp->repouri, - rp->basedir); - /* need to check if it was already loaded */ - if (repo_state(rp) != REPO_LOADING) - entityq_flush(&rp->queue, rp); - } else - entityq_flush(&rp->queue, rp); + if (vp != rp->ta && vp != rp->rsync && vp != rp->rrdp) + continue; + + /* for rrdp try to fall back to rsync */ + if (vp == rp->rrdp && !ok && !nofetch) { + rp->rrdp = NULL; + rp->rsync = rsync_get(rp->repouri, rp->basedir); + /* need to check if it was already loaded */ + if (repo_state(rp) == REPO_LOADING) + continue; } + + entityq_flush(&rp->queue, rp); + clock_gettime(CLOCK_MONOTONIC, &flush_time); + timespecsub(&flush_time, &rp->start_time, &rp->stats.sync_time); } } @@ -600,6 +602,7 @@ repo_alloc(int talid) rp->alarm = getmonotime() + repo_timeout; TAILQ_INIT(&rp->queue); SLIST_INSERT_HEAD(&repos, rp, entry); + clock_gettime(CLOCK_MONOTONIC, &rp->start_time); stats.repos++; return rp; @@ -1179,6 +1182,38 @@ repo_uri(const struct repo *rp) return rp->repouri; } +/* + * Return the repository URI. + */ +void +repo_fetch_uris(const struct repo *rp, const char **carepo, + const char **notifyuri) +{ + *carepo = rp->repouri; + *notifyuri = rp->notifyuri; +} + +/* + * Return 1 if repository is synced else 0. + */ +int +repo_synced(const struct repo *rp) +{ + if (repo_state(rp) == REPO_DONE && + !(rp->rrdp == NULL && rp->rsync == NULL && rp->ta == NULL)) + return 1; + return 0; +} + +/* + * Return the repository tal ID. + */ +int +repo_talid(const struct repo *rp) +{ + return rp->talid; +} + int repo_queued(struct repo *rp, struct entity *p) { @@ -1262,6 +1297,112 @@ repo_check_timeout(int timeout) return timeout; } +/* + * Update stats object of repository depending on rtype and subtype. + */ +void +repo_stat_inc(struct repo *rp, enum rtype type, enum stype subtype) +{ + if (rp == NULL) + return; + switch (type) { + case RTYPE_CER: + if (subtype == STYPE_OK) + rp->stats.certs++; + if (subtype == STYPE_FAIL) + rp->stats.certs_fail++; + if (subtype == STYPE_BGPSEC) { + rp->stats.certs--; + rp->stats.brks++; + } + break; + case RTYPE_MFT: + if (subtype == STYPE_OK) + rp->stats.mfts++; + if (subtype == STYPE_FAIL) + rp->stats.mfts_fail++; + if (subtype == STYPE_STALE) + rp->stats.mfts_stale++; + break; + case RTYPE_ROA: + switch (subtype) { + case STYPE_OK: + rp->stats.roas++; + break; + case STYPE_FAIL: + rp->stats.roas_fail++; + break; + case STYPE_INVALID: + rp->stats.roas_invalid++; + break; + case STYPE_TOTAL: + rp->stats.vrps++; + break; + case STYPE_UNIQUE: + rp->stats.vrps_uniqs++; + break; + case STYPE_DEC_UNIQUE: + rp->stats.vrps_uniqs--; + break; + default: + break; + } + break; + case RTYPE_ASPA: + switch (subtype) { + case STYPE_OK: + rp->stats.aspas++; + break; + case STYPE_FAIL: + rp->stats.aspas_fail++; + break; + case STYPE_INVALID: + rp->stats.aspas_invalid++; + break; + case STYPE_TOTAL: + rp->stats.vaps++; + break; + case STYPE_UNIQUE: + rp->stats.vaps_uniqs++; + break; + case STYPE_BOTH: + rp->stats.vaps_pas++; + break; + case STYPE_ONLY_IPV4: + rp->stats.vaps_pas4++; + break; + case STYPE_ONLY_IPV6: + rp->stats.vaps_pas6++; + break; + default: + break; + } + break; + case RTYPE_CRL: + rp->stats.crls++; + break; + case RTYPE_GBR: + rp->stats.gbrs++; + break; + case RTYPE_TAK: + rp->stats.taks++; + break; + default: + break; + } +} + +void +repo_stats_collect(void (*cb)(const struct repo *, const struct repostats *, + void *), void *arg) +{ + struct repo *rp; + + SLIST_FOREACH(rp, &repos, entry) { + cb(rp, &rp->stats, arg); + } +} + /* * Delayed delete of files from RRDP. Since RRDP has no security built-in * this code needs to check if this RRDP repository is actually allowed to diff --git a/usr.sbin/rpki-client/roa.c b/usr.sbin/rpki-client/roa.c index b2367dc1684..a18daf187d7 100644 --- a/usr.sbin/rpki-client/roa.c +++ b/usr.sbin/rpki-client/roa.c @@ -1,4 +1,4 @@ -/* $OpenBSD: roa.c,v 1.58 2022/11/29 20:41:32 job Exp $ */ +/* $OpenBSD: roa.c,v 1.59 2022/12/15 12:02:29 claudio Exp $ */ /* * Copyright (c) 2022 Theo Buehler * Copyright (c) 2019 Kristaps Dzonsons @@ -360,8 +360,7 @@ roa_read(struct ibuf *b) * number of addresses. */ void -roa_insert_vrps(struct vrp_tree *tree, struct roa *roa, size_t *vrps, - size_t *uniqs) +roa_insert_vrps(struct vrp_tree *tree, struct roa *roa, struct repo *rp) { struct vrp *v, *found; size_t i; @@ -374,6 +373,10 @@ roa_insert_vrps(struct vrp_tree *tree, struct roa *roa, size_t *vrps, v->maxlength = roa->ips[i].maxlength; v->asid = roa->asid; v->talid = roa->talid; + if (rp != NULL) + v->repoid = repo_id(rp); + else + v->repoid = 0; v->expires = roa->expires; /* @@ -387,12 +390,17 @@ roa_insert_vrps(struct vrp_tree *tree, struct roa *roa, size_t *vrps, /* update found with preferred data */ found->talid = v->talid; found->expires = v->expires; + /* adjust unique count */ + repo_stat_inc(repo_byid(found->repoid), + RTYPE_ROA, STYPE_DEC_UNIQUE); + found->repoid = v->repoid; + repo_stat_inc(rp, RTYPE_ROA, STYPE_UNIQUE); } free(v); } else - (*uniqs)++; + repo_stat_inc(rp, RTYPE_ROA, STYPE_UNIQUE); - (*vrps)++; + repo_stat_inc(rp, RTYPE_ROA, STYPE_TOTAL); } } diff --git a/usr.sbin/rpki-client/rpki-client.8 b/usr.sbin/rpki-client/rpki-client.8 index ccdbb114ee1..4c8fdd88706 100644 --- a/usr.sbin/rpki-client/rpki-client.8 +++ b/usr.sbin/rpki-client/rpki-client.8 @@ -1,4 +1,4 @@ -.\" $OpenBSD: rpki-client.8,v 1.81 2022/11/26 12:02:37 job Exp $ +.\" $OpenBSD: rpki-client.8,v 1.82 2022/12/15 12:02:29 claudio Exp $ .\" .\" Copyright (c) 2019 Kristaps Dzonsons .\" @@ -14,7 +14,7 @@ .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. .\" -.Dd $Mdocdate: November 26 2022 $ +.Dd $Mdocdate: December 15 2022 $ .Dt RPKI-CLIENT 8 .Os .Sh NAME @@ -22,7 +22,7 @@ .Nd RPKI validator to support BGP routing security .Sh SYNOPSIS .Nm -.Op Fl BcjnoRrVv +.Op Fl BcjmnoRrVv .Op Fl b Ar sourceaddr .Op Fl d Ar cachedir .Op Fl e Ar rsync_prog @@ -138,6 +138,10 @@ in the output directory as JSON object. See .Fl c for a description of the fields. +.It Fl m +Create output in the file +.Pa metrics +in the output directory in OpenMetrics format. .It Fl n Offline mode. Validate the contents of