From f1da507fdd28f0ef6cf390c0be3a759d812fde8b Mon Sep 17 00:00:00 2001 From: schwarze Date: Sun, 5 Jan 2014 20:26:27 +0000 Subject: [PATCH] Add an option -Q (quick) to mandocdb(8) for accelerated generation of reduced-size databases. Implement this by allowing the parsers to optionally abort the parse sequence after the NAME section. While here, garbage collect the unused void *arg attribute of struct mparse and mparse_alloc(). This reduces the processing time of mandocdb(8) on /usr/share/man by a factor of 2 and the database size by a factor of 4. However, it still takes 5 times the time and 6 times the space of makewhatis(8), so more work is clearly needed. --- usr.bin/mandoc/libman.h | 3 ++- usr.bin/mandoc/libmandoc.h | 6 +++--- usr.bin/mandoc/libmdoc.h | 3 ++- usr.bin/mandoc/main.c | 4 ++-- usr.bin/mandoc/man.c | 12 ++++++++++-- usr.bin/mandoc/mandoc.h | 4 ++-- usr.bin/mandoc/mandocdb.c | 20 ++++++++++++-------- usr.bin/mandoc/mdoc.c | 14 +++++++++++--- usr.bin/mandoc/read.c | 20 +++++++++++--------- 9 files changed, 55 insertions(+), 31 deletions(-) diff --git a/usr.bin/mandoc/libman.h b/usr.bin/mandoc/libman.h index fcff91d550b..45a78bc344c 100644 --- a/usr.bin/mandoc/libman.h +++ b/usr.bin/mandoc/libman.h @@ -1,4 +1,4 @@ -/* $Id: libman.h,v 1.32 2012/11/17 00:25:20 schwarze Exp $ */ +/* $Id: libman.h,v 1.33 2014/01/05 20:26:27 schwarze Exp $ */ /* * Copyright (c) 2009, 2010, 2011 Kristaps Dzonsons * @@ -24,6 +24,7 @@ enum man_next { struct man { struct mparse *parse; /* parse pointer */ + int quick; /* abort parse early */ int flags; /* parse flags */ #define MAN_HALT (1 << 0) /* badness happened: die */ #define MAN_ELINE (1 << 1) /* Next-line element scope. */ diff --git a/usr.bin/mandoc/libmandoc.h b/usr.bin/mandoc/libmandoc.h index cb6162a985f..6ca80c1e08a 100644 --- a/usr.bin/mandoc/libmandoc.h +++ b/usr.bin/mandoc/libmandoc.h @@ -1,4 +1,4 @@ -/* $Id: libmandoc.h,v 1.23 2013/12/30 00:52:18 schwarze Exp $ */ +/* $Id: libmandoc.h,v 1.24 2014/01/05 20:26:27 schwarze Exp $ */ /* * Copyright (c) 2009, 2010, 2011, 2012 Kristaps Dzonsons * Copyright (c) 2013 Ingo Schwarze @@ -47,7 +47,7 @@ int mandoc_strntoi(const char *, size_t, int); const char *mandoc_a2msec(const char*); void mdoc_free(struct mdoc *); -struct mdoc *mdoc_alloc(struct roff *, struct mparse *, char *); +struct mdoc *mdoc_alloc(struct roff *, struct mparse *, char *, int); void mdoc_reset(struct mdoc *); int mdoc_parseln(struct mdoc *, int, char *, int); int mdoc_endparse(struct mdoc *); @@ -55,7 +55,7 @@ int mdoc_addspan(struct mdoc *, const struct tbl_span *); int mdoc_addeqn(struct mdoc *, const struct eqn *); void man_free(struct man *); -struct man *man_alloc(struct roff *, struct mparse *); +struct man *man_alloc(struct roff *, struct mparse *, int); void man_reset(struct man *); int man_parseln(struct man *, int, char *, int); int man_endparse(struct man *); diff --git a/usr.bin/mandoc/libmdoc.h b/usr.bin/mandoc/libmdoc.h index 1a51844470c..cd74739dfb8 100644 --- a/usr.bin/mandoc/libmdoc.h +++ b/usr.bin/mandoc/libmdoc.h @@ -1,4 +1,4 @@ -/* $Id: libmdoc.h,v 1.52 2013/10/21 23:32:32 schwarze Exp $ */ +/* $Id: libmdoc.h,v 1.53 2014/01/05 20:26:27 schwarze Exp $ */ /* * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons * Copyright (c) 2013 Ingo Schwarze @@ -26,6 +26,7 @@ enum mdoc_next { struct mdoc { struct mparse *parse; /* parse pointer */ char *defos; /* default argument for .Os */ + int quick; /* abort parse early */ int flags; /* parse flags */ #define MDOC_HALT (1 << 0) /* error in parse: halt */ #define MDOC_LITERAL (1 << 1) /* in a literal scope */ diff --git a/usr.bin/mandoc/main.c b/usr.bin/mandoc/main.c index 0e7195ef7b1..1e0f0d38388 100644 --- a/usr.bin/mandoc/main.c +++ b/usr.bin/mandoc/main.c @@ -1,4 +1,4 @@ -/* $Id: main.c,v 1.84 2012/11/19 08:46:24 jmc Exp $ */ +/* $Id: main.c,v 1.85 2014/01/05 20:26:27 schwarze Exp $ */ /* * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons * Copyright (c) 2010, 2011, 2012 Ingo Schwarze @@ -141,7 +141,7 @@ main(int argc, char *argv[]) /* NOTREACHED */ } - curp.mp = mparse_alloc(type, curp.wlevel, mmsg, &curp, defos); + curp.mp = mparse_alloc(type, curp.wlevel, mmsg, defos, 0); /* * Conditionally start up the lookaside buffer before parsing. diff --git a/usr.bin/mandoc/man.c b/usr.bin/mandoc/man.c index 932590223b9..867e226645e 100644 --- a/usr.bin/mandoc/man.c +++ b/usr.bin/mandoc/man.c @@ -1,6 +1,7 @@ -/* $Id: man.c,v 1.72 2013/12/30 00:52:18 schwarze Exp $ */ +/* $Id: man.c,v 1.73 2014/01/05 20:26:27 schwarze Exp $ */ /* * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons + * Copyright (c) 2013, 2014 Ingo Schwarze * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -93,7 +94,7 @@ man_free(struct man *man) struct man * -man_alloc(struct roff *roff, struct mparse *parse) +man_alloc(struct roff *roff, struct mparse *parse, int quick) { struct man *p; @@ -101,6 +102,7 @@ man_alloc(struct roff *roff, struct mparse *parse) man_hash_init(); p->parse = parse; + p->quick = quick; p->roff = roff; man_alloc1(p); @@ -600,6 +602,12 @@ man_pmacro(struct man *man, int ln, char *buf, int offs) if ( ! (*man_macros[tok].fp)(man, tok, ln, ppos, &offs, buf)) goto err; + /* In quick mode (for mandocdb), abort after the NAME section. */ + + if (man->quick && MAN_SH == tok && + strcmp(man->last->prev->child->string, "NAME")) + return(2); + /* * We weren't in a block-line scope when entering the * above-parsed macro, so return. diff --git a/usr.bin/mandoc/mandoc.h b/usr.bin/mandoc/mandoc.h index 1dcce57be28..9a3362a9c8d 100644 --- a/usr.bin/mandoc/mandoc.h +++ b/usr.bin/mandoc/mandoc.h @@ -1,4 +1,4 @@ -/* $Id: mandoc.h,v 1.57 2014/01/02 16:29:46 schwarze Exp $ */ +/* $Id: mandoc.h,v 1.58 2014/01/05 20:26:27 schwarze Exp $ */ /* * Copyright (c) 2010, 2011 Kristaps Dzonsons * Copyright (c) 2012, 2013, 2014 Ingo Schwarze @@ -422,7 +422,7 @@ int mchars_spec2cp(const struct mchars *, const char *mchars_spec2str(const struct mchars *, const char *, size_t, size_t *); struct mparse *mparse_alloc(enum mparset, enum mandoclevel, - mandocmsg, void *, char *); + mandocmsg, char *, int); void mparse_free(struct mparse *); void mparse_keep(struct mparse *); enum mandoclevel mparse_readfd(struct mparse *, int, const char *); diff --git a/usr.bin/mandoc/mandocdb.c b/usr.bin/mandoc/mandocdb.c index 0e37ea9cb57..eb239764af4 100644 --- a/usr.bin/mandoc/mandocdb.c +++ b/usr.bin/mandoc/mandocdb.c @@ -1,4 +1,4 @@ -/* $Id: mandocdb.c,v 1.58 2014/01/05 04:48:35 schwarze Exp $ */ +/* $Id: mandocdb.c,v 1.59 2014/01/05 20:26:27 schwarze Exp $ */ /* * Copyright (c) 2011, 2012 Kristaps Dzonsons * Copyright (c) 2011, 2012, 2013, 2014 Ingo Schwarze @@ -159,8 +159,9 @@ static int treescan(void); static size_t utf8(unsigned int, char [7]); static char *progname; -static int use_all; /* use all found files */ static int nodb; /* no database changes */ +static int quick; /* abort the parse early */ +static int use_all; /* use all found files */ static int verb; /* print what we're doing */ static int warnings; /* warn about crap */ static int write_utf8; /* write UTF-8 output; else ASCII */ @@ -339,7 +340,7 @@ mandocdb(int argc, char *argv[]) path_arg = NULL; op = OP_DEFAULT; - while (-1 != (ch = getopt(argc, argv, "aC:d:nT:tu:vW"))) + while (-1 != (ch = getopt(argc, argv, "aC:d:nQT:tu:vW"))) switch (ch) { case ('a'): use_all = 1; @@ -357,6 +358,9 @@ mandocdb(int argc, char *argv[]) case ('n'): nodb = 1; break; + case ('Q'): + quick = 1; + break; case ('T'): if (strcmp(optarg, "utf8")) { fprintf(stderr, "-T%s: Unsupported " @@ -396,7 +400,7 @@ mandocdb(int argc, char *argv[]) exitcode = (int)MANDOCLEVEL_OK; mp = mparse_alloc(MPARSE_AUTO, - MANDOCLEVEL_FATAL, NULL, NULL, NULL); + MANDOCLEVEL_FATAL, NULL, NULL, quick); mc = mchars_alloc(); ohash_init(&mpages, 6, &mpages_info); @@ -486,11 +490,11 @@ out: ohash_delete(&mlinks); return(exitcode); usage: - fprintf(stderr, "usage: %s [-anvW] [-C file] [-Tutf8]\n" - " %s [-anvW] [-Tutf8] dir ...\n" - " %s [-nvW] [-Tutf8] -d dir [file ...]\n" + fprintf(stderr, "usage: %s [-anQvW] [-C file] [-Tutf8]\n" + " %s [-anQvW] [-Tutf8] dir ...\n" + " %s [-nQvW] [-Tutf8] -d dir [file ...]\n" " %s [-nvW] -u dir [file ...]\n" - " %s -t file ...\n", + " %s [-Q] -t file ...\n", progname, progname, progname, progname, progname); diff --git a/usr.bin/mandoc/mdoc.c b/usr.bin/mandoc/mdoc.c index 40595d95729..3c4370aa29c 100644 --- a/usr.bin/mandoc/mdoc.c +++ b/usr.bin/mandoc/mdoc.c @@ -1,7 +1,7 @@ -/* $Id: mdoc.c,v 1.97 2013/12/30 00:52:18 schwarze Exp $ */ +/* $Id: mdoc.c,v 1.98 2014/01/05 20:26:27 schwarze Exp $ */ /* * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons - * Copyright (c) 2010, 2012, 2013 Ingo Schwarze + * Copyright (c) 2010, 2012, 2013, 2014 Ingo Schwarze * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -193,7 +193,8 @@ mdoc_free(struct mdoc *mdoc) * Allocate volatile and non-volatile parse resources. */ struct mdoc * -mdoc_alloc(struct roff *roff, struct mparse *parse, char *defos) +mdoc_alloc(struct roff *roff, struct mparse *parse, + char *defos, int quick) { struct mdoc *p; @@ -201,6 +202,7 @@ mdoc_alloc(struct roff *roff, struct mparse *parse, char *defos) p->parse = parse; p->defos = defos; + p->quick = quick; p->roff = roff; mdoc_hash_init(); @@ -957,6 +959,12 @@ mdoc_pmacro(struct mdoc *mdoc, int ln, char *buf, int offs) if ( ! mdoc_macro(mdoc, tok, ln, sv, &offs, buf)) goto err; + /* In quick mode (for mandocdb), abort after the NAME section. */ + + if (mdoc->quick && MDOC_Sh == tok && + SEC_NAME != mdoc->last->sec) + return(2); + return(1); err: /* Error out. */ diff --git a/usr.bin/mandoc/read.c b/usr.bin/mandoc/read.c index 370f8cd525b..b0ef73ed895 100644 --- a/usr.bin/mandoc/read.c +++ b/usr.bin/mandoc/read.c @@ -1,4 +1,4 @@ -/* $Id: read.c,v 1.18 2014/01/02 16:29:46 schwarze Exp $ */ +/* $Id: read.c,v 1.19 2014/01/05 20:26:27 schwarze Exp $ */ /* * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons * Copyright (c) 2010-2014 Ingo Schwarze @@ -52,10 +52,10 @@ struct mparse { struct roff *roff; /* roff parser (!NULL) */ int reparse_count; /* finite interp. stack */ mandocmsg mmsg; /* warning/error message handler */ - void *arg; /* argument to mmsg */ const char *file; struct buf *secondary; char *defos; /* default operating system */ + int quick; /* abort the parse early */ }; static void resize_buf(struct buf *, size_t); @@ -250,13 +250,14 @@ pset(const char *buf, int pos, struct mparse *curp) case (MPARSE_MDOC): if (NULL == curp->pmdoc) curp->pmdoc = mdoc_alloc(curp->roff, curp, - curp->defos); + curp->defos, curp->quick); assert(curp->pmdoc); curp->mdoc = curp->pmdoc; return; case (MPARSE_MAN): if (NULL == curp->pman) - curp->pman = man_alloc(curp->roff, curp); + curp->pman = man_alloc(curp->roff, curp, + curp->quick); assert(curp->pman); curp->man = curp->pman; return; @@ -267,14 +268,14 @@ pset(const char *buf, int pos, struct mparse *curp) if (pos >= 3 && 0 == memcmp(buf, ".Dd", 3)) { if (NULL == curp->pmdoc) curp->pmdoc = mdoc_alloc(curp->roff, curp, - curp->defos); + curp->defos, curp->quick); assert(curp->pmdoc); curp->mdoc = curp->pmdoc; return; } if (NULL == curp->pman) - curp->pman = man_alloc(curp->roff, curp); + curp->pman = man_alloc(curp->roff, curp, curp->quick); assert(curp->pman); curp->man = curp->pman; } @@ -552,7 +553,8 @@ rerun: if (0 == rc) { assert(MANDOCLEVEL_FATAL <= curp->file_status); break; - } + } else if (2 == rc) + break; /* Temporary buffers typically are not full. */ @@ -738,7 +740,7 @@ out: struct mparse * mparse_alloc(enum mparset inttype, enum mandoclevel wlevel, - mandocmsg mmsg, void *arg, char *defos) + mandocmsg mmsg, char *defos, int quick) { struct mparse *curp; @@ -748,9 +750,9 @@ mparse_alloc(enum mparset inttype, enum mandoclevel wlevel, curp->wlevel = wlevel; curp->mmsg = mmsg; - curp->arg = arg; curp->inttype = inttype; curp->defos = defos; + curp->quick = quick; curp->roff = roff_alloc(inttype, curp); return(curp); -- 2.20.1