Add an option -Q (quick) to mandocdb(8)

for accelerated generation of reduced-size databases.

Implement this by allowing the parsers to optionally
abort the parse sequence after the NAME section.

While here, garbage collect the unused void *arg attribute
of struct mparse and mparse_alloc().

This reduces the processing time of mandocdb(8) on /usr/share/man
by a factor of 2 and the database size by a factor of 4.
However, it still takes 5 times the time and 6 times the space
of makewhatis(8), so more work is clearly needed.

diff --git a/usr.bin/mandoc/libman.h b/usr.bin/mandoc/libman.h
index fcff91d..45a78bc 100644 (file)
--- a/usr.bin/mandoc/libman.h
+++ b/usr.bin/mandoc/libman.h
@@ -1,4 +1,4 @@
-/*     $Id: libman.h,v 1.32 2012/11/17 00:25:20 schwarze Exp $ */
+/*     $Id: libman.h,v 1.33 2014/01/05 20:26:27 schwarze Exp $ */
 /*
  * Copyright (c) 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  *
@@ -24,6 +24,7 @@ enum  man_next {
 
 struct man {
        struct mparse   *parse; /* parse pointer */
+       int              quick; /* abort parse early */
        int              flags; /* parse flags */
 #define        MAN_HALT        (1 << 0) /* badness happened: die */
 #define        MAN_ELINE       (1 << 1) /* Next-line element scope. */

diff --git a/usr.bin/mandoc/libmandoc.h b/usr.bin/mandoc/libmandoc.h
index cb6162a..6ca80c1 100644 (file)
--- a/usr.bin/mandoc/libmandoc.h
+++ b/usr.bin/mandoc/libmandoc.h
@@ -1,4 +1,4 @@
-/*     $Id: libmandoc.h,v 1.23 2013/12/30 00:52:18 schwarze Exp $ */
+/*     $Id: libmandoc.h,v 1.24 2014/01/05 20:26:27 schwarze Exp $ */
 /*
  * Copyright (c) 2009, 2010, 2011, 2012 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2013 Ingo Schwarze <schwarze@openbsd.org>
@@ -47,7 +47,7 @@ int            mandoc_strntoi(const char *, size_t, int);
 const char     *mandoc_a2msec(const char*);
 
 void            mdoc_free(struct mdoc *);
-struct mdoc    *mdoc_alloc(struct roff *, struct mparse *, char *);
+struct mdoc    *mdoc_alloc(struct roff *, struct mparse *, char *, int);
 void            mdoc_reset(struct mdoc *);
 int             mdoc_parseln(struct mdoc *, int, char *, int);
 int             mdoc_endparse(struct mdoc *);
@@ -55,7 +55,7 @@ int            mdoc_addspan(struct mdoc *, const struct tbl_span *);
 int             mdoc_addeqn(struct mdoc *, const struct eqn *);
 
 void            man_free(struct man *);
-struct man     *man_alloc(struct roff *, struct mparse *);
+struct man     *man_alloc(struct roff *, struct mparse *, int);
 void            man_reset(struct man *);
 int             man_parseln(struct man *, int, char *, int);
 int             man_endparse(struct man *);

diff --git a/usr.bin/mandoc/libmdoc.h b/usr.bin/mandoc/libmdoc.h
index 1a51844..cd74739 100644 (file)
--- a/usr.bin/mandoc/libmdoc.h
+++ b/usr.bin/mandoc/libmdoc.h
@@ -1,4 +1,4 @@
-/*     $Id: libmdoc.h,v 1.52 2013/10/21 23:32:32 schwarze Exp $ */
+/*     $Id: libmdoc.h,v 1.53 2014/01/05 20:26:27 schwarze Exp $ */
 /*
  * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2013 Ingo Schwarze <schwarze@openbsd.org>
@@ -26,6 +26,7 @@ enum  mdoc_next {
 struct mdoc {
        struct mparse    *parse; /* parse pointer */
        char             *defos; /* default argument for .Os */
+       int               quick; /* abort parse early */
        int               flags; /* parse flags */
 #define        MDOC_HALT        (1 << 0) /* error in parse: halt */
 #define        MDOC_LITERAL     (1 << 1) /* in a literal scope */

diff --git a/usr.bin/mandoc/main.c b/usr.bin/mandoc/main.c
index 0e7195e..1e0f0d3 100644 (file)
--- a/usr.bin/mandoc/main.c
+++ b/usr.bin/mandoc/main.c
@@ -1,4 +1,4 @@
-/*     $Id: main.c,v 1.84 2012/11/19 08:46:24 jmc Exp $ */
+/*     $Id: main.c,v 1.85 2014/01/05 20:26:27 schwarze Exp $ */
 /*
  * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2010, 2011, 2012 Ingo Schwarze <schwarze@openbsd.org>
@@ -141,7 +141,7 @@ main(int argc, char *argv[])
                        /* NOTREACHED */
                }
 
-       curp.mp = mparse_alloc(type, curp.wlevel, mmsg, &curp, defos);
+       curp.mp = mparse_alloc(type, curp.wlevel, mmsg, defos, 0);
 
        /*
         * Conditionally start up the lookaside buffer before parsing.

diff --git a/usr.bin/mandoc/man.c b/usr.bin/mandoc/man.c
index 9325902..867e226 100644 (file)
--- a/usr.bin/mandoc/man.c
+++ b/usr.bin/mandoc/man.c
@@ -1,6 +1,7 @@
-/*     $Id: man.c,v 1.72 2013/12/30 00:52:18 schwarze Exp $ */
+/*     $Id: man.c,v 1.73 2014/01/05 20:26:27 schwarze Exp $ */
 /*
  * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
+ * Copyright (c) 2013, 2014 Ingo Schwarze <schwarze@openbsd.org>
  *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -93,7 +94,7 @@ man_free(struct man *man)
 
 
 struct man *
-man_alloc(struct roff *roff, struct mparse *parse)
+man_alloc(struct roff *roff, struct mparse *parse, int quick)
 {
        struct man      *p;
 
@@ -101,6 +102,7 @@ man_alloc(struct roff *roff, struct mparse *parse)
 
        man_hash_init();
        p->parse = parse;
+       p->quick = quick;
        p->roff = roff;
 
        man_alloc1(p);
@@ -600,6 +602,12 @@ man_pmacro(struct man *man, int ln, char *buf, int offs)
        if ( ! (*man_macros[tok].fp)(man, tok, ln, ppos, &offs, buf))
                goto err;
 
+       /* In quick mode (for mandocdb), abort after the NAME section. */
+
+       if (man->quick && MAN_SH == tok &&
+           strcmp(man->last->prev->child->string, "NAME"))
+               return(2);
+
        /* 
         * We weren't in a block-line scope when entering the
         * above-parsed macro, so return.

diff --git a/usr.bin/mandoc/mandoc.h b/usr.bin/mandoc/mandoc.h
index 1dcce57..9a3362a 100644 (file)
--- a/usr.bin/mandoc/mandoc.h
+++ b/usr.bin/mandoc/mandoc.h
@@ -1,4 +1,4 @@
-/*     $Id: mandoc.h,v 1.57 2014/01/02 16:29:46 schwarze Exp $ */
+/*     $Id: mandoc.h,v 1.58 2014/01/05 20:26:27 schwarze Exp $ */
 /*
  * Copyright (c) 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2012, 2013, 2014 Ingo Schwarze <schwarze@openbsd.org>
@@ -422,7 +422,7 @@ int           mchars_spec2cp(const struct mchars *,
 const char      *mchars_spec2str(const struct mchars *, 
                        const char *, size_t, size_t *);
 struct mparse   *mparse_alloc(enum mparset, enum mandoclevel,
-                       mandocmsg, void *, char *);
+                       mandocmsg, char *, int);
 void             mparse_free(struct mparse *);
 void             mparse_keep(struct mparse *);
 enum mandoclevel  mparse_readfd(struct mparse *, int, const char *);

diff --git a/usr.bin/mandoc/mandocdb.c b/usr.bin/mandoc/mandocdb.c
index 0e37ea9..eb23976 100644 (file)
--- a/usr.bin/mandoc/mandocdb.c
+++ b/usr.bin/mandoc/mandocdb.c
@@ -1,4 +1,4 @@
-/*     $Id: mandocdb.c,v 1.58 2014/01/05 04:48:35 schwarze Exp $ */
+/*     $Id: mandocdb.c,v 1.59 2014/01/05 20:26:27 schwarze Exp $ */
 /*
  * Copyright (c) 2011, 2012 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2011, 2012, 2013, 2014 Ingo Schwarze <schwarze@openbsd.org>
@@ -159,8 +159,9 @@ static      int      treescan(void);
 static size_t   utf8(unsigned int, char [7]);
 
 static char            *progname;
-static int              use_all; /* use all found files */
 static int              nodb; /* no database changes */
+static int              quick; /* abort the parse early */
+static int              use_all; /* use all found files */
 static int              verb; /* print what we're doing */
 static int              warnings; /* warn about crap */
 static int              write_utf8; /* write UTF-8 output; else ASCII */
@@ -339,7 +340,7 @@ mandocdb(int argc, char *argv[])
        path_arg = NULL;
        op = OP_DEFAULT;
 
-       while (-1 != (ch = getopt(argc, argv, "aC:d:nT:tu:vW")))
+       while (-1 != (ch = getopt(argc, argv, "aC:d:nQT:tu:vW")))
                switch (ch) {
                case ('a'):
                        use_all = 1;
@@ -357,6 +358,9 @@ mandocdb(int argc, char *argv[])
                case ('n'):
                        nodb = 1;
                        break;
+               case ('Q'):
+                       quick = 1;
+                       break;
                case ('T'):
                        if (strcmp(optarg, "utf8")) {
                                fprintf(stderr, "-T%s: Unsupported "
@@ -396,7 +400,7 @@ mandocdb(int argc, char *argv[])
 
        exitcode = (int)MANDOCLEVEL_OK;
        mp = mparse_alloc(MPARSE_AUTO, 
-               MANDOCLEVEL_FATAL, NULL, NULL, NULL);
+               MANDOCLEVEL_FATAL, NULL, NULL, quick);
        mc = mchars_alloc();
 
        ohash_init(&mpages, 6, &mpages_info);
@@ -486,11 +490,11 @@ out:
        ohash_delete(&mlinks);
        return(exitcode);
 usage:
-       fprintf(stderr, "usage: %s [-anvW] [-C file] [-Tutf8]\n"
-                       "       %s [-anvW] [-Tutf8] dir ...\n"
-                       "       %s [-nvW] [-Tutf8] -d dir [file ...]\n"
+       fprintf(stderr, "usage: %s [-anQvW] [-C file] [-Tutf8]\n"
+                       "       %s [-anQvW] [-Tutf8] dir ...\n"
+                       "       %s [-nQvW] [-Tutf8] -d dir [file ...]\n"
                        "       %s [-nvW] -u dir [file ...]\n"
-                       "       %s -t file ...\n",
+                       "       %s [-Q] -t file ...\n",
                       progname, progname, progname, 
                       progname, progname);
 

diff --git a/usr.bin/mandoc/mdoc.c b/usr.bin/mandoc/mdoc.c
index 40595d9..3c4370a 100644 (file)
--- a/usr.bin/mandoc/mdoc.c
+++ b/usr.bin/mandoc/mdoc.c
@@ -1,7 +1,7 @@
-/*     $Id: mdoc.c,v 1.97 2013/12/30 00:52:18 schwarze Exp $ */
+/*     $Id: mdoc.c,v 1.98 2014/01/05 20:26:27 schwarze Exp $ */
 /*
  * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
- * Copyright (c) 2010, 2012, 2013 Ingo Schwarze <schwarze@openbsd.org>
+ * Copyright (c) 2010, 2012, 2013, 2014 Ingo Schwarze <schwarze@openbsd.org>
  *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -193,7 +193,8 @@ mdoc_free(struct mdoc *mdoc)
  * Allocate volatile and non-volatile parse resources.  
  */
 struct mdoc *
-mdoc_alloc(struct roff *roff, struct mparse *parse, char *defos)
+mdoc_alloc(struct roff *roff, struct mparse *parse,
+       char *defos, int quick)
 {
        struct mdoc     *p;
 
@@ -201,6 +202,7 @@ mdoc_alloc(struct roff *roff, struct mparse *parse, char *defos)
 
        p->parse = parse;
        p->defos = defos;
+       p->quick = quick;
        p->roff = roff;
 
        mdoc_hash_init();
@@ -957,6 +959,12 @@ mdoc_pmacro(struct mdoc *mdoc, int ln, char *buf, int offs)
        if ( ! mdoc_macro(mdoc, tok, ln, sv, &offs, buf)) 
                goto err;
 
+       /* In quick mode (for mandocdb), abort after the NAME section. */
+
+       if (mdoc->quick && MDOC_Sh == tok &&
+           SEC_NAME != mdoc->last->sec)
+               return(2);
+
        return(1);
 
 err:   /* Error out. */

diff --git a/usr.bin/mandoc/read.c b/usr.bin/mandoc/read.c
index 370f8cd..b0ef73e 100644 (file)
--- a/usr.bin/mandoc/read.c
+++ b/usr.bin/mandoc/read.c
@@ -1,4 +1,4 @@
-/*     $Id: read.c,v 1.18 2014/01/02 16:29:46 schwarze Exp $ */
+/*     $Id: read.c,v 1.19 2014/01/05 20:26:27 schwarze Exp $ */
 /*
  * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2010-2014 Ingo Schwarze <schwarze@openbsd.org>
@@ -52,10 +52,10 @@ struct      mparse {
        struct roff      *roff; /* roff parser (!NULL) */
        int               reparse_count; /* finite interp. stack */
        mandocmsg         mmsg; /* warning/error message handler */
-       void             *arg; /* argument to mmsg */
        const char       *file; 
        struct buf       *secondary;
        char             *defos; /* default operating system */
+       int               quick; /* abort the parse early */
 };
 
 static void      resize_buf(struct buf *, size_t);
@@ -250,13 +250,14 @@ pset(const char *buf, int pos, struct mparse *curp)
        case (MPARSE_MDOC):
                if (NULL == curp->pmdoc) 
                        curp->pmdoc = mdoc_alloc(curp->roff, curp,
-                                       curp->defos);
+                                       curp->defos, curp->quick);
                assert(curp->pmdoc);
                curp->mdoc = curp->pmdoc;
                return;
        case (MPARSE_MAN):
                if (NULL == curp->pman) 
-                       curp->pman = man_alloc(curp->roff, curp);
+                       curp->pman = man_alloc(curp->roff, curp,
+                                       curp->quick);
                assert(curp->pman);
                curp->man = curp->pman;
                return;
@@ -267,14 +268,14 @@ pset(const char *buf, int pos, struct mparse *curp)
        if (pos >= 3 && 0 == memcmp(buf, ".Dd", 3))  {
                if (NULL == curp->pmdoc) 
                        curp->pmdoc = mdoc_alloc(curp->roff, curp,
-                                       curp->defos);
+                                       curp->defos, curp->quick);
                assert(curp->pmdoc);
                curp->mdoc = curp->pmdoc;
                return;
        } 
 
        if (NULL == curp->pman) 
-               curp->pman = man_alloc(curp->roff, curp);
+               curp->pman = man_alloc(curp->roff, curp, curp->quick);
        assert(curp->pman);
        curp->man = curp->pman;
 }
@@ -552,7 +553,8 @@ rerun:
                if (0 == rc) {
                        assert(MANDOCLEVEL_FATAL <= curp->file_status);
                        break;
-               }
+               } else if (2 == rc)
+                       break;
 
                /* Temporary buffers typically are not full. */
 
@@ -738,7 +740,7 @@ out:
 
 struct mparse *
 mparse_alloc(enum mparset inttype, enum mandoclevel wlevel,
-               mandocmsg mmsg, void *arg, char *defos)
+               mandocmsg mmsg, char *defos, int quick)
 {
        struct mparse   *curp;
 
@@ -748,9 +750,9 @@ mparse_alloc(enum mparset inttype, enum mandoclevel wlevel,
 
        curp->wlevel = wlevel;
        curp->mmsg = mmsg;
-       curp->arg = arg;
        curp->inttype = inttype;
        curp->defos = defos;
+       curp->quick = quick;
 
        curp->roff = roff_alloc(inttype, curp);
        return(curp);