Distinguish between escape sequences that produce no output

whatsoever (for example \fR) and escape sequences that produce
invisible zero-width output (for example \&).  No, i'm not joking,
groff does make that distinction, and it has consequences in some
situations, for example for vertical spacing in no-fill mode.
Heirloom and Plan 9 behaviour is subtly different, but in case of
doubt, we want to follow groff.

While this fixes the behaviour for the majority of escape sequences,
in particular for those most likely to occur in practice, it is not
perfect yet because some of the more exotic ESCAPE_IGNORE sequences
are actually of the "no output whatsoever" type but treated
as "invisible zero-width" for now.  With the new ASCII_NBRZW mechanism
in place, switching them over one by one when the need arises will
no longer be very difficult.

diff --git a/regress/usr.bin/mandoc/mdoc/Bd/blank.in b/regress/usr.bin/mandoc/mdoc/Bd/blank.in
index c7e9b54..fb06256 100644 (file)
--- a/regress/usr.bin/mandoc/mdoc/Bd/blank.in
+++ b/regress/usr.bin/mandoc/mdoc/Bd/blank.in
@@ -1,5 +1,5 @@
-.\" $OpenBSD: blank.in,v 1.6 2017/07/04 14:53:24 schwarze Exp $
-.Dd $Mdocdate: July 4 2017 $
+.\" $OpenBSD: blank.in,v 1.7 2022/08/15 17:59:00 schwarze Exp $
+.Dd $Mdocdate: August 15 2022 $
 .Dt BD-BLANK 1
 .Os
 .Sh NAME
@@ -15,6 +15,14 @@ line containing two blank characters:
   
 line containing space tab space:
         
+line containing a zero-width space:
+\&
+line containing an invalid numbered character escape:
+\N'257'
+line containing an invalid named character escape:
+\[foobar]
+line containing a font escape:
+\fR
 line starting with a blank character:
  x
 line starting with two blank characters:

diff --git a/regress/usr.bin/mandoc/mdoc/Bd/blank.out_ascii b/regress/usr.bin/mandoc/mdoc/Bd/blank.out_ascii
index 116063c..8d3becf 100644 (file)
--- a/regress/usr.bin/mandoc/mdoc/Bd/blank.out_ascii
+++ b/regress/usr.bin/mandoc/mdoc/Bd/blank.out_ascii
@@ -12,6 +12,13 @@ D\bDE\bES\bSC\bCR\bRI\bIP\bPT\bTI\bIO\bON\bN
 
      line containing space tab space:
 
+     line containing a zero-width space:
+
+     line containing an invalid numbered character escape:
+
+     line containing an invalid named character escape:
+
+     line containing a font escape:
      line starting with a blank character:
       x
      line starting with two blank characters:

diff --git a/regress/usr.bin/mandoc/mdoc/Bd/blank.out_lint b/regress/usr.bin/mandoc/mdoc/Bd/blank.out_lint
index b261432..22643d7 100644 (file)
--- a/regress/usr.bin/mandoc/mdoc/Bd/blank.out_lint
+++ b/regress/usr.bin/mandoc/mdoc/Bd/blank.out_lint
@@ -1,8 +1,9 @@
 mandoc: blank.in:13:1: STYLE: whitespace at end of input line
 mandoc: blank.in:15:1: STYLE: whitespace at end of input line
 mandoc: blank.in:17:1: STYLE: whitespace at end of input line
-mandoc: blank.in:22:36: STYLE: whitespace at end of input line
-mandoc: blank.in:23:37: STYLE: whitespace at end of input line
-mandoc: blank.in:24:32: STYLE: whitespace at end of input line
-mandoc: blank.in:31:8: STYLE: whitespace at end of input line
-mandoc: blank.in:31:2: WARNING: empty block: Dl
+mandoc: blank.in:23:1: ERROR: unknown special character: \[foobar]
+mandoc: blank.in:30:36: STYLE: whitespace at end of input line
+mandoc: blank.in:31:37: STYLE: whitespace at end of input line
+mandoc: blank.in:32:32: STYLE: whitespace at end of input line
+mandoc: blank.in:39:8: STYLE: whitespace at end of input line
+mandoc: blank.in:39:2: WARNING: empty block: Dl

diff --git a/regress/usr.bin/mandoc/mdoc/Bd/blank.out_markdown b/regress/usr.bin/mandoc/mdoc/Bd/blank.out_markdown
index 215d045..8463c17 100644 (file)
--- a/regress/usr.bin/mandoc/mdoc/Bd/blank.out_markdown
+++ b/regress/usr.bin/mandoc/mdoc/Bd/blank.out_markdown
@@ -14,6 +14,14 @@ BD-BLANK(1) - General Commands Manual
        
        line containing space tab space:
        
+       line containing a zero-width space:
+       
+       line containing an invalid numbered character escape:
+       <?>
+       line containing an invalid named character escape:
+       <?>
+       line containing a font escape:
+       
        line starting with a blank character:
         x
        line starting with two blank characters:

diff --git a/usr.bin/mandoc/mandoc.h b/usr.bin/mandoc/mandoc.h
index 154bb50..e989422 100644 (file)
--- a/usr.bin/mandoc/mandoc.h
+++ b/usr.bin/mandoc/mandoc.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: mandoc.h,v 1.221 2022/06/07 09:51:03 schwarze Exp $ */
+/* $OpenBSD: mandoc.h,v 1.222 2022/08/15 17:59:00 schwarze Exp $ */
 /*
  * Copyright (c) 2012-2022 Ingo Schwarze <schwarze@openbsd.org>
  * Copyright (c) 2010, 2011, 2014 Kristaps Dzonsons <kristaps@bsd.lv>
@@ -20,8 +20,9 @@
  */
 
 #define ASCII_NBRSP     31  /* non-breaking space */
-#define        ASCII_HYPH       30  /* breakable hyphen */
-#define        ASCII_BREAK      29  /* breakable zero-width space */
+#define ASCII_NBRZW     30  /* non-breaking zero-width space */
+#define ASCII_BREAK     29  /* breakable zero-width space */
+#define ASCII_HYPH      28  /* breakable hyphen */
 
 /*
  * Status level.  This refers to both internal status (i.e., whilst

diff --git a/usr.bin/mandoc/term.c b/usr.bin/mandoc/term.c
index fe098f7..b300c53 100644 (file)
--- a/usr.bin/mandoc/term.c
+++ b/usr.bin/mandoc/term.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: term.c,v 1.148 2022/08/15 13:01:40 schwarze Exp $ */
+/* $OpenBSD: term.c,v 1.149 2022/08/15 17:59:00 schwarze Exp $ */
 /*
  * Copyright (c) 2010-2022 Ingo Schwarze <schwarze@openbsd.org>
  * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
@@ -206,7 +206,6 @@ term_flushln(struct termp *p)
                        return;
 
                endline(p);
-               p->viscol = 0;
 
                /*
                 * Normally, start the next line at the same indentation
@@ -312,6 +311,8 @@ term_fill(struct termp *p, size_t *nbr, size_t *vbr, size_t vtarget)
                                vis = term_tab_next(vis);
                                vis -= p->tcol->taboff;
                                break;
+                       case ASCII_NBRZW:  /* Non-breakable zero-width. */
+                               break;
                        case ASCII_NBRSP:  /* Non-breakable space. */
                                p->tcol->buf[ic] = ' ';
                                /* FALLTHROUGH */
@@ -363,6 +364,7 @@ term_field(struct termp *p, size_t vbl, size_t nbr)
                switch (p->tcol->buf[ic]) {
                case '\n':
                case ASCII_BREAK:
+               case ASCII_NBRZW:
                        continue;
                case '\t':
                case ' ':
@@ -569,18 +571,23 @@ term_word(struct termp *p, const char *word)
                        break;
                case ESCAPE_NUMBERED:
                        uc = mchars_num2char(seq, sz);
-                       if (uc < 0)
-                               continue;
-                       break;
+                       if (uc >= 0)
+                               break;
+                       bufferc(p, ASCII_NBRZW);
+                       continue;
                case ESCAPE_SPECIAL:
                        if (p->enc == TERMENC_ASCII) {
                                cp = mchars_spec2str(seq, sz, &ssz);
                                if (cp != NULL)
                                        encode(p, cp, ssz);
+                               else
+                                       bufferc(p, ASCII_NBRZW);
                        } else {
                                uc = mchars_spec2cp(seq, sz);
                                if (uc > 0)
                                        encode1(p, uc);
+                               else
+                                       bufferc(p, ASCII_NBRZW);
                        }
                        continue;
                case ESCAPE_UNDEF:
@@ -742,6 +749,9 @@ term_word(struct termp *p, const char *word)
                        if (p->col > p->tcol->lastcol)
                                p->col = p->tcol->lastcol;
                        continue;
+               case ESCAPE_IGNORE:
+                       bufferc(p, ASCII_NBRZW);
+                       continue;
                default:
                        continue;
                }
@@ -933,8 +943,8 @@ term_strlen(const struct termp *p, const char *cp)
        int              ssz, skip, uc;
        const char      *seq, *rhs;
        enum mandoc_esc  esc;
-       static const char rej[] = { '\\', ASCII_NBRSP, ASCII_HYPH,
-                       ASCII_BREAK, '\0' };
+       static const char rej[] = { '\\', ASCII_NBRSP, ASCII_NBRZW,
+               ASCII_BREAK, ASCII_HYPH, '\0' };
 
        /*
         * Account for escaped sequences within string length

diff --git a/usr.bin/mandoc/term_ascii.c b/usr.bin/mandoc/term_ascii.c
index 8b89deb..19f4c26 100644 (file)
--- a/usr.bin/mandoc/term_ascii.c
+++ b/usr.bin/mandoc/term_ascii.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: term_ascii.c,v 1.52 2020/09/09 13:40:24 schwarze Exp $ */
+/* $OpenBSD: term_ascii.c,v 1.53 2022/08/15 17:59:00 schwarze Exp $ */
 /*
  * Copyright (c) 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2014,2015,2017,2018,2020 Ingo Schwarze <schwarze@openbsd.org>
@@ -186,7 +186,7 @@ terminal_sepline(void *arg)
 static size_t
 ascii_width(const struct termp *p, int c)
 {
-       return c != ASCII_BREAK;
+       return c != ASCII_BREAK && c != ASCII_NBRZW;
 }
 
 void