===================================================================
RCS file: /cvs/mandoc/mdoc_macro.c,v
retrieving revision 1.203
retrieving revision 1.217
diff -u -p -r1.203 -r1.217
--- mandoc/mdoc_macro.c	2015/10/12 00:08:15	1.203
+++ mandoc/mdoc_macro.c	2017/02/16 09:47:31	1.217
@@ -1,7 +1,7 @@
-/*	$Id: mdoc_macro.c,v 1.203 2015/10/12 00:08:15 schwarze Exp $ */
+/*	$Id: mdoc_macro.c,v 1.217 2017/02/16 09:47:31 schwarze Exp $ */
 /*
  * Copyright (c) 2008-2012 Kristaps Dzonsons <kristaps@bsd.lv>
- * Copyright (c) 2010, 2012-2015 Ingo Schwarze <schwarze@openbsd.org>
+ * Copyright (c) 2010, 2012-2017 Ingo Schwarze <schwarze@openbsd.org>
  *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -50,6 +50,8 @@ static	int		find_pending(struct roff_man *, int, int, 
 				struct roff_node *);
 static	int		lookup(struct roff_man *, int, int, int, const char *);
 static	int		macro_or_word(MACRO_PROT_ARGS, int);
+static	void		break_intermediate(struct roff_node *,
+			    struct roff_node *);
 static	int		parse_rest(struct roff_man *, int, int, int *, char *);
 static	int		rew_alt(int);
 static	void		rew_elem(struct roff_man *, int);
@@ -216,7 +218,7 @@ mdoc_endparse(struct roff_man *mdoc)
 
 	/* Scan for open explicit scopes. */
 
-	n = mdoc->last->flags & MDOC_VALID ?
+	n = mdoc->last->flags & NODE_VALID ?
 	    mdoc->last->parent : mdoc->last;
 
 	for ( ; n; n = n->parent)
@@ -228,6 +230,7 @@ mdoc_endparse(struct roff_man *mdoc)
 	/* Rewind to the first. */
 
 	rew_last(mdoc, mdoc->first);
+	mdoc_state_reset(mdoc);
 }
 
 /*
@@ -239,6 +242,10 @@ lookup(struct roff_man *mdoc, int from, int line, int 
 {
 	int	 res;
 
+	if (mdoc->flags & MDOC_PHRASEQF) {
+		mdoc->flags &= ~MDOC_PHRASEQF;
+		return TOKEN_NONE;
+	}
 	if (from == TOKEN_NONE || mdoc_macros[from].flags & MDOC_PARSED) {
 		res = mdoc_hash_find(p);
 		if (res != TOKEN_NONE) {
@@ -258,25 +265,18 @@ lookup(struct roff_man *mdoc, int from, int line, int 
 static void
 rew_last(struct roff_man *mdoc, const struct roff_node *to)
 {
-	struct roff_node	*np;
 
-	if (to->flags & MDOC_VALID)
+	if (to->flags & NODE_VALID)
 		return;
 
 	while (mdoc->last != to) {
-		/*
-		 * Save the parent here, because we may delete the
-		 * mdoc->last node in the post-validation phase and reset
-		 * it to mdoc->last->parent, causing a step in the closing
-		 * out to be lost.
-		 */
-		np = mdoc->last->parent;
-		mdoc_valid_post(mdoc);
-		mdoc->last = np;
-		assert(mdoc->last);
+		mdoc_state(mdoc, mdoc->last);
+		mdoc->last->flags |= NODE_VALID | NODE_ENDED;
+		mdoc->last = mdoc->last->parent;
 	}
+	mdoc_state(mdoc, mdoc->last);
+	mdoc->last->flags |= NODE_VALID | NODE_ENDED;
 	mdoc->next = ROFF_NEXT_SIBLING;
-	mdoc_valid_post(mdoc);
 }
 
 /*
@@ -294,13 +294,13 @@ rew_pending(struct roff_man *mdoc, const struct roff_n
 			case ROFFT_HEAD:
 				roff_body_alloc(mdoc, n->line, n->pos,
 				    n->tok);
-				return;
+				break;
 			case ROFFT_BLOCK:
 				break;
 			default:
 				return;
 			}
-			if ( ! (n->flags & MDOC_BROKEN))
+			if ( ! (n->flags & NODE_BROKEN))
 				return;
 		} else
 			n = mdoc->last;
@@ -311,7 +311,7 @@ rew_pending(struct roff_man *mdoc, const struct roff_n
 
 			if (n->type == ROFFT_BLOCK ||
 			    n->type == ROFFT_HEAD) {
-				if (n->flags & MDOC_ENDED)
+				if (n->flags & NODE_ENDED)
 					break;
 				else
 					return;
@@ -378,6 +378,20 @@ rew_elem(struct roff_man *mdoc, int tok)
 	rew_last(mdoc, n);
 }
 
+static void
+break_intermediate(struct roff_node *n, struct roff_node *breaker)
+{
+	if (n != breaker &&
+	    n->type != ROFFT_BLOCK && n->type != ROFFT_HEAD &&
+	    (n->type != ROFFT_BODY || n->end != ENDBODY_NOT))
+		n = n->parent;
+	while (n != breaker) {
+		if ( ! (n->flags & NODE_VALID))
+			n->flags |= NODE_BROKEN;
+		n = n->parent;
+	}
+}
+
 /*
  * If there is an open sub-block of the target requiring
  * explicit close-out, postpone closing out the target until
@@ -390,26 +404,26 @@ find_pending(struct roff_man *mdoc, int tok, int line,
 	struct roff_node	*n;
 	int			 irc;
 
+	if (target->flags & NODE_VALID)
+		return 0;
+
 	irc = 0;
 	for (n = mdoc->last; n != NULL && n != target; n = n->parent) {
-		if (n->flags & MDOC_ENDED) {
-			if ( ! (n->flags & MDOC_VALID))
-				n->flags |= MDOC_BROKEN;
+		if (n->flags & NODE_ENDED)
 			continue;
-		}
 		if (n->type == ROFFT_BLOCK &&
 		    mdoc_macros[n->tok].flags & MDOC_EXPLICIT) {
 			irc = 1;
-			n->flags = MDOC_BROKEN;
+			break_intermediate(mdoc->last, target);
 			if (target->type == ROFFT_HEAD)
-				target->flags = MDOC_ENDED;
-			else if ( ! (target->flags & MDOC_ENDED)) {
+				target->flags |= NODE_ENDED;
+			else if ( ! (target->flags & NODE_ENDED)) {
 				mandoc_vmsg(MANDOCERR_BLK_NEST,
 				    mdoc->parse, line, ppos,
 				    "%s breaks %s", mdoc_macronames[tok],
 				    mdoc_macronames[n->tok]);
 				mdoc_endbody_alloc(mdoc, line, ppos,
-				    tok, target, ENDBODY_NOSPACE);
+				    tok, target);
 			}
 		}
 	}
@@ -446,11 +460,11 @@ dword(struct roff_man *mdoc, int line, int col, const 
 	 */
 
 	if (d == DELIM_OPEN)
-		mdoc->last->flags |= MDOC_DELIMO;
+		mdoc->last->flags |= NODE_DELIMO;
 	else if (d == DELIM_CLOSE &&
 	    ! (mdoc->flags & MDOC_NODELIMC) &&
 	    mdoc->last->parent->tok != MDOC_Fd)
-		mdoc->last->flags |= MDOC_DELIMC;
+		mdoc->last->flags |= NODE_DELIMC;
 	mdoc->flags &= ~MDOC_NODELIMC;
 }
 
@@ -483,7 +497,7 @@ append_delims(struct roff_man *mdoc, int line, int *po
 		 */
 
 		if (mandoc_eos(p, strlen(p)))
-			mdoc->last->flags |= MDOC_EOS;
+			mdoc->last->flags |= NODE_EOS;
 	}
 }
 
@@ -551,30 +565,46 @@ blk_exp_close(MACRO_PROT_ARGS)
 		break;
 	}
 
+	/* Search backwards for the beginning of our own body. */
+
+	atok = rew_alt(tok);
+	body = NULL;
+	for (n = mdoc->last; n; n = n->parent) {
+		if (n->flags & NODE_ENDED || n->tok != atok ||
+		    n->type != ROFFT_BODY || n->end != ENDBODY_NOT)
+			continue;
+		body = n;
+		break;
+	}
+
 	/*
 	 * Search backwards for beginnings of blocks,
 	 * both of our own and of pending sub-blocks.
 	 */
 
-	atok = rew_alt(tok);
-	body = endbody = itblk = later = NULL;
+	endbody = itblk = later = NULL;
 	for (n = mdoc->last; n; n = n->parent) {
-		if (n->flags & MDOC_ENDED) {
-			if ( ! (n->flags & MDOC_VALID))
-				n->flags |= MDOC_BROKEN;
+		if (n->flags & NODE_ENDED)
 			continue;
-		}
 
-		/* Remember the start of our own body. */
+		/*
+		 * Mismatching end macros can never break anything
+		 * and we only care about the breaking of BLOCKs.
+		 */
 
-		if (n->type == ROFFT_BODY && atok == n->tok) {
-			if (n->end == ENDBODY_NOT)
-				body = n;
+		if (body == NULL || n->type != ROFFT_BLOCK)
 			continue;
-		}
 
-		if (n->type != ROFFT_BLOCK || n->tok == MDOC_Nm)
+		/*
+		 * SYNOPSIS name blocks can not be broken themselves,
+		 * but they do get broken together with a broken child.
+		 */
+
+		if (n->tok == MDOC_Nm) {
+			if (later != NULL)
+				n->flags |= NODE_BROKEN | NODE_ENDED;
 			continue;
+		}
 
 		if (n->tok == MDOC_It) {
 			itblk = n;
@@ -582,7 +612,6 @@ blk_exp_close(MACRO_PROT_ARGS)
 		}
 
 		if (atok == n->tok) {
-			assert(body);
 
 			/*
 			 * Found the start of our own block.
@@ -608,10 +637,10 @@ blk_exp_close(MACRO_PROT_ARGS)
 			    mdoc_macronames[later->tok]);
 
 			endbody = mdoc_endbody_alloc(mdoc, line, ppos,
-			    atok, body, ENDBODY_SPACE);
+			    atok, body);
 
 			if (tok == MDOC_El)
-				itblk->flags |= MDOC_ENDED | MDOC_BROKEN;
+				itblk->flags |= NODE_ENDED | NODE_BROKEN;
 
 			/*
 			 * If a block closing macro taking arguments
@@ -624,16 +653,23 @@ blk_exp_close(MACRO_PROT_ARGS)
 			break;
 		}
 
-		/* Explicit blocks close out description lines. */
+		/*
+		 * Explicit blocks close out description lines, but
+		 * even those can get broken together with a child.
+		 */
 
 		if (n->tok == MDOC_Nd) {
-			rew_last(mdoc, n);
+			if (later != NULL)
+				n->flags |= NODE_BROKEN | NODE_ENDED;
+			else
+				rew_last(mdoc, n);
 			continue;
 		}
 
 		/* Breaking an open sub block. */
 
-		n->flags |= MDOC_BROKEN;
+		break_intermediate(mdoc->last, body);
+		n->flags |= NODE_BROKEN;
 		if (later == NULL)
 			later = n;
 	}
@@ -641,8 +677,6 @@ blk_exp_close(MACRO_PROT_ARGS)
 	if (body == NULL) {
 		mandoc_msg(MANDOCERR_BLK_NOTOPEN, mdoc->parse,
 		    line, ppos, mdoc_macronames[tok]);
-		if (later != NULL)
-			later->flags &= ~MDOC_BROKEN;
 		if (maxargs && endbody == NULL) {
 			/*
 			 * Stray .Ec without previous .Eo:
@@ -699,15 +733,14 @@ blk_exp_close(MACRO_PROT_ARGS)
 	}
 
 	if (n != NULL) {
-		if (ntok != TOKEN_NONE && n->flags & MDOC_BROKEN) {
+		pending = 0;
+		if (ntok != TOKEN_NONE && n->flags & NODE_BROKEN) {
 			target = n;
 			do
 				target = target->parent;
-			while ( ! (target->flags & MDOC_ENDED));
-			pending = find_pending(mdoc, ntok, line, ppos,
-			    target);
-		} else
-			pending = 0;
+			while ( ! (target->flags & NODE_ENDED));
+			pending = find_pending(mdoc, ntok, line, ppos, target);
+		}
 		if ( ! pending)
 			rew_pending(mdoc, n);
 	}
@@ -762,7 +795,7 @@ in_line(MACRO_PROT_ARGS)
 
 		if (ac == ARGS_EOLN) {
 			if (d == DELIM_OPEN)
-				mdoc->last->flags &= ~MDOC_DELIMO;
+				mdoc->last->flags &= ~NODE_DELIMO;
 			break;
 		}
 
@@ -856,7 +889,7 @@ in_line(MACRO_PROT_ARGS)
 		 */
 
 		if (firstarg && d == DELIM_CLOSE && !nc)
-			mdoc->last->flags &= ~MDOC_DELIMC;
+			mdoc->last->flags &= ~NODE_DELIMC;
 		firstarg = 0;
 
 		/*
@@ -919,9 +952,9 @@ blk_full(MACRO_PROT_ARGS)
 
 		blk = NULL;
 		for (n = mdoc->last; n != NULL; n = n->parent) {
-			if (n->flags & MDOC_ENDED) {
-				if ( ! (n->flags & MDOC_VALID))
-					n->flags |= MDOC_BROKEN;
+			if (n->flags & NODE_ENDED) {
+				if ( ! (n->flags & NODE_VALID))
+					n->flags |= NODE_BROKEN;
 				continue;
 			}
 			if (n->type != ROFFT_BLOCK)
@@ -980,7 +1013,7 @@ blk_full(MACRO_PROT_ARGS)
 
 			/* Close out prior implicit scopes. */
 
-			rew_last(mdoc, n);
+			rew_pending(mdoc, n);
 		}
 
 		/* Skip items outside lists. */
@@ -1030,26 +1063,39 @@ blk_full(MACRO_PROT_ARGS)
 	if (tok == MDOC_Bk)
 		mdoc->flags |= MDOC_KEEP;
 
-	ac = ARGS_PEND;
+	ac = ARGS_EOLN;
 	for (;;) {
+
+		/*
+		 * If we are right after a tab character,
+		 * do not parse the first word for macros.
+		 */
+
+		if (mdoc->flags & MDOC_PHRASEQN) {
+			mdoc->flags &= ~MDOC_PHRASEQN;
+			mdoc->flags |= MDOC_PHRASEQF;
+		}
+
 		la = *pos;
 		lac = ac;
 		ac = mdoc_args(mdoc, line, pos, buf, tok, &p);
 		if (ac == ARGS_EOLN) {
-			if (lac != ARGS_PPHRASE && lac != ARGS_PHRASE)
+			if (lac != ARGS_PHRASE ||
+			    ! (mdoc->flags & MDOC_PHRASEQF))
 				break;
+
 			/*
-			 * This is necessary: if the last token on a
-			 * line is a `Ta' or tab, then we'll get
-			 * ARGS_EOLN, so we must be smart enough to
-			 * reopen our scope if the last parse was a
-			 * phrase or partial phrase.
+			 * This line ends in a tab; start the next
+			 * column now, with a leading blank.
 			 */
+
 			if (body != NULL)
 				rew_last(mdoc, body);
 			body = roff_body_alloc(mdoc, line, ppos, tok);
+			roff_word_alloc(mdoc, line, ppos, "\\&");
 			break;
 		}
+
 		if (tok == MDOC_Bd || tok == MDOC_Bk) {
 			mandoc_vmsg(MANDOCERR_ARG_EXCESS,
 			    mdoc->parse, line, la, "%s ... %s",
@@ -1070,9 +1116,7 @@ blk_full(MACRO_PROT_ARGS)
 		 */
 
 		if (head == NULL &&
-		    ac != ARGS_PEND &&
 		    ac != ARGS_PHRASE &&
-		    ac != ARGS_PPHRASE &&
 		    ac != ARGS_QWORD &&
 		    mdoc_isdelim(p) == DELIM_OPEN) {
 			dword(mdoc, line, la, p, DELIM_OPEN, 0);
@@ -1084,9 +1128,7 @@ blk_full(MACRO_PROT_ARGS)
 		if (head == NULL)
 			head = roff_head_alloc(mdoc, line, ppos, tok);
 
-		if (ac == ARGS_PHRASE ||
-		    ac == ARGS_PEND ||
-		    ac == ARGS_PPHRASE) {
+		if (ac == ARGS_PHRASE) {
 
 			/*
 			 * If we haven't opened a body yet, rewind the
@@ -1096,18 +1138,16 @@ blk_full(MACRO_PROT_ARGS)
 			rew_last(mdoc, body == NULL ? head : body);
 			body = roff_body_alloc(mdoc, line, ppos, tok);
 
-			/*
-			 * Process phrases: set whether we're in a
-			 * partial-phrase (this effects line handling)
-			 * then call down into the phrase parser.
-			 */
+			/* Process to the tab or to the end of the line. */
 
-			if (ac == ARGS_PPHRASE)
-				mdoc->flags |= MDOC_PPHRASE;
-			if (ac == ARGS_PEND && lac == ARGS_PPHRASE)
-				mdoc->flags |= MDOC_PPHRASE;
+			mdoc->flags |= MDOC_PHRASE;
 			parse_rest(mdoc, TOKEN_NONE, line, &la, buf);
-			mdoc->flags &= ~MDOC_PPHRASE;
+			mdoc->flags &= ~MDOC_PHRASE;
+
+			/* There may have been `Ta' macros. */
+
+			while (body->next != NULL)
+				body = body->next;
 			continue;
 		}
 
@@ -1115,7 +1155,7 @@ blk_full(MACRO_PROT_ARGS)
 			break;
 	}
 
-	if (blk->flags & MDOC_VALID)
+	if (blk->flags & NODE_VALID)
 		return;
 	if (head == NULL)
 		head = roff_head_alloc(mdoc, line, ppos, tok);
@@ -1450,11 +1490,11 @@ phrase_ta(MACRO_PROT_ARGS)
 
 	body = NULL;
 	for (n = mdoc->last; n != NULL; n = n->parent) {
-		if (n->flags & MDOC_ENDED)
+		if (n->flags & NODE_ENDED)
 			continue;
 		if (n->tok == MDOC_It && n->type == ROFFT_BODY)
 			body = n;
-		if (n->tok == MDOC_Bl)
+		if (n->tok == MDOC_Bl && n->end == ENDBODY_NOT)
 			break;
 	}