===================================================================
RCS file: /cvs/mandoc/roff.c,v
retrieving revision 1.1
retrieving revision 1.52
diff -u -p -r1.1 -r1.52
--- mandoc/roff.c	2008/11/24 14:24:55	1.1
+++ mandoc/roff.c	2008/12/08 16:29:57	1.52
@@ -1,4 +1,4 @@
-/* $Id: roff.c,v 1.1 2008/11/24 14:24:55 kristaps Exp $ */
+/* $Id: roff.c,v 1.52 2008/12/08 16:29:57 kristaps Exp $ */
 /*
  * Copyright (c) 2008 Kristaps Dzonsons <kristaps@kth.se>
  *
@@ -16,9 +16,13 @@
  * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
  * PERFORMANCE OF THIS SOFTWARE.
  */
+#include <sys/param.h>
+#include <sys/types.h>
+
 #include <assert.h>
 #include <ctype.h>
 #include <err.h>
+#include <stdarg.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
@@ -26,359 +30,455 @@
 
 #include "libmdocml.h"
 #include "private.h"
+#include "roff.h"
 
-#define	ROFF_MAXARG	  10
+/* FIXME: First letters of quoted-text interpreted in rofffindtok. */
+/* FIXME: `No' not implemented. */
+/* TODO: warn if Pp occurs before/after Sh etc. (see mdoc.samples). */
+/* TODO: warn about empty lists. */
+/* TODO: (warn) some sections need specific elements. */
+/* TODO: (warn) NAME section has particular order. */
+/* TODO: unify empty-content tags a la <br />. */
+/* TODO: macros with a set number of arguments? */
+/* TODO: validate Dt macro arguments. */
+/* FIXME: Bl -diag supposed to ignore callable children. */
+/* FIXME: Nm has newline when used in NAME section. */
 
-enum	roffd { 
-	ROFF_ENTER = 0, 
-	ROFF_EXIT 
-};
-
-enum	rofftype { 
-	ROFF_TITLE, 
-	ROFF_COMMENT, 
-	ROFF_TEXT, 
-	ROFF_LAYOUT 
-};
-
-#define	ROFFCALL_ARGS \
-	struct rofftree *tree, const char *argv[], enum roffd type
-
-struct	rofftree;
-
-struct	rofftok {
-	char		 *name;
-	int		(*cb)(ROFFCALL_ARGS);
-	enum rofftype	  type;
-	int		  flags;
-#define	ROFF_NESTED	 (1 << 0) 
-#define	ROFF_PARSED	 (1 << 1)
-#define	ROFF_CALLABLE	 (1 << 2)
-#define	ROFF_QUOTES	 (1 << 3)
-};
-
-struct	roffarg {
-	char		 *name;
-	int		  flags;
-#define	ROFF_VALUE	 (1 << 0)
-};
-
 struct	roffnode {
-	int		  tok;	
-	struct roffnode	 *parent;
-	size_t		  line;
+	int		  tok;			/* Token id. */
+	struct roffnode	 *parent;		/* Parent (or NULL). */
 };
 
 struct	rofftree {
-	struct roffnode	 *last;
-	time_t		  date;
-	char		  title[256];
-	char		  section[256];
-	char		  volume[256];
+	struct roffnode	 *last;			/* Last parsed node. */
+	char		 *cur;			/* Line start. */
+	struct tm	  tm;			/* `Dd' results. */
+	char		  name[64];		/* `Nm' results. */
+	char		  os[64];		/* `Os' results. */
+	char		  title[64];		/* `Dt' results. */
+	enum roffmsec	  section;
+	char		  volume[64];		/* `Dt' results. */
 	int		  state;
-#define	ROFF_PRELUDE	 (1 << 1)
-#define	ROFF_PRELUDE_Os	 (1 << 2)
-#define	ROFF_PRELUDE_Dt	 (1 << 3)
-#define	ROFF_PRELUDE_Dd	 (1 << 4)
-#define	ROFF_BODY	 (1 << 5)
-	struct md_mbuf	*mbuf; /* NULL if ROFF_EXIT and error. */
-
-	const struct md_args	*args;
-	const struct md_rbuf	*rbuf;
+#define	ROFF_PRELUDE	 (1 << 1)		/* In roff prelude. */
+#define	ROFF_PRELUDE_Os	 (1 << 2)		/* `Os' is parsed. */
+#define	ROFF_PRELUDE_Dt	 (1 << 3)		/* `Dt' is parsed. */
+#define	ROFF_PRELUDE_Dd	 (1 << 4)		/* `Dd' is parsed. */
+#define	ROFF_BODY	 (1 << 5)		/* In roff body. */
+	struct roffcb	  cb;			/* Callbacks. */
+	void		 *arg;			/* Callbacks' arg. */
+	int		  csec;			/* Current section. */
+	int		  asec;			/* Thus-far sections. */
 };
 
-#define	ROFF___	 	  0
-#define	ROFF_Dd		  1
-#define	ROFF_Dt		  2
-#define	ROFF_Os		  3
-#define	ROFF_Sh		  4
-#define	ROFF_An		  5
-#define	ROFF_Li		  6
-#define	ROFF_MAX	  7
-
-static	int		  roff_Dd(ROFFCALL_ARGS);
-static	int		  roff_Dt(ROFFCALL_ARGS);
-static	int		  roff_Os(ROFFCALL_ARGS);
-static	int		  roff_Sh(ROFFCALL_ARGS);
-static	int		  roff_An(ROFFCALL_ARGS);
-static	int		  roff_Li(ROFFCALL_ARGS);
-
-static	struct roffnode	 *roffnode_new(int, size_t, 
-				struct rofftree *);
-static	void		  roffnode_free(int, struct rofftree *);
-
+static	struct roffnode	 *roffnode_new(int, struct rofftree *);
+static	void		  roffnode_free(struct rofftree *);
+static	void		  roff_warn(const struct rofftree *, 
+				const char *, char *, ...);
+static	void		  roff_err(const struct rofftree *, 
+				const char *, char *, ...);
+static	int		  roffpurgepunct(struct rofftree *, char **);
+static	int		  roffscan(int, const int *);
 static	int		  rofffindtok(const char *);
 static	int		  rofffindarg(const char *);
-static	int		  roffargs(int, char *, char **);
-static	int 		  roffparse(struct rofftree *, char *, size_t);
-static	int		  textparse(const struct rofftree *,
-				const char *, size_t);
+static	int		  rofffindcallable(const char *);
+static	int		  roffismsec(const char *);
+static	int		  roffissec(const char **);
+static	int		  roffispunct(const char *);
+static	int		  roffchecksec(struct rofftree *, 
+				const char *, int);
+static	int		  roffargs(const struct rofftree *,
+				int, char *, char **);
+static	int		  roffargok(int, int);
+static	int		  roffnextopt(const struct rofftree *,
+				int, char ***, char **);
+static	int		  roffparseopts(struct rofftree *, int, 
+				char ***, int *, char **);
+static	int		  roffcall(struct rofftree *, int, char **);
+static	int		  roffexit(struct rofftree *, int);
+static	int 		  roffparse(struct rofftree *, char *);
+static	int		  textparse(struct rofftree *, char *);
+static	int		  roffdata(struct rofftree *, int, char *);
+static	int		  roffspecial(struct rofftree *, int, 
+				const char *, const int *,
+				const char **, size_t, char **);
+static	int		  roffsetname(struct rofftree *, char **);
 
-static	void		  dbg_enter(const struct md_args *, int);
-static	void		  dbg_leave(const struct md_args *, int);
+#ifdef __linux__ 
+extern	size_t		  strlcat(char *, const char *, size_t);
+extern	size_t		  strlcpy(char *, const char *, size_t);
+extern	int		  vsnprintf(char *, size_t, 
+				const char *, va_list);
+extern	char		 *strptime(const char *, const char *,
+				struct tm *);
+#endif
 
-
-static const struct rofftok tokens[ROFF_MAX] = {
-	{ "\\\"",    NULL, ROFF_COMMENT, 0 },
-	{   "Dd", roff_Dd, ROFF_TITLE, 0 },
-	{   "Dt", roff_Dt, ROFF_TITLE, 0 },
-	{   "Os", roff_Os, ROFF_TITLE, 0 },
-	{   "Sh", roff_Sh, ROFF_LAYOUT, 0 },
-	{   "An", roff_An, ROFF_TEXT, ROFF_PARSED },
-	{   "Li", roff_Li, ROFF_TEXT, ROFF_PARSED | ROFF_CALLABLE },
-};
-
-#define	ROFF_Split	  0
-#define	ROFF_Nosplit	  1
-#define	ROFF_ARGMAX	  2
-
-static const struct roffarg tokenargs[ROFF_ARGMAX] = {
-	{  "split", 	0 },
-	{  "nosplit",	0 },
-};
-
-
 int
 roff_free(struct rofftree *tree, int flush)
 {
-	int		 error;
+	int		 error, t;
+	struct roffnode	*n;
 
-	assert(tree->mbuf);
+	error = 0;
+
 	if ( ! flush)
-		tree->mbuf = NULL;
+		goto end;
 
-	/* LINTED */
-	while (tree->last)
-		if ( ! (*tokens[tree->last->tok].cb)
-				(tree, NULL, ROFF_EXIT))
-			/* Disallow flushing. */
-			tree->mbuf = NULL;
+	error = 1;
 
-	error = tree->mbuf ? 0 : 1;
+	if (ROFF_PRELUDE & tree->state) {
+		roff_err(tree, NULL, "prelude never finished");
+		goto end;
+	} else if ( ! (ROFFSec_NAME & tree->asec)) {
+		roff_err(tree, NULL, "missing `NAME' section");
+		goto end;
+	} else if ( ! (ROFFSec_NMASK & tree->asec))
+		roff_warn(tree, NULL, "missing suggested `NAME', "
+				"`SYNOPSIS', `DESCRIPTION' sections");
 
-	if (tree->mbuf && (ROFF_PRELUDE & tree->state)) {
-		warnx("%s: prelude never finished", 
-				tree->rbuf->name);
-		error = 1;
+	for (n = tree->last; n; n = n->parent) {
+		if (0 != tokens[n->tok].ctx) 
+			continue;
+		roff_err(tree, NULL, "closing explicit scope `%s'", 
+				toknames[n->tok]);
+		goto end;
 	}
 
+	while (tree->last) {
+		t = tree->last->tok;
+		if ( ! roffexit(tree, t))
+			goto end;
+	}
+
+	if ( ! (*tree->cb.rofftail)(tree->arg))
+		goto end;
+
+	error = 0;
+
+end:
+
+	while (tree->last) 
+		roffnode_free(tree);
+
 	free(tree);
+
 	return(error ? 0 : 1);
 }
 
 
 struct rofftree *
-roff_alloc(const struct md_args *args, struct md_mbuf *out, 
-		const struct md_rbuf *in)
+roff_alloc(const struct roffcb *cb, void *args)
 {
 	struct rofftree	*tree;
 
-	if (NULL == (tree = calloc(1, sizeof(struct rofftree)))) {
-		warn("malloc");
-		return(NULL);
-	}
+	assert(args);
+	assert(cb);
 
+	if (NULL == (tree = calloc(1, sizeof(struct rofftree))))
+		err(1, "calloc");
+
 	tree->state = ROFF_PRELUDE;
-	tree->args = args;
-	tree->mbuf = out;
-	tree->rbuf = in;
+	tree->arg = args;
+	tree->section = ROFF_MSEC_MAX;
 
+	(void)memcpy(&tree->cb, cb, sizeof(struct roffcb));
+
 	return(tree);
 }
 
 
 int
-roff_engine(struct rofftree *tree, char *buf, size_t sz)
+roff_engine(struct rofftree *tree, char *buf)
 {
 
-	if (0 == sz) {
-		warnx("%s: blank line (line %zu)", 
-				tree->rbuf->name, 
-				tree->rbuf->line);
+	tree->cur = buf;
+	assert(buf);
+
+	if (0 == *buf) {
+		roff_err(tree, buf, "blank line");
 		return(0);
 	} else if ('.' != *buf)
-		return(textparse(tree, buf, sz));
+		return(textparse(tree, buf));
 
-	return(roffparse(tree, buf, sz));
+	return(roffparse(tree, buf));
 }
 
 
 static int
-textparse(const struct rofftree *tree, const char *buf, size_t sz)
+textparse(struct rofftree *tree, char *buf)
 {
-	
-	if (NULL == tree->last) {
-		warnx("%s: unexpected text (line %zu)",
-				tree->rbuf->name, 
-				tree->rbuf->line);
+	char		*bufp;
+
+	/* TODO: literal parsing. */
+
+	if ( ! (ROFF_BODY & tree->state)) {
+		roff_err(tree, buf, "data not in body");
 		return(0);
-	} else if (NULL == tree->last->parent) {
-		warnx("%s: disallowed text (line %zu)",
-				tree->rbuf->name, 
-				tree->rbuf->line);
-		return(0);
 	}
 
-	/* Print text. */
+	/* LINTED */
+	while (*buf) {
+		while (*buf && isspace(*buf))
+			buf++;
 
+		if (0 == *buf)
+			break;
+
+		bufp = buf++;
+
+		while (*buf && ! isspace(*buf))
+			buf++;
+
+		if (0 != *buf) {
+			*buf++ = 0;
+			if ( ! roffdata(tree, 1, bufp))
+				return(0);
+			continue;
+		}
+
+		if ( ! roffdata(tree, 1, bufp))
+			return(0);
+		break;
+	}
+
 	return(1);
 }
 
 
 static int
-roffargs(int tok, char *buf, char **argv)
+roffargs(const struct rofftree *tree, 
+		int tok, char *buf, char **argv)
 {
 	int		 i;
+	char		*p;
 
-	(void)tok;/* FIXME: quotable strings? */
-
 	assert(tok >= 0 && tok < ROFF_MAX);
 	assert('.' == *buf);
 
+	p = buf;
+
+	/*
+	 * This is an ugly little loop.  It parses a line into
+	 * space-delimited tokens.  If a quote mark is encountered, a
+	 * token is alloted the entire quoted text.  If whitespace is
+	 * escaped, it's included in the prior alloted token.
+	 */
+
 	/* LINTED */
-	for (i = 0; *buf && i < ROFF_MAXARG; i++) {
-		argv[i] = buf++;
-		while (*buf && ! isspace(*buf))
-			buf++;
-		if (0 == *buf) {
-			continue;
+	for (i = 0; *buf && i < ROFF_MAXLINEARG; i++) {
+		if ('\"' == *buf) {
+			argv[i] = ++buf;
+			while (*buf && '\"' != *buf)
+				buf++;
+			if (0 == *buf) {
+				roff_err(tree, argv[i], "unclosed "
+						"quote in argument "
+						"list for `%s'", 
+						toknames[tok]);
+				return(0);
+			}
+		} else { 
+			argv[i] = buf++;
+			while (*buf) {
+				if ( ! isspace(*buf)) {
+					buf++;
+					continue;
+				}
+				if (*(buf - 1) == '\\') {
+					buf++;
+					continue;
+				}
+				break;
+			}
+			if (0 == *buf)
+				continue;
 		}
 		*buf++ = 0;
 		while (*buf && isspace(*buf))
 			buf++;
 	}
-	
+
 	assert(i > 0);
-	if (i < ROFF_MAXARG)
-		argv[i] = NULL;
+	if (ROFF_MAXLINEARG == i && *buf) {
+		roff_err(tree, p, "too many arguments for `%s'", toknames
+				[tok]);
+		return(0);
+	}
 
-	return(ROFF_MAXARG > i);
+	argv[i] = NULL;
+	return(1);
 }
 
 
 static int
-roffparse(struct rofftree *tree, char *buf, size_t sz)
+roffscan(int tok, const int *tokv)
 {
-	int		  tok, t;
-	struct roffnode	 *node;
-	char		 *argv[ROFF_MAXARG];
-	const char	**argvp;
 
-	assert(sz > 0);
+	if (NULL == tokv)
+		return(1);
 
-	/*
-	 * Extract the token identifier from the buffer.  If there's no
-	 * callback for the token (comment, etc.) then exit immediately.
-	 * We don't do any error handling (yet), so if the token doesn't
-	 * exist, die.
-	 */
+	for ( ; ROFF_MAX != *tokv; tokv++) 
+		if (tok == *tokv)
+			return(1);
 
-	if (3 > sz) {
-		warnx("%s: malformed line (line %zu)", 
-				tree->rbuf->name, 
-				tree->rbuf->line);
+	return(0);
+}
+
+
+static int
+roffparse(struct rofftree *tree, char *buf)
+{
+	int		  tok, t;
+	struct roffnode	 *n;
+	char		 *argv[ROFF_MAXLINEARG];
+	char		**argvp;
+
+	if (0 != *buf && 0 != *(buf + 1) && 0 != *(buf + 2))
+		if (0 == strncmp(buf, ".\\\"", 3))
+			return(1);
+
+	if (ROFF_MAX == (tok = rofffindtok(buf + 1))) {
+		roff_err(tree, buf, "bogus line macro");
 		return(0);
-	} else if (ROFF_MAX == (tok = rofffindtok(buf + 1))) {
-		warnx("%s: unknown line token `%c%c' (line %zu)",
-				tree->rbuf->name, 
-				*(buf + 1), *(buf + 2), 
-				tree->rbuf->line);
+	} else if ( ! roffargs(tree, tok, buf, argv)) 
 		return(0);
-	} else if (ROFF_COMMENT == tokens[tok].type) 
-		/* Ignore comment tokens. */
-		return(1);
+
+	argvp = (char **)argv;
+
+	/* 
+	 * Prelude macros break some assumptions, so branch now. 
+	 */
 	
-	if ( ! roffargs(tok, buf, argv)) {
-		warnx("%s: too many arguments to `%s' (line %zu)",
-				tree->rbuf->name, tokens[tok].name, 
-				tree->rbuf->line);
-		return(0);
-	}
+	if (ROFF_PRELUDE & tree->state) {
+		assert(NULL == tree->last);
+		return(roffcall(tree, tok, argvp));
+	} 
 
-	/* Domain cross-contamination (and sanity) checks. */
+	assert(ROFF_BODY & tree->state);
 
-	switch (tokens[tok].type) {
-	case (ROFF_TITLE):
-		if (ROFF_PRELUDE & tree->state) {
-			assert( ! (ROFF_BODY & tree->state));
-			break;
-		}
-		assert(ROFF_BODY & tree->state);
-		warnx("%s: prelude token `%s' in body (line %zu)",
-				tree->rbuf->name, tokens[tok].name, 
-				tree->rbuf->line);
+	/* 
+	 * First check that our possible parents and parent's possible
+	 * children are satisfied.  
+	 */
+
+	if (tree->last && ! roffscan
+			(tree->last->tok, tokens[tok].parents)) {
+		roff_err(tree, *argvp, "`%s' has invalid parent `%s'",
+				toknames[tok], 
+				toknames[tree->last->tok]);
 		return(0);
-	case (ROFF_LAYOUT):
-		/* FALLTHROUGH */
-	case (ROFF_TEXT):
-		if (ROFF_BODY & tree->state) {
-			assert( ! (ROFF_PRELUDE & tree->state));
-			break;
-		}
-		assert(ROFF_PRELUDE & tree->state);
-		warnx("%s: body token `%s' in prelude (line %zu)",
-				tree->rbuf->name, tokens[tok].name, 
-				tree->rbuf->line);
+	} 
+
+	if (tree->last && ! roffscan
+			(tok, tokens[tree->last->tok].children)) {
+		roff_err(tree, *argvp, "`%s' is invalid child of `%s'",
+				toknames[tok],
+				toknames[tree->last->tok]);
 		return(0);
-	case (ROFF_COMMENT):
-		return(1);
-	default:
-		abort();
 	}
 
 	/*
-	 * If this is a non-nestable layout token and we're below a
-	 * token of the same type, then recurse upward to the token,
-	 * closing out the interim scopes.
-	 *
-	 * If there's a nested token on the chain, then raise an error
-	 * as nested tokens have corresponding "ending" tokens and we're
-	 * breaking their scope.
+	 * Branch if we're not a layout token.
 	 */
 
-	node = NULL;
+	if (ROFF_LAYOUT != tokens[tok].type)
+		return(roffcall(tree, tok, argvp));
+	if (0 == tokens[tok].ctx)
+		return(roffcall(tree, tok, argvp));
 
-	if (ROFF_LAYOUT == tokens[tok].type && 
-			! (ROFF_NESTED & tokens[tok].flags)) {
-		for (node = tree->last; node; node = node->parent) {
-			if (node->tok == tok)
-				break;
+	/*
+	 * First consider implicit-end tags, like as follows:
+	 *	.Sh SECTION 1
+	 *	.Sh SECTION 2
+	 * In this, we want to close the scope of the NAME section.  If
+	 * there's an intermediary implicit-end tag, such as
+	 *	.Sh SECTION 1
+	 *	.Ss Subsection 1
+	 *	.Sh SECTION 2
+	 * then it must be closed as well.
+	 */
 
-			/* Don't break nested scope. */
+	if (tok == tokens[tok].ctx) {
+		/* 
+		 * First search up to the point where we must close.
+		 * If one doesn't exist, then we can open a new scope.
+		 */
 
-			if ( ! (ROFF_NESTED & tokens[node->tok].flags))
+		for (n = tree->last; n; n = n->parent) {
+			assert(0 == tokens[n->tok].ctx ||
+					n->tok == tokens[n->tok].ctx);
+			if (n->tok == tok)
+				break;
+			if (ROFF_SHALLOW & tokens[tok].flags) {
+				n = NULL;
+				break;
+			}
+			if (tokens[n->tok].ctx == n->tok)
 				continue;
-			warnx("%s: scope of %s (line %zu) broken by "
-					"%s (line %zu)", 
-					tree->rbuf->name, 
-					tokens[tok].name,
-					node->line, 
-					tokens[node->tok].name,
-					tree->rbuf->line);
+			roff_err(tree, *argv, "`%s' breaks `%s' scope",
+					toknames[tok], toknames[n->tok]);
 			return(0);
 		}
-	}
 
-	if (node) {
-		assert(ROFF_LAYOUT == tokens[tok].type);
-		assert( ! (ROFF_NESTED & tokens[tok].flags));
-		assert(node->tok == tok);
+		/*
+		 * Create a new scope, as no previous one exists to
+		 * close out.
+		 */
 
-		/* Clear up to last scoped token. */
+		if (NULL == n)
+			return(roffcall(tree, tok, argvp));
 
-		/* LINTED */
+		/* 
+		 * Close out all intermediary scoped blocks, then hang
+		 * the current scope from our predecessor's parent.
+		 */
+
 		do {
 			t = tree->last->tok;
-			if ( ! (*tokens[tree->last->tok].cb)
-					(tree, NULL, ROFF_EXIT))
+			if ( ! roffexit(tree, t))
 				return(0);
 		} while (t != tok);
+
+		return(roffcall(tree, tok, argvp));
 	}
 
-	/* Proceed with actual token processing. */
+	/*
+	 * Now consider explicit-end tags, where we want to close back
+	 * to a specific tag.  Example:
+	 * 	.Bl
+	 * 	.It Item.
+	 * 	.El
+	 * In this, the `El' tag closes out the scope of `Bl'.
+	 */
 
-	argvp = (const char **)&argv[1];
-	return((*tokens[tok].cb)(tree, argvp, ROFF_ENTER));
+	assert(tok != tokens[tok].ctx && 0 != tokens[tok].ctx);
+
+	/* LINTED */
+	for (n = tree->last; n; n = n->parent)
+		if (n->tok != tokens[tok].ctx) {
+			if (n->tok == tokens[n->tok].ctx)
+				continue;
+			roff_err(tree, *argv, "`%s' breaks `%s' scope",
+					toknames[tok], toknames[n->tok]);
+			return(0);
+		} else
+			break;
+
+
+	if (NULL == n) {
+		roff_err(tree, *argv, "`%s' has no starting tag `%s'",
+				toknames[tok], 
+				toknames[tokens[tok].ctx]);
+		return(0);
+	}
+
+	/* LINTED */
+	do {
+		t = tree->last->tok;
+		if ( ! roffexit(tree, t))
+			return(0);
+	} while (t != tokens[tok].ctx);
+
+	return(1);
 }
 
 
@@ -392,7 +492,7 @@ rofffindarg(const char *name)
 	/* LINTED */
 	for (i = 0; i < ROFF_ARGMAX; i++)
 		/* LINTED */
-		if (0 == strcmp(name, tokenargs[i].name))
+		if (0 == strcmp(name, tokargnames[i]))
 			return((int)i);
 	
 	return(ROFF_ARGMAX);
@@ -400,48 +500,277 @@ rofffindarg(const char *name)
 
 
 static int
-rofffindtok(const char *name)
+rofffindtok(const char *buf)
 {
-	size_t		 i;
+	char		 token[4];
+	int		 i;
 
+	for (i = 0; *buf && ! isspace(*buf) && i < 3; i++, buf++)
+		token[i] = *buf;
+
+	if (i == 3) 
+		return(ROFF_MAX);
+
+	token[i] = 0;
+
 	/* FIXME: use a table, this is slow but ok for now. */
 
 	/* LINTED */
 	for (i = 0; i < ROFF_MAX; i++)
 		/* LINTED */
-		if (0 == strncmp(name, tokens[i].name, 2))
+		if (0 == strcmp(toknames[i], token))
 			return((int)i);
-	
+
 	return(ROFF_MAX);
 }
 
 
-/* FIXME: accept only struct rofftree *. */
+static int
+roffchecksec(struct rofftree *tree, const char *start, int sec)
+{
+	int		 prior;
+
+	switch (sec) {
+	case(ROFFSec_SYNOP):
+		if ((prior = ROFFSec_NAME) & tree->asec)
+			return(1);
+		break;
+	case(ROFFSec_DESC):
+		if ((prior = ROFFSec_SYNOP) & tree->asec)
+			return(1);
+		break;
+	case(ROFFSec_RETVAL):
+		if ((prior = ROFFSec_DESC) & tree->asec)
+			return(1);
+		break;
+	case(ROFFSec_ENV):
+		if ((prior = ROFFSec_RETVAL) & tree->asec)
+			return(1);
+		break;
+	case(ROFFSec_FILES):
+		if ((prior = ROFFSec_ENV) & tree->asec)
+			return(1);
+		break;
+	case(ROFFSec_EX):
+		if ((prior = ROFFSec_FILES) & tree->asec)
+			return(1);
+		break;
+	case(ROFFSec_DIAG):
+		if ((prior = ROFFSec_EX) & tree->asec)
+			return(1);
+		break;
+	case(ROFFSec_ERRS):
+		if ((prior = ROFFSec_DIAG) & tree->asec)
+			return(1);
+		break;
+	case(ROFFSec_SEEALSO):
+		if ((prior = ROFFSec_ERRS) & tree->asec)
+			return(1);
+		break;
+	case(ROFFSec_STAND):
+		if ((prior = ROFFSec_SEEALSO) & tree->asec)
+			return(1);
+		break;
+	case(ROFFSec_HIST):
+		if ((prior = ROFFSec_STAND) & tree->asec)
+			return(1);
+		break;
+	case(ROFFSec_AUTH):
+		if ((prior = ROFFSec_HIST) & tree->asec)
+			return(1);
+		break;
+	case(ROFFSec_CAVEATS):
+		if ((prior = ROFFSec_AUTH) & tree->asec)
+			return(1);
+		break;
+	case(ROFFSec_BUGS):
+		if ((prior = ROFFSec_CAVEATS) & tree->asec)
+			return(1);
+		break;
+	default:
+		return(1);
+	}
+
+	roff_warn(tree, start, "section violates conventional order");
+	return(1);
+}
+
+
+static int
+roffissec(const char **p)
+{
+
+	assert(*p);
+	if (NULL != *(p + 1)) {
+		if (NULL != *(p + 2))
+			return(ROFFSec_OTHER);
+		if (0 == strcmp(*p, "RETURN") &&
+				0 == strcmp(*(p + 1), "VALUES"))
+			return(ROFFSec_RETVAL);
+		if (0 == strcmp(*p, "SEE") &&
+				0 == strcmp(*(p + 1), "ALSO"))
+			return(ROFFSec_SEEALSO);
+		return(ROFFSec_OTHER);
+	}
+
+	if (0 == strcmp(*p, "NAME"))
+		return(ROFFSec_NAME);
+	else if (0 == strcmp(*p, "SYNOPSIS"))
+		return(ROFFSec_SYNOP);
+	else if (0 == strcmp(*p, "DESCRIPTION"))
+		return(ROFFSec_DESC);
+	else if (0 == strcmp(*p, "ENVIRONMENT"))
+		return(ROFFSec_ENV);
+	else if (0 == strcmp(*p, "FILES"))
+		return(ROFFSec_FILES);
+	else if (0 == strcmp(*p, "EXAMPLES"))
+		return(ROFFSec_EX);
+	else if (0 == strcmp(*p, "DIAGNOSTICS")) 
+		return(ROFFSec_DIAG);
+	else if (0 == strcmp(*p, "ERRORS"))
+		return(ROFFSec_ERRS);
+	else if (0 == strcmp(*p, "STANDARDS"))
+		return(ROFFSec_STAND);
+	else if (0 == strcmp(*p, "HISTORY"))
+		return(ROFFSec_HIST);
+	else if (0 == strcmp(*p, "AUTHORS"))
+		return(ROFFSec_AUTH);
+	else if (0 == strcmp(*p, "CAVEATS"))
+		return(ROFFSec_CAVEATS);
+	else if (0 == strcmp(*p, "BUGS"))
+		return(ROFFSec_BUGS);
+
+	return(ROFFSec_OTHER);
+}
+
+
+static int
+roffismsec(const char *p)
+{
+
+	if (0 == strcmp(p, "1"))
+		return(ROFF_MSEC_1);
+	else if (0 == strcmp(p, "2"))
+		return(ROFF_MSEC_2);
+	else if (0 == strcmp(p, "3"))
+		return(ROFF_MSEC_3);
+	else if (0 == strcmp(p, "3p"))
+		return(ROFF_MSEC_3p);
+	else if (0 == strcmp(p, "4"))
+		return(ROFF_MSEC_4);
+	else if (0 == strcmp(p, "5"))
+		return(ROFF_MSEC_5);
+	else if (0 == strcmp(p, "6"))
+		return(ROFF_MSEC_6);
+	else if (0 == strcmp(p, "7"))
+		return(ROFF_MSEC_7);
+	else if (0 == strcmp(p, "8"))
+		return(ROFF_MSEC_8);
+	else if (0 == strcmp(p, "9"))
+		return(ROFF_MSEC_9);
+	else if (0 == strcmp(p, "unass"))
+		return(ROFF_MSEC_UNASS);
+	else if (0 == strcmp(p, "draft"))
+		return(ROFF_MSEC_DRAFT);
+	else if (0 == strcmp(p, "paper"))
+		return(ROFF_MSEC_PAPER);
+
+	return(ROFF_MSEC_MAX);
+}
+
+
+static int
+roffispunct(const char *p)
+{
+
+	if (0 == *p)
+		return(0);
+	if (0 != *(p + 1))
+		return(0);
+
+	switch (*p) {
+	case('{'):
+		/* FALLTHROUGH */
+	case('.'):
+		/* FALLTHROUGH */
+	case(','):
+		/* FALLTHROUGH */
+	case(';'):
+		/* FALLTHROUGH */
+	case(':'):
+		/* FALLTHROUGH */
+	case('?'):
+		/* FALLTHROUGH */
+	case('!'):
+		/* FALLTHROUGH */
+	case('('):
+		/* FALLTHROUGH */
+	case(')'):
+		/* FALLTHROUGH */
+	case('['):
+		/* FALLTHROUGH */
+	case(']'):
+		/* FALLTHROUGH */
+	case('}'):
+		return(1);
+	default:
+		break;
+	}
+
+	return(0);
+}
+
+
+static int
+rofffindcallable(const char *name)
+{
+	int		 c;
+
+	if (ROFF_MAX == (c = rofffindtok(name)))
+		return(ROFF_MAX);
+	assert(c >= 0 && c < ROFF_MAX);
+	return(ROFF_CALLABLE & tokens[c].flags ? c : ROFF_MAX);
+}
+
+
 static struct roffnode *
-roffnode_new(int tokid, size_t line, struct rofftree *tree)
+roffnode_new(int tokid, struct rofftree *tree)
 {
 	struct roffnode	*p;
 	
-	if (NULL == (p = malloc(sizeof(struct roffnode)))) {
-		warn("malloc");
-		return(NULL);
-	}
+	if (NULL == (p = malloc(sizeof(struct roffnode))))
+		err(1, "malloc");
 
-	p->line = line;
 	p->tok = tokid;
 	p->parent = tree->last;
 	tree->last = p;
+
 	return(p);
 }
 
 
+static int
+roffargok(int tokid, int argid)
+{
+	const int	*c;
+
+	if (NULL == (c = tokens[tokid].args))
+		return(0);
+
+	for ( ; ROFF_ARGMAX != *c; c++) 
+		if (argid == *c)
+			return(1);
+
+	return(0);
+}
+
+
 static void
-roffnode_free(int tokid, struct rofftree *tree)
+roffnode_free(struct rofftree *tree)
 {
 	struct roffnode	*p;
 
 	assert(tree->last);
-	assert(tree->last->tok == tokid);
 
 	p = tree->last;
 	tree->last = tree->last->parent;
@@ -449,83 +778,328 @@ roffnode_free(int tokid, struct rofftree *tree)
 }
 
 
-static int dbg_lvl = 0;
+static int
+roffspecial(struct rofftree *tree, int tok, const char *start, 
+		const int *argc, const char **argv, 
+		size_t sz, char **ordp)
+{
 
+	switch (tok) {
+	case (ROFF_At):
+		if (0 == sz)
+			break;
+		if (0 == strcmp(*ordp, "v1"))
+			break;
+		else if (0 == strcmp(*ordp, "v2")) 
+			break;
+		else if (0 == strcmp(*ordp, "v3")) 
+			break;
+		else if (0 == strcmp(*ordp, "v6")) 
+			break;
+		else if (0 == strcmp(*ordp, "v7")) 
+			break;
+		else if (0 == strcmp(*ordp, "32v"))
+			break;
+		else if (0 == strcmp(*ordp, "V.1"))
+			break;
+		else if (0 == strcmp(*ordp, "V.4"))
+			break;
+		roff_err(tree, *ordp, "invalid `At' arg");
+		return(0);
+	
+	case (ROFF_Xr):
+		if (2 == sz) {
+			assert(ordp[1]);
+			if (ROFF_MSEC_MAX != roffismsec(ordp[1]))
+				break;
+			roff_warn(tree, start, "invalid `%s' manual "
+					"section", toknames[tok]);
+		}
+		/* FALLTHROUGH */
 
-static void
-dbg_enter(const struct md_args *args, int tokid)
-{
-	int		 i;
-	static char	 buf[72];
+	case (ROFF_Sx):
+		/* FALLTHROUGH*/
+	case (ROFF_Fn):
+		if (0 != sz) 
+			break;
+		roff_err(tree, start, "`%s' expects at least "
+				"one arg", toknames[tok]);
+		return(0);
 
-	assert(args);
-	if ( ! (args->dbg & MD_DBG_TREE))
-		return;
-	assert(tokid >= 0 && tokid <= ROFF_MAX);
+	case (ROFF_Nm):
+		if (0 == sz) {
+			if (0 == tree->name[0]) {
+				roff_err(tree, start, "`Nm' not set");
+				return(0);
+			}
+			ordp[0] = tree->name;
+			ordp[1] = NULL;
+		} else if ( ! roffsetname(tree, ordp))
+			return(0);
+		break;
 
-	buf[0] = buf[71] = 0;
+	case (ROFF_Rv):
+		/* FALLTHROUGH*/
+	case (ROFF_Ex):
+		if (1 == sz) 
+			break;
+		roff_err(tree, start, "`%s' expects one arg", 
+				toknames[tok]);
+		return(0);
 
-	switch (tokens[tokid].type) {
-	case (ROFF_LAYOUT):
-		(void)strncat(buf, "[body-layout] ", sizeof(buf) - 1);
+	case (ROFF_Sm):
+		if (1 != sz) {
+			roff_err(tree, start, "`Sm' expects one arg");
+			return(0);
+		} 
+		
+		if (0 != strcmp(ordp[0], "on") &&
+				0 != strcmp(ordp[0], "off")) {
+			roff_err(tree, start, "`Sm' has invalid argument");
+			return(0);
+		}
 		break;
-	case (ROFF_TEXT):
-		(void)strncat(buf, "[  body-text] ", sizeof(buf) - 1);
+	
+	case (ROFF_Ud):
+		/* FALLTHROUGH */
+	case (ROFF_Ux):
+		/* FALLTHROUGH */
+	case (ROFF_Bt):
+		if (0 != sz) {
+			roff_err(tree, start, "`%s' expects no args",
+					toknames[tok]);
+			return(0);
+		}
 		break;
-	case (ROFF_TITLE):
-		(void)strncat(buf, "[    prelude] ", sizeof(buf) - 1);
-		break;
 	default:
-		abort();
+		break;
 	}
 
+	return((*tree->cb.roffspecial)(tree->arg, tok, tree->cur, 
+				argc, argv, (const char **)ordp));
+}
+
+
+static int
+roffexit(struct rofftree *tree, int tok)
+{
+
+	assert(tokens[tok].cb);
+	return((*tokens[tok].cb)(tok, tree, NULL, ROFF_EXIT));
+}
+
+
+static int
+roffcall(struct rofftree *tree, int tok, char **argv)
+{
+	int		 i;
+	enum roffmsec	 c;
+
+	if (NULL == tokens[tok].cb) {
+		roff_err(tree, *argv, "`%s' is unsupported", 
+				toknames[tok]);
+		return(0);
+	}
+	if (tokens[tok].sections && ROFF_MSEC_MAX != tree->section) {
+		i = 0;
+		while (ROFF_MSEC_MAX != 
+				(c = tokens[tok].sections[i++]))
+			if (c == tree->section)
+				break;
+		if (ROFF_MSEC_MAX == c) {
+			roff_warn(tree, *argv, "`%s' is not a valid "
+					"macro in this manual section",
+					toknames[tok]);
+		}
+	}
+
+	return((*tokens[tok].cb)(tok, tree, argv, ROFF_ENTER));
+}
+
+
+static int
+roffnextopt(const struct rofftree *tree, int tok, 
+		char ***in, char **val)
+{
+	char		*arg, **argv;
+	int		 v;
+
+	*val = NULL;
+	argv = *in;
+	assert(argv);
+
+	if (NULL == (arg = *argv))
+		return(-1);
+	if ('-' != *arg)
+		return(-1);
+
+	if (ROFF_ARGMAX == (v = rofffindarg(arg + 1))) {
+		roff_warn(tree, arg, "argument-like parameter `%s' to "
+				"`%s'", arg, toknames[tok]);
+		return(-1);
+	} 
+	
+	if ( ! roffargok(tok, v)) {
+		roff_warn(tree, arg, "invalid argument parameter `%s' to "
+				"`%s'", tokargnames[v], toknames[tok]);
+		return(-1);
+	} 
+	
+	if ( ! (ROFF_VALUE & tokenargs[v]))
+		return(v);
+
+	*in = ++argv;
+
+	if (NULL == *argv) {
+		roff_err(tree, arg, "empty value of `%s' for `%s'",
+				tokargnames[v], toknames[tok]);
+		return(ROFF_ARGMAX);
+	}
+
+	return(v);
+}
+
+
+static int
+roffpurgepunct(struct rofftree *tree, char **argv)
+{
+	int		 i;
+
+	i = 0;
+	while (argv[i])
+		i++;
+	assert(i > 0);
+	if ( ! roffispunct(argv[--i]))
+		return(1);
+	while (i >= 0 && roffispunct(argv[i]))
+		i--;
+	i++;
+
 	/* LINTED */
-	for (i = 0; i < dbg_lvl; i++)
-		(void)strncat(buf, "  ", sizeof(buf) - 1);
+	while (argv[i])
+		if ( ! roffdata(tree, 0, argv[i++]))
+			return(0);
+	return(1);
+}
 
-	(void)strncat(buf, tokens[tokid].name, sizeof(buf) - 1);
 
-	(void)printf("%s\n", buf);
+static int
+roffparseopts(struct rofftree *tree, int tok, 
+		char ***args, int *argc, char **argv)
+{
+	int		 i, c;
+	char		*v;
 
-	dbg_lvl++;
+	i = 0;
+
+	while (-1 != (c = roffnextopt(tree, tok, args, &v))) {
+		if (ROFF_ARGMAX == c) 
+			return(0);
+
+		argc[i] = c;
+		argv[i] = v;
+		i++;
+		*args = *args + 1;
+	}
+
+	argc[i] = ROFF_ARGMAX;
+	argv[i] = NULL;
+	return(1);
 }
 
 
-/* FIXME: accept only struct rofftree *. */
-static void
-dbg_leave(const struct md_args *args, int tokid)
+static int
+roffdata(struct rofftree *tree, int space, char *buf)
 {
-	assert(args);
-	if ( ! (args->dbg & MD_DBG_TREE))
-		return;
 
-	assert(tokid >= 0 && tokid <= ROFF_MAX);
-	assert(dbg_lvl > 0);
-	dbg_lvl--;
+	if (0 == *buf)
+		return(1);
+	return((*tree->cb.roffdata)(tree->arg, 
+				space != 0, tree->cur, buf));
 }
 
 
-/* FIXME: accept only struct rofftree *. */
 /* ARGSUSED */
 static	int
 roff_Dd(ROFFCALL_ARGS)
 {
+	time_t		 t;
+	char		*p, buf[32];
 
-	dbg_enter(tree->args, ROFF_Dd);
+	if (ROFF_BODY & tree->state) {
+		assert( ! (ROFF_PRELUDE & tree->state));
+		assert(ROFF_PRELUDE_Dd & tree->state);
+		return(roff_text(tok, tree, argv, type));
+	}
 
 	assert(ROFF_PRELUDE & tree->state);
-	if (ROFF_PRELUDE_Dt & tree->state ||
-			ROFF_PRELUDE_Dd & tree->state) {
-		warnx("%s: prelude `Dd' out-of-order (line %zu)",
-				tree->rbuf->name, tree->rbuf->line);
+	assert( ! (ROFF_BODY & tree->state));
+
+	if (ROFF_PRELUDE_Dd & tree->state) {
+		roff_err(tree, *argv, "repeated `Dd' in prelude");
 		return(0);
+	} else if (ROFF_PRELUDE_Dt & tree->state) {
+		roff_err(tree, *argv, "out-of-order `Dd' in prelude");
+		return(0);
 	}
 
 	assert(NULL == tree->last);
-	tree->state |= ROFF_PRELUDE_Dd;
 
-	dbg_leave(tree->args, ROFF_Dd);
+	argv++;
 
+	if (0 == strcmp(*argv, "$Mdocdate: December 8 2008 $")) {
+		t = time(NULL);
+		if (NULL == localtime_r(&t, &tree->tm))
+			err(1, "localtime_r");
+		tree->state |= ROFF_PRELUDE_Dd;
+		return(1);
+	} 
+
+	/* Build this from Mdocdate or raw date. */
+	
+	buf[0] = 0;
+	p = *argv;
+
+	if (0 != strcmp(*argv, "$Mdocdate:")) {
+		while (*argv) {
+			if (strlcat(buf, *argv++, sizeof(buf))
+					< sizeof(buf)) 
+				continue;
+			roff_err(tree, p, "bad `Dd' date");
+			return(0);
+		}
+		if (strptime(buf, "%b%d,%Y", &tree->tm)) {
+			tree->state |= ROFF_PRELUDE_Dd;
+			return(1);
+		}
+		roff_err(tree, *argv, "bad `Dd' date");
+		return(0);
+	}
+
+	argv++;
+	while (*argv && **argv != '$') {
+		if (strlcat(buf, *argv++, sizeof(buf))
+				>= sizeof(buf)) {
+			roff_err(tree, p, "bad `Dd' Mdocdate");
+			return(0);
+		} 
+		if (strlcat(buf, " ", sizeof(buf))
+				>= sizeof(buf)) {
+			roff_err(tree, p, "bad `Dd' Mdocdate");
+			return(0);
+		}
+	}
+	if (NULL == *argv) {
+		roff_err(tree, p, "bad `Dd' Mdocdate");
+		return(0);
+	}
+
+	if (NULL == strptime(buf, "%b %d %Y", &tree->tm)) {
+		roff_err(tree, *argv, "bad `Dd' Mdocdate");
+		return(0);
+	}
+
+	tree->state |= ROFF_PRELUDE_Dd;
 	return(1);
 }
 
@@ -535,167 +1109,584 @@ static	int
 roff_Dt(ROFFCALL_ARGS)
 {
 
-	dbg_enter(tree->args, ROFF_Dt);
+	if (ROFF_BODY & tree->state) {
+		assert( ! (ROFF_PRELUDE & tree->state));
+		assert(ROFF_PRELUDE_Dt & tree->state);
+		return(roff_text(tok, tree, argv, type));
+	}
 
 	assert(ROFF_PRELUDE & tree->state);
-	if ( ! (ROFF_PRELUDE_Dd & tree->state) ||
-			(ROFF_PRELUDE_Dt & tree->state)) {
-		warnx("%s: prelude `Dt' out-of-order (line %zu)",
-				tree->rbuf->name, tree->rbuf->line);
+	assert( ! (ROFF_BODY & tree->state));
+
+	if ( ! (ROFF_PRELUDE_Dd & tree->state)) {
+		roff_err(tree, *argv, "out-of-order `Dt' in prelude");
 		return(0);
+	} else if (ROFF_PRELUDE_Dt & tree->state) {
+		roff_err(tree, *argv, "repeated `Dt' in prelude");
+		return(0);
 	}
 
+	argv++;
+	if (NULL == *argv) {
+		roff_err(tree, *argv, "`Dt' needs document title");
+		return(0);
+	} else if (strlcpy(tree->title, *argv, sizeof(tree->title))
+			>= sizeof(tree->title)) {
+		roff_err(tree, *argv, "`Dt' document title too long");
+		return(0);
+	}
+
+	argv++;
+	if (NULL == *argv) {
+		roff_err(tree, *argv, "`Dt' needs section");
+		return(0);
+	} 
+
+	if (ROFF_MSEC_MAX == (tree->section = roffismsec(*argv))) {
+		roff_err(tree, *argv, "bad `Dt' section");
+		return(0);
+	}
+
+	argv++;
+	if (NULL == *argv) {
+		tree->volume[0] = 0;
+	} else if (strlcpy(tree->volume, *argv, sizeof(tree->volume))
+			>= sizeof(tree->volume)) {
+		roff_err(tree, *argv, "`Dt' volume too long");
+		return(0);
+	}
+
 	assert(NULL == tree->last);
 	tree->state |= ROFF_PRELUDE_Dt;
 
-	dbg_leave(tree->args, ROFF_Dt);
+	return(1);
+}
 
+
+static int
+roffsetname(struct rofftree *tree, char **ordp)
+{
+	
+	assert(*ordp);
+
+	/* FIXME: not all sections can set this. */
+
+	if (NULL != *(ordp + 1)) {
+		roff_err(tree, *ordp, "too many `Nm' args");
+		return(0);
+	} 
+	
+	if (strlcpy(tree->name, *ordp, sizeof(tree->name)) 
+			>= sizeof(tree->name)) {
+		roff_err(tree, *ordp, "`Nm' arg too long");
+		return(0);
+	}
+
 	return(1);
 }
 
 
 /* ARGSUSED */
 static	int
-roff_Os(ROFFCALL_ARGS)
+roff_Ns(ROFFCALL_ARGS)
 {
+	int		 j, c, first;
+	char		*morep[1];
 
-	if (ROFF_EXIT == type) {
-		roffnode_free(ROFF_Os, tree);
-		dbg_leave(tree->args, ROFF_Os);
+	first = (*argv++ == tree->cur);
+	morep[0] = NULL;
+
+	if ( ! roffspecial(tree, tok, *argv, NULL, NULL, 0, morep))
+		return(0);
+
+	while (*argv) {
+		if (ROFF_MAX != (c = rofffindcallable(*argv))) {
+			if ( ! roffcall(tree, c, argv))
+				return(0);
+			break;
+		}
+
+		if ( ! roffispunct(*argv)) {
+			if ( ! roffdata(tree, 1, *argv++))
+				return(0);
+			continue;
+		}
+
+		for (j = 0; argv[j]; j++)
+			if ( ! roffispunct(argv[j]))
+				break;
+
+		if (argv[j]) {
+			if ( ! roffdata(tree, 0, *argv++))
+				return(0);
+			continue;
+		}
+
+		break;
+	}
+
+	if ( ! first)
 		return(1);
-	} 
 
-	dbg_enter(tree->args, ROFF_Os);
+	return(roffpurgepunct(tree, argv));
+}
 
+
+/* ARGSUSED */
+static	int
+roff_Os(ROFFCALL_ARGS)
+{
+	char		*p;
+
+	if (ROFF_BODY & tree->state) {
+		assert( ! (ROFF_PRELUDE & tree->state));
+		assert(ROFF_PRELUDE_Os & tree->state);
+		return(roff_text(tok, tree, argv, type));
+	}
+
 	assert(ROFF_PRELUDE & tree->state);
 	if ( ! (ROFF_PRELUDE_Dt & tree->state) ||
 			! (ROFF_PRELUDE_Dd & tree->state)) {
-		warnx("%s: prelude `Os' out-of-order (line %zu)",
-				tree->rbuf->name, tree->rbuf->line);
+		roff_err(tree, *argv, "out-of-order `Os' in prelude");
 		return(0);
 	}
 
-	assert(NULL == tree->last);
-	if (NULL == roffnode_new(ROFF_Os, tree->rbuf->line, tree))
+	tree->os[0] = 0;
+
+	p = *++argv;
+
+	while (*argv) {
+		if (strlcat(tree->os, *argv++, sizeof(tree->os))
+				< sizeof(tree->os)) 
+			continue;
+		roff_err(tree, p, "`Os' value too long");
 		return(0);
+	}
 
+	if (0 == tree->os[0])
+		if (strlcpy(tree->os, "LOCAL", sizeof(tree->os))
+				>= sizeof(tree->os)) {
+			roff_err(tree, p, "`Os' value too long");
+			return(0);
+		}
+
 	tree->state |= ROFF_PRELUDE_Os;
 	tree->state &= ~ROFF_PRELUDE;
 	tree->state |= ROFF_BODY;
 
-	return(1);
+	assert(ROFF_MSEC_MAX != tree->section);
+	assert(0 != tree->title[0]);
+	assert(0 != tree->os[0]);
+
+	assert(NULL == tree->last);
+
+	return((*tree->cb.roffhead)(tree->arg, &tree->tm,
+				tree->os, tree->title, tree->section,
+				tree->volume));
 }
 
 
 /* ARGSUSED */
 static int
-roff_Sh(ROFFCALL_ARGS)
+roff_layout(ROFFCALL_ARGS) 
 {
+	int		 i, c, argcp[ROFF_MAXLINEARG];
+	char		*argvp[ROFF_MAXLINEARG];
 
-	if (ROFF_EXIT == type) {
-		roffnode_free(ROFF_Sh, tree);
-		dbg_leave(tree->args, ROFF_Sh);
-		return(1);
+	/*
+	 * The roff_layout function is for multi-line macros.  A layout
+	 * has a start and end point, which is either declared
+	 * explicitly or implicitly.  An explicit start and end is
+	 * embodied by `.Bl' and `.El', with the former being the start
+	 * and the latter being an end.  The `.Sh' and `.Ss' tags, on
+	 * the other hand, are implicit.  The scope of a layout is the
+	 * space between start and end.  Explicit layouts may not close
+	 * out implicit ones and vice versa; implicit layouts may close
+	 * out other implicit layouts.
+	 */
+
+	assert( ! (ROFF_CALLABLE & tokens[tok].flags));
+
+	if (ROFF_PRELUDE & tree->state) {
+		roff_err(tree, *argv, "bad `%s' in prelude", 
+				toknames[tok]);
+		return(0);
+	} else if (ROFF_EXIT == type) {
+		roffnode_free(tree);
+		if ( ! (*tree->cb.roffblkbodyout)(tree->arg, tok))
+			return(0);
+		return((*tree->cb.roffblkout)(tree->arg, tok));
 	} 
 
-	dbg_enter(tree->args, ROFF_Sh);
+	assert( ! (ROFF_CALLABLE & tokens[tok].flags));
 
-	if (NULL == roffnode_new(ROFF_Sh, tree->rbuf->line, tree))
+	++argv;
+
+	if ( ! roffparseopts(tree, tok, &argv, argcp, argvp))
 		return(0);
+	if (NULL == roffnode_new(tok, tree))
+		return(0);
 
-	return(1);
+	/*
+	 * Layouts have two parts: the layout body and header.  The
+	 * layout header is the trailing text of the line macro, while
+	 * the layout body is everything following until termination.
+	 * Example:
+	 *
+	 * .It Fl f ) ;
+	 * Bar.
+	 *
+	 * ...Produces...
+	 *
+	 * <block>
+	 * 	<head>
+	 * 		<!Fl f!> ;
+	 * 	</head>
+	 * 	
+	 *	<body>
+	 *		Bar.
+	 *	</body>
+	 * </block>
+	 */
+
+	if ( ! (*tree->cb.roffblkin)(tree->arg, tok, argcp, 
+				(const char **)argvp))
+		return(0);
+
+	/* +++ Begin run macro-specific hooks over argv. */
+
+	switch (tok) {
+	case (ROFF_Sh):
+		if (NULL == *argv) {
+			roff_err(tree, *(argv - 1), 
+					"`Sh' expects arguments");
+			return(0);
+		}
+		tree->csec = roffissec((const char **)argv);
+		if ( ! (ROFFSec_OTHER & tree->csec) &&
+				tree->asec & tree->csec) 
+			roff_warn(tree, *argv, "section repeated");
+		if (0 == tree->asec && ! (ROFFSec_NAME & tree->csec)) {
+			roff_err(tree, *argv, "`NAME' section "
+					"must be first");
+			return(0);
+		} else if ( ! roffchecksec(tree, *argv, tree->csec))
+			return(0);
+
+		tree->asec |= tree->csec;
+		break;
+	default:
+		break;
+	}
+
+	/* --- End run macro-specific hooks over argv. */
+
+	if (NULL == *argv)
+		return((*tree->cb.roffblkbodyin)
+				(tree->arg, tok, argcp, 
+				 (const char **)argvp));
+
+	if ( ! (*tree->cb.roffblkheadin)(tree->arg, tok, argcp, 
+				(const char **)argvp))
+		return(0);
+
+	/*
+	 * If there are no parsable parts, then write remaining tokens
+	 * into the layout header and exit.
+	 */
+
+	if ( ! (ROFF_PARSED & tokens[tok].flags)) {
+		i = 0;
+		while (*argv)
+			if ( ! roffdata(tree, i++, *argv++))
+				return(0);
+
+		if ( ! (*tree->cb.roffblkheadout)(tree->arg, tok))
+			return(0);
+		return((*tree->cb.roffblkbodyin)(tree->arg, tok, argcp,
+				 (const char **)argvp));
+	}
+
+	/*
+	 * Parsable elements may be in the header (or be the header, for
+	 * that matter).  Follow the regular parsing rules for these.
+	 */
+
+	i = 0;
+	while (*argv) {
+		if (ROFF_MAX == (c = rofffindcallable(*argv))) {
+			assert(tree->arg);
+			if ( ! roffdata(tree, i++, *argv++))
+				return(0);
+			continue;
+		}
+		if ( ! roffcall(tree, c, argv))
+			return(0);
+		break;
+	}
+
+	/* 
+	 * If there's trailing punctuation in the header, then write it
+	 * out now.  Here we mimic the behaviour of a line-dominant text
+	 * macro.
+	 */
+
+	if (NULL == *argv) {
+		if ( ! (*tree->cb.roffblkheadout)(tree->arg, tok))
+			return(0);
+		return((*tree->cb.roffblkbodyin)
+				(tree->arg, tok, argcp, 
+				 (const char **)argvp));
+	}
+
+	/*
+	 * Expensive.  Scan to the end of line then work backwards until
+	 * a token isn't punctuation.
+	 */
+
+	if ( ! roffpurgepunct(tree, argv))
+		return(0);
+	if ( ! (*tree->cb.roffblkheadout)(tree->arg, tok))
+		return(0);
+	return((*tree->cb.roffblkbodyin)(tree->arg, 
+				tok, argcp, (const char **)argvp));
 }
 
 
 /* ARGSUSED */
 static int
-roff_Li(ROFFCALL_ARGS) 
+roff_ordered(ROFFCALL_ARGS) 
 {
+	int		 i, first, c, argcp[ROFF_MAXLINEARG];
+	char		*ordp[ROFF_MAXLINEARG], *p,
+			*argvp[ROFF_MAXLINEARG];
 
-	dbg_enter(tree->args, ROFF_Li);
-	dbg_leave(tree->args, ROFF_Li);
+	/*
+	 * Ordered macros pass their arguments directly to handlers,
+	 * instead of considering it free-form text.  Thus, the
+	 * following macro looks as follows:
+	 *
+	 * .Xr foo 1 ) ,
+	 *
+	 * .Xr arg1 arg2 punctuation
+	 */
 
-	return(1);
-}
+	if (ROFF_PRELUDE & tree->state) {
+		roff_err(tree, *argv, "`%s' disallowed in prelude", 
+				toknames[tok]);
+		return(0);
+	}
 
+	first = (*argv == tree->cur);
+	p = *argv++;
+	ordp[0] = NULL;
 
-static int
-roffnextopt(const char ***in, char **val)
-{
-	const char	*arg, **argv;
-	int		 v;
+	if ( ! roffparseopts(tree, tok, &argv, argcp, argvp))
+		return(0);
 
-	*val = NULL;
-	argv = *in;
-	assert(argv);
+	if (NULL == *argv)
+		return(roffspecial(tree, tok, p, argcp, 
+					(const char **)argvp, 0, ordp));
 
-	if (NULL == (arg = *argv))
-		return(-1);
-	if ('-' != *arg)
-		return(-1);
-	if (ROFF_ARGMAX == (v = rofffindarg(&arg[1])))
-		return(-1);
-	if ( ! (ROFF_VALUE & tokenargs[v].flags))
-		return(v);
+	i = 0;
+	while (*argv && i < ROFF_MAXLINEARG) {
+		c = ROFF_PARSED & tokens[tok].flags ?
+			rofffindcallable(*argv) : ROFF_MAX;
 
-	*in = ++argv;
+		if (ROFF_MAX == c && ! roffispunct(*argv)) {
+			ordp[i++] = *argv++;
+			continue;
+		}
+		ordp[i] = NULL;
 
-	/* FIXME: what if this looks like a roff token or argument? */
+		if (ROFF_MAX == c)
+			break;
 
-	return(*argv ? v : ROFF_ARGMAX);
+		if ( ! roffspecial(tree, tok, p, argcp, 
+					(const char **)argvp,
+					(size_t)i, ordp))
+			return(0);
+
+		return(roffcall(tree, c, argv));
+	}
+
+	assert(i != ROFF_MAXLINEARG);
+	ordp[i] = NULL;
+
+	if ( ! roffspecial(tree, tok, p, argcp, 
+				(const char**)argvp,
+				(size_t)i, ordp))
+		return(0);
+
+	/* FIXME: error if there's stuff after the punctuation. */
+
+	if ( ! first || NULL == *argv)
+		return(1);
+
+	return(roffpurgepunct(tree, argv));
 }
 
 
 /* ARGSUSED */
 static int
-roff_An(ROFFCALL_ARGS) 
+roff_text(ROFFCALL_ARGS) 
 {
-	int		 c;
-	char		*val;
+	int		 i, j, first, c, argcp[ROFF_MAXLINEARG];
+	char		*argvp[ROFF_MAXLINEARG];
 
-	dbg_enter(tree->args, ROFF_An);
+	/*
+	 * Text macros are similar to special tokens, except that
+	 * arguments are instead flushed as pure data: we're only
+	 * concerned with the macro and its arguments.  Example:
+	 * 
+	 * .Fl v W f ;
+	 *
+	 * ...Produces...
+	 *
+	 * <fl> v W f </fl> ;
+	 */
 
-	while (-1 != (c = roffnextopt(&argv, &val))) {
-		switch (c) {
-		case (ROFF_Split):
-			/* Process argument. */
-			break;
-		case (ROFF_Nosplit):
-			/* Process argument. */
-			break;
-		default:
-			warnx("%s: error parsing `An' args (line %zu)", 
-					tree->rbuf->name, 
-					tree->rbuf->line);
-			return(0);
-		}
-		argv++;
+	if (ROFF_PRELUDE & tree->state) {
+		roff_err(tree, *argv, "`%s' disallowed in prelude", 
+				toknames[tok]);
+		return(0);
 	}
 
-	/* Print header. */
-  
+	first = (*argv == tree->cur);
+	argv++;
+
+	if ( ! roffparseopts(tree, tok, &argv, argcp, argvp))
+		return(0);
+	if ( ! (*tree->cb.roffin)(tree->arg, tok, argcp, 
+				(const char **)argvp))
+		return(0);
+	if (NULL == *argv)
+		return((*tree->cb.roffout)(tree->arg, tok));
+
+	if ( ! (ROFF_PARSED & tokens[tok].flags)) {
+		i = 0;
+		while (*argv)
+			if ( ! roffdata(tree, i++, *argv++))
+				return(0);
+
+		return((*tree->cb.roffout)(tree->arg, tok));
+	}
+
+	/*
+	 * Deal with punctuation.  Ugly.  Work ahead until we encounter
+	 * terminating punctuation.  If we encounter it and all
+	 * subsequent tokens are punctuation, then stop processing (the
+	 * line-dominant macro will print these tokens after closure).
+	 * If the punctuation is followed by non-punctuation, then close
+	 * and re-open our scope, then continue.
+	 */
+
+	i = 0;
 	while (*argv) {
-		if (/* is_parsable && */ 2 >= strlen(*argv)) {
-			if (ROFF_MAX != (c = rofffindtok(*argv))) {
-				if (ROFF_CALLABLE & tokens[c].flags) {
-					/* Call to token. */
-					if ( ! (*tokens[c].cb)(tree, (const char **)argv + 1, ROFF_ENTER))
-						return(0);
-				} 
-				/* Print token. */
-			} else {
-				/* Print token. */
+		if (ROFF_MAX != (c = rofffindcallable(*argv))) {
+			if ( ! (ROFF_LSCOPE & tokens[tok].flags))
+				if ( ! (*tree->cb.roffout)(tree->arg, tok))
+					return(0);
+	
+			if ( ! roffcall(tree, c, argv))
+				return(0);
+	
+			if (ROFF_LSCOPE & tokens[tok].flags)
+				if ( ! (*tree->cb.roffout)(tree->arg, tok))
+					return(0);
+	
+			break;
+		}
+
+		if ( ! roffispunct(*argv)) {
+			if ( ! roffdata(tree, i++, *argv++))
+				return(0);
+			continue;
+		}
+
+		i = 1;
+		for (j = 0; argv[j]; j++)
+			if ( ! roffispunct(argv[j]))
+				break;
+
+		if (argv[j]) {
+			if (ROFF_LSCOPE & tokens[tok].flags) {
+				if ( ! roffdata(tree, 0, *argv++))
+					return(0);
+				continue;
 			}
-		} else {
-			/* Print token. */
+			if ( ! (*tree->cb.roffout)(tree->arg, tok))
+				return(0);
+			if ( ! roffdata(tree, 0, *argv++))
+				return(0);
+			if ( ! (*tree->cb.roffin)(tree->arg, tok, 
+						argcp, 
+						(const char **)argvp))
+				return(0);
+
+			i = 0;
+			continue;
 		}
-		argv++;
+
+		if ( ! (*tree->cb.roffout)(tree->arg, tok))
+			return(0);
+		break;
 	}
 
-	/* Print footer. */
+	if (NULL == *argv)
+		return((*tree->cb.roffout)(tree->arg, tok));
+	if ( ! first)
+		return(1);
 
-	dbg_leave(tree->args, ROFF_An);
+	return(roffpurgepunct(tree, argv));
+}
 
+
+/* ARGSUSED */
+static int
+roff_noop(ROFFCALL_ARGS)
+{
+
 	return(1);
+}
+
+
+/* ARGSUSED */
+static int
+roff_depr(ROFFCALL_ARGS)
+{
+
+	roff_err(tree, *argv, "`%s' is deprecated", toknames[tok]);
+	return(0);
+}
+
+
+static void
+roff_warn(const struct rofftree *tree, const char *pos, char *fmt, ...)
+{
+	va_list		 ap;
+	char		 buf[128];
+
+	va_start(ap, fmt);
+	(void)vsnprintf(buf, sizeof(buf), fmt, ap);
+	va_end(ap);
+
+	(*tree->cb.roffmsg)(tree->arg, 
+			ROFF_WARN, tree->cur, pos, buf);
+}
+
+
+static void
+roff_err(const struct rofftree *tree, const char *pos, char *fmt, ...)
+{
+	va_list		 ap;
+	char		 buf[128];
+
+	va_start(ap, fmt);
+	(void)vsnprintf(buf, sizeof(buf), fmt, ap);
+	va_end(ap);
+
+	(*tree->cb.roffmsg)(tree->arg, 
+			ROFF_ERROR, tree->cur, pos, buf);
 }