=================================================================== RCS file: /cvs/mandoc/mandocdb.c,v retrieving revision 1.33 retrieving revision 1.197 diff -u -p -r1.33 -r1.197 --- mandoc/mandocdb.c 2011/12/10 22:01:06 1.33 +++ mandoc/mandocdb.c 2015/10/06 18:32:19 1.197 @@ -1,914 +1,1621 @@ -/* $Id: mandocdb.c,v 1.33 2011/12/10 22:01:06 schwarze Exp $ */ +/* $Id: mandocdb.c,v 1.197 2015/10/06 18:32:19 schwarze Exp $ */ /* - * Copyright (c) 2011 Kristaps Dzonsons - * Copyright (c) 2011 Ingo Schwarze + * Copyright (c) 2011, 2012 Kristaps Dzonsons + * Copyright (c) 2011-2015 Ingo Schwarze * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -#ifdef HAVE_CONFIG_H #include "config.h" -#endif -#include #include #include +#include #include -#include +#include +#include #include +#if HAVE_FTS +#include +#else +#include "compat_fts.h" +#endif #include +#include +#include #include #include #include #include #include -#if defined(__linux__) -# include -# include -#elif defined(__APPLE__) -# include -# include +#if HAVE_OHASH +#include #else -# include +#include "compat_ohash.h" #endif +#include -#include "man.h" -#include "mdoc.h" +#include "mandoc_aux.h" #include "mandoc.h" -#include "mandocdb.h" -#include "manpath.h" +#include "roff.h" +#include "mdoc.h" +#include "man.h" +#include "manconf.h" +#include "mansearch.h" -#define MANDOC_BUFSZ BUFSIZ -#define MANDOC_SLOP 1024 +extern int mansearch_keymax; +extern const char *const mansearch_keynames[]; -#define MANDOC_SRC 0x1 -#define MANDOC_FORM 0x2 +#define SQL_EXEC(_v) \ + if (SQLITE_OK != sqlite3_exec(db, (_v), NULL, NULL, NULL)) \ + say("", "%s: %s", (_v), sqlite3_errmsg(db)) +#define SQL_BIND_TEXT(_s, _i, _v) \ + if (SQLITE_OK != sqlite3_bind_text \ + ((_s), (_i)++, (_v), -1, SQLITE_STATIC)) \ + say(mlink->file, "%s", sqlite3_errmsg(db)) +#define SQL_BIND_INT(_s, _i, _v) \ + if (SQLITE_OK != sqlite3_bind_int \ + ((_s), (_i)++, (_v))) \ + say(mlink->file, "%s", sqlite3_errmsg(db)) +#define SQL_BIND_INT64(_s, _i, _v) \ + if (SQLITE_OK != sqlite3_bind_int64 \ + ((_s), (_i)++, (_v))) \ + say(mlink->file, "%s", sqlite3_errmsg(db)) +#define SQL_STEP(_s) \ + if (SQLITE_DONE != sqlite3_step((_s))) \ + say(mlink->file, "%s", sqlite3_errmsg(db)) -/* Tiny list for files. No need to bring in QUEUE. */ +enum op { + OP_DEFAULT = 0, /* new dbs from dir list or default config */ + OP_CONFFILE, /* new databases from custom config file */ + OP_UPDATE, /* delete/add entries in existing database */ + OP_DELETE, /* delete entries from existing database */ + OP_TEST /* change no databases, report potential problems */ +}; -struct of { - char *fname; /* heap-allocated */ - char *sec; - char *arch; - char *title; - int src_form; - struct of *next; /* NULL for last one */ - struct of *first; /* first in list */ +struct str { + const struct mpage *mpage; /* if set, the owning parse */ + uint64_t mask; /* bitmask in sequence */ + char key[]; /* rendered text */ }; -/* Buffer for storing growable data. */ +struct inodev { + ino_t st_ino; + dev_t st_dev; +}; -struct buf { - char *cp; - size_t len; /* current length */ - size_t size; /* total buffer size */ +struct mpage { + struct inodev inodev; /* used for hashing routine */ + int64_t pageid; /* pageid in mpages SQL table */ + char *sec; /* section from file content */ + char *arch; /* architecture from file content */ + char *title; /* title from file content */ + char *desc; /* description from file content */ + struct mlink *mlinks; /* singly linked list */ + int form; /* format from file content */ + int name_head_done; }; -/* Operation we're going to perform. */ +struct mlink { + char file[PATH_MAX]; /* filename rel. to manpath */ + char *dsec; /* section from directory */ + char *arch; /* architecture from directory */ + char *name; /* name from file name (not empty) */ + char *fsec; /* section from file name suffix */ + struct mlink *next; /* singly linked list */ + struct mpage *mpage; /* parent */ + int dform; /* format from directory */ + int fform; /* format from file name suffix */ + int gzip; /* filename has a .gz suffix */ +}; -enum op { - OP_NEW = 0, /* new database */ - OP_UPDATE, /* delete/add entries in existing database */ - OP_DELETE /* delete entries from existing database */ +enum stmt { + STMT_DELETE_PAGE = 0, /* delete mpage */ + STMT_INSERT_PAGE, /* insert mpage */ + STMT_INSERT_LINK, /* insert mlink */ + STMT_INSERT_NAME, /* insert name */ + STMT_SELECT_NAME, /* retrieve existing name flags */ + STMT_INSERT_KEY, /* insert parsed key */ + STMT__MAX }; -#define MAN_ARGS DB *hash, \ - struct buf *buf, \ - struct buf *dbuf, \ - const struct man_node *n -#define MDOC_ARGS DB *hash, \ - struct buf *buf, \ - struct buf *dbuf, \ - const struct mdoc_node *n, \ - const struct mdoc_meta *m +typedef int (*mdoc_fp)(struct mpage *, const struct roff_meta *, + const struct roff_node *); -static void buf_appendmdoc(struct buf *, - const struct mdoc_node *, int); -static void buf_append(struct buf *, const char *); -static void buf_appendb(struct buf *, - const void *, size_t); -static void dbt_put(DB *, const char *, DBT *, DBT *); -static void hash_put(DB *, const struct buf *, uint64_t); -static void hash_reset(DB **); -static void index_merge(const struct of *, struct mparse *, - struct buf *, struct buf *, DB *, - DB *, const char *, DB *, const char *, - recno_t, const recno_t *, size_t); -static void index_prune(const struct of *, DB *, - const char *, DB *, const char *, - recno_t *, recno_t **, size_t *, - size_t *); -static void ofile_argbuild(int, char *[], struct of **); -static int ofile_dirbuild(const char *, const char *, - const char *, int, struct of **); -static void ofile_free(struct of *); -static void pformatted(DB *, struct buf *, struct buf *, - const struct of *); -static int pman_node(MAN_ARGS); -static void pmdoc_node(MDOC_ARGS); -static int pmdoc_head(MDOC_ARGS); -static int pmdoc_body(MDOC_ARGS); -static int pmdoc_Fd(MDOC_ARGS); -static int pmdoc_In(MDOC_ARGS); -static int pmdoc_Fn(MDOC_ARGS); -static int pmdoc_Nd(MDOC_ARGS); -static int pmdoc_Nm(MDOC_ARGS); -static int pmdoc_Sh(MDOC_ARGS); -static int pmdoc_St(MDOC_ARGS); -static int pmdoc_Xr(MDOC_ARGS); -static void usage(void); - -#define MDOCF_CHILD 0x01 /* Automatically index child nodes. */ - struct mdoc_handler { - int (*fp)(MDOC_ARGS); /* Optional handler. */ - uint64_t mask; /* Set unless handler returns 0. */ - int flags; /* For use by pmdoc_node. */ + mdoc_fp fp; /* optional handler */ + uint64_t mask; /* set unless handler returns 0 */ }; +static void dbclose(int); +static void dbadd(struct mpage *); +static void dbadd_mlink(const struct mlink *mlink); +static void dbadd_mlink_name(const struct mlink *mlink); +static int dbopen(int); +static void dbprune(void); +static void filescan(const char *); +static void *hash_alloc(size_t, void *); +static void hash_free(void *, void *); +static void *hash_calloc(size_t, size_t, void *); +static void mlink_add(struct mlink *, const struct stat *); +static void mlink_check(struct mpage *, struct mlink *); +static void mlink_free(struct mlink *); +static void mlinks_undupe(struct mpage *); +static void mpages_free(void); +static void mpages_merge(struct mparse *); +static void names_check(void); +static void parse_cat(struct mpage *, int); +static void parse_man(struct mpage *, const struct roff_meta *, + const struct roff_node *); +static void parse_mdoc(struct mpage *, const struct roff_meta *, + const struct roff_node *); +static int parse_mdoc_body(struct mpage *, const struct roff_meta *, + const struct roff_node *); +static int parse_mdoc_head(struct mpage *, const struct roff_meta *, + const struct roff_node *); +static int parse_mdoc_Fd(struct mpage *, const struct roff_meta *, + const struct roff_node *); +static void parse_mdoc_fname(struct mpage *, const struct roff_node *); +static int parse_mdoc_Fn(struct mpage *, const struct roff_meta *, + const struct roff_node *); +static int parse_mdoc_Fo(struct mpage *, const struct roff_meta *, + const struct roff_node *); +static int parse_mdoc_Nd(struct mpage *, const struct roff_meta *, + const struct roff_node *); +static int parse_mdoc_Nm(struct mpage *, const struct roff_meta *, + const struct roff_node *); +static int parse_mdoc_Sh(struct mpage *, const struct roff_meta *, + const struct roff_node *); +static int parse_mdoc_Xr(struct mpage *, const struct roff_meta *, + const struct roff_node *); +static void putkey(const struct mpage *, char *, uint64_t); +static void putkeys(const struct mpage *, char *, size_t, uint64_t); +static void putmdockey(const struct mpage *, + const struct roff_node *, uint64_t); +static int render_string(char **, size_t *); +static void say(const char *, const char *, ...); +static int set_basedir(const char *, int); +static int treescan(void); +static size_t utf8(unsigned int, char [7]); + +static char tempfilename[32]; +static char *progname; +static int nodb; /* no database changes */ +static int mparse_options; /* abort the parse early */ +static int use_all; /* use all found files */ +static int debug; /* print what we're doing */ +static int warnings; /* warn about crap */ +static int write_utf8; /* write UTF-8 output; else ASCII */ +static int exitcode; /* to be returned by main */ +static enum op op; /* operational mode */ +static char basedir[PATH_MAX]; /* current base directory */ +static struct mchars *mchars; /* table of named characters */ +static struct ohash mpages; /* table of distinct manual pages */ +static struct ohash mlinks; /* table of directory entries */ +static struct ohash names; /* table of all names */ +static struct ohash strings; /* table of all strings */ +static sqlite3 *db = NULL; /* current database */ +static sqlite3_stmt *stmts[STMT__MAX]; /* current statements */ +static uint64_t name_mask; + static const struct mdoc_handler mdocs[MDOC_MAX] = { - { NULL, 0, 0 }, /* Ap */ - { NULL, 0, 0 }, /* Dd */ - { NULL, 0, 0 }, /* Dt */ - { NULL, 0, 0 }, /* Os */ - { pmdoc_Sh, TYPE_Sh, MDOCF_CHILD }, /* Sh */ - { pmdoc_head, TYPE_Ss, MDOCF_CHILD }, /* Ss */ - { NULL, 0, 0 }, /* Pp */ - { NULL, 0, 0 }, /* D1 */ - { NULL, 0, 0 }, /* Dl */ - { NULL, 0, 0 }, /* Bd */ - { NULL, 0, 0 }, /* Ed */ - { NULL, 0, 0 }, /* Bl */ - { NULL, 0, 0 }, /* El */ - { NULL, 0, 0 }, /* It */ - { NULL, 0, 0 }, /* Ad */ - { NULL, TYPE_An, MDOCF_CHILD }, /* An */ - { NULL, TYPE_Ar, MDOCF_CHILD }, /* Ar */ - { NULL, TYPE_Cd, MDOCF_CHILD }, /* Cd */ - { NULL, TYPE_Cm, MDOCF_CHILD }, /* Cm */ - { NULL, TYPE_Dv, MDOCF_CHILD }, /* Dv */ - { NULL, TYPE_Er, MDOCF_CHILD }, /* Er */ - { NULL, TYPE_Ev, MDOCF_CHILD }, /* Ev */ - { NULL, 0, 0 }, /* Ex */ - { NULL, TYPE_Fa, MDOCF_CHILD }, /* Fa */ - { pmdoc_Fd, TYPE_In, 0 }, /* Fd */ - { NULL, TYPE_Fl, MDOCF_CHILD }, /* Fl */ - { pmdoc_Fn, 0, 0 }, /* Fn */ - { NULL, TYPE_Ft, MDOCF_CHILD }, /* Ft */ - { NULL, TYPE_Ic, MDOCF_CHILD }, /* Ic */ - { pmdoc_In, TYPE_In, 0 }, /* In */ - { NULL, TYPE_Li, MDOCF_CHILD }, /* Li */ - { pmdoc_Nd, TYPE_Nd, MDOCF_CHILD }, /* Nd */ - { pmdoc_Nm, TYPE_Nm, MDOCF_CHILD }, /* Nm */ - { NULL, 0, 0 }, /* Op */ - { NULL, 0, 0 }, /* Ot */ - { NULL, TYPE_Pa, MDOCF_CHILD }, /* Pa */ - { NULL, 0, 0 }, /* Rv */ - { pmdoc_St, TYPE_St, 0 }, /* St */ - { NULL, TYPE_Va, MDOCF_CHILD }, /* Va */ - { pmdoc_body, TYPE_Va, MDOCF_CHILD }, /* Vt */ - { pmdoc_Xr, TYPE_Xr, 0 }, /* Xr */ - { NULL, 0, 0 }, /* %A */ - { NULL, 0, 0 }, /* %B */ - { NULL, 0, 0 }, /* %D */ - { NULL, 0, 0 }, /* %I */ - { NULL, 0, 0 }, /* %J */ - { NULL, 0, 0 }, /* %N */ - { NULL, 0, 0 }, /* %O */ - { NULL, 0, 0 }, /* %P */ - { NULL, 0, 0 }, /* %R */ - { NULL, 0, 0 }, /* %T */ - { NULL, 0, 0 }, /* %V */ - { NULL, 0, 0 }, /* Ac */ - { NULL, 0, 0 }, /* Ao */ - { NULL, 0, 0 }, /* Aq */ - { NULL, TYPE_At, MDOCF_CHILD }, /* At */ - { NULL, 0, 0 }, /* Bc */ - { NULL, 0, 0 }, /* Bf */ - { NULL, 0, 0 }, /* Bo */ - { NULL, 0, 0 }, /* Bq */ - { NULL, TYPE_Bsx, MDOCF_CHILD }, /* Bsx */ - { NULL, TYPE_Bx, MDOCF_CHILD }, /* Bx */ - { NULL, 0, 0 }, /* Db */ - { NULL, 0, 0 }, /* Dc */ - { NULL, 0, 0 }, /* Do */ - { NULL, 0, 0 }, /* Dq */ - { NULL, 0, 0 }, /* Ec */ - { NULL, 0, 0 }, /* Ef */ - { NULL, TYPE_Em, MDOCF_CHILD }, /* Em */ - { NULL, 0, 0 }, /* Eo */ - { NULL, TYPE_Fx, MDOCF_CHILD }, /* Fx */ - { NULL, TYPE_Ms, MDOCF_CHILD }, /* Ms */ - { NULL, 0, 0 }, /* No */ - { NULL, 0, 0 }, /* Ns */ - { NULL, TYPE_Nx, MDOCF_CHILD }, /* Nx */ - { NULL, TYPE_Ox, MDOCF_CHILD }, /* Ox */ - { NULL, 0, 0 }, /* Pc */ - { NULL, 0, 0 }, /* Pf */ - { NULL, 0, 0 }, /* Po */ - { NULL, 0, 0 }, /* Pq */ - { NULL, 0, 0 }, /* Qc */ - { NULL, 0, 0 }, /* Ql */ - { NULL, 0, 0 }, /* Qo */ - { NULL, 0, 0 }, /* Qq */ - { NULL, 0, 0 }, /* Re */ - { NULL, 0, 0 }, /* Rs */ - { NULL, 0, 0 }, /* Sc */ - { NULL, 0, 0 }, /* So */ - { NULL, 0, 0 }, /* Sq */ - { NULL, 0, 0 }, /* Sm */ - { NULL, 0, 0 }, /* Sx */ - { NULL, TYPE_Sy, MDOCF_CHILD }, /* Sy */ - { NULL, TYPE_Tn, MDOCF_CHILD }, /* Tn */ - { NULL, 0, 0 }, /* Ux */ - { NULL, 0, 0 }, /* Xc */ - { NULL, 0, 0 }, /* Xo */ - { pmdoc_head, TYPE_Fn, 0 }, /* Fo */ - { NULL, 0, 0 }, /* Fc */ - { NULL, 0, 0 }, /* Oo */ - { NULL, 0, 0 }, /* Oc */ - { NULL, 0, 0 }, /* Bk */ - { NULL, 0, 0 }, /* Ek */ - { NULL, 0, 0 }, /* Bt */ - { NULL, 0, 0 }, /* Hf */ - { NULL, 0, 0 }, /* Fr */ - { NULL, 0, 0 }, /* Ud */ - { NULL, TYPE_Lb, MDOCF_CHILD }, /* Lb */ - { NULL, 0, 0 }, /* Lp */ - { NULL, TYPE_Lk, MDOCF_CHILD }, /* Lk */ - { NULL, TYPE_Mt, MDOCF_CHILD }, /* Mt */ - { NULL, 0, 0 }, /* Brq */ - { NULL, 0, 0 }, /* Bro */ - { NULL, 0, 0 }, /* Brc */ - { NULL, 0, 0 }, /* %C */ - { NULL, 0, 0 }, /* Es */ - { NULL, 0, 0 }, /* En */ - { NULL, TYPE_Dx, MDOCF_CHILD }, /* Dx */ - { NULL, 0, 0 }, /* %Q */ - { NULL, 0, 0 }, /* br */ - { NULL, 0, 0 }, /* sp */ - { NULL, 0, 0 }, /* %U */ - { NULL, 0, 0 }, /* Ta */ + { NULL, 0 }, /* Ap */ + { NULL, 0 }, /* Dd */ + { NULL, 0 }, /* Dt */ + { NULL, 0 }, /* Os */ + { parse_mdoc_Sh, TYPE_Sh }, /* Sh */ + { parse_mdoc_head, TYPE_Ss }, /* Ss */ + { NULL, 0 }, /* Pp */ + { NULL, 0 }, /* D1 */ + { NULL, 0 }, /* Dl */ + { NULL, 0 }, /* Bd */ + { NULL, 0 }, /* Ed */ + { NULL, 0 }, /* Bl */ + { NULL, 0 }, /* El */ + { NULL, 0 }, /* It */ + { NULL, 0 }, /* Ad */ + { NULL, TYPE_An }, /* An */ + { NULL, TYPE_Ar }, /* Ar */ + { NULL, TYPE_Cd }, /* Cd */ + { NULL, TYPE_Cm }, /* Cm */ + { NULL, TYPE_Dv }, /* Dv */ + { NULL, TYPE_Er }, /* Er */ + { NULL, TYPE_Ev }, /* Ev */ + { NULL, 0 }, /* Ex */ + { NULL, TYPE_Fa }, /* Fa */ + { parse_mdoc_Fd, 0 }, /* Fd */ + { NULL, TYPE_Fl }, /* Fl */ + { parse_mdoc_Fn, 0 }, /* Fn */ + { NULL, TYPE_Ft }, /* Ft */ + { NULL, TYPE_Ic }, /* Ic */ + { NULL, TYPE_In }, /* In */ + { NULL, TYPE_Li }, /* Li */ + { parse_mdoc_Nd, 0 }, /* Nd */ + { parse_mdoc_Nm, 0 }, /* Nm */ + { NULL, 0 }, /* Op */ + { NULL, 0 }, /* Ot */ + { NULL, TYPE_Pa }, /* Pa */ + { NULL, 0 }, /* Rv */ + { NULL, TYPE_St }, /* St */ + { NULL, TYPE_Va }, /* Va */ + { parse_mdoc_body, TYPE_Va }, /* Vt */ + { parse_mdoc_Xr, 0 }, /* Xr */ + { NULL, 0 }, /* %A */ + { NULL, 0 }, /* %B */ + { NULL, 0 }, /* %D */ + { NULL, 0 }, /* %I */ + { NULL, 0 }, /* %J */ + { NULL, 0 }, /* %N */ + { NULL, 0 }, /* %O */ + { NULL, 0 }, /* %P */ + { NULL, 0 }, /* %R */ + { NULL, 0 }, /* %T */ + { NULL, 0 }, /* %V */ + { NULL, 0 }, /* Ac */ + { NULL, 0 }, /* Ao */ + { NULL, 0 }, /* Aq */ + { NULL, TYPE_At }, /* At */ + { NULL, 0 }, /* Bc */ + { NULL, 0 }, /* Bf */ + { NULL, 0 }, /* Bo */ + { NULL, 0 }, /* Bq */ + { NULL, TYPE_Bsx }, /* Bsx */ + { NULL, TYPE_Bx }, /* Bx */ + { NULL, 0 }, /* Db */ + { NULL, 0 }, /* Dc */ + { NULL, 0 }, /* Do */ + { NULL, 0 }, /* Dq */ + { NULL, 0 }, /* Ec */ + { NULL, 0 }, /* Ef */ + { NULL, TYPE_Em }, /* Em */ + { NULL, 0 }, /* Eo */ + { NULL, TYPE_Fx }, /* Fx */ + { NULL, TYPE_Ms }, /* Ms */ + { NULL, 0 }, /* No */ + { NULL, 0 }, /* Ns */ + { NULL, TYPE_Nx }, /* Nx */ + { NULL, TYPE_Ox }, /* Ox */ + { NULL, 0 }, /* Pc */ + { NULL, 0 }, /* Pf */ + { NULL, 0 }, /* Po */ + { NULL, 0 }, /* Pq */ + { NULL, 0 }, /* Qc */ + { NULL, 0 }, /* Ql */ + { NULL, 0 }, /* Qo */ + { NULL, 0 }, /* Qq */ + { NULL, 0 }, /* Re */ + { NULL, 0 }, /* Rs */ + { NULL, 0 }, /* Sc */ + { NULL, 0 }, /* So */ + { NULL, 0 }, /* Sq */ + { NULL, 0 }, /* Sm */ + { NULL, 0 }, /* Sx */ + { NULL, TYPE_Sy }, /* Sy */ + { NULL, TYPE_Tn }, /* Tn */ + { NULL, 0 }, /* Ux */ + { NULL, 0 }, /* Xc */ + { NULL, 0 }, /* Xo */ + { parse_mdoc_Fo, 0 }, /* Fo */ + { NULL, 0 }, /* Fc */ + { NULL, 0 }, /* Oo */ + { NULL, 0 }, /* Oc */ + { NULL, 0 }, /* Bk */ + { NULL, 0 }, /* Ek */ + { NULL, 0 }, /* Bt */ + { NULL, 0 }, /* Hf */ + { NULL, 0 }, /* Fr */ + { NULL, 0 }, /* Ud */ + { NULL, TYPE_Lb }, /* Lb */ + { NULL, 0 }, /* Lp */ + { NULL, TYPE_Lk }, /* Lk */ + { NULL, TYPE_Mt }, /* Mt */ + { NULL, 0 }, /* Brq */ + { NULL, 0 }, /* Bro */ + { NULL, 0 }, /* Brc */ + { NULL, 0 }, /* %C */ + { NULL, 0 }, /* Es */ + { NULL, 0 }, /* En */ + { NULL, TYPE_Dx }, /* Dx */ + { NULL, 0 }, /* %Q */ + { NULL, 0 }, /* br */ + { NULL, 0 }, /* sp */ + { NULL, 0 }, /* %U */ + { NULL, 0 }, /* Ta */ + { NULL, 0 }, /* ll */ }; -static const char *progname; -static int use_all; /* Use all directories and files. */ -static int verb; /* Output verbosity level. */ int -main(int argc, char *argv[]) +mandocdb(int argc, char *argv[]) { - struct mparse *mp; /* parse sequence */ - struct manpaths dirs; - enum op op; /* current operation */ - const char *dir; - char *cp; - char pbuf[PATH_MAX], - ibuf[MAXPATHLEN], /* index fname */ - fbuf[MAXPATHLEN]; /* btree fname */ - int ch, i, flags; - DB *idx, /* index database */ - *db, /* keyword database */ - *hash; /* temporary keyword hashtable */ - BTREEINFO info; /* btree configuration */ - recno_t maxrec; /* last record number in the index */ - recno_t *recs; /* the numbers of all empty records */ - size_t sz1, sz2, - recsz, /* number of allocated slots in recs */ - reccur; /* current number of empty records */ - struct buf buf, /* keyword buffer */ - dbuf; /* description buffer */ - struct of *of; /* list of files for processing */ - extern int optind; - extern char *optarg; + struct manconf conf; + struct ohash_info mpages_info, mlinks_info; + struct mparse *mp; + const char *path_arg; + size_t j, sz; + int ch, i; + memset(&conf, 0, sizeof(conf)); + memset(stmts, 0, STMT__MAX * sizeof(sqlite3_stmt *)); + + mpages_info.alloc = mlinks_info.alloc = hash_alloc; + mpages_info.calloc = mlinks_info.calloc = hash_calloc; + mpages_info.free = mlinks_info.free = hash_free; + mpages_info.data = mlinks_info.data = NULL; + + mpages_info.key_offset = offsetof(struct mpage, inodev); + mlinks_info.key_offset = offsetof(struct mlink, file); + progname = strrchr(argv[0], '/'); if (progname == NULL) progname = argv[0]; else ++progname; - memset(&dirs, 0, sizeof(struct manpaths)); + /* + * We accept a few different invocations. + * The CHECKOP macro makes sure that invocation styles don't + * clobber each other. + */ +#define CHECKOP(_op, _ch) do \ + if (OP_DEFAULT != (_op)) { \ + fprintf(stderr, "%s: -%c: Conflicting option\n", \ + progname, (_ch)); \ + goto usage; \ + } while (/*CONSTCOND*/0) - verb = 0; - use_all = 0; - of = NULL; - db = idx = NULL; - mp = NULL; - hash = NULL; - recs = NULL; - recsz = reccur = 0; - maxrec = 0; - op = OP_NEW; - dir = NULL; + path_arg = NULL; + op = OP_DEFAULT; - while (-1 != (ch = getopt(argc, argv, "ad:u:v"))) + while (-1 != (ch = getopt(argc, argv, "aC:Dd:npQT:tu:v"))) switch (ch) { - case ('a'): + case 'a': use_all = 1; break; - case ('d'): - dir = optarg; + case 'C': + CHECKOP(op, ch); + path_arg = optarg; + op = OP_CONFFILE; + break; + case 'D': + debug++; + break; + case 'd': + CHECKOP(op, ch); + path_arg = optarg; op = OP_UPDATE; break; - case ('u'): - dir = optarg; + case 'n': + nodb = 1; + break; + case 'p': + warnings = 1; + break; + case 'Q': + mparse_options |= MPARSE_QUICK; + break; + case 'T': + if (strcmp(optarg, "utf8")) { + fprintf(stderr, "%s: -T%s: " + "Unsupported output format\n", + progname, optarg); + goto usage; + } + write_utf8 = 1; + break; + case 't': + CHECKOP(op, ch); + dup2(STDOUT_FILENO, STDERR_FILENO); + op = OP_TEST; + nodb = warnings = 1; + break; + case 'u': + CHECKOP(op, ch); + path_arg = optarg; op = OP_DELETE; break; - case ('v'): - verb++; + case 'v': + /* Compatibility with espie@'s makewhatis. */ break; default: - usage(); - return((int)MANDOCLEVEL_BADARG); + goto usage; } argc -= optind; argv += optind; - memset(&info, 0, sizeof(BTREEINFO)); - info.flags = R_DUP; + if (OP_CONFFILE == op && argc > 0) { + fprintf(stderr, "%s: -C: Too many arguments\n", + progname); + goto usage; + } - mp = mparse_alloc(MPARSE_AUTO, MANDOCLEVEL_FATAL, NULL, NULL); + exitcode = (int)MANDOCLEVEL_OK; + mchars = mchars_alloc(); + mp = mparse_alloc(mparse_options, MANDOCLEVEL_BADARG, NULL, + mchars, NULL); + ohash_init(&mpages, 6, &mpages_info); + ohash_init(&mlinks, 6, &mlinks_info); - memset(&buf, 0, sizeof(struct buf)); - memset(&dbuf, 0, sizeof(struct buf)); + if (OP_UPDATE == op || OP_DELETE == op || OP_TEST == op) { - buf.size = dbuf.size = MANDOC_BUFSZ; + /* + * Most of these deal with a specific directory. + * Jump into that directory first. + */ + if (OP_TEST != op && 0 == set_basedir(path_arg, 1)) + goto out; - buf.cp = mandoc_malloc(buf.size); - dbuf.cp = mandoc_malloc(dbuf.size); + if (dbopen(1)) { + /* + * The existing database is usable. Process + * all files specified on the command-line. + */ + use_all = 1; + for (i = 0; i < argc; i++) + filescan(argv[i]); + if (OP_TEST != op) + dbprune(); + } else { + /* + * Database missing or corrupt. + * Recreate from scratch. + */ + exitcode = (int)MANDOCLEVEL_OK; + op = OP_DEFAULT; + if (0 == treescan()) + goto out; + if (0 == dbopen(0)) + goto out; + } + if (OP_DELETE != op) + mpages_merge(mp); + dbclose(OP_DEFAULT == op ? 0 : 1); + } else { + /* + * If we have arguments, use them as our manpaths. + * If we don't, grok from manpath(1) or however else + * manconf_parse() wants to do it. + */ + if (argc > 0) { + conf.manpath.paths = mandoc_reallocarray(NULL, + argc, sizeof(char *)); + conf.manpath.sz = (size_t)argc; + for (i = 0; i < argc; i++) + conf.manpath.paths[i] = mandoc_strdup(argv[i]); + } else + manconf_parse(&conf, path_arg, NULL, NULL); - flags = OP_NEW == op ? O_CREAT|O_TRUNC|O_RDWR : O_CREAT|O_RDWR; + if (conf.manpath.sz == 0) { + exitcode = (int)MANDOCLEVEL_BADARG; + say("", "Empty manpath"); + } - if (OP_UPDATE == op || OP_DELETE == op) { - ibuf[0] = fbuf[0] = '\0'; + /* + * First scan the tree rooted at a base directory, then + * build a new database and finally move it into place. + * Ignore zero-length directories and strip trailing + * slashes. + */ + for (j = 0; j < conf.manpath.sz; j++) { + sz = strlen(conf.manpath.paths[j]); + if (sz && conf.manpath.paths[j][sz - 1] == '/') + conf.manpath.paths[j][--sz] = '\0'; + if (0 == sz) + continue; - strlcat(fbuf, dir, MAXPATHLEN); - strlcat(fbuf, "/", MAXPATHLEN); - sz1 = strlcat(fbuf, MANDOC_DB, MAXPATHLEN); + if (j) { + ohash_init(&mpages, 6, &mpages_info); + ohash_init(&mlinks, 6, &mlinks_info); + } - strlcat(ibuf, dir, MAXPATHLEN); - strlcat(ibuf, "/", MAXPATHLEN); - sz2 = strlcat(ibuf, MANDOC_IDX, MAXPATHLEN); + if ( ! set_basedir(conf.manpath.paths[j], argc > 0)) + continue; + if (0 == treescan()) + continue; + if (0 == dbopen(0)) + continue; - if (sz1 >= MAXPATHLEN || sz2 >= MAXPATHLEN) { - fprintf(stderr, "%s: Path too long\n", dir); - exit((int)MANDOCLEVEL_BADARG); + mpages_merge(mp); + if (warnings && !nodb && + ! (MPARSE_QUICK & mparse_options)) + names_check(); + dbclose(0); + + if (j + 1 < conf.manpath.sz) { + mpages_free(); + ohash_delete(&mpages); + ohash_delete(&mlinks); + } } + } +out: + manconf_free(&conf); + mparse_free(mp); + mchars_free(mchars); + mpages_free(); + ohash_delete(&mpages); + ohash_delete(&mlinks); + return exitcode; +usage: + fprintf(stderr, "usage: %s [-aDnpQ] [-C file] [-Tutf8]\n" + " %s [-aDnpQ] [-Tutf8] dir ...\n" + " %s [-DnpQ] [-Tutf8] -d dir [file ...]\n" + " %s [-Dnp] -u dir [file ...]\n" + " %s [-Q] -t file ...\n", + progname, progname, progname, + progname, progname); - db = dbopen(fbuf, flags, 0644, DB_BTREE, &info); - idx = dbopen(ibuf, flags, 0644, DB_RECNO, NULL); + return (int)MANDOCLEVEL_BADARG; +} - if (NULL == db) { - perror(fbuf); - exit((int)MANDOCLEVEL_SYSERR); - } else if (NULL == idx) { - perror(ibuf); - exit((int)MANDOCLEVEL_SYSERR); - } +/* + * Scan a directory tree rooted at "basedir" for manpages. + * We use fts(), scanning directory parts along the way for clues to our + * section and architecture. + * + * If use_all has been specified, grok all files. + * If not, sanitise paths to the following: + * + * [./]man*[/]/.
+ * or + * [./]cat
[/]/.0 + * + * TODO: accomodate for multi-language directories. + */ +static int +treescan(void) +{ + char buf[PATH_MAX]; + FTS *f; + FTSENT *ff; + struct mlink *mlink; + int dform, gzip; + char *dsec, *arch, *fsec, *cp; + const char *path; + const char *argv[2]; - if (verb > 2) { - printf("%s: Opened\n", fbuf); - printf("%s: Opened\n", ibuf); - } + argv[0] = "."; + argv[1] = (char *)NULL; - ofile_argbuild(argc, argv, &of); - if (NULL == of) - goto out; + f = fts_open((char * const *)argv, + FTS_PHYSICAL | FTS_NOCHDIR, NULL); + if (NULL == f) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say("", "&fts_open"); + return 0; + } - of = of->first; + dsec = arch = NULL; + dform = FORM_NONE; - index_prune(of, db, fbuf, idx, ibuf, - &maxrec, &recs, &recsz, &reccur); + while (NULL != (ff = fts_read(f))) { + path = ff->fts_path + 2; + switch (ff->fts_info) { /* - * Go to the root of the respective manual tree - * such that .so links work. In case of failure, - * just prod on, even though .so links won't work. + * Symbolic links require various sanity checks, + * then get handled just like regular files. */ + case FTS_SL: + if (NULL == realpath(path, buf)) { + if (warnings) + say(path, "&realpath"); + continue; + } + if (strstr(buf, basedir) != buf +#ifdef HOMEBREWDIR + && strstr(buf, HOMEBREWDIR) != buf +#endif + ) { + if (warnings) say("", + "%s: outside base directory", buf); + continue; + } + /* Use logical inode to avoid mpages dupe. */ + if (-1 == stat(path, ff->fts_statp)) { + if (warnings) + say(path, "&stat"); + continue; + } + /* FALLTHROUGH */ - if (OP_UPDATE == op) { - chdir(dir); - index_merge(of, mp, &dbuf, &buf, hash, - db, fbuf, idx, ibuf, - maxrec, recs, reccur); + /* + * If we're a regular file, add an mlink by using the + * stored directory data and handling the filename. + */ + case FTS_F: + if (0 == strcmp(path, MANDOC_DB)) + continue; + if ( ! use_all && ff->fts_level < 2) { + if (warnings) + say(path, "Extraneous file"); + continue; + } + gzip = 0; + fsec = NULL; + while (NULL == fsec) { + fsec = strrchr(ff->fts_name, '.'); + if (NULL == fsec || strcmp(fsec+1, "gz")) + break; + gzip = 1; + *fsec = '\0'; + fsec = NULL; + } + if (NULL == fsec) { + if ( ! use_all) { + if (warnings) + say(path, + "No filename suffix"); + continue; + } + } else if (0 == strcmp(++fsec, "html")) { + if (warnings) + say(path, "Skip html"); + continue; + } else if (0 == strcmp(fsec, "ps")) { + if (warnings) + say(path, "Skip ps"); + continue; + } else if (0 == strcmp(fsec, "pdf")) { + if (warnings) + say(path, "Skip pdf"); + continue; + } else if ( ! use_all && + ((FORM_SRC == dform && + strncmp(fsec, dsec, strlen(dsec))) || + (FORM_CAT == dform && strcmp(fsec, "0")))) { + if (warnings) + say(path, "Wrong filename suffix"); + continue; + } else + fsec[-1] = '\0'; + + mlink = mandoc_calloc(1, sizeof(struct mlink)); + if (strlcpy(mlink->file, path, + sizeof(mlink->file)) >= + sizeof(mlink->file)) { + say(path, "Filename too long"); + free(mlink); + continue; + } + mlink->dform = dform; + mlink->dsec = dsec; + mlink->arch = arch; + mlink->name = ff->fts_name; + mlink->fsec = fsec; + mlink->gzip = gzip; + mlink_add(mlink, ff->fts_statp); + continue; + + case FTS_D: + /* FALLTHROUGH */ + case FTS_DP: + break; + + default: + if (warnings) + say(path, "Not a regular file"); + continue; } - goto out; + switch (ff->fts_level) { + case 0: + /* Ignore the root directory. */ + break; + case 1: + /* + * This might contain manX/ or catX/. + * Try to infer this from the name. + * If we're not in use_all, enforce it. + */ + cp = ff->fts_name; + if (FTS_DP == ff->fts_info) + break; + + if (0 == strncmp(cp, "man", 3)) { + dform = FORM_SRC; + dsec = cp + 3; + } else if (0 == strncmp(cp, "cat", 3)) { + dform = FORM_CAT; + dsec = cp + 3; + } else { + dform = FORM_NONE; + dsec = NULL; + } + + if (NULL != dsec || use_all) + break; + + if (warnings) + say(path, "Unknown directory part"); + fts_set(f, ff, FTS_SKIP); + break; + case 2: + /* + * Possibly our architecture. + * If we're descending, keep tabs on it. + */ + if (FTS_DP != ff->fts_info && NULL != dsec) + arch = ff->fts_name; + else + arch = NULL; + break; + default: + if (FTS_DP == ff->fts_info || use_all) + break; + if (warnings) + say(path, "Extraneous directory part"); + fts_set(f, ff, FTS_SKIP); + break; + } } + fts_close(f); + return 1; +} + +/* + * Add a file to the mlinks table. + * Do not verify that it's a "valid" looking manpage (we'll do that + * later). + * + * Try to infer the manual section, architecture, and page name from the + * path, assuming it looks like + * + * [./]man*[/]/.
+ * or + * [./]cat
[/]/.0 + * + * See treescan() for the fts(3) version of this. + */ +static void +filescan(const char *file) +{ + char buf[PATH_MAX]; + struct stat st; + struct mlink *mlink; + char *p, *start; + + assert(use_all); + + if (0 == strncmp(file, "./", 2)) + file += 2; + /* - * Configure the directories we're going to scan. - * If we have command-line arguments, use them. - * If not, we use man(1)'s method (see mandocdb.8). + * We have to do lstat(2) before realpath(3) loses + * the information whether this is a symbolic link. + * We need to know that because for symbolic links, + * we want to use the orginal file name, while for + * regular files, we want to use the real path. */ + if (-1 == lstat(file, &st)) { + exitcode = (int)MANDOCLEVEL_BADARG; + say(file, "&lstat"); + return; + } else if (0 == ((S_IFREG | S_IFLNK) & st.st_mode)) { + exitcode = (int)MANDOCLEVEL_BADARG; + say(file, "Not a regular file"); + return; + } - if (argc > 0) { - dirs.paths = mandoc_calloc(argc, sizeof(char *)); - dirs.sz = argc; - for (i = 0; i < argc; i++) { - if (NULL == (cp = realpath(argv[i], pbuf))) { - perror(argv[i]); - goto out; - } - dirs.paths[i] = mandoc_strdup(cp); - } - } else - manpath_parse(&dirs, NULL, NULL); + /* + * We have to resolve the file name to the real path + * in any case for the base directory check. + */ + if (NULL == realpath(file, buf)) { + exitcode = (int)MANDOCLEVEL_BADARG; + say(file, "&realpath"); + return; + } - for (i = 0; i < dirs.sz; i++) { - ibuf[0] = fbuf[0] = '\0'; + if (OP_TEST == op) + start = buf; + else if (strstr(buf, basedir) == buf) + start = buf + strlen(basedir); +#ifdef HOMEBREWDIR + else if (strstr(buf, HOMEBREWDIR) == buf) + start = buf; +#endif + else { + exitcode = (int)MANDOCLEVEL_BADARG; + say("", "%s: outside base directory", buf); + return; + } - strlcat(fbuf, dirs.paths[i], MAXPATHLEN); - strlcat(fbuf, "/", MAXPATHLEN); - sz1 = strlcat(fbuf, MANDOC_DB, MAXPATHLEN); + /* + * Now we are sure the file is inside our tree. + * If it is a symbolic link, ignore the real path + * and use the original name. + * This implies passing stuff like "cat1/../man1/foo.1" + * on the command line won't work. So don't do that. + * Note the stat(2) can still fail if the link target + * doesn't exist. + */ + if (S_IFLNK & st.st_mode) { + if (-1 == stat(buf, &st)) { + exitcode = (int)MANDOCLEVEL_BADARG; + say(file, "&stat"); + return; + } + if (strlcpy(buf, file, sizeof(buf)) >= sizeof(buf)) { + say(file, "Filename too long"); + return; + } + start = buf; + if (OP_TEST != op && strstr(buf, basedir) == buf) + start += strlen(basedir); + } - strlcat(ibuf, dirs.paths[i], MAXPATHLEN); - strlcat(ibuf, "/", MAXPATHLEN); - sz2 = strlcat(ibuf, MANDOC_IDX, MAXPATHLEN); + mlink = mandoc_calloc(1, sizeof(struct mlink)); + mlink->dform = FORM_NONE; + if (strlcpy(mlink->file, start, sizeof(mlink->file)) >= + sizeof(mlink->file)) { + say(start, "Filename too long"); + free(mlink); + return; + } - if (sz1 >= MAXPATHLEN || sz2 >= MAXPATHLEN) { - fprintf(stderr, "%s: Path too long\n", - dirs.paths[i]); - exit((int)MANDOCLEVEL_BADARG); + /* + * First try to guess our directory structure. + * If we find a separator, try to look for man* or cat*. + * If we find one of these and what's underneath is a directory, + * assume it's an architecture. + */ + if (NULL != (p = strchr(start, '/'))) { + *p++ = '\0'; + if (0 == strncmp(start, "man", 3)) { + mlink->dform = FORM_SRC; + mlink->dsec = start + 3; + } else if (0 == strncmp(start, "cat", 3)) { + mlink->dform = FORM_CAT; + mlink->dsec = start + 3; } - if (db) - (*db->close)(db); - if (idx) - (*idx->close)(idx); + start = p; + if (NULL != mlink->dsec && NULL != (p = strchr(start, '/'))) { + *p++ = '\0'; + mlink->arch = start; + start = p; + } + } - db = dbopen(fbuf, flags, 0644, DB_BTREE, &info); - idx = dbopen(ibuf, flags, 0644, DB_RECNO, NULL); + /* + * Now check the file suffix. + * Suffix of `.0' indicates a catpage, `.1-9' is a manpage. + */ + p = strrchr(start, '\0'); + while (p-- > start && '/' != *p && '.' != *p) + /* Loop. */ ; - if (NULL == db) { - perror(fbuf); - exit((int)MANDOCLEVEL_SYSERR); - } else if (NULL == idx) { - perror(ibuf); - exit((int)MANDOCLEVEL_SYSERR); + if ('.' == *p) { + *p++ = '\0'; + mlink->fsec = p; + } + + /* + * Now try to parse the name. + * Use the filename portion of the path. + */ + mlink->name = start; + if (NULL != (p = strrchr(start, '/'))) { + mlink->name = p + 1; + *p = '\0'; + } + mlink_add(mlink, &st); +} + +static void +mlink_add(struct mlink *mlink, const struct stat *st) +{ + struct inodev inodev; + struct mpage *mpage; + unsigned int slot; + + assert(NULL != mlink->file); + + mlink->dsec = mandoc_strdup(mlink->dsec ? mlink->dsec : ""); + mlink->arch = mandoc_strdup(mlink->arch ? mlink->arch : ""); + mlink->name = mandoc_strdup(mlink->name ? mlink->name : ""); + mlink->fsec = mandoc_strdup(mlink->fsec ? mlink->fsec : ""); + + if ('0' == *mlink->fsec) { + free(mlink->fsec); + mlink->fsec = mandoc_strdup(mlink->dsec); + mlink->fform = FORM_CAT; + } else if ('1' <= *mlink->fsec && '9' >= *mlink->fsec) + mlink->fform = FORM_SRC; + else + mlink->fform = FORM_NONE; + + slot = ohash_qlookup(&mlinks, mlink->file); + assert(NULL == ohash_find(&mlinks, slot)); + ohash_insert(&mlinks, slot, mlink); + + memset(&inodev, 0, sizeof(inodev)); /* Clear padding. */ + inodev.st_ino = st->st_ino; + inodev.st_dev = st->st_dev; + slot = ohash_lookup_memory(&mpages, (char *)&inodev, + sizeof(struct inodev), inodev.st_ino); + mpage = ohash_find(&mpages, slot); + if (NULL == mpage) { + mpage = mandoc_calloc(1, sizeof(struct mpage)); + mpage->inodev.st_ino = inodev.st_ino; + mpage->inodev.st_dev = inodev.st_dev; + ohash_insert(&mpages, slot, mpage); + } else + mlink->next = mpage->mlinks; + mpage->mlinks = mlink; + mlink->mpage = mpage; +} + +static void +mlink_free(struct mlink *mlink) +{ + + free(mlink->dsec); + free(mlink->arch); + free(mlink->name); + free(mlink->fsec); + free(mlink); +} + +static void +mpages_free(void) +{ + struct mpage *mpage; + struct mlink *mlink; + unsigned int slot; + + mpage = ohash_first(&mpages, &slot); + while (NULL != mpage) { + while (NULL != (mlink = mpage->mlinks)) { + mpage->mlinks = mlink->next; + mlink_free(mlink); } + free(mpage->sec); + free(mpage->arch); + free(mpage->title); + free(mpage->desc); + free(mpage); + mpage = ohash_next(&mpages, &slot); + } +} - if (verb > 2) { - printf("%s: Truncated\n", fbuf); - printf("%s: Truncated\n", ibuf); +/* + * For each mlink to the mpage, check whether the path looks like + * it is formatted, and if it does, check whether a source manual + * exists by the same name, ignoring the suffix. + * If both conditions hold, drop the mlink. + */ +static void +mlinks_undupe(struct mpage *mpage) +{ + char buf[PATH_MAX]; + struct mlink **prev; + struct mlink *mlink; + char *bufp; + + mpage->form = FORM_CAT; + prev = &mpage->mlinks; + while (NULL != (mlink = *prev)) { + if (FORM_CAT != mlink->dform) { + mpage->form = FORM_NONE; + goto nextlink; } + (void)strlcpy(buf, mlink->file, sizeof(buf)); + bufp = strstr(buf, "cat"); + assert(NULL != bufp); + memcpy(bufp, "man", 3); + if (NULL != (bufp = strrchr(buf, '.'))) + *++bufp = '\0'; + (void)strlcat(buf, mlink->dsec, sizeof(buf)); + if (NULL == ohash_find(&mlinks, + ohash_qlookup(&mlinks, buf))) + goto nextlink; + if (warnings) + say(mlink->file, "Man source exists: %s", buf); + if (use_all) + goto nextlink; + *prev = mlink->next; + mlink_free(mlink); + continue; +nextlink: + prev = &(*prev)->next; + } +} - ofile_free(of); - of = NULL; +static void +mlink_check(struct mpage *mpage, struct mlink *mlink) +{ + struct str *str; + unsigned int slot; - if ( ! ofile_dirbuild(dirs.paths[i], NULL, NULL, - 0, &of)) - exit((int)MANDOCLEVEL_SYSERR); + /* + * Check whether the manual section given in a file + * agrees with the directory where the file is located. + * Some manuals have suffixes like (3p) on their + * section number either inside the file or in the + * directory name, some are linked into more than one + * section, like encrypt(1) = makekey(8). + */ - if (NULL == of) - continue; + if (FORM_SRC == mpage->form && + strcasecmp(mpage->sec, mlink->dsec)) + say(mlink->file, "Section \"%s\" manual in %s directory", + mpage->sec, mlink->dsec); - of = of->first; + /* + * Manual page directories exist for each kernel + * architecture as returned by machine(1). + * However, many manuals only depend on the + * application architecture as returned by arch(1). + * For example, some (2/ARM) manuals are shared + * across the "armish" and "zaurus" kernel + * architectures. + * A few manuals are even shared across completely + * different architectures, for example fdformat(1) + * on amd64, i386, sparc, and sparc64. + */ - /* - * Go to the root of the respective manual tree - * such that .so links work. In case of failure, - * just prod on, even though .so links won't work. - */ + if (strcasecmp(mpage->arch, mlink->arch)) + say(mlink->file, "Architecture \"%s\" manual in " + "\"%s\" directory", mpage->arch, mlink->arch); - chdir(dirs.paths[i]); - index_merge(of, mp, &dbuf, &buf, hash, db, fbuf, - idx, ibuf, maxrec, recs, reccur); - } + /* + * XXX + * parse_cat() doesn't set NAME_TITLE yet. + */ -out: - if (db) - (*db->close)(db); - if (idx) - (*idx->close)(idx); - if (hash) - (*hash->close)(hash); - if (mp) - mparse_free(mp); + if (FORM_CAT == mpage->form) + return; - manpath_free(&dirs); - ofile_free(of); - free(buf.cp); - free(dbuf.cp); - free(recs); + /* + * Check whether this mlink + * appears as a name in the NAME section. + */ - return(MANDOCLEVEL_OK); + slot = ohash_qlookup(&names, mlink->name); + str = ohash_find(&names, slot); + assert(NULL != str); + if ( ! (NAME_TITLE & str->mask)) + say(mlink->file, "Name missing in NAME section"); } -void -index_merge(const struct of *of, struct mparse *mp, - struct buf *dbuf, struct buf *buf, DB *hash, - DB *db, const char *dbf, DB *idx, const char *idxf, - recno_t maxrec, const recno_t *recs, size_t reccur) +/* + * Run through the files in the global vector "mpages" + * and add them to the database specified in "basedir". + * + * This handles the parsing scheme itself, using the cues of directory + * and filename to determine whether the file is parsable or not. + */ +static void +mpages_merge(struct mparse *mp) { - recno_t rec; - int ch; - DBT key, val; - struct mdoc *mdoc; - struct man *man; - const char *fn, *msec, *mtitle, *arch; - size_t sv; - unsigned seq; - struct db_val vbuf; + char any[] = "any"; + struct ohash_info str_info; + struct mpage *mpage, *mpage_dest; + struct mlink *mlink, *mlink_dest; + struct roff_man *man; + char *sodest; + char *cp; + int fd; + unsigned int pslot; - for (rec = 0; of; of = of->next) { - fn = of->fname; + str_info.alloc = hash_alloc; + str_info.calloc = hash_calloc; + str_info.free = hash_free; + str_info.data = NULL; + str_info.key_offset = offsetof(struct str, key); - /* - * Try interpreting the file as mdoc(7) or man(7) - * source code, unless it is already known to be - * formatted. Fall back to formatted mode. - */ + if ( ! nodb) + SQL_EXEC("BEGIN TRANSACTION"); + mpage = ohash_first(&mpages, &pslot); + while (mpage != NULL) { + mlinks_undupe(mpage); + if ((mlink = mpage->mlinks) == NULL) { + mpage = ohash_next(&mpages, &pslot); + continue; + } + + name_mask = NAME_MASK; + ohash_init(&names, 4, &str_info); + ohash_init(&strings, 6, &str_info); mparse_reset(mp); - mdoc = NULL; man = NULL; + sodest = NULL; - if ((MANDOC_SRC & of->src_form || - ! (MANDOC_FORM & of->src_form)) && - MANDOCLEVEL_FATAL > mparse_readfd(mp, -1, fn)) - mparse_result(mp, &mdoc, &man); - - if (NULL != mdoc) { - msec = mdoc_meta(mdoc)->msec; - arch = mdoc_meta(mdoc)->arch; - mtitle = mdoc_meta(mdoc)->title; - } else if (NULL != man) { - msec = man_meta(man)->msec; - arch = NULL; - mtitle = man_meta(man)->title; - } else { - msec = of->sec; - arch = of->arch; - mtitle = of->title; + mparse_open(mp, &fd, mlink->file); + if (fd == -1) { + say(mlink->file, "&open"); + goto nextpage; } /* - * By default, skip a file if the manual section - * and architecture given in the file disagree - * with the directory where the file is located. + * Interpret the file as mdoc(7) or man(7) source + * code, unless it is known to be formatted. */ - - if (0 == use_all) { - assert(of->sec); - assert(msec); - if (strcasecmp(msec, of->sec)) - continue; - - if (NULL == arch) { - if (NULL != of->arch) - continue; - } else if (NULL == of->arch || - strcasecmp(arch, of->arch)) - continue; + if (mlink->dform != FORM_CAT || mlink->fform != FORM_CAT) { + mparse_readfd(mp, fd, mlink->file); + mparse_result(mp, &man, &sodest); } - if (NULL == arch) - arch = ""; + if (sodest != NULL) { + mlink_dest = ohash_find(&mlinks, + ohash_qlookup(&mlinks, sodest)); + if (mlink_dest == NULL) { + mandoc_asprintf(&cp, "%s.gz", sodest); + mlink_dest = ohash_find(&mlinks, + ohash_qlookup(&mlinks, cp)); + free(cp); + } + if (mlink_dest != NULL) { - /* - * By default, skip a file if the title given - * in the file disagrees with the file name. - * If both agree, use the file name as the title, - * because the one in the file usually is all caps. - */ + /* The .so target exists. */ - assert(of->title); - assert(mtitle); + mpage_dest = mlink_dest->mpage; + while (1) { + mlink->mpage = mpage_dest; - if (0 == strcasecmp(mtitle, of->title)) - mtitle = of->title; - else if (0 == use_all) - continue; + /* + * If the target was already + * processed, add the links + * to the database now. + * Otherwise, this will + * happen when we come + * to the target. + */ - /* - * The index record value consists of a nil-terminated - * filename, a nil-terminated manual section, and a - * nil-terminated description. Since the description - * may not be set, we set a sentinel to see if we're - * going to write a nil byte in its place. - */ + if (mpage_dest->pageid) + dbadd_mlink_name(mlink); - dbuf->len = 0; - buf_append(dbuf, mdoc ? "mdoc" : (man ? "man" : "cat")); - buf_appendb(dbuf, fn, strlen(fn) + 1); - buf_appendb(dbuf, msec, strlen(msec) + 1); - buf_appendb(dbuf, mtitle, strlen(mtitle) + 1); - buf_appendb(dbuf, arch, strlen(arch) + 1); + if (mlink->next == NULL) + break; + mlink = mlink->next; + } - sv = dbuf->len; + /* Move all links to the target. */ - /* - * Collect keyword/mask pairs. - * Each pair will become a new btree node. - */ + mlink->next = mlink_dest->next; + mlink_dest->next = mpage->mlinks; + mpage->mlinks = NULL; + } + goto nextpage; + } else if (man != NULL && man->macroset == MACROSET_MDOC) { + mpage->form = FORM_SRC; + mpage->sec = man->meta.msec; + mpage->sec = mandoc_strdup( + mpage->sec == NULL ? "" : mpage->sec); + mpage->arch = man->meta.arch; + mpage->arch = mandoc_strdup( + mpage->arch == NULL ? "" : mpage->arch); + mpage->title = mandoc_strdup(man->meta.title); + } else if (man != NULL && man->macroset == MACROSET_MAN) { + mpage->form = FORM_SRC; + mpage->sec = mandoc_strdup(man->meta.msec); + mpage->arch = mandoc_strdup(mlink->arch); + mpage->title = mandoc_strdup(man->meta.title); + } else { + mpage->form = FORM_CAT; + mpage->sec = mandoc_strdup(mlink->dsec); + mpage->arch = mandoc_strdup(mlink->arch); + mpage->title = mandoc_strdup(mlink->name); + } + putkey(mpage, mpage->sec, TYPE_sec); + if (*mpage->arch != '\0') + putkey(mpage, mpage->arch, TYPE_arch); - hash_reset(&hash); - if (mdoc) - pmdoc_node(hash, buf, dbuf, - mdoc_node(mdoc), mdoc_meta(mdoc)); - else if (man) - pman_node(hash, buf, dbuf, man_node(man)); + for ( ; mlink != NULL; mlink = mlink->next) { + if ('\0' != *mlink->dsec) + putkey(mpage, mlink->dsec, TYPE_sec); + if ('\0' != *mlink->fsec) + putkey(mpage, mlink->fsec, TYPE_sec); + putkey(mpage, '\0' == *mlink->arch ? + any : mlink->arch, TYPE_arch); + putkey(mpage, mlink->name, NAME_FILE); + } + + assert(mpage->desc == NULL); + if (man != NULL && man->macroset == MACROSET_MDOC) + parse_mdoc(mpage, &man->meta, man->first); + else if (man != NULL) + parse_man(mpage, &man->meta, man->first); else - pformatted(hash, buf, dbuf, of); + parse_cat(mpage, fd); + if (mpage->desc == NULL) + mpage->desc = mandoc_strdup(mpage->mlinks->name); - /* - * Reclaim an empty index record, if available. - * Use its record number for all new btree nodes. - */ + if (warnings && !use_all) + for (mlink = mpage->mlinks; mlink; + mlink = mlink->next) + mlink_check(mpage, mlink); - if (reccur > 0) { - --reccur; - rec = recs[(int)reccur]; - } else if (maxrec > 0) { - rec = maxrec; - maxrec = 0; - } else - rec++; - vbuf.rec = htobe32(rec); + dbadd(mpage); + mlink = mpage->mlinks; - /* - * Copy from the in-memory hashtable of pending - * keyword/mask pairs into the database. - */ +nextpage: + ohash_delete(&strings); + ohash_delete(&names); + mpage = ohash_next(&mpages, &pslot); + } - seq = R_FIRST; - while (0 == (ch = (*hash->seq)(hash, &key, &val, seq))) { - seq = R_NEXT; - vbuf.mask = htobe64(*(uint64_t *)val.data); - val.size = sizeof(struct db_val); - val.data = &vbuf; - dbt_put(db, dbf, &key, &val); - } - if (ch < 0) { - perror("hash"); - exit((int)MANDOCLEVEL_SYSERR); - } - - /* - * Apply to the index. If we haven't had a description - * set, put an empty one in now. - */ + if (0 == nodb) + SQL_EXEC("END TRANSACTION"); +} - if (dbuf->len == sv) - buf_appendb(dbuf, "", 1); +static void +names_check(void) +{ + sqlite3_stmt *stmt; + const char *name, *sec, *arch, *key; + int irc; - key.data = &rec; - key.size = sizeof(recno_t); + sqlite3_prepare_v2(db, + "SELECT name, sec, arch, key FROM (" + "SELECT name AS key, pageid FROM names " + "WHERE bits & ? AND NOT EXISTS (" + "SELECT pageid FROM mlinks " + "WHERE mlinks.pageid == names.pageid " + "AND mlinks.name == names.name" + ")" + ") JOIN (" + "SELECT sec, arch, name, pageid FROM mlinks " + "GROUP BY pageid" + ") USING (pageid);", + -1, &stmt, NULL); - val.data = dbuf->cp; - val.size = dbuf->len; + if (SQLITE_OK != sqlite3_bind_int64(stmt, 1, NAME_TITLE)) + say("", "%s", sqlite3_errmsg(db)); - if (verb) - printf("%s: Added index\n", fn); - - dbt_put(idx, idxf, &key, &val); + while (SQLITE_ROW == (irc = sqlite3_step(stmt))) { + name = (const char *)sqlite3_column_text(stmt, 0); + sec = (const char *)sqlite3_column_text(stmt, 1); + arch = (const char *)sqlite3_column_text(stmt, 2); + key = (const char *)sqlite3_column_text(stmt, 3); + say("", "%s(%s%s%s) lacks mlink \"%s\"", name, sec, + '\0' == *arch ? "" : "/", + '\0' == *arch ? "" : arch, key); } + sqlite3_finalize(stmt); } -/* - * Scan through all entries in the index file `idx' and prune those - * entries in `ofile'. - * Pruning consists of removing from `db', then invalidating the entry - * in `idx' (zeroing its value size). - */ static void -index_prune(const struct of *ofile, DB *db, const char *dbf, - DB *idx, const char *idxf, recno_t *maxrec, - recno_t **recs, size_t *recsz, size_t *reccur) +parse_cat(struct mpage *mpage, int fd) { - const struct of *of; - const char *fn, *cp; - struct db_val *vbuf; - unsigned seq, sseq; - DBT key, val; - int ch; + FILE *stream; + char *line, *p, *title; + size_t len, plen, titlesz; - *reccur = 0; - seq = R_FIRST; - while (0 == (ch = (*idx->seq)(idx, &key, &val, seq))) { - seq = R_NEXT; - *maxrec = *(recno_t *)key.data; - cp = val.data; + stream = (-1 == fd) ? + fopen(mpage->mlinks->file, "r") : + fdopen(fd, "r"); + if (NULL == stream) { + if (-1 != fd) + close(fd); + if (warnings) + say(mpage->mlinks->file, "&fopen"); + return; + } - /* Deleted records are zero-sized. Skip them. */ + /* Skip to first blank line. */ - if (0 == val.size) - goto cont; + while (NULL != (line = fgetln(stream, &len))) + if ('\n' == *line) + break; - /* - * Make sure we're sane. - * Read past our mdoc/man/cat type to the next string, - * then make sure it's bounded by a NUL. - * Failing any of these, we go into our error handler. - */ + /* + * Assume the first line that is not indented + * is the first section header. Skip to it. + */ - if (NULL == (fn = memchr(cp, '\0', val.size))) + while (NULL != (line = fgetln(stream, &len))) + if ('\n' != *line && ' ' != *line) break; - if (++fn - cp >= (int)val.size) - break; - if (NULL == memchr(fn, '\0', val.size - (fn - cp))) - break; - /* - * Search for the file in those we care about. - * XXX: build this into a tree. Too slow. - */ + /* + * Read up until the next section into a buffer. + * Strip the leading and trailing newline from each read line, + * appending a trailing space. + * Ignore empty (whitespace-only) lines. + */ - for (of = ofile; of; of = of->next) - if (0 == strcmp(fn, of->fname)) - break; + titlesz = 0; + title = NULL; - if (NULL == of) + while (NULL != (line = fgetln(stream, &len))) { + if (' ' != *line || '\n' != line[len - 1]) + break; + while (len > 0 && isspace((unsigned char)*line)) { + line++; + len--; + } + if (1 == len) continue; + title = mandoc_realloc(title, titlesz + len); + memcpy(title + titlesz, line, len); + titlesz += len; + title[titlesz - 1] = ' '; + } - /* - * Search through the keyword database, throwing out all - * references to our file. - */ + /* + * If no page content can be found, or the input line + * is already the next section header, or there is no + * trailing newline, reuse the page title as the page + * description. + */ - sseq = R_FIRST; - while (0 == (ch = (*db->seq)(db, &key, &val, sseq))) { - sseq = R_NEXT; - if (sizeof(struct db_val) != val.size) - break; + if (NULL == title || '\0' == *title) { + if (warnings) + say(mpage->mlinks->file, + "Cannot find NAME section"); + fclose(stream); + free(title); + return; + } - vbuf = val.data; - if (*maxrec != betoh32(vbuf->rec)) - continue; + title = mandoc_realloc(title, titlesz + 1); + title[titlesz] = '\0'; - if ((ch = (*db->del)(db, &key, R_CURSOR)) < 0) - break; - } + /* + * Skip to the first dash. + * Use the remaining line as the description (no more than 70 + * bytes). + */ - if (ch < 0) { - perror(dbf); - exit((int)MANDOCLEVEL_SYSERR); - } else if (1 != ch) { - fprintf(stderr, "%s: Corrupt database\n", dbf); - exit((int)MANDOCLEVEL_SYSERR); - } + if (NULL != (p = strstr(title, "- "))) { + for (p += 2; ' ' == *p || '\b' == *p; p++) + /* Skip to next word. */ ; + } else { + if (warnings) + say(mpage->mlinks->file, + "No dash in title line"); + p = title; + } - if (verb) - printf("%s: Deleted index\n", fn); + plen = strlen(p); - val.size = 0; - ch = (*idx->put)(idx, &key, &val, R_CURSOR); + /* Strip backspace-encoding from line. */ - if (ch < 0) - break; -cont: - if (*reccur >= *recsz) { - *recsz += MANDOC_SLOP; - *recs = mandoc_realloc - (*recs, *recsz * sizeof(recno_t)); + while (NULL != (line = memchr(p, '\b', plen))) { + len = line - p; + if (0 == len) { + memmove(line, line + 1, plen--); + continue; } - - (*recs)[(int)*reccur] = *maxrec; - (*reccur)++; + memmove(line - 1, line + 1, plen - len); + plen -= 2; } - if (ch < 0) { - perror(idxf); - exit((int)MANDOCLEVEL_SYSERR); - } else if (1 != ch) { - fprintf(stderr, "%s: Corrupt index\n", idxf); - exit((int)MANDOCLEVEL_SYSERR); - } - - (*maxrec)++; + mpage->desc = mandoc_strdup(p); + fclose(stream); + free(title); } /* - * Grow the buffer (if necessary) and copy in a binary string. + * Put a type/word pair into the word database for this particular file. */ static void -buf_appendb(struct buf *buf, const void *cp, size_t sz) +putkey(const struct mpage *mpage, char *value, uint64_t type) { + char *cp; - /* Overshoot by MANDOC_BUFSZ. */ - - while (buf->len + sz >= buf->size) { - buf->size = buf->len + sz + MANDOC_BUFSZ; - buf->cp = mandoc_realloc(buf->cp, buf->size); - } - - memcpy(buf->cp + (int)buf->len, cp, sz); - buf->len += sz; + assert(NULL != value); + if (TYPE_arch == type) + for (cp = value; *cp; cp++) + if (isupper((unsigned char)*cp)) + *cp = _tolower((unsigned char)*cp); + putkeys(mpage, value, strlen(value), type); } /* - * Append a nil-terminated string to the buffer. - * This can be invoked multiple times. - * The buffer string will be nil-terminated. - * If invoked multiple times, a space is put between strings. + * Grok all nodes at or below a certain mdoc node into putkey(). */ static void -buf_append(struct buf *buf, const char *cp) +putmdockey(const struct mpage *mpage, + const struct roff_node *n, uint64_t m) { + + for ( ; NULL != n; n = n->next) { + if (NULL != n->child) + putmdockey(mpage, n->child, m); + if (n->type == ROFFT_TEXT) + putkey(mpage, n->string, m); + } +} + +static void +parse_man(struct mpage *mpage, const struct roff_meta *meta, + const struct roff_node *n) +{ + const struct roff_node *head, *body; + char *start, *title; + char byte; size_t sz; - if (0 == (sz = strlen(cp))) + if (NULL == n) return; - if (buf->len) - buf->cp[(int)buf->len - 1] = ' '; + /* + * We're only searching for one thing: the first text child in + * the BODY of a NAME section. Since we don't keep track of + * sections in -man, run some hoops to find out whether we're in + * the correct section or not. + */ - buf_appendb(buf, cp, sz + 1); -} + if (n->type == ROFFT_BODY && n->tok == MAN_SH) { + body = n; + assert(body->parent); + if (NULL != (head = body->parent->head) && + 1 == head->nchild && + NULL != (head = (head->child)) && + head->type == ROFFT_TEXT && + 0 == strcmp(head->string, "NAME") && + NULL != body->child) { -/* - * Recursively add all text from a given node. - * This is optimised for general mdoc nodes in this context, which do - * not consist of subexpressions and having a recursive call for n->next - * would be wasteful. - * The "f" variable should be 0 unless called from pmdoc_Nd for the - * description buffer, which does not start at the beginning of the - * buffer. - */ -static void -buf_appendmdoc(struct buf *buf, const struct mdoc_node *n, int f) -{ + /* + * Suck the entire NAME section into memory. + * Yes, we might run away. + * But too many manuals have big, spread-out + * NAME sections over many lines. + */ - for ( ; n; n = n->next) { - if (n->child) - buf_appendmdoc(buf, n->child, f); + title = NULL; + deroff(&title, body); + if (NULL == title) + return; - if (MDOC_TEXT == n->type && f) { - f = 0; - buf_appendb(buf, n->string, - strlen(n->string) + 1); - } else if (MDOC_TEXT == n->type) - buf_append(buf, n->string); + /* + * Go through a special heuristic dance here. + * Conventionally, one or more manual names are + * comma-specified prior to a whitespace, then a + * dash, then a description. Try to puzzle out + * the name parts here. + */ - } -} + start = title; + for ( ;; ) { + sz = strcspn(start, " ,"); + if ('\0' == start[sz]) + break; -static void -hash_reset(DB **db) -{ - DB *hash; + byte = start[sz]; + start[sz] = '\0'; - if (NULL != (hash = *db)) - (*hash->close)(hash); + /* + * Assume a stray trailing comma in the + * name list if a name begins with a dash. + */ - *db = dbopen(NULL, O_CREAT|O_RDWR, 0644, DB_HASH, NULL); - if (NULL == *db) { - perror("hash"); - exit((int)MANDOCLEVEL_SYSERR); - } -} + if ('-' == start[0] || + ('\\' == start[0] && '-' == start[1])) + break; -/* ARGSUSED */ -static int -pmdoc_head(MDOC_ARGS) -{ + putkey(mpage, start, NAME_TITLE); + if ( ! (mpage->name_head_done || + strcasecmp(start, meta->title))) { + putkey(mpage, start, NAME_HEAD); + mpage->name_head_done = 1; + } - return(MDOC_HEAD == n->type); + if (' ' == byte) { + start += sz + 1; + break; + } + + assert(',' == byte); + start += sz + 1; + while (' ' == *start) + start++; + } + + if (start == title) { + putkey(mpage, start, NAME_TITLE); + if ( ! (mpage->name_head_done || + strcasecmp(start, meta->title))) { + putkey(mpage, start, NAME_HEAD); + mpage->name_head_done = 1; + } + free(title); + return; + } + + while (isspace((unsigned char)*start)) + start++; + + if (0 == strncmp(start, "-", 1)) + start += 1; + else if (0 == strncmp(start, "\\-\\-", 4)) + start += 4; + else if (0 == strncmp(start, "\\-", 2)) + start += 2; + else if (0 == strncmp(start, "\\(en", 4)) + start += 4; + else if (0 == strncmp(start, "\\(em", 4)) + start += 4; + + while (' ' == *start) + start++; + + mpage->desc = mandoc_strdup(start); + free(title); + return; + } + } + + for (n = n->child; n; n = n->next) { + if (NULL != mpage->desc) + break; + parse_man(mpage, meta, n); + } } -/* ARGSUSED */ -static int -pmdoc_body(MDOC_ARGS) +static void +parse_mdoc(struct mpage *mpage, const struct roff_meta *meta, + const struct roff_node *n) { - return(MDOC_BODY == n->type); + assert(NULL != n); + for (n = n->child; NULL != n; n = n->next) { + switch (n->type) { + case ROFFT_ELEM: + /* FALLTHROUGH */ + case ROFFT_BLOCK: + /* FALLTHROUGH */ + case ROFFT_HEAD: + /* FALLTHROUGH */ + case ROFFT_BODY: + /* FALLTHROUGH */ + case ROFFT_TAIL: + if (NULL != mdocs[n->tok].fp) + if (0 == (*mdocs[n->tok].fp)(mpage, meta, n)) + break; + if (mdocs[n->tok].mask) + putmdockey(mpage, n->child, + mdocs[n->tok].mask); + break; + default: + assert(n->type != ROFFT_ROOT); + continue; + } + if (NULL != n->child) + parse_mdoc(mpage, meta, n); + } } -/* ARGSUSED */ static int -pmdoc_Fd(MDOC_ARGS) +parse_mdoc_Fd(struct mpage *mpage, const struct roff_meta *meta, + const struct roff_node *n) { - const char *start, *end; + char *start, *end; size_t sz; - if (SEC_SYNOPSIS != n->sec) - return(0); - if (NULL == (n = n->child) || MDOC_TEXT != n->type) - return(0); + if (SEC_SYNOPSIS != n->sec || + NULL == (n = n->child) || + n->type != ROFFT_TEXT) + return 0; /* * Only consider those `Fd' macro fields that begin with an * "inclusion" token (versus, e.g., #define). */ + if (strcmp("#include", n->string)) - return(0); + return 0; - if (NULL == (n = n->next) || MDOC_TEXT != n->type) - return(0); + if ((n = n->next) == NULL || n->type != ROFFT_TEXT) + return 0; /* * Strip away the enclosing angle brackets and make sure we're @@ -920,747 +1627,913 @@ pmdoc_Fd(MDOC_ARGS) start++; if (0 == (sz = strlen(start))) - return(0); + return 0; end = &start[(int)sz - 1]; if ('>' == *end || '"' == *end) end--; - assert(end >= start); - - buf_appendb(buf, start, (size_t)(end - start + 1)); - buf_appendb(buf, "", 1); - return(1); + if (end > start) + putkeys(mpage, start, end - start + 1, TYPE_In); + return 0; } -/* ARGSUSED */ -static int -pmdoc_In(MDOC_ARGS) +static void +parse_mdoc_fname(struct mpage *mpage, const struct roff_node *n) { + char *cp; + size_t sz; - if (NULL == n->child || MDOC_TEXT != n->child->type) - return(0); + if (n->type != ROFFT_TEXT) + return; - buf_append(buf, n->child->string); - return(1); + /* Skip function pointer punctuation. */ + + cp = n->string; + while (*cp == '(' || *cp == '*') + cp++; + sz = strcspn(cp, "()"); + + putkeys(mpage, cp, sz, TYPE_Fn); + if (n->sec == SEC_SYNOPSIS) + putkeys(mpage, cp, sz, NAME_SYN); } -/* ARGSUSED */ static int -pmdoc_Fn(MDOC_ARGS) +parse_mdoc_Fn(struct mpage *mpage, const struct roff_meta *meta, + const struct roff_node *n) { - struct mdoc_node *nn; - const char *cp; - nn = n->child; + if (n->child == NULL) + return 0; - if (NULL == nn || MDOC_TEXT != nn->type) - return(0); + parse_mdoc_fname(mpage, n->child); - /* .Fn "struct type *name" "char *arg" */ + for (n = n->child->next; n != NULL; n = n->next) + if (n->type == ROFFT_TEXT) + putkey(mpage, n->string, TYPE_Fa); - cp = strrchr(nn->string, ' '); - if (NULL == cp) - cp = nn->string; + return 0; +} - /* Strip away pointer symbol. */ +static int +parse_mdoc_Fo(struct mpage *mpage, const struct roff_meta *meta, + const struct roff_node *n) +{ - while ('*' == *cp) - cp++; + if (n->type != ROFFT_HEAD) + return 1; - /* Store the function name. */ + if (n->child != NULL) + parse_mdoc_fname(mpage, n->child); - buf_append(buf, cp); - hash_put(hash, buf, TYPE_Fn); + return 0; +} - /* Store the function type. */ +static int +parse_mdoc_Xr(struct mpage *mpage, const struct roff_meta *meta, + const struct roff_node *n) +{ + char *cp; - if (nn->string < cp) { - buf->len = 0; - buf_appendb(buf, nn->string, cp - nn->string); - buf_appendb(buf, "", 1); - hash_put(hash, buf, TYPE_Ft); - } + if (NULL == (n = n->child)) + return 0; - /* Store the arguments. */ - - for (nn = nn->next; nn; nn = nn->next) { - if (MDOC_TEXT != nn->type) - continue; - buf->len = 0; - buf_append(buf, nn->string); - hash_put(hash, buf, TYPE_Fa); + if (NULL == n->next) { + putkey(mpage, n->string, TYPE_Xr); + return 0; } - return(0); + mandoc_asprintf(&cp, "%s(%s)", n->string, n->next->string); + putkey(mpage, cp, TYPE_Xr); + free(cp); + return 0; } -/* ARGSUSED */ static int -pmdoc_St(MDOC_ARGS) +parse_mdoc_Nd(struct mpage *mpage, const struct roff_meta *meta, + const struct roff_node *n) { - if (NULL == n->child || MDOC_TEXT != n->child->type) - return(0); - - buf_append(buf, n->child->string); - return(1); + if (n->type == ROFFT_BODY) + deroff(&mpage->desc, n); + return 0; } -/* ARGSUSED */ static int -pmdoc_Xr(MDOC_ARGS) +parse_mdoc_Nm(struct mpage *mpage, const struct roff_meta *meta, + const struct roff_node *n) { - if (NULL == (n = n->child)) - return(0); - - buf_appendb(buf, n->string, strlen(n->string)); - - if (NULL != (n = n->next)) { - buf_appendb(buf, ".", 1); - buf_appendb(buf, n->string, strlen(n->string) + 1); - } else - buf_appendb(buf, ".", 2); - - return(1); + if (SEC_NAME == n->sec) + putmdockey(mpage, n->child, NAME_TITLE); + else if (n->sec == SEC_SYNOPSIS && n->type == ROFFT_HEAD) { + if (n->child == NULL) + putkey(mpage, meta->name, NAME_SYN); + else + putmdockey(mpage, n->child, NAME_SYN); + } + if ( ! (mpage->name_head_done || + n->child == NULL || n->child->string == NULL || + strcasecmp(n->child->string, meta->title))) { + putkey(mpage, n->child->string, ROFFT_HEAD); + mpage->name_head_done = 1; + } + return 0; } -/* ARGSUSED */ static int -pmdoc_Nd(MDOC_ARGS) +parse_mdoc_Sh(struct mpage *mpage, const struct roff_meta *meta, + const struct roff_node *n) { - if (MDOC_BODY != n->type) - return(0); - - buf_appendmdoc(dbuf, n->child, 1); - return(1); + return n->sec == SEC_CUSTOM && n->type == ROFFT_HEAD; } -/* ARGSUSED */ static int -pmdoc_Nm(MDOC_ARGS) +parse_mdoc_head(struct mpage *mpage, const struct roff_meta *meta, + const struct roff_node *n) { - if (SEC_NAME == n->sec) - return(1); - else if (SEC_SYNOPSIS != n->sec || MDOC_HEAD != n->type) - return(0); - - if (NULL == n->child) - buf_append(buf, m->name); - - return(1); + return n->type == ROFFT_HEAD; } -/* ARGSUSED */ static int -pmdoc_Sh(MDOC_ARGS) +parse_mdoc_body(struct mpage *mpage, const struct roff_meta *meta, + const struct roff_node *n) { - return(SEC_CUSTOM == n->sec && MDOC_HEAD == n->type); + return n->type == ROFFT_BODY; } +/* + * Add a string to the hash table for the current manual. + * Each string has a bitmask telling which macros it belongs to. + * When we finish the manual, we'll dump the table. + */ static void -hash_put(DB *db, const struct buf *buf, uint64_t mask) +putkeys(const struct mpage *mpage, char *cp, size_t sz, uint64_t v) { - DBT key, val; - int rc; + struct ohash *htab; + struct str *s; + const char *end; + unsigned int slot; + int i, mustfree; - if (buf->len < 2) + if (0 == sz) return; - key.data = buf->cp; - key.size = buf->len; + mustfree = render_string(&cp, &sz); - if ((rc = (*db->get)(db, &key, &val, 0)) < 0) { - perror("hash"); - exit((int)MANDOCLEVEL_SYSERR); - } else if (0 == rc) - mask |= *(uint64_t *)val.data; + if (TYPE_Nm & v) { + htab = &names; + v &= name_mask; + if (v & NAME_FIRST) + name_mask &= ~NAME_FIRST; + if (debug > 1) + say(mpage->mlinks->file, + "Adding name %*s, bits=%d", sz, cp, v); + } else { + htab = &strings; + if (debug > 1) + for (i = 0; i < mansearch_keymax; i++) + if ((uint64_t)1 << i & v) + say(mpage->mlinks->file, + "Adding key %s=%*s", + mansearch_keynames[i], sz, cp); + } - val.data = &mask; - val.size = sizeof(uint64_t); + end = cp + sz; + slot = ohash_qlookupi(htab, cp, &end); + s = ohash_find(htab, slot); - if ((rc = (*db->put)(db, &key, &val, 0)) < 0) { - perror("hash"); - exit((int)MANDOCLEVEL_SYSERR); - } + if (NULL != s && mpage == s->mpage) { + s->mask |= v; + return; + } else if (NULL == s) { + s = mandoc_calloc(1, sizeof(struct str) + sz + 1); + memcpy(s->key, cp, sz); + ohash_insert(htab, slot, s); + } + s->mpage = mpage; + s->mask = v; + + if (mustfree) + free(cp); } -static void -dbt_put(DB *db, const char *dbn, DBT *key, DBT *val) +/* + * Take a Unicode codepoint and produce its UTF-8 encoding. + * This isn't the best way to do this, but it works. + * The magic numbers are from the UTF-8 packaging. + * They're not as scary as they seem: read the UTF-8 spec for details. + */ +static size_t +utf8(unsigned int cp, char out[7]) { + size_t rc; - assert(key->size); - assert(val->size); + rc = 0; + if (cp <= 0x0000007F) { + rc = 1; + out[0] = (char)cp; + } else if (cp <= 0x000007FF) { + rc = 2; + out[0] = (cp >> 6 & 31) | 192; + out[1] = (cp & 63) | 128; + } else if (cp <= 0x0000FFFF) { + rc = 3; + out[0] = (cp >> 12 & 15) | 224; + out[1] = (cp >> 6 & 63) | 128; + out[2] = (cp & 63) | 128; + } else if (cp <= 0x001FFFFF) { + rc = 4; + out[0] = (cp >> 18 & 7) | 240; + out[1] = (cp >> 12 & 63) | 128; + out[2] = (cp >> 6 & 63) | 128; + out[3] = (cp & 63) | 128; + } else if (cp <= 0x03FFFFFF) { + rc = 5; + out[0] = (cp >> 24 & 3) | 248; + out[1] = (cp >> 18 & 63) | 128; + out[2] = (cp >> 12 & 63) | 128; + out[3] = (cp >> 6 & 63) | 128; + out[4] = (cp & 63) | 128; + } else if (cp <= 0x7FFFFFFF) { + rc = 6; + out[0] = (cp >> 30 & 1) | 252; + out[1] = (cp >> 24 & 63) | 128; + out[2] = (cp >> 18 & 63) | 128; + out[3] = (cp >> 12 & 63) | 128; + out[4] = (cp >> 6 & 63) | 128; + out[5] = (cp & 63) | 128; + } else + return 0; - if (0 == (*db->put)(db, key, val, 0)) - return; - - perror(dbn); - exit((int)MANDOCLEVEL_SYSERR); - /* NOTREACHED */ + out[rc] = '\0'; + return rc; } /* - * Call out to per-macro handlers after clearing the persistent database - * key. If the macro sets the database key, flush it to the database. + * If the string contains escape sequences, + * replace it with an allocated rendering and return 1, + * such that the caller can free it after use. + * Otherwise, do nothing and return 0. */ -static void -pmdoc_node(MDOC_ARGS) +static int +render_string(char **public, size_t *psz) { + const char *src, *scp, *addcp, *seq; + char *dst; + size_t ssz, dsz, addsz; + char utfbuf[7], res[6]; + int seqlen, unicode; - if (NULL == n) - return; + res[0] = '\\'; + res[1] = '\t'; + res[2] = ASCII_NBRSP; + res[3] = ASCII_HYPH; + res[4] = ASCII_BREAK; + res[5] = '\0'; - switch (n->type) { - case (MDOC_HEAD): - /* FALLTHROUGH */ - case (MDOC_BODY): - /* FALLTHROUGH */ - case (MDOC_TAIL): - /* FALLTHROUGH */ - case (MDOC_BLOCK): - /* FALLTHROUGH */ - case (MDOC_ELEM): - buf->len = 0; + src = scp = *public; + ssz = *psz; + dst = NULL; + dsz = 0; + while (scp < src + *psz) { + + /* Leave normal characters unchanged. */ + + if (strchr(res, *scp) == NULL) { + if (dst != NULL) + dst[dsz++] = *scp; + scp++; + continue; + } + /* - * Both NULL handlers and handlers returning true - * request using the data. Only skip the element - * when the handler returns false. + * Found something that requires replacing, + * make sure we have a destination buffer. */ - if (NULL != mdocs[n->tok].fp && - 0 == (*mdocs[n->tok].fp)(hash, buf, dbuf, n, m)) + if (dst == NULL) { + dst = mandoc_malloc(ssz + 1); + dsz = scp - src; + memcpy(dst, src, dsz); + } + + /* Handle single-char special characters. */ + + switch (*scp) { + case '\\': break; + case '\t': + /* FALLTHROUGH */ + case ASCII_NBRSP: + dst[dsz++] = ' '; + scp++; + continue; + case ASCII_HYPH: + dst[dsz++] = '-'; + /* FALLTHROUGH */ + case ASCII_BREAK: + scp++; + continue; + default: + abort(); + } /* - * For many macros, use the text from all children. - * Set zero flags for macros not needing this. - * In that case, the handler must fill the buffer. + * Found an escape sequence. + * Read past the slash, then parse it. + * Ignore everything except characters. */ - if (MDOCF_CHILD & mdocs[n->tok].flags) - buf_appendmdoc(buf, n->child, 0); + scp++; + if (mandoc_escape(&scp, &seq, &seqlen) != ESCAPE_SPECIAL) + continue; /* - * Cover the most common case: - * Automatically stage one string per element. - * Set a zero mask for macros not needing this. - * Additional staging can be done in the handler. + * Render the special character + * as either UTF-8 or ASCII. */ - if (mdocs[n->tok].mask) - hash_put(hash, buf, mdocs[n->tok].mask); - break; - default: - break; + if (write_utf8) { + unicode = mchars_spec2cp(mchars, seq, seqlen); + if (unicode <= 0) + continue; + addsz = utf8(unicode, utfbuf); + if (addsz == 0) + continue; + addcp = utfbuf; + } else { + addcp = mchars_spec2str(mchars, seq, seqlen, &addsz); + if (addcp == NULL) + continue; + if (*addcp == ASCII_NBRSP) { + addcp = " "; + addsz = 1; + } + } + + /* Copy the rendered glyph into the stream. */ + + ssz += addsz; + dst = mandoc_realloc(dst, ssz + 1); + memcpy(dst + dsz, addcp, addsz); + dsz += addsz; } + if (dst != NULL) { + *public = dst; + *psz = dsz; + } - pmdoc_node(hash, buf, dbuf, n->child, m); - pmdoc_node(hash, buf, dbuf, n->next, m); + /* Trim trailing whitespace and NUL-terminate. */ + + while (*psz > 0 && (*public)[*psz - 1] == ' ') + --*psz; + if (dst != NULL) { + (*public)[*psz] = '\0'; + return 1; + } else + return 0; } -static int -pman_node(MAN_ARGS) +static void +dbadd_mlink(const struct mlink *mlink) { - const struct man_node *head, *body; - const char *start, *sv; - size_t sz; + size_t i; - if (NULL == n) - return(0); + i = 1; + SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->dsec); + SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->arch); + SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->name); + SQL_BIND_INT64(stmts[STMT_INSERT_LINK], i, mlink->mpage->pageid); + SQL_STEP(stmts[STMT_INSERT_LINK]); + sqlite3_reset(stmts[STMT_INSERT_LINK]); +} - /* - * We're only searching for one thing: the first text child in - * the BODY of a NAME section. Since we don't keep track of - * sections in -man, run some hoops to find out whether we're in - * the correct section or not. - */ +static void +dbadd_mlink_name(const struct mlink *mlink) +{ + uint64_t bits; + size_t i; - if (MAN_BODY == n->type && MAN_SH == n->tok) { - body = n; - assert(body->parent); - if (NULL != (head = body->parent->head) && - 1 == head->nchild && - NULL != (head = (head->child)) && - MAN_TEXT == head->type && - 0 == strcmp(head->string, "NAME") && - NULL != (body = body->child) && - MAN_TEXT == body->type) { + dbadd_mlink(mlink); - assert(body->string); - start = sv = body->string; + i = 1; + SQL_BIND_INT64(stmts[STMT_SELECT_NAME], i, mlink->mpage->pageid); + bits = NAME_FILE & NAME_MASK; + if (sqlite3_step(stmts[STMT_SELECT_NAME]) == SQLITE_ROW) { + bits |= sqlite3_column_int64(stmts[STMT_SELECT_NAME], 0); + sqlite3_reset(stmts[STMT_SELECT_NAME]); + } - /* - * Go through a special heuristic dance here. - * This is why -man manuals are great! - * (I'm being sarcastic: my eyes are bleeding.) - * Conventionally, one or more manual names are - * comma-specified prior to a whitespace, then a - * dash, then a description. Try to puzzle out - * the name parts here. - */ + i = 1; + SQL_BIND_INT64(stmts[STMT_INSERT_NAME], i, bits); + SQL_BIND_TEXT(stmts[STMT_INSERT_NAME], i, mlink->name); + SQL_BIND_INT64(stmts[STMT_INSERT_NAME], i, mlink->mpage->pageid); + SQL_STEP(stmts[STMT_INSERT_NAME]); + sqlite3_reset(stmts[STMT_INSERT_NAME]); +} - for ( ;; ) { - sz = strcspn(start, " ,"); - if ('\0' == start[(int)sz]) - break; +/* + * Flush the current page's terms (and their bits) into the database. + * Wrap the entire set of additions in a transaction to make sqlite be a + * little faster. + * Also, handle escape sequences at the last possible moment. + */ +static void +dbadd(struct mpage *mpage) +{ + struct mlink *mlink; + struct str *key; + char *cp; + size_t i; + unsigned int slot; + int mustfree; - buf->len = 0; - buf_appendb(buf, start, sz); - buf_appendb(buf, "", 1); + mlink = mpage->mlinks; - hash_put(hash, buf, TYPE_Nm); - - if (' ' == start[(int)sz]) { - start += (int)sz + 1; - break; - } - - assert(',' == start[(int)sz]); - start += (int)sz + 1; - while (' ' == *start) - start++; + if (nodb) { + for (key = ohash_first(&names, &slot); NULL != key; + key = ohash_next(&names, &slot)) + free(key); + for (key = ohash_first(&strings, &slot); NULL != key; + key = ohash_next(&strings, &slot)) + free(key); + if (0 == debug) + return; + while (NULL != mlink) { + fputs(mlink->name, stdout); + if (NULL == mlink->next || + strcmp(mlink->dsec, mlink->next->dsec) || + strcmp(mlink->fsec, mlink->next->fsec) || + strcmp(mlink->arch, mlink->next->arch)) { + putchar('('); + if ('\0' == *mlink->dsec) + fputs(mlink->fsec, stdout); + else + fputs(mlink->dsec, stdout); + if ('\0' != *mlink->arch) + printf("/%s", mlink->arch); + putchar(')'); } + mlink = mlink->next; + if (NULL != mlink) + fputs(", ", stdout); + } + printf(" - %s\n", mpage->desc); + return; + } - buf->len = 0; + if (debug) + say(mlink->file, "Adding to database"); - if (sv == start) { - buf_append(buf, start); - return(1); - } + cp = mpage->desc; + i = strlen(cp); + mustfree = render_string(&cp, &i); + i = 1; + SQL_BIND_TEXT(stmts[STMT_INSERT_PAGE], i, cp); + SQL_BIND_INT(stmts[STMT_INSERT_PAGE], i, mpage->form); + SQL_STEP(stmts[STMT_INSERT_PAGE]); + mpage->pageid = sqlite3_last_insert_rowid(db); + sqlite3_reset(stmts[STMT_INSERT_PAGE]); + if (mustfree) + free(cp); - while (' ' == *start) - start++; + while (NULL != mlink) { + dbadd_mlink(mlink); + mlink = mlink->next; + } + mlink = mpage->mlinks; - if (0 == strncmp(start, "-", 1)) - start += 1; - else if (0 == strncmp(start, "\\-", 2)) - start += 2; - else if (0 == strncmp(start, "\\(en", 4)) - start += 4; - else if (0 == strncmp(start, "\\(em", 4)) - start += 4; + for (key = ohash_first(&names, &slot); NULL != key; + key = ohash_next(&names, &slot)) { + assert(key->mpage == mpage); + i = 1; + SQL_BIND_INT64(stmts[STMT_INSERT_NAME], i, key->mask); + SQL_BIND_TEXT(stmts[STMT_INSERT_NAME], i, key->key); + SQL_BIND_INT64(stmts[STMT_INSERT_NAME], i, mpage->pageid); + SQL_STEP(stmts[STMT_INSERT_NAME]); + sqlite3_reset(stmts[STMT_INSERT_NAME]); + free(key); + } + for (key = ohash_first(&strings, &slot); NULL != key; + key = ohash_next(&strings, &slot)) { + assert(key->mpage == mpage); + i = 1; + SQL_BIND_INT64(stmts[STMT_INSERT_KEY], i, key->mask); + SQL_BIND_TEXT(stmts[STMT_INSERT_KEY], i, key->key); + SQL_BIND_INT64(stmts[STMT_INSERT_KEY], i, mpage->pageid); + SQL_STEP(stmts[STMT_INSERT_KEY]); + sqlite3_reset(stmts[STMT_INSERT_KEY]); + free(key); + } +} - while (' ' == *start) - start++; +static void +dbprune(void) +{ + struct mpage *mpage; + struct mlink *mlink; + size_t i; + unsigned int slot; - sz = strlen(start) + 1; - buf_appendb(dbuf, start, sz); - buf_appendb(buf, start, sz); + if (0 == nodb) + SQL_EXEC("BEGIN TRANSACTION"); - hash_put(hash, buf, TYPE_Nd); + for (mpage = ohash_first(&mpages, &slot); NULL != mpage; + mpage = ohash_next(&mpages, &slot)) { + mlink = mpage->mlinks; + if (debug) + say(mlink->file, "Deleting from database"); + if (nodb) + continue; + for ( ; NULL != mlink; mlink = mlink->next) { + i = 1; + SQL_BIND_TEXT(stmts[STMT_DELETE_PAGE], + i, mlink->dsec); + SQL_BIND_TEXT(stmts[STMT_DELETE_PAGE], + i, mlink->arch); + SQL_BIND_TEXT(stmts[STMT_DELETE_PAGE], + i, mlink->name); + SQL_STEP(stmts[STMT_DELETE_PAGE]); + sqlite3_reset(stmts[STMT_DELETE_PAGE]); } } - for (n = n->child; n; n = n->next) - if (pman_node(hash, buf, dbuf, n)) - return(1); - - return(0); + if (0 == nodb) + SQL_EXEC("END TRANSACTION"); } /* - * Parse a formatted manual page. - * By necessity, this involves rather crude guesswork. + * Close an existing database and its prepared statements. + * If "real" is not set, rename the temporary file into the real one. */ static void -pformatted(DB *hash, struct buf *buf, struct buf *dbuf, - const struct of *of) +dbclose(int real) { - FILE *stream; - char *line, *p; - size_t len, plen; + size_t i; + int status; + pid_t child; - if (NULL == (stream = fopen(of->fname, "r"))) { - perror(of->fname); + if (nodb) return; + + for (i = 0; i < STMT__MAX; i++) { + sqlite3_finalize(stmts[i]); + stmts[i] = NULL; } - /* - * Always use the title derived from the filename up front, - * do not even try to find it in the file. This also makes - * sure we don't end up with an orphan index record, even if - * the file content turns out to be completely unintelligible. - */ + sqlite3_close(db); + db = NULL; - buf->len = 0; - buf_append(buf, of->title); - hash_put(hash, buf, TYPE_Nm); + if (real) + return; - /* Skip to first blank line. */ + if ('\0' == *tempfilename) { + if (-1 == rename(MANDOC_DB "~", MANDOC_DB)) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say(MANDOC_DB, "&rename"); + } + return; + } - while (NULL != (line = fgetln(stream, &len))) - if ('\n' == *line) - break; + switch (child = fork()) { + case -1: + exitcode = (int)MANDOCLEVEL_SYSERR; + say("", "&fork cmp"); + return; + case 0: + execlp("cmp", "cmp", "-s", + tempfilename, MANDOC_DB, (char *)NULL); + say("", "&exec cmp"); + exit(0); + default: + break; + } + if (-1 == waitpid(child, &status, 0)) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say("", "&wait cmp"); + } else if (WIFSIGNALED(status)) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say("", "cmp died from signal %d", WTERMSIG(status)); + } else if (WEXITSTATUS(status)) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say(MANDOC_DB, + "Data changed, but cannot replace database"); + } - /* - * Assume the first line that is not indented - * is the first section header. Skip to it. - */ - - while (NULL != (line = fgetln(stream, &len))) - if ('\n' != *line && ' ' != *line) - break; - - /* - * If no page content can be found, or the input line - * is already the next section header, or there is no - * trailing newline, reuse the page title as the page - * description. - */ - - line = fgetln(stream, &len); - if (NULL == line || ' ' != *line || '\n' != line[(int)len - 1]) { - buf_appendb(dbuf, buf->cp, buf->size); - hash_put(hash, buf, TYPE_Nd); - fclose(stream); + *strrchr(tempfilename, '/') = '\0'; + switch (child = fork()) { + case -1: + exitcode = (int)MANDOCLEVEL_SYSERR; + say("", "&fork rm"); return; + case 0: + execlp("rm", "rm", "-rf", tempfilename, (char *)NULL); + say("", "&exec rm"); + exit((int)MANDOCLEVEL_SYSERR); + default: + break; } + if (-1 == waitpid(child, &status, 0)) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say("", "&wait rm"); + } else if (WIFSIGNALED(status) || WEXITSTATUS(status)) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say("", "%s: Cannot remove temporary directory", + tempfilename); + } +} - line[(int)--len] = '\0'; +/* + * This is straightforward stuff. + * Open a database connection to a "temporary" database, then open a set + * of prepared statements we'll use over and over again. + * If "real" is set, we use the existing database; if not, we truncate a + * temporary one. + * Must be matched by dbclose(). + */ +static int +dbopen(int real) +{ + const char *sql; + int rc, ofl; - /* - * Skip to the first dash. - * Use the remaining line as the description (no more than 70 - * bytes). - */ + if (nodb) + return 1; - if (NULL != (p = strstr(line, "- "))) { - for (p += 2; ' ' == *p || '\b' == *p; p++) - /* Skip to next word. */ ; - } else - p = line; + *tempfilename = '\0'; + ofl = SQLITE_OPEN_READWRITE; - if ((plen = strlen(p)) > 70) { - plen = 70; - p[plen] = '\0'; + if (real) { + rc = sqlite3_open_v2(MANDOC_DB, &db, ofl, NULL); + if (SQLITE_OK != rc) { + exitcode = (int)MANDOCLEVEL_SYSERR; + if (SQLITE_CANTOPEN != rc) + say(MANDOC_DB, "%s", sqlite3_errstr(rc)); + return 0; + } + goto prepare_statements; } - /* Strip backspace-encoding from line. */ + ofl |= SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE; - while (NULL != (line = memchr(p, '\b', plen))) { - len = line - p; - if (0 == len) { - memmove(line, line + 1, plen--); - continue; - } - memmove(line - 1, line + 1, plen - len); - plen -= 2; + remove(MANDOC_DB "~"); + rc = sqlite3_open_v2(MANDOC_DB "~", &db, ofl, NULL); + if (SQLITE_OK == rc) + goto create_tables; + if (MPARSE_QUICK & mparse_options) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say(MANDOC_DB "~", "%s", sqlite3_errstr(rc)); + return 0; } - buf_appendb(dbuf, p, plen + 1); - buf->len = 0; - buf_appendb(buf, p, plen + 1); - hash_put(hash, buf, TYPE_Nd); - fclose(stream); -} + (void)strlcpy(tempfilename, "/tmp/mandocdb.XXXXXX", + sizeof(tempfilename)); + if (NULL == mkdtemp(tempfilename)) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say("", "&%s", tempfilename); + return 0; + } + (void)strlcat(tempfilename, "/" MANDOC_DB, + sizeof(tempfilename)); + rc = sqlite3_open_v2(tempfilename, &db, ofl, NULL); + if (SQLITE_OK != rc) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say("", "%s: %s", tempfilename, sqlite3_errstr(rc)); + return 0; + } -static void -ofile_argbuild(int argc, char *argv[], struct of **of) -{ - char buf[MAXPATHLEN]; - char *sec, *arch, *title, *p; - int i, src_form; - struct of *nof; +create_tables: + sql = "CREATE TABLE \"mpages\" (\n" + " \"desc\" TEXT NOT NULL,\n" + " \"form\" INTEGER NOT NULL,\n" + " \"pageid\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n" + ");\n" + "\n" + "CREATE TABLE \"mlinks\" (\n" + " \"sec\" TEXT NOT NULL,\n" + " \"arch\" TEXT NOT NULL,\n" + " \"name\" TEXT NOT NULL,\n" + " \"pageid\" INTEGER NOT NULL REFERENCES mpages(pageid) " + "ON DELETE CASCADE\n" + ");\n" + "CREATE INDEX mlinks_pageid_idx ON mlinks (pageid);\n" + "\n" + "CREATE TABLE \"names\" (\n" + " \"bits\" INTEGER NOT NULL,\n" + " \"name\" TEXT NOT NULL,\n" + " \"pageid\" INTEGER NOT NULL REFERENCES mpages(pageid) " + "ON DELETE CASCADE,\n" + " UNIQUE (\"name\", \"pageid\") ON CONFLICT REPLACE\n" + ");\n" + "\n" + "CREATE TABLE \"keys\" (\n" + " \"bits\" INTEGER NOT NULL,\n" + " \"key\" TEXT NOT NULL,\n" + " \"pageid\" INTEGER NOT NULL REFERENCES mpages(pageid) " + "ON DELETE CASCADE\n" + ");\n" + "CREATE INDEX keys_pageid_idx ON keys (pageid);\n"; - for (i = 0; i < argc; i++) { + if (SQLITE_OK != sqlite3_exec(db, sql, NULL, NULL, NULL)) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say(MANDOC_DB, "%s", sqlite3_errmsg(db)); + sqlite3_close(db); + return 0; + } - /* - * Try to infer the manual section, architecture and - * page title from the path, assuming it looks like - * man*[/]/.<section> or - * cat<section>[/<arch>]/<title>.0 - */ +prepare_statements: + if (SQLITE_OK != sqlite3_exec(db, + "PRAGMA foreign_keys = ON", NULL, NULL, NULL)) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say(MANDOC_DB, "PRAGMA foreign_keys: %s", + sqlite3_errmsg(db)); + sqlite3_close(db); + return 0; + } - if (strlcpy(buf, argv[i], sizeof(buf)) >= sizeof(buf)) { - fprintf(stderr, "%s: Path too long\n", argv[i]); - continue; - } - sec = arch = title = NULL; - src_form = 0; - p = strrchr(buf, '\0'); - while (p-- > buf) { - if (NULL == sec && '.' == *p) { - sec = p + 1; - *p = '\0'; - if ('0' == *sec) - src_form |= MANDOC_FORM; - else if ('1' <= *sec && '9' >= *sec) - src_form |= MANDOC_SRC; - continue; - } - if ('/' != *p) - continue; - if (NULL == title) { - title = p + 1; - *p = '\0'; - continue; - } - if (0 == strncmp("man", p + 1, 3)) - src_form |= MANDOC_SRC; - else if (0 == strncmp("cat", p + 1, 3)) - src_form |= MANDOC_FORM; - else - arch = p + 1; - break; - } - if (NULL == title) - title = buf; + sql = "DELETE FROM mpages WHERE pageid IN " + "(SELECT pageid FROM mlinks WHERE " + "sec=? AND arch=? AND name=?)"; + sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_DELETE_PAGE], NULL); + sql = "INSERT INTO mpages " + "(desc,form) VALUES (?,?)"; + sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_PAGE], NULL); + sql = "INSERT INTO mlinks " + "(sec,arch,name,pageid) VALUES (?,?,?,?)"; + sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_LINK], NULL); + sql = "SELECT bits FROM names where pageid = ?"; + sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_SELECT_NAME], NULL); + sql = "INSERT INTO names " + "(bits,name,pageid) VALUES (?,?,?)"; + sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_NAME], NULL); + sql = "INSERT INTO keys " + "(bits,key,pageid) VALUES (?,?,?)"; + sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_KEY], NULL); - /* - * Build the file structure. - */ +#ifndef __APPLE__ + /* + * When opening a new database, we can turn off + * synchronous mode for much better performance. + */ - nof = mandoc_calloc(1, sizeof(struct of)); - nof->fname = mandoc_strdup(argv[i]); - if (NULL != sec) - nof->sec = mandoc_strdup(sec); - if (NULL != arch) - nof->arch = mandoc_strdup(arch); - nof->title = mandoc_strdup(title); - nof->src_form = src_form; - - /* - * Add the structure to the list. - */ - - if (verb > 2) - printf("%s: Scheduling\n", argv[i]); - if (NULL == *of) { - *of = nof; - (*of)->first = nof; - } else { - nof->first = (*of)->first; - (*of)->next = nof; - *of = nof; - } + if (real && SQLITE_OK != sqlite3_exec(db, + "PRAGMA synchronous = OFF", NULL, NULL, NULL)) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say(MANDOC_DB, "PRAGMA synchronous: %s", + sqlite3_errmsg(db)); + sqlite3_close(db); + return 0; } +#endif + + return 1; } -/* - * Recursively build up a list of files to parse. - * We use this instead of ftw() and so on because I don't want global - * variables hanging around. - * This ignores the mandoc.db and mandoc.index files, but assumes that - * everything else is a manual. - * Pass in a pointer to a NULL structure for the first invocation. - */ -static int -ofile_dirbuild(const char *dir, const char* psec, const char *parch, - int p_src_form, struct of **of) +static void * +hash_calloc(size_t n, size_t sz, void *arg) { - char buf[MAXPATHLEN]; - struct stat sb; - size_t sz; - DIR *d; - const char *fn, *sec, *arch; - char *p, *q, *suffix; - struct of *nof; - struct dirent *dp; - int src_form; - if (NULL == (d = opendir(dir))) { - perror(dir); - return(0); - } + return mandoc_calloc(n, sz); +} - while (NULL != (dp = readdir(d))) { - fn = dp->d_name; +static void * +hash_alloc(size_t sz, void *arg) +{ - if ('.' == *fn) - continue; + return mandoc_malloc(sz); +} - src_form = p_src_form; +static void +hash_free(void *p, void *arg) +{ - if (DT_DIR == dp->d_type) { - sec = psec; - arch = parch; + free(p); +} - /* - * By default, only use directories called: - * man<section>/[<arch>/] or - * cat<section>/[<arch>/] - */ +static int +set_basedir(const char *targetdir, int report_baddir) +{ + static char startdir[PATH_MAX]; + static int getcwd_status; /* 1 = ok, 2 = failure */ + static int chdir_status; /* 1 = changed directory */ + char *cp; - if (NULL == sec) { - if(0 == strncmp("man", fn, 3)) { - src_form |= MANDOC_SRC; - sec = fn + 3; - } else if (0 == strncmp("cat", fn, 3)) { - src_form |= MANDOC_FORM; - sec = fn + 3; - } else if (use_all) - sec = fn; - else - continue; - } else if (NULL == arch && (use_all || - NULL == strchr(fn, '.'))) - arch = fn; - else if (0 == use_all) - continue; + /* + * Remember the original working directory, if possible. + * This will be needed if the second or a later directory + * on the command line is given as a relative path. + * Do not error out if the current directory is not + * searchable: Maybe it won't be needed after all. + */ + if (0 == getcwd_status) { + if (NULL == getcwd(startdir, sizeof(startdir))) { + getcwd_status = 2; + (void)strlcpy(startdir, strerror(errno), + sizeof(startdir)); + } else + getcwd_status = 1; + } - buf[0] = '\0'; - strlcat(buf, dir, MAXPATHLEN); - strlcat(buf, "/", MAXPATHLEN); - sz = strlcat(buf, fn, MAXPATHLEN); + /* + * We are leaving the old base directory. + * Do not use it any longer, not even for messages. + */ + *basedir = '\0'; - if (MAXPATHLEN <= sz) { - fprintf(stderr, "%s: Path too long\n", dir); - return(0); - } - - if (verb > 2) - printf("%s: Scanning\n", buf); - - if ( ! ofile_dirbuild(buf, sec, arch, - src_form, of)) - return(0); + /* + * If and only if the directory was changed earlier and + * the next directory to process is given as a relative path, + * first go back, or bail out if that is impossible. + */ + if (chdir_status && '/' != *targetdir) { + if (2 == getcwd_status) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say("", "getcwd: %s", startdir); + return 0; } - if (DT_REG != dp->d_type || - (NULL == psec && !use_all) || - !strcmp(MANDOC_DB, fn) || - !strcmp(MANDOC_IDX, fn)) - continue; - - /* - * By default, skip files where the file name suffix - * does not agree with the section directory - * they are located in. - */ - - suffix = strrchr(fn, '.'); - if (0 == use_all) { - if (NULL == suffix) - continue; - if ((MANDOC_SRC & src_form && - strcmp(suffix + 1, psec)) || - (MANDOC_FORM & src_form && - strcmp(suffix + 1, "0"))) - continue; + if (-1 == chdir(startdir)) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say("", "&chdir %s", startdir); + return 0; } - if (NULL != suffix) { - if ('0' == suffix[1]) - src_form |= MANDOC_FORM; - else if ('1' <= suffix[1] && '9' >= suffix[1]) - src_form |= MANDOC_SRC; - } + } - - /* - * Skip formatted manuals if a source version is - * available. Ignore the age: it is very unlikely - * that people install newer formatted base manuals - * when they used to have source manuals before, - * and in ports, old manuals get removed on update. - */ - if (0 == use_all && MANDOC_FORM & src_form && - NULL != psec) { - buf[0] = '\0'; - strlcat(buf, dir, MAXPATHLEN); - p = strrchr(buf, '/'); - if (NULL != parch && NULL != p) - for (p--; p > buf; p--) - if ('/' == *p) - break; - if (NULL == p) - p = buf; - else - p++; - if (0 == strncmp("cat", p, 3)) - memcpy(p, "man", 3); - strlcat(buf, "/", MAXPATHLEN); - sz = strlcat(buf, fn, MAXPATHLEN); - if (sz >= MAXPATHLEN) { - fprintf(stderr, "%s: Path too long\n", buf); - continue; - } - q = strrchr(buf, '.'); - if (NULL != q && p < q++) { - *q = '\0'; - sz = strlcat(buf, psec, MAXPATHLEN); - if (sz >= MAXPATHLEN) { - fprintf(stderr, - "%s: Path too long\n", buf); - continue; - } - if (0 == stat(buf, &sb)) - continue; - } + /* + * Always resolve basedir to the canonicalized absolute + * pathname and append a trailing slash, such that + * we can reliably check whether files are inside. + */ + if (NULL == realpath(targetdir, basedir)) { + if (report_baddir || errno != ENOENT) { + exitcode = (int)MANDOCLEVEL_BADARG; + say("", "&%s: realpath", targetdir); } - - buf[0] = '\0'; - strlcat(buf, dir, MAXPATHLEN); - strlcat(buf, "/", MAXPATHLEN); - sz = strlcat(buf, fn, MAXPATHLEN); - if (sz >= MAXPATHLEN) { - fprintf(stderr, "%s: Path too long\n", dir); - continue; + return 0; + } else if (-1 == chdir(basedir)) { + if (report_baddir || errno != ENOENT) { + exitcode = (int)MANDOCLEVEL_BADARG; + say("", "&chdir"); } - - nof = mandoc_calloc(1, sizeof(struct of)); - nof->fname = mandoc_strdup(buf); - if (NULL != psec) - nof->sec = mandoc_strdup(psec); - if (NULL != parch) - nof->arch = mandoc_strdup(parch); - nof->src_form = src_form; - - /* - * Remember the file name without the extension, - * to be used as the page title in the database. - */ - - if (NULL != suffix) - *suffix = '\0'; - nof->title = mandoc_strdup(fn); - - /* - * Add the structure to the list. - */ - - if (verb > 2) - printf("%s: Scheduling\n", buf); - if (NULL == *of) { - *of = nof; - (*of)->first = nof; - } else { - nof->first = (*of)->first; - (*of)->next = nof; - *of = nof; + return 0; + } + chdir_status = 1; + cp = strchr(basedir, '\0'); + if ('/' != cp[-1]) { + if (cp - basedir >= PATH_MAX - 1) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say("", "Filename too long"); + return 0; } + *cp++ = '/'; + *cp = '\0'; } - - closedir(d); - return(1); + return 1; } static void -ofile_free(struct of *of) +say(const char *file, const char *format, ...) { - struct of *nof; + va_list ap; + int use_errno; - while (of) { - nof = of->next; - free(of->fname); - free(of->sec); - free(of->arch); - free(of->title); - free(of); - of = nof; - } -} + if ('\0' != *basedir) + fprintf(stderr, "%s", basedir); + if ('\0' != *basedir && '\0' != *file) + fputc('/', stderr); + if ('\0' != *file) + fprintf(stderr, "%s", file); -static void -usage(void) -{ - - fprintf(stderr, "usage: %s [-v] " - "[-d dir [files...] |" - " -u dir [files...] |" - " dir...]\n", progname); + use_errno = 1; + if (NULL != format) { + switch (*format) { + case '&': + format++; + break; + case '\0': + format = NULL; + break; + default: + use_errno = 0; + break; + } + } + if (NULL != format) { + if ('\0' != *basedir || '\0' != *file) + fputs(": ", stderr); + va_start(ap, format); + vfprintf(stderr, format, ap); + va_end(ap); + } + if (use_errno) { + if ('\0' != *basedir || '\0' != *file || NULL != format) + fputs(": ", stderr); + perror(NULL); + } else + fputc('\n', stderr); }