mandoc/mandocdb.c - diff

Return to mandocdb.c CVS log

Up to [cvsweb.bsd.lv] / mandoc

Diff for /mandoc/mandocdb.c between version 1.90 and 1.150

version 1.90, 2013/12/27 23:41:55

version 1.150, 2014/06/19 00:45:37

Line 1

/* $Id$ */

* Permission to use, copy, modify, and distribute this software for any

* purpose with or without fee is hereby granted, provided that the above

Line 20

#endif

#include <sys/stat.h>

#include <sys/wait.h>

#include <assert.h>

#include <ctype.h>

Line 45

Line 46

#include "mdoc.h"

#include "man.h"

#include "mandoc.h"

#include "mandoc_aux.h"

#include "manpath.h"

#include "mansearch.h"

extern int mansearch_keymax;

extern const char *const mansearch_keynames[];

#define SQL_EXEC(_v) \

if (SQLITE_OK != sqlite3_exec(db, (_v), NULL, NULL, NULL)) \

fprintf(stderr, "%s\n", sqlite3_errmsg(db))

say("", "%s: %s", (_v), sqlite3_errmsg(db))

#define SQL_BIND_TEXT(_s, _i, _v) \

if (SQLITE_OK != sqlite3_bind_text \

((_s), (_i)++, (_v), -1, SQLITE_STATIC)) \

fprintf(stderr, "%s\n", sqlite3_errmsg(db))

say(mlink->file, "%s", sqlite3_errmsg(db))

#define SQL_BIND_INT(_s, _i, _v) \

if (SQLITE_OK != sqlite3_bind_int \

((_s), (_i)++, (_v))) \

fprintf(stderr, "%s\n", sqlite3_errmsg(db))

say(mlink->file, "%s", sqlite3_errmsg(db))

#define SQL_BIND_INT64(_s, _i, _v) \

if (SQLITE_OK != sqlite3_bind_int64 \

((_s), (_i)++, (_v))) \

fprintf(stderr, "%s\n", sqlite3_errmsg(db))

say(mlink->file, "%s", sqlite3_errmsg(db))

#define SQL_STEP(_s) \

if (SQLITE_DONE != sqlite3_step((_s))) \

fprintf(stderr, "%s\n", sqlite3_errmsg(db))

say(mlink->file, "%s", sqlite3_errmsg(db))

enum op {

OP_DEFAULT = 0, /* new dbs from dir list or default config */

Line 82 enum form {

Line 87 enum form {

};

struct str {

char *utf8; /* key in UTF-8 form */

char *rendered; /* key in UTF-8 or ASCII form */

const struct mpage *mpage; /* if set, the owning parse */

uint64_t mask; /* bitmask in sequence */

char key[]; /* the string itself */

char key[]; /* may contain escape sequences */

};

struct inodev {

Line 95 struct inodev {

Line 100 struct inodev {

struct mpage {

struct inodev inodev; /* used for hashing routine */

int64_t pageid; /* pageid in mpages SQL table */

enum form form; /* format from file content */

char *sec; /* section from file content */

char *arch; /* architecture from file content */

Line 112 struct mlink {

Line 118 struct mlink {

char *name; /* name from file name (not empty) */

char *fsec; /* section from file name suffix */

struct mlink *next; /* singly linked list */

struct mpage *mpage; /* parent */

int gzip; /* filename has a .gz suffix */

};

struct title {

char *title; /* name(sec/arch) given inside the file */

char *file; /* file name in case of mismatch */

};

enum stmt {

STMT_DELETE_PAGE = 0, /* delete mpage */

STMT_INSERT_PAGE, /* insert mpage */

STMT_INSERT_LINK, /* insert mlink */

STMT_INSERT_NAME, /* insert name */

STMT_INSERT_KEY, /* insert parsed key */

STMT__MAX

};

Line 135 struct mdoc_handler {

Line 139 struct mdoc_handler {

};

static void dbclose(int);

static void dbindex(const struct mpage *, struct mchars *);

static void dbadd(struct mpage *, struct mchars *);

static void dbadd_mlink(const struct mlink *mlink);

static int dbopen(int);

static void dbprune(void);

static void filescan(const char *);

Line 143 static void *hash_alloc(size_t, void *);

Line 148 static void *hash_alloc(size_t, void *);

static void hash_free(void *, size_t, void *);

static void *hash_halloc(size_t, void *);

static void mlink_add(struct mlink *, const struct stat *);

static void mlink_check(struct mpage *, struct mlink *);

static void mlink_free(struct mlink *);

static void mlinks_undupe(struct mpage *);

static void mpages_free(void);

static void mpages_merge(struct mchars *, struct mparse *, int);

static void mpages_merge(struct mchars *, struct mparse *);

static void parse_cat(struct mpage *);

static void names_check(void);

static void parse_cat(struct mpage *, int);

static void parse_man(struct mpage *, const struct man_node *);

static void parse_mdoc(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_body(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_head(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_Fd(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_Fn(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_In(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_Nd(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_Nm(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_Sh(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_St(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_Xr(struct mpage *, const struct mdoc_node *);

static void putkey(const struct mpage *,

static void putkey(const struct mpage *, char *, uint64_t);

const char *, uint64_t);

static void putkeys(const struct mpage *,

const char *, size_t, uint64_t);

static void putmdockey(const struct mpage *,

const struct mdoc_node *, uint64_t);

static void render_key(struct mchars *, struct str *);

static void say(const char *, const char *, ...);

static int set_basedir(const char *);

static int treescan(void);

static size_t utf8(unsigned int, char [7]);

static void utf8key(struct mchars *, struct str *);

static char tempfilename[32];

static char *progname;

static int use_all; /* use all found files */

static int nodb; /* no database changes */

static int verb; /* print what we're doing */

static int mparse_options; /* abort the parse early */

static int warnings; /* warn about crap */

static int use_all; /* use all found files */

static int debug; /* print what we're doing */

static int warnings; /* warn about crap */

static int write_utf8; /* write UTF-8 output; else ASCII */

static int exitcode; /* to be returned by main */

static enum op op; /* operational mode */

static char basedir[PATH_MAX]; /* current base directory */

static struct ohash mpages; /* table of distinct manual pages */

static struct ohash mlinks; /* table of directory entries */

static struct ohash names; /* table of all names */

static struct ohash strings; /* table of all strings */

static sqlite3 *db = NULL; /* current database */

static sqlite3_stmt *stmts[STMT__MAX]; /* current statements */

static uint64_t name_mask;

static const struct mdoc_handler mdocs[MDOC_MAX] = {

{ NULL, 0 }, /* Ap */

Line 216 static const struct mdoc_handler mdocs[MDOC_MAX] = {

Line 225 static const struct mdoc_handler mdocs[MDOC_MAX] = {

{ parse_mdoc_Fn, 0 }, /* Fn */

{ NULL, TYPE_Ft }, /* Ft */

{ NULL, TYPE_Ic }, /* Ic */

{ parse_mdoc_In, TYPE_In }, /* In */

{ NULL, TYPE_In }, /* In */

{ NULL, TYPE_Li }, /* Li */

{ parse_mdoc_Nd, TYPE_Nd }, /* Nd */

{ parse_mdoc_Nd, 0 }, /* Nd */

{ parse_mdoc_Nm, TYPE_Nm }, /* Nm */

{ parse_mdoc_Nm, 0 }, /* Nm */

{ NULL, 0 }, /* Op */

{ NULL, 0 }, /* Ot */

{ NULL, TYPE_Pa }, /* Pa */

{ NULL, 0 }, /* Rv */

{ parse_mdoc_St, 0 }, /* St */

{ NULL, TYPE_St }, /* St */

{ NULL, TYPE_Va }, /* Va */

{ parse_mdoc_body, TYPE_Va }, /* Vt */

{ parse_mdoc_Xr, 0 }, /* Xr */

Line 311 static const struct mdoc_handler mdocs[MDOC_MAX] = {

Line 320 static const struct mdoc_handler mdocs[MDOC_MAX] = {

{ NULL, 0 }, /* Ta */

};

int

main(int argc, char *argv[])

{

Line 339 main(int argc, char *argv[])

Line 349 main(int argc, char *argv[])

++progname;

* We accept a few different invocations.

* The CHECKOP macro makes sure that invocation styles don't

* clobber each other.

Line 352 main(int argc, char *argv[])

Line 362 main(int argc, char *argv[])

path_arg = NULL;

op = OP_DEFAULT;

while (-1 != (ch = getopt(argc, argv, "aC:d:ntu:vW")))

while (-1 != (ch = getopt(argc, argv, "aC:Dd:npQT:tu:v")))

switch (ch) {

case ('a'):

case 'a':

use_all = 1;

break;

case ('C'):

case 'C':

CHECKOP(op, ch);

path_arg = optarg;

op = OP_CONFFILE;

break;

case ('d'):

case 'D':

debug++;

break;

case 'd':

CHECKOP(op, ch);

path_arg = optarg;

op = OP_UPDATE;

break;

case ('n'):

case 'n':

nodb = 1;

break;

case ('t'):

case 'p':

warnings = 1;

break;

case 'Q':

mparse_options |= MPARSE_QUICK;

break;

case 'T':

if (strcmp(optarg, "utf8")) {

fprintf(stderr, "-T%s: Unsupported "

"output format\n", optarg);

goto usage;

}

write_utf8 = 1;

break;

case 't':

CHECKOP(op, ch);

dup2(STDOUT_FILENO, STDERR_FILENO);

op = OP_TEST;

nodb = warnings = 1;

break;

case ('u'):

case 'u':

CHECKOP(op, ch);

path_arg = optarg;

op = OP_DELETE;

break;

case ('v'):

case 'v':

verb++;

/* Compatibility with espie@'s makewhatis. */

break;

case ('W'):

warnings = 1;

break;

default:

goto usage;

}

Line 400 main(int argc, char *argv[])

Line 424 main(int argc, char *argv[])

}

exitcode = (int)MANDOCLEVEL_OK;

mp = mparse_alloc(MPARSE_AUTO,

mp = mparse_alloc(mparse_options, MANDOCLEVEL_FATAL, NULL, NULL);

MANDOCLEVEL_FATAL, NULL, NULL, NULL);

mc = mchars_alloc();

ohash_init(&mpages, 6, &mpages_info);

ohash_init(&mlinks, 6, &mlinks_info);

if (OP_UPDATE == op || OP_DELETE == op || OP_TEST == op) {

* Force processing all files.

use_all = 1;

* All of these deal with a specific directory.

* Most of these deal with a specific directory.

* Jump into that directory then collect files specified

* Jump into that directory first.

* on the command-line.

if (0 == set_basedir(path_arg))

if (OP_TEST != op && 0 == set_basedir(path_arg))

goto out;

for (i = 0; i < argc; i++)

filescan(argv[i]);

if (dbopen(1)) {

if (0 == dbopen(1))

goto out;

* The existing database is usable. Process

if (OP_TEST != op)

* all files specified on the command-line.

dbprune();

use_all = 1;

for (i = 0; i < argc; i++)

filescan(argv[i]);

if (OP_TEST != op)

dbprune();

} else {

* Database missing or corrupt.

* Recreate from scratch.

exitcode = (int)MANDOCLEVEL_OK;

op = OP_DEFAULT;

if (0 == treescan())

goto out;

if (0 == dbopen(0))

goto out;

}

if (OP_DELETE != op)

mpages_merge(mc, mp, 0);

mpages_merge(mc, mp);

dbclose(1);

dbclose(OP_DEFAULT == op ? 0 : 1);

} else {

* If we have arguments, use them as our manpaths.

Line 436 main(int argc, char *argv[])

Line 471 main(int argc, char *argv[])

* manpath_parse() wants to do it.

if (argc > 0) {

dirs.paths = mandoc_calloc

dirs.paths = mandoc_reallocarray(NULL,

(argc, sizeof(char *));

argc, sizeof(char *));

dirs.sz = (size_t)argc;

for (i = 0; i < argc; i++)

dirs.paths[i] = mandoc_strdup(argv[i]);

} else

manpath_parse(&dirs, path_arg, NULL, NULL);

if (0 == dirs.sz) {

exitcode = (int)MANDOCLEVEL_BADARG;

say("", "Empty manpath");

}

* First scan the tree rooted at a base directory, then

* build a new database and finally move it into place.

Line 471 main(int argc, char *argv[])

Line 511 main(int argc, char *argv[])

if (0 == dbopen(0))

goto out;

mpages_merge(mc, mp, warnings && !use_all);

mpages_merge(mc, mp);

if (warnings && !nodb &&

! (MPARSE_QUICK & mparse_options))

names_check();

dbclose(0);

if (j + 1 < dirs.sz) {

Line 482 main(int argc, char *argv[])

Line 525 main(int argc, char *argv[])

}

out:

set_basedir(NULL);

manpath_free(&dirs);

mchars_free(mc);

mparse_free(mp);

Line 491 out:

Line 533 out:

ohash_delete(&mlinks);

return(exitcode);

usage:

fprintf(stderr, "usage: %s [-anvW] [-C file]\n"

fprintf(stderr, "usage: %s [-aDnpQ] [-C file] [-Tutf8]\n"

" %s [-anvW] dir ...\n"

" %s [-aDnpQ] [-Tutf8] dir ...\n"

" %s [-nvW] -d dir [file ...]\n"

" %s [-DnpQ] [-Tutf8] -d dir [file ...]\n"

" %s [-nvW] -u dir [file ...]\n"

" %s [-Dnp] -u dir [file ...]\n"

" %s -t file ...\n",

" %s [-Q] -t file ...\n",

progname, progname, progname,

progname, progname);

return((int)MANDOCLEVEL_BADARG);

Line 510 usage:

Line 552 usage:

* If use_all has been specified, grok all files.

* If not, sanitise paths to the following:

* [./]man*[/<arch>]/<name>.<section>

* or

* [./]cat<section>[/<arch>]/<name>.0

Line 519 usage:

Line 561 usage:

static int

treescan(void)

{

char buf[PATH_MAX];

FTS *f;

FTSENT *ff;

struct mlink *mlink;

int dform;

int dform, gzip;

char *fsec;

char *dsec, *arch, *fsec, *cp;

const char *dsec, *arch, *cp, *path;

const char *path;

const char *argv[2];

argv[0] = ".";

argv[1] = (char *)NULL;

f = fts_open((char * const *)argv,

* Walk through all components under the directory, using the

FTS_PHYSICAL | FTS_NOCHDIR, NULL);

* logical descent of files.

f = fts_open((char * const *)argv, FTS_LOGICAL, NULL);

if (NULL == f) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say("", NULL);

say("", "&fts_open");

return(0);

}

Line 546 treescan(void)

Line 586 treescan(void)

while (NULL != (ff = fts_read(f))) {

path = ff->fts_path + 2;

switch (ff->fts_info) {

* Symbolic links require various sanity checks,

* then get handled just like regular files.

case FTS_SL:

if (NULL == realpath(path, buf)) {

if (warnings)

say(path, "&realpath");

continue;

}

if (strstr(buf, basedir) != buf) {

if (warnings) say("",

"%s: outside base directory", buf);

continue;

}

/* Use logical inode to avoid mpages dupe. */

if (-1 == stat(path, ff->fts_statp)) {

if (warnings)

say(path, "&stat");

continue;

}

/* FALLTHROUGH */

* If we're a regular file, add an mlink by using the

* stored directory data and handling the filename.

if (FTS_F == ff->fts_info) {

case FTS_F:

if (0 == strcmp(path, MANDOC_DB))

continue;

if ( ! use_all && ff->fts_level < 2) {

if (warnings)

say(path, "Extraneous file");

continue;

} else if (NULL == (fsec =

}

strrchr(ff->fts_name, '.'))) {

gzip = 0;

fsec = NULL;

while (NULL == fsec) {

fsec = strrchr(ff->fts_name, '.');

if (NULL == fsec || strcmp(fsec+1, "gz"))

break;

gzip = 1;

*fsec = '\0';

fsec = NULL;

}

if (NULL == fsec) {

if ( ! use_all) {

if (warnings)

say(path,

Line 569 treescan(void)

Line 644 treescan(void)

if (warnings)

say(path, "Skip html");

continue;

} else if (0 == strcmp(fsec, "gz")) {

if (warnings)

say(path, "Skip gz");

continue;

} else if (0 == strcmp(fsec, "ps")) {

if (warnings)

say(path, "Skip ps");

Line 589 treescan(void)

Line 660 treescan(void)

continue;

} else

fsec[-1] = '\0';

mlink = mandoc_calloc(1, sizeof(struct mlink));

strlcpy(mlink->file, path, sizeof(mlink->file));

if (strlcpy(mlink->file, path,

sizeof(mlink->file)) >=

sizeof(mlink->file)) {

say(path, "Filename too long");

free(mlink);

continue;

}

mlink->dform = dform;

if (NULL != dsec)

mlink->dsec = dsec;

mlink->dsec = mandoc_strdup(dsec);

mlink->arch = arch;

if (NULL != arch)

mlink->name = ff->fts_name;

mlink->arch = mandoc_strdup(arch);

mlink->fsec = fsec;

mlink->name = mandoc_strdup(ff->fts_name);

mlink->gzip = gzip;

if (NULL != fsec)

mlink->fsec = mandoc_strdup(fsec);

mlink_add(mlink, ff->fts_statp);

continue;

} else if (FTS_D != ff->fts_info &&

FTS_DP != ff->fts_info) {

case FTS_D:

/* FALLTHROUGH */

case FTS_DP:

break;

default:

if (warnings)

say(path, "Not a regular file");

continue;

}

switch (ff->fts_level) {

case (0):

case 0:

/* Ignore the root directory. */

break;

case (1):

case 1:

* This might contain manX/ or catX/.

* Try to infer this from the name.

* If we're not in use_all, enforce it.

dsec = NULL;

dform = FORM_NONE;

cp = ff->fts_name;

if (FTS_DP == ff->fts_info)

break;

Line 630 treescan(void)

Line 709 treescan(void)

} else if (0 == strncmp(cp, "cat", 3)) {

dform = FORM_CAT;

dsec = cp + 3;

} else {

dform = FORM_NONE;

dsec = NULL;

}

if (NULL != dsec || use_all)

break;

if (warnings)

say(path, "Unknown directory part");

fts_set(f, ff, FTS_SKIP);

break;

case (2):

case 2:

* Possibly our architecture.

* If we're descending, keep tabs on it.

arch = NULL;

if (FTS_DP != ff->fts_info && NULL != dsec)

arch = ff->fts_name;

else

arch = NULL;

break;

default:

if (FTS_DP == ff->fts_info || use_all)

Line 670 treescan(void)

Line 753 treescan(void)

* Try to infer the manual section, architecture, and page name from the

* path, assuming it looks like

* [./]man*[/<arch>]/<name>.<section>

* or

* [./]cat<section>[/<arch>]/<name>.0

Line 689 filescan(const char *file)

Line 772 filescan(const char *file)

if (0 == strncmp(file, "./", 2))

file += 2;

if (NULL == realpath(file, buf)) {

* We have to do lstat(2) before realpath(3) loses

* the information whether this is a symbolic link.

* We need to know that because for symbolic links,

* we want to use the orginal file name, while for

* regular files, we want to use the real path.

if (-1 == lstat(file, &st)) {

exitcode = (int)MANDOCLEVEL_BADARG;

say(file, NULL);

say(file, "&lstat");

return;

} else if (OP_TEST != op && strstr(buf, basedir) != buf) {

} else if (0 == ((S_IFREG | S_IFLNK) & st.st_mode)) {

exitcode = (int)MANDOCLEVEL_BADARG;

say("", "%s: outside base directory", buf);

say(file, "Not a regular file");

return;

} else if (-1 == stat(buf, &st)) {

}

* We have to resolve the file name to the real path

* in any case for the base directory check.

if (NULL == realpath(file, buf)) {

exitcode = (int)MANDOCLEVEL_BADARG;

say(file, NULL);

say(file, "&realpath");

return;

} else if ( ! (S_IFREG & st.st_mode)) {

}

if (OP_TEST == op)

start = buf;

else if (strstr(buf, basedir) == buf)

start = buf + strlen(basedir);

else {

exitcode = (int)MANDOCLEVEL_BADARG;

say(file, "Not a regular file");

say("", "%s: outside base directory", buf);

return;

}

start = buf + strlen(basedir);

* Now we are sure the file is inside our tree.

* If it is a symbolic link, ignore the real path

* and use the original name.

* This implies passing stuff like "cat1/../man1/foo.1"

* on the command line won't work. So don't do that.

* Note the stat(2) can still fail if the link target

* doesn't exist.

if (S_IFLNK & st.st_mode) {

if (-1 == stat(buf, &st)) {

exitcode = (int)MANDOCLEVEL_BADARG;

say(file, "&stat");

return;

}

if (strlcpy(buf, file, sizeof(buf)) >= sizeof(buf)) {

say(file, "Filename too long");

return;

}

start = buf;

if (OP_TEST != op && strstr(buf, basedir) == buf)

start += strlen(basedir);

}

mlink = mandoc_calloc(1, sizeof(struct mlink));

strlcpy(mlink->file, start, sizeof(mlink->file));

if (strlcpy(mlink->file, start, sizeof(mlink->file)) >=

sizeof(mlink->file)) {

say(start, "Filename too long");

return;

}

* First try to guess our directory structure.

Line 720 filescan(const char *file)

Line 850 filescan(const char *file)

*p++ = '\0';

if (0 == strncmp(start, "man", 3)) {

mlink->dform = FORM_SRC;

mlink->dsec = mandoc_strdup(start + 3);

mlink->dsec = start + 3;

} else if (0 == strncmp(start, "cat", 3)) {

mlink->dform = FORM_CAT;

mlink->dsec = mandoc_strdup(start + 3);

mlink->dsec = start + 3;

}

start = p;

if (NULL != mlink->dsec && NULL != (p = strchr(start, '/'))) {

*p++ = '\0';

mlink->arch = mandoc_strdup(start);

mlink->arch = start;

start = p;

}

Line 744 filescan(const char *file)

Line 874 filescan(const char *file)

if ('.' == *p) {

*p++ = '\0';

mlink->fsec = mandoc_strdup(p);

mlink->fsec = p;

}

Line 756 filescan(const char *file)

Line 886 filescan(const char *file)

mlink->name = p + 1;

*p = '\0';

}

mlink->name = mandoc_strdup(mlink->name);

mlink_add(mlink, &st);

}

Line 770 mlink_add(struct mlink *mlink, const struct stat *st)

Line 898 mlink_add(struct mlink *mlink, const struct stat *st)

assert(NULL != mlink->file);

if (NULL == mlink->dsec)

mlink->dsec = mandoc_strdup(mlink->dsec ? mlink->dsec : "");

mlink->dsec = mandoc_strdup("");

mlink->arch = mandoc_strdup(mlink->arch ? mlink->arch : "");

if (NULL == mlink->arch)

mlink->name = mandoc_strdup(mlink->name ? mlink->name : "");

mlink->arch = mandoc_strdup("");

mlink->fsec = mandoc_strdup(mlink->fsec ? mlink->fsec : "");

if (NULL == mlink->name)

mlink->name = mandoc_strdup("");

if (NULL == mlink->fsec)

mlink->fsec = mandoc_strdup("");

if ('0' == *mlink->fsec) {

free(mlink->fsec);

Line 805 mlink_add(struct mlink *mlink, const struct stat *st)

Line 929 mlink_add(struct mlink *mlink, const struct stat *st)

} else

mlink->next = mpage->mlinks;

mpage->mlinks = mlink;

mlink->mpage = mpage;

}

static void

Line 861 mlinks_undupe(struct mpage *mpage)

Line 986 mlinks_undupe(struct mpage *mpage)

mpage->form = FORM_NONE;

goto nextlink;

}

if (strlcpy(buf, mlink->file, PATH_MAX) >= PATH_MAX) {

(void)strlcpy(buf, mlink->file, sizeof(buf));

if (warnings)

say(mlink->file, "Filename too long");

goto nextlink;

}

bufp = strstr(buf, "cat");

assert(NULL != bufp);

memcpy(bufp, "man", 3);

if (NULL != (bufp = strrchr(buf, '.')))

*++bufp = '\0';

strlcat(buf, mlink->dsec, PATH_MAX);

(void)strlcat(buf, mlink->dsec, sizeof(buf));

if (NULL == ohash_find(&mlinks,

ohash_qlookup(&mlinks, buf)))

goto nextlink;

if (warnings)

say(mlink->file, "Man source exists: %s", buf);

Line 887 nextlink:

Line 1008 nextlink:

}

static void

mlink_check(struct mpage *mpage, struct mlink *mlink)

{

struct str *str;

unsigned int slot;

* Check whether the manual section given in a file

* agrees with the directory where the file is located.

* Some manuals have suffixes like (3p) on their

* section number either inside the file or in the

* directory name, some are linked into more than one

* section, like encrypt(1) = makekey(8).

if (FORM_SRC == mpage->form &&

strcasecmp(mpage->sec, mlink->dsec))

say(mlink->file, "Section \"%s\" manual in %s directory",

mpage->sec, mlink->dsec);

* Manual page directories exist for each kernel

* architecture as returned by machine(1).

* However, many manuals only depend on the

* application architecture as returned by arch(1).

* For example, some (2/ARM) manuals are shared

* across the "armish" and "zaurus" kernel

* architectures.

* A few manuals are even shared across completely

* different architectures, for example fdformat(1)

* on amd64, i386, sparc, and sparc64.

if (strcasecmp(mpage->arch, mlink->arch))

say(mlink->file, "Architecture \"%s\" manual in "

"\"%s\" directory", mpage->arch, mlink->arch);

* XXX

* parse_cat() doesn't set NAME_TITLE yet.

if (FORM_CAT == mpage->form)

return;

* Check whether this mlink

* appears as a name in the NAME section.

slot = ohash_qlookup(&names, mlink->name);

str = ohash_find(&names, slot);

assert(NULL != str);

if ( ! (NAME_TITLE & str->mask))

say(mlink->file, "Name missing in NAME section");

}

* Run through the files in the global vector "mpages"

* and add them to the database specified in "basedir".

Line 895 nextlink:

Line 1073 nextlink:

* and filename to determine whether the file is parsable or not.

static void

mpages_merge(struct mchars *mc, struct mparse *mp, int check_reachable)

mpages_merge(struct mchars *mc, struct mparse *mp)

{

struct ohash title_table;

char any[] = "any";

struct ohash_info title_info, str_info;

struct ohash_info str_info;

struct mpage *mpage;

int fd[2];

struct mpage *mpage, *mpage_dest;

struct mlink *mlink, *mlink_dest;

struct mdoc *mdoc;

struct man *man;

struct title *title_entry;

char *sodest;

char *title_str;

char *cp;

const char *cp;

pid_t child_pid;

int match;

int status;

unsigned int pslot, tslot;

unsigned int pslot;

enum mandoclevel lvl;

str_info.alloc = hash_alloc;

Line 914 mpages_merge(struct mchars *mc, struct mparse *mp, int

Line 1094 mpages_merge(struct mchars *mc, struct mparse *mp, int

str_info.hfree = hash_free;

str_info.key_offset = offsetof(struct str, key);

if (check_reachable) {

if (0 == nodb)

title_info.alloc = hash_alloc;

SQL_EXEC("BEGIN TRANSACTION");

title_info.halloc = hash_halloc;

title_info.hfree = hash_free;

title_info.key_offset = offsetof(struct title, title);

ohash_init(&title_table, 6, &title_info);

}

mpage = ohash_first(&mpages, &pslot);

while (NULL != mpage) {

Line 930 mpages_merge(struct mchars *mc, struct mparse *mp, int

Line 1105 mpages_merge(struct mchars *mc, struct mparse *mp, int

continue;

}

name_mask = NAME_MASK;

ohash_init(&names, 4, &str_info);

ohash_init(&strings, 6, &str_info);

mparse_reset(mp);

mdoc = NULL;

man = NULL;

match = 1;

sodest = NULL;

child_pid = 0;

fd[0] = -1;

fd[1] = -1;

if (mpage->mlinks->gzip) {

if (-1 == pipe(fd)) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say(mpage->mlinks->file, "&pipe gunzip");

goto nextpage;

}

switch (child_pid = fork()) {

case -1:

exitcode = (int)MANDOCLEVEL_SYSERR;

say(mpage->mlinks->file, "&fork gunzip");

child_pid = 0;

close(fd[1]);

close(fd[0]);

goto nextpage;

case 0:

close(fd[0]);

if (-1 == dup2(fd[1], STDOUT_FILENO)) {

say(mpage->mlinks->file,

"&dup gunzip");

exit(1);

}

execlp("gunzip", "gunzip", "-c",

mpage->mlinks->file, NULL);

say(mpage->mlinks->file, "&exec gunzip");

exit(1);

default:

close(fd[1]);

break;

}

* Try interpreting the file as mdoc(7) or man(7)

* source code, unless it is already known to be

Line 943 mpages_merge(struct mchars *mc, struct mparse *mp, int

Line 1154 mpages_merge(struct mchars *mc, struct mparse *mp, int

if (FORM_CAT != mpage->mlinks->dform ||

FORM_CAT != mpage->mlinks->fform) {

lvl = mparse_readfd(mp, -1, mpage->mlinks->file);

lvl = mparse_readfd(mp, fd[0], mpage->mlinks->file);

if (lvl < MANDOCLEVEL_FATAL)

mparse_result(mp, &mdoc, &man);

mparse_result(mp, &mdoc, &man, &sodest);

}

if (NULL != mdoc) {

if (NULL != sodest) {

mlink_dest = ohash_find(&mlinks,

ohash_qlookup(&mlinks, sodest));

if (NULL != mlink_dest) {

/* The .so target exists. */

mpage_dest = mlink_dest->mpage;

mlink = mpage->mlinks;

while (1) {

mlink->mpage = mpage_dest;

* If the target was already

* processed, add the links

* to the database now.

* Otherwise, this will

* happen when we come

* to the target.

if (mpage_dest->pageid)

dbadd_mlink(mlink);

if (NULL == mlink->next)

break;

mlink = mlink->next;

}

/* Move all links to the target. */

mlink->next = mlink_dest->next;

mlink_dest->next = mpage->mlinks;

mpage->mlinks = NULL;

}

goto nextpage;

} else if (NULL != mdoc) {

mpage->form = FORM_SRC;

mpage->sec =

mandoc_strdup(mdoc_meta(mdoc)->msec);

Line 974 mpages_merge(struct mchars *mc, struct mparse *mp, int

Line 1221 mpages_merge(struct mchars *mc, struct mparse *mp, int

mpage->title =

mandoc_strdup(mpage->mlinks->name);

}

putkey(mpage, mpage->sec, TYPE_sec);

putkey(mpage, '\0' == *mpage->arch ?

any : mpage->arch, TYPE_arch);

for (mlink = mpage->mlinks; mlink; mlink = mlink->next) {

* Check whether the manual section given in a file

if ('\0' != *mlink->dsec)

* agrees with the directory where the file is located.

putkey(mpage, mlink->dsec, TYPE_sec);

* Some manuals have suffixes like (3p) on their

if ('\0' != *mlink->fsec)

* section number either inside the file or in the

putkey(mpage, mlink->fsec, TYPE_sec);

* directory name, some are linked into more than one

putkey(mpage, '\0' == *mlink->arch ?

* section, like encrypt(1) = makekey(8). Do not skip

any : mlink->arch, TYPE_arch);

* manuals for such reasons.

putkey(mpage, mlink->name, NAME_FILE);

if (warnings && !use_all && FORM_SRC == mpage->form &&

strcasecmp(mpage->sec, mpage->mlinks->dsec)) {

match = 0;

say(mpage->mlinks->file, "Section \"%s\" "

"manual in %s directory",

mpage->sec, mpage->mlinks->dsec);

}

assert(NULL == mpage->desc);

* Manual page directories exist for each kernel

* architecture as returned by machine(1).

* However, many manuals only depend on the

* application architecture as returned by arch(1).

* For example, some (2/ARM) manuals are shared

* across the "armish" and "zaurus" kernel

* architectures.

* A few manuals are even shared across completely

* different architectures, for example fdformat(1)

* on amd64, i386, sparc, and sparc64.

* Thus, warn about architecture mismatches,

* but don't skip manuals for this reason.

if (warnings && !use_all &&

strcasecmp(mpage->arch, mpage->mlinks->arch)) {

match = 0;

say(mpage->mlinks->file, "Architecture \"%s\" "

"manual in \"%s\" directory",

mpage->arch, mpage->mlinks->arch);

}

if (warnings && !use_all &&

strcasecmp(mpage->title, mpage->mlinks->name))

match = 0;

putkey(mpage, mpage->mlinks->name, TYPE_Nm);

if (NULL != mdoc) {

if (NULL != (cp = mdoc_meta(mdoc)->name))

putkey(mpage, cp, TYPE_Nm);

putkey(mpage, cp, NAME_HEAD);

assert(NULL == mpage->desc);

parse_mdoc(mpage, mdoc_node(mdoc));

putkey(mpage, NULL != mpage->desc ?

mpage->desc : mpage->mlinks->name, TYPE_Nd);

} else if (NULL != man)

parse_man(mpage, man_node(man));

else

parse_cat(mpage);

parse_cat(mpage, fd[0]);

if (NULL == mpage->desc)

mpage->desc = mandoc_strdup(mpage->mlinks->name);

if (warnings && !use_all)

* Build a title string for the file. If it matches

for (mlink = mpage->mlinks; mlink;

* the location of the file, remember the title as

mlink = mlink->next)

* found; else, remember it as missing.

mlink_check(mpage, mlink);

if (check_reachable) {

dbadd(mpage, mc);

if (-1 == asprintf(&title_str, "%s(%s%s%s)",

mpage->title, mpage->sec,

'\0' == *mpage->arch ? "" : "/",

if (child_pid) {

mpage->arch)) {

if (-1 == waitpid(child_pid, &status, 0)) {

perror(NULL);

exitcode = (int)MANDOCLEVEL_SYSERR;

exit((int)MANDOCLEVEL_SYSERR);

say(mpage->mlinks->file, "&wait gunzip");

} else if (WIFSIGNALED(status)) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say(mpage->mlinks->file,

"gunzip died from signal %d",

WTERMSIG(status));

} else if (WEXITSTATUS(status)) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say(mpage->mlinks->file,

"gunzip failed with code %d",

WEXITSTATUS(status));

}

tslot = ohash_qlookup(&title_table, title_str);

title_entry = ohash_find(&title_table, tslot);

if (NULL == title_entry) {

title_entry = mandoc_malloc(

sizeof(struct title));

title_entry->title = title_str;

title_entry->file = mandoc_strdup(

match ? "" : mpage->mlinks->file);

ohash_insert(&title_table, tslot,

title_entry);

} else {

if (match)

*title_entry->file = '\0';

free(title_str);

}

dbindex(mpage, mc);

ohash_delete(&strings);

ohash_delete(&names);

mpage = ohash_next(&mpages, &pslot);

}

if (check_reachable) {

if (0 == nodb)

title_entry = ohash_first(&title_table, &tslot);

SQL_EXEC("END TRANSACTION");

while (NULL != title_entry) {

}

if ('\0' != *title_entry->file)

say(title_entry->file,

static void

"Probably unreachable, title is %s",

names_check(void)

title_entry->title);

{

free(title_entry->title);

sqlite3_stmt *stmt;

free(title_entry->file);

const char *name, *sec, *arch, *key;

free(title_entry);

int irc;

title_entry = ohash_next(&title_table, &tslot);

}

sqlite3_prepare_v2(db,

ohash_delete(&title_table);

"SELECT name, sec, arch, key FROM ("

"SELECT name AS key, pageid FROM names "

"WHERE bits & ? AND NOT EXISTS ("

"SELECT pageid FROM mlinks "

"WHERE mlinks.pageid == names.pageid "

"AND mlinks.name == names.name"

")"

") JOIN ("

"SELECT sec, arch, name, pageid FROM mlinks "

"GROUP BY pageid"

") USING (pageid);",

-1, &stmt, NULL);

if (SQLITE_OK != sqlite3_bind_int64(stmt, 1, NAME_TITLE))

say("", "%s", sqlite3_errmsg(db));

while (SQLITE_ROW == (irc = sqlite3_step(stmt))) {

name = sqlite3_column_text(stmt, 0);

sec = sqlite3_column_text(stmt, 1);

arch = sqlite3_column_text(stmt, 2);

key = sqlite3_column_text(stmt, 3);

say("", "%s(%s%s%s) lacks mlink \"%s\"", name, sec,

'\0' == *arch ? "" : "/",

'\0' == *arch ? "" : arch, key);

}

sqlite3_finalize(stmt);

}

static void

parse_cat(struct mpage *mpage)

parse_cat(struct mpage *mpage, int fd)

{

FILE *stream;

char *line, *p, *title;

size_t len, plen, titlesz;

if (NULL == (stream = fopen(mpage->mlinks->file, "r"))) {

stream = (-1 == fd) ?

fopen(mpage->mlinks->file, "r") :

fdopen(fd, "r");

if (NULL == stream) {

if (warnings)

say(mpage->mlinks->file, NULL);

say(mpage->mlinks->file, "&fopen");

return;

}

Line 1110 parse_cat(struct mpage *mpage)

Line 1346 parse_cat(struct mpage *mpage)

while (NULL != (line = fgetln(stream, &len)))

if ('\n' != *line && ' ' != *line)

break;

* Read up until the next section into a buffer.

* Strip the leading and trailing newline from each read line,

Line 1147 parse_cat(struct mpage *mpage)

Line 1383 parse_cat(struct mpage *mpage)

if (warnings)

say(mpage->mlinks->file,

"Cannot find NAME section");

assert(NULL == mpage->desc);

mpage->desc = mandoc_strdup(mpage->mlinks->name);

putkey(mpage, mpage->mlinks->name, TYPE_Nd);

fclose(stream);

free(title);

return;

Line 1183 parse_cat(struct mpage *mpage)

Line 1416 parse_cat(struct mpage *mpage)

if (0 == len) {

memmove(line, line + 1, plen--);

continue;

}

memmove(line - 1, line + 1, plen - len);

plen -= 2;

}

assert(NULL == mpage->desc);

mpage->desc = mandoc_strdup(p);

putkey(mpage, mpage->desc, TYPE_Nd);

fclose(stream);

free(title);

}

Line 1199 parse_cat(struct mpage *mpage)

Line 1430 parse_cat(struct mpage *mpage)

* Put a type/word pair into the word database for this particular file.

static void

putkey(const struct mpage *mpage, const char *value, uint64_t type)

putkey(const struct mpage *mpage, char *value, uint64_t type)

{

char *cp;

assert(NULL != value);

if (TYPE_arch == type)

for (cp = value; *cp; cp++)

if (isupper((unsigned char)*cp))

*cp = _tolower((unsigned char)*cp);

putkeys(mpage, value, strlen(value), type);

}

Line 1226 static void

Line 1462 static void

parse_man(struct mpage *mpage, const struct man_node *n)

{

const struct man_node *head, *body;

char *start, *sv, *title;

char *start, *title;

char byte;

size_t sz, titlesz;

size_t sz;

if (NULL == n)

return;

Line 1244 parse_man(struct mpage *mpage, const struct man_node *

Line 1480 parse_man(struct mpage *mpage, const struct man_node *

body = n;

assert(body->parent);

if (NULL != (head = body->parent->head) &&

1 == head->nchild &&

NULL != (head = (head->child)) &&

MAN_TEXT == head->type &&

0 == strcmp(head->string, "NAME") &&

NULL != (body = body->child) &&

NULL != body->child) {

MAN_TEXT == body->type) {

title = NULL;

titlesz = 0;

* Suck the entire NAME section into memory.

* Yes, we might run away.

Line 1261 parse_man(struct mpage *mpage, const struct man_node *

Line 1493 parse_man(struct mpage *mpage, const struct man_node *

* NAME sections over many lines.

for ( ; NULL != body; body = body->next) {

title = NULL;

if (MAN_TEXT != body->type)

man_deroff(&title, body);

break;

if (0 == (sz = strlen(body->string)))

continue;

title = mandoc_realloc

(title, titlesz + sz + 1);

memcpy(title + titlesz, body->string, sz);

titlesz += sz + 1;

title[titlesz - 1] = ' ';

}

if (NULL == title)

return;

title = mandoc_realloc(title, titlesz + 1);

title[titlesz] = '\0';

/* Skip leading space. */

sv = title;

while (isspace((unsigned char)*sv))

sv++;

if (0 == (sz = strlen(sv))) {

free(title);

return;

}

/* Erase trailing space. */

start = &sv[sz - 1];

while (start > sv && isspace((unsigned char)*start))

*start-- = '\0';

if (start == sv) {

free(title);

return;

}

start = sv;

* Go through a special heuristic dance here.

* Conventionally, one or more manual names are

* comma-specified prior to a whitespace, then a

Line 1310 parse_man(struct mpage *mpage, const struct man_node *

Line 1506 parse_man(struct mpage *mpage, const struct man_node *

* the name parts here.

start = title;

for ( ;; ) {

sz = strcspn(start, " ,");

if ('\0' == start[sz])

Line 1318 parse_man(struct mpage *mpage, const struct man_node *

Line 1515 parse_man(struct mpage *mpage, const struct man_node *

byte = start[sz];

start[sz] = '\0';

putkey(mpage, start, TYPE_Nm);

* Assume a stray trailing comma in the

* name list if a name begins with a dash.

if ('-' == start[0] ||

('\\' == start[0] && '-' == start[1]))

break;

putkey(mpage, start, NAME_TITLE);

if (' ' == byte) {

start += sz + 1;

break;

Line 1331 parse_man(struct mpage *mpage, const struct man_node *

Line 1537 parse_man(struct mpage *mpage, const struct man_node *

start++;

}

if (sv == start) {

if (start == title) {

putkey(mpage, start, TYPE_Nm);

putkey(mpage, start, NAME_TITLE);

free(title);

return;

}

Line 1354 parse_man(struct mpage *mpage, const struct man_node *

Line 1560 parse_man(struct mpage *mpage, const struct man_node *

while (' ' == *start)

start++;

assert(NULL == mpage->desc);

mpage->desc = mandoc_strdup(start);

putkey(mpage, mpage->desc, TYPE_Nd);

free(title);

return;

}

Line 1376 parse_mdoc(struct mpage *mpage, const struct mdoc_node

Line 1580 parse_mdoc(struct mpage *mpage, const struct mdoc_node

assert(NULL != n);

for (n = n->child; NULL != n; n = n->next) {

switch (n->type) {

case (MDOC_ELEM):

case MDOC_ELEM:

/* FALLTHROUGH */

case (MDOC_BLOCK):

case MDOC_BLOCK:

/* FALLTHROUGH */

case (MDOC_HEAD):

case MDOC_HEAD:

/* FALLTHROUGH */

case (MDOC_BODY):

case MDOC_BODY:

/* FALLTHROUGH */

case (MDOC_TAIL):

case MDOC_TAIL:

if (NULL != mdocs[n->tok].fp)

if (0 == (*mdocs[n->tok].fp)(mpage, n))

break;

Line 1408 parse_mdoc_Fd(struct mpage *mpage, const struct mdoc_n

Line 1612 parse_mdoc_Fd(struct mpage *mpage, const struct mdoc_n

size_t sz;

if (SEC_SYNOPSIS != n->sec ||

NULL == (n = n->child) ||

MDOC_TEXT != n->type)

return(0);

Line 1441 parse_mdoc_Fd(struct mpage *mpage, const struct mdoc_n

Line 1645 parse_mdoc_Fd(struct mpage *mpage, const struct mdoc_n

if (end > start)

putkeys(mpage, start, end - start + 1, TYPE_In);

return(1);

return(0);

}

static int

parse_mdoc_In(struct mpage *mpage, const struct mdoc_node *n)

{

if (NULL != n->child && MDOC_TEXT == n->child->type)

return(0);

putkey(mpage, n->child->string, TYPE_In);

return(1);

}

static int

parse_mdoc_Fn(struct mpage *mpage, const struct mdoc_node *n)

{

const char *cp;

char *cp;

if (NULL == (n = n->child) || MDOC_TEXT != n->type)

return(0);

* Parse: .Fn "struct type *name" "char *arg".

* First strip away pointer symbol.

* Then store the function name, then type.

* Finally, store the arguments.

if (NULL == (cp = strrchr(n->string, ' ')))

Line 1489 parse_mdoc_Fn(struct mpage *mpage, const struct mdoc_n

Line 1682 parse_mdoc_Fn(struct mpage *mpage, const struct mdoc_n

}

static int

parse_mdoc_St(struct mpage *mpage, const struct mdoc_node *n)

{

if (NULL == n->child || MDOC_TEXT != n->child->type)

return(0);

putkey(mpage, n->child->string, TYPE_St);

return(1);

}

static int

parse_mdoc_Xr(struct mpage *mpage, const struct mdoc_node *n)

{

char *cp;

Line 1512 parse_mdoc_Xr(struct mpage *mpage, const struct mdoc_n

Line 1694 parse_mdoc_Xr(struct mpage *mpage, const struct mdoc_n

return(0);

}

if (-1 == asprintf(&cp, "%s(%s)", n->string, n->next->string)) {

mandoc_asprintf(&cp, "%s(%s)", n->string, n->next->string);

perror(NULL);

exit((int)MANDOCLEVEL_SYSERR);

}

putkey(mpage, cp, TYPE_Xr);

free(cp);

return(0);

Line 1524 parse_mdoc_Xr(struct mpage *mpage, const struct mdoc_n

Line 1703 parse_mdoc_Xr(struct mpage *mpage, const struct mdoc_n

static int

parse_mdoc_Nd(struct mpage *mpage, const struct mdoc_node *n)

{

size_t sz;

if (MDOC_BODY != n->type)

if (MDOC_BODY == n->type)

return(0);

mdoc_deroff(&mpage->desc, n);

return(0);

* Special-case the `Nd' because we need to put the description

* into the document table.

for (n = n->child; NULL != n; n = n->next) {

if (MDOC_TEXT == n->type) {

if (NULL != mpage->desc) {

sz = strlen(mpage->desc) +

strlen(n->string) + 2;

mpage->desc = mandoc_realloc(

mpage->desc, sz);

strlcat(mpage->desc, " ", sz);

strlcat(mpage->desc, n->string, sz);

} else

mpage->desc = mandoc_strdup(n->string);

}

if (NULL != n->child)

parse_mdoc_Nd(mpage, n);

}

return(1);

}

static int

Line 1557 parse_mdoc_Nm(struct mpage *mpage, const struct mdoc_n

Line 1714 parse_mdoc_Nm(struct mpage *mpage, const struct mdoc_n

{

if (SEC_NAME == n->sec)

return(1);

putmdockey(mpage, n->child, NAME_TITLE);

else if (SEC_SYNOPSIS != n->sec || MDOC_HEAD != n->type)

else if (SEC_SYNOPSIS == n->sec && MDOC_HEAD == n->type)

return(0);

putmdockey(mpage, n->child, NAME_SYN);

return(0);

return(1);

}

static int

Line 1594 static void

Line 1750 static void

putkeys(const struct mpage *mpage,

const char *cp, size_t sz, uint64_t v)

{

struct ohash *htab;

struct str *s;

unsigned int slot;

const char *end;

unsigned int slot;

int i;

if (0 == sz)

return;

if (TYPE_Nm & v) {

htab = &names;

v &= name_mask;

name_mask &= ~NAME_FIRST;

if (debug > 1)

say(mpage->mlinks->file,

"Adding name %*s", sz, cp);

} else {

htab = &strings;

if (debug > 1)

for (i = 0; i < mansearch_keymax; i++)

if (1 << i & v)

say(mpage->mlinks->file,

"Adding key %s=%*s",

mansearch_keynames[i], sz, cp);

}

end = cp + sz;

slot = ohash_qlookupi(&strings, cp, &end);

slot = ohash_qlookupi(htab, cp, &end);

s = ohash_find(&strings, slot);

s = ohash_find(htab, slot);

if (NULL != s && mpage == s->mpage) {

s->mask |= v;

return;

} else if (NULL == s) {

s = mandoc_calloc(sizeof(struct str) + sz + 1, 1);

s = mandoc_calloc(1, sizeof(struct str) + sz + 1);

memcpy(s->key, cp, sz);

ohash_insert(&strings, slot, s);

ohash_insert(htab, slot, s);

}

s->mpage = mpage;

s->mask = v;

Line 1670 utf8(unsigned int cp, char out[7])

Line 1845 utf8(unsigned int cp, char out[7])

}

* Store the UTF-8 version of a key, or alias the pointer if the key has

* Store the rendered version of a key, or alias the pointer

* no UTF-8 transcription marks in it.

* if the key contains no escape sequences.

static void

utf8key(struct mchars *mc, struct str *key)

render_key(struct mchars *mc, struct str *key)

{

size_t sz, bsz, pos;

char utfbuf[7], res[5];

char utfbuf[7], res[6];

char *buf;

const char *seq, *cpp, *val;

int len, u;

enum mandoc_esc esc;

assert(NULL == key->utf8);

assert(NULL == key->rendered);

res[0] = '\\';

res[1] = '\t';

res[2] = ASCII_NBRSP;

res[3] = ASCII_HYPH;

res[4] = '\0';

res[4] = ASCII_BREAK;

res[5] = '\0';

val = key->key;

bsz = strlen(val);

Line 1699 utf8key(struct mchars *mc, struct str *key)

Line 1875 utf8key(struct mchars *mc, struct str *key)

* pointer as ourselvse and get out of here.

if (strcspn(val, res) == bsz) {

key->utf8 = key->key;

key->rendered = key->key;

return;

}

/* Pre-allocate by the length of the input */

Line 1720 utf8key(struct mchars *mc, struct str *key)

Line 1896 utf8key(struct mchars *mc, struct str *key)

val += sz;

}

if (ASCII_HYPH == *val) {

switch (*val) {

case ASCII_HYPH:

buf[pos++] = '-';

val++;

continue;

} else if ('\t' == *val || ASCII_NBRSP == *val) {

case '\t':

/* FALLTHROUGH */

case ASCII_NBRSP:

buf[pos++] = ' ';

val++;

/* FALLTHROUGH */

case ASCII_BREAK:

continue;

} else if ('\\' != *val)

default:

break;

}

if ('\\' != *val)

break;

/* Read past the slash. */

val++;

u = 0;

* Parse the escape sequence and see if it's a

* predefined character or special character.

esc = mandoc_escape

((const char **)&val, &seq, &len);

esc = mandoc_escape((const char **)&val,

&seq, &len);

if (ESCAPE_ERROR == esc)

break;

if (ESCAPE_SPECIAL != esc)

continue;

if (0 == (u = mchars_spec2cp(mc, seq, len)))

continue;

* If we have a Unicode codepoint, try to convert that

* Render the special character

* to a UTF-8 byte string.

* as either UTF-8 or ASCII.

cpp = utfbuf;

if (0 == (sz = utf8(u, utfbuf)))

continue;

if (write_utf8) {

if (0 == (u = mchars_spec2cp(mc, seq, len)))

continue;

cpp = utfbuf;

if (0 == (sz = utf8(u, utfbuf)))

continue;

sz = strlen(cpp);

} else {

cpp = mchars_spec2str(mc, seq, len, &sz);

if (NULL == cpp)

continue;

if (ASCII_NBRSP == *cpp) {

cpp = " ";

sz = 1;

}

/* Copy the rendered glyph into the stream. */

sz = strlen(cpp);

bsz += sz;

buf = mandoc_realloc(buf, bsz);

memcpy(&buf[pos], cpp, sz);

pos += sz;

}

buf[pos] = '\0';

key->utf8 = buf;

key->rendered = buf;

}

static void

dbadd_mlink(const struct mlink *mlink)

{

size_t i;

i = 1;

SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->dsec);

SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->arch);

SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->name);

SQL_BIND_INT64(stmts[STMT_INSERT_LINK], i, mlink->mpage->pageid);

SQL_STEP(stmts[STMT_INSERT_LINK]);

sqlite3_reset(stmts[STMT_INSERT_LINK]);

}

* Flush the current page's terms (and their bits) into the database.

* Wrap the entire set of additions in a transaction to make sqlite be a

* little faster.

* Also, UTF-8-encode the description at the last possible moment.

* Also, handle escape sequences at the last possible moment.

static void

dbindex(const struct mpage *mpage, struct mchars *mc)

dbadd(struct mpage *mpage, struct mchars *mc)

{

struct mlink *mlink;

struct str *key;

const char *desc;

int64_t recno;

size_t i;

unsigned int slot;

if (verb)

mlink = mpage->mlinks;

say(mpage->mlinks->file, "Adding to index");

if (nodb)

if (nodb) {

for (key = ohash_first(&names, &slot); NULL != key;

key = ohash_next(&names, &slot)) {

if (key->rendered != key->key)

free(key->rendered);

free(key);

}

for (key = ohash_first(&strings, &slot); NULL != key;

key = ohash_next(&strings, &slot)) {

if (key->rendered != key->key)

free(key->rendered);

free(key);

}

if (0 == debug)

return;

while (NULL != mlink) {

fputs(mlink->name, stdout);

if (NULL == mlink->next ||

strcmp(mlink->dsec, mlink->next->dsec) ||

strcmp(mlink->fsec, mlink->next->fsec) ||

strcmp(mlink->arch, mlink->next->arch)) {

putchar('(');

if ('\0' == *mlink->dsec)

fputs(mlink->fsec, stdout);

else

fputs(mlink->dsec, stdout);

if ('\0' != *mlink->arch)

printf("/%s", mlink->arch);

putchar(')');

}

mlink = mlink->next;

if (NULL != mlink)

fputs(", ", stdout);

}

printf(" - %s\n", mpage->desc);

return;

desc = "";

if (NULL != mpage->desc && '\0' != *mpage->desc) {

key = ohash_find(&strings,

ohash_qlookup(&strings, mpage->desc));

assert(NULL != key);

if (NULL == key->utf8)

utf8key(mc, key);

desc = key->utf8;

}

SQL_EXEC("BEGIN TRANSACTION");

if (debug)

say(mlink->file, "Adding to database");

i = strlen(mpage->desc) + 1;

key = mandoc_calloc(1, sizeof(struct str) + i);

memcpy(key->key, mpage->desc, i);

render_key(mc, key);

i = 1;

SQL_BIND_TEXT(stmts[STMT_INSERT_PAGE], i, key->rendered);

* XXX The following three lines are obsolete

* and only kept for backward compatibility

* until apropos(1) and friends have caught up.

SQL_BIND_TEXT(stmts[STMT_INSERT_PAGE], i, mpage->mlinks->file);

SQL_BIND_TEXT(stmts[STMT_INSERT_PAGE], i, mpage->mlinks->dsec);

SQL_BIND_TEXT(stmts[STMT_INSERT_PAGE], i, mpage->mlinks->arch);

SQL_BIND_TEXT(stmts[STMT_INSERT_PAGE], i, desc);

SQL_BIND_INT(stmts[STMT_INSERT_PAGE], i, FORM_SRC == mpage->form);

SQL_STEP(stmts[STMT_INSERT_PAGE]);

recno = sqlite3_last_insert_rowid(db);

mpage->pageid = sqlite3_last_insert_rowid(db);

sqlite3_reset(stmts[STMT_INSERT_PAGE]);

for (mlink = mpage->mlinks; mlink; mlink = mlink->next) {

if (key->rendered != key->key)

i = 1;

free(key->rendered);

SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->file);

free(key);

SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->dsec);

SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->arch);

while (NULL != mlink) {

SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->name);

dbadd_mlink(mlink);

SQL_BIND_INT64(stmts[STMT_INSERT_LINK], i, recno);

mlink = mlink->next;

SQL_STEP(stmts[STMT_INSERT_LINK]);

sqlite3_reset(stmts[STMT_INSERT_LINK]);

}

mlink = mpage->mlinks;

for (key = ohash_first(&names, &slot); NULL != key;

key = ohash_next(&names, &slot)) {

assert(key->mpage == mpage);

if (NULL == key->rendered)

render_key(mc, key);

i = 1;

SQL_BIND_INT64(stmts[STMT_INSERT_NAME], i, key->mask);

SQL_BIND_TEXT(stmts[STMT_INSERT_NAME], i, key->rendered);

SQL_BIND_INT64(stmts[STMT_INSERT_NAME], i, mpage->pageid);

SQL_STEP(stmts[STMT_INSERT_NAME]);

sqlite3_reset(stmts[STMT_INSERT_NAME]);

if (key->rendered != key->key)

free(key->rendered);

free(key);

}

for (key = ohash_first(&strings, &slot); NULL != key;

key = ohash_next(&strings, &slot)) {

assert(key->mpage == mpage);

if (NULL == key->utf8)

if (NULL == key->rendered)

utf8key(mc, key);

render_key(mc, key);

i = 1;

SQL_BIND_INT64(stmts[STMT_INSERT_KEY], i, key->mask);

SQL_BIND_TEXT(stmts[STMT_INSERT_KEY], i, key->utf8);

SQL_BIND_TEXT(stmts[STMT_INSERT_KEY], i, key->rendered);

SQL_BIND_INT64(stmts[STMT_INSERT_KEY], i, recno);

SQL_BIND_INT64(stmts[STMT_INSERT_KEY], i, mpage->pageid);

SQL_STEP(stmts[STMT_INSERT_KEY]);

sqlite3_reset(stmts[STMT_INSERT_KEY]);

if (key->utf8 != key->key)

if (key->rendered != key->key)

free(key->utf8);

free(key->rendered);

free(key);

}

SQL_EXEC("END TRANSACTION");

}

static void

Line 1860 dbprune(void)

Line 2098 dbprune(void)

size_t i;

unsigned int slot;

if (nodb)

if (0 == nodb)

return;

SQL_EXEC("BEGIN TRANSACTION");

mpage = ohash_first(&mpages, &slot);

for (mpage = ohash_first(&mpages, &slot); NULL != mpage;

while (NULL != mpage) {

mpage = ohash_next(&mpages, &slot)) {

mlink = mpage->mlinks;

i = 1;

if (debug)

SQL_BIND_TEXT(stmts[STMT_DELETE_PAGE], i, mlink->file);

say(mlink->file, "Deleting from database");

SQL_STEP(stmts[STMT_DELETE_PAGE]);

if (nodb)

sqlite3_reset(stmts[STMT_DELETE_PAGE]);

continue;

if (verb)

for ( ; NULL != mlink; mlink = mlink->next) {

say(mlink->file, "Deleted from index");

i = 1;

mpage = ohash_next(&mpages, &slot);

SQL_BIND_TEXT(stmts[STMT_DELETE_PAGE],

i, mlink->dsec);

SQL_BIND_TEXT(stmts[STMT_DELETE_PAGE],

i, mlink->arch);

SQL_BIND_TEXT(stmts[STMT_DELETE_PAGE],

i, mlink->name);

SQL_STEP(stmts[STMT_DELETE_PAGE]);

sqlite3_reset(stmts[STMT_DELETE_PAGE]);

}

if (0 == nodb)

SQL_EXEC("END TRANSACTION");

}

Line 1884 static void

Line 2133 static void

dbclose(int real)

{

size_t i;

int status;

pid_t child;

if (nodb)

return;

Line 1899 dbclose(int real)

Line 2150 dbclose(int real)

if (real)

return;

if (-1 == rename(MANDOC_DB "~", MANDOC_DB)) {

if ('\0' == *tempfilename) {

if (-1 == rename(MANDOC_DB "~", MANDOC_DB)) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say(MANDOC_DB, "&rename");

}

return;

}

switch (child = fork()) {

case -1:

exitcode = (int)MANDOCLEVEL_SYSERR;

say(MANDOC_DB, NULL);

say("", "&fork cmp");

return;

case 0:

execlp("cmp", "cmp", "-s",

tempfilename, MANDOC_DB, NULL);

say("", "&exec cmp");

exit(0);

default:

break;

}

if (-1 == waitpid(child, &status, 0)) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say("", "&wait cmp");

} else if (WIFSIGNALED(status)) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say("", "cmp died from signal %d", WTERMSIG(status));

} else if (WEXITSTATUS(status)) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say(MANDOC_DB,

"Data changed, but cannot replace database");

}

*strrchr(tempfilename, '/') = '\0';

switch (child = fork()) {

case -1:

exitcode = (int)MANDOCLEVEL_SYSERR;

say("", "&fork rm");

return;

case 0:

execlp("rm", "rm", "-rf", tempfilename, NULL);

say("", "&exec rm");

exit((int)MANDOCLEVEL_SYSERR);

default:

break;

}

if (-1 == waitpid(child, &status, 0)) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say("", "&wait rm");

} else if (WIFSIGNALED(status) || WEXITSTATUS(status)) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say("", "%s: Cannot remove temporary directory",

tempfilename);

}

Line 1916 dbclose(int real)

Line 2217 dbclose(int real)

static int

dbopen(int real)

{

const char *file, *sql;

const char *sql;

int rc, ofl;

if (nodb)

return(1);

*tempfilename = '\0';

ofl = SQLITE_OPEN_READWRITE;

if (0 == real) {

file = MANDOC_DB "~";

if (real) {

if (-1 == remove(file) && ENOENT != errno) {

rc = sqlite3_open_v2(MANDOC_DB, &db, ofl, NULL);

if (SQLITE_OK != rc) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say(file, NULL);

if (SQLITE_CANTOPEN != rc)

say(MANDOC_DB, "%s", sqlite3_errstr(rc));

return(0);

}

ofl |= SQLITE_OPEN_EXCLUSIVE;

} else

file = MANDOC_DB;

rc = sqlite3_open_v2(file, &db, ofl, NULL);

if (SQLITE_OK == rc)

goto prepare_statements;

if (SQLITE_CANTOPEN != rc) {

}

ofl |= SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE;

remove(MANDOC_DB "~");

rc = sqlite3_open_v2(MANDOC_DB "~", &db, ofl, NULL);

if (SQLITE_OK == rc)

goto create_tables;

if (MPARSE_QUICK & mparse_options) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say(file, NULL);

say(MANDOC_DB "~", "%s", sqlite3_errstr(rc));

return(0);

}

sqlite3_close(db);

(void)strlcpy(tempfilename, "/tmp/mandocdb.XXXXXX",

db = NULL;

sizeof(tempfilename));

if (NULL == mkdtemp(tempfilename)) {

if (SQLITE_OK != (rc = sqlite3_open(file, &db))) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say(file, NULL);

say("", "&%s", tempfilename);

return(0);

}

(void)strlcat(tempfilename, "/" MANDOC_DB,

sizeof(tempfilename));

rc = sqlite3_open_v2(tempfilename, &db, ofl, NULL);

if (SQLITE_OK != rc) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say("", "%s: %s", tempfilename, sqlite3_errstr(rc));

return(0);

}

create_tables:

* XXX The first three columns in table mpages are obsolete

* and only kept for backward compatibility

* until apropos(1) and friends have caught up.

sql = "CREATE TABLE \"mpages\" (\n"

" \"file\" TEXT NOT NULL,\n"

" \"sec\" TEXT NOT NULL,\n"

" \"arch\" TEXT NOT NULL,\n"

" \"desc\" TEXT NOT NULL,\n"

" \"form\" INTEGER NOT NULL,\n"

" \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n"

" \"pageid\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n"

");\n"

"\n"

"CREATE TABLE \"mlinks\" (\n"

" \"file\" TEXT NOT NULL,\n"

" \"sec\" TEXT NOT NULL,\n"

" \"arch\" TEXT NOT NULL,\n"

" \"name\" TEXT NOT NULL,\n"

" \"pageid\" INTEGER NOT NULL REFERENCES mpages(id) "

" \"pageid\" INTEGER NOT NULL REFERENCES mpages(pageid) "

"ON DELETE CASCADE,\n"

"ON DELETE CASCADE\n"

" \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n"

");\n"

"CREATE INDEX mlinks_pageid_idx ON mlinks (pageid);\n"

"\n"

"CREATE TABLE \"names\" (\n"

" \"bits\" INTEGER NOT NULL,\n"

" \"name\" TEXT NOT NULL,\n"

" \"pageid\" INTEGER NOT NULL REFERENCES mpages(pageid) "

"ON DELETE CASCADE\n"

");\n"

"\n"

"CREATE TABLE \"keys\" (\n"

" \"bits\" INTEGER NOT NULL,\n"

" \"key\" TEXT NOT NULL,\n"

" \"pageid\" INTEGER NOT NULL REFERENCES mpages(id) "

" \"pageid\" INTEGER NOT NULL REFERENCES mpages(pageid) "

"ON DELETE CASCADE,\n"

"ON DELETE CASCADE\n"

" \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n"

");\n"

"\n"

"CREATE INDEX keys_pageid_idx ON keys (pageid);\n";

"CREATE INDEX \"key_index\" ON keys (key);\n";

if (SQLITE_OK != sqlite3_exec(db, sql, NULL, NULL, NULL)) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say(file, "%s", sqlite3_errmsg(db));

say(MANDOC_DB, "%s", sqlite3_errmsg(db));

sqlite3_close(db);

return(0);

}

prepare_statements:

SQL_EXEC("PRAGMA foreign_keys = ON");

if (SQLITE_OK != sqlite3_exec(db,

sql = "DELETE FROM mpages where file=?";

"PRAGMA foreign_keys = ON", NULL, NULL, NULL)) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say(MANDOC_DB, "PRAGMA foreign_keys: %s",

sqlite3_errmsg(db));

sqlite3_close(db);

return(0);

}

sql = "DELETE FROM mpages WHERE pageid IN "

"(SELECT pageid FROM mlinks WHERE "

"sec=? AND arch=? AND name=?)";

sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_DELETE_PAGE], NULL);

sql = "INSERT INTO mpages "

"(file,sec,arch,desc,form) VALUES (?,?,?,?,?)";

"(desc,form) VALUES (?,?)";

sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_PAGE], NULL);

sql = "INSERT INTO mlinks "

"(file,sec,arch,name,pageid) VALUES (?,?,?,?,?)";

"(sec,arch,name,pageid) VALUES (?,?,?,?)";

sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_LINK], NULL);

sql = "INSERT INTO names "

"(bits,name,pageid) VALUES (?,?,?)";

sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_NAME], NULL);

sql = "INSERT INTO keys "

"(bits,key,pageid) VALUES (?,?,?)";

sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_KEY], NULL);

Line 2012 prepare_statements:

Line 2336 prepare_statements:

* synchronous mode for much better performance.

if (real)

if (real && SQLITE_OK != sqlite3_exec(db,

SQL_EXEC("PRAGMA synchronous = OFF");

"PRAGMA synchronous = OFF", NULL, NULL, NULL)) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say(MANDOC_DB, "PRAGMA synchronous: %s",

sqlite3_errmsg(db));

sqlite3_close(db);

return(0);

}

#endif

return(1);

Line 2023 static void *

Line 2353 static void *

hash_halloc(size_t sz, void *arg)

{

return(mandoc_calloc(sz, 1));

return(mandoc_calloc(1, sz));

}

static void *

Line 2045 set_basedir(const char *targetdir)

Line 2375 set_basedir(const char *targetdir)

{

static char startdir[PATH_MAX];

static int fd;

char *cp;

* Remember where we started by keeping a fd open to the origin

Line 2055 set_basedir(const char *targetdir)

Line 2386 set_basedir(const char *targetdir)

if ('\0' == *startdir) {

if (NULL == getcwd(startdir, PATH_MAX)) {

exitcode = (int)MANDOCLEVEL_SYSERR;

if (NULL != targetdir)

say("", "&getcwd");

say(".", NULL);

return(0);

}

if (-1 == (fd = open(startdir, O_RDONLY, 0))) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say(startdir, NULL);

say("", "&open %s", startdir);

return(0);

}

if (NULL == targetdir)

targetdir = startdir;

} else {

if (-1 == fd)

return(0);

Line 2073 set_basedir(const char *targetdir)

Line 2401 set_basedir(const char *targetdir)

close(fd);

basedir[0] = '\0';

exitcode = (int)MANDOCLEVEL_SYSERR;

say(startdir, NULL);

say("", "&chdir %s", startdir);

return(0);

}

if (NULL == targetdir) {

close(fd);

return(1);

}

if (NULL == realpath(targetdir, basedir)) {

basedir[0] = '\0';

exitcode = (int)MANDOCLEVEL_BADARG;

say(targetdir, NULL);

say("", "&%s: realpath", targetdir);

return(0);

} else if (-1 == chdir(basedir)) {

exitcode = (int)MANDOCLEVEL_BADARG;

say("", NULL);

say("", "&chdir");

return(0);

}

cp = strchr(basedir, '\0');

if ('/' != cp[-1]) {

if (cp - basedir >= PATH_MAX - 1) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say("", "Filename too long");

return(0);

}

*cp++ = '/';

*cp = '\0';

}

return(1);

}

Line 2098 static void

Line 2432 static void

say(const char *file, const char *format, ...)

{

va_list ap;

int use_errno;

if ('\0' != *basedir)

fprintf(stderr, "%s", basedir);

Line 2105 say(const char *file, const char *format, ...)

Line 2440 say(const char *file, const char *format, ...)

fputs("//", stderr);

if ('\0' != *file)

fprintf(stderr, "%s", file);

fputs(": ", stderr);

if (NULL == format) {

use_errno = 1;

perror(NULL);

if (NULL != format) {

return;

switch (*format) {

case '&':

format++;

break;

case '\0':

format = NULL;

break;

default:

use_errno = 0;

break;

}

if (NULL != format) {

va_start(ap, format);

if ('\0' != *basedir || '\0' != *file)

vfprintf(stderr, format, ap);

fputs(": ", stderr);

va_end(ap);

va_start(ap, format);

vfprintf(stderr, format, ap);

fputc('\n', stderr);

va_end(ap);

}

if (use_errno) {

if ('\0' != *basedir || '\0' != *file || NULL != format)

fputs(": ", stderr);

perror(NULL);

} else

fputc('\n', stderr);

}

CVSweb