mandoc/mandocdb.c - diff

Return to mandocdb.c CVS log

Up to [cvsweb.bsd.lv] / mandoc

Diff for /mandoc/mandocdb.c between version 1.69 and 1.99

version 1.69, 2013/07/02 11:40:40

version 1.99, 2014/01/05 03:25:51

Line 1

/* $Id$ */

* Permission to use, copy, modify, and distribute this software for any

* purpose with or without fee is hereby granted, provided that the above

Line 76 enum op {

};

enum form {

FORM_SRC, /* format is -man or -mdoc */

FORM_NONE, /* format is unknown */

FORM_CAT, /* format is cat */

FORM_SRC, /* format is -man or -mdoc */

FORM_NONE /* format is unknown */

FORM_CAT /* format is cat */

};

struct str {

char *utf8; /* key in UTF-8 form */

char *rendered; /* key in UTF-8 or ASCII form */

const struct of *of; /* if set, the owning parse */

const struct mpage *mpage; /* if set, the owning parse */

uint64_t mask; /* bitmask in sequence */

char key[]; /* the string itself */

char key[]; /* may contain escape sequences */

};

struct id {

struct inodev {

ino_t ino;

ino_t st_ino;

dev_t dev;

dev_t st_dev;

};

struct of {

struct mpage {

struct id id; /* used for hashing routine */

struct inodev inodev; /* used for hashing routine */

struct of *next; /* next in ofs */

enum form form; /* format from file content */

enum form dform; /* path-cued form */

char *sec; /* section from file content */

enum form sform; /* suffix-cued form */

char *arch; /* architecture from file content */

char *title; /* title from file content */

char *desc; /* description from file content */

struct mlink *mlinks; /* singly linked list */

};

struct mlink {

char file[PATH_MAX]; /* filename rel. to manpath */

char *desc; /* parsed description */

enum form dform; /* format from directory */

char *name; /* name (from filename) (not empty) */

enum form fform; /* format from file name suffix */

char *sec; /* suffix-cued section (or empty) */

char *dsec; /* section from directory */

char *dsec; /* path-cued section (or empty) */

char *arch; /* architecture from directory */

char *arch; /* path-cued arch. (or empty) */

char *name; /* name from file name (not empty) */

char *fsec; /* section from file name suffix */

struct mlink *next; /* singly linked list */

};

struct title {

Line 112 struct title {

Line 120 struct title {

};

enum stmt {

STMT_DELETE = 0, /* delete manpage */

STMT_DELETE_PAGE = 0, /* delete mpage */

STMT_INSERT_DOC, /* insert manpage */

STMT_INSERT_PAGE, /* insert mpage */

STMT_INSERT_KEY, /* insert parsed key */

STMT_INSERT_LINK, /* insert mlink */

STMT_INSERT_KEY, /* insert parsed key */

STMT__MAX

};

typedef int (*mdoc_fp)(struct of *, const struct mdoc_node *);

typedef int (*mdoc_fp)(struct mpage *, const struct mdoc_node *);

struct mdoc_handler {

mdoc_fp fp; /* optional handler */

Line 126 struct mdoc_handler {

Line 135 struct mdoc_handler {

};

static void dbclose(int);

static void dbindex(struct mchars *, int, const struct of *);

static void dbindex(const struct mpage *, struct mchars *);

static int dbopen(int);

static void dbprune(void);

static void fileadd(struct of *);

static int filecheck(const char *);

static void filescan(const char *);

static void *hash_alloc(size_t, void *);

static void hash_free(void *, size_t, void *);

static void *hash_halloc(size_t, void *);

static void inoadd(const struct stat *, struct of *);

static void mlink_add(struct mlink *, const struct stat *);

static int inocheck(const struct stat *);

static int mlink_check(struct mpage *, struct mlink *);

static void ofadd(int, const char *, const char *, const char *,

static void mlink_free(struct mlink *);

const char *, const char *, const struct stat *);

static void mlinks_undupe(struct mpage *);

static void offree(void);

static void mpages_free(void);

static void ofmerge(struct mchars *, struct mparse *,

static void mpages_merge(struct mchars *, struct mparse *, int);

struct ohash_info*, int);

static void parse_cat(struct mpage *);

static void parse_catpage(struct of *);

static void parse_man(struct mpage *, const struct man_node *);

static void parse_man(struct of *, const struct man_node *);

static void parse_mdoc(struct mpage *, const struct mdoc_node *);

static void parse_mdoc(struct of *, const struct mdoc_node *);

static int parse_mdoc_body(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_body(struct of *, const struct mdoc_node *);

static int parse_mdoc_head(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_head(struct of *, const struct mdoc_node *);

static int parse_mdoc_Fd(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_Fd(struct of *, const struct mdoc_node *);

static int parse_mdoc_Fn(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_Fn(struct of *, const struct mdoc_node *);

static int parse_mdoc_Nd(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_In(struct of *, const struct mdoc_node *);

static int parse_mdoc_Nm(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_Nd(struct of *, const struct mdoc_node *);

static int parse_mdoc_Sh(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_Nm(struct of *, const struct mdoc_node *);

static int parse_mdoc_Xr(struct mpage *, const struct mdoc_node *);

static int parse_mdoc_Sh(struct of *, const struct mdoc_node *);

static void putkey(const struct mpage *,

static int parse_mdoc_St(struct of *, const struct mdoc_node *);

static int parse_mdoc_Xr(struct of *, const struct mdoc_node *);

static int set_basedir(const char *);

static void putkey(const struct of *,

const char *, uint64_t);

static void putkeys(const struct of *,

static void putkeys(const struct mpage *,

const char *, size_t, uint64_t);

static void putmdockey(const struct of *,

static void putmdockey(const struct mpage *,

const struct mdoc_node *, uint64_t);

static void render_key(struct mchars *, struct str *);

static void say(const char *, const char *, ...);

static int set_basedir(const char *);

static int treescan(void);

static size_t utf8(unsigned int, char [7]);

static void utf8key(struct mchars *, struct str *);

static char *progname;

static int use_all; /* use all found files */

static int nodb; /* no database changes */

static int verb; /* print what we're doing */

static int warnings; /* warn about crap */

static int write_utf8; /* write UTF-8 output; else ASCII */

static int exitcode; /* to be returned by main */

static enum op op; /* operational mode */

static char basedir[PATH_MAX]; /* current base directory */

static struct ohash inos; /* table of inodes/devices */

static struct ohash mpages; /* table of distinct manual pages */

static struct ohash filenames; /* table of filenames */

static struct ohash mlinks; /* table of directory entries */

static struct ohash strings; /* table of all strings */

static struct of *ofs = NULL; /* vector of files to parse */

static sqlite3 *db = NULL; /* current database */

static sqlite3_stmt *stmts[STMT__MAX]; /* current statements */

Line 212 static const struct mdoc_handler mdocs[MDOC_MAX] = {

Line 216 static const struct mdoc_handler mdocs[MDOC_MAX] = {

{ parse_mdoc_Fn, 0 }, /* Fn */

{ NULL, TYPE_Ft }, /* Ft */

{ NULL, TYPE_Ic }, /* Ic */

{ parse_mdoc_In, TYPE_In }, /* In */

{ NULL, TYPE_In }, /* In */

{ NULL, TYPE_Li }, /* Li */

{ parse_mdoc_Nd, TYPE_Nd }, /* Nd */

{ parse_mdoc_Nm, TYPE_Nm }, /* Nm */

Line 220 static const struct mdoc_handler mdocs[MDOC_MAX] = {

Line 224 static const struct mdoc_handler mdocs[MDOC_MAX] = {

{ NULL, 0 }, /* Ot */

{ NULL, TYPE_Pa }, /* Pa */

{ NULL, 0 }, /* Rv */

{ parse_mdoc_St, 0 }, /* St */

{ NULL, TYPE_St }, /* St */

{ NULL, TYPE_Va }, /* Va */

{ parse_mdoc_body, TYPE_Va }, /* Vt */

{ parse_mdoc_Xr, 0 }, /* Xr */

Line 316 main(int argc, char *argv[])

Line 320 main(int argc, char *argv[])

struct mchars *mc;

struct manpaths dirs;

struct mparse *mp;

struct ohash_info ino_info, filename_info, str_info;

struct ohash_info mpages_info, mlinks_info;

memset(stmts, 0, STMT__MAX * sizeof(sqlite3_stmt *));

memset(&dirs, 0, sizeof(struct manpaths));

ino_info.halloc = filename_info.halloc =

mpages_info.alloc = mlinks_info.alloc = hash_alloc;

str_info.halloc = hash_halloc;

mpages_info.halloc = mlinks_info.halloc = hash_halloc;

ino_info.hfree = filename_info.hfree =

mpages_info.hfree = mlinks_info.hfree = hash_free;

str_info.hfree = hash_free;

ino_info.alloc = filename_info.alloc =

str_info.alloc = hash_alloc;

ino_info.key_offset = offsetof(struct of, id);

mpages_info.key_offset = offsetof(struct mpage, inodev);

filename_info.key_offset = offsetof(struct of, file);

mlinks_info.key_offset = offsetof(struct mlink, file);

str_info.key_offset = offsetof(struct str, key);

progname = strrchr(argv[0], '/');

if (progname == NULL)

Line 352 main(int argc, char *argv[])

path_arg = NULL;

op = OP_DEFAULT;

while (-1 != (ch = getopt(argc, argv, "aC:d:ntu:vW")))

while (-1 != (ch = getopt(argc, argv, "aC:d:nT:tu:vW")))

switch (ch) {

case ('a'):

use_all = 1;

Line 370 main(int argc, char *argv[])

case ('n'):

nodb = 1;

break;

case ('T'):

if (strcmp(optarg, "utf8")) {

fprintf(stderr, "-T%s: Unsupported "

"output format\n", optarg);

goto usage;

}

write_utf8 = 1;

break;

case ('t'):

CHECKOP(op, ch);

dup2(STDOUT_FILENO, STDERR_FILENO);

Line 404 main(int argc, char *argv[])

Line 412 main(int argc, char *argv[])

MANDOCLEVEL_FATAL, NULL, NULL, NULL);

mc = mchars_alloc();

ohash_init(&inos, 6, &ino_info);

ohash_init(&mpages, 6, &mpages_info);

ohash_init(&filenames, 6, &filename_info);

ohash_init(&mlinks, 6, &mlinks_info);

if (OP_UPDATE == op || OP_DELETE == op || OP_TEST == op) {

Line 427 main(int argc, char *argv[])

Line 435 main(int argc, char *argv[])

if (OP_TEST != op)

dbprune();

if (OP_DELETE != op)

ofmerge(mc, mp, &str_info, 0);

mpages_merge(mc, mp, 0);

dbclose(1);

} else {

Line 458 main(int argc, char *argv[])

Line 466 main(int argc, char *argv[])

continue;

if (j) {

ohash_init(&inos, 6, &ino_info);

ohash_init(&mpages, 6, &mpages_info);

ohash_init(&filenames, 6, &filename_info);

ohash_init(&mlinks, 6, &mlinks_info);

}

if (0 == set_basedir(dirs.paths[j]))

Line 471 main(int argc, char *argv[])

Line 479 main(int argc, char *argv[])

if (0 == dbopen(0))

goto out;

mpages_merge(mc, mp, warnings && !use_all);

* Since we're opening up a new database, we can

* turn off synchronous mode for much better

* performance.

#ifndef __APPLE__

SQL_EXEC("PRAGMA synchronous = OFF");

#endif

ofmerge(mc, mp, &str_info, warnings && !use_all);

dbclose(0);

if (j + 1 < dirs.sz) {

ohash_delete(&inos);

mpages_free();

ohash_delete(&filenames);

ohash_delete(&mpages);

offree();

ohash_delete(&mlinks);

}

Line 495 out:

Line 494 out:

manpath_free(&dirs);

mchars_free(mc);

mparse_free(mp);

ohash_delete(&inos);

mpages_free();

ohash_delete(&filenames);

ohash_delete(&mpages);

offree();

ohash_delete(&mlinks);

return(exitcode);

usage:

fprintf(stderr, "usage: %s [-anvW] [-C file]\n"

fprintf(stderr, "usage: %s [-anvW] [-C file] [-Tutf8]\n"

" %s [-anvW] dir ...\n"

" %s [-anvW] [-Tutf8] dir ...\n"

" %s [-nvW] -d dir [file ...]\n"

" %s [-nvW] [-Tutf8] -d dir [file ...]\n"

" %s [-nvW] -u dir [file ...]\n"

" %s -t file ...\n",

progname, progname, progname,

Line 530 treescan(void)

Line 529 treescan(void)

{

FTS *f;

FTSENT *ff;

struct mlink *mlink;

int dform;

char *sec;

char *dsec, *arch, *fsec, *cp;

const char *dsec, *arch, *cp, *path;

const char *path;

const char *argv[2];

argv[0] = ".";

Line 555 treescan(void)

while (NULL != (ff = fts_read(f))) {

path = ff->fts_path + 2;

* If we're a regular file, add an "of" by using the

* If we're a regular file, add an mlink by using the

* stored directory data and handling the filename.

* Disallow duplicate (hard-linked) files.

if (FTS_F == ff->fts_info) {

if (0 == strcmp(path, MANDOC_DB))

Line 566 treescan(void)

Line 565 treescan(void)

if (warnings)

say(path, "Extraneous file");

continue;

} else if (inocheck(ff->fts_statp)) {

} else if (NULL == (fsec =

if (warnings)

say(path, "Duplicate file");

continue;

} else if (NULL == (sec =

strrchr(ff->fts_name, '.'))) {

if ( ! use_all) {

if (warnings)

Line 578 treescan(void)

Line 573 treescan(void)

"No filename suffix");

continue;

}

} else if (0 == strcmp(++sec, "html")) {

} else if (0 == strcmp(++fsec, "html")) {

if (warnings)

say(path, "Skip html");

continue;

} else if (0 == strcmp(sec, "gz")) {

} else if (0 == strcmp(fsec, "gz")) {

if (warnings)

say(path, "Skip gz");

continue;

} else if (0 == strcmp(sec, "ps")) {

} else if (0 == strcmp(fsec, "ps")) {

if (warnings)

say(path, "Skip ps");

continue;

} else if (0 == strcmp(sec, "pdf")) {

} else if (0 == strcmp(fsec, "pdf")) {

if (warnings)

say(path, "Skip pdf");

continue;

} else if ( ! use_all &&

((FORM_SRC == dform && strcmp(sec, dsec)) ||

((FORM_SRC == dform && strcmp(fsec, dsec)) ||

(FORM_CAT == dform && strcmp(sec, "0")))) {

(FORM_CAT == dform && strcmp(fsec, "0")))) {

if (warnings)

say(path, "Wrong filename suffix");

continue;

} else

sec[-1] = '\0';

fsec[-1] = '\0';

ofadd(dform, path, ff->fts_name, dsec, sec,

arch, ff->fts_statp);

mlink = mandoc_calloc(1, sizeof(struct mlink));

strlcpy(mlink->file, path, sizeof(mlink->file));

mlink->dform = dform;

mlink->dsec = dsec;

mlink->arch = arch;

mlink->name = ff->fts_name;

mlink->fsec = fsec;

mlink_add(mlink, ff->fts_statp);

continue;

} else if (FTS_D != ff->fts_info &&

FTS_DP != ff->fts_info) {

if (warnings)

say(path, "Not a regular file");

Line 622 treescan(void)

Line 624 treescan(void)

* Try to infer this from the name.

* If we're not in use_all, enforce it.

dsec = NULL;

dform = FORM_NONE;

cp = ff->fts_name;

if (FTS_DP == ff->fts_info)

break;

Line 634 treescan(void)

} else if (0 == strncmp(cp, "cat", 3)) {

dform = FORM_CAT;

dsec = cp + 3;

} else {

dform = FORM_NONE;

dsec = NULL;

}

if (NULL != dsec || use_all)

Line 648 treescan(void)

Line 651 treescan(void)

* Possibly our architecture.

* If we're descending, keep tabs on it.

arch = NULL;

if (FTS_DP != ff->fts_info && NULL != dsec)

arch = ff->fts_name;

else

arch = NULL;

break;

default:

if (FTS_DP == ff->fts_info || use_all)

Line 667 treescan(void)

Line 671 treescan(void)

}

* Add a file to the file vector.

* Add a file to the mlinks table.

* Do not verify that it's a "valid" looking manpage (we'll do that

* later).

Line 678 treescan(void)

Line 682 treescan(void)

* or

* [./]cat<section>[/<arch>]/<name>.0

* Stuff this information directly into the "of" vector.

* See treescan() for the fts(3) version of this.

static void

filescan(const char *file)

{

char buf[PATH_MAX];

const char *sec, *arch, *name, *dsec;

char *p, *start;

int dform;

struct stat st;

struct mlink *mlink;

char *p, *start;

assert(use_all);

Line 699 filescan(const char *file)

Line 701 filescan(const char *file)

exitcode = (int)MANDOCLEVEL_BADARG;

say(file, NULL);

return;

} else if (strstr(buf, basedir) != buf) {

} else if (OP_TEST != op && strstr(buf, basedir) != buf) {

exitcode = (int)MANDOCLEVEL_BADARG;

say("", "%s: outside base directory", buf);

return;

Line 711 filescan(const char *file)

Line 713 filescan(const char *file)

exitcode = (int)MANDOCLEVEL_BADARG;

say(file, "Not a regular file");

return;

} else if (inocheck(&st)) {

if (warnings)

say(file, "Duplicate file");

return;

}

start = buf + strlen(basedir);

sec = arch = name = dsec = NULL;

mlink = mandoc_calloc(1, sizeof(struct mlink));

dform = FORM_NONE;

strlcpy(mlink->file, start, sizeof(mlink->file));

* First try to guess our directory structure.

Line 729 filescan(const char *file)

Line 727 filescan(const char *file)

if (NULL != (p = strchr(start, '/'))) {

*p++ = '\0';

if (0 == strncmp(start, "man", 3)) {

dform = FORM_SRC;

mlink->dform = FORM_SRC;

dsec = start + 3;

mlink->dsec = start + 3;

} else if (0 == strncmp(start, "cat", 3)) {

dform = FORM_CAT;

mlink->dform = FORM_CAT;

dsec = start + 3;

mlink->dsec = start + 3;

}

start = p;

if (NULL != dsec && NULL != (p = strchr(start, '/'))) {

if (NULL != mlink->dsec && NULL != (p = strchr(start, '/'))) {

*p++ = '\0';

arch = start;

mlink->arch = start;

start = p;

}

Line 754 filescan(const char *file)

Line 752 filescan(const char *file)

if ('.' == *p) {

*p++ = '\0';

sec = p;

mlink->fsec = p;

}

* Now try to parse the name.

* Use the filename portion of the path.

name = start;

mlink->name = start;

if (NULL != (p = strrchr(start, '/'))) {

name = p + 1;

mlink->name = p + 1;

*p = '\0';

}

mlink_add(mlink, &st);

ofadd(dform, file, name, dsec, sec, arch, &st);

}

static void

* See fileadd().

mlink_add(struct mlink *mlink, const struct stat *st)

static int

filecheck(const char *name)

{

struct inodev inodev;

struct mpage *mpage;

unsigned int slot;

return(NULL != ohash_find(&filenames,

assert(NULL != mlink->file);

ohash_qlookup(&filenames, name)));

mlink->dsec = mandoc_strdup(mlink->dsec ? mlink->dsec : "");

mlink->arch = mandoc_strdup(mlink->arch ? mlink->arch : "");

mlink->name = mandoc_strdup(mlink->name ? mlink->name : "");

mlink->fsec = mandoc_strdup(mlink->fsec ? mlink->fsec : "");

if ('0' == *mlink->fsec) {

free(mlink->fsec);

mlink->fsec = mandoc_strdup(mlink->dsec);

mlink->fform = FORM_CAT;

} else if ('1' <= *mlink->fsec && '9' >= *mlink->fsec)

mlink->fform = FORM_SRC;

else

mlink->fform = FORM_NONE;

slot = ohash_qlookup(&mlinks, mlink->file);

assert(NULL == ohash_find(&mlinks, slot));

ohash_insert(&mlinks, slot, mlink);

inodev.st_ino = st->st_ino;

inodev.st_dev = st->st_dev;

slot = ohash_lookup_memory(&mpages, (char *)&inodev,

sizeof(struct inodev), inodev.st_ino);

mpage = ohash_find(&mpages, slot);

if (NULL == mpage) {

mpage = mandoc_calloc(1, sizeof(struct mpage));

mpage->inodev.st_ino = inodev.st_ino;

mpage->inodev.st_dev = inodev.st_dev;

ohash_insert(&mpages, slot, mpage);

} else

mlink->next = mpage->mlinks;

mpage->mlinks = mlink;

}

* Use the standard hashing mechanism (K&R) to see if the given filename

* already exists.

static void

fileadd(struct of *of)

mlink_free(struct mlink *mlink)

{

unsigned int slot;

slot = ohash_qlookup(&filenames, of->file);

free(mlink->dsec);

assert(NULL == ohash_find(&filenames, slot));

free(mlink->arch);

ohash_insert(&filenames, slot, of);

free(mlink->name);

free(mlink->fsec);

free(mlink);

}

static void

* See inoadd().

mpages_free(void)

static int

inocheck(const struct stat *st)

{

struct id id;

struct mpage *mpage;

uint32_t hash;

struct mlink *mlink;

unsigned int slot;

memset(&id, 0, sizeof(id));

mpage = ohash_first(&mpages, &slot);

id.ino = hash = st->st_ino;

while (NULL != mpage) {

id.dev = st->st_dev;

while (NULL != (mlink = mpage->mlinks)) {

mpage->mlinks = mlink->next;

return(NULL != ohash_find(&inos, ohash_lookup_memory(

mlink_free(mlink);

&inos, (char *)&id, sizeof(id), hash)));

}

free(mpage->sec);

free(mpage->arch);

free(mpage->title);

free(mpage->desc);

free(mpage);

mpage = ohash_next(&mpages, &slot);

}

* The hashing function used here is quite simple: simply take the inode

* For each mlink to the mpage, check whether the path looks like

* and use uint32_t of its bits.

* it is formatted, and if it does, check whether a source manual

* Then when we do the lookup, use both the inode and device identifier.

* exists by the same name, ignoring the suffix.

* If both conditions hold, drop the mlink.

static void

inoadd(const struct stat *st, struct of *of)

mlinks_undupe(struct mpage *mpage)

{

uint32_t hash;

char buf[PATH_MAX];

unsigned int slot;

struct mlink **prev;

struct mlink *mlink;

char *bufp;

of->id.ino = hash = st->st_ino;

mpage->form = FORM_CAT;

of->id.dev = st->st_dev;

prev = &mpage->mlinks;

slot = ohash_lookup_memory

while (NULL != (mlink = *prev)) {

(&inos, (char *)&of->id, sizeof(of->id), hash);

if (FORM_CAT != mlink->dform) {

mpage->form = FORM_NONE;

assert(NULL == ohash_find(&inos, slot));

goto nextlink;

ohash_insert(&inos, slot, of);

}

if (strlcpy(buf, mlink->file, PATH_MAX) >= PATH_MAX) {

if (warnings)

say(mlink->file, "Filename too long");

goto nextlink;

}

bufp = strstr(buf, "cat");

assert(NULL != bufp);

memcpy(bufp, "man", 3);

if (NULL != (bufp = strrchr(buf, '.')))

*++bufp = '\0';

strlcat(buf, mlink->dsec, PATH_MAX);

if (NULL == ohash_find(&mlinks,

ohash_qlookup(&mlinks, buf)))

goto nextlink;

if (warnings)

say(mlink->file, "Man source exists: %s", buf);

if (use_all)

goto nextlink;

*prev = mlink->next;

mlink_free(mlink);

continue;

nextlink:

prev = &(*prev)->next;

}

static void

static int

ofadd(int dform, const char *file, const char *name, const char *dsec,

mlink_check(struct mpage *mpage, struct mlink *mlink)

const char *sec, const char *arch, const struct stat *st)

{

struct of *of;

int match;

int sform;

assert(NULL != file);

match = 1;

if (NULL == name)

name = "";

* Check whether the manual section given in a file

if (NULL == sec)

* agrees with the directory where the file is located.

sec = "";

* Some manuals have suffixes like (3p) on their

if (NULL == dsec)

* section number either inside the file or in the

dsec = "";

* directory name, some are linked into more than one

if (NULL == arch)

* section, like encrypt(1) = makekey(8).

arch = "";

sform = FORM_NONE;

if (FORM_SRC == mpage->form &&

if (NULL != sec && *sec <= '9' && *sec >= '1')

strcasecmp(mpage->sec, mlink->dsec)) {

sform = FORM_SRC;

match = 0;

else if (NULL != sec && *sec == '0') {

say(mlink->file, "Section \"%s\" manual in %s directory",

sec = dsec;

mpage->sec, mlink->dsec);

sform = FORM_CAT;

}

of = mandoc_calloc(1, sizeof(struct of));

strlcpy(of->file, file, PATH_MAX);

of->name = mandoc_strdup(name);

of->sec = mandoc_strdup(sec);

of->dsec = mandoc_strdup(dsec);

of->arch = mandoc_strdup(arch);

of->sform = sform;

of->dform = dform;

of->next = ofs;

ofs = of;

* Add to unique identifier hash.

* Manual page directories exist for each kernel

* Then if it's a source manual and we're going to use source in

* architecture as returned by machine(1).

* favour of catpages, add it to that hash.

* However, many manuals only depend on the

* application architecture as returned by arch(1).

* For example, some (2/ARM) manuals are shared

* across the "armish" and "zaurus" kernel

* architectures.

* A few manuals are even shared across completely

* different architectures, for example fdformat(1)

* on amd64, i386, sparc, and sparc64.

inoadd(st, of);

fileadd(of);

}

static void

if (strcasecmp(mpage->arch, mlink->arch)) {

offree(void)

match = 0;

{

say(mlink->file, "Architecture \"%s\" manual in "

struct of *of;

"\"%s\" directory", mpage->arch, mlink->arch);

while (NULL != (of = ofs)) {

ofs = of->next;

free(of->name);

free(of->sec);

free(of->dsec);

free(of->arch);

free(of);

}

if (strcasecmp(mpage->title, mlink->name))

match = 0;

return(match);

}

* Run through the files in the global vector "ofs" and add them to the

* Run through the files in the global vector "mpages"

* database specified in "basedir".

* and add them to the database specified in "basedir".

* This handles the parsing scheme itself, using the cues of directory

* and filename to determine whether the file is parsable or not.

static void

ofmerge(struct mchars *mc, struct mparse *mp,

mpages_merge(struct mchars *mc, struct mparse *mp, int check_reachable)

struct ohash_info *infop, int check_reachable)

{

struct ohash title_table;

struct ohash_info title_info;

struct ohash_info title_info, str_info;

char buf[PATH_MAX];

struct mpage *mpage;

struct of *of;

struct mlink *mlink;

struct mdoc *mdoc;

struct man *man;

struct title *title_entry;

char *bufp, *title_str;

char *title_str;

const char *msec, *march, *mtitle, *cp;

const char *cp;

size_t sz;

int form;

int match;

unsigned int slot;

unsigned int pslot, tslot;

enum mandoclevel lvl;

str_info.alloc = hash_alloc;

str_info.halloc = hash_halloc;

str_info.hfree = hash_free;

str_info.key_offset = offsetof(struct str, key);

if (check_reachable) {

title_info.alloc = hash_alloc;

title_info.halloc = hash_halloc;

Line 927 ofmerge(struct mchars *mc, struct mparse *mp,

Line 973 ofmerge(struct mchars *mc, struct mparse *mp,

ohash_init(&title_table, 6, &title_info);

}

for (of = ofs; NULL != of; of = of->next) {

mpage = ohash_first(&mpages, &pslot);

while (NULL != mpage) {

* If we're a catpage (as defined by our path), then see

mlinks_undupe(mpage);

* if a manpage exists by the same name (ignoring the

if (NULL == mpage->mlinks) {

* suffix).

mpage = ohash_next(&mpages, &pslot);

* If it does, then we want to use it instead of our

continue;

* own.

if ( ! use_all && FORM_CAT == of->dform) {

sz = strlcpy(buf, of->file, PATH_MAX);

if (sz >= PATH_MAX) {

if (warnings)

say(of->file, "Filename too long");

continue;

}

bufp = strstr(buf, "cat");

assert(NULL != bufp);

memcpy(bufp, "man", 3);

if (NULL != (bufp = strrchr(buf, '.')))

*++bufp = '\0';

strlcat(buf, of->dsec, PATH_MAX);

if (filecheck(buf)) {

if (warnings)

say(of->file, "Man "

"source exists: %s", buf);

continue;

}

ohash_init(&strings, 6, infop);

ohash_init(&strings, 6, &str_info);

mparse_reset(mp);

mdoc = NULL;

man = NULL;

form = 0;

msec = of->dsec;

march = of->arch;

mtitle = of->name;

match = 1;

* Try interpreting the file as mdoc(7) or man(7)

* source code, unless it is already known to be

* formatted. Fall back to formatted mode.

if (FORM_SRC == of->dform || FORM_SRC == of->sform) {

if (FORM_CAT != mpage->mlinks->dform ||

lvl = mparse_readfd(mp, -1, of->file);

FORM_CAT != mpage->mlinks->fform) {

lvl = mparse_readfd(mp, -1, mpage->mlinks->file);

if (lvl < MANDOCLEVEL_FATAL)

mparse_result(mp, &mdoc, &man);

}

if (NULL != mdoc) {

form = 1;

mpage->form = FORM_SRC;

msec = mdoc_meta(mdoc)->msec;

mpage->sec =

march = mdoc_meta(mdoc)->arch;

mandoc_strdup(mdoc_meta(mdoc)->msec);

mtitle = mdoc_meta(mdoc)->title;

mpage->arch = mdoc_meta(mdoc)->arch;

mpage->arch = mandoc_strdup(

NULL == mpage->arch ? "" : mpage->arch);

mpage->title =

mandoc_strdup(mdoc_meta(mdoc)->title);

} else if (NULL != man) {

form = 1;

mpage->form = FORM_SRC;

msec = man_meta(man)->msec;

mpage->sec =

march = "";

mandoc_strdup(man_meta(man)->msec);

mtitle = man_meta(man)->title;

mpage->arch =

}

mandoc_strdup(mpage->mlinks->arch);

mpage->title =

mandoc_strdup(man_meta(man)->title);

} else {

mpage->form = FORM_CAT;

mpage->sec =

mandoc_strdup(mpage->mlinks->dsec);

mpage->arch =

mandoc_strdup(mpage->mlinks->arch);

mpage->title =

mandoc_strdup(mpage->mlinks->name);

}

putkey(mpage, mpage->sec, TYPE_sec);

putkey(mpage, '\0' == *mpage->arch ?

"any" : mpage->arch, TYPE_arch);

if (NULL == msec)

for (mlink = mpage->mlinks; mlink; mlink = mlink->next) {

msec = "";

if ('\0' != *mlink->dsec)

if (NULL == march)

putkey(mpage, mlink->dsec, TYPE_sec);

march = "";

if ('\0' != *mlink->fsec)

if (NULL == mtitle)

putkey(mpage, mlink->fsec, TYPE_sec);

mtitle = "";

putkey(mpage, '\0' == *mlink->arch ?

"any" : mlink->arch, TYPE_arch);

putkey(mpage, mlink->name, TYPE_Nm);

* Check whether the manual section given in a file

* agrees with the directory where the file is located.

* Some manuals have suffixes like (3p) on their

* section number either inside the file or in the

* directory name, some are linked into more than one

* section, like encrypt(1) = makekey(8). Do not skip

* manuals for such reasons.

if (warnings && !use_all && form &&

strcasecmp(msec, of->dsec)) {

match = 0;

say(of->file, "Section \"%s\" "

"manual in %s directory",

msec, of->dsec);

}

if (warnings && !use_all) {

* Manual page directories exist for each kernel

* architecture as returned by machine(1).

* However, many manuals only depend on the

* application architecture as returned by arch(1).

* For example, some (2/ARM) manuals are shared

* across the "armish" and "zaurus" kernel

* architectures.

* A few manuals are even shared across completely

* different architectures, for example fdformat(1)

* on amd64, i386, sparc, and sparc64.

* Thus, warn about architecture mismatches,

* but don't skip manuals for this reason.

if (warnings && !use_all && strcasecmp(march, of->arch)) {

match = 0;

say(of->file, "Architecture \"%s\" "

for (mlink = mpage->mlinks; mlink;

"manual in \"%s\" directory",

mlink = mlink->next)

march, of->arch);

if (mlink_check(mpage, mlink))

}

match = 1;

if (warnings && !use_all && strcasecmp(mtitle, of->name))

} else

match = 0;

match = 1;

putkey(of, of->name, TYPE_Nm);

if (NULL != mdoc) {

if (NULL != (cp = mdoc_meta(mdoc)->name))

putkey(of, cp, TYPE_Nm);

putkey(mpage, cp, TYPE_Nm);

assert(NULL == of->desc);

assert(NULL == mpage->desc);

parse_mdoc(of, mdoc_node(mdoc));

parse_mdoc(mpage, mdoc_node(mdoc));

putkey(of, NULL != of->desc ?

putkey(mpage, NULL != mpage->desc ?

of->desc : of->name, TYPE_Nd);

mpage->desc : mpage->mlinks->name, TYPE_Nd);

} else if (NULL != man)

parse_man(of, man_node(man));

parse_man(mpage, man_node(man));

else

parse_catpage(of);

parse_cat(mpage);

* Build a title string for the file. If it matches

Line 1057 ofmerge(struct mchars *mc, struct mparse *mp,

Line 1066 ofmerge(struct mchars *mc, struct mparse *mp,

if (check_reachable) {

if (-1 == asprintf(&title_str, "%s(%s%s%s)", mtitle,

if (-1 == asprintf(&title_str, "%s(%s%s%s)",

msec, '\0' == *march ? "" : "/", march)) {

mpage->title, mpage->sec,

'\0' == *mpage->arch ? "" : "/",

mpage->arch)) {

perror(NULL);

exit((int)MANDOCLEVEL_SYSERR);

}

slot = ohash_qlookup(&title_table, title_str);

tslot = ohash_qlookup(&title_table, title_str);

title_entry = ohash_find(&title_table, slot);

title_entry = ohash_find(&title_table, tslot);

if (NULL == title_entry) {

title_entry = mandoc_malloc(

sizeof(struct title));

title_entry->title = title_str;

title_entry->file = mandoc_strdup(

match ? "" : of->file);

match ? "" : mpage->mlinks->file);

ohash_insert(&title_table, slot,

ohash_insert(&title_table, tslot,

title_entry);

} else {

if (match)

Line 1079 ofmerge(struct mchars *mc, struct mparse *mp,

Line 1090 ofmerge(struct mchars *mc, struct mparse *mp,

}

dbindex(mc, form, of);

dbindex(mpage, mc);

ohash_delete(&strings);

mpage = ohash_next(&mpages, &pslot);

}

if (check_reachable) {

title_entry = ohash_first(&title_table, &slot);

title_entry = ohash_first(&title_table, &tslot);

while (NULL != title_entry) {

if ('\0' != *title_entry->file)

say(title_entry->file,

Line 1093 ofmerge(struct mchars *mc, struct mparse *mp,

Line 1105 ofmerge(struct mchars *mc, struct mparse *mp,

free(title_entry->title);

free(title_entry->file);

free(title_entry);

title_entry = ohash_next(&title_table, &slot);

title_entry = ohash_next(&title_table, &tslot);

}

ohash_delete(&title_table);

}

static void

parse_catpage(struct of *of)

parse_cat(struct mpage *mpage)

{

FILE *stream;

char *line, *p, *title;

size_t len, plen, titlesz;

if (NULL == (stream = fopen(of->file, "r"))) {

if (NULL == (stream = fopen(mpage->mlinks->file, "r"))) {

if (warnings)

say(of->file, NULL);

say(mpage->mlinks->file, NULL);

return;

}

Line 1161 parse_catpage(struct of *of)

Line 1173 parse_catpage(struct of *of)

if (NULL == title || '\0' == *title) {

if (warnings)

say(of->file, "Cannot find NAME section");

say(mpage->mlinks->file,

assert(NULL == of->desc);

"Cannot find NAME section");

of->desc = mandoc_strdup(of->name);

assert(NULL == mpage->desc);

putkey(of, of->name, TYPE_Nd);

mpage->desc = mandoc_strdup(mpage->mlinks->name);

putkey(mpage, mpage->mlinks->name, TYPE_Nd);

fclose(stream);

free(title);

return;

Line 1184 parse_catpage(struct of *of)

Line 1197 parse_catpage(struct of *of)

/* Skip to next word. */ ;

} else {

if (warnings)

say(of->file, "No dash in title line");

say(mpage->mlinks->file,

"No dash in title line");

p = title;

}

Line 1202 parse_catpage(struct of *of)

Line 1216 parse_catpage(struct of *of)

plen -= 2;

}

assert(NULL == of->desc);

assert(NULL == mpage->desc);

of->desc = mandoc_strdup(p);

mpage->desc = mandoc_strdup(p);

putkey(of, of->desc, TYPE_Nd);

putkey(mpage, mpage->desc, TYPE_Nd);

fclose(stream);

free(title);

}

Line 1213 parse_catpage(struct of *of)

Line 1227 parse_catpage(struct of *of)

* Put a type/word pair into the word database for this particular file.

static void

putkey(const struct of *of, const char *value, uint64_t type)

putkey(const struct mpage *mpage, const char *value, uint64_t type)

{

assert(NULL != value);

putkeys(of, value, strlen(value), type);

putkeys(mpage, value, strlen(value), type);

}

* Grok all nodes at or below a certain mdoc node into putkey().

static void

putmdockey(const struct of *of, const struct mdoc_node *n, uint64_t m)

putmdockey(const struct mpage *mpage,

const struct mdoc_node *n, uint64_t m)

{

for ( ; NULL != n; n = n->next) {

if (NULL != n->child)

putmdockey(of, n->child, m);

putmdockey(mpage, n->child, m);

if (MDOC_TEXT == n->type)

putkey(of, n->string, m);

putkey(mpage, n->string, m);

}

static void

parse_man(struct of *of, const struct man_node *n)

parse_man(struct mpage *mpage, const struct man_node *n)

{

const struct man_node *head, *body;

char *start, *sv, *title;

Line 1331 parse_man(struct of *of, const struct man_node *n)

Line 1346 parse_man(struct of *of, const struct man_node *n)

byte = start[sz];

start[sz] = '\0';

putkey(of, start, TYPE_Nm);

putkey(mpage, start, TYPE_Nm);

if (' ' == byte) {

start += sz + 1;

Line 1345 parse_man(struct of *of, const struct man_node *n)

Line 1360 parse_man(struct of *of, const struct man_node *n)

}

if (sv == start) {

putkey(of, start, TYPE_Nm);

putkey(mpage, start, TYPE_Nm);

free(title);

return;

}

Line 1367 parse_man(struct of *of, const struct man_node *n)

Line 1382 parse_man(struct of *of, const struct man_node *n)

while (' ' == *start)

start++;

assert(NULL == of->desc);

assert(NULL == mpage->desc);

of->desc = mandoc_strdup(start);

mpage->desc = mandoc_strdup(start);

putkey(of, of->desc, TYPE_Nd);

putkey(mpage, mpage->desc, TYPE_Nd);

free(title);

return;

}

for (n = n->child; n; n = n->next)

for (n = n->child; n; n = n->next) {

parse_man(of, n);

if (NULL != mpage->desc)

break;

parse_man(mpage, n);

}

static void

parse_mdoc(struct of *of, const struct mdoc_node *n)

parse_mdoc(struct mpage *mpage, const struct mdoc_node *n)

{

assert(NULL != n);

Line 1396 parse_mdoc(struct of *of, const struct mdoc_node *n)

Line 1414 parse_mdoc(struct of *of, const struct mdoc_node *n)

/* FALLTHROUGH */

case (MDOC_TAIL):

if (NULL != mdocs[n->tok].fp)

if (0 == (*mdocs[n->tok].fp)(of, n))

if (0 == (*mdocs[n->tok].fp)(mpage, n))

break;

if (mdocs[n->tok].mask)

putmdockey(of, n->child, mdocs[n->tok].mask);

putmdockey(mpage, n->child,

mdocs[n->tok].mask);

break;

default:

assert(MDOC_ROOT != n->type);

continue;

}

if (NULL != n->child)

parse_mdoc(of, n);

parse_mdoc(mpage, n);

}

static int

parse_mdoc_Fd(struct of *of, const struct mdoc_node *n)

parse_mdoc_Fd(struct mpage *mpage, const struct mdoc_node *n)

{

const char *start, *end;

size_t sz;

Line 1449 parse_mdoc_Fd(struct of *of, const struct mdoc_node *n

Line 1468 parse_mdoc_Fd(struct of *of, const struct mdoc_node *n

end--;

if (end > start)

putkeys(of, start, end - start + 1, TYPE_In);

putkeys(mpage, start, end - start + 1, TYPE_In);

return(1);

return(0);

}

static int

parse_mdoc_In(struct of *of, const struct mdoc_node *n)

parse_mdoc_Fn(struct mpage *mpage, const struct mdoc_node *n)

{

if (NULL != n->child && MDOC_TEXT == n->child->type)

return(0);

putkey(of, n->child->string, TYPE_In);

return(1);

}

static int

parse_mdoc_Fn(struct of *of, const struct mdoc_node *n)

{

const char *cp;

if (NULL == (n = n->child) || MDOC_TEXT != n->type)

Line 1485 parse_mdoc_Fn(struct of *of, const struct mdoc_node *n

Line 1493 parse_mdoc_Fn(struct of *of, const struct mdoc_node *n

while ('*' == *cp)

cp++;

putkey(of, cp, TYPE_Fn);

putkey(mpage, cp, TYPE_Fn);

if (n->string < cp)

putkeys(of, n->string, cp - n->string, TYPE_Ft);

putkeys(mpage, n->string, cp - n->string, TYPE_Ft);

for (n = n->next; NULL != n; n = n->next)

if (MDOC_TEXT == n->type)

putkey(of, n->string, TYPE_Fa);

putkey(mpage, n->string, TYPE_Fa);

return(0);

}

static int

parse_mdoc_St(struct of *of, const struct mdoc_node *n)

parse_mdoc_Xr(struct mpage *mpage, const struct mdoc_node *n)

{

if (NULL == n->child || MDOC_TEXT != n->child->type)

return(0);

putkey(of, n->child->string, TYPE_St);

return(1);

}

static int

parse_mdoc_Xr(struct of *of, const struct mdoc_node *n)

{

char *cp;

if (NULL == (n = n->child))

return(0);

if (NULL == n->next) {

putkey(of, n->string, TYPE_Xr);

putkey(mpage, n->string, TYPE_Xr);

return(0);

}

Line 1525 parse_mdoc_Xr(struct of *of, const struct mdoc_node *n

Line 1522 parse_mdoc_Xr(struct of *of, const struct mdoc_node *n

perror(NULL);

exit((int)MANDOCLEVEL_SYSERR);

}

putkey(of, cp, TYPE_Xr);

putkey(mpage, cp, TYPE_Xr);

free(cp);

return(0);

}

static int

parse_mdoc_Nd(struct of *of, const struct mdoc_node *n)

parse_mdoc_Nd(struct mpage *mpage, const struct mdoc_node *n)

{

size_t sz;

Line 1545 parse_mdoc_Nd(struct of *of, const struct mdoc_node *n

Line 1542 parse_mdoc_Nd(struct of *of, const struct mdoc_node *n

for (n = n->child; NULL != n; n = n->next) {

if (MDOC_TEXT == n->type) {

if (NULL != of->desc) {

if (NULL != mpage->desc) {

sz = strlen(of->desc) +

sz = strlen(mpage->desc) +

strlen(n->string) + 2;

of->desc = mandoc_realloc(of->desc, sz);

mpage->desc = mandoc_realloc(

strlcat(of->desc, " ", sz);

mpage->desc, sz);

strlcat(of->desc, n->string, sz);

strlcat(mpage->desc, " ", sz);

strlcat(mpage->desc, n->string, sz);

} else

of->desc = mandoc_strdup(n->string);

mpage->desc = mandoc_strdup(n->string);

}

if (NULL != n->child)

parse_mdoc_Nd(of, n);

parse_mdoc_Nd(mpage, n);

}

return(1);

}

static int

parse_mdoc_Nm(struct of *of, const struct mdoc_node *n)

parse_mdoc_Nm(struct mpage *mpage, const struct mdoc_node *n)

{

if (SEC_NAME == n->sec)

return(SEC_NAME == n->sec ||

return(1);

(SEC_SYNOPSIS == n->sec && MDOC_HEAD == n->type));

else if (SEC_SYNOPSIS != n->sec || MDOC_HEAD != n->type)

return(0);

return(1);

}

static int

parse_mdoc_Sh(struct of *of, const struct mdoc_node *n)

parse_mdoc_Sh(struct mpage *mpage, const struct mdoc_node *n)

{

return(SEC_CUSTOM == n->sec && MDOC_HEAD == n->type);

}

static int

parse_mdoc_head(struct of *of, const struct mdoc_node *n)

parse_mdoc_head(struct mpage *mpage, const struct mdoc_node *n)

{

return(MDOC_HEAD == n->type);

}

static int

parse_mdoc_body(struct of *of, const struct mdoc_node *n)

parse_mdoc_body(struct mpage *mpage, const struct mdoc_node *n)

{

return(MDOC_BODY == n->type);

Line 1599 parse_mdoc_body(struct of *of, const struct mdoc_node

Line 1593 parse_mdoc_body(struct of *of, const struct mdoc_node

* When we finish the manual, we'll dump the table.

static void

putkeys(const struct of *of, const char *cp, size_t sz, uint64_t v)

putkeys(const struct mpage *mpage,

const char *cp, size_t sz, uint64_t v)

{

struct str *s;

unsigned int slot;

Line 1612 putkeys(const struct of *of, const char *cp, size_t sz

Line 1607 putkeys(const struct of *of, const char *cp, size_t sz

slot = ohash_qlookupi(&strings, cp, &end);

s = ohash_find(&strings, slot);

if (NULL != s && of == s->of) {

if (NULL != s && mpage == s->mpage) {

s->mask |= v;

return;

} else if (NULL == s) {

Line 1620 putkeys(const struct of *of, const char *cp, size_t sz

Line 1615 putkeys(const struct of *of, const char *cp, size_t sz

memcpy(s->key, cp, sz);

ohash_insert(&strings, slot, s);

}

s->of = of;

s->mpage = mpage;

s->mask = v;

}

Line 1677 utf8(unsigned int cp, char out[7])

Line 1672 utf8(unsigned int cp, char out[7])

}

* Store the UTF-8 version of a key, or alias the pointer if the key has

* Store the rendered version of a key, or alias the pointer

* no UTF-8 transcription marks in it.

* if the key contains no escape sequences.

static void

utf8key(struct mchars *mc, struct str *key)

render_key(struct mchars *mc, struct str *key)

{

size_t sz, bsz, pos;

char utfbuf[7], res[5];

Line 1690 utf8key(struct mchars *mc, struct str *key)

Line 1685 utf8key(struct mchars *mc, struct str *key)

int len, u;

enum mandoc_esc esc;

assert(NULL == key->utf8);

assert(NULL == key->rendered);

res[0] = '\\';

res[1] = '\t';

Line 1706 utf8key(struct mchars *mc, struct str *key)

Line 1701 utf8key(struct mchars *mc, struct str *key)

* pointer as ourselvse and get out of here.

if (strcspn(val, res) == bsz) {

key->utf8 = key->key;

key->rendered = key->key;

return;

}

Line 1741 utf8key(struct mchars *mc, struct str *key)

Line 1736 utf8key(struct mchars *mc, struct str *key)

/* Read past the slash. */

val++;

u = 0;

* Parse the escape sequence and see if it's a

* predefined character or special character.

esc = mandoc_escape

((const char **)&val, &seq, &len);

if (ESCAPE_ERROR == esc)

break;

if (ESCAPE_SPECIAL != esc)

continue;

if (0 == (u = mchars_spec2cp(mc, seq, len)))

continue;

* If we have a Unicode codepoint, try to convert that

* Render the special character

* to a UTF-8 byte string.

* as either UTF-8 or ASCII.

cpp = utfbuf;

if (0 == (sz = utf8(u, utfbuf)))

continue;

if (write_utf8) {

if (0 == (u = mchars_spec2cp(mc, seq, len)))

continue;

cpp = utfbuf;

if (0 == (sz = utf8(u, utfbuf)))

continue;

sz = strlen(cpp);

} else {

cpp = mchars_spec2str(mc, seq, len, &sz);

if (NULL == cpp)

continue;

if (ASCII_NBRSP == *cpp) {

cpp = " ";

sz = 1;

}

/* Copy the rendered glyph into the stream. */

sz = strlen(cpp);

bsz += sz;

buf = mandoc_realloc(buf, bsz);

memcpy(&buf[pos], cpp, sz);

pos += sz;

}

buf[pos] = '\0';

key->utf8 = buf;

key->rendered = buf;

}

* Flush the current page's terms (and their bits) into the database.

* Wrap the entire set of additions in a transaction to make sqlite be a

* little faster.

* Also, UTF-8-encode the description at the last possible moment.

* Also, handle escape sequences at the last possible moment.

static void

dbindex(struct mchars *mc, int form, const struct of *of)

dbindex(const struct mpage *mpage, struct mchars *mc)

{

struct mlink *mlink;

struct str *key;

const char *desc;

int64_t recno;

Line 1796 dbindex(struct mchars *mc, int form, const struct of *

Line 1800 dbindex(struct mchars *mc, int form, const struct of *

unsigned int slot;

if (verb)

say(of->file, "Adding to index");

say(mpage->mlinks->file, "Adding to index");

if (nodb)

return;

desc = "";

if (NULL != of->desc) {

if (NULL != mpage->desc && '\0' != *mpage->desc) {

key = ohash_find(&strings,

ohash_qlookup(&strings, of->desc));

ohash_qlookup(&strings, mpage->desc));

assert(NULL != key);

if (NULL == key->utf8)

if (NULL == key->rendered)

utf8key(mc, key);

render_key(mc, key);

desc = key->utf8;

desc = key->rendered;

}

SQL_EXEC("BEGIN TRANSACTION");

i = 1;

SQL_BIND_TEXT(stmts[STMT_INSERT_DOC], i, of->file);

SQL_BIND_TEXT(stmts[STMT_INSERT_DOC], i, of->sec);

* XXX The following line is obsolete

SQL_BIND_TEXT(stmts[STMT_INSERT_DOC], i, of->arch);

* and only kept for backward compatibility

SQL_BIND_TEXT(stmts[STMT_INSERT_DOC], i, desc);

* until apropos(1) and friends have caught up.

SQL_BIND_INT(stmts[STMT_INSERT_DOC], i, form);

SQL_STEP(stmts[STMT_INSERT_DOC]);

SQL_BIND_TEXT(stmts[STMT_INSERT_PAGE], i, mpage->mlinks->file);

SQL_BIND_TEXT(stmts[STMT_INSERT_PAGE], i, desc);

SQL_BIND_INT(stmts[STMT_INSERT_PAGE], i, FORM_SRC == mpage->form);

SQL_STEP(stmts[STMT_INSERT_PAGE]);

recno = sqlite3_last_insert_rowid(db);

sqlite3_reset(stmts[STMT_INSERT_DOC]);

sqlite3_reset(stmts[STMT_INSERT_PAGE]);

for (mlink = mpage->mlinks; mlink; mlink = mlink->next) {

i = 1;

SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->file);

SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->dsec);

SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->arch);

SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->name);

SQL_BIND_INT64(stmts[STMT_INSERT_LINK], i, recno);

SQL_STEP(stmts[STMT_INSERT_LINK]);

sqlite3_reset(stmts[STMT_INSERT_LINK]);

}

for (key = ohash_first(&strings, &slot); NULL != key;

key = ohash_next(&strings, &slot)) {

assert(key->of == of);

assert(key->mpage == mpage);

if (NULL == key->utf8)

if (NULL == key->rendered)

utf8key(mc, key);

render_key(mc, key);

i = 1;

SQL_BIND_INT64(stmts[STMT_INSERT_KEY], i, key->mask);

SQL_BIND_TEXT(stmts[STMT_INSERT_KEY], i, key->utf8);

SQL_BIND_TEXT(stmts[STMT_INSERT_KEY], i, key->rendered);

SQL_BIND_INT64(stmts[STMT_INSERT_KEY], i, recno);

SQL_STEP(stmts[STMT_INSERT_KEY]);

sqlite3_reset(stmts[STMT_INSERT_KEY]);

if (key->utf8 != key->key)

if (key->rendered != key->key)

free(key->utf8);

free(key->rendered);

free(key);

}

Line 1845 dbindex(struct mchars *mc, int form, const struct of *

Line 1863 dbindex(struct mchars *mc, int form, const struct of *

static void

dbprune(void)

{

struct of *of;

struct mpage *mpage;

struct mlink *mlink;

size_t i;

unsigned int slot;

if (nodb)

return;

for (of = ofs; NULL != of; of = of->next) {

mpage = ohash_first(&mpages, &slot);

while (NULL != mpage) {

mlink = mpage->mlinks;

i = 1;

SQL_BIND_TEXT(stmts[STMT_DELETE], i, of->file);

SQL_BIND_TEXT(stmts[STMT_DELETE_PAGE], i, mlink->file);

SQL_STEP(stmts[STMT_DELETE]);

SQL_STEP(stmts[STMT_DELETE_PAGE]);

sqlite3_reset(stmts[STMT_DELETE]);

sqlite3_reset(stmts[STMT_DELETE_PAGE]);

if (verb)

say(of->file, "Deleted from index");

say(mlink->file, "Deleted from index");

mpage = ohash_next(&mpages, &slot);

}

Line 1937 dbopen(int real)

Line 1960 dbopen(int real)

return(0);

}

sql = "CREATE TABLE \"docs\" (\n"

* XXX The first column in table mpages is obsolete

* and only kept for backward compatibility

* until apropos(1) and friends have caught up.

sql = "CREATE TABLE \"mpages\" (\n"

" \"file\" TEXT NOT NULL,\n"

" \"sec\" TEXT NOT NULL,\n"

" \"arch\" TEXT NOT NULL,\n"

" \"desc\" TEXT NOT NULL,\n"

" \"form\" INTEGER NOT NULL,\n"

" \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n"

");\n"

"\n"

"CREATE TABLE \"mlinks\" (\n"

" \"file\" TEXT NOT NULL,\n"

" \"sec\" TEXT NOT NULL,\n"

" \"arch\" TEXT NOT NULL,\n"

" \"name\" TEXT NOT NULL,\n"

" \"pageid\" INTEGER NOT NULL REFERENCES mpages(id) "

"ON DELETE CASCADE,\n"

" \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n"

");\n"

"\n"

"CREATE TABLE \"keys\" (\n"

" \"bits\" INTEGER NOT NULL,\n"

" \"key\" TEXT NOT NULL,\n"

" \"docid\" INTEGER NOT NULL REFERENCES docs(id) "

" \"pageid\" INTEGER NOT NULL REFERENCES mpages(id) "

"ON DELETE CASCADE,\n"

" \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n"

");\n"

"\n"

Line 1964 dbopen(int real)

Line 2000 dbopen(int real)

prepare_statements:

SQL_EXEC("PRAGMA foreign_keys = ON");

sql = "DELETE FROM docs where file=?";

sql = "DELETE FROM mpages where file=?";

sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_DELETE], NULL);

sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_DELETE_PAGE], NULL);

sql = "INSERT INTO docs "

sql = "INSERT INTO mpages "

"(file,sec,arch,desc,form) VALUES (?,?,?,?,?)";

"(file,desc,form) VALUES (?,?,?)";

sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_DOC], NULL);

sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_PAGE], NULL);

sql = "INSERT INTO mlinks "

"(file,sec,arch,name,pageid) VALUES (?,?,?,?,?)";

sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_LINK], NULL);

sql = "INSERT INTO keys "

"(bits,key,docid) VALUES (?,?,?)";

"(bits,key,pageid) VALUES (?,?,?)";

sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_KEY], NULL);

#ifndef __APPLE__

* When opening a new database, we can turn off

* synchronous mode for much better performance.

if (real)

SQL_EXEC("PRAGMA synchronous = OFF");

#endif

return(1);

}

CVSweb