mandoc/mandocdb.c - diff

Return to mandocdb.c CVS log

Up to [cvsweb.bsd.lv] / mandoc

Diff for /mandoc/mandocdb.c between version 1.13 and 1.18

version 1.13, 2011/11/26 22:38:11

version 1.18, 2011/12/01 21:05:49

Line 20

#endif

#include <sys/param.h>

#include <sys/types.h>

#include <sys/stat.h>

#include <assert.h>

#include <dirent.h>

Line 29

Line 31

#include <stdint.h>

#include <stdlib.h>

#include <string.h>

#include <unistd.h>

#ifdef __linux__

# include <db_185.h>

Line 45

Line 48

#define MANDOC_BUFSZ BUFSIZ

#define MANDOC_SLOP 1024

#define MANDOC_SRC 0x1

#define MANDOC_FORM 0x2

/* Tiny list for files. No need to bring in QUEUE. */

struct of {

Line 52 struct of {

Line 58 struct of {

char *sec;

char *arch;

char *title;

int src_form;

struct of *next; /* NULL for last one */

struct of *first; /* first in list */

};

Line 91 static void dbt_put(DB *, const char *, DBT *, DBT

Line 98 static void dbt_put(DB *, const char *, DBT *, DBT

static void hash_put(DB *, const struct buf *, uint64_t);

static void hash_reset(DB **);

static void index_merge(const struct of *, struct mparse *,

struct buf *, struct buf *,

struct buf *, struct buf *, DB *,

DB *, DB *, const char *,

DB *, const char *, DB *, const char *,

DB *, const char *, int, int,

recno_t, const recno_t *, size_t);

static void index_prune(const struct of *, DB *,

const char *, DB *, const char *,

int, recno_t *, recno_t **, size_t *);

recno_t *, recno_t **, size_t *);

static void ofile_argbuild(char *[], int, int, int,

static void ofile_argbuild(int, char *[], struct of **);

struct of **);

static int ofile_dirbuild(const char *, const char *,

const char *, int, int, struct of **);

const char *, int, struct of **);

static void ofile_free(struct of *);

static void pformatted(DB *, struct buf *, struct buf *,

const struct of *);

static int pman_node(MAN_ARGS);

static void pmdoc_node(MDOC_ARGS);

static void pmdoc_An(MDOC_ARGS);

Line 249 static const pmdoc_nf mdocs[MDOC_MAX] = {

Line 256 static const pmdoc_nf mdocs[MDOC_MAX] = {

};

static const char *progname;

static int use_all; /* Use all directories and files. */

static int verb; /* Output verbosity level. */

int

main(int argc, char *argv[])

Line 259 main(int argc, char *argv[])

Line 268 main(int argc, char *argv[])

const char *dir;

char ibuf[MAXPATHLEN], /* index fname */

fbuf[MAXPATHLEN]; /* btree fname */

int verb, /* output verbosity */

int ch, i, flags;

use_all, /* use all directories and files */

ch, i, flags;

DB *idx, /* index database */

*db, /* keyword database */

*hash; /* temporary keyword hashtable */

Line 368 main(int argc, char *argv[])

Line 375 main(int argc, char *argv[])

printf("%s: Opened\n", ibuf);

}

ofile_argbuild(argv, argc, use_all, verb, &of);

ofile_argbuild(argc, argv, &of);

if (NULL == of)

goto out;

of = of->first;

index_prune(of, db, fbuf, idx, ibuf, verb,

index_prune(of, db, fbuf, idx, ibuf,

&maxrec, &recs, &recsz);

if (OP_UPDATE == op)

* Go to the root of the respective manual tree

* such that .so links work. In case of failure,

* just prod on, even though .so links won't work.

if (OP_UPDATE == op) {

chdir(dir);

index_merge(of, mp, &dbuf, &buf, hash,

db, fbuf, idx, ibuf, use_all,

db, fbuf, idx, ibuf,

verb, maxrec, recs, reccur);

maxrec, recs, reccur);

}

goto out;

}

Line 441 main(int argc, char *argv[])

Line 456 main(int argc, char *argv[])

of = NULL;

if ( ! ofile_dirbuild(dirs.paths[i], NULL, NULL,

use_all, verb, &of))

0, &of))

exit((int)MANDOCLEVEL_SYSERR);

if (NULL == of)

Line 449 main(int argc, char *argv[])

Line 464 main(int argc, char *argv[])

of = of->first;

* Go to the root of the respective manual tree

* such that .so links work. In case of failure,

* just prod on, even though .so links won't work.

chdir(dirs.paths[i]);

index_merge(of, mp, &dbuf, &buf, hash, db, fbuf,

idx, ibuf, use_all, verb,

idx, ibuf, maxrec, recs, reccur);

maxrec, recs, reccur);

}

out:

Line 475 out:

Line 496 out:

void

index_merge(const struct of *of, struct mparse *mp,

struct buf *dbuf, struct buf *buf,

struct buf *dbuf, struct buf *buf, DB *hash,

DB *hash, DB *db, const char *dbf,

DB *db, const char *dbf, DB *idx, const char *idxf,

DB *idx, const char *idxf, int use_all, int verb,

recno_t maxrec, const recno_t *recs, size_t reccur)

{

recno_t rec;

Line 492 index_merge(const struct of *of, struct mparse *mp,

Line 512 index_merge(const struct of *of, struct mparse *mp,

for (rec = 0; of; of = of->next) {

fn = of->fname;

* Reclaim an empty index record, if available.

if (reccur > 0) {

--reccur;

rec = recs[(int)reccur];

Line 503 index_merge(const struct of *of, struct mparse *mp,

Line 528 index_merge(const struct of *of, struct mparse *mp,

mparse_reset(mp);

hash_reset(&hash);

mdoc = NULL;

man = NULL;

if (mparse_readfd(mp, -1, fn) >= MANDOCLEVEL_FATAL) {

fprintf(stderr, "%s: Parse failure\n", fn);

* Try interpreting the file as mdoc(7) or man(7)

continue;

* source code, unless it is already known to be

}

* formatted. Fall back to formatted mode.

mparse_result(mp, &mdoc, &man);

if ((MANDOC_SRC & of->src_form ||

if (NULL == mdoc && NULL == man)

! (MANDOC_FORM & of->src_form)) &&

continue;

MANDOCLEVEL_FATAL > mparse_readfd(mp, -1, fn))

mparse_result(mp, &mdoc, &man);

if (NULL != mdoc) {

msec = mdoc_meta(mdoc)->msec;

arch = mdoc_meta(mdoc)->arch;

mtitle = mdoc_meta(mdoc)->title;

} else if (NULL != man) {

msec = man_meta(man)->msec;

arch = NULL;

mtitle = man_meta(man)->title;

} else {

msec = of->sec;

arch = of->arch;

mtitle = of->title;

}

* By default, skip a file if the manual section

* and architecture given in the file disagree

* with the directory where the file is located.

msec = NULL != mdoc ?

mdoc_meta(mdoc)->msec : man_meta(man)->msec;

arch = NULL != mdoc ?

mdoc_meta(mdoc)->arch : NULL;

if (0 == use_all) {

assert(of->sec);

assert(msec);

Line 548 index_merge(const struct of *of, struct mparse *mp,

Line 586 index_merge(const struct of *of, struct mparse *mp,

* because the one in the file usually is all caps.

mtitle = NULL != mdoc ?

mdoc_meta(mdoc)->title : man_meta(man)->title;

assert(of->title);

assert(mtitle);

Line 568 index_merge(const struct of *of, struct mparse *mp,

Line 603 index_merge(const struct of *of, struct mparse *mp,

dbuf->len = 0;

buf_append(dbuf, mdoc ? "mdoc" : (man ? "man" : "cat"));

buf_appendb(dbuf, fn, strlen(fn) + 1);

buf_appendb(dbuf, msec, strlen(msec) + 1);

buf_appendb(dbuf, mtitle, strlen(mtitle) + 1);

Line 580 index_merge(const struct of *of, struct mparse *mp,

Line 616 index_merge(const struct of *of, struct mparse *mp,

if (mdoc)

pmdoc_node(hash, buf, dbuf,

mdoc_node(mdoc), mdoc_meta(mdoc));

else

else if (man)

pman_node(hash, buf, dbuf, man_node(man));

else

pformatted(hash, buf, dbuf, of);

* Copy from the in-memory hashtable of pending keywords

Line 597 index_merge(const struct of *of, struct mparse *mp,

Line 635 index_merge(const struct of *of, struct mparse *mp,

val.size = sizeof(struct db_val);

val.data = &vbuf;

if (verb > 1)

printf("%s: Added keyword: %s\n",

fn, (char *)key.data);

dbt_put(db, dbf, &key, &val);

}

if (ch < 0) {

Line 623 index_merge(const struct of *of, struct mparse *mp,

Line 658 index_merge(const struct of *of, struct mparse *mp,

if (verb)

printf("%s: Added index\n", fn);

dbt_put(idx, idxf, &key, &val);

}

Line 635 index_merge(const struct of *of, struct mparse *mp,

Line 671 index_merge(const struct of *of, struct mparse *mp,

static void

index_prune(const struct of *ofile, DB *db, const char *dbf,

DB *idx, const char *idxf, int verb,

DB *idx, const char *idxf,

recno_t *maxrec, recno_t **recs, size_t *recsz)

{

const struct of *of;

const char *fn;

const char *fn, *cp;

struct db_val *vbuf;

unsigned seq, sseq;

DBT key, val;

Line 651 index_prune(const struct of *ofile, DB *db, const char

Line 687 index_prune(const struct of *ofile, DB *db, const char

while (0 == (ch = (*idx->seq)(idx, &key, &val, seq))) {

seq = R_NEXT;

*maxrec = *(recno_t *)key.data;

if (0 == val.size) {

cp = val.data;

if (reccur >= *recsz) {

*recsz += MANDOC_SLOP;

*recs = mandoc_realloc(*recs,

*recsz * sizeof(recno_t));

}

(*recs)[(int)reccur] = *maxrec;

reccur++;

continue;

}

fn = (char *)val.data;

/* Deleted records are zero-sized. Skip them. */

if (0 == val.size)

goto cont;

* Make sure we're sane.

* Read past our mdoc/man/cat type to the next string,

* then make sure it's bounded by a NUL.

* Failing any of these, we go into our error handler.

if (NULL == (fn = memchr(cp, '\0', val.size)))

break;

if (++fn - cp >= (int)val.size)

break;

if (NULL == memchr(fn, '\0', val.size - (fn - cp)))

break;

* Search for the file in those we care about.

* XXX: build this into a tree. Too slow.

for (of = ofile; of; of = of->next)

if (0 == strcmp(fn, of->fname))

break;

Line 670 index_prune(const struct of *ofile, DB *db, const char

Line 720 index_prune(const struct of *ofile, DB *db, const char

if (NULL == of)

continue;

* Search through the keyword database, throwing out all

* references to our file.

sseq = R_FIRST;

while (0 == (ch = (*db->seq)(db, &key, &val, sseq))) {

sseq = R_NEXT;

assert(sizeof(struct db_val) == val.size);

if (sizeof(struct db_val) != val.size)

break;

vbuf = val.data;

if (*maxrec != vbuf->rec)

continue;

if (verb)

printf("%s: Deleted keyword: %s\n",

if ((ch = (*db->del)(db, &key, R_CURSOR)) < 0)

fn, (char *)key.data);

ch = (*db->del)(db, &key, R_CURSOR);

if (ch < 0)

break;

}

if (ch < 0) {

perror(dbf);

exit((int)MANDOCLEVEL_SYSERR);

} else if (1 != ch) {

fprintf(stderr, "%s: Corrupt database\n", dbf);

exit((int)MANDOCLEVEL_SYSERR);

}

if (verb)

Line 694 index_prune(const struct of *ofile, DB *db, const char

Line 752 index_prune(const struct of *ofile, DB *db, const char

val.size = 0;

ch = (*idx->put)(idx, &key, &val, R_CURSOR);

if (ch < 0) {

perror(idxf);

exit((int)MANDOCLEVEL_SYSERR);

}

if (ch < 0)

break;

cont:

if (reccur >= *recsz) {

*recsz += MANDOC_SLOP;

*recs = mandoc_realloc

Line 708 index_prune(const struct of *ofile, DB *db, const char

Line 765 index_prune(const struct of *ofile, DB *db, const char

(*recs)[(int)reccur] = *maxrec;

reccur++;

}

if (ch < 0) {

perror(idxf);

exit((int)MANDOCLEVEL_SYSERR);

} else if (1 != ch) {

fprintf(stderr, "%s: Corrupt index\n", idxf);

exit((int)MANDOCLEVEL_SYSERR);

}

(*maxrec)++;

}

Line 1232 pman_node(MAN_ARGS)

Line 1298 pman_node(MAN_ARGS)

return(0);

}

* Parse a formatted manual page.

* By necessity, this involves rather crude guesswork.

static void

ofile_argbuild(char *argv[], int argc, int use_all, int verb,

pformatted(DB *hash, struct buf *buf, struct buf *dbuf,

struct of **of)

const struct of *of)

{

FILE *stream;

char *line, *p;

size_t len, plen;

if (NULL == (stream = fopen(of->fname, "r"))) {

perror(of->fname);

return;

}

* Always use the title derived from the filename up front,

* do not even try to find it in the file. This also makes

* sure we don't end up with an orphan index record, even if

* the file content turns out to be completely unintelligible.

buf->len = 0;

buf_append(buf, of->title);

hash_put(hash, buf, TYPE_Nm);

while (NULL != (line = fgetln(stream, &len)) && '\n' != *line)

/* Skip to first blank line. */ ;

while (NULL != (line = fgetln(stream, &len)) &&

('\n' == *line || ' ' == *line))

/* Skip to first section header. */ ;

* If no page content can be found,

* reuse the page title as the page description.

if (NULL == (line = fgetln(stream, &len))) {

buf_appendb(dbuf, buf->cp, buf->size);

hash_put(hash, buf, TYPE_Nd);

fclose(stream);

return;

}

fclose(stream);

* If there is a dash, skip to the text following it.

for (p = line, plen = len; plen; p++, plen--)

if ('-' == *p)

break;

for ( ; plen; p++, plen--)

if ('-' != *p && ' ' != *p && 8 != *p)

break;

if (0 == plen) {

p = line;

plen = len;

}

* Copy the rest of the line, but no more than 70 bytes.

if (70 < plen)

plen = 70;

p[plen-1] = '\0';

buf_appendb(dbuf, p, plen);

buf->len = 0;

buf_appendb(buf, p, plen);

hash_put(hash, buf, TYPE_Nd);

}

static void

ofile_argbuild(int argc, char *argv[], struct of **of)

{

char buf[MAXPATHLEN];

char *sec, *arch, *title, *p;

int i;

int i, src_form;

struct of *nof;

for (i = 0; i < argc; i++) {

Line 1246 ofile_argbuild(char *argv[], int argc, int use_all, in

Line 1387 ofile_argbuild(char *argv[], int argc, int use_all, in

* Try to infer the manual section, architecture and

* page title from the path, assuming it looks like

* man*[/<arch>]/<title>.<section>

* man*[/<arch>]/<title>.<section> or

* cat<section>[/<arch>]/<title>.0

if (strlcpy(buf, argv[i], sizeof(buf)) >= sizeof(buf)) {

Line 1254 ofile_argbuild(char *argv[], int argc, int use_all, in

Line 1396 ofile_argbuild(char *argv[], int argc, int use_all, in

continue;

}

sec = arch = title = NULL;

src_form = 0;

p = strrchr(buf, '\0');

while (p-- > buf) {

if (NULL == sec && '.' == *p) {

sec = p + 1;

*p = '\0';

if ('0' == *sec)

src_form |= MANDOC_FORM;

else if ('1' <= *sec && '9' >= *sec)

src_form |= MANDOC_SRC;

continue;

}

if ('/' != *p)

Line 1268 ofile_argbuild(char *argv[], int argc, int use_all, in

Line 1415 ofile_argbuild(char *argv[], int argc, int use_all, in

*p = '\0';

continue;

}

if (strncmp("man", p + 1, 3))

if (strncmp("man", p + 1, 3)) {

src_form |= MANDOC_SRC;

arch = p + 1;

} else if (strncmp("cat", p + 1, 3)) {

src_form |= MANDOC_FORM;

arch = p + 1;

}

break;

}

if (NULL == title)

Line 1286 ofile_argbuild(char *argv[], int argc, int use_all, in

Line 1438 ofile_argbuild(char *argv[], int argc, int use_all, in

if (NULL != arch)

nof->arch = mandoc_strdup(arch);

nof->title = mandoc_strdup(title);

nof->src_form = src_form;

* Add the structure to the list.

Line 1314 ofile_argbuild(char *argv[], int argc, int use_all, in

Line 1467 ofile_argbuild(char *argv[], int argc, int use_all, in

static int

ofile_dirbuild(const char *dir, const char* psec, const char *parch,

int use_all, int verb, struct of **of)

int p_src_form, struct of **of)

{

char buf[MAXPATHLEN];

struct stat sb;

size_t sz;

DIR *d;

const char *fn, *sec, *arch;

char *suffix;

char *p, *q, *suffix;

struct of *nof;

struct dirent *dp;

int src_form;

if (NULL == (d = opendir(dir))) {

perror(dir);

Line 1335 ofile_dirbuild(const char *dir, const char* psec, cons

Line 1490 ofile_dirbuild(const char *dir, const char* psec, cons

if ('.' == *fn)

continue;

src_form = p_src_form;

if (DT_DIR == dp->d_type) {

sec = psec;

arch = parch;

* By default, only use directories called:

* man<section>/[<arch>/]

* man<section>/[<arch>/] or

* cat<section>/[<arch>/]

if (NULL == sec) {

if(0 == strncmp("man", fn, 3))

if(0 == strncmp("man", fn, 3)) {

src_form |= MANDOC_SRC;

sec = fn + 3;

else if (use_all)

} else if (0 == strncmp("cat", fn, 3)) {

src_form |= MANDOC_FORM;

sec = fn + 3;

} else if (use_all)

sec = fn;

else

continue;

Line 1371 ofile_dirbuild(const char *dir, const char* psec, cons

Line 1533 ofile_dirbuild(const char *dir, const char* psec, cons

printf("%s: Scanning\n", buf);

if ( ! ofile_dirbuild(buf, sec, arch,

use_all, verb, of))

src_form, of))

return(0);

}

if (DT_REG != dp->d_type ||

Line 1390 ofile_dirbuild(const char *dir, const char* psec, cons

Line 1552 ofile_dirbuild(const char *dir, const char* psec, cons

if (0 == use_all) {

if (NULL == suffix)

continue;

if (strcmp(suffix + 1, psec))

if ((MANDOC_SRC & src_form &&

strcmp(suffix + 1, psec)) ||

(MANDOC_FORM & src_form &&

strcmp(suffix + 1, "0")))

continue;

}

if (NULL != suffix) {

if ('0' == suffix[1])

src_form |= MANDOC_FORM;

else if ('1' <= suffix[1] && '9' >= suffix[1])

src_form |= MANDOC_SRC;

}

* Skip formatted manuals if a source version is

* available. Ignore the age: it is very unlikely

* that people install newer formatted base manuals

* when they used to have source manuals before,

* and in ports, old manuals get removed on update.

if (0 == use_all && MANDOC_FORM & src_form &&

NULL != psec) {

buf[0] = '\0';

strlcat(buf, dir, MAXPATHLEN);

p = strrchr(buf, '/');

if (NULL == p)

p = buf;

else

p++;

if (0 == strncmp("cat", p, 3))

memcpy(p, "man", 3);

strlcat(buf, "/", MAXPATHLEN);

sz = strlcat(buf, fn, MAXPATHLEN);

if (sz >= MAXPATHLEN) {

fprintf(stderr, "%s: Path too long\n", buf);

continue;

}

q = strrchr(buf, '.');

if (NULL != q && p < q++) {

*q = '\0';

sz = strlcat(buf, psec, MAXPATHLEN);

if (sz >= MAXPATHLEN) {

fprintf(stderr,

"%s: Path too long\n", buf);

continue;

}

if (0 == stat(buf, &sb))

continue;

}

buf[0] = '\0';

Line 1400 ofile_dirbuild(const char *dir, const char* psec, cons

Line 1610 ofile_dirbuild(const char *dir, const char* psec, cons

sz = strlcat(buf, fn, MAXPATHLEN);

if (sz >= MAXPATHLEN) {

fprintf(stderr, "%s: Path too long\n", dir);

return(0);

continue;

}

nof = mandoc_calloc(1, sizeof(struct of));

Line 1409 ofile_dirbuild(const char *dir, const char* psec, cons

Line 1619 ofile_dirbuild(const char *dir, const char* psec, cons

nof->sec = mandoc_strdup(psec);

if (NULL != parch)

nof->arch = mandoc_strdup(parch);

nof->src_form = src_form;

* Remember the file name without the extension,

Line 1419 ofile_dirbuild(const char *dir, const char* psec, cons

Line 1630 ofile_dirbuild(const char *dir, const char* psec, cons

*suffix = '\0';

nof->title = mandoc_strdup(fn);

* Add the structure to the list.

if (verb > 2)

printf("%s: Scheduling\n", buf);

if (NULL == *of) {

*of = nof;

(*of)->first = nof;

CVSweb