=================================================================== RCS file: /cvs/mandoc/mandocdb.c,v retrieving revision 1.13 retrieving revision 1.22 diff -u -p -r1.13 -r1.22 --- mandoc/mandocdb.c 2011/11/26 22:38:11 1.13 +++ mandoc/mandocdb.c 2011/12/03 12:09:07 1.22 @@ -1,4 +1,4 @@ -/* $Id: mandocdb.c,v 1.13 2011/11/26 22:38:11 schwarze Exp $ */ +/* $Id: mandocdb.c,v 1.22 2011/12/03 12:09:07 kristaps Exp $ */ /* * Copyright (c) 2011 Kristaps Dzonsons * Copyright (c) 2011 Ingo Schwarze @@ -20,6 +20,8 @@ #endif #include +#include +#include #include #include @@ -29,9 +31,14 @@ #include #include #include +#include -#ifdef __linux__ +#if defined(__linux__) +# include # include +#elif defined(__APPLE__) +# include +# include #else # include #endif @@ -45,6 +52,9 @@ #define MANDOC_BUFSZ BUFSIZ #define MANDOC_SLOP 1024 +#define MANDOC_SRC 0x1 +#define MANDOC_FORM 0x2 + /* Tiny list for files. No need to bring in QUEUE. */ struct of { @@ -52,6 +62,7 @@ struct of { char *sec; char *arch; char *title; + int src_form; struct of *next; /* NULL for last one */ struct of *first; /* first in list */ }; @@ -91,18 +102,18 @@ static void dbt_put(DB *, const char *, DBT *, DBT static void hash_put(DB *, const struct buf *, uint64_t); static void hash_reset(DB **); static void index_merge(const struct of *, struct mparse *, - struct buf *, struct buf *, - DB *, DB *, const char *, - DB *, const char *, int, int, + struct buf *, struct buf *, DB *, + DB *, const char *, DB *, const char *, recno_t, const recno_t *, size_t); static void index_prune(const struct of *, DB *, const char *, DB *, const char *, - int, recno_t *, recno_t **, size_t *); -static void ofile_argbuild(char *[], int, int, int, - struct of **); + recno_t *, recno_t **, size_t *); +static void ofile_argbuild(int, char *[], struct of **); static int ofile_dirbuild(const char *, const char *, - const char *, int, int, struct of **); + const char *, int, struct of **); static void ofile_free(struct of *); +static void pformatted(DB *, struct buf *, struct buf *, + const struct of *); static int pman_node(MAN_ARGS); static void pmdoc_node(MDOC_ARGS); static void pmdoc_An(MDOC_ARGS); @@ -249,6 +260,8 @@ static const pmdoc_nf mdocs[MDOC_MAX] = { }; static const char *progname; +static int use_all; /* Use all directories and files. */ +static int verb; /* Output verbosity level. */ int main(int argc, char *argv[]) @@ -259,9 +272,7 @@ main(int argc, char *argv[]) const char *dir; char ibuf[MAXPATHLEN], /* index fname */ fbuf[MAXPATHLEN]; /* btree fname */ - int verb, /* output verbosity */ - use_all, /* use all directories and files */ - ch, i, flags; + int ch, i, flags; DB *idx, /* index database */ *db, /* keyword database */ *hash; /* temporary keyword hashtable */ @@ -322,6 +333,7 @@ main(int argc, char *argv[]) argv += optind; memset(&info, 0, sizeof(BTREEINFO)); + info.lorder = 4321; info.flags = R_DUP; mp = mparse_alloc(MPARSE_AUTO, MANDOCLEVEL_FATAL, NULL, NULL); @@ -368,19 +380,27 @@ main(int argc, char *argv[]) printf("%s: Opened\n", ibuf); } - ofile_argbuild(argv, argc, use_all, verb, &of); + ofile_argbuild(argc, argv, &of); if (NULL == of) goto out; of = of->first; - index_prune(of, db, fbuf, idx, ibuf, verb, + index_prune(of, db, fbuf, idx, ibuf, &maxrec, &recs, &recsz); - if (OP_UPDATE == op) + /* + * Go to the root of the respective manual tree + * such that .so links work. In case of failure, + * just prod on, even though .so links won't work. + */ + + if (OP_UPDATE == op) { + chdir(dir); index_merge(of, mp, &dbuf, &buf, hash, - db, fbuf, idx, ibuf, use_all, - verb, maxrec, recs, reccur); + db, fbuf, idx, ibuf, + maxrec, recs, reccur); + } goto out; } @@ -441,7 +461,7 @@ main(int argc, char *argv[]) of = NULL; if ( ! ofile_dirbuild(dirs.paths[i], NULL, NULL, - use_all, verb, &of)) + 0, &of)) exit((int)MANDOCLEVEL_SYSERR); if (NULL == of) @@ -449,9 +469,15 @@ main(int argc, char *argv[]) of = of->first; + /* + * Go to the root of the respective manual tree + * such that .so links work. In case of failure, + * just prod on, even though .so links won't work. + */ + + chdir(dirs.paths[i]); index_merge(of, mp, &dbuf, &buf, hash, db, fbuf, - idx, ibuf, use_all, verb, - maxrec, recs, reccur); + idx, ibuf, maxrec, recs, reccur); } out: @@ -475,9 +501,8 @@ out: void index_merge(const struct of *of, struct mparse *mp, - struct buf *dbuf, struct buf *buf, - DB *hash, DB *db, const char *dbf, - DB *idx, const char *idxf, int use_all, int verb, + struct buf *dbuf, struct buf *buf, DB *hash, + DB *db, const char *dbf, DB *idx, const char *idxf, recno_t maxrec, const recno_t *recs, size_t reccur) { recno_t rec; @@ -492,6 +517,11 @@ index_merge(const struct of *of, struct mparse *mp, for (rec = 0; of; of = of->next) { fn = of->fname; + + /* + * Reclaim an empty index record, if available. + */ + if (reccur > 0) { --reccur; rec = recs[(int)reccur]; @@ -503,27 +533,40 @@ index_merge(const struct of *of, struct mparse *mp, mparse_reset(mp); hash_reset(&hash); + mdoc = NULL; + man = NULL; - if (mparse_readfd(mp, -1, fn) >= MANDOCLEVEL_FATAL) { - fprintf(stderr, "%s: Parse failure\n", fn); - continue; - } + /* + * Try interpreting the file as mdoc(7) or man(7) + * source code, unless it is already known to be + * formatted. Fall back to formatted mode. + */ - mparse_result(mp, &mdoc, &man); - if (NULL == mdoc && NULL == man) - continue; + if ((MANDOC_SRC & of->src_form || + ! (MANDOC_FORM & of->src_form)) && + MANDOCLEVEL_FATAL > mparse_readfd(mp, -1, fn)) + mparse_result(mp, &mdoc, &man); + if (NULL != mdoc) { + msec = mdoc_meta(mdoc)->msec; + arch = mdoc_meta(mdoc)->arch; + mtitle = mdoc_meta(mdoc)->title; + } else if (NULL != man) { + msec = man_meta(man)->msec; + arch = NULL; + mtitle = man_meta(man)->title; + } else { + msec = of->sec; + arch = of->arch; + mtitle = of->title; + } + /* * By default, skip a file if the manual section * and architecture given in the file disagree * with the directory where the file is located. */ - msec = NULL != mdoc ? - mdoc_meta(mdoc)->msec : man_meta(man)->msec; - arch = NULL != mdoc ? - mdoc_meta(mdoc)->arch : NULL; - if (0 == use_all) { assert(of->sec); assert(msec); @@ -548,9 +591,6 @@ index_merge(const struct of *of, struct mparse *mp, * because the one in the file usually is all caps. */ - mtitle = NULL != mdoc ? - mdoc_meta(mdoc)->title : man_meta(man)->title; - assert(of->title); assert(mtitle); @@ -568,6 +608,7 @@ index_merge(const struct of *of, struct mparse *mp, */ dbuf->len = 0; + buf_append(dbuf, mdoc ? "mdoc" : (man ? "man" : "cat")); buf_appendb(dbuf, fn, strlen(fn) + 1); buf_appendb(dbuf, msec, strlen(msec) + 1); buf_appendb(dbuf, mtitle, strlen(mtitle) + 1); @@ -580,26 +621,23 @@ index_merge(const struct of *of, struct mparse *mp, if (mdoc) pmdoc_node(hash, buf, dbuf, mdoc_node(mdoc), mdoc_meta(mdoc)); - else + else if (man) pman_node(hash, buf, dbuf, man_node(man)); + else + pformatted(hash, buf, dbuf, of); /* * Copy from the in-memory hashtable of pending keywords * into the database. */ - vbuf.rec = rec; + vbuf.rec = htobe32(rec); seq = R_FIRST; while (0 == (ch = (*hash->seq)(hash, &key, &val, seq))) { seq = R_NEXT; - - vbuf.mask = *(uint64_t *)val.data; + vbuf.mask = htobe64(*(uint64_t *)val.data); val.size = sizeof(struct db_val); val.data = &vbuf; - - if (verb > 1) - printf("%s: Added keyword: %s\n", - fn, (char *)key.data); dbt_put(db, dbf, &key, &val); } if (ch < 0) { @@ -623,6 +661,7 @@ index_merge(const struct of *of, struct mparse *mp, if (verb) printf("%s: Added index\n", fn); + dbt_put(idx, idxf, &key, &val); } } @@ -635,11 +674,11 @@ index_merge(const struct of *of, struct mparse *mp, */ static void index_prune(const struct of *ofile, DB *db, const char *dbf, - DB *idx, const char *idxf, int verb, + DB *idx, const char *idxf, recno_t *maxrec, recno_t **recs, size_t *recsz) { const struct of *of; - const char *fn; + const char *fn, *cp; struct db_val *vbuf; unsigned seq, sseq; DBT key, val; @@ -651,18 +690,32 @@ index_prune(const struct of *ofile, DB *db, const char while (0 == (ch = (*idx->seq)(idx, &key, &val, seq))) { seq = R_NEXT; *maxrec = *(recno_t *)key.data; - if (0 == val.size) { - if (reccur >= *recsz) { - *recsz += MANDOC_SLOP; - *recs = mandoc_realloc(*recs, - *recsz * sizeof(recno_t)); - } - (*recs)[(int)reccur] = *maxrec; - reccur++; - continue; - } + cp = val.data; - fn = (char *)val.data; + /* Deleted records are zero-sized. Skip them. */ + + if (0 == val.size) + goto cont; + + /* + * Make sure we're sane. + * Read past our mdoc/man/cat type to the next string, + * then make sure it's bounded by a NUL. + * Failing any of these, we go into our error handler. + */ + + if (NULL == (fn = memchr(cp, '\0', val.size))) + break; + if (++fn - cp >= (int)val.size) + break; + if (NULL == memchr(fn, '\0', val.size - (fn - cp))) + break; + + /* + * Search for the file in those we care about. + * XXX: build this into a tree. Too slow. + */ + for (of = ofile; of; of = of->next) if (0 == strcmp(fn, of->fname)) break; @@ -670,23 +723,31 @@ index_prune(const struct of *ofile, DB *db, const char if (NULL == of) continue; + /* + * Search through the keyword database, throwing out all + * references to our file. + */ + sseq = R_FIRST; while (0 == (ch = (*db->seq)(db, &key, &val, sseq))) { sseq = R_NEXT; - assert(sizeof(struct db_val) == val.size); + if (sizeof(struct db_val) != val.size) + break; + vbuf = val.data; - if (*maxrec != vbuf->rec) + if (*maxrec != betoh32(vbuf->rec)) continue; - if (verb) - printf("%s: Deleted keyword: %s\n", - fn, (char *)key.data); - ch = (*db->del)(db, &key, R_CURSOR); - if (ch < 0) + + if ((ch = (*db->del)(db, &key, R_CURSOR)) < 0) break; } + if (ch < 0) { perror(dbf); exit((int)MANDOCLEVEL_SYSERR); + } else if (1 != ch) { + fprintf(stderr, "%s: Corrupt database\n", dbf); + exit((int)MANDOCLEVEL_SYSERR); } if (verb) @@ -694,11 +755,10 @@ index_prune(const struct of *ofile, DB *db, const char val.size = 0; ch = (*idx->put)(idx, &key, &val, R_CURSOR); - if (ch < 0) { - perror(idxf); - exit((int)MANDOCLEVEL_SYSERR); - } + if (ch < 0) + break; +cont: if (reccur >= *recsz) { *recsz += MANDOC_SLOP; *recs = mandoc_realloc @@ -708,6 +768,15 @@ index_prune(const struct of *ofile, DB *db, const char (*recs)[(int)reccur] = *maxrec; reccur++; } + + if (ch < 0) { + perror(idxf); + exit((int)MANDOCLEVEL_SYSERR); + } else if (1 != ch) { + fprintf(stderr, "%s: Corrupt index\n", idxf); + exit((int)MANDOCLEVEL_SYSERR); + } + (*maxrec)++; } @@ -1232,13 +1301,88 @@ pman_node(MAN_ARGS) return(0); } +/* + * Parse a formatted manual page. + * By necessity, this involves rather crude guesswork. + */ static void -ofile_argbuild(char *argv[], int argc, int use_all, int verb, - struct of **of) +pformatted(DB *hash, struct buf *buf, struct buf *dbuf, + const struct of *of) { + FILE *stream; + char *line, *p; + size_t len, plen; + + if (NULL == (stream = fopen(of->fname, "r"))) { + perror(of->fname); + return; + } + + /* + * Always use the title derived from the filename up front, + * do not even try to find it in the file. This also makes + * sure we don't end up with an orphan index record, even if + * the file content turns out to be completely unintelligible. + */ + + buf->len = 0; + buf_append(buf, of->title); + hash_put(hash, buf, TYPE_Nm); + + while (NULL != (line = fgetln(stream, &len)) && '\n' != *line) + /* Skip to first blank line. */ ; + + while (NULL != (line = fgetln(stream, &len)) && + ('\n' == *line || ' ' == *line)) + /* Skip to first section header. */ ; + + /* + * If no page content can be found, + * reuse the page title as the page description. + */ + + if (NULL == (line = fgetln(stream, &len))) { + buf_appendb(dbuf, buf->cp, buf->size); + hash_put(hash, buf, TYPE_Nd); + fclose(stream); + return; + } + fclose(stream); + + /* + * If there is a dash, skip to the text following it. + */ + + for (p = line, plen = len; plen; p++, plen--) + if ('-' == *p) + break; + for ( ; plen; p++, plen--) + if ('-' != *p && ' ' != *p && 8 != *p) + break; + if (0 == plen) { + p = line; + plen = len; + } + + /* + * Copy the rest of the line, but no more than 70 bytes. + */ + + if (70 < plen) + plen = 70; + p[plen-1] = '\0'; + buf_appendb(dbuf, p, plen); + buf->len = 0; + buf_appendb(buf, p, plen); + hash_put(hash, buf, TYPE_Nd); +} + +static void +ofile_argbuild(int argc, char *argv[], struct of **of) +{ char buf[MAXPATHLEN]; char *sec, *arch, *title, *p; - int i; + int i, src_form; struct of *nof; for (i = 0; i < argc; i++) { @@ -1246,7 +1390,8 @@ ofile_argbuild(char *argv[], int argc, int use_all, in /* * Try to infer the manual section, architecture and * page title from the path, assuming it looks like - * man*[/]/.<section> + * man*[/<arch>]/<title>.<section> or + * cat<section>[/<arch>]/<title>.0 */ if (strlcpy(buf, argv[i], sizeof(buf)) >= sizeof(buf)) { @@ -1254,11 +1399,16 @@ ofile_argbuild(char *argv[], int argc, int use_all, in continue; } sec = arch = title = NULL; + src_form = 0; p = strrchr(buf, '\0'); while (p-- > buf) { if (NULL == sec && '.' == *p) { sec = p + 1; *p = '\0'; + if ('0' == *sec) + src_form |= MANDOC_FORM; + else if ('1' <= *sec && '9' >= *sec) + src_form |= MANDOC_SRC; continue; } if ('/' != *p) @@ -1268,8 +1418,13 @@ ofile_argbuild(char *argv[], int argc, int use_all, in *p = '\0'; continue; } - if (strncmp("man", p + 1, 3)) + if (strncmp("man", p + 1, 3)) { + src_form |= MANDOC_SRC; arch = p + 1; + } else if (strncmp("cat", p + 1, 3)) { + src_form |= MANDOC_FORM; + arch = p + 1; + } break; } if (NULL == title) @@ -1286,6 +1441,7 @@ ofile_argbuild(char *argv[], int argc, int use_all, in if (NULL != arch) nof->arch = mandoc_strdup(arch); nof->title = mandoc_strdup(title); + nof->src_form = src_form; /* * Add the structure to the list. @@ -1314,15 +1470,17 @@ ofile_argbuild(char *argv[], int argc, int use_all, in */ static int ofile_dirbuild(const char *dir, const char* psec, const char *parch, - int use_all, int verb, struct of **of) + int p_src_form, struct of **of) { char buf[MAXPATHLEN]; + struct stat sb; size_t sz; DIR *d; const char *fn, *sec, *arch; - char *suffix; + char *p, *q, *suffix; struct of *nof; struct dirent *dp; + int src_form; if (NULL == (d = opendir(dir))) { perror(dir); @@ -1335,19 +1493,26 @@ ofile_dirbuild(const char *dir, const char* psec, cons if ('.' == *fn) continue; + src_form = p_src_form; + if (DT_DIR == dp->d_type) { sec = psec; arch = parch; /* * By default, only use directories called: - * man<section>/[<arch>/] + * man<section>/[<arch>/] or + * cat<section>/[<arch>/] */ if (NULL == sec) { - if(0 == strncmp("man", fn, 3)) + if(0 == strncmp("man", fn, 3)) { + src_form |= MANDOC_SRC; sec = fn + 3; - else if (use_all) + } else if (0 == strncmp("cat", fn, 3)) { + src_form |= MANDOC_FORM; + sec = fn + 3; + } else if (use_all) sec = fn; else continue; @@ -1371,7 +1536,7 @@ ofile_dirbuild(const char *dir, const char* psec, cons printf("%s: Scanning\n", buf); if ( ! ofile_dirbuild(buf, sec, arch, - use_all, verb, of)) + src_form, of)) return(0); } if (DT_REG != dp->d_type || @@ -1390,8 +1555,56 @@ ofile_dirbuild(const char *dir, const char* psec, cons if (0 == use_all) { if (NULL == suffix) continue; - if (strcmp(suffix + 1, psec)) + if ((MANDOC_SRC & src_form && + strcmp(suffix + 1, psec)) || + (MANDOC_FORM & src_form && + strcmp(suffix + 1, "0"))) + continue; + } + if (NULL != suffix) { + if ('0' == suffix[1]) + src_form |= MANDOC_FORM; + else if ('1' <= suffix[1] && '9' >= suffix[1]) + src_form |= MANDOC_SRC; + } + + + /* + * Skip formatted manuals if a source version is + * available. Ignore the age: it is very unlikely + * that people install newer formatted base manuals + * when they used to have source manuals before, + * and in ports, old manuals get removed on update. + */ + if (0 == use_all && MANDOC_FORM & src_form && + NULL != psec) { + buf[0] = '\0'; + strlcat(buf, dir, MAXPATHLEN); + p = strrchr(buf, '/'); + if (NULL == p) + p = buf; + else + p++; + if (0 == strncmp("cat", p, 3)) + memcpy(p, "man", 3); + strlcat(buf, "/", MAXPATHLEN); + sz = strlcat(buf, fn, MAXPATHLEN); + if (sz >= MAXPATHLEN) { + fprintf(stderr, "%s: Path too long\n", buf); continue; + } + q = strrchr(buf, '.'); + if (NULL != q && p < q++) { + *q = '\0'; + sz = strlcat(buf, psec, MAXPATHLEN); + if (sz >= MAXPATHLEN) { + fprintf(stderr, + "%s: Path too long\n", buf); + continue; + } + if (0 == stat(buf, &sb)) + continue; + } } buf[0] = '\0'; @@ -1400,7 +1613,7 @@ ofile_dirbuild(const char *dir, const char* psec, cons sz = strlcat(buf, fn, MAXPATHLEN); if (sz >= MAXPATHLEN) { fprintf(stderr, "%s: Path too long\n", dir); - return(0); + continue; } nof = mandoc_calloc(1, sizeof(struct of)); @@ -1409,6 +1622,7 @@ ofile_dirbuild(const char *dir, const char* psec, cons nof->sec = mandoc_strdup(psec); if (NULL != parch) nof->arch = mandoc_strdup(parch); + nof->src_form = src_form; /* * Remember the file name without the extension, @@ -1419,9 +1633,12 @@ ofile_dirbuild(const char *dir, const char* psec, cons *suffix = '\0'; nof->title = mandoc_strdup(fn); + /* + * Add the structure to the list. + */ + if (verb > 2) printf("%s: Scheduling\n", buf); - if (NULL == *of) { *of = nof; (*of)->first = nof;