=================================================================== RCS file: /cvs/mandoc/mandocdb.c,v retrieving revision 1.101 retrieving revision 1.113 diff -u -p -r1.101 -r1.113 --- mandoc/mandocdb.c 2014/01/05 04:48:40 1.101 +++ mandoc/mandocdb.c 2014/01/19 22:48:16 1.113 @@ -1,4 +1,4 @@ -/* $Id: mandocdb.c,v 1.101 2014/01/05 04:48:40 schwarze Exp $ */ +/* $Id: mandocdb.c,v 1.113 2014/01/19 22:48:16 schwarze Exp $ */ /* * Copyright (c) 2011, 2012 Kristaps Dzonsons * Copyright (c) 2011, 2012, 2013, 2014 Ingo Schwarze @@ -48,6 +48,9 @@ #include "manpath.h" #include "mansearch.h" +extern int mansearch_keymax; +extern const char *const mansearch_keynames[]; + #define SQL_EXEC(_v) \ if (SQLITE_OK != sqlite3_exec(db, (_v), NULL, NULL, NULL)) \ fprintf(stderr, "%s\n", sqlite3_errmsg(db)) @@ -130,7 +133,7 @@ struct mdoc_handler { }; static void dbclose(int); -static void dbindex(const struct mpage *, struct mchars *); +static void dbadd(const struct mpage *, struct mchars *); static int dbopen(int); static void dbprune(void); static void filescan(const char *); @@ -154,8 +157,7 @@ static int parse_mdoc_Nd(struct mpage *, const struct static int parse_mdoc_Nm(struct mpage *, const struct mdoc_node *); static int parse_mdoc_Sh(struct mpage *, const struct mdoc_node *); static int parse_mdoc_Xr(struct mpage *, const struct mdoc_node *); -static void putkey(const struct mpage *, - const char *, uint64_t); +static void putkey(const struct mpage *, char *, uint64_t); static void putkeys(const struct mpage *, const char *, size_t, uint64_t); static void putmdockey(const struct mpage *, @@ -167,8 +169,9 @@ static int treescan(void); static size_t utf8(unsigned int, char [7]); static char *progname; -static int use_all; /* use all found files */ static int nodb; /* no database changes */ +static int quick; /* abort the parse early */ +static int use_all; /* use all found files */ static int verb; /* print what we're doing */ static int warnings; /* warn about crap */ static int write_utf8; /* write UTF-8 output; else ASCII */ @@ -347,7 +350,7 @@ main(int argc, char *argv[]) path_arg = NULL; op = OP_DEFAULT; - while (-1 != (ch = getopt(argc, argv, "aC:d:nT:tu:vW"))) + while (-1 != (ch = getopt(argc, argv, "aC:d:nQT:tu:vW"))) switch (ch) { case ('a'): use_all = 1; @@ -365,6 +368,9 @@ main(int argc, char *argv[]) case ('n'): nodb = 1; break; + case ('Q'): + quick = 1; + break; case ('T'): if (strcmp(optarg, "utf8")) { fprintf(stderr, "-T%s: Unsupported " @@ -404,7 +410,7 @@ main(int argc, char *argv[]) exitcode = (int)MANDOCLEVEL_OK; mp = mparse_alloc(MPARSE_AUTO, - MANDOCLEVEL_FATAL, NULL, NULL, NULL); + MANDOCLEVEL_FATAL, NULL, NULL, quick); mc = mchars_alloc(); ohash_init(&mpages, 6, &mpages_info); @@ -494,11 +500,11 @@ out: ohash_delete(&mlinks); return(exitcode); usage: - fprintf(stderr, "usage: %s [-anvW] [-C file] [-Tutf8]\n" - " %s [-anvW] [-Tutf8] dir ...\n" - " %s [-nvW] [-Tutf8] -d dir [file ...]\n" + fprintf(stderr, "usage: %s [-anQvW] [-C file] [-Tutf8]\n" + " %s [-anQvW] [-Tutf8] dir ...\n" + " %s [-nQvW] [-Tutf8] -d dir [file ...]\n" " %s [-nvW] -u dir [file ...]\n" - " %s -t file ...\n", + " %s [-Q] -t file ...\n", progname, progname, progname, progname, progname); @@ -696,11 +702,19 @@ filescan(const char *file) exitcode = (int)MANDOCLEVEL_BADARG; say(file, NULL); return; - } else if (OP_TEST != op && strstr(buf, basedir) != buf) { + } + + if (strstr(buf, basedir) == buf) + start = buf + strlen(basedir) + 1; + else if (OP_TEST == op) + start = buf; + else { exitcode = (int)MANDOCLEVEL_BADARG; say("", "%s: outside base directory", buf); return; - } else if (-1 == stat(buf, &st)) { + } + + if (-1 == stat(buf, &st)) { exitcode = (int)MANDOCLEVEL_BADARG; say(file, NULL); return; @@ -709,7 +723,7 @@ filescan(const char *file) say(file, "Not a regular file"); return; } - start = buf + strlen(basedir); + mlink = mandoc_calloc(1, sizeof(struct mlink)); strlcpy(mlink->file, start, sizeof(mlink->file)); @@ -942,12 +956,13 @@ mlink_check(struct mpage *mpage, struct mlink *mlink) static void mpages_merge(struct mchars *mc, struct mparse *mp) { + char any[] = "any"; struct ohash_info str_info; struct mpage *mpage; struct mlink *mlink; struct mdoc *mdoc; struct man *man; - const char *cp; + char *cp; int match; unsigned int pslot; enum mandoclevel lvl; @@ -957,6 +972,9 @@ mpages_merge(struct mchars *mc, struct mparse *mp) str_info.hfree = hash_free; str_info.key_offset = offsetof(struct str, key); + if (0 == nodb) + SQL_EXEC("BEGIN TRANSACTION"); + mpage = ohash_first(&mpages, &pslot); while (NULL != mpage) { mlinks_undupe(mpage); @@ -1010,7 +1028,7 @@ mpages_merge(struct mchars *mc, struct mparse *mp) } putkey(mpage, mpage->sec, TYPE_sec); putkey(mpage, '\0' == *mpage->arch ? - "any" : mpage->arch, TYPE_arch); + any : mpage->arch, TYPE_arch); for (mlink = mpage->mlinks; mlink; mlink = mlink->next) { if ('\0' != *mlink->dsec) @@ -1018,7 +1036,7 @@ mpages_merge(struct mchars *mc, struct mparse *mp) if ('\0' != *mlink->fsec) putkey(mpage, mlink->fsec, TYPE_sec); putkey(mpage, '\0' == *mlink->arch ? - "any" : mlink->arch, TYPE_arch); + any : mlink->arch, TYPE_arch); putkey(mpage, mlink->name, TYPE_Nm); } @@ -1043,10 +1061,13 @@ mpages_merge(struct mchars *mc, struct mparse *mp) else parse_cat(mpage); - dbindex(mpage, mc); + dbadd(mpage, mc); ohash_delete(&strings); mpage = ohash_next(&mpages, &pslot); } + + if (0 == nodb) + SQL_EXEC("END TRANSACTION"); } static void @@ -1165,10 +1186,15 @@ parse_cat(struct mpage *mpage) * Put a type/word pair into the word database for this particular file. */ static void -putkey(const struct mpage *mpage, const char *value, uint64_t type) +putkey(const struct mpage *mpage, char *value, uint64_t type) { + char *cp; assert(NULL != value); + if (TYPE_arch == type) + for (cp = value; *cp; cp++) + if (isupper((unsigned char)*cp)) + *cp = _tolower((unsigned char)*cp); putkeys(mpage, value, strlen(value), type); } @@ -1284,6 +1310,15 @@ parse_man(struct mpage *mpage, const struct man_node * byte = start[sz]; start[sz] = '\0'; + /* + * Assume a stray trailing comma in the + * name list if a name begins with a dash. + */ + + if ('-' == start[0] || + ('\\' == start[0] && '-' == start[1])) + break; + putkey(mpage, start, TYPE_Nm); if (' ' == byte) { @@ -1413,7 +1448,7 @@ parse_mdoc_Fd(struct mpage *mpage, const struct mdoc_n static int parse_mdoc_Fn(struct mpage *mpage, const struct mdoc_node *n) { - const char *cp; + char *cp; if (NULL == (n = n->child) || MDOC_TEXT != n->type) return(0); @@ -1535,12 +1570,24 @@ putkeys(const struct mpage *mpage, const char *cp, size_t sz, uint64_t v) { struct str *s; - unsigned int slot; const char *end; + uint64_t mask; + unsigned int slot; + int i; if (0 == sz) return; + if (verb > 1) { + for (i = 0, mask = 1; + i < mansearch_keymax; + i++, mask <<= 1) + if (mask & v) + break; + say(mpage->mlinks->file, "Adding key %s=%*s", + mansearch_keynames[i], sz, cp); + } + end = cp + sz; slot = ohash_qlookupi(&strings, cp, &end); s = ohash_find(&strings, slot); @@ -1728,35 +1775,21 @@ render_key(struct mchars *mc, struct str *key) * Also, handle escape sequences at the last possible moment. */ static void -dbindex(const struct mpage *mpage, struct mchars *mc) +dbadd(const struct mpage *mpage, struct mchars *mc) { struct mlink *mlink; struct str *key; - const char *desc; int64_t recno; size_t i; unsigned int slot; if (verb) - say(mpage->mlinks->file, "Adding to index"); + say(mpage->mlinks->file, "Adding to database"); if (nodb) return; - desc = ""; - if (NULL != mpage->desc && '\0' != *mpage->desc) { - key = ohash_find(&strings, - ohash_qlookup(&strings, mpage->desc)); - assert(NULL != key); - if (NULL == key->rendered) - render_key(mc, key); - desc = key->rendered; - } - - SQL_EXEC("BEGIN TRANSACTION"); - i = 1; - SQL_BIND_TEXT(stmts[STMT_INSERT_PAGE], i, desc); SQL_BIND_INT(stmts[STMT_INSERT_PAGE], i, FORM_SRC == mpage->form); SQL_STEP(stmts[STMT_INSERT_PAGE]); recno = sqlite3_last_insert_rowid(db); @@ -1764,7 +1797,6 @@ dbindex(const struct mpage *mpage, struct mchars *mc) for (mlink = mpage->mlinks; mlink; mlink = mlink->next) { i = 1; - SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->file); SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->dsec); SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->arch); SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->name); @@ -1788,8 +1820,6 @@ dbindex(const struct mpage *mpage, struct mchars *mc) free(key->rendered); free(key); } - - SQL_EXEC("END TRANSACTION"); } static void @@ -1800,20 +1830,31 @@ dbprune(void) size_t i; unsigned int slot; - if (nodb) - return; + if (0 == nodb) + SQL_EXEC("BEGIN TRANSACTION"); - mpage = ohash_first(&mpages, &slot); - while (NULL != mpage) { + for (mpage = ohash_first(&mpages, &slot); NULL != mpage; + mpage = ohash_next(&mpages, &slot)) { mlink = mpage->mlinks; - i = 1; - SQL_BIND_TEXT(stmts[STMT_DELETE_PAGE], i, mlink->file); - SQL_STEP(stmts[STMT_DELETE_PAGE]); - sqlite3_reset(stmts[STMT_DELETE_PAGE]); if (verb) - say(mlink->file, "Deleted from index"); - mpage = ohash_next(&mpages, &slot); + say(mlink->file, "Deleting from database"); + if (nodb) + continue; + for ( ; NULL != mlink; mlink = mlink->next) { + i = 1; + SQL_BIND_TEXT(stmts[STMT_DELETE_PAGE], + i, mlink->dsec); + SQL_BIND_TEXT(stmts[STMT_DELETE_PAGE], + i, mlink->arch); + SQL_BIND_TEXT(stmts[STMT_DELETE_PAGE], + i, mlink->name); + SQL_STEP(stmts[STMT_DELETE_PAGE]); + sqlite3_reset(stmts[STMT_DELETE_PAGE]); + } } + + if (0 == nodb) + SQL_EXEC("END TRANSACTION"); } /* @@ -1893,30 +1934,24 @@ dbopen(int real) } sql = "CREATE TABLE \"mpages\" (\n" - " \"desc\" TEXT NOT NULL,\n" " \"form\" INTEGER NOT NULL,\n" " \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n" ");\n" "\n" "CREATE TABLE \"mlinks\" (\n" - " \"file\" TEXT NOT NULL,\n" " \"sec\" TEXT NOT NULL,\n" " \"arch\" TEXT NOT NULL,\n" " \"name\" TEXT NOT NULL,\n" " \"pageid\" INTEGER NOT NULL REFERENCES mpages(id) " - "ON DELETE CASCADE,\n" - " \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n" + "ON DELETE CASCADE\n" ");\n" "\n" "CREATE TABLE \"keys\" (\n" " \"bits\" INTEGER NOT NULL,\n" " \"key\" TEXT NOT NULL,\n" " \"pageid\" INTEGER NOT NULL REFERENCES mpages(id) " - "ON DELETE CASCADE,\n" - " \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n" - ");\n" - "\n" - "CREATE INDEX \"key_index\" ON keys (key);\n"; + "ON DELETE CASCADE\n" + ");\n"; if (SQLITE_OK != sqlite3_exec(db, sql, NULL, NULL, NULL)) { exitcode = (int)MANDOCLEVEL_SYSERR; @@ -1926,13 +1961,15 @@ dbopen(int real) prepare_statements: SQL_EXEC("PRAGMA foreign_keys = ON"); - sql = "DELETE FROM mpages where file=?"; + sql = "DELETE FROM mpages WHERE id IN " + "(SELECT pageid FROM mlinks WHERE " + "sec=? AND arch=? AND name=?)"; sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_DELETE_PAGE], NULL); sql = "INSERT INTO mpages " - "(desc,form) VALUES (?,?)"; + "(form) VALUES (?)"; sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_PAGE], NULL); sql = "INSERT INTO mlinks " - "(file,sec,arch,name,pageid) VALUES (?,?,?,?,?)"; + "(sec,arch,name,pageid) VALUES (?,?,?,?)"; sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_LINK], NULL); sql = "INSERT INTO keys " "(bits,key,pageid) VALUES (?,?,?)";