=================================================================== RCS file: /cvs/mandoc/mandocdb.c,v retrieving revision 1.51 retrieving revision 1.57 diff -u -p -r1.51 -r1.57 --- mandoc/mandocdb.c 2012/06/08 12:05:27 1.51 +++ mandoc/mandocdb.c 2013/06/03 22:51:14 1.57 @@ -1,7 +1,7 @@ -/* $Id: mandocdb.c,v 1.51 2012/06/08 12:05:27 kristaps Exp $ */ +/* $Id: mandocdb.c,v 1.57 2013/06/03 22:51:14 schwarze Exp $ */ /* * Copyright (c) 2011, 2012 Kristaps Dzonsons - * Copyright (c) 2011, 2012 Ingo Schwarze + * Copyright (c) 2011, 2012, 2013 Ingo Schwarze * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -34,31 +34,37 @@ #include #include +#ifdef HAVE_OHASH #include +#else +#include "compat_ohash.h" +#endif #include #include "mdoc.h" #include "man.h" #include "mandoc.h" -#include "mandocdb.h" #include "manpath.h" +#include "mansearch.h" -/* Post a warning to stderr. */ -#define WARNING(_f, _b, _fmt, _args...) \ - do if (warnings) { \ - fprintf(stderr, "%s: ", (_b)); \ - fprintf(stderr, (_fmt), ##_args); \ - if ('\0' != *(_f)) \ - fprintf(stderr, ": %s", (_f)); \ - fprintf(stderr, "\n"); \ - } while (/* CONSTCOND */ 0) -/* Post a "verbose" message to stderr. */ -#define DEBUG(_f, _b, _fmt, _args...) \ - do if (verb) { \ - fprintf(stderr, "%s: ", (_b)); \ - fprintf(stderr, (_fmt), ##_args); \ - fprintf(stderr, ": %s\n", (_f)); \ - } while (/* CONSTCOND */ 0) +#define SQL_EXEC(_v) \ + if (SQLITE_OK != sqlite3_exec(db, (_v), NULL, NULL, NULL)) \ + fprintf(stderr, "%s\n", sqlite3_errmsg(db)) +#define SQL_BIND_TEXT(_s, _i, _v) \ + if (SQLITE_OK != sqlite3_bind_text \ + ((_s), (_i)++, (_v), -1, SQLITE_STATIC)) \ + fprintf(stderr, "%s\n", sqlite3_errmsg(db)) +#define SQL_BIND_INT(_s, _i, _v) \ + if (SQLITE_OK != sqlite3_bind_int \ + ((_s), (_i)++, (_v))) \ + fprintf(stderr, "%s\n", sqlite3_errmsg(db)) +#define SQL_BIND_INT64(_s, _i, _v) \ + if (SQLITE_OK != sqlite3_bind_int64 \ + ((_s), (_i)++, (_v))) \ + fprintf(stderr, "%s\n", sqlite3_errmsg(db)) +#define SQL_STEP(_s) \ + if (SQLITE_DONE != sqlite3_step((_s))) \ + fprintf(stderr, "%s\n", sqlite3_errmsg(db)) enum op { OP_DEFAULT = 0, /* new dbs from dir list or default config */ @@ -156,6 +162,7 @@ static void putkeys(const struct of *, const char *, int, uint64_t); static void putmdockey(const struct of *, const struct mdoc_node *, uint64_t); +static void say(const char *, const char *, const char *, ...); static char *stradd(const char *); static char *straddbuf(const char *, size_t); static int treescan(const char *); @@ -308,6 +315,7 @@ main(int argc, char *argv[]) { char cwd[MAXPATHLEN]; int ch, rc, fd, i; + unsigned int index; size_t j, sz; const char *dir; struct str *s; @@ -479,6 +487,16 @@ main(int argc, char *argv[]) goto out; if (0 == dbopen(dirs.paths[j], 0)) goto out; + + /* + * Since we're opening up a new database, we can + * turn off synchronous mode for much better + * performance. + */ +#ifndef __APPLE__ + SQL_EXEC("PRAGMA synchronous = OFF"); +#endif + if (0 == ofmerge(mc, mp, dirs.paths[j])) goto out; dbclose(dirs.paths[j], 0); @@ -494,8 +512,8 @@ out: manpath_free(&dirs); mchars_free(mc); mparse_free(mp); - for (s = ohash_first(&strings, &ch); - NULL != s; s = ohash_next(&strings, &ch)) { + for (s = ohash_first(&strings, &index); + NULL != s; s = ohash_next(&strings, &index)) { if (s->utf8 != s->key) free(s->utf8); free(s); @@ -566,30 +584,37 @@ treescan(const char *base) */ if (FTS_F == ff->fts_info) { if ( ! use_all && ff->fts_level < 2) { - WARNING(path, base, "Extraneous file"); + if (warnings) + say(base, path, "Extraneous file"); continue; } else if (inocheck(ff->fts_statp)) { - WARNING(path, base, "Duplicate file"); + if (warnings) + say(base, path, "Duplicate file"); continue; } cp = ff->fts_name; if (0 == strcmp(cp, "mandocdb.db")) { - WARNING(path, base, "Skip database"); + if (warnings) + say(base, path, "Skip database"); continue; } else if (NULL != (cp = strrchr(cp, '.'))) { if (0 == strcmp(cp + 1, "html")) { - WARNING(path, base, "Skip html"); + if (warnings) + say(base, path, "Skip html"); continue; } else if (0 == strcmp(cp + 1, "gz")) { - WARNING(path, base, "Skip gz"); + if (warnings) + say(base, path, "Skip gz"); continue; } else if (0 == strcmp(cp + 1, "ps")) { - WARNING(path, base, "Skip ps"); + if (warnings) + say(base, path, "Skip ps"); continue; } else if (0 == strcmp(cp + 1, "pdf")) { - WARNING(path, base, "Skip pdf"); + if (warnings) + say(base, path, "Skip pdf"); continue; } } @@ -633,7 +658,8 @@ treescan(const char *base) if (NULL != dsec || use_all) break; - WARNING(path, base, "Unknown directory part"); + if (warnings) + say(base, path, "Unknown directory part"); fts_set(f, ff, FTS_SKIP); break; case (2): @@ -648,7 +674,8 @@ treescan(const char *base) default: if (FTS_DP == ff->fts_info || use_all) break; - WARNING(path, base, "Extraneous directory part"); + if (warnings) + say(base, path, "Extraneous directory part"); fts_set(f, ff, FTS_SKIP); break; } @@ -687,13 +714,16 @@ filescan(const char *file, const char *base) file += 2; if (-1 == stat(file, &st)) { - WARNING(file, base, "%s", strerror(errno)); + if (warnings) + say(base, file, "%s", strerror(errno)); return; } else if ( ! (S_IFREG & st.st_mode)) { - WARNING(file, base, "Not a regular file"); + if (warnings) + say(base, file, "Not a regular file"); return; } else if (inocheck(&st)) { - WARNING(file, base, "Duplicate file"); + if (warnings) + say(base, file, "Duplicate file"); return; } @@ -907,8 +937,9 @@ ofmerge(struct mchars *mc, struct mparse *mp, const ch if ( ! use_all && FORM_CAT == of->dform) { sz = strlcpy(buf, of->file, MAXPATHLEN); if (sz >= MAXPATHLEN) { - WARNING(of->file, base, - "Filename too long"); + if (warnings) + say(base, of->file, + "Filename too long"); continue; } bufp = strstr(buf, "cat"); @@ -918,8 +949,9 @@ ofmerge(struct mchars *mc, struct mparse *mp, const ch *++bufp = '\0'; strlcat(buf, of->dsec, MAXPATHLEN); if (filecheck(buf)) { - WARNING(of->file, base, "Man " - "source exists: %s", buf); + if (warnings) + say(base, of->file, "Man " + "source exists: %s", buf); continue; } } @@ -972,8 +1004,9 @@ ofmerge(struct mchars *mc, struct mparse *mp, const ch * section, like encrypt(1) = makekey(8). Do not skip * manuals for such reasons. */ - if ( ! use_all && form && strcasecmp(msec, of->dsec)) - WARNING(of->file, base, "Section \"%s\" " + if (warnings && !use_all && form && + strcasecmp(msec, of->dsec)) + say(base, of->file, "Section \"%s\" " "manual in %s directory", msec, of->dsec); @@ -991,8 +1024,8 @@ ofmerge(struct mchars *mc, struct mparse *mp, const ch * Thus, warn about architecture mismatches, * but don't skip manuals for this reason. */ - if ( ! use_all && strcasecmp(march, of->arch)) - WARNING(of->file, base, "Architecture \"%s\" " + if (warnings && !use_all && strcasecmp(march, of->arch)) + say(base, of->file, "Architecture \"%s\" " "manual in \"%s\" directory", march, of->arch); @@ -1021,7 +1054,8 @@ parse_catpage(struct of *of, const char *base) size_t len, plen, titlesz; if (NULL == (stream = fopen(of->file, "r"))) { - WARNING(of->file, base, "%s", strerror(errno)); + if (warnings) + say(base, of->file, "%s", strerror(errno)); return; } @@ -1073,7 +1107,8 @@ parse_catpage(struct of *of, const char *base) */ if (NULL == title || '\0' == *title) { - WARNING(of->file, base, "Cannot find NAME section"); + if (warnings) + say(base, of->file, "Cannot find NAME section"); fclose(stream); free(title); return; @@ -1092,7 +1127,8 @@ parse_catpage(struct of *of, const char *base) for (p += 2; ' ' == *p || '\b' == *p; p++) /* Skip to next word. */ ; } else { - WARNING(of->file, base, "No dash in title line"); + if (warnings) + say(base, of->file, "No dash in title line"); p = title; } @@ -1768,8 +1804,10 @@ dbindex(struct mchars *mc, int form, struct str *key; const char *desc; int64_t recno; + size_t i; - DEBUG(of->file, base, "Adding to index"); + if (verb) + say(base, of->file, "Adding to index"); if (nodb) return; @@ -1783,23 +1821,15 @@ dbindex(struct mchars *mc, int form, desc = key->utf8; } - sqlite3_exec(db, "BEGIN TRANSACTION", NULL, NULL, NULL); + SQL_EXEC("BEGIN TRANSACTION"); - sqlite3_bind_text - (stmts[STMT_INSERT_DOC], 1, - of->file, -1, SQLITE_STATIC); - sqlite3_bind_text - (stmts[STMT_INSERT_DOC], 2, - of->sec, -1, SQLITE_STATIC); - sqlite3_bind_text - (stmts[STMT_INSERT_DOC], 3, - of->arch, -1, SQLITE_STATIC); - sqlite3_bind_text - (stmts[STMT_INSERT_DOC], 4, - desc, -1, SQLITE_STATIC); - sqlite3_bind_int - (stmts[STMT_INSERT_DOC], 5, form); - sqlite3_step(stmts[STMT_INSERT_DOC]); + i = 1; + SQL_BIND_TEXT(stmts[STMT_INSERT_DOC], i, of->file); + SQL_BIND_TEXT(stmts[STMT_INSERT_DOC], i, of->sec); + SQL_BIND_TEXT(stmts[STMT_INSERT_DOC], i, of->arch); + SQL_BIND_TEXT(stmts[STMT_INSERT_DOC], i, desc); + SQL_BIND_INT(stmts[STMT_INSERT_DOC], i, form); + SQL_STEP(stmts[STMT_INSERT_DOC]); recno = sqlite3_last_insert_rowid(db); sqlite3_reset(stmts[STMT_INSERT_DOC]); @@ -1807,36 +1837,33 @@ dbindex(struct mchars *mc, int form, assert(key->of == of); if (NULL == key->utf8) utf8key(mc, key); - sqlite3_bind_int64 - (stmts[STMT_INSERT_KEY], 1, key->mask); - sqlite3_bind_text - (stmts[STMT_INSERT_KEY], 2, - key->utf8, -1, SQLITE_STATIC); - sqlite3_bind_int64 - (stmts[STMT_INSERT_KEY], 3, recno); - sqlite3_step(stmts[STMT_INSERT_KEY]); + i = 1; + SQL_BIND_INT64(stmts[STMT_INSERT_KEY], i, key->mask); + SQL_BIND_TEXT(stmts[STMT_INSERT_KEY], i, key->utf8); + SQL_BIND_INT64(stmts[STMT_INSERT_KEY], i, recno); + SQL_STEP(stmts[STMT_INSERT_KEY]); sqlite3_reset(stmts[STMT_INSERT_KEY]); } - sqlite3_exec(db, "COMMIT TRANSACTION", NULL, NULL, NULL); - + SQL_EXEC("END TRANSACTION"); } static void dbprune(const char *base) { struct of *of; + size_t i; if (nodb) return; for (of = ofs; NULL != of; of = of->next) { - sqlite3_bind_text - (stmts[STMT_DELETE], 1, - of->file, -1, SQLITE_STATIC); - sqlite3_step(stmts[STMT_DELETE]); + i = 1; + SQL_BIND_TEXT(stmts[STMT_DELETE], i, of->file); + SQL_STEP(stmts[STMT_DELETE]); sqlite3_reset(stmts[STMT_DELETE]); - DEBUG(of->file, base, "Deleted from index"); + if (verb) + say(base, of->file, "Deleted from index"); } } @@ -1901,12 +1928,12 @@ dbopen(const char *base, int real) if ( ! real) remove(file); - ofl = SQLITE_OPEN_PRIVATECACHE | SQLITE_OPEN_READWRITE | + ofl = SQLITE_OPEN_READWRITE | (0 == real ? SQLITE_OPEN_EXCLUSIVE : 0); rc = sqlite3_open_v2(file, &db, ofl, NULL); if (SQLITE_OK == rc) - return(1); + goto prepare_statements; if (SQLITE_CANTOPEN != rc) { perror(file); return(0); @@ -1944,6 +1971,8 @@ dbopen(const char *base, int real) return(0); } +prepare_statements: + SQL_EXEC("PRAGMA foreign_keys = ON"); sql = "DELETE FROM docs where file=?"; sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_DELETE], NULL); sql = "INSERT INTO docs " @@ -1988,4 +2017,21 @@ path_reset(const char *cwd, int fd, const char *base) return(0); } return(1); +} + +static void +say(const char *dir, const char *file, const char *format, ...) +{ + va_list ap; + + fprintf(stderr, "%s", dir); + if ('\0' != *file) + fprintf(stderr, "//%s", file); + fputs(": ", stderr); + + va_start(ap, format); + vfprintf(stderr, format, ap); + va_end(ap); + + fputc('\n', stderr); }