=================================================================== RCS file: /cvs/mandoc/mansearch.c,v retrieving revision 1.15 retrieving revision 1.33 diff -u -p -r1.15 -r1.33 --- mandoc/mansearch.c 2014/01/05 03:06:43 1.15 +++ mandoc/mansearch.c 2014/04/17 19:20:01 1.33 @@ -1,4 +1,4 @@ -/* $Id: mansearch.c,v 1.15 2014/01/05 03:06:43 schwarze Exp $ */ +/* $Id: mansearch.c,v 1.33 2014/04/17 19:20:01 schwarze Exp $ */ /* * Copyright (c) 2012 Kristaps Dzonsons * Copyright (c) 2013, 2014 Ingo Schwarze @@ -19,6 +19,7 @@ #include "config.h" #endif +#include #include #include #include @@ -39,9 +40,13 @@ #include #include "mandoc.h" +#include "mandoc_aux.h" #include "manpath.h" #include "mansearch.h" +extern int mansearch_keymax; +extern const char *const mansearch_keynames[]; + #define SQL_BIND_TEXT(_db, _s, _i, _v) \ do { if (SQLITE_OK != sqlite3_bind_text \ ((_s), (_i)++, (_v), -1, SQLITE_STATIC)) \ @@ -69,64 +74,14 @@ struct expr { }; struct match { - uint64_t id; /* identifier in database */ - char *file; /* relative filepath of manpage */ - char *desc; /* description of manpage */ + uint64_t pageid; /* identifier in database */ + char *desc; /* manual page description */ int form; /* 0 == catpage */ }; -struct type { - uint64_t bits; - const char *name; -}; - -static const struct type types[] = { - { TYPE_An, "An" }, - { TYPE_Ar, "Ar" }, - { TYPE_At, "At" }, - { TYPE_Bsx, "Bsx" }, - { TYPE_Bx, "Bx" }, - { TYPE_Cd, "Cd" }, - { TYPE_Cm, "Cm" }, - { TYPE_Dv, "Dv" }, - { TYPE_Dx, "Dx" }, - { TYPE_Em, "Em" }, - { TYPE_Er, "Er" }, - { TYPE_Ev, "Ev" }, - { TYPE_Fa, "Fa" }, - { TYPE_Fl, "Fl" }, - { TYPE_Fn, "Fn" }, - { TYPE_Fn, "Fo" }, - { TYPE_Ft, "Ft" }, - { TYPE_Fx, "Fx" }, - { TYPE_Ic, "Ic" }, - { TYPE_In, "In" }, - { TYPE_Lb, "Lb" }, - { TYPE_Li, "Li" }, - { TYPE_Lk, "Lk" }, - { TYPE_Ms, "Ms" }, - { TYPE_Mt, "Mt" }, - { TYPE_Nd, "Nd" }, - { TYPE_Nm, "Nm" }, - { TYPE_Nx, "Nx" }, - { TYPE_Ox, "Ox" }, - { TYPE_Pa, "Pa" }, - { TYPE_Rs, "Rs" }, - { TYPE_Sh, "Sh" }, - { TYPE_Ss, "Ss" }, - { TYPE_St, "St" }, - { TYPE_Sy, "Sy" }, - { TYPE_Tn, "Tn" }, - { TYPE_Va, "Va" }, - { TYPE_Va, "Vt" }, - { TYPE_Xr, "Xr" }, - { TYPE_sec, "sec" }, - { TYPE_arch,"arch" }, - { ~0ULL, "any" }, - { 0ULL, NULL } -}; - -static char *buildnames(sqlite3 *, sqlite3_stmt *, uint64_t); +static void buildnames(struct manpage *, sqlite3 *, + sqlite3_stmt *, uint64_t, + const char *, int form); static char *buildoutput(sqlite3 *, sqlite3_stmt *, uint64_t, uint64_t); static void *hash_alloc(size_t, void *); @@ -147,15 +102,62 @@ static void sql_regexp(sqlite3_context *context, static char *sql_statement(const struct expr *); int +mansearch_setup(int start) +{ + static void *pagecache; + int c; + +#define PC_PAGESIZE 1280 +#define PC_NUMPAGES 256 + + if (start) { + if (NULL != pagecache) { + fprintf(stderr, "pagecache already enabled\n"); + return((int)MANDOCLEVEL_BADARG); + } + + pagecache = mmap(NULL, PC_PAGESIZE * PC_NUMPAGES, + PROT_READ | PROT_WRITE, MAP_ANON, -1, 0); + + if (MAP_FAILED == pagecache) { + perror("mmap"); + pagecache = NULL; + return((int)MANDOCLEVEL_SYSERR); + } + + c = sqlite3_config(SQLITE_CONFIG_PAGECACHE, + pagecache, PC_PAGESIZE, PC_NUMPAGES); + + if (SQLITE_OK == c) + return((int)MANDOCLEVEL_OK); + + fprintf(stderr, "pagecache: %s\n", sqlite3_errstr(c)); + + } else if (NULL == pagecache) { + fprintf(stderr, "pagecache missing\n"); + return((int)MANDOCLEVEL_BADARG); + } + + if (-1 == munmap(pagecache, PC_PAGESIZE * PC_NUMPAGES)) { + perror("munmap"); + pagecache = NULL; + return((int)MANDOCLEVEL_SYSERR); + } + + pagecache = NULL; + return((int)MANDOCLEVEL_OK); +} + +int mansearch(const struct mansearch *search, const struct manpaths *paths, int argc, char *argv[], const char *outkey, struct manpage **res, size_t *sz) { - int fd, rc, c, ibit; - int64_t id; - uint64_t outbit; + int fd, rc, c, indexbit; + int64_t pageid; + uint64_t outbit, iterbit; char buf[PATH_MAX]; char *sql; struct manpage *mpage; @@ -173,7 +175,7 @@ mansearch(const struct mansearch *search, info.halloc = hash_halloc; info.alloc = hash_alloc; info.hfree = hash_free; - info.key_offset = offsetof(struct match, id); + info.key_offset = offsetof(struct match, pageid); *sz = cur = maxres = 0; sql = NULL; @@ -189,9 +191,12 @@ mansearch(const struct mansearch *search, outbit = 0; if (NULL != outkey) { - for (ibit = 0; types[ibit].bits; ibit++) { - if (0 == strcasecmp(types[ibit].name, outkey)) { - outbit = types[ibit].bits; + for (indexbit = 0, iterbit = 1; + indexbit < mansearch_keymax; + indexbit++, iterbit <<= 1) { + if (0 == strcasecmp(outkey, + mansearch_keynames[indexbit])) { + outbit = iterbit; break; } } @@ -205,7 +210,7 @@ mansearch(const struct mansearch *search, */ if (NULL == getcwd(buf, PATH_MAX)) { - perror(NULL); + perror("getcwd"); goto out; } else if (-1 == (fd = open(buf, O_RDONLY, 0))) { perror(buf); @@ -248,10 +253,12 @@ mansearch(const struct mansearch *search, */ c = sqlite3_create_function(db, "match", 2, - SQLITE_ANY, NULL, sql_match, NULL, NULL); + SQLITE_UTF8 | SQLITE_DETERMINISTIC, + NULL, sql_match, NULL, NULL); assert(SQLITE_OK == c); c = sqlite3_create_function(db, "regexp", 2, - SQLITE_ANY, NULL, sql_regexp, NULL, NULL); + SQLITE_UTF8 | SQLITE_DETERMINISTIC, + NULL, sql_regexp, NULL, NULL); assert(SQLITE_OK == c); j = 1; @@ -264,7 +271,8 @@ mansearch(const struct mansearch *search, SQL_BIND_BLOB(db, s, j, ep->regexp); } else SQL_BIND_TEXT(db, s, j, ep->substr); - SQL_BIND_INT64(db, s, j, ep->bits); + if (0 == ((TYPE_Nd | TYPE_Nm) & ep->bits)) + SQL_BIND_INT64(db, s, j, ep->bits); } memset(&htab, 0, sizeof(struct ohash)); @@ -279,21 +287,20 @@ mansearch(const struct mansearch *search, * distribution of buckets in the table. */ while (SQLITE_ROW == (c = sqlite3_step(s))) { - id = sqlite3_column_int64(s, 5); + pageid = sqlite3_column_int64(s, 2); idx = ohash_lookup_memory - (&htab, (char *)&id, - sizeof(uint64_t), (uint32_t)id); + (&htab, (char *)&pageid, + sizeof(uint64_t), (uint32_t)pageid); if (NULL != ohash_find(&htab, idx)) continue; mp = mandoc_calloc(1, sizeof(struct match)); - mp->id = id; - mp->file = mandoc_strdup - ((char *)sqlite3_column_text(s, 0)); - mp->desc = mandoc_strdup - ((char *)sqlite3_column_text(s, 3)); - mp->form = sqlite3_column_int(s, 4); + mp->pageid = pageid; + mp->form = sqlite3_column_int(s, 1); + if (TYPE_Nd == outbit) + mp->desc = mandoc_strdup( + sqlite3_column_text(s, 0)); ohash_insert(&htab, idx, mp); } @@ -303,7 +310,8 @@ mansearch(const struct mansearch *search, sqlite3_finalize(s); c = sqlite3_prepare_v2(db, - "SELECT * FROM mlinks WHERE pageid=?", + "SELECT * FROM mlinks WHERE pageid=?" + " ORDER BY sec, arch, name", -1, &s, NULL); if (SQLITE_OK != c) fprintf(stderr, "%s\n", sqlite3_errmsg(db)); @@ -323,18 +331,13 @@ mansearch(const struct mansearch *search, (*res, maxres * sizeof(struct manpage)); } mpage = *res + cur; - if (-1 == asprintf(&mpage->file, "%s/%s", - paths->paths[i], mp->file)) { - perror(0); - exit((int)MANDOCLEVEL_SYSERR); - } - mpage->desc = mp->desc; mpage->form = mp->form; - mpage->names = buildnames(db, s, mp->id); - mpage->output = outbit ? - buildoutput(db, s2, mp->id, outbit) : NULL; + buildnames(mpage, db, s, mp->pageid, + paths->paths[i], mp->form); + mpage->output = TYPE_Nd & outbit ? + mp->desc : outbit ? + buildoutput(db, s2, mp->pageid, outbit) : NULL; - free(mp->file); free(mp); cur++; } @@ -346,53 +349,113 @@ mansearch(const struct mansearch *search, } rc = 1; out: - exprfree(e); - if (-1 != fd) + if (-1 != fd) { + if (-1 == fchdir(fd)) + perror(buf); close(fd); + } + exprfree(e); free(sql); *sz = cur; return(rc); } -static char * -buildnames(sqlite3 *db, sqlite3_stmt *s, uint64_t id) +static void +buildnames(struct manpage *mpage, sqlite3 *db, sqlite3_stmt *s, + uint64_t pageid, const char *path, int form) { - char *names, *newnames; - const char *oldnames, *sep1, *name, *sec, *sep2, *arch; + char *newnames, *prevsec, *prevarch; + const char *oldnames, *sep1, *name, *sec, *sep2, *arch, *fsec; size_t i; int c; - names = NULL; + mpage->file = NULL; + mpage->names = NULL; + prevsec = prevarch = NULL; i = 1; - SQL_BIND_INT64(db, s, i, id); + SQL_BIND_INT64(db, s, i, pageid); while (SQLITE_ROW == (c = sqlite3_step(s))) { - if (NULL == names) { + + /* Decide whether we already have some names. */ + + if (NULL == mpage->names) { oldnames = ""; sep1 = ""; } else { - oldnames = names; + oldnames = mpage->names; sep1 = ", "; } - sec = sqlite3_column_text(s, 1); - arch = sqlite3_column_text(s, 2); - name = sqlite3_column_text(s, 3); - sep2 = '\0' == *arch ? "" : "/"; - if (-1 == asprintf(&newnames, "%s%s%s(%s%s%s)", - oldnames, sep1, name, sec, sep2, arch)) { - perror(0); - exit((int)MANDOCLEVEL_SYSERR); + + /* Fetch the next name. */ + + sec = sqlite3_column_text(s, 0); + arch = sqlite3_column_text(s, 1); + name = sqlite3_column_text(s, 2); + + /* If the section changed, append the old one. */ + + if (NULL != prevsec && + (strcmp(sec, prevsec) || + strcmp(arch, prevarch))) { + sep2 = '\0' == *prevarch ? "" : "/"; + mandoc_asprintf(&newnames, "%s(%s%s%s)", + oldnames, prevsec, sep2, prevarch); + free(mpage->names); + oldnames = mpage->names = newnames; + free(prevsec); + free(prevarch); + prevsec = prevarch = NULL; } - free(names); - names = newnames; + + /* Save the new section, to append it later. */ + + if (NULL == prevsec) { + prevsec = mandoc_strdup(sec); + prevarch = mandoc_strdup(arch); + } + + /* Append the new name. */ + + mandoc_asprintf(&newnames, "%s%s%s", + oldnames, sep1, name); + free(mpage->names); + mpage->names = newnames; + + /* Also save the first file name encountered. */ + + if (NULL != mpage->file) + continue; + + if (form) { + sep1 = "man"; + fsec = sec; + } else { + sep1 = "cat"; + fsec = "0"; + } + sep2 = '\0' == *arch ? "" : "/"; + mandoc_asprintf(&mpage->file, "%s/%s%s%s%s/%s.%s", + path, sep1, sec, sep2, arch, name, fsec); } if (SQLITE_DONE != c) fprintf(stderr, "%s\n", sqlite3_errmsg(db)); sqlite3_reset(s); - return(names); + + /* Append one final section to the names. */ + + if (NULL != prevsec) { + sep2 = '\0' == *prevarch ? "" : "/"; + mandoc_asprintf(&newnames, "%s(%s%s%s)", + mpage->names, prevsec, sep2, prevarch); + free(mpage->names); + mpage->names = newnames; + free(prevsec); + free(prevarch); + } } static char * -buildoutput(sqlite3 *db, sqlite3_stmt *s, uint64_t id, uint64_t outbit) +buildoutput(sqlite3 *db, sqlite3_stmt *s, uint64_t pageid, uint64_t outbit) { char *output, *newoutput; const char *oldoutput, *sep1, *data; @@ -401,7 +464,7 @@ buildoutput(sqlite3 *db, sqlite3_stmt *s, uint64_t id, output = NULL; i = 1; - SQL_BIND_INT64(db, s, i, id); + SQL_BIND_INT64(db, s, i, pageid); SQL_BIND_INT64(db, s, i, outbit); while (SQLITE_ROW == (c = sqlite3_step(s))) { if (NULL == output) { @@ -412,11 +475,8 @@ buildoutput(sqlite3 *db, sqlite3_stmt *s, uint64_t id, sep1 = " # "; } data = sqlite3_column_text(s, 1); - if (-1 == asprintf(&newoutput, "%s%s%s", - oldoutput, sep1, data)) { - perror(0); - exit((int)MANDOCLEVEL_SYSERR); - } + mandoc_asprintf(&newoutput, "%s%s%s", + oldoutput, sep1, data); free(output); output = newoutput; } @@ -492,11 +552,22 @@ sql_statement(const struct expr *e) sql_append(&sql, &sz, " OR ", 1); if (e->open) sql_append(&sql, &sz, "(", e->open); - sql_append(&sql, &sz, NULL == e->substr ? - "id IN (SELECT pageid FROM keys " - "WHERE key REGEXP ? AND bits & ?)" : - "id IN (SELECT pageid FROM keys " - "WHERE key MATCH ? AND bits & ?)", 1); + sql_append(&sql, &sz, + TYPE_Nd & e->bits + ? (NULL == e->substr + ? "desc REGEXP ?" + : "desc MATCH ?") + : TYPE_Nm == e->bits + ? (NULL == e->substr + ? "pageid IN (SELECT pageid FROM names " + "WHERE name REGEXP ?)" + : "pageid IN (SELECT pageid FROM names " + "WHERE name MATCH ?)") + : (NULL == e->substr + ? "pageid IN (SELECT pageid FROM keys " + "WHERE key REGEXP ? AND bits & ?)" + : "pageid IN (SELECT pageid FROM keys " + "WHERE key MATCH ? AND bits & ?)"), 1); if (e->close) sql_append(&sql, &sz, ")", e->close); needop = 1; @@ -513,12 +584,13 @@ sql_statement(const struct expr *e) static struct expr * exprcomp(const struct mansearch *search, int argc, char *argv[]) { + uint64_t mask; int i, toopen, logic, igncase, toclose; - struct expr *first, *next, *cur; + struct expr *first, *prev, *cur, *next; first = cur = NULL; logic = igncase = toclose = 0; - toopen = 1; + toopen = NULL != search->sec || NULL != search->arch; for (i = 0; i < argc; i++) { if (0 == strcmp("(", argv[i])) { @@ -553,21 +625,46 @@ exprcomp(const struct mansearch *search, int argc, cha next = exprterm(search, argv[i], !igncase); if (NULL == next) goto fail; - next->open = toopen; - next->and = (1 == logic); - if (NULL != first) { + if (NULL == first) + first = next; + else cur->next = next; - cur = next; - } else - cur = first = next; + prev = cur = next; + + /* + * Searching for descriptions must be split out + * because they are stored in the mpages table, + * not in the keys table. + */ + + for (mask = TYPE_Nm; mask <= TYPE_Nd; mask <<= 1) { + if (mask & cur->bits && ~mask & cur->bits) { + next = mandoc_calloc(1, + sizeof(struct expr)); + memcpy(next, cur, sizeof(struct expr)); + prev->open = 1; + cur->bits = mask; + cur->next = next; + cur = next; + cur->bits &= ~mask; + } + } + prev->and = (1 == logic); + prev->open += toopen; + if (cur != prev) + cur->close = 1; + toopen = logic = igncase = 0; } if (toopen || logic || igncase || toclose) goto fail; - cur->close++; - cur = exprspec(cur, TYPE_arch, search->arch, "^(%s|any)$"); - exprspec(cur, TYPE_sec, search->sec, "^%s$"); + if (NULL != search->sec || NULL != search->arch) + cur->close++; + if (NULL != search->arch) + cur = exprspec(cur, TYPE_arch, search->arch, "^(%s|any)$"); + if (NULL != search->sec) + exprspec(cur, TYPE_sec, search->sec, "^%s$"); return(first); @@ -585,13 +682,7 @@ exprspec(struct expr *cur, uint64_t key, const char *v char *cp; int irc; - if (NULL == value) - return(cur); - - if (-1 == asprintf(&cp, format, value)) { - perror(0); - exit((int)MANDOCLEVEL_SYSERR); - } + mandoc_asprintf(&cp, format, value); cur->next = mandoc_calloc(1, sizeof(struct expr)); cur = cur->next; cur->and = 1; @@ -612,8 +703,8 @@ exprterm(const struct mansearch *search, char *buf, in char errbuf[BUFSIZ]; struct expr *e; char *key, *v; - size_t i; - int irc; + uint64_t iterbit; + int i, irc; if ('\0' == *buf) return(NULL); @@ -642,6 +733,8 @@ exprterm(const struct mansearch *search, char *buf, in e->bits = search->deftype; if ('~' == *v++) { + if (NULL != strstr(buf, "arch")) + cs = 0; if (0 != (irc = regcomp(&e->regexp, v, REG_EXTENDED | REG_NOSUB | (cs ? 0 : REG_ICASE)))) { regerror(irc, &e->regexp, errbuf, sizeof(errbuf)); @@ -661,15 +754,22 @@ exprterm(const struct mansearch *search, char *buf, in while (NULL != (key = strsep(&buf, ","))) { if ('\0' == *key) continue; - i = 0; - while (types[i].bits && - strcasecmp(types[i].name, key)) - i++; - if (0 == types[i].bits) { - free(e); - return(NULL); + for (i = 0, iterbit = 1; + i < mansearch_keymax; + i++, iterbit <<= 1) { + if (0 == strcasecmp(key, + mansearch_keynames[i])) { + e->bits |= iterbit; + break; + } } - e->bits |= types[i].bits; + if (i == mansearch_keymax) { + if (strcasecmp(key, "any")) { + free(e); + return(NULL); + } + e->bits |= ~0ULL; + } } return(e);