=================================================================== RCS file: /cvs/mandoc/mansearch.c,v retrieving revision 1.10 retrieving revision 1.25 diff -u -p -r1.10 -r1.25 --- mandoc/mansearch.c 2013/12/27 18:51:25 1.10 +++ mandoc/mansearch.c 2014/03/28 19:17:12 1.25 @@ -1,7 +1,7 @@ -/* $Id: mansearch.c,v 1.10 2013/12/27 18:51:25 schwarze Exp $ */ +/* $Id: mansearch.c,v 1.25 2014/03/28 19:17:12 schwarze Exp $ */ /* * Copyright (c) 2012 Kristaps Dzonsons - * Copyright (c) 2013 Ingo Schwarze + * Copyright (c) 2013, 2014 Ingo Schwarze * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -39,9 +39,13 @@ #include #include "mandoc.h" +#include "mandoc_aux.h" #include "manpath.h" #include "mansearch.h" +extern int mansearch_keymax; +extern const char *const mansearch_keynames[]; + #define SQL_BIND_TEXT(_db, _s, _i, _v) \ do { if (SQLITE_OK != sqlite3_bind_text \ ((_s), (_i)++, (_v), -1, SQLITE_STATIC)) \ @@ -62,94 +66,55 @@ struct expr { uint64_t bits; /* type-mask */ const char *substr; /* to search for, if applicable */ regex_t regexp; /* compiled regexp, if applicable */ + int open; /* opening parentheses before */ + int and; /* logical AND before */ + int close; /* closing parentheses after */ struct expr *next; /* next in sequence */ }; struct match { uint64_t id; /* identifier in database */ - char *file; /* relative filepath of manpage */ - char *desc; /* description of manpage */ int form; /* 0 == catpage */ }; -struct type { - uint64_t bits; - const char *name; -}; - -static const struct type types[] = { - { TYPE_An, "An" }, - { TYPE_Ar, "Ar" }, - { TYPE_At, "At" }, - { TYPE_Bsx, "Bsx" }, - { TYPE_Bx, "Bx" }, - { TYPE_Cd, "Cd" }, - { TYPE_Cm, "Cm" }, - { TYPE_Dv, "Dv" }, - { TYPE_Dx, "Dx" }, - { TYPE_Em, "Em" }, - { TYPE_Er, "Er" }, - { TYPE_Ev, "Ev" }, - { TYPE_Fa, "Fa" }, - { TYPE_Fl, "Fl" }, - { TYPE_Fn, "Fn" }, - { TYPE_Fn, "Fo" }, - { TYPE_Ft, "Ft" }, - { TYPE_Fx, "Fx" }, - { TYPE_Ic, "Ic" }, - { TYPE_In, "In" }, - { TYPE_Lb, "Lb" }, - { TYPE_Li, "Li" }, - { TYPE_Lk, "Lk" }, - { TYPE_Ms, "Ms" }, - { TYPE_Mt, "Mt" }, - { TYPE_Nd, "Nd" }, - { TYPE_Nm, "Nm" }, - { TYPE_Nx, "Nx" }, - { TYPE_Ox, "Ox" }, - { TYPE_Pa, "Pa" }, - { TYPE_Rs, "Rs" }, - { TYPE_Sh, "Sh" }, - { TYPE_Ss, "Ss" }, - { TYPE_St, "St" }, - { TYPE_Sy, "Sy" }, - { TYPE_Tn, "Tn" }, - { TYPE_Va, "Va" }, - { TYPE_Va, "Vt" }, - { TYPE_Xr, "Xr" }, - { ~0ULL, "any" }, - { 0ULL, NULL } -}; - +static void buildnames(struct manpage *, sqlite3 *, + sqlite3_stmt *, uint64_t, + const char *, int form); +static char *buildoutput(sqlite3 *, sqlite3_stmt *, + uint64_t, uint64_t); static void *hash_alloc(size_t, void *); static void hash_free(void *, size_t, void *); static void *hash_halloc(size_t, void *); static struct expr *exprcomp(const struct mansearch *, int, char *[]); static void exprfree(struct expr *); +static struct expr *exprspec(struct expr *, uint64_t, + const char *, const char *); static struct expr *exprterm(const struct mansearch *, char *, int); +static void sql_append(char **sql, size_t *sz, + const char *newstr, int count); static void sql_match(sqlite3_context *context, int argc, sqlite3_value **argv); static void sql_regexp(sqlite3_context *context, int argc, sqlite3_value **argv); -static char *sql_statement(const struct expr *, - const char *, const char *); +static char *sql_statement(const struct expr *); int mansearch(const struct mansearch *search, - const struct manpaths *paths, - int argc, char *argv[], + const struct manpaths *paths, + int argc, char *argv[], + const char *outkey, struct manpage **res, size_t *sz) { - int fd, rc, c; + int fd, rc, c, indexbit; int64_t id; + uint64_t outbit, iterbit; char buf[PATH_MAX]; - char *sql, *newnames; - const char *oldnames, *sep1, *name, *sec, *sep2, *arch; + char *sql; struct manpage *mpage; struct expr *e, *ep; sqlite3 *db; - sqlite3_stmt *s; + sqlite3_stmt *s, *s2; struct match *mp; struct ohash_info info; struct ohash htab; @@ -175,6 +140,19 @@ mansearch(const struct mansearch *search, if (NULL == (e = exprcomp(search, argc, argv))) goto out; + outbit = 0; + if (NULL != outkey) { + for (indexbit = 0, iterbit = 1; + indexbit < mansearch_keymax; + indexbit++, iterbit <<= 1) { + if (0 == strcasecmp(outkey, + mansearch_keynames[indexbit])) { + outbit = iterbit; + break; + } + } + } + /* * Save a descriptor to the current working directory. * Since pathnames in the "paths" variable might be relative, @@ -190,7 +168,7 @@ mansearch(const struct mansearch *search, goto out; } - sql = sql_statement(e, search->arch, search->sec); + sql = sql_statement(e); /* * Loop over the directories (containing databases) for us to @@ -237,11 +215,6 @@ mansearch(const struct mansearch *search, if (SQLITE_OK != c) fprintf(stderr, "%s\n", sqlite3_errmsg(db)); - if (NULL != search->arch) - SQL_BIND_TEXT(db, s, j, search->arch); - if (NULL != search->sec) - SQL_BIND_TEXT(db, s, j, search->sec); - for (ep = e; NULL != ep; ep = ep->next) { if (NULL == ep->substr) { SQL_BIND_BLOB(db, s, j, ep->regexp); @@ -262,7 +235,7 @@ mansearch(const struct mansearch *search, * distribution of buckets in the table. */ while (SQLITE_ROW == (c = sqlite3_step(s))) { - id = sqlite3_column_int64(s, 0); + id = sqlite3_column_int64(s, 1); idx = ohash_lookup_memory (&htab, (char *)&id, sizeof(uint64_t), (uint32_t)id); @@ -272,11 +245,7 @@ mansearch(const struct mansearch *search, mp = mandoc_calloc(1, sizeof(struct match)); mp->id = id; - mp->file = mandoc_strdup - ((char *)sqlite3_column_text(s, 3)); - mp->desc = mandoc_strdup - ((char *)sqlite3_column_text(s, 4)); - mp->form = sqlite3_column_int(s, 5); + mp->form = sqlite3_column_int(s, 0); ohash_insert(&htab, idx, mp); } @@ -286,11 +255,18 @@ mansearch(const struct mansearch *search, sqlite3_finalize(s); c = sqlite3_prepare_v2(db, - "SELECT * FROM mlinks WHERE pageid=?", + "SELECT * FROM mlinks WHERE pageid=?" + " ORDER BY sec, arch, name", -1, &s, NULL); if (SQLITE_OK != c) fprintf(stderr, "%s\n", sqlite3_errmsg(db)); + c = sqlite3_prepare_v2(db, + "SELECT * FROM keys WHERE pageid=? AND bits & ?", + -1, &s2, NULL); + if (SQLITE_OK != c) + fprintf(stderr, "%s\n", sqlite3_errmsg(db)); + for (mp = ohash_first(&htab, &idx); NULL != mp; mp = ohash_next(&htab, &idx)) { @@ -300,48 +276,18 @@ mansearch(const struct mansearch *search, (*res, maxres * sizeof(struct manpage)); } mpage = *res + cur; - if (-1 == asprintf(&mpage->file, "%s/%s", - paths->paths[i], mp->file)) { - perror(0); - exit((int)MANDOCLEVEL_SYSERR); - } - mpage->names = NULL; - mpage->desc = mp->desc; mpage->form = mp->form; + buildnames(mpage, db, s, mp->id, + paths->paths[i], mp->form); + mpage->output = outbit ? + buildoutput(db, s2, mp->id, outbit) : NULL; - j = 1; - SQL_BIND_INT64(db, s, j, mp->id); - while (SQLITE_ROW == (c = sqlite3_step(s))) { - if (NULL == mpage->names) { - oldnames = ""; - sep1 = ""; - } else { - oldnames = mpage->names; - sep1 = ", "; - } - sec = sqlite3_column_text(s, 1); - arch = sqlite3_column_text(s, 2); - name = sqlite3_column_text(s, 3); - sep2 = '\0' == *arch ? "" : "/"; - if (-1 == asprintf(&newnames, - "%s%s%s(%s%s%s)", oldnames, sep1, - name, sec, sep2, arch)) { - perror(0); - exit((int)MANDOCLEVEL_SYSERR); - } - free(mpage->names); - mpage->names = newnames; - } - if (SQLITE_DONE != c) - fprintf(stderr, "%s\n", sqlite3_errmsg(db)); - sqlite3_reset(s); - - free(mp->file); free(mp); cur++; } sqlite3_finalize(s); + sqlite3_finalize(s2); sqlite3_close(db); ohash_delete(&htab); } @@ -355,6 +301,132 @@ out: return(rc); } +static void +buildnames(struct manpage *mpage, sqlite3 *db, sqlite3_stmt *s, + uint64_t id, const char *path, int form) +{ + char *newnames, *prevsec, *prevarch; + const char *oldnames, *sep1, *name, *sec, *sep2, *arch, *fsec; + size_t i; + int c; + + mpage->file = NULL; + mpage->names = NULL; + prevsec = prevarch = NULL; + i = 1; + SQL_BIND_INT64(db, s, i, id); + while (SQLITE_ROW == (c = sqlite3_step(s))) { + + /* Decide whether we already have some names. */ + + if (NULL == mpage->names) { + oldnames = ""; + sep1 = ""; + } else { + oldnames = mpage->names; + sep1 = ", "; + } + + /* Fetch the next name. */ + + sec = sqlite3_column_text(s, 0); + arch = sqlite3_column_text(s, 1); + name = sqlite3_column_text(s, 2); + + /* If the section changed, append the old one. */ + + if (NULL != prevsec && + (strcmp(sec, prevsec) || + strcmp(arch, prevarch))) { + sep2 = '\0' == *prevarch ? "" : "/"; + mandoc_asprintf(&newnames, "%s(%s%s%s)", + oldnames, prevsec, sep2, prevarch); + free(mpage->names); + oldnames = mpage->names = newnames; + free(prevsec); + free(prevarch); + prevsec = prevarch = NULL; + } + + /* Save the new section, to append it later. */ + + if (NULL == prevsec) { + prevsec = mandoc_strdup(sec); + prevarch = mandoc_strdup(arch); + } + + /* Append the new name. */ + + mandoc_asprintf(&newnames, "%s%s%s", + oldnames, sep1, name); + free(mpage->names); + mpage->names = newnames; + + /* Also save the first file name encountered. */ + + if (NULL != mpage->file) + continue; + + if (form) { + sep1 = "man"; + fsec = sec; + } else { + sep1 = "cat"; + fsec = "0"; + } + sep2 = '\0' == *arch ? "" : "/"; + mandoc_asprintf(&mpage->file, "%s/%s%s%s%s/%s.%s", + path, sep1, sec, sep2, arch, name, fsec); + } + if (SQLITE_DONE != c) + fprintf(stderr, "%s\n", sqlite3_errmsg(db)); + sqlite3_reset(s); + + /* Append one final section to the names. */ + + if (NULL != prevsec) { + sep2 = '\0' == *prevarch ? "" : "/"; + mandoc_asprintf(&newnames, "%s(%s%s%s)", + mpage->names, prevsec, sep2, prevarch); + free(mpage->names); + mpage->names = newnames; + free(prevsec); + free(prevarch); + } +} + +static char * +buildoutput(sqlite3 *db, sqlite3_stmt *s, uint64_t id, uint64_t outbit) +{ + char *output, *newoutput; + const char *oldoutput, *sep1, *data; + size_t i; + int c; + + output = NULL; + i = 1; + SQL_BIND_INT64(db, s, i, id); + SQL_BIND_INT64(db, s, i, outbit); + while (SQLITE_ROW == (c = sqlite3_step(s))) { + if (NULL == output) { + oldoutput = ""; + sep1 = ""; + } else { + oldoutput = output; + sep1 = " # "; + } + data = sqlite3_column_text(s, 1); + mandoc_asprintf(&newoutput, "%s%s%s", + oldoutput, sep1, data); + free(output); + output = newoutput; + } + if (SQLITE_DONE != c) + fprintf(stderr, "%s\n", sqlite3_errmsg(db)); + sqlite3_reset(s); + return(output); +} + /* * Implement substring match as an application-defined SQL function. * Using the SQL LIKE or GLOB operators instead would be a bad idea @@ -386,56 +458,51 @@ sql_regexp(sqlite3_context *context, int argc, sqlite3 0, NULL, 0)); } +static void +sql_append(char **sql, size_t *sz, const char *newstr, int count) +{ + size_t newsz; + + newsz = 1 < count ? (size_t)count : strlen(newstr); + *sql = mandoc_realloc(*sql, *sz + newsz + 1); + if (1 < count) + memset(*sql + *sz, *newstr, (size_t)count); + else + memcpy(*sql + *sz, newstr, newsz); + *sz += newsz; + (*sql)[*sz] = '\0'; +} + /* * Prepare the search SQL statement. - * We search for any of the words specified in our match expression. - * We filter the per-doc AND expressions when collecting results. */ static char * -sql_statement(const struct expr *e, const char *arch, const char *sec) +sql_statement(const struct expr *e) { char *sql; - const char *substr = "(key MATCH ? AND bits & ?)"; - const char *regexp = "(key REGEXP ? AND bits & ?)"; - const char *andarch = "arch = ? AND "; - const char *andsec = "sec = ? AND "; - size_t substrsz; - size_t regexpsz; size_t sz; + int needop; - sql = mandoc_strdup - ("SELECT pageid,bits,key,file,desc,form,sec,arch " - "FROM keys " - "INNER JOIN mpages ON mpages.id=keys.pageid " - "WHERE "); + sql = mandoc_strdup("SELECT * FROM mpages WHERE "); sz = strlen(sql); - substrsz = strlen(substr); - regexpsz = strlen(regexp); - if (NULL != arch) { - sz += strlen(andarch) + 1; - sql = mandoc_realloc(sql, sz); - strlcat(sql, andarch, sz); + for (needop = 0; NULL != e; e = e->next) { + if (e->and) + sql_append(&sql, &sz, " AND ", 1); + else if (needop) + sql_append(&sql, &sz, " OR ", 1); + if (e->open) + sql_append(&sql, &sz, "(", e->open); + sql_append(&sql, &sz, NULL == e->substr ? + "id IN (SELECT pageid FROM keys " + "WHERE key REGEXP ? AND bits & ?)" : + "id IN (SELECT pageid FROM keys " + "WHERE key MATCH ? AND bits & ?)", 1); + if (e->close) + sql_append(&sql, &sz, ")", e->close); + needop = 1; } - if (NULL != sec) { - sz += strlen(andsec) + 1; - sql = mandoc_realloc(sql, sz); - strlcat(sql, andsec, sz); - } - - sz += 2; - sql = mandoc_realloc(sql, sz); - strlcat(sql, "(", sz); - - for ( ; NULL != e; e = e->next) { - sz += (NULL == e->substr ? regexpsz : substrsz) + - (NULL == e->next ? 3 : 5); - sql = mandoc_realloc(sql, sz); - strlcat(sql, NULL == e->substr ? regexp : substr, sz); - strlcat(sql, NULL == e->next ? ");" : " OR ", sz); - } - return(sql); } @@ -447,39 +514,104 @@ sql_statement(const struct expr *e, const char *arch, static struct expr * exprcomp(const struct mansearch *search, int argc, char *argv[]) { - int i, cs; + int i, toopen, logic, igncase, toclose; struct expr *first, *next, *cur; first = cur = NULL; + logic = igncase = toclose = 0; + toopen = 1; for (i = 0; i < argc; i++) { - if (0 == strcmp("-i", argv[i])) { - if (++i >= argc) - return(NULL); - cs = 0; - } else - cs = 1; - next = exprterm(search, argv[i], cs); - if (NULL == next) { - exprfree(first); - return(NULL); + if (0 == strcmp("(", argv[i])) { + if (igncase) + goto fail; + toopen++; + toclose++; + continue; + } else if (0 == strcmp(")", argv[i])) { + if (toopen || logic || igncase || NULL == cur) + goto fail; + cur->close++; + if (0 > --toclose) + goto fail; + continue; + } else if (0 == strcmp("-a", argv[i])) { + if (toopen || logic || igncase || NULL == cur) + goto fail; + logic = 1; + continue; + } else if (0 == strcmp("-o", argv[i])) { + if (toopen || logic || igncase || NULL == cur) + goto fail; + logic = 2; + continue; + } else if (0 == strcmp("-i", argv[i])) { + if (igncase) + goto fail; + igncase = 1; + continue; } + next = exprterm(search, argv[i], !igncase); + if (NULL == next) + goto fail; + next->open = toopen; + next->and = (1 == logic); if (NULL != first) { cur->next = next; cur = next; } else cur = first = next; + toopen = logic = igncase = 0; } + if (toopen || logic || igncase || toclose) + goto fail; + cur->close++; + cur = exprspec(cur, TYPE_arch, search->arch, "^(%s|any)$"); + exprspec(cur, TYPE_sec, search->sec, "^%s$"); + return(first); + +fail: + if (NULL != first) + exprfree(first); + return(NULL); } static struct expr * +exprspec(struct expr *cur, uint64_t key, const char *value, + const char *format) +{ + char errbuf[BUFSIZ]; + char *cp; + int irc; + + if (NULL == value) + return(cur); + + mandoc_asprintf(&cp, format, value); + cur->next = mandoc_calloc(1, sizeof(struct expr)); + cur = cur->next; + cur->and = 1; + cur->bits = key; + if (0 != (irc = regcomp(&cur->regexp, cp, + REG_EXTENDED | REG_NOSUB | REG_ICASE))) { + regerror(irc, &cur->regexp, errbuf, sizeof(errbuf)); + fprintf(stderr, "regcomp: %s\n", errbuf); + cur->substr = value; + } + free(cp); + return(cur); +} + +static struct expr * exprterm(const struct mansearch *search, char *buf, int cs) { + char errbuf[BUFSIZ]; struct expr *e; char *key, *v; - size_t i; + uint64_t iterbit; + int i, irc; if ('\0' == *buf) return(NULL); @@ -508,8 +640,12 @@ exprterm(const struct mansearch *search, char *buf, in e->bits = search->deftype; if ('~' == *v++) { - if (regcomp(&e->regexp, v, - REG_EXTENDED | REG_NOSUB | (cs ? 0 : REG_ICASE))) { + if (NULL != strstr(buf, "arch")) + cs = 0; + if (0 != (irc = regcomp(&e->regexp, v, + REG_EXTENDED | REG_NOSUB | (cs ? 0 : REG_ICASE)))) { + regerror(irc, &e->regexp, errbuf, sizeof(errbuf)); + fprintf(stderr, "regcomp: %s\n", errbuf); free(e); return(NULL); } @@ -525,15 +661,22 @@ exprterm(const struct mansearch *search, char *buf, in while (NULL != (key = strsep(&buf, ","))) { if ('\0' == *key) continue; - i = 0; - while (types[i].bits && - strcasecmp(types[i].name, key)) - i++; - if (0 == types[i].bits) { - free(e); - return(NULL); + for (i = 0, iterbit = 1; + i < mansearch_keymax; + i++, iterbit <<= 1) { + if (0 == strcasecmp(key, + mansearch_keynames[i])) { + e->bits |= iterbit; + break; + } } - e->bits |= types[i].bits; + if (i == mansearch_keymax) { + if (strcasecmp(key, "any")) { + free(e); + return(NULL); + } + e->bits |= ~0ULL; + } } return(e);