=================================================================== RCS file: /cvs/mandoc/mansearch.c,v retrieving revision 1.1 retrieving revision 1.10 diff -u -p -r1.1 -r1.10 --- mandoc/mansearch.c 2012/06/08 10:36:23 1.1 +++ mandoc/mansearch.c 2013/12/27 18:51:25 1.10 @@ -1,6 +1,7 @@ -/* $Id: mansearch.c,v 1.1 2012/06/08 10:36:23 kristaps Exp $ */ +/* $Id: mansearch.c,v 1.10 2013/12/27 18:51:25 schwarze Exp $ */ /* * Copyright (c) 2012 Kristaps Dzonsons + * Copyright (c) 2013 Ingo Schwarze * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -18,11 +19,11 @@ #include "config.h" #endif -#include - #include #include #include +#include +#include #include #include #include @@ -30,19 +31,38 @@ #include #include +#ifdef HAVE_OHASH #include +#else +#include "compat_ohash.h" +#endif #include #include "mandoc.h" #include "manpath.h" -#include "mandocdb.h" #include "mansearch.h" +#define SQL_BIND_TEXT(_db, _s, _i, _v) \ + do { if (SQLITE_OK != sqlite3_bind_text \ + ((_s), (_i)++, (_v), -1, SQLITE_STATIC)) \ + fprintf(stderr, "%s\n", sqlite3_errmsg((_db))); \ + } while (0) +#define SQL_BIND_INT64(_db, _s, _i, _v) \ + do { if (SQLITE_OK != sqlite3_bind_int64 \ + ((_s), (_i)++, (_v))) \ + fprintf(stderr, "%s\n", sqlite3_errmsg((_db))); \ + } while (0) +#define SQL_BIND_BLOB(_db, _s, _i, _v) \ + do { if (SQLITE_OK != sqlite3_bind_blob \ + ((_s), (_i)++, (&_v), sizeof(_v), SQLITE_STATIC)) \ + fprintf(stderr, "%s\n", sqlite3_errmsg((_db))); \ + } while (0) + struct expr { - int glob; /* is glob? */ - uint64_t bits; /* type-mask */ - const char *v; /* search value */ - struct expr *next; /* next in sequence */ + uint64_t bits; /* type-mask */ + const char *substr; /* to search for, if applicable */ + regex_t regexp; /* compiled regexp, if applicable */ + struct expr *next; /* next in sequence */ }; struct match { @@ -104,22 +124,29 @@ static const struct type types[] = { static void *hash_alloc(size_t, void *); static void hash_free(void *, size_t, void *); static void *hash_halloc(size_t, void *); -static struct expr *exprcomp(int, char *[]); +static struct expr *exprcomp(const struct mansearch *, + int, char *[]); static void exprfree(struct expr *); -static struct expr *exprterm(char *); +static struct expr *exprterm(const struct mansearch *, char *, int); +static void sql_match(sqlite3_context *context, + int argc, sqlite3_value **argv); +static void sql_regexp(sqlite3_context *context, + int argc, sqlite3_value **argv); static char *sql_statement(const struct expr *, const char *, const char *); int -mansearch(const struct manpaths *paths, - const char *arch, const char *sec, +mansearch(const struct mansearch *search, + const struct manpaths *paths, int argc, char *argv[], struct manpage **res, size_t *sz) { - int fd, rc; + int fd, rc, c; int64_t id; - char buf[MAXPATHLEN]; - char *sql; + char buf[PATH_MAX]; + char *sql, *newnames; + const char *oldnames, *sep1, *name, *sec, *sep2, *arch; + struct manpage *mpage; struct expr *e, *ep; sqlite3 *db; sqlite3_stmt *s; @@ -136,16 +163,16 @@ mansearch(const struct manpaths *paths, info.hfree = hash_free; info.key_offset = offsetof(struct match, id); - *sz = 0; + *sz = cur = maxres = 0; sql = NULL; *res = NULL; fd = -1; e = NULL; - cur = maxres = 0; + rc = 0; if (0 == argc) goto out; - if (NULL == (e = exprcomp(argc, argv))) + if (NULL == (e = exprcomp(search, argc, argv))) goto out; /* @@ -155,7 +182,7 @@ mansearch(const struct manpaths *paths, * on our current directory from which to start the chdir(). */ - if (NULL == getcwd(buf, MAXPATHLEN)) { + if (NULL == getcwd(buf, PATH_MAX)) { perror(NULL); goto out; } else if (-1 == (fd = open(buf, O_RDONLY, 0))) { @@ -163,7 +190,7 @@ mansearch(const struct manpaths *paths, goto out; } - sql = sql_statement(e, arch, sec); + sql = sql_statement(e, search->arch, search->sec); /* * Loop over the directories (containing databases) for us to @@ -175,7 +202,6 @@ mansearch(const struct manpaths *paths, for (i = 0; i < paths->sz; i++) { if (-1 == fchdir(fd)) { - /* FIXME: will return success */ perror(buf); free(*res); break; @@ -184,30 +210,44 @@ mansearch(const struct manpaths *paths, continue; } - rc = sqlite3_open_v2 - (MANDOC_DB, &db, SQLITE_OPEN_READONLY, NULL); + c = sqlite3_open_v2 + (MANDOC_DB, &db, + SQLITE_OPEN_READONLY, NULL); - if (SQLITE_OK != rc) { + if (SQLITE_OK != c) { perror(MANDOC_DB); sqlite3_close(db); continue; } + /* + * Define the SQL functions for substring + * and regular expression matching. + */ + + c = sqlite3_create_function(db, "match", 2, + SQLITE_ANY, NULL, sql_match, NULL, NULL); + assert(SQLITE_OK == c); + c = sqlite3_create_function(db, "regexp", 2, + SQLITE_ANY, NULL, sql_regexp, NULL, NULL); + assert(SQLITE_OK == c); + j = 1; - sqlite3_prepare_v2(db, sql, -1, &s, NULL); + c = sqlite3_prepare_v2(db, sql, -1, &s, NULL); + if (SQLITE_OK != c) + fprintf(stderr, "%s\n", sqlite3_errmsg(db)); - if (NULL != arch) - sqlite3_bind_text - (s, j++, arch, -1, SQLITE_STATIC); - if (NULL != sec) - sqlite3_bind_text - (s, j++, sec, -1, SQLITE_STATIC); + if (NULL != search->arch) + SQL_BIND_TEXT(db, s, j, search->arch); + if (NULL != search->sec) + SQL_BIND_TEXT(db, s, j, search->sec); for (ep = e; NULL != ep; ep = ep->next) { - sqlite3_bind_text - (s, j++, ep->v, -1, SQLITE_STATIC); - sqlite3_bind_int64 - (s, j++, ep->bits); + if (NULL == ep->substr) { + SQL_BIND_BLOB(db, s, j, ep->regexp); + } else + SQL_BIND_TEXT(db, s, j, ep->substr); + SQL_BIND_INT64(db, s, j, ep->bits); } memset(&htab, 0, sizeof(struct ohash)); @@ -221,7 +261,7 @@ mansearch(const struct manpaths *paths, * This gives good performance and preserves the * distribution of buckets in the table. */ - while (SQLITE_ROW == sqlite3_step(s)) { + while (SQLITE_ROW == (c = sqlite3_step(s))) { id = sqlite3_column_int64(s, 0); idx = ohash_lookup_memory (&htab, (char *)&id, @@ -240,9 +280,17 @@ mansearch(const struct manpaths *paths, ohash_insert(&htab, idx, mp); } + if (SQLITE_DONE != c) + fprintf(stderr, "%s\n", sqlite3_errmsg(db)); + sqlite3_finalize(s); - sqlite3_close(db); + c = sqlite3_prepare_v2(db, + "SELECT * FROM mlinks WHERE pageid=?", + -1, &s, NULL); + if (SQLITE_OK != c) + fprintf(stderr, "%s\n", sqlite3_errmsg(db)); + for (mp = ohash_first(&htab, &idx); NULL != mp; mp = ohash_next(&htab, &idx)) { @@ -251,28 +299,94 @@ mansearch(const struct manpaths *paths, *res = mandoc_realloc (*res, maxres * sizeof(struct manpage)); } - strlcpy((*res)[cur].file, - paths->paths[i], MAXPATHLEN); - strlcat((*res)[cur].file, "/", MAXPATHLEN); - strlcat((*res)[cur].file, mp->file, MAXPATHLEN); - (*res)[cur].desc = mp->desc; - (*res)[cur].form = mp->form; + mpage = *res + cur; + if (-1 == asprintf(&mpage->file, "%s/%s", + paths->paths[i], mp->file)) { + perror(0); + exit((int)MANDOCLEVEL_SYSERR); + } + mpage->names = NULL; + mpage->desc = mp->desc; + mpage->form = mp->form; + + j = 1; + SQL_BIND_INT64(db, s, j, mp->id); + while (SQLITE_ROW == (c = sqlite3_step(s))) { + if (NULL == mpage->names) { + oldnames = ""; + sep1 = ""; + } else { + oldnames = mpage->names; + sep1 = ", "; + } + sec = sqlite3_column_text(s, 1); + arch = sqlite3_column_text(s, 2); + name = sqlite3_column_text(s, 3); + sep2 = '\0' == *arch ? "" : "/"; + if (-1 == asprintf(&newnames, + "%s%s%s(%s%s%s)", oldnames, sep1, + name, sec, sep2, arch)) { + perror(0); + exit((int)MANDOCLEVEL_SYSERR); + } + free(mpage->names); + mpage->names = newnames; + } + if (SQLITE_DONE != c) + fprintf(stderr, "%s\n", sqlite3_errmsg(db)); + sqlite3_reset(s); + free(mp->file); free(mp); cur++; } + + sqlite3_finalize(s); + sqlite3_close(db); ohash_delete(&htab); } + rc = 1; out: exprfree(e); if (-1 != fd) close(fd); free(sql); *sz = cur; - return(1); + return(rc); } /* + * Implement substring match as an application-defined SQL function. + * Using the SQL LIKE or GLOB operators instead would be a bad idea + * because that would require escaping metacharacters in the string + * being searched for. + */ +static void +sql_match(sqlite3_context *context, int argc, sqlite3_value **argv) +{ + + assert(2 == argc); + sqlite3_result_int(context, NULL != strcasestr( + (const char *)sqlite3_value_text(argv[1]), + (const char *)sqlite3_value_text(argv[0]))); +} + +/* + * Implement regular expression match + * as an application-defined SQL function. + */ +static void +sql_regexp(sqlite3_context *context, int argc, sqlite3_value **argv) +{ + + assert(2 == argc); + sqlite3_result_int(context, !regexec( + (regex_t *)sqlite3_value_blob(argv[0]), + (const char *)sqlite3_value_text(argv[1]), + 0, NULL, 0)); +} + +/* * Prepare the search SQL statement. * We search for any of the words specified in our match expression. * We filter the per-doc AND expressions when collecting results. @@ -281,26 +395,29 @@ static char * sql_statement(const struct expr *e, const char *arch, const char *sec) { char *sql; - const char *glob = "(key GLOB ? AND bits & ?)"; - const char *eq = "(key = ? AND bits & ?)"; + const char *substr = "(key MATCH ? AND bits & ?)"; + const char *regexp = "(key REGEXP ? AND bits & ?)"; const char *andarch = "arch = ? AND "; const char *andsec = "sec = ? AND "; - const size_t globsz = 27; - const size_t eqsz = 22; + size_t substrsz; + size_t regexpsz; size_t sz; sql = mandoc_strdup - ("SELECT docid,bits,key,file,desc,form,sec,arch " + ("SELECT pageid,bits,key,file,desc,form,sec,arch " "FROM keys " - "INNER JOIN docs ON docs.id=keys.docid " + "INNER JOIN mpages ON mpages.id=keys.pageid " "WHERE "); sz = strlen(sql); + substrsz = strlen(substr); + regexpsz = strlen(regexp); if (NULL != arch) { sz += strlen(andarch) + 1; sql = mandoc_realloc(sql, sz); strlcat(sql, andarch, sz); } + if (NULL != sec) { sz += strlen(andsec) + 1; sql = mandoc_realloc(sql, sz); @@ -312,10 +429,10 @@ sql_statement(const struct expr *e, const char *arch, strlcat(sql, "(", sz); for ( ; NULL != e; e = e->next) { - sz += (e->glob ? globsz : eqsz) + + sz += (NULL == e->substr ? regexpsz : substrsz) + (NULL == e->next ? 3 : 5); sql = mandoc_realloc(sql, sz); - strlcat(sql, e->glob ? glob : eq, sz); + strlcat(sql, NULL == e->substr ? regexp : substr, sz); strlcat(sql, NULL == e->next ? ");" : " OR ", sz); } @@ -328,15 +445,21 @@ sql_statement(const struct expr *e, const char *arch, * "(", "foo=bar", etc.). */ static struct expr * -exprcomp(int argc, char *argv[]) +exprcomp(const struct mansearch *search, int argc, char *argv[]) { - int i; + int i, cs; struct expr *first, *next, *cur; first = cur = NULL; for (i = 0; i < argc; i++) { - next = exprterm(argv[i]); + if (0 == strcmp("-i", argv[i])) { + if (++i >= argc) + return(NULL); + cs = 0; + } else + cs = 1; + next = exprterm(search, argv[i], cs); if (NULL == next) { exprfree(first); return(NULL); @@ -352,7 +475,7 @@ exprcomp(int argc, char *argv[]) } static struct expr * -exprterm(char *buf) +exprterm(const struct mansearch *search, char *buf, int cs) { struct expr *e; char *key, *v; @@ -363,6 +486,14 @@ exprterm(char *buf) e = mandoc_calloc(1, sizeof(struct expr)); + /*"whatis" mode uses an opaque string and default fields. */ + + if (MANSEARCH_WHATIS & search->flags) { + e->substr = buf; + e->bits = search->deftype; + return(e); + } + /* * If no =~ is specified, search with equality over names and * descriptions. @@ -370,15 +501,21 @@ exprterm(char *buf) */ if (NULL == (v = strpbrk(buf, "=~"))) { - e->v = buf; - e->bits = TYPE_Nm | TYPE_Nd; + e->substr = buf; + e->bits = search->deftype; return(e); } else if (v == buf) - e->bits = TYPE_Nm | TYPE_Nd; + e->bits = search->deftype; - e->glob = '~' == *v; - *v++ = '\0'; - e->v = v; + if ('~' == *v++) { + if (regcomp(&e->regexp, v, + REG_EXTENDED | REG_NOSUB | (cs ? 0 : REG_ICASE))) { + free(e); + return(NULL); + } + } else + e->substr = v; + v[-1] = '\0'; /* * Parse out all possible fields.