mandoc/mandocdb.c - diff

Return to mandocdb.c CVS log

Up to [cvsweb.bsd.lv] / mandoc

Diff for /mandoc/mandocdb.c between version 1.62 and 1.67

version 1.62, 2013/06/06 02:40:37

version 1.67, 2013/06/07 04:51:26

Line 23

#include <assert.h>

#include <ctype.h>

#include <errno.h>

#include <fcntl.h>

#include <fts.h>

#include <getopt.h>

Line 83 enum form {

Line 84 enum form {

struct str {

char *utf8; /* key in UTF-8 form */

const struct of *of; /* if set, the owning parse */

struct str *next; /* next in owning parse sequence */

uint64_t mask; /* bitmask in sequence */

char key[]; /* the string itself */

};

Line 99 struct of {

enum form dform; /* path-cued form */

enum form sform; /* suffix-cued form */

char file[PATH_MAX]; /* filename rel. to manpath */

const char *desc; /* parsed description */

char *desc; /* parsed description */

const char *sec; /* suffix-cued section (or empty) */

char *name; /* name (from filename) (not empty) */

const char *dsec; /* path-cued section (or empty) */

char *sec; /* suffix-cued section (or empty) */

const char *arch; /* path-cued arch. (or empty) */

char *dsec; /* path-cued section (or empty) */

const char *name; /* name (from filename) (not empty) */

char *arch; /* path-cued arch. (or empty) */

};

enum stmt {

Line 129 static void dbprune(void);

static void fileadd(struct of *);

static int filecheck(const char *);

static void filescan(const char *);

static struct str *hashget(const char *, size_t);

static void *hash_alloc(size_t, void *);

static void hash_free(void *, size_t, void *);

static void *hash_halloc(size_t, void *);

Line 138 static int inocheck(const struct stat *);

Line 137 static int inocheck(const struct stat *);

static void ofadd(int, const char *, const char *, const char *,

const char *, const char *, const struct stat *);

static void offree(void);

static void ofmerge(struct mchars *, struct mparse *);

static void ofmerge(struct mchars *, struct mparse *,

struct ohash_info*);

static void parse_catpage(struct of *);

static void parse_man(struct of *, const struct man_node *);

static void parse_mdoc(struct of *, const struct mdoc_node *);

Line 156 static int set_basedir(const char *);

static void putkey(const struct of *,

const char *, uint64_t);

static void putkeys(const struct of *,

const char *, int, uint64_t);

const char *, size_t, uint64_t);

static void putmdockey(const struct of *,

const struct mdoc_node *, uint64_t);

static void say(const char *, const char *, ...);

static char *stradd(const char *);

static char *straddbuf(const char *, size_t);

static int treescan(void);

static size_t utf8(unsigned int, char [7]);

static void utf8key(struct mchars *, struct str *);

static void wordaddbuf(const struct of *,

const char *, size_t, uint64_t);

static char *progname;

static int use_all; /* use all found files */

Line 180 static struct ohash inos; /* table of inodes/devices

Line 176 static struct ohash inos; /* table of inodes/devices

static struct ohash filenames; /* table of filenames */

static struct ohash strings; /* table of all strings */

static struct of *ofs = NULL; /* vector of files to parse */

static struct str *words = NULL; /* word list in current parse */

static sqlite3 *db = NULL; /* current database */

static sqlite3_stmt *stmts[STMT__MAX]; /* current statements */

Line 313 int

Line 308 int

main(int argc, char *argv[])

{

int ch, i;

unsigned int index;

size_t j, sz;

const char *path_arg;

struct str *s;

struct mchars *mc;

struct manpaths dirs;

struct mparse *mp;

Line 408 main(int argc, char *argv[])

Line 401 main(int argc, char *argv[])

MANDOCLEVEL_FATAL, NULL, NULL, NULL);

mc = mchars_alloc();

ohash_init(&strings, 6, &str_info);

ohash_init(&inos, 6, &ino_info);

ohash_init(&filenames, 6, &filename_info);

Line 432 main(int argc, char *argv[])

Line 424 main(int argc, char *argv[])

if (OP_TEST != op)

dbprune();

if (OP_DELETE != op)

ofmerge(mc, mp);

ofmerge(mc, mp, &str_info);

dbclose(1);

} else {

Line 462 main(int argc, char *argv[])

Line 454 main(int argc, char *argv[])

dirs.paths[j][--sz] = '\0';

if (0 == sz)

continue;

if (j) {

ohash_init(&inos, 6, &ino_info);

ohash_init(&filenames, 6, &filename_info);

}

if (0 == set_basedir(dirs.paths[j]))

goto out;

if (0 == treescan())

Line 480 main(int argc, char *argv[])

Line 478 main(int argc, char *argv[])

SQL_EXEC("PRAGMA synchronous = OFF");

#endif

ofmerge(mc, mp);

ofmerge(mc, mp, &str_info);

dbclose(0);

offree();

ohash_delete(&inos);

if (j + 1 < dirs.sz) {

ohash_init(&inos, 6, &ino_info);

ohash_delete(&inos);

ohash_delete(&filenames);

ohash_init(&filenames, 6, &filename_info);

offree();

}

out:

Line 494 out:

Line 493 out:

manpath_free(&dirs);

mchars_free(mc);

mparse_free(mp);

for (s = ohash_first(&strings, &index);

NULL != s; s = ohash_next(&strings, &index)) {

if (s->utf8 != s->key)

free(s->utf8);

free(s);

}

ohash_delete(&strings);

ohash_delete(&inos);

ohash_delete(&filenames);

offree();

Line 538 treescan(void)

Line 530 treescan(void)

FTSENT *ff;

int dform;

char *sec;

const char *dsec, *arch, *cp, *name, *path;

const char *dsec, *arch, *cp, *path;

const char *argv[2];

argv[0] = ".";

Line 606 treescan(void)

Line 598 treescan(void)

if (warnings)

say(path, "Wrong filename suffix");

continue;

} else {

} else

sec[-1] = '\0';

sec = stradd(sec);

ofadd(dform, path, ff->fts_name, dsec, sec,

}

arch, ff->fts_statp);

name = stradd(ff->fts_name);

ofadd(dform, path,

name, dsec, sec, arch, ff->fts_statp);

continue;

} else if (FTS_D != ff->fts_info &&

FTS_DP != ff->fts_info) {

Line 639 treescan(void)

Line 628 treescan(void)

if (0 == strncmp(cp, "man", 3)) {

dform = FORM_SRC;

dsec = stradd(cp + 3);

dsec = cp + 3;

} else if (0 == strncmp(cp, "cat", 3)) {

dform = FORM_CAT;

dsec = stradd(cp + 3);

dsec = cp + 3;

}

if (NULL != dsec || use_all)

Line 659 treescan(void)

Line 648 treescan(void)

arch = NULL;

if (FTS_DP != ff->fts_info && NULL != dsec)

arch = stradd(ff->fts_name);

arch = ff->fts_name;

break;

default:

if (FTS_DP == ff->fts_info || use_all)

Line 785 filescan(const char *file)

Line 774 filescan(const char *file)

static int

filecheck(const char *name)

{

unsigned int index;

index = ohash_qlookup(&filenames, name);

return(NULL != ohash_find(&filenames,

return(NULL != ohash_find(&filenames, index));

ohash_qlookup(&filenames, name)));

}

Line 798 filecheck(const char *name)

Line 786 filecheck(const char *name)

static void

fileadd(struct of *of)

{

unsigned int index;

unsigned int slot;

index = ohash_qlookup(&filenames, of->file);

slot = ohash_qlookup(&filenames, of->file);

assert(NULL == ohash_find(&filenames, index));

assert(NULL == ohash_find(&filenames, slot));

ohash_insert(&filenames, index, of);

ohash_insert(&filenames, slot, of);

}

Line 813 inocheck(const struct stat *st)

Line 801 inocheck(const struct stat *st)

{

struct id id;

uint32_t hash;

unsigned int index;

memset(&id, 0, sizeof(id));

id.ino = hash = st->st_ino;

id.dev = st->st_dev;

index = ohash_lookup_memory

(&inos, (char *)&id, sizeof(id), hash);

return(NULL != ohash_find(&inos, index));

return(NULL != ohash_find(&inos, ohash_lookup_memory(

&inos, (char *)&id, sizeof(id), hash)));

}

Line 833 static void

Line 819 static void

inoadd(const struct stat *st, struct of *of)

{

uint32_t hash;

unsigned int index;

unsigned int slot;

of->id.ino = hash = st->st_ino;

of->id.dev = st->st_dev;

index = ohash_lookup_memory

slot = ohash_lookup_memory

(&inos, (char *)&of->id, sizeof(of->id), hash);

assert(NULL == ohash_find(&inos, index));

assert(NULL == ohash_find(&inos, slot));

ohash_insert(&inos, index, of);

ohash_insert(&inos, slot, of);

}

static void

Line 872 ofadd(int dform, const char *file, const char *name, c

Line 858 ofadd(int dform, const char *file, const char *name, c

of = mandoc_calloc(1, sizeof(struct of));

strlcpy(of->file, file, PATH_MAX);

of->name = name;

of->name = mandoc_strdup(name);

of->sec = sec;

of->sec = mandoc_strdup(sec);

of->dsec = dsec;

of->dsec = mandoc_strdup(dsec);

of->arch = arch;

of->arch = mandoc_strdup(arch);

of->sform = sform;

of->dform = dform;

of->next = ofs;

Line 897 offree(void)

Line 883 offree(void)

while (NULL != (of = ofs)) {

ofs = of->next;

free(of->name);

free(of->sec);

free(of->dsec);

free(of->arch);

free(of);

}

Line 909 offree(void)

Line 899 offree(void)

* and filename to determine whether the file is parsable or not.

static void

ofmerge(struct mchars *mc, struct mparse *mp)

ofmerge(struct mchars *mc, struct mparse *mp,

struct ohash_info *infop)

{

int form;

size_t sz;

Line 950 ofmerge(struct mchars *mc, struct mparse *mp)

Line 941 ofmerge(struct mchars *mc, struct mparse *mp)

}

words = NULL;

ohash_init(&strings, 6, infop);

mparse_reset(mp);

mdoc = NULL;

man = NULL;

Line 1028 ofmerge(struct mchars *mc, struct mparse *mp)

Line 1019 ofmerge(struct mchars *mc, struct mparse *mp)

if (NULL != mdoc) {

if (NULL != (cp = mdoc_meta(mdoc)->name))

putkey(of, cp, TYPE_Nm);

assert(NULL == of->desc);

parse_mdoc(of, mdoc_node(mdoc));

putkey(of, NULL != of->desc ?

of->desc : of->name, TYPE_Nd);

} else if (NULL != man)

parse_man(of, man_node(man));

else

parse_catpage(of);

dbindex(mc, form, of);

ohash_delete(&strings);

}

Line 1101 parse_catpage(struct of *of)

Line 1096 parse_catpage(struct of *of)

if (NULL == title || '\0' == *title) {

if (warnings)

say(of->file, "Cannot find NAME section");

assert(NULL == of->desc);

of->desc = mandoc_strdup(of->name);

putkey(of, of->name, TYPE_Nd);

fclose(stream);

free(title);

Line 1139 parse_catpage(struct of *of)

Line 1136 parse_catpage(struct of *of)

plen -= 2;

}

of->desc = stradd(p);

assert(NULL == of->desc);

putkey(of, p, TYPE_Nd);

of->desc = mandoc_strdup(p);

putkey(of, of->desc, TYPE_Nd);

fclose(stream);

free(title);

}

Line 1153 putkey(const struct of *of, const char *value, uint64_

Line 1151 putkey(const struct of *of, const char *value, uint64_

{

assert(NULL != value);

wordaddbuf(of, value, strlen(value), type);

putkeys(of, value, strlen(value), type);

}

* Like putkey() but for unterminated strings.

static void

putkeys(const struct of *of, const char *value, int sz, uint64_t type)

{

wordaddbuf(of, value, sz, type);

}

* Grok all nodes at or below a certain mdoc node into putkey().

static void

Line 1314 parse_man(struct of *of, const struct man_node *n)

Line 1302 parse_man(struct of *of, const struct man_node *n)

start++;

assert(NULL == of->desc);

of->desc = stradd(start);

of->desc = mandoc_strdup(start);

putkey(of, start, TYPE_Nd);

putkey(of, of->desc, TYPE_Nd);

free(title);

return;

}

Line 1458 parse_mdoc_St(struct of *of, const struct mdoc_node *n

Line 1446 parse_mdoc_St(struct of *of, const struct mdoc_node *n

static int

parse_mdoc_Xr(struct of *of, const struct mdoc_node *n)

{

char *cp;

if (NULL == (n = n->child))

return(0);

putkey(of, n->string, TYPE_Xr);

if (NULL == n->next) {

return(1);

putkey(of, n->string, TYPE_Xr);

return(0);

}

if (-1 == asprintf(&cp, "%s(%s)", n->string, n->next->string)) {

perror(NULL);

exit((int)MANDOCLEVEL_SYSERR);

}

putkey(of, cp, TYPE_Xr);

free(cp);

return(0);

}

static int

parse_mdoc_Nd(struct of *of, const struct mdoc_node *n)

{

size_t sz;

char *sv, *desc;

if (MDOC_BODY != n->type)

return(0);

Line 1480 parse_mdoc_Nd(struct of *of, const struct mdoc_node *n

Line 1478 parse_mdoc_Nd(struct of *of, const struct mdoc_node *n

* into the document table.

desc = NULL;

for (n = n->child; NULL != n; n = n->next) {

if (MDOC_TEXT == n->type) {

sz = strlen(n->string) + 1;

if (NULL != of->desc) {

if (NULL != (sv = desc))

sz = strlen(of->desc) +

sz += strlen(desc) + 1;

strlen(n->string) + 2;

desc = mandoc_realloc(desc, sz);

of->desc = mandoc_realloc(of->desc, sz);

if (NULL != sv)

strlcat(of->desc, " ", sz);

strlcat(desc, " ", sz);

strlcat(of->desc, n->string, sz);

else

} else

*desc = '\0';

of->desc = mandoc_strdup(n->string);

strlcat(desc, n->string, sz);

}

if (NULL != n->child)

parse_mdoc_Nd(of, n);

}

of->desc = NULL != desc ? stradd(desc) : NULL;

free(desc);

return(1);

}

Line 1536 parse_mdoc_body(struct of *of, const struct mdoc_node

Line 1529 parse_mdoc_body(struct of *of, const struct mdoc_node

}

* See straddbuf().

* Add a string to the hash table for the current manual.

* Each string has a bitmask telling which macros it belongs to.

* When we finish the manual, we'll dump the table.

static char *

stradd(const char *cp)

{

return(straddbuf(cp, strlen(cp)));

}

* This looks up or adds a string to the string table.

* The string table is a table of all strings encountered during parse

* or file scan.

* In using it, we avoid having thousands of (e.g.) "cat1" string

* allocations for the "of" table.

* We also have a layer atop the string table for keeping track of words

* in a parse sequence (see wordaddbuf()).

static char *

straddbuf(const char *cp, size_t sz)

{

struct str *s;

unsigned int index;

const char *end;

if (NULL != (s = hashget(cp, sz)))

return(s->key);

s = mandoc_calloc(sizeof(struct str) + sz + 1, 1);

memcpy(s->key, cp, sz);

end = cp + sz;

index = ohash_qlookupi(&strings, cp, &end);

assert(NULL == ohash_find(&strings, index));

ohash_insert(&strings, index, s);

return(s->key);

}

static struct str *

hashget(const char *cp, size_t sz)

{

unsigned int index;

const char *end;

end = cp + sz;

index = ohash_qlookupi(&strings, cp, &end);

return(ohash_find(&strings, index));

}

* Add a word to the current parse sequence.

* Within the hashtable of strings, we maintain a list of strings that

* are currently indexed.

* Each of these ("words") has a bitmask modified within the parse.

* When we finish a parse, we'll dump the list, then remove the head

* entry -- since the next parse will have a new "of", it can keep track

* of its entries without conflict.

static void

wordaddbuf(const struct of *of,

putkeys(const struct of *of, const char *cp, size_t sz, uint64_t v)

const char *cp, size_t sz, uint64_t v)

{

struct str *s;

unsigned int index;

unsigned int slot;

const char *end;

if (0 == sz)

return;

s = hashget(cp, sz);

end = cp + sz;

slot = ohash_qlookupi(&strings, cp, &end);

s = ohash_find(&strings, slot);

if (NULL != s && of == s->of) {

s->mask |= v;

Line 1613 wordaddbuf(const struct of *of,

Line 1553 wordaddbuf(const struct of *of,

} else if (NULL == s) {

s = mandoc_calloc(sizeof(struct str) + sz + 1, 1);

memcpy(s->key, cp, sz);

end = cp + sz;

ohash_insert(&strings, slot, s);

index = ohash_qlookupi(&strings, cp, &end);

assert(NULL == ohash_find(&strings, index));

ohash_insert(&strings, index, s);

}

s->next = words;

s->of = of;

s->mask = v;

words = s;

}

Line 1794 dbindex(struct mchars *mc, int form, const struct of *

Line 1728 dbindex(struct mchars *mc, int form, const struct of *

const char *desc;

int64_t recno;

size_t i;

unsigned int slot;

if (verb)

say(of->file, "Adding to index");

Line 1803 dbindex(struct mchars *mc, int form, const struct of *

Line 1738 dbindex(struct mchars *mc, int form, const struct of *

desc = "";

if (NULL != of->desc) {

key = hashget(of->desc, strlen(of->desc));

key = ohash_find(&strings,

ohash_qlookup(&strings, of->desc));

assert(NULL != key);

if (NULL == key->utf8)

utf8key(mc, key);

Line 1822 dbindex(struct mchars *mc, int form, const struct of *

Line 1758 dbindex(struct mchars *mc, int form, const struct of *

recno = sqlite3_last_insert_rowid(db);

sqlite3_reset(stmts[STMT_INSERT_DOC]);

for (key = words; NULL != key; key = key->next) {

for (key = ohash_first(&strings, &slot); NULL != key;

key = ohash_next(&strings, &slot)) {

assert(key->of == of);

if (NULL == key->utf8)

utf8key(mc, key);

Line 1832 dbindex(struct mchars *mc, int form, const struct of *

Line 1769 dbindex(struct mchars *mc, int form, const struct of *

SQL_BIND_INT64(stmts[STMT_INSERT_KEY], i, recno);

SQL_STEP(stmts[STMT_INSERT_KEY]);

sqlite3_reset(stmts[STMT_INSERT_KEY]);

if (key->utf8 != key->key)

free(key->utf8);

free(key);

}

SQL_EXEC("END TRANSACTION");

Line 1864 static void

Line 1804 static void

dbclose(int real)

{

size_t i;

char file[PATH_MAX];

if (nodb)

return;

Line 1880 dbclose(int real)

Line 1819 dbclose(int real)

if (real)

return;

strlcpy(file, MANDOC_DB, PATH_MAX);

if (-1 == rename(MANDOC_DB "~", MANDOC_DB)) {

strlcat(file, "~", PATH_MAX);

if (-1 == rename(file, MANDOC_DB)) {

exitcode = (int)MANDOCLEVEL_SYSERR;

say(MANDOC_DB, NULL);

}

Line 1899 dbclose(int real)

Line 1836 dbclose(int real)

static int

dbopen(int real)

{

char file[PATH_MAX];

const char *file, *sql;

const char *sql;

int rc, ofl;

size_t sz;

if (nodb)

return(1);

sz = strlcpy(file, MANDOC_DB, PATH_MAX);

ofl = SQLITE_OPEN_READWRITE;

if ( ! real)

if (0 == real) {

sz = strlcat(file, "~", PATH_MAX);

file = MANDOC_DB "~";

if (-1 == remove(file) && ENOENT != errno) {

if (sz >= PATH_MAX) {

exitcode = (int)MANDOCLEVEL_SYSERR;

fprintf(stderr, "%s: Path too long\n", file);

say(file, NULL);

return(0);

}

ofl |= SQLITE_OPEN_EXCLUSIVE;

if ( ! real)

} else

remove(file);

file = MANDOC_DB;

ofl = SQLITE_OPEN_READWRITE |

(0 == real ? SQLITE_OPEN_EXCLUSIVE : 0);

rc = sqlite3_open_v2(file, &db, ofl, NULL);

if (SQLITE_OK == rc)

CVSweb