=================================================================== RCS file: /cvs/mandoc/preconv.c,v retrieving revision 1.1 retrieving revision 1.5 diff -u -p -r1.1 -r1.5 --- mandoc/preconv.c 2011/05/26 00:30:11 1.1 +++ mandoc/preconv.c 2011/07/24 18:15:14 1.5 @@ -1,4 +1,4 @@ -/* $Id: preconv.c,v 1.1 2011/05/26 00:30:11 kristaps Exp $ */ +/* $Id: preconv.c,v 1.5 2011/07/24 18:15:14 kristaps Exp $ */ /* * Copyright (c) 2011 Kristaps Dzonsons * @@ -18,8 +18,10 @@ #include "config.h" #endif +#ifdef HAVE_MMAP #include #include +#endif #include #include @@ -55,6 +57,7 @@ struct encode { int (*conv)(const struct buf *); }; +static int cue_enc(const struct buf *, size_t *, enum enc *); static int conv_latin_1(const struct buf *); static int conv_us_ascii(const struct buf *); static int conv_utf_8(const struct buf *); @@ -85,7 +88,7 @@ static int conv_latin_1(const struct buf *b) { size_t i; - unsigned char c; + unsigned char cu; const char *cp; cp = b->buf + (int)b->offs; @@ -94,11 +97,13 @@ conv_latin_1(const struct buf *b) * Latin-1 falls into the first 256 code-points of Unicode, so * there's no need for any sort of translation. Just make the * 8-bit characters use the Unicode escape. + * Note that binary values 128 < v < 160 are passed through + * unmodified to mandoc. */ for (i = b->offs; i < b->sz; i++) { - c = (unsigned char)*cp++; - c < 128 ? putchar(c) : printf("\\[u%.4X]", c); + cu = (unsigned char)*cp++; + cu < 160U ? putchar(cu) : printf("\\[u%.4X]", cu); } return(1); @@ -120,7 +125,106 @@ conv_us_ascii(const struct buf *b) static int conv_utf_8(const struct buf *b) { + int state, be; + unsigned int accum; + size_t i; + unsigned char cu; + const char *cp; + const long one = 1L; + cp = b->buf + (int)b->offs; + state = 0; + accum = 0U; + be = 0; + + /* Quick test for big-endian value. */ + + if ( ! (*((const char *)(&one)))) + be = 1; + + for (i = b->offs; i < b->sz; i++) { + cu = (unsigned char)*cp++; + if (state) { + if ( ! (cu & 128) || (cu & 64)) { + /* Bad sequence header. */ + return(0); + } + + /* Accept only legitimate bit patterns. */ + + if (cu > 191 || cu < 128) { + /* Bad in-sequence bits. */ + return(0); + } + + accum |= (cu & 63) << --state * 6; + + /* + * Accum is held in little-endian order as + * stipulated by the UTF-8 sequence coding. We + * need to convert to a native big-endian if our + * architecture requires it. + */ + + if (0 == state && be) + accum = (accum >> 24) | + ((accum << 8) & 0x00FF0000) | + ((accum >> 8) & 0x0000FF00) | + (accum << 24); + + if (0 == state) { + accum < 128U ? putchar(accum) : + printf("\\[u%.4X]", accum); + accum = 0U; + } + } else if (cu & (1 << 7)) { + /* + * Entering a UTF-8 state: if we encounter a + * UTF-8 bitmask, calculate the expected UTF-8 + * state from it. + */ + for (state = 0; state < 7; state++) + if ( ! (cu & (1 << (7 - state)))) + break; + + /* Accept only legitimate bit patterns. */ + + switch (state) { + case (4): + if (cu <= 244 && cu >= 240) { + accum = (cu & 7) << 18; + break; + } + /* Bad 4-sequence start bits. */ + return(0); + case (3): + if (cu <= 239 && cu >= 224) { + accum = (cu & 15) << 12; + break; + } + /* Bad 3-sequence start bits. */ + return(0); + case (2): + if (cu <= 223 && cu >= 194) { + accum = (cu & 31) << 6; + break; + } + /* Bad 2-sequence start bits. */ + return(0); + default: + /* Bad sequence bit mask. */ + return(0); + } + state--; + } else + putchar(cu); + } + + if (0 != state) { + /* Bad trailing bits. */ + return(0); + } + return(1); } @@ -142,10 +246,11 @@ static int read_whole_file(const char *f, int fd, struct buf *fb, int *with_mmap) { - struct stat st; size_t off; ssize_t ssz; +#ifdef HAVE_MMAP + struct stat st; if (-1 == fstat(fd, &st)) { perror(f); return(0); @@ -171,6 +276,7 @@ read_whole_file(const char *f, int fd, if (fb->buf != MAP_FAILED) return(1); } +#endif /* * If this isn't a regular file (like, say, stdin), then we must @@ -207,13 +313,102 @@ read_whole_file(const char *f, int fd, return(0); } +static int +cue_enc(const struct buf *b, size_t *offs, enum enc *enc) +{ + const char *ln, *eoln, *eoph; + size_t sz, phsz, nsz; + int i; + + ln = b->buf + (int)*offs; + sz = b->sz - *offs; + + /* Look for the end-of-line. */ + + if (NULL == (eoln = memchr(ln, '\n', sz))) + return(-1); + + /* Set next-line marker. */ + + *offs = (size_t)((eoln + 1) - b->buf); + + /* Check if we have the correct header/trailer. */ + + if ((sz = (size_t)(eoln - ln)) < 10 || + memcmp(ln, ".\\\" -*-", 7) || + memcmp(eoln - 3, "-*-", 3)) + return(0); + + /* Move after the header and adjust for the trailer. */ + + ln += 7; + sz -= 10; + + while (sz > 0) { + while (sz > 0 && ' ' == *ln) { + ln++; + sz--; + } + if (0 == sz) + break; + + /* Find the end-of-phrase marker (or eoln). */ + + if (NULL == (eoph = memchr(ln, ';', sz))) + eoph = eoln - 3; + else + eoph++; + + /* Only account for the "coding" phrase. */ + + if ((phsz = (size_t)(eoph - ln)) < 7 || + strncasecmp(ln, "coding:", 7)) { + sz -= phsz; + ln += phsz; + continue; + } + + sz -= 7; + ln += 7; + + while (sz > 0 && ' ' == *ln) { + ln++; + sz--; + } + if (0 == sz) + break; + + /* Check us against known encodings. */ + + for (i = 0; i < (int)ENC__MAX; i++) { + nsz = strlen(encs[i].name); + if (phsz < nsz) + continue; + if (strncasecmp(ln, encs[i].name, nsz)) + continue; + + *enc = (enum enc)i; + return(1); + } + + /* Unknown encoding. */ + + *enc = ENC__MAX; + return(1); + } + + return(0); +} + int main(int argc, char *argv[]) { int i, ch, map, fd, rc; - struct buf buf; + struct buf b; const char *fn; enum enc enc, def; + unsigned char bom[3] = { 0xEF, 0xBB, 0xBF }; + size_t offs; extern int optind; extern char *optarg; @@ -229,19 +424,19 @@ main(int argc, char *argv[]) enc = def = ENC__MAX; map = 0; - memset(&buf, 0, sizeof(struct buf)); + memset(&b, 0, sizeof(struct buf)); while (-1 != (ch = getopt(argc, argv, "D:e:rdvh"))) switch (ch) { case ('D'): /* FALLTHROUGH */ case ('e'): - for (i = 0; i < ENC__MAX; i++) { + for (i = 0; i < (int)ENC__MAX; i++) { if (strcasecmp(optarg, encs[i].name)) continue; break; } - if (i < ENC__MAX) { + if (i < (int)ENC__MAX) { if ('D' == ch) def = (enum enc)i; else @@ -283,11 +478,24 @@ main(int argc, char *argv[]) } } - if ( ! read_whole_file(fn, fd, &buf, &map)) + if ( ! read_whole_file(fn, fd, &b, &map)) goto out; + /* Try to read the UTF-8 BOM. */ + + if (ENC__MAX == enc) + if (b.sz > 3 && 0 == memcmp(b.buf, bom, 3)) { + b.offs = 3; + enc = ENC_UTF_8; + } + + /* Try reading from the "-*-" cue. */ + if (ENC__MAX == enc) { - /* TODO: search for BOM. */ + offs = b.offs; + ch = cue_enc(&b, &offs, &enc); + if (0 == ch) + ch = cue_enc(&b, &offs, &enc); } /* @@ -299,15 +507,19 @@ main(int argc, char *argv[]) if (ENC__MAX == enc) enc = ENC__MAX == def ? ENC_LATIN_1 : def; - if ( ! (*encs[(int)enc].conv)(&buf)) + if ( ! (*encs[(int)enc].conv)(&b)) { + fprintf(stderr, "%s: Bad encoding\n", fn); goto out; + } rc = EXIT_SUCCESS; out: +#ifdef HAVE_MMAP if (map) - munmap(buf.buf, buf.sz); + munmap(b.buf, b.sz); else - free(buf.buf); +#endif + free(b.buf); if (fd > STDIN_FILENO) close(fd);