=================================================================== RCS file: /cvs/mandoc/preconv.c,v retrieving revision 1.4 retrieving revision 1.17 diff -u -p -r1.4 -r1.17 --- mandoc/preconv.c 2011/05/26 21:13:07 1.4 +++ mandoc/preconv.c 2018/12/13 11:55:47 1.17 @@ -1,6 +1,7 @@ -/* $Id: preconv.c,v 1.4 2011/05/26 21:13:07 kristaps Exp $ */ +/* $Id: preconv.c,v 1.17 2018/12/13 11:55:47 schwarze Exp $ */ /* * Copyright (c) 2011 Kristaps Dzonsons + * Copyright (c) 2014 Ingo Schwarze * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,326 +15,118 @@ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -#ifdef HAVE_CONFIG_H #include "config.h" -#endif -#include -#include +#include #include -#include #include -#include #include -#include -/* - * The read_whole_file() and resize_buf() functions are copied from - * read.c, including all dependency code (MAP_FILE, etc.). - */ +#include "mandoc.h" +#include "roff.h" +#include "mandoc_parse.h" +#include "libmandoc.h" -#ifndef MAP_FILE -#define MAP_FILE 0 -#endif - -enum enc { - ENC_UTF_8, /* UTF-8 */ - ENC_US_ASCII, /* US-ASCII */ - ENC_LATIN_1, /* Latin-1 */ - ENC__MAX -}; - -struct buf { - char *buf; /* binary input buffer */ - size_t sz; /* size of binary buffer */ - size_t offs; /* starting buffer offset */ -}; - -struct encode { - const char *name; - int (*conv)(const struct buf *); -}; - -static int cue_enc(const struct buf *, size_t *, enum enc *); -static int conv_latin_1(const struct buf *); -static int conv_us_ascii(const struct buf *); -static int conv_utf_8(const struct buf *); -static int read_whole_file(const char *, int, - struct buf *, int *); -static void resize_buf(struct buf *, size_t); -static void usage(void); - -static const struct encode encs[ENC__MAX] = { - { "utf-8", conv_utf_8 }, /* ENC_UTF_8 */ - { "us-ascii", conv_us_ascii }, /* ENC_US_ASCII */ - { "latin-1", conv_latin_1 }, /* ENC_LATIN_1 */ -}; - -static const char *progname; - -static void -usage(void) +int +preconv_encode(const struct buf *ib, size_t *ii, struct buf *ob, size_t *oi, + int *filenc) { + const unsigned char *cu; + int nby; + unsigned int accum; - fprintf(stderr, "usage: %s " - "[-D enc] " - "[-e ENC] " - "[file]\n", progname); -} + cu = (const unsigned char *)ib->buf + *ii; + assert(*cu & 0x80); -static int -conv_latin_1(const struct buf *b) -{ - size_t i; - unsigned char cu; - const char *cp; + if ( ! (*filenc & MPARSE_UTF8)) + goto latin; - cp = b->buf + (int)b->offs; + nby = 1; + while (nby < 5 && *cu & (1 << (7 - nby))) + nby++; - /* - * Latin-1 falls into the first 256 code-points of Unicode, so - * there's no need for any sort of translation. Just make the - * 8-bit characters use the Unicode escape. - * Note that binary values 128 < v < 160 are passed through - * unmodified to mandoc. - */ - - for (i = b->offs; i < b->sz; i++) { - cu = (unsigned char)*cp++; - cu < 160U ? putchar(cu) : printf("\\[u%.4X]", cu); + switch (nby) { + case 2: + accum = *cu & 0x1f; + if (accum < 0x02) /* Obfuscated ASCII. */ + goto latin; + break; + case 3: + accum = *cu & 0x0f; + break; + case 4: + accum = *cu & 0x07; + if (accum > 0x04) /* Beyond Unicode. */ + goto latin; + break; + default: /* Bad sequence header. */ + goto latin; } - return(1); -} - -static int -conv_us_ascii(const struct buf *b) -{ - - /* - * US-ASCII has no conversion since it falls into the first 128 - * bytes of Unicode. - */ - - fwrite(b->buf, 1, b->sz, stdout); - return(1); -} - -static int -conv_utf_8(const struct buf *b) -{ - int state, be; - unsigned int accum; - size_t i; - unsigned char cu; - const char *cp; - const long one = 1L; - - cp = b->buf + (int)b->offs; - state = 0; - accum = 0U; - be = 0; - - /* Quick test for big-endian value. */ - - if ( ! (*((const char *)(&one)))) - be = 1; - - for (i = b->offs; i < b->sz; i++) { - cu = (unsigned char)*cp++; - if (state) { - if ( ! (cu & 128) || (cu & 64)) { - /* Bad sequence header. */ - return(0); - } - - /* Accept only legitimate bit patterns. */ - - if (cu > 191 || cu < 128) { - /* Bad in-sequence bits. */ - return(0); - } - - accum |= (cu & 63) << --state * 6; - - /* - * Accum is held in little-endian order as - * stipulated by the UTF-8 sequence coding. We - * need to convert to a native big-endian if our - * architecture requires it. - */ - - if (0 == state && be) - accum = (accum >> 24) | - ((accum << 8) & 0x00FF0000) | - ((accum >> 8) & 0x0000FF00) | - (accum << 24); - - if (0 == state) { - accum < 128U ? putchar(accum) : - printf("\\[u%.4X]", accum); - accum = 0U; - } - } else if (cu & (1 << 7)) { - /* - * Entering a UTF-8 state: if we encounter a - * UTF-8 bitmask, calculate the expected UTF-8 - * state from it. - */ - for (state = 0; state < 7; state++) - if ( ! (cu & (1 << (7 - state)))) - break; - - /* Accept only legitimate bit patterns. */ - - switch (state) { - case (4): - if (cu <= 244 && cu >= 240) { - accum = (cu & 7) << 18; - break; - } - /* Bad 4-sequence start bits. */ - return(0); - case (3): - if (cu <= 239 && cu >= 224) { - accum = (cu & 15) << 12; - break; - } - /* Bad 3-sequence start bits. */ - return(0); - case (2): - if (cu <= 223 && cu >= 194) { - accum = (cu & 31) << 6; - break; - } - /* Bad 2-sequence start bits. */ - return(0); - default: - /* Bad sequence bit mask. */ - return(0); - } - state--; - } else - putchar(cu); + cu++; + switch (nby) { + case 3: + if ((accum == 0x00 && ! (*cu & 0x20)) || /* Use 2-byte. */ + (accum == 0x0d && *cu & 0x20)) /* Surrogates. */ + goto latin; + break; + case 4: + if ((accum == 0x00 && ! (*cu & 0x30)) || /* Use 3-byte. */ + (accum == 0x04 && *cu & 0x30)) /* Beyond Unicode. */ + goto latin; + break; + default: + break; } - if (0 != state) { - /* Bad trailing bits. */ - return(0); + while (--nby) { + if ((*cu & 0xc0) != 0x80) /* Invalid continuation. */ + goto latin; + accum <<= 6; + accum += *cu & 0x3f; + cu++; } - return(1); -} + assert(accum > 0x7f); + assert(accum < 0x110000); + assert(accum < 0xd800 || accum > 0xdfff); -static void -resize_buf(struct buf *buf, size_t initial) -{ + *oi += snprintf(ob->buf + *oi, 11, "\\[u%.4X]", accum); + *ii = (const char *)cu - ib->buf; + *filenc &= ~MPARSE_LATIN1; + return 1; - buf->sz = buf->sz > initial / 2 ? - 2 * buf->sz : initial; +latin: + if ( ! (*filenc & MPARSE_LATIN1)) + return 0; - buf->buf = realloc(buf->buf, buf->sz); - if (NULL == buf->buf) { - perror(NULL); - exit(EXIT_FAILURE); - } -} + *oi += snprintf(ob->buf + *oi, 11, + "\\[u%.4X]", (unsigned char)ib->buf[(*ii)++]); -static int -read_whole_file(const char *f, int fd, - struct buf *fb, int *with_mmap) -{ - struct stat st; - size_t off; - ssize_t ssz; - - if (-1 == fstat(fd, &st)) { - perror(f); - return(0); - } - - /* - * If we're a regular file, try just reading in the whole entry - * via mmap(). This is faster than reading it into blocks, and - * since each file is only a few bytes to begin with, I'm not - * concerned that this is going to tank any machines. - */ - - if (S_ISREG(st.st_mode) && st.st_size >= (1U << 31)) { - fprintf(stderr, "%s: input too large\n", f); - return(0); - } - - if (S_ISREG(st.st_mode)) { - *with_mmap = 1; - fb->sz = (size_t)st.st_size; - fb->buf = mmap(NULL, fb->sz, PROT_READ, - MAP_FILE|MAP_SHARED, fd, 0); - if (fb->buf != MAP_FAILED) - return(1); - } - - /* - * If this isn't a regular file (like, say, stdin), then we must - * go the old way and just read things in bit by bit. - */ - - *with_mmap = 0; - off = 0; - fb->sz = 0; - fb->buf = NULL; - for (;;) { - if (off == fb->sz && fb->sz == (1U << 31)) { - fprintf(stderr, "%s: input too large\n", f); - break; - } - - if (off == fb->sz) - resize_buf(fb, 65536); - - ssz = read(fd, fb->buf + (int)off, fb->sz - off); - if (ssz == 0) { - fb->sz = off; - return(1); - } - if (ssz == -1) { - perror(f); - break; - } - off += (size_t)ssz; - } - - free(fb->buf); - fb->buf = NULL; - return(0); + *filenc &= ~MPARSE_UTF8; + return 1; } -static int -cue_enc(const struct buf *b, size_t *offs, enum enc *enc) +int +preconv_cue(const struct buf *b, size_t offset) { const char *ln, *eoln, *eoph; - size_t sz, phsz, nsz; - int i; + size_t sz, phsz; - ln = b->buf + (int)*offs; - sz = b->sz - *offs; + ln = b->buf + offset; + sz = b->sz - offset; /* Look for the end-of-line. */ if (NULL == (eoln = memchr(ln, '\n', sz))) - return(-1); + eoln = ln + sz; - /* Set next-line marker. */ - - *offs = (size_t)((eoln + 1) - b->buf); - /* Check if we have the correct header/trailer. */ - if ((sz = (size_t)(eoln - ln)) < 10 || - memcmp(ln, ".\\\" -*-", 7) || - memcmp(eoln - 3, "-*-", 3)) - return(0); + if ((sz = (size_t)(eoln - ln)) < 10 || + memcmp(ln, ".\\\" -*-", 7) || memcmp(eoln - 3, "-*-", 3)) + return MPARSE_UTF8 | MPARSE_LATIN1; /* Move after the header and adjust for the trailer. */ @@ -357,12 +150,12 @@ cue_enc(const struct buf *b, size_t *offs, enum enc *e /* Only account for the "coding" phrase. */ - if ((phsz = (size_t)(eoph - ln)) < 7 || - strncasecmp(ln, "coding:", 7)) { + if ((phsz = eoph - ln) < 7 || + strncasecmp(ln, "coding:", 7)) { sz -= phsz; ln += phsz; continue; - } + } sz -= 7; ln += 7; @@ -372,151 +165,15 @@ cue_enc(const struct buf *b, size_t *offs, enum enc *e sz--; } if (0 == sz) - break; + return 0; /* Check us against known encodings. */ - for (i = 0; i < (int)ENC__MAX; i++) { - nsz = strlen(encs[i].name); - if (phsz < nsz) - continue; - if (strncasecmp(ln, encs[i].name, nsz)) - continue; - - *enc = (enum enc)i; - return(1); - } - - /* Unknown encoding. */ - - *enc = ENC__MAX; - return(1); + if (phsz > 4 && !strncasecmp(ln, "utf-8", 5)) + return MPARSE_UTF8; + if (phsz > 10 && !strncasecmp(ln, "iso-latin-1", 11)) + return MPARSE_LATIN1; + return 0; } - - return(0); -} - -int -main(int argc, char *argv[]) -{ - int i, ch, map, fd, rc; - struct buf b; - const char *fn; - enum enc enc, def; - unsigned char bom[3] = { 0xEF, 0xBB, 0xBF }; - size_t offs; - extern int optind; - extern char *optarg; - - progname = strrchr(argv[0], '/'); - if (progname == NULL) - progname = argv[0]; - else - ++progname; - - fn = ""; - fd = STDIN_FILENO; - rc = EXIT_FAILURE; - enc = def = ENC__MAX; - map = 0; - - memset(&b, 0, sizeof(struct buf)); - - while (-1 != (ch = getopt(argc, argv, "D:e:rdvh"))) - switch (ch) { - case ('D'): - /* FALLTHROUGH */ - case ('e'): - for (i = 0; i < (int)ENC__MAX; i++) { - if (strcasecmp(optarg, encs[i].name)) - continue; - break; - } - if (i < (int)ENC__MAX) { - if ('D' == ch) - def = (enum enc)i; - else - enc = (enum enc)i; - break; - } - - fprintf(stderr, "%s: Bad encoding\n", optarg); - return(EXIT_FAILURE); - case ('r'): - /* FALLTHROUGH */ - case ('d'): - /* FALLTHROUGH */ - case ('v'): - /* Compatibility with GNU preconv. */ - break; - case ('h'): - /* Compatibility with GNU preconv. */ - /* FALLTHROUGH */ - default: - usage(); - return(EXIT_FAILURE); - } - - argc -= optind; - argv += optind; - - /* - * Open and read the first argument on the command-line. - * If we don't have one, we default to stdin. - */ - - if (argc > 0) { - fn = *argv; - fd = open(fn, O_RDONLY, 0); - if (-1 == fd) { - perror(fn); - return(EXIT_FAILURE); - } - } - - if ( ! read_whole_file(fn, fd, &b, &map)) - goto out; - - /* Try to read the UTF-8 BOM. */ - - if (ENC__MAX == enc) - if (b.sz > 3 && 0 == memcmp(b.buf, bom, 3)) { - b.offs = 3; - enc = ENC_UTF_8; - } - - /* Try reading from the "-*-" cue. */ - - if (ENC__MAX == enc) { - offs = b.offs; - ch = cue_enc(&b, &offs, &enc); - if (0 == ch) - ch = cue_enc(&b, &offs, &enc); - } - - /* - * No encoding has been detected. - * Thus, we either fall into our default encoder, if specified, - * or use Latin-1 if all else fails. - */ - - if (ENC__MAX == enc) - enc = ENC__MAX == def ? ENC_LATIN_1 : def; - - if ( ! (*encs[(int)enc].conv)(&b)) { - fprintf(stderr, "%s: Bad encoding\n", fn); - goto out; - } - - rc = EXIT_SUCCESS; -out: - if (map) - munmap(b.buf, b.sz); - else - free(b.buf); - - if (fd > STDIN_FILENO) - close(fd); - - return(rc); + return MPARSE_UTF8 | MPARSE_LATIN1; }