Annotation of mandoc/preconv.c, Revision 1.5
1.5 ! kristaps 1: /* $Id: preconv.c,v 1.4 2011/05/26 21:13:07 kristaps Exp $ */
1.1 kristaps 2: /*
3: * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
4: *
5: * Permission to use, copy, modify, and distribute this software for any
6: * purpose with or without fee is hereby granted, provided that the above
7: * copyright notice and this permission notice appear in all copies.
8: *
9: * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10: * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11: * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12: * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13: * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14: * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15: * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16: */
17: #ifdef HAVE_CONFIG_H
18: #include "config.h"
19: #endif
20:
1.5 ! kristaps 21: #ifdef HAVE_MMAP
1.1 kristaps 22: #include <sys/stat.h>
23: #include <sys/mman.h>
1.5 ! kristaps 24: #endif
1.1 kristaps 25:
26: #include <assert.h>
27: #include <fcntl.h>
28: #include <stdio.h>
29: #include <stdlib.h>
30: #include <string.h>
31: #include <unistd.h>
32:
33: /*
34: * The read_whole_file() and resize_buf() functions are copied from
35: * read.c, including all dependency code (MAP_FILE, etc.).
36: */
37:
38: #ifndef MAP_FILE
39: #define MAP_FILE 0
40: #endif
41:
42: enum enc {
43: ENC_UTF_8, /* UTF-8 */
44: ENC_US_ASCII, /* US-ASCII */
45: ENC_LATIN_1, /* Latin-1 */
46: ENC__MAX
47: };
48:
49: struct buf {
50: char *buf; /* binary input buffer */
51: size_t sz; /* size of binary buffer */
52: size_t offs; /* starting buffer offset */
53: };
54:
55: struct encode {
56: const char *name;
57: int (*conv)(const struct buf *);
58: };
59:
1.3 kristaps 60: static int cue_enc(const struct buf *, size_t *, enum enc *);
1.1 kristaps 61: static int conv_latin_1(const struct buf *);
62: static int conv_us_ascii(const struct buf *);
63: static int conv_utf_8(const struct buf *);
64: static int read_whole_file(const char *, int,
65: struct buf *, int *);
66: static void resize_buf(struct buf *, size_t);
67: static void usage(void);
68:
69: static const struct encode encs[ENC__MAX] = {
70: { "utf-8", conv_utf_8 }, /* ENC_UTF_8 */
71: { "us-ascii", conv_us_ascii }, /* ENC_US_ASCII */
72: { "latin-1", conv_latin_1 }, /* ENC_LATIN_1 */
73: };
74:
75: static const char *progname;
76:
77: static void
78: usage(void)
79: {
80:
81: fprintf(stderr, "usage: %s "
82: "[-D enc] "
83: "[-e ENC] "
84: "[file]\n", progname);
85: }
86:
87: static int
88: conv_latin_1(const struct buf *b)
89: {
90: size_t i;
1.2 kristaps 91: unsigned char cu;
1.1 kristaps 92: const char *cp;
93:
94: cp = b->buf + (int)b->offs;
95:
96: /*
97: * Latin-1 falls into the first 256 code-points of Unicode, so
98: * there's no need for any sort of translation. Just make the
99: * 8-bit characters use the Unicode escape.
1.3 kristaps 100: * Note that binary values 128 < v < 160 are passed through
101: * unmodified to mandoc.
1.1 kristaps 102: */
103:
104: for (i = b->offs; i < b->sz; i++) {
1.2 kristaps 105: cu = (unsigned char)*cp++;
1.3 kristaps 106: cu < 160U ? putchar(cu) : printf("\\[u%.4X]", cu);
1.1 kristaps 107: }
108:
109: return(1);
110: }
111:
112: static int
113: conv_us_ascii(const struct buf *b)
114: {
115:
116: /*
117: * US-ASCII has no conversion since it falls into the first 128
118: * bytes of Unicode.
119: */
120:
121: fwrite(b->buf, 1, b->sz, stdout);
122: return(1);
123: }
124:
125: static int
126: conv_utf_8(const struct buf *b)
127: {
1.2 kristaps 128: int state, be;
129: unsigned int accum;
130: size_t i;
131: unsigned char cu;
132: const char *cp;
133: const long one = 1L;
134:
135: cp = b->buf + (int)b->offs;
136: state = 0;
137: accum = 0U;
138: be = 0;
139:
140: /* Quick test for big-endian value. */
141:
1.4 kristaps 142: if ( ! (*((const char *)(&one))))
1.2 kristaps 143: be = 1;
144:
145: for (i = b->offs; i < b->sz; i++) {
146: cu = (unsigned char)*cp++;
147: if (state) {
148: if ( ! (cu & 128) || (cu & 64)) {
149: /* Bad sequence header. */
150: return(0);
151: }
152:
153: /* Accept only legitimate bit patterns. */
154:
155: if (cu > 191 || cu < 128) {
156: /* Bad in-sequence bits. */
157: return(0);
158: }
159:
160: accum |= (cu & 63) << --state * 6;
161:
162: /*
163: * Accum is held in little-endian order as
164: * stipulated by the UTF-8 sequence coding. We
165: * need to convert to a native big-endian if our
166: * architecture requires it.
167: */
168:
169: if (0 == state && be)
170: accum = (accum >> 24) |
171: ((accum << 8) & 0x00FF0000) |
172: ((accum >> 8) & 0x0000FF00) |
173: (accum << 24);
174:
175: if (0 == state) {
176: accum < 128U ? putchar(accum) :
177: printf("\\[u%.4X]", accum);
178: accum = 0U;
179: }
180: } else if (cu & (1 << 7)) {
181: /*
182: * Entering a UTF-8 state: if we encounter a
183: * UTF-8 bitmask, calculate the expected UTF-8
184: * state from it.
185: */
186: for (state = 0; state < 7; state++)
187: if ( ! (cu & (1 << (7 - state))))
188: break;
189:
190: /* Accept only legitimate bit patterns. */
191:
192: switch (state) {
193: case (4):
194: if (cu <= 244 && cu >= 240) {
195: accum = (cu & 7) << 18;
196: break;
197: }
198: /* Bad 4-sequence start bits. */
199: return(0);
200: case (3):
201: if (cu <= 239 && cu >= 224) {
202: accum = (cu & 15) << 12;
203: break;
204: }
205: /* Bad 3-sequence start bits. */
206: return(0);
207: case (2):
208: if (cu <= 223 && cu >= 194) {
209: accum = (cu & 31) << 6;
210: break;
211: }
212: /* Bad 2-sequence start bits. */
213: return(0);
214: default:
215: /* Bad sequence bit mask. */
216: return(0);
217: }
218: state--;
219: } else
220: putchar(cu);
221: }
222:
223: if (0 != state) {
224: /* Bad trailing bits. */
225: return(0);
226: }
1.1 kristaps 227:
228: return(1);
229: }
230:
231: static void
232: resize_buf(struct buf *buf, size_t initial)
233: {
234:
235: buf->sz = buf->sz > initial / 2 ?
236: 2 * buf->sz : initial;
237:
238: buf->buf = realloc(buf->buf, buf->sz);
239: if (NULL == buf->buf) {
240: perror(NULL);
241: exit(EXIT_FAILURE);
242: }
243: }
244:
245: static int
246: read_whole_file(const char *f, int fd,
247: struct buf *fb, int *with_mmap)
248: {
249: size_t off;
250: ssize_t ssz;
251:
1.5 ! kristaps 252: #ifdef HAVE_MMAP
! 253: struct stat st;
1.1 kristaps 254: if (-1 == fstat(fd, &st)) {
255: perror(f);
256: return(0);
257: }
258:
259: /*
260: * If we're a regular file, try just reading in the whole entry
261: * via mmap(). This is faster than reading it into blocks, and
262: * since each file is only a few bytes to begin with, I'm not
263: * concerned that this is going to tank any machines.
264: */
265:
266: if (S_ISREG(st.st_mode) && st.st_size >= (1U << 31)) {
267: fprintf(stderr, "%s: input too large\n", f);
268: return(0);
269: }
270:
271: if (S_ISREG(st.st_mode)) {
272: *with_mmap = 1;
273: fb->sz = (size_t)st.st_size;
274: fb->buf = mmap(NULL, fb->sz, PROT_READ,
275: MAP_FILE|MAP_SHARED, fd, 0);
276: if (fb->buf != MAP_FAILED)
277: return(1);
278: }
1.5 ! kristaps 279: #endif
1.1 kristaps 280:
281: /*
282: * If this isn't a regular file (like, say, stdin), then we must
283: * go the old way and just read things in bit by bit.
284: */
285:
286: *with_mmap = 0;
287: off = 0;
288: fb->sz = 0;
289: fb->buf = NULL;
290: for (;;) {
291: if (off == fb->sz && fb->sz == (1U << 31)) {
292: fprintf(stderr, "%s: input too large\n", f);
293: break;
294: }
295:
296: if (off == fb->sz)
297: resize_buf(fb, 65536);
298:
299: ssz = read(fd, fb->buf + (int)off, fb->sz - off);
300: if (ssz == 0) {
301: fb->sz = off;
302: return(1);
303: }
304: if (ssz == -1) {
305: perror(f);
306: break;
307: }
308: off += (size_t)ssz;
309: }
310:
311: free(fb->buf);
312: fb->buf = NULL;
313: return(0);
314: }
315:
1.3 kristaps 316: static int
317: cue_enc(const struct buf *b, size_t *offs, enum enc *enc)
318: {
319: const char *ln, *eoln, *eoph;
320: size_t sz, phsz, nsz;
321: int i;
322:
323: ln = b->buf + (int)*offs;
324: sz = b->sz - *offs;
325:
326: /* Look for the end-of-line. */
327:
328: if (NULL == (eoln = memchr(ln, '\n', sz)))
329: return(-1);
330:
331: /* Set next-line marker. */
332:
333: *offs = (size_t)((eoln + 1) - b->buf);
334:
335: /* Check if we have the correct header/trailer. */
336:
337: if ((sz = (size_t)(eoln - ln)) < 10 ||
338: memcmp(ln, ".\\\" -*-", 7) ||
339: memcmp(eoln - 3, "-*-", 3))
340: return(0);
341:
342: /* Move after the header and adjust for the trailer. */
343:
344: ln += 7;
345: sz -= 10;
346:
347: while (sz > 0) {
348: while (sz > 0 && ' ' == *ln) {
349: ln++;
350: sz--;
351: }
352: if (0 == sz)
353: break;
354:
355: /* Find the end-of-phrase marker (or eoln). */
356:
357: if (NULL == (eoph = memchr(ln, ';', sz)))
358: eoph = eoln - 3;
359: else
360: eoph++;
361:
362: /* Only account for the "coding" phrase. */
363:
364: if ((phsz = (size_t)(eoph - ln)) < 7 ||
365: strncasecmp(ln, "coding:", 7)) {
366: sz -= phsz;
367: ln += phsz;
368: continue;
369: }
370:
371: sz -= 7;
372: ln += 7;
373:
374: while (sz > 0 && ' ' == *ln) {
375: ln++;
376: sz--;
377: }
378: if (0 == sz)
379: break;
380:
381: /* Check us against known encodings. */
382:
1.4 kristaps 383: for (i = 0; i < (int)ENC__MAX; i++) {
1.3 kristaps 384: nsz = strlen(encs[i].name);
385: if (phsz < nsz)
386: continue;
387: if (strncasecmp(ln, encs[i].name, nsz))
388: continue;
389:
390: *enc = (enum enc)i;
391: return(1);
392: }
393:
394: /* Unknown encoding. */
395:
396: *enc = ENC__MAX;
397: return(1);
398: }
399:
400: return(0);
401: }
402:
1.1 kristaps 403: int
404: main(int argc, char *argv[])
405: {
406: int i, ch, map, fd, rc;
1.2 kristaps 407: struct buf b;
1.1 kristaps 408: const char *fn;
409: enum enc enc, def;
1.4 kristaps 410: unsigned char bom[3] = { 0xEF, 0xBB, 0xBF };
1.3 kristaps 411: size_t offs;
1.1 kristaps 412: extern int optind;
413: extern char *optarg;
414:
415: progname = strrchr(argv[0], '/');
416: if (progname == NULL)
417: progname = argv[0];
418: else
419: ++progname;
420:
421: fn = "<stdin>";
422: fd = STDIN_FILENO;
423: rc = EXIT_FAILURE;
424: enc = def = ENC__MAX;
425: map = 0;
426:
1.2 kristaps 427: memset(&b, 0, sizeof(struct buf));
1.1 kristaps 428:
429: while (-1 != (ch = getopt(argc, argv, "D:e:rdvh")))
430: switch (ch) {
431: case ('D'):
432: /* FALLTHROUGH */
433: case ('e'):
1.4 kristaps 434: for (i = 0; i < (int)ENC__MAX; i++) {
1.1 kristaps 435: if (strcasecmp(optarg, encs[i].name))
436: continue;
437: break;
438: }
1.4 kristaps 439: if (i < (int)ENC__MAX) {
1.1 kristaps 440: if ('D' == ch)
441: def = (enum enc)i;
442: else
443: enc = (enum enc)i;
444: break;
445: }
446:
447: fprintf(stderr, "%s: Bad encoding\n", optarg);
448: return(EXIT_FAILURE);
449: case ('r'):
450: /* FALLTHROUGH */
451: case ('d'):
452: /* FALLTHROUGH */
453: case ('v'):
454: /* Compatibility with GNU preconv. */
455: break;
456: case ('h'):
457: /* Compatibility with GNU preconv. */
458: /* FALLTHROUGH */
459: default:
460: usage();
461: return(EXIT_FAILURE);
462: }
463:
464: argc -= optind;
465: argv += optind;
466:
467: /*
468: * Open and read the first argument on the command-line.
469: * If we don't have one, we default to stdin.
470: */
471:
472: if (argc > 0) {
473: fn = *argv;
474: fd = open(fn, O_RDONLY, 0);
475: if (-1 == fd) {
476: perror(fn);
477: return(EXIT_FAILURE);
478: }
479: }
480:
1.2 kristaps 481: if ( ! read_whole_file(fn, fd, &b, &map))
1.1 kristaps 482: goto out;
483:
1.2 kristaps 484: /* Try to read the UTF-8 BOM. */
485:
486: if (ENC__MAX == enc)
487: if (b.sz > 3 && 0 == memcmp(b.buf, bom, 3)) {
488: b.offs = 3;
489: enc = ENC_UTF_8;
490: }
1.1 kristaps 491:
1.3 kristaps 492: /* Try reading from the "-*-" cue. */
493:
494: if (ENC__MAX == enc) {
495: offs = b.offs;
496: ch = cue_enc(&b, &offs, &enc);
497: if (0 == ch)
498: ch = cue_enc(&b, &offs, &enc);
499: }
500:
1.1 kristaps 501: /*
502: * No encoding has been detected.
503: * Thus, we either fall into our default encoder, if specified,
504: * or use Latin-1 if all else fails.
505: */
506:
507: if (ENC__MAX == enc)
508: enc = ENC__MAX == def ? ENC_LATIN_1 : def;
509:
1.3 kristaps 510: if ( ! (*encs[(int)enc].conv)(&b)) {
511: fprintf(stderr, "%s: Bad encoding\n", fn);
1.1 kristaps 512: goto out;
1.3 kristaps 513: }
1.1 kristaps 514:
515: rc = EXIT_SUCCESS;
516: out:
1.5 ! kristaps 517: #ifdef HAVE_MMAP
1.1 kristaps 518: if (map)
1.2 kristaps 519: munmap(b.buf, b.sz);
1.1 kristaps 520: else
1.5 ! kristaps 521: #endif
1.2 kristaps 522: free(b.buf);
1.1 kristaps 523:
524: if (fd > STDIN_FILENO)
525: close(fd);
526:
527: return(rc);
528: }
CVSweb