/* * This code is derivative of guess.c of Gauche-0.8.3. * The following is the original copyright notice. */ /* * guess.c - guessing character encoding * * Copyright (c) 2000-2003 Shiro Kawai, All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * 3. Neither the name of the authors nor the names of its contributors * may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ #include "libguess.h" #include "dfa.h" #include /* take precedence if scores are same. you can customize the order as: */ /* ORDER_** &highest, &second, ... &lowest */ #define ORDER_JP &utf8, &sjis, &eucj #define ORDER_TW &utf8, &big5 #define ORDER_CN &utf8, &gb2312, &gb18030 #define ORDER_KR &utf8, &euck, &johab #define ORDER_AR &utf8, &iso8859_6, &cp1256 #define ORDER_GR &utf8, &iso8859_7, &cp1253 #define ORDER_RU &utf8, &cp1251, &koi8_u, &koi8_r, &cp866, &iso8859_2, &iso8859_5 #define ORDER_HW &utf8, &iso8859_8, &cp1255 #define ORDER_PL &utf8, &cp1250, &iso8859_2 #define ORDER_TR &utf8, &iso8859_9, &cp1254 #define ORDER_BL &utf8, &iso8859_13, &cp1257 /* include DFA table generated by guess.scm */ #include "guess_tab.c" int libguess_validate_utf8(const char *buf, int buflen) { int i; guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar, "UTF-8"); for (i = 0; i < buflen; i++) { int c = (unsigned char) buf[i]; if (DFA_ALIVE(utf8)) DFA_NEXT(utf8, c); else break; } DFA_NEXT(utf8, '\0'); //Bug #53 if(DFA_ALIVE(utf8)) return 1; else return 0; } const char *guess_jp(const char *buf, int buflen) { int i; const char *rv = NULL; guess_dfa eucj = DFA_INIT(guess_eucj_st, guess_eucj_ar, "EUC-JP"); guess_dfa sjis = DFA_INIT(guess_sjis_st, guess_sjis_ar, "SJIS"); guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar, "UTF-8"); guess_dfa *top = NULL; guess_dfa *order[] = { ORDER_JP, NULL }; for (i = 0; i < buflen; i++) { int c = (unsigned char) buf[i]; /* special treatment of iso-2022 escape sequence */ if (c == 0x1b) { if (i < buflen - 1) { c = (unsigned char) buf[++i]; if (c == '$' || c == '(') return "ISO-2022-JP"; } } /* special treatment of BOM */ if (i == 0 && c == 0xff) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xfe) return UCS_2LE; } } if (i == 0 && c == 0xfe) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xff) return UCS_2BE; } } rv = dfa_process(order, c); if(rv) return rv; if (dfa_none(order)) { /* we ran out the possibilities */ return NULL; } } top = dfa_top(order); if(top) return top->name; else return NULL; } const char *guess_tw(const char *buf, int buflen) { int i; const char *rv = NULL; guess_dfa big5 = DFA_INIT(guess_big5_st, guess_big5_ar, "BIG5"); guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar, "UTF-8"); guess_dfa *top = NULL; guess_dfa *order[] = { ORDER_TW, NULL }; for (i = 0; i < buflen; i++) { int c = (unsigned char) buf[i]; /* special treatment of iso-2022 escape sequence */ if (c == 0x1b) { if (i < buflen - 1) { c = (unsigned char) buf[++i]; if (c == '$' || c == '(') return "ISO-2022-TW"; } } /* special treatment of BOM */ if (i == 0 && c == 0xff) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xfe) return UCS_2LE; } } if (i == 0 && c == 0xfe) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xff) return UCS_2BE; } } rv = dfa_process(order, c); if(rv) return rv; if (dfa_none(order)) { /* we ran out the possibilities */ return NULL; } } top = dfa_top(order); if (top) return top->name; else return NULL; } const char *guess_cn(const char *buf, int buflen) { int i; const char *rv = NULL; guess_dfa gb2312 = DFA_INIT(guess_gb2312_st, guess_gb2312_ar, "GB2312"); guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar, "UTF-8"); guess_dfa gb18030 = DFA_INIT(guess_gb18030_st, guess_gb18030_ar, "GB18030"); guess_dfa *top = NULL; guess_dfa *order[] = { ORDER_CN, NULL }; for (i = 0; i < buflen; i++) { int c = (unsigned char) buf[i]; int c2; /* special treatment of iso-2022 escape sequence */ if (c == 0x1b) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; c2 = (unsigned char) buf[i + 2]; if (c == '$' && (c2 == ')' || c2 == '+')) return "ISO-2022-CN"; } } /* special treatment of BOM */ if (i == 0 && c == 0xff) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xfe) return UCS_2LE; } } if (i == 0 && c == 0xfe) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xff) return UCS_2BE; } } rv = dfa_process(order, c); if(rv) return rv; if (dfa_none(order)) { /* we ran out the possibilities */ return NULL; } } top = dfa_top(order); if(top) return top->name; else return NULL; } const char *guess_kr(const char *buf, int buflen) { int i; const char *rv = NULL; guess_dfa euck = DFA_INIT(guess_euck_st, guess_euck_ar, "EUC-KR"); guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar, "UTF-8"); guess_dfa johab = DFA_INIT(guess_johab_st, guess_johab_ar, "JOHAB"); guess_dfa *top = NULL; guess_dfa *order[] = { ORDER_KR, NULL }; for (i = 0; i < buflen; i++) { int c = (unsigned char) buf[i]; int c2; /* special treatment of iso-2022 escape sequence */ if (c == 0x1b) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; c2 = (unsigned char) buf[i + 2]; if (c == '$' && c2 == ')') return "ISO-2022-KR"; } } /* special treatment of BOM */ if (i == 0 && c == 0xff) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xfe) return UCS_2LE; } } if (i == 0 && c == 0xfe) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xff) return UCS_2BE; } } rv = dfa_process(order, c); if(rv) return rv; if (dfa_none(order)) { /* we ran out the possibilities */ return NULL; } } top = dfa_top(order); if(top) return top->name; else return NULL; } const char *guess_ar(const char *buf, int buflen) { int i; const char *rv = NULL; /* encodings */ guess_dfa cp1256 = DFA_INIT(guess_cp1256_st, guess_cp1256_ar, "CP1256"); guess_dfa iso8859_6 = DFA_INIT(guess_iso8859_6_st, guess_iso8859_6_ar, "ISO-8859-6"); guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar, "UTF-8"); guess_dfa *top = NULL; guess_dfa *order[] = { ORDER_AR, NULL }; for (i = 0; i < buflen; i++) { int c = (unsigned char) buf[i]; /* special treatment of BOM */ if (i == 0 && c == 0xff) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xfe) return UCS_2LE; } } if (i == 0 && c == 0xfe) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xff) return UCS_2BE; } } rv = dfa_process(order, c); if(rv) return rv; if (dfa_none(order)) { /* we ran out the possibilities */ return NULL; } } top = dfa_top(order); if (top) return top->name; else return NULL; } const char *guess_gr(const char *buf, int buflen) { int i; const char *rv = NULL; /* encodings */ guess_dfa cp1253 = DFA_INIT(guess_cp1253_st, guess_cp1253_ar, "CP1253"); guess_dfa iso8859_7 = DFA_INIT(guess_iso8859_7_st, guess_iso8859_7_ar, "ISO-8859-7"); guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar, "UTF-8"); guess_dfa *top = NULL; guess_dfa *order[] = { ORDER_GR, NULL }; for (i = 0; i < buflen; i++) { int c = (unsigned char) buf[i]; /* special treatment of BOM */ if (i == 0 && c == 0xff) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xfe) return UCS_2LE; } } if (i == 0 && c == 0xfe) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xff) return UCS_2BE; } } rv = dfa_process(order, c); if(rv) return rv; if (dfa_none(order)) { /* we ran out the possibilities */ return NULL; } } top = dfa_top(order); if (top) return top->name; else return NULL; } const char *guess_ru(const char *buf, int buflen) { int i; const char *rv = NULL; /* encodings */ guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar, "UTF-8"); guess_dfa cp1251 = DFA_INIT(guess_cp1251_st, guess_cp1251_ar, "CP1251"); guess_dfa cp866 = DFA_INIT(guess_cp866_st, guess_cp866_ar, "CP866"); guess_dfa koi8_u = DFA_INIT(guess_koi8_u_st, guess_koi8_u_ar, "KOI8-U"); guess_dfa koi8_r = DFA_INIT(guess_koi8_r_st, guess_koi8_r_ar, "KOI8-R"); guess_dfa iso8859_2 = DFA_INIT(guess_iso8859_2_st, guess_iso8859_2_ar, "ISO-8859-2"); guess_dfa iso8859_5 = DFA_INIT(guess_iso8859_5_st, guess_iso8859_5_ar, "ISO-8859-5"); guess_dfa *top = NULL; guess_dfa *order[] = { ORDER_RU, NULL }; for (i = 0; i < buflen; i++) { int c = (unsigned char) buf[i]; /* special treatment of BOM */ if (i == 0 && c == 0xff) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xfe) return UCS_2LE; } } if (i == 0 && c == 0xfe) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xff) return UCS_2BE; } } rv = dfa_process(order, c); if(rv) return rv; if (dfa_none(order)) { /* we ran out the possibilities */ return NULL; } } top = dfa_top(order); if (top) return top->name; else return NULL; } const char *guess_hw(const char *buf, int buflen) { int i; const char *rv = NULL; /* encodings */ guess_dfa cp1255 = DFA_INIT(guess_cp1255_st, guess_cp1255_ar, "CP1255"); guess_dfa iso8859_8 = DFA_INIT(guess_iso8859_8_st, guess_iso8859_8_ar, "ISO-8859-8-I"); guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar, "UTF-8"); guess_dfa *top = NULL; guess_dfa *order[] = { ORDER_HW, NULL }; for (i = 0; i < buflen; i++) { int c = (unsigned char) buf[i]; /* special treatment of BOM */ if (i == 0 && c == 0xff) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xfe) return UCS_2LE; } } if (i == 0 && c == 0xfe) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xff) return UCS_2BE; } } rv = dfa_process(order, c); if(rv) return rv; if (dfa_none(order)) { /* we ran out the possibilities */ return NULL; } } top = dfa_top(order); if (top) return top->name; else return NULL; } const char *guess_pl(const char *buf, int buflen) { int i; const char *rv = NULL; /* encodings */ guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar, "UTF-8"); guess_dfa cp1250 = DFA_INIT(guess_cp1250_st, guess_cp1250_ar, "CP1250"); guess_dfa iso8859_2 = DFA_INIT(guess_iso8859_2_st, guess_iso8859_2_ar, "ISO-8859-2"); guess_dfa *top = NULL; guess_dfa *order[] = { ORDER_PL, NULL }; for (i = 0; i < buflen; i++) { int c = (unsigned char) buf[i]; /* special treatment of BOM */ if (i == 0 && c == 0xff) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xfe) return UCS_2LE; } } if (i == 0 && c == 0xfe) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xff) return UCS_2BE; } } rv = dfa_process(order, c); if(rv) return rv; if (dfa_none(order)) { /* we ran out the possibilities */ return NULL; } } top = dfa_top(order); if (top) return top->name; else return NULL; } const char *guess_tr(const char *buf, int buflen) { int i; const char *rv = NULL; /* encodings */ guess_dfa cp1254 = DFA_INIT(guess_cp1254_st, guess_cp1254_ar, "CP1254"); guess_dfa iso8859_9 = DFA_INIT(guess_iso8859_9_st, guess_iso8859_9_ar, "ISO-8859-9"); guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar, "UTF-8"); guess_dfa *top = NULL; guess_dfa *order[] = { ORDER_TR, NULL }; for (i = 0; i < buflen; i++) { int c = (unsigned char) buf[i]; /* special treatment of BOM */ if (i == 0 && c == 0xff) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xfe) return UCS_2LE; } } if (i == 0 && c == 0xfe) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xff) return UCS_2BE; } } rv = dfa_process(order, c); if(rv) return rv; if (dfa_none(order)) { /* we ran out the possibilities */ return NULL; } } top = dfa_top(order); if (top) return top->name; else return NULL; } const char *guess_bl(const char *buf, int buflen) { int i; const char *rv = NULL; /* encodings */ guess_dfa cp1257 = DFA_INIT(guess_cp1257_st, guess_cp1257_ar, "CP1257"); guess_dfa iso8859_13 = DFA_INIT(guess_iso8859_13_st, guess_iso8859_13_ar, "ISO-8859-13"); guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar, "UTF-8"); guess_dfa *top = NULL; guess_dfa *order[] = { ORDER_BL, NULL }; for (i = 0; i < buflen; i++) { int c = (unsigned char) buf[i]; /* special treatment of BOM */ if (i == 0 && c == 0xff) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xfe) return UCS_2LE; } } if (i == 0 && c == 0xfe) { if (i < buflen - 1) { c = (unsigned char) buf[i + 1]; if (c == 0xff) return UCS_2BE; } } rv = dfa_process(order, c); if(rv) return rv; if (dfa_none(order)) { /* we ran out the possibilities */ return NULL; } } top = dfa_top(order); if (top) return top->name; else return NULL; }