/*
* This file is part of Hubbub.
* Licensed under the MIT License,
* http://www.opensource.org/licenses/mit-license.php
* Copyright 2007 John-Mark Bell <jmb@netsurf-browser.org>
*/
#include <assert.h>
#include <stdbool.h>
#include <string.h>
#include <parserutils/charset/mibenum.h>
#include <hubbub/types.h>
#include "utils/utils.h"
#include "detect.h"
static uint16_t hubbub_charset_read_bom(const uint8_t *data, size_t len);
static uint16_t hubbub_charset_scan_meta(const uint8_t *data, size_t len);
static uint16_t hubbub_charset_parse_attributes(const uint8_t **pos,
const uint8_t *end);
static bool hubbub_charset_get_attribute(const uint8_t **data,
const uint8_t *end,
const uint8_t **name, uint32_t *namelen,
const uint8_t **value, uint32_t *valuelen);
/**
* Extract a charset from a chunk of data
*
* \param data Pointer to buffer containing data
* \param len Buffer length
* \param mibenum Pointer to location containing current MIB enum
* \param source Pointer to location containint current charset source
* \return PARSERUTILS_OK on success, appropriate error otherwise
*
* ::mibenum and ::source will be updated on exit
*
* The larger a chunk of data fed to this routine, the better, as it allows
* charset autodetection access to a larger dataset for analysis.
*/
parserutils_error hubbub_charset_extract(const uint8_t *data, size_t len,
uint16_t *mibenum, uint32_t *source)
{
uint16_t charset = 0;
if (data == NULL || mibenum == NULL || source == NULL)
return PARSERUTILS_BADPARM;
/**
* Meaning of *source on entry:
*
* CONFIDENT - Do not pass Go, do not attempt auto-detection.
* TENTATIVE - We've tried to autodetect already, but subsequently
* discovered that we don't actually support the detected
* charset. Thus, we've defaulted to Windows-1252. Don't
* perform auto-detection again, as it would be futile.
* (This bit diverges from the spec)
* UNKNOWN - No autodetection performed yet. Get on with it.
*/
/* 1. */
/* If the source is dictated, there's nothing for us to do */
if (*source == HUBBUB_CHARSET_CONFIDENT ||
*source == HUBBUB_CHARSET_TENTATIVE) {
return PARSERUTILS_OK;
}
/* 2. */
/** \todo We probably want to wait for ~512 bytes of data / 500ms here */
/* 3. */
/* We need at least 3 bytes of data */
if (len < 3)
goto default_encoding;
/* First, look for a BOM */
charset = hubbub_charset_read_bom(data, len);
if (charset != 0) {
*mibenum = charset;
*source = HUBBUB_CHARSET_CONFIDENT;
return PARSERUTILS_OK;
}
/* 4. */
/* No BOM was found, so we must look for a meta charset within
* the document itself. */
charset = hubbub_charset_scan_meta(data, len);
if (charset != 0) {
/* Fix charsets according to HTML5,
* section 8.2.2.2. Character encoding requirements */
hubbub_charset_fix_charset(&charset);
/* If we've encountered a meta charset for a non-ASCII-
* compatible encoding, don't trust it.
*
* Firstly, it should have been sent with a BOM (and thus
* detected above).
*
* Secondly, we've just used an ASCII-only parser to
* extract the encoding from the document. Therefore,
* the document plainly isn't what the meta charset
* claims it is.
*
* What we do in this case is to ignore the meta charset's
* claims and leave the charset determination to the
* autodetection routines (or the fallback case if they
* fail).
*/
if (charset != parserutils_charset_mibenum_from_name(
"UTF-32", SLEN("UTF-32")) &&
charset != parserutils_charset_mibenum_from_name(
"UTF-32LE", SLEN("UTF-32LE")) &&
charset != parserutils_charset_mibenum_from_name(
"UTF-32BE", SLEN("UTF-32BE"))) {
*mibenum = charset;
*source = HUBBUB_CHARSET_TENTATIVE;
return PARSERUTILS_OK;
}
}
/* No charset was specified within the document, attempt to
* autodetect the encoding from the data that we have available. */
/** \todo Charset autodetection */
/* We failed to autodetect a charset, so use the default fallback */
default_encoding:
/* 7. */
charset = parserutils_charset_mibenum_from_name("Windows-1252",
SLEN("Windows-1252"));
if (charset == 0)
charset = parserutils_charset_mibenum_from_name("ISO-8859-1",
SLEN("ISO-8859-1"));
*mibenum = charset;
*source = HUBBUB_CHARSET_TENTATIVE;
return PARSERUTILS_OK;
}
/**
* Inspect the beginning of a buffer of data for the presence of a
* UTF Byte Order Mark.
*
* \param data Pointer to buffer containing data
* \param len Buffer length
* \return MIB enum representing encoding described by BOM, or 0 if not found
*/
uint16_t hubbub_charset_read_bom(const uint8_t *data, size_t len)
{
if (data == NULL)
return 0;
/* We require at least 3 bytes of data */
if (len < 3)
return 0;
if (data[0] == 0xFE && data[1] == 0xFF) {
return parserutils_charset_mibenum_from_name("UTF-16BE",
SLEN("UTF-16BE"));
} else if (data[0] == 0xFF && data[1] == 0xFE) {
return parserutils_charset_mibenum_from_name("UTF-16LE",
SLEN("UTF-16LE"));
} else if (data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF) {
return parserutils_charset_mibenum_from_name("UTF-8",
SLEN("UTF-8"));
}
return 0;
}
#define PEEK(a) \
(pos < end - SLEN(a) && \
strncasecmp((const char *) pos, a, SLEN(a)) == 0)
#define ADVANCE(a) \
while (pos < end - SLEN(a)) { \
if (PEEK(a)) \
break; \
pos++; \
} \
\
if (pos == end - SLEN(a)) \
return 0;
#define ISSPACE(a) \
(a == 0x09 || a == 0x0a || a == 0x0c || \
a == 0x0d || a == 0x20 || a == 0x2f)
/**
* Search for a meta charset within a buffer of data
*
* \param data Pointer to buffer containing data
* \param len Length of buffer
* \return MIB enum representing encoding, or 0 if none found
*/
uint16_t hubbub_charset_scan_meta(const uint8_t *data, size_t len)
{
const uint8_t *pos = data;
const uint8_t *end;
uint16_t mibenum;
if (data == NULL)
return 0;
end = pos + min(512, len);
/* 1. */
while (pos < end) {
/* a */
if (PEEK("<!--")) {
pos += SLEN("<!--");
ADVANCE("-->");
/* b */
} else if (PEEK("<meta")) {
if (pos + SLEN("<meta") >= end - 1)
return 0;
if (ISSPACE(*(pos + SLEN("<meta")))) {
/* 1 */
pos += SLEN("<meta");
mibenum = hubbub_charset_parse_attributes(
&pos, end);
if (mibenum != 0)
return mibenum;
if (pos >= end)
return 0;
}
/* c */
} else if ((PEEK("</") && (pos < end - 3 &&
(0x41 <= (*(pos + 2) & ~ 0x20) &&
(*(pos + 2) & ~ 0x20) <= 0x5A))) ||
(pos < end - 2 && *pos == '<' &&
(0x41 <= (*(pos + 1) & ~ 0x20) &&
(*(pos + 1) & ~ 0x20) <= 0x5A))) {
/* skip '<' */
pos++;
/* 1. */
while (pos < end) {
if (ISSPACE(*pos) ||
*pos == '>' || *pos == '<')
break;
pos++;
}
if (pos >= end)
return 0;
/* 3 */
if (*pos != '<') {
const uint8_t *n;
const uint8_t *v;
uint32_t nl, vl;
while (hubbub_charset_get_attribute(&pos, end,
&n, &nl, &v, &vl))
; /* do nothing */
/* 2 */
} else
continue;
/* d */
} else if (PEEK("<!") || PEEK("</") || PEEK("<?")) {
pos++;
ADVANCE(">");
}
/* e - do nothing */
/* 2 */
pos++;
}
return 0;
}
/**
* Parse attributes on a meta tag
*
* \param pos Pointer to pointer to current location (updated on exit)
* \param end Pointer to end of data stream
* \return MIB enum of detected encoding, or 0 if none found
*/
uint16_t hubbub_charset_parse_attributes(const uint8_t **pos,
const uint8_t *end)
{
const uint8_t *name;
const uint8_t *value;
uint32_t namelen, valuelen;
uint16_t mibenum = 0;
if (pos == NULL || *pos == NULL || end == NULL)
return 0;
/* 2 */
while (hubbub_charset_get_attribute(pos, end,
&name, &namelen, &value, &valuelen)) {
/* 3 done by default */
/* 4 */
if (namelen == SLEN("charset") && valuelen > 0 &&
strncasecmp((const char *) name, "charset",
SLEN("charset")) == 0) {
/* strip value */
while (ISSPACE(*value)) {
value++;
valuelen--;
}
while (valuelen > 0 && ISSPACE(value[valuelen - 1]))
valuelen--;
mibenum = parserutils_charset_mibenum_from_name(
(const char *) value, valuelen);
/* 5 */
} else if (namelen == SLEN("content") && valuelen > 0 &&
strncasecmp((const char *) name, "content",
SLEN("content")) == 0) {
mibenum = hubbub_charset_parse_content(value,
valuelen);
}
/* 6 */
if (mibenum == parserutils_charset_mibenum_from_name(
"UTF-16LE", SLEN("UTF-16LE")) ||
mibenum ==
parserutils_charset_mibenum_from_name(
"UTF-16BE", SLEN("UTF-16BE")) ||
mibenum ==
parserutils_charset_mibenum_from_name(
"UTF-16", SLEN("UTF-16"))) {
mibenum = parserutils_charset_mibenum_from_name(
"UTF-8", SLEN("UTF-8"));
}
/* 7 */
if (mibenum != 0) {
/* confidence = tentative; */
return mibenum;
}
}
return 0;
}
/**
* Parse a content= attribute's value
*
* \param value Attribute's value
* \param valuelen Length of value
* \return MIB enum of detected encoding, or 0 if none found
*/
uint16_t hubbub_charset_parse_content(const uint8_t *value,
uint32_t valuelen)
{
const uint8_t *end;
const uint8_t *tentative = NULL;
uint32_t tentative_len = 0;
if (value == NULL)
return 0;
end = value + valuelen;
/* 1 */
while (value < end) {
if (*value == ';') {
value++;
break;
}
value++;
}
if (value >= end)
return 0;
/* 2 */
while (value < end && ISSPACE(*value)) {
value++;
}
if (value >= end)
return 0;
/* 3 */
if (value < end - SLEN("charset") &&
strncasecmp((const char *) value,
"charset", SLEN("charset")) != 0)
return 0;
value += SLEN("charset");
/* 4 */
while (value < end && ISSPACE(*value)) {
value++;
}
if (value >= end)
return 0;
/* 5 */
if (*value != '=')
return 0;
/* skip '=' */
value++;
/* 6 */
while (value < end && ISSPACE(*value)) {
value++;
}
if (value >= end)
return 0;
/* 7 */
tentative = value;
/* a */
if (*value == '"') {
while (++value < end && *value != '"') {
tentative_len++;
}
if (value < end)
tentative++;
else
tentative = NULL;
/* b */
} else if (*value == '\'') {
while (++value < end && *value != '\'') {
tentative_len++;
}
if (value < end)
tentative++;
else
tentative = NULL;
/* c */
} else {
while (value < end && !ISSPACE(*value)) {
value++;
tentative_len++;
}
}
/* 8 */
if (tentative != NULL) {
return parserutils_charset_mibenum_from_name(
(const char *) tentative, tentative_len);
}
/* 9 */
return 0;
}
/**
* Extract an attribute from the data stream
*
* \param data Pointer to pointer to current location (updated on exit)
* \param end Pointer to end of data stream
* \param name Pointer to location to receive attribute name
* \param namelen Pointer to location to receive attribute name length
* \param value Pointer to location to receive attribute value
* \param valuelen Pointer to location to receive attribute value langth
* \return true if attribute extracted, false otherwise.
*
* Note: The caller should heed the returned lengths; these are the only
* indicator that useful content resides in name or value.
*/
bool hubbub_charset_get_attribute(const uint8_t **data, const uint8_t *end,
const uint8_t **name, uint32_t *namelen,
const uint8_t **value, uint32_t *valuelen)
{
const uint8_t *pos;
if (data == NULL || *data == NULL || end == NULL || name == NULL ||
namelen == NULL || value == NULL || valuelen == NULL)
return false;
pos = *data;
/* 1. Skip leading spaces or '/' characters */
while (pos < end && (ISSPACE(*pos) || *pos == '/')) {
pos++;
}
if (pos >= end) {
*data = pos;
return false;
}
/* 2. Invalid element open character */
if (*pos == '<') {
pos--;
*data = pos;
return false;
}
/* 3. End of element */
if (*pos == '>') {
*data = pos;
return false;
}
/* 4. Initialise name & value to empty string */
*name = pos;
*namelen = 0;
*value = (const uint8_t *) "";
*valuelen = 0;
/* 5. Extract name */
while (pos < end) {
/* a */
if (*pos == '=') {
break;
}
/* b */
if (ISSPACE(*pos)) {
break;
}
/* c */
if (*pos == '/' || *pos == '<' || *pos == '>') {
*data = pos;
return true;
}
/* d is handled by strncasecmp in _parse_attributes */
/* e */
(*namelen)++;
/* 6 */
pos++;
}
if (pos >= end) {
*data = pos;
return false;
}
if (ISSPACE(*pos)) {
/* 7. Skip trailing spaces */
while (pos < end && ISSPACE(*pos)) {
pos++;
}
if (pos >= end) {
*data = pos;
return false;
}
/* 8. Must be '=' */
if (*pos != '=') {
pos--;
*data = pos;
return true;
}
}
/* 9. Skip '=' */
pos++;
/* 10. Skip any spaces after '=' */
while (pos < end && ISSPACE(*pos)) {
pos++;
}
if (pos >= end) {
*data = pos;
return false;
}
/* 11. Extract value, if quoted */
/* a */
if (*pos == '\'' || *pos == '"') {
/* 1 */
const uint8_t *quote = pos;
/* 2 */
while (++pos < end) {
/* 3 */
if (*pos == *quote) {
*value = (quote + 1);
*data = ++pos;
return true;
}
/* 4 is handled by strncasecmp */
/* 5 */
(*valuelen)++;
/* 6 */
}
if (pos >= end) {
*data = pos;
return false;
}
}
/* b */
if (*pos == '<' || *pos == '>') {
*data = pos;
return true;
}
/* c is handled by strncasecmp */
/* d */
*value = pos;
while (pos < end) {
/* 12. Extract unquoted value */
/* a */
if (ISSPACE(*pos) || *pos == '<' || *pos == '>') {
*data = pos;
return true;
}
/* b is handled by strncasecmp */
/* c */
(*valuelen)++;
/* 13. Advance */
pos++;
}
if (pos >= end) {
*data = pos;
return false;
}
/* should never be reached */
return false;
}
/**
* Fix charsets, according to the override table in HTML5,
* section 8.2.2.2. Character encoding requirements
* <http://www.whatwg.org/specs/web-apps/current-work/#character0>
*
* \param charset Pointer to charset value to fix
*/
void hubbub_charset_fix_charset(uint16_t *charset)
{
uint16_t tmp = 0;
/* ISO-8859-1 -> Windows-1252 */
if (*charset == parserutils_charset_mibenum_from_name(
"ISO-8859-1", SLEN("ISO-8859-1"))) {
tmp = parserutils_charset_mibenum_from_name(
"Windows-1252", SLEN("Windows-1252"));
assert(tmp
!= 0 && "Windows-1252 MUST be supported");
/* ISO-8859-9 -> Windows-1254 */
} else if (*charset == parserutils_charset_mibenum_from_name(
"ISO-8859-9", SLEN("ISO-8859-9"))) {
tmp = parserutils_charset_mibenum_from_name(
"Windows-1254", SLEN("Windows-1254"));
/* ISO-8859-11 -> Windows-874 */
} else if (*charset == parserutils_charset_mibenum_from_name(
"ISO-8859-11", SLEN("ISO-8859-11"))) {
tmp = parserutils_charset_mibenum_from_name(
"Windows-874", SLEN("Windows-874"));
/* KS_C_5601-1987 and EUC-KR -> Windows-949 */
} else if (*charset == parserutils_charset_mibenum_from_name(
"KS_C_5601-1987", SLEN("KS_C_5601-1987")) ||
*charset == parserutils_charset_mibenum_from_name(
"EUC-KR", SLEN("EUC-KR"))) {
tmp = parserutils_charset_mibenum_from_name(
"Windows-949", SLEN("Windows-949"));
/* TIS-620 -> Windows-874 */
} else if (*charset == parserutils_charset_mibenum_from_name(
"TIS-620", SLEN("TIS-620"))) {
tmp = parserutils_charset_mibenum_from_name(
"Windows-874", SLEN("Windows-874"));
/* x-x-big5 -> Big5 */
} else if (*charset == parserutils_charset_mibenum_from_name(
"x-x-big5", SLEN("x-x-big5"))) {
tmp = parserutils_charset_mibenum_from_name(
"Big5", SLEN("Big5"));
/* GB2312 and GB_2312-80 -> GBK */
} else if (*charset == parserutils_charset_mibenum_from_name(
"GB2312", SLEN("GB2312")) ||
*charset == parserutils_charset_mibenum_from_name(
"GB_2312-80", SLEN("GB_2312-80"))) {
tmp = parserutils_charset_mibenum_from_name(
"GBK", SLEN("GBK"));
}
if (tmp != 0)
*charset = tmp;
}