Initial add.
This commit is contained in:
Ryan C. Gordon
parent
49f0f536cb
commit
95cbc6bed2
|
|
@ -0,0 +1,338 @@
|
|||
#if HAVE_CONFIG_H
|
||||
# include <config.h>
|
||||
#endif
|
||||
|
||||
#include "physfs.h"
|
||||
|
||||
#define __PHYSICSFS_INTERNAL__
|
||||
#include "physfs_internal.h"
|
||||
|
||||
|
||||
/*
|
||||
* From rfc3629, the UTF-8 spec:
|
||||
* http://www.ietf.org/rfc/rfc3629.txt
|
||||
*
|
||||
* Char. number range | UTF-8 octet sequence
|
||||
* (hexadecimal) | (binary)
|
||||
* --------------------+---------------------------------------------
|
||||
* 0000 0000-0000 007F | 0xxxxxxx
|
||||
* 0000 0080-0000 07FF | 110xxxxx 10xxxxxx
|
||||
* 0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
|
||||
* 0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
|
||||
*/
|
||||
|
||||
|
||||
/*
|
||||
* This may not be the best value, but it's one that isn't represented
|
||||
* in Unicode (0x10FFFF is the largest codepoint value). We return this
|
||||
* value from utf8codepoint() if there's bogus bits in the
|
||||
* stream. utf8codepoint() will turn this value into something
|
||||
* reasonable (like a question mark), for text that wants to try to recover,
|
||||
* whereas utf8valid() will use the value to determine if a string has bad
|
||||
* bits.
|
||||
*/
|
||||
#define UNICODE_BOGUS_CHAR_VALUE 0xFFFFFFFF
|
||||
|
||||
/*
|
||||
* This is the codepoint we currently return when there was bogus bits in a
|
||||
* UTF-8 string. May not fly in Asian locales?
|
||||
*/
|
||||
#define UNICODE_BOGUS_CHAR_CODEPOINT '?'
|
||||
|
||||
static PHYSFS_uint32 utf8codepoint(const char **_str)
|
||||
{
|
||||
const char *str = *_str;
|
||||
PHYSFS_uint32 retval = 0;
|
||||
PHYSFS_uint32 octet = (PHYSFS_uint32) ((PHYSFS_uint8) *str);
|
||||
PHYSFS_uint32 octet2, octet3, octet4;
|
||||
|
||||
if (octet == 0) /* null terminator, end of string. */
|
||||
return 0;
|
||||
|
||||
else if (octet < 128) /* one octet char: 0 to 127 */
|
||||
{
|
||||
(*_str)++; /* skip to next possible start of codepoint. */
|
||||
return(octet);
|
||||
} /* else if */
|
||||
|
||||
else if ((octet > 127) && (octet < 192)) /* bad (starts with 10xxxxxx). */
|
||||
{
|
||||
/*
|
||||
* Apparently each of these is supposed to be flagged as a bogus
|
||||
* char, instead of just resyncing to the next valid codepoint.
|
||||
*/
|
||||
(*_str)++; /* skip to next possible start of codepoint. */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
} /* else if */
|
||||
|
||||
else if (octet < 224) /* two octets */
|
||||
{
|
||||
octet -= (128+64);
|
||||
octet2 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
|
||||
if ((octet2 & (128+64)) != 128) /* Format isn't 10xxxxxx? */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
|
||||
*_str += 2; /* skip to next possible start of codepoint. */
|
||||
retval = ((octet << 6) | (octet2 - 128));
|
||||
if ((retval >= 0x80) && (retval <= 0x7FF))
|
||||
return retval;
|
||||
} /* else if */
|
||||
|
||||
else if (octet < 240) /* three octets */
|
||||
{
|
||||
octet -= (128+64+32);
|
||||
octet2 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
|
||||
if ((octet2 & (128+64)) != 128) /* Format isn't 10xxxxxx? */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
|
||||
octet3 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
|
||||
if ((octet3 & (128+64)) != 128) /* Format isn't 10xxxxxx? */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
|
||||
*_str += 3; /* skip to next possible start of codepoint. */
|
||||
retval = ( ((octet << 12)) | ((octet2-128) << 6) | ((octet3-128)) );
|
||||
|
||||
/* There are seven "UTF-16 surrogates" that are illegal in UTF-8. */
|
||||
switch (retval)
|
||||
{
|
||||
case 0xD800:
|
||||
case 0xDB7F:
|
||||
case 0xDB80:
|
||||
case 0xDBFF:
|
||||
case 0xDC00:
|
||||
case 0xDF80:
|
||||
case 0xDFFF:
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
} /* switch */
|
||||
|
||||
/* 0xFFFE and 0xFFFF are illegal, too, so we check them at the edge. */
|
||||
if ((retval >= 0x800) && (retval <= 0xFFFD))
|
||||
return retval;
|
||||
} /* else if */
|
||||
|
||||
else if (octet < 248) /* four octets */
|
||||
{
|
||||
octet -= (128+64+32+16);
|
||||
octet2 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
|
||||
if ((octet2 & (128+64)) != 128) /* Format isn't 10xxxxxx? */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
|
||||
octet3 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
|
||||
if ((octet3 & (128+64)) != 128) /* Format isn't 10xxxxxx? */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
|
||||
octet4 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
|
||||
if ((octet4 & (128+64)) != 128) /* Format isn't 10xxxxxx? */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
|
||||
*_str += 4; /* skip to next possible start of codepoint. */
|
||||
retval = ( ((octet << 18)) | ((octet2 - 128) << 12) |
|
||||
((octet3 - 128) << 6) | ((octet4 - 128)) );
|
||||
if ((retval >= 0x10000) && (retval <= 0x10FFFF))
|
||||
return retval;
|
||||
} /* else if */
|
||||
|
||||
/*
|
||||
* Five and six octet sequences became illegal in rfc3629.
|
||||
* We throw the codepoint away, but parse them to make sure we move
|
||||
* ahead the right number of bytes and don't overflow the buffer.
|
||||
*/
|
||||
|
||||
else if (octet < 252) /* five octets */
|
||||
{
|
||||
octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
|
||||
if ((octet & (128+64)) != 128) /* Format isn't 10xxxxxx? */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
|
||||
octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
|
||||
if ((octet & (128+64)) != 128) /* Format isn't 10xxxxxx? */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
|
||||
octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
|
||||
if ((octet & (128+64)) != 128) /* Format isn't 10xxxxxx? */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
|
||||
octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
|
||||
if ((octet & (128+64)) != 128) /* Format isn't 10xxxxxx? */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
|
||||
*_str += 5; /* skip to next possible start of codepoint. */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
} /* else if */
|
||||
|
||||
else /* six octets */
|
||||
{
|
||||
octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
|
||||
if ((octet & (128+64)) != 128) /* Format isn't 10xxxxxx? */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
|
||||
octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
|
||||
if ((octet & (128+64)) != 128) /* Format isn't 10xxxxxx? */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
|
||||
octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
|
||||
if ((octet & (128+64)) != 128) /* Format isn't 10xxxxxx? */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
|
||||
octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
|
||||
if ((octet & (128+64)) != 128) /* Format isn't 10xxxxxx? */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
|
||||
octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
|
||||
if ((octet & (128+64)) != 128) /* Format isn't 10xxxxxx? */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
|
||||
*_str += 6; /* skip to next possible start of codepoint. */
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
} /* else if */
|
||||
|
||||
return UNICODE_BOGUS_CHAR_VALUE;
|
||||
} /* utf8codepoint */
|
||||
|
||||
void PHYSFS_utf8toucs4(const char *src, PHYSFS_uint32 *dst, PHYSFS_uint64 len)
|
||||
{
|
||||
len -= sizeof (PHYSFS_uint32); /* save room for null char. */
|
||||
while (len >= sizeof (PHYSFS_uint32))
|
||||
{
|
||||
PHYSFS_uint32 cp = utf8codepoint(&src);
|
||||
if (cp == 0)
|
||||
break;
|
||||
else if (cp == UNICODE_BOGUS_CHAR_VALUE)
|
||||
cp = UNICODE_BOGUS_CHAR_CODEPOINT;
|
||||
*(dst++) = cp;
|
||||
len -= sizeof (PHYSFS_uint32);
|
||||
} /* while */
|
||||
|
||||
*dst = 0;
|
||||
} /* PHYSFS_utf8toucs4 */
|
||||
|
||||
void PHYSFS_utf8toucs2(const char *src, PHYSFS_uint16 *dst, PHYSFS_uint64 len)
|
||||
{
|
||||
len -= sizeof (PHYSFS_uint16); /* save room for null char. */
|
||||
while (len >= sizeof (PHYSFS_uint16))
|
||||
{
|
||||
PHYSFS_uint32 cp = utf8codepoint(&src);
|
||||
if (cp == 0)
|
||||
break;
|
||||
else if (cp == UNICODE_BOGUS_CHAR_VALUE)
|
||||
cp = UNICODE_BOGUS_CHAR_CODEPOINT;
|
||||
|
||||
/* !!! BLUESKY: UTF-16 surrogates? */
|
||||
if (cp > 0xFFFF)
|
||||
cp = UNICODE_BOGUS_CHAR_CODEPOINT;
|
||||
|
||||
*(dst++) = cp;
|
||||
len -= sizeof (PHYSFS_uint16);
|
||||
} /* while */
|
||||
|
||||
*dst = 0;
|
||||
} /* PHYSFS_utf8toucs2 */
|
||||
|
||||
static void utf8fromcodepoint(PHYSFS_uint32 cp, char **_dst, PHYSFS_uint64 *_len)
|
||||
{
|
||||
char *dst = *_dst;
|
||||
PHYSFS_uint64 len = *_len;
|
||||
|
||||
if (len == 0)
|
||||
return;
|
||||
|
||||
if (cp > 0x10FFFF)
|
||||
cp = UNICODE_BOGUS_CHAR_CODEPOINT;
|
||||
else if ((cp == 0xFFFE) || (cp == 0xFFFF)) /* illegal values. */
|
||||
cp = UNICODE_BOGUS_CHAR_CODEPOINT;
|
||||
else
|
||||
{
|
||||
/* There are seven "UTF-16 surrogates" that are illegal in UTF-8. */
|
||||
switch (cp)
|
||||
{
|
||||
case 0xD800:
|
||||
case 0xDB7F:
|
||||
case 0xDB80:
|
||||
case 0xDBFF:
|
||||
case 0xDC00:
|
||||
case 0xDF80:
|
||||
case 0xDFFF:
|
||||
cp = UNICODE_BOGUS_CHAR_CODEPOINT;
|
||||
} /* switch */
|
||||
} /* else */
|
||||
|
||||
/* Do the encoding... */
|
||||
if (cp < 0x80)
|
||||
{
|
||||
*(dst++) = (char) cp;
|
||||
len--;
|
||||
} /* if */
|
||||
|
||||
else if (cp < 0x800)
|
||||
{
|
||||
if (len < 2)
|
||||
len = 0;
|
||||
else
|
||||
{
|
||||
*(dst++) = (char) ((cp >> 6) | 128 | 64);
|
||||
*(dst++) = (char) (cp & 0x3F) | 128;
|
||||
len -= 2;
|
||||
} /* else */
|
||||
} /* else if */
|
||||
|
||||
else if (cp < 0x10000)
|
||||
{
|
||||
if (len < 3)
|
||||
len = 0;
|
||||
else
|
||||
{
|
||||
*(dst++) = (char) ((cp >> 12) | 128 | 64 | 32);
|
||||
*(dst++) = (char) ((cp >> 6) & 0x3F) | 128;
|
||||
*(dst++) = (char) (cp & 0x3F) | 128;
|
||||
len -= 3;
|
||||
} /* else */
|
||||
} /* else if */
|
||||
|
||||
else
|
||||
{
|
||||
if (len < 4)
|
||||
len = 0;
|
||||
else
|
||||
{
|
||||
*(dst++) = (char) ((cp >> 18) | 128 | 64 | 32 | 16);
|
||||
*(dst++) = (char) ((cp >> 12) & 0x3F) | 128;
|
||||
*(dst++) = (char) ((cp >> 6) & 0x3F) | 128;
|
||||
*(dst++) = (char) (cp & 0x3F) | 128;
|
||||
len -= 4;
|
||||
} /* else if */
|
||||
} /* else */
|
||||
|
||||
*_dst = dst;
|
||||
*_len = len;
|
||||
} /* utf8fromcodepoint */
|
||||
|
||||
#define UTF8FROMTYPE(typ, src, dst, len) \
|
||||
len--; \
|
||||
while (len) \
|
||||
{ \
|
||||
const PHYSFS_uint32 cp = (PHYSFS_uint32) *(src++); \
|
||||
if (cp == 0) break; \
|
||||
utf8fromcodepoint(cp, &dst, &len); \
|
||||
} \
|
||||
*dst = '\0'; \
|
||||
|
||||
void PHYSFS_utf8fromucs4(const PHYSFS_uint32 *src, char *dst, PHYSFS_uint64 len)
|
||||
{
|
||||
UTF8FROMTYPE(PHYSFS_uint32, src, dst, len);
|
||||
} /* PHYSFS_utf8fromucs4 */
|
||||
|
||||
void PHYSFS_utf8fromucs2(const PHYSFS_uint16 *src, char *dst, PHYSFS_uint64 len)
|
||||
{
|
||||
UTF8FROMTYPE(PHYSFS_uint64, src, dst, len);
|
||||
} /* PHYSFS_utf8fromucs4 */
|
||||
|
||||
/* latin1 maps to unicode codepoints directly, we just utf-8 encode it. */
|
||||
void PHYSFS_utf8fromlatin1(const char *src, char *dst, PHYSFS_uint64 len)
|
||||
{
|
||||
UTF8FROMTYPE(PHYSFS_uint8, src, dst, len);
|
||||
} /* PHYSFS_utf8fromlatin1 */
|
||||
|
||||
#undef UTF8FROMTYPE
|
||||
|
||||
/* end of physfs_unicode.c ... */
|
||||
|
||||
Loading…
Reference in New Issue