Menu
▾
▴
SF.net SVN: frontierkernel: [1269] Frontier/branches/Conversant/Common/source
|
From: <set...@us...> - 2006-04-15 04:58:16
|
Revision: 1269 Author: sethdill Date: 2006-04-14 21:58:06 -0700 (Fri, 14 Apr 2006) ViewCVS: http://svn.sourceforge.net/frontierkernel/?rev=1269&view=rev Log Message: ----------- Big udate for character encoding. Added cross-platform verb, string.convertTextEncoding( charsetIn, charsetOut, s ) Platform-specific codes for a subset of the known character sets are now stored in the compiler table. For example, system.compiler.language.charsets.[UTF-16] has the value 256 on Mac OS, corresponding to the constant kCFStringEncodingUnicode in CFString.h. However, on Windows, the value of system.compiler.language.charsets.[UTF-16] is 0. (Actually, bad example, because UTF-16 is the ONLY character set on Windows that isn't a code page.) Modified Paths: -------------- Frontier/branches/Conversant/Common/headers/lang.h Frontier/branches/Conversant/Common/headers/strings.h Frontier/branches/Conversant/Common/resources/Mac/kernelverbs.r Frontier/branches/Conversant/Common/source/langstartup.c Frontier/branches/Conversant/Common/source/strings.c Frontier/branches/Conversant/Common/source/stringverbs.c Modified: Frontier/branches/Conversant/Common/headers/lang.h =================================================================== --- Frontier/branches/Conversant/Common/headers/lang.h 2006-04-13 21:45:39 UTC (rev 1268) +++ Frontier/branches/Conversant/Common/headers/lang.h 2006-04-15 04:58:06 UTC (rev 1269) @@ -725,6 +725,8 @@ extern hdlhashtable hconsttable; /*holds the names of built-in constants, like true and up*/ +extern hdlhashtable hcharsetconsttable; /* holds the names of the built-in character sets, like iso-8859-1, MacRoman, and UTF-8 */ + extern hdlhashtable hbuiltinfunctions; /*holds the names of the built-in functions*/ extern unsigned long ctscanlines; /*number of lines that have been scanned, for error reporting*/ Modified: Frontier/branches/Conversant/Common/headers/strings.h =================================================================== --- Frontier/branches/Conversant/Common/headers/strings.h 2006-04-13 21:45:39 UTC (rev 1268) +++ Frontier/branches/Conversant/Common/headers/strings.h 2006-04-15 04:58:06 UTC (rev 1269) @@ -266,5 +266,7 @@ extern boolean utf8tomacroman (Handle, Handle); /* 2006-02-25 creedon */ +extern boolean converttextencoding (Handle, Handle, long, long); + #endif Modified: Frontier/branches/Conversant/Common/resources/Mac/kernelverbs.r =================================================================== --- Frontier/branches/Conversant/Common/resources/Mac/kernelverbs.r 2006-04-13 21:45:39 UTC (rev 1268) +++ Frontier/branches/Conversant/Common/resources/Mac/kernelverbs.r 2006-04-15 04:58:06 UTC (rev 1269) @@ -600,7 +600,8 @@ "ansitoutf16", "multipleReplaceAll", "macromantoutf8", // 2006-02-25 creedon - "utf8tomacroman" // 2006-02-25 creedon + "utf8tomacroman", // 2006-02-25 creedon + "convertcharset" /* 2006-04-13 */ } } }; Modified: Frontier/branches/Conversant/Common/source/langstartup.c =================================================================== --- Frontier/branches/Conversant/Common/source/langstartup.c 2006-04-13 21:45:39 UTC (rev 1268) +++ Frontier/branches/Conversant/Common/source/langstartup.c 2006-04-15 04:58:06 UTC (rev 1269) @@ -579,6 +579,100 @@ #define addstring(x,y) if (!langaddstringconst ((ptrstring) x, y)) return (false) +static boolean langinit_charset_consttable (void) { + + hdlhashtable hcharsetconsttable; + + if (!tablenewsystemtable (langtable, (ptrstring) "\x08" "charsets", &hcharsetconsttable)) + return (false); + + pushhashtable (hcharsetconsttable); + +#if MACVERSION + addlong( "ASCII", kCFStringEncodingASCII ); + addlong( "MacRoman", kCFStringEncodingMacRoman ); + addlong( "MACINTOSH", kCFStringEncodingMacRoman ); + + addlong( "iso-8859-1", kCFStringEncodingISOLatin1 ); + addlong( "iso-latin-1", kCFStringEncodingISOLatin1 ); + addlong( "iso-8859-2", kCFStringEncodingISOLatin2 ); + addlong( "iso-8859-3", kCFStringEncodingISOLatin3 ); + addlong( "iso-8859-4", kCFStringEncodingISOLatin4 ); + addlong( "iso-8859-5", kCFStringEncodingISOLatin5 ); + addlong( "iso-8859-6", kCFStringEncodingISOLatin6 ); + addlong( "iso-8859-7", kCFStringEncodingISOLatin7 ); + addlong( "iso-8859-8", kCFStringEncodingISOLatin8 ); + addlong( "iso-8859-9", kCFStringEncodingISOLatin9 ); + addlong( "iso-8859-10", kCFStringEncodingISOLatin10 ); + + addlong( "windows-1250", kCFStringEncodingWindowsLatin2 ); + addlong( "windows-latin-2", kCFStringEncodingWindowsLatin2 ); /* code page 1250, Central Europe */ + + addlong( "windows-1251", kCFStringEncodingWindowsCyrillic ); /* code page 1251, Slavic Cyrillic */ + + addlong( "windows-1252", kCFStringEncodingWindowsLatin1 ); + addlong( "windows-latin-1", kCFStringEncodingWindowsLatin1 ); /* ANSI 1252 */ + + addlong( "windows-1253", kCFStringEncodingWindowsLatin5 ); /* code page 1253 */ + addlong( "windows-1254", kCFStringEncodingWindowsLatin5 ); /* code page 1254, Turkish */ + addlong( "windows-1255", kCFStringEncodingWindowsHebrew ); /* code page 1255 */ + addlong( "windows-1256", kCFStringEncodingWindowsArabic ); /* code page 1256 */ + addlong( "windows-1257", kCFStringEncodingWindowsBalticRim ); /* code page 1257 */ + addlong( "windows-1258", kCFStringEncodingWindowsVietnamese ); /* code page 1258 */ + addlong( "windows-1361", kCFStringEncodingWindowsKoreanJohab ); /* code page 1361, for Windows NT */ + + addlong( "UTF-7", kCFStringEncodingNonLossyASCII ); + addlong( "UTF-8", kCFStringEncodingUTF8 ); + addlong( "UTF-16", kCFStringEncodingUnicode ); + addlong( "UTF-16le", kCFStringEncodingUTF16LE ); + addlong( "UTF-16be", kCFStringEncodingUTF16BE ); +#endif + +#if WIN95VERSION + addlong( "ASCII", 20127 ); + addlong( "MacRoman", 10000 ); + addlong( "MACINTOSH", 10000 ); + + addlong( "iso-8859-1", 28591 ); + addlong( "iso-latin-1", 28591 ); + addlong( "iso-8859-2", 28592 ); + addlong( "iso-8859-3", 28593 ); + addlong( "iso-8859-4", 28594 ); + addlong( "iso-8859-5", 28595 ); + addlong( "iso-8859-6", 28596 ); + addlong( "iso-8859-7", 28597 ); + addlong( "iso-8859-8", 28598 ); + addlong( "iso-8859-9", 28599 ); + addlong( "iso-8859-10", 28592 ); + + addlong( "windows-1250", 1250 ); + addlong( "windows-latin-2", 1250 ); /* code page 1250, Central Europe */ + + addlong( "windows-1251", 1251 ); /* code page 1251, Slavic Cyrillic */ + + addlong( "windows-1252", 1252 ); + addlong( "windows-latin-1", 1252 ); + + addlong( "windows-1253", 1253 ); /* code page 1253 */ + addlong( "windows-1254", 1254 ); /* code page 1254, Turkish */ + addlong( "windows-1255", 1255 ); /* code page 1255 */ + addlong( "windows-1256", 1256 ); /* code page 1256 */ + addlong( "windows-1257", 1257 ); /* code page 1257 */ + addlong( "windows-1258", 1258 ); /* code page 1258 */ + addlong( "windows-1361", 1361 ); /* code page 1361, for Windows NT */ + + addlong( "UTF-7", 65000 ); + addlong( "UTF-8", 65001 ); + addlong( "UTF-16", 0 ); + addlong( "UTF-16le", 1200 ); + addlong( "UTF-16be", 1201 ); +#endif + + pophashtable(); + + return (true); + } /* langinit_charset_consttable */ + static boolean langinitconsttable (void) { /* @@ -591,7 +685,7 @@ tyvaluetype type; bigstring bs; - if (!tablenewsystemtable (langtable, (ptrstring) BIGSTRING ("\x09" "constants"), &hconsttable)) + if (!tablenewsystemtable (langtable, (ptrstring) "\x09" "constants", &hconsttable)) return (false); pushhashtable (hconsttable); /*converted to constants by the scanner*/ @@ -649,7 +743,7 @@ lastword (bs, chspace, bs); - pushstring ((ptrstring) BIGSTRING ("\x04Type"), bs); + pushstring ((ptrstring) "\x04Type", bs); addtype (bs, type); @@ -808,6 +902,9 @@ if (!langinitconsttable ()) return (false); + if (!langinit_charset_consttable ()) + return (false); + if (!langinitbuiltintable ()) return (false); Modified: Frontier/branches/Conversant/Common/source/strings.c =================================================================== --- Frontier/branches/Conversant/Common/source/strings.c 2006-04-13 21:45:39 UTC (rev 1268) +++ Frontier/branches/Conversant/Common/source/strings.c 2006-04-15 04:58:06 UTC (rev 1269) @@ -2265,10 +2265,10 @@ ULONG cbAnsi, cCharacters; long lentext; - + cCharacters = wcslen((const unsigned short*) *h)+1; - cbAnsi = cCharacters*2; + cbAnsi = cCharacters * 2; sethandlesize (hresult, cbAnsi); @@ -2313,16 +2313,17 @@ char ch = (*hresult) [ix]; if (ch == '\0') { + + if ((*hresult) [ix + 1] == '\0') { - if (lastch == '\0') { + sethandlesize (hresult, ix + 2); - sethandlesize (hresult, ix); - break; - } /*if*/ + } /* if */ + } /*if*/ - ix++; + ix += 2; if (ix > cbUni) break; @@ -2337,10 +2338,10 @@ #endif /*end Windows character conversion routines*/ +boolean converttextencoding (Handle h, Handle hresult, long inputcharset, long outputcharset) { + #ifdef MACVERSION /*Mac character conversion common routine*/ -static boolean converttextencoding (Handle h, Handle hresult, long inputcharset, long outputcharset) { - TECObjectRef converter; OSStatus status; ByteCount ctorigbytes, ctoutputbytes, ctflushedbytes; @@ -2407,12 +2408,76 @@ // kwchange 2005-05-23 --- commented out to see if this is the missing trailing char bug // kwchange 2005-05-27 --- ran this the last week and have no complains //sethandlesize (hresult, gethandlesize (hresult) - 1); /*pop trailing terminator*/ - } /*converttextencoding*/ - #endif /*end Mac character conversion routine(s)*/ +#ifdef WIN95VERSION + long lentext; + + switch ( inputcharset ) { + + case 0: /* unicode with BOM, not handled as a code page. I Hatesk Windows. */ + lentext = gethandlesize (h); + + sethandlesize (h,lentext + 2); + + (*h) [lentext] = '\0'; + (*h) [lentext + 1] = '\0'; + + if (UnicodeToAnsi (h, hresult, outputcharset)) + return (false); + + if ((*h) [0] == '\xFF') + pullfromhandle (hresult,0, 1, NULL); /* Pop off the BOM */ + + default: { + lentext = gethandlesize (h); + + switch ( outputcharset ) { + case 1200: /* wide, utf-16 */ + case 1201: /* wide, utf-16 */ + + if (AnsiToUnicode (h, hresult, inputcharset)) + return (false); + + default: { + Handle htemp; + boolean flResult = true; + + newemptyhandle (&htemp); + + sethandlesize (h, lentext + 1); + (*h) [lentext] = '\0'; + + if (AnsiToUnicode (h, htemp, inputcharset)) + flResult = false; + + if (flResult && ! UnicodeToAnsi (htemp, hresult, outputcharset)) { + + if ((*hresult) [0] == '\xFF') + pullfromhandle (hresult, 0, 1, NULL); /* pop BOM ? */ + + } /* unicodetoansi */ + else + flResult = false; + + disposehandle (htemp); + + return (flResult); + + } /* case default */ + + } /* switch outputcharset */ + + } /* case default */ + + } /* switch inputcharset */ + +#endif + } /*converttextencoding*/ + + boolean utf16toansi (Handle h, Handle hresult) { /* @@ -2530,7 +2595,7 @@ #ifdef MACVERSION - if (!converttextencoding (h, hresult, 0x08000100, kTextEncodingWindowsLatin1)) + if (!converttextencoding (h, hresult, kCFStringEncodingUTF8, kTextEncodingWindowsLatin1)) return (false); /* OSErr err = noErr; @@ -2600,7 +2665,7 @@ #ifdef MACVERSION - if (!converttextencoding (h, hresult, kTextEncodingWindowsLatin1, 0x08000100)) + if (!converttextencoding (h, hresult, kTextEncodingWindowsLatin1, kCFStringEncodingUTF8)) return (false); #endif @@ -2713,13 +2778,14 @@ #ifdef WIN95VERSION - langerrormessage ("\x92" "Direct Mac Roman to UTF-8 string conversion currently only works on Mac OS. On Windows string.macToLatin and string.ansiToUtf8 verbs can be used."); + if (!converttextencoding (h, hresult, 10000, CP_UTF8)) + return (false); #endif #ifdef MACVERSION - if (!converttextencoding (h, hresult, kTextEncodingMacRoman, 0x08000100)) + if (!converttextencoding (h, hresult, kTextEncodingMacRoman, kCFStringEncodingUTF8)) return (false); #endif @@ -2736,13 +2802,14 @@ #ifdef WIN95VERSION - langerrormessage ("\x92" "Direct UTF-8 to Mac Roman string conversion currently only works on Mac OS. On Windows string.utf8ToAnsi and string.latinToMac verbs can be used."); + if (!converttextencoding (h, hresult, CP_UTF8, 10000)) + return (false); #endif #ifdef MACVERSION - if (!converttextencoding (h, hresult, 0x08000100, kTextEncodingMacRoman)) + if (!converttextencoding (h, hresult, kCFStringEncodingUTF8, kTextEncodingMacRoman)) return (false); #endif Modified: Frontier/branches/Conversant/Common/source/stringverbs.c =================================================================== --- Frontier/branches/Conversant/Common/source/stringverbs.c 2006-04-13 21:45:39 UTC (rev 1268) +++ Frontier/branches/Conversant/Common/source/stringverbs.c 2006-04-15 04:58:06 UTC (rev 1269) @@ -310,6 +310,8 @@ macromantoutf8func, // 2006-02-25 creedon utf8tomacromanfunc, // 2006-02-25 creedon + + convertcharsetfunc, /* 2006-04-14 smd */ ctstringverbs } tystringtoken; @@ -2303,6 +2305,32 @@ return (setheapvalue (hresult, stringvaluetype, v)); } + + case convertcharsetfunc: { /* 2006-04-13 smd: convert any character set to any other character set, if the os supports it */ + + Handle h, hresult; + long charsetIn, charsetOut; + + if (!getlongvalue (hp1, 1, &charsetIn)) + goto error; + + if (!getlongvalue (hp1, 2, &charsetOut)) + goto error; + + flnextparamislast = true; + + if (!getexempttextvalue (hp1, 3, &h)) + goto error; + + newemptyhandle (&hresult); + + if (! converttextencoding (h, hresult, charsetIn, charsetOut)) + goto error; + + disposehandle(h); + + return (setheapvalue (hresult, stringvaluetype, v)); + } default: errornum = notimplementederror; This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
