[OMPT-svn-commit] SF.net SVN: modplug:[4351] trunk/OpenMPT

[<man...@us...>]

Revision: 4351
          http://sourceforge.net/p/modplug/code/4351
Author:   manxorist
Date:     2014-10-02 07:21:29 +0000 (Thu, 02 Oct 2014)
Log Message:
-----------
[Ref] Add MPT_UTF8 macro which allows defining mpt::ustring strings with non-ASCII characters by explicitely writing them in UTF-8 encoded form.
[Ref] test: Add conversion test cases for mpt::ustring.

Modified Paths:
--------------
    trunk/OpenMPT/common/mptString.h
    trunk/OpenMPT/test/test.cpp

Modified: trunk/OpenMPT/common/mptString.h
===================================================================
--- trunk/OpenMPT/common/mptString.h	2014-10-02 06:49:00 UTC (rev 4350)
+++ trunk/OpenMPT/common/mptString.h	2014-10-02 07:21:29 UTC (rev 4351)
@@ -340,6 +340,13 @@
 #endif // MFC
 #endif // MPT_USTRING_MODE_WIDE
 
+// The MPT_UTF8 allows specifying UTF8 char arrays.
+// The resulting type is mpt::ustring and the construction might require runtime translation,
+// i.e. it is NOT generally available at compile time.
+// Use explicit UTF8 encoding,
+// i.e. U+00FC (LATIN SMALL LETTER U WITH DIAERESIS) would be written as "\xC3\xBC".
+#define MPT_UTF8(x) mpt::ToUnicode(mpt::CharsetUTF8, x )
+
 } // namespace mpt
 
 

Modified: trunk/OpenMPT/test/test.cpp
===================================================================
--- trunk/OpenMPT/test/test.cpp	2014-10-02 06:49:00 UTC (rev 4350)
+++ trunk/OpenMPT/test/test.cpp	2014-10-02 07:21:29 UTC (rev 4351)
@@ -64,6 +64,7 @@
 static noinline void TestVersion();
 static noinline void TestTypes();
 static noinline void TestMisc();
+static noinline void TestCharsets();
 static noinline void TestStringFormatting();
 static noinline void TestSettings();
 static noinline void TestStringIO();
@@ -103,6 +104,7 @@
 	DO_TEST(TestVersion);
 	DO_TEST(TestTypes);
 	DO_TEST(TestMisc);
+	DO_TEST(TestCharsets);
 	DO_TEST(TestStringFormatting);
 	DO_TEST(TestSettings);
 	DO_TEST(TestStringIO);
@@ -584,7 +586,125 @@
 		VERIFY_EQUAL(strlen(ModSpecs::Collection[i]->volcommands), (size_t)MAX_VOLCMDS);
 	}
 	
+	// UUID
+#ifdef MODPLUG_TRACKER
+	VERIFY_EQUAL(Util::IsValid(Util::CreateGUID()), true);
+	VERIFY_EQUAL(Util::IsValid(Util::CreateUUID()), true);
+	VERIFY_EQUAL(Util::IsValid(Util::CreateLocalUUID()), true);
+	UUID uuid = Util::CreateUUID();
+	VERIFY_EQUAL(IsEqualUUID(uuid, Util::StringToUUID(Util::UUIDToString(uuid))), true);
+	VERIFY_EQUAL(IsEqualUUID(uuid, Util::StringToGUID(Util::GUIDToString(uuid))), true);
+	VERIFY_EQUAL(IsEqualUUID(uuid, Util::StringToIID(Util::IIDToString(uuid))), true);
+	VERIFY_EQUAL(IsEqualUUID(uuid, Util::StringToCLSID(Util::CLSIDToString(uuid))), true);
+#endif
+
+}
+
+
+static noinline void TestCharsets()
+//---------------------------------
+{
+
+	// MPT_UTF8 version
+
 	// Charset conversions (basic sanity checks)
+	VERIFY_EQUAL(mpt::ToCharset(mpt::CharsetUTF8, MPT_USTRING("a")), "a");
+	VERIFY_EQUAL(mpt::ToCharset(mpt::CharsetISO8859_1, MPT_USTRING("a")), "a");
+	VERIFY_EQUAL(mpt::ToCharset(mpt::CharsetASCII, MPT_USTRING("a")), "a");
+	VERIFY_EQUAL(mpt::ToWide(mpt::CharsetUTF8, "a"), MPT_USTRING("a"));
+	VERIFY_EQUAL(mpt::ToWide(mpt::CharsetISO8859_1, "a"), MPT_USTRING("a"));
+	VERIFY_EQUAL(mpt::ToWide(mpt::CharsetASCII, "a"), MPT_USTRING("a"));
+#if defined(MPT_WITH_CHARSET_LOCALE)
+	VERIFY_EQUAL(mpt::ToLocale(MPT_USTRING("a")), "a");
+	VERIFY_EQUAL(mpt::ToWide(mpt::CharsetLocale, "a"), MPT_USTRING("a"));
+#endif
+	VERIFY_EQUAL(mpt::ToCharset(mpt::CharsetUTF8, MPT_UTF8("a")), "a");
+	VERIFY_EQUAL(mpt::ToCharset(mpt::CharsetISO8859_1, MPT_UTF8("a")), "a");
+	VERIFY_EQUAL(mpt::ToCharset(mpt::CharsetASCII, MPT_UTF8("a")), "a");
+	VERIFY_EQUAL(mpt::ToWide(mpt::CharsetUTF8, "a"), MPT_UTF8("a"));
+	VERIFY_EQUAL(mpt::ToWide(mpt::CharsetISO8859_1, "a"), MPT_UTF8("a"));
+	VERIFY_EQUAL(mpt::ToWide(mpt::CharsetASCII, "a"), MPT_UTF8("a"));
+#if defined(MPT_WITH_CHARSET_LOCALE)
+	VERIFY_EQUAL(mpt::ToLocale(MPT_UTF8("a")), "a");
+	VERIFY_EQUAL(mpt::ToWide(mpt::CharsetLocale, "a"), MPT_UTF8("a"));
+#endif
+	
+	// Check that some character replacement is done (and not just empty strings or truncated strings are returned)
+	// We test german umlaut-a (U+00E4) (\xC3\xA4) and CJK U+5BB6 (\xE5\xAE\xB6)
+
+	VERIFY_EQUAL(EndsWith(mpt::ToCharset(mpt::CharsetASCII,MPT_UTF8("abc\xC3\xA4xyz")),"xyz"),true);
+	VERIFY_EQUAL(EndsWith(mpt::ToCharset(mpt::CharsetISO8859_1,MPT_UTF8("abc\xC3\xA4xyz")),"xyz"),true);
+	VERIFY_EQUAL(EndsWith(mpt::ToCharset(mpt::CharsetCP437,MPT_UTF8("abc\xC3\xA4xyz")),"xyz"),true);
+	VERIFY_EQUAL(EndsWith(mpt::ToCharset(mpt::CharsetUTF8,MPT_UTF8("abc\xC3\xA4xyz")),"xyz"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToCharset(mpt::CharsetASCII,MPT_UTF8("abc\xC3\xA4xyz")),"abc"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToCharset(mpt::CharsetISO8859_1,MPT_UTF8("abc\xC3\xA4xyz")),"abc"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToCharset(mpt::CharsetCP437,MPT_UTF8("abc\xC3\xA4xyz")),"abc"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToCharset(mpt::CharsetUTF8,MPT_UTF8("abc\xC3\xA4xyz")),"abc"),true);
+#if defined(MPT_WITH_CHARSET_LOCALE)
+	VERIFY_EQUAL(EndsWith(mpt::ToCharset(mpt::CharsetLocale,MPT_UTF8("abc\xC3\xA4xyz")),"xyz"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToCharset(mpt::CharsetLocale,MPT_UTF8("abc\xC3\xA4xyz")),"abc"),true);
+#endif
+
+	VERIFY_EQUAL(EndsWith(mpt::ToCharset(mpt::CharsetASCII,MPT_UTF8("abc\xE5\xAE\xB6xyz")),"xyz"),true);
+	VERIFY_EQUAL(EndsWith(mpt::ToCharset(mpt::CharsetISO8859_1,MPT_UTF8("abc\xE5\xAE\xB6xyz")),"xyz"),true);
+	VERIFY_EQUAL(EndsWith(mpt::ToCharset(mpt::CharsetCP437,MPT_UTF8("abc\xE5\xAE\xB6xyz")),"xyz"),true);
+	VERIFY_EQUAL(EndsWith(mpt::ToCharset(mpt::CharsetUTF8,MPT_UTF8("abc\xE5\xAE\xB6xyz")),"xyz"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToCharset(mpt::CharsetASCII,MPT_UTF8("abc\xE5\xAE\xB6xyz")),"abc"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToCharset(mpt::CharsetISO8859_1,MPT_UTF8("abc\xE5\xAE\xB6xyz")),"abc"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToCharset(mpt::CharsetCP437,MPT_UTF8("abc\xE5\xAE\xB6xyz")),"abc"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToCharset(mpt::CharsetUTF8,MPT_UTF8("abc\xE5\xAE\xB6xyz")),"abc"),true);
+#if defined(MPT_WITH_CHARSET_LOCALE)
+	VERIFY_EQUAL(EndsWith(mpt::ToCharset(mpt::CharsetLocale,MPT_UTF8("abc\xE5\xAE\xB6xyz")),"xyz"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToCharset(mpt::CharsetLocale,MPT_UTF8("abc\xE5\xAE\xB6xyz")),"abc"),true);
+#endif
+
+	VERIFY_EQUAL(EndsWith(mpt::ToWide(mpt::CharsetASCII,"abc\xC3\xA4xyz"),L"xyz"),true);
+	VERIFY_EQUAL(EndsWith(mpt::ToWide(mpt::CharsetISO8859_1,"abc\xC3\xA4xyz"),L"xyz"),true);
+	VERIFY_EQUAL(EndsWith(mpt::ToWide(mpt::CharsetCP437,"abc\xC3\xA4xyz"),L"xyz"),true);
+	VERIFY_EQUAL(EndsWith(mpt::ToWide(mpt::CharsetUTF8,"abc\xC3\xA4xyz"),L"xyz"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToWide(mpt::CharsetASCII,"abc\xC3\xA4xyz"),L"abc"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToWide(mpt::CharsetISO8859_1,"abc\xC3\xA4xyz"),L"abc"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToWide(mpt::CharsetCP437,"abc\xC3\xA4xyz"),L"abc"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToWide(mpt::CharsetUTF8,"abc\xC3\xA4xyz"),L"abc"),true);
+#if defined(MPT_WITH_CHARSET_LOCALE)
+	VERIFY_EQUAL(EndsWith(mpt::ToWide(mpt::CharsetLocale,"abc\xC3\xA4xyz"),L"xyz"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToWide(mpt::CharsetLocale,"abc\xC3\xA4xyz"),L"abc"),true);
+#endif
+
+	VERIFY_EQUAL(EndsWith(mpt::ToWide(mpt::CharsetASCII,"abc\xE5\xAE\xB6xyz"),L"xyz"),true);
+	VERIFY_EQUAL(EndsWith(mpt::ToWide(mpt::CharsetISO8859_1,"abc\xE5\xAE\xB6xyz"),L"xyz"),true);
+	VERIFY_EQUAL(EndsWith(mpt::ToWide(mpt::CharsetCP437,"abc\xE5\xAE\xB6xyz"),L"xyz"),true);
+	VERIFY_EQUAL(EndsWith(mpt::ToWide(mpt::CharsetUTF8,"abc\xE5\xAE\xB6xyz"),L"xyz"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToWide(mpt::CharsetASCII,"abc\xE5\xAE\xB6xyz"),L"abc"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToWide(mpt::CharsetISO8859_1,"abc\xE5\xAE\xB6xyz"),L"abc"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToWide(mpt::CharsetCP437,"abc\xE5\xAE\xB6xyz"),L"abc"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToWide(mpt::CharsetUTF8,"abc\xE5\xAE\xB6xyz"),L"abc"),true);
+#if defined(MPT_WITH_CHARSET_LOCALE)
+	VERIFY_EQUAL(EndsWith(mpt::ToWide(mpt::CharsetLocale,"abc\xE5\xAE\xB6xyz"),L"xyz"),true);
+	VERIFY_EQUAL(BeginsWith(mpt::ToWide(mpt::CharsetLocale,"abc\xE5\xAE\xB6xyz"),L"abc"),true);
+#endif
+
+	// Check that characters are correctly converted
+	// We test german umlaut-a (U+00E4) and CJK U+5BB6
+
+	// cp437
+	VERIFY_EQUAL(mpt::ToCharset(mpt::CharsetCP437,MPT_UTF8("abc\xC3\xA4xyz")),"abc\x84xyz");
+	VERIFY_EQUAL(MPT_UTF8("abc\xC3\xA4xyz"),mpt::ToWide(mpt::CharsetCP437,"abc\x84xyz"));
+	
+	// iso8859
+	VERIFY_EQUAL(mpt::ToCharset(mpt::CharsetISO8859_1,MPT_UTF8("abc\xC3\xA4xyz")),"abc\xE4xyz");
+	VERIFY_EQUAL(MPT_UTF8("abc\xC3\xA4xyz"),mpt::ToWide(mpt::CharsetISO8859_1,"abc\xE4xyz"));
+
+	// utf8
+	VERIFY_EQUAL(mpt::ToCharset(mpt::CharsetUTF8,MPT_UTF8("abc\xC3\xA4xyz")),"abc\xC3\xA4xyz");
+	VERIFY_EQUAL(MPT_UTF8("abc\xC3\xA4xyz"),mpt::ToWide(mpt::CharsetUTF8,"abc\xC3\xA4xyz"));
+	VERIFY_EQUAL(mpt::ToCharset(mpt::CharsetUTF8,MPT_UTF8("abc\xE5\xAE\xB6xyz")),"abc\xE5\xAE\xB6xyz");
+	VERIFY_EQUAL(MPT_UTF8("abc\xE5\xAE\xB6xyz"),mpt::ToWide(mpt::CharsetUTF8,"abc\xE5\xAE\xB6xyz"));
+
+
+	// wide L"" version
+
+	// Charset conversions (basic sanity checks)
 	VERIFY_EQUAL(mpt::ToCharset(mpt::CharsetUTF8, L"a"), "a");
 	VERIFY_EQUAL(mpt::ToCharset(mpt::CharsetISO8859_1, L"a"), "a");
 	VERIFY_EQUAL(mpt::ToCharset(mpt::CharsetASCII, L"a"), "a");
@@ -681,18 +801,6 @@
 	VERIFY_EQUAL(MPT_PATHSTRING("\\\\server\\path\\file").RelativePathToAbsolute(exePath), MPT_PATHSTRING("\\\\server\\path\\file"));
 #endif
 
-	// UUID
-#ifdef MODPLUG_TRACKER
-	VERIFY_EQUAL(Util::IsValid(Util::CreateGUID()), true);
-	VERIFY_EQUAL(Util::IsValid(Util::CreateUUID()), true);
-	VERIFY_EQUAL(Util::IsValid(Util::CreateLocalUUID()), true);
-	UUID uuid = Util::CreateUUID();
-	VERIFY_EQUAL(IsEqualUUID(uuid, Util::StringToUUID(Util::UUIDToString(uuid))), true);
-	VERIFY_EQUAL(IsEqualUUID(uuid, Util::StringToGUID(Util::GUIDToString(uuid))), true);
-	VERIFY_EQUAL(IsEqualUUID(uuid, Util::StringToIID(Util::IIDToString(uuid))), true);
-	VERIFY_EQUAL(IsEqualUUID(uuid, Util::StringToCLSID(Util::CLSIDToString(uuid))), true);
-#endif
-
 }
 
 

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.