#1033 __mingw_printf fails on UTF-16 surrogates in %S

[Jan]

Consider the following code:

#define __USE_MINGW_ANSI_STDIO 1
#include <cstdio>
#include <clocale>
#include <cstdlib>

int main()
{
    setlocale(LC_ALL, ".UTF-8"); 

    const wchar_t* data_utf16[] = {
            L"Mixed: Hello \u4E16\u754C",                       // Chinese characters
            L"Unicode: \U0001F389\u2728",                       // Emojis
            L"Japanese: \u3053\u3093\u306B\u3061\u306F",        // Japanese greeting
            L"Korean: \uC548\uB155\uD558\uC138\uC694",          // Korean greeting
            L"Arabic: \u0645\u0631\u062D\u0628\u0627",          // Arabic greeting
            L"Emoji: \U0001F600\U0001F601\U0001F602",           // Smiley emojis
            L"Cyrillic: \u041F\u0440\u0438\u0432\u0435\u0442",  // Russian greeting
            L"Chinese: \u4E2D\u6587",                           // Chinese characters
            L"Mixed2: Hello \u4E16\u754C \U0001F30D"            // Hello with globe emoji
    };

    //the same data as utf8
    const char* data_utf8[] = {
            "Mixed: Hello \xE4\xB8\x96\xE7\x95\x8C",          
            "Unicode: \xF0\x9F\x8E\x89\xE2\x9C\xA8",          
            "Japanese: \xE3\x81\x93\xE3\x82\x93\xE3\x81\xAB\xE3\x81\xA1\xE3\x81\xAF", 
            "Korean: \xEC\x95\x88\xEB\x85\x95\xED\x95\x98\xEC\x84\xB8\xEC\x9A\x94", 
            "Arabic: \xD9\x85\xD8\xB1\xD8\xAD\xD8\xA8\xD8\xA7",
            "Emoji: \xF0\x9F\x98\x80\xF0\x9F\x98\x81\xF0\x9F\x98\x82", 
            "Cyrillic: \xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82", 
            "Chinese: \xE4\xB8\xAD\xE6\x96\x87",               
            "Mixed2: Hello \xE4\xB8\x96\xE7\x95\x8C \xF0\x9F\x8C\x8D"
    };

    freopen("out.txt", "wt", stdout);

    for (int i = 0; i < 9; i++)
    {   
        char conv[1024];
        auto ret = wcstombs(conv, data_utf16[i], 1024);
        __mingw_printf("'%S' '%s' '%s'\n", data_utf16[i], data_utf8[i], conv);
        __ms_printf("'%S' '%s' '%s'\n", data_utf16[i], data_utf8[i], conv);
    }

    return 0;
}

It generates out.txt with content:

'Mixed: Hello 世界' 'Mixed: Hello 世界' 'Mixed: Hello 世界'
'Mixed: Hello 世界' 'Mixed: Hello 世界' 'Mixed: Hello 世界'
'Unicode: ' 'Unicode: 🎉✨' 'Unicode: 🎉✨'
'Unicode: 🎉✨' 'Unicode: 🎉✨' 'Unicode: 🎉✨'
'Japanese: こんにちは' 'Japanese: こんにちは' 'Japanese: こんにちは'
'Japanese: こんにちは' 'Japanese: こんにちは' 'Japanese: こんにちは'
'Korean: 안녕하세요' 'Korean: 안녕하세요' 'Korean: 안녕하세요'
'Korean: 안녕하세요' 'Korean: 안녕하세요' 'Korean: 안녕하세요'
'Arabic: مرحبا' 'Arabic: مرحبا' 'Arabic: مرحبا'
'Arabic: مرحبا' 'Arabic: مرحبا' 'Arabic: مرحبا'
'Emoji: ' 'Emoji: 😀😁😂' 'Emoji: 😀😁😂'
'Emoji: 😀😁😂' 'Emoji: 😀😁😂' 'Emoji: 😀😁😂'
'Cyrillic: Привет' 'Cyrillic: Привет' 'Cyrillic: Привет'
'Cyrillic: Привет' 'Cyrillic: Привет' 'Cyrillic: Привет'
'Chinese: 中文' 'Chinese: 中文' 'Chinese: 中文'
'Chinese: 中文' 'Chinese: 中文' 'Chinese: 中文'
'Mixed2: Hello 世界 ' 'Mixed2: Hello 世界 🌍' 'Mixed2: Hello 世界 🌍'
'Mixed2: Hello 世界 🌍' 'Mixed2: Hello 世界 🌍' 'Mixed2: Hello 世界 🌍'

From this output we can see that MinGW's printf cannot correctly handle surrogate pairs, while Microsoft's printf can. The function wcstombs also correctly handles surrogate pairs.
The reason of this bug lies in the following code in the file named mingw_pformat.c, in the function __pformat_wputchars, which outputs wchar_t* type string:

while( (count-- > 0) && ((len = wcrtomb( buf, *s++, &state )) > 0) )
  {
    char *p = buf;
    while( len-- > 0 )
      __pformat_putc( *p++, stream );
  }

On Windows, wchar_t is 16 bits (UTF-16). When a surrogate pair (such as an emoji or extended Chinese character) is encountered, *s++ extracts only the high surrogate code unit. Passing an isolated surrogate to wcrtomb results in an encoding error, i.e., len == -1, causing the loop to exit early and silently truncate the remaining output string. The correct approach should use wcstombs, i.e.,

while (count > 0 && *s)
{
    char *p = buf;
    wchar_t high = *s;

    if (high >= 0xD800 && high <= 0xDBFF) 
    {
        /* surrogate */
        if (count > 1 && *(s + 1))
        {
            wchar_t low = *(s + 1);
            if (low >= 0xDC00 && low <= 0xDFFF)
            {
                /* valid surrogate */
                wchar_t wcs[3] = { high, low, 0 };
                len = wcsrtombs(buf, &wcs, sizeof(buf), &state); 

                if (len <= 0) 
                    break; /* utf16 -> our locale conversion error */

                s += 2;
                count -= 2;
            }
            else /* invalid surrogate */
                break;
        }
        else /* we have only high part of surrogate pair, but do not have low one */
            break;
    }
    else
    {
        /* normal wchar_t */
        len = wcrtomb(buf, high, &state);
        if (len <= 0)  /* utf16 -> our locale conversion error */
            break;

        s++;
        count--;
    }

    while (len-- > 0) __pformat_putc(*p++, stream);
}

To reproduce it is essential to use g++ in the UCRT environment, since otherwise the utf-8 locale does not work.