c++unicodebyte-order-marklibiconv

Character conversion using iconv without the Unicode Byte Order Mark


How do I make iconv do conversions to UTF-* encodings without adding a byte order mark?

The problem is shown by this code, compiled with G++ 4.8 under 64-bit Linux:

#include <iostream>
#include <iconv.h>
#include <cstdint>
#include <iomanip>
using namespace std;

int main() {
        const wchar_t *ws = L"Hello, world.\n";
        for(size_t ii = 0; ii < 15; ii++) {
                cout << setw(2) << hex << uint32_t(ws[ii]) << " ";
        }
        cout << endl;
        iconv_t conv = iconv_open("UTF-16", "UTF-32");
        char *outbuf = new char[14 * 4];
        char *inptr = const_cast<char*>(reinterpret_cast<const char *>(ws));
        char *outptr = outbuf;
        size_t in_len = 14 * 4;
        size_t out_len = 14 * 4;
        size_t result = iconv(conv, &inptr, &in_len, &outptr, &out_len);
        uint16_t *encoded = reinterpret_cast<uint16_t*>(outbuf);
        for(size_t ii = 0; ii < 15; ii++) {
                cout << setw(2) << hex << encoded[ii] << " ";
        }
        cout << endl;
}

This outputs:

48 65 6c 6c 6f 2c 20 77 6f 72 6c 64 2e  a 0
feff 48 65 6c 6c 6f 2c 20 77 6f 72 6c 64 2e a

clearly showing the BOM at the start of the resulting string where it is not present at the


Solution

  • I found that this works for me:

    iconv_t conv = iconv_open("UTF16LE", "UTF32");
    

    Though I believe it's implementation-dependent.

    There is a similar question: Using iconv to convert from UTF-16BE to UTF-8 without BOM