UnicodeTest.cpp File Reference

#include <folly/Unicode.h>
#include <initializer_list>
#include <stdexcept>
#include <folly/Range.h>
#include <folly/portability/GTest.h>

Go to the source code of this file.

Functions
void	testValid (std::initializer_list< unsigned char > data, char32_t expected)
void	testInvalid (std::initializer_list< unsigned char > data)
	TEST (InvalidUtf8ToCodePoint, rfc3629Overlong)
	TEST (InvalidUtf8ToCodePoint, rfc3629SurrogatePair)
	TEST (InvalidUtf8ToCodePoint, MarkusKuhnSingleUTF16Surrogates)
	TEST (InvalidUtf8ToCodePoint, MarkusKuhnPairedUTF16Surrogates)
	TEST (ValidUtf8ToCodePoint, FourCloverLeaf)
	TEST (InvalidUtf8ToCodePoint, FourCloverLeafAsSurrogates)
	TEST (ValidUtf8ToCodePoint, LastCodePoint)

Function Documentation

TEST	(	InvalidUtf8ToCodePoint	,
		rfc3629Overlong
	)

Definition at line 61 of file UnicodeTest.cpp.

References testInvalid().

                                              {
  // https://tools.ietf.org/html/rfc3629
  // Implementations of the decoding algorithm above MUST protect against
  // decoding invalid sequences.  For instance, a naive implementation may
  // decode the overlong UTF-8 sequence C0 80 into the character U+0000 [...]
  // Decoding invalid sequences may have security consequences or cause other
  // problems.
  testInvalid({0xC0, 0x80});
}

TEST	(	InvalidUtf8ToCodePoint	,
		rfc3629SurrogatePair
	)

Definition at line 71 of file UnicodeTest.cpp.

References testInvalid().

                                                   {
  // https://tools.ietf.org/html/rfc3629
  // Implementations of the decoding algorithm above MUST protect against
  // decoding invalid sequences.  For instance, a naive implementation may
  // decode [...] the surrogate pair ED A1 8C ED BE B4 into U+233B4.
  // Decoding invalid sequences may have security consequences or cause other
  // problems.
  testInvalid({0xED, 0xA1, 0x8C, 0xED, 0xBE, 0xB4});
}

TEST	(	InvalidUtf8ToCodePoint	,
		MarkusKuhnSingleUTF16Surrogates
	)

Definition at line 81 of file UnicodeTest.cpp.

References testInvalid().

                                                              {
  // https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
  // 5.1.1  U+D800 = ed a0 80
  // 5.1.2  U+DB7F = ed ad bf
  // 5.1.3  U+DB80 = ed ae 80
  // 5.1.4  U+DBFF = ed af bf
  // 5.1.5  U+DC00 = ed b0 80
  // 5.1.6  U+DF80 = ed be 80
  // 5.1.7  U+DFFF = ed bf bf
  testInvalid({0xed, 0xa0, 0x80});
  testInvalid({0xed, 0xad, 0xbf});
  testInvalid({0xed, 0xae, 0x80});
  testInvalid({0xed, 0xaf, 0xbf});
  testInvalid({0xed, 0xb0, 0x80});
  testInvalid({0xed, 0xbe, 0x80});
  testInvalid({0xed, 0xbf, 0xbf});
}

TEST	(	InvalidUtf8ToCodePoint	,
		MarkusKuhnPairedUTF16Surrogates
	)

Definition at line 99 of file UnicodeTest.cpp.

References testInvalid().

                                                              {
  // https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
  // 5.2.1  U+D800 U+DC00 = ed a0 80 ed b0 80
  // 5.2.2  U+D800 U+DFFF = ed a0 80 ed bf bf
  // 5.2.3  U+DB7F U+DC00 = ed ad bf ed b0 80
  // 5.2.4  U+DB7F U+DFFF = ed ad bf ed bf bf
  // 5.2.5  U+DB80 U+DC00 = ed ae 80 ed b0 80
  // 5.2.6  U+DB80 U+DFFF = ed ae 80 ed bf bf
  // 5.2.7  U+DBFF U+DC00 = ed af bf ed b0 80
  // 5.2.8  U+DBFF U+DFFF = ed af bf ed bf bf
  testInvalid({0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80});
  testInvalid({0xed, 0xa0, 0x80, 0xed, 0xbf, 0xbf});
  testInvalid({0xed, 0xad, 0xbf, 0xed, 0xb0, 0x80});
  testInvalid({0xed, 0xad, 0xbf, 0xed, 0xbf, 0xbf});
  testInvalid({0xed, 0xae, 0x80, 0xed, 0xb0, 0x80});
  testInvalid({0xed, 0xae, 0x80, 0xed, 0xbf, 0xbf});
  testInvalid({0xed, 0xaf, 0xbf, 0xed, 0xb0, 0x80});
  testInvalid({0xed, 0xaf, 0xbf, 0xed, 0xbf, 0xbf});
}

TEST	(	ValidUtf8ToCodePoint	,
		FourCloverLeaf
	)

Definition at line 119 of file UnicodeTest.cpp.

References testValid().

                                           {
  testValid({0xF0, 0x9F, 0x8D, 0x80}, 0x1F340); // u8"\U0001F340";
}

TEST	(	InvalidUtf8ToCodePoint	,
		FourCloverLeafAsSurrogates
	)

Definition at line 123 of file UnicodeTest.cpp.

References testInvalid().

                                                         {
  testInvalid({0xd8, 0x3c, 0xdf, 0x40}); // u8"\U0001F340";
}

TEST	(	ValidUtf8ToCodePoint	,
		LastCodePoint
	)

Definition at line 127 of file UnicodeTest.cpp.

References testValid().

                                          {
  testValid({0xF4, 0x8F, 0xBF, 0xBF}, 0x10FFFF); // u8"\U0010FFFF";
}

void testInvalid

(

std::initializer_list< unsigned char >

data

)

Definition at line 43 of file UnicodeTest.cpp.

References folly::Range< Iter >::begin(), EXPECT_EQ, EXPECT_THROW, and folly::utf8ToCodePoint().

Referenced by TEST().

                                                          {
  {
    const unsigned char* p = data.begin();
    const unsigned char* e = data.end();
    EXPECT_THROW(
        utf8ToCodePoint(p, e, /* skipOnError */ false), std::runtime_error)
        << folly::StringPiece(
               (const char*)data.begin(), (const char*)data.end());
  }
  {
    const unsigned char* p = data.begin();
    const unsigned char* e = data.end();
    EXPECT_EQ(utf8ToCodePoint(p, e, /* skipOnError */ true), 0xfffd)
        << folly::StringPiece(
               (const char*)data.begin(), (const char*)data.end());
  }
}

void testValid	(	std::initializer_list< unsigned char >	data,
		char32_t	expected
	)

Definition at line 26 of file UnicodeTest.cpp.

References folly::Range< Iter >::begin(), EXPECT_EQ, and folly::utf8ToCodePoint().

Referenced by TEST().

                                                                           {
  {
    const unsigned char* p = data.begin();
    const unsigned char* e = data.end();
    EXPECT_EQ(utf8ToCodePoint(p, e, /* skipOnError */ false), expected)
        << folly::StringPiece(
               (const char*)data.begin(), (const char*)data.end());
  }
  {
    const unsigned char* p = data.begin();
    const unsigned char* e = data.end();
    EXPECT_EQ(utf8ToCodePoint(p, e, /* skipOnError */ true), expected)
        << folly::StringPiece(
               (const char*)data.begin(), (const char*)data.end());
  }
}