utf8-parse-test.cpp

Go to the documentation of this file.

/*
 * utf8-parse-test.cpp
 *
 * Copyright (C) 2010  Thomas A. Vaughan
 * All rights reserved.
 *
 * Quick test of parsing routines, with UTF-8 examples.
 */

// includes --------------------------------------------------------------------
#include <iostream>
#include <sstream>

#include "perf/perf.h"
#include "util/parsing.h"


struct test_line_t {
        const char *    line;           // a UTF-8 line
        int             nTokens;        // number of tokens in line
        int             nChars;         // number of characters in line
};


// much test data from http://www.columbia.edu/kermit/utf8.html

static test_line_t s_testLines[] = {
          // utf-8 string                               # of tokens     chars
        { "This is a test in English",                          6,      25 },
        { "I can eat glass.",                                   4,      16 },

        // Erzian
        { "Мон ярсан суликадо, ды зыян эйстэнзэ а ули.",      8,      43 },

        // Hindi
        { "मैं काँच खा सकता हूँ और मुझे उससे कोई चोट नहीं पहुंचती",
        12, 54 },

        // Chinese (Traditional)
        { "我能吞下玻璃而不傷身體",                          1,      11 },

        // Russian
        { "Я могу есть стекло, оно мне не вредит.",                8,      38 },

        // French
        { "Je peux manger du verre, ça ne me fait pas  mal.",  11,     48 },

        // keep this last!
        { NULL, 0, 0 }
};



// invalid lines are very important!  Not handling these safely is a
//      security risk
struct invalid_line_t {
        const char      line[16];
};


static const invalid_line_t s_invalidLines[] = {
        // continuation bytes
        { { 0x80, 0x00 } },
        { { 0x81, 0x00 } },

        // lonely leading characters
        { { 0xc0, 0x20, 0x00 } },       // two-byte sequences
        { { 0xc1, 0x30, 0x00 } },
        { { 0xc1, 0x00 } },
        { { 0xe0, 0x20, 0x00 } }, // 3-byte sequences
        { { 0xe1, 0x00 } },
        { { 0xe1, 0x20, 0x00 } },
        { { 0xf0, 0x20, 0x00 } }, // 4-byte sequences
        { { 0xf0, 0x00 } },

        // impossible bytes
        { { 0xfe, 0x00 } },
        { { 0xff, 0x00 } },
        { { 0xfe, 0xfe, 0xff, 0xff, 0x00 } },

        // keep this last!
        { { 0x00 } }
};


////////////////////////////////////////////////////////////////////////////////
//
//      static helper methods
//
////////////////////////////////////////////////////////////////////////////////

static void
testLine
(
IN const test_line_t * p
)
{
        ASSERT(p, "null");
        ASSERT(p->line, "null");

        int count = 0;
        std::string token;
        const char * txt = p->line;
        DPRINTF("Parsing string: %s", txt);
        while (*txt) {
                txt = getNextTokenFromString(txt, token, eParse_None);
                ++count;
                DPRINTF("  Token: '%s'", token.c_str());
        }

        // did we count the right number of tokens?
        ASSERT_THROW(p->nTokens == count,
            "Failed to correctly parse tokens in string: '" << p->line <<
            "', read " << count << " tokens instead of " << p->nTokens);

        // now parse characters
        DPRINTF("  Parsing characters from string...");
        utf8_char_t c;
        count = 0;      // reset count
        txt = p->line;
        while (*txt) {
                txt = getUTF8CharacterFromString(txt, c);
                ++count;
                c.dump("    Character");
        }

        // did we count the right number of characters?
        ASSERT_THROW(p->nChars == count,
            "Failed to correctly parse characters in string: '" << p->line <<
            "', read " << count << " characters instead of " << p->nChars);

        // now create a stringstream from the line
        DPRINTF("  Parsing characters from stream...");
        std::istringstream iss(p->line);
        count = 0;
        while (getUTF8CharacterFromStream(iss, c)) {
                ++count;
                c.dump("    Character");
        }

        // did we count the right number of characters?
        ASSERT_THROW(p->nChars == count,
            "Failed to correctly parse characters in stream: '" << p->line <<
            "', read " << count << " characters instead of " << p->nChars);
}



static void
invalidLine
(
IN const invalid_line_t * p
)
{
        ASSERT(p, "null");

        int replaceChar = 0xFFFD;       // Unicode replacement character

        // parse this line
        utf8_char_t c;
        const char * txt = p->line;
        while (*txt) {
                txt = getUTF8CharacterFromString(txt, c);
                c.dump("  invalid character");
                ASSERT_THROW(c.getValue() == replaceChar,
                    "Value of parsed character should be replacement char!");
        }
}



static void
doTest
(
void
)
{
        for (const test_line_t * p = s_testLines; p->line; ++p) {
                testLine(p);
        }

        for (const invalid_line_t * p = s_invalidLines; p->line[0]; ++p) {
                invalidLine(p);
        }
}



////////////////////////////////////////////////////////////////////////////////
//
//      entry point
//
////////////////////////////////////////////////////////////////////////////////

int
main
(
IN int argc,
IN const char * argv[]
)
{
        int retval = 0;

        try {
                perf::Timer timer("full timer");

                doTest();

        } catch (std::exception& e) {
                DPRINTF("Exception: %s", e.what());
                retval = 1;
        }

        perf::dumpTimingSummary(std::cerr);

        return retval;
}