parsing.cpp

Go to the documentation of this file.

/*
 * parsing.cpp
 *
 * Copyright (C) 2007,2010  Thomas A. Vaughan
 * All rights reserved.
 *
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the <organization> nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THOMAS A. VAUGHAN ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THOMAS A. VAUGHAN BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *
 * Implementation of routines in parsing.h
 */

// includes --------------------------------------------------------------------
#include "parsing.h"            // always include our own header first!

#include "common/wave_ex.h"     // exceptions



static const byte_t s_maskHigh          = 0x80; // high bit set
static const byte_t s_maskTwoHigh       = 0xC0; // both high bits set


////////////////////////////////////////////////////////////////////////////////
//
//      static helper methods
//
////////////////////////////////////////////////////////////////////////////////


////////////////////////////////////////////////////////////////////////////////
//
//      public API
//
////////////////////////////////////////////////////////////////////////////////

void
utf8_char_t::setToReplacement
(
void
)
throw()
{
        // DPRINTF("Setting replacement!");
        // replacement character: Unicode 0xFFFD
        // That's 3 bytes in UTF-8
        nBytes = 3;
        value[0] = (char) 0xEF;
        value[1] = (char) 0xBF;
        value[2] = (char) 0xBD;
        value[3] = 0;
}



int
utf8_char_t::getValue
(
void
)
const
throw()
{
        if (1 == nBytes) {
                return value[0];
        } else if (nBytes > 1 && nBytes <= eMaxChars) {
                int nBits = 7 - nBytes;
                int retval = 0;
                for (int i = 0; i < nBytes; ++i) {
        //              DPRINTF("  byte[%d] = 0x%02x", i, (byte_t) value[i]);
                        int mask = 1 << (nBits - 1);
                        for (int j = 0; j < nBits; ++j) {
                                retval *= 2;
                                if (((byte_t) value[i]) & mask) {
                                        retval += 1;
                                }
                                mask /= 2;
                        }
                        nBits = 6;
                }
                return retval;
        } else {
                ASSERT(false, "Bad byte count for utf-8 character: %d", nBytes);
        }

        ASSERT(false, "should never get here!");
        return 0;
}



void
utf8_char_t::dump
(
IN const char * title
)
const
throw()
{
        ASSERT(title, "null");

        DPRINTF("%s (utf-8 char): %d bytes value=0x%04x '%s'",
                title, nBytes, getValue(), value);
}



bool
getUTF8CharacterFromStream
(
IN std::istream& stream,
OUT utf8_char_t& c
)
{
        ASSERT_THROW(stream.good(), "bad stream reading UTF8 character");
        c.clear();

        // get the first character
        char a;
        stream.read(&a, 1);
        if (stream.eof()) {
                return false;
        }

        // valid first character?
        if (isLeadingUTF8Byte(a)) {
                c.value[0] = a;
                c.nBytes = getUTF8ByteCount(a);
                if (c.nBytes < 1) {
                        c.setToReplacement();
                        return true;
                }
        } else {
                c.setToReplacement();
                return true;
        }

//      DPRINTF("Character 0x%02x has byte count: %d", a, c.nBytes);
        for (int i = 1; i < c.nBytes; ++i) {
                ASSERT_THROW(stream.good(),
                    "bad stream reading next byte of UTF8 character");
                stream.read(&a, 1);
                if (!isTrailingUTF8Byte(a)) {
                        c.setToReplacement();
                        return true;
                }
                c.value[i] = a;
        }
        c.value[c.nBytes] = 0;
        return true;
}



const char *
getUTF8CharacterFromString
(
IN const char * input,
OUT utf8_char_t& c
)
{
        ASSERT(input, "null");
        c.clear();

        // get the first character
        char a = *input;
        ++input;

        if (isLeadingUTF8Byte(a)) {
                c.value[0] = a;
                c.nBytes = getUTF8ByteCount(a);
                if (c.nBytes < 1) {
                        c.setToReplacement();
                        return input;
                }
        } else {
                c.setToReplacement();
                return input;
        }

//      DPRINTF("Character 0x%02x has byte count: %d", a, c.nBytes);
        for (int i = 1; i < c.nBytes; ++i) {
                char a = *input;
                ++input;
                if (!isTrailingUTF8Byte(a)) {
                        c.setToReplacement();
                        return input;
                }
                c.value[i] = a;
        }
        c.value[c.nBytes] = 0;

        // all done--returning new head of string
        return input;
}



int
getUTF8ByteCount
(
IN char a
)
{
        byte_t b = (byte_t) a;

        // first, a quick test: if high bit isn't set, this is ASCII
        if (!(s_maskHigh & b))
                return 1;

        // if high bit is set, both high bits should be set!
        ASSERT_THROW((s_maskTwoHigh & b) == s_maskTwoHigh,
                "bad leading UTF8 character: " << (int) b);

        // okay, look at number of set high bits
        byte_t mask = 0x20;     // third bit set
        int count = 2;
        while (mask & b) {
                mask /= 2;
                ++count;
        }
        if (count > utf8_char_t::eMaxChars) {
                return -1;
        }

        return count;
}



std::string
getNextLineFromStream
(
IN std::istream& stream,
IN eParseBehavior behavior
)
{
        ASSERT(stream.good(), "bad stream?");
        ASSERT(!(eParse_Invalid & behavior), "bad behavior");

        // DPRINTF("Parsing line...");

        std::string output;

        bool in_quote = false;
        bool in_comment = false;

        while (true) {
                if (stream.fail() || !stream.good()) {
                        WAVE_EX(wex);
                        wex << "Failure reading line from stream.";
                }

                char a;
                stream.read(&a, 1);
                if (stream.eof())
                        break;  // end of stream!

                if ('\n' == a)
                        break;  // end of line!

                if ((eParse_RespectQuotes & behavior) &&
                    '\"' == a)
                        in_quote = !in_quote;   // toggle

                if (!in_quote &&
                    '#' == a &&
                    (eParse_StripComments & behavior))
                        in_comment = true;      // hit a comment

                if (!in_comment) {
                        // not in a comment--should we add this?
                        if (!(eParse_StripBogus & behavior) ||
                            !isBogus(a)) {
                                output += a;
                        }
                }
        }

        return output;
}



const char *
getNextTokenFromString
(
IN const char * input,
OUT std::string& token,
IN eParseBehavior behavior
)
{
        ASSERT(input, "null input buffer");
        ASSERT(!(eParse_Invalid & behavior), "invalid behavior");
        token.clear();

        // DPRINTF("Parsing line for next token: '%s'...", input);

        const char * p = input;

        // skip leading whitespace (safe for UTF-8!)
        while (*p && isSpace(*p)) {
                ++p;
        }

        // is the first character a quotes?
        if ((eParse_RespectQuotes & behavior) && '\"' == *p) {
                ++p;    // go to next character
                while (*p && '\"' != *p) {
                        token += *p;
                        ++p;
                }
                if ('"' == *p) {
                        ++p;    // skip the closing quotes
                } else {
                        DPRINTF("WARNING unterminated quote token: %s",
                            token.c_str());
                }
        } else {
                // NOT in quotes
                // keep adding to token until hit end or buffer
                while (*p && !isSpace(*p)) {
                        token += *p;
                        ++p;
                }
        }

        // if we still have non-whitespace characters, traverse
        while (*p && !isSpace(*p)) {
                ++p;
        }

        // DPRINTF("  extracted token = '%s'", token.c_str());

        // return new position
        return p;
}



const char *
getNextTokenFromString
(
IN const char * input,
IO char * buffer,
IN int buffer_size,
IN eParseBehavior behavior,
OUT int& chars
)
throw()
{
        ASSERT(input, "null");
        ASSERT(buffer, "null");
        ASSERT(buffer_size > 0, "Bad buffer size: %d", buffer_size);
        ASSERT(eParse_None == behavior, "unsupported parse behavior!");
        chars = 0;

        const char * p = input;

        // skip leading whitespace
        for (; *p && isSpace(*p); ++p) { }

        // read input
        for (; *p && !isSpace(*p); ++p) {
                if (chars < buffer_size) {
                        buffer[chars] = *p;
                }
                ++chars;
        }

        // null terminate
        if (chars < buffer_size) {
                buffer[chars] = 0;
        } else {
                buffer[buffer_size - 1] = 0;
        }

        // all done
        return p;
}



const char *
expectFromString
(
IN const char * input,
IN const char * expect,
IN eParseBehavior parse
)
{
        ASSERT(input, "null");
        ASSERT(expect, "null");

        // skip any whitespace
        while (*input && isSpace(*input)) {
                ++input;
        }

        // now everything should match
        const char * p = expect;
        while (*p) {
                ASSERT_THROW(*input == *p,
                    "Expected token '" << expect << "' not found");
                ++p;
                ++input;
        }

        // should be end of input, or space
        ASSERT_THROW(!*input || isSpace(*input),
            "Expected token '" << expect << "' has trailing characters");

        // must be good!
        return input;
}



bool
isBogus
(
IN char a
)
throw()
{
        return (a == '\r');
}



char *
stripComments
(
IN char * line
)
throw()
{
        ASSERT(line, "null");

        bool in_quotes = false;
        char * p = line;
        while (*p) {
                if ('\"' == *p) {
                        in_quotes = !in_quotes;
                } else if (!in_quotes && '#' == *p) {
                        *p = 0;
                        return line;
                }
                ++p;
        }

        // no comments encountered
        return line;
}



char *
stripBogus
(
IN char * line
)
throw()
{
        char * src = line;
        char * dst = line;

        while (*src) {
                if (!isBogus(*src)) {
                        *dst = *src;
                        ++dst;
                }
                ++src;
        }
        *dst = 0;
        return line;
}



char *
lowercase
(
IN char * wext
)
throw()
{
        ASSERT(wext, "null");

        char * p = wext;
        while (*p) {
                *p = tolower(*p);
                ++p;
        }

        return wext;
}



void
getDictionaryFromString
(
IN const char * cursor,
IN const char * info,
OUT dictionary_t& dict
)
{
        ASSERT(cursor, "null string passed to getDictionaryFromString()");
        ASSERT(info, "null debug info passed to getDictionaryFromString()");
        dict.clear();

        std::string key, value;

        // keep looping through pairs of tokens
        while (true) {
                cursor = getNextTokenFromString(cursor, key, eParse_None);
                if ("" == key) {
                        break;  // no more key/value pairs
                }

                // should only have a given key once
                dictionary_t::iterator i = dict.find(key);
                if (dict.end() != i) {
                        WAVE_EX(wex);
                        wex << "Invalid key/value string for '" << info;
                        wex << "'.  Key '" << key << "' appears more than once.";
                }

                cursor = getNextTokenFromString(cursor, value,
                    eParse_RespectQuotes);
                dict[key] = value;
        }
}



const char *
getValue
(
IN const dictionary_t& data,
IN const char * key
)
{
        dictionary_t::const_iterator i = data.find(key);
        if (data.end() == i)
                return NULL;
        return i->second.c_str();
}



const char *
getRequiredValue
(
IN const dictionary_t& data,
IN const char * key
)
{
        ASSERT(key, "null key passed in to getRequiredValue()");

        const char * value = getValue(data, key);
        if (!value) {
                // couldn't find an entry with this key
                // throw a std::exception
                WAVE_EX(wex);
                wex << "Missing a required parameter: '" << key << "'";
        }

        // found it!
        return value;
}




const char *
getOptionalValue
(
IN const dictionary_t& data,
IN const char * key,
IN const char * default_value
)
throw()
{
        ASSERT(key, "null key passed in to getOptionalValue()");
        ASSERT(default_value,
            "null default value passed in to getOptionalValue()");

        const char * value = getValue(data, key);
        return (value) ? value : default_value;
}



const char *
getFloatFromString
(
IN const char * input,
OUT float& x
)
throw()
{
        ASSERT(input, "null");
        x = 0;

        static const int s_maxSize = 15;
        static const int s_bufSize = s_maxSize + 1;
        char buffer[s_bufSize];

        int nChars = 0;
        input = getNextTokenFromString(input, buffer, s_bufSize, eParse_None,
            nChars);
        if (nChars > s_maxSize) {
                DPRINTF("Warning: float was truncated!  "
                    "(read %d out of %d characters)", s_maxSize, nChars);
        }
        x = atof(buffer);

        return input;
}



int
readFloatsFromString
(
IN const char * input,
IN int nFloats,
OUT float * output
)
throw()
{
        ASSERT(input, "null");
        ASSERT(nFloats > 0, "Bad float count: %d", nFloats);
        ASSERT(output, "null");

        for (int i = 0; i < nFloats; ++i, ++output) {
                if (!*input) {
                        return i;
                }

                input = getFloatFromString(input, *output);
        }

        // read all requested!
        return nFloats;
}



bool
getBooleanFromString
(
IN const char * input
)
throw()
{
        ASSERT(input, "null");

        // empty string?  means false
        if (!*input)
                return false;

        // single character string?
        if (!*(input + 1)) {
                if ('F' == *input || 'f' == *input || '0' == *input)
                        return false;
                return true;
        }

        // long string.  Only false value is the word "false"!
        return (0 != strcasecmp("false", input));
}



const char *
getNextWord
(
IN const char * line,
OUT std::string& word
)
{
        ASSERT(line, "null");
        word.clear();

        // UTF-8 compliant
        const char * p = line;

        // skip any leading whitespace
        for (; *p && '\n' != *p && isSpace(*p); ++p) { }

        // is this a newline?
        if ('\n' == *p) {
                word = "\n";
                return p + 1;
        }

        // add any non-whitespace characters, except newlines
        for (; *p && '\n' != *p && !isSpace(*p); ++p) {
                word += *p;
        }

        return p;
}