i18n.cpp

Go to the documentation of this file.

/*
 * i18n.cpp
 *
 * Copyright (c) 2010 Thomas A. Vaughan
 * All rights reserved.
 *
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the <organization> nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THOMAS A. VAUGHAN ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THOMAS A. VAUGHAN BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *
 * Basic localized string management.  See i18n.h
 */

// includes --------------------------------------------------------------------
#include "i18n.h"               // always include our own header first

#include "common/wave_ex.h"
#include "datahash/datahash_text.h"
#include "datahash/datahash_util.h"
#include "perf/perf.h"
#include "resources/resources-internal.h"
#include "util/parsing.h"
#include "util/string-buffer.h"


// needed to force resource linking
void registerWaveResource_i18n_country_codes_txt(void) throw();
void registerWaveResource_i18n_language_codes_txt(void) throw();

namespace i18n {

// interface destructors
Manager::~Manager(void) throw() { }


struct two_character_map_t {
        enum eConstants {
                eDimension      = 26,   //
                eSize           = eDimension * eDimension
        };

        void clear(void) throw() {
                        base = 'a';
                        memset(map, 0, eSize);
                }

        // WARNING: returns -1 for bad input
        int getIndex(IN const char * code) const throw() {
                        ASSERT(code, "null");
                        char c0 = *code;
                        char c1 = *(code + 1);

                        if (c0 < base || c0 >= base + eDimension) {
                                DPRINTF("Bad first character: %s", code);
                                return -1;
                        }
                        if (c1 < base || c1 >= base + eDimension) {
                                DPRINTF("Bad second character: %s", code);
                                return -1;
                        }

                        int offset = (c0 - base) * eDimension + (c1 - base);
                        return offset;
                }

        bool isValid(IN const char * code) const throw() {
                        int index = this->getIndex(code);
                        if (index < 0 || !map[index]) {
                                DPRINTF("Not a valid %s code: '%s'",
                                    type, code);
                                return false;
                        }
                        return true;
                }

        // data fields
        char            base;   // 'a' or 'A', typically
        byte_t          map[eSize];
        char            type[16];
};


static two_character_map_t s_countryCodes;
static two_character_map_t s_languageCodes;

static const char * s_resourceNamespace         = "i18n";


////////////////////////////////////////////////////////////////////////////////
//
//      static helper methods
//
////////////////////////////////////////////////////////////////////////////////

static void
loadTable
(
IO two_character_map_t& table,
IN char base,
IN const char * name
)
{
        ASSERT(base, "bad character base");
        ASSERT(name, "null");

        table.clear();
        table.base = base;

        DPRINTF("Number of resource namespaces: %d",
            getResourceNamespaceCount());
        DPRINTF("Name of first namespace: %s",
            getResourceNamespaceName(0));
        DPRINTF("Namespace: %s", s_resourceNamespace);
        DPRINTF("  name: %s", name);

        int bytes = getResourceSize(s_resourceNamespace, name);
        ASSERT(bytes > 0, "null");

        const byte_t * raw = getResourceData(s_resourceNamespace, name);
        const char * value = (const char *) raw;
        ASSERT(value, "null");

//      DPRINTF("Resource '%s':\n%s", name, value);

        // okay, now run through each line...
        while (bytes > 0) {
                int offset = table.getIndex(value);
//              DPRINTF("%c%c : index %d", value[0], value[1], offset);

                table.map[offset] = 1;

                // walk forward to end of line
                while (*value && '\n' != *value && bytes > 0) {
                        ++value;
                        --bytes;
                }
                if (*value) {
                        ++value;        // skip newline character
                        --bytes;
                }
        }
}



static void
loadTables
(
void
)
throw()
{
        // this static methods loads our static tables from linked-in resource
        //      files
        static bool s_loaded = false;
        if (s_loaded)
                return; // already loaded!
        s_loaded = true;        // set flag

        // register our resources manually
        registerWaveResource_i18n_country_codes_txt();
        registerWaveResource_i18n_language_codes_txt();

        strcpy(s_countryCodes.type, "country");
        strcpy(s_languageCodes.type, "language");

        loadTable(s_countryCodes, 'A', "country-codes.txt");
        loadTable(s_languageCodes, 'a', "language-codes.txt");
}



////////////////////////////////////////////////////////////////////////////////
//
//      locale_t -- method implementations
//
////////////////////////////////////////////////////////////////////////////////

bool
locale_t::isValid
(
void
)
const
throw()
{
        char buffer[eBufferSize];
        this->getLanguageCode(buffer);
//      DPRINTF("Language code: %s", buffer);
        if (!isValidLanguageCode(buffer)) {
                DPRINTF("Language code not valid: '%s'", buffer);
                return false;
        }

        this->getCountryCode(buffer);
//      DPRINTF("Country code: %s", buffer);
        if (!isValidCountryCode(buffer)) {
                DPRINTF("Country code not valid: '%s'", buffer);
                return false;
        }

        this->getEncoding(buffer);
//      DPRINTF("Encoding: %s", buffer);
        if (!isValidEncoding(buffer)) {
                DPRINTF("Encoding not valid: '%s'", buffer);
                DPRINTF("  Only 'UTF-8' is supported for now!");
                return false;
        }

        // got here?  All elements must be valid!
        return true;
}



////////////////////////////////////////////////////////////////////////////////
//
//      Mgr -- class that implements the i18n::Manager interface
//
////////////////////////////////////////////////////////////////////////////////

class Mgr : public Manager {
public:
        // constructor, destructor ---------------------------------------------
        ~Mgr(void) throw() { }

        // public class methods ------------------------------------------------
        void initialize(IN const char * locale);

        // i18n::Manager class interface methods -------------------------------
        const char * getLocale(void) const throw() { return m_locale.string; }
        void parseStrings(IO nstream::Stream * stream);
        void parseFolder(IN nstream::Folder * folder,
                                IN const SetString * extensions,
                                IN const char * filter);
        const char * getString(IN const char * id) const throw();

private:
        // private typedefs ----------------------------------------------------
        struct string_record_t {
                int     valueOffset;    // localized string
                int     filenameOffset; // pointer to filename
        };

        typedef std::map<std::string, string_record_t> string_map_t;
        typedef std::map<std::string, int> filename_map_t;

        // private helper methods ----------------------------------------------
        int getFilenameOffset(IN const char * filename);
        const string_record_t * findString(IN const char * id) const throw();
        static nstream::eIterationFlag parseEntry(IN nstream::Entry * entry,
                                IN void * context);

        // private member data -------------------------------------------------
        locale_t                m_locale;
        filename_map_t          m_filenames;
        smart_ptr<StringBuffer> m_sbuf; // list of all strings
        string_map_t            m_map;
};



void
Mgr::initialize
(
IN const char * locale
)
{
        ASSERT(locale, "null");

        getLocaleFromString(locale, m_locale);
        ASSERT_THROW(m_locale.isValid(), "Invalid locale: " << locale);

        m_sbuf = StringBuffer::create();
        ASSERT(m_sbuf, "out of memory");
}



////////////////////////////////////////////////////////////////////////////////
//
//      Mgr -- i18n::Manager class interface methods
//
////////////////////////////////////////////////////////////////////////////////

void
Mgr::parseStrings
(
IN nstream::Stream * stream
)
{
        ASSERT(m_sbuf, "null");
        ASSERT(stream, "null");

        // get the File object associated with this stream
        smart_ptr<nstream::File> file = stream->getFile();
        ASSERT(file, "null file associated with named stream");
        const char * name = file->getName();
        //DPRINTF("Parsing file for strings: %s", name);
        int filenameOffset = this->getFilenameOffset(name);

        std::istream& instream = stream->getStream();
        ASSERT_THROW(!instream.bad(), "bad stream?");

        // keep parsing until eof or we hit "localeInfo"
        // NOTE: this isn't a simple hash structure!  We require "localeInfo"
        // to be the first token on the line, and we won't pay attention to
        // nesting of brackets etc.
        std::string line;
        eParseBehavior parse = eParse_Strip;
        while (instream.good()) {
                line = getNextLineFromStream(instream, parse);
                if (strncmp(line.c_str(), "localeInfo", 10))
                        continue;       // skip this line

                // okay this is a locale ...maybe
                const char * p = line.c_str();
                p = expectFromString(p, "localeInfo", parse);
                p = expectFromString(p, "{", parse);

                // okay, read the rest as a hash
                smart_ptr<Datahash> hash =
                    readHashFromStream("localeInfo", instream);
                ASSERT(hash, "null");

                // extract locale
                const char * locale = ::getString(hash, "locale");
                ASSERT_THROW(locale,
                    "No 'locale' specified in 'localeInfo' block?");
                if (strcmp(locale, m_locale.getString())) {
                        DPRINTF("Skipping locale '%s'", locale);
                        continue;       // skip this block--wrong locale
                }

                // okay, this is our locale!
                smart_ptr<Datahash> strings =
                    getSubhash(hash, "localizedStrings");
                if (!strings)
                        break;  // no localized strings--skip

                Datahash::iterator_t i;
                strings->getIterator(i);
                std::string key;
                const hash_value_t * phv;
                while ((phv = strings->getNextElementUnsafe(i, key))) {
                        if (eHashDataType_String != phv->type)
                                continue;       // not a string--skip

                        const char * val = phv->text.c_str();
                        //DPRINTF("string %s --> '%s'", key.c_str(), val);

                        // already seen this key?
                        const string_record_t * psr =
                            this->findString(key.c_str());
                        ASSERT_THROW(!psr, "Duplicate string definition!  " <<
                            "String id = '" << key << "'.  Previous definition "
                            << "in file " <<
                            m_sbuf->getString(psr->filenameOffset) << ", second"
                            << " definition in file " <<
                            m_sbuf->getString(filenameOffset));

                        string_record_t sr;
                        sr.filenameOffset = filenameOffset;
                        sr.valueOffset = m_sbuf->appendString(val);
                        m_map[key] = sr;
                }
        }
        //DPRINTF("%d strings in map", (int) m_map.size());
}



void
Mgr::parseFolder
(
IN nstream::Folder * folder,
IN const SetString * extensions,
IN const char * filter
)
{
        ASSERT(m_sbuf, "null");
        ASSERT(folder, "null");
        // ASSERT(extensions) -- can be null!
        // ASSERT(filter) -- can be null!

        // recursively walk the folder
        bool visitHidden = false; // skip entries beginning with '.'
        walkChildFolders(folder, parseEntry, this, extensions, filter,
            visitHidden);
}



const char *
Mgr::getString
(
IN const char * id
)
const
throw()
{
        ASSERT(id, "null");

        const string_record_t * psr = this->findString(id);
        if (!psr)
                return NULL;

        ASSERT(m_sbuf, "null");
        return m_sbuf->getString(psr->valueOffset);
}



////////////////////////////////////////////////////////////////////////////////
//
//      Mgr -- private helper methods
//
////////////////////////////////////////////////////////////////////////////////

int
Mgr::getFilenameOffset
(
IN const char * filename
)
{
        ASSERT(filename, "null");

        // is this already in our set?
        filename_map_t::iterator i = m_filenames.find(filename);
        if (m_filenames.end() != i) {
                return i->second;
        }

        // not already encountered--add it!
        int offset = m_sbuf->appendString(filename);
        m_filenames[filename] = offset;
        return offset;
}



const Mgr::string_record_t *
Mgr::findString
(
IN const char * id
)
const
throw()
{
        ASSERT(id, "null");

        string_map_t::const_iterator i = m_map.find(id);
        if (m_map.end() == i)
                return NULL;    // no such string
        return &i->second;
}



nstream::eIterationFlag
Mgr::parseEntry
(
IN nstream::Entry * entry,
IN void * context
)
{
        ASSERT(entry, "null");
        Mgr * pThis = (Mgr *) context;
        ASSERT(pThis, "null context?");

        nstream::File * file = dynamic_cast<nstream::File *>(entry);
        if (!file)
                return nstream::eIterate_Continue;

        // okay, this is a file we care about!
        smart_ptr<nstream::Stream> stream = file->openStream();
        ASSERT_THROW(stream, "failed to open stream");

        pThis->parseStrings(stream);
        return nstream::eIterate_Continue;
}



////////////////////////////////////////////////////////////////////////////////
//
//      Public APIs
//
////////////////////////////////////////////////////////////////////////////////

smart_ptr<Manager>
Manager::create
(
IN const char * locale
)
{
        ASSERT(locale, "null");

        smart_ptr<Mgr> local = new Mgr;
        ASSERT(local, "out of memory");

        local->initialize(locale);

        return local;
}



void
getLocaleFromString
(
IN const char * localeString,
OUT locale_t& locale
)
{
        ASSERT(localeString, "null");
        locale.clear();

        ASSERT_THROW(strlen(localeString) <= locale_t::eMaxLength,
            "Locale string is too long: " << localeString);

        strcpy(locale.string, localeString);

        ASSERT_THROW(locale.isValid(),
            "Locale string is not valid: " << localeString);
}



bool
isValidCountryCode
(
IN const char * code
)
{
        ASSERT(code, "null");

        loadTables();

        return s_countryCodes.isValid(code);
}



bool
isValidLanguageCode
(
IN const char * code
)
{
        ASSERT(code, "null");

        loadTables();

        return s_languageCodes.isValid(code);
}



bool
isValidEncoding
(
IN const char * encoding
)
{
        ASSERT(encoding, "null");

        // only support 'UTF-8' for now!
        return (!strcmp(encoding, "UTF-8"));
}



const char *
getString
(
IN const Manager * mgr,
IN const char * id
)
{
        ASSERT(id, "null");
        ASSERT(mgr, "null");

        const char * val = mgr->getString(id);
        if (val)
                return val;

        // string wasn't found!  Return a temp
        const int bufsize = 256;
        static char buffer[bufsize];

#ifdef WIN32
        _snprintf(
#else   // WIN32
        snprintf(
#endif  // WIN32
            buffer, bufsize,
            "Missing a localized string (id='%s') for locale '%s'",
            id, mgr->getLocale());

        return buffer;
}



const char *
getHostLocale
(
void
)
{
        // try environment
        const char * val = getenv("LANG");
        if (val)
                return val;

#ifdef WIN32
        // windows: make a win32 API call.
        // NOTE: this call only works for Vista+.  need to support XP?
        const int bufsize = LOCALE_NAME_MAX_LENGTH;
        WCHAR locale[bufsize];
        int nChars = GetUserDefaultLocaleName(locale, bufsize);
        ASSERT_THROW(nChars > 0, "Failed to get default locale name");

        // need to translate back to UTF-8
        UINT codePage = CP_UTF8;
        DWORD dwFlags = 0;
        LPCSTR defaultChar = NULL;
        LPBOOL usedDefault = NULL;
        const int maxLocaleSize = 2 * LOCALE_NAME_MAX_LENGTH;
        static char s_localeUTF8[maxLocaleSize];
        int nBytes = WideCharToMultiByte(codePage, dwFlags,
            locale, nChars, s_localeUTF8, maxLocaleSize,
            defaultChar, usedDefault);
        ASSERT_THROW(nBytes > 0, "Failed to translate locale to UTF8");
        // Windows uses "-" (hyphen) rather than "_" (underscore)
        // translate that too...  (assumes RFC639)
        if (nBytes > 2 && '-' == s_localeUTF8[2]) {
                s_localeUTF8[2] = '_';
        }

        // mandate UTF8 encoding
        strcpy(s_localeUTF8 + nBytes - 1, ".UTF-8");
        DPRINTF("win32 locale: '%s'", s_localeUTF8);

        return s_localeUTF8;
#endif  // WIN32

        // unable to determine locale!
        DPRINTF("Unable to determine host's locale!");
        return NULL;
}




};      // i18n namespace