aleajactaest/khanat-code-old
1
0
Fork 0
mirror of https://port.numenaute.org/aleajactaest/khanat-code-old.git synced 2024-12-28 19:30:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
khanat-code-old/code/nel/src/misc/i18n.cpp
botanic b09fb394da Backed out merge changeset: b2d97621fa3b 
Backed out merge revision to its first parent (99840e8413f2)
2014-02-13 12:12:28 -08:00

1161 lines
28 KiB
C++
Raw Blame History

// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
// Copyright (C) 2010 Winch Gate Property Limited
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
#include "stdmisc.h"
#include "nel/misc/path.h"
#include "nel/misc/i18n.h"
using namespace std;
#ifdef DEBUG_NEW
#define new DEBUG_NEW
#endif
namespace NLMISC {
CI18N::StrMapContainer CI18N::_StrMap;
CI18N::StrMapContainer CI18N::_StrMapFallback;
bool CI18N::_StrMapLoaded = false;
const ucstring CI18N::_NotTranslatedValue("<Not Translated>");
bool CI18N::_LanguagesNamesLoaded = false;
string CI18N::_SelectedLanguageCode;
CI18N::ILoadProxy *CI18N::_LoadProxy = 0;
vector<string> CI18N::_LanguageCodes;
vector<ucstring> CI18N::_LanguageNames;
bool CI18N::noResolution = false;
void CI18N::setLoadProxy(ILoadProxy *loadProxy)
{
_LoadProxy = loadProxy;
}
void CI18N::initLanguages()
{
if (!_LanguagesNamesLoaded)
{
_LanguageCodes.push_back("en");
_LanguageCodes.push_back("fr");
_LanguageCodes.push_back("de");
_LanguageCodes.push_back("ru");
_LanguageNames.push_back(ucstring("English"));
_LanguageNames.push_back(ucstring("French"));
_LanguageNames.push_back(ucstring("German"));
_LanguageNames.push_back(ucstring("Russian"));
_LanguagesNamesLoaded = true;
}
}
const std::vector<ucstring> &CI18N::getLanguageNames()
{
initLanguages();
return _LanguageNames;
}
const std::vector<std::string> &CI18N::getLanguageCodes()
{
initLanguages();
return _LanguageCodes;
}
void CI18N::load (const string &languageCode, const string &fallbackLanguageCode)
{
if (_StrMapLoaded) _StrMap.clear ();
else _StrMapLoaded = true;
_SelectedLanguageCode = languageCode;
loadFileIntoMap(languageCode + ".uxt", _StrMap);
_StrMapFallback.clear();
if(!fallbackLanguageCode.empty())
{
loadFileIntoMap(fallbackLanguageCode + ".uxt", _StrMapFallback);
}
}
bool CI18N::loadFileIntoMap(const string &fileName, StrMapContainer &destMap)
{
ucstring text;
// read in the text
if (_LoadProxy)
_LoadProxy->loadStringFile(fileName, text);
else
readTextFile(fileName, text);
// remove any comment
removeCComment(text);
ucstring::const_iterator first(text.begin()), last(text.end());
string lastReadLabel("nothing");
while (first != last)
{
skipWhiteSpace(first, last);
string label;
ucstring ucs;
if (!parseLabel(first, last, label))
{
nlwarning("I18N: Error reading label field in %s. Stop reading after %s.", fileName.c_str(), lastReadLabel.c_str());
return false;
}
lastReadLabel = label;
skipWhiteSpace(first, last);
if (!parseMarkedString('[', ']', first, last, ucs))
{
nlwarning("I18N: Error reading text for label %s in %s. Stop reading.", label.c_str(), fileName.c_str());
return false;
}
// ok, a line read.
pair<map<string, ucstring>::iterator, bool> ret;
ret = destMap.insert(make_pair(label, ucs));
if (!ret.second)
{
nlwarning("I18N: Error in %s, the label %s exists twice !", fileName.c_str(), label.c_str());
}
skipWhiteSpace(first, last);
}
// a little check to ensure that the lang name has been set.
StrMapContainer::iterator it(destMap.find("LanguageName"));
if (it == destMap.end())
{
nlwarning("I18N: In file %s, missing LanguageName translation (should be first in file)", fileName.c_str());
}
return true;
}
void CI18N::loadFromFilename(const string &filename, bool reload)
{
StrMapContainer destMap;
if (!loadFileIntoMap(filename, destMap))
{
return;
}
// merge with existing map
for(StrMapContainer::iterator it = destMap.begin(); it != destMap.end(); ++it)
{
if (!reload)
{
if (_StrMap.count(it->first))
{
nlwarning("I18N: Error in %s, the label %s exist twice !", filename.c_str(), it->first.c_str());
}
}
_StrMap[it->first] = it->second;
}
}
const ucstring &CI18N::get (const string &label)
{
if( noResolution )
{
static ucstring labelString;
labelString = label;
return labelString;
}
if (label.empty())
{
static ucstring emptyString;
return emptyString;
}
StrMapContainer::iterator it(_StrMap.find(label));
if (it != _StrMap.end())
return it->second;
static CHashSet<string> missingStrings;
if (missingStrings.find(label) == missingStrings.end())
{
nlwarning("I18N: The string %s did not exist in language %s (display once)", label.c_str(), _SelectedLanguageCode.c_str());
missingStrings.insert(label);
}
// use the fall back language if it exists
it = _StrMapFallback.find(label);
if (it != _StrMapFallback.end())
return it->second;
static ucstring badString;
badString = ucstring(string("<NotExist:")+label+">");
return badString;
}
bool CI18N::hasTranslation(const string &label)
{
if (label.empty()) return true;
if(_StrMap.find(label) != _StrMap.end())
return true;
// use the fall back language if it exists
if (_StrMapFallback.find(label) != _StrMapFallback.end())
return true;
return false;
}
ucstring CI18N::getCurrentLanguageName ()
{
return get("LanguageName");
}
string CI18N::getCurrentLanguageCode ()
{
return _SelectedLanguageCode;
}
void CI18N::removeCComment(ucstring &commentedString)
{
ucstring temp;
temp.reserve(commentedString.size());
ucstring::const_iterator first(commentedString.begin()), last(commentedString.end());
for (;first != last; ++first)
{
temp.push_back(*first);
if (*first == '[')
{
// no comment inside string literal
while (++first != last)
{
temp.push_back(*first);
if (*first == ']')
break;
}
}
else if (*first == '/')
{
// start of comment ?
++first;
if (first != last && *first == '/')
{
temp.resize(temp.size()-1);
// one line comment, skip until end of line
while (first != last && *first != '\n')
++first;
}
else if (first != last && *first == '*')
{
temp.resize(temp.size()-1);
// start of multi line comment, skip until we found '*/'
while (first != last && !(*first == '*' && (first+1) != last && *(first+1) == '/'))
++first;
// skip the closing '/'
if (first != last)
++first;
}
else
{
temp.push_back(*first);
}
}
}
commentedString.swap(temp);
}
void CI18N::skipWhiteSpace(ucstring::const_iterator &it, ucstring::const_iterator &last, ucstring *storeComments, bool newLineAsWhiteSpace)
{
while (it != last &&
(
(*it == 0xa && newLineAsWhiteSpace)
|| (*it == 0xd && newLineAsWhiteSpace)
|| *it == ' '
|| *it == '\t'
|| (storeComments && *it == '/' && it+1 != last && *(it+1) == '/')
|| (storeComments && *it == '/' && it+1 != last && *(it+1) == '*')
))
{
if (storeComments && *it == '/' && it+1 != last && *(it+1) == '/')
{
// found a one line C comment. Store it until end of line.
while (it != last && *it != '\n')
storeComments->push_back(*it++);
// store the final '\n'
if (it != last)
storeComments->push_back(*it++);
}
else if (storeComments && *it == '/' && it+1 != last && *(it+1) == '*')
{
// found a multi line C++ comment. store until we found the closing '*/'
while (it != last && !(*it == '*' && it+1 != last && *(it+1) == '/'))
storeComments->push_back(*it++);
// store the final '*'
if (it != last)
storeComments->push_back(*it++);
// store the final '/'
if (it != last)
storeComments->push_back(*it++);
// and a new line.
storeComments->push_back('\r');
storeComments->push_back('\n');
}
else
{
// just skip white space or don't store comments
++it;
}
}
}
bool CI18N::parseLabel(ucstring::const_iterator &it, ucstring::const_iterator &last, string &label)
{
ucstring::const_iterator rewind = it;
label.erase();
// first char must be A-Za-z@_
if (it != last &&
(
(*it >= '0' && *it <= '9')
|| (*it >= 'A' && *it <= 'Z')
|| (*it >= 'a' && *it <= 'z')
|| (*it == '_')
|| (*it == '@')
)
)
label.push_back(char(*it++));
else
{
it = rewind;
return false;
}
// other char must be [0-9A-Za-z@_]*
while (it != last &&
(
(*it >= '0' && *it <= '9')
|| (*it >= 'A' && *it <= 'Z')
|| (*it >= 'a' && *it <= 'z')
|| (*it == '_')
|| (*it == '@')
)
)
label.push_back(char(*it++));
return true;
}
bool CI18N::parseMarkedString(ucchar openMark, ucchar closeMark, ucstring::const_iterator &it, ucstring::const_iterator &last, ucstring &result, uint32 *lineCounter, bool allowNewline)
{
result.erase();
// parse a string delimited by the specified opening and closing mark
if (it != last && *it == openMark)
{
++it;
while (it != last && *it != closeMark && (allowNewline || *it != '\n'))
{
// ignore tab, new lines and line feed
if (*it == openMark)
{
nlwarning("I18N: Found a non escaped openmark %c in a delimited string (Delimiters : '%c' - '%c')", char(openMark), char(openMark), char(closeMark));
return false;
}
if (*it == '\t'
|| (*it == '\n' && allowNewline)
|| *it == '\r')
++it;
else if (*it == '\\' && it+1 != last && *(it+1) != '\\')
{
++it;
// this is an escape sequence !
switch(*it)
{
case 't':
result.push_back('\t');
break;
case 'n':
result.push_back('\n');
break;
case 'd':
// insert a delete
result.push_back(8);
break;
default:
// escape the close mark ?
if(*it == closeMark)
result.push_back(closeMark);
// escape the open mark ?
else if(*it == openMark)
result.push_back(openMark);
else
{
nlwarning("I18N: Ignoring unknown escape code \\%c (char value : %u)", char(*it), *it);
return false;
}
}
++it;
}
else if (*it == '\\' && it+1 != last && *(it+1) == '\\')
{
// escape the \ char
++it;
result.push_back(*it);
++it;
}
else
{
if (*it == '\n' && lineCounter != NULL)
// update line counter
++(*lineCounter);
result.push_back(*it++);
}
}
if (it == last || *it != closeMark)
{
nlwarning("I18N: Missing end of delimited string (Delimiters : '%c' - '%c')", char(openMark), char(closeMark));
return false;
}
else
++it;
}
else
{
nlwarning("I18N: Malformed or non existent delimited string (Delimiters : '%c' - '%c')", char(openMark), char(closeMark));
return false;
}
return true;
}
void CI18N::readTextFile(const string &filename,
ucstring &result,
bool forceUtf8,
bool fileLookup,
bool preprocess,
TLineFormat lineFmt,
bool warnIfIncludesNotFound)
{
// create the read context
TReadContext readContext;
// call the inner function
_readTextFile(filename, result, forceUtf8, fileLookup, preprocess, lineFmt, warnIfIncludesNotFound, readContext);
if (!readContext.IfStack.empty())
{
nlwarning("Preprocess: Missing %u closing #endif after parsing %s", readContext.IfStack.size(), filename.c_str() );
}
}
bool CI18N::matchToken(const char* token, ucstring::const_iterator &it, ucstring::const_iterator end)
{
ucstring::const_iterator rewind = it;
skipWhiteSpace(it, end, NULL, false);
while (it != end && *token != 0 && *it == *token)
{
++it;
++token;
}
if (*token == 0)
{
// we fund the token
return true;
}
// not found
it = rewind;
return false;
}
void CI18N::skipLine(ucstring::const_iterator &it, ucstring::const_iterator end, uint32 &lineCounter)
{
while (it != end && *it != '\n')
++it;
if (it != end)
{
++lineCounter;
++it;
}
}
void CI18N::_readTextFile(const string &filename,
ucstring &result,
bool forceUtf8,
bool fileLookup,
bool preprocess,
TLineFormat lineFmt,
bool warnIfIncludesNotFound,
TReadContext &readContext)
{
string fullName;
if (fileLookup)
fullName = CPath::lookup(filename, false,warnIfIncludesNotFound);
else
fullName = filename;
if (fullName.empty())
return;
// If ::lookup is used, the file can be in a bnp and CFile::fileExists fails.
bool isInBnp = fullName.find('@') != string::npos;
if (!isInBnp && !CFile::fileExists(fullName))
{
nlwarning("CI18N::readTextFile : file '%s' does not exist, returning empty string", fullName.c_str());
return;
}
NLMISC::CIFile file(fullName);
// Fast read all the text in binary mode.
string text;
text.resize(file.getFileSize());
if (file.getFileSize() > 0)
file.serialBuffer((uint8*)(&text[0]), (uint)text.size());
// Transform the string in ucstring according to format header
if (!text.empty())
readTextBuffer((uint8*)&text[0], (uint)text.size(), result, forceUtf8);
if (preprocess)
{
// a string to old the result of the preprocess
ucstring final;
// make rooms to reduce allocation cost
final.reserve(raiseToNextPowerOf2((uint)result.size()));
// parse the file, looking for preprocessor command.
ucstring::const_iterator it(result.begin()), end(result.end());
// input line counter
uint32 currentLine = 1;
// set the current file and line info
final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
while (it != end)
{
// remember the begin of the line
ucstring::const_iterator beginOfLine = it;
// advance in the line, looking for a preprocessor command
skipWhiteSpace(it, end, NULL, false);
if (it != end && *it == '#')
{
// skip the '#' symbol
++it;
// we found a preprocessor command !
skipWhiteSpace(it, end, NULL, false);
if (matchToken("include", it, end))
{
if (readContext.IfStack.empty() || readContext.IfStack.back())
{
// we have an include command
skipWhiteSpace(it, end, NULL, false);
// read the file name between quote
ucstring str;
breakable
{
if (!parseMarkedString(ucchar('\"'), ucchar('\"'), it, end, str, &currentLine, false))
{
nlwarning("Preprocess: In file %s(%u) : Error parsing include file command", filename.c_str(), currentLine);
break;
}
else
{
// ok, read the subfile
string subFilename = str.toString();
// check is file exist
if (!CFile::fileExists(subFilename))
{
// look for the file relative to current file
subFilename = CFile::getPath(filename)+subFilename;
if (!CFile::fileExists(subFilename))
{
// the include file is not found, issue a warning
nlwarning("Preprocess: In file %s(%u) : Cannot include file '%s'",
filename.c_str(), currentLine,
str.toString().c_str());
break;
}
}
nlinfo("Preprocess: In file %s(%u) : Including '%s'",
filename.c_str(), currentLine,
subFilename.c_str());
ucstring inserted;
_readTextFile(subFilename, inserted, forceUtf8, fileLookup, preprocess, lineFmt, warnIfIncludesNotFound, readContext);
final += inserted;
}
}
// advance to next line
skipLine(it, end, currentLine);
// reset filename and line counter
final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
}
}
else if (matchToken("optional", it, end))
{
if (readContext.IfStack.empty() || readContext.IfStack.back())
{
// we have an optional include command
skipWhiteSpace(it, end, NULL, false);
// read the file name between quote
ucstring str;
breakable
{
if (!parseMarkedString('\"', '\"', it, end, str, &currentLine, false))
{
nlwarning("Preprocess: In file %s(%u) : Error parsing optional file command", filename.c_str(), currentLine);
break;
}
else
{
// ok, read the subfile
string subFilename = str.toString();
// check is file exist
if (!CFile::fileExists(subFilename))
{
// look for the file relative to current file
subFilename = CFile::getPath(filename)+subFilename;
if (!CFile::fileExists(subFilename))
{
// not found but optional, only emit a debug log
// the include file is not found, issue a warning
nldebug("Preprocess: In file %s(%u) : Cannot include optional file '%s'",
filename.c_str(), currentLine,
str.toString().c_str());
break;
}
}
nlinfo("Preprocess: In file %s(%u) : Including optional '%s'",
filename.c_str(), currentLine,
subFilename.c_str());
ucstring inserted;
_readTextFile(subFilename, inserted, forceUtf8, fileLookup, preprocess, lineFmt, warnIfIncludesNotFound, readContext);
final += inserted;
}
}
// advance to next line
skipLine(it, end, currentLine);
// reset filename and line counter
final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
}
}
else if (matchToken("define", it, end))
{
if (readContext.IfStack.empty() || readContext.IfStack.back())
{
skipWhiteSpace(it, end, NULL, false);
string label;
if (parseLabel(it, end, label))
{
if (readContext.Defines.find(label) != readContext.Defines.end())
{
nlinfo("Preprocess: In file %s(%u) : symbol '%s' already defined",
filename.c_str(), currentLine,
label.c_str());
}
else
{
readContext.Defines.insert(label);
}
}
else
{
nlwarning("Preprocess: In file %s(%u) : Error parsing #define command", filename.c_str(), currentLine);
}
// advance to next line
skipLine(it, end, currentLine);
// update filename and line number
final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
}
}
else if (matchToken("ifdef", it, end))
{
if (readContext.IfStack.empty() || readContext.IfStack.back())
{
skipWhiteSpace(it, end, NULL, false);
string label;
if (parseLabel(it, end, label))
{
if (readContext.Defines.find(label) != readContext.Defines.end())
{
// symbol defined, push a true
readContext.IfStack.push_back(true);
}
else
{
// symbol not defines, push a false
readContext.IfStack.push_back(false);
}
}
else
{
nlwarning("Preprocess: In file %s(%u) : Error parsing #ifdef command", filename.c_str(), currentLine);
}
// advance to next line
skipLine(it, end, currentLine);
// update filename and line number
final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
}
else
{
// just push to false
readContext.IfStack.push_back(false);
skipLine(it, end, currentLine);
}
}
else if (matchToken("ifndef", it, end))
{
if (readContext.IfStack.empty() || readContext.IfStack.back())
{
skipWhiteSpace(it, end, NULL, false);
string label;
if (parseLabel(it, end, label))
{
if (readContext.Defines.find(label) == readContext.Defines.end())
{
// symbol defined, push a true
readContext.IfStack.push_back(true);
}
else
{
// symbol not defines, push a false
readContext.IfStack.push_back(false);
}
}
else
{
nlwarning("Preprocess: In file %s(%u) : Error parsing #ifndef command", filename.c_str(), currentLine);
}
// advance to next line
skipLine(it, end, currentLine);
// update filename and line number
final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
}
else
{
// just push to false
readContext.IfStack.push_back(false);
skipLine(it, end, currentLine);
}
}
else if (matchToken("endif", it, end))
{
bool previous = false;
if (readContext.IfStack.empty())
{
nlwarning("Preprocess: In file %s(%u) : Error found '#endif' without matching #if", filename.c_str(), currentLine);
}
else
{
previous = readContext.IfStack.back();
readContext.IfStack.pop_back();
}
skipLine(it, end, currentLine);
if (!previous && (readContext.IfStack.empty() || readContext.IfStack.back()))
{
// end of ignored file part, restore the file and line number
final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
}
// update filename and line number
// final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
}
else
{
// unrecognized command, ignore line
nlwarning("Preprocess: In file %s(%u) : Error unrecognized preprocessor command",
filename.c_str(), currentLine);
skipLine(it, end, currentLine);
// update filename and line number
final += toString("#fileline \"%s\" %u\n", filename.c_str(), currentLine);
}
}
else
{
// normal line
skipLine(it, end, currentLine);
if (readContext.IfStack.empty() || readContext.IfStack.back())
{
// copy the line to the final string
final.append(beginOfLine, it);
}
}
}
// set the result with the preprocessed content
result.swap(final);
}
// apply line delimiter conversion if needed
if (lineFmt != LINE_FMT_NO_CARE)
{
if (lineFmt == LINE_FMT_LF)
{
// we only want \n
// easy, just remove or replace any \r code
string::size_type pos;
string::size_type lastPos = 0;
ucstring temp;
// reserve some place to reduce re-allocation
temp.reserve(result.size() +result.size()/10);
// look for the first \r
pos = result.find('\r');
while (pos != string::npos)
{
if (pos < result.size()-1 && result[pos+1] == '\n')
{
temp.append(result.begin()+lastPos, result.begin()+pos);
pos += 1;
}
else
{
temp.append(result.begin()+lastPos, result.begin()+pos);
temp[temp.size()-1] = '\n';
}
lastPos = pos;
// look for next \r
pos = result.find('\r', pos);
}
// copy the rest
temp.append(result.begin()+lastPos, result.end());
result.swap(temp);
}
else if (lineFmt == LINE_FMT_CRLF)
{
// need to replace simple '\n' or '\r' with a '\r\n' double
string::size_type pos = 0;
string::size_type lastPos = 0;
ucstring temp;
// reserve some place to reduce re-allocation
temp.reserve(result.size() +result.size()/10);
// first loop with the '\r'
pos = result.find('\r', pos);
while (pos != string::npos)
{
if (pos >= result.size()-1 || result[pos+1] != '\n')
{
temp.append(result.begin()+lastPos, result.begin()+pos+1);
temp += '\n';
lastPos = pos+1;
}
// skip this char
pos++;
// look the next '\r'
pos = result.find('\r', pos);
}
// copy the rest
temp.append(result.begin()+lastPos, result.end());
result.swap(temp);
temp = "";
// second loop with the '\n'
pos = 0;
lastPos = 0;
pos = result.find('\n', pos);
while (pos != string::npos)
{
if (pos == 0 || result[pos-1] != '\r')
{
temp.append(result.begin()+lastPos, result.begin()+pos);
temp += '\r';
temp += '\n';
lastPos = pos+1;
}
// skip this char
pos++;
pos = result.find('\n', pos);
}
// copy the rest
temp.append(result.begin()+lastPos, result.end());
result.swap(temp);
}
}
}
void CI18N::readTextBuffer(uint8 *buffer, uint size, ucstring &result, bool forceUtf8)
{
static uint8 utf16Header[] = { 0xffu, 0xfeu };
static uint8 utf16RevHeader[] = { 0xfeu, 0xffu };
static uint8 utf8Header[] = { 0xefu, 0xbbu, 0xbfu };
if (forceUtf8)
{
if (size>=3 &&
buffer[0]==utf8Header[0] &&
buffer[1]==utf8Header[1] &&
buffer[2]==utf8Header[2]
)
{
// remove utf8 header
buffer+= 3;
size-=3;
}
string text((char*)buffer, size);
result.fromUtf8(text);
}
else if (size>=3 &&
buffer[0]==utf8Header[0] &&
buffer[1]==utf8Header[1] &&
buffer[2]==utf8Header[2]
)
{
// remove utf8 header
buffer+= 3;
size-=3;
string text((char*)buffer, size);
result.fromUtf8(text);
}
else if (size>=2 &&
buffer[0]==utf16Header[0] &&
buffer[1]==utf16Header[1]
)
{
// remove utf16 header
buffer+= 2;
size-= 2;
// check pair number of bytes
nlassert((size & 1) == 0);
// and do manual conversion
uint16 *src = (uint16*)(buffer);
result.resize(size/2);
for (uint j=0; j<result.size(); j++)
result[j]= *src++;
}
else if (size>=2 &&
buffer[0]==utf16RevHeader[0] &&
buffer[1]==utf16RevHeader[1]
)
{
// remove utf16 header
buffer+= 2;
size-= 2;
// check pair number of bytes
nlassert((size & 1) == 0);
// and do manual conversion
uint16 *src = (uint16*)(buffer);
result.resize(size/2);
uint j;
for (j=0; j<result.size(); j++)
result[j]= *src++;
// Reverse byte order
for (j=0; j<result.size(); j++)
{
uint8 *pc = (uint8*) &result[j];
swap(pc[0], pc[1]);
}
}
else
{
// hum.. ascii read ?
// so, just do a direct conversion
string text((char*)buffer, size);
result = text;
}
}
void CI18N::writeTextFile(const string filename, const ucstring &content, bool utf8)
{
COFile file(filename);
if (!utf8)
{
// write the Unicode 16 bits tag
uint16 unicodeTag = 0xfeff;
file.serial(unicodeTag);
uint i;
for (i=0; i<content.size(); ++i)
{
uint16 c = content[i];
file.serial(c);
}
}
else
{
static char utf8Header[] = {char(0xef), char(0xbb), char(0xbf), 0};
string str = encodeUTF8(content);
// add the UTF-8 'not official' header
str = utf8Header + str;
uint i;
for (i=0; i<str.size(); ++i)
{
file.serial(str[i]);
}
}
}
ucstring CI18N::makeMarkedString(ucchar openMark, ucchar closeMark, const ucstring &text)
{
ucstring ret;
ret.push_back(openMark);
ucstring::const_iterator first(text.begin()), last(text.end());
for (; first != last; ++first)
{
if (*first == '\n')
{
ret += '\\';
ret += 'n';
}
else if (*first == '\t')
{
ret += '\\';
ret += 't';
}
else if (*first == closeMark)
{
// escape the embedded closing mark
ret += '\\';
ret += closeMark;
}
else
{
ret += *first;
}
}
ret += closeMark;
return ret;
}
string CI18N::encodeUTF8(const ucstring &str)
{
return str.toUtf8();
}
/* UTF-8 conversion table
U-00000000 - U-0000007F: 0xxxxxxx
U-00000080 - U-000007FF: 110xxxxx 10xxxxxx
U-00000800 - U-0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx
// not used as we convert from 16 bits unicode
U-00010000 - U-001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
U-00200000 - U-03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
U-04000000 - U-7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
*/
uint64 CI18N::makeHash(const ucstring &str)
{
// we do at least 8 pass on each result byte
if (str.empty())
return 0;
const uint32 MIN_TURN = 8*8;
uint64 hash = 0;
uint8 *ph = (uint8*)&hash;
uint8 *pc = (uint8*)str.data();
uint nbLoop = max(uint32(str.size()*2), MIN_TURN);
uint roll = 0;
for (uint i=0; i<nbLoop; ++i)
{
ph[(i/2) & 0x7] = uint8((ph[(i/2) & 0x7] + (pc[i%(str.size()*2)] << roll)) & 0xff);
ph[(i/2) & 0x7] = uint8((ph[(i/2) & 0x7] + (pc[i%(str.size()*2)] >> (8-roll))) & 0xff);
roll++;
roll &= 0x7;
}
return hash;
}
// convert a hash value to a readable string
string CI18N::hashToString(uint64 hash)
{
char temp[] = "0011223344556677";
sprintf(temp, "%08X%08X", (uint32)(hash & 0xffffffff), (uint32)(hash >> 32));
return string(temp);
}
// fast convert a hash value to a ucstring
void CI18N::hashToUCString(uint64 hash, ucstring &dst)
{
static ucchar cvtTable[]= {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
dst.resize(16);
for(sint i=15;i>=0;i--)
{
// Must decal dest of 8, cause of hashToString code (Little Endian)
dst[(i+8)&15]= cvtTable[hash&15];
hash>>=4;
}
}
// convert a readable string into a hash value.
uint64 CI18N::stringToHash(const string &str)
{
nlassert(str.size() == 16);
uint32 low, high;
string sl, sh;
sh = str.substr(0, 8);
sl = str.substr(8, 8);
sscanf(sh.c_str(), "%08X", &high);
sscanf(sl.c_str(), "%08X", &low);
uint64 hash;
memcpy(&hash, &high, sizeof(high));
memcpy((uint32*)&hash + 1, &low, sizeof(low));
return hash;
}
} // namespace NLMISC
Reference in a new issue View git blame Copy permalink