khanat-code-old/code/ryzom/tools/translation_tools/extract_bot_names.cpp
botanic 80660b5146 FIX #200
dont crash when neldb is missing
2014-09-11 10:50:34 -07:00

755 lines
20 KiB
C++

// Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
// Copyright (C) 2010 Winch Gate Property Limited
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
#include "nel/misc/types_nl.h"
#include "nel/misc/config_file.h"
#include "nel/misc/sheet_id.h"
#include "nel/misc/path.h"
#include "nel/misc/diff_tool.h"
#include "nel/georges/u_form.h"
#include "nel/georges/u_form_elm.h"
#include "nel/georges/load_form.h"
#include "nel/ligo/ligo_config.h"
#include "nel/ligo/primitive.h"
#include "nel/ligo/primitive_utils.h"
using namespace std;
using namespace NLMISC;
using namespace NLLIGO;
using namespace STRING_MANAGER;
vector<string> Filters;
static CLigoConfig LigoConfig;
static bool RemoveOlds = false;
struct TCreatureInfo
{
CSheetId SheetId;
bool ForceSheetName;
bool DisplayName;
void readGeorges (const NLMISC::CSmartPtr<NLGEORGES::UForm> &form, const NLMISC::CSheetId &sheetId)
{
const NLGEORGES::UFormElm &item=form->getRootNode();
SheetId=sheetId;
item.getValueByName(ForceSheetName, "3d data.ForceDisplayCreatureName");
item.getValueByName(DisplayName, "3d data.DisplayName");
}
void serial(NLMISC::IStream &f)
{
f.serial(SheetId);
f.serial(ForceSheetName);
f.serial(DisplayName);
}
static uint getVersion ()
{
return 1;
}
void removed()
{
}
};
std::map<CSheetId, TCreatureInfo> Creatures;
TCreatureInfo *getCreature(const std::string &sheetName)
{
CSheetId id(sheetName+".creature");
if (Creatures.find(id) != Creatures.end())
return &(Creatures.find(id)->second);
else
return NULL;
}
string cleanupName(const std::string &name)
{
string ret;
for (uint i=0; i<name.size(); ++i)
{
if (name[i] != ' ')
ret += name[i];
else
ret += '_';
}
return ret;
}
ucstring cleanupUcName(const ucstring &name)
{
ucstring ret;
for (uint i=0; i<name.size(); ++i)
{
if (name[i] != ' ')
ret += name[i];
else
ret += '_';
}
return ret;
}
/*
Removes first and last '$'
*/
ucstring makeGroupName(const ucstring & translationName)
{
ucstring ret = translationName;
if (ret.size() >= 2)
{
if ( *ret.begin() == ucchar('$'))
{
ret=ret.substr(1);
}
if ( *ret.rbegin() == ucchar('$'))
{
ret = ret.substr(0, ret.size()-1);
}
}
ret = cleanupUcName(ret);
return ret;
}
struct TEntryInfo
{
string SheetName;
};
set<string> GenericNames;
map<string, TEntryInfo> SimpleNames;
set<string> Functions;
string removeAndStoreFunction(const std::string &fullName)
{
string::size_type pos = fullName.find("$");
if (pos == string::npos)
return fullName;
else
{
// extract and store the function name
string ret;
ret = fullName.substr(0, pos);
string::size_type pos2 = fullName.find("$", pos+1);
string fct = fullName.substr(pos+1, pos2-(pos+1));
ret += fullName.substr(pos2+1);
if (Functions.find(fct) == Functions.end())
{
nldebug("Adding function '%s'", fct.c_str());
Functions.insert(fct);
}
return ret;
}
}
void addGenericName(const std::string &name, const std::string &sheetName)
{
TCreatureInfo *c = getCreature(sheetName);
if (!c || c->ForceSheetName || !c->DisplayName)
return;
if (SimpleNames.find(name) != SimpleNames.end())
{
nldebug("Name '%s' is now a generic name", name.c_str());
GenericNames.insert(name);
SimpleNames.erase(name);
}
else if (GenericNames.find(name) == GenericNames.end())
{
nldebug("Adding generic name '%s'", name.c_str());
GenericNames.insert(name);
}
}
void addSimpleName(const std::string &name, const std::string &sheetName)
{
TCreatureInfo *c = getCreature(sheetName);
if (!c || c->ForceSheetName || !c->DisplayName)
return;
if (SimpleNames.find(name) != SimpleNames.end())
{
addGenericName(name, sheetName);
}
else if (GenericNames.find(name) != GenericNames.end())
{
return;
}
else
{
nldebug("Adding simple name '%s'", name.c_str());
TEntryInfo ei;
ei.SheetName = sheetName;
SimpleNames.insert(make_pair(name, ei));
}
}
int extractBotNames(int argc, char *argv[])
{
//-------------------------------------------------------------------
// read the parameters
for (int i=2; i<argc; ++i)
{
string s = argv[i];
if (s == "-r")
{
// active remove mode
RemoveOlds = true;
}
else
{
nlwarning("Unknow option '%s'", argv[i]);
return -1;
}
}
//-------------------------------------------------------------------
// read the configuration file
CConfigFile cf;
cf.load("bin/translation_tools.cfg");
//-------------------------------------------------------------------
// read the vars
CConfigFile::CVar &paths = cf.getVar("Paths");
CConfigFile::CVar &filtersVar = cf.getVar("Filters");
CConfigFile::CVar &ligoClassFile= cf.getVar("LigoClassFile");
CConfigFile::CVar &georgesPaths= cf.getVar("GeorgesPaths");
CConfigFile::CVar &pathNoRecurse= cf.getVar("PathsNoRecurse");
CConfigFile::CVar &workBotNamesFile= cf.getVar("WorkBotNamesFile");
CConfigFile::CVar &transBotNamesFile= cf.getVar("TransBotNamesFile");
CConfigFile::CVar &workTitleFile= cf.getVar("WorkTitleFile");
for (uint i=0; i<paths.size(); ++i)
{
CPath::addSearchPath(paths.asString(i), true, false);
}
for (uint i=0; i<pathNoRecurse.size(); ++i)
{
CPath::addSearchPath(pathNoRecurse.asString(i), false, false);
}
for (uint i=0; i<filtersVar.size(); ++i)
{
Filters.push_back(filtersVar.asString(i));
}
//-------------------------------------------------------------------
// init the sheets
CSheetId::init(false);
const string PACKED_SHEETS_NAME = "bin/translation_tools_creature.packed_sheets";
loadForm("creature", PACKED_SHEETS_NAME, Creatures, false, false);
if (Creatures.empty())
{
for (uint i=0;i<georgesPaths.size();++i)
CPath::addSearchPath(georgesPaths.asString(i).c_str(), true, false);
loadForm("creature", PACKED_SHEETS_NAME, Creatures, true);
}
//-------------------------------------------------------------------
// init ligo config
string ligoPath = CPath::lookup(ligoClassFile.asString(), true, true);
LigoConfig.readPrimitiveClass(ligoPath.c_str(), false);
NLLIGO::Register();
CPrimitiveContext::instance().CurrentLigoConfig = &LigoConfig;
//-------------------------------------------------------------------
// ok, ready for the real work,
// first, read the primitives files and parse the primitives
vector<string> files;
CPath::getFileList("primitive", files);
for (uint i=0; i<files.size(); ++i)
{
string pathName = files[i];
pathName = CPath::lookup(pathName);
// check filters
uint j=0;
for (j=0; j<Filters.size(); ++j)
{
if (pathName.find(Filters[j]) != string::npos)
break;
}
if (j != Filters.size())
// skip this file
continue;
nlinfo("Loading file '%s'...", CFile::getFilename(pathName).c_str());
CPrimitives primDoc;
CPrimitiveContext::instance().CurrentPrimitive = &primDoc;
loadXmlPrimitiveFile(primDoc, pathName, LigoConfig);
// now parse the file
// look for group template
{
TPrimitiveClassPredicate pred("group_template_npc");
TPrimitiveSet result;
CPrimitiveSet<TPrimitiveClassPredicate> ps;
ps.buildSet(primDoc.RootNode, pred, result);
for (uint i=0; i<result.size(); ++i)
{
string name;
string countStr;
string sheetStr;
result[i]->getPropertyByName("name", name);
result[i]->getPropertyByName("count", countStr);
result[i]->getPropertyByName("bot_sheet_look", sheetStr);
uint32 count;
NLMISC::fromString(countStr, count);
if (count != 0)
{
if (sheetStr.empty())
{
nlwarning("In '%s', empty sheet !", buildPrimPath(result[i]).c_str());
}
else
{
addGenericName(removeAndStoreFunction(name), sheetStr);
}
}
}
}
// look for bot template
{
TPrimitiveClassPredicate pred("bot_template_npc");
TPrimitiveSet result;
CPrimitiveSet<TPrimitiveClassPredicate> ps;
ps.buildSet(primDoc.RootNode, pred, result);
for (uint i=0; i<result.size(); ++i)
{
string name;
string sheetStr;
result[i]->getPropertyByName("name", name);
result[i]->getPropertyByName("sheet_look", sheetStr);
if (sheetStr.empty())
{
// take the sheet in the parent
result[i]->getParent()->getPropertyByName("bot_sheet_look", sheetStr);
}
if (sheetStr.empty())
{
nlwarning("In '%s', empty sheet !", buildPrimPath(result[i]).c_str());
}
else
{
addGenericName(removeAndStoreFunction(name), sheetStr);
}
}
}
// look for npc_group
{
TPrimitiveClassPredicate pred("npc_group");
TPrimitiveSet result;
CPrimitiveSet<TPrimitiveClassPredicate> ps;
ps.buildSet(primDoc.RootNode, pred, result);
for (uint i=0; i<result.size(); ++i)
{
string name;
string countStr;
string sheetStr;
result[i]->getPropertyByName("name", name);
result[i]->getPropertyByName("count", countStr);
result[i]->getPropertyByName("bot_sheet_client", sheetStr);
uint32 count;
NLMISC::fromString(countStr, count);
if (count > 0 && sheetStr.empty())
{
nlwarning("In '%s', empty sheet !", buildPrimPath(result[i]).c_str());
}
else
{
if (count == 1)
{
addSimpleName(removeAndStoreFunction(name), sheetStr);
}
else if (count > 1)
{
addGenericName(removeAndStoreFunction(name), sheetStr);
}
}
}
}
// look for bot
{
TPrimitiveClassPredicate pred("npc_bot");
TPrimitiveSet result;
CPrimitiveSet<TPrimitiveClassPredicate> ps;
ps.buildSet(primDoc.RootNode, pred, result);
for (uint i=0; i<result.size(); ++i)
{
string name;
string sheetStr;
result[i]->getPropertyByName("name", name);
result[i]->getPropertyByName("sheet_client", sheetStr);
if (sheetStr.empty())
{
// take the sheet in the parent
result[i]->getParent()->getPropertyByName("bot_sheet_client", sheetStr);
}
if (sheetStr.empty())
{
nlwarning("In '%s', empty sheet !", buildPrimPath(result[i]).c_str());
}
else
{
TEntryInfo ei;
addSimpleName(removeAndStoreFunction(name), sheetStr);
}
}
}
}
//-------------------------------------------------------------------
// step 2 : load the reference file
nlinfo("Looking for missing translation:");
TWorksheet botNames;
loadExcelSheet(workBotNamesFile.asString(), botNames, true);
TWorksheet transBotNames;
loadExcelSheet(transBotNamesFile.asString(), transBotNames, true);
TWorksheet fcts;
loadExcelSheet(workTitleFile.asString(), fcts, true);
// add missing element
uint nbAddSimpleName = 0;
uint nbAddFunction = 0;
uint nbAddGenericName = 0;
uint botIdCol;
nlverify(botNames.findId(botIdCol));
uint transIdCol;
nlverify(transBotNames.findId(transIdCol));
uint fctsIdCol;
nlverify(fcts.findId(fctsIdCol));
// special treatment to add the sheet_name col
{
uint sheetCol;
if (!botNames.findCol(ucstring("sheet_name"), sheetCol))
{
botNames.insertColumn(botNames.ColCount);
botNames.setData(0, botNames.ColCount-1, ucstring("sheet_name"));
}
if (!transBotNames.findCol(ucstring("sheet_name"), sheetCol))
{
transBotNames.insertColumn(transBotNames.ColCount);
transBotNames.setData(0, transBotNames.ColCount-1, ucstring("sheet_name"));
}
}
// 1 - simple names
{
nlinfo(" Simple names...");
map<string, TEntryInfo>::iterator first(SimpleNames.begin()), last(SimpleNames.end());
for (; first != last; ++first)
{
uint rowIdx = 0;
if (!botNames.findRow(botIdCol, first->first, rowIdx))
{
// we need to add the entry
rowIdx = botNames.size();
botNames.resize(botNames.size()+1);
botNames.setData(rowIdx, ucstring("bot name"), first->first);
botNames.setData(rowIdx, ucstring("translated name"), first->first);
botNames.setData(rowIdx, ucstring("sheet_name"), first->second.SheetName);
nbAddSimpleName++;
}
else
{
// set/update the sheet name info
// try to restore the existing translation
uint transRowIdx = 0;
if (transBotNames.findRow(transIdCol, first->first, transRowIdx))
{
ucstring wkBotName = botNames.getData(rowIdx, ucstring("bot name"));
ucstring wkSheetName = botNames.getData(rowIdx, ucstring("sheet_name"));
ucstring wkTranslationName = botNames.getData(rowIdx, ucstring("translated name"));
ucstring ucWkHash;
uint64 hash = CI18N::makeHash(wkBotName + wkTranslationName +wkSheetName);
CI18N::hashToUCString(hash, ucWkHash);
ucstring trUcHash = transBotNames[transRowIdx][0];
bool isWkTranslationNameAGroupName = wkTranslationName.find(ucstring("$")) != ucstring::npos;
bool hashIsValide = std::equal(ucWkHash.begin(), ucWkHash.end(), trUcHash.begin()+1);
// Hash is equal get the translation
if (hashIsValide && !isWkTranslationNameAGroupName)
{
wkTranslationName = transBotNames.getData(transRowIdx, ucstring("translated name"));
wkSheetName = transBotNames.getData(transRowIdx, ucstring("sheet_name"));
botNames.setData(rowIdx, ucstring("translated name"), wkTranslationName);
botNames.setData(rowIdx, ucstring("sheet_name"), wkSheetName);
hash = CI18N::makeHash(wkBotName + wkTranslationName + wkSheetName);
// update the hash code
CI18N::hashToUCString(hash, transBotNames[transRowIdx][0]);
}
// bots_name.txt has been manually changed. We trust what the Level Designer has done. We don't destroy is work.
// or it is a simple
else
{
//use the "translated name" of the manually changed work/bot_name.txt
botNames.setData(rowIdx, ucstring("translated name"), wkTranslationName);
botNames.setData(rowIdx, ucstring("sheet_name"), wkSheetName);
}
}
}
}
}
// 2 - generic names
{
nlinfo(" Generic names...");
set<string>::iterator first(GenericNames.begin()), last(GenericNames.end());
for (; first != last; ++first)
{
string gnName = "gn_" + cleanupName(*first);
ucstring fctsTitleId;
ucstring fctsName;
// add or modify the bot names
uint rowIdx;
if (!botNames.findRow(botIdCol, *first, rowIdx))
{
// we need to add the entry
rowIdx = botNames.size();
botNames.resize(botNames.size()+1);
botNames.setData(rowIdx, ucstring("bot name"), *first);
botNames.setData(rowIdx, ucstring("translated name"), ucstring("$") + gnName + "$");
botNames.setData(rowIdx, ucstring("sheet_name"), ucstring());
fctsTitleId = gnName;
fctsName = *first;
nbAddSimpleName++;
}
else
{
// look in the translated table to remember the translated name to write it in the string file
ucstring wkBotName = botNames.getData(rowIdx, ucstring("bot name"));
ucstring wkTranslationName = botNames.getData(rowIdx, ucstring("translated name"));
ucstring wkSheetName = botNames.getData(rowIdx, ucstring("sheet_name"));
nlinfo("Bot name:%s\n",wkBotName.toString().c_str());
bool isWkTranslationNameAGroupName = wkTranslationName.find(ucstring("$")) != ucstring::npos;
if ( isWkTranslationNameAGroupName ) //work name looks like "$gn_***$: do not modify
{
//Do not change work/bot_name.txt
// update work/world_title.txt
ucstring transName;
fctsTitleId = makeGroupName(wkTranslationName);
uint transRowIdx;
if (transBotNames.findRow(transIdCol, *first, transRowIdx))
{
transName = transBotNames.getData(transRowIdx, ucstring("translated name"));
if (transName.find(ucstring("$")) != ucstring::npos)
{
transName = fctsTitleId;
}
}
else
{
transName = fctsTitleId;
}
//Do not touch anything
botNames.setData(rowIdx, ucstring("translated name"), wkTranslationName);
botNames.setData(rowIdx, ucstring("sheet_name"), wkSheetName);
// fctsTitleId = makeGroupName(wkTranslationName);
fctsName = transName;
}
else // WkTranslationName != "$gn*$"
{
uint transRowIdx;
ucstring transName;
ucstring wkSheetName;
// Get the translation as a simple name.
if (transBotNames.findRow(transIdCol, *first, transRowIdx))
{
transName = transBotNames.getData(transRowIdx, ucstring("translated name"));
ucstring trSheetName = transBotNames.getData(transRowIdx, ucstring("sheet_name"));
//tr."translation name" is
if (transName.find(ucstring("$")) != ucstring::npos)
{
//get Translation, update hash
botNames[rowIdx][1] = transName;
botNames[rowIdx][2] = trSheetName;
fctsTitleId = makeGroupName(transName);
fctsName = makeGroupName(transName);
ucstring trNewUcHash;
uint64 hash = CI18N::makeHash(wkBotName + transName +trSheetName);
CI18N::hashToUCString(hash, trNewUcHash);
transBotNames[transRowIdx][0] = ucstring("_") + trNewUcHash;
}
else //botNames."translated name" != $gn_$ && tansName."translated name" != $gn_$
{
// get the translation back
//update work/bot_name.txt
wkTranslationName = ucstring("$")+gnName+"$";
botNames[rowIdx][0] = wkBotName;
botNames[rowIdx][1] = wkTranslationName;
botNames[rowIdx][2] = wkSheetName;
//update translated/bot_name.txt
fctsName = transName; //transName
fctsTitleId = gnName;
ucstring trNewUcHash;
uint64 hash = CI18N::makeHash(botNames[rowIdx][0] + botNames[rowIdx][1] +botNames[rowIdx][2]);
CI18N::hashToUCString(hash, trNewUcHash);
transBotNames[transRowIdx][0] = ucstring("_") + trNewUcHash;
}
}
else //There is no translation yet
{
fctsName = wkTranslationName;
wkTranslationName = ucstring("$")+gnName+"$";
botNames[rowIdx][0] = wkBotName;
botNames[rowIdx][1] = wkTranslationName;
botNames[rowIdx][2] = wkSheetName;
fctsTitleId = gnName;
}
}
}
// look for a corresponding entry
uint gnNameRow;
if (!fcts.findRow(fctsIdCol, fctsTitleId, gnNameRow))
{
// not found, add it
gnNameRow = fcts.size();
fcts.resize(fcts.size()+1);
fcts.setData(gnNameRow, ucstring("title_id"), fctsTitleId);
fcts.setData(gnNameRow, ucstring("name"), fctsName);
nbAddGenericName++;
}
else //Update
{
}
}
}
// 3 - functions
{
nlinfo(" Functions...");
set<string>::iterator first(Functions.begin()), last(Functions.end());
for (; first != last; ++first)
{
string fctName = *first;
// look for a corresponding entry
uint functionRow;
if (!fcts.findRow(fctsIdCol, fctName, functionRow))
{
// not found, add it
functionRow = fcts.size();
fcts.resize(fcts.size()+1);
fcts.setData(functionRow, ucstring("title_id"), fctName);
fcts.setData(functionRow, ucstring("name"), *first);
nbAddFunction++;
}
}
}
// display summary
nlinfo("Adding %u new simple name", nbAddSimpleName);
nlinfo("Adding %u new generic name", nbAddGenericName);
nlinfo("Adding %u new function name", nbAddFunction);
// saving the modified files
ucstring s = prepareExcelSheet(botNames);
CI18N::writeTextFile(workBotNamesFile.asString(), s, false);
s = prepareExcelSheet(transBotNames);
CI18N::writeTextFile(transBotNamesFile.asString(), s, false);
s = prepareExcelSheet(fcts);
CI18N::writeTextFile(workTitleFile.asString(), s, false);
return 0;
}