Khaganat-NeL/khanat-opennel-code
2
0
Fork 0
Code Issues 21 Pull requests Projects Releases 1 Packages Wiki Activity

feature #36 patch_gen multi-thread

Browse source
This commit is contained in:
AleaJactaEst 2018-11-29 22:18:09 +01:00
parent cfb5b71728
commit 8091bc8fb1
4 changed files with 252 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
code/ryzom/tools/patch_gen/CMakeLists.txt
View file

@ -1,7 +1,7 @@
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/ryzom/client/src/seven_zip) INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/ryzom/client/src/seven_zip)
SET(MAIN_SRC patch_gen_common.cpp patch_gen_main.cpp patch_gen_main.h) SET(MAIN_SRC patch_gen_common.cpp patch_gen_main.cpp patch_gen_main.h patch_gen_common.h)
SET(SERVICE_SRC patch_gen_common.cpp patch_gen_service.cpp patch_gen_service.h) SET(SERVICE_SRC patch_gen_common.cpp patch_gen_service.cpp patch_gen_service.h patch_gen_common.h)
ADD_EXECUTABLE(patch_gen ${MAIN_SRC}) ADD_EXECUTABLE(patch_gen ${MAIN_SRC})
TARGET_LINK_LIBRARIES(patch_gen ryzom_sevenzip ryzom_gameshare nelmisc nelnet nelligo nelgeorges) TARGET_LINK_LIBRARIES(patch_gen ryzom_sevenzip ryzom_gameshare nelmisc nelnet nelligo nelgeorges)

302
code/ryzom/tools/patch_gen/patch_gen_common.cpp
View file

@ -20,6 +20,11 @@
#include <cstdio> #include <cstdio>
#include <limits> #include <limits>
#include <string>
#include <vector>
#include <queue>
#include <mutex>
#include <thread>
#include "game_share/bnp_patch.h" #include "game_share/bnp_patch.h"
#include "nel/misc/path.h" #include "nel/misc/path.h"
@ -28,12 +33,14 @@
#include "nel/misc/sstring.h" #include "nel/misc/sstring.h"
#include "game_share/singleton_registry.h" #include "game_share/singleton_registry.h"
#include "seven_zip.h" #include "seven_zip.h"
#include "patch_gen_main.h"
using namespace std; using namespace std;
using namespace NLMISC; using namespace NLMISC;
#define PERSISTENT_TOKEN_FAMILY RyzomTokenFamily #define PERSISTENT_TOKEN_FAMILY RyzomTokenFamily
unsigned int jobs_simultaneously = 1;
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Handy utility functions // Handy utility functions
@ -47,11 +54,11 @@ void normalisePackageDescriptionFileName(std::string& fileName)
fileName+=".xml"; fileName+=".xml";
} }
void GeneratePatch(const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName) void GeneratePatch(const uint32 id, const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName)
{ {
std::string cmd = toString("xdelta delta %s %s %s", srcFileName.c_str(), destFileName.c_str(), patchFileName.c_str()); std::string cmd = toString("xdelta delta %s %s %s", srcFileName.c_str(), destFileName.c_str(), patchFileName.c_str());
nlinfo("Executing system command: %s", cmd.c_str()); nlinfo("[id:%d] Executing system command: %s", id, cmd.c_str());
#ifdef NL_OS_WINDOWS #ifdef NL_OS_WINDOWS
_spawnlp(_P_WAIT, "xdelta.exe", "xdelta.exe", "delta", srcFileName.c_str(), destFileName.c_str(), patchFileName.c_str(), NULL); _spawnlp(_P_WAIT, "xdelta.exe", "xdelta.exe", "delta", srcFileName.c_str(), destFileName.c_str(), patchFileName.c_str(), NULL);
@ -60,15 +67,15 @@ void GeneratePatch(const std::string& srcFileName,const std::string& destFileNam
sint error = system (cmd.c_str()); sint error = system (cmd.c_str());
if (error == 2) if (error == 2)
nlwarning("'%s' failed with error code %d", cmd.c_str(), error); nlwarning("[id:%d] '%s' failed with error code %d", id, cmd.c_str(), error);
#endif // NL_OS_WINDOWS #endif // NL_OS_WINDOWS
} }
void ApplyPatch(const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName=std::string()) void ApplyPatch(const uint32 id, const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName=std::string())
{ {
std::string cmd = toString("xdelta patch %s %s %s", patchFileName.c_str(), srcFileName.c_str(), destFileName.c_str()); std::string cmd = toString("xdelta patch %s %s %s", patchFileName.c_str(), srcFileName.c_str(), destFileName.c_str());
nlinfo("Executing system command: %s", cmd.c_str()); nlinfo("[id:%d] Executing system command: %s", id, cmd.c_str());
#ifdef NL_OS_WINDOWS #ifdef NL_OS_WINDOWS
_spawnlp(_P_WAIT, "xdelta.exe", "xdelta.exe", "patch",patchFileName.c_str(), srcFileName.c_str(), destFileName.c_str(), NULL); _spawnlp(_P_WAIT, "xdelta.exe", "xdelta.exe", "patch",patchFileName.c_str(), srcFileName.c_str(), destFileName.c_str(), NULL);
@ -77,22 +84,157 @@ void ApplyPatch(const std::string& srcFileName,const std::string& destFileName,c
sint error = system (cmd.c_str()); sint error = system (cmd.c_str());
if (error == 2) if (error == 2)
nlwarning("'%s' failed with error code %d", cmd.c_str(), error); nlwarning("[id:%d] '%s' failed with error code %d", id, cmd.c_str(), error);
#endif // NL_OS_WINDOWS #endif // NL_OS_WINDOWS
} }
void GenerateLZMA(const std::string &sourceFile, const std::string &outputFile) void GenerateLZMA(const uint32 id, const std::string &sourceFile, const std::string &outputFile)
{ {
{ {
nlinfo("Compressing %s to %s using LZMA...", sourceFile.c_str(), outputFile.c_str()); nlinfo("[id:%d] Compressing %s to %s using LZMA...", id, sourceFile.c_str(), outputFile.c_str());
} }
if (!packLZMA(sourceFile, outputFile)) if (!packLZMA(sourceFile, outputFile))
{ {
nlwarning("LZMA compress failed"); nlwarning("[id:%d, source:%s] LZMA compress failed", id, sourceFile.c_str());
} }
} }
//-----------------------------------------------------------------------------
// class CTaskPackageDescription
//-----------------------------------------------------------------------------
class CTaskPackageDescription
{
bool deleteRefAfterDelta;
bool usingTemporaryFile;
uint32 id;
CBNPFile * _pbnpPtr;
std::string _BnpDirectory;
std::string _RefDirectory;
std::string _PatchDirectory;
std::string _RootDirectory;
const CBNPCategorySet * _pCategories;
public:
CTaskPackageDescription(uint32 id, bool deleteRefAfterDelta, bool usingTemporaryFile, CBNPFile * _pbnpPtr,
std::string _BnpDirectory, std::string _RefDirectory, std::string _PatchDirectory,
std::string _RootDirectory, const CBNPCategorySet * _pCategories)
{
this->id = id;
this->deleteRefAfterDelta = deleteRefAfterDelta;
this->usingTemporaryFile = usingTemporaryFile;
this->_pbnpPtr = _pbnpPtr;
this->_BnpDirectory = _BnpDirectory;
this->_RefDirectory = _RefDirectory;
this->_PatchDirectory = _PatchDirectory;
this->_RootDirectory = _RootDirectory;
this->_pCategories = _pCategories;
}
void action()
{
nldebug("[id:%d] thread for %s : start", id, this->_pbnpPtr->getFileName().c_str());
std::string bnpFileName = _BnpDirectory + _pbnpPtr->getFileName();
std::string refNameRoot = _RefDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName);
std::string patchNameRoot = _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName);
// get the last version number and the related file name
const CBNPFileVersion& curVersion= _pbnpPtr->getVersion(_pbnpPtr->versionCount()-1);
std::string curVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",curVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str());
std::string patchFileName= _PatchDirectory + toString("%05u/",curVersion.getVersionNumber())+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+toString("_%05u",curVersion.getVersionNumber())+".patch";
// get the second last version number and the related file name
std::string prevVersionFileName;
if (_pbnpPtr->versionCount()==1)
{
prevVersionFileName= _RootDirectory + "empty_" + std::to_string(this->id);
CFile::createEmptyFile(prevVersionFileName);
usingTemporaryFile = true;
deleteRefAfterDelta= false;
}
else
{
const CBNPFileVersion& prevVersion= _pbnpPtr->getVersion(_pbnpPtr->versionCount()-2);
prevVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",prevVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str());
}
std::string refVersionFileName= prevVersionFileName;
// create the subdirectory for this patch number
string versionSubDir = _PatchDirectory + toString("%05u/", curVersion.getVersionNumber());
CFile::createDirectory(versionSubDir);
// generate the lzma packed version of the bnp if needed (lzma file are slow to generate)
string lzmaFile = versionSubDir+CFile::getFilename(bnpFileName)+".lzma";
if (!CFile::fileExists(lzmaFile))
{
// build the lzma compression in a temp file (avoid leaving dirty file if the
// process cannot terminate)
GenerateLZMA(id, bnpFileName, lzmaFile+".tmp");
// rename the tmp file
CFile::moveFile(lzmaFile, lzmaFile+".tmp");
}
// store the lzma file size in the descriptor
_pbnpPtr->getVersion(_pbnpPtr->versionCount()-1).set7ZipFileSize(CFile::getFileSize(lzmaFile));
// if we need to generate a new patch then do it and create the new ref file
if (!NLMISC::CFile::fileExists(curVersionFileName))
{
nlinfo("- Creating patch: %s",patchFileName.c_str());
// in the case where we compress against a ref file...
if (!_pCategories->isFileIncremental(NLMISC::CFile::getFilename(bnpFileName)))
{
// setup the name of the reference file to patch against
refVersionFileName= _BnpDirectory+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+"_.ref";
// delete the previous patch - because we only need the latest patch for non-incremental files
std::string lastPatch= _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(prevVersionFileName)+".patch";
if (NLMISC::CFile::fileExists(lastPatch.c_str()))
NLMISC::CFile::deleteFile(lastPatch.c_str());
}
// call xdelta to generate the patch
GeneratePatch(id, refVersionFileName, bnpFileName, patchFileName);
nlassert(NLMISC::CFile::fileExists(patchFileName));
uint32 nPatchSize = NLMISC::CFile::getFileSize(patchFileName);
_pbnpPtr->getVersion(_pbnpPtr->versionCount()-1).setPatchSize(nPatchSize);
// apply the incremental patch to the old ref file to create the new ref file
// and ensure that the new ref file matches the BNP
ApplyPatch(id, refVersionFileName, curVersionFileName, patchFileName);
nlassert(NLMISC::CFile::fileExists(curVersionFileName));
nlassert(NLMISC::CFile::thoroughFileCompare(bnpFileName, curVersionFileName));
}
// if we have a ref file still hanging about from the previous patch then delete it
if (NLMISC::CFile::fileExists(prevVersionFileName))
{
NLMISC::CFile::deleteFile(prevVersionFileName);
}
nldebug("[id:%d] thread for %s : end", id, this->_pbnpPtr->getFileName().c_str());
}
};
void threadWorker(std::mutex & mtx, std::queue<CTaskPackageDescription> & queueTask)
{
while(true)
{
mtx.lock();
if ( queueTask.empty())
{
mtx.unlock();
break;
}
CTaskPackageDescription task = queueTask.front();
queueTask.pop();
mtx.unlock();
task.action();
}
};
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// class CPackageDescription // class CPackageDescription
@ -125,6 +267,8 @@ public:
void updatePatchSizes(CBNPFileSet& packageIndex) const; void updatePatchSizes(CBNPFileSet& packageIndex) const;
//void threadWorker(mutex & mtx, queue<CTaskPackageDescription> & queueTask) const;
// specialisation of IVersionNumberGenerator // specialisation of IVersionNumberGenerator
void grabVersionNumber(); void grabVersionNumber();
uint32 getPackageVersionNumber(); uint32 getPackageVersionNumber();
@ -334,96 +478,76 @@ void CPackageDescription::generatePatches(CBNPFileSet& packageIndex) const
{ {
nlinfo("Generating patches ..."); nlinfo("Generating patches ...");
for (uint32 i = packageIndex.fileCount(); i--;) // Generate patch with Leader/Follower pattern
uint32 maxThread=jobs_simultaneously;
uint32 i;
std::mutex mtx;
struct SPrepareTask
{
uint32 fileSize;
uint32 id;
};
struct SComparePrepareTask
{
bool operator()(const SPrepareTask& left, const SPrepareTask& right) const
{
return left.fileSize < right.fileSize;
}
};
std::priority_queue<struct SPrepareTask, std::vector<struct SPrepareTask>, SComparePrepareTask > queuePrepareTask;
std::queue<CTaskPackageDescription> queueTask;
std::vector<thread> workers;
// Initialize queue task
if ( packageIndex.fileCount() < maxThread )
maxThread = packageIndex.fileCount();
for (i = packageIndex.fileCount(); i--;)
{
struct SPrepareTask stask;
stask.id = i;
stask.fileSize = packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1).getFileSize();
queuePrepareTask.push(stask);
}
// Reorder task Big file to small file
//getFileSize
std::priority_queue<struct SPrepareTask, std::vector<struct SPrepareTask>, SComparePrepareTask > queueSortedPrepareTask;
while ( ! queuePrepareTask.empty() )
{ {
bool deleteRefAfterDelta = true; bool deleteRefAfterDelta = true;
bool usingTemporaryFile = false; bool usingTemporaryFile = false;
// generate file name root
std::string bnpFileName = _BnpDirectory + packageIndex.getFile(i).getFileName(); struct SPrepareTask task = queuePrepareTask.top();
std::string refNameRoot = _RefDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName); queuePrepareTask.pop();
std::string patchNameRoot = _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName);
// if the file has no versions then skip on to the next file // if the file has no versions then skip on to the next file
if (packageIndex.getFile(i).versionCount()==0) if (packageIndex.getFile(task.id).versionCount()==0)
continue; continue;
// get the last version number and the related file name CTaskPackageDescription data(task.id,
const CBNPFileVersion& curVersion= packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1); deleteRefAfterDelta,
std::string curVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",curVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str()); usingTemporaryFile,
// std::string patchFileName= patchNameRoot+NLMISC::toString("_%05d.patch",curVersion.getVersionNumber()); & packageIndex.getFile(task.id),
std::string patchFileName= _PatchDirectory + toString("%05u/",curVersion.getVersionNumber())+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+toString("_%05u",curVersion.getVersionNumber())+".patch"; std::ref(_BnpDirectory),
std::ref(_RefDirectory),
// get the second last version number and the related file name std::ref(_PatchDirectory),
std::string prevVersionFileName; std::ref(_RootDirectory),
if (packageIndex.getFile(i).versionCount()==1) & _Categories);
{ queueTask.push(data);
prevVersionFileName= _RootDirectory + "empty";
CFile::createEmptyFile(prevVersionFileName);
usingTemporaryFile = true;
deleteRefAfterDelta= false;
}
else
{
const CBNPFileVersion& prevVersion= packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-2);
prevVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",prevVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str());
}
std::string refVersionFileName= prevVersionFileName;
// create the subdirectory for this patch number
string versionSubDir = _PatchDirectory + toString("%05u/", curVersion.getVersionNumber());
CFile::createDirectory(versionSubDir);
// generate the lzma packed version of the bnp if needed (lzma file are slow to generate)
string lzmaFile = versionSubDir+CFile::getFilename(bnpFileName)+".lzma";
if (!CFile::fileExists(lzmaFile))
{
// build the lzma compression in a temp file (avoid leaving dirty file if the
// process cannot terminate)
GenerateLZMA(bnpFileName, lzmaFile+".tmp");
// rename the tmp file
CFile::moveFile(lzmaFile, lzmaFile+".tmp");
} }
// store the lzma file size in the descriptor // launch thread workers
packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1).set7ZipFileSize(CFile::getFileSize(lzmaFile)); for(i=0;i<maxThread;++i)
workers.push_back(std::thread(threadWorker, std::ref(mtx), std::ref(queueTask)));
// if we need to generate a new patch then do it and create the new ref file // wait all workers
if (!NLMISC::CFile::fileExists(curVersionFileName)) for (std::vector<std::thread>::iterator it = workers.begin(); it != workers.end(); ++it)
{ it->join();
nlinfo("- Creating patch: %s",patchFileName.c_str()); NLMISC::DebugLog->displayNL("End thread");
// in the case where we compress against a ref file...
if (!_Categories.isFileIncremental(NLMISC::CFile::getFilename(bnpFileName)))
{
// setup the name of the reference file to patch against
refVersionFileName= _BnpDirectory+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+"_.ref";
// delete the previous patch - because we only need the latest patch for non-incremental files
std::string lastPatch= _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(prevVersionFileName)+".patch";
if (NLMISC::CFile::fileExists(lastPatch.c_str()))
NLMISC::CFile::deleteFile(lastPatch.c_str());
}
// call xdelta to generate the patch
GeneratePatch(refVersionFileName, bnpFileName, patchFileName);
nlassert(NLMISC::CFile::fileExists(patchFileName));
uint32 nPatchSize = NLMISC::CFile::getFileSize(patchFileName);
packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1).setPatchSize(nPatchSize);
// apply the incremental patch to the old ref file to create the new ref file
// and ensure that the new ref file matches the BNP
ApplyPatch(refVersionFileName, curVersionFileName, patchFileName);
nlassert(NLMISC::CFile::fileExists(curVersionFileName));
nlassert(NLMISC::CFile::thoroughFileCompare(bnpFileName, curVersionFileName));
}
// if we have a ref file still hanging about from the previous patch then delete it
if (NLMISC::CFile::fileExists(prevVersionFileName))
{
NLMISC::CFile::deleteFile(prevVersionFileName);
}
}
} }
void CPackageDescription::createDirectories() const void CPackageDescription::createDirectories() const

7
code/ryzom/tools/patch_gen/patch_gen_common.h Normal file
View file

@ -0,0 +1,7 @@
#ifndef PATCH_GEN_COMMON_H
#define PATCH_GEN_COMMON_H
// We need to find other method to send global parameter/argument
extern unsigned int jobs_simultaneously;
#endif // PATCH_GEN_COMMON_H

38
code/ryzom/tools/patch_gen/patch_gen_main.cpp
View file

@ -17,6 +17,8 @@
#include "nel/misc/debug.h" #include "nel/misc/debug.h"
#include "nel/misc/command.h" #include "nel/misc/command.h"
#include "nel/misc/sstring.h" #include "nel/misc/sstring.h"
#include "nel/misc/cmd_args.h"
#include "patch_gen_common.h"
//#include "game_share/handy_commands.h" //#include "game_share/handy_commands.h"
//----------------------------------------------- //-----------------------------------------------
@ -26,27 +28,43 @@ int main(int argc,char** argv)
{ {
NLMISC::createDebug(); NLMISC::createDebug();
// Parse Command Line.
NLMISC::CCmdArgs args;
args.setDescription("Generate patch");
args.addArg("j", "jobs", "jobs", "Specifies the number of jobs (commands) to run simultaneously");
args.addAdditionalArg("CommandLine","command line (For a list of valid commands and their paramaters try help)",false,true);
// if there are no command line args then display a friendly message and exit // if there are no command line args then display a friendly message and exit
if (argc==1) if (!args.parse(argc, argv))
{ {
NLMISC::InfoLog->displayNL("SYNTAX: %s <command line>",argv[0]); args.displayHelp();
NLMISC::InfoLog->displayNL(""); return 1;
NLMISC::InfoLog->displayNL("For a list of valid commands and their paramaters try: %s help",argv[0]); }
NLMISC::InfoLog->displayNL("");
return 0; // Analyze option
try
{
std::string cjobs = args.haveArg("j") ? args.getArg("j").front() : "1";
jobs_simultaneously = (uint32) std::stoul(cjobs);
} catch (const std::invalid_argument) {
args.displayHelp();
NLMISC::ErrorLog->displayNL("Bad value for parameter jobs");
return 2;
} }
// build the command line by concatnating input arguments (separating with spaces) // build the command line by concatnating input arguments (separating with spaces)
NLMISC::DebugLog->displayNL("param jobs:%u", jobs_simultaneously);
std::vector<std::string> listargs = args.getAdditionalArg("CommandLine");
NLMISC::CSString commandline; NLMISC::CSString commandline;
for (int i=1;i<argc;++i) for (std::vector<std::string>::iterator it = listargs.begin() ; it != listargs.end(); ++it)
{ {
NLMISC::CSString s= argv[i]; NLMISC::CSString s = *it;
// check whether the argument needs to be quote encapsulated // check whether the argument needs to be quote encapsulated
if (s.contains(' ') && s[0]!='\"') if (s.contains(' ') && s[0]!='\"')
s= "\""+ s+ "\""; s= "\""+ s+ "\"";
if (i>1) if ( it != listargs.begin() )
commandline+=' '; commandline+=' ';
commandline+= s; commandline+= s;
} }