// Ryzom - MMORPG Framework // Copyright (C) 2010 Winch Gate Property Limited // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as // published by the Free Software Foundation, either version 3 of the // License, or (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . //----------------------------------------------------------------------------- // includes //----------------------------------------------------------------------------- #include #include #include "game_share/bnp_patch.h" #include "nel/misc/path.h" #include "nel/misc/file.h" #include "nel/misc/command.h" #include "nel/misc/sstring.h" #include "game_share/singleton_registry.h" #include "seven_zip.h" using namespace std; using namespace NLMISC; #define PERSISTENT_TOKEN_FAMILY RyzomTokenFamily //----------------------------------------------------------------------------- // Handy utility functions //----------------------------------------------------------------------------- void normalisePackageDescriptionFileName(std::string& fileName) { if (fileName.empty()) fileName="package_description"; if (NLMISC::CFile::getExtension(fileName).empty() && fileName[fileName.size()-1]!='.') fileName+=".xml"; } void GeneratePatch(const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName) { std::string cmd="xdelta delta"; cmd+=" "+srcFileName+" "+destFileName+" "+patchFileName; nlinfo("executing system command: %s",cmd.c_str()); #ifdef NL_OS_WINDOWS _spawnlp(_P_WAIT, "xdelta.exe","xdelta.exe","delta",srcFileName.c_str(),destFileName.c_str(),patchFileName.c_str(),NULL); #else // NL_OS_WINDOWS // xdelta-1.x behaves like "diff" and returns 0 for identical files, 1 for different files, 2 for errors sint error = system (cmd.c_str()); if (error == 2) nlwarning("'%s' failed with error code %d", cmd.c_str(), error); #endif // NL_OS_WINDOWS } void ApplyPatch(const std::string& srcFileName,const std::string& destFileName,const std::string& patchFileName=std::string()) { std::string cmd="xdelta patch"; cmd+=" "+patchFileName+" "+srcFileName+" "+destFileName; nlinfo("executing system command: %s",cmd.c_str()); #ifdef NL_OS_WINDOWS _spawnlp(_P_WAIT, "xdelta.exe","xdelta.exe","patch",patchFileName.c_str(),srcFileName.c_str(),destFileName.c_str(),NULL); #else // NL_OS_WINDOWS // xdelta-1.x behaves like "diff" and returns 0 for identical files, 1 for different files, 2 for errors sint error = system (cmd.c_str()); if (error == 2) nlwarning("'%s' failed with error code %d", cmd.c_str(), error); #endif // NL_OS_WINDOWS } void GenerateLZMA(const std::string &sourceFile, const std::string &outputFile) { { nlinfo("Compressing %s to %s using LZMA...", sourceFile.c_str(), outputFile.c_str()); } if (!packLZMA(sourceFile, outputFile)) { nlwarning("LZMA compress failed"); } } //----------------------------------------------------------------------------- // class CPackageDescription //----------------------------------------------------------------------------- class CPackageDescription: public IVersionNumberGenerator { private: DECLARE_PERSISTENCE_METHODS public: CPackageDescription(); void clear(); void setup(const std::string& packageName); void storeToPdr(CPersistentDataRecord& pdr) const; void readIndex(CBNPFileSet& packageIndex) const; void writeIndex(const CBNPFileSet& packageIndex) const; void getCategories(CPersistentDataRecord &pdr) const; void updateIndexFileList(CBNPFileSet& packageIndex) const; void generateClientIndex(CProductDescriptionForClient& theClientPackage,const CBNPFileSet& packageIndex) const; void addVersion(CBNPFileSet& packageIndex); void generatePatches(CBNPFileSet& packageIndex) const; void createDirectories() const; void buildDefaultFileList(); void updatePatchSizes(CBNPFileSet& packageIndex) const; // specialisation of IVersionNumberGenerator void grabVersionNumber(); uint32 getPackageVersionNumber(); private: CBNPCategorySet _Categories; std::string _IndexFileName; std::string _ClientIndexFileName; std::string _RootDirectory; std::string _PatchDirectory; std::string _BnpDirectory; std::string _RefDirectory; std::string _NextVersionFile; uint32 _NextVersionNumber; bool _VersionNumberReserved; }; //----------------------------------------------------------------------------- // methods CPackageDescription //----------------------------------------------------------------------------- CPackageDescription::CPackageDescription() { clear(); } void CPackageDescription::clear() { _NextVersionNumber= std::numeric_limits::max(); _VersionNumberReserved = false; _Categories.clear(); _IndexFileName.clear(); _ClientIndexFileName.clear(); _PatchDirectory.clear(); _BnpDirectory.clear(); _RefDirectory.clear(); } void CPackageDescription::setup(const std::string& packageName) { nlinfo("Reading package description: %s ...",packageName.c_str()); // clear out old contents before reading from input file clear(); // read new contents from input file static CPersistentDataRecord pdr; pdr.clear(); pdr.readFromTxtFile(packageName); apply(pdr); // root directory if (_RootDirectory.empty()) _RootDirectory = NLMISC::CFile::getPath(packageName); _RootDirectory = NLMISC::CPath::standardizePath(_RootDirectory, true); // patch directory if (_PatchDirectory.empty()) _PatchDirectory = _RootDirectory + "patch"; _PatchDirectory = NLMISC::CPath::standardizePath(_PatchDirectory, true); // BNP directory if (_BnpDirectory.empty()) _BnpDirectory = _RootDirectory+"bnp"; _BnpDirectory = NLMISC::CPath::standardizePath(_BnpDirectory, true); // ref directory if (_RefDirectory.empty()) _RefDirectory = _RootDirectory+"ref"; _RefDirectory = NLMISC::CPath::standardizePath(_RefDirectory, true); // client index file if (_ClientIndexFileName.empty()) _ClientIndexFileName = NLMISC::CFile::getFilenameWithoutExtension(packageName)+".idx"; // index file if (_IndexFileName.empty()) _IndexFileName = NLMISC::CFile::getFilenameWithoutExtension(_ClientIndexFileName)+".hist"; } void CPackageDescription::storeToPdr(CPersistentDataRecord& pdr) const { pdr.clear(); store(pdr); } void CPackageDescription::readIndex(CBNPFileSet& packageIndex) const { std::string indexPath = _RootDirectory + _IndexFileName; nlinfo("Reading history file: %s ...", indexPath.c_str()); // clear out old contents before reading from input file packageIndex.clear(); // read new contents from input file if (NLMISC::CFile::fileExists(indexPath)) { static CPersistentDataRecord pdr; pdr.clear(); pdr.readFromTxtFile(indexPath); packageIndex.apply(pdr); } } void CPackageDescription::writeIndex(const CBNPFileSet& packageIndex) const { std::string indexPath = _RootDirectory + _IndexFileName; nlinfo("Writing history file: %s ...", indexPath.c_str()); // write contents to output file static CPersistentDataRecordRyzomStore pdr; pdr.clear(); packageIndex.store(pdr); pdr.writeToTxtFile(indexPath); } void CPackageDescription::getCategories(CPersistentDataRecord &pdr) const { pdr.clear(); _Categories.store(pdr); } void CPackageDescription::updateIndexFileList(CBNPFileSet& packageIndex) const { nlinfo("Updating file list from package categories (%d files) ...",_Categories.fileCount()); for (uint32 i=_Categories.fileCount();i--;) { const std::string& fileName= _Categories.getFile(i); packageIndex.addFile(fileName,_Categories.isFileIncremental(fileName)); // if the file is flagged as non-incremental then we need to add its refference file too if (!_Categories.isFileIncremental(fileName)) { std::string refName= NLMISC::CFile::getFilenameWithoutExtension(_Categories.getFile(i))+"_.ref"; packageIndex.addFile(refName); // if the ref file doesn't exist then create it by copying the original if (NLMISC::CFile::fileExists(_BnpDirectory+fileName) && !NLMISC::CFile::fileExists(_BnpDirectory+refName)) { NLMISC::CFile::copyFile(_BnpDirectory+refName,_BnpDirectory+fileName); nlassert(NLMISC::CFile::getFileSize(_BnpDirectory+refName)== NLMISC::CFile::getFileSize(_BnpDirectory+fileName)); } } } } void CPackageDescription::generateClientIndex(CProductDescriptionForClient& theClientPackage, const CBNPFileSet& packageIndex) const { std::string patchNumber = toString("%05u", packageIndex.getVersionNumber()); std::string patchDirectory = _PatchDirectory + patchNumber; std::string patchFile = patchDirectory + "/" + _ClientIndexFileName; nlinfo("Generating client index: %s...", patchFile.c_str()); // make sure the version sub directory exist CFile::createDirectory(patchDirectory); // clear out the client package before we start theClientPackage.clear(); // copy the categories using a pdr record static CPersistentDataRecordRyzomStore pdr; pdr.clear(); _Categories.store(pdr); theClientPackage.setCategories(pdr); // copy the files using a pdr record pdr.clear(); packageIndex.store(pdr); theClientPackage.setFiles(pdr); // create the output file pdr.clear(); theClientPackage.store(pdr); std::string newName = patchDirectory + "/" + NLMISC::CFile::getFilenameWithoutExtension(_ClientIndexFileName) + "_" + patchNumber; pdr.writeToBinFile(newName + ".idx"); pdr.writeToTxtFile(newName + "_debug.xml"); } void CPackageDescription::addVersion(CBNPFileSet& packageIndex) { // calculate the last version number in the index file // nlinfo("Calculating package version number..."); // uint32 versionNumber= packageIndex.getVersionNumber(); // nlinfo("Last version number = %d",versionNumber); // uint32 newVersionNumber= packageIndex.addVersion(_BnpDirectory,versionNumber+1); // nlinfo("New version number = %d",newVersionNumber); // setup the default next version number by scanning the package index for the highest existing version number nlinfo("Calculating package version number..."); _NextVersionNumber= packageIndex.getVersionNumber(); nlinfo("Last version number = %u",_NextVersionNumber); ++_NextVersionNumber; // have the package index check its file list to see if a new version is required uint32 newVersionNumber= packageIndex.addVersion(_BnpDirectory,_RefDirectory,*this); nlinfo("Added files for version: %u",newVersionNumber); } void CPackageDescription::generatePatches(CBNPFileSet& packageIndex) const { nlinfo("Generating patches ..."); for (uint32 i = packageIndex.fileCount(); i--;) { bool deleteRefAfterDelta = true; bool usingTemporaryFile = false; // generate file name root std::string bnpFileName = _BnpDirectory + packageIndex.getFile(i).getFileName(); std::string refNameRoot = _RefDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName); std::string patchNameRoot = _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(bnpFileName); // if the file has no versions then skip on to the next file if (packageIndex.getFile(i).versionCount()==0) continue; // get the last version number and the related file name const CBNPFileVersion& curVersion= packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1); std::string curVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",curVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str()); // std::string patchFileName= patchNameRoot+NLMISC::toString("_%05d.patch",curVersion.getVersionNumber()); std::string patchFileName= _PatchDirectory + toString("%05u/",curVersion.getVersionNumber())+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+toString("_%05u",curVersion.getVersionNumber())+".patch"; // get the second last version number and the related file name std::string prevVersionFileName; if (packageIndex.getFile(i).versionCount()==1) { prevVersionFileName= _RootDirectory + "empty"; CFile::createEmptyFile(prevVersionFileName); usingTemporaryFile = true; deleteRefAfterDelta= false; } else { const CBNPFileVersion& prevVersion= packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-2); prevVersionFileName= refNameRoot+NLMISC::toString("_%05u.%s",prevVersion.getVersionNumber(),NLMISC::CFile::getExtension(bnpFileName).c_str()); } std::string refVersionFileName= prevVersionFileName; // create the subdirectory for this patch number string versionSubDir = _PatchDirectory + toString("%05u/", curVersion.getVersionNumber()); CFile::createDirectory(versionSubDir); // generate the lzma packed version of the bnp if needed (lzma file are slow to generate) string lzmaFile = versionSubDir+CFile::getFilename(bnpFileName)+".lzma"; if (!CFile::fileExists(lzmaFile)) { // build the lzma compression in a temp file (avoid leaving dirty file if the // process cannot terminate) GenerateLZMA(bnpFileName, lzmaFile+".tmp"); // rename the tmp file CFile::moveFile(lzmaFile, lzmaFile+".tmp"); } // store the lzma file size in the descriptor packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1).set7ZipFileSize(CFile::getFileSize(lzmaFile)); // if we need to generate a new patch then do it and create the new ref file if (!NLMISC::CFile::fileExists(curVersionFileName)) { nlinfo("- Creating patch: %s",patchFileName.c_str()); // in the case where we compress against a ref file... if (!_Categories.isFileIncremental(NLMISC::CFile::getFilename(bnpFileName))) { // setup the name of the reference file to patch against refVersionFileName= _BnpDirectory+NLMISC::CFile::getFilenameWithoutExtension(bnpFileName)+"_.ref"; // delete the previous patch - because we only need the latest patch for non-incremental files std::string lastPatch= _PatchDirectory + NLMISC::CFile::getFilenameWithoutExtension(prevVersionFileName)+".patch"; if (NLMISC::CFile::fileExists(lastPatch.c_str())) NLMISC::CFile::deleteFile(lastPatch.c_str()); } // call xdelta to generate the patch GeneratePatch(refVersionFileName, bnpFileName, patchFileName); nlassert(NLMISC::CFile::fileExists(patchFileName)); uint32 nPatchSize = NLMISC::CFile::getFileSize(patchFileName); packageIndex.getFile(i).getVersion(packageIndex.getFile(i).versionCount()-1).setPatchSize(nPatchSize); // apply the incremental patch to the old ref file to create the new ref file // and ensure that the new ref file matches the BNP ApplyPatch(refVersionFileName, curVersionFileName, patchFileName); nlassert(NLMISC::CFile::fileExists(curVersionFileName)); nlassert(NLMISC::CFile::thoroughFileCompare(bnpFileName, curVersionFileName)); } // if we have a ref file still hanging about from the previous patch then delete it if (NLMISC::CFile::fileExists(prevVersionFileName)) { NLMISC::CFile::deleteFile(prevVersionFileName); } } } void CPackageDescription::createDirectories() const { NLMISC::CFile::createDirectoryTree(_RootDirectory); NLMISC::CFile::createDirectoryTree(_PatchDirectory); NLMISC::CFile::createDirectoryTree(_BnpDirectory); NLMISC::CFile::createDirectoryTree(_RefDirectory); } void CPackageDescription::buildDefaultFileList() { // make sure the default categories exist CBNPCategory* packedCategory= _Categories.getCategory("main", true); packedCategory->setOptional(false); packedCategory->setIncremental(true); CBNPCategory* unpackedCategory= _Categories.getCategory("unpacked", true); unpackedCategory->setUnpackTo("./"); unpackedCategory->setOptional(false); unpackedCategory->setIncremental(false); CBNPCategory* optionCategory= _Categories.getCategory("optional", true); optionCategory->setOptional(true); optionCategory->setIncremental(true); // look for BNP files in the BNP directry and add them to the main category std::vector fileList; NLMISC::CPath::getPathContent(_BnpDirectory,false,false,true,fileList); for (uint32 i=0;i") { if (args.size()!=1) return false; createNewProduct(args[0]); return true; } NLMISC_COMMAND(updateProduct,"process a package","") { if (args.size()!=1) return false; updateProduct(args[0]); return true; } NLMISC_COMMAND(go,"perform a 'createNewProduct' if required and 'updateProduct' on patch_test/test0/test_package.xml","") { if (args.size()!=0) return false; if (!NLMISC::CFile::fileExists("patch_test/test0/test_package.xml")) createNewProduct("patch_test/test0/test_package.xml"); updateProduct("patch_test/test0/test_package.xml"); return true; }