Changed: Centralize BNP unpacking and packing methods, to be reused in other tools/programs

This commit is contained in:
kervala 2016-01-23 10:18:09 +01:00
parent 94b033caf9
commit 9d319a5688
2 changed files with 445 additions and 162 deletions

View file

@ -20,7 +20,7 @@
#include "types_nl.h" #include "types_nl.h"
#include "tds.h" #include "tds.h"
#include "singleton.h" #include "singleton.h"
#include "callback.h"
namespace NLMISC { namespace NLMISC {
@ -86,6 +86,66 @@ public:
// Used for CPath only for the moment ! // Used for CPath only for the moment !
char *getFileNamePtr(const std::string &sFileName, const std::string &sBigFileName); char *getFileNamePtr(const std::string &sFileName, const std::string &sBigFileName);
typedef CCallback<bool /* continue */, const std::string &/* filename */, uint32 /* currentSize */, uint32 /* totalSize */> TUnpackProgressCallback;
// Unpack all files in sBigFileName to sDestDir and send progress notifications to optional callback
static bool unpack(const std::string &sBigFileName, const std::string &sDestDir, TUnpackProgressCallback *callback = NULL);
// A BNPFile header (filename is a char* pointing on FileNames and is always lowercase)
struct BNPFile
{
BNPFile() : Name(NULL), Size(0), Pos(0) { }
char* Name;
uint32 Size;
uint32 Pos;
};
// A SBNPFile header (filename is a std::string and keeps the original case)
struct SBNPFile
{
SBNPFile() : Size(0), Pos(0) { }
std::string Name;
uint32 Size;
uint32 Pos;
};
// A BNP structure
struct BNP
{
BNP() : FileNames(NULL), ThreadFileId(0), CacheFileOnOpen(false), AlwaysOpened(false), InternalUse(false), OffsetFromBeginning(0) { }
// FileName of the BNP. important to open it in getFile() (for other threads or if not always opened).
std::string BigFileName;
// map of files in the BNP.
char *FileNames;
std::vector<BNPFile> Files;
std::vector<SBNPFile> SFiles;
// Since many seek may be done on a FILE*, each thread should have its own FILE opened.
uint32 ThreadFileId;
bool CacheFileOnOpen;
bool AlwaysOpened;
bool InternalUse;
// Offset written in BNP header
uint32 OffsetFromBeginning;
// Read BNP header from FILE* and init member variables
bool readHeader(FILE* file);
// Read BNP header from BigFileName and init member variables
bool readHeader();
// Append BNP header to the big file BigFileName (to use after appendFile calls)
bool appendHeader();
// Append a file to BigFileName
bool appendFile(const std::string &filename);
// Unpack BigFileName to sDestDir and send progress notifications to optional callback
bool unpack(const std::string &sDestDir, TUnpackProgressCallback *callback = NULL);
};
// *************** // ***************
private: private:
class CThreadFileArray; class CThreadFileArray;
@ -118,38 +178,6 @@ private:
uint32 _CurrentId; uint32 _CurrentId;
}; };
// A BNPFile header
struct BNPFile
{
BNPFile() : Name(NULL), Size(0), Pos(0) { }
char *Name;
uint32 Size;
uint32 Pos;
};
struct CBNPFileComp
{
bool operator()(const BNPFile &f, const BNPFile &s )
{
return strcmp( f.Name, s.Name ) < 0;
}
};
// A BNP structure
struct BNP
{
BNP() : FileNames(NULL) { }
// FileName of the BNP. important to open it in getFile() (for other threads or if not always opened).
std::string BigFileName;
// map of files in the BNP.
char *FileNames;
std::vector<BNPFile> Files;
// Since many seek may be done on a FILE*, each thread should have its own FILE opened.
uint32 ThreadFileId;
bool CacheFileOnOpen;
bool AlwaysOpened;
};
private: private:
// CBigFile(); // Singleton mode -> access it with the getInstance function // CBigFile(); // Singleton mode -> access it with the getInstance function

View file

@ -16,6 +16,7 @@
#include "stdmisc.h" #include "stdmisc.h"
#include "nel/misc/file.h"
#include "nel/misc/big_file.h" #include "nel/misc/big_file.h"
#include "nel/misc/path.h" #include "nel/misc/path.h"
@ -138,141 +139,17 @@ bool CBigFile::add (const std::string &sBigFileName, uint32 nOptions)
handle.File = fopen (sBigFileName.c_str(), "rb"); handle.File = fopen (sBigFileName.c_str(), "rb");
if (handle.File == NULL) if (handle.File == NULL)
return false; return false;
uint32 nFileSize=CFile::getFileSize (handle.File);
//nlfseek64 (handle.File, 0, SEEK_END);
//uint32 nFileSize = ftell (handle.File);
// Result // Used internally by CBigFile, use optimizations and lower case of filenames
if (nlfseek64 (handle.File, nFileSize-4, SEEK_SET) != 0) bnp.InternalUse = true;
// read BNP header
if (!bnp.readHeader(handle.File))
{ {
fclose (handle.File); fclose (handle.File);
handle.File = NULL; handle.File = NULL;
return false; return false;
} }
uint32 nOffsetFromBeginning;
if (fread (&nOffsetFromBeginning, sizeof(uint32), 1, handle.File) != 1)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nOffsetFromBeginning);
#endif
if (nlfseek64 (handle.File, nOffsetFromBeginning, SEEK_SET) != 0)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
// Read the file count
uint32 nNbFile;
if (fread (&nNbFile, sizeof(uint32), 1, handle.File) != 1)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nNbFile);
#endif
map<string,BNPFile> tempMap;
for (uint32 i = 0; i < nNbFile; ++i)
{
char FileName[256];
uint8 nStringSize;
if (fread (&nStringSize, 1, 1, handle.File) != 1)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
if (fread (FileName, nStringSize, 1, handle.File) != 1)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
FileName[nStringSize] = 0;
uint32 nFileSize2;
if (fread (&nFileSize2, sizeof(uint32), 1, handle.File) != 1)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nFileSize2);
#endif
uint32 nFilePos;
if (fread (&nFilePos, sizeof(uint32), 1, handle.File) != 1)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nFilePos);
#endif
BNPFile bnpfTmp;
bnpfTmp.Pos = nFilePos;
bnpfTmp.Size = nFileSize2;
tempMap.insert (make_pair(toLower(string(FileName)), bnpfTmp));
}
if (nlfseek64 (handle.File, 0, SEEK_SET) != 0)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
// Convert temp map
if (nNbFile > 0)
{
uint nSize = 0, nNb = 0;
map<string,BNPFile>::iterator it = tempMap.begin();
while (it != tempMap.end())
{
nSize += (uint)it->first.size() + 1;
nNb++;
it++;
}
bnp.FileNames = new char[nSize];
memset(bnp.FileNames, 0, nSize);
bnp.Files.resize(nNb);
it = tempMap.begin();
nSize = 0;
nNb = 0;
while (it != tempMap.end())
{
strcpy(bnp.FileNames+nSize, it->first.c_str());
bnp.Files[nNb].Name = bnp.FileNames+nSize;
bnp.Files[nNb].Size = it->second.Size;
bnp.Files[nNb].Pos = it->second.Pos;
nSize += (uint)it->first.size() + 1;
nNb++;
it++;
}
}
// End of temp map conversion
if (nOptions&BF_CACHE_FILE_ON_OPEN) if (nOptions&BF_CACHE_FILE_ON_OPEN)
bnp.CacheFileOnOpen = true; bnp.CacheFileOnOpen = true;
else else
@ -314,6 +191,369 @@ void CBigFile::remove (const std::string &sBigFileName)
} }
} }
//// ***************************************************************************
bool CBigFile::BNP::readHeader()
{
// Only external use
if (InternalUse || BigFileName.empty()) return false;
FILE *f = fopen (BigFileName.c_str(), "rb");
if (f == NULL) return false;
bool res = readHeader(f);
fclose (f);
return res;
}
//// ***************************************************************************
bool CBigFile::BNP::readHeader(FILE *file)
{
if (file == NULL) return false;
uint32 nFileSize=CFile::getFileSize (file);
// Result
if (nlfseek64 (file, nFileSize-4, SEEK_SET) != 0)
{
return false;
}
if (fread (&OffsetFromBeginning, sizeof(uint32), 1, file) != 1)
{
return false;
}
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(OffsetFromBeginning);
#endif
if (nlfseek64 (file, OffsetFromBeginning, SEEK_SET) != 0)
{
return false;
}
// Read the file count
uint32 nNbFile;
if (fread (&nNbFile, sizeof(uint32), 1, file) != 1)
{
return false;
}
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nNbFile);
#endif
map<string, BNPFile> tempMap;
if (!InternalUse) SFiles.clear();
for (uint32 i = 0; i < nNbFile; ++i)
{
uint8 nStringSize;
if (fread (&nStringSize, 1, 1, file) != 1)
{
return false;
}
char sFileName[256];
if (fread (sFileName, 1, nStringSize, file) != nStringSize)
{
return false;
}
sFileName[nStringSize] = 0;
uint32 nFileSize2;
if (fread (&nFileSize2, sizeof(uint32), 1, file) != 1)
{
return false;
}
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nFileSize2);
#endif
uint32 nFilePos;
if (fread (&nFilePos, sizeof(uint32), 1, file) != 1)
{
return false;
}
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nFilePos);
#endif
if (InternalUse)
{
BNPFile bnpfTmp;
bnpfTmp.Pos = nFilePos;
bnpfTmp.Size = nFileSize2;
tempMap.insert (make_pair(toLower(string(sFileName)), bnpfTmp));
}
else
{
SBNPFile bnpfTmp;
bnpfTmp.Name = sFileName;
bnpfTmp.Pos = nFilePos;
bnpfTmp.Size = nFileSize2;
SFiles.push_back(bnpfTmp);
}
}
if (nlfseek64 (file, 0, SEEK_SET) != 0)
{
return false;
}
// Convert temp map
if (InternalUse && nNbFile > 0)
{
uint nSize = 0, nNb = 0;
map<string,BNPFile>::iterator it = tempMap.begin();
while (it != tempMap.end())
{
nSize += (uint)it->first.size() + 1;
nNb++;
it++;
}
FileNames = new char[nSize];
memset(FileNames, 0, nSize);
Files.resize(nNb);
it = tempMap.begin();
nSize = 0;
nNb = 0;
while (it != tempMap.end())
{
strcpy(FileNames+nSize, it->first.c_str());
Files[nNb].Name = FileNames+nSize;
Files[nNb].Size = it->second.Size;
Files[nNb].Pos = it->second.Pos;
nSize += (uint)it->first.size() + 1;
nNb++;
it++;
}
}
// End of temp map conversion
return true;
}
bool CBigFile::BNP::appendHeader()
{
// Only external use
if (InternalUse || BigFileName.empty()) return false;
FILE *f = fopen (BigFileName.c_str(), "ab");
if (f == NULL) return false;
uint32 nNbFile = (uint32)SFiles.size();
// value to be serialized
uint32 nNbFile2 = nNbFile;
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nNbFile2);
#endif
if (fwrite (&nNbFile2, sizeof(uint32), 1, f) != 1)
{
fclose(f);
return false;
}
for (uint32 i = 0; i < nNbFile; ++i)
{
uint8 nStringSize = (uint8)SFiles[i].Name.length();
if (fwrite (&nStringSize, 1, 1, f) != 1)
{
fclose(f);
return false;
}
if (fwrite (SFiles[i].Name.c_str(), 1, nStringSize, f) != nStringSize)
{
fclose(f);
return false;
}
uint32 nFileSize = SFiles[i].Size;
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nFileSize);
#endif
if (fwrite (&nFileSize, sizeof(uint32), 1, f) != 1)
{
fclose(f);
return false;
}
uint32 nFilePos = SFiles[i].Pos;
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nFilePos);
#endif
if (fwrite (&nFilePos, sizeof(uint32), 1, f) != 1)
{
fclose(f);
return false;
}
}
uint32 nOffsetFromBeginning = OffsetFromBeginning;
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nOffsetFromBeginning);
#endif
if (fwrite (&nOffsetFromBeginning, sizeof(uint32), 1, f) != 1)
{
fclose(f);
return false;
}
fclose (f);
return true;
}
// ***************************************************************************
bool CBigFile::BNP::appendFile(const std::string &filename)
{
// Only external use
if (InternalUse || BigFileName.empty()) return false;
// Check if we can read the source file
if (!CFile::fileExists(filename)) return false;
SBNPFile ftmp;
ftmp.Name = CFile::getFilename(filename);
ftmp.Size = CFile::getFileSize(filename);
ftmp.Pos = OffsetFromBeginning;
SFiles.push_back(ftmp);
OffsetFromBeginning += ftmp.Size;
FILE *f1 = fopen(BigFileName.c_str(), "ab");
if (f1 == NULL) return false;
FILE *f2 = fopen(filename.c_str(), "rb");
if (f2 == NULL)
{
fclose(f1);
return false;
}
uint8 *ptr = new uint8[ftmp.Size];
if (fread (ptr, ftmp.Size, 1, f2) != 1)
{
nlwarning("%s read error", filename.c_str());
}
else if (fwrite (ptr, ftmp.Size, 1, f1) != 1)
{
nlwarning("%s write error", BigFileName.c_str());
}
delete [] ptr;
fclose(f2);
fclose(f1);
return true;
}
// ***************************************************************************
bool CBigFile::BNP::unpack(const std::string &sDestDir, TUnpackProgressCallback *callback)
{
// Only external use
if (InternalUse || BigFileName.empty()) return false;
FILE *bnp = fopen (BigFileName.c_str(), "rb");
if (bnp == NULL)
return false;
// only read header is not already read
if (SFiles.empty() && !readHeader(bnp))
{
fclose (bnp);
return false;
}
CFile::createDirectory(sDestDir);
uint32 totalUncompressed = 0, total = 0;
for (uint32 i = 0; i < SFiles.size(); ++i)
{
total += SFiles[i].Size;
}
FILE *out = NULL;
for (uint32 i = 0; i < SFiles.size(); ++i)
{
const SBNPFile &rBNPFile = SFiles[i];
string filename = CPath::standardizePath(sDestDir) + rBNPFile.Name;
if (callback && !(*callback)(filename, totalUncompressed, total))
{
fclose (bnp);
return false;
}
out = fopen (filename.c_str(), "wb");
if (out != NULL)
{
nlfseek64 (bnp, rBNPFile.Pos, SEEK_SET);
uint8 *ptr = new uint8[rBNPFile.Size];
bool readError = fread (ptr, rBNPFile.Size, 1, bnp) != 1;
if (readError)
{
nlwarning("%s read error errno = %d: %s", filename.c_str(), errno, strerror(errno));
}
bool writeError = fwrite (ptr, rBNPFile.Size, 1, out) != 1;
if (writeError)
{
nlwarning("%s write error errno = %d: %s", filename.c_str(), errno, strerror(errno));
}
bool diskFull = ferror(out) && errno == 28 /* ENOSPC*/;
fclose (out);
delete [] ptr;
if (diskFull)
{
fclose (bnp);
throw NLMISC::EDiskFullError(filename);
}
if (writeError)
{
fclose (bnp);
throw NLMISC::EWriteError(filename);
}
if (readError)
{
fclose (bnp);
throw NLMISC::EReadError(filename);
}
}
totalUncompressed += rBNPFile.Size;
if (callback && !(*callback)(filename, totalUncompressed, total))
{
fclose (bnp);
return false;
}
}
fclose (bnp);
return true;
}
// *************************************************************************** // ***************************************************************************
bool CBigFile::isBigFileAdded(const std::string &sBigFileName) const bool CBigFile::isBigFileAdded(const std::string &sBigFileName) const
{ {
@ -359,6 +599,14 @@ void CBigFile::removeAll ()
} }
} }
struct CBNPFileComp
{
bool operator()(const CBigFile::BNPFile &f, const CBigFile::BNPFile &s )
{
return strcmp( f.Name, s.Name ) < 0;
}
};
// *************************************************************************** // ***************************************************************************
bool CBigFile::getFileInternal (const std::string &sFileName, BNP *&zeBnp, BNPFile *&zeBnpFile) bool CBigFile::getFileInternal (const std::string &sFileName, BNP *&zeBnp, BNPFile *&zeBnpFile)
{ {
@ -504,5 +752,12 @@ void CBigFile::getBigFilePaths(std::vector<std::string> &bigFilePaths)
} }
} }
// ***************************************************************************
bool CBigFile::unpack(const std::string &sBigFileName, const std::string &sDestDir, TUnpackProgressCallback *callback)
{
BNP bnpFile;
bnpFile.BigFileName = sBigFileName;
return bnpFile.unpack(sDestDir, callback);
}
} // namespace NLMISC } // namespace NLMISC