Changed: Centralize BNP unpacking and packing methods, to be reused in other tools/programs

code/nel/include/nel/misc/big_file.h

@@ -20,7 +20,7 @@
 #include "types_nl.h"
 #include "tds.h"
 #include "singleton.h"
-
+#include "callback.h"
 
 namespace NLMISC {
 
@@ -86,6 +86,66 @@ public:
 	// Used for CPath only for the moment !
 	char *getFileNamePtr(const std::string &sFileName, const std::string &sBigFileName);
 
+	typedef CCallback<bool /* continue */, const std::string &/* filename */, uint32 /* currentSize */, uint32 /* totalSize */> TUnpackProgressCallback;
+
+	// Unpack all files in sBigFileName to sDestDir and send progress notifications to optional callback
+	static bool unpack(const std::string &sBigFileName, const std::string &sDestDir, TUnpackProgressCallback *callback = NULL);
+
+	// A BNPFile header (filename is a char* pointing on FileNames and is always lowercase)
+	struct BNPFile
+	{
+		BNPFile() : Name(NULL), Size(0), Pos(0) { }
+		char*		Name;
+		uint32		Size;
+		uint32		Pos;
+	};
+
+	// A SBNPFile header (filename is a std::string and keeps the original case)
+	struct SBNPFile
+	{
+		SBNPFile() : Size(0), Pos(0) { }
+		std::string	Name;
+		uint32		Size;
+		uint32		Pos;
+	};
+
+	// A BNP structure
+	struct BNP
+	{
+		BNP() : FileNames(NULL), ThreadFileId(0), CacheFileOnOpen(false), AlwaysOpened(false), InternalUse(false), OffsetFromBeginning(0) { }
+
+		// FileName of the BNP. important to open it in getFile() (for other threads or if not always opened).
+		std::string						BigFileName;
+		// map of files in the BNP.
+		char							*FileNames;
+		std::vector<BNPFile>			Files;
+		std::vector<SBNPFile>			SFiles;
+
+		// Since many seek may be done on a FILE*, each thread should have its own FILE opened.
+		uint32							ThreadFileId;
+		bool							CacheFileOnOpen;
+		bool							AlwaysOpened;
+		bool							InternalUse;
+
+		// Offset written in BNP header
+		uint32							OffsetFromBeginning;
+
+		// Read BNP header from FILE* and init member variables
+		bool readHeader(FILE* file);
+
+		// Read BNP header from BigFileName and init member variables
+		bool readHeader();
+
+		// Append BNP header to the big file BigFileName (to use after appendFile calls)
+		bool appendHeader();
+
+		// Append a file to BigFileName
+		bool appendFile(const std::string &filename);
+
+		// Unpack BigFileName to sDestDir and send progress notifications to optional callback
+		bool unpack(const std::string &sDestDir, TUnpackProgressCallback *callback = NULL);
+	};
+
 // ***************
 private:
 	class	CThreadFileArray;
@@ -118,38 +178,6 @@ private:
 		uint32		_CurrentId;
 	};
 
-	// A BNPFile header
-	struct BNPFile
-	{
-		BNPFile() : Name(NULL), Size(0), Pos(0) { }
-		char		*Name;
-		uint32		Size;
-		uint32		Pos;
-	};
-
-	struct CBNPFileComp
-	{
-		bool operator()(const BNPFile &f, const BNPFile &s )
-		{
-			return strcmp( f.Name, s.Name ) < 0;
-		}
-	};
-
-	// A BNP structure
-	struct BNP
-	{
-		BNP() : FileNames(NULL) { }
-
-		// FileName of the BNP. important to open it in getFile() (for other threads or if not always opened).
-		std::string						BigFileName;
-		// map of files in the BNP.
-		char							*FileNames;
-		std::vector<BNPFile>			Files;
-		// Since many seek may be done on a FILE*, each thread should have its own FILE opened.
-		uint32							ThreadFileId;
-		bool							CacheFileOnOpen;
-		bool							AlwaysOpened;
-	};
 private:
 
 //	CBigFile(); // Singleton mode -> access it with the getInstance function

code/nel/src/misc/big_file.cpp

@@ -16,6 +16,7 @@
 
 #include "stdmisc.h"
 
+#include "nel/misc/file.h"
 #include "nel/misc/big_file.h"
 #include "nel/misc/path.h"
 
@@ -138,141 +139,17 @@ bool CBigFile::add (const std::string &sBigFileName, uint32 nOptions)
 	handle.File = fopen (sBigFileName.c_str(), "rb");
 	if (handle.File == NULL)
 		return false;
-	uint32 nFileSize=CFile::getFileSize (handle.File);
-	//nlfseek64 (handle.File, 0, SEEK_END);
-	//uint32 nFileSize = ftell (handle.File);
 
-	// Result
+	// Used internally by CBigFile, use optimizations and lower case of filenames
-	if (nlfseek64 (handle.File, nFileSize-4, SEEK_SET) != 0)
+	bnp.InternalUse = true;
+
+	// read BNP header
+	if (!bnp.readHeader(handle.File))
 	{
 		fclose (handle.File);
 		handle.File = NULL;
 		return false;
 	}
-
-	uint32 nOffsetFromBeginning;
-	if (fread (&nOffsetFromBeginning, sizeof(uint32), 1, handle.File) != 1)
-	{
-		fclose (handle.File);
-		handle.File = NULL;
-		return false;
-	}
-
-#ifdef NL_BIG_ENDIAN
-	NLMISC_BSWAP32(nOffsetFromBeginning);
-#endif
-
-	if (nlfseek64 (handle.File, nOffsetFromBeginning, SEEK_SET) != 0)
-	{
-		fclose (handle.File);
-		handle.File = NULL;
-		return false;
-	}
-
-	// Read the file count
-	uint32 nNbFile;
-	if (fread (&nNbFile, sizeof(uint32), 1, handle.File) != 1)
-	{
-		fclose (handle.File);
-		handle.File = NULL;
-		return false;
-	}
-
-#ifdef NL_BIG_ENDIAN
-	NLMISC_BSWAP32(nNbFile);
-#endif
-
-	map<string,BNPFile> tempMap;
-	for (uint32 i = 0; i < nNbFile; ++i)
-	{
-		char FileName[256];
-		uint8 nStringSize;
-		if (fread (&nStringSize, 1, 1, handle.File) != 1)
-		{
-			fclose (handle.File);
-			handle.File = NULL;
-			return false;
-		}
-
-		if (fread (FileName, nStringSize, 1, handle.File) != 1)
-		{
-			fclose (handle.File);
-			handle.File = NULL;
-			return false;
-		}
-
-		FileName[nStringSize] = 0;
-		uint32 nFileSize2;
-		if (fread (&nFileSize2, sizeof(uint32), 1, handle.File) != 1)
-		{
-			fclose (handle.File);
-			handle.File = NULL;
-			return false;
-		}
-
-#ifdef NL_BIG_ENDIAN
-		NLMISC_BSWAP32(nFileSize2);
-#endif
-
-		uint32 nFilePos;
-		if (fread (&nFilePos, sizeof(uint32), 1, handle.File) != 1)
-		{
-			fclose (handle.File);
-			handle.File = NULL;
-			return false;
-		}
-
-#ifdef NL_BIG_ENDIAN
-		NLMISC_BSWAP32(nFilePos);
-#endif
-
-		BNPFile bnpfTmp;
-		bnpfTmp.Pos = nFilePos;
-		bnpfTmp.Size = nFileSize2;
-		tempMap.insert (make_pair(toLower(string(FileName)), bnpfTmp));
-	}
-
-	if (nlfseek64 (handle.File, 0, SEEK_SET) != 0)
-	{
-		fclose (handle.File);
-		handle.File = NULL;
-		return false;
-	}
-
-	// Convert temp map
-	if (nNbFile > 0)
-	{
-		uint nSize = 0, nNb = 0;
-		map<string,BNPFile>::iterator it = tempMap.begin();
-		while (it != tempMap.end())
-		{
-			nSize += (uint)it->first.size() + 1;
-			nNb++;
-			it++;
-		}
-
-		bnp.FileNames = new char[nSize];
-		memset(bnp.FileNames, 0, nSize);
-		bnp.Files.resize(nNb);
-
-		it = tempMap.begin();
-		nSize = 0;
-		nNb = 0;
-		while (it != tempMap.end())
-		{
-			strcpy(bnp.FileNames+nSize, it->first.c_str());
-
-			bnp.Files[nNb].Name = bnp.FileNames+nSize;
-			bnp.Files[nNb].Size = it->second.Size;
-			bnp.Files[nNb].Pos = it->second.Pos;
-
-			nSize += (uint)it->first.size() + 1;
-			nNb++;
-			it++;
-		}
-	}
-	// End of temp map conversion
-
 	if (nOptions&BF_CACHE_FILE_ON_OPEN)
 		bnp.CacheFileOnOpen = true;
 	else
@@ -314,6 +191,369 @@ void CBigFile::remove (const std::string &sBigFileName)
 	}
 }
 
+//// ***************************************************************************
+bool CBigFile::BNP::readHeader()
+{
+	// Only external use
+	if (InternalUse || BigFileName.empty()) return false;
+
+	FILE *f = fopen (BigFileName.c_str(), "rb");
+	if (f == NULL) return false;
+
+	bool res = readHeader(f);
+	fclose (f);
+
+	return res;
+}
+
+//// ***************************************************************************
+bool CBigFile::BNP::readHeader(FILE *file)
+{
+	if (file == NULL) return false;
+
+	uint32 nFileSize=CFile::getFileSize (file);
+
+	// Result
+	if (nlfseek64 (file, nFileSize-4, SEEK_SET) != 0)
+	{
+		return false;
+	}
+
+	if (fread (&OffsetFromBeginning, sizeof(uint32), 1, file) != 1)
+	{
+		return false;
+	}
+
+#ifdef NL_BIG_ENDIAN
+	NLMISC_BSWAP32(OffsetFromBeginning);
+#endif
+
+	if (nlfseek64 (file, OffsetFromBeginning, SEEK_SET) != 0)
+	{
+		return false;
+	}
+
+	// Read the file count
+	uint32 nNbFile;
+	if (fread (&nNbFile, sizeof(uint32), 1, file) != 1)
+	{
+		return false;
+	}
+
+#ifdef NL_BIG_ENDIAN
+	NLMISC_BSWAP32(nNbFile);
+#endif
+
+	map<string, BNPFile> tempMap;
+
+	if (!InternalUse) SFiles.clear();
+
+	for (uint32 i = 0; i < nNbFile; ++i)
+	{
+		uint8 nStringSize;
+		if (fread (&nStringSize, 1, 1, file) != 1)
+		{
+			return false;
+		}
+
+		char sFileName[256];
+		if (fread (sFileName, 1, nStringSize, file) != nStringSize)
+		{
+			return false;
+		}
+
+		sFileName[nStringSize] = 0;
+
+		uint32 nFileSize2;
+		if (fread (&nFileSize2, sizeof(uint32), 1, file) != 1)
+		{
+			return false;
+		}
+
+#ifdef NL_BIG_ENDIAN
+		NLMISC_BSWAP32(nFileSize2);
+#endif
+
+		uint32 nFilePos;
+		if (fread (&nFilePos, sizeof(uint32), 1, file) != 1)
+		{
+			return false;
+		}
+
+#ifdef NL_BIG_ENDIAN
+		NLMISC_BSWAP32(nFilePos);
+#endif
+
+		if (InternalUse)
+		{
+			BNPFile bnpfTmp;
+			bnpfTmp.Pos = nFilePos;
+			bnpfTmp.Size = nFileSize2;
+			tempMap.insert (make_pair(toLower(string(sFileName)), bnpfTmp));
+		}
+		else
+		{
+			SBNPFile bnpfTmp;
+			bnpfTmp.Name = sFileName;
+			bnpfTmp.Pos = nFilePos;
+			bnpfTmp.Size = nFileSize2;
+			SFiles.push_back(bnpfTmp);
+		}
+	}
+
+	if (nlfseek64 (file, 0, SEEK_SET) != 0)
+	{
+		return false;
+	}
+
+	// Convert temp map
+	if (InternalUse && nNbFile > 0)
+	{
+		uint nSize = 0, nNb = 0;
+		map<string,BNPFile>::iterator it = tempMap.begin();
+		while (it != tempMap.end())
+		{
+			nSize += (uint)it->first.size() + 1;
+			nNb++;
+			it++;
+		}
+
+		FileNames = new char[nSize];
+		memset(FileNames, 0, nSize);
+		Files.resize(nNb);
+
+		it = tempMap.begin();
+		nSize = 0;
+		nNb = 0;
+		while (it != tempMap.end())
+		{
+			strcpy(FileNames+nSize, it->first.c_str());
+
+			Files[nNb].Name = FileNames+nSize;
+			Files[nNb].Size = it->second.Size;
+			Files[nNb].Pos = it->second.Pos;
+
+			nSize += (uint)it->first.size() + 1;
+			nNb++;
+			it++;
+		}
+	}
+	// End of temp map conversion
+
+	return true;
+}
+
+bool CBigFile::BNP::appendHeader()
+{
+	// Only external use
+	if (InternalUse || BigFileName.empty()) return false;
+
+	FILE *f = fopen (BigFileName.c_str(), "ab");
+	if (f == NULL) return false;
+
+	uint32 nNbFile = (uint32)SFiles.size();
+
+	// value to be serialized
+	uint32 nNbFile2 = nNbFile;
+
+#ifdef NL_BIG_ENDIAN
+	NLMISC_BSWAP32(nNbFile2);
+#endif
+
+	if (fwrite (&nNbFile2, sizeof(uint32), 1, f) != 1)
+	{
+		fclose(f);
+		return false;
+	}
+
+	for (uint32 i = 0; i < nNbFile; ++i)
+	{
+		uint8 nStringSize = (uint8)SFiles[i].Name.length();
+		if (fwrite (&nStringSize, 1, 1, f) != 1)
+		{
+			fclose(f);
+			return false;
+		}
+
+		if (fwrite (SFiles[i].Name.c_str(), 1, nStringSize, f) != nStringSize)
+		{
+			fclose(f);
+			return false;
+		}
+
+		uint32 nFileSize = SFiles[i].Size;
+
+#ifdef NL_BIG_ENDIAN
+		NLMISC_BSWAP32(nFileSize);
+#endif
+
+		if (fwrite (&nFileSize, sizeof(uint32), 1, f) != 1)
+		{
+			fclose(f);
+			return false;
+		}
+
+		uint32 nFilePos = SFiles[i].Pos;
+
+#ifdef NL_BIG_ENDIAN
+		NLMISC_BSWAP32(nFilePos);
+#endif
+
+		if (fwrite (&nFilePos, sizeof(uint32), 1, f) != 1)
+		{
+			fclose(f);
+			return false;
+		}
+	}
+
+	uint32 nOffsetFromBeginning = OffsetFromBeginning;
+
+#ifdef NL_BIG_ENDIAN
+	NLMISC_BSWAP32(nOffsetFromBeginning);
+#endif
+
+	if (fwrite (&nOffsetFromBeginning, sizeof(uint32), 1, f) != 1)
+	{
+		fclose(f);
+		return false;
+	}
+
+	fclose (f);
+	return true;
+}
+
+// ***************************************************************************
+bool CBigFile::BNP::appendFile(const std::string &filename)
+{
+	// Only external use
+	if (InternalUse || BigFileName.empty()) return false;
+
+	// Check if we can read the source file
+	if (!CFile::fileExists(filename)) return false;
+
+	SBNPFile ftmp;
+	ftmp.Name = CFile::getFilename(filename);
+	ftmp.Size = CFile::getFileSize(filename);
+	ftmp.Pos = OffsetFromBeginning;
+	SFiles.push_back(ftmp);
+	OffsetFromBeginning += ftmp.Size;
+
+	FILE *f1 = fopen(BigFileName.c_str(), "ab");
+	if (f1 == NULL) return false;
+
+	FILE *f2 = fopen(filename.c_str(), "rb");
+	if (f2 == NULL)
+	{
+		fclose(f1);
+		return false;
+	}
+	
+	uint8 *ptr = new uint8[ftmp.Size];
+
+	if (fread (ptr, ftmp.Size, 1, f2) != 1)
+	{
+		nlwarning("%s read error", filename.c_str());
+	}
+	else if (fwrite (ptr, ftmp.Size, 1, f1) != 1)
+	{
+		nlwarning("%s write error", BigFileName.c_str());
+	}
+
+	delete [] ptr;
+	
+	fclose(f2);
+	fclose(f1);
+
+	return true;
+}
+
+// ***************************************************************************
+bool CBigFile::BNP::unpack(const std::string &sDestDir, TUnpackProgressCallback *callback)
+{
+	// Only external use
+	if (InternalUse || BigFileName.empty()) return false;
+
+	FILE *bnp = fopen (BigFileName.c_str(), "rb");
+	if (bnp == NULL)
+		return false;
+
+	// only read header is not already read
+	if (SFiles.empty() && !readHeader(bnp))
+	{
+		fclose (bnp);
+		return false;
+	}
+
+	CFile::createDirectory(sDestDir);
+
+	uint32 totalUncompressed = 0, total = 0;
+
+	for (uint32 i = 0; i < SFiles.size(); ++i)
+	{
+		total += SFiles[i].Size;
+	}
+
+	FILE *out = NULL;
+
+	for (uint32 i = 0; i < SFiles.size(); ++i)
+	{
+		const SBNPFile &rBNPFile = SFiles[i];
+		string filename = CPath::standardizePath(sDestDir) + rBNPFile.Name;
+
+		if (callback && !(*callback)(filename, totalUncompressed, total))
+		{
+			fclose (bnp);
+			return false;
+		}
+
+		out = fopen (filename.c_str(), "wb");
+		if (out != NULL)
+		{
+			nlfseek64 (bnp, rBNPFile.Pos, SEEK_SET);
+			uint8 *ptr = new uint8[rBNPFile.Size];
+			bool readError = fread (ptr, rBNPFile.Size, 1, bnp) != 1;
+			if (readError)
+			{
+				nlwarning("%s read error errno = %d: %s", filename.c_str(), errno, strerror(errno));
+			}
+			bool writeError = fwrite (ptr, rBNPFile.Size, 1, out) != 1;
+			if (writeError)
+			{
+				nlwarning("%s write error errno = %d: %s", filename.c_str(), errno, strerror(errno));
+			}
+			bool diskFull = ferror(out) && errno == 28 /* ENOSPC*/;
+			fclose (out);
+			delete [] ptr;
+			if (diskFull)
+			{
+				fclose (bnp);
+				throw NLMISC::EDiskFullError(filename);
+			}
+			if (writeError)
+			{
+				fclose (bnp);
+				throw NLMISC::EWriteError(filename);
+			}
+			if (readError)
+			{
+				fclose (bnp);
+				throw NLMISC::EReadError(filename);
+			}
+		}
+
+		totalUncompressed += rBNPFile.Size;
+
+		if (callback && !(*callback)(filename, totalUncompressed, total))
+		{
+			fclose (bnp);
+			return false;
+		}
+	}
+
+	fclose (bnp);
+	return true;
+}
+
 // ***************************************************************************
 bool CBigFile::isBigFileAdded(const std::string &sBigFileName) const
 {
@@ -359,6 +599,14 @@ void CBigFile::removeAll ()
 	}
 }
 
+struct CBNPFileComp
+{
+	bool operator()(const CBigFile::BNPFile &f, const CBigFile::BNPFile &s )
+	{
+		return strcmp( f.Name, s.Name ) < 0;
+	}
+};
+
 // ***************************************************************************
 bool CBigFile::getFileInternal (const std::string &sFileName, BNP *&zeBnp, BNPFile *&zeBnpFile)
 {
@@ -504,5 +752,12 @@ void CBigFile::getBigFilePaths(std::vector<std::string> &bigFilePaths)
 	}
 }
 
+// ***************************************************************************
+bool CBigFile::unpack(const std::string &sBigFileName, const std::string &sDestDir, TUnpackProgressCallback *callback)
+{
+	BNP bnpFile;
+	bnpFile.BigFileName = sBigFileName;
+	return bnpFile.unpack(sDestDir, callback);
+}
 
 } // namespace NLMISC