khanat-opennel-code/code/nel/tools/misc/extract_filename/extract_filename.cpp

// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
// Copyright (C) 2010  Winch Gate Property Limited
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

#include "nel/misc/file.h"
#include "nel/misc/config_file.h"

using namespace std;
using namespace NLMISC;

#ifdef NL_DEBUG
#define INFO nlinfo
#else // NL_DEBUG
#	if defined(NL_COMP_VC) && NL_COMP_VC_VERSION >= 71
#		define INFO __noop
#	else
#		define INFO 0&&
#	endif
#endif // NL_DEBUG

bool readTheFile (const char *filename, vector<char> &fileArray)
{
	// Open the file
	CIFile file;
	if (file.open (filename))
	{
		// Go to the end
		file.seek (0, NLMISC::IStream::end);

		// Get file size
		sint32 size = file.getPos ();

		// Resize the array
		fileArray.resize (size);

		// Go to the begin
		file.seek (0, NLMISC::IStream::begin);

		if (size)
		{
			// Read the file
			NLMISC::IStream *stream = &file;
			stream->serialBuffer ((uint8*)&fileArray[0], size);
		}

		return true;
	}
	else
	{
		// Error, can't open the file
		nlwarning ("Can't open the file %s for reading.", filename);

		return false;
	}
}

inline bool isASCII (uint8 c)
{
	return ( (c>=0x20 /*&& c<=0x7E*/) || (c == 0x09) || (c == 0x0A) || (c == 0x0D) );
}

bool isASCII(vector<char> &fileArray)
{
	// Array size
	uint size = (uint)fileArray.size();

	if (size)
	{
		// Array pointer
		const char *arrayPointer = &fileArray[0];

		// Line count
		uint line = 0;

		for (uint c=0; c<size; c++)
		{
			if (!isASCII (arrayPointer[c]))
			{
				INFO ("Binary character: 0x%02x (line %d)", (uint)(uint8)arrayPointer[c], line);
				return false;
			}
			if (arrayPointer[c] == '\n')
				line++;
		}
	}

	return true;
}

inline bool isStringChar (char c)
{
	return ( ( (c>='a') && (c<='z') ) || ( (c>='A') && (c<='Z') ) || ( (c>='0') && (c<='9') ) || (c=='-') || (c=='_') || (c=='.') );
}

void removeBeginEndSpaces (string &str)
{
	// Remove begin space
	sint index=0;
	while ( (index<(sint)str.length ()) && (str[index]==' ') )
		index++;
	if (index != 0)
		str = str.substr (index, str.length ()-index);

	// Remove end space
	index=(sint)str.length ()-1;
	while ( (index>0) && (str[index]==' ') )
		index--;
	str.resize (index+1);
}

void removeChar (string &str, char ch)
{
	for (uint c=0; c<str.size (); c++)
	{
		if (str[c] == ch)
			str.erase (c, 1);
	}
}

void removeDirectory (string &str)
{
	// Remove begin space
	string::size_type pos = str.rfind ('\\');
	string::size_type pos2 = str.rfind ('/');

	if (pos == string::npos)
		pos = pos2;

	if (pos != string::npos)
		str = str.substr (pos+1, str.size());
}

bool filterExtension (const char *str, const vector<string> &extensions)
{
	// String size
	uint size = (uint)strlen (str);

	// For each filter
	for (uint i=0; i<extensions.size(); i++)
	{
		if ( (str+(size - extensions[i].size())) == extensions[i])
			return true;
	}

	// Not found
	return false;
}

bool validateFilename (const char *str)
{
	// Look for space
	return (strchr (str, ' ') == NULL);
}

const char *getExtension (const char *filename)
{
	return strrchr (filename, '.');
}

void removeExtension (string &filename)
{
	string::size_type index = filename.rfind ('.');
	filename.resize (index);
}

void extractStringsFromBinary (const vector<char> &fileArray, set<string> &filenameStrings, const vector<string> &extensions)
{
	// Array size
	uint size = (uint)fileArray.size();

	// Array pointer
	const char *arrayPointer = &fileArray[0];

	// run through the data buffer looking for plausible looking strings
	uint i,j;
	for (i=0; (j=i+sizeof(uint))<size; i++)
	{
		// if we're pointing at a string the first 4 bytes ar the string length
		uint len=*(uint *)(arrayPointer+i);

		// if the string length could be valid
		if (len>0 && len+i<=size)
		{
			uint k;
			for (k=j; k<len+j && isASCII (arrayPointer[k]);k++);
			if (k==len+j)
			{
				// build a string
				string str;

				// Resize the string
				str.resize (len);

				// Copy the string
				for (k=0; k<len; k++)
					str[k] = arrayPointer[j+k];

				// Remove space
				removeBeginEndSpaces (str);

				// Not empty ?
				if (str != "")
				{
					// Lower case
					str = toLower (str);

					// Filter extensions
					if (filterExtension (str.c_str(), extensions))
					{
						// Remove directory
						removeDirectory (str);

						// Validate filename
						if (validateFilename (str.c_str()))
						{
							// Add a string
							filenameStrings.insert (str);

							// Information
							INFO ("Binary filename extracted: \"%s\" (0x%08x)", str.c_str(), j);
						}
						else
						{
							INFO ("Invalid filename: \"%s\" (0x%08x)", str.c_str(), j);
						}
					}
				}
			}
		}
	}
}

void extractStringsFromASCII (const vector<char> &fileArray, set<string> &filenameStrings, const vector<string> &extensions)
{
	// Array size
	uint size = (uint)fileArray.size();
	if (size)
	{
		// Array pointer
		const char *arrayPointer = &fileArray[0];

		// Temp string
		string temp;

		// Begin of a valid string
		const char *begin = arrayPointer;
		while ((begin<arrayPointer) && (!isStringChar (*begin)))
			begin++;
		const char *end = begin;
		while (end<arrayPointer+size)
		{
			// End char is ok ?
			if (isStringChar (*end))
			{
				end++;
			}
			else
			{
				// Found a string ?
				if (end != begin)
				{
					// String size
					uint size = (uint)(end-begin);
					temp.resize (size);

					// Copy the string
					for (uint c=0; c<size; c++)
						temp[c] = begin[c];

					// Lower case
					temp = toLower (temp);

					// Filter extensions
					if (filterExtension (temp.c_str(), extensions))
					{
						// Remove directory
						removeDirectory (temp);

						// Validate filename
						if (validateFilename (temp.c_str()))
						{
							// Add a string
							filenameStrings.insert (temp);

							// Information
							INFO ("ASCII filename extracted: \"%s\"", temp.c_str());
						}
						else
						{
							INFO ("Invalid filename: \"%s\"", temp.c_str());
						}
					}
				}
				end++;
				begin = end;
			}
		}
	}
}

bool loadConfigFiles (const char *ext, const char *input_files, const char *available_files, vector<string> &extensions, set<string> &inputFiles, map<string, string> &availableFiles)
{
	// Get a string
	string temp;
	string temp2;

	// Extensions files
	FILE *file = fopen (ext, "r");
	if (file)
	{
		bool cont = true;
		char name[512];
		while (fgets (name, 512, file))
		{
			// To string and lower
			temp = toLower (string(name));

			// Remove return
			removeChar (temp, '\n');

			// Valid extension ?
			if (temp.size() && temp[0] == '.')
				// Add the extension
				extensions.push_back (temp);
			else
			{
				nlwarning ("ERROR extension %s must begin with a '.' character.", temp.c_str());
				cont = false;
				break;
			}
		}

		// Close
		fclose (file);
		if (cont)
		{
			// Input files
			file = fopen (input_files, "r");
			if (file)
			{
				char name[512];
				while (fgets (name, 512, file))
				{
					// To string
					temp = name;

					// Remove return
					removeChar (temp, '\n');

					// Add the extension
					inputFiles.insert (temp);
				}

				// Close
				fclose (file);

				// Available files
				file = fopen (available_files, "r");
				if (file)
				{
					char name[512];
					while (fgets (name, 512, file))
					{
						// To lower
						temp = toLower (string(name));
						temp2 = toLower (string(name));

						// Remove space
						removeBeginEndSpaces (temp);
						removeBeginEndSpaces (temp2);

						// Remove return
						removeChar (temp, '\n');
						removeChar (temp2, '\n');

						// Remove directory
						removeDirectory (temp);

						// Good extension
						if (filterExtension (temp.c_str (), extensions))
						{
							if (validateFilename (temp.c_str ()))
							{
								// Add the extension
								if (!availableFiles.insert (map<string, string>::value_type (temp, temp2)).second)
								{
									fprintf (stderr, "DUBLING: %s %s\n", temp.c_str (), temp2.c_str());
								}
							}
							else
							{
								fprintf (stderr, "INVALIDE NAME: %s\n", temp.c_str ());
							}
						}
						else
						{
							fprintf (stderr, "INVALIDE EXT: %s\n", temp.c_str ());
						}
					}

					// Close
					fclose (file);

					// Ok
					return true;
				}
				else
				{
					nlwarning ("ERROR can't load available files %s", ext);
				}
			}
			else
			{
				nlwarning ("ERROR can't load input files %s", ext);
			}
		}
	}
	else
	{
		nlwarning ("ERROR can't load extension file %s", ext);
	}

	return false;
}


#define BAR_LENGTH 21

const char *progressbar[BAR_LENGTH]=
{
	"[                    ]",
	"[.                   ]",
	"[..                  ]",
	"[...                 ]",
	"[....                ]",
	"[.....               ]",
	"[......              ]",
	"[.......             ]",
	"[........            ]",
	"[.........           ]",
	"[..........          ]",
	"[...........         ]",
	"[............        ]",
	"[.............       ]",
	"[..............      ]",
	"[...............     ]",
	"[................    ]",
	"[.................   ]",
	"[..................  ]",
	"[................... ]",
	"[....................]"
};

void progress (const char *message, float progress)
{
	// Progress bar
	char msg[512];
	uint	pgId= (uint)(progress*(float)BAR_LENGTH);
	pgId= min(pgId, (uint)(BAR_LENGTH-1));
	sprintf (msg, "\r%s %s", progressbar[pgId], message );
	uint i;
	for (i=(uint)strlen(msg); i<79; i++)
		msg[i]=' ';
	msg[i]=0;
	printf ("%s", msg);
	printf ("\r");
}

int main(int argc, char* argv[])
{
	if (argc==4)
	{
		// Load the config file
		printf ("Loading config files...");

		// Extensions
		vector<string> extensions;

		// Extensions
		map<string, string> alternateExtensions;
		alternateExtensions.insert (map<string, string>::value_type (".tga", ".dds"));

		// String array
		set<string> inputFiles;

		// String array
		set<string> usedFiles;

		// String array
		map<string, string> availableFiles;

		if (loadConfigFiles (argv[1], argv[2], argv[3], extensions, inputFiles, availableFiles))
		{
			// Load the config file
			printf ("\rScaning files...                 \n");

			uint size = 0;
			uint maxSize = (uint)inputFiles.size();
			set<string>::iterator ite = inputFiles.begin();
			while (ite != inputFiles.end())
			{
				// Update
				string label = ite->c_str();

				// Remove directory
				removeDirectory (label);

				progress (label.c_str(), (float)size/(float)maxSize);

				// String array
				set<string> outputFilename;

				// File array
				vector<char> fileArray;

				// Read the file
				INFO ("Open file: %s", ite->c_str());
				if (readTheFile (ite->c_str(), fileArray))
				{
					// Is it a ASCII file ?
					if (isASCII (fileArray))
					{
						INFO ("ASCII mode");

						// Extract strings
						extractStringsFromASCII (fileArray, outputFilename, extensions);
					}
					else
					{
						INFO ("BINARY mode");

						// Extract strings
						extractStringsFromBinary (fileArray, outputFilename, extensions);
					}
				}

				// Check this files exists
				set<string>::iterator found = outputFilename.begin();
				while (found != outputFilename.end())
				{
					// Look for this file
					if (availableFiles.find (*found) == availableFiles.end())
					{
						// Found ?
						bool foundIt = false;

						// Try alternate extension
						string ext = getExtension ((*found).c_str());
						std::map<string, string>::iterator alternIte = alternateExtensions.find (ext);
						if (alternIte != alternateExtensions.end ())
						{
							string name = *found;
							removeExtension (name);
							name += alternIte->second;
							if (availableFiles.find (name) != availableFiles.end())
								foundIt = true;
						}

						// Not found ?
						if (!foundIt)
						{
							fprintf (stderr, "MISSING: %s (needed by file %s)\n", found->c_str(), ite->c_str());
						}
					}
					else
					{
						// Add to used files
						usedFiles.insert (*found);
					}

					// Next file found
					found++;
				}

				size++;

				// Next file
				ite++;
			}

			// Look for unused files
			map<string, string>::iterator available = availableFiles.begin();
			while (available != availableFiles.end())
			{
				// It is used ?
				if (usedFiles.find (available->first) == usedFiles.end())
				{
					string temp = toLower (available->second);
					fprintf (stderr, "UNUSED: %s\n", temp.c_str());
				}

				// Next
				available++;
			}

			// Ok
			return 1;
		}
	}
	else
	{
		printf ("extract_filename [extensions.txt] [inputFiles.txt] [availableFiles.txt]\n");
	}
	return 0;
}