Changed: Use http cache headers for image cache

--HG--
branch : develop

code/nel/include/nel/gui/http_cache.h

@@ -0,0 +1,77 @@
+// Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
+// Copyright (C) 2010  Winch Gate Property Limited
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#ifndef CL_HTTP_CACHE_H
+#define CL_HTTP_CACHE_H
+
+#include "nel/misc/types_nl.h"
+
+namespace NLGUI
+{
+	struct CHttpCacheObject
+	{
+		CHttpCacheObject(uint32 expires = 0, const std::string& lastModified = "", const std::string& etag = "")
+			: Expires(expires)
+			, LastModified(lastModified)
+			, Etag(etag){};
+
+		uint32 Expires;
+		std::string LastModified;
+		std::string Etag;
+
+		void serial(NLMISC::IStream& f);
+	};
+
+	/**
+	 * Keeping track of downloaded files cache related headers
+	 * \author Meelis Mägi (nimetu)
+	 * \date 2017
+	 */
+	class CHttpCache
+	{
+		typedef std::map<std::string, CHttpCacheObject> THttpCacheMap;
+
+	public:
+		static CHttpCache* getInstance();
+		static void release();
+
+	public:
+		void setCacheIndex(const std::string& fname);
+		void init();
+
+		CHttpCacheObject lookup(const std::string& fname);
+		void store(const std::string& fname, const CHttpCacheObject& data);
+
+		void flushCache();
+
+		void serial(NLMISC::IStream& f);
+
+	private:
+		CHttpCache();
+		~CHttpCache();
+
+		void pruneCache();
+
+		static CHttpCache* instance;
+
+		THttpCacheMap _List;
+
+		std::string _IndexFilename;
+		bool _Initialized;
+		size_t _MaxObjects;
+	};
+}
+#endif

code/nel/src/gui/group_html.cpp

@@ -45,6 +45,7 @@
 #include "nel/3d/texture_file.h"
 #include "nel/misc/big_file.h"
 #include "nel/gui/url_parser.h"
+#include "nel/gui/http_cache.h"
 
 using namespace std;
 using namespace NLMISC;
@@ -87,6 +88,15 @@ namespace NLGUI
 					curl_slist_free_all(HeadersSent);
 			}
 
+			void sendHeaders(const std::vector<std::string> headers)
+			{
+				for(uint i = 0; i < headers.size(); ++i)
+				{
+					HeadersSent = curl_slist_append(HeadersSent, headers[i].c_str());
+				}
+				curl_easy_setopt(Request, CURLOPT_HTTPHEADER, HeadersSent);
+			}
+
 			void setRecvHeader(const std::string &header)
 			{
 				size_t pos = header.find(": ");
@@ -110,12 +120,42 @@ namespace NLGUI
 				return "";
 			}
 
+			const uint32 getExpires()
+			{
+				time_t ret = 0;
+				if (HeadersRecv.count("expires") > 0)
+					ret = curl_getdate(HeadersRecv["expires"].c_str(), NULL);
+
+				return ret > -1 ? ret : 0;
+			}
+
+			const std::string getLastModified()
+			{
+				if (HeadersRecv.count("last-modified") > 0)
+				{
+					return HeadersRecv["last-modified"];
+				}
+
+				return "";
+			}
+
+			const std::string getEtag()
+			{
+				if (HeadersRecv.count("etag") > 0)
+				{
+					return HeadersRecv["etag"];
+				}
+
+				return "";
+			}
+
 		public:
 			CURL *Request;
 
 			std::string Url;
 			std::string Content;
 
+		private:
 			// headers sent with curl request, must be released after transfer
 			curl_slist * HeadersSent;
 
@@ -274,19 +314,16 @@ namespace NLGUI
 			return false;
 		}
 
-		// TODO: replace with expire and etag headers
+		time_t currentTime;
-		if (CFile::fileExists(download.dest))
+		time(&currentTime);
+
+		CHttpCacheObject cache = CHttpCache::getInstance()->lookup(download.dest);
+		if (cache.Expires > currentTime)
 		{
-			time_t currentTime;
-			time(&currentTime);
-			uint32 mtime = CFile::getFileModificationDate(download.dest);
-			if (mtime + 3600 > currentTime)
-			{
 	#ifdef LOG_DL
-				nlwarning("Cache for (%s) is not expired (%s, age:%d)", download.url.c_str(), download.dest.c_str(), currentTime - mtime);
+			nlwarning("Cache for (%s) is not expired (%s, expires:%d)", download.url.c_str(), download.dest.c_str(), cache.Expires - currentTime);
 	#endif
-				return false;
+			return false;
-			}
 		}
 
 		string tmpdest = download.dest + ".tmp";
@@ -322,6 +359,16 @@ namespace NLGUI
 		curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
 		curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
 
+		std::vector<std::string> headers;
+		if (!cache.Etag.empty())
+			headers.push_back("If-None-Match: " + cache.Etag);
+
+		if (!cache.LastModified.empty())
+			headers.push_back("If-Modified-Since: " + cache.LastModified);
+
+		if (headers.size() > 0)
+			download.data->sendHeaders(headers);
+
 		// catch headers
 		curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback);
 		curl_easy_setopt(curl, CURLOPT_WRITEHEADER, download.data);
@@ -601,9 +648,27 @@ namespace NLGUI
 							if(res != CURLE_OK || r < 200 || r >= 300 || (!it->md5sum.empty() && (it->md5sum != getMD5(tmpfile).toString())))
 							{
 								NLMISC::CFile::deleteFile(tmpfile.c_str());
+
+								// 304 Not Modified
+								if (res == CURLE_OK && r == 304)
+								{
+									CHttpCacheObject obj;
+									obj.Expires = it->data->getExpires();
+									obj.Etag = it->data->getEtag();
+									obj.LastModified = it->data->getLastModified();
+
+									CHttpCache::getInstance()->store(it->dest, obj);
+								}
 							}
 							else
 							{
+								CHttpCacheObject obj;
+								obj.Expires = it->data->getExpires();
+								obj.Etag = it->data->getEtag();
+								obj.LastModified = it->data->getLastModified();
+
+								CHttpCache::getInstance()->store(it->dest, obj);
+
 								string finalUrl;
 								if (it->type == ImgType)
 								{
@@ -5179,11 +5244,7 @@ namespace NLGUI
 		std::vector<std::string> headers;
 		headers.push_back("Accept-Language: "+options.languageCode);
 		headers.push_back("Accept-Charset: utf-8");
-		for(uint i=0; i< headers.size(); ++i)
+		_CurlWWW->sendHeaders(headers);
-		{
-			_CurlWWW->HeadersSent = curl_slist_append(_CurlWWW->HeadersSent, headers[i].c_str());
-		}
-		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, _CurlWWW->HeadersSent);
 
 		// catch headers for redirect
 		curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback);

code/nel/src/gui/http_cache.cpp

@@ -0,0 +1,200 @@
+// Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
+// Copyright (C) 2010  Winch Gate Property Limited
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "stdpch.h"
+#include "nel/gui/http_cache.h"
+
+using namespace std;
+using namespace NLMISC;
+
+#ifdef DEBUG_NEW
+#define new DEBUG_NEW
+#endif
+
+namespace NLGUI
+{
+	CHttpCache* CHttpCache::instance = NULL;
+
+	CHttpCache* CHttpCache::getInstance()
+	{
+		if (!instance)
+		{
+			instance = new CHttpCache();
+		}
+
+		return instance;
+	}
+
+	void CHttpCache::release()
+	{
+		delete instance;
+		instance = NULL;
+	}
+
+	CHttpCache::CHttpCache()
+		: _Initialized(false)
+		, _MaxObjects(100)
+	{ };
+
+	CHttpCache::~CHttpCache()
+	{
+		flushCache();
+	}
+
+	void CHttpCache::setCacheIndex(const std::string& fname)
+	{
+		_IndexFilename = fname;
+		_Initialized = false;
+	}
+
+	CHttpCacheObject CHttpCache::lookup(const std::string& fname)
+	{
+		if (!_Initialized)
+			init();
+
+		if (_List.count(fname) > 0)
+			return _List[fname];
+
+		return CHttpCacheObject();
+	}
+
+	void CHttpCache::store(const std::string& fname, const CHttpCacheObject& data)
+	{
+		if (!_Initialized)
+			init();
+
+		_List[fname] = data;
+	}
+
+	void CHttpCache::init()
+	{
+		if (_Initialized)
+			return;
+
+		_Initialized = true;
+
+		if (_IndexFilename.empty() || !CFile::fileExists(_IndexFilename))
+			return;
+
+		CIFile in;
+		if (!in.open(_IndexFilename)) {
+			nlwarning("Unable to open %s for reading", _IndexFilename.c_str());
+			return;
+		}
+
+		serial(in);
+	}
+
+	void CHttpCacheObject::serial(NLMISC::IStream& f)
+	{
+		f.serialVersion(1);
+		f.serial(Expires);
+		f.serial(LastModified);
+		f.serial(Etag);
+	}
+
+	void CHttpCache::serial(NLMISC::IStream& f)
+	{
+		// saved state is ignored when version checks fail
+		try {
+			f.serialVersion(1);
+
+			// CacheIdx
+			f.serialCheck(NELID("hcaC"));
+			f.serialCheck(NELID("xdIe"));
+
+			if (f.isReading())
+			{
+				uint32 numFiles;
+				f.serial(numFiles);
+
+				_List.clear();
+				for (uint k = 0; k < numFiles; ++k)
+				{
+					std::string fname;
+					f.serial(fname);
+
+					CHttpCacheObject obj;
+					obj.serial(f);
+
+					_List[fname] = obj;
+				}
+			}
+			else
+			{
+				uint32 numFiles = _List.size();
+				f.serial(numFiles);
+
+				for (THttpCacheMap::iterator it = _List.begin(); it != _List.end(); ++it)
+				{
+					std::string fname(it->first);
+					f.serial(fname);
+
+					(*it).second.serial(f);
+				}
+			}
+		} catch (...) {
+			_List.clear();
+			nlwarning("Invalid cache index format (%s)", _IndexFilename.c_str());
+			return;
+		}
+	}
+
+	void CHttpCache::pruneCache()
+	{
+		if (_List.size() < _MaxObjects)
+			return;
+
+		size_t mustDrop = _List.size() - _MaxObjects;
+
+		time_t currentTime;
+		time(&currentTime);
+
+		// if we over object limit, then start removing expired objects
+		// this does not guarantee that max limit is reached
+		for (THttpCacheMap::iterator it = _List.begin(); it != _List.end();) {
+			if (it->second.Expires <= currentTime) {
+				it = _List.erase(it);
+
+				--mustDrop;
+				if (mustDrop == 0)
+					break;
+			}
+			else
+			{
+				++it;
+			}
+		}
+	}
+
+	void CHttpCache::flushCache()
+	{
+		if (_IndexFilename.empty())
+			return;
+
+		pruneCache();
+
+		COFile out;
+		if (!out.open(_IndexFilename))
+		{
+			nlwarning("Unable to open %s for writing", _IndexFilename.c_str());
+			return;
+		}
+
+		serial(out);
+		out.close();
+	}
+}

code/nel/src/gui/stdpch.h

@@ -26,6 +26,7 @@
 
 #include <string>
 #include <limits>
+#include <ctime>
 
 #include "nel/misc/types_nl.h"
 #include "nel/misc/algo.h"

code/ryzom/client/src/init.cpp

@@ -68,6 +68,7 @@
 
 #include "interface_v3/sbrick_manager.h"
 #include "nel/gui/widget_manager.h"
+#include "nel/gui/http_cache.h"
 //
 #include "gabarit.h"
 #include "hair_set.h"
@@ -1365,6 +1366,10 @@ void prelogInit()
 			//nlinfo ("PROFILE: %d seconds for Add search paths Data", (uint32)(ryzomGetLocalTime ()-initPaths)/1000);
 		}
 
+		// Initialize HTTP cache
+		CHttpCache::getInstance()->setCacheIndex("cache/cache.index");
+		CHttpCache::getInstance()->init();
+
 		// Register the reflected classes
 		registerInterfaceElements();
 

code/ryzom/client/src/release.cpp

@@ -94,7 +94,7 @@
 #include "bg_downloader_access.h"
 #include "nel/gui/lua_manager.h"
 #include "item_group_manager.h"
-
+#include "nel/gui/http_cache.h"
 
 ///////////
 // USING //
@@ -687,6 +687,7 @@ void release()
 	CWidgetManager::release();
 	CViewRenderer::release();
 	CIXml::releaseLibXml();
+	CHttpCache::release();
 
 #if FINAL_VERSION
 	// openURL ("http://www.ryzomcore.org/exit/");