From 3d48368de679631e1831b443aaccebae20a22937 Mon Sep 17 00:00:00 2001 From: Nimetu Date: Sun, 23 Apr 2017 20:01:35 +0300 Subject: [PATCH 1/3] Changed: Workaround for dancing trees under direct3d (issue #160) --HG-- branch : develop --- code/nel/src/3d/meshvp_wind_tree.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/nel/src/3d/meshvp_wind_tree.cpp b/code/nel/src/3d/meshvp_wind_tree.cpp index 1b81af721..ba27216f0 100644 --- a/code/nel/src/3d/meshvp_wind_tree.cpp +++ b/code/nel/src/3d/meshvp_wind_tree.cpp @@ -226,7 +226,8 @@ void CMeshVPWindTree::initVertexPrograms() { // setup of the VPLight fragment uint numPls= i/4; - bool normalize= (i&1)!=0; + // FIXME: normalize=true makes trees dance, workaround for issue #160 + bool normalize= false; //(i&1)!=0; bool specular= (i&2)!=0; // combine From 3d09dd209422020e42474727b953cf9b1ef05cb9 Mon Sep 17 00:00:00 2001 From: Nimetu Date: Thu, 20 Apr 2017 21:19:01 +0300 Subject: [PATCH 2/3] Changed: Save cookies from image request --HG-- branch : develop --- code/nel/include/nel/gui/group_html.h | 4 ++-- code/nel/src/gui/group_html.cpp | 27 +++++++++++++++++---------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/code/nel/include/nel/gui/group_html.h b/code/nel/include/nel/gui/group_html.h index 2dc5e80bc..ead873497 100644 --- a/code/nel/include/nel/gui/group_html.h +++ b/code/nel/include/nel/gui/group_html.h @@ -821,13 +821,13 @@ namespace NLGUI { public: CDataDownload(const std::string &u, const std::string &d, TDataType t, CViewBase *i, const std::string &s, const std::string &m, const CStyleParams &style = CStyleParams()) - : curl(NULL), fp(NULL), url(u), dest(d), type(t), luaScript(s), md5sum(m) + : data(NULL), fp(NULL), url(u), dest(d), type(t), luaScript(s), md5sum(m) { if (t == ImgType) imgs.push_back(CDataImageDownload(i, style)); } public: - CURL *curl; + CCurlWWWData *data; std::string url; std::string dest; std::string luaScript; diff --git a/code/nel/src/gui/group_html.cpp b/code/nel/src/gui/group_html.cpp index 319feae92..acc33357a 100644 --- a/code/nel/src/gui/group_html.cpp +++ b/code/nel/src/gui/group_html.cpp @@ -312,7 +312,7 @@ namespace NLGUI return false; } - download.curl = curl; + download.data = new CCurlWWWData(curl, download.url); download.fp = fp; curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true); @@ -322,6 +322,10 @@ namespace NLGUI curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + // catch headers + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback); + curl_easy_setopt(curl, CURLOPT_WRITEHEADER, download.data); + std::string userAgent = options.appName + "/" + options.appVersion; curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgent.c_str()); @@ -369,7 +373,7 @@ namespace NLGUI RunningCurls++; #ifdef LOG_DL - nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().curl, Curls.size()); + nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().data->Request, Curls.size()); } else { @@ -443,7 +447,7 @@ namespace NLGUI } RunningCurls++; #ifdef LOG_DL - nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().curl, Curls.size()); + nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().data->Request, Curls.size()); } else { @@ -580,18 +584,18 @@ namespace NLGUI for (vector::iterator it=Curls.begin(); iteasy_handle == it->curl) + if(it->data && it->data->Request == msg->easy_handle) { CURLcode res = msg->data.result; long r; - curl_easy_getinfo(it->curl, CURLINFO_RESPONSE_CODE, &r); + curl_easy_getinfo(it->data->Request, CURLINFO_RESPONSE_CODE, &r); fclose(it->fp); + receiveCookies(it->data->Request, _DocumentDomain, _TrustedDomain); #ifdef LOG_DL - nlwarning("(%s) transfer '%p' completed with status %d, http %d, url (len %d) '%s'", _Id.c_str(), it->curl, res, r, it->url.size(), it->url.c_str()); + nlwarning("(%s) transfer '%p' completed with status %d, http %d, url (len %d) '%s'", _Id.c_str(), it->data->Request, res, r, it->url.size(), it->url.c_str()); #endif - curl_multi_remove_handle(MultiCurl, it->curl); - curl_easy_cleanup(it->curl); + curl_multi_remove_handle(MultiCurl, it->data->Request); string tmpfile = it->dest + ".tmp"; if(res != CURLE_OK || r < 200 || r >= 300 || (!it->md5sum.empty() && (it->md5sum != getMD5(tmpfile).toString()))) @@ -642,6 +646,9 @@ namespace NLGUI } } + // release CCurlWWWData + delete it->data; + Curls.erase(it); break; } @@ -656,7 +663,7 @@ namespace NLGUI { for (vector::iterator it=Curls.begin(); itcurl == NULL) { + if (it->data == NULL) { #ifdef LOG_DL nlwarning("(%s) starting new download '%s'", _Id.c_str(), it->url.c_str()); #endif @@ -4575,7 +4582,7 @@ namespace NLGUI // remove download that are still queued for (vector::iterator it=Curls.begin(); itcurl == NULL) { + if (it->data == NULL) { #ifdef LOG_DL nlwarning("Remove waiting curl download (%s)", it->url.c_str()); #endif From 108759952e0dc7082d8fd8d220541bea8a55d2d1 Mon Sep 17 00:00:00 2001 From: Nimetu Date: Thu, 20 Apr 2017 21:19:51 +0300 Subject: [PATCH 3/3] Changed: Use http cache headers for image cache --HG-- branch : develop --- code/nel/include/nel/gui/http_cache.h | 77 ++++++++++ code/nel/src/gui/group_html.cpp | 91 ++++++++++-- code/nel/src/gui/http_cache.cpp | 200 ++++++++++++++++++++++++++ code/nel/src/gui/stdpch.h | 1 + code/ryzom/client/src/init.cpp | 5 + code/ryzom/client/src/release.cpp | 3 +- 6 files changed, 361 insertions(+), 16 deletions(-) create mode 100644 code/nel/include/nel/gui/http_cache.h create mode 100644 code/nel/src/gui/http_cache.cpp diff --git a/code/nel/include/nel/gui/http_cache.h b/code/nel/include/nel/gui/http_cache.h new file mode 100644 index 000000000..0921b2f64 --- /dev/null +++ b/code/nel/include/nel/gui/http_cache.h @@ -0,0 +1,77 @@ +// Ryzom - MMORPG Framework +// Copyright (C) 2010 Winch Gate Property Limited +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef CL_HTTP_CACHE_H +#define CL_HTTP_CACHE_H + +#include "nel/misc/types_nl.h" + +namespace NLGUI +{ + struct CHttpCacheObject + { + CHttpCacheObject(uint32 expires = 0, const std::string& lastModified = "", const std::string& etag = "") + : Expires(expires) + , LastModified(lastModified) + , Etag(etag){}; + + uint32 Expires; + std::string LastModified; + std::string Etag; + + void serial(NLMISC::IStream& f); + }; + + /** + * Keeping track of downloaded files cache related headers + * \author Meelis Mägi (nimetu) + * \date 2017 + */ + class CHttpCache + { + typedef std::map THttpCacheMap; + + public: + static CHttpCache* getInstance(); + static void release(); + + public: + void setCacheIndex(const std::string& fname); + void init(); + + CHttpCacheObject lookup(const std::string& fname); + void store(const std::string& fname, const CHttpCacheObject& data); + + void flushCache(); + + void serial(NLMISC::IStream& f); + + private: + CHttpCache(); + ~CHttpCache(); + + void pruneCache(); + + static CHttpCache* instance; + + THttpCacheMap _List; + + std::string _IndexFilename; + bool _Initialized; + size_t _MaxObjects; + }; +} +#endif diff --git a/code/nel/src/gui/group_html.cpp b/code/nel/src/gui/group_html.cpp index acc33357a..0c1fba8d0 100644 --- a/code/nel/src/gui/group_html.cpp +++ b/code/nel/src/gui/group_html.cpp @@ -45,6 +45,7 @@ #include "nel/3d/texture_file.h" #include "nel/misc/big_file.h" #include "nel/gui/url_parser.h" +#include "nel/gui/http_cache.h" using namespace std; using namespace NLMISC; @@ -87,6 +88,15 @@ namespace NLGUI curl_slist_free_all(HeadersSent); } + void sendHeaders(const std::vector headers) + { + for(uint i = 0; i < headers.size(); ++i) + { + HeadersSent = curl_slist_append(HeadersSent, headers[i].c_str()); + } + curl_easy_setopt(Request, CURLOPT_HTTPHEADER, HeadersSent); + } + void setRecvHeader(const std::string &header) { size_t pos = header.find(": "); @@ -110,12 +120,42 @@ namespace NLGUI return ""; } + const uint32 getExpires() + { + time_t ret = 0; + if (HeadersRecv.count("expires") > 0) + ret = curl_getdate(HeadersRecv["expires"].c_str(), NULL); + + return ret > -1 ? ret : 0; + } + + const std::string getLastModified() + { + if (HeadersRecv.count("last-modified") > 0) + { + return HeadersRecv["last-modified"]; + } + + return ""; + } + + const std::string getEtag() + { + if (HeadersRecv.count("etag") > 0) + { + return HeadersRecv["etag"]; + } + + return ""; + } + public: CURL *Request; std::string Url; std::string Content; + private: // headers sent with curl request, must be released after transfer curl_slist * HeadersSent; @@ -274,19 +314,16 @@ namespace NLGUI return false; } - // TODO: replace with expire and etag headers - if (CFile::fileExists(download.dest)) + time_t currentTime; + time(¤tTime); + + CHttpCacheObject cache = CHttpCache::getInstance()->lookup(download.dest); + if (cache.Expires > currentTime) { - time_t currentTime; - time(¤tTime); - uint32 mtime = CFile::getFileModificationDate(download.dest); - if (mtime + 3600 > currentTime) - { #ifdef LOG_DL - nlwarning("Cache for (%s) is not expired (%s, age:%d)", download.url.c_str(), download.dest.c_str(), currentTime - mtime); + nlwarning("Cache for (%s) is not expired (%s, expires:%d)", download.url.c_str(), download.dest.c_str(), cache.Expires - currentTime); #endif - return false; - } + return false; } string tmpdest = download.dest + ".tmp"; @@ -322,6 +359,16 @@ namespace NLGUI curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + std::vector headers; + if (!cache.Etag.empty()) + headers.push_back("If-None-Match: " + cache.Etag); + + if (!cache.LastModified.empty()) + headers.push_back("If-Modified-Since: " + cache.LastModified); + + if (headers.size() > 0) + download.data->sendHeaders(headers); + // catch headers curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback); curl_easy_setopt(curl, CURLOPT_WRITEHEADER, download.data); @@ -601,9 +648,27 @@ namespace NLGUI if(res != CURLE_OK || r < 200 || r >= 300 || (!it->md5sum.empty() && (it->md5sum != getMD5(tmpfile).toString()))) { NLMISC::CFile::deleteFile(tmpfile.c_str()); + + // 304 Not Modified + if (res == CURLE_OK && r == 304) + { + CHttpCacheObject obj; + obj.Expires = it->data->getExpires(); + obj.Etag = it->data->getEtag(); + obj.LastModified = it->data->getLastModified(); + + CHttpCache::getInstance()->store(it->dest, obj); + } } else { + CHttpCacheObject obj; + obj.Expires = it->data->getExpires(); + obj.Etag = it->data->getEtag(); + obj.LastModified = it->data->getLastModified(); + + CHttpCache::getInstance()->store(it->dest, obj); + string finalUrl; if (it->type == ImgType) { @@ -5179,11 +5244,7 @@ namespace NLGUI std::vector headers; headers.push_back("Accept-Language: "+options.languageCode); headers.push_back("Accept-Charset: utf-8"); - for(uint i=0; i< headers.size(); ++i) - { - _CurlWWW->HeadersSent = curl_slist_append(_CurlWWW->HeadersSent, headers[i].c_str()); - } - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, _CurlWWW->HeadersSent); + _CurlWWW->sendHeaders(headers); // catch headers for redirect curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback); diff --git a/code/nel/src/gui/http_cache.cpp b/code/nel/src/gui/http_cache.cpp new file mode 100644 index 000000000..f0839752c --- /dev/null +++ b/code/nel/src/gui/http_cache.cpp @@ -0,0 +1,200 @@ +// Ryzom - MMORPG Framework +// Copyright (C) 2010 Winch Gate Property Limited +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdpch.h" +#include "nel/gui/http_cache.h" + +using namespace std; +using namespace NLMISC; + +#ifdef DEBUG_NEW +#define new DEBUG_NEW +#endif + +namespace NLGUI +{ + CHttpCache* CHttpCache::instance = NULL; + + CHttpCache* CHttpCache::getInstance() + { + if (!instance) + { + instance = new CHttpCache(); + } + + return instance; + } + + void CHttpCache::release() + { + delete instance; + instance = NULL; + } + + CHttpCache::CHttpCache() + : _Initialized(false) + , _MaxObjects(100) + { }; + + CHttpCache::~CHttpCache() + { + flushCache(); + } + + void CHttpCache::setCacheIndex(const std::string& fname) + { + _IndexFilename = fname; + _Initialized = false; + } + + CHttpCacheObject CHttpCache::lookup(const std::string& fname) + { + if (!_Initialized) + init(); + + if (_List.count(fname) > 0) + return _List[fname]; + + return CHttpCacheObject(); + } + + void CHttpCache::store(const std::string& fname, const CHttpCacheObject& data) + { + if (!_Initialized) + init(); + + _List[fname] = data; + } + + void CHttpCache::init() + { + if (_Initialized) + return; + + _Initialized = true; + + if (_IndexFilename.empty() || !CFile::fileExists(_IndexFilename)) + return; + + CIFile in; + if (!in.open(_IndexFilename)) { + nlwarning("Unable to open %s for reading", _IndexFilename.c_str()); + return; + } + + serial(in); + } + + void CHttpCacheObject::serial(NLMISC::IStream& f) + { + f.serialVersion(1); + f.serial(Expires); + f.serial(LastModified); + f.serial(Etag); + } + + void CHttpCache::serial(NLMISC::IStream& f) + { + // saved state is ignored when version checks fail + try { + f.serialVersion(1); + + // CacheIdx + f.serialCheck(NELID("hcaC")); + f.serialCheck(NELID("xdIe")); + + if (f.isReading()) + { + uint32 numFiles; + f.serial(numFiles); + + _List.clear(); + for (uint k = 0; k < numFiles; ++k) + { + std::string fname; + f.serial(fname); + + CHttpCacheObject obj; + obj.serial(f); + + _List[fname] = obj; + } + } + else + { + uint32 numFiles = _List.size(); + f.serial(numFiles); + + for (THttpCacheMap::iterator it = _List.begin(); it != _List.end(); ++it) + { + std::string fname(it->first); + f.serial(fname); + + (*it).second.serial(f); + } + } + } catch (...) { + _List.clear(); + nlwarning("Invalid cache index format (%s)", _IndexFilename.c_str()); + return; + } + } + + void CHttpCache::pruneCache() + { + if (_List.size() < _MaxObjects) + return; + + size_t mustDrop = _List.size() - _MaxObjects; + + time_t currentTime; + time(¤tTime); + + // if we over object limit, then start removing expired objects + // this does not guarantee that max limit is reached + for (THttpCacheMap::iterator it = _List.begin(); it != _List.end();) { + if (it->second.Expires <= currentTime) { + it = _List.erase(it); + + --mustDrop; + if (mustDrop == 0) + break; + } + else + { + ++it; + } + } + } + + void CHttpCache::flushCache() + { + if (_IndexFilename.empty()) + return; + + pruneCache(); + + COFile out; + if (!out.open(_IndexFilename)) + { + nlwarning("Unable to open %s for writing", _IndexFilename.c_str()); + return; + } + + serial(out); + out.close(); + } +} diff --git a/code/nel/src/gui/stdpch.h b/code/nel/src/gui/stdpch.h index e0be5837e..a4ba0ecac 100644 --- a/code/nel/src/gui/stdpch.h +++ b/code/nel/src/gui/stdpch.h @@ -26,6 +26,7 @@ #include #include +#include #include "nel/misc/types_nl.h" #include "nel/misc/algo.h" diff --git a/code/ryzom/client/src/init.cpp b/code/ryzom/client/src/init.cpp index 267042b7e..900f3c2c1 100644 --- a/code/ryzom/client/src/init.cpp +++ b/code/ryzom/client/src/init.cpp @@ -68,6 +68,7 @@ #include "interface_v3/sbrick_manager.h" #include "nel/gui/widget_manager.h" +#include "nel/gui/http_cache.h" // #include "gabarit.h" #include "hair_set.h" @@ -1365,6 +1366,10 @@ void prelogInit() //nlinfo ("PROFILE: %d seconds for Add search paths Data", (uint32)(ryzomGetLocalTime ()-initPaths)/1000); } + // Initialize HTTP cache + CHttpCache::getInstance()->setCacheIndex("cache/cache.index"); + CHttpCache::getInstance()->init(); + // Register the reflected classes registerInterfaceElements(); diff --git a/code/ryzom/client/src/release.cpp b/code/ryzom/client/src/release.cpp index 689f0bf56..a1f3a29c2 100644 --- a/code/ryzom/client/src/release.cpp +++ b/code/ryzom/client/src/release.cpp @@ -94,7 +94,7 @@ #include "bg_downloader_access.h" #include "nel/gui/lua_manager.h" #include "item_group_manager.h" - +#include "nel/gui/http_cache.h" /////////// // USING // @@ -687,6 +687,7 @@ void release() CWidgetManager::release(); CViewRenderer::release(); CIXml::releaseLibXml(); + CHttpCache::release(); #if FINAL_VERSION // openURL ("http://www.ryzomcore.org/exit/");