From 8e1ada36b0e4d39dd5c0d3a981bc75c9f3b73f6e Mon Sep 17 00:00:00 2001 From: Nimetu Date: Fri, 17 Apr 2015 17:41:01 +0300 Subject: [PATCH] Use cURL as http transport --HG-- branch : develop --- code/nel/include/nel/gui/group_html.h | 56 +- code/nel/include/nel/gui/libwww.h | 10 + code/nel/src/gui/group_html.cpp | 609 ++++++++++++++++-- code/nel/src/gui/libwww.cpp | 137 +++- .../client/src/interface_v3/group_html_cs.cpp | 4 +- .../client/src/interface_v3/group_html_cs.h | 2 +- .../src/interface_v3/group_html_forum.cpp | 10 +- .../src/interface_v3/group_html_forum.h | 2 +- .../src/interface_v3/group_html_mail.cpp | 10 +- .../client/src/interface_v3/group_html_mail.h | 2 +- .../src/interface_v3/group_html_webig.cpp | 16 +- .../src/interface_v3/group_html_webig.h | 4 +- 12 files changed, 790 insertions(+), 72 deletions(-) diff --git a/code/nel/include/nel/gui/group_html.h b/code/nel/include/nel/gui/group_html.h index 76caa2caa..d7c1cd30a 100644 --- a/code/nel/include/nel/gui/group_html.h +++ b/code/nel/include/nel/gui/group_html.h @@ -25,6 +25,7 @@ #include "nel/gui/group_tree.h" #include "nel/gui/ctrl_button.h" #include "nel/gui/group_table.h" +#include "nel/gui/libwww_types.h" typedef std::map TStyle; @@ -36,7 +37,8 @@ namespace NLGUI class CDBGroupComboBox; class CGroupParagraph; - + extern std::string CurrentCookie; + extern std::string HTTPCurrentDomain; // HTML group /** @@ -164,6 +166,34 @@ namespace NLGUI std::string DefaultBackgroundBitmapView; std::string CurrentLinkTitle; + struct TFormField { + public: + TFormField(const std::string &k, const std::string &v) + :name(k),value(v) + {} + std::string name; + std::string value; + }; + + struct SFormFields { + public: + SFormFields() + { + } + + void clear() + { + Values.clear(); + } + + void add(const std::string &key, const std::string &value) + { + Values.push_back(TFormField(key, value)); + } + + std::vector Values; + }; + // Browser home std::string Home; @@ -237,10 +267,10 @@ namespace NLGUI virtual void addHTTPGetParams (std::string &url, bool trustedDomain); // Add POST params to the libwww list - virtual void addHTTPPostParams (HTAssocList *formfields, bool trustedDomain); + virtual void addHTTPPostParams (SFormFields &formfields, bool trustedDomain); // the current request is terminated - virtual void requestTerminated(HTRequest *request); + virtual void requestTerminated(); // libxml2 html parser functions void htmlElement(xmlNode *node, int element_number); @@ -333,6 +363,7 @@ namespace NLGUI bool _Connecting; double _TimeoutValue; // the timeout in seconds double _ConnectingTimeout; + uint32 _RedirectsRemaining; // minimal embeded lua script support // Note : any embeded script is executed immediately after the closing @@ -346,6 +377,9 @@ namespace NLGUI bool _Object; std::string _ObjectScript; + // Data container for active curl transfer + class CCurlWWWData * _CurlWWW; + // Current paragraph std::string _DivName; CGroupParagraph* _Paragraph; @@ -660,9 +694,15 @@ namespace NLGUI // load and render local html file (from bnp for example) void doBrowseLocalFile(const std::string &filename); + // load remote content using either GET or POST + void doBrowseRemoteUrl(const std::string &url, const std::string &referer, bool doPost = false, const SFormFields &formfields = SFormFields()); + // render html string as new browser page bool renderHtmlString(const std::string &html); + // initialize formfields list from form elements on page + void buildHTTPPostParams (SFormFields &formfields); + private: // decode all HTML entities static ucstring decodeHTMLEntities(const ucstring &str); @@ -672,11 +712,13 @@ namespace NLGUI struct CDataDownload { + public: CDataDownload(CURL *c, const std::string &u, FILE *f, TDataType t, CViewBase *i, const std::string &s, const std::string &m) : curl(c), url(u), luaScript(s), md5sum(m), type(t), fp(f) { if (t == ImgType) imgs.push_back(i); } + public: CURL *curl; std::string url; std::string luaScript; @@ -708,6 +750,13 @@ namespace NLGUI void releaseDownloads(); void checkDownloads(); + // HtmlType download finished + void htmlDownloadFinished(const std::string &content, const std::string &type, long code); + + // cURL transfer callbacks + static size_t curlHeaderCallback(char *buffer, size_t size, size_t nmemb, void *pCCurlWWWData); + static size_t curlDataCallback(char *buffer, size_t size, size_t nmemb, void *pCCurlWWWData); + static size_t curlProgressCallback(void *pCCurlWWWData, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow); }; // adapter group that store y offset for inputs inside an html form @@ -721,7 +770,6 @@ namespace NLGUI xmlNodePtr serialize( xmlNodePtr parentNode, const char *type ) const; virtual bool parse (xmlNodePtr cur, CInterfaceGroup *parentGroup); }; - } #endif diff --git a/code/nel/include/nel/gui/libwww.h b/code/nel/include/nel/gui/libwww.h index d3e4eeec9..be3d02242 100644 --- a/code/nel/include/nel/gui/libwww.h +++ b/code/nel/include/nel/gui/libwww.h @@ -20,7 +20,10 @@ #ifndef CL_LIB_WWW_H #define CL_LIB_WWW_H +#include + #include "nel/misc/rgba.h" +#include "nel/gui/libwww_types.h" namespace NLGUI { @@ -30,6 +33,9 @@ namespace NLGUI // *************************************************************************** + // Legacy function from libwww + SGML_dtd * HTML_dtd (void); + // Init the libwww void initLibWWW(); @@ -230,6 +236,10 @@ namespace NLGUI // *************************************************************************** + const std::string &setCurrentDomain(const std::string &uri); + void receiveCookies (CURL *curl, const std::string &domain, bool trusted); + void sendCookies(CURL *curl, const std::string &domain, bool trusted); + } #endif diff --git a/code/nel/src/gui/group_html.cpp b/code/nel/src/gui/group_html.cpp index d99d706b1..5d3ae2570 100644 --- a/code/nel/src/gui/group_html.cpp +++ b/code/nel/src/gui/group_html.cpp @@ -43,13 +43,16 @@ #include "nel/misc/md5.h" #include "nel/3d/texture_file.h" #include "nel/misc/big_file.h" +#include using namespace std; using namespace NLMISC; // Default timeout to connect a server -#define DEFAULT_RYZOM_CONNECTION_TIMEOUT (10.0) +#define DEFAULT_RYZOM_CONNECTION_TIMEOUT (30.0) +// Allow up to 10 redirects, then give up +#define DEFAULT_RYZOM_REDIRECT_LIMIT (10) namespace NLGUI { @@ -60,6 +63,59 @@ namespace NLGUI CGroupHTML::SWebOptions CGroupHTML::options; + // Active cURL www transfer + class CCurlWWWData + { + public: + CCurlWWWData(CURL *curl, const std::string &url) + : Request(curl), Url(url), Content(""), HeadersSent(NULL) + { + } + ~CCurlWWWData() + { + if (Request) + curl_easy_cleanup(Request); + + if (HeadersSent) + curl_slist_free_all(HeadersSent); + } + + void setRecvHeader(const std::string &header) + { + size_t pos = header.find(": "); + if (pos == std::string::npos) + return; + + std::string key = toLower(header.substr(0, pos)); + if (pos != std::string::npos) + { + HeadersRecv[key] = header.substr(pos + 2); + //nlinfo(">> received header '%s' = '%s'", key.c_str(), HeadersRecv[key].c_str()); + } + } + + // return last received "Location: " header or empty string if no header set + const std::string getLocationHeader() + { + if (HeadersRecv.count("location") > 0) + return HeadersRecv["location"]; + + return ""; + } + + public: + CURL *Request; + + std::string Url; + std::string Content; + + // headers sent with curl request, must be released after transfer + curl_slist * HeadersSent; + + // headers received from curl transfer + std::map HeadersRecv; + }; + // Check if domain is on TrustedDomain bool CGroupHTML::isTrustedDomain(const string &domain) { @@ -285,7 +341,7 @@ namespace NLGUI { //nlassert(_CrtCheckMemory()); - if(RunningCurls == 0) + if(Curls.empty() && _CurlWWW == NULL) return; int NewRunningCurls = 0; @@ -306,8 +362,82 @@ namespace NLGUI int msgs_left; while ((msg = curl_multi_info_read(MultiCurl, &msgs_left))) { + #ifdef LOG_DL + nlwarning("> (%s) msgs_left %d", _Id.c_str(), msgs_left); + #endif if (msg->msg == CURLMSG_DONE) { + if (_CurlWWW && _CurlWWW->Request && _CurlWWW->Request == msg->easy_handle) + { + CURLcode res = msg->data.result; + long code; + curl_easy_getinfo(_CurlWWW->Request, CURLINFO_RESPONSE_CODE, &code); + #ifdef LOG_DL + nlwarning("(%s) web transfer '%p' completed with status %d, http %d, url (len %d) '%s'", _Id.c_str(), _CurlWWW->Request, res, code, _CurlWWW->Url.size(), _CurlWWW->Url.c_str()); + #endif + + if (res != CURLE_OK) + { + browseError(string("Connection failed with curl error " + string(curl_easy_strerror(res))).c_str()); + } + else + if ((code >= 301 && code <= 303) || code == 307 || code == 308) + { + if (_RedirectsRemaining < 0) + { + browseError(string("Redirect limit reached : " + _URL).c_str()); + } + else + { + // redirect, get the location and try browse again + // we cant use curl redirection because 'addHTTPGetParams()' must be called on new destination + std::string location(_CurlWWW->getLocationHeader()); + if (location.size() > 0) + { + #ifdef LOG_DL + nlwarning("(%s) request (%d) redirected to (len %d) '%s'", _Id.c_str(), _RedirectsRemaining, location.size(), location.c_str()); + #endif + location = getAbsoluteUrl(location); + // throw away this handle and start with new one (easier than reusing) + requestTerminated(); + + _PostNextTime = false; + _RedirectsRemaining--; + + doBrowse(location.c_str()); + } + else + { + browseError(string("Request was redirected, but location was not set : "+_URL).c_str()); + } + } + } + else + { + _RedirectsRemaining = DEFAULT_RYZOM_REDIRECT_LIMIT; + + if ( (code < 200 || code >= 300) ) + { + browseError(string("Connection failed (curl code " + toString((sint32)res) + "), http code " + toString(code) + ") : " + _CurlWWW->Url).c_str()); + } + else + { + receiveCookies(_CurlWWW->Request, HTTPCurrentDomain, _TrustedDomain); + + char *ch; + std::string contentType; + res = curl_easy_getinfo(_CurlWWW->Request, CURLINFO_CONTENT_TYPE, &ch); + if (res == CURLE_OK) + { + contentType = ch; + } + + htmlDownloadFinished(_CurlWWW->Content, contentType, code); + } + requestTerminated(); + } + } + for (vector::iterator it=Curls.begin(); iteasy_handle == it->curl) @@ -318,8 +448,9 @@ namespace NLGUI fclose(it->fp); #ifdef LOG_DL - nlwarning("transfer %x completed with status res %d r %d - %d curls", msg->easy_handle, res, r, Curls.size()); + nlwarning("(%s) transfer '%p' completed with status %d, http %d, url (len %d) '%s'", _Id.c_str(), it->curl, res, r, it->url.size(), it->url.c_str()); #endif + curl_multi_remove_handle(MultiCurl, it->curl); curl_easy_cleanup(it->curl); string file; @@ -342,11 +473,6 @@ namespace NLGUI { for(uint i = 0; i < it->imgs.size(); i++) { - // don't display image that are not power of 2 - //uint32 w, h; - //CBitmap::loadSize (file, w, h); - //if (w == 0 || h == 0 || ((!NLMISC::isPowerOf2(w) || !NLMISC::isPowerOf2(h)) && !NL3D::CTextureFile::supportNonPowerOfTwoTextures())) - // file.clear(); setImage(it->imgs[i], file); } } @@ -360,6 +486,7 @@ namespace NLGUI } } } + Curls.erase(it); break; } @@ -368,6 +495,10 @@ namespace NLGUI } } RunningCurls = NewRunningCurls; + #ifdef LOG_DL + if (RunningCurls > 0 || Curls.size() > 0) + nlwarning("(%s) RunningCurls %d, _Curls %d", _Id.c_str(), RunningCurls, Curls.size()); + #endif } @@ -436,11 +567,12 @@ namespace NLGUI { if (_Browsing) { - nlassert (_Connecting); _Connecting = false; removeContent (); } + else + nlwarning("_Browsing = FALSE"); } @@ -1043,12 +1175,10 @@ namespace NLGUI if (present[HTML_FORM_ACTION] && value[HTML_FORM_ACTION]) { form.Action = getAbsoluteUrl(string(value[HTML_FORM_ACTION])); - nlinfo("(%s) form.action '%s' (converted)", _Id.c_str(), form.Action.c_str()); } else { form.Action = _URL; - nlinfo("(%s) using _URL for form.action '%s'", _Id.c_str(), form.Action.c_str()); } _Forms.push_back(form); } @@ -1243,7 +1373,16 @@ namespace NLGUI // Translate the tooltip if (tooltip) - ctrlButton->setDefaultContextHelp (CI18N::get (tooltip)); + { + if (CI18N::hasTranslation(tooltip)) + { + ctrlButton->setDefaultContextHelp(CI18N::get(tooltip)); + } + else + { + ctrlButton->setDefaultContextHelp(ucstring(tooltip)); + } + } ctrlButton->setText(ucstring::makeFromUtf8(text)); } @@ -1336,24 +1475,11 @@ namespace NLGUI if (!(_Forms.empty())) { // A select box - - // read general property - string templateName; - string minWidth; - - // Widget template name - if (present[MY_HTML_INPUT_Z_INPUT_TMPL] && value[MY_HTML_INPUT_Z_INPUT_TMPL]) - templateName = value[MY_HTML_INPUT_Z_INPUT_TMPL]; - // Widget minimal width - if (present[MY_HTML_INPUT_Z_INPUT_WIDTH] && value[MY_HTML_INPUT_Z_INPUT_WIDTH]) - minWidth = value[MY_HTML_INPUT_Z_INPUT_WIDTH]; - string name; if (present[HTML_SELECT_NAME] && value[HTML_SELECT_NAME]) name = value[HTML_SELECT_NAME]; - string formTemplate = templateName.empty() ? DefaultFormSelectGroup : templateName; - CDBGroupComboBox *cb = addComboBox(formTemplate, name.c_str()); + CDBGroupComboBox *cb = addComboBox(DefaultFormSelectGroup, name.c_str()); CGroupHTML::CForm::CEntry entry; entry.Name = name; entry.ComboBox = cb; @@ -1824,7 +1950,8 @@ namespace NLGUI // *************************************************************************** CGroupHTML::CGroupHTML(const TCtorParam ¶m) : CGroupScrollText(param), - _TimeoutValue(DEFAULT_RYZOM_CONNECTION_TIMEOUT) + _TimeoutValue(DEFAULT_RYZOM_CONNECTION_TIMEOUT), + _RedirectsRemaining(DEFAULT_RYZOM_REDIRECT_LIMIT) { // add it to map of group html created _GroupHtmlUID= ++_GroupHtmlUIDPool; // valid assigned Id begin to 1! @@ -1894,9 +2021,11 @@ namespace NLGUI MultiCurl = curl_multi_init(); RunningCurls = 0; + _CurlWWW = NULL; initImageDownload(); initBnpDownload(); + initLibWWW(); } // *************************************************************************** @@ -1921,6 +2050,8 @@ namespace NLGUI // this is why the call to 'updateRefreshButton' has been removed from stopBrowse clearContext(); + if (_CurlWWW) + delete _CurlWWW; } std::string CGroupHTML::getProperty( const std::string &name ) const @@ -2849,20 +2980,20 @@ namespace NLGUI clearContext(); _Browsing = false; - _Connecting = false; updateRefreshButton(); #ifdef LOG_DL - nlwarning("*** ALREADY BROWSING, break first"); + nlwarning("(%s) *** ALREADY BROWSING, break first", _Id.c_str()); #endif } #ifdef LOG_DL - nlwarning("Browsing URL : '%s'", url); + nlwarning("(%s) Browsing URL : '%s'", _Id.c_str(), url); #endif // go _URL = url; + _Connecting = false; _BrowseNextTime = true; // if a BrowseTree is bound to us, try to select the node that opens this URL (auto-locate) @@ -2910,13 +3041,15 @@ namespace NLGUI void CGroupHTML::stopBrowse () { #ifdef LOG_DL - nlwarning("*** STOP BROWSE"); + nlwarning("*** STOP BROWSE (%s)", _Id.c_str()); #endif // Clear all the context clearContext(); _Browsing = false; + + requestTerminated(); } // *************************************************************************** @@ -3539,7 +3672,6 @@ namespace NLGUI p->setTopSpace(beginSpace); else group->setY(-(sint32)beginSpace); - parentGroup->addGroup (group); } @@ -3703,17 +3835,64 @@ namespace NLGUI const CWidgetManager::SInterfaceTimes × = CWidgetManager::getInstance()->getInterfaceTimes(); + // handle curl downloads + checkDownloads(); + if (_Connecting) { // Check timeout if needed if (_TimeoutValue != 0 && _ConnectingTimeout <= ( times.thisFrameMs / 1000.0f ) ) { browseError(("Connection timeout : "+_URL).c_str()); + + _Connecting = false; } } else if (_BrowseNextTime || _PostNextTime) { + // Set timeout + _Connecting = true; + _ConnectingTimeout = ( times.thisFrameMs / 1000.0f ) + _TimeoutValue; + + // freeze form buttons + CButtonFreezer freezer; + this->visit(&freezer); + + // Home ? + if (_URL == "home") + _URL = home(); + + string finalUrl; + bool isLocal = lookupLocalFile (finalUrl, _URL.c_str(), true); + + // Save new url + _URL = finalUrl; + + // file is probably from bnp (ingame help) + if (isLocal) + { + doBrowseLocalFile(finalUrl); + } + else + { + _TrustedDomain = isTrustedDomain(setCurrentDomain(finalUrl)); + + SFormFields formfields; + if (_PostNextTime) + { + buildHTTPPostParams(formfields); + // _URL is set from form.Action + finalUrl = _URL; + } + else + { + // Add custom get params from child classes + addHTTPGetParams (finalUrl, _TrustedDomain); + } + + doBrowseRemoteUrl(finalUrl, "", _PostNextTime, formfields); + } _BrowseNextTime = false; _PostNextTime = false; @@ -3721,8 +3900,118 @@ namespace NLGUI } // *************************************************************************** - void CGroupHTML::doBrowseLocalFile(const std::string &filename) + void CGroupHTML::buildHTTPPostParams (SFormFields &formfields) { + // Add text area text + uint i; + + if (_PostFormId >= _Forms.size()) + { + nlwarning("(%s) invalid form index %d, _Forms %d", _Id.c_str(), _PostFormId, _Forms.size()); + return; + } + // Ref the form + CForm &form = _Forms[_PostFormId]; + + // Save new url + _URL = form.Action; + _TrustedDomain = isTrustedDomain(setCurrentDomain(_URL)); + + for (i=0; igetGroup ("eb"); + if (group) + { + // Should be a CGroupEditBox + CGroupEditBox *editBox = dynamic_cast(group); + if (editBox) + { + entryData = editBox->getViewText()->getText(); + addEntry = true; + } + } + } + else if (form.Entries[i].Checkbox) + { + // todo handle unicode POST here + if (form.Entries[i].Checkbox->getPushed ()) + { + entryData = ucstring ("on"); + addEntry = true; + } + } + else if (form.Entries[i].ComboBox) + { + CDBGroupComboBox *cb = form.Entries[i].ComboBox; + entryData.fromUtf8(form.Entries[i].SelectValues[cb->getSelection()]); + addEntry = true; + } + // This is a hidden value + else + { + entryData = form.Entries[i].Value; + addEntry = true; + } + + // Add this entry + if (addEntry) + { + formfields.add(form.Entries[i].Name, CI18N::encodeUTF8(entryData)); + } + } + + if (_PostFormSubmitType == "image") + { + // Add the button coordinates + if (_PostFormSubmitButton.find_first_of("[") == string::npos) + { + formfields.add(_PostFormSubmitButton + "_x", NLMISC::toString(_PostFormSubmitX)); + formfields.add(_PostFormSubmitButton + "_y", NLMISC::toString(_PostFormSubmitY)); + } + else + { + formfields.add(_PostFormSubmitButton, NLMISC::toString(_PostFormSubmitX)); + formfields.add(_PostFormSubmitButton, NLMISC::toString(_PostFormSubmitY)); + } + } + else + formfields.add(_PostFormSubmitButton, _PostFormSubmitValue); + + // Add custom params from child classes + addHTTPPostParams(formfields, _TrustedDomain); + } + + // *************************************************************************** + void CGroupHTML::doBrowseLocalFile(const std::string &uri) + { + std::string filename; + if (strlwr(uri).find("file:/") == 0) + { + filename = uri.substr(6, uri.size() - 6); + } + else + { + filename = uri; + } + + #if LOG_DL + nlwarning("(%s) browse local file '%s'", filename.c_str()); + #endif + + _TrustedDomain = true; + + // Stop previous browse, remove content + stopBrowse (); + + _Browsing = true; + updateRefreshButton(); + CIFile in; if (in.open(filename)) { @@ -3746,12 +4035,182 @@ namespace NLGUI } } + // *************************************************************************** + void CGroupHTML::doBrowseRemoteUrl(const std::string &url, const std::string &referer, bool doPost, const SFormFields &formfields) + { + // Stop previous request and remove content + stopBrowse (); + + _Browsing = true; + updateRefreshButton(); + + // Reset the title + if(_TitlePrefix.empty()) + setTitle (CI18N::get("uiPleaseWait")); + else + setTitle (_TitlePrefix + " - " + CI18N::get("uiPleaseWait")); + + #if LOG_DL + nlwarning("(%s) browse url (trusted=%s) '%s', referer='%s', post='%s', nb form values %d", + _Id.c_str(), (_TrustedDomain ? "true" :"false"), url.c_str(), referer.c_str(), (doPost ? "true" : "false"), formfields.Values.size()); + #endif + + if (!MultiCurl) + { + browseError(string("Invalid MultCurl handle, loading url failed : "+url).c_str()); + return; + } + + CURL *curl = curl_easy_init(); + if (!curl) + { + nlwarning("(%s) failed to create curl handle", _Id.c_str()); + browseError(string("Failed to create cURL handle : " + url).c_str()); + return; + } + + // do not follow redirects, we have own handler + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0); + // after redirect + curl_easy_setopt(curl, CURLOPT_FRESH_CONNECT, 1); + + // tell curl to use compression if possible (gzip, deflate) + // leaving this empty allows all encodings that curl supports + //curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, ""); + + // limit curl to HTTP and HTTPS protocols only + curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + + // Destination + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + + // User-Agent: + std::string userAgent = options.appName + "/" + options.appVersion; + curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgent.c_str()); + + // Cookies + sendCookies(curl, HTTPCurrentDomain, _TrustedDomain); + + // Referer + if (!referer.empty()) + { + curl_easy_setopt(curl, CURLOPT_REFERER, referer.c_str()); + #ifdef LOG_DL + nlwarning("(%s) set referer '%s'", _Id.c_str(), referer.c_str()); + #endif + } + + if (doPost) + { + // serialize form data and add it to curl + std::string data; + for(uint i=0; i0) + data += "&"; + + data += std::string(escapedName) + "=" + escapedValue; + + curl_free(escapedName); + curl_free(escapedValue); + } + curl_easy_setopt(curl, CURLOPT_POST, 1); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, data.size()); + curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, data.c_str()); + } + else + { + curl_easy_setopt(curl, CURLOPT_HTTPGET, 1); + } + + // transfer handle + _CurlWWW = new CCurlWWWData(curl, url); + + // set the language code used by the client + std::vector headers; + headers.push_back("Accept-Language: "+options.languageCode); + headers.push_back("Accept-Charset: utf-8"); + for(uint i=0; i< headers.size(); ++i) + { + _CurlWWW->HeadersSent = curl_slist_append(_CurlWWW->HeadersSent, headers[i].c_str()); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, _CurlWWW->HeadersSent); + + // catch headers for redirect + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback); + curl_easy_setopt(curl, CURLOPT_WRITEHEADER, _CurlWWW); + + // catch body + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlDataCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, _CurlWWW); + + #if LOG_DL + // progress callback + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); + curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, curlProgressCallback); + curl_easy_setopt(curl, CURLOPT_XFERINFODATA, _CurlWWW); + #else + // progress off + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1); + #endif + + // + curl_multi_add_handle(MultiCurl, curl); + + // start the transfer + int NewRunningCurls = 0; + curl_multi_perform(MultiCurl, &NewRunningCurls); + RunningCurls++; + } + + // *************************************************************************** + void CGroupHTML::htmlDownloadFinished(const std::string &content, const std::string &type, long code) + { + #ifdef LOG_DL + nlwarning("(%s) HTML download finished, content length %d, type '%s', code %d", _Id.c_str(), content.size(), type.c_str(), code); + #endif + + // set trusted domain for parsing + _TrustedDomain = isTrustedDomain(setCurrentDomain(_URL)); + + // create markup for image downloads + if (type.find("image/") == 0 && content.size() > 0) + { + try + { + std::string dest = localImageName(_URL); + COFile out; + out.open(dest); + out.serialBuffer((uint8 *)(content.c_str()), content.size()); + out.close(); + #ifdef LOG_DL + nlwarning("(%s) image saved to '%s', url '%s'", _Id.c_str(), dest.c_str(), _URL.c_str()); + #endif + } + catch(...) { } + + // create html code with image url inside and do the request again + renderHtmlString(""+_URL+""); + } + else + { + renderHtmlString(content); + } + } + // *************************************************************************** bool CGroupHTML::renderHtmlString(const std::string &html) { bool success; + // + _Browsing = true; + // clear content beginBuild(); @@ -3760,14 +4219,16 @@ namespace NLGUI // invalidate coords endBuild(); - // libwww would call requestTerminated() here + // set the browser as complete _Browsing = false; + updateRefreshButton(); + // check that the title is set, or reset it (in the case the page + // does not provide a title) if (_TitleString.empty()) { setTitle(_TitlePrefix); } - updateRefreshButton(); return success; } @@ -3776,7 +4237,6 @@ namespace NLGUI void CGroupHTML::draw () { - checkDownloads(); CGroupScrollText::draw (); } @@ -3795,14 +4255,26 @@ namespace NLGUI // *************************************************************************** - void CGroupHTML::addHTTPPostParams (HTAssocList * /* formfields */, bool /*trustedDomain*/) + void CGroupHTML::addHTTPPostParams (SFormFields &/* formfields */, bool /*trustedDomain*/) { } // *************************************************************************** - - void CGroupHTML::requestTerminated(HTRequest * request ) + void CGroupHTML::requestTerminated() { + if (_CurlWWW) + { + #if LOG_DL + nlwarning("(%s) stop curl, url '%s'", _Id.c_str(), _CurlWWW->Url.c_str()); + #endif + if (MultiCurl) + curl_multi_remove_handle(MultiCurl, _CurlWWW->Request); + + delete _CurlWWW; + + _CurlWWW = NULL; + _Connecting = false; + } } // *************************************************************************** @@ -3830,7 +4302,9 @@ namespace NLGUI _GroupListAdaptor->clearViews(); CWidgetManager::getInstance()->clearViewUnders(); CWidgetManager::getInstance()->clearCtrlsUnders(); - _Paragraph = NULL; + + // Clear all the context + clearContext(); // Reset default background color setBackgroundColor (BgColor); @@ -4305,10 +4779,20 @@ namespace NLGUI // *************************************************************************** std::string CGroupHTML::getAbsoluteUrl(const std::string &url) { - if (HTURL_isAbsolute(url.c_str())) + if (_URL.size() == 0 || url.find("http://") != std::string::npos || url.find("https://") != std::string::npos) return url; - return std::string(HTParse(url.c_str(), _URL.c_str(), PARSE_ALL)); + xmlChar * uri; + uri = xmlBuildURI(reinterpret_cast(url.c_str()), reinterpret_cast(_URL.c_str())); + if (uri) + { + std::string ret(reinterpret_cast(uri)); + xmlFree(uri); + + return ret; + } + + return url; } // *************************************************************************** @@ -4389,5 +4873,46 @@ namespace NLGUI style.StrikeThrough = getFontStrikeThrough() || style.StrikeThrough; } } + + // *************************************************************************** + size_t CGroupHTML::curlHeaderCallback(char *buffer, size_t size, size_t nmemb, void *pCCurlWWWData) + { + CCurlWWWData * me = static_cast(pCCurlWWWData); + if (me) + { + std::string header; + header.append(buffer, size * nmemb); + me->setRecvHeader(header.substr(0, header.find_first_of("\n\r"))); + } + + return size * nmemb; + } + + // *************************************************************************** + size_t CGroupHTML::curlDataCallback(char *buffer, size_t size, size_t nmemb, void *pCCurlWWWData) + { + CCurlWWWData * me = static_cast(pCCurlWWWData); + if (me) + me->Content.append(buffer, size * nmemb); + + return size * nmemb; + } + + // *************************************************************************** + size_t CGroupHTML::curlProgressCallback(void *pCCurlWWWData, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) + { + CCurlWWWData * me = static_cast(pCCurlWWWData); + if (me) + { + if (dltotal > 0 || dlnow > 0 || ultotal > 0 || ulnow > 0) + { + nlwarning("> dltotal %d, dlnow %d, ultotal %d, ulnow %d, url '%s'", dltotal, dlnow, ultotal, ulnow, me->Url.c_str()); + } + } + + // return 1 to cancel download + return 0; + } + } diff --git a/code/nel/src/gui/libwww.cpp b/code/nel/src/gui/libwww.cpp index e5a587f4d..ff73bb59a 100644 --- a/code/nel/src/gui/libwww.cpp +++ b/code/nel/src/gui/libwww.cpp @@ -16,6 +16,7 @@ #include "stdpch.h" +#include "nel/gui/libwww.h" #include "nel/gui/group_html.h" using namespace NLMISC; @@ -23,11 +24,13 @@ using namespace NLMISC; namespace NLGUI { + // *************************************************************************** + /// the cookie value for session identification (nel cookie) std::string CurrentCookie; /// store all cookies we receive and resent them depending of the domain - std::map > HTTPCookies; + static std::map > HTTPCookies; std::string HTTPCurrentDomain; // The current domain that will be used to get which cookies to send // *************************************************************************** @@ -281,6 +284,138 @@ namespace NLGUI return dst; } + // set current HTTPCurrentDomain for cookie selection, return new domain + const std::string &setCurrentDomain(const std::string &uri) + { + if (uri.find("http://") == 0) + HTTPCurrentDomain = uri.substr(7, uri.find("/", 7) - 7); + else + if (uri.find("https://") == 0) + HTTPCurrentDomain = uri.substr(8, uri.find("/", 8) - 8); + else + if (uri.find("//") == 0) + HTTPCurrentDomain = uri.substr(2, uri.find("/", 2) - 2); + else + if (uri.find("/") != std::string::npos) + HTTPCurrentDomain = uri.substr(0, uri.find("/") - 1); + + return HTTPCurrentDomain; + } + + // update HTTPCookies list + static void receiveCookie(const char *nsformat, const std::string &domain, bool trusted) + { + // 0 1 2 3 4 5 6 + // domain tailmatch path secure expires name value + // .app.ryzom.com TRUE / FALSE 1234 ryzomId AAAAAAAA|BBBBBBBB|CCCCCCCC + // #HttpOnly_app.ryzom.com FALSE / FALSE 0 PHPSESSID sess-id-value + std::string cookie(nsformat); + + std::vector chunks; + splitString(cookie, "\t", chunks); + if (chunks.size() < 6) + { + nlwarning("invalid cookie format '%s'", cookie.c_str()); + } + + if (chunks[0].find("#HttpOnly_") == 0) + { + chunks[0] = chunks[0].substr(10); + } + + if (chunks[0] != domain && chunks[0] != std::string("." + domain)) + { + // cookie is for different domain + //nlinfo("cookie for different domain ('%s')", nsformat); + return; + } + + if (chunks[5] == "ryzomId") + { + // we receive this cookie because we are telling curl about this on send + // normally, this cookie should be set from client and not from headers + // it's used for R2 sessions + if (trusted && CurrentCookie != chunks[6]) + { + CurrentCookie = chunks[6]; + nlwarning("received ryzomId cookie '%s' from trusted domain '%s'", CurrentCookie.c_str(), domain.c_str()); + } + } + else + { + uint32 expires = 0; + fromString(chunks[4], expires); + // expires == 0 is session cookie + if (expires > 0) + { + time_t now; + time(&now); + if (expires < now) + { + nlwarning("cookie expired, remove from list '%s'", nsformat); + HTTPCookies[domain].erase(chunks[5]); + + return; + } + } + + // this overrides cookies with same name, but different paths + //nlwarning("save domain '%s' cookie '%s' value '%s'", domain.c_str(), chunks[5].c_str(), nsformat); + HTTPCookies[domain][chunks[5]] = nsformat; + } + } + + // update HTTPCookies with cookies received from curl + void receiveCookies (CURL *curl, const std::string &domain, bool trusted) + { + struct curl_slist *cookies = NULL; + if (curl_easy_getinfo(curl, CURLINFO_COOKIELIST, &cookies) == CURLE_OK) + { + struct curl_slist *nc; + nc = cookies; + while(nc) + { + //nlwarning("received cookie '%s'", nc->data); + receiveCookie(nc->data, domain, trusted); + nc = nc->next; + } + + curl_slist_free_all(cookies); + } + } + + // add all cookies for domain to curl handle + void sendCookies(CURL *curl, const std::string &domain, bool trusted) + { + if (domain.empty()) + return; + + if (trusted && !CurrentCookie.empty()) + { + // domain tailmatch path secure expires name value + // .app.ryzom.com TRUE / FALSE 1234 ryzomId AAAAAAAA|BBBBBBBB|CCCCCCCC + // #HttpOnly_app.ryzom.com FALSE / FALSE 0 PHPSESSID sess-id-value + std::string cookie; + // set tailmatch + if (domain[0] != '.' && domain[0] != '#') + cookie = "." + domain + "\tTRUE"; + else + cookie = domain + "\tFALSE"; + cookie += "\t/\tFALSE\t0\tryzomId\t" + CurrentCookie; + curl_easy_setopt(curl, CURLOPT_COOKIELIST, cookie.c_str()); + //nlwarning("domain '%s', cookie '%s'", domain.c_str(), cookie.c_str()); + } + + if(!HTTPCookies[domain].empty()) + { + for(std::map::iterator it = HTTPCookies[domain].begin(); it != HTTPCookies[domain].end(); it++) + { + curl_easy_setopt(curl, CURLOPT_COOKIELIST, it->second.c_str()); + //nlwarning("set domain '%s' cookie '%s'", domain.c_str(), it->second.c_str()); + } + } + } + void initLibWWW() { static bool initialized = false; diff --git a/code/ryzom/client/src/interface_v3/group_html_cs.cpp b/code/ryzom/client/src/interface_v3/group_html_cs.cpp index 8dc9fdb2b..ef72eaae6 100644 --- a/code/ryzom/client/src/interface_v3/group_html_cs.cpp +++ b/code/ryzom/client/src/interface_v3/group_html_cs.cpp @@ -70,7 +70,7 @@ void CGroupHTMLCS::addHTTPGetParams (string &url, bool /*trustedDomain*/) // *************************************************************************** -void CGroupHTMLCS::addHTTPPostParams (HTAssocList *formfields, bool /*trustedDomain*/) +void CGroupHTMLCS::addHTTPPostParams (SFormFields &formfields, bool /*trustedDomain*/) { std::vector parameters; getParameters (parameters, false); @@ -78,7 +78,7 @@ void CGroupHTMLCS::addHTTPPostParams (HTAssocList *formfields, bool /*trustedDom uint i; for (i=0; igetLoginName (); - HTParseFormInput(formfields, ("shard="+toString(CharacterHomeSessionId)).c_str()); - HTParseFormInput(formfields, ("user_login="+user_name.toString()).c_str()); - HTParseFormInput(formfields, ("session_cookie="+NetMngr.getLoginCookie().toString()).c_str()); - HTParseFormInput(formfields, ("lang="+CI18N::getCurrentLanguageCode()).c_str()); + formfields.add("shard", toString(CharacterHomeSessionId)); + formfields.add("user_login", user_name.toString()); + formfields.add("session_cookie", NetMngr.getLoginCookie().toString()); + formfields.add("lang", CI18N::getCurrentLanguageCode()); } // *************************************************************************** diff --git a/code/ryzom/client/src/interface_v3/group_html_mail.h b/code/ryzom/client/src/interface_v3/group_html_mail.h index 675a580c4..4731e1c3c 100644 --- a/code/ryzom/client/src/interface_v3/group_html_mail.h +++ b/code/ryzom/client/src/interface_v3/group_html_mail.h @@ -40,7 +40,7 @@ public: // From CGroupHTML virtual void addHTTPGetParams (std::string &url, bool trustedDomain); - virtual void addHTTPPostParams (HTAssocList *formfields, bool trustedDomain); + virtual void addHTTPPostParams (SFormFields &formfields, bool trustedDomain); virtual std::string home(); virtual void handle (); diff --git a/code/ryzom/client/src/interface_v3/group_html_webig.cpp b/code/ryzom/client/src/interface_v3/group_html_webig.cpp index 680a26273..1b74233e7 100644 --- a/code/ryzom/client/src/interface_v3/group_html_webig.cpp +++ b/code/ryzom/client/src/interface_v3/group_html_webig.cpp @@ -307,19 +307,19 @@ void CGroupHTMLAuth::addHTTPGetParams (string &url, bool trustedDomain) // *************************************************************************** -void CGroupHTMLAuth::addHTTPPostParams (HTAssocList *formfields, bool trustedDomain) +void CGroupHTMLAuth::addHTTPPostParams (SFormFields &formfields, bool trustedDomain) { if(!UserEntity) return; uint32 cid = NetMngr.getLoginCookie().getUserId() * 16 + PlayerSelectedSlot; - HTParseFormInput(formfields, ("shardid="+toString(CharacterHomeSessionId)).c_str()); - HTParseFormInput(formfields, ("name="+UserEntity->getLoginName().toUtf8()).c_str()); - HTParseFormInput(formfields, ("lang="+CI18N::getCurrentLanguageCode()).c_str()); - HTParseFormInput(formfields, "ig=1"); + formfields.add("shardid", toString(CharacterHomeSessionId)); + formfields.add("name", UserEntity->getLoginName().toUtf8()); + formfields.add("lang", CI18N::getCurrentLanguageCode()); + formfields.add("ig", "1"); if (trustedDomain) { - HTParseFormInput(formfields, ("cid="+toString(cid)).c_str()); - HTParseFormInput(formfields, ("authkey="+getWebAuthKey()).c_str()); + formfields.add("cid", toString(cid)); + formfields.add("authkey", getWebAuthKey()); } } @@ -365,7 +365,7 @@ void CGroupHTMLWebIG::addHTTPGetParams (string &url, bool trustedDomain) // *************************************************************************** -void CGroupHTMLWebIG::addHTTPPostParams (HTAssocList *formfields, bool trustedDomain) +void CGroupHTMLWebIG::addHTTPPostParams (SFormFields &formfields, bool trustedDomain) { CGroupHTMLAuth::addHTTPPostParams(formfields, trustedDomain); } diff --git a/code/ryzom/client/src/interface_v3/group_html_webig.h b/code/ryzom/client/src/interface_v3/group_html_webig.h index 9da5adee5..49aac7153 100644 --- a/code/ryzom/client/src/interface_v3/group_html_webig.h +++ b/code/ryzom/client/src/interface_v3/group_html_webig.h @@ -33,7 +33,7 @@ public: // From CGroupHTML virtual void addHTTPGetParams (std::string &url, bool trustedDomain); - virtual void addHTTPPostParams (HTAssocList *formfields, bool trustedDomain); + virtual void addHTTPPostParams (SFormFields &formfields, bool trustedDomain); virtual std::string home(); virtual void handle (); @@ -55,7 +55,7 @@ public: /// From CGroupHTMLAuth virtual void addHTTPGetParams (std::string &url, bool trustedDomain); - virtual void addHTTPPostParams (HTAssocList *formfields, bool trustedDomain); + virtual void addHTTPPostParams (SFormFields &formfields, bool trustedDomain); virtual std::string home(); virtual void handle ();