Use cURL as http transport

--HG--
branch : develop

code/nel/include/nel/gui/group_html.h

@@ -25,6 +25,7 @@
 #include "nel/gui/group_tree.h"
 #include "nel/gui/ctrl_button.h"
 #include "nel/gui/group_table.h"
+#include "nel/gui/libwww_types.h"
 
 typedef std::map<std::string, std::string>	TStyle;
 
@@ -36,7 +37,8 @@ namespace NLGUI
 	class CDBGroupComboBox;
 	class CGroupParagraph;
 
-
+	extern std::string CurrentCookie;
+	extern std::string HTTPCurrentDomain;
 
 	// HTML group
 	/**
@@ -164,6 +166,34 @@ namespace NLGUI
 		std::string		DefaultBackgroundBitmapView;
 		std::string		CurrentLinkTitle;
 
+		struct TFormField {
+		public:
+			TFormField(const std::string &k, const std::string &v)
+				:name(k),value(v)
+			{}
+			std::string name;
+			std::string value;
+		};
+
+		struct SFormFields {
+		public:
+			SFormFields()
+			{
+			}
+
+			void clear()
+			{
+				Values.clear();
+			}
+
+			void add(const std::string &key, const std::string &value)
+			{
+				Values.push_back(TFormField(key, value));
+			}
+
+			std::vector<TFormField> Values;
+		};
+
 		// Browser home
 		std::string		Home;
 
@@ -237,10 +267,10 @@ namespace NLGUI
 		virtual void addHTTPGetParams (std::string &url, bool trustedDomain);
 
 		// Add POST params to the libwww list
-		virtual void addHTTPPostParams (HTAssocList *formfields, bool trustedDomain);
+		virtual void addHTTPPostParams (SFormFields &formfields, bool trustedDomain);
 
 		// the current request is terminated
-		virtual void requestTerminated(HTRequest *request);
+		virtual void requestTerminated();
 
 		// libxml2 html parser functions
 		void htmlElement(xmlNode *node, int element_number);
@@ -333,6 +363,7 @@ namespace NLGUI
 		bool			_Connecting;
 		double			_TimeoutValue;			// the timeout in seconds
 		double			_ConnectingTimeout;
+		uint32			_RedirectsRemaining;
 
 		// minimal embeded lua script support
 		// Note : any embeded script is executed immediately after the closing
@@ -346,6 +377,9 @@ namespace NLGUI
 		bool			_Object;
 		std::string		_ObjectScript;
 
+		// Data container for active curl transfer
+		class CCurlWWWData *	_CurlWWW;
+
 		// Current paragraph
 		std::string		_DivName;
 		CGroupParagraph*	_Paragraph;
@@ -660,9 +694,15 @@ namespace NLGUI
 		// load and render local html file (from bnp for example)
 		void doBrowseLocalFile(const std::string &filename);
 
+		// load remote content using either GET or POST
+		void doBrowseRemoteUrl(const std::string &url, const std::string &referer, bool doPost = false, const SFormFields &formfields = SFormFields());
+
 		// render html string as new browser page
 		bool renderHtmlString(const std::string &html);
 
+		// initialize formfields list from form elements on page
+		void buildHTTPPostParams (SFormFields &formfields);
+
 	private:
 		// decode all HTML entities
 		static ucstring decodeHTMLEntities(const ucstring &str);
@@ -672,11 +712,13 @@ namespace NLGUI
 
 		struct CDataDownload
 		{
+		public:
 			CDataDownload(CURL *c, const std::string &u, FILE *f, TDataType t, CViewBase *i, const std::string &s, const std::string &m) : curl(c), url(u), luaScript(s), md5sum(m), type(t), fp(f)
 			{
 				if (t == ImgType) imgs.push_back(i);
 			}
 
+		public:
 			CURL *curl;
 			std::string url;
 			std::string luaScript;
@@ -708,6 +750,13 @@ namespace NLGUI
 		void releaseDownloads();
 		void checkDownloads();
 
+		// HtmlType download finished
+		void htmlDownloadFinished(const std::string &content, const std::string &type, long code);
+
+		// cURL transfer callbacks
+		static size_t curlHeaderCallback(char *buffer, size_t size, size_t nmemb, void *pCCurlWWWData);
+		static size_t curlDataCallback(char *buffer, size_t size, size_t nmemb, void *pCCurlWWWData);
+		static size_t curlProgressCallback(void *pCCurlWWWData, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow);
 	};
 
 	// adapter group that store y offset for inputs inside an html form
@@ -721,7 +770,6 @@ namespace NLGUI
 		xmlNodePtr serialize( xmlNodePtr parentNode, const char *type ) const;
 		virtual bool parse (xmlNodePtr cur, CInterfaceGroup *parentGroup);
 	};
-
 }
 
 #endif

code/nel/include/nel/gui/libwww.h

@@ -20,7 +20,10 @@
 #ifndef CL_LIB_WWW_H
 #define CL_LIB_WWW_H
 
+#include <curl/curl.h>
+
 #include "nel/misc/rgba.h"
+#include "nel/gui/libwww_types.h"
 
 namespace NLGUI
 {
@@ -30,6 +33,9 @@ namespace NLGUI
 
 	// ***************************************************************************
 
+	// Legacy function from libwww
+	SGML_dtd * HTML_dtd (void);
+
 	// Init the libwww
 	void initLibWWW();
 
@@ -230,6 +236,10 @@ namespace NLGUI
 
 	// ***************************************************************************
 
+	const std::string &setCurrentDomain(const std::string &uri);
+	void receiveCookies (CURL *curl, const std::string &domain, bool trusted);
+	void sendCookies(CURL *curl, const std::string &domain, bool trusted);
+
 }
 
 #endif

code/nel/src/gui/group_html.cpp

@@ -43,13 +43,16 @@
 #include "nel/misc/md5.h"
 #include "nel/3d/texture_file.h"
 #include "nel/misc/big_file.h"
+#include <libxml/uri.h>
 
 using namespace std;
 using namespace NLMISC;
 
 
 // Default timeout to connect a server
-#define DEFAULT_RYZOM_CONNECTION_TIMEOUT (10.0)
+#define DEFAULT_RYZOM_CONNECTION_TIMEOUT (30.0)
+// Allow up to 10 redirects, then give up
+#define DEFAULT_RYZOM_REDIRECT_LIMIT (10)
 
 namespace NLGUI
 {
@@ -60,6 +63,59 @@ namespace NLGUI
 	CGroupHTML::SWebOptions CGroupHTML::options;
 
 
+	// Active cURL www transfer
+	class CCurlWWWData
+	{
+		public:
+			CCurlWWWData(CURL *curl, const std::string &url)
+				: Request(curl), Url(url), Content(""), HeadersSent(NULL)
+			{
+			}
+			~CCurlWWWData()
+			{
+				if (Request)
+					curl_easy_cleanup(Request);
+
+				if (HeadersSent)
+					curl_slist_free_all(HeadersSent);
+			}
+
+			void setRecvHeader(const std::string &header)
+			{
+				size_t pos = header.find(": ");
+				if (pos == std::string::npos)
+					return;
+
+				std::string key = toLower(header.substr(0, pos));
+				if (pos != std::string::npos)
+				{
+					HeadersRecv[key] = header.substr(pos + 2);
+					//nlinfo(">> received header '%s' = '%s'", key.c_str(), HeadersRecv[key].c_str());
+				}
+			}
+
+			// return last received "Location: <url>" header or empty string if no header set
+			const std::string getLocationHeader()
+			{
+				if (HeadersRecv.count("location") > 0)
+					return HeadersRecv["location"];
+
+				return "";
+			}
+
+		public:
+			CURL *Request;
+
+			std::string Url;
+			std::string Content;
+
+			// headers sent with curl request, must be released after transfer
+			curl_slist * HeadersSent;
+
+			// headers received from curl transfer
+			std::map<std::string, std::string> HeadersRecv;
+	};
+
 	// Check if domain is on TrustedDomain
 	bool CGroupHTML::isTrustedDomain(const string &domain)
 	{
@@ -285,7 +341,7 @@ namespace NLGUI
 	{
 		//nlassert(_CrtCheckMemory());
 
-		if(RunningCurls == 0)
+		if(Curls.empty() && _CurlWWW == NULL)
 			return;
 
 		int NewRunningCurls = 0;
@@ -306,8 +362,82 @@ namespace NLGUI
 			int msgs_left;
 			while ((msg = curl_multi_info_read(MultiCurl, &msgs_left)))
 			{
+	#ifdef LOG_DL
+				nlwarning("> (%s) msgs_left %d", _Id.c_str(), msgs_left);
+	#endif
 				if (msg->msg == CURLMSG_DONE)
 				{
+					if (_CurlWWW && _CurlWWW->Request && _CurlWWW->Request == msg->easy_handle)
+					{
+						CURLcode res = msg->data.result;
+						long code;
+						curl_easy_getinfo(_CurlWWW->Request, CURLINFO_RESPONSE_CODE, &code);
+	#ifdef LOG_DL
+						nlwarning("(%s) web transfer '%p' completed with status %d, http %d, url (len %d) '%s'", _Id.c_str(), _CurlWWW->Request, res, code, _CurlWWW->Url.size(), _CurlWWW->Url.c_str());
+	#endif
+
+						if (res != CURLE_OK)
+						{
+							browseError(string("Connection failed with curl error " + string(curl_easy_strerror(res))).c_str());
+						}
+						else
+						if ((code >= 301 && code <= 303) || code == 307 || code == 308)
+						{
+							if (_RedirectsRemaining < 0)
+							{
+								browseError(string("Redirect limit reached : " + _URL).c_str());
+							}
+							else
+							{
+								// redirect, get the location and try browse again
+								// we cant use curl redirection because 'addHTTPGetParams()' must be called on new destination
+								std::string location(_CurlWWW->getLocationHeader());
+								if (location.size() > 0)
+								{
+	#ifdef LOG_DL
+									nlwarning("(%s) request (%d) redirected to (len %d) '%s'", _Id.c_str(), _RedirectsRemaining, location.size(), location.c_str());
+	#endif
+									location = getAbsoluteUrl(location);
+									// throw away this handle and start with new one (easier than reusing)
+									requestTerminated();
+
+									_PostNextTime = false;
+									_RedirectsRemaining--;
+
+									doBrowse(location.c_str());
+								}
+								else
+								{
+									browseError(string("Request was redirected, but location was not set : "+_URL).c_str());
+								}
+							}
+						}
+						else
+						{
+							_RedirectsRemaining = DEFAULT_RYZOM_REDIRECT_LIMIT;
+
+							if ( (code < 200 || code >= 300) )
+							{
+								browseError(string("Connection failed (curl code " + toString((sint32)res) + "), http code " + toString(code) + ") : " + _CurlWWW->Url).c_str());
+							}
+							else
+							{
+								receiveCookies(_CurlWWW->Request, HTTPCurrentDomain, _TrustedDomain);
+
+								char *ch;
+								std::string contentType;
+								res = curl_easy_getinfo(_CurlWWW->Request, CURLINFO_CONTENT_TYPE, &ch);
+								if (res == CURLE_OK)
+								{
+									contentType = ch;
+								}
+
+								htmlDownloadFinished(_CurlWWW->Content, contentType, code);
+							}
+							requestTerminated();
+						}
+					}
+
 					for (vector<CDataDownload>::iterator it=Curls.begin(); it<Curls.end(); it++)
 					{
 						if(msg->easy_handle == it->curl)
@@ -318,8 +448,9 @@ namespace NLGUI
 							fclose(it->fp);
 
 	#ifdef LOG_DL
-							nlwarning("transfer %x completed with status res %d r %d - %d curls", msg->easy_handle, res, r, Curls.size());
+							nlwarning("(%s) transfer '%p' completed with status %d, http %d, url (len %d) '%s'", _Id.c_str(), it->curl, res, r, it->url.size(), it->url.c_str());
 	#endif
+							curl_multi_remove_handle(MultiCurl, it->curl);
 							curl_easy_cleanup(it->curl);
 
 							string file;
@@ -342,11 +473,6 @@ namespace NLGUI
 									{
 										for(uint i = 0; i < it->imgs.size(); i++)
 										{
-											// don't display image that are not power of 2
-											//uint32 w, h;
-											//CBitmap::loadSize (file, w, h);
-											//if (w == 0 || h == 0 || ((!NLMISC::isPowerOf2(w) || !NLMISC::isPowerOf2(h)) && !NL3D::CTextureFile::supportNonPowerOfTwoTextures()))
-											//	file.clear();
 											setImage(it->imgs[i], file);
 										}
 									}
@@ -360,6 +486,7 @@ namespace NLGUI
 									}
 								}
 							}
+
 							Curls.erase(it);
 							break;
 						}
@@ -368,6 +495,10 @@ namespace NLGUI
 			}
 		}
 		RunningCurls = NewRunningCurls;
+	#ifdef LOG_DL
+		if (RunningCurls > 0 || Curls.size() > 0)
+			nlwarning("(%s) RunningCurls %d, _Curls %d", _Id.c_str(), RunningCurls, Curls.size());
+	#endif
 	}
 
 
@@ -436,11 +567,12 @@ namespace NLGUI
 	{
 		if (_Browsing)
 		{
-			nlassert (_Connecting);
 			_Connecting = false;
 
 			removeContent ();
 		}
+		else
+			nlwarning("_Browsing = FALSE");
 	}
 
 
@@ -1043,12 +1175,10 @@ namespace NLGUI
 					if (present[HTML_FORM_ACTION] && value[HTML_FORM_ACTION])
 					{
 						form.Action = getAbsoluteUrl(string(value[HTML_FORM_ACTION]));
-						nlinfo("(%s) form.action '%s' (converted)", _Id.c_str(), form.Action.c_str());
 					}
 					else
 					{
 						form.Action = _URL;
-						nlinfo("(%s) using _URL for form.action '%s'", _Id.c_str(), form.Action.c_str());
 					}
 					_Forms.push_back(form);
 				}
@@ -1243,7 +1373,16 @@ namespace NLGUI
 
 									// Translate the tooltip
 									if (tooltip)
+									{
+										if (CI18N::hasTranslation(tooltip))
+										{
 											ctrlButton->setDefaultContextHelp(CI18N::get(tooltip));
+										}
+										else
+										{
+											ctrlButton->setDefaultContextHelp(ucstring(tooltip));
+										}
+									}
 
 									ctrlButton->setText(ucstring::makeFromUtf8(text));
 								}
@@ -1336,24 +1475,11 @@ namespace NLGUI
 				if (!(_Forms.empty()))
 				{
 					// A select box
-
-					// read general property
-					string templateName;
-					string minWidth;
-
-					// Widget template name
-					if (present[MY_HTML_INPUT_Z_INPUT_TMPL] && value[MY_HTML_INPUT_Z_INPUT_TMPL])
-						templateName = value[MY_HTML_INPUT_Z_INPUT_TMPL];
-					// Widget minimal width
-					if (present[MY_HTML_INPUT_Z_INPUT_WIDTH] && value[MY_HTML_INPUT_Z_INPUT_WIDTH])
-						minWidth = value[MY_HTML_INPUT_Z_INPUT_WIDTH];
-
 					string name;
 					if (present[HTML_SELECT_NAME] && value[HTML_SELECT_NAME])
 						name = value[HTML_SELECT_NAME];
 
-					string formTemplate = templateName.empty() ? DefaultFormSelectGroup : templateName;
+					CDBGroupComboBox *cb = addComboBox(DefaultFormSelectGroup, name.c_str());
-					CDBGroupComboBox *cb = addComboBox(formTemplate, name.c_str());
 					CGroupHTML::CForm::CEntry entry;
 					entry.Name = name;
 					entry.ComboBox = cb;
@@ -1824,7 +1950,8 @@ namespace NLGUI
 	// ***************************************************************************
 	CGroupHTML::CGroupHTML(const TCtorParam &param)
 	:	CGroupScrollText(param),
-		_TimeoutValue(DEFAULT_RYZOM_CONNECTION_TIMEOUT)
+		_TimeoutValue(DEFAULT_RYZOM_CONNECTION_TIMEOUT),
+		_RedirectsRemaining(DEFAULT_RYZOM_REDIRECT_LIMIT)
 	{
 		// add it to map of group html created
 		_GroupHtmlUID= ++_GroupHtmlUIDPool; // valid assigned Id begin to 1!
@@ -1894,9 +2021,11 @@ namespace NLGUI
 
 		MultiCurl = curl_multi_init();
 		RunningCurls = 0;
+		_CurlWWW = NULL;
 
 		initImageDownload();
 		initBnpDownload();
+		initLibWWW();
 	}
 
 	// ***************************************************************************
@@ -1921,6 +2050,8 @@ namespace NLGUI
 					   //     this is why the call to 'updateRefreshButton' has been removed from stopBrowse
 
 		clearContext();
+		if (_CurlWWW)
+			delete _CurlWWW;
 	}
 
 	std::string CGroupHTML::getProperty( const std::string &name ) const
@@ -2849,20 +2980,20 @@ namespace NLGUI
 			clearContext();
 
 			_Browsing = false;
-			_Connecting = false;
 			updateRefreshButton();
 
 	#ifdef LOG_DL
-			nlwarning("*** ALREADY BROWSING, break first");
+			nlwarning("(%s) *** ALREADY BROWSING, break first", _Id.c_str());
 	#endif
 		}
 
 	#ifdef LOG_DL
-		nlwarning("Browsing URL : '%s'", url);
+		nlwarning("(%s) Browsing URL : '%s'", _Id.c_str(), url);
 	#endif
 
 		// go
 		_URL = url;
+		_Connecting = false;
 		_BrowseNextTime = true;
 
 		// if a BrowseTree is bound to us, try to select the node that opens this URL (auto-locate)
@@ -2910,13 +3041,15 @@ namespace NLGUI
 	void CGroupHTML::stopBrowse ()
 	{
 	#ifdef LOG_DL
-		nlwarning("*** STOP BROWSE");
+		nlwarning("*** STOP BROWSE (%s)", _Id.c_str());
 	#endif
 
 		// Clear all the context
 		clearContext();
 
 		_Browsing = false;
+
+		requestTerminated();
 	}
 
 	// ***************************************************************************
@@ -3539,7 +3672,6 @@ namespace NLGUI
 			p->setTopSpace(beginSpace);
 		else
 			group->setY(-(sint32)beginSpace);
-
 		parentGroup->addGroup (group);
 	}
 
@@ -3703,17 +3835,64 @@ namespace NLGUI
 
 		const CWidgetManager::SInterfaceTimes &times = CWidgetManager::getInstance()->getInterfaceTimes();
 
+		// handle curl downloads
+		checkDownloads();
+
 		if (_Connecting)
 		{
 			// Check timeout if needed
 			if (_TimeoutValue != 0 && _ConnectingTimeout <= ( times.thisFrameMs / 1000.0f ) )
 			{
 				browseError(("Connection timeout : "+_URL).c_str());
+
+				_Connecting = false;
 			}
 		}
 		else
 		if (_BrowseNextTime || _PostNextTime)
 		{
+			// Set timeout
+			_Connecting = true;
+			_ConnectingTimeout = ( times.thisFrameMs / 1000.0f ) + _TimeoutValue;
+
+			// freeze form buttons
+			CButtonFreezer freezer;
+			this->visit(&freezer);
+
+			// Home ?
+			if (_URL == "home")
+				_URL = home();
+
+			string finalUrl;
+			bool isLocal = lookupLocalFile (finalUrl, _URL.c_str(), true);
+
+			// Save new url
+			_URL = finalUrl;
+
+			// file is probably from bnp (ingame help)
+			if (isLocal)
+			{
+				doBrowseLocalFile(finalUrl);
+			}
+			else
+			{
+				_TrustedDomain = isTrustedDomain(setCurrentDomain(finalUrl));
+
+				SFormFields formfields;
+				if (_PostNextTime)
+				{
+					buildHTTPPostParams(formfields);
+					// _URL is set from form.Action
+					finalUrl = _URL;
+				}
+				else
+				{
+					// Add custom get params from child classes
+					addHTTPGetParams (finalUrl, _TrustedDomain);
+				}
+
+				doBrowseRemoteUrl(finalUrl, "", _PostNextTime, formfields);
+			}
 
 			_BrowseNextTime = false;
 			_PostNextTime = false;
@@ -3721,8 +3900,118 @@ namespace NLGUI
 	}
 
 	// ***************************************************************************
-	void CGroupHTML::doBrowseLocalFile(const std::string &filename)
+	void CGroupHTML::buildHTTPPostParams (SFormFields &formfields)
 	{
+		// Add text area text
+		uint i;
+
+		if (_PostFormId >= _Forms.size())
+		{
+			nlwarning("(%s) invalid form index %d, _Forms %d", _Id.c_str(), _PostFormId, _Forms.size());
+			return;
+		}
+		// Ref the form
+		CForm &form = _Forms[_PostFormId];
+
+		// Save new url
+		_URL = form.Action;
+		_TrustedDomain = isTrustedDomain(setCurrentDomain(_URL));
+
+		for (i=0; i<form.Entries.size(); i++)
+		{
+			// Text area ?
+			bool addEntry = false;
+			ucstring entryData;
+			if (form.Entries[i].TextArea)
+			{
+				// Get the edit box view
+				CInterfaceGroup *group = form.Entries[i].TextArea->getGroup ("eb");
+				if (group)
+				{
+					// Should be a CGroupEditBox
+					CGroupEditBox *editBox = dynamic_cast<CGroupEditBox*>(group);
+					if (editBox)
+					{
+						entryData = editBox->getViewText()->getText();
+						addEntry = true;
+					}
+				}
+			}
+			else if (form.Entries[i].Checkbox)
+			{
+				// todo handle unicode POST here
+				if (form.Entries[i].Checkbox->getPushed ())
+				{
+					entryData = ucstring ("on");
+					addEntry = true;
+				}
+			}
+			else if (form.Entries[i].ComboBox)
+			{
+				CDBGroupComboBox *cb = form.Entries[i].ComboBox;
+				entryData.fromUtf8(form.Entries[i].SelectValues[cb->getSelection()]);
+				addEntry = true;
+			}
+			// This is a hidden value
+			else
+			{
+				entryData = form.Entries[i].Value;
+				addEntry = true;
+			}
+
+			// Add this entry
+			if (addEntry)
+			{
+				formfields.add(form.Entries[i].Name, CI18N::encodeUTF8(entryData));
+			}
+		}
+
+		if (_PostFormSubmitType == "image")
+		{
+			// Add the button coordinates
+			if (_PostFormSubmitButton.find_first_of("[") == string::npos)
+			{
+				formfields.add(_PostFormSubmitButton + "_x", NLMISC::toString(_PostFormSubmitX));
+				formfields.add(_PostFormSubmitButton + "_y", NLMISC::toString(_PostFormSubmitY));
+			}
+			else
+			{
+				formfields.add(_PostFormSubmitButton, NLMISC::toString(_PostFormSubmitX));
+				formfields.add(_PostFormSubmitButton, NLMISC::toString(_PostFormSubmitY));
+			}
+		}
+		else
+			formfields.add(_PostFormSubmitButton, _PostFormSubmitValue);
+
+		// Add custom params from child classes
+		addHTTPPostParams(formfields, _TrustedDomain);
+	}
+
+	// ***************************************************************************
+	void CGroupHTML::doBrowseLocalFile(const std::string &uri)
+	{
+		std::string filename;
+		if (strlwr(uri).find("file:/") == 0)
+		{
+			filename = uri.substr(6, uri.size() - 6);
+		}
+		else
+		{
+			filename = uri;
+		}
+
+	#if LOG_DL
+		nlwarning("(%s) browse local file '%s'", filename.c_str());
+	#endif
+
+		_TrustedDomain = true;
+
+		// Stop previous browse, remove content
+		stopBrowse ();
+
+		_Browsing = true;
+		updateRefreshButton();
+
 		CIFile in;
 		if (in.open(filename))
 		{
@@ -3746,12 +4035,182 @@ namespace NLGUI
 		}
 	}
 
+	// ***************************************************************************
+	void CGroupHTML::doBrowseRemoteUrl(const std::string &url, const std::string &referer, bool doPost, const SFormFields &formfields)
+	{
+		// Stop previous request and remove content
+		stopBrowse ();
+
+		_Browsing = true;
+		updateRefreshButton();
+
+		// Reset the title
+		if(_TitlePrefix.empty())
+			setTitle (CI18N::get("uiPleaseWait"));
+		else
+			setTitle (_TitlePrefix + " - " + CI18N::get("uiPleaseWait"));
+
+	#if LOG_DL
+		nlwarning("(%s) browse url (trusted=%s) '%s', referer='%s', post='%s', nb form values %d",
+				_Id.c_str(), (_TrustedDomain ? "true" :"false"), url.c_str(), referer.c_str(), (doPost ? "true" : "false"), formfields.Values.size());
+	#endif
+
+		if (!MultiCurl)
+		{
+			browseError(string("Invalid MultCurl handle, loading url failed : "+url).c_str());
+			return;
+		}
+
+		CURL *curl = curl_easy_init();
+		if (!curl)
+		{
+			nlwarning("(%s) failed to create curl handle", _Id.c_str());
+			browseError(string("Failed to create cURL handle : " + url).c_str());
+			return;
+		}
+
+		// do not follow redirects, we have own handler
+		curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0);
+		// after redirect
+		curl_easy_setopt(curl, CURLOPT_FRESH_CONNECT, 1);
+
+		// tell curl to use compression if possible (gzip, deflate)
+		// leaving this empty allows all encodings that curl supports
+		//curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "");
+
+		// limit curl to HTTP and HTTPS protocols only
+		curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
+		curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
+
+		// Destination
+		curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+
+		// User-Agent:
+		std::string userAgent = options.appName + "/" + options.appVersion;
+		curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgent.c_str());
+
+		// Cookies
+		sendCookies(curl, HTTPCurrentDomain, _TrustedDomain);
+
+		// Referer
+		if (!referer.empty())
+		{
+			curl_easy_setopt(curl, CURLOPT_REFERER, referer.c_str());
+	#ifdef LOG_DL
+			nlwarning("(%s) set referer '%s'", _Id.c_str(), referer.c_str());
+	#endif
+		}
+
+		if (doPost)
+		{
+			// serialize form data and add it to curl
+			std::string data;
+			for(uint i=0; i<formfields.Values.size(); ++i)
+			{
+				char * escapedName = curl_easy_escape(curl, formfields.Values[i].name.c_str(), formfields.Values[i].name.size());
+				char * escapedValue = curl_easy_escape(curl, formfields.Values[i].value.c_str(), formfields.Values[i].value.size());
+
+				if (i>0)
+					data += "&";
+
+				data += std::string(escapedName) + "=" + escapedValue;
+
+				curl_free(escapedName);
+				curl_free(escapedValue);
+			}
+			curl_easy_setopt(curl, CURLOPT_POST, 1);
+			curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, data.size());
+			curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, data.c_str());
+		}
+		else
+		{
+			curl_easy_setopt(curl, CURLOPT_HTTPGET, 1);
+		}
+
+		// transfer handle
+		_CurlWWW = new CCurlWWWData(curl, url);
+
+		// set the language code used by the client
+		std::vector<std::string> headers;
+		headers.push_back("Accept-Language: "+options.languageCode);
+		headers.push_back("Accept-Charset: utf-8");
+		for(uint i=0; i< headers.size(); ++i)
+		{
+			_CurlWWW->HeadersSent = curl_slist_append(_CurlWWW->HeadersSent, headers[i].c_str());
+		}
+		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, _CurlWWW->HeadersSent);
+
+		// catch headers for redirect
+		curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback);
+		curl_easy_setopt(curl, CURLOPT_WRITEHEADER, _CurlWWW);
+
+		// catch body
+		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlDataCallback);
+		curl_easy_setopt(curl, CURLOPT_WRITEDATA, _CurlWWW);
+
+	#if LOG_DL
+		// progress callback
+		curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
+		curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, curlProgressCallback);
+		curl_easy_setopt(curl, CURLOPT_XFERINFODATA, _CurlWWW);
+	#else
+		// progress off
+		curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
+	#endif
+
+		//
+		curl_multi_add_handle(MultiCurl, curl);
+
+		// start the transfer
+		int NewRunningCurls = 0;
+		curl_multi_perform(MultiCurl, &NewRunningCurls);
+		RunningCurls++;
+	}
+
+	// ***************************************************************************
+	void CGroupHTML::htmlDownloadFinished(const std::string &content, const std::string &type, long code)
+	{
+	#ifdef LOG_DL
+		nlwarning("(%s) HTML download finished, content length %d, type '%s', code %d", _Id.c_str(), content.size(), type.c_str(), code);
+	#endif
+
+		// set trusted domain for parsing
+		_TrustedDomain = isTrustedDomain(setCurrentDomain(_URL));
+
+		// create <html> markup for image downloads
+		if (type.find("image/") == 0 && content.size() > 0)
+		{
+			try
+			{
+				std::string dest = localImageName(_URL);
+				COFile out;
+				out.open(dest);
+				out.serialBuffer((uint8 *)(content.c_str()), content.size());
+				out.close();
+	#ifdef LOG_DL
+				nlwarning("(%s) image saved to '%s', url '%s'", _Id.c_str(), dest.c_str(), _URL.c_str());
+	#endif
+			}
+			catch(...) { }
+
+			// create html code with image url inside and do the request again
+			renderHtmlString("<html><head><title>"+_URL+"</title></head><body><img src=\"" + _URL + "\"></body></html>");
+		}
+		else
+		{
+			renderHtmlString(content);
+		}
+	}
+
 	// ***************************************************************************
 
 	bool CGroupHTML::renderHtmlString(const std::string &html)
 	{
 		bool success;
 
+		//
+		_Browsing = true;
+
 		// clear content
 		beginBuild();
 
@@ -3760,14 +4219,16 @@ namespace NLGUI
 		// invalidate coords
 		endBuild();
 
-		// libwww would call requestTerminated() here
+		// set the browser as complete
 		_Browsing = false;
+		updateRefreshButton();
 
+		// check that the title is set, or reset it (in the case the page
+		// does not provide a title)
 		if (_TitleString.empty())
 		{
 			setTitle(_TitlePrefix);
 		}
-		updateRefreshButton();
 
 		return success;
 	}
@@ -3776,7 +4237,6 @@ namespace NLGUI
 
 	void CGroupHTML::draw ()
 	{
-		checkDownloads();
 		CGroupScrollText::draw ();
 	}
 
@@ -3795,14 +4255,26 @@ namespace NLGUI
 
 	// ***************************************************************************
 
-	void CGroupHTML::addHTTPPostParams (HTAssocList * /* formfields */, bool /*trustedDomain*/)
+	void CGroupHTML::addHTTPPostParams (SFormFields &/* formfields */, bool /*trustedDomain*/)
 	{
 	}
 
 	// ***************************************************************************
-
+	void CGroupHTML::requestTerminated()
-	void CGroupHTML::requestTerminated(HTRequest * request )
 	{
+		if (_CurlWWW)
+		{
+	#if LOG_DL
+			nlwarning("(%s) stop curl, url '%s'", _Id.c_str(), _CurlWWW->Url.c_str());
+	#endif
+			if (MultiCurl)
+				curl_multi_remove_handle(MultiCurl, _CurlWWW->Request);
+
+			delete _CurlWWW;
+
+			_CurlWWW = NULL;
+			_Connecting = false;
+		}
 	}
 
 	// ***************************************************************************
@@ -3830,7 +4302,9 @@ namespace NLGUI
 		_GroupListAdaptor->clearViews();
 		CWidgetManager::getInstance()->clearViewUnders();
 		CWidgetManager::getInstance()->clearCtrlsUnders();
-		_Paragraph = NULL;
+
+		// Clear all the context
+		clearContext();
 
 		// Reset default background color
 		setBackgroundColor (BgColor);
@@ -4305,10 +4779,20 @@ namespace NLGUI
 	// ***************************************************************************
 	std::string CGroupHTML::getAbsoluteUrl(const std::string &url)
 	{
-		if (HTURL_isAbsolute(url.c_str()))
+		if (_URL.size() == 0 || url.find("http://") != std::string::npos || url.find("https://") != std::string::npos)
 			return url;
 
-		return std::string(HTParse(url.c_str(), _URL.c_str(), PARSE_ALL));
+		xmlChar * uri;
+		uri = xmlBuildURI(reinterpret_cast<const xmlChar *>(url.c_str()), reinterpret_cast<const xmlChar *>(_URL.c_str()));
+		if (uri)
+		{
+			std::string ret(reinterpret_cast<char *>(uri));
+			xmlFree(uri);
+
+			return ret;
+		}
+
+		return url;
 	}
 
 	// ***************************************************************************
@@ -4389,5 +4873,46 @@ namespace NLGUI
 			style.StrikeThrough = getFontStrikeThrough() || style.StrikeThrough;
 		}
 	}
+
+	// ***************************************************************************
+	size_t CGroupHTML::curlHeaderCallback(char *buffer, size_t size, size_t nmemb, void *pCCurlWWWData)
+	{
+		CCurlWWWData * me = static_cast<CCurlWWWData *>(pCCurlWWWData);
+		if (me)
+		{
+			std::string header;
+			header.append(buffer, size * nmemb);
+			me->setRecvHeader(header.substr(0, header.find_first_of("\n\r")));
+		}
+
+		return size * nmemb;
+	}
+
+	// ***************************************************************************
+	size_t CGroupHTML::curlDataCallback(char *buffer, size_t size, size_t nmemb, void *pCCurlWWWData)
+	{
+		CCurlWWWData * me = static_cast<CCurlWWWData *>(pCCurlWWWData);
+		if (me)
+			me->Content.append(buffer, size * nmemb);
+
+		return size * nmemb;
+	}
+
+	// ***************************************************************************
+	size_t CGroupHTML::curlProgressCallback(void *pCCurlWWWData, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
+	{
+		CCurlWWWData * me = static_cast<CCurlWWWData *>(pCCurlWWWData);
+		if (me)
+		{
+			if (dltotal > 0 || dlnow > 0 || ultotal > 0 || ulnow > 0)
+			{
+				nlwarning("> dltotal %d, dlnow %d, ultotal %d, ulnow %d, url '%s'", dltotal, dlnow, ultotal, ulnow, me->Url.c_str());
+			}
+		}
+
+		// return 1 to cancel download
+		return 0;
+	}
+
 }
 

code/nel/src/gui/libwww.cpp

@@ -16,6 +16,7 @@
 
 #include "stdpch.h"
 
+#include "nel/gui/libwww.h"
 #include "nel/gui/group_html.h"
 
 using namespace NLMISC;
@@ -23,11 +24,13 @@ using namespace NLMISC;
 namespace NLGUI
 {
 
+	// ***************************************************************************
+
 	/// the cookie value for session identification (nel cookie)
 	std::string CurrentCookie;
 
 	/// store all cookies we receive and resent them depending of the domain
-	std::map<std::string, std::map<std::string, std::string> > HTTPCookies;
+	static std::map<std::string, std::map<std::string, std::string> > HTTPCookies;
 	std::string HTTPCurrentDomain;	// The current domain that will be used to get which cookies to send
 
 	// ***************************************************************************
@@ -281,6 +284,138 @@ namespace NLGUI
 		return dst;
 	}
 
+	// set current HTTPCurrentDomain for cookie selection, return new domain
+	const std::string &setCurrentDomain(const std::string &uri)
+	{
+		if (uri.find("http://") == 0)
+			HTTPCurrentDomain = uri.substr(7, uri.find("/", 7) - 7);
+		else
+		if (uri.find("https://") == 0)
+			HTTPCurrentDomain = uri.substr(8, uri.find("/", 8) - 8);
+		else
+		if (uri.find("//") == 0)
+			HTTPCurrentDomain = uri.substr(2, uri.find("/", 2) - 2);
+		else
+		if (uri.find("/") != std::string::npos)
+			HTTPCurrentDomain = uri.substr(0, uri.find("/") - 1);
+
+		return HTTPCurrentDomain;
+	}
+
+	// update HTTPCookies list
+	static void receiveCookie(const char *nsformat, const std::string &domain, bool trusted)
+	{
+		// 0        1           2       3       4       5       6
+		// domain	tailmatch	path	secure	expires	name	value
+		// .app.ryzom.com	TRUE	/	 FALSE	1234	ryzomId	AAAAAAAA|BBBBBBBB|CCCCCCCC
+		// #HttpOnly_app.ryzom.com	FALSE	/	FALSE	0	PHPSESSID	sess-id-value
+		std::string cookie(nsformat);
+
+		std::vector<std::string> chunks;
+		splitString(cookie, "\t", chunks);
+		if (chunks.size() < 6)
+		{
+			nlwarning("invalid cookie format '%s'", cookie.c_str());
+		}
+
+		if (chunks[0].find("#HttpOnly_") == 0)
+		{
+			chunks[0] = chunks[0].substr(10);
+		}
+
+		if (chunks[0] != domain && chunks[0] != std::string("." + domain))
+		{
+			// cookie is for different domain
+			//nlinfo("cookie for different domain ('%s')", nsformat);
+			return;
+		}
+
+		if (chunks[5] == "ryzomId")
+		{
+			// we receive this cookie because we are telling curl about this on send
+			// normally, this cookie should be set from client and not from headers
+			// it's used for R2 sessions
+			if (trusted && CurrentCookie != chunks[6])
+			{
+				CurrentCookie = chunks[6];
+				nlwarning("received ryzomId cookie '%s' from trusted domain '%s'", CurrentCookie.c_str(), domain.c_str());
+			}
+		}
+		else
+		{
+			uint32 expires = 0;
+			fromString(chunks[4], expires);
+			// expires == 0 is session cookie
+			if (expires > 0)
+			{
+				time_t now;
+				time(&now);
+				if (expires < now)
+				{
+					nlwarning("cookie expired, remove from list '%s'", nsformat);
+					HTTPCookies[domain].erase(chunks[5]);
+
+					return;
+				}
+			}
+
+			// this overrides cookies with same name, but different paths
+			//nlwarning("save domain '%s' cookie '%s' value '%s'", domain.c_str(), chunks[5].c_str(), nsformat);
+			HTTPCookies[domain][chunks[5]] = nsformat;
+		}
+	}
+
+	// update HTTPCookies with cookies received from curl
+	void receiveCookies (CURL *curl, const std::string &domain, bool trusted)
+	{
+		struct curl_slist *cookies = NULL;
+		if (curl_easy_getinfo(curl, CURLINFO_COOKIELIST, &cookies) == CURLE_OK)
+		{
+			struct curl_slist *nc;
+			nc = cookies;
+			while(nc)
+			{
+				//nlwarning("received cookie '%s'", nc->data);
+				receiveCookie(nc->data, domain, trusted);
+				nc = nc->next;
+			}
+
+			curl_slist_free_all(cookies);
+		}
+	}
+
+	// add all cookies for domain to curl handle
+	void sendCookies(CURL *curl, const std::string &domain, bool trusted)
+	{
+		if (domain.empty())
+			return;
+
+		if (trusted && !CurrentCookie.empty())
+		{
+			// domain	tailmatch	path	secure	expires	name	value
+			// .app.ryzom.com	TRUE	/	 FALSE	1234	ryzomId	AAAAAAAA|BBBBBBBB|CCCCCCCC
+			// #HttpOnly_app.ryzom.com	FALSE	/	FALSE	0	PHPSESSID	sess-id-value
+			std::string cookie;
+			// set tailmatch
+			if (domain[0] != '.' && domain[0] != '#')
+				cookie = "." + domain + "\tTRUE";
+			else
+				cookie = domain + "\tFALSE";
+			cookie += "\t/\tFALSE\t0\tryzomId\t" + CurrentCookie;
+			curl_easy_setopt(curl, CURLOPT_COOKIELIST, cookie.c_str());
+			//nlwarning("domain '%s', cookie '%s'", domain.c_str(), cookie.c_str());
+		}
+
+		if(!HTTPCookies[domain].empty())
+		{
+			for(std::map<std::string, std::string>::iterator it = HTTPCookies[domain].begin(); it != HTTPCookies[domain].end(); it++)
+			{
+				curl_easy_setopt(curl, CURLOPT_COOKIELIST, it->second.c_str());
+				//nlwarning("set domain '%s' cookie '%s'", domain.c_str(), it->second.c_str());
+			}
+		}
+	}
+
 	void initLibWWW()
 	{
 		static bool initialized = false;

code/ryzom/client/src/interface_v3/group_html_cs.cpp

@@ -70,7 +70,7 @@ void CGroupHTMLCS::addHTTPGetParams (string &url, bool /*trustedDomain*/)
 
 // ***************************************************************************
 
-void CGroupHTMLCS::addHTTPPostParams (HTAssocList *formfields, bool /*trustedDomain*/)
+void CGroupHTMLCS::addHTTPPostParams (SFormFields &formfields, bool /*trustedDomain*/)
 {
 	std::vector<CParameter> parameters;
 	getParameters (parameters, false);
@@ -78,7 +78,7 @@ void CGroupHTMLCS::addHTTPPostParams (HTAssocList *formfields, bool /*trustedDom
 	uint i;
 	for (i=0; i<parameters.size(); i++)
 	{
-		HTParseFormInput(formfields, (parameters[i].Name+"="+parameters[i].Value).c_str());
+		formfields.add(parameters[i].Name, parameters[i].Value);
 	}
 }
 

code/ryzom/client/src/interface_v3/group_html_cs.h

@@ -40,7 +40,7 @@ public:
 
 	// From CGroupHTML
 	virtual void addHTTPGetParams (std::string &url, bool trustedDomain);
-	virtual void addHTTPPostParams (HTAssocList *formfields, bool trustedDomain);
+	virtual void addHTTPPostParams (SFormFields &formfields, bool trustedDomain);
 	virtual std::string	home();
 
 private:

code/ryzom/client/src/interface_v3/group_html_forum.cpp

@@ -83,7 +83,7 @@ void CGroupHTMLForum::addHTTPGetParams (string &url, bool /*trustedDomain*/)
 
 // ***************************************************************************
 
-void CGroupHTMLForum::addHTTPPostParams (HTAssocList *formfields, bool /*trustedDomain*/)
+void CGroupHTMLForum::addHTTPPostParams (SFormFields &formfields, bool /*trustedDomain*/)
 {
 	ucstring user_name = UserEntity->getLoginName ();
 	const SGuild &guild = CGuildManager::getInstance()->getGuild();
@@ -91,10 +91,10 @@ void CGroupHTMLForum::addHTTPPostParams (HTAssocList *formfields, bool /*trusted
 
 	if (!gname.empty())
 	{
-		HTParseFormInput(formfields, ("shard="+toString(CharacterHomeSessionId)).c_str());
+		formfields.add("shard", toString(CharacterHomeSessionId));
-		HTParseFormInput(formfields, ("user_login="+user_name.toString()).c_str());
+		formfields.add("user_login", user_name.toString());
-		HTParseFormInput(formfields, ("forum="+gname).c_str());
+		formfields.add("forum", gname);
-		HTParseFormInput(formfields, ("session_cookie="+NetMngr.getLoginCookie().toString()).c_str());
+		formfields.add("session_cookie", NetMngr.getLoginCookie().toString());
 	}
 	else
 	{

code/ryzom/client/src/interface_v3/group_html_forum.h

@@ -40,7 +40,7 @@ public:
 
 	// From CGroupHTML
 	virtual void addHTTPGetParams (std::string &url, bool trustedDomain);
-	virtual void addHTTPPostParams (HTAssocList *formfields, bool trustedDomain);
+	virtual void addHTTPPostParams (SFormFields &formfields, bool trustedDomain);
 	virtual std::string	home();
 	virtual void handle ();
 

code/ryzom/client/src/interface_v3/group_html_mail.cpp

@@ -62,13 +62,13 @@ void CGroupHTMLMail::addHTTPGetParams (string &url, bool /*trustedDomain*/)
 
 // ***************************************************************************
 
-void CGroupHTMLMail::addHTTPPostParams (HTAssocList *formfields, bool /*trustedDomain*/)
+void CGroupHTMLMail::addHTTPPostParams (SFormFields &formfields, bool /*trustedDomain*/)
 {
 	ucstring user_name = UserEntity->getLoginName ();
-	HTParseFormInput(formfields, ("shard="+toString(CharacterHomeSessionId)).c_str());
+	formfields.add("shard", toString(CharacterHomeSessionId));
-	HTParseFormInput(formfields, ("user_login="+user_name.toString()).c_str());
+	formfields.add("user_login", user_name.toString());
-	HTParseFormInput(formfields, ("session_cookie="+NetMngr.getLoginCookie().toString()).c_str());
+	formfields.add("session_cookie", NetMngr.getLoginCookie().toString());
-	HTParseFormInput(formfields, ("lang="+CI18N::getCurrentLanguageCode()).c_str());
+	formfields.add("lang", CI18N::getCurrentLanguageCode());
 }
 
 // ***************************************************************************

code/ryzom/client/src/interface_v3/group_html_mail.h

@@ -40,7 +40,7 @@ public:
 
 	// From CGroupHTML
 	virtual void addHTTPGetParams (std::string &url, bool trustedDomain);
-	virtual void addHTTPPostParams (HTAssocList *formfields, bool trustedDomain);
+	virtual void addHTTPPostParams (SFormFields &formfields, bool trustedDomain);
 	virtual std::string	home();
 	virtual void handle ();
 

code/ryzom/client/src/interface_v3/group_html_webig.cpp

@@ -307,19 +307,19 @@ void CGroupHTMLAuth::addHTTPGetParams (string &url, bool trustedDomain)
 
 // ***************************************************************************
 
-void CGroupHTMLAuth::addHTTPPostParams (HTAssocList *formfields, bool trustedDomain)
+void CGroupHTMLAuth::addHTTPPostParams (SFormFields &formfields, bool trustedDomain)
 {
 	if(!UserEntity) return;
 
 	uint32 cid = NetMngr.getLoginCookie().getUserId() * 16 + PlayerSelectedSlot;
-	HTParseFormInput(formfields, ("shardid="+toString(CharacterHomeSessionId)).c_str());
+	formfields.add("shardid", toString(CharacterHomeSessionId));
-	HTParseFormInput(formfields, ("name="+UserEntity->getLoginName().toUtf8()).c_str());
+	formfields.add("name", UserEntity->getLoginName().toUtf8());
-	HTParseFormInput(formfields, ("lang="+CI18N::getCurrentLanguageCode()).c_str());
+	formfields.add("lang", CI18N::getCurrentLanguageCode());
-	HTParseFormInput(formfields, "ig=1");
+	formfields.add("ig", "1");
 	if (trustedDomain)
 	{
-		HTParseFormInput(formfields, ("cid="+toString(cid)).c_str());
+		formfields.add("cid", toString(cid));
-		HTParseFormInput(formfields, ("authkey="+getWebAuthKey()).c_str());
+		formfields.add("authkey", getWebAuthKey());
 	}
 }
 
@@ -365,7 +365,7 @@ void CGroupHTMLWebIG::addHTTPGetParams (string &url, bool trustedDomain)
 
 // ***************************************************************************
 
-void CGroupHTMLWebIG::addHTTPPostParams (HTAssocList *formfields, bool trustedDomain)
+void CGroupHTMLWebIG::addHTTPPostParams (SFormFields &formfields, bool trustedDomain)
 {
 	CGroupHTMLAuth::addHTTPPostParams(formfields, trustedDomain);
 }

code/ryzom/client/src/interface_v3/group_html_webig.h

@@ -33,7 +33,7 @@ public:
 
 	// From CGroupHTML
 	virtual void addHTTPGetParams (std::string &url, bool trustedDomain);
-	virtual void addHTTPPostParams (HTAssocList *formfields, bool trustedDomain);
+	virtual void addHTTPPostParams (SFormFields &formfields, bool trustedDomain);
 	virtual std::string	home();
 	virtual void handle ();
 
@@ -55,7 +55,7 @@ public:
 
 	/// From CGroupHTMLAuth
 	virtual void addHTTPGetParams (std::string &url, bool trustedDomain);
-	virtual void addHTTPPostParams (HTAssocList *formfields, bool trustedDomain);
+	virtual void addHTTPPostParams (SFormFields &formfields, bool trustedDomain);
 	virtual std::string	home();
 	virtual void handle ();