mirror of
https://port.numenaute.org/aleajactaest/khanat-opennel-code.git
synced 2024-11-10 01:09:50 +00:00
Added: Basic queue for curl downloads
--HG-- branch : develop
This commit is contained in:
Nimetu
parent
9e7258d96e
commit
20c216ecda
2 changed files with 122 additions and 76 deletions
|
|
@ -820,7 +820,8 @@ namespace NLGUI
|
|||
struct CDataDownload
|
||||
{
|
||||
public:
|
||||
CDataDownload(CURL *c, const std::string &u, const std::string &d, FILE *f, TDataType t, CViewBase *i, const std::string &s, const std::string &m, const CStyleParams &style = CStyleParams()) : curl(c), url(u), dest(d), luaScript(s), md5sum(m), type(t), fp(f)
|
||||
CDataDownload(const std::string &u, const std::string &d, TDataType t, CViewBase *i, const std::string &s, const std::string &m, const CStyleParams &style = CStyleParams())
|
||||
: curl(NULL), fp(NULL), url(u), dest(d), type(t), luaScript(s), md5sum(m)
|
||||
{
|
||||
if (t == ImgType) imgs.push_back(CDataImageDownload(i, style));
|
||||
}
|
||||
|
|
@ -840,6 +841,8 @@ namespace NLGUI
|
|||
CURLM *MultiCurl;
|
||||
int RunningCurls;
|
||||
|
||||
bool startCurlDownload(CDataDownload &download);
|
||||
|
||||
void initImageDownload();
|
||||
void checkImageDownload();
|
||||
void addImageDownload(const std::string &url, CViewBase *img, const CStyleParams &style = CStyleParams());
|
||||
|
|
|
|||
|
|
@ -265,6 +265,61 @@ namespace NLGUI
|
|||
return dest;
|
||||
}
|
||||
|
||||
// Add url to MultiCurl queue and return cURL handle
|
||||
bool CGroupHTML::startCurlDownload(CDataDownload &download)
|
||||
{
|
||||
if (!MultiCurl)
|
||||
{
|
||||
nlwarning("Invalid MultiCurl handle, unable to download '%s'", download.url.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
string tmpdest = download.dest + ".tmp";
|
||||
|
||||
// erase the tmp file if exists
|
||||
if (CFile::fileExists(tmpdest))
|
||||
CFile::deleteFile(tmpdest);
|
||||
|
||||
FILE *fp = nlfopen (tmpdest, "wb");
|
||||
if (fp == NULL)
|
||||
{
|
||||
nlwarning("Can't open file '%s' for writing: code=%d '%s'", tmpdest.c_str (), errno, strerror(errno));
|
||||
return false;
|
||||
}
|
||||
|
||||
CURL *curl = curl_easy_init();
|
||||
if (!curl)
|
||||
{
|
||||
fclose(fp);
|
||||
CFile::deleteFile(tmpdest);
|
||||
|
||||
nlwarning("Creating cURL handle failed, unable to download '%s'", download.url.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
download.curl = curl;
|
||||
download.fp = fp;
|
||||
|
||||
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
|
||||
curl_easy_setopt(curl, CURLOPT_URL, download.url.c_str());
|
||||
|
||||
// limit curl to HTTP and HTTPS protocols only
|
||||
curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
|
||||
curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
|
||||
|
||||
std::string userAgent = options.appName + "/" + options.appVersion;
|
||||
curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgent.c_str());
|
||||
|
||||
sendCookies(curl, _DocumentDomain, _TrustedDomain);
|
||||
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite);
|
||||
|
||||
curl_multi_add_handle(MultiCurl, curl);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// Add a image download request in the multi_curl
|
||||
void CGroupHTML::addImageDownload(const string &url, CViewBase *img, const CStyleParams &style)
|
||||
{
|
||||
|
|
@ -285,55 +340,29 @@ namespace NLGUI
|
|||
|
||||
// use requested url for local name
|
||||
string dest = localImageName(url);
|
||||
string tmpdest = localImageName(url)+".tmp";
|
||||
#ifdef LOG_DL
|
||||
nlwarning("add to download '%s' dest '%s' img %p", finalUrl.c_str(), dest.c_str(), img);
|
||||
#endif
|
||||
|
||||
// erase the tmp file if exists
|
||||
if (NLMISC::CFile::fileExists(tmpdest))
|
||||
NLMISC::CFile::deleteFile(tmpdest);
|
||||
|
||||
if (!NLMISC::CFile::fileExists(dest))
|
||||
{
|
||||
if (!MultiCurl)
|
||||
Curls.push_back(CDataDownload(finalUrl, dest, ImgType, img, "", "", style));
|
||||
if (Curls.size() < options.curlMaxConnections) {
|
||||
if (!startCurlDownload(Curls.back()))
|
||||
{
|
||||
nlwarning("Invalid MultiCurl handle, unable to download '%s'", finalUrl.c_str());
|
||||
Curls.pop_back();
|
||||
return;
|
||||
}
|
||||
|
||||
CURL *curl = curl_easy_init();
|
||||
if (!curl)
|
||||
{
|
||||
nlwarning("Creating cURL handle failed, unable to download '%s'", finalUrl.c_str());
|
||||
return;
|
||||
}
|
||||
|
||||
FILE *fp = nlfopen(tmpdest, "wb");
|
||||
if (fp == NULL)
|
||||
{
|
||||
curl_easy_cleanup(curl);
|
||||
|
||||
nlwarning("Can't open file '%s' for writing: code=%d '%s'", tmpdest.c_str (), errno, strerror(errno));
|
||||
return;
|
||||
}
|
||||
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
|
||||
curl_easy_setopt(curl, CURLOPT_URL, finalUrl.c_str());
|
||||
|
||||
std::string userAgent = options.appName + "/" + options.appVersion;
|
||||
curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgent.c_str());
|
||||
|
||||
sendCookies(curl, _DocumentDomain, _TrustedDomain);
|
||||
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite);
|
||||
|
||||
curl_multi_add_handle(MultiCurl, curl);
|
||||
Curls.push_back(CDataDownload(curl, finalUrl, dest, fp, ImgType, img, "", "", style));
|
||||
#ifdef LOG_DL
|
||||
nlwarning("adding handle %x, %d curls", curl, Curls.size());
|
||||
#endif
|
||||
RunningCurls++;
|
||||
#ifdef LOG_DL
|
||||
nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().curl, Curls.size());
|
||||
}
|
||||
else
|
||||
{
|
||||
nlwarning("(%s) download queued, %d curls", _Id.c_str(), Curls.size());
|
||||
#endif
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -378,15 +407,10 @@ namespace NLGUI
|
|||
}
|
||||
|
||||
string dest = localBnpName(url);
|
||||
string tmpdest = localBnpName(url)+".tmp";
|
||||
#ifdef LOG_DL
|
||||
nlwarning("add to download '%s' dest '%s'", url.c_str(), dest.c_str());
|
||||
#endif
|
||||
|
||||
// erase the tmp file if exists
|
||||
if (NLMISC::CFile::fileExists(tmpdest))
|
||||
NLMISC::CFile::deleteFile(tmpdest);
|
||||
|
||||
// create/delete the local file
|
||||
if (NLMISC::CFile::fileExists(dest))
|
||||
{
|
||||
|
|
@ -402,39 +426,23 @@ namespace NLGUI
|
|||
}
|
||||
if (action != "delete")
|
||||
{
|
||||
if (!MultiCurl)
|
||||
Curls.push_back(CDataDownload(url, dest, BnpType, NULL, script, md5sum));
|
||||
if (Curls.size() < options.curlMaxConnections)
|
||||
{
|
||||
nlwarning("Invalid MultiCurl handle, unable to download '%s'", url.c_str());
|
||||
if (!startCurlDownload(Curls.back()))
|
||||
{
|
||||
Curls.pop_back();
|
||||
return false;
|
||||
}
|
||||
|
||||
CURL *curl = curl_easy_init();
|
||||
if (!curl)
|
||||
{
|
||||
nlwarning("Creating cURL handle failed, unable to download '%s'", url.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
FILE *fp = nlfopen (tmpdest, "wb");
|
||||
if (fp == NULL)
|
||||
{
|
||||
curl_easy_cleanup(curl);
|
||||
nlwarning("Can't open file '%s' for writing: code=%d '%s'", tmpdest.c_str (), errno, strerror(errno));
|
||||
return false;
|
||||
}
|
||||
|
||||
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
|
||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite);
|
||||
|
||||
curl_multi_add_handle(MultiCurl, curl);
|
||||
Curls.push_back(CDataDownload(curl, url, dest, fp, BnpType, NULL, script, md5sum));
|
||||
#ifdef LOG_DL
|
||||
nlwarning("adding handle %x, %d curls", curl, Curls.size());
|
||||
#endif
|
||||
RunningCurls++;
|
||||
#ifdef LOG_DL
|
||||
nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().curl, Curls.size());
|
||||
}
|
||||
else
|
||||
{
|
||||
nlwarning("(%s) download queued, %d curls", _Id.c_str(), Curls.size());
|
||||
#endif
|
||||
}
|
||||
}
|
||||
else
|
||||
return true;
|
||||
|
|
@ -631,7 +639,30 @@ namespace NLGUI
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
RunningCurls = NewRunningCurls;
|
||||
|
||||
if (RunningCurls < options.curlMaxConnections)
|
||||
{
|
||||
for (vector<CDataDownload>::iterator it=Curls.begin(); it<Curls.end(); it++)
|
||||
{
|
||||
if (it->curl == NULL) {
|
||||
#ifdef LOG_DL
|
||||
nlwarning("(%s) starting new download '%s'", _Id.c_str(), it->url.c_str());
|
||||
#endif
|
||||
if (!startCurlDownload(*it))
|
||||
{
|
||||
Curls.erase(it);
|
||||
break;
|
||||
}
|
||||
|
||||
RunningCurls++;
|
||||
if (RunningCurls >= options.curlMaxConnections)
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef LOG_DL
|
||||
if (RunningCurls > 0 || !Curls.empty())
|
||||
nlwarning("(%s) RunningCurls %d, _Curls %d", _Id.c_str(), RunningCurls, Curls.size());
|
||||
|
|
@ -4531,6 +4562,18 @@ namespace NLGUI
|
|||
Curls[i].imgs.clear();
|
||||
}
|
||||
|
||||
// remove download that are still queued
|
||||
for (vector<CDataDownload>::iterator it=Curls.begin(); it<Curls.end(); )
|
||||
{
|
||||
if (it->curl == NULL) {
|
||||
#ifdef LOG_DL
|
||||
nlwarning("Remove waiting curl download (%s)", it->url.c_str());
|
||||
#endif
|
||||
it = Curls.erase(it);
|
||||
} else {
|
||||
++it;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ***************************************************************************
|
||||
|
|
|
|||
Loading…
Reference in a new issue