mirror of
https://port.numenaute.org/aleajactaest/khanat-opennel-code.git
synced 2024-11-10 01:09:50 +00:00
Added: Basic queue for curl downloads
--HG-- branch : develop
This commit is contained in:
parent
9e7258d96e
commit
20c216ecda
2 changed files with 122 additions and 76 deletions
|
@ -820,7 +820,8 @@ namespace NLGUI
|
||||||
struct CDataDownload
|
struct CDataDownload
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
CDataDownload(CURL *c, const std::string &u, const std::string &d, FILE *f, TDataType t, CViewBase *i, const std::string &s, const std::string &m, const CStyleParams &style = CStyleParams()) : curl(c), url(u), dest(d), luaScript(s), md5sum(m), type(t), fp(f)
|
CDataDownload(const std::string &u, const std::string &d, TDataType t, CViewBase *i, const std::string &s, const std::string &m, const CStyleParams &style = CStyleParams())
|
||||||
|
: curl(NULL), fp(NULL), url(u), dest(d), type(t), luaScript(s), md5sum(m)
|
||||||
{
|
{
|
||||||
if (t == ImgType) imgs.push_back(CDataImageDownload(i, style));
|
if (t == ImgType) imgs.push_back(CDataImageDownload(i, style));
|
||||||
}
|
}
|
||||||
|
@ -840,6 +841,8 @@ namespace NLGUI
|
||||||
CURLM *MultiCurl;
|
CURLM *MultiCurl;
|
||||||
int RunningCurls;
|
int RunningCurls;
|
||||||
|
|
||||||
|
bool startCurlDownload(CDataDownload &download);
|
||||||
|
|
||||||
void initImageDownload();
|
void initImageDownload();
|
||||||
void checkImageDownload();
|
void checkImageDownload();
|
||||||
void addImageDownload(const std::string &url, CViewBase *img, const CStyleParams &style = CStyleParams());
|
void addImageDownload(const std::string &url, CViewBase *img, const CStyleParams &style = CStyleParams());
|
||||||
|
|
|
@ -265,6 +265,61 @@ namespace NLGUI
|
||||||
return dest;
|
return dest;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Add url to MultiCurl queue and return cURL handle
|
||||||
|
bool CGroupHTML::startCurlDownload(CDataDownload &download)
|
||||||
|
{
|
||||||
|
if (!MultiCurl)
|
||||||
|
{
|
||||||
|
nlwarning("Invalid MultiCurl handle, unable to download '%s'", download.url.c_str());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
string tmpdest = download.dest + ".tmp";
|
||||||
|
|
||||||
|
// erase the tmp file if exists
|
||||||
|
if (CFile::fileExists(tmpdest))
|
||||||
|
CFile::deleteFile(tmpdest);
|
||||||
|
|
||||||
|
FILE *fp = nlfopen (tmpdest, "wb");
|
||||||
|
if (fp == NULL)
|
||||||
|
{
|
||||||
|
nlwarning("Can't open file '%s' for writing: code=%d '%s'", tmpdest.c_str (), errno, strerror(errno));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
CURL *curl = curl_easy_init();
|
||||||
|
if (!curl)
|
||||||
|
{
|
||||||
|
fclose(fp);
|
||||||
|
CFile::deleteFile(tmpdest);
|
||||||
|
|
||||||
|
nlwarning("Creating cURL handle failed, unable to download '%s'", download.url.c_str());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
download.curl = curl;
|
||||||
|
download.fp = fp;
|
||||||
|
|
||||||
|
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_URL, download.url.c_str());
|
||||||
|
|
||||||
|
// limit curl to HTTP and HTTPS protocols only
|
||||||
|
curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
|
||||||
|
|
||||||
|
std::string userAgent = options.appName + "/" + options.appVersion;
|
||||||
|
curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgent.c_str());
|
||||||
|
|
||||||
|
sendCookies(curl, _DocumentDomain, _TrustedDomain);
|
||||||
|
|
||||||
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite);
|
||||||
|
|
||||||
|
curl_multi_add_handle(MultiCurl, curl);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
// Add a image download request in the multi_curl
|
// Add a image download request in the multi_curl
|
||||||
void CGroupHTML::addImageDownload(const string &url, CViewBase *img, const CStyleParams &style)
|
void CGroupHTML::addImageDownload(const string &url, CViewBase *img, const CStyleParams &style)
|
||||||
{
|
{
|
||||||
|
@ -285,55 +340,29 @@ namespace NLGUI
|
||||||
|
|
||||||
// use requested url for local name
|
// use requested url for local name
|
||||||
string dest = localImageName(url);
|
string dest = localImageName(url);
|
||||||
string tmpdest = localImageName(url)+".tmp";
|
|
||||||
#ifdef LOG_DL
|
#ifdef LOG_DL
|
||||||
nlwarning("add to download '%s' dest '%s' img %p", finalUrl.c_str(), dest.c_str(), img);
|
nlwarning("add to download '%s' dest '%s' img %p", finalUrl.c_str(), dest.c_str(), img);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
// erase the tmp file if exists
|
|
||||||
if (NLMISC::CFile::fileExists(tmpdest))
|
|
||||||
NLMISC::CFile::deleteFile(tmpdest);
|
|
||||||
|
|
||||||
if (!NLMISC::CFile::fileExists(dest))
|
if (!NLMISC::CFile::fileExists(dest))
|
||||||
{
|
{
|
||||||
if (!MultiCurl)
|
Curls.push_back(CDataDownload(finalUrl, dest, ImgType, img, "", "", style));
|
||||||
|
if (Curls.size() < options.curlMaxConnections) {
|
||||||
|
if (!startCurlDownload(Curls.back()))
|
||||||
{
|
{
|
||||||
nlwarning("Invalid MultiCurl handle, unable to download '%s'", finalUrl.c_str());
|
Curls.pop_back();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
CURL *curl = curl_easy_init();
|
|
||||||
if (!curl)
|
|
||||||
{
|
|
||||||
nlwarning("Creating cURL handle failed, unable to download '%s'", finalUrl.c_str());
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
FILE *fp = nlfopen(tmpdest, "wb");
|
|
||||||
if (fp == NULL)
|
|
||||||
{
|
|
||||||
curl_easy_cleanup(curl);
|
|
||||||
|
|
||||||
nlwarning("Can't open file '%s' for writing: code=%d '%s'", tmpdest.c_str (), errno, strerror(errno));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
|
|
||||||
curl_easy_setopt(curl, CURLOPT_URL, finalUrl.c_str());
|
|
||||||
|
|
||||||
std::string userAgent = options.appName + "/" + options.appVersion;
|
|
||||||
curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgent.c_str());
|
|
||||||
|
|
||||||
sendCookies(curl, _DocumentDomain, _TrustedDomain);
|
|
||||||
|
|
||||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
|
|
||||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite);
|
|
||||||
|
|
||||||
curl_multi_add_handle(MultiCurl, curl);
|
|
||||||
Curls.push_back(CDataDownload(curl, finalUrl, dest, fp, ImgType, img, "", "", style));
|
|
||||||
#ifdef LOG_DL
|
|
||||||
nlwarning("adding handle %x, %d curls", curl, Curls.size());
|
|
||||||
#endif
|
|
||||||
RunningCurls++;
|
RunningCurls++;
|
||||||
|
#ifdef LOG_DL
|
||||||
|
nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().curl, Curls.size());
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
nlwarning("(%s) download queued, %d curls", _Id.c_str(), Curls.size());
|
||||||
|
#endif
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
@ -378,15 +407,10 @@ namespace NLGUI
|
||||||
}
|
}
|
||||||
|
|
||||||
string dest = localBnpName(url);
|
string dest = localBnpName(url);
|
||||||
string tmpdest = localBnpName(url)+".tmp";
|
|
||||||
#ifdef LOG_DL
|
#ifdef LOG_DL
|
||||||
nlwarning("add to download '%s' dest '%s'", url.c_str(), dest.c_str());
|
nlwarning("add to download '%s' dest '%s'", url.c_str(), dest.c_str());
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
// erase the tmp file if exists
|
|
||||||
if (NLMISC::CFile::fileExists(tmpdest))
|
|
||||||
NLMISC::CFile::deleteFile(tmpdest);
|
|
||||||
|
|
||||||
// create/delete the local file
|
// create/delete the local file
|
||||||
if (NLMISC::CFile::fileExists(dest))
|
if (NLMISC::CFile::fileExists(dest))
|
||||||
{
|
{
|
||||||
|
@ -402,39 +426,23 @@ namespace NLGUI
|
||||||
}
|
}
|
||||||
if (action != "delete")
|
if (action != "delete")
|
||||||
{
|
{
|
||||||
if (!MultiCurl)
|
Curls.push_back(CDataDownload(url, dest, BnpType, NULL, script, md5sum));
|
||||||
|
if (Curls.size() < options.curlMaxConnections)
|
||||||
{
|
{
|
||||||
nlwarning("Invalid MultiCurl handle, unable to download '%s'", url.c_str());
|
if (!startCurlDownload(Curls.back()))
|
||||||
|
{
|
||||||
|
Curls.pop_back();
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
CURL *curl = curl_easy_init();
|
|
||||||
if (!curl)
|
|
||||||
{
|
|
||||||
nlwarning("Creating cURL handle failed, unable to download '%s'", url.c_str());
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
FILE *fp = nlfopen (tmpdest, "wb");
|
|
||||||
if (fp == NULL)
|
|
||||||
{
|
|
||||||
curl_easy_cleanup(curl);
|
|
||||||
nlwarning("Can't open file '%s' for writing: code=%d '%s'", tmpdest.c_str (), errno, strerror(errno));
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
|
|
||||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
|
||||||
|
|
||||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
|
|
||||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite);
|
|
||||||
|
|
||||||
curl_multi_add_handle(MultiCurl, curl);
|
|
||||||
Curls.push_back(CDataDownload(curl, url, dest, fp, BnpType, NULL, script, md5sum));
|
|
||||||
#ifdef LOG_DL
|
|
||||||
nlwarning("adding handle %x, %d curls", curl, Curls.size());
|
|
||||||
#endif
|
|
||||||
RunningCurls++;
|
RunningCurls++;
|
||||||
|
#ifdef LOG_DL
|
||||||
|
nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().curl, Curls.size());
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
nlwarning("(%s) download queued, %d curls", _Id.c_str(), Curls.size());
|
||||||
|
#endif
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
return true;
|
return true;
|
||||||
|
@ -631,7 +639,30 @@ namespace NLGUI
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
RunningCurls = NewRunningCurls;
|
RunningCurls = NewRunningCurls;
|
||||||
|
|
||||||
|
if (RunningCurls < options.curlMaxConnections)
|
||||||
|
{
|
||||||
|
for (vector<CDataDownload>::iterator it=Curls.begin(); it<Curls.end(); it++)
|
||||||
|
{
|
||||||
|
if (it->curl == NULL) {
|
||||||
|
#ifdef LOG_DL
|
||||||
|
nlwarning("(%s) starting new download '%s'", _Id.c_str(), it->url.c_str());
|
||||||
|
#endif
|
||||||
|
if (!startCurlDownload(*it))
|
||||||
|
{
|
||||||
|
Curls.erase(it);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
RunningCurls++;
|
||||||
|
if (RunningCurls >= options.curlMaxConnections)
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#ifdef LOG_DL
|
#ifdef LOG_DL
|
||||||
if (RunningCurls > 0 || !Curls.empty())
|
if (RunningCurls > 0 || !Curls.empty())
|
||||||
nlwarning("(%s) RunningCurls %d, _Curls %d", _Id.c_str(), RunningCurls, Curls.size());
|
nlwarning("(%s) RunningCurls %d, _Curls %d", _Id.c_str(), RunningCurls, Curls.size());
|
||||||
|
@ -4531,6 +4562,18 @@ namespace NLGUI
|
||||||
Curls[i].imgs.clear();
|
Curls[i].imgs.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// remove download that are still queued
|
||||||
|
for (vector<CDataDownload>::iterator it=Curls.begin(); it<Curls.end(); )
|
||||||
|
{
|
||||||
|
if (it->curl == NULL) {
|
||||||
|
#ifdef LOG_DL
|
||||||
|
nlwarning("Remove waiting curl download (%s)", it->url.c_str());
|
||||||
|
#endif
|
||||||
|
it = Curls.erase(it);
|
||||||
|
} else {
|
||||||
|
++it;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ***************************************************************************
|
// ***************************************************************************
|
||||||
|
|
Loading…
Reference in a new issue