michael_ul
/
odm
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Added mirror action and COPERNICUS source for it with caches, fucnctions, etc

master
Michael Uleysky 3 months ago
parent
be7103e718
commit
4274cda5b3
9 changed files with 756 additions and 1 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 23
      actions/actionmirror.h
  2. 144
      include/cache.h
  3. 29
      include/curlfuncs.h
  4. 62
      include/mirrorfuncs.h
  5. 282
      sources/COPERNICUS.cpp
  6. 36
      sources/COPERNICUS.h
  7. 10
      src/CMakeLists.txt
  8. 41
      src/curlfuncs.cpp
  9. 130
      src/mirrorfuncs.cpp

23
actions/actionmirror.h
Unescape View File

@ -0,0 +1,23 @@
#pragma once
#include "actiondep.h"
#include "merrors.h"
using michlib::message;
template<class T>
concept MirrorSupported = requires(T t, const CLArgs& args) {
{
t.Mirror(args)
} -> std::convertible_to<MString>;
};
ADD_ACTION(Mirror, mirror, MirrorSupported<Source>);
template<class D> MString ActionMirror::DoAction(const CLArgs& args, D& data)
{
//auto resop = data.Open(args);
//if(resop.Exist()) return "Can't open source: " + resop;
auto res = data.Mirror(args);
if(res.Exist()) return "Mirroring failed: " + res;
return "";
};

144
include/cache.h
Unescape View File

@ -0,0 +1,144 @@
#pragma once
#include "MString.h"
#include <sqlite3.h>
#include <time.h>
#include <variant>
using michlib::MString;
class GenericCache
{
public:
virtual bool Put(const MString& key, const MString& value, size_t ttl) const = 0;
virtual std::pair<MString, bool> Get(const MString& key) const = 0;
virtual ~GenericCache() {}
};
class FakeCache: public GenericCache
{
public:
virtual bool Put([[maybe_unused]] const MString& key, [[maybe_unused]] const MString& value, [[maybe_unused]] size_t ttl) const override { return false; }
virtual std::pair<MString, bool> Get([[maybe_unused]] const MString& key) const override { return {"", false}; }
virtual ~FakeCache() override {}
};
class SQLiteCache: public GenericCache
{
sqlite3* db = nullptr;
public:
bool Init(const MString& name)
{
Close();
auto ret = sqlite3_open(name.Buf(), &db);
if(ret != SQLITE_OK)
{
Close();
return false;
}
// Create table
sqlite3_stmt* sqst;
int i;
i = sqlite3_prepare_v2(db,
"CREATE TABLE IF NOT EXISTS `cache`('key' TEXT PRIMARY KEY ON CONFLICT REPLACE NOT NULL ON CONFLICT FAIL, 'value' BLOB NOT NULL ON CONFLICT FAIL, "
"'exptime' INTEGER NOT NULL ON CONFLICT FAIL) WITHOUT ROWID, STRICT;",
-1, &sqst, 0);
i = sqlite3_step(sqst);
if(i != SQLITE_DONE)
{
sqlite3_finalize(sqst);
Close();
return false;
}
sqlite3_finalize(sqst);
sqlite3_busy_timeout(db, 1000);
return true;
}
void Close()
{
if(db != nullptr) sqlite3_close(db);
db = nullptr;
}
virtual bool Put(const MString& key, const MString& value, size_t ttl) const override
{
if(!*this) return false;
sqlite3_stmt* sqst = nullptr;
int i = SQLITE_OK;
if(i == SQLITE_OK) i = sqlite3_prepare_v2(db, "INSERT OR REPLACE into `cache` VALUES(?1,?2,?3);", -1, &sqst, 0);
if(i == SQLITE_OK) i = sqlite3_bind_text(sqst, 1, key.Buf(), -1, SQLITE_STATIC);
if(i == SQLITE_OK) i = sqlite3_bind_blob64(sqst, 2, value.Buf(), value.Len(), SQLITE_STATIC);
if(i == SQLITE_OK) i = sqlite3_bind_int64(sqst, 3, time(nullptr) + ttl);
if(i == SQLITE_OK) i = sqlite3_step(sqst);
sqlite3_finalize(sqst);
return i == SQLITE_OK;
}
virtual std::pair<MString, bool> Get(const MString& key) const override
{
if(!*this) return {"", false};
sqlite3_stmt* sqst = nullptr;
int i = SQLITE_OK;
if(i == SQLITE_OK) i = sqlite3_prepare_v2(db, "SELECT value from `cache` WHERE key=?1 AND exptime>?2;", -1, &sqst, 0);
if(i == SQLITE_OK) i = sqlite3_bind_text(sqst, 1, key.Buf(), -1, SQLITE_STATIC);
if(i == SQLITE_OK) i = sqlite3_bind_int64(sqst, 2, time(nullptr));
if(i == SQLITE_OK) i = sqlite3_step(sqst);
if(i == SQLITE_ROW)
{
auto p = sqlite3_column_blob(sqst, 0);
auto sz = sqlite3_column_bytes(sqst, 0);
if(p != nullptr)
{
MString out(p, sz);
sqlite3_finalize(sqst);
return {std::move(out), true};
}
}
sqlite3_finalize(sqst);
return {"", false};
}
virtual ~SQLiteCache() override
{
if(!*this) return;
sqlite3_stmt* sqst = nullptr;
int i = SQLITE_OK;
if(i == SQLITE_OK) i = sqlite3_prepare_v2(db, "DELETE from `cache` WHERE exptime<?1;", -1, &sqst, 0);
if(i == SQLITE_OK) i = sqlite3_bind_int64(sqst, 1, time(nullptr));
if(i == SQLITE_OK) i = sqlite3_step(sqst);
sqlite3_finalize(sqst);
}
explicit operator bool() const { return db != nullptr; }
};
inline GenericCache* CreateCache(const MString& cachedesc)
{
auto i = cachedesc.GetPos(':');
if(i == 0)
{
if(cachedesc == "no") return new FakeCache;
return nullptr;
}
auto name = cachedesc.SubStr(1, i - 1);
auto par = cachedesc.SubStr(i + 1, cachedesc.Len() - i);
if(name == "sqlite")
{
auto ret = new SQLiteCache;
ret->Init(par);
if(*ret) return ret;
delete ret;
}
return nullptr;
}

29
include/curlfuncs.h
Unescape View File

@ -0,0 +1,29 @@
#pragma once
#include "MString.h"
#include <curl/curl.h>
#include <memory>
using michlib::MString;
class CURLRAIIDT
{
public:
// TODO: make static
void operator()(CURL* c) { curl_easy_cleanup(c); }
};
class CURLRAII: public std::unique_ptr<CURL, CURLRAIIDT>
{
public:
CURLRAII() { reset(curl_easy_init()); }
operator CURL*() const { return get(); }
};
// Curl writeback function, write to MString
size_t Write2String(char* ptr, size_t size, size_t n, void* data);
// Curl writeback function, write to file descriptor
size_t Write2File(char* ptr, size_t size, size_t n, void* data);
// Get content of url to MString
std::pair<MString, CURLcode> GetUrl(const CURLRAII& chandle, const MString& url);

62
include/mirrorfuncs.h
Unescape View File

@ -0,0 +1,62 @@
#pragma once
#include "curlfuncs.h"
#include "mdatetime.h"
#include <dirent.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <vector>
using michlib::MDateTime;
class DIRRAIIDT
{
public:
// TODO: make static
void operator()(DIR* d) { closedir(d); }
};
class DIRRAII: public std::unique_ptr<DIR, DIRRAIIDT>
{
public:
operator DIR*() const { return get(); }
};
struct FileInfo
{
MString url;
MString name;
MDateTime mtime;
size_t size;
};
// Remove last element from path
inline MString DirName(const MString& name)
{
auto p = name.GetPos('/', false);
if(p == 0) return name;
return name.SubStr(1, p - 1);
}
// Get last element from path
inline MString FileName(const MString& name)
{
auto p = name.GetPos('/', false);
if(p == 0) return name;
return name.SubStr(p + 1, name.Len() - p);
}
// Check and, if necessary, create the path to the file
bool MakePath(const MString& dname);
// Get local file list
std::pair<std::vector<struct FileInfo>, MString> ReadLocalFileList(const MString& dir, const MString& path = "");
// Download file to the local mirror
MString DownloadFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const MString& root);
// Remove file from the local mirror
MString RemoveFile(const struct FileInfo& linfo);
// Updare file in the local mirror
MString UpdateFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const struct FileInfo& linfo, const MString& root);

282
sources/COPERNICUS.cpp
Unescape View File

@ -0,0 +1,282 @@
#define MICHLIB_NOSOURCE
#include "COPERNICUS.h"
#include "mirrorfuncs.h"
#include <libxml/parser.h>
#include <libxml/tree.h>
using michlib::GPL;
const MString COPERNICUSData::caturl = "https://stac.marine.copernicus.eu/metadata/catalog.stac.json";
std::pair<Json::Value, MString> COPERNICUSData::GetJSON(const MString& url)
{
Json::Reader reader;
Json::Value obj;
MString content;
auto [val, suc] = cache->Get(url);
if(suc)
content = std::move(val);
else
{
michlib::message(url + " not found in cache, downloading");
auto [out, res] = GetUrl(chandle, url);
if(res != CURLE_OK) return {obj, MString("Can't download JSON: ") + curlerr};
cache->Put(url, out, 3600);
content = std::move(out);
}
reader.parse(content.Buf(), content.Buf() + content.Len(), obj, false);
return {obj, ""};
}
MString COPERNICUSData::ReadURL(const Json::Value& cat, const MString& prod)
{
const auto& links = cat["links"];
if(links.type() != Json::arrayValue) return "";
for(Json::ArrayIndex i = 0; i < links.size(); i++)
{
const auto& titl = links[i]["title"];
const auto& href = links[i]["href"];
if(titl.type() == Json::stringValue && href.type() == Json::stringValue)
{
MString str(titl.asString().c_str());
if(str == prod) return MString(href.asString().c_str());
}
}
return "";
}
std::pair<std::vector<struct FileInfo>, MString> COPERNICUSData::ReadRemoteFileList(const MString& url)
{
LIBXML_TEST_VERSION
std::vector<struct FileInfo> out;
MString bucket, prefix;
// Split url on prefix and bucket
{
size_t pos = url.Len();
size_t count = 0;
for(size_t i = 0; i < url.Len(); i++)
{
if(url[i] == '/') count++;
if(count == 4)
{
pos = i;
break;
}
}
if(pos == url.Len()) return {out, "Can't parse url: " + url};
bucket = url.SubStr(1, pos);
prefix = url.SubStr(pos + 2, url.Len() - pos - 1);
}
MString cont;
bool next = true;
while(next)
{
MString url = bucket + "?list-type=2&prefix=" + prefix;
if(cont.Exist()) url += "&continuation-token=" + cont;
cont = "";
auto [data, res] = GetUrl(chandle, url);
if(res != CURLE_OK) return {out, MString("Can't download ") + url + ": " + curlerr};
xmlDocPtr doc = xmlReadMemory(data.Buf(), data.Len(), "data.xml", nullptr, 0);
if(doc == nullptr) return {out, MString("Can't download ") + url + ": XML parse error"};
auto cur = xmlDocGetRootElement(doc);
if(cur == nullptr)
{
xmlFreeDoc(doc);
return {out, MString("Can't download ") + url + ": empty XML"};
}
if(xmlStrEqual(cur->name, (const xmlChar*)"ListBucketResult") == 0)
{
xmlFreeDoc(doc);
return {out, MString("Can't download ") + url + ": unknown XML"};
}
for(const auto* n = cur->children; n; n = n->next)
{
if(xmlStrEqual(n->name, (const xmlChar*)"NextContinuationToken") == 1)
{
auto* content = xmlNodeGetContent(n);
cont = (char*)content;
xmlFree(content);
}
if(xmlStrEqual(n->name, (const xmlChar*)"Contents") == 1)
{
MString fname;
MDateTime mtime;
size_t size = 0;
for(const auto* c = n->children; c; c = c->next)
{
if(xmlStrEqual(c->name, (const xmlChar*)"Key") == 1)
{
auto* content = xmlNodeGetContent(c);
fname = (char*)content;
xmlFree(content);
}
if(xmlStrEqual(c->name, (const xmlChar*)"LastModified") == 1)
{
auto* content = xmlNodeGetContent(c);
mtime.FromString((char*)content);
xmlFree(content);
}
if(xmlStrEqual(c->name, (const xmlChar*)"Size") == 1)
{
auto* content = xmlNodeGetContent(c);
size = MString((char*)content).ToInteger<size_t>();
xmlFree(content);
}
}
out.emplace_back(bucket + "/" + fname, fname.SubStr(prefix.Len() + 2, fname.Len() - prefix.Len() - 1), mtime, size);
}
}
xmlFreeDoc(doc);
next = cont.Exist();
}
std::sort(out.begin(), out.end(), [](const struct FileInfo& a, const struct FileInfo& b) { return a.name < b.name; });
return {out, ""};
}
MString COPERNICUSData::Mirror(const CLArgs& args)
{
GPL.UsePrefix("COPERNICUS");
// Local directory
MString mirrorroot = GPL.ParameterSValue("MirrorTo", "");
if(!mirrorroot.Exist()) return "Local mirror directory not specified";
// Cache
cache.reset(CreateCache(GPL.ParameterSValue("Cache", "")));
if(!cache)
{
michlib::errmessage("Can't init cache");
cache.reset(new FakeCache);
}
curl_easy_setopt(chandle, CURLOPT_ERRORBUFFER, curlerr);
if(!args.contains("product")) return "Copernicus product not specified";
MString prod = args.at("product");
Json::Value product;
MString produrl;
// Get catalog
{
auto [cat, err] = GetJSON(caturl);
if(err.Exist()) return "Can't download catalog: " + err;
if(cat["title"].type() != Json::stringValue || cat["title"].asString() != "Copernicus Marine Data Store") return "Can't parse catalog";
catalog = std::move(cat);
}
// Get product
{
auto url = ReadURL(catalog, prod);
if(!url.Exist()) return "Url for product " + prod + " not found in catalog";
produrl = DirName(caturl) + "/" + url;
auto [pr, err] = GetJSON(produrl);
if(err.Exist()) return "Can't download product information from " + produrl + ": " + err;
product = std::move(pr);
}
std::vector<MString> dsets;
if(args.contains("dataset"))
dsets.push_back(args.at("dataset"));
else
{
const auto& links = product["links"];
if(links.type() != Json::arrayValue) return "Can't find information about datasets";
for(Json::ArrayIndex i = 0; i < links.size(); i++)
{
const auto& rel = links[i]["rel"];
const auto& titl = links[i]["title"];
if(rel.type() == Json::stringValue && titl.type() == Json::stringValue && rel.asString() == "item") dsets.push_back(titl.asString().c_str());
}
}
for(const auto& dset: dsets)
{
michlib::message("Mirroring " + dset);
auto url = ReadURL(product, dset);
if(!url.Exist()) return "Url for dataset " + dset + " not found in product description";
MString dseturl = DirName(produrl) + "/" + url;
auto [ds, err] = GetJSON(dseturl);
if(err.Exist()) return "Can't download dataset information from " + dseturl + ": " + err;
const auto& href = ds["assets"]["native"]["href"];
if(href.type() != Json::stringValue) return "Can't find data for dataset " + dset + " from product " + prod;
url = href.asString().c_str();
MString locroot = mirrorroot + "/" + prod + "/" + dset;
auto [lfiles, lerr] = ReadLocalFileList(locroot);
if(lerr.Exist()) return lerr;
auto [rfiles, rerr] = ReadRemoteFileList(url);
if(rerr.Exist()) return rerr;
std::vector<size_t> down, rem;
std::vector<std::pair<size_t, size_t>> upd;
{
size_t rpos = 0, lpos = 0;
while(rpos != rfiles.size() || lpos != lfiles.size())
{
if(rpos == rfiles.size())
while(lpos != lfiles.size()) rem.push_back(lpos++);
if(lpos == lfiles.size())
while(rpos != rfiles.size()) down.push_back(rpos++);
if(rpos == rfiles.size() || lpos == lfiles.size()) continue;
if(rfiles[rpos].name < lfiles[lpos].name)
down.push_back(rpos++);
else if(lfiles[lpos].name < rfiles[rpos].name)
rem.push_back(lpos++);
else
{
auto delta = rfiles[rpos].mtime.Epoch() - lfiles[lpos].mtime.Epoch();
if(delta < 0) delta = -delta;
if(delta > 0 || rfiles[rpos].size != lfiles[lpos].size) upd.emplace_back(rpos, lpos);
lpos++;
rpos++;
}
}
}
michlib::message(MString("New files: ") + down.size());
michlib::message(MString("Obsolete files: ") + rem.size());
michlib::message(MString("Modified files: ") + upd.size());
for(size_t i = 0; i < down.size(); i++)
{
size_t ri = down[i];
auto err = DownloadFile(chandle, rfiles[ri], locroot);
if(err.Exist()) return err;
}
for(size_t i = 0; i < rem.size(); i++)
{
size_t li = rem[i];
auto err = RemoveFile(lfiles[li]);
if(err.Exist()) return err;
}
for(size_t i = 0; i < upd.size(); i++)
{
size_t ri = upd[i].first;
size_t li = upd[i].second;
auto err = UpdateFile(chandle, rfiles[ri], lfiles[li], locroot);
if(err.Exist()) return err;
}
}
return "";
}

36
sources/COPERNICUS.h
Unescape View File

@ -0,0 +1,36 @@
#pragma once
#include "ParseArgs.h"
#include "cache.h"
#include "curlfuncs.h"
#include "mdatetime.h"
#include <json/json.h>
using michlib::MDateTime;
using michlib::MString;
class COPERNICUSData
{
static const MString caturl;
std::unique_ptr<GenericCache> cache;
CURLRAII chandle;
Json::Value catalog;
char curlerr[CURL_ERROR_SIZE];
// Get url for product or dataset from catalog
static MString ReadURL(const Json::Value& cat, const MString& prod);
// Download JSON from url
std::pair<Json::Value, MString> GetJSON(const MString& url);
// Get remote file list from url
std::pair<std::vector<struct FileInfo>,MString> ReadRemoteFileList(const MString& url);
public:
static constexpr const char* name = "COPERNICUS";
COPERNICUSData() = default;
// Main mirror function
MString Mirror(const CLArgs& args);
};

10
src/CMakeLists.txt
Unescape View File

@ -2,15 +2,23 @@ set(EXENAME odm)
set(ACTIONLISTINC ${CMAKE_CURRENT_BINARY_DIR}/../include/actionlist.h) # Include actions files and define the actions classes list
set(DATALISTINC ${CMAKE_CURRENT_BINARY_DIR}/../include/datalist.h) # Include data sources files and define the data sources classes list
find_package(PkgConfig REQUIRED)
find_library(netcdf netcdf REQUIRED)
find_package(OpenMP REQUIRED)
find_package(CURL REQUIRED)
find_package(LibXml2 REQUIRED)
find_package(SQLite3 REQUIRED)
pkg_check_modules(JSONCPP REQUIRED jsoncpp)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
include_directories(${JSONCPP_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIRS} ${SQLite3_INCLUDE_DIRS})
file(GLOB srcs CONFIGURE_DEPENDS *.cpp)
add_executable(${EXENAME} ${srcs} ${ACTIONLISTINC} ${SOURCELISTINC})
target_include_directories(${EXENAME} PRIVATE ../michlib/michlib ${CMAKE_CURRENT_BINARY_DIR}/../include)
target_link_libraries(${EXENAME} ${linker_options} ${netcdf} OpenMP::OpenMP_CXX teos)
target_link_libraries(${EXENAME} ${linker_options} ${netcdf} OpenMP::OpenMP_CXX CURL::libcurl ${JSONCPP_LINK_LIBRARIES} LibXml2::LibXml2 SQLite::SQLite3 teos)
set_target_properties(${EXENAME} PROPERTIES POSITION_INDEPENDENT_CODE ON)
install(TARGETS ${EXENAME})

41
src/curlfuncs.cpp
Unescape View File

@ -0,0 +1,41 @@
#define MICHLIB_NOSOURCE
#include "curlfuncs.h"
#include <unistd.h>
using michlib::pointer_cast;
using michlib::uint1;
size_t Write2String(char* ptr, size_t size, size_t n, void* data)
{
MString* out = pointer_cast<MString*>(data);
*out += MString(ptr, size * n);
return size * n;
}
size_t Write2File(char* ptr, size_t size, size_t n, void* data)
{
const int* fd = pointer_cast<const int*>(data);
size_t count = size * n;
const uint1* buf = pointer_cast<const uint1*>(ptr);
while(count != 0)
{
auto wr = write(*fd, buf, count);
if(wr == -1) return 0;
count -= wr;
buf += wr;
}
return size * n;
}
std::pair<MString, CURLcode> GetUrl(const CURLRAII& chandle, const MString& url)
{
MString out;
curl_easy_setopt(chandle, CURLOPT_URL, url.Buf());
curl_easy_setopt(chandle, CURLOPT_WRITEFUNCTION, Write2String);
curl_easy_setopt(chandle, CURLOPT_WRITEDATA, &out);
auto res = curl_easy_perform(chandle);
return {out, res};
}

130
src/mirrorfuncs.cpp
Unescape View File

@ -0,0 +1,130 @@
#define MICHLIB_NOSOURCE
#include "mirrorfuncs.h"
#include "StringFunctions.h"
#include "filehelpers.h"
#include "merrors.h"
using michlib::FD;
using michlib::message;
bool MakePath(const MString& dname)
{
struct stat st;
int ret = stat(dname.Buf(), &st);
if(ret == 0) return S_ISDIR(st.st_mode);
auto dirs = michlib::Split_on_words(dname, "/", false);
MString cdir = "";
for(const auto& dir: dirs)
{
cdir += "/" + dir;
ret = stat(cdir.Buf(), &st);
if(ret == 0 && S_ISDIR(st.st_mode)) continue;
if(ret == 0 && !S_ISDIR(st.st_mode)) return false;
ret = mkdir(cdir.Buf(), 0755);
if(ret != 0) return false;
}
return true;
}
std::pair<std::vector<struct FileInfo>, MString> ReadLocalFileList(const MString& dir, const MString& path)
{
std::vector<struct FileInfo> out;
DIRRAII dhandle;
MakePath(dir);
dhandle.reset(opendir(dir.Buf()));
if(!dhandle) return {out, "Can't open directory " + path + (path.Exist() ? "/" : "") + dir};
int dfd = dirfd(dhandle);
errno = 0;
struct dirent* dent = readdir(dhandle);
if(errno != 0) return {out, "Can't read directory " + path + (path.Exist() ? "/" : "") + dir};
struct stat st;
do {
if(dent->d_name[0] != '.')
{
int ret = fstatat(dfd, dent->d_name, &st, AT_SYMLINK_NOFOLLOW);
if(ret != 0) return {out, "Can't stat " + path + "/" + dir + "/" + dent->d_name};
if(S_ISDIR(st.st_mode)) // Directory, recurse
{
auto [list, err] = ReadLocalFileList(dir + "/" + dent->d_name, path + (path.Exist() ? "/" : "") + dent->d_name);
if(err.Exist()) return {out, err};
out.insert(out.end(), list.begin(), list.end());
}
if(S_ISREG(st.st_mode)) // Regular file
{
out.emplace_back(dir + "/" + dent->d_name, path + (path.Exist() ? "/" : "") + dent->d_name, MDateTime(st.st_mtim.tv_sec, st.st_mtim.tv_nsec), st.st_size);
}
// Ignore non-directories, non-files
}
dent = readdir(dhandle);
} while(dent != nullptr || errno != 0);
if(errno != 0) return {out, "Can't read directory " + path + "/" + dir};
std::sort(out.begin(), out.end(), [](const struct FileInfo& a, const struct FileInfo& b) { return a.name < b.name; });
return {out, ""};
}
MString DownloadFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const MString& root)
{
message("Downloading " + rinfo.url);
MString dname = DirName(rinfo.name), fname = FileName(rinfo.name);
FD fd;
if(!MakePath(root + "/" + dname)) return "Can't create directory " + root + "/" + dname;
fd.Reset(creat((root + "/" + rinfo.name).Buf(), 0644));
if(!fd) return "Can't create file " + root + "/" + rinfo.name;
char errbuf[CURL_ERROR_SIZE];
int cfd = fd.Get();
curl_easy_setopt(chandle, CURLOPT_ERRORBUFFER, errbuf);
curl_easy_setopt(chandle, CURLOPT_WRITEFUNCTION, Write2File);
curl_easy_setopt(chandle, CURLOPT_WRITEDATA, &cfd);
curl_easy_setopt(chandle, CURLOPT_URL, rinfo.url.Buf());
auto res = curl_easy_perform(chandle);
if(res != CURLE_OK)
{
unlink((root + "/" + rinfo.name).Buf());
return MString("Can't download file: ") + errbuf;
}
{
struct timespec times[2];
times[0].tv_sec = times[1].tv_sec = rinfo.mtime.Epoch();
times[0].tv_nsec = times[1].tv_nsec = 0;
int ret = futimens(fd, times);
if(ret != 0)
{
unlink((root + "/" + rinfo.name).Buf());
return "Can't set mtime for file: " + root + "/" + rinfo.name;
}
}
return "";
}
MString RemoveFile(const struct FileInfo& linfo)
{
message("Remove " + linfo.url);
int ret = unlink(linfo.url.Buf());
if(ret != 0) return "Can't remove file " + linfo.url;
return "";
}
MString UpdateFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const struct FileInfo& linfo, const MString& root)
{
MString err;
message("Update " + linfo.url);
err = RemoveFile(linfo);
if(err.Exist()) return err;
err = DownloadFile(chandle, rinfo, root);
if(err.Exist()) return err;
return "";
}
Write Preview
Loading…
Cancel
Save