Added mirror action and COPERNICUS source for it with caches, fucnctions, etc

actions/actionmirror.h

@@ -0,0 +1,23 @@
+#pragma once
+#include "actiondep.h"
+#include "merrors.h"
+
+using michlib::message;
+
+template<class T>
+concept MirrorSupported = requires(T t, const CLArgs& args) {
+ {
+  t.Mirror(args)
+ } -> std::convertible_to<MString>;
+};
+
+ADD_ACTION(Mirror, mirror, MirrorSupported<Source>);
+
+template<class D> MString ActionMirror::DoAction(const CLArgs& args, D& data)
+{
+ //auto resop = data.Open(args);
+ //if(resop.Exist()) return "Can't open source: " + resop;
+ auto res = data.Mirror(args);
+ if(res.Exist()) return "Mirroring failed: " + res;
+ return "";
+};

include/cache.h

@@ -0,0 +1,144 @@
+#pragma once
+#include "MString.h"
+#include <sqlite3.h>
+#include <time.h>
+#include <variant>
+
+using michlib::MString;
+
+class GenericCache
+{
+ public:
+ virtual bool                     Put(const MString& key, const MString& value, size_t ttl) const = 0;
+ virtual std::pair<MString, bool> Get(const MString& key) const                                   = 0;
+ virtual ~GenericCache() {}
+};
+
+class FakeCache: public GenericCache
+{
+ public:
+ virtual bool                     Put([[maybe_unused]] const MString& key, [[maybe_unused]] const MString& value, [[maybe_unused]] size_t ttl) const override { return false; }
+ virtual std::pair<MString, bool> Get([[maybe_unused]] const MString& key) const override { return {"", false}; }
+ virtual ~FakeCache() override {}
+};
+
+class SQLiteCache: public GenericCache
+{
+ sqlite3* db = nullptr;
+
+ public:
+ bool Init(const MString& name)
+ {
+  Close();
+  auto ret = sqlite3_open(name.Buf(), &db);
+  if(ret != SQLITE_OK)
+  {
+   Close();
+   return false;
+  }
+  // Create table
+  sqlite3_stmt* sqst;
+  int           i;
+
+  i = sqlite3_prepare_v2(db,
+                         "CREATE TABLE IF NOT EXISTS `cache`('key' TEXT PRIMARY KEY ON CONFLICT REPLACE NOT NULL ON CONFLICT FAIL, 'value' BLOB NOT NULL ON CONFLICT FAIL, "
+                         "'exptime' INTEGER NOT NULL ON CONFLICT FAIL) WITHOUT ROWID, STRICT;",
+                         -1, &sqst, 0);
+  i = sqlite3_step(sqst);
+  if(i != SQLITE_DONE)
+  {
+   sqlite3_finalize(sqst);
+   Close();
+   return false;
+  }
+  sqlite3_finalize(sqst);
+  sqlite3_busy_timeout(db, 1000);
+  return true;
+ }
+
+ void Close()
+ {
+  if(db != nullptr) sqlite3_close(db);
+  db = nullptr;
+ }
+
+ virtual bool Put(const MString& key, const MString& value, size_t ttl) const override
+ {
+  if(!*this) return false;
+  sqlite3_stmt* sqst = nullptr;
+  int           i    = SQLITE_OK;
+
+  if(i == SQLITE_OK) i = sqlite3_prepare_v2(db, "INSERT OR REPLACE into `cache` VALUES(?1,?2,?3);", -1, &sqst, 0);
+  if(i == SQLITE_OK) i = sqlite3_bind_text(sqst, 1, key.Buf(), -1, SQLITE_STATIC);
+  if(i == SQLITE_OK) i = sqlite3_bind_blob64(sqst, 2, value.Buf(), value.Len(), SQLITE_STATIC);
+  if(i == SQLITE_OK) i = sqlite3_bind_int64(sqst, 3, time(nullptr) + ttl);
+  if(i == SQLITE_OK) i = sqlite3_step(sqst);
+  sqlite3_finalize(sqst);
+
+  return i == SQLITE_OK;
+ }
+
+ virtual std::pair<MString, bool> Get(const MString& key) const override
+ {
+  if(!*this) return {"", false};
+
+  sqlite3_stmt* sqst = nullptr;
+  int           i    = SQLITE_OK;
+
+  if(i == SQLITE_OK) i = sqlite3_prepare_v2(db, "SELECT value from `cache` WHERE key=?1 AND exptime>?2;", -1, &sqst, 0);
+  if(i == SQLITE_OK) i = sqlite3_bind_text(sqst, 1, key.Buf(), -1, SQLITE_STATIC);
+  if(i == SQLITE_OK) i = sqlite3_bind_int64(sqst, 2, time(nullptr));
+  if(i == SQLITE_OK) i = sqlite3_step(sqst);
+  if(i == SQLITE_ROW)
+  {
+   auto p  = sqlite3_column_blob(sqst, 0);
+   auto sz = sqlite3_column_bytes(sqst, 0);
+   if(p != nullptr)
+   {
+    MString out(p, sz);
+    sqlite3_finalize(sqst);
+    return {std::move(out), true};
+   }
+  }
+  sqlite3_finalize(sqst);
+  return {"", false};
+ }
+
+ virtual ~SQLiteCache() override
+ {
+  if(!*this) return;
+
+  sqlite3_stmt* sqst = nullptr;
+  int           i    = SQLITE_OK;
+
+  if(i == SQLITE_OK) i = sqlite3_prepare_v2(db, "DELETE from `cache` WHERE exptime<?1;", -1, &sqst, 0);
+  if(i == SQLITE_OK) i = sqlite3_bind_int64(sqst, 1, time(nullptr));
+  if(i == SQLITE_OK) i = sqlite3_step(sqst);
+  sqlite3_finalize(sqst);
+ }
+
+ explicit operator bool() const { return db != nullptr; }
+};
+
+inline GenericCache* CreateCache(const MString& cachedesc)
+{
+ auto i = cachedesc.GetPos(':');
+ if(i == 0)
+ {
+  if(cachedesc == "no") return new FakeCache;
+  return nullptr;
+ }
+
+ auto name = cachedesc.SubStr(1, i - 1);
+ auto par  = cachedesc.SubStr(i + 1, cachedesc.Len() - i);
+
+ if(name == "sqlite")
+ {
+  auto ret = new SQLiteCache;
+  ret->Init(par);
+  if(*ret) return ret;
+  delete ret;
+ }
+
+ return nullptr;
+}

include/curlfuncs.h

@@ -0,0 +1,29 @@
+#pragma once
+#include "MString.h"
+#include <curl/curl.h>
+#include <memory>
+
+using michlib::MString;
+
+class CURLRAIIDT
+{
+ public:
+ // TODO: make static
+ void operator()(CURL* c) { curl_easy_cleanup(c); }
+};
+
+class CURLRAII: public std::unique_ptr<CURL, CURLRAIIDT>
+{
+ public:
+ CURLRAII() { reset(curl_easy_init()); }
+ operator CURL*() const { return get(); }
+};
+
+// Curl writeback function, write to MString
+size_t Write2String(char* ptr, size_t size, size_t n, void* data);
+
+// Curl writeback function, write to file descriptor
+size_t Write2File(char* ptr, size_t size, size_t n, void* data);
+
+// Get content of url to MString
+std::pair<MString, CURLcode> GetUrl(const CURLRAII& chandle, const MString& url);

include/mirrorfuncs.h

@@ -0,0 +1,62 @@
+#pragma once
+#include "curlfuncs.h"
+#include "mdatetime.h"
+#include <dirent.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <vector>
+
+using michlib::MDateTime;
+
+class DIRRAIIDT
+{
+ public:
+ // TODO: make static
+ void operator()(DIR* d) { closedir(d); }
+};
+
+class DIRRAII: public std::unique_ptr<DIR, DIRRAIIDT>
+{
+ public:
+ operator DIR*() const { return get(); }
+};
+
+struct FileInfo
+{
+ MString   url;
+ MString   name;
+ MDateTime mtime;
+ size_t    size;
+};
+
+// Remove last element from path
+inline MString DirName(const MString& name)
+{
+ auto p = name.GetPos('/', false);
+ if(p == 0) return name;
+ return name.SubStr(1, p - 1);
+}
+
+// Get last element from path
+inline MString FileName(const MString& name)
+{
+ auto p = name.GetPos('/', false);
+ if(p == 0) return name;
+ return name.SubStr(p + 1, name.Len() - p);
+}
+
+// Check and, if necessary, create the path to the file
+bool MakePath(const MString& dname);
+
+// Get local file list
+std::pair<std::vector<struct FileInfo>, MString> ReadLocalFileList(const MString& dir, const MString& path = "");
+
+// Download file to the local mirror
+MString DownloadFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const MString& root);
+
+// Remove file from the local mirror
+MString RemoveFile(const struct FileInfo& linfo);
+
+// Updare file in the local mirror
+MString UpdateFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const struct FileInfo& linfo, const MString& root);

sources/COPERNICUS.cpp

@@ -0,0 +1,282 @@
+#define MICHLIB_NOSOURCE
+#include "COPERNICUS.h"
+#include "mirrorfuncs.h"
+#include <libxml/parser.h>
+#include <libxml/tree.h>
+
+using michlib::GPL;
+
+const MString COPERNICUSData::caturl = "https://stac.marine.copernicus.eu/metadata/catalog.stac.json";
+
+std::pair<Json::Value, MString> COPERNICUSData::GetJSON(const MString& url)
+{
+ Json::Reader reader;
+ Json::Value  obj;
+ MString      content;
+
+ auto [val, suc] = cache->Get(url);
+ if(suc)
+  content = std::move(val);
+ else
+ {
+  michlib::message(url + " not found in cache, downloading");
+  auto [out, res] = GetUrl(chandle, url);
+  if(res != CURLE_OK) return {obj, MString("Can't download JSON: ") + curlerr};
+  cache->Put(url, out, 3600);
+  content = std::move(out);
+ }
+
+ reader.parse(content.Buf(), content.Buf() + content.Len(), obj, false);
+
+ return {obj, ""};
+}
+
+MString COPERNICUSData::ReadURL(const Json::Value& cat, const MString& prod)
+{
+ const auto& links = cat["links"];
+ if(links.type() != Json::arrayValue) return "";
+ for(Json::ArrayIndex i = 0; i < links.size(); i++)
+ {
+  const auto& titl = links[i]["title"];
+  const auto& href = links[i]["href"];
+  if(titl.type() == Json::stringValue && href.type() == Json::stringValue)
+  {
+   MString str(titl.asString().c_str());
+   if(str == prod) return MString(href.asString().c_str());
+  }
+ }
+ return "";
+}
+
+std::pair<std::vector<struct FileInfo>, MString> COPERNICUSData::ReadRemoteFileList(const MString& url)
+{
+ LIBXML_TEST_VERSION
+
+ std::vector<struct FileInfo> out;
+ MString                      bucket, prefix;
+
+ // Split url on prefix and bucket
+ {
+  size_t pos   = url.Len();
+  size_t count = 0;
+  for(size_t i = 0; i < url.Len(); i++)
+  {
+   if(url[i] == '/') count++;
+   if(count == 4)
+   {
+    pos = i;
+    break;
+   }
+  }
+  if(pos == url.Len()) return {out, "Can't parse url: " + url};
+
+  bucket = url.SubStr(1, pos);
+  prefix = url.SubStr(pos + 2, url.Len() - pos - 1);
+ }
+
+ MString cont;
+ bool    next = true;
+
+ while(next)
+ {
+  MString url = bucket + "?list-type=2&prefix=" + prefix;
+  if(cont.Exist()) url += "&continuation-token=" + cont;
+  cont = "";
+
+  auto [data, res] = GetUrl(chandle, url);
+  if(res != CURLE_OK) return {out, MString("Can't download ") + url + ": " + curlerr};
+
+  xmlDocPtr doc = xmlReadMemory(data.Buf(), data.Len(), "data.xml", nullptr, 0);
+  if(doc == nullptr) return {out, MString("Can't download ") + url + ": XML parse error"};
+  auto cur = xmlDocGetRootElement(doc);
+  if(cur == nullptr)
+  {
+   xmlFreeDoc(doc);
+   return {out, MString("Can't download ") + url + ": empty XML"};
+  }
+  if(xmlStrEqual(cur->name, (const xmlChar*)"ListBucketResult") == 0)
+  {
+   xmlFreeDoc(doc);
+   return {out, MString("Can't download ") + url + ": unknown XML"};
+  }
+
+  for(const auto* n = cur->children; n; n = n->next)
+  {
+   if(xmlStrEqual(n->name, (const xmlChar*)"NextContinuationToken") == 1)
+   {
+    auto* content = xmlNodeGetContent(n);
+    cont          = (char*)content;
+    xmlFree(content);
+   }
+   if(xmlStrEqual(n->name, (const xmlChar*)"Contents") == 1)
+   {
+    MString   fname;
+    MDateTime mtime;
+    size_t    size = 0;
+    for(const auto* c = n->children; c; c = c->next)
+    {
+     if(xmlStrEqual(c->name, (const xmlChar*)"Key") == 1)
+     {
+      auto* content = xmlNodeGetContent(c);
+      fname         = (char*)content;
+      xmlFree(content);
+     }
+     if(xmlStrEqual(c->name, (const xmlChar*)"LastModified") == 1)
+     {
+      auto* content = xmlNodeGetContent(c);
+      mtime.FromString((char*)content);
+      xmlFree(content);
+     }
+     if(xmlStrEqual(c->name, (const xmlChar*)"Size") == 1)
+     {
+      auto* content = xmlNodeGetContent(c);
+      size          = MString((char*)content).ToInteger<size_t>();
+      xmlFree(content);
+     }
+    }
+    out.emplace_back(bucket + "/" + fname, fname.SubStr(prefix.Len() + 2, fname.Len() - prefix.Len() - 1), mtime, size);
+   }
+  }
+  xmlFreeDoc(doc);
+  next = cont.Exist();
+ }
+
+ std::sort(out.begin(), out.end(), [](const struct FileInfo& a, const struct FileInfo& b) { return a.name < b.name; });
+ return {out, ""};
+}
+
+MString COPERNICUSData::Mirror(const CLArgs& args)
+{
+ GPL.UsePrefix("COPERNICUS");
+
+ // Local directory
+ MString mirrorroot = GPL.ParameterSValue("MirrorTo", "");
+ if(!mirrorroot.Exist()) return "Local mirror directory not specified";
+
+ // Cache
+ cache.reset(CreateCache(GPL.ParameterSValue("Cache", "")));
+ if(!cache)
+ {
+  michlib::errmessage("Can't init cache");
+  cache.reset(new FakeCache);
+ }
+
+ curl_easy_setopt(chandle, CURLOPT_ERRORBUFFER, curlerr);
+
+ if(!args.contains("product")) return "Copernicus product not specified";
+ MString     prod = args.at("product");
+ Json::Value product;
+ MString     produrl;
+
+ // Get catalog
+ {
+  auto [cat, err] = GetJSON(caturl);
+  if(err.Exist()) return "Can't download catalog: " + err;
+  if(cat["title"].type() != Json::stringValue || cat["title"].asString() != "Copernicus Marine Data Store") return "Can't parse catalog";
+  catalog = std::move(cat);
+ }
+
+ // Get product
+ {
+  auto url = ReadURL(catalog, prod);
+  if(!url.Exist()) return "Url for product " + prod + " not found in catalog";
+  produrl        = DirName(caturl) + "/" + url;
+  auto [pr, err] = GetJSON(produrl);
+  if(err.Exist()) return "Can't download product information from " + produrl + ": " + err;
+  product = std::move(pr);
+ }
+
+ std::vector<MString> dsets;
+ if(args.contains("dataset"))
+  dsets.push_back(args.at("dataset"));
+ else
+ {
+  const auto& links = product["links"];
+  if(links.type() != Json::arrayValue) return "Can't find information about datasets";
+  for(Json::ArrayIndex i = 0; i < links.size(); i++)
+  {
+   const auto& rel  = links[i]["rel"];
+   const auto& titl = links[i]["title"];
+   if(rel.type() == Json::stringValue && titl.type() == Json::stringValue && rel.asString() == "item") dsets.push_back(titl.asString().c_str());
+  }
+ }
+
+ for(const auto& dset: dsets)
+ {
+  michlib::message("Mirroring " + dset);
+  auto url = ReadURL(product, dset);
+  if(!url.Exist()) return "Url for dataset " + dset + " not found in product description";
+  MString dseturl = DirName(produrl) + "/" + url;
+  auto [ds, err]  = GetJSON(dseturl);
+  if(err.Exist()) return "Can't download dataset information from " + dseturl + ": " + err;
+
+  const auto& href = ds["assets"]["native"]["href"];
+  if(href.type() != Json::stringValue) return "Can't find data for dataset " + dset + " from product " + prod;
+
+  url = href.asString().c_str();
+
+  MString locroot = mirrorroot + "/" + prod + "/" + dset;
+
+  auto [lfiles, lerr] = ReadLocalFileList(locroot);
+  if(lerr.Exist()) return lerr;
+
+  auto [rfiles, rerr] = ReadRemoteFileList(url);
+  if(rerr.Exist()) return rerr;
+
+  std::vector<size_t>                    down, rem;
+  std::vector<std::pair<size_t, size_t>> upd;
+
+  {
+   size_t rpos = 0, lpos = 0;
+   while(rpos != rfiles.size() || lpos != lfiles.size())
+   {
+    if(rpos == rfiles.size())
+     while(lpos != lfiles.size()) rem.push_back(lpos++);
+    if(lpos == lfiles.size())
+     while(rpos != rfiles.size()) down.push_back(rpos++);
+    if(rpos == rfiles.size() || lpos == lfiles.size()) continue;
+
+    if(rfiles[rpos].name < lfiles[lpos].name)
+     down.push_back(rpos++);
+    else if(lfiles[lpos].name < rfiles[rpos].name)
+     rem.push_back(lpos++);
+    else
+    {
+     auto delta = rfiles[rpos].mtime.Epoch() - lfiles[lpos].mtime.Epoch();
+     if(delta < 0) delta = -delta;
+     if(delta > 0 || rfiles[rpos].size != lfiles[lpos].size) upd.emplace_back(rpos, lpos);
+     lpos++;
+     rpos++;
+    }
+   }
+  }
+
+  michlib::message(MString("New files: ") + down.size());
+  michlib::message(MString("Obsolete files: ") + rem.size());
+  michlib::message(MString("Modified files: ") + upd.size());
+
+  for(size_t i = 0; i < down.size(); i++)
+  {
+   size_t ri  = down[i];
+   auto   err = DownloadFile(chandle, rfiles[ri], locroot);
+   if(err.Exist()) return err;
+  }
+
+  for(size_t i = 0; i < rem.size(); i++)
+  {
+   size_t li  = rem[i];
+   auto   err = RemoveFile(lfiles[li]);
+   if(err.Exist()) return err;
+  }
+
+  for(size_t i = 0; i < upd.size(); i++)
+  {
+   size_t ri  = upd[i].first;
+   size_t li  = upd[i].second;
+   auto   err = UpdateFile(chandle, rfiles[ri], lfiles[li], locroot);
+   if(err.Exist()) return err;
+  }
+ }
+
+ return "";
+}

sources/COPERNICUS.h

@@ -0,0 +1,36 @@
+#pragma once
+#include "ParseArgs.h"
+#include "cache.h"
+#include "curlfuncs.h"
+#include "mdatetime.h"
+#include <json/json.h>
+
+using michlib::MDateTime;
+using michlib::MString;
+
+class COPERNICUSData
+{
+ static const MString caturl;
+
+ std::unique_ptr<GenericCache> cache;
+ CURLRAII                      chandle;
+ Json::Value                   catalog;
+ char                          curlerr[CURL_ERROR_SIZE];
+
+ // Get url for product or dataset from catalog
+ static MString ReadURL(const Json::Value& cat, const MString& prod);
+
+ // Download JSON from url
+ std::pair<Json::Value, MString> GetJSON(const MString& url);
+
+ // Get remote file list from url
+ std::pair<std::vector<struct FileInfo>,MString> ReadRemoteFileList(const MString& url);
+
+ public:
+ static constexpr const char* name = "COPERNICUS";
+
+ COPERNICUSData() = default;
+
+ // Main mirror function
+ MString Mirror(const CLArgs& args);
+};

src/CMakeLists.txt

@@ -2,15 +2,23 @@ set(EXENAME odm)
 set(ACTIONLISTINC ${CMAKE_CURRENT_BINARY_DIR}/../include/actionlist.h) # Include actions files and define the actions classes list
 set(DATALISTINC ${CMAKE_CURRENT_BINARY_DIR}/../include/datalist.h) # Include data sources files and define the data sources classes list
 
+find_package(PkgConfig REQUIRED)
 find_library(netcdf netcdf REQUIRED)
 find_package(OpenMP REQUIRED)
+find_package(CURL REQUIRED)
+find_package(LibXml2 REQUIRED)
+find_package(SQLite3 REQUIRED)
+pkg_check_modules(JSONCPP REQUIRED jsoncpp)
+
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
 
+include_directories(${JSONCPP_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIRS} ${SQLite3_INCLUDE_DIRS})
+
 file(GLOB srcs CONFIGURE_DEPENDS *.cpp)
 add_executable(${EXENAME} ${srcs} ${ACTIONLISTINC} ${SOURCELISTINC})
 target_include_directories(${EXENAME} PRIVATE ../michlib/michlib ${CMAKE_CURRENT_BINARY_DIR}/../include)
 
-target_link_libraries(${EXENAME} ${linker_options} ${netcdf} OpenMP::OpenMP_CXX teos)
+target_link_libraries(${EXENAME} ${linker_options} ${netcdf} OpenMP::OpenMP_CXX CURL::libcurl ${JSONCPP_LINK_LIBRARIES} LibXml2::LibXml2 SQLite::SQLite3 teos)
 set_target_properties(${EXENAME} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 install(TARGETS ${EXENAME})
 

src/curlfuncs.cpp

@@ -0,0 +1,41 @@
+#define MICHLIB_NOSOURCE
+#include "curlfuncs.h"
+#include <unistd.h>
+
+using michlib::pointer_cast;
+using michlib::uint1;
+
+size_t Write2String(char* ptr, size_t size, size_t n, void* data)
+{
+ MString* out = pointer_cast<MString*>(data);
+ *out += MString(ptr, size * n);
+ return size * n;
+}
+
+size_t Write2File(char* ptr, size_t size, size_t n, void* data)
+{
+ const int*   fd    = pointer_cast<const int*>(data);
+ size_t       count = size * n;
+ const uint1* buf   = pointer_cast<const uint1*>(ptr);
+
+ while(count != 0)
+ {
+  auto wr = write(*fd, buf, count);
+  if(wr == -1) return 0;
+  count -= wr;
+  buf += wr;
+ }
+
+ return size * n;
+}
+
+std::pair<MString, CURLcode> GetUrl(const CURLRAII& chandle, const MString& url)
+{
+ MString out;
+
+ curl_easy_setopt(chandle, CURLOPT_URL, url.Buf());
+ curl_easy_setopt(chandle, CURLOPT_WRITEFUNCTION, Write2String);
+ curl_easy_setopt(chandle, CURLOPT_WRITEDATA, &out);
+ auto res = curl_easy_perform(chandle);
+ return {out, res};
+}

src/mirrorfuncs.cpp

@@ -0,0 +1,130 @@
+#define MICHLIB_NOSOURCE
+#include "mirrorfuncs.h"
+#include "StringFunctions.h"
+#include "filehelpers.h"
+#include "merrors.h"
+
+using michlib::FD;
+using michlib::message;
+
+bool MakePath(const MString& dname)
+{
+ struct stat st;
+ int         ret = stat(dname.Buf(), &st);
+ if(ret == 0) return S_ISDIR(st.st_mode);
+
+ auto    dirs = michlib::Split_on_words(dname, "/", false);
+ MString cdir = "";
+ for(const auto& dir: dirs)
+ {
+  cdir += "/" + dir;
+  ret = stat(cdir.Buf(), &st);
+  if(ret == 0 && S_ISDIR(st.st_mode)) continue;
+  if(ret == 0 && !S_ISDIR(st.st_mode)) return false;
+  ret = mkdir(cdir.Buf(), 0755);
+  if(ret != 0) return false;
+ }
+ return true;
+}
+
+std::pair<std::vector<struct FileInfo>, MString> ReadLocalFileList(const MString& dir, const MString& path)
+{
+ std::vector<struct FileInfo> out;
+ DIRRAII                      dhandle;
+
+ MakePath(dir);
+ dhandle.reset(opendir(dir.Buf()));
+
+ if(!dhandle) return {out, "Can't open directory " + path + (path.Exist() ? "/" : "") + dir};
+
+ int dfd             = dirfd(dhandle);
+ errno               = 0;
+ struct dirent* dent = readdir(dhandle);
+ if(errno != 0) return {out, "Can't read directory " + path + (path.Exist() ? "/" : "") + dir};
+ struct stat st;
+
+ do {
+  if(dent->d_name[0] != '.')
+  {
+   int ret = fstatat(dfd, dent->d_name, &st, AT_SYMLINK_NOFOLLOW);
+   if(ret != 0) return {out, "Can't stat " + path + "/" + dir + "/" + dent->d_name};
+   if(S_ISDIR(st.st_mode)) // Directory, recurse
+   {
+    auto [list, err] = ReadLocalFileList(dir + "/" + dent->d_name, path + (path.Exist() ? "/" : "") + dent->d_name);
+    if(err.Exist()) return {out, err};
+    out.insert(out.end(), list.begin(), list.end());
+   }
+   if(S_ISREG(st.st_mode)) // Regular file
+   {
+    out.emplace_back(dir + "/" + dent->d_name, path + (path.Exist() ? "/" : "") + dent->d_name, MDateTime(st.st_mtim.tv_sec, st.st_mtim.tv_nsec), st.st_size);
+   }
+   // Ignore non-directories, non-files
+  }
+  dent = readdir(dhandle);
+ } while(dent != nullptr || errno != 0);
+
+ if(errno != 0) return {out, "Can't read directory " + path + "/" + dir};
+ std::sort(out.begin(), out.end(), [](const struct FileInfo& a, const struct FileInfo& b) { return a.name < b.name; });
+ return {out, ""};
+}
+
+MString DownloadFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const MString& root)
+{
+ message("Downloading " + rinfo.url);
+
+ MString dname = DirName(rinfo.name), fname = FileName(rinfo.name);
+ FD      fd;
+
+ if(!MakePath(root + "/" + dname)) return "Can't create directory " + root + "/" + dname;
+ fd.Reset(creat((root + "/" + rinfo.name).Buf(), 0644));
+ if(!fd) return "Can't create file " + root + "/" + rinfo.name;
+
+ char errbuf[CURL_ERROR_SIZE];
+ int  cfd = fd.Get();
+ curl_easy_setopt(chandle, CURLOPT_ERRORBUFFER, errbuf);
+ curl_easy_setopt(chandle, CURLOPT_WRITEFUNCTION, Write2File);
+ curl_easy_setopt(chandle, CURLOPT_WRITEDATA, &cfd);
+ curl_easy_setopt(chandle, CURLOPT_URL, rinfo.url.Buf());
+ auto res = curl_easy_perform(chandle);
+ if(res != CURLE_OK)
+ {
+  unlink((root + "/" + rinfo.name).Buf());
+  return MString("Can't download file: ") + errbuf;
+ }
+
+ {
+  struct timespec times[2];
+  times[0].tv_sec = times[1].tv_sec = rinfo.mtime.Epoch();
+  times[0].tv_nsec = times[1].tv_nsec = 0;
+
+  int ret = futimens(fd, times);
+  if(ret != 0)
+  {
+   unlink((root + "/" + rinfo.name).Buf());
+   return "Can't set mtime for file: " + root + "/" + rinfo.name;
+  }
+ }
+
+ return "";
+}
+
+MString RemoveFile(const struct FileInfo& linfo)
+{
+ message("Remove " + linfo.url);
+ int ret = unlink(linfo.url.Buf());
+ if(ret != 0) return "Can't remove file " + linfo.url;
+ return "";
+}
+
+MString UpdateFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const struct FileInfo& linfo, const MString& root)
+{
+ MString err;
+
+ message("Update " + linfo.url);
+ err = RemoveFile(linfo);
+ if(err.Exist()) return err;
+ err = DownloadFile(chandle, rinfo, root);
+ if(err.Exist()) return err;
+
+ return "";
+}