You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

222 lines
6.3 KiB

#define MICHLIB_NOSOURCE
#include "COPERNICUS.h"
#include "mirrorfuncs.h"
#include <libxml/parser.h>
#include <libxml/tree.h>
using michlib::GPL;
RetVal<std::vector<struct FileInfo>> COPERNICUSData::ReadRemoteFileList(const MString& url) const
{
const static MString pref = "COPERNICUSData::ReadRemoteFileList";
LIBXML_TEST_VERSION
std::vector<struct FileInfo> out;
MString bucket, prefix;
// Split url on prefix and bucket
{
size_t pos = url.Len();
size_t count = 0;
for(size_t i = 0; i < url.Len(); i++)
{
if(url[i] == '/') count++;
if(count == 4)
{
pos = i;
break;
}
}
if(pos == url.Len()) return {pref, "Can't parse url: " + url};
bucket = url.SubStr(1, pos);
prefix = url.SubStr(pos + 2, url.Len() - pos - 1);
}
MString cont;
bool next = true;
CURLRAII chandle;
while(next)
{
MString url = bucket + "?list-type=2&prefix=" + prefix;
if(cont.Exist()) url += "&continuation-token=" + cont;
cont = "";
auto [data, res] = GetUrl(chandle, url);
if(res != CURLE_OK) return {pref, MString("Can't download ") + url + ": " + chandle.Err()};
xmlDocPtr doc = xmlReadMemory(data.Buf(), data.Len(), "data.xml", nullptr, 0);
if(doc == nullptr) return {pref, MString("Can't download ") + url + ": XML parse error"};
auto cur = xmlDocGetRootElement(doc);
if(cur == nullptr)
{
xmlFreeDoc(doc);
return {pref, MString("Can't download ") + url + ": empty XML"};
}
if(xmlStrEqual(cur->name, (const xmlChar*)"ListBucketResult") == 0)
{
xmlFreeDoc(doc);
return {pref, MString("Can't download ") + url + ": unknown XML"};
}
for(const auto* n = cur->children; n; n = n->next)
{
if(xmlStrEqual(n->name, (const xmlChar*)"NextContinuationToken") == 1)
{
auto* content = xmlNodeGetContent(n);
cont = (char*)content;
xmlFree(content);
}
if(xmlStrEqual(n->name, (const xmlChar*)"Contents") == 1)
{
MString fname;
MDateTime mtime;
size_t size = 0;
for(const auto* c = n->children; c; c = c->next)
{
if(xmlStrEqual(c->name, (const xmlChar*)"Key") == 1)
{
auto* content = xmlNodeGetContent(c);
fname = (char*)content;
xmlFree(content);
}
if(xmlStrEqual(c->name, (const xmlChar*)"LastModified") == 1)
{
auto* content = xmlNodeGetContent(c);
mtime.FromString((char*)content);
xmlFree(content);
}
if(xmlStrEqual(c->name, (const xmlChar*)"Size") == 1)
{
auto* content = xmlNodeGetContent(c);
size = MString((char*)content).ToInteger<size_t>();
xmlFree(content);
}
}
out.emplace_back(bucket + "/" + fname, fname.SubStr(prefix.Len() + 2, fname.Len() - prefix.Len() - 1), mtime, size);
}
}
xmlFreeDoc(doc);
next = cont.Exist();
}
std::sort(out.begin(), out.end(), [](const struct FileInfo& a, const struct FileInfo& b) { return a.name < b.name; });
return out;
}
Error COPERNICUSData::Mirror(const CLArgs& args) const
{
const static MString pref = "COPERNICUSData::Mirror";
GPL.UsePrefix("COPERNICUS");
// Local directory
MString mirrorroot = GPL.ParameterSValue("MirrorTo", "");
if(!mirrorroot.Exist()) return {pref, "Local mirror directory not specified"};
if(!args.contains("product")) return {pref, "Copernicus product not specified"};
MString prod = args.at("product");
CopernicusCatalog cat;
std::vector<MString> dsets;
if(args.contains("dataset"))
dsets.push_back(args.at("dataset"));
else
{
auto dlist = cat.DatasetList(prod);
if(!dlist) return dlist.Add(pref, "Can't get list of datasets");
dsets = dlist.Value();
}
michlib::RegExpSimple filter((args.contains("filter") ? args.at("filter") : ".*").Buf());
if(filter.Compile() != 0) return Error(pref, MString("Can't compile regular expression ") + filter.RegStr());
CURLRAII chandle;
for(const auto& dset: dsets)
{
michlib::message("Mirroring " + dset);
auto url = cat.DatasetNativeURL(prod, dset);
if(!url) return {pref, "Can't find data for dataset " + dset + " from product " + prod};
MString locroot = mirrorroot + "/" + prod + "/" + dset;
auto lfilesret = ReadLocalFileList(locroot);
if(!lfilesret) return lfilesret.Add(pref, "Can't get local file list");
const auto& lfiles = lfilesret.Value();
auto rfilesret = ReadRemoteFileList(url.Value());
if(!rfilesret) return rfilesret.Add(pref, "Can't get remote file list");
const auto& rfiles = rfilesret.Value();
std::vector<size_t> down, rem;
std::vector<std::pair<size_t, size_t>> upd;
{
size_t rpos = 0, lpos = 0;
while(rpos != rfiles.size() || lpos != lfiles.size())
{
if(rpos == rfiles.size())
while(lpos != lfiles.size())
{
if(filter.Match(lfiles[lpos].name.Buf())) rem.push_back(lpos);
lpos++;
}
if(lpos == lfiles.size())
while(rpos != rfiles.size())
{
if(filter.Match(rfiles[rpos].name.Buf())) down.push_back(rpos);
rpos++;
}
if(rpos == rfiles.size() || lpos == lfiles.size()) continue;
if(rfiles[rpos].name < lfiles[lpos].name)
{
if(filter.Match(rfiles[rpos].name.Buf())) down.push_back(rpos);
rpos++;
}
else if(lfiles[lpos].name < rfiles[rpos].name)
{
if(filter.Match(lfiles[lpos].name.Buf())) rem.push_back(lpos);
lpos++;
}
else
{
auto delta = rfiles[rpos].mtime.Epoch() - lfiles[lpos].mtime.Epoch();
if(delta < 0) delta = -delta;
if((delta > 0 || rfiles[rpos].size != lfiles[lpos].size) && filter.Match(lfiles[lpos].name.Buf())) upd.emplace_back(rpos, lpos);
lpos++;
rpos++;
}
}
}
michlib::message(MString("New files: ") + down.size());
michlib::message(MString("Obsolete files: ") + rem.size());
michlib::message(MString("Modified files: ") + upd.size());
for(size_t i = 0; i < down.size(); i++)
{
size_t ri = down[i];
auto err = DownloadFile(chandle, rfiles[ri], locroot);
if(!err) return err.Add(pref, "Can't download file");
}
for(size_t i = 0; i < rem.size(); i++)
{
size_t li = rem[i];
auto err = RemoveFile(lfiles[li]);
if(!err) return err.Add(pref, "Can't remove file");
}
for(size_t i = 0; i < upd.size(); i++)
{
size_t ri = upd[i].first;
size_t li = upd[i].second;
auto err = UpdateFile(chandle, rfiles[ri], lfiles[li], locroot);
if(!err) return err.Add(pref, "Can't update file");
}
}
return Error();
}