michael_ul
/
odm
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: michael_ul:4274cda5b3fd15b873ba8c40e5c47e1de1a5654d
Branches Tags
michael_ul:interpolate
michael_ul:lintest
michael_ul:master
..
compare: michael_ul:04f45ba459d8fd5dd1d0a5ef12b7d8a0b3ef8885
Branches Tags
michael_ul:interpolate
michael_ul:lintest
michael_ul:master

No commits in common. '4274cda5b3fd15b873ba8c40e5c47e1de1a5654d' and '04f45ba459d8fd5dd1d0a5ef12b7d8a0b3ef8885' have entirely different histories.
4274cda5b3 ... 04f45ba459

17 changed files with 8 additions and 1190 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats
  1. 210
      actions/actionint.h
  2. 23
      actions/actionmirror.h
  3. 12
      actions/actionuv.h
  4. 13
      include/basedata.h
  5. 144
      include/cache.h
  6. 29
      include/curlfuncs.h
  7. 51
      include/interpolation.h
  8. 62
      include/mirrorfuncs.h
  9. 17
      include/simple2ddata.h
  10. 2
      michlib
  11. 282
      sources/COPERNICUS.cpp
  12. 36
      sources/COPERNICUS.h
  13. 86
      sources/GRIDVEL.cpp
  14. 50
      sources/GRIDVEL.h
  15. 10
      src/CMakeLists.txt
  16. 41
      src/curlfuncs.cpp
  17. 130
      src/mirrorfuncs.cpp

210
actions/actionint.h
Unescape View File

@ -1,210 +0,0 @@
#pragma once
#include "BFileR.h"
#include "BFileW.h"
#include "actiondep.h"
#include "interpolation.h"
#include "ncfilew.h"
#include "ncfuncs.h"
#include <memory>
template<class T>
concept ReadIsInterpolable = requires {
{
std::declval<ReadType<T>>().GridPos(0.0, 0.0)
} -> std::convertible_to<struct GridPoint>;
};
ADD_ACTION(INT, int, (ReadPSupported<Source> || ReadSupported<Source>)&&ReadIsInterpolable<Source>);
template<class D> MString ActionINT::DoAction(const CLArgs& args, D& ds)
{
struct TPoint
{
struct Point2D p;
MDateTime t;
real lon0, lat0, dtime;
};
MDateTime refdate("1980-01-01");
if(!args.contains("input")) return "No input file given";
michlib::BFileR in;
if(in.Open(args.at("input")) != ERR_NOERR) return "Can't open input file " + args.at("input");
std::vector<TPoint> points;
struct Region reg;
{
michlib::uint loncol = in.Columns(), latcol = in.Columns(), timecol = in.Columns();
michlib::uint lon0col = in.Columns(), lat0col = in.Columns(), dtimecol = in.Columns();
for(uint i = 0; i < in.Columns(); i++)
{
if(in.ColumnName(i + 1) == "lon") loncol = i;
if(in.ColumnName(i + 1) == "lat") latcol = i;
if(in.ColumnName(i + 1) == "rtime") timecol = i;
if(in.ColumnName(i + 1) == "lon0") lon0col = i;
if(in.ColumnName(i + 1) == "lat0") lat0col = i;
if(in.ColumnName(i + 1) == "time") dtimecol = i;
}
if(loncol >= in.Columns()) return "Lon column not found in input file " + args.at("input");
if(latcol >= in.Columns()) return "Lat column not found in input file " + args.at("input");
if(timecol >= in.Columns()) return "Time column not found in input file " + args.at("input");
if(lon0col >= in.Columns()) return "Lon0 column not found in input file " + args.at("input");
if(lat0col >= in.Columns()) return "Lat0 column not found in input file " + args.at("input");
if(dtimecol >= in.Columns()) return "Age column not found in input file " + args.at("input");
points.resize(in.Rows());
reg.lonb = reg.lone = in[loncol][0];
reg.latb = reg.late = in[latcol][0];
for(uint i = 0; i < in.Rows(); i++)
{
points[i].p.x = in[loncol][i];
points[i].p.y = in[latcol][i];
points[i].t = refdate + static_cast<time_t>(michlib::Round(in[timecol][i] * 86400));
points[i].lon0 = in[lon0col][i];
points[i].lat0 = in[lat0col][i];
points[i].dtime = in[dtimecol][i];
reg.lonb = std::min(reg.lonb, points[i].p.x);
reg.lone = std::max(reg.lone, points[i].p.x);
reg.latb = std::min(reg.latb, points[i].p.y);
reg.late = std::max(reg.late, points[i].p.y);
}
std::ranges::sort(points, {}, &TPoint::t);
}
in.Close();
auto resop = ds.Open(args);
if(resop.Exist()) return "Can't open source: " + resop;
michlib_internal::ParameterListEx pars;
pars.UsePrefix("");
pars.SetParameter("source", args.at("source"));
pars.SetParameter("history", args.at("_cmdline"));
MString varstring;
if(args.contains("var"))
varstring = args.at("var");
else
{
if(args.contains("vars"))
varstring = args.at("vars");
else
{
if constexpr(HasDefVars<D>)
varstring = ds.DefaultVars();
else
return "Variables not specified";
}
}
auto vlist = michlib::Split_on_words(varstring, " ,", false);
std::vector<MString> vnames{std::make_move_iterator(std::begin(vlist)), std::make_move_iterator(std::end(vlist))};
{
std::set<MString> used;
for(const auto& vname: vnames)
{
if(used.contains(vname)) return "Duplicate variable " + vname + " in list " + varstring;
if(ds.CheckVar(vname) == VarPresence::NONE) return "Variable " + vname + " not exists in this dataset";
used.insert(vname);
}
}
pars.SetParameter("variables", varstring);
//int compress = 3;
//if(args.contains("compress")) compress = args.at("compress").ToInt();
std::unique_ptr<const BaseParameters> sourcepars;
if constexpr(ParametersSupported<D>)
{
if constexpr(ParametersRequiredRegion<D>)
{
auto [p, err] = ds.Parameters(pars, args, reg);
if(err.Exist()) return err;
sourcepars.reset(p);
}
else
{
auto [p, err] = ds.Parameters(pars, args);
if(err.Exist()) return err;
sourcepars.reset(p);
}
}
auto p = sourcepars.get();
MString name = args.contains("out") ? args.at("out") : "out.bin";
//MString outfmt = args.contains("format") ? args.at("format") : (GetExt(name) == "nc" ? "nc" : "bin");
size_t loind = ds.NTimes(), hiind = ds.NTimes(), curind = 0;
michlib::BFileW fw;
const MDateTime first = ds.Time(0), last = ds.Time(ds.NTimes() - 1);
std::vector<LinearInterpolator<ReadType<D>>> lo, hi;
{
size_t ic = 0;
fw.Create(name, 6 + vnames.size());
fw.SetColumnName(++ic, "lon0");
fw.SetColumnName(++ic, "lat0");
fw.SetColumnName(++ic, "lon");
fw.SetColumnName(++ic, "lat");
fw.SetColumnName(++ic, "time");
fw.SetColumnName(++ic, "rtime");
for(size_t i = 0; i < vnames.size(); i++) fw.SetColumnName(++ic, vnames[i]);
fw.SetParameters(pars);
}
for(size_t i = 0; i < points.size(); i++)
{
if(points[i].t < first || points[i].t > last) continue;
while(!(points[i].t >= ds.Time(curind) && points[i].t <= ds.Time(curind + 1))) curind++;
if(curind != loind && curind != hiind)
{
loind = curind;
hiind = curind + 1;
{
auto temp = Read(ds, vnames, p, loind);
if(temp.size() != vnames.size()) return "Can't read data";
//lo.resize(temp.size());
for(size_t j = 0; j < temp.size(); j++) lo.emplace_back(std::move(temp[j]));
}
{
auto temp = Read(ds, vnames, p, hiind);
if(temp.size() != vnames.size()) return "Can't read data";
//hi.resize(temp.size());
for(size_t j = 0; j < temp.size(); j++) hi.emplace_back(std::move(temp[j]));
}
}
else if(curind == hiind)
{
loind = curind;
hiind = curind + 1;
lo = std::move(hi);
{
auto temp = Read(ds, vnames, p, hiind);
if(temp.size() != vnames.size()) return "Can't read data";
//hi.resize(temp.size());
for(size_t j = 0; j < temp.size(); j++) hi.emplace_back(std::move(temp[j]));
}
}
auto step = ds.Time(hiind) - ds.Time(loind);
auto delta = points[i].t - ds.Time(loind);
real trel = static_cast<real>(delta) / static_cast<real>(step);
fw.Write(points[i].lon0);
fw.Write(points[i].lat0);
fw.Write(points[i].p.x);
fw.Write(points[i].p.y);
fw.Write(points[i].dtime);
fw.Write(DeltaDates(refdate, points[i].t));
for(size_t iv = 0; iv < lo.size(); iv++)
{
real vlo = lo[iv](points[i].p);
real vhi = hi[iv](points[i].p);
fw.Write(vlo + (vhi - vlo) * trel);
}
}
fw.Finalize();
fw.Close();
return "";
};

23
actions/actionmirror.h
Unescape View File

@ -1,23 +0,0 @@
#pragma once
#include "actiondep.h"
#include "merrors.h"
using michlib::message;
template<class T>
concept MirrorSupported = requires(T t, const CLArgs& args) {
{
t.Mirror(args)
} -> std::convertible_to<MString>;
};
ADD_ACTION(Mirror, mirror, MirrorSupported<Source>);
template<class D> MString ActionMirror::DoAction(const CLArgs& args, D& data)
{
//auto resop = data.Open(args);
//if(resop.Exist()) return "Can't open source: " + resop;
auto res = data.Mirror(args);
if(res.Exist()) return "Mirroring failed: " + res;
return "";
};

12
actions/actionuv.h
Unescape View File

@ -152,9 +152,9 @@ template<class D> MString ActionUV::DoAction(const CLArgs& args, D& ds)
MString name = args.contains("out") ? args.at("out") : "";
MString outfmt = args.contains("outformat") ? args.at("outformat") : (GetExt(name) == "nc" ? "nc" : "bin");
MString namevel = args.contains("velout") ? args.at("velout") : "";
MString outfmtvel = args.contains("veloutformat") ? args.at("veloutformat") : (GetExt(namevel) == "nc" ? "nc" : "bin");
MString outfmtvel = args.contains("veloutformat") ? args.at("veloutformat") : (GetExt(name) == "nc" ? "nc" : "bin");
MString namestp = args.contains("stpout") ? args.at("stpout") : "";
MString outfmtstp = args.contains("stpoutformat") ? args.at("stpoutformat") : (GetExt(namestp) == "nc" ? "nc" : "bin");
MString outfmtstp = args.contains("stpoutformat") ? args.at("stpoutformat") : (GetExt(name) == "nc" ? "nc" : "bin");
size_t shiftx = args.contains("shiftx") ? args.at("shiftx").ToInteger<size_t>() : 0;
size_t shifty = args.contains("shifty") ? args.at("shifty").ToInteger<size_t>() : 0;
@ -263,7 +263,7 @@ template<class D> MString ActionUV::DoAction(const CLArgs& args, D& ds)
if(outfmtvel == "bin")
{
BFileW vel;
vel.Create(namevel, 4);
vel.Create(name, 4);
vel.SetColumnName(1, "Longitude");
vel.SetColumnName(2, "Latitude");
vel.SetColumnName(3, "u, " + velunit);
@ -285,7 +285,7 @@ template<class D> MString ActionUV::DoAction(const CLArgs& args, D& ds)
{
MString err;
if(!err.Exist() && !headwrited) err = fwfilt.Create(sdata, namevel, compress);
if(!err.Exist() && !headwrited) err = fwfilt.Create(sdata, name, compress);
if(!err.Exist() && !headwrited) err = fwfilt.AddTimeData(tdata, !average);
if(!err.Exist() && !headwrited) err = fwfilt.AddAtts(pars);
if(!err.Exist() && !headwrited) err = fwfilt.AddVariable("u", "", "Eastward velocity", velunit, "");
@ -312,11 +312,11 @@ template<class D> MString ActionUV::DoAction(const CLArgs& args, D& ds)
for(size_t iy = 0; iy < data.Ny() - 1; iy++) stp.Add(data.StablePoints(ix, iy));
if(outfmtstp == "bin")
stp.WriteBinBile(namestp, pars);
stp.WriteBinBile(name, pars);
else if(outfmtstp == "nc" || outfmtstp == "netcdf")
{
MString err;
if(!err.Exist() && !headwrited) err = stp.CreateNcFile(namestp, pars, args.at("_cmdline"), compress, tdata, !average);
if(!err.Exist() && !headwrited) err = stp.CreateNcFile(name, pars, args.at("_cmdline"), compress, tdata, !average);
if(!err.Exist()) err = stp.WriteNcFile(it);
if(err.Exist()) return err;
}

13
include/basedata.h
Unescape View File

@ -15,19 +15,6 @@ struct Region
real lonb, lone, latb, late;
};
struct GridPoint
{
size_t ix = 0, iy = 0;
real x = -1.0, y = -1.0;
bool Valid() const { return x >= 0.0 && y >= 0.0; }
};
struct Point2D
{
real x, y;
};
enum class VarPresence
{
NONE,

144
include/cache.h
Unescape View File

@ -1,144 +0,0 @@
#pragma once
#include "MString.h"
#include <sqlite3.h>
#include <time.h>
#include <variant>
using michlib::MString;
class GenericCache
{
public:
virtual bool Put(const MString& key, const MString& value, size_t ttl) const = 0;
virtual std::pair<MString, bool> Get(const MString& key) const = 0;
virtual ~GenericCache() {}
};
class FakeCache: public GenericCache
{
public:
virtual bool Put([[maybe_unused]] const MString& key, [[maybe_unused]] const MString& value, [[maybe_unused]] size_t ttl) const override { return false; }
virtual std::pair<MString, bool> Get([[maybe_unused]] const MString& key) const override { return {"", false}; }
virtual ~FakeCache() override {}
};
class SQLiteCache: public GenericCache
{
sqlite3* db = nullptr;
public:
bool Init(const MString& name)
{
Close();
auto ret = sqlite3_open(name.Buf(), &db);
if(ret != SQLITE_OK)
{
Close();
return false;
}
// Create table
sqlite3_stmt* sqst;
int i;
i = sqlite3_prepare_v2(db,
"CREATE TABLE IF NOT EXISTS `cache`('key' TEXT PRIMARY KEY ON CONFLICT REPLACE NOT NULL ON CONFLICT FAIL, 'value' BLOB NOT NULL ON CONFLICT FAIL, "
"'exptime' INTEGER NOT NULL ON CONFLICT FAIL) WITHOUT ROWID, STRICT;",
-1, &sqst, 0);
i = sqlite3_step(sqst);
if(i != SQLITE_DONE)
{
sqlite3_finalize(sqst);
Close();
return false;
}
sqlite3_finalize(sqst);
sqlite3_busy_timeout(db, 1000);
return true;
}
void Close()
{
if(db != nullptr) sqlite3_close(db);
db = nullptr;
}
virtual bool Put(const MString& key, const MString& value, size_t ttl) const override
{
if(!*this) return false;
sqlite3_stmt* sqst = nullptr;
int i = SQLITE_OK;
if(i == SQLITE_OK) i = sqlite3_prepare_v2(db, "INSERT OR REPLACE into `cache` VALUES(?1,?2,?3);", -1, &sqst, 0);
if(i == SQLITE_OK) i = sqlite3_bind_text(sqst, 1, key.Buf(), -1, SQLITE_STATIC);
if(i == SQLITE_OK) i = sqlite3_bind_blob64(sqst, 2, value.Buf(), value.Len(), SQLITE_STATIC);
if(i == SQLITE_OK) i = sqlite3_bind_int64(sqst, 3, time(nullptr) + ttl);
if(i == SQLITE_OK) i = sqlite3_step(sqst);
sqlite3_finalize(sqst);
return i == SQLITE_OK;
}
virtual std::pair<MString, bool> Get(const MString& key) const override
{
if(!*this) return {"", false};
sqlite3_stmt* sqst = nullptr;
int i = SQLITE_OK;
if(i == SQLITE_OK) i = sqlite3_prepare_v2(db, "SELECT value from `cache` WHERE key=?1 AND exptime>?2;", -1, &sqst, 0);
if(i == SQLITE_OK) i = sqlite3_bind_text(sqst, 1, key.Buf(), -1, SQLITE_STATIC);
if(i == SQLITE_OK) i = sqlite3_bind_int64(sqst, 2, time(nullptr));
if(i == SQLITE_OK) i = sqlite3_step(sqst);
if(i == SQLITE_ROW)
{
auto p = sqlite3_column_blob(sqst, 0);
auto sz = sqlite3_column_bytes(sqst, 0);
if(p != nullptr)
{
MString out(p, sz);
sqlite3_finalize(sqst);
return {std::move(out), true};
}
}
sqlite3_finalize(sqst);
return {"", false};
}
virtual ~SQLiteCache() override
{
if(!*this) return;
sqlite3_stmt* sqst = nullptr;
int i = SQLITE_OK;
if(i == SQLITE_OK) i = sqlite3_prepare_v2(db, "DELETE from `cache` WHERE exptime<?1;", -1, &sqst, 0);
if(i == SQLITE_OK) i = sqlite3_bind_int64(sqst, 1, time(nullptr));
if(i == SQLITE_OK) i = sqlite3_step(sqst);
sqlite3_finalize(sqst);
}
explicit operator bool() const { return db != nullptr; }
};
inline GenericCache* CreateCache(const MString& cachedesc)
{
auto i = cachedesc.GetPos(':');
if(i == 0)
{
if(cachedesc == "no") return new FakeCache;
return nullptr;
}
auto name = cachedesc.SubStr(1, i - 1);
auto par = cachedesc.SubStr(i + 1, cachedesc.Len() - i);
if(name == "sqlite")
{
auto ret = new SQLiteCache;
ret->Init(par);
if(*ret) return ret;
delete ret;
}
return nullptr;
}

29
include/curlfuncs.h
Unescape View File

@ -1,29 +0,0 @@
#pragma once
#include "MString.h"
#include <curl/curl.h>
#include <memory>
using michlib::MString;
class CURLRAIIDT
{
public:
// TODO: make static
void operator()(CURL* c) { curl_easy_cleanup(c); }
};
class CURLRAII: public std::unique_ptr<CURL, CURLRAIIDT>
{
public:
CURLRAII() { reset(curl_easy_init()); }
operator CURL*() const { return get(); }
};
// Curl writeback function, write to MString
size_t Write2String(char* ptr, size_t size, size_t n, void* data);
// Curl writeback function, write to file descriptor
size_t Write2File(char* ptr, size_t size, size_t n, void* data);
// Get content of url to MString
std::pair<MString, CURLcode> GetUrl(const CURLRAII& chandle, const MString& url);

51
include/interpolation.h
Unescape View File

@ -1,51 +0,0 @@
#pragma once
#include "basedata.h"
#include "comdefs.h"
using michlib::real;
template<class D> class LinearInterpolator: public D
{
public:
LinearInterpolator(D&& d): D(std::move(d)) {}
real operator()(const struct Point2D& in) const
{
const auto gp = D::GridPos(in.x, in.y);
if(!gp.Valid()) return NAN;
if(gp.x == 0.0 && gp.y == 0.0) return D::V(gp.ix, gp.iy);
real v00;
real v10;
real v01;
real v11;
bool isfill = false;
size_t fx, fy;
// Count fills
for(size_t ix = 0; ix <= 1; ix++)
for(size_t iy = 0; iy <= 1; iy++)
{
if(isfill && D::IsFill(gp.ix + ix, gp.iy + iy)) return NAN;
if(D::IsFill(gp.ix + ix, gp.iy + iy))
{
fx = ix;
fy = iy;
isfill = true;
}
}
v00 = D::V(gp.ix, gp.iy);
v10 = D::V(gp.ix + 1, gp.iy);
v01 = D::V(gp.ix, gp.iy + 1);
v11 = D::V(gp.ix + 1, gp.iy + 1);
if(isfill && fx == 0 && fy == 0) v00 = (v10 + v01 + v11) / 3.0;
if(isfill && fx == 1 && fy == 0) v10 = (v00 + v01 + v11) / 3.0;
if(isfill && fx == 0 && fy == 1) v01 = (v10 + v00 + v11) / 3.0;
if(isfill && fx == 1 && fy == 1) v11 = (v10 + v01 + v00) / 3.0;
return v00 + gp.y * (v01 - v00) + gp.x * ((v10 - v00) + gp.y * (v00 - v01 + v11 - v10));
};
};

62
include/mirrorfuncs.h
Unescape View File

@ -1,62 +0,0 @@
#pragma once
#include "curlfuncs.h"
#include "mdatetime.h"
#include <dirent.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <vector>
using michlib::MDateTime;
class DIRRAIIDT
{
public:
// TODO: make static
void operator()(DIR* d) { closedir(d); }
};
class DIRRAII: public std::unique_ptr<DIR, DIRRAIIDT>
{
public:
operator DIR*() const { return get(); }
};
struct FileInfo
{
MString url;
MString name;
MDateTime mtime;
size_t size;
};
// Remove last element from path
inline MString DirName(const MString& name)
{
auto p = name.GetPos('/', false);
if(p == 0) return name;
return name.SubStr(1, p - 1);
}
// Get last element from path
inline MString FileName(const MString& name)
{
auto p = name.GetPos('/', false);
if(p == 0) return name;
return name.SubStr(p + 1, name.Len() - p);
}
// Check and, if necessary, create the path to the file
bool MakePath(const MString& dname);
// Get local file list
std::pair<std::vector<struct FileInfo>, MString> ReadLocalFileList(const MString& dir, const MString& path = "");
// Download file to the local mirror
MString DownloadFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const MString& root);
// Remove file from the local mirror
MString RemoveFile(const struct FileInfo& linfo);
// Updare file in the local mirror
MString UpdateFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const struct FileInfo& linfo, const MString& root);

17
include/simple2ddata.h
Unescape View File

@ -42,23 +42,6 @@ class Simple2DData: public BaseData
real XStep() const { return xstep; }
real YStep() const { return ystep; }
struct GridPoint GridPos(real x, real y) const
{
struct GridPoint out;
if(!*this) return out;
if(x < x0 || x > x0 + (nx - 1) * xstep || y < y0 || y > y0 + (ny - 1) * ystep) return out;
real rx = x - x0;
real ry = y - y0;
out.ix = static_cast<decltype(out.ix)>(michlib::Floor(rx / xstep));
out.iy = static_cast<decltype(out.ix)>(michlib::Floor(ry / ystep));
out.x = rx - out.ix * xstep;
out.y = ry - out.iy * ystep;
return out;
}
};
class Rect2DData: public BaseData

2
michlib

@ -1 +1 @@
Subproject commit e2882902b88229bb4b0e6fbeb76c79ac6d46d53d
Subproject commit 6268edea843cae04337fe6493b611f3a4fe7cc9b

282
sources/COPERNICUS.cpp
Unescape View File

@ -1,282 +0,0 @@
#define MICHLIB_NOSOURCE
#include "COPERNICUS.h"
#include "mirrorfuncs.h"
#include <libxml/parser.h>
#include <libxml/tree.h>
using michlib::GPL;
const MString COPERNICUSData::caturl = "https://stac.marine.copernicus.eu/metadata/catalog.stac.json";
std::pair<Json::Value, MString> COPERNICUSData::GetJSON(const MString& url)
{
Json::Reader reader;
Json::Value obj;
MString content;
auto [val, suc] = cache->Get(url);
if(suc)
content = std::move(val);
else
{
michlib::message(url + " not found in cache, downloading");
auto [out, res] = GetUrl(chandle, url);
if(res != CURLE_OK) return {obj, MString("Can't download JSON: ") + curlerr};
cache->Put(url, out, 3600);
content = std::move(out);
}
reader.parse(content.Buf(), content.Buf() + content.Len(), obj, false);
return {obj, ""};
}
MString COPERNICUSData::ReadURL(const Json::Value& cat, const MString& prod)
{
const auto& links = cat["links"];
if(links.type() != Json::arrayValue) return "";
for(Json::ArrayIndex i = 0; i < links.size(); i++)
{
const auto& titl = links[i]["title"];
const auto& href = links[i]["href"];
if(titl.type() == Json::stringValue && href.type() == Json::stringValue)
{
MString str(titl.asString().c_str());
if(str == prod) return MString(href.asString().c_str());
}
}
return "";
}
std::pair<std::vector<struct FileInfo>, MString> COPERNICUSData::ReadRemoteFileList(const MString& url)
{
LIBXML_TEST_VERSION
std::vector<struct FileInfo> out;
MString bucket, prefix;
// Split url on prefix and bucket
{
size_t pos = url.Len();
size_t count = 0;
for(size_t i = 0; i < url.Len(); i++)
{
if(url[i] == '/') count++;
if(count == 4)
{
pos = i;
break;
}
}
if(pos == url.Len()) return {out, "Can't parse url: " + url};
bucket = url.SubStr(1, pos);
prefix = url.SubStr(pos + 2, url.Len() - pos - 1);
}
MString cont;
bool next = true;
while(next)
{
MString url = bucket + "?list-type=2&prefix=" + prefix;
if(cont.Exist()) url += "&continuation-token=" + cont;
cont = "";
auto [data, res] = GetUrl(chandle, url);
if(res != CURLE_OK) return {out, MString("Can't download ") + url + ": " + curlerr};
xmlDocPtr doc = xmlReadMemory(data.Buf(), data.Len(), "data.xml", nullptr, 0);
if(doc == nullptr) return {out, MString("Can't download ") + url + ": XML parse error"};
auto cur = xmlDocGetRootElement(doc);
if(cur == nullptr)
{
xmlFreeDoc(doc);
return {out, MString("Can't download ") + url + ": empty XML"};
}
if(xmlStrEqual(cur->name, (const xmlChar*)"ListBucketResult") == 0)
{
xmlFreeDoc(doc);
return {out, MString("Can't download ") + url + ": unknown XML"};
}
for(const auto* n = cur->children; n; n = n->next)
{
if(xmlStrEqual(n->name, (const xmlChar*)"NextContinuationToken") == 1)
{
auto* content = xmlNodeGetContent(n);
cont = (char*)content;
xmlFree(content);
}
if(xmlStrEqual(n->name, (const xmlChar*)"Contents") == 1)
{
MString fname;
MDateTime mtime;
size_t size = 0;
for(const auto* c = n->children; c; c = c->next)
{
if(xmlStrEqual(c->name, (const xmlChar*)"Key") == 1)
{
auto* content = xmlNodeGetContent(c);
fname = (char*)content;
xmlFree(content);
}
if(xmlStrEqual(c->name, (const xmlChar*)"LastModified") == 1)
{
auto* content = xmlNodeGetContent(c);
mtime.FromString((char*)content);
xmlFree(content);
}
if(xmlStrEqual(c->name, (const xmlChar*)"Size") == 1)
{
auto* content = xmlNodeGetContent(c);
size = MString((char*)content).ToInteger<size_t>();
xmlFree(content);
}
}
out.emplace_back(bucket + "/" + fname, fname.SubStr(prefix.Len() + 2, fname.Len() - prefix.Len() - 1), mtime, size);
}
}
xmlFreeDoc(doc);
next = cont.Exist();
}
std::sort(out.begin(), out.end(), [](const struct FileInfo& a, const struct FileInfo& b) { return a.name < b.name; });
return {out, ""};
}
MString COPERNICUSData::Mirror(const CLArgs& args)
{
GPL.UsePrefix("COPERNICUS");
// Local directory
MString mirrorroot = GPL.ParameterSValue("MirrorTo", "");
if(!mirrorroot.Exist()) return "Local mirror directory not specified";
// Cache
cache.reset(CreateCache(GPL.ParameterSValue("Cache", "")));
if(!cache)
{
michlib::errmessage("Can't init cache");
cache.reset(new FakeCache);
}
curl_easy_setopt(chandle, CURLOPT_ERRORBUFFER, curlerr);
if(!args.contains("product")) return "Copernicus product not specified";
MString prod = args.at("product");
Json::Value product;
MString produrl;
// Get catalog
{
auto [cat, err] = GetJSON(caturl);
if(err.Exist()) return "Can't download catalog: " + err;
if(cat["title"].type() != Json::stringValue || cat["title"].asString() != "Copernicus Marine Data Store") return "Can't parse catalog";
catalog = std::move(cat);
}
// Get product
{
auto url = ReadURL(catalog, prod);
if(!url.Exist()) return "Url for product " + prod + " not found in catalog";
produrl = DirName(caturl) + "/" + url;
auto [pr, err] = GetJSON(produrl);
if(err.Exist()) return "Can't download product information from " + produrl + ": " + err;
product = std::move(pr);
}
std::vector<MString> dsets;
if(args.contains("dataset"))
dsets.push_back(args.at("dataset"));
else
{
const auto& links = product["links"];
if(links.type() != Json::arrayValue) return "Can't find information about datasets";
for(Json::ArrayIndex i = 0; i < links.size(); i++)
{
const auto& rel = links[i]["rel"];
const auto& titl = links[i]["title"];
if(rel.type() == Json::stringValue && titl.type() == Json::stringValue && rel.asString() == "item") dsets.push_back(titl.asString().c_str());
}
}
for(const auto& dset: dsets)
{
michlib::message("Mirroring " + dset);
auto url = ReadURL(product, dset);
if(!url.Exist()) return "Url for dataset " + dset + " not found in product description";
MString dseturl = DirName(produrl) + "/" + url;
auto [ds, err] = GetJSON(dseturl);
if(err.Exist()) return "Can't download dataset information from " + dseturl + ": " + err;
const auto& href = ds["assets"]["native"]["href"];
if(href.type() != Json::stringValue) return "Can't find data for dataset " + dset + " from product " + prod;
url = href.asString().c_str();
MString locroot = mirrorroot + "/" + prod + "/" + dset;
auto [lfiles, lerr] = ReadLocalFileList(locroot);
if(lerr.Exist()) return lerr;
auto [rfiles, rerr] = ReadRemoteFileList(url);
if(rerr.Exist()) return rerr;
std::vector<size_t> down, rem;
std::vector<std::pair<size_t, size_t>> upd;
{
size_t rpos = 0, lpos = 0;
while(rpos != rfiles.size() || lpos != lfiles.size())
{
if(rpos == rfiles.size())
while(lpos != lfiles.size()) rem.push_back(lpos++);
if(lpos == lfiles.size())
while(rpos != rfiles.size()) down.push_back(rpos++);
if(rpos == rfiles.size() || lpos == lfiles.size()) continue;
if(rfiles[rpos].name < lfiles[lpos].name)
down.push_back(rpos++);
else if(lfiles[lpos].name < rfiles[rpos].name)
rem.push_back(lpos++);
else
{
auto delta = rfiles[rpos].mtime.Epoch() - lfiles[lpos].mtime.Epoch();
if(delta < 0) delta = -delta;
if(delta > 0 || rfiles[rpos].size != lfiles[lpos].size) upd.emplace_back(rpos, lpos);
lpos++;
rpos++;
}
}
}
michlib::message(MString("New files: ") + down.size());
michlib::message(MString("Obsolete files: ") + rem.size());
michlib::message(MString("Modified files: ") + upd.size());
for(size_t i = 0; i < down.size(); i++)
{
size_t ri = down[i];
auto err = DownloadFile(chandle, rfiles[ri], locroot);
if(err.Exist()) return err;
}
for(size_t i = 0; i < rem.size(); i++)
{
size_t li = rem[i];
auto err = RemoveFile(lfiles[li]);
if(err.Exist()) return err;
}
for(size_t i = 0; i < upd.size(); i++)
{
size_t ri = upd[i].first;
size_t li = upd[i].second;
auto err = UpdateFile(chandle, rfiles[ri], lfiles[li], locroot);
if(err.Exist()) return err;
}
}
return "";
}

36
sources/COPERNICUS.h
Unescape View File

@ -1,36 +0,0 @@
#pragma once
#include "ParseArgs.h"
#include "cache.h"
#include "curlfuncs.h"
#include "mdatetime.h"
#include <json/json.h>
using michlib::MDateTime;
using michlib::MString;
class COPERNICUSData
{
static const MString caturl;
std::unique_ptr<GenericCache> cache;
CURLRAII chandle;
Json::Value catalog;
char curlerr[CURL_ERROR_SIZE];
// Get url for product or dataset from catalog
static MString ReadURL(const Json::Value& cat, const MString& prod);
// Download JSON from url
std::pair<Json::Value, MString> GetJSON(const MString& url);
// Get remote file list from url
std::pair<std::vector<struct FileInfo>,MString> ReadRemoteFileList(const MString& url);
public:
static constexpr const char* name = "COPERNICUS";
COPERNICUSData() = default;
// Main mirror function
MString Mirror(const CLArgs& args);
};

86
sources/GRIDVEL.cpp
Unescape View File

@ -1,86 +0,0 @@
#define MICHLIB_NOSOURCE
#include "GRIDVEL.h"
MString GRIDVELData::Info() const
{
if(!isOk()) return "";
// clang-format off
return MString() +
" Region: (" + u.Xb() + " : " + u.Xe() + ") x (" + u.Yb() + " : " + u.Ye() + ")\n" +
" Grid: " + u.Nx() + "x" + u.Ny() + "\n" +
" Supported variables: u, v, U2";
// clang-format on
}
MString GRIDVELData::Open(const CLArgs& args)
{
MString uname = args.contains("u") ? args.at("u") : "";
MString vname = args.contains("v") ? args.at("v") : "";
if(!uname.Exist()) return "File with u component not specified";
if(!vname.Exist()) return "File with v component not specified";
unit = args.contains("unit") ? args.at("unit") : "cm/s";
fill = args.contains("fill") ? args.at("fill").ToReal() : 1e10;
u.Open(uname);
if(!u.Opened()) return "Can't open file: " + uname;
v.Open(vname);
if(!v.Opened()) return "Can't open file: " + vname;
if(u != v) return "Files " + uname + " and " + vname + " have different grids";
return "";
}
bool GRIDVELData::Read(const MString& vname, std::map<MString, GRIDVELData::Data>& cache, size_t i) const
{
if(cache.contains(vname)) return true;
if(!isOk()) return false;
// Only rectangular grids are supported
real xs = (u.Xe() - u.Xb()) / (u.Nx() - 1);
real ys = (u.Ye() - u.Yb()) / (u.Ny() - 1);
Data out(u.Nx(), u.Ny(), u.Xb(), u.Yb(), xs, ys, MString(unit));
// U and U2 from u and v
if(vname == "U" || vname == "U2")
{
bool square = vname == "U2";
if(!(Read("u", cache, i) && Read("v", cache, i))) return false;
cache[vname] = cache.at("u").CopyGrid();
auto& U = cache.at(vname);
const auto& udata = cache.at("u");
const auto& vdata = cache.at("v");
if(udata.Unit().Exist()) U.SetUnit(square ? ("(" + udata.Unit() + ")2") : udata.Unit());
for(size_t ind = 0; ind < U.N(); ind++)
{
if(udata.IsFill(ind) || vdata.IsFill(ind))
U.V(ind) = U.Fillval();
else
U.V(ind) = square ? (udata(ind) * udata(ind) + vdata(ind) * vdata(ind)) : michlib::Hypot(udata(ind), vdata(ind));
}
return true;
}
if(vname == "u" || vname == "v")
{
bool isu = vname == "u";
for(IType ix = 0; ix < u.Nx(); ix++)
for(IType iy = 0; iy < u.Ny(); iy++)
{
if(isu ? (u(ix, iy) == NAN || u(ix, iy) >= 1e10) : (v(ix, iy) == NAN || v(ix, iy) >= 1e10))
out.V(ix, iy) = out.Fillval();
else
out.V(ix, iy) = isu ? u(ix, iy) : v(ix, iy);
}
if(out)
{
cache[vname] = std::move(out);
return true;
}
}
return false;
}

50
sources/GRIDVEL.h
Unescape View File

@ -1,50 +0,0 @@
#pragma once
#include "DataAdapters/gridfile.h"
#include "mdatetime.h"
#include "simple2ddata.h"
using michlib::MDateTime;
class GRIDVELData
{
michlib::GridFile u,v;
MString unit;
real fill;
using IType=decltype(u.Nx());
public:
static constexpr const char* name = "GRIDVEL";
static constexpr const char* disabledactions = "genintfile";
using Data = Simple2DData;
GRIDVELData() = default;
MString Info() const;
// TODO: RetVal
MString Open(const CLArgs& args);
bool isOk() const { return u.Opened() &&v.Opened(); }
size_t NTimes() const { return 1; }
MDateTime Time(size_t i) const
{
return MDateTime();
}
//time_t Timestep() const { return isOk() ? (times[1] - times[0]) : 0; }
explicit operator bool() const { return u.Opened() &&v.Opened(); }
VarPresence CheckVar(const MString& vname) const
{
if(vname == "u" || vname == "v") return VarPresence::INTERNAL;
if(vname == "U" || vname == "U2") return VarPresence::DERIVED;
return VarPresence::NONE;
}
bool Read(const MString& vname, std::map<MString, Data>& cache, size_t i) const;
};

10
src/CMakeLists.txt
Unescape View File

@ -2,23 +2,15 @@ set(EXENAME odm)
set(ACTIONLISTINC ${CMAKE_CURRENT_BINARY_DIR}/../include/actionlist.h) # Include actions files and define the actions classes list
set(DATALISTINC ${CMAKE_CURRENT_BINARY_DIR}/../include/datalist.h) # Include data sources files and define the data sources classes list
find_package(PkgConfig REQUIRED)
find_library(netcdf netcdf REQUIRED)
find_package(OpenMP REQUIRED)
find_package(CURL REQUIRED)
find_package(LibXml2 REQUIRED)
find_package(SQLite3 REQUIRED)
pkg_check_modules(JSONCPP REQUIRED jsoncpp)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
include_directories(${JSONCPP_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIRS} ${SQLite3_INCLUDE_DIRS})
file(GLOB srcs CONFIGURE_DEPENDS *.cpp)
add_executable(${EXENAME} ${srcs} ${ACTIONLISTINC} ${SOURCELISTINC})
target_include_directories(${EXENAME} PRIVATE ../michlib/michlib ${CMAKE_CURRENT_BINARY_DIR}/../include)
target_link_libraries(${EXENAME} ${linker_options} ${netcdf} OpenMP::OpenMP_CXX CURL::libcurl ${JSONCPP_LINK_LIBRARIES} LibXml2::LibXml2 SQLite::SQLite3 teos)
target_link_libraries(${EXENAME} ${linker_options} ${netcdf} OpenMP::OpenMP_CXX teos)
set_target_properties(${EXENAME} PROPERTIES POSITION_INDEPENDENT_CODE ON)
install(TARGETS ${EXENAME})

41
src/curlfuncs.cpp
Unescape View File

@ -1,41 +0,0 @@
#define MICHLIB_NOSOURCE
#include "curlfuncs.h"
#include <unistd.h>
using michlib::pointer_cast;
using michlib::uint1;
size_t Write2String(char* ptr, size_t size, size_t n, void* data)
{
MString* out = pointer_cast<MString*>(data);
*out += MString(ptr, size * n);
return size * n;
}
size_t Write2File(char* ptr, size_t size, size_t n, void* data)
{
const int* fd = pointer_cast<const int*>(data);
size_t count = size * n;
const uint1* buf = pointer_cast<const uint1*>(ptr);
while(count != 0)
{
auto wr = write(*fd, buf, count);
if(wr == -1) return 0;
count -= wr;
buf += wr;
}
return size * n;
}
std::pair<MString, CURLcode> GetUrl(const CURLRAII& chandle, const MString& url)
{
MString out;
curl_easy_setopt(chandle, CURLOPT_URL, url.Buf());
curl_easy_setopt(chandle, CURLOPT_WRITEFUNCTION, Write2String);
curl_easy_setopt(chandle, CURLOPT_WRITEDATA, &out);
auto res = curl_easy_perform(chandle);
return {out, res};
}

130
src/mirrorfuncs.cpp
Unescape View File

@ -1,130 +0,0 @@
#define MICHLIB_NOSOURCE
#include "mirrorfuncs.h"
#include "StringFunctions.h"
#include "filehelpers.h"
#include "merrors.h"
using michlib::FD;
using michlib::message;
bool MakePath(const MString& dname)
{
struct stat st;
int ret = stat(dname.Buf(), &st);
if(ret == 0) return S_ISDIR(st.st_mode);
auto dirs = michlib::Split_on_words(dname, "/", false);
MString cdir = "";
for(const auto& dir: dirs)
{
cdir += "/" + dir;
ret = stat(cdir.Buf(), &st);
if(ret == 0 && S_ISDIR(st.st_mode)) continue;
if(ret == 0 && !S_ISDIR(st.st_mode)) return false;
ret = mkdir(cdir.Buf(), 0755);
if(ret != 0) return false;
}
return true;
}
std::pair<std::vector<struct FileInfo>, MString> ReadLocalFileList(const MString& dir, const MString& path)
{
std::vector<struct FileInfo> out;
DIRRAII dhandle;
MakePath(dir);
dhandle.reset(opendir(dir.Buf()));
if(!dhandle) return {out, "Can't open directory " + path + (path.Exist() ? "/" : "") + dir};
int dfd = dirfd(dhandle);
errno = 0;
struct dirent* dent = readdir(dhandle);
if(errno != 0) return {out, "Can't read directory " + path + (path.Exist() ? "/" : "") + dir};
struct stat st;
do {
if(dent->d_name[0] != '.')
{
int ret = fstatat(dfd, dent->d_name, &st, AT_SYMLINK_NOFOLLOW);
if(ret != 0) return {out, "Can't stat " + path + "/" + dir + "/" + dent->d_name};
if(S_ISDIR(st.st_mode)) // Directory, recurse
{
auto [list, err] = ReadLocalFileList(dir + "/" + dent->d_name, path + (path.Exist() ? "/" : "") + dent->d_name);
if(err.Exist()) return {out, err};
out.insert(out.end(), list.begin(), list.end());
}
if(S_ISREG(st.st_mode)) // Regular file
{
out.emplace_back(dir + "/" + dent->d_name, path + (path.Exist() ? "/" : "") + dent->d_name, MDateTime(st.st_mtim.tv_sec, st.st_mtim.tv_nsec), st.st_size);
}
// Ignore non-directories, non-files
}
dent = readdir(dhandle);
} while(dent != nullptr || errno != 0);
if(errno != 0) return {out, "Can't read directory " + path + "/" + dir};
std::sort(out.begin(), out.end(), [](const struct FileInfo& a, const struct FileInfo& b) { return a.name < b.name; });
return {out, ""};
}
MString DownloadFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const MString& root)
{
message("Downloading " + rinfo.url);
MString dname = DirName(rinfo.name), fname = FileName(rinfo.name);
FD fd;
if(!MakePath(root + "/" + dname)) return "Can't create directory " + root + "/" + dname;
fd.Reset(creat((root + "/" + rinfo.name).Buf(), 0644));
if(!fd) return "Can't create file " + root + "/" + rinfo.name;
char errbuf[CURL_ERROR_SIZE];
int cfd = fd.Get();
curl_easy_setopt(chandle, CURLOPT_ERRORBUFFER, errbuf);
curl_easy_setopt(chandle, CURLOPT_WRITEFUNCTION, Write2File);
curl_easy_setopt(chandle, CURLOPT_WRITEDATA, &cfd);
curl_easy_setopt(chandle, CURLOPT_URL, rinfo.url.Buf());
auto res = curl_easy_perform(chandle);
if(res != CURLE_OK)
{
unlink((root + "/" + rinfo.name).Buf());
return MString("Can't download file: ") + errbuf;
}
{
struct timespec times[2];
times[0].tv_sec = times[1].tv_sec = rinfo.mtime.Epoch();
times[0].tv_nsec = times[1].tv_nsec = 0;
int ret = futimens(fd, times);
if(ret != 0)
{
unlink((root + "/" + rinfo.name).Buf());
return "Can't set mtime for file: " + root + "/" + rinfo.name;
}
}
return "";
}
MString RemoveFile(const struct FileInfo& linfo)
{
message("Remove " + linfo.url);
int ret = unlink(linfo.url.Buf());
if(ret != 0) return "Can't remove file " + linfo.url;
return "";
}
MString UpdateFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const struct FileInfo& linfo, const MString& root)
{
MString err;
message("Update " + linfo.url);
err = RemoveFile(linfo);
if(err.Exist()) return err;
err = DownloadFile(chandle, rinfo, root);
if(err.Exist()) return err;
return "";
}
Write Preview
Loading…
Cancel
Save