Added mirror action and COPERNICUS source for it with caches, fucnctions, etc

actions/actionint.h

@@ -0,0 +1,210 @@
+#pragma once
+#include "BFileR.h"
+#include "BFileW.h"
+#include "actiondep.h"
+#include "interpolation.h"
+#include "ncfilew.h"
+#include "ncfuncs.h"
+#include <memory>
+
+template<class T>
+concept ReadIsInterpolable = requires {
+ {
+  std::declval<ReadType<T>>().GridPos(0.0, 0.0)
+ } -> std::convertible_to<struct GridPoint>;
+};
+
+ADD_ACTION(INT, int, (ReadPSupported<Source> || ReadSupported<Source>)&&ReadIsInterpolable<Source>);
+
+template<class D> MString ActionINT::DoAction(const CLArgs& args, D& ds)
+{
+ struct TPoint
+ {
+  struct Point2D p;
+  MDateTime      t;
+  real           lon0, lat0, dtime;
+ };
+
+ MDateTime refdate("1980-01-01");
+
+ if(!args.contains("input")) return "No input file given";
+ michlib::BFileR in;
+ if(in.Open(args.at("input")) != ERR_NOERR) return "Can't open input file " + args.at("input");
+ std::vector<TPoint> points;
+ struct Region       reg;
+
+ {
+  michlib::uint loncol = in.Columns(), latcol = in.Columns(), timecol = in.Columns();
+  michlib::uint lon0col = in.Columns(), lat0col = in.Columns(), dtimecol = in.Columns();
+  for(uint i = 0; i < in.Columns(); i++)
+  {
+   if(in.ColumnName(i + 1) == "lon") loncol = i;
+   if(in.ColumnName(i + 1) == "lat") latcol = i;
+   if(in.ColumnName(i + 1) == "rtime") timecol = i;
+   if(in.ColumnName(i + 1) == "lon0") lon0col = i;
+   if(in.ColumnName(i + 1) == "lat0") lat0col = i;
+   if(in.ColumnName(i + 1) == "time") dtimecol = i;
+  }
+  if(loncol >= in.Columns()) return "Lon column not found in input file " + args.at("input");
+  if(latcol >= in.Columns()) return "Lat column not found in input file " + args.at("input");
+  if(timecol >= in.Columns()) return "Time column not found in input file " + args.at("input");
+  if(lon0col >= in.Columns()) return "Lon0 column not found in input file " + args.at("input");
+  if(lat0col >= in.Columns()) return "Lat0 column not found in input file " + args.at("input");
+  if(dtimecol >= in.Columns()) return "Age column not found in input file " + args.at("input");
+
+  points.resize(in.Rows());
+  reg.lonb = reg.lone = in[loncol][0];
+  reg.latb = reg.late = in[latcol][0];
+  for(uint i = 0; i < in.Rows(); i++)
+  {
+   points[i].p.x   = in[loncol][i];
+   points[i].p.y   = in[latcol][i];
+   points[i].t     = refdate + static_cast<time_t>(michlib::Round(in[timecol][i] * 86400));
+   points[i].lon0  = in[lon0col][i];
+   points[i].lat0  = in[lat0col][i];
+   points[i].dtime = in[dtimecol][i];
+
+   reg.lonb = std::min(reg.lonb, points[i].p.x);
+   reg.lone = std::max(reg.lone, points[i].p.x);
+   reg.latb = std::min(reg.latb, points[i].p.y);
+   reg.late = std::max(reg.late, points[i].p.y);
+  }
+  std::ranges::sort(points, {}, &TPoint::t);
+ }
+ in.Close();
+
+ auto resop = ds.Open(args);
+ if(resop.Exist()) return "Can't open source: " + resop;
+
+ michlib_internal::ParameterListEx pars;
+ pars.UsePrefix("");
+ pars.SetParameter("source", args.at("source"));
+ pars.SetParameter("history", args.at("_cmdline"));
+
+ MString varstring;
+ if(args.contains("var"))
+  varstring = args.at("var");
+ else
+ {
+  if(args.contains("vars"))
+   varstring = args.at("vars");
+  else
+  {
+   if constexpr(HasDefVars<D>)
+    varstring = ds.DefaultVars();
+   else
+    return "Variables not specified";
+  }
+ }
+
+ auto                 vlist = michlib::Split_on_words(varstring, " ,", false);
+ std::vector<MString> vnames{std::make_move_iterator(std::begin(vlist)), std::make_move_iterator(std::end(vlist))};
+ {
+  std::set<MString> used;
+  for(const auto& vname: vnames)
+  {
+   if(used.contains(vname)) return "Duplicate variable " + vname + " in list " + varstring;
+   if(ds.CheckVar(vname) == VarPresence::NONE) return "Variable " + vname + " not exists in this dataset";
+   used.insert(vname);
+  }
+ }
+
+ pars.SetParameter("variables", varstring);
+
+ //int compress = 3;
+ //if(args.contains("compress")) compress = args.at("compress").ToInt();
+
+ std::unique_ptr<const BaseParameters> sourcepars;
+ if constexpr(ParametersSupported<D>)
+ {
+  if constexpr(ParametersRequiredRegion<D>)
+  {
+   auto [p, err] = ds.Parameters(pars, args, reg);
+   if(err.Exist()) return err;
+   sourcepars.reset(p);
+  }
+  else
+  {
+   auto [p, err] = ds.Parameters(pars, args);
+   if(err.Exist()) return err;
+   sourcepars.reset(p);
+  }
+ }
+ auto p = sourcepars.get();
+
+ MString name = args.contains("out") ? args.at("out") : "out.bin";
+ //MString outfmt = args.contains("format") ? args.at("format") : (GetExt(name) == "nc" ? "nc" : "bin");
+
+ size_t          loind = ds.NTimes(), hiind = ds.NTimes(), curind = 0;
+ michlib::BFileW fw;
+ const MDateTime first = ds.Time(0), last = ds.Time(ds.NTimes() - 1);
+
+ std::vector<LinearInterpolator<ReadType<D>>> lo, hi;
+
+ {
+  size_t ic = 0;
+  fw.Create(name, 6 + vnames.size());
+  fw.SetColumnName(++ic, "lon0");
+  fw.SetColumnName(++ic, "lat0");
+  fw.SetColumnName(++ic, "lon");
+  fw.SetColumnName(++ic, "lat");
+  fw.SetColumnName(++ic, "time");
+  fw.SetColumnName(++ic, "rtime");
+  for(size_t i = 0; i < vnames.size(); i++) fw.SetColumnName(++ic, vnames[i]);
+  fw.SetParameters(pars);
+ }
+
+ for(size_t i = 0; i < points.size(); i++)
+ {
+  if(points[i].t < first || points[i].t > last) continue;
+  while(!(points[i].t >= ds.Time(curind) && points[i].t <= ds.Time(curind + 1))) curind++;
+  if(curind != loind && curind != hiind)
+  {
+   loind = curind;
+   hiind = curind + 1;
+   {
+    auto temp = Read(ds, vnames, p, loind);
+    if(temp.size() != vnames.size()) return "Can't read data";
+    //lo.resize(temp.size());
+    for(size_t j = 0; j < temp.size(); j++) lo.emplace_back(std::move(temp[j]));
+   }
+   {
+    auto temp = Read(ds, vnames, p, hiind);
+    if(temp.size() != vnames.size()) return "Can't read data";
+    //hi.resize(temp.size());
+    for(size_t j = 0; j < temp.size(); j++) hi.emplace_back(std::move(temp[j]));
+   }
+  }
+  else if(curind == hiind)
+  {
+   loind = curind;
+   hiind = curind + 1;
+   lo    = std::move(hi);
+   {
+    auto temp = Read(ds, vnames, p, hiind);
+    if(temp.size() != vnames.size()) return "Can't read data";
+    //hi.resize(temp.size());
+    for(size_t j = 0; j < temp.size(); j++) hi.emplace_back(std::move(temp[j]));
+   }
+  }
+  auto step  = ds.Time(hiind) - ds.Time(loind);
+  auto delta = points[i].t - ds.Time(loind);
+  real trel  = static_cast<real>(delta) / static_cast<real>(step);
+
+  fw.Write(points[i].lon0);
+  fw.Write(points[i].lat0);
+  fw.Write(points[i].p.x);
+  fw.Write(points[i].p.y);
+  fw.Write(points[i].dtime);
+  fw.Write(DeltaDates(refdate, points[i].t));
+  for(size_t iv = 0; iv < lo.size(); iv++)
+  {
+   real vlo = lo[iv](points[i].p);
+   real vhi = hi[iv](points[i].p);
+   fw.Write(vlo + (vhi - vlo) * trel);
+  }
+ }
+ fw.Finalize();
+ fw.Close();
+ return "";
+};

actions/actionmirror.h

@@ -0,0 +1,23 @@
+#pragma once
+#include "actiondep.h"
+#include "merrors.h"
+
+using michlib::message;
+
+template<class T>
+concept MirrorSupported = requires(T t, const CLArgs& args) {
+ {
+  t.Mirror(args)
+ } -> std::convertible_to<MString>;
+};
+
+ADD_ACTION(Mirror, mirror, MirrorSupported<Source>);
+
+template<class D> MString ActionMirror::DoAction(const CLArgs& args, D& data)
+{
+ //auto resop = data.Open(args);
+ //if(resop.Exist()) return "Can't open source: " + resop;
+ auto res = data.Mirror(args);
+ if(res.Exist()) return "Mirroring failed: " + res;
+ return "";
+};

actions/actionuv.h

@@ -152,9 +152,9 @@ template<class D> MString ActionUV::DoAction(const CLArgs& args, D& ds)
  MString name      = args.contains("out") ? args.at("out") : "";
  MString outfmt    = args.contains("outformat") ? args.at("outformat") : (GetExt(name) == "nc" ? "nc" : "bin");
  MString namevel   = args.contains("velout") ? args.at("velout") : "";
- MString outfmtvel = args.contains("veloutformat") ? args.at("veloutformat") : (GetExt(name) == "nc" ? "nc" : "bin");
+ MString outfmtvel = args.contains("veloutformat") ? args.at("veloutformat") : (GetExt(namevel) == "nc" ? "nc" : "bin");
  MString namestp   = args.contains("stpout") ? args.at("stpout") : "";
- MString outfmtstp = args.contains("stpoutformat") ? args.at("stpoutformat") : (GetExt(name) == "nc" ? "nc" : "bin");
+ MString outfmtstp = args.contains("stpoutformat") ? args.at("stpoutformat") : (GetExt(namestp) == "nc" ? "nc" : "bin");
 
  size_t shiftx = args.contains("shiftx") ? args.at("shiftx").ToInteger<size_t>() : 0;
  size_t shifty = args.contains("shifty") ? args.at("shifty").ToInteger<size_t>() : 0;
@@ -263,7 +263,7 @@ template<class D> MString ActionUV::DoAction(const CLArgs& args, D& ds)
    if(outfmtvel == "bin")
    {
     BFileW vel;
-    vel.Create(name, 4);
+    vel.Create(namevel, 4);
     vel.SetColumnName(1, "Longitude");
     vel.SetColumnName(2, "Latitude");
     vel.SetColumnName(3, "u, " + velunit);
@@ -285,7 +285,7 @@ template<class D> MString ActionUV::DoAction(const CLArgs& args, D& ds)
    {
     MString err;
 
-    if(!err.Exist() && !headwrited) err = fwfilt.Create(sdata, name, compress);
+    if(!err.Exist() && !headwrited) err = fwfilt.Create(sdata, namevel, compress);
     if(!err.Exist() && !headwrited) err = fwfilt.AddTimeData(tdata, !average);
     if(!err.Exist() && !headwrited) err = fwfilt.AddAtts(pars);
     if(!err.Exist() && !headwrited) err = fwfilt.AddVariable("u", "", "Eastward velocity", velunit, "");
@@ -312,11 +312,11 @@ template<class D> MString ActionUV::DoAction(const CLArgs& args, D& ds)
     for(size_t iy = 0; iy < data.Ny() - 1; iy++) stp.Add(data.StablePoints(ix, iy));
 
    if(outfmtstp == "bin")
-    stp.WriteBinBile(name, pars);
+    stp.WriteBinBile(namestp, pars);
    else if(outfmtstp == "nc" || outfmtstp == "netcdf")
    {
     MString err;
-    if(!err.Exist() && !headwrited) err = stp.CreateNcFile(name, pars, args.at("_cmdline"), compress, tdata, !average);
+    if(!err.Exist() && !headwrited) err = stp.CreateNcFile(namestp, pars, args.at("_cmdline"), compress, tdata, !average);
     if(!err.Exist()) err = stp.WriteNcFile(it);
     if(err.Exist()) return err;
    }

include/basedata.h

@@ -15,6 +15,19 @@ struct Region
  real lonb, lone, latb, late;
 };
 
+struct GridPoint
+{
+ size_t ix = 0, iy = 0;
+ real   x = -1.0, y = -1.0;
+
+ bool Valid() const { return x >= 0.0 && y >= 0.0; }
+};
+
+struct Point2D
+{
+ real x, y;
+};
+
 enum class VarPresence
 {
  NONE,

include/cache.h

@@ -0,0 +1,144 @@
+#pragma once
+#include "MString.h"
+#include <sqlite3.h>
+#include <time.h>
+#include <variant>
+
+using michlib::MString;
+
+class GenericCache
+{
+ public:
+ virtual bool                     Put(const MString& key, const MString& value, size_t ttl) const = 0;
+ virtual std::pair<MString, bool> Get(const MString& key) const                                   = 0;
+ virtual ~GenericCache() {}
+};
+
+class FakeCache: public GenericCache
+{
+ public:
+ virtual bool                     Put([[maybe_unused]] const MString& key, [[maybe_unused]] const MString& value, [[maybe_unused]] size_t ttl) const override { return false; }
+ virtual std::pair<MString, bool> Get([[maybe_unused]] const MString& key) const override { return {"", false}; }
+ virtual ~FakeCache() override {}
+};
+
+class SQLiteCache: public GenericCache
+{
+ sqlite3* db = nullptr;
+
+ public:
+ bool Init(const MString& name)
+ {
+  Close();
+  auto ret = sqlite3_open(name.Buf(), &db);
+  if(ret != SQLITE_OK)
+  {
+   Close();
+   return false;
+  }
+  // Create table
+  sqlite3_stmt* sqst;
+  int           i;
+
+  i = sqlite3_prepare_v2(db,
+                         "CREATE TABLE IF NOT EXISTS `cache`('key' TEXT PRIMARY KEY ON CONFLICT REPLACE NOT NULL ON CONFLICT FAIL, 'value' BLOB NOT NULL ON CONFLICT FAIL, "
+                         "'exptime' INTEGER NOT NULL ON CONFLICT FAIL) WITHOUT ROWID, STRICT;",
+                         -1, &sqst, 0);
+  i = sqlite3_step(sqst);
+  if(i != SQLITE_DONE)
+  {
+   sqlite3_finalize(sqst);
+   Close();
+   return false;
+  }
+  sqlite3_finalize(sqst);
+  sqlite3_busy_timeout(db, 1000);
+  return true;
+ }
+
+ void Close()
+ {
+  if(db != nullptr) sqlite3_close(db);
+  db = nullptr;
+ }
+
+ virtual bool Put(const MString& key, const MString& value, size_t ttl) const override
+ {
+  if(!*this) return false;
+  sqlite3_stmt* sqst = nullptr;
+  int           i    = SQLITE_OK;
+
+  if(i == SQLITE_OK) i = sqlite3_prepare_v2(db, "INSERT OR REPLACE into `cache` VALUES(?1,?2,?3);", -1, &sqst, 0);
+  if(i == SQLITE_OK) i = sqlite3_bind_text(sqst, 1, key.Buf(), -1, SQLITE_STATIC);
+  if(i == SQLITE_OK) i = sqlite3_bind_blob64(sqst, 2, value.Buf(), value.Len(), SQLITE_STATIC);
+  if(i == SQLITE_OK) i = sqlite3_bind_int64(sqst, 3, time(nullptr) + ttl);
+  if(i == SQLITE_OK) i = sqlite3_step(sqst);
+  sqlite3_finalize(sqst);
+
+  return i == SQLITE_OK;
+ }
+
+ virtual std::pair<MString, bool> Get(const MString& key) const override
+ {
+  if(!*this) return {"", false};
+
+  sqlite3_stmt* sqst = nullptr;
+  int           i    = SQLITE_OK;
+
+  if(i == SQLITE_OK) i = sqlite3_prepare_v2(db, "SELECT value from `cache` WHERE key=?1 AND exptime>?2;", -1, &sqst, 0);
+  if(i == SQLITE_OK) i = sqlite3_bind_text(sqst, 1, key.Buf(), -1, SQLITE_STATIC);
+  if(i == SQLITE_OK) i = sqlite3_bind_int64(sqst, 2, time(nullptr));
+  if(i == SQLITE_OK) i = sqlite3_step(sqst);
+  if(i == SQLITE_ROW)
+  {
+   auto p  = sqlite3_column_blob(sqst, 0);
+   auto sz = sqlite3_column_bytes(sqst, 0);
+   if(p != nullptr)
+   {
+    MString out(p, sz);
+    sqlite3_finalize(sqst);
+    return {std::move(out), true};
+   }
+  }
+  sqlite3_finalize(sqst);
+  return {"", false};
+ }
+
+ virtual ~SQLiteCache() override
+ {
+  if(!*this) return;
+
+  sqlite3_stmt* sqst = nullptr;
+  int           i    = SQLITE_OK;
+
+  if(i == SQLITE_OK) i = sqlite3_prepare_v2(db, "DELETE from `cache` WHERE exptime<?1;", -1, &sqst, 0);
+  if(i == SQLITE_OK) i = sqlite3_bind_int64(sqst, 1, time(nullptr));
+  if(i == SQLITE_OK) i = sqlite3_step(sqst);
+  sqlite3_finalize(sqst);
+ }
+
+ explicit operator bool() const { return db != nullptr; }
+};
+
+inline GenericCache* CreateCache(const MString& cachedesc)
+{
+ auto i = cachedesc.GetPos(':');
+ if(i == 0)
+ {
+  if(cachedesc == "no") return new FakeCache;
+  return nullptr;
+ }
+
+ auto name = cachedesc.SubStr(1, i - 1);
+ auto par  = cachedesc.SubStr(i + 1, cachedesc.Len() - i);
+
+ if(name == "sqlite")
+ {
+  auto ret = new SQLiteCache;
+  ret->Init(par);
+  if(*ret) return ret;
+  delete ret;
+ }
+
+ return nullptr;
+}

include/curlfuncs.h

@@ -0,0 +1,29 @@
+#pragma once
+#include "MString.h"
+#include <curl/curl.h>
+#include <memory>
+
+using michlib::MString;
+
+class CURLRAIIDT
+{
+ public:
+ // TODO: make static
+ void operator()(CURL* c) { curl_easy_cleanup(c); }
+};
+
+class CURLRAII: public std::unique_ptr<CURL, CURLRAIIDT>
+{
+ public:
+ CURLRAII() { reset(curl_easy_init()); }
+ operator CURL*() const { return get(); }
+};
+
+// Curl writeback function, write to MString
+size_t Write2String(char* ptr, size_t size, size_t n, void* data);
+
+// Curl writeback function, write to file descriptor
+size_t Write2File(char* ptr, size_t size, size_t n, void* data);
+
+// Get content of url to MString
+std::pair<MString, CURLcode> GetUrl(const CURLRAII& chandle, const MString& url);

include/interpolation.h

@@ -0,0 +1,51 @@
+#pragma once
+#include "basedata.h"
+#include "comdefs.h"
+
+using michlib::real;
+
+template<class D> class LinearInterpolator: public D
+{
+ public:
+ LinearInterpolator(D&& d): D(std::move(d)) {}
+
+ real operator()(const struct Point2D& in) const
+ {
+  const auto gp = D::GridPos(in.x, in.y);
+  if(!gp.Valid()) return NAN;
+
+  if(gp.x == 0.0 && gp.y == 0.0) return D::V(gp.ix, gp.iy);
+
+  real   v00;
+  real   v10;
+  real   v01;
+  real   v11;
+  bool   isfill = false;
+  size_t fx, fy;
+
+  // Count fills
+  for(size_t ix = 0; ix <= 1; ix++)
+   for(size_t iy = 0; iy <= 1; iy++)
+   {
+    if(isfill && D::IsFill(gp.ix + ix, gp.iy + iy)) return NAN;
+    if(D::IsFill(gp.ix + ix, gp.iy + iy))
+    {
+     fx     = ix;
+     fy     = iy;
+     isfill = true;
+    }
+   }
+
+  v00 = D::V(gp.ix, gp.iy);
+  v10 = D::V(gp.ix + 1, gp.iy);
+  v01 = D::V(gp.ix, gp.iy + 1);
+  v11 = D::V(gp.ix + 1, gp.iy + 1);
+
+  if(isfill && fx == 0 && fy == 0) v00 = (v10 + v01 + v11) / 3.0;
+  if(isfill && fx == 1 && fy == 0) v10 = (v00 + v01 + v11) / 3.0;
+  if(isfill && fx == 0 && fy == 1) v01 = (v10 + v00 + v11) / 3.0;
+  if(isfill && fx == 1 && fy == 1) v11 = (v10 + v01 + v00) / 3.0;
+
+  return v00 + gp.y * (v01 - v00) + gp.x * ((v10 - v00) + gp.y * (v00 - v01 + v11 - v10));
+ };
+};

include/mirrorfuncs.h

@@ -0,0 +1,62 @@
+#pragma once
+#include "curlfuncs.h"
+#include "mdatetime.h"
+#include <dirent.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <vector>
+
+using michlib::MDateTime;
+
+class DIRRAIIDT
+{
+ public:
+ // TODO: make static
+ void operator()(DIR* d) { closedir(d); }
+};
+
+class DIRRAII: public std::unique_ptr<DIR, DIRRAIIDT>
+{
+ public:
+ operator DIR*() const { return get(); }
+};
+
+struct FileInfo
+{
+ MString   url;
+ MString   name;
+ MDateTime mtime;
+ size_t    size;
+};
+
+// Remove last element from path
+inline MString DirName(const MString& name)
+{
+ auto p = name.GetPos('/', false);
+ if(p == 0) return name;
+ return name.SubStr(1, p - 1);
+}
+
+// Get last element from path
+inline MString FileName(const MString& name)
+{
+ auto p = name.GetPos('/', false);
+ if(p == 0) return name;
+ return name.SubStr(p + 1, name.Len() - p);
+}
+
+// Check and, if necessary, create the path to the file
+bool MakePath(const MString& dname);
+
+// Get local file list
+std::pair<std::vector<struct FileInfo>, MString> ReadLocalFileList(const MString& dir, const MString& path = "");
+
+// Download file to the local mirror
+MString DownloadFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const MString& root);
+
+// Remove file from the local mirror
+MString RemoveFile(const struct FileInfo& linfo);
+
+// Updare file in the local mirror
+MString UpdateFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const struct FileInfo& linfo, const MString& root);

include/simple2ddata.h

@@ -42,6 +42,23 @@ class Simple2DData: public BaseData
 
  real XStep() const { return xstep; }
  real YStep() const { return ystep; }
+
+ struct GridPoint GridPos(real x, real y) const
+ {
+  struct GridPoint out;
+
+  if(!*this) return out;
+  if(x < x0 || x > x0 + (nx - 1) * xstep || y < y0 || y > y0 + (ny - 1) * ystep) return out;
+
+  real rx = x - x0;
+  real ry = y - y0;
+  out.ix  = static_cast<decltype(out.ix)>(michlib::Floor(rx / xstep));
+  out.iy  = static_cast<decltype(out.ix)>(michlib::Floor(ry / ystep));
+  out.x   = rx - out.ix * xstep;
+  out.y   = ry - out.iy * ystep;
+
+  return out;
+ }
 };
 
 class Rect2DData: public BaseData

michlib

@@ -1 +1 @@
-Subproject commit 6268edea843cae04337fe6493b611f3a4fe7cc9b
+Subproject commit e2882902b88229bb4b0e6fbeb76c79ac6d46d53d

sources/COPERNICUS.cpp

@@ -0,0 +1,282 @@
+#define MICHLIB_NOSOURCE
+#include "COPERNICUS.h"
+#include "mirrorfuncs.h"
+#include <libxml/parser.h>
+#include <libxml/tree.h>
+
+using michlib::GPL;
+
+const MString COPERNICUSData::caturl = "https://stac.marine.copernicus.eu/metadata/catalog.stac.json";
+
+std::pair<Json::Value, MString> COPERNICUSData::GetJSON(const MString& url)
+{
+ Json::Reader reader;
+ Json::Value  obj;
+ MString      content;
+
+ auto [val, suc] = cache->Get(url);
+ if(suc)
+  content = std::move(val);
+ else
+ {
+  michlib::message(url + " not found in cache, downloading");
+  auto [out, res] = GetUrl(chandle, url);
+  if(res != CURLE_OK) return {obj, MString("Can't download JSON: ") + curlerr};
+  cache->Put(url, out, 3600);
+  content = std::move(out);
+ }
+
+ reader.parse(content.Buf(), content.Buf() + content.Len(), obj, false);
+
+ return {obj, ""};
+}
+
+MString COPERNICUSData::ReadURL(const Json::Value& cat, const MString& prod)
+{
+ const auto& links = cat["links"];
+ if(links.type() != Json::arrayValue) return "";
+ for(Json::ArrayIndex i = 0; i < links.size(); i++)
+ {
+  const auto& titl = links[i]["title"];
+  const auto& href = links[i]["href"];
+  if(titl.type() == Json::stringValue && href.type() == Json::stringValue)
+  {
+   MString str(titl.asString().c_str());
+   if(str == prod) return MString(href.asString().c_str());
+  }
+ }
+ return "";
+}
+
+std::pair<std::vector<struct FileInfo>, MString> COPERNICUSData::ReadRemoteFileList(const MString& url)
+{
+ LIBXML_TEST_VERSION
+
+ std::vector<struct FileInfo> out;
+ MString                      bucket, prefix;
+
+ // Split url on prefix and bucket
+ {
+  size_t pos   = url.Len();
+  size_t count = 0;
+  for(size_t i = 0; i < url.Len(); i++)
+  {
+   if(url[i] == '/') count++;
+   if(count == 4)
+   {
+    pos = i;
+    break;
+   }
+  }
+  if(pos == url.Len()) return {out, "Can't parse url: " + url};
+
+  bucket = url.SubStr(1, pos);
+  prefix = url.SubStr(pos + 2, url.Len() - pos - 1);
+ }
+
+ MString cont;
+ bool    next = true;
+
+ while(next)
+ {
+  MString url = bucket + "?list-type=2&prefix=" + prefix;
+  if(cont.Exist()) url += "&continuation-token=" + cont;
+  cont = "";
+
+  auto [data, res] = GetUrl(chandle, url);
+  if(res != CURLE_OK) return {out, MString("Can't download ") + url + ": " + curlerr};
+
+  xmlDocPtr doc = xmlReadMemory(data.Buf(), data.Len(), "data.xml", nullptr, 0);
+  if(doc == nullptr) return {out, MString("Can't download ") + url + ": XML parse error"};
+  auto cur = xmlDocGetRootElement(doc);
+  if(cur == nullptr)
+  {
+   xmlFreeDoc(doc);
+   return {out, MString("Can't download ") + url + ": empty XML"};
+  }
+  if(xmlStrEqual(cur->name, (const xmlChar*)"ListBucketResult") == 0)
+  {
+   xmlFreeDoc(doc);
+   return {out, MString("Can't download ") + url + ": unknown XML"};
+  }
+
+  for(const auto* n = cur->children; n; n = n->next)
+  {
+   if(xmlStrEqual(n->name, (const xmlChar*)"NextContinuationToken") == 1)
+   {
+    auto* content = xmlNodeGetContent(n);
+    cont          = (char*)content;
+    xmlFree(content);
+   }
+   if(xmlStrEqual(n->name, (const xmlChar*)"Contents") == 1)
+   {
+    MString   fname;
+    MDateTime mtime;
+    size_t    size = 0;
+    for(const auto* c = n->children; c; c = c->next)
+    {
+     if(xmlStrEqual(c->name, (const xmlChar*)"Key") == 1)
+     {
+      auto* content = xmlNodeGetContent(c);
+      fname         = (char*)content;
+      xmlFree(content);
+     }
+     if(xmlStrEqual(c->name, (const xmlChar*)"LastModified") == 1)
+     {
+      auto* content = xmlNodeGetContent(c);
+      mtime.FromString((char*)content);
+      xmlFree(content);
+     }
+     if(xmlStrEqual(c->name, (const xmlChar*)"Size") == 1)
+     {
+      auto* content = xmlNodeGetContent(c);
+      size          = MString((char*)content).ToInteger<size_t>();
+      xmlFree(content);
+     }
+    }
+    out.emplace_back(bucket + "/" + fname, fname.SubStr(prefix.Len() + 2, fname.Len() - prefix.Len() - 1), mtime, size);
+   }
+  }
+  xmlFreeDoc(doc);
+  next = cont.Exist();
+ }
+
+ std::sort(out.begin(), out.end(), [](const struct FileInfo& a, const struct FileInfo& b) { return a.name < b.name; });
+ return {out, ""};
+}
+
+MString COPERNICUSData::Mirror(const CLArgs& args)
+{
+ GPL.UsePrefix("COPERNICUS");
+
+ // Local directory
+ MString mirrorroot = GPL.ParameterSValue("MirrorTo", "");
+ if(!mirrorroot.Exist()) return "Local mirror directory not specified";
+
+ // Cache
+ cache.reset(CreateCache(GPL.ParameterSValue("Cache", "")));
+ if(!cache)
+ {
+  michlib::errmessage("Can't init cache");
+  cache.reset(new FakeCache);
+ }
+
+ curl_easy_setopt(chandle, CURLOPT_ERRORBUFFER, curlerr);
+
+ if(!args.contains("product")) return "Copernicus product not specified";
+ MString     prod = args.at("product");
+ Json::Value product;
+ MString     produrl;
+
+ // Get catalog
+ {
+  auto [cat, err] = GetJSON(caturl);
+  if(err.Exist()) return "Can't download catalog: " + err;
+  if(cat["title"].type() != Json::stringValue || cat["title"].asString() != "Copernicus Marine Data Store") return "Can't parse catalog";
+  catalog = std::move(cat);
+ }
+
+ // Get product
+ {
+  auto url = ReadURL(catalog, prod);
+  if(!url.Exist()) return "Url for product " + prod + " not found in catalog";
+  produrl        = DirName(caturl) + "/" + url;
+  auto [pr, err] = GetJSON(produrl);
+  if(err.Exist()) return "Can't download product information from " + produrl + ": " + err;
+  product = std::move(pr);
+ }
+
+ std::vector<MString> dsets;
+ if(args.contains("dataset"))
+  dsets.push_back(args.at("dataset"));
+ else
+ {
+  const auto& links = product["links"];
+  if(links.type() != Json::arrayValue) return "Can't find information about datasets";
+  for(Json::ArrayIndex i = 0; i < links.size(); i++)
+  {
+   const auto& rel  = links[i]["rel"];
+   const auto& titl = links[i]["title"];
+   if(rel.type() == Json::stringValue && titl.type() == Json::stringValue && rel.asString() == "item") dsets.push_back(titl.asString().c_str());
+  }
+ }
+
+ for(const auto& dset: dsets)
+ {
+  michlib::message("Mirroring " + dset);
+  auto url = ReadURL(product, dset);
+  if(!url.Exist()) return "Url for dataset " + dset + " not found in product description";
+  MString dseturl = DirName(produrl) + "/" + url;
+  auto [ds, err]  = GetJSON(dseturl);
+  if(err.Exist()) return "Can't download dataset information from " + dseturl + ": " + err;
+
+  const auto& href = ds["assets"]["native"]["href"];
+  if(href.type() != Json::stringValue) return "Can't find data for dataset " + dset + " from product " + prod;
+
+  url = href.asString().c_str();
+
+  MString locroot = mirrorroot + "/" + prod + "/" + dset;
+
+  auto [lfiles, lerr] = ReadLocalFileList(locroot);
+  if(lerr.Exist()) return lerr;
+
+  auto [rfiles, rerr] = ReadRemoteFileList(url);
+  if(rerr.Exist()) return rerr;
+
+  std::vector<size_t>                    down, rem;
+  std::vector<std::pair<size_t, size_t>> upd;
+
+  {
+   size_t rpos = 0, lpos = 0;
+   while(rpos != rfiles.size() || lpos != lfiles.size())
+   {
+    if(rpos == rfiles.size())
+     while(lpos != lfiles.size()) rem.push_back(lpos++);
+    if(lpos == lfiles.size())
+     while(rpos != rfiles.size()) down.push_back(rpos++);
+    if(rpos == rfiles.size() || lpos == lfiles.size()) continue;
+
+    if(rfiles[rpos].name < lfiles[lpos].name)
+     down.push_back(rpos++);
+    else if(lfiles[lpos].name < rfiles[rpos].name)
+     rem.push_back(lpos++);
+    else
+    {
+     auto delta = rfiles[rpos].mtime.Epoch() - lfiles[lpos].mtime.Epoch();
+     if(delta < 0) delta = -delta;
+     if(delta > 0 || rfiles[rpos].size != lfiles[lpos].size) upd.emplace_back(rpos, lpos);
+     lpos++;
+     rpos++;
+    }
+   }
+  }
+
+  michlib::message(MString("New files: ") + down.size());
+  michlib::message(MString("Obsolete files: ") + rem.size());
+  michlib::message(MString("Modified files: ") + upd.size());
+
+  for(size_t i = 0; i < down.size(); i++)
+  {
+   size_t ri  = down[i];
+   auto   err = DownloadFile(chandle, rfiles[ri], locroot);
+   if(err.Exist()) return err;
+  }
+
+  for(size_t i = 0; i < rem.size(); i++)
+  {
+   size_t li  = rem[i];
+   auto   err = RemoveFile(lfiles[li]);
+   if(err.Exist()) return err;
+  }
+
+  for(size_t i = 0; i < upd.size(); i++)
+  {
+   size_t ri  = upd[i].first;
+   size_t li  = upd[i].second;
+   auto   err = UpdateFile(chandle, rfiles[ri], lfiles[li], locroot);
+   if(err.Exist()) return err;
+  }
+ }
+
+ return "";
+}

sources/COPERNICUS.h

@@ -0,0 +1,36 @@
+#pragma once
+#include "ParseArgs.h"
+#include "cache.h"
+#include "curlfuncs.h"
+#include "mdatetime.h"
+#include <json/json.h>
+
+using michlib::MDateTime;
+using michlib::MString;
+
+class COPERNICUSData
+{
+ static const MString caturl;
+
+ std::unique_ptr<GenericCache> cache;
+ CURLRAII                      chandle;
+ Json::Value                   catalog;
+ char                          curlerr[CURL_ERROR_SIZE];
+
+ // Get url for product or dataset from catalog
+ static MString ReadURL(const Json::Value& cat, const MString& prod);
+
+ // Download JSON from url
+ std::pair<Json::Value, MString> GetJSON(const MString& url);
+
+ // Get remote file list from url
+ std::pair<std::vector<struct FileInfo>,MString> ReadRemoteFileList(const MString& url);
+
+ public:
+ static constexpr const char* name = "COPERNICUS";
+
+ COPERNICUSData() = default;
+
+ // Main mirror function
+ MString Mirror(const CLArgs& args);
+};

sources/GRIDVEL.cpp

@@ -0,0 +1,86 @@
+#define MICHLIB_NOSOURCE
+#include "GRIDVEL.h"
+
+MString GRIDVELData::Info() const
+{
+ if(!isOk()) return "";
+
+ // clang-format off
+ return MString() +
+  " Region: (" + u.Xb() + " : " + u.Xe() + ") x (" + u.Yb() + " : " + u.Ye() + ")\n" +
+  " Grid: " + u.Nx() + "x" + u.Ny() + "\n" +
+  " Supported variables: u, v, U2";
+ // clang-format on
+}
+
+MString GRIDVELData::Open(const CLArgs& args)
+{
+ MString uname = args.contains("u") ? args.at("u") : "";
+ MString vname = args.contains("v") ? args.at("v") : "";
+ if(!uname.Exist()) return "File with u component not specified";
+ if(!vname.Exist()) return "File with v component not specified";
+
+ unit = args.contains("unit") ? args.at("unit") : "cm/s";
+ fill = args.contains("fill") ? args.at("fill").ToReal() : 1e10;
+
+ u.Open(uname);
+ if(!u.Opened()) return "Can't open file: " + uname;
+ v.Open(vname);
+ if(!v.Opened()) return "Can't open file: " + vname;
+ if(u != v) return "Files " + uname + " and " + vname + " have different grids";
+
+ return "";
+}
+
+bool GRIDVELData::Read(const MString& vname, std::map<MString, GRIDVELData::Data>& cache, size_t i) const
+{
+ if(cache.contains(vname)) return true;
+ if(!isOk()) return false;
+
+ // Only rectangular grids are supported
+ real xs = (u.Xe() - u.Xb()) / (u.Nx() - 1);
+ real ys = (u.Ye() - u.Yb()) / (u.Ny() - 1);
+
+ Data out(u.Nx(), u.Ny(), u.Xb(), u.Yb(), xs, ys, MString(unit));
+
+ // U and U2 from u and v
+ if(vname == "U" || vname == "U2")
+ {
+  bool square = vname == "U2";
+  if(!(Read("u", cache, i) && Read("v", cache, i))) return false;
+  cache[vname]      = cache.at("u").CopyGrid();
+  auto&       U     = cache.at(vname);
+  const auto& udata = cache.at("u");
+  const auto& vdata = cache.at("v");
+  if(udata.Unit().Exist()) U.SetUnit(square ? ("(" + udata.Unit() + ")2") : udata.Unit());
+  for(size_t ind = 0; ind < U.N(); ind++)
+  {
+   if(udata.IsFill(ind) || vdata.IsFill(ind))
+    U.V(ind) = U.Fillval();
+   else
+    U.V(ind) = square ? (udata(ind) * udata(ind) + vdata(ind) * vdata(ind)) : michlib::Hypot(udata(ind), vdata(ind));
+  }
+  return true;
+ }
+
+ if(vname == "u" || vname == "v")
+ {
+  bool isu = vname == "u";
+  for(IType ix = 0; ix < u.Nx(); ix++)
+   for(IType iy = 0; iy < u.Ny(); iy++)
+   {
+    if(isu ? (u(ix, iy) == NAN || u(ix, iy) >= 1e10) : (v(ix, iy) == NAN || v(ix, iy) >= 1e10))
+     out.V(ix, iy) = out.Fillval();
+    else
+     out.V(ix, iy) = isu ? u(ix, iy) : v(ix, iy);
+   }
+
+  if(out)
+  {
+   cache[vname] = std::move(out);
+   return true;
+  }
+ }
+
+ return false;
+}

sources/GRIDVEL.h

@@ -0,0 +1,50 @@
+#pragma once
+#include "DataAdapters/gridfile.h"
+#include "mdatetime.h"
+#include "simple2ddata.h"
+
+using michlib::MDateTime;
+
+class GRIDVELData
+{
+ michlib::GridFile u,v;
+ MString unit;
+ real fill;
+
+ using IType=decltype(u.Nx());
+
+ public:
+ static constexpr const char* name = "GRIDVEL";
+
+ static constexpr const char* disabledactions = "genintfile";
+
+ using Data = Simple2DData;
+
+ GRIDVELData() = default;
+
+ MString Info() const;
+ // TODO: RetVal
+ MString Open(const CLArgs& args);
+
+ bool isOk() const { return u.Opened() &&v.Opened(); }
+
+ size_t NTimes() const { return 1; }
+
+ MDateTime Time(size_t i) const
+ {
+  return MDateTime();
+ }
+
+ //time_t Timestep() const { return isOk() ? (times[1] - times[0]) : 0; }
+
+ explicit operator bool() const { return u.Opened() &&v.Opened(); }
+
+ VarPresence CheckVar(const MString& vname) const
+ {
+  if(vname == "u" || vname == "v") return VarPresence::INTERNAL;
+  if(vname == "U" || vname == "U2") return VarPresence::DERIVED;
+  return VarPresence::NONE;
+ }
+
+ bool Read(const MString& vname, std::map<MString, Data>& cache, size_t i) const;
+};

src/CMakeLists.txt

@@ -2,15 +2,23 @@ set(EXENAME odm)
 set(ACTIONLISTINC ${CMAKE_CURRENT_BINARY_DIR}/../include/actionlist.h) # Include actions files and define the actions classes list
 set(DATALISTINC ${CMAKE_CURRENT_BINARY_DIR}/../include/datalist.h) # Include data sources files and define the data sources classes list
 
+find_package(PkgConfig REQUIRED)
 find_library(netcdf netcdf REQUIRED)
 find_package(OpenMP REQUIRED)
+find_package(CURL REQUIRED)
+find_package(LibXml2 REQUIRED)
+find_package(SQLite3 REQUIRED)
+pkg_check_modules(JSONCPP REQUIRED jsoncpp)
+
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
 
+include_directories(${JSONCPP_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIRS} ${SQLite3_INCLUDE_DIRS})
+
 file(GLOB srcs CONFIGURE_DEPENDS *.cpp)
 add_executable(${EXENAME} ${srcs} ${ACTIONLISTINC} ${SOURCELISTINC})
 target_include_directories(${EXENAME} PRIVATE ../michlib/michlib ${CMAKE_CURRENT_BINARY_DIR}/../include)
 
-target_link_libraries(${EXENAME} ${linker_options} ${netcdf} OpenMP::OpenMP_CXX teos)
+target_link_libraries(${EXENAME} ${linker_options} ${netcdf} OpenMP::OpenMP_CXX CURL::libcurl ${JSONCPP_LINK_LIBRARIES} LibXml2::LibXml2 SQLite::SQLite3 teos)
 set_target_properties(${EXENAME} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 install(TARGETS ${EXENAME})
 

src/curlfuncs.cpp

@@ -0,0 +1,41 @@
+#define MICHLIB_NOSOURCE
+#include "curlfuncs.h"
+#include <unistd.h>
+
+using michlib::pointer_cast;
+using michlib::uint1;
+
+size_t Write2String(char* ptr, size_t size, size_t n, void* data)
+{
+ MString* out = pointer_cast<MString*>(data);
+ *out += MString(ptr, size * n);
+ return size * n;
+}
+
+size_t Write2File(char* ptr, size_t size, size_t n, void* data)
+{
+ const int*   fd    = pointer_cast<const int*>(data);
+ size_t       count = size * n;
+ const uint1* buf   = pointer_cast<const uint1*>(ptr);
+
+ while(count != 0)
+ {
+  auto wr = write(*fd, buf, count);
+  if(wr == -1) return 0;
+  count -= wr;
+  buf += wr;
+ }
+
+ return size * n;
+}
+
+std::pair<MString, CURLcode> GetUrl(const CURLRAII& chandle, const MString& url)
+{
+ MString out;
+
+ curl_easy_setopt(chandle, CURLOPT_URL, url.Buf());
+ curl_easy_setopt(chandle, CURLOPT_WRITEFUNCTION, Write2String);
+ curl_easy_setopt(chandle, CURLOPT_WRITEDATA, &out);
+ auto res = curl_easy_perform(chandle);
+ return {out, res};
+}

src/mirrorfuncs.cpp

@@ -0,0 +1,130 @@
+#define MICHLIB_NOSOURCE
+#include "mirrorfuncs.h"
+#include "StringFunctions.h"
+#include "filehelpers.h"
+#include "merrors.h"
+
+using michlib::FD;
+using michlib::message;
+
+bool MakePath(const MString& dname)
+{
+ struct stat st;
+ int         ret = stat(dname.Buf(), &st);
+ if(ret == 0) return S_ISDIR(st.st_mode);
+
+ auto    dirs = michlib::Split_on_words(dname, "/", false);
+ MString cdir = "";
+ for(const auto& dir: dirs)
+ {
+  cdir += "/" + dir;
+  ret = stat(cdir.Buf(), &st);
+  if(ret == 0 && S_ISDIR(st.st_mode)) continue;
+  if(ret == 0 && !S_ISDIR(st.st_mode)) return false;
+  ret = mkdir(cdir.Buf(), 0755);
+  if(ret != 0) return false;
+ }
+ return true;
+}
+
+std::pair<std::vector<struct FileInfo>, MString> ReadLocalFileList(const MString& dir, const MString& path)
+{
+ std::vector<struct FileInfo> out;
+ DIRRAII                      dhandle;
+
+ MakePath(dir);
+ dhandle.reset(opendir(dir.Buf()));
+
+ if(!dhandle) return {out, "Can't open directory " + path + (path.Exist() ? "/" : "") + dir};
+
+ int dfd             = dirfd(dhandle);
+ errno               = 0;
+ struct dirent* dent = readdir(dhandle);
+ if(errno != 0) return {out, "Can't read directory " + path + (path.Exist() ? "/" : "") + dir};
+ struct stat st;
+
+ do {
+  if(dent->d_name[0] != '.')
+  {
+   int ret = fstatat(dfd, dent->d_name, &st, AT_SYMLINK_NOFOLLOW);
+   if(ret != 0) return {out, "Can't stat " + path + "/" + dir + "/" + dent->d_name};
+   if(S_ISDIR(st.st_mode)) // Directory, recurse
+   {
+    auto [list, err] = ReadLocalFileList(dir + "/" + dent->d_name, path + (path.Exist() ? "/" : "") + dent->d_name);
+    if(err.Exist()) return {out, err};
+    out.insert(out.end(), list.begin(), list.end());
+   }
+   if(S_ISREG(st.st_mode)) // Regular file
+   {
+    out.emplace_back(dir + "/" + dent->d_name, path + (path.Exist() ? "/" : "") + dent->d_name, MDateTime(st.st_mtim.tv_sec, st.st_mtim.tv_nsec), st.st_size);
+   }
+   // Ignore non-directories, non-files
+  }
+  dent = readdir(dhandle);
+ } while(dent != nullptr || errno != 0);
+
+ if(errno != 0) return {out, "Can't read directory " + path + "/" + dir};
+ std::sort(out.begin(), out.end(), [](const struct FileInfo& a, const struct FileInfo& b) { return a.name < b.name; });
+ return {out, ""};
+}
+
+MString DownloadFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const MString& root)
+{
+ message("Downloading " + rinfo.url);
+
+ MString dname = DirName(rinfo.name), fname = FileName(rinfo.name);
+ FD      fd;
+
+ if(!MakePath(root + "/" + dname)) return "Can't create directory " + root + "/" + dname;
+ fd.Reset(creat((root + "/" + rinfo.name).Buf(), 0644));
+ if(!fd) return "Can't create file " + root + "/" + rinfo.name;
+
+ char errbuf[CURL_ERROR_SIZE];
+ int  cfd = fd.Get();
+ curl_easy_setopt(chandle, CURLOPT_ERRORBUFFER, errbuf);
+ curl_easy_setopt(chandle, CURLOPT_WRITEFUNCTION, Write2File);
+ curl_easy_setopt(chandle, CURLOPT_WRITEDATA, &cfd);
+ curl_easy_setopt(chandle, CURLOPT_URL, rinfo.url.Buf());
+ auto res = curl_easy_perform(chandle);
+ if(res != CURLE_OK)
+ {
+  unlink((root + "/" + rinfo.name).Buf());
+  return MString("Can't download file: ") + errbuf;
+ }
+
+ {
+  struct timespec times[2];
+  times[0].tv_sec = times[1].tv_sec = rinfo.mtime.Epoch();
+  times[0].tv_nsec = times[1].tv_nsec = 0;
+
+  int ret = futimens(fd, times);
+  if(ret != 0)
+  {
+   unlink((root + "/" + rinfo.name).Buf());
+   return "Can't set mtime for file: " + root + "/" + rinfo.name;
+  }
+ }
+
+ return "";
+}
+
+MString RemoveFile(const struct FileInfo& linfo)
+{
+ message("Remove " + linfo.url);
+ int ret = unlink(linfo.url.Buf());
+ if(ret != 0) return "Can't remove file " + linfo.url;
+ return "";
+}
+
+MString UpdateFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const struct FileInfo& linfo, const MString& root)
+{
+ MString err;
+
+ message("Update " + linfo.url);
+ err = RemoveFile(linfo);
+ if(err.Exist()) return err;
+ err = DownloadFile(chandle, rinfo, root);
+ if(err.Exist()) return err;
+
+ return "";
+}