actions/actionint.h

@@ -1,210 +0,0 @@
-#pragma once
-#include "BFileR.h"
-#include "BFileW.h"
-#include "actiondep.h"
-#include "interpolation.h"
-#include "ncfilew.h"
-#include "ncfuncs.h"
-#include <memory>
-
-template<class T>
-concept ReadIsInterpolable = requires {
- {
-  std::declval<ReadType<T>>().GridPos(0.0, 0.0)
- } -> std::convertible_to<struct GridPoint>;
-};
-
-ADD_ACTION(INT, int, (ReadPSupported<Source> || ReadSupported<Source>)&&ReadIsInterpolable<Source>);
-
-template<class D> MString ActionINT::DoAction(const CLArgs& args, D& ds)
-{
- struct TPoint
- {
-  struct Point2D p;
-  MDateTime      t;
-  real           lon0, lat0, dtime;
- };
-
- MDateTime refdate("1980-01-01");
-
- if(!args.contains("input")) return "No input file given";
- michlib::BFileR in;
- if(in.Open(args.at("input")) != ERR_NOERR) return "Can't open input file " + args.at("input");
- std::vector<TPoint> points;
- struct Region       reg;
-
- {
-  michlib::uint loncol = in.Columns(), latcol = in.Columns(), timecol = in.Columns();
-  michlib::uint lon0col = in.Columns(), lat0col = in.Columns(), dtimecol = in.Columns();
-  for(uint i = 0; i < in.Columns(); i++)
-  {
-   if(in.ColumnName(i + 1) == "lon") loncol = i;
-   if(in.ColumnName(i + 1) == "lat") latcol = i;
-   if(in.ColumnName(i + 1) == "rtime") timecol = i;
-   if(in.ColumnName(i + 1) == "lon0") lon0col = i;
-   if(in.ColumnName(i + 1) == "lat0") lat0col = i;
-   if(in.ColumnName(i + 1) == "time") dtimecol = i;
-  }
-  if(loncol >= in.Columns()) return "Lon column not found in input file " + args.at("input");
-  if(latcol >= in.Columns()) return "Lat column not found in input file " + args.at("input");
-  if(timecol >= in.Columns()) return "Time column not found in input file " + args.at("input");
-  if(lon0col >= in.Columns()) return "Lon0 column not found in input file " + args.at("input");
-  if(lat0col >= in.Columns()) return "Lat0 column not found in input file " + args.at("input");
-  if(dtimecol >= in.Columns()) return "Age column not found in input file " + args.at("input");
-
-  points.resize(in.Rows());
-  reg.lonb = reg.lone = in[loncol][0];
-  reg.latb = reg.late = in[latcol][0];
-  for(uint i = 0; i < in.Rows(); i++)
-  {
-   points[i].p.x   = in[loncol][i];
-   points[i].p.y   = in[latcol][i];
-   points[i].t     = refdate + static_cast<time_t>(michlib::Round(in[timecol][i] * 86400));
-   points[i].lon0  = in[lon0col][i];
-   points[i].lat0  = in[lat0col][i];
-   points[i].dtime = in[dtimecol][i];
-
-   reg.lonb = std::min(reg.lonb, points[i].p.x);
-   reg.lone = std::max(reg.lone, points[i].p.x);
-   reg.latb = std::min(reg.latb, points[i].p.y);
-   reg.late = std::max(reg.late, points[i].p.y);
-  }
-  std::ranges::sort(points, {}, &TPoint::t);
- }
- in.Close();
-
- auto resop = ds.Open(args);
- if(resop.Exist()) return "Can't open source: " + resop;
-
- michlib_internal::ParameterListEx pars;
- pars.UsePrefix("");
- pars.SetParameter("source", args.at("source"));
- pars.SetParameter("history", args.at("_cmdline"));
-
- MString varstring;
- if(args.contains("var"))
-  varstring = args.at("var");
- else
- {
-  if(args.contains("vars"))
-   varstring = args.at("vars");
-  else
-  {
-   if constexpr(HasDefVars<D>)
-    varstring = ds.DefaultVars();
-   else
-    return "Variables not specified";
-  }
- }
-
- auto                 vlist = michlib::Split_on_words(varstring, " ,", false);
- std::vector<MString> vnames{std::make_move_iterator(std::begin(vlist)), std::make_move_iterator(std::end(vlist))};
- {
-  std::set<MString> used;
-  for(const auto& vname: vnames)
-  {
-   if(used.contains(vname)) return "Duplicate variable " + vname + " in list " + varstring;
-   if(ds.CheckVar(vname) == VarPresence::NONE) return "Variable " + vname + " not exists in this dataset";
-   used.insert(vname);
-  }
- }
-
- pars.SetParameter("variables", varstring);
-
- //int compress = 3;
- //if(args.contains("compress")) compress = args.at("compress").ToInt();
-
- std::unique_ptr<const BaseParameters> sourcepars;
- if constexpr(ParametersSupported<D>)
- {
-  if constexpr(ParametersRequiredRegion<D>)
-  {
-   auto [p, err] = ds.Parameters(pars, args, reg);
-   if(err.Exist()) return err;
-   sourcepars.reset(p);
-  }
-  else
-  {
-   auto [p, err] = ds.Parameters(pars, args);
-   if(err.Exist()) return err;
-   sourcepars.reset(p);
-  }
- }
- auto p = sourcepars.get();
-
- MString name = args.contains("out") ? args.at("out") : "out.bin";
- //MString outfmt = args.contains("format") ? args.at("format") : (GetExt(name) == "nc" ? "nc" : "bin");
-
- size_t          loind = ds.NTimes(), hiind = ds.NTimes(), curind = 0;
- michlib::BFileW fw;
- const MDateTime first = ds.Time(0), last = ds.Time(ds.NTimes() - 1);
-
- std::vector<LinearInterpolator<ReadType<D>>> lo, hi;
-
- {
-  size_t ic = 0;
-  fw.Create(name, 6 + vnames.size());
-  fw.SetColumnName(++ic, "lon0");
-  fw.SetColumnName(++ic, "lat0");
-  fw.SetColumnName(++ic, "lon");
-  fw.SetColumnName(++ic, "lat");
-  fw.SetColumnName(++ic, "time");
-  fw.SetColumnName(++ic, "rtime");
-  for(size_t i = 0; i < vnames.size(); i++) fw.SetColumnName(++ic, vnames[i]);
-  fw.SetParameters(pars);
- }
-
- for(size_t i = 0; i < points.size(); i++)
- {
-  if(points[i].t < first || points[i].t > last) continue;
-  while(!(points[i].t >= ds.Time(curind) && points[i].t <= ds.Time(curind + 1))) curind++;
-  if(curind != loind && curind != hiind)
-  {
-   loind = curind;
-   hiind = curind + 1;
-   {
-    auto temp = Read(ds, vnames, p, loind);
-    if(temp.size() != vnames.size()) return "Can't read data";
-    //lo.resize(temp.size());
-    for(size_t j = 0; j < temp.size(); j++) lo.emplace_back(std::move(temp[j]));
-   }
-   {
-    auto temp = Read(ds, vnames, p, hiind);
-    if(temp.size() != vnames.size()) return "Can't read data";
-    //hi.resize(temp.size());
-    for(size_t j = 0; j < temp.size(); j++) hi.emplace_back(std::move(temp[j]));
-   }
-  }
-  else if(curind == hiind)
-  {
-   loind = curind;
-   hiind = curind + 1;
-   lo    = std::move(hi);
-   {
-    auto temp = Read(ds, vnames, p, hiind);
-    if(temp.size() != vnames.size()) return "Can't read data";
-    //hi.resize(temp.size());
-    for(size_t j = 0; j < temp.size(); j++) hi.emplace_back(std::move(temp[j]));
-   }
-  }
-  auto step  = ds.Time(hiind) - ds.Time(loind);
-  auto delta = points[i].t - ds.Time(loind);
-  real trel  = static_cast<real>(delta) / static_cast<real>(step);
-
-  fw.Write(points[i].lon0);
-  fw.Write(points[i].lat0);
-  fw.Write(points[i].p.x);
-  fw.Write(points[i].p.y);
-  fw.Write(points[i].dtime);
-  fw.Write(DeltaDates(refdate, points[i].t));
-  for(size_t iv = 0; iv < lo.size(); iv++)
-  {
-   real vlo = lo[iv](points[i].p);
-   real vhi = hi[iv](points[i].p);
-   fw.Write(vlo + (vhi - vlo) * trel);
-  }
- }
- fw.Finalize();
- fw.Close();
- return "";
-};

actions/actionmirror.h

@@ -1,23 +0,0 @@
-#pragma once
-#include "actiondep.h"
-#include "merrors.h"
-
-using michlib::message;
-
-template<class T>
-concept MirrorSupported = requires(T t, const CLArgs& args) {
- {
-  t.Mirror(args)
- } -> std::convertible_to<MString>;
-};
-
-ADD_ACTION(Mirror, mirror, MirrorSupported<Source>);
-
-template<class D> MString ActionMirror::DoAction(const CLArgs& args, D& data)
-{
- //auto resop = data.Open(args);
- //if(resop.Exist()) return "Can't open source: " + resop;
- auto res = data.Mirror(args);
- if(res.Exist()) return "Mirroring failed: " + res;
- return "";
-};

actions/actionuv.h

@@ -152,9 +152,9 @@ template<class D> MString ActionUV::DoAction(const CLArgs& args, D& ds)
  MString name      = args.contains("out") ? args.at("out") : "";
  MString outfmt    = args.contains("outformat") ? args.at("outformat") : (GetExt(name) == "nc" ? "nc" : "bin");
  MString namevel   = args.contains("velout") ? args.at("velout") : "";
- MString outfmtvel = args.contains("veloutformat") ? args.at("veloutformat") : (GetExt(namevel) == "nc" ? "nc" : "bin");
+ MString outfmtvel = args.contains("veloutformat") ? args.at("veloutformat") : (GetExt(name) == "nc" ? "nc" : "bin");
  MString namestp   = args.contains("stpout") ? args.at("stpout") : "";
- MString outfmtstp = args.contains("stpoutformat") ? args.at("stpoutformat") : (GetExt(namestp) == "nc" ? "nc" : "bin");
+ MString outfmtstp = args.contains("stpoutformat") ? args.at("stpoutformat") : (GetExt(name) == "nc" ? "nc" : "bin");
 
  size_t shiftx = args.contains("shiftx") ? args.at("shiftx").ToInteger<size_t>() : 0;
  size_t shifty = args.contains("shifty") ? args.at("shifty").ToInteger<size_t>() : 0;
@@ -263,7 +263,7 @@ template<class D> MString ActionUV::DoAction(const CLArgs& args, D& ds)
    if(outfmtvel == "bin")
    {
     BFileW vel;
-    vel.Create(namevel, 4);
+    vel.Create(name, 4);
     vel.SetColumnName(1, "Longitude");
     vel.SetColumnName(2, "Latitude");
     vel.SetColumnName(3, "u, " + velunit);
@@ -285,7 +285,7 @@ template<class D> MString ActionUV::DoAction(const CLArgs& args, D& ds)
    {
     MString err;
 
-    if(!err.Exist() && !headwrited) err = fwfilt.Create(sdata, namevel, compress);
+    if(!err.Exist() && !headwrited) err = fwfilt.Create(sdata, name, compress);
     if(!err.Exist() && !headwrited) err = fwfilt.AddTimeData(tdata, !average);
     if(!err.Exist() && !headwrited) err = fwfilt.AddAtts(pars);
     if(!err.Exist() && !headwrited) err = fwfilt.AddVariable("u", "", "Eastward velocity", velunit, "");
@@ -312,11 +312,11 @@ template<class D> MString ActionUV::DoAction(const CLArgs& args, D& ds)
     for(size_t iy = 0; iy < data.Ny() - 1; iy++) stp.Add(data.StablePoints(ix, iy));
 
    if(outfmtstp == "bin")
-    stp.WriteBinBile(namestp, pars);
+    stp.WriteBinBile(name, pars);
    else if(outfmtstp == "nc" || outfmtstp == "netcdf")
    {
     MString err;
-    if(!err.Exist() && !headwrited) err = stp.CreateNcFile(namestp, pars, args.at("_cmdline"), compress, tdata, !average);
+    if(!err.Exist() && !headwrited) err = stp.CreateNcFile(name, pars, args.at("_cmdline"), compress, tdata, !average);
     if(!err.Exist()) err = stp.WriteNcFile(it);
     if(err.Exist()) return err;
    }

include/basedata.h

@@ -15,19 +15,6 @@ struct Region
  real lonb, lone, latb, late;
 };
 
-struct GridPoint
-{
- size_t ix = 0, iy = 0;
- real   x = -1.0, y = -1.0;
-
- bool Valid() const { return x >= 0.0 && y >= 0.0; }
-};
-
-struct Point2D
-{
- real x, y;
-};
-
 enum class VarPresence
 {
  NONE,

include/cache.h

@@ -1,144 +0,0 @@
-#pragma once
-#include "MString.h"
-#include <sqlite3.h>
-#include <time.h>
-#include <variant>
-
-using michlib::MString;
-
-class GenericCache
-{
- public:
- virtual bool                     Put(const MString& key, const MString& value, size_t ttl) const = 0;
- virtual std::pair<MString, bool> Get(const MString& key) const                                   = 0;
- virtual ~GenericCache() {}
-};
-
-class FakeCache: public GenericCache
-{
- public:
- virtual bool                     Put([[maybe_unused]] const MString& key, [[maybe_unused]] const MString& value, [[maybe_unused]] size_t ttl) const override { return false; }
- virtual std::pair<MString, bool> Get([[maybe_unused]] const MString& key) const override { return {"", false}; }
- virtual ~FakeCache() override {}
-};
-
-class SQLiteCache: public GenericCache
-{
- sqlite3* db = nullptr;
-
- public:
- bool Init(const MString& name)
- {
-  Close();
-  auto ret = sqlite3_open(name.Buf(), &db);
-  if(ret != SQLITE_OK)
-  {
-   Close();
-   return false;
-  }
-  // Create table
-  sqlite3_stmt* sqst;
-  int           i;
-
-  i = sqlite3_prepare_v2(db,
-                         "CREATE TABLE IF NOT EXISTS `cache`('key' TEXT PRIMARY KEY ON CONFLICT REPLACE NOT NULL ON CONFLICT FAIL, 'value' BLOB NOT NULL ON CONFLICT FAIL, "
-                         "'exptime' INTEGER NOT NULL ON CONFLICT FAIL) WITHOUT ROWID, STRICT;",
-                         -1, &sqst, 0);
-  i = sqlite3_step(sqst);
-  if(i != SQLITE_DONE)
-  {
-   sqlite3_finalize(sqst);
-   Close();
-   return false;
-  }
-  sqlite3_finalize(sqst);
-  sqlite3_busy_timeout(db, 1000);
-  return true;
- }
-
- void Close()
- {
-  if(db != nullptr) sqlite3_close(db);
-  db = nullptr;
- }
-
- virtual bool Put(const MString& key, const MString& value, size_t ttl) const override
- {
-  if(!*this) return false;
-  sqlite3_stmt* sqst = nullptr;
-  int           i    = SQLITE_OK;
-
-  if(i == SQLITE_OK) i = sqlite3_prepare_v2(db, "INSERT OR REPLACE into `cache` VALUES(?1,?2,?3);", -1, &sqst, 0);
-  if(i == SQLITE_OK) i = sqlite3_bind_text(sqst, 1, key.Buf(), -1, SQLITE_STATIC);
-  if(i == SQLITE_OK) i = sqlite3_bind_blob64(sqst, 2, value.Buf(), value.Len(), SQLITE_STATIC);
-  if(i == SQLITE_OK) i = sqlite3_bind_int64(sqst, 3, time(nullptr) + ttl);
-  if(i == SQLITE_OK) i = sqlite3_step(sqst);
-  sqlite3_finalize(sqst);
-
-  return i == SQLITE_OK;
- }
-
- virtual std::pair<MString, bool> Get(const MString& key) const override
- {
-  if(!*this) return {"", false};
-
-  sqlite3_stmt* sqst = nullptr;
-  int           i    = SQLITE_OK;
-
-  if(i == SQLITE_OK) i = sqlite3_prepare_v2(db, "SELECT value from `cache` WHERE key=?1 AND exptime>?2;", -1, &sqst, 0);
-  if(i == SQLITE_OK) i = sqlite3_bind_text(sqst, 1, key.Buf(), -1, SQLITE_STATIC);
-  if(i == SQLITE_OK) i = sqlite3_bind_int64(sqst, 2, time(nullptr));
-  if(i == SQLITE_OK) i = sqlite3_step(sqst);
-  if(i == SQLITE_ROW)
-  {
-   auto p  = sqlite3_column_blob(sqst, 0);
-   auto sz = sqlite3_column_bytes(sqst, 0);
-   if(p != nullptr)
-   {
-    MString out(p, sz);
-    sqlite3_finalize(sqst);
-    return {std::move(out), true};
-   }
-  }
-  sqlite3_finalize(sqst);
-  return {"", false};
- }
-
- virtual ~SQLiteCache() override
- {
-  if(!*this) return;
-
-  sqlite3_stmt* sqst = nullptr;
-  int           i    = SQLITE_OK;
-
-  if(i == SQLITE_OK) i = sqlite3_prepare_v2(db, "DELETE from `cache` WHERE exptime<?1;", -1, &sqst, 0);
-  if(i == SQLITE_OK) i = sqlite3_bind_int64(sqst, 1, time(nullptr));
-  if(i == SQLITE_OK) i = sqlite3_step(sqst);
-  sqlite3_finalize(sqst);
- }
-
- explicit operator bool() const { return db != nullptr; }
-};
-
-inline GenericCache* CreateCache(const MString& cachedesc)
-{
- auto i = cachedesc.GetPos(':');
- if(i == 0)
- {
-  if(cachedesc == "no") return new FakeCache;
-  return nullptr;
- }
-
- auto name = cachedesc.SubStr(1, i - 1);
- auto par  = cachedesc.SubStr(i + 1, cachedesc.Len() - i);
-
- if(name == "sqlite")
- {
-  auto ret = new SQLiteCache;
-  ret->Init(par);
-  if(*ret) return ret;
-  delete ret;
- }
-
- return nullptr;
-}

include/curlfuncs.h

@@ -1,29 +0,0 @@
-#pragma once
-#include "MString.h"
-#include <curl/curl.h>
-#include <memory>
-
-using michlib::MString;
-
-class CURLRAIIDT
-{
- public:
- // TODO: make static
- void operator()(CURL* c) { curl_easy_cleanup(c); }
-};
-
-class CURLRAII: public std::unique_ptr<CURL, CURLRAIIDT>
-{
- public:
- CURLRAII() { reset(curl_easy_init()); }
- operator CURL*() const { return get(); }
-};
-
-// Curl writeback function, write to MString
-size_t Write2String(char* ptr, size_t size, size_t n, void* data);
-
-// Curl writeback function, write to file descriptor
-size_t Write2File(char* ptr, size_t size, size_t n, void* data);
-
-// Get content of url to MString
-std::pair<MString, CURLcode> GetUrl(const CURLRAII& chandle, const MString& url);

include/interpolation.h

@@ -1,51 +0,0 @@
-#pragma once
-#include "basedata.h"
-#include "comdefs.h"
-
-using michlib::real;
-
-template<class D> class LinearInterpolator: public D
-{
- public:
- LinearInterpolator(D&& d): D(std::move(d)) {}
-
- real operator()(const struct Point2D& in) const
- {
-  const auto gp = D::GridPos(in.x, in.y);
-  if(!gp.Valid()) return NAN;
-
-  if(gp.x == 0.0 && gp.y == 0.0) return D::V(gp.ix, gp.iy);
-
-  real   v00;
-  real   v10;
-  real   v01;
-  real   v11;
-  bool   isfill = false;
-  size_t fx, fy;
-
-  // Count fills
-  for(size_t ix = 0; ix <= 1; ix++)
-   for(size_t iy = 0; iy <= 1; iy++)
-   {
-    if(isfill && D::IsFill(gp.ix + ix, gp.iy + iy)) return NAN;
-    if(D::IsFill(gp.ix + ix, gp.iy + iy))
-    {
-     fx     = ix;
-     fy     = iy;
-     isfill = true;
-    }
-   }
-
-  v00 = D::V(gp.ix, gp.iy);
-  v10 = D::V(gp.ix + 1, gp.iy);
-  v01 = D::V(gp.ix, gp.iy + 1);
-  v11 = D::V(gp.ix + 1, gp.iy + 1);
-
-  if(isfill && fx == 0 && fy == 0) v00 = (v10 + v01 + v11) / 3.0;
-  if(isfill && fx == 1 && fy == 0) v10 = (v00 + v01 + v11) / 3.0;
-  if(isfill && fx == 0 && fy == 1) v01 = (v10 + v00 + v11) / 3.0;
-  if(isfill && fx == 1 && fy == 1) v11 = (v10 + v01 + v00) / 3.0;
-
-  return v00 + gp.y * (v01 - v00) + gp.x * ((v10 - v00) + gp.y * (v00 - v01 + v11 - v10));
- };
-};

include/mirrorfuncs.h

@@ -1,62 +0,0 @@
-#pragma once
-#include "curlfuncs.h"
-#include "mdatetime.h"
-#include <dirent.h>
-#include <fcntl.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <vector>
-
-using michlib::MDateTime;
-
-class DIRRAIIDT
-{
- public:
- // TODO: make static
- void operator()(DIR* d) { closedir(d); }
-};
-
-class DIRRAII: public std::unique_ptr<DIR, DIRRAIIDT>
-{
- public:
- operator DIR*() const { return get(); }
-};
-
-struct FileInfo
-{
- MString   url;
- MString   name;
- MDateTime mtime;
- size_t    size;
-};
-
-// Remove last element from path
-inline MString DirName(const MString& name)
-{
- auto p = name.GetPos('/', false);
- if(p == 0) return name;
- return name.SubStr(1, p - 1);
-}
-
-// Get last element from path
-inline MString FileName(const MString& name)
-{
- auto p = name.GetPos('/', false);
- if(p == 0) return name;
- return name.SubStr(p + 1, name.Len() - p);
-}
-
-// Check and, if necessary, create the path to the file
-bool MakePath(const MString& dname);
-
-// Get local file list
-std::pair<std::vector<struct FileInfo>, MString> ReadLocalFileList(const MString& dir, const MString& path = "");
-
-// Download file to the local mirror
-MString DownloadFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const MString& root);
-
-// Remove file from the local mirror
-MString RemoveFile(const struct FileInfo& linfo);
-
-// Updare file in the local mirror
-MString UpdateFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const struct FileInfo& linfo, const MString& root);

include/simple2ddata.h

@@ -42,23 +42,6 @@ class Simple2DData: public BaseData
 
  real XStep() const { return xstep; }
  real YStep() const { return ystep; }
-
- struct GridPoint GridPos(real x, real y) const
- {
-  struct GridPoint out;
-
-  if(!*this) return out;
-  if(x < x0 || x > x0 + (nx - 1) * xstep || y < y0 || y > y0 + (ny - 1) * ystep) return out;
-
-  real rx = x - x0;
-  real ry = y - y0;
-  out.ix  = static_cast<decltype(out.ix)>(michlib::Floor(rx / xstep));
-  out.iy  = static_cast<decltype(out.ix)>(michlib::Floor(ry / ystep));
-  out.x   = rx - out.ix * xstep;
-  out.y   = ry - out.iy * ystep;
-
-  return out;
- }
 };
 
 class Rect2DData: public BaseData

michlib

@@ -1 +1 @@
-Subproject commit e2882902b88229bb4b0e6fbeb76c79ac6d46d53d
+Subproject commit 6268edea843cae04337fe6493b611f3a4fe7cc9b

sources/COPERNICUS.cpp

@@ -1,282 +0,0 @@
-#define MICHLIB_NOSOURCE
-#include "COPERNICUS.h"
-#include "mirrorfuncs.h"
-#include <libxml/parser.h>
-#include <libxml/tree.h>
-
-using michlib::GPL;
-
-const MString COPERNICUSData::caturl = "https://stac.marine.copernicus.eu/metadata/catalog.stac.json";
-
-std::pair<Json::Value, MString> COPERNICUSData::GetJSON(const MString& url)
-{
- Json::Reader reader;
- Json::Value  obj;
- MString      content;
-
- auto [val, suc] = cache->Get(url);
- if(suc)
-  content = std::move(val);
- else
- {
-  michlib::message(url + " not found in cache, downloading");
-  auto [out, res] = GetUrl(chandle, url);
-  if(res != CURLE_OK) return {obj, MString("Can't download JSON: ") + curlerr};
-  cache->Put(url, out, 3600);
-  content = std::move(out);
- }
-
- reader.parse(content.Buf(), content.Buf() + content.Len(), obj, false);
-
- return {obj, ""};
-}
-
-MString COPERNICUSData::ReadURL(const Json::Value& cat, const MString& prod)
-{
- const auto& links = cat["links"];
- if(links.type() != Json::arrayValue) return "";
- for(Json::ArrayIndex i = 0; i < links.size(); i++)
- {
-  const auto& titl = links[i]["title"];
-  const auto& href = links[i]["href"];
-  if(titl.type() == Json::stringValue && href.type() == Json::stringValue)
-  {
-   MString str(titl.asString().c_str());
-   if(str == prod) return MString(href.asString().c_str());
-  }
- }
- return "";
-}
-
-std::pair<std::vector<struct FileInfo>, MString> COPERNICUSData::ReadRemoteFileList(const MString& url)
-{
- LIBXML_TEST_VERSION
-
- std::vector<struct FileInfo> out;
- MString                      bucket, prefix;
-
- // Split url on prefix and bucket
- {
-  size_t pos   = url.Len();
-  size_t count = 0;
-  for(size_t i = 0; i < url.Len(); i++)
-  {
-   if(url[i] == '/') count++;
-   if(count == 4)
-   {
-    pos = i;
-    break;
-   }
-  }
-  if(pos == url.Len()) return {out, "Can't parse url: " + url};
-
-  bucket = url.SubStr(1, pos);
-  prefix = url.SubStr(pos + 2, url.Len() - pos - 1);
- }
-
- MString cont;
- bool    next = true;
-
- while(next)
- {
-  MString url = bucket + "?list-type=2&prefix=" + prefix;
-  if(cont.Exist()) url += "&continuation-token=" + cont;
-  cont = "";
-
-  auto [data, res] = GetUrl(chandle, url);
-  if(res != CURLE_OK) return {out, MString("Can't download ") + url + ": " + curlerr};
-
-  xmlDocPtr doc = xmlReadMemory(data.Buf(), data.Len(), "data.xml", nullptr, 0);
-  if(doc == nullptr) return {out, MString("Can't download ") + url + ": XML parse error"};
-  auto cur = xmlDocGetRootElement(doc);
-  if(cur == nullptr)
-  {
-   xmlFreeDoc(doc);
-   return {out, MString("Can't download ") + url + ": empty XML"};
-  }
-  if(xmlStrEqual(cur->name, (const xmlChar*)"ListBucketResult") == 0)
-  {
-   xmlFreeDoc(doc);
-   return {out, MString("Can't download ") + url + ": unknown XML"};
-  }
-
-  for(const auto* n = cur->children; n; n = n->next)
-  {
-   if(xmlStrEqual(n->name, (const xmlChar*)"NextContinuationToken") == 1)
-   {
-    auto* content = xmlNodeGetContent(n);
-    cont          = (char*)content;
-    xmlFree(content);
-   }
-   if(xmlStrEqual(n->name, (const xmlChar*)"Contents") == 1)
-   {
-    MString   fname;
-    MDateTime mtime;
-    size_t    size = 0;
-    for(const auto* c = n->children; c; c = c->next)
-    {
-     if(xmlStrEqual(c->name, (const xmlChar*)"Key") == 1)
-     {
-      auto* content = xmlNodeGetContent(c);
-      fname         = (char*)content;
-      xmlFree(content);
-     }
-     if(xmlStrEqual(c->name, (const xmlChar*)"LastModified") == 1)
-     {
-      auto* content = xmlNodeGetContent(c);
-      mtime.FromString((char*)content);
-      xmlFree(content);
-     }
-     if(xmlStrEqual(c->name, (const xmlChar*)"Size") == 1)
-     {
-      auto* content = xmlNodeGetContent(c);
-      size          = MString((char*)content).ToInteger<size_t>();
-      xmlFree(content);
-     }
-    }
-    out.emplace_back(bucket + "/" + fname, fname.SubStr(prefix.Len() + 2, fname.Len() - prefix.Len() - 1), mtime, size);
-   }
-  }
-  xmlFreeDoc(doc);
-  next = cont.Exist();
- }
-
- std::sort(out.begin(), out.end(), [](const struct FileInfo& a, const struct FileInfo& b) { return a.name < b.name; });
- return {out, ""};
-}
-
-MString COPERNICUSData::Mirror(const CLArgs& args)
-{
- GPL.UsePrefix("COPERNICUS");
-
- // Local directory
- MString mirrorroot = GPL.ParameterSValue("MirrorTo", "");
- if(!mirrorroot.Exist()) return "Local mirror directory not specified";
-
- // Cache
- cache.reset(CreateCache(GPL.ParameterSValue("Cache", "")));
- if(!cache)
- {
-  michlib::errmessage("Can't init cache");
-  cache.reset(new FakeCache);
- }
-
- curl_easy_setopt(chandle, CURLOPT_ERRORBUFFER, curlerr);
-
- if(!args.contains("product")) return "Copernicus product not specified";
- MString     prod = args.at("product");
- Json::Value product;
- MString     produrl;
-
- // Get catalog
- {
-  auto [cat, err] = GetJSON(caturl);
-  if(err.Exist()) return "Can't download catalog: " + err;
-  if(cat["title"].type() != Json::stringValue || cat["title"].asString() != "Copernicus Marine Data Store") return "Can't parse catalog";
-  catalog = std::move(cat);
- }
-
- // Get product
- {
-  auto url = ReadURL(catalog, prod);
-  if(!url.Exist()) return "Url for product " + prod + " not found in catalog";
-  produrl        = DirName(caturl) + "/" + url;
-  auto [pr, err] = GetJSON(produrl);
-  if(err.Exist()) return "Can't download product information from " + produrl + ": " + err;
-  product = std::move(pr);
- }
-
- std::vector<MString> dsets;
- if(args.contains("dataset"))
-  dsets.push_back(args.at("dataset"));
- else
- {
-  const auto& links = product["links"];
-  if(links.type() != Json::arrayValue) return "Can't find information about datasets";
-  for(Json::ArrayIndex i = 0; i < links.size(); i++)
-  {
-   const auto& rel  = links[i]["rel"];
-   const auto& titl = links[i]["title"];
-   if(rel.type() == Json::stringValue && titl.type() == Json::stringValue && rel.asString() == "item") dsets.push_back(titl.asString().c_str());
-  }
- }
-
- for(const auto& dset: dsets)
- {
-  michlib::message("Mirroring " + dset);
-  auto url = ReadURL(product, dset);
-  if(!url.Exist()) return "Url for dataset " + dset + " not found in product description";
-  MString dseturl = DirName(produrl) + "/" + url;
-  auto [ds, err]  = GetJSON(dseturl);
-  if(err.Exist()) return "Can't download dataset information from " + dseturl + ": " + err;
-
-  const auto& href = ds["assets"]["native"]["href"];
-  if(href.type() != Json::stringValue) return "Can't find data for dataset " + dset + " from product " + prod;
-
-  url = href.asString().c_str();
-
-  MString locroot = mirrorroot + "/" + prod + "/" + dset;
-
-  auto [lfiles, lerr] = ReadLocalFileList(locroot);
-  if(lerr.Exist()) return lerr;
-
-  auto [rfiles, rerr] = ReadRemoteFileList(url);
-  if(rerr.Exist()) return rerr;
-
-  std::vector<size_t>                    down, rem;
-  std::vector<std::pair<size_t, size_t>> upd;
-
-  {
-   size_t rpos = 0, lpos = 0;
-   while(rpos != rfiles.size() || lpos != lfiles.size())
-   {
-    if(rpos == rfiles.size())
-     while(lpos != lfiles.size()) rem.push_back(lpos++);
-    if(lpos == lfiles.size())
-     while(rpos != rfiles.size()) down.push_back(rpos++);
-    if(rpos == rfiles.size() || lpos == lfiles.size()) continue;
-
-    if(rfiles[rpos].name < lfiles[lpos].name)
-     down.push_back(rpos++);
-    else if(lfiles[lpos].name < rfiles[rpos].name)
-     rem.push_back(lpos++);
-    else
-    {
-     auto delta = rfiles[rpos].mtime.Epoch() - lfiles[lpos].mtime.Epoch();
-     if(delta < 0) delta = -delta;
-     if(delta > 0 || rfiles[rpos].size != lfiles[lpos].size) upd.emplace_back(rpos, lpos);
-     lpos++;
-     rpos++;
-    }
-   }
-  }
-
-  michlib::message(MString("New files: ") + down.size());
-  michlib::message(MString("Obsolete files: ") + rem.size());
-  michlib::message(MString("Modified files: ") + upd.size());
-
-  for(size_t i = 0; i < down.size(); i++)
-  {
-   size_t ri  = down[i];
-   auto   err = DownloadFile(chandle, rfiles[ri], locroot);
-   if(err.Exist()) return err;
-  }
-
-  for(size_t i = 0; i < rem.size(); i++)
-  {
-   size_t li  = rem[i];
-   auto   err = RemoveFile(lfiles[li]);
-   if(err.Exist()) return err;
-  }
-
-  for(size_t i = 0; i < upd.size(); i++)
-  {
-   size_t ri  = upd[i].first;
-   size_t li  = upd[i].second;
-   auto   err = UpdateFile(chandle, rfiles[ri], lfiles[li], locroot);
-   if(err.Exist()) return err;
-  }
- }
-
- return "";
-}

sources/COPERNICUS.h

@@ -1,36 +0,0 @@
-#pragma once
-#include "ParseArgs.h"
-#include "cache.h"
-#include "curlfuncs.h"
-#include "mdatetime.h"
-#include <json/json.h>
-
-using michlib::MDateTime;
-using michlib::MString;
-
-class COPERNICUSData
-{
- static const MString caturl;
-
- std::unique_ptr<GenericCache> cache;
- CURLRAII                      chandle;
- Json::Value                   catalog;
- char                          curlerr[CURL_ERROR_SIZE];
-
- // Get url for product or dataset from catalog
- static MString ReadURL(const Json::Value& cat, const MString& prod);
-
- // Download JSON from url
- std::pair<Json::Value, MString> GetJSON(const MString& url);
-
- // Get remote file list from url
- std::pair<std::vector<struct FileInfo>,MString> ReadRemoteFileList(const MString& url);
-
- public:
- static constexpr const char* name = "COPERNICUS";
-
- COPERNICUSData() = default;
-
- // Main mirror function
- MString Mirror(const CLArgs& args);
-};

sources/GRIDVEL.cpp

@@ -1,86 +0,0 @@
-#define MICHLIB_NOSOURCE
-#include "GRIDVEL.h"
-
-MString GRIDVELData::Info() const
-{
- if(!isOk()) return "";
-
- // clang-format off
- return MString() +
-  " Region: (" + u.Xb() + " : " + u.Xe() + ") x (" + u.Yb() + " : " + u.Ye() + ")\n" +
-  " Grid: " + u.Nx() + "x" + u.Ny() + "\n" +
-  " Supported variables: u, v, U2";
- // clang-format on
-}
-
-MString GRIDVELData::Open(const CLArgs& args)
-{
- MString uname = args.contains("u") ? args.at("u") : "";
- MString vname = args.contains("v") ? args.at("v") : "";
- if(!uname.Exist()) return "File with u component not specified";
- if(!vname.Exist()) return "File with v component not specified";
-
- unit = args.contains("unit") ? args.at("unit") : "cm/s";
- fill = args.contains("fill") ? args.at("fill").ToReal() : 1e10;
-
- u.Open(uname);
- if(!u.Opened()) return "Can't open file: " + uname;
- v.Open(vname);
- if(!v.Opened()) return "Can't open file: " + vname;
- if(u != v) return "Files " + uname + " and " + vname + " have different grids";
-
- return "";
-}
-
-bool GRIDVELData::Read(const MString& vname, std::map<MString, GRIDVELData::Data>& cache, size_t i) const
-{
- if(cache.contains(vname)) return true;
- if(!isOk()) return false;
-
- // Only rectangular grids are supported
- real xs = (u.Xe() - u.Xb()) / (u.Nx() - 1);
- real ys = (u.Ye() - u.Yb()) / (u.Ny() - 1);
-
- Data out(u.Nx(), u.Ny(), u.Xb(), u.Yb(), xs, ys, MString(unit));
-
- // U and U2 from u and v
- if(vname == "U" || vname == "U2")
- {
-  bool square = vname == "U2";
-  if(!(Read("u", cache, i) && Read("v", cache, i))) return false;
-  cache[vname]      = cache.at("u").CopyGrid();
-  auto&       U     = cache.at(vname);
-  const auto& udata = cache.at("u");
-  const auto& vdata = cache.at("v");
-  if(udata.Unit().Exist()) U.SetUnit(square ? ("(" + udata.Unit() + ")2") : udata.Unit());
-  for(size_t ind = 0; ind < U.N(); ind++)
-  {
-   if(udata.IsFill(ind) || vdata.IsFill(ind))
-    U.V(ind) = U.Fillval();
-   else
-    U.V(ind) = square ? (udata(ind) * udata(ind) + vdata(ind) * vdata(ind)) : michlib::Hypot(udata(ind), vdata(ind));
-  }
-  return true;
- }
-
- if(vname == "u" || vname == "v")
- {
-  bool isu = vname == "u";
-  for(IType ix = 0; ix < u.Nx(); ix++)
-   for(IType iy = 0; iy < u.Ny(); iy++)
-   {
-    if(isu ? (u(ix, iy) == NAN || u(ix, iy) >= 1e10) : (v(ix, iy) == NAN || v(ix, iy) >= 1e10))
-     out.V(ix, iy) = out.Fillval();
-    else
-     out.V(ix, iy) = isu ? u(ix, iy) : v(ix, iy);
-   }
-
-  if(out)
-  {
-   cache[vname] = std::move(out);
-   return true;
-  }
- }
-
- return false;
-}

sources/GRIDVEL.h

@@ -1,50 +0,0 @@
-#pragma once
-#include "DataAdapters/gridfile.h"
-#include "mdatetime.h"
-#include "simple2ddata.h"
-
-using michlib::MDateTime;
-
-class GRIDVELData
-{
- michlib::GridFile u,v;
- MString unit;
- real fill;
-
- using IType=decltype(u.Nx());
-
- public:
- static constexpr const char* name = "GRIDVEL";
-
- static constexpr const char* disabledactions = "genintfile";
-
- using Data = Simple2DData;
-
- GRIDVELData() = default;
-
- MString Info() const;
- // TODO: RetVal
- MString Open(const CLArgs& args);
-
- bool isOk() const { return u.Opened() &&v.Opened(); }
-
- size_t NTimes() const { return 1; }
-
- MDateTime Time(size_t i) const
- {
-  return MDateTime();
- }
-
- //time_t Timestep() const { return isOk() ? (times[1] - times[0]) : 0; }
-
- explicit operator bool() const { return u.Opened() &&v.Opened(); }
-
- VarPresence CheckVar(const MString& vname) const
- {
-  if(vname == "u" || vname == "v") return VarPresence::INTERNAL;
-  if(vname == "U" || vname == "U2") return VarPresence::DERIVED;
-  return VarPresence::NONE;
- }
-
- bool Read(const MString& vname, std::map<MString, Data>& cache, size_t i) const;
-};

src/CMakeLists.txt

@@ -2,23 +2,15 @@ set(EXENAME odm)
 set(ACTIONLISTINC ${CMAKE_CURRENT_BINARY_DIR}/../include/actionlist.h) # Include actions files and define the actions classes list
 set(DATALISTINC ${CMAKE_CURRENT_BINARY_DIR}/../include/datalist.h) # Include data sources files and define the data sources classes list
 
-find_package(PkgConfig REQUIRED)
 find_library(netcdf netcdf REQUIRED)
 find_package(OpenMP REQUIRED)
-find_package(CURL REQUIRED)
-find_package(LibXml2 REQUIRED)
-find_package(SQLite3 REQUIRED)
-pkg_check_modules(JSONCPP REQUIRED jsoncpp)
-
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
 
-include_directories(${JSONCPP_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIRS} ${SQLite3_INCLUDE_DIRS})
-
 file(GLOB srcs CONFIGURE_DEPENDS *.cpp)
 add_executable(${EXENAME} ${srcs} ${ACTIONLISTINC} ${SOURCELISTINC})
 target_include_directories(${EXENAME} PRIVATE ../michlib/michlib ${CMAKE_CURRENT_BINARY_DIR}/../include)
 
-target_link_libraries(${EXENAME} ${linker_options} ${netcdf} OpenMP::OpenMP_CXX CURL::libcurl ${JSONCPP_LINK_LIBRARIES} LibXml2::LibXml2 SQLite::SQLite3 teos)
+target_link_libraries(${EXENAME} ${linker_options} ${netcdf} OpenMP::OpenMP_CXX teos)
 set_target_properties(${EXENAME} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 install(TARGETS ${EXENAME})
 

src/curlfuncs.cpp

@@ -1,41 +0,0 @@
-#define MICHLIB_NOSOURCE
-#include "curlfuncs.h"
-#include <unistd.h>
-
-using michlib::pointer_cast;
-using michlib::uint1;
-
-size_t Write2String(char* ptr, size_t size, size_t n, void* data)
-{
- MString* out = pointer_cast<MString*>(data);
- *out += MString(ptr, size * n);
- return size * n;
-}
-
-size_t Write2File(char* ptr, size_t size, size_t n, void* data)
-{
- const int*   fd    = pointer_cast<const int*>(data);
- size_t       count = size * n;
- const uint1* buf   = pointer_cast<const uint1*>(ptr);
-
- while(count != 0)
- {
-  auto wr = write(*fd, buf, count);
-  if(wr == -1) return 0;
-  count -= wr;
-  buf += wr;
- }
-
- return size * n;
-}
-
-std::pair<MString, CURLcode> GetUrl(const CURLRAII& chandle, const MString& url)
-{
- MString out;
-
- curl_easy_setopt(chandle, CURLOPT_URL, url.Buf());
- curl_easy_setopt(chandle, CURLOPT_WRITEFUNCTION, Write2String);
- curl_easy_setopt(chandle, CURLOPT_WRITEDATA, &out);
- auto res = curl_easy_perform(chandle);
- return {out, res};
-}

src/mirrorfuncs.cpp

@@ -1,130 +0,0 @@
-#define MICHLIB_NOSOURCE
-#include "mirrorfuncs.h"
-#include "StringFunctions.h"
-#include "filehelpers.h"
-#include "merrors.h"
-
-using michlib::FD;
-using michlib::message;
-
-bool MakePath(const MString& dname)
-{
- struct stat st;
- int         ret = stat(dname.Buf(), &st);
- if(ret == 0) return S_ISDIR(st.st_mode);
-
- auto    dirs = michlib::Split_on_words(dname, "/", false);
- MString cdir = "";
- for(const auto& dir: dirs)
- {
-  cdir += "/" + dir;
-  ret = stat(cdir.Buf(), &st);
-  if(ret == 0 && S_ISDIR(st.st_mode)) continue;
-  if(ret == 0 && !S_ISDIR(st.st_mode)) return false;
-  ret = mkdir(cdir.Buf(), 0755);
-  if(ret != 0) return false;
- }
- return true;
-}
-
-std::pair<std::vector<struct FileInfo>, MString> ReadLocalFileList(const MString& dir, const MString& path)
-{
- std::vector<struct FileInfo> out;
- DIRRAII                      dhandle;
-
- MakePath(dir);
- dhandle.reset(opendir(dir.Buf()));
-
- if(!dhandle) return {out, "Can't open directory " + path + (path.Exist() ? "/" : "") + dir};
-
- int dfd             = dirfd(dhandle);
- errno               = 0;
- struct dirent* dent = readdir(dhandle);
- if(errno != 0) return {out, "Can't read directory " + path + (path.Exist() ? "/" : "") + dir};
- struct stat st;
-
- do {
-  if(dent->d_name[0] != '.')
-  {
-   int ret = fstatat(dfd, dent->d_name, &st, AT_SYMLINK_NOFOLLOW);
-   if(ret != 0) return {out, "Can't stat " + path + "/" + dir + "/" + dent->d_name};
-   if(S_ISDIR(st.st_mode)) // Directory, recurse
-   {
-    auto [list, err] = ReadLocalFileList(dir + "/" + dent->d_name, path + (path.Exist() ? "/" : "") + dent->d_name);
-    if(err.Exist()) return {out, err};
-    out.insert(out.end(), list.begin(), list.end());
-   }
-   if(S_ISREG(st.st_mode)) // Regular file
-   {
-    out.emplace_back(dir + "/" + dent->d_name, path + (path.Exist() ? "/" : "") + dent->d_name, MDateTime(st.st_mtim.tv_sec, st.st_mtim.tv_nsec), st.st_size);
-   }
-   // Ignore non-directories, non-files
-  }
-  dent = readdir(dhandle);
- } while(dent != nullptr || errno != 0);
-
- if(errno != 0) return {out, "Can't read directory " + path + "/" + dir};
- std::sort(out.begin(), out.end(), [](const struct FileInfo& a, const struct FileInfo& b) { return a.name < b.name; });
- return {out, ""};
-}
-
-MString DownloadFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const MString& root)
-{
- message("Downloading " + rinfo.url);
-
- MString dname = DirName(rinfo.name), fname = FileName(rinfo.name);
- FD      fd;
-
- if(!MakePath(root + "/" + dname)) return "Can't create directory " + root + "/" + dname;
- fd.Reset(creat((root + "/" + rinfo.name).Buf(), 0644));
- if(!fd) return "Can't create file " + root + "/" + rinfo.name;
-
- char errbuf[CURL_ERROR_SIZE];
- int  cfd = fd.Get();
- curl_easy_setopt(chandle, CURLOPT_ERRORBUFFER, errbuf);
- curl_easy_setopt(chandle, CURLOPT_WRITEFUNCTION, Write2File);
- curl_easy_setopt(chandle, CURLOPT_WRITEDATA, &cfd);
- curl_easy_setopt(chandle, CURLOPT_URL, rinfo.url.Buf());
- auto res = curl_easy_perform(chandle);
- if(res != CURLE_OK)
- {
-  unlink((root + "/" + rinfo.name).Buf());
-  return MString("Can't download file: ") + errbuf;
- }
-
- {
-  struct timespec times[2];
-  times[0].tv_sec = times[1].tv_sec = rinfo.mtime.Epoch();
-  times[0].tv_nsec = times[1].tv_nsec = 0;
-
-  int ret = futimens(fd, times);
-  if(ret != 0)
-  {
-   unlink((root + "/" + rinfo.name).Buf());
-   return "Can't set mtime for file: " + root + "/" + rinfo.name;
-  }
- }
-
- return "";
-}
-
-MString RemoveFile(const struct FileInfo& linfo)
-{
- message("Remove " + linfo.url);
- int ret = unlink(linfo.url.Buf());
- if(ret != 0) return "Can't remove file " + linfo.url;
- return "";
-}
-
-MString UpdateFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const struct FileInfo& linfo, const MString& root)
-{
- MString err;
-
- message("Update " + linfo.url);
- err = RemoveFile(linfo);
- if(err.Exist()) return err;
- err = DownloadFile(chandle, rinfo, root);
- if(err.Exist()) return err;
-
- return "";
-}