New dataset for NEMO (temporary fix)

include/cache.h

@@ -1,14 +1,149 @@
 #pragma once
-#include "merrors.h"
+#include "GPL.h"
+#include <functional>
 #include <libpq-fe.h>
 #include <sqlite3.h>
 #include <time.h>
 #include <variant>
 
+using michlib::GPL;
 using michlib::int_cast;
 using michlib::MString;
 using michlib::pointer_cast;
 
+class SQLiteConnection
+{
+ public:
+ using DBType   = sqlite3*;
+ using FuncType = std::function<void(DBType)>;
+
+ private:
+ static DBType                db;
+ static size_t                count;
+ static std::vector<FuncType> destructs;
+
+ public:
+ SQLiteConnection()
+ {
+  count++;
+  if(db == nullptr)
+  {
+   MString oldprefix = GPL.UsePrefix("SQLITE");
+   MString name      = GPL.ParameterSValue("db", "");
+   GPL.UsePrefix(oldprefix);
+
+   auto ret = sqlite3_open(name.Buf(), &db);
+   if(ret != SQLITE_OK)
+   {
+    sqlite3_close(db);
+    db = nullptr;
+   }
+  }
+ }
+
+ SQLiteConnection([[maybe_unused]] const SQLiteConnection& sq): SQLiteConnection() {}
+ SQLiteConnection(SQLiteConnection&&) = default;
+
+ SQLiteConnection& operator=([[maybe_unused]] const SQLiteConnection& sq)
+ {
+  *this = {};
+  return *this;
+ }
+ SQLiteConnection& operator=(SQLiteConnection&&) = default;
+
+ ~SQLiteConnection()
+ {
+  if(count == 0) michlib::errmessage("Destructor of SQLiteConnection called on count==0");
+  if(count > 1)
+   count--;
+  else
+  {
+   count = 0;
+   if(db != nullptr)
+   {
+    for(const auto& f: destructs) f(db);
+    sqlite3_close(db);
+   }
+   db = nullptr;
+  }
+ }
+
+ static void AddDestructor(FuncType&& f) { destructs.emplace_back(std::move(f)); }
+
+ operator DBType() const { return db; }
+
+ static DBType GetDB() { return db; }
+
+ explicit operator bool() const { return db != nullptr; }
+};
+
+class PostgreSQLConnection
+{
+ public:
+ using DBType   = PGconn*;
+ using FuncType = std::function<void(DBType)>;
+
+ private:
+ static DBType                conn;
+ static size_t                count;
+ static std::vector<FuncType> destructs;
+
+ public:
+ PostgreSQLConnection()
+ {
+  count++;
+  if(conn == nullptr)
+  {
+   MString oldprefix = GPL.UsePrefix("POSTGRES");
+   MString name      = GPL.ParameterSValue("connection", "");
+   GPL.UsePrefix(oldprefix);
+
+   conn = PQconnectdb(name.Buf());
+   if(PQstatus(conn) != CONNECTION_OK)
+   {
+    michlib::errmessage(PQerrorMessage(conn));
+    PQfinish(conn);
+    conn = nullptr;
+   }
+  }
+ }
+
+ PostgreSQLConnection([[maybe_unused]] const PostgreSQLConnection& pq): PostgreSQLConnection() {}
+ PostgreSQLConnection(PostgreSQLConnection&&) = default;
+
+ PostgreSQLConnection& operator=([[maybe_unused]] const PostgreSQLConnection& pq)
+ {
+  *this = {};
+  return *this;
+ }
+ PostgreSQLConnection& operator=(PostgreSQLConnection&&) = default;
+
+ ~PostgreSQLConnection()
+ {
+  if(count == 0) michlib::errmessage("Destructor of PostgreSQLConnection called on count==0");
+  if(count > 1)
+   count--;
+  else
+  {
+   count = 0;
+   if(conn != nullptr)
+   {
+    for(const auto& f: destructs) f(conn);
+    PQfinish(conn);
+   }
+   conn = nullptr;
+  }
+ }
+
+ static void AddDestructor(FuncType&& f) { destructs.emplace_back(std::move(f)); }
+
+ operator DBType() const { return conn; }
+
+ static DBType GetDB() { return conn; }
+
+ explicit operator bool() const { return conn != nullptr; }
+};
+
 class GenericCache
 {
  public:
@@ -27,18 +162,17 @@ class FakeCache: public GenericCache
 
 class SQLiteCache: public GenericCache
 {
- sqlite3* db = nullptr;
+ static bool regdest;
+
+ SQLiteConnection db;
 
  public:
- bool Init(const MString& name)
+ bool Init()
  {
-  Close();
+  if(!db) return false;
-  auto ret = sqlite3_open(name.Buf(), &db);
+
-  if(ret != SQLITE_OK)
+  if(!regdest)
   {
-   Close();
-   return false;
-  }
    // Create table
    sqlite3_stmt* sqst;
    int           i;
@@ -51,18 +185,25 @@ class SQLiteCache: public GenericCache
    if(i != SQLITE_DONE)
    {
     sqlite3_finalize(sqst);
-   Close();
     return false;
    }
    sqlite3_finalize(sqst);
    sqlite3_busy_timeout(db, 1000);
-  return true;
- }
 
- void Close()
+   db.AddDestructor(
+       [](SQLiteConnection::DBType db)
        {
-  if(db != nullptr) sqlite3_close(db);
+        sqlite3_stmt* sqst = nullptr;
-  db = nullptr;
+        int           i    = SQLITE_OK;
+
+        if(i == SQLITE_OK) i = sqlite3_prepare_v2(db, "DELETE from `cache` WHERE exptime<?1;", -1, &sqst, 0);
+        if(i == SQLITE_OK) i = sqlite3_bind_int64(sqst, 1, time(nullptr));
+        if(i == SQLITE_OK) i = sqlite3_step(sqst);
+        sqlite3_finalize(sqst);
+       });
+   regdest = true;
+  }
+  return true;
  }
 
  virtual bool Put(const MString& key, const MString& value, size_t ttl) const override
@@ -107,26 +248,16 @@ class SQLiteCache: public GenericCache
   return {"", false};
  }
 
- virtual ~SQLiteCache() override
+ virtual ~SQLiteCache() override = default;
- {
-  if(!*this) return;
-
-  sqlite3_stmt* sqst = nullptr;
-  int           i    = SQLITE_OK;
-
-  if(i == SQLITE_OK) i = sqlite3_prepare_v2(db, "DELETE from `cache` WHERE exptime<?1;", -1, &sqst, 0);
-  if(i == SQLITE_OK) i = sqlite3_bind_int64(sqst, 1, time(nullptr));
-  if(i == SQLITE_OK) i = sqlite3_step(sqst);
-  sqlite3_finalize(sqst);
-  sqlite3_close_v2(db);
- }
 
  explicit operator bool() const { return db != nullptr; }
 };
 
 class PostgreSQLCache: public GenericCache
 {
- PGconn* conn = nullptr;
+ static bool regdest;
+
+ PostgreSQLConnection conn;
 
  bool CheckCon() const
  {
@@ -148,39 +279,41 @@ class PostgreSQLCache: public GenericCache
  }
 
  public:
- bool Init(const MString& name)
+ bool Init()
  {
-  Close();
+  if(!conn) return false;
-  conn = PQconnectdb(name.Buf());
 
-  if(PQstatus(conn) != CONNECTION_OK)
+  if(!regdest)
   {
-   michlib::errmessage(PQerrorMessage(conn));
-   Close();
-   return false;
-  }
-
    // Create table
-  if(false)
+   {
+    auto* res = PQexec(conn, "SET client_min_messages=WARNING;");
+    PQclear(res);
+   }
    {
     auto* res = PQexec(conn, "CREATE TABLE IF NOT EXISTS cache(key TEXT PRIMARY KEY NOT NULL, value BYTEA, exptime TIMESTAMP(0) NOT NULL);");
     if(PQresultStatus(res) != PGRES_COMMAND_OK)
     {
      michlib::errmessage(PQresStatus(PQresultStatus(res)));
      michlib::errmessage(PQerrorMessage(conn));
-    PQclear(res);
-    Close();
     }
-   else
     PQclear(res);
    }
-  return true;
+   {
+    auto* res = PQexec(conn, "SET client_min_messages=NOTICE;");
+    PQclear(res);
    }
 
- void Close()
+   conn.AddDestructor(
+       [](PostgreSQLConnection::DBType conn)
        {
-  if(conn != nullptr) PQfinish(conn);
+        auto* res = PQexec(conn, "DELETE FROM cache WHERE exptime<localtimestamp;");
-  conn = nullptr;
+        PQclear(res);
+       });
+
+   regdest = true;
+  }
+  return true;
  }
 
  virtual bool Put(const MString& key, const MString& value, size_t ttl) const override
@@ -235,14 +368,7 @@ class PostgreSQLCache: public GenericCache
   return {std::move(val), true};
  }
 
- virtual ~PostgreSQLCache() override
+ virtual ~PostgreSQLCache() override = default;
- {
-  if(!CheckCon()) return;
-
-  auto* res = PQexec(conn, "DELETE FROM cache WHERE exptime<localtimestamp;");
-  PQclear(res);
-  Close();
- }
 
  explicit operator bool() const { return conn != nullptr; }
 };
@@ -250,26 +376,23 @@ class PostgreSQLCache: public GenericCache
 inline GenericCache* CreateCache(const MString& cachedesc)
 {
  auto i    = cachedesc.GetPos(':');
- if(i == 0)
+ auto name = i == 0 ? cachedesc : cachedesc.SubStr(1, i - 1);
- {
+ auto par  = i == 0 ? "" : cachedesc.SubStr(i + 1, cachedesc.Len() - i);
-  if(cachedesc == "no") return new FakeCache;
-  return nullptr;
- }
 
- auto name = cachedesc.SubStr(1, i - 1);
+ if(name == "no") return new FakeCache;
- auto par  = cachedesc.SubStr(i + 1, cachedesc.Len() - i);
 
  if(name == "sqlite")
  {
   auto ret = new SQLiteCache;
-  ret->Init(par);
+  ret->Init();
   if(*ret) return ret;
   delete ret;
  }
+
  if(name == "postgre" || name == "postgres" || name == "postgresql")
  {
   auto ret = new PostgreSQLCache;
-  ret->Init(par);
+  ret->Init();
   if(*ret) return ret;
   delete ret;
  }

sources/COPERNICUS.cpp

@@ -129,6 +129,9 @@ Error COPERNICUSData::Mirror(const CLArgs& args) const
   dsets = dlist.Value();
  }
 
+ michlib::RegExpSimple filter((args.contains("filter") ? args.at("filter") : ".*").Buf());
+ if(filter.Compile() != 0) return Error(pref, MString("Can't compile regular expression ") + filter.RegStr());
+
  CURLRAII chandle;
  for(const auto& dset: dsets)
  {
@@ -154,20 +157,34 @@ Error COPERNICUSData::Mirror(const CLArgs& args) const
    while(rpos != rfiles.size() || lpos != lfiles.size())
    {
     if(rpos == rfiles.size())
-     while(lpos != lfiles.size()) rem.push_back(lpos++);
+     while(lpos != lfiles.size())
+     {
+      if(filter.Match(lfiles[lpos].name.Buf())) rem.push_back(lpos);
+      lpos++;
+     }
     if(lpos == lfiles.size())
-     while(rpos != rfiles.size()) down.push_back(rpos++);
+     while(rpos != rfiles.size())
+     {
+      if(filter.Match(rfiles[rpos].name.Buf())) down.push_back(rpos);
+      rpos++;
+     }
     if(rpos == rfiles.size() || lpos == lfiles.size()) continue;
 
     if(rfiles[rpos].name < lfiles[lpos].name)
-     down.push_back(rpos++);
+    {
+     if(filter.Match(rfiles[rpos].name.Buf())) down.push_back(rpos);
+     rpos++;
+    }
     else if(lfiles[lpos].name < rfiles[rpos].name)
-     rem.push_back(lpos++);
+    {
+     if(filter.Match(lfiles[lpos].name.Buf())) rem.push_back(lpos);
+     lpos++;
+    }
     else
     {
      auto delta = rfiles[rpos].mtime.Epoch() - lfiles[lpos].mtime.Epoch();
      if(delta < 0) delta = -delta;
-     if(delta > 0 || rfiles[rpos].size != lfiles[lpos].size) upd.emplace_back(rpos, lpos);
+     if((delta > 0 || rfiles[rpos].size != lfiles[lpos].size) && filter.Match(lfiles[lpos].name.Buf())) upd.emplace_back(rpos, lpos);
      lpos++;
      rpos++;
     }

sources/NEMO.h

@@ -7,6 +7,7 @@ class NEMOData: public LayeredDataZ
  {
   TYPE_UNKNOWN,
   TYPE_DT,
+  TYPE_DT1,
   TYPE_NRT,
   TYPE_NRT6,
   TYPE_BALTICDT,
@@ -34,6 +35,7 @@ class NEMOData: public LayeredDataZ
   switch(type)
   {
   case(TYPE_DT): return "NEMO Delayed time, daily mean (DT)";
+  case(TYPE_DT1): return "NEMO Delayed time, daily mean, part 2 (DT1)";
   case(TYPE_NRT): return "NEMO Near-real time, daily mean (NRT)";
   case(TYPE_NRT6): return "NEMO Near-real time, 6h resolution (NRT6)";
   case(TYPE_BALTICDT): return "NEMO Delayed time, Baltic region, daily mean (BALTICDT)";
@@ -59,6 +61,8 @@ class NEMOData: public LayeredDataZ
   GPL.UsePrefix("NEMO");
   if(dataset == "DT")
    type = TYPE_DT;
+  else if(dataset == "DT1")
+   type = TYPE_DT1;
   else if(dataset == "NRT")
    type = TYPE_NRT;
   else if(dataset == "NRT6")

src/cache.cpp

@@ -0,0 +1,14 @@
+#define MICHLIB_NOSOURCE
+#include "cache.h"
+
+SQLiteConnection::DBType                SQLiteConnection::db        = nullptr;
+size_t                                  SQLiteConnection::count     = 0;
+std::vector<SQLiteConnection::FuncType> SQLiteConnection::destructs = {};
+
+PostgreSQLConnection::DBType                PostgreSQLConnection::conn      = nullptr;
+size_t                                      PostgreSQLConnection::count     = 0;
+std::vector<PostgreSQLConnection::FuncType> PostgreSQLConnection::destructs = {};
+
+bool SQLiteCache::regdest = false;
+
+bool PostgreSQLCache::regdest = false;

src/copcat.cpp

@@ -7,16 +7,20 @@ const MString CopernicusCatalog::caturl = "https://stac.marine.copernicus.eu/met
 
 CopernicusCatalog::CopernicusCatalog()
 {
- // Cache
  auto oldprefix = michlib::GPL.UsePrefix("COPERNICUS");
+ // Cache
  cache.reset(CreateCache(michlib::GPL.ParameterSValue("Cache", "")));
- michlib::GPL.UsePrefix(oldprefix);
  if(!cache)
  {
   michlib::errmessage("Can't init cache");
   cache.reset(new FakeCache);
  }
 
+ // Proxy
+ auto proxyurl = michlib::GPL.ParameterSValue("Proxy", "");
+ if(proxyurl.Exist()) curl_easy_setopt(chandle, CURLOPT_PROXY, proxyurl.Buf());
+
+ michlib::GPL.UsePrefix(oldprefix);
  GetCatalog();
 }
 
@@ -44,9 +48,14 @@ RetVal<std::vector<MString>> CopernicusCatalog::ProductList() const
  for(Json::ArrayIndex i = 0; i < links.size(); i++)
  {
   const auto& rel  = links[i]["rel"];
-  const auto& titl = links[i]["title"];
+  const auto& href = links[i]["href"];
 
-  if(rel.type() == Json::stringValue && titl.type() == Json::stringValue && rel.asString() == "child") out.emplace_back(titl.asString().c_str());
+  if(rel.type() == Json::stringValue && href.type() == Json::stringValue && rel.asString() == "child")
+  {
+   auto str = href.asString();
+   str.erase(str.find('/'));
+   out.emplace_back(str.c_str());
+  }
  }
  return out;
 }
@@ -62,9 +71,8 @@ RetVal<MString> CopernicusCatalog::ProductURL(const MString& prod) const
 
  for(Json::ArrayIndex i = 0; i < links.size(); i++)
  {
-  const auto& titl = links[i]["title"];
   const auto& href = links[i]["href"];
-  if(titl.type() == Json::stringValue && href.type() == Json::stringValue && titl.asString().c_str() == prod) return DirName(caturl) + "/" + MString(href.asString().c_str());
+  if(href.type() == Json::stringValue && href.asString() == (prod + "/product.stac.json").Buf()) return DirName(caturl) + "/" + MString(href.asString().c_str());
  }
  return {pref, "unknown product: " + prod};
 }
@@ -90,9 +98,14 @@ RetVal<std::vector<MString>> CopernicusCatalog::DatasetList(const MString& prod)
  for(Json::ArrayIndex i = 0; i < links.size(); i++)
  {
   const auto& rel  = links[i]["rel"];
-  const auto& titl = links[i]["title"];
+  const auto& href = links[i]["href"];
 
-  if(rel.type() == Json::stringValue && titl.type() == Json::stringValue && rel.asString() == "item") out.emplace_back(titl.asString().c_str());
+  if(rel.type() == Json::stringValue && href.type() == Json::stringValue && rel.asString() == "item")
+  {
+   auto str = href.asString();
+   str.erase(str.find('/'));
+   out.emplace_back(str.c_str());
+  }
  }
  return out;
 }
@@ -116,9 +129,8 @@ RetVal<MString> CopernicusCatalog::DatasetURL(const MString& prod, const MString
 
  for(Json::ArrayIndex i = 0; i < links.size(); i++)
  {
-  const auto& titl = links[i]["title"];
   const auto& href = links[i]["href"];
-  if(titl.type() == Json::stringValue && href.type() == Json::stringValue && titl.asString().c_str() == dataset) return DirName(url) + "/" + MString(href.asString().c_str());
+  if(href.type() == Json::stringValue && href.asString() == (dataset + "/dataset.stac.json").Buf()) return DirName(url) + "/" + MString(href.asString().c_str());
  }
  return {pref, "unknown dataset: " + dataset};
 }