7 changed files with 297 additions and 157 deletions
@ -0,0 +1,55 @@
|
||||
#pragma once |
||||
#include "cache.h" |
||||
#include "curlfuncs.h" |
||||
#include "merrors.h" |
||||
#include <json/json.h> |
||||
|
||||
using michlib::Error; |
||||
using michlib::RetVal; |
||||
|
||||
class CopernicusCatalog |
||||
{ |
||||
static const MString caturl; |
||||
|
||||
std::unique_ptr<GenericCache> cache; |
||||
CURLRAII chandle; |
||||
Json::Value catalog; |
||||
char curlerr[CURL_ERROR_SIZE]; |
||||
|
||||
// Download JSON from url
|
||||
RetVal<Json::Value> GetJSON(const MString& url) const; |
||||
|
||||
// Download catalog
|
||||
Error GetCatalog(); |
||||
|
||||
// Asset url from dataset
|
||||
RetVal<MString> AssetURL(const MString& prod, const MString& dataset, const MString& asset) const; |
||||
|
||||
public: |
||||
CopernicusCatalog(); |
||||
|
||||
// List of products
|
||||
RetVal<std::vector<MString>> ProductList() const; |
||||
|
||||
// List of datasets in product
|
||||
RetVal<std::vector<MString>> DatasetList(const MString& prod) const; |
||||
|
||||
// URL of product
|
||||
RetVal<MString> ProductURL(const MString& prod) const; |
||||
|
||||
// URL of dataset
|
||||
RetVal<MString> DatasetURL(const MString& prod, const MString& dataset) const; |
||||
|
||||
// URL of native data (files) in dataset
|
||||
RetVal<MString> DatasetNativeURL(const MString& prod, const MString& dataset) const { return AssetURL(prod, dataset, "native"); } |
||||
|
||||
// URL of timechuncked data (files) in dataset
|
||||
RetVal<MString> DatasetTimeURL(const MString& prod, const MString& dataset) const { return AssetURL(prod, dataset, "timeChunked"); } |
||||
|
||||
// URL of geochuncked data (files) in dataset
|
||||
RetVal<MString> DatasetGeoURL(const MString& prod, const MString& dataset) const { return AssetURL(prod, dataset, "geoChunked"); } |
||||
|
||||
bool Valid() const { return catalog.isObject(); } |
||||
|
||||
explicit operator bool() const { return Valid(); } |
||||
}; |
||||
@ -0,0 +1,169 @@
|
||||
#define MICHLIB_NOSOURCE |
||||
#include "copcat.h" |
||||
#include "GPL.h" |
||||
#include "mirrorfuncs.h" |
||||
|
||||
const MString CopernicusCatalog::caturl = "https://stac.marine.copernicus.eu/metadata/catalog.stac.json"; |
||||
|
||||
CopernicusCatalog::CopernicusCatalog() |
||||
{ |
||||
// Cache
|
||||
michlib::GPL.UsePrefix("COPERNICUS"); |
||||
cache.reset(CreateCache(michlib::GPL.ParameterSValue("Cache", ""))); |
||||
if(!cache) |
||||
{ |
||||
michlib::errmessage("Can't init cache"); |
||||
cache.reset(new FakeCache); |
||||
} |
||||
|
||||
curl_easy_setopt(chandle, CURLOPT_ERRORBUFFER, curlerr); |
||||
|
||||
GetCatalog(); |
||||
} |
||||
|
||||
Error CopernicusCatalog::GetCatalog() |
||||
{ |
||||
if(Valid()) return Error(); |
||||
auto ret = GetJSON(caturl); |
||||
if(ret) |
||||
catalog = ret.Value(); |
||||
else |
||||
return ret.Add("CopernicusCatalog::GetCatalog", "can't download catalog"); |
||||
return Error(); |
||||
} |
||||
|
||||
RetVal<std::vector<MString>> CopernicusCatalog::ProductList() const |
||||
{ |
||||
static const MString pref = "CopernicusCatalog::ProductList"; |
||||
|
||||
if(!Valid()) return {pref, "no catalog"}; |
||||
|
||||
const auto& links = catalog["links"]; |
||||
if(links.type() != Json::arrayValue) return {pref, "no \"links\" section in the catalog"}; |
||||
|
||||
std::vector<MString> out; |
||||
for(Json::ArrayIndex i = 0; i < links.size(); i++) |
||||
{ |
||||
const auto& rel = links[i]["rel"]; |
||||
const auto& titl = links[i]["title"]; |
||||
|
||||
if(rel.type() == Json::stringValue && titl.type() == Json::stringValue && rel.asString() == "child") out.emplace_back(titl.asString().c_str()); |
||||
} |
||||
return out; |
||||
} |
||||
|
||||
RetVal<MString> CopernicusCatalog::ProductURL(const MString& prod) const |
||||
{ |
||||
static const MString pref = "CopernicusCatalog::ProductURL"; |
||||
|
||||
if(!Valid()) return {pref, "no catalog"}; |
||||
|
||||
const auto& links = catalog["links"]; |
||||
if(links.type() != Json::arrayValue) return {pref, "no \"links\" section in the catalog"}; |
||||
|
||||
for(Json::ArrayIndex i = 0; i < links.size(); i++) |
||||
{ |
||||
const auto& titl = links[i]["title"]; |
||||
const auto& href = links[i]["href"]; |
||||
if(titl.type() == Json::stringValue && href.type() == Json::stringValue && titl.asString().c_str() == prod) return DirName(caturl) + "/" + MString(href.asString().c_str()); |
||||
} |
||||
return {pref, "unknown product: " + prod}; |
||||
} |
||||
|
||||
RetVal<std::vector<MString>> CopernicusCatalog::DatasetList(const MString& prod) const |
||||
{ |
||||
static const MString pref = "CopernicusCatalog::DatasetList"; |
||||
|
||||
MString url; |
||||
{ |
||||
auto ret = ProductURL(prod); |
||||
if(!ret) return ret.Add(pref, "Can't get url for the product " + prod); |
||||
url = ret.Value(); |
||||
} |
||||
|
||||
auto ret = GetJSON(url); |
||||
if(!ret) return ret.Add(pref, "Can't download product " + prod); |
||||
|
||||
const auto& links = ret.Value()["links"]; |
||||
if(links.type() != Json::arrayValue) return {pref, "no \"links\" section in the product " + prod + " description"}; |
||||
|
||||
std::vector<MString> out; |
||||
for(Json::ArrayIndex i = 0; i < links.size(); i++) |
||||
{ |
||||
const auto& rel = links[i]["rel"]; |
||||
const auto& titl = links[i]["title"]; |
||||
|
||||
if(rel.type() == Json::stringValue && titl.type() == Json::stringValue && rel.asString() == "item") out.emplace_back(titl.asString().c_str()); |
||||
} |
||||
return out; |
||||
} |
||||
|
||||
RetVal<MString> CopernicusCatalog::DatasetURL(const MString& prod, const MString& dataset) const |
||||
{ |
||||
static const MString pref = "CopernicusCatalog::DatasetURL"; |
||||
|
||||
MString url; |
||||
{ |
||||
auto ret = ProductURL(prod); |
||||
if(!ret) return ret.Add(pref, "Can't get url for the product " + prod); |
||||
url = ret.Value(); |
||||
} |
||||
|
||||
auto ret = GetJSON(url); |
||||
if(!ret) return ret.Add(pref, "Can't download product " + prod); |
||||
|
||||
const auto& links = ret.Value()["links"]; |
||||
if(links.type() != Json::arrayValue) return {pref, "no \"links\" section in the product " + prod + " description"}; |
||||
|
||||
for(Json::ArrayIndex i = 0; i < links.size(); i++) |
||||
{ |
||||
const auto& titl = links[i]["title"]; |
||||
const auto& href = links[i]["href"]; |
||||
if(titl.type() == Json::stringValue && href.type() == Json::stringValue && titl.asString().c_str() == dataset) return DirName(url) + "/" + MString(href.asString().c_str()); |
||||
} |
||||
return {pref, "unknown dataset: " + dataset}; |
||||
} |
||||
|
||||
RetVal<MString> CopernicusCatalog::AssetURL(const MString& prod, const MString& dataset, const MString& asset) const |
||||
{ |
||||
static const MString pref = "CopernicusCatalog::AssetURL"; |
||||
|
||||
MString url; |
||||
{ |
||||
auto ret = DatasetURL(prod, dataset); |
||||
if(!ret) return ret.Add(pref, "Can't get url for the dataset " + dataset); |
||||
url = ret.Value(); |
||||
} |
||||
|
||||
auto ret = GetJSON(url); |
||||
if(!ret) return ret.Add(pref, "Can't download dataset " + dataset); |
||||
|
||||
const auto& href = ret.Value()["assets"][asset.Buf()]["href"]; |
||||
if(!href || href.type() != Json::stringValue) return {pref, "href for the asset " + asset + " not found"}; |
||||
return MString(href.asString().c_str()); |
||||
} |
||||
|
||||
RetVal<Json::Value> CopernicusCatalog::GetJSON(const MString& url) const |
||||
{ |
||||
const static MString pref = "CopernicusCatalog::GetJSON"; |
||||
|
||||
Json::Reader reader; |
||||
Json::Value obj; |
||||
MString content; |
||||
|
||||
auto [val, suc] = cache->Get(url); |
||||
if(suc) |
||||
content = std::move(val); |
||||
else |
||||
{ |
||||
michlib::message(url + " not found in cache, downloading"); |
||||
auto [out, res] = GetUrl(chandle, url); |
||||
if(res != CURLE_OK) return Error(pref, MString("can't download JSON: ") + curlerr); |
||||
cache->Put(url, out, 3600); |
||||
content = std::move(out); |
||||
} |
||||
|
||||
reader.parse(content.Buf(), content.Buf() + content.Len(), obj, false); |
||||
|
||||
return obj; |
||||
} |
||||
Loading…
Reference in new issue