michael_ul
/
odm
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Support for multiple open files or datasets in the NCZarr interface.

lintest
Michael Uleysky 5 months ago
parent
c46abb091d
commit
7c3c254b72
2 changed files with 234 additions and 3 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 31
      include/nczarr.h
  2. 206
      include/nczarrmulti.h

31
include/nczarr.h
Unescape View File

@ -1,9 +1,8 @@
#pragma once #pragma once
#include "ncsimple.h" #include "nczarrmulti.h"
#include "zarr.h"
#include <variant> #include <variant>
using NCZarrBase = std::variant<Zarr, NCSimple>; using NCZarrBase = std::variant<Zarr, NCSimple, ZarrMulti, NCMulti>;
#define DEFFUNC(NAME) \ #define DEFFUNC(NAME) \
auto NAME() const \ auto NAME() const \
@ -134,6 +133,32 @@ class NCZarr: private NCZarrBase, private DimReqDef
if(p == nullptr) return Error("NCZarr::OpenNC", "Impossible error!"); if(p == nullptr) return Error("NCZarr::OpenNC", "Impossible error!");
return p->Open(product, dataset, time); return p->Open(product, dataset, time);
} }
Error OpenMultiNC(const std::vector<MString>& names)
{
if(names.size() == 0) return Error("NCZarr::OpenMultiNC", "no names");
if(names.size() == 1) return OpenNC(names[0]);
auto pv = static_cast<NCZarrBase*>(this);
*pv = NCZarrBase(NCMulti());
auto p = std::get_if<NCMulti>(pv);
if(p == nullptr) return Error("NCZarr::OpenMultiNC", "Impossible error!");
return p->Open(names);
}
Error OpenMultiZarr(const MString& product, const std::vector<MString>& dsets, bool time = true)
{
if(dsets.size() == 0) return Error("NCZarr::OpenMultiZarr", "no datasets");
if(dsets.size() == 1) return OpenZarr(product, dsets[0], time);
auto pv = static_cast<NCZarrBase*>(this);
*pv = NCZarrBase(ZarrMulti());
auto p = std::get_if<ZarrMulti>(pv);
if(p == nullptr) return Error("NCZarr::OpenMultiZarr", "Impossible error!");
return p->Open(product, dsets, time);
}
}; };
#undef DEFFUNC #undef DEFFUNC

206
include/nczarrmulti.h
Unescape View File

@ -0,0 +1,206 @@
#pragma once
#include "ncsimple.h"
#include "zarr.h"
#include <set>
#define DEFFUNC(func, def) \
auto func(const MString& var) const \
{ \
const auto ind = FindVar(var); \
return ind < sources.size() ? sources[ind].func(var) : (def); \
}
#define DEFFUNCA(func, def) \
auto func(const MString& att) const \
{ \
const auto ind = FindAtt(att); \
return ind < sources.size() ? sources[ind].func(att) : (def); \
}
#define DEFFUNC2(func, def) \
auto func(const MString& var, const MString& name) const \
{ \
const auto ind = FindVar(var); \
return ind < sources.size() ? sources[ind].func(var, name) : (def); \
}
#define DEFFUNCNAMES(func) \
auto func() const \
{ \
std::set<MString> names; \
for(const auto& s: sources) \
{ \
auto snames = s.func(); \
for(const auto& name: snames) names.insert(name); \
} \
std::vector<MString> out; \
for(const auto& name: names) out.push_back(name); \
return out; \
}
template<class T> class NCZarrMultiCommon: private DimReqDef
{
protected:
std::vector<T> sources;
private:
size_t FindVar(const MString& var) const
{
for(size_t i = 0; i < sources.size(); i++)
if(sources[i].HasVar(var)) return i;
return sources.size();
}
size_t FindAtt(const MString& att) const
{
for(size_t i = 0; i < sources.size(); i++)
if(sources[i].HasAtt(att)) return i;
return sources.size();
}
public:
explicit operator bool() const
{
if(sources.empty()) return false;
for(const auto& s: sources)
if(!s) return false;
return true;
}
size_t NDim() const { return DimNames().size(); }
DEFFUNC(NDim, 0)
size_t NAtt() const { return AttNames().size(); }
DEFFUNC(NAtt, 0)
DEFFUNCNAMES(AttNames)
DEFFUNC(AttNames, decltype(AttNames())())
DEFFUNCNAMES(VarNames)
DEFFUNC(VarT, T::VarType::UNDEF)
DEFFUNC(VarFill, decltype(T().VarFill(MString()))())
DEFFUNC(DimNames, decltype(T().DimNames(MString()))())
DEFFUNC2(DimSize, 0)
DEFFUNC2(AttT, T::AttType::UNDEF)
DEFFUNC2(AttInt, 0)
DEFFUNC2(AttUInt, 0)
DEFFUNC2(AttReal, 0.0)
DEFFUNC2(AttString, MString())
DEFFUNC2(AttBool, false)
DEFFUNCA(AttT, T::AttType::UNDEF)
DEFFUNCA(AttInt, 0)
DEFFUNCA(AttUInt, 0)
DEFFUNCA(AttReal, 0.0)
DEFFUNCA(AttString, MString())
DEFFUNCA(AttBool, false)
DEFFUNC2(HasDim, false)
DEFFUNC(HasVar, false)
DEFFUNCA(HasAtt, false)
DEFFUNC2(HasAtt, false)
// Request is string
template<class Data, class Transform> Error Read(const MString& vname, Data& data, Transform transform, const char* request) const
{
return Read(vname, data, transform, MString(request));
}
// Request by one dimension
template<class Data, class Transform> Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1) const
{
return Read(vname, data, transform, std::vector<DimReq>{std::move(req1)});
}
// Request by two dimension
template<class Data, class Transform> Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1, DimReq&& req2) const
{
return Read(vname, data, transform, std::vector<DimReq>{std::move(req1), std::move(req2)});
}
// Request by three dimension
template<class Data, class Transform> Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1, DimReq&& req2, DimReq&& req3) const
{
return Read(vname, data, transform, std::vector<DimReq>{std::move(req1), std::move(req2), std::move(req3)});
}
// Request by four dimension
template<class Data, class Transform> Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1, DimReq&& req2, DimReq&& req3, DimReq&& req4) const
{
return Read(vname, data, transform, std::vector<DimReq>{std::move(req1), std::move(req2), std::move(req3), std::move(req4)});
}
// Request full variable
template<class Data, class Transform> Error Read(const MString& vname, Data& data, Transform transform) const
{
auto ind = FindVar(vname);
if(ind < sources.size()) return sources[ind].Read(vname, data, transform);
return Error("NCZarrMultiCommon::Read", "Variable " + vname + " not found");
}
// Base function for all Read's
template<class Data, class Transform> Error Read(const MString& vname, Data& data, Transform transform, const std::vector<DimReqDef::DimReq>& reqs) const
{
auto ind = FindVar(vname);
if(ind < sources.size()) return sources[ind].Read(vname, data, transform, reqs);
return Error("NCZarrMultiCommon::Read", "Variable " + vname + " not found");
}
// Request by string argument
template<class Data, class Transform> Error Read(const MString& vname, Data& data, Transform transform, const MString& request) const
{
auto ind = FindVar(vname);
if(ind < sources.size()) return sources[ind].Read(vname, data, transform, request);
return Error("NCZarrMultiCommon::Read", "Variable " + vname + " not found");
}
// Request full one-dimensional variable
template<class Type> Error Read(const MString& vname, std::vector<Type>& out) const
{
auto ind = FindVar(vname);
if(ind < sources.size()) return sources[ind].Read(vname, out);
return Error("NCZarrMultiCommon::Read", "Variable " + vname + " not found");
}
};
#undef DEFFUNC
#undef DEFFUNCA
#undef DEFFUNC2
#undef DEFFUNCNAME
class NCMulti: public NCZarrMultiCommon<NCSimple>
{
public:
Error Open(const std::vector<MString>& names)
{
sources.clear();
decltype(sources) newsources;
static const MString pref = "NcMulti::Open";
if(names.size() == 0) return Error(pref, "empty file list");
for(const auto& name: names)
{
newsources.emplace_back();
auto ret = newsources.back().Open(name);
if(!ret) return ret.Add(pref, "Can't open file " + name);
}
sources = std::move(newsources);
return Error();
}
};
class ZarrMulti: public NCZarrMultiCommon<Zarr>
{
public:
Error Open(const MString& product, const std::vector<MString>& dsets, bool time = true)
{
sources.clear();
decltype(sources) newsources;
static const MString pref = "ZarrMulti::Open";
if(dsets.size() == 0) return Error(pref, "empty datasets list");
for(const auto& dset: dsets)
{
newsources.emplace_back();
auto ret = newsources.back().Open(product, dset, time);
if(!ret) return ret.Add(pref, "Can't open dataset " + dset);
}
sources = std::move(newsources);
return Error();
}
};
Write Preview
Loading…
Cancel
Save