Browse Source

Support for multiple open files or datasets in the NCZarr interface.

lintest
Michael Uleysky 5 months ago
parent
commit
7c3c254b72
  1. 31
      include/nczarr.h
  2. 206
      include/nczarrmulti.h

31
include/nczarr.h

@ -1,9 +1,8 @@
#pragma once
#include "ncsimple.h"
#include "zarr.h"
#include "nczarrmulti.h"
#include <variant>
using NCZarrBase = std::variant<Zarr, NCSimple>;
using NCZarrBase = std::variant<Zarr, NCSimple, ZarrMulti, NCMulti>;
#define DEFFUNC(NAME) \
auto NAME() const \
@ -134,6 +133,32 @@ class NCZarr: private NCZarrBase, private DimReqDef
if(p == nullptr) return Error("NCZarr::OpenNC", "Impossible error!");
return p->Open(product, dataset, time);
}
Error OpenMultiNC(const std::vector<MString>& names)
{
if(names.size() == 0) return Error("NCZarr::OpenMultiNC", "no names");
if(names.size() == 1) return OpenNC(names[0]);
auto pv = static_cast<NCZarrBase*>(this);
*pv = NCZarrBase(NCMulti());
auto p = std::get_if<NCMulti>(pv);
if(p == nullptr) return Error("NCZarr::OpenMultiNC", "Impossible error!");
return p->Open(names);
}
Error OpenMultiZarr(const MString& product, const std::vector<MString>& dsets, bool time = true)
{
if(dsets.size() == 0) return Error("NCZarr::OpenMultiZarr", "no datasets");
if(dsets.size() == 1) return OpenZarr(product, dsets[0], time);
auto pv = static_cast<NCZarrBase*>(this);
*pv = NCZarrBase(ZarrMulti());
auto p = std::get_if<ZarrMulti>(pv);
if(p == nullptr) return Error("NCZarr::OpenMultiZarr", "Impossible error!");
return p->Open(product, dsets, time);
}
};
#undef DEFFUNC

206
include/nczarrmulti.h

@ -0,0 +1,206 @@
#pragma once
#include "ncsimple.h"
#include "zarr.h"
#include <set>
#define DEFFUNC(func, def) \
auto func(const MString& var) const \
{ \
const auto ind = FindVar(var); \
return ind < sources.size() ? sources[ind].func(var) : (def); \
}
#define DEFFUNCA(func, def) \
auto func(const MString& att) const \
{ \
const auto ind = FindAtt(att); \
return ind < sources.size() ? sources[ind].func(att) : (def); \
}
#define DEFFUNC2(func, def) \
auto func(const MString& var, const MString& name) const \
{ \
const auto ind = FindVar(var); \
return ind < sources.size() ? sources[ind].func(var, name) : (def); \
}
#define DEFFUNCNAMES(func) \
auto func() const \
{ \
std::set<MString> names; \
for(const auto& s: sources) \
{ \
auto snames = s.func(); \
for(const auto& name: snames) names.insert(name); \
} \
std::vector<MString> out; \
for(const auto& name: names) out.push_back(name); \
return out; \
}
template<class T> class NCZarrMultiCommon: private DimReqDef
{
protected:
std::vector<T> sources;
private:
size_t FindVar(const MString& var) const
{
for(size_t i = 0; i < sources.size(); i++)
if(sources[i].HasVar(var)) return i;
return sources.size();
}
size_t FindAtt(const MString& att) const
{
for(size_t i = 0; i < sources.size(); i++)
if(sources[i].HasAtt(att)) return i;
return sources.size();
}
public:
explicit operator bool() const
{
if(sources.empty()) return false;
for(const auto& s: sources)
if(!s) return false;
return true;
}
size_t NDim() const { return DimNames().size(); }
DEFFUNC(NDim, 0)
size_t NAtt() const { return AttNames().size(); }
DEFFUNC(NAtt, 0)
DEFFUNCNAMES(AttNames)
DEFFUNC(AttNames, decltype(AttNames())())
DEFFUNCNAMES(VarNames)
DEFFUNC(VarT, T::VarType::UNDEF)
DEFFUNC(VarFill, decltype(T().VarFill(MString()))())
DEFFUNC(DimNames, decltype(T().DimNames(MString()))())
DEFFUNC2(DimSize, 0)
DEFFUNC2(AttT, T::AttType::UNDEF)
DEFFUNC2(AttInt, 0)
DEFFUNC2(AttUInt, 0)
DEFFUNC2(AttReal, 0.0)
DEFFUNC2(AttString, MString())
DEFFUNC2(AttBool, false)
DEFFUNCA(AttT, T::AttType::UNDEF)
DEFFUNCA(AttInt, 0)
DEFFUNCA(AttUInt, 0)
DEFFUNCA(AttReal, 0.0)
DEFFUNCA(AttString, MString())
DEFFUNCA(AttBool, false)
DEFFUNC2(HasDim, false)
DEFFUNC(HasVar, false)
DEFFUNCA(HasAtt, false)
DEFFUNC2(HasAtt, false)
// Request is string
template<class Data, class Transform> Error Read(const MString& vname, Data& data, Transform transform, const char* request) const
{
return Read(vname, data, transform, MString(request));
}
// Request by one dimension
template<class Data, class Transform> Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1) const
{
return Read(vname, data, transform, std::vector<DimReq>{std::move(req1)});
}
// Request by two dimension
template<class Data, class Transform> Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1, DimReq&& req2) const
{
return Read(vname, data, transform, std::vector<DimReq>{std::move(req1), std::move(req2)});
}
// Request by three dimension
template<class Data, class Transform> Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1, DimReq&& req2, DimReq&& req3) const
{
return Read(vname, data, transform, std::vector<DimReq>{std::move(req1), std::move(req2), std::move(req3)});
}
// Request by four dimension
template<class Data, class Transform> Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1, DimReq&& req2, DimReq&& req3, DimReq&& req4) const
{
return Read(vname, data, transform, std::vector<DimReq>{std::move(req1), std::move(req2), std::move(req3), std::move(req4)});
}
// Request full variable
template<class Data, class Transform> Error Read(const MString& vname, Data& data, Transform transform) const
{
auto ind = FindVar(vname);
if(ind < sources.size()) return sources[ind].Read(vname, data, transform);
return Error("NCZarrMultiCommon::Read", "Variable " + vname + " not found");
}
// Base function for all Read's
template<class Data, class Transform> Error Read(const MString& vname, Data& data, Transform transform, const std::vector<DimReqDef::DimReq>& reqs) const
{
auto ind = FindVar(vname);
if(ind < sources.size()) return sources[ind].Read(vname, data, transform, reqs);
return Error("NCZarrMultiCommon::Read", "Variable " + vname + " not found");
}
// Request by string argument
template<class Data, class Transform> Error Read(const MString& vname, Data& data, Transform transform, const MString& request) const
{
auto ind = FindVar(vname);
if(ind < sources.size()) return sources[ind].Read(vname, data, transform, request);
return Error("NCZarrMultiCommon::Read", "Variable " + vname + " not found");
}
// Request full one-dimensional variable
template<class Type> Error Read(const MString& vname, std::vector<Type>& out) const
{
auto ind = FindVar(vname);
if(ind < sources.size()) return sources[ind].Read(vname, out);
return Error("NCZarrMultiCommon::Read", "Variable " + vname + " not found");
}
};
#undef DEFFUNC
#undef DEFFUNCA
#undef DEFFUNC2
#undef DEFFUNCNAME
class NCMulti: public NCZarrMultiCommon<NCSimple>
{
public:
Error Open(const std::vector<MString>& names)
{
sources.clear();
decltype(sources) newsources;
static const MString pref = "NcMulti::Open";
if(names.size() == 0) return Error(pref, "empty file list");
for(const auto& name: names)
{
newsources.emplace_back();
auto ret = newsources.back().Open(name);
if(!ret) return ret.Add(pref, "Can't open file " + name);
}
sources = std::move(newsources);
return Error();
}
};
class ZarrMulti: public NCZarrMultiCommon<Zarr>
{
public:
Error Open(const MString& product, const std::vector<MString>& dsets, bool time = true)
{
sources.clear();
decltype(sources) newsources;
static const MString pref = "ZarrMulti::Open";
if(dsets.size() == 0) return Error(pref, "empty datasets list");
for(const auto& dset: dsets)
{
newsources.emplace_back();
auto ret = newsources.back().Open(product, dset, time);
if(!ret) return ret.Add(pref, "Can't open dataset " + dset);
}
sources = std::move(newsources);
return Error();
}
};
Loading…
Cancel
Save