From 7c3c254b7221b6cfa1025f7509c8573e16148ce4 Mon Sep 17 00:00:00 2001 From: Michael Uleysky Date: Fri, 14 Jun 2024 16:00:11 +1000 Subject: [PATCH] Support for multiple open files or datasets in the NCZarr interface. --- include/nczarr.h | 31 ++++++- include/nczarrmulti.h | 206 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 234 insertions(+), 3 deletions(-) create mode 100644 include/nczarrmulti.h diff --git a/include/nczarr.h b/include/nczarr.h index 50886c5..555f30a 100644 --- a/include/nczarr.h +++ b/include/nczarr.h @@ -1,9 +1,8 @@ #pragma once -#include "ncsimple.h" -#include "zarr.h" +#include "nczarrmulti.h" #include -using NCZarrBase = std::variant; +using NCZarrBase = std::variant; #define DEFFUNC(NAME) \ auto NAME() const \ @@ -134,6 +133,32 @@ class NCZarr: private NCZarrBase, private DimReqDef if(p == nullptr) return Error("NCZarr::OpenNC", "Impossible error!"); return p->Open(product, dataset, time); } + + Error OpenMultiNC(const std::vector& names) + { + if(names.size() == 0) return Error("NCZarr::OpenMultiNC", "no names"); + if(names.size() == 1) return OpenNC(names[0]); + + auto pv = static_cast(this); + + *pv = NCZarrBase(NCMulti()); + auto p = std::get_if(pv); + if(p == nullptr) return Error("NCZarr::OpenMultiNC", "Impossible error!"); + return p->Open(names); + } + + Error OpenMultiZarr(const MString& product, const std::vector& dsets, bool time = true) + { + if(dsets.size() == 0) return Error("NCZarr::OpenMultiZarr", "no datasets"); + if(dsets.size() == 1) return OpenZarr(product, dsets[0], time); + + auto pv = static_cast(this); + + *pv = NCZarrBase(ZarrMulti()); + auto p = std::get_if(pv); + if(p == nullptr) return Error("NCZarr::OpenMultiZarr", "Impossible error!"); + return p->Open(product, dsets, time); + } }; #undef DEFFUNC diff --git a/include/nczarrmulti.h b/include/nczarrmulti.h new file mode 100644 index 0000000..5cf92f4 --- /dev/null +++ b/include/nczarrmulti.h @@ -0,0 +1,206 @@ +#pragma once +#include "ncsimple.h" +#include "zarr.h" +#include + +#define DEFFUNC(func, def) \ + auto func(const MString& var) const \ + { \ + const auto ind = FindVar(var); \ + return ind < sources.size() ? sources[ind].func(var) : (def); \ + } +#define DEFFUNCA(func, def) \ + auto func(const MString& att) const \ + { \ + const auto ind = FindAtt(att); \ + return ind < sources.size() ? sources[ind].func(att) : (def); \ + } +#define DEFFUNC2(func, def) \ + auto func(const MString& var, const MString& name) const \ + { \ + const auto ind = FindVar(var); \ + return ind < sources.size() ? sources[ind].func(var, name) : (def); \ + } +#define DEFFUNCNAMES(func) \ + auto func() const \ + { \ + std::set names; \ + for(const auto& s: sources) \ + { \ + auto snames = s.func(); \ + for(const auto& name: snames) names.insert(name); \ + } \ + std::vector out; \ + for(const auto& name: names) out.push_back(name); \ + return out; \ + } + +template class NCZarrMultiCommon: private DimReqDef +{ + protected: + std::vector sources; + + private: + size_t FindVar(const MString& var) const + { + for(size_t i = 0; i < sources.size(); i++) + if(sources[i].HasVar(var)) return i; + return sources.size(); + } + + size_t FindAtt(const MString& att) const + { + for(size_t i = 0; i < sources.size(); i++) + if(sources[i].HasAtt(att)) return i; + return sources.size(); + } + + public: + explicit operator bool() const + { + if(sources.empty()) return false; + for(const auto& s: sources) + if(!s) return false; + return true; + } + + size_t NDim() const { return DimNames().size(); } + DEFFUNC(NDim, 0) + + size_t NAtt() const { return AttNames().size(); } + DEFFUNC(NAtt, 0) + + DEFFUNCNAMES(AttNames) + DEFFUNC(AttNames, decltype(AttNames())()) + + DEFFUNCNAMES(VarNames) + DEFFUNC(VarT, T::VarType::UNDEF) + DEFFUNC(VarFill, decltype(T().VarFill(MString()))()) + + DEFFUNC(DimNames, decltype(T().DimNames(MString()))()) + + DEFFUNC2(DimSize, 0) + + DEFFUNC2(AttT, T::AttType::UNDEF) + DEFFUNC2(AttInt, 0) + DEFFUNC2(AttUInt, 0) + DEFFUNC2(AttReal, 0.0) + DEFFUNC2(AttString, MString()) + DEFFUNC2(AttBool, false) + + DEFFUNCA(AttT, T::AttType::UNDEF) + DEFFUNCA(AttInt, 0) + DEFFUNCA(AttUInt, 0) + DEFFUNCA(AttReal, 0.0) + DEFFUNCA(AttString, MString()) + DEFFUNCA(AttBool, false) + + DEFFUNC2(HasDim, false) + DEFFUNC(HasVar, false) + DEFFUNCA(HasAtt, false) + DEFFUNC2(HasAtt, false) + + // Request is string + template Error Read(const MString& vname, Data& data, Transform transform, const char* request) const + { + return Read(vname, data, transform, MString(request)); + } + + // Request by one dimension + template Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1) const + { + return Read(vname, data, transform, std::vector{std::move(req1)}); + } + // Request by two dimension + template Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1, DimReq&& req2) const + { + return Read(vname, data, transform, std::vector{std::move(req1), std::move(req2)}); + } + // Request by three dimension + template Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1, DimReq&& req2, DimReq&& req3) const + { + return Read(vname, data, transform, std::vector{std::move(req1), std::move(req2), std::move(req3)}); + } + // Request by four dimension + template Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1, DimReq&& req2, DimReq&& req3, DimReq&& req4) const + { + return Read(vname, data, transform, std::vector{std::move(req1), std::move(req2), std::move(req3), std::move(req4)}); + } + + // Request full variable + template Error Read(const MString& vname, Data& data, Transform transform) const + { + auto ind = FindVar(vname); + if(ind < sources.size()) return sources[ind].Read(vname, data, transform); + return Error("NCZarrMultiCommon::Read", "Variable " + vname + " not found"); + } + + // Base function for all Read's + template Error Read(const MString& vname, Data& data, Transform transform, const std::vector& reqs) const + { + auto ind = FindVar(vname); + if(ind < sources.size()) return sources[ind].Read(vname, data, transform, reqs); + return Error("NCZarrMultiCommon::Read", "Variable " + vname + " not found"); + } + + // Request by string argument + template Error Read(const MString& vname, Data& data, Transform transform, const MString& request) const + { + auto ind = FindVar(vname); + if(ind < sources.size()) return sources[ind].Read(vname, data, transform, request); + return Error("NCZarrMultiCommon::Read", "Variable " + vname + " not found"); + } + + // Request full one-dimensional variable + template Error Read(const MString& vname, std::vector& out) const + { + auto ind = FindVar(vname); + if(ind < sources.size()) return sources[ind].Read(vname, out); + return Error("NCZarrMultiCommon::Read", "Variable " + vname + " not found"); + } +}; + +#undef DEFFUNC +#undef DEFFUNCA +#undef DEFFUNC2 +#undef DEFFUNCNAME + +class NCMulti: public NCZarrMultiCommon +{ + public: + Error Open(const std::vector& names) + { + sources.clear(); + decltype(sources) newsources; + static const MString pref = "NcMulti::Open"; + if(names.size() == 0) return Error(pref, "empty file list"); + for(const auto& name: names) + { + newsources.emplace_back(); + auto ret = newsources.back().Open(name); + if(!ret) return ret.Add(pref, "Can't open file " + name); + } + sources = std::move(newsources); + return Error(); + } +}; + +class ZarrMulti: public NCZarrMultiCommon +{ + public: + Error Open(const MString& product, const std::vector& dsets, bool time = true) + { + sources.clear(); + decltype(sources) newsources; + static const MString pref = "ZarrMulti::Open"; + if(dsets.size() == 0) return Error(pref, "empty datasets list"); + for(const auto& dset: dsets) + { + newsources.emplace_back(); + auto ret = newsources.back().Open(product, dset, time); + if(!ret) return ret.Add(pref, "Can't open dataset " + dset); + } + sources = std::move(newsources); + return Error(); + } +};