#pragma once #include "merrors.h" #include #include using michlib::Error; using michlib::int1; using michlib::int2; using michlib::int4; using michlib::int8; using michlib::int_cast; using michlib::MString; using michlib::RetVal; using michlib::uint1; using michlib::uint8; class NcZarrTypes { protected: using AttVT = std::variant; class ArrCounter { using VT = std::vector; const VT count; VT ind; bool end; public: static size_t Index(const VT& i, const VT& c) { size_t out = 0; size_t mul = 1; for(size_t ii = i.size(); ii != 0; ii--) { out += mul * i[ii - 1]; mul *= c[ii - 1]; } return out; } static VT Index(size_t lind, const VT& c) { VT out(c.size()); size_t j = lind; for(auto i = c.size(); i > 0; i--) { out[i - 1] = j % c[i - 1]; j = j / c[i - 1]; } return out; } ArrCounter() = delete; ArrCounter(const VT& cnt): count(cnt), ind(cnt.size(), 0), end(false) {} size_t operator[](size_t i) const { return ind[i]; } ArrCounter& operator++() { size_t curind = count.size(); while(curind != 0) { ind[curind - 1]++; if(ind[curind - 1] >= count[curind - 1]) { ind[curind - 1] = 0; curind--; } else return *this; } ind = count; end = true; return *this; } explicit operator bool() const { return !end; } size_t Index() const { return Index(ind, count); } size_t Index(const VT& i) const { return Index(i, count); } VT Index(size_t lind) const { return Index(lind, count); } size_t Count(size_t i) const { return count[i]; } const VT& VIndex() const { return ind; } VT VIndex(const VT& start) const { VT out(ind.size()); for(size_t i = 0; i < ind.size(); i++) out[i] = ind[i] + start[i]; return out; } const auto& Count() const { return count; } size_t N() const { size_t out = 1; for(size_t i = 0; i < count.size(); i++) out *= count[i]; return out; } }; public: enum class AttType { UNDEF, INT, UINT, REAL, STRING, BOOL }; enum class VarType { UNDEF, FLOAT, DOUBLE, INT1, INT2, INT4, INT8, UINT1 }; protected: template struct VarType2Type; template struct VarType2Type { using type = float; }; template struct VarType2Type { using type = double; }; template struct VarType2Type { using type = int1; }; template struct VarType2Type { using type = int2; }; template struct VarType2Type { using type = int4; }; template struct VarType2Type { using type = int8; }; template struct VarType2Type { using type = uint1; }; template using Type = VarType2Type::type; static constexpr size_t SizeOf(VarType vt) { switch(vt) { case(VarType::UNDEF): return 0; case(VarType::FLOAT): return sizeof(Type); case(VarType::DOUBLE): return sizeof(Type); case(VarType::INT1): return sizeof(Type); case(VarType::INT2): return sizeof(Type); case(VarType::INT4): return sizeof(Type); case(VarType::INT8): return sizeof(Type); case(VarType::UINT1): return sizeof(Type); } return 0; } template static size_t FindInd(const MString& name, const std::vector& arr) { for(size_t i = 0; i < arr.size(); i++) if(arr[i].Name() == name) return i; return arr.size(); } class Attribute: public AttVT { MString name; public: Attribute(const MString& n, AttVT&& v): AttVT(std::move(v)), name(n) {} Attribute(const std::string& n, AttVT&& v): AttVT(std::move(v)), name(n.c_str(), n.size()) {} const MString& Name() const { return name; } AttType Type() const { if(std::holds_alternative(*this)) return AttType::INT; else if(std::holds_alternative(*this)) return AttType::UINT; else if(std::holds_alternative(*this)) return AttType::REAL; else if(std::holds_alternative(*this)) return AttType::STRING; else if(std::holds_alternative(*this)) return AttType::BOOL; return AttType::UNDEF; } int8 I() const { if(std::holds_alternative(*this)) return std::get(*this); else if(std::holds_alternative(*this)) return int_cast(std::get(*this)); else if(std::holds_alternative(*this)) return static_cast(std::get(*this)); else if(std::holds_alternative(*this)) return std::get(*this).ToInteger(); else if(std::holds_alternative(*this)) return std::get(*this) ? 1 : 0; return 0; } uint8 U() const { if(std::holds_alternative(*this)) return int_cast(std::get(*this)); else if(std::holds_alternative(*this)) return std::get(*this); else if(std::holds_alternative(*this)) return static_cast(std::get(*this)); else if(std::holds_alternative(*this)) return std::get(*this).ToInteger(); else if(std::holds_alternative(*this)) return std::get(*this) ? 1 : 0; return 0; } double D() const { if(std::holds_alternative(*this)) return std::get(*this); else if(std::holds_alternative(*this)) return std::get(*this); else if(std::holds_alternative(*this)) return std::get(*this); else if(std::holds_alternative(*this)) return michlib_internal::RealType::String2Real(std::get(*this).Buf()); else if(std::holds_alternative(*this)) return std::get(*this) ? 1 : 0; return 0; } MString S() const { if(std::holds_alternative(*this)) return MString().FromInt(std::get(*this)); else if(std::holds_alternative(*this)) return MString().FromUInt(std::get(*this)); else if(std::holds_alternative(*this)) return MString().FromReal(std::get(*this)); else if(std::holds_alternative(*this)) return std::get(*this); else if(std::holds_alternative(*this)) return MString().FromBool(std::get(*this)); return ""; } bool B() const { if(std::holds_alternative(*this)) return std::get(*this) != 0; else if(std::holds_alternative(*this)) return std::get(*this) != 0; else if(std::holds_alternative(*this)) return std::get(*this) != 0.0; else if(std::holds_alternative(*this)) return std::get(*this).ToBool(); else if(std::holds_alternative(*this)) return std::get(*this); return false; } }; class Dimension { MString name; size_t size; public: Dimension(const MString& str, size_t num): name(str), size(num) {} const MString& Name() const { return name; } size_t Size() const { return size; } }; class Variable { public: using FillType = std::variant; private: MString name; VarType type = VarType::UNDEF; std::vector dims; std::vector atts; FillType fill; public: Variable(const MString& name_, VarType type_, std::vector&& dims_, std::vector&& atts_, FillType fill_ = 0): name(name_), type(type_), dims(std::move(dims_)), atts(std::move(atts_)), fill(fill_) { } explicit operator bool() const { return type != VarType::UNDEF; } const auto& Dims() const { return dims; } size_t NDim() const { return dims.size(); } size_t NAtt() const { return atts.size(); } auto AttNames() const { std::vector out; std::transform(atts.cbegin(), atts.cend(), std::back_inserter(out), [](const Attribute& a) { return a.Name(); }); return out; } AttType AttT(const MString& name) const { size_t ind = FindInd(name, atts); return ind < atts.size() ? atts[ind].Type() : AttType::UNDEF; } int8 AttInt(const MString& name) const { size_t ind = FindInd(name, atts); return ind < atts.size() ? atts[ind].I() : 0; } uint8 AttUInt(const MString& name) const { size_t ind = FindInd(name, atts); return ind < atts.size() ? atts[ind].U() : 0; } double AttReal(const MString& name) const { size_t ind = FindInd(name, atts); return ind < atts.size() ? atts[ind].D() : 0.0; } MString AttString(const MString& name) const { size_t ind = FindInd(name, atts); return ind < atts.size() ? atts[ind].S() : MString(); } bool AttBool(const MString& name) const { size_t ind = FindInd(name, atts); return ind < atts.size() ? atts[ind].B() : false; } const MString& Name() const { return name; } auto Type() const { return type; } const auto& Fill() const { return fill; } }; protected: std::vector gats; std::vector dims; std::vector vars; public: explicit operator bool() const { return !vars.empty(); } size_t NDim() const { return dims.size(); } size_t NDim(const MString& var) const { size_t ind = FindInd(var, vars); return ind < vars.size() ? vars[ind].NDim() : 0; } size_t NAtt() const { return gats.size(); } auto AttNames() const { std::vector out; std::transform(gats.cbegin(), gats.cend(), std::back_inserter(out), [](const Attribute& a) { return a.Name(); }); return out; } size_t NAtt(const MString& var) const { if(!var.Exist()) return NAtt(); size_t ind = FindInd(var, vars); return ind < vars.size() ? vars[ind].NAtt() : 0; } auto AttNames(const MString& var) const { if(!var.Exist()) return AttNames(); size_t ind = FindInd(var, vars); return ind < vars.size() ? vars[ind].AttNames() : decltype(AttNames())(); } auto VarNames() const { std::vector out; std::transform(vars.cbegin(), vars.cend(), std::back_inserter(out), [](const Variable& v) { return v.Name(); }); return out; } VarType VarT(const MString& var) const { size_t ind = FindInd(var, vars); return ind < vars.size() ? vars[ind].Type() : VarType::UNDEF; } auto VarFill(const MString& var) const { size_t ind = FindInd(var, vars); return ind < vars.size() ? vars[ind].Fill() : Variable::FillType(); } auto DimNames() const { std::vector out; std::transform(dims.cbegin(), dims.cend(), std::back_inserter(out), [](const Dimension& d) { return d.Name(); }); return out; } auto DimNames(const MString& var) const { size_t ind = FindInd(var, vars); std::vector out; if(ind >= vars.size()) return out; auto vdims = vars[ind].Dims(); std::transform(vdims.cbegin(), vdims.cend(), std::back_inserter(out), [&dims = std::as_const(dims)](const size_t& i) { return dims[i].Name(); }); return out; } size_t DimSize(const MString& dim) const { size_t ind = FindInd(dim, dims); return ind < dims.size() ? dims[ind].Size() : 0; } AttType AttT(const MString& var, const MString& name) const { if(!var.Exist()) { size_t ind = FindInd(name, gats); return ind < gats.size() ? gats[ind].Type() : AttType::UNDEF; } size_t ind = FindInd(var, vars); return ind < vars.size() ? vars[ind].AttT(name) : AttType::UNDEF; } int8 AttInt(const MString& var, const MString& name) const { if(!var.Exist()) { size_t ind = FindInd(name, gats); return ind < gats.size() ? gats[ind].I() : 0; } size_t ind = FindInd(var, vars); return ind < vars.size() ? vars[ind].AttInt(name) : 0; } uint8 AttUInt(const MString& var, const MString& name) const { if(!var.Exist()) { size_t ind = FindInd(name, gats); return ind < gats.size() ? gats[ind].U() : 0; } size_t ind = FindInd(var, vars); return ind < vars.size() ? vars[ind].AttUInt(name) : 0; } double AttReal(const MString& var, const MString& name) const { if(!var.Exist()) { size_t ind = FindInd(name, gats); return ind < gats.size() ? gats[ind].D() : 0.0; } size_t ind = FindInd(var, vars); return ind < vars.size() ? vars[ind].AttReal(name) : 0.0; } MString AttString(const MString& var, const MString& name) const { if(!var.Exist()) { size_t ind = FindInd(name, gats); return ind < gats.size() ? gats[ind].S() : MString(); } size_t ind = FindInd(var, vars); return ind < vars.size() ? vars[ind].AttString(name) : MString(); } bool AttBool(const MString& var, const MString& name) const { if(!var.Exist()) { size_t ind = FindInd(name, gats); return ind < gats.size() ? gats[ind].B() : false; } size_t ind = FindInd(var, vars); return ind < vars.size() ? vars[ind].AttBool(name) : false; } auto AttT(const MString& name) const { return AttT("", name); } auto AttInt(const MString& name) const { return AttInt("", name); } auto AttUInt(const MString& name) const { return AttUInt("", name); } auto AttReal(const MString& name) const { return AttReal("", name); } auto AttString(const MString& name) const { return AttString("", name); } auto AttBool(const MString& name) const { return AttBool("", name); } bool HasDim(const MString& name) const { return FindInd(name, dims) < dims.size(); } bool HasVar(const MString& name) const { return FindInd(name, vars) < vars.size(); } bool HasAtt(const MString& vname, const MString& aname) const { return AttT(vname, aname) != AttType::UNDEF; } bool HasAtt(const MString& aname) const { return AttT(aname) != AttType::UNDEF; } const auto& Vars() const { return vars; } const auto& Dims() const { return dims; } }; class DimReqDef { protected: struct DimReq { static const auto fill = std::numeric_limits::max(); MString name; size_t beg, count; DimReq(): name(MString()), beg(fill), count(fill) {} DimReq(const char* n): name(n), beg(fill), count(fill) {} DimReq(const MString& n): name(n), beg(fill), count(fill) {} DimReq(MString&& n): name(std::move(n)), beg(fill), count(fill) {} DimReq(const char* n, size_t s): name(n), beg(s), count(fill) {} DimReq(const MString& n, size_t s): name(n), beg(s), count(fill) {} DimReq(MString&& n, size_t s): name(std::move(n)), beg(s), count(fill) {} DimReq(const char* n, size_t s, size_t c): name(n), beg(s), count(c) {} DimReq(const MString& n, size_t s, size_t c): name(n), beg(s), count(c) {} DimReq(MString&& n, size_t s, size_t c): name(std::move(n)), beg(s), count(c) {} const MString& Name() const { return name; } }; }; template class NcZarrRead: public C, public DimReqDef { template static constexpr size_t Dimensionity() { if constexpr(requires(Data& d) { d(0, 0, 0, 0); }) return 4; if constexpr(requires(Data& d) { d(0, 0, 0); }) return 3; if constexpr(requires(Data& d) { d(0, 0); }) return 2; if constexpr(requires(Data& d) { d(0); }) return 1; return 0; } template struct DataTypeExtractorS; template struct DataTypeExtractorS { using type = std::decay_t()(0))>; }; template struct DataTypeExtractorS { using type = std::decay_t()(0, 0))>; }; template struct DataTypeExtractorS { using type = std::decay_t()(0, 0, 0))>; }; template struct DataTypeExtractorS { using type = std::decay_t()(0, 0, 0, 0))>; }; template using DataTypeExtractor = DataTypeExtractorS()>::type; template Error Read(const MString& vname, const std::vector& transindex, Data& data, Transform transform, std::vector reqs) const { const size_t indim = reqs.size(); constexpr size_t outdim = Dimensionity(); std::vector start; std::vector count; start.resize(indim); count.resize(indim); for(size_t i = 0; i < indim; i++) { start[i] = reqs[i].beg; count[i] = reqs[i].count; } using DataType = DataTypeExtractor; DataType fillout; bool havefill = C::VarFill(vname).index() > 0; VType fillin = std::visit( [](auto v) { if constexpr(std::is_convertible_v) return static_cast(v); else return std::numeric_limits::max(); }, C::VarFill(vname)); if constexpr(requires(Data& d) { // Data have own fillvalue { d.Fillval() } -> std::convertible_to; }) fillout = data.Fillval(); else // Data does'nt have own fillvalue, using variable fillvalue fillout = static_cast(fillin); auto ret = C::template Read(vname, start.data(), count.data()); if(!ret) return ret; const auto& rawdata = ret.Value(); std::vector mul(indim, 1); for(size_t i = indim - 1; i > 0; i--) mul[i - 1] = mul[i] * count[i]; size_t inind = 0; for(typename C::ArrCounter i(count); i; ++i) { // TODO: Remove this testing block size_t cind = 0; for(size_t j = 0; j < indim; j++) cind += i[j] * mul[j]; if(cind != inind) return {"NcZarrRead::Read", "Internal error"}; if(i.Index() != inind) return {"NcZarrRead::Read", "Internal error"}; if(inind != i.Index(i.Index(inind, count), count)) return {"NcZarrRead::Read", "Internal error"}; DataType out; const VType& in = rawdata(inind); if(havefill && in == fillin) out = fillout; else out = transform(in); if constexpr(outdim == 1) data(i[transindex[0]]) = out; else if constexpr(outdim == 2) data(i[transindex[0]], i[transindex[1]]) = out; else if constexpr(outdim == 3) data(i[transindex[0]], i[transindex[1]], i[transindex[2]]) = out; else if constexpr(outdim == 4) data(i[transindex[0]], i[transindex[1]], i[transindex[2]], i[transindex[3]]) = out; inind++; } return Error(); } public: // Request is string template Error Read(const MString& vname, Data& data, Transform transform, const char* request) const { return Read(vname, data, transform, MString(request)); } // Request by one dimension template Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1) const { return Read(vname, data, transform, std::vector{std::move(req1)}); } // Request by two dimension template Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1, DimReq&& req2) const { return Read(vname, data, transform, std::vector{std::move(req1), std::move(req2)}); } // Request by three dimension template Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1, DimReq&& req2, DimReq&& req3) const { return Read(vname, data, transform, std::vector{std::move(req1), std::move(req2), std::move(req3)}); } // Request by four dimension template Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1, DimReq&& req2, DimReq&& req3, DimReq&& req4) const { return Read(vname, data, transform, std::vector{std::move(req1), std::move(req2), std::move(req3), std::move(req4)}); } // Request full variable template Error Read(const MString& vname, Data& data, Transform transform) const { static const MString pref = "NcZarrRead::Read"; if(!C::HasVar(vname)) return {pref, "Variable " + vname + " not found"}; std::vector pdims; const auto vdims = C::DimNames(vname); std::transform(vdims.cbegin(), vdims.cend(), std::back_inserter(pdims), [this](const MString& n) -> struct DimReq { return {n, 0, C::DimSize(n)}; }); return Read(vname, data, transform, pdims); } // Base function for all Read's template Error Read(const MString& vname, Data& data, Transform transform, const std::vector& reqs) const { static const MString pref = "NcZarrRead::Read"; if(!C::HasVar(vname)) return {pref, "Variable " + vname + " not found"}; std::vector pdims; { const auto vdims = C::DimNames(vname); std::transform(vdims.cbegin(), vdims.cend(), std::back_inserter(pdims), [](const MString& n) -> struct DimReq { return {n, 0, 1}; }); } std::vector transindex; // Parse request if(reqs.size() == 0) return {pref, "Empty request"}; for(const auto& req: reqs) { size_t ind = C::FindInd(req.name, pdims); if(ind >= pdims.size()) return {pref, "Variable " + vname + " has no dimension " + req.name}; for(size_t i = 0; i < transindex.size(); i++) if(transindex[i] == ind) return {pref, "Parameters for dimension " + req.name + " already defined"}; transindex.push_back(ind); size_t dlen = C::DimSize(pdims[ind].name); if(req.beg == req.fill && req.count == req.fill) // Only name, so, we request full length { pdims[ind].beg = 0; pdims[ind].count = dlen; } else if(req.count == req.fill) // Name and first index { pdims[ind].beg = req.beg; pdims[ind].count = 1; } else // Name, first index, count { pdims[ind].beg = req.beg; pdims[ind].count = req.count; } // Sanity checks if(pdims[ind].count <= 0) return {pref, "Error parsing request: count must be greter then zero"}; if(pdims[ind].beg >= dlen) return {pref, MString("Error parsing request: start index ") + pdims[ind].beg + " must be lesser then " + pdims[ind].name + " size " + dlen}; if(pdims[ind].beg + pdims[ind].count > dlen) return {pref, MString("Error parsing request: start index ") + pdims[ind].beg + " with count " + pdims[ind].count + " exceeds " + pdims[ind].name + " size " + dlen}; // Ignore hyperplanes in requests for calculation of data dimensionality if(pdims[transindex.back()].count == 1) transindex.pop_back(); } if(transindex.size() != Dimensionity()) return {pref, MString("Output data dimensions (") + Dimensionity() + ") not corresponding request dimensions (" + transindex.size() + ")"}; switch(C::VarT(vname)) { case(C::VarType::UNDEF): return {pref, "No variable with name " + vname + " (impossible)"}; case(C::VarType::FLOAT): return Read>(vname, transindex, data, transform, pdims); case(C::VarType::DOUBLE): return Read>(vname, transindex, data, transform, pdims); case(C::VarType::INT1): return Read>(vname, transindex, data, transform, pdims); case(C::VarType::INT2): return Read>(vname, transindex, data, transform, pdims); case(C::VarType::INT4): return Read>(vname, transindex, data, transform, pdims); case(C::VarType::INT8): return Read>(vname, transindex, data, transform, pdims); case(C::VarType::UINT1): return Read>(vname, transindex, data, transform, pdims); } return {pref, "Internal error (impossible)"}; } // Request by string argument template Error Read(const MString& vname, Data& data, Transform transform, const MString& request) const { static const MString pref = "NcZarrRead::Read"; std::vector pdims; // Parse request const auto dimdesc = request.Split(";, \t"); if(dimdesc.size() == 0) return {pref, "Empty request"}; for(const auto& dd: dimdesc) { const auto dimpar = dd.Split(":", true); if(dimpar.size() == 1) // Only name, so, we request full length pdims.emplace_back(dimpar[0]); else if(dimpar.size() == 2) // Name and first index pdims.emplace_back(dimpar[0], dimpar[1].ToInteger()); else if(dimpar.size() == 3) // Name, first index, count pdims.emplace_back(dimpar[0], dimpar[1].ToInteger(), dimpar[2].ToInteger()); else return {pref, "Can't parse expression " + dd}; } return Read(vname, data, transform, pdims); } // Request full one-dimensional variable template Error Read(const MString& vname, std::vector& out) const { const auto& dnames = C::DimNames(vname); if(dnames.size() > 0) out.resize(C::DimSize(dnames[0])); auto data = [&vec = out](size_t i) -> Type& { return vec[i]; }; return Read(vname, data, std::identity()); } };