From 01af4109c196ec816587b055ef0a785d2d1954bc Mon Sep 17 00:00:00 2001 From: Michael Uleysky Date: Tue, 2 Apr 2024 14:10:59 +1000 Subject: [PATCH] ZARR-compatible interface to the netcdf files --- include/ncsimple.h | 87 +++++++++++++++++ src/ncsimple.cpp | 231 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 318 insertions(+) create mode 100644 include/ncsimple.h create mode 100644 src/ncsimple.cpp diff --git a/include/ncsimple.h b/include/ncsimple.h new file mode 100644 index 0000000..ef93a1a --- /dev/null +++ b/include/ncsimple.h @@ -0,0 +1,87 @@ +#pragma once +#include "nczarrcommon.h" +#include +#include + +class NCSimpleTypes: public NcZarrTypes +{ + protected: + template class ReadedData + { + using Vec = std::vector; + + private: + std::unique_ptr data; + + public: + ReadedData() = default; + + ReadedData(std::unique_ptr&& d): data(std::move(d)) {} + + VType operator()(size_t lini) const { return data[lini]; } + }; +}; + +class NCSimpleFunctions: public NCSimpleTypes +{ + int ncid; + + RetVal> ReadAtts(int vid) const; + + protected: + NCSimpleFunctions(): ncid(0) {} + + template RetVal> Read(const MString& var, const size_t* start, const size_t* count) const + { + static const MString pref = "NCSimpleFunctions::Read"; + + size_t ind = FindInd(var, vars); + const size_t N = vars[ind].NDim(); + + std::unique_ptr cdata; + + size_t dsize = 1; + for(size_t i = 0; i < N; i++) dsize *= count[i]; + cdata.reset(new VType[dsize]); + + int vid; + int res = nc_inq_varid(ncid, var.Buf(), &vid); + if(res != NC_NOERR) return Error(pref, MString("nc_inq_varid error: ") + nc_strerror(res)); + + if constexpr(std::is_same_v) + res = nc_get_vara_float(ncid, vid, start, count, cdata.get()); + else if constexpr(std::is_same_v) + res = nc_get_vara_double(ncid, vid, start, count, cdata.get()); + else if constexpr(std::is_same_v) + res = nc_get_vara_int(ncid, vid, start, count, cdata.get()); + else if constexpr(std::is_same_v) + res = nc_get_vara_long(ncid, vid, start, count, cdata.get()); + else if constexpr(std::is_same_v) + res = nc_get_vara_longlong(ncid, vid, start, count, cdata.get()); + else if constexpr(std::is_same_v) + res = nc_get_vara_short(ncid, vid, start, count, cdata.get()); + else if constexpr(std::is_same_v) + res = nc_get_vara_schar(ncid, vid, start, count, cdata.get()); + else if constexpr(std::is_same_v) + res = nc_get_vara_uint(ncid, vid, start, count, cdata.get()); + else if constexpr(std::is_same_v) + res = nc_get_vara_ulonglong(ncid, vid, start, count, cdata.get()); + else if constexpr(std::is_same_v) + res = nc_get_vara_ushort(ncid, vid, start, count, cdata.get()); + else if constexpr(std::is_same_v) + res = nc_get_vara_ubyte(ncid, vid, start, count, cdata.get()); + else + return Error(pref, "Unsupported variable type"); + + if(res != NC_NOERR) return Error(pref, MString("nc_get_vara error: ") + nc_strerror(res)); + + return ReadedData(std::move(cdata)); + } + + public: + ~NCSimpleFunctions() { nc_close(ncid); } + + Error Open(const MString& filename); +}; + +using NCSimple = NcZarrRead; diff --git a/src/ncsimple.cpp b/src/ncsimple.cpp new file mode 100644 index 0000000..86e1c72 --- /dev/null +++ b/src/ncsimple.cpp @@ -0,0 +1,231 @@ +#define MICHLIB_NOSOURCE +#include "ncsimple.h" + +RetVal> NCSimpleFunctions::ReadAtts(int vid) const +{ + static const MString pref = "NCSimple::ReadAtts"; + + int natt; + int ret; + if(vid == NC_GLOBAL) + ret = nc_inq_natts(ncid, &natt); + else + ret = nc_inq_var(ncid, vid, nullptr, nullptr, nullptr, nullptr, &natt); + if(ret != NC_NOERR) return Error(pref, MString("Can't inquire number of attributes: ") + nc_strerror(ret)); + + std::vector out; + char name[NC_MAX_NAME + 1]; + + for(int aid = 0; aid < natt; aid++) + { + nc_type type; + size_t len; + + ret = nc_inq_attname(ncid, vid, aid, name); + if(ret != NC_NOERR) return Error(pref, MString("Can't inquire attribute name: ") + nc_strerror(ret)); + ret = nc_inq_atttype(ncid, vid, name, &type); + if(ret != NC_NOERR) return Error(pref, MString("Can't inquire attribute type: ") + nc_strerror(ret)); + ret = nc_inq_attlen(ncid, vid, name, &len); + if(ret != NC_NOERR) return Error(pref, MString("Can't inquire attribute length: ") + nc_strerror(ret)); + + if(type == NC_DOUBLE || type == NC_FLOAT) + { + if(len == 1) + { + double d; + ret = nc_get_att_double(ncid, vid, name, &d); + if(ret != NC_NOERR) return Error(pref, MString("Can't read attribute ") + name + ": " + nc_strerror(ret)); + out.emplace_back(MString(name), d); + } + else + { + std::vector dd(len); + ret = nc_get_att_double(ncid, vid, name, dd.data()); + if(ret != NC_NOERR) return Error(pref, MString("Can't read attribute ") + name + ": " + nc_strerror(ret)); + for(size_t i = 0; i < dd.size(); i++) out.emplace_back(MString(name) + "[" + i + "]", dd[i]); + } + } + else if(type == NC_BYTE || type == NC_SHORT || type == NC_INT || type == NC_INT64) + { + if(len == 1) + { + long long i; + ret = nc_get_att_longlong(ncid, vid, name, &i); + if(ret != NC_NOERR) return Error(pref, MString("Can't read attribute ") + name + ": " + nc_strerror(ret)); + out.emplace_back(MString(name), int_cast(i)); + } + else + { + std::vector ii(len); + ret = nc_get_att_longlong(ncid, vid, name, ii.data()); + if(ret != NC_NOERR) return Error(pref, MString("Can't read attribute ") + name + ": " + nc_strerror(ret)); + for(size_t i = 0; i < ii.size(); i++) out.emplace_back(MString(name) + "[" + i + "]", int_cast(ii[i])); + } + } + else if(type == NC_UBYTE || type == NC_USHORT || type == NC_UINT || type == NC_UINT64) + { + if(len == 1) + { + unsigned long long u; + ret = nc_get_att_ulonglong(ncid, vid, name, &u); + if(ret != NC_NOERR) return Error(pref, MString("Can't read attribute ") + name + ": " + nc_strerror(ret)); + out.emplace_back(MString(name), int_cast(u)); + } + else + { + std::vector uu(len); + ret = nc_get_att_ulonglong(ncid, vid, name, uu.data()); + if(ret != NC_NOERR) return Error(pref, MString("Can't read attribute ") + name + ": " + nc_strerror(ret)); + for(size_t i = 0; i < uu.size(); i++) out.emplace_back(MString(name) + "[" + i + "]", int_cast(uu[i])); + } + } + else if(type == NC_CHAR) + { + std::vector ss(len + 1, 0); + ret = nc_get_att_text(ncid, vid, name, ss.data()); + if(ret != NC_NOERR) return Error(pref, MString("Can't read attribute ") + name + ": " + nc_strerror(ret)); + out.emplace_back(MString(name), MString(ss.data())); + } + else + return Error(pref, MString("Unsupported type of attribute ") + name); + + // Ignore all other types + } + + return out; +} + +Error NCSimpleFunctions::Open(const MString& filename) +{ + static const MString pref = "NCSimple::Open"; + + // Cleanup + gats.clear(); + dims.clear(); + vars.clear(); + + nc_close(ncid); + + std::vector newgats; + std::vector newdims; + std::vector newvars; + + char name[NC_MAX_NAME + 1]; + + // Open + { + auto ret = nc_open(filename.Buf(), 0, &ncid); + if(ret != NC_NOERR) return Error(pref, "Can't open file " + filename + ": " + nc_strerror(ret)); + } + + // Dimensions + { + int ndim; + auto ret = nc_inq_dimids(ncid, &ndim, nullptr, 1); + if(ret != NC_NOERR) return Error(pref, "Can't inquire number of dimensions in file " + filename + ": " + nc_strerror(ret)); + std::vector dimids(ndim); + ret = nc_inq_dimids(ncid, nullptr, dimids.data(), 1); + if(ret != NC_NOERR) return Error(pref, "Can't inquire dimension ids in file " + filename + ": " + nc_strerror(ret)); + + size_t len; + for(const auto id: dimids) + { + ret = nc_inq_dim(ncid, id, name, &len); + if(ret != NC_NOERR) return Error(pref, "Can't inquire dimension name and size in file " + filename + ": " + nc_strerror(ret)); + newdims.emplace_back(name, len); + } + } + + // Global attributes + { + auto ret = ReadAtts(NC_GLOBAL); + if(!ret) return ret.Add(pref, "Can't read global attributes in file " + filename); + newgats = std::move(ret.Value()); + } + + // Variables + { + int nvar; + auto ret = nc_inq_varids(ncid, &nvar, nullptr); + if(ret != NC_NOERR) return Error(pref, "Can't inquire number of variables in file " + filename + ": " + nc_strerror(ret)); + std::vector varids(nvar); + ret = nc_inq_varids(ncid, nullptr, varids.data()); + if(ret != NC_NOERR) return Error(pref, "Can't inquire variables ids in file " + filename + ": " + nc_strerror(ret)); + + for(const auto vid: varids) + { + nc_type nctype; + int ndim; + + auto ret = nc_inq_var(ncid, vid, name, &nctype, &ndim, nullptr, nullptr); + if(ret != NC_NOERR) return Error(pref, "Can't inquire variable info in file " + filename + ": " + nc_strerror(ret)); + + VarType vt = VarType::UNDEF; + + if(nctype == NC_FLOAT) vt = VarType::FLOAT; + if(nctype == NC_DOUBLE) vt = VarType::DOUBLE; + if(nctype == NC_BYTE) vt = VarType::INT1; + if(nctype == NC_SHORT) vt = VarType::INT2; + if(nctype == NC_INT) vt = VarType::INT4; + if(nctype == NC_INT64) vt = VarType::INT8; + if(nctype == NC_UBYTE) vt = VarType::UINT1; + + if(vt == VarType::UNDEF) return Error(pref, "Unsupported type of variable " + MString(name) + " in file " + filename); + + std::vector dimids(ndim); + ret = nc_inq_vardimid(ncid, vid, dimids.data()); + if(ret != NC_NOERR) return Error(pref, "Can't inquire variable dimensions in file " + filename + ": " + nc_strerror(ret)); + + std::vector dims; + char dname[NC_MAX_NAME + 1]; + for(const auto did: dimids) + { + auto ret = nc_inq_dimname(ncid, did, dname); + if(ret != NC_NOERR) return Error(pref, "Can't inquire dimension name in file " + filename + ": " + nc_strerror(ret)); + size_t ind = newdims.size(); + for(size_t i = 0; i < newdims.size() && ind == newdims.size(); i++) + if(dname == newdims[i].Name()) ind = i; + if(ind == newdims.size()) return Error(pref, "Can't find dimension " + MString(dname) + " of variable " + name + " in file " + filename); + dims.push_back(ind); + } + + auto atts = ReadAtts(vid); + const char fname[] = "_FillValue"; + if(!atts) return Error(pref, "Can't get attributes of variable " + MString(name) + " in file " + filename); + std::vector vatts = std::move(atts.Value()); + + Variable::FillType fill; + if(FindInd(fname, vatts) != vatts.size()) + { + if(nctype == NC_FLOAT || nctype == NC_DOUBLE) + { + double d; + auto ret = nc_get_att_double(ncid, vid, fname, &d); + if(ret != NC_NOERR) return Error(pref, "Can't get fill value for the variable " + MString(name) + " in file " + filename + ": " + nc_strerror(ret)); + fill = d; + } + if(nctype == NC_BYTE || nctype == NC_SHORT || nctype == NC_INT || nctype == NC_INT64) + { + long long l; + auto ret = nc_get_att_longlong(ncid, vid, fname, &l); + if(ret != NC_NOERR) return Error(pref, "Can't get fill value for the variable " + MString(name) + " in file " + filename + ": " + nc_strerror(ret)); + fill = int_cast(l); + } + if(nctype == NC_UBYTE || nctype == NC_USHORT || nctype == NC_UINT || nctype == NC_UINT64) + { + unsigned long long u; + auto ret = nc_get_att_ulonglong(ncid, vid, fname, &u); + if(ret != NC_NOERR) return Error(pref, "Can't get fill value for the variable " + MString(name) + " in file " + filename + ": " + nc_strerror(ret)); + fill = int_cast(u); + } + } + + newvars.emplace_back(name, vt, std::move(dims), std::move(vatts), fill); + } + } + + gats = std::move(newgats); + dims = std::move(newdims); + vars = std::move(newvars); + return Error(); +}