michael_ul
/
odm
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

ZARR-compatible interface to the netcdf files

lintest
Michael Uleysky 8 months ago
parent
8a4fbea40a
commit
01af4109c1
2 changed files with 318 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 87
      include/ncsimple.h
  2. 231
      src/ncsimple.cpp

87
include/ncsimple.h
Unescape View File

@ -0,0 +1,87 @@
#pragma once
#include "nczarrcommon.h"
#include <netcdf.h>
#include <variant>
class NCSimpleTypes: public NcZarrTypes
{
protected:
template<class VType> class ReadedData
{
using Vec = std::vector<size_t>;
private:
std::unique_ptr<VType[]> data;
public:
ReadedData() = default;
ReadedData(std::unique_ptr<VType[]>&& d): data(std::move(d)) {}
VType operator()(size_t lini) const { return data[lini]; }
};
};
class NCSimpleFunctions: public NCSimpleTypes
{
int ncid;
RetVal<std::vector<Attribute>> ReadAtts(int vid) const;
protected:
NCSimpleFunctions(): ncid(0) {}
template<class VType> RetVal<ReadedData<VType>> Read(const MString& var, const size_t* start, const size_t* count) const
{
static const MString pref = "NCSimpleFunctions::Read";
size_t ind = FindInd(var, vars);
const size_t N = vars[ind].NDim();
std::unique_ptr<VType[]> cdata;
size_t dsize = 1;
for(size_t i = 0; i < N; i++) dsize *= count[i];
cdata.reset(new VType[dsize]);
int vid;
int res = nc_inq_varid(ncid, var.Buf(), &vid);
if(res != NC_NOERR) return Error(pref, MString("nc_inq_varid error: ") + nc_strerror(res));
if constexpr(std::is_same_v<VType, float>)
res = nc_get_vara_float(ncid, vid, start, count, cdata.get());
else if constexpr(std::is_same_v<VType, double>)
res = nc_get_vara_double(ncid, vid, start, count, cdata.get());
else if constexpr(std::is_same_v<VType, int>)
res = nc_get_vara_int(ncid, vid, start, count, cdata.get());
else if constexpr(std::is_same_v<VType, long>)
res = nc_get_vara_long(ncid, vid, start, count, cdata.get());
else if constexpr(std::is_same_v<VType, long long>)
res = nc_get_vara_longlong(ncid, vid, start, count, cdata.get());
else if constexpr(std::is_same_v<VType, short>)
res = nc_get_vara_short(ncid, vid, start, count, cdata.get());
else if constexpr(std::is_same_v<VType, signed char>)
res = nc_get_vara_schar(ncid, vid, start, count, cdata.get());
else if constexpr(std::is_same_v<VType, unsigned int>)
res = nc_get_vara_uint(ncid, vid, start, count, cdata.get());
else if constexpr(std::is_same_v<VType, unsigned long long>)
res = nc_get_vara_ulonglong(ncid, vid, start, count, cdata.get());
else if constexpr(std::is_same_v<VType, unsigned short>)
res = nc_get_vara_ushort(ncid, vid, start, count, cdata.get());
else if constexpr(std::is_same_v<VType, unsigned char>)
res = nc_get_vara_ubyte(ncid, vid, start, count, cdata.get());
else
return Error(pref, "Unsupported variable type");
if(res != NC_NOERR) return Error(pref, MString("nc_get_vara error: ") + nc_strerror(res));
return ReadedData<VType>(std::move(cdata));
}
public:
~NCSimpleFunctions() { nc_close(ncid); }
Error Open(const MString& filename);
};
using NCSimple = NcZarrRead<NCSimpleFunctions>;

231
src/ncsimple.cpp
Unescape View File

@ -0,0 +1,231 @@
#define MICHLIB_NOSOURCE
#include "ncsimple.h"
RetVal<std::vector<NCSimpleFunctions::Attribute>> NCSimpleFunctions::ReadAtts(int vid) const
{
static const MString pref = "NCSimple::ReadAtts";
int natt;
int ret;
if(vid == NC_GLOBAL)
ret = nc_inq_natts(ncid, &natt);
else
ret = nc_inq_var(ncid, vid, nullptr, nullptr, nullptr, nullptr, &natt);
if(ret != NC_NOERR) return Error(pref, MString("Can't inquire number of attributes: ") + nc_strerror(ret));
std::vector<Attribute> out;
char name[NC_MAX_NAME + 1];
for(int aid = 0; aid < natt; aid++)
{
nc_type type;
size_t len;
ret = nc_inq_attname(ncid, vid, aid, name);
if(ret != NC_NOERR) return Error(pref, MString("Can't inquire attribute name: ") + nc_strerror(ret));
ret = nc_inq_atttype(ncid, vid, name, &type);
if(ret != NC_NOERR) return Error(pref, MString("Can't inquire attribute type: ") + nc_strerror(ret));
ret = nc_inq_attlen(ncid, vid, name, &len);
if(ret != NC_NOERR) return Error(pref, MString("Can't inquire attribute length: ") + nc_strerror(ret));
if(type == NC_DOUBLE || type == NC_FLOAT)
{
if(len == 1)
{
double d;
ret = nc_get_att_double(ncid, vid, name, &d);
if(ret != NC_NOERR) return Error(pref, MString("Can't read attribute ") + name + ": " + nc_strerror(ret));
out.emplace_back(MString(name), d);
}
else
{
std::vector<double> dd(len);
ret = nc_get_att_double(ncid, vid, name, dd.data());
if(ret != NC_NOERR) return Error(pref, MString("Can't read attribute ") + name + ": " + nc_strerror(ret));
for(size_t i = 0; i < dd.size(); i++) out.emplace_back(MString(name) + "[" + i + "]", dd[i]);
}
}
else if(type == NC_BYTE || type == NC_SHORT || type == NC_INT || type == NC_INT64)
{
if(len == 1)
{
long long i;
ret = nc_get_att_longlong(ncid, vid, name, &i);
if(ret != NC_NOERR) return Error(pref, MString("Can't read attribute ") + name + ": " + nc_strerror(ret));
out.emplace_back(MString(name), int_cast<int8>(i));
}
else
{
std::vector<long long> ii(len);
ret = nc_get_att_longlong(ncid, vid, name, ii.data());
if(ret != NC_NOERR) return Error(pref, MString("Can't read attribute ") + name + ": " + nc_strerror(ret));
for(size_t i = 0; i < ii.size(); i++) out.emplace_back(MString(name) + "[" + i + "]", int_cast<int8>(ii[i]));
}
}
else if(type == NC_UBYTE || type == NC_USHORT || type == NC_UINT || type == NC_UINT64)
{
if(len == 1)
{
unsigned long long u;
ret = nc_get_att_ulonglong(ncid, vid, name, &u);
if(ret != NC_NOERR) return Error(pref, MString("Can't read attribute ") + name + ": " + nc_strerror(ret));
out.emplace_back(MString(name), int_cast<uint8>(u));
}
else
{
std::vector<unsigned long long> uu(len);
ret = nc_get_att_ulonglong(ncid, vid, name, uu.data());
if(ret != NC_NOERR) return Error(pref, MString("Can't read attribute ") + name + ": " + nc_strerror(ret));
for(size_t i = 0; i < uu.size(); i++) out.emplace_back(MString(name) + "[" + i + "]", int_cast<uint8>(uu[i]));
}
}
else if(type == NC_CHAR)
{
std::vector<char> ss(len + 1, 0);
ret = nc_get_att_text(ncid, vid, name, ss.data());
if(ret != NC_NOERR) return Error(pref, MString("Can't read attribute ") + name + ": " + nc_strerror(ret));
out.emplace_back(MString(name), MString(ss.data()));
}
else
return Error(pref, MString("Unsupported type of attribute ") + name);
// Ignore all other types
}
return out;
}
Error NCSimpleFunctions::Open(const MString& filename)
{
static const MString pref = "NCSimple::Open";
// Cleanup
gats.clear();
dims.clear();
vars.clear();
nc_close(ncid);
std::vector<Attribute> newgats;
std::vector<Dimension> newdims;
std::vector<Variable> newvars;
char name[NC_MAX_NAME + 1];
// Open
{
auto ret = nc_open(filename.Buf(), 0, &ncid);
if(ret != NC_NOERR) return Error(pref, "Can't open file " + filename + ": " + nc_strerror(ret));
}
// Dimensions
{
int ndim;
auto ret = nc_inq_dimids(ncid, &ndim, nullptr, 1);
if(ret != NC_NOERR) return Error(pref, "Can't inquire number of dimensions in file " + filename + ": " + nc_strerror(ret));
std::vector<int> dimids(ndim);
ret = nc_inq_dimids(ncid, nullptr, dimids.data(), 1);
if(ret != NC_NOERR) return Error(pref, "Can't inquire dimension ids in file " + filename + ": " + nc_strerror(ret));
size_t len;
for(const auto id: dimids)
{
ret = nc_inq_dim(ncid, id, name, &len);
if(ret != NC_NOERR) return Error(pref, "Can't inquire dimension name and size in file " + filename + ": " + nc_strerror(ret));
newdims.emplace_back(name, len);
}
}
// Global attributes
{
auto ret = ReadAtts(NC_GLOBAL);
if(!ret) return ret.Add(pref, "Can't read global attributes in file " + filename);
newgats = std::move(ret.Value());
}
// Variables
{
int nvar;
auto ret = nc_inq_varids(ncid, &nvar, nullptr);
if(ret != NC_NOERR) return Error(pref, "Can't inquire number of variables in file " + filename + ": " + nc_strerror(ret));
std::vector<int> varids(nvar);
ret = nc_inq_varids(ncid, nullptr, varids.data());
if(ret != NC_NOERR) return Error(pref, "Can't inquire variables ids in file " + filename + ": " + nc_strerror(ret));
for(const auto vid: varids)
{
nc_type nctype;
int ndim;
auto ret = nc_inq_var(ncid, vid, name, &nctype, &ndim, nullptr, nullptr);
if(ret != NC_NOERR) return Error(pref, "Can't inquire variable info in file " + filename + ": " + nc_strerror(ret));
VarType vt = VarType::UNDEF;
if(nctype == NC_FLOAT) vt = VarType::FLOAT;
if(nctype == NC_DOUBLE) vt = VarType::DOUBLE;
if(nctype == NC_BYTE) vt = VarType::INT1;
if(nctype == NC_SHORT) vt = VarType::INT2;
if(nctype == NC_INT) vt = VarType::INT4;
if(nctype == NC_INT64) vt = VarType::INT8;
if(nctype == NC_UBYTE) vt = VarType::UINT1;
if(vt == VarType::UNDEF) return Error(pref, "Unsupported type of variable " + MString(name) + " in file " + filename);
std::vector<int> dimids(ndim);
ret = nc_inq_vardimid(ncid, vid, dimids.data());
if(ret != NC_NOERR) return Error(pref, "Can't inquire variable dimensions in file " + filename + ": " + nc_strerror(ret));
std::vector<size_t> dims;
char dname[NC_MAX_NAME + 1];
for(const auto did: dimids)
{
auto ret = nc_inq_dimname(ncid, did, dname);
if(ret != NC_NOERR) return Error(pref, "Can't inquire dimension name in file " + filename + ": " + nc_strerror(ret));
size_t ind = newdims.size();
for(size_t i = 0; i < newdims.size() && ind == newdims.size(); i++)
if(dname == newdims[i].Name()) ind = i;
if(ind == newdims.size()) return Error(pref, "Can't find dimension " + MString(dname) + " of variable " + name + " in file " + filename);
dims.push_back(ind);
}
auto atts = ReadAtts(vid);
const char fname[] = "_FillValue";
if(!atts) return Error(pref, "Can't get attributes of variable " + MString(name) + " in file " + filename);
std::vector<Attribute> vatts = std::move(atts.Value());
Variable::FillType fill;
if(FindInd(fname, vatts) != vatts.size())
{
if(nctype == NC_FLOAT || nctype == NC_DOUBLE)
{
double d;
auto ret = nc_get_att_double(ncid, vid, fname, &d);
if(ret != NC_NOERR) return Error(pref, "Can't get fill value for the variable " + MString(name) + " in file " + filename + ": " + nc_strerror(ret));
fill = d;
}
if(nctype == NC_BYTE || nctype == NC_SHORT || nctype == NC_INT || nctype == NC_INT64)
{
long long l;
auto ret = nc_get_att_longlong(ncid, vid, fname, &l);
if(ret != NC_NOERR) return Error(pref, "Can't get fill value for the variable " + MString(name) + " in file " + filename + ": " + nc_strerror(ret));
fill = int_cast<int8>(l);
}
if(nctype == NC_UBYTE || nctype == NC_USHORT || nctype == NC_UINT || nctype == NC_UINT64)
{
unsigned long long u;
auto ret = nc_get_att_ulonglong(ncid, vid, fname, &u);
if(ret != NC_NOERR) return Error(pref, "Can't get fill value for the variable " + MString(name) + " in file " + filename + ": " + nc_strerror(ret));
fill = int_cast<uint8>(u);
}
}
newvars.emplace_back(name, vt, std::move(dims), std::move(vatts), fill);
}
}
gats = std::move(newgats);
dims = std::move(newdims);
vars = std::move(newvars);
return Error();
}
Write Preview
Loading…
Cancel
Save