From fd320973b80b505e73b213e1be119df718ab855e Mon Sep 17 00:00:00 2001 From: Michael Uleysky Date: Tue, 25 Apr 2023 15:37:04 +1000 Subject: [PATCH] Added new data adapter for binary files --- include/BINFILE.h | 44 +++++++++++++++++++++ include/data.h | 3 +- include/odm.h | 3 +- michlib | 2 +- src/BINFILE.cpp | 97 +++++++++++++++++++++++++++++++++++++++++++++++ src/data.cpp | 7 ++++ src/odm.cpp | 8 ++-- 7 files changed, 158 insertions(+), 6 deletions(-) create mode 100644 include/BINFILE.h create mode 100644 src/BINFILE.cpp diff --git a/include/BINFILE.h b/include/BINFILE.h new file mode 100644 index 0000000..7dc0c0c --- /dev/null +++ b/include/BINFILE.h @@ -0,0 +1,44 @@ +#pragma once +#include "DataAdapters/datafile.cpp" +#include "DataAdapters/findfilebyid.cpp" +#include "ParameterList.h" +#include "mregex.h" +#include "simple2ddata.h" + +using michlib::GPL; +using michlib::MDateTime; +using michlib::RegExp; + +class BINFILEData +{ + std::vector times; + MString datapath; + std::unique_ptr data; + + public: + using Data = Simple2DData; + + BINFILEData() = default; + + MString Info() const; + // TODO: RetVal + MString Open(const CLArgs& args); + + bool isOk() const { return times.size() > 0; } + + size_t NTimes() const { return times.size(); } + + MDateTime Time(size_t i) const + { + if(!isOk() || i >= times.size()) return MDateTime(); + return times[i]; + } + + time_t Timestep() const { return isOk() ? (times[1] - times[0]) : 0; } + + explicit operator bool() const { return times.size() > 0; } + + bool CheckVar(const MString& vname) const { return vname == "u" || vname == "v" || vname == "U" || vname == "U2"; } + + Data Read(const MString& vname, size_t i) const; +}; diff --git a/include/data.h b/include/data.h index 80a64c1..b30ea8b 100644 --- a/include/data.h +++ b/include/data.h @@ -1,11 +1,12 @@ #pragma once #include "AVISO.h" #include "AVISOLOCAL.h" +#include "BINFILE.h" #include "HYCOM.h" #include "NEMO.h" #include -using DataVariants = std::variant; +using DataVariants = std::variant; class Data: public DataVariants { public: diff --git a/include/odm.h b/include/odm.h index ac6fdd3..d9ce488 100644 --- a/include/odm.h +++ b/include/odm.h @@ -15,4 +15,5 @@ template consteval bool MustSupported() constexpr bool supported = MustSupported() && MustSupported() && MustSupported() && MustSupported() && MustSupported() && MustSupported() && MustSupported() && MustSupported() && MustSupported() && - MustSupported() && MustSupported() && MustSupported(); + MustSupported() && MustSupported() && MustSupported() && + MustSupported() && MustSupported() && MustSupported(); diff --git a/michlib b/michlib index 020fb92..a210541 160000 --- a/michlib +++ b/michlib @@ -1 +1 @@ -Subproject commit 020fb92bf796a1f66d10b488b2e0768d8ed72cb2 +Subproject commit a2105413f8a10e8695c85706487d4db5d2f54a61 diff --git a/src/BINFILE.cpp b/src/BINFILE.cpp new file mode 100644 index 0000000..92a0f1b --- /dev/null +++ b/src/BINFILE.cpp @@ -0,0 +1,97 @@ +#define MICHLIB_NOSOURCE +#include "BINFILE.h" + +MString BINFILEData::Info() const +{ + if(!isOk()) return ""; + + michlib_internal::ParameterListEx pars; + pars.AddParameters(data->DataFileParameters()); + pars.UsePrefix("Datafile_Info"); + + MString title = pars.ParameterSValue("DataSource", "no information"); + MString lonb = pars.ParameterSValue("lonb", ""); + MString lone = pars.ParameterSValue("lone", ""); + MString latb = pars.ParameterSValue("latb", ""); + MString late = pars.ParameterSValue("late", ""); + MString dataid = pars.ParameterSValue("DataID", "no information"); + + // clang-format off + return + "Dataset: " + title + "\n" + + " DataId: " + dataid + "\n" + + " Begin date: " + Time(0).ToString() + "\n" + + " End date: " + Time(NTimes()-1).ToString() + "\n" + + " Time step: " + Timestep() + " seconds\n" + + " Time moments: " + NTimes() + "\n" + + " Region: (" + lonb + " : " + lone + ") x (" + latb + " : " + late + ")\n" + + " Grid: " + data->Nx() + "x" + data->Ny() + "\n" + + " Supported variables: u, v, U2"; + // clang-format on +} + +MString BINFILEData::Open(const CLArgs& args) +{ + GPL.UsePrefix("BINFILE"); + datapath = GPL.ParameterSValue("Datapath", ""); + MString dataset = args.contains("dataset") ? args.at("dataset") : ""; + if(!dataset.Exist()) return "No data file"; + + MString file; + { + RegExp regex("^[0-9a-f]{64}$"); + regex.Compile(); + if(regex.Match(dataset.Buf())) + file = michlib::FindFileByID(datapath, dataset); + else + file = (dataset[0] != '/') ? (datapath + "/" + dataset) : dataset; + } + data.reset(new michlib::IntData); + if(!data->Open(file)) return "Can't open file " + file; + for(size_t it = 0; it < data->Nt(); it++) times.push_back(data->Date(it)); + + return ""; +} + +BINFILEData::Data BINFILEData::Read(const MString& vname, size_t i) const +{ + if(!isOk()) return Data(); + + // Only rectangular grids are supported + real xs = (data->Lon(data->Nx() - 1, data->Ny() - 1) - data->Lon(0, 0)) / (data->Nx() - 1); + real ys = (data->Lat(data->Nx() - 1, data->Ny() - 1) - data->Lat(0, 0)) / (data->Ny() - 1); + Data out(data->Nx(), data->Ny(), data->Lon(0, 0), data->Lat(0, 0), xs, ys); + + // U and U2 from u and v + if(vname == "U" || vname == "U2") + { + bool square = vname == "U2"; + auto u = Read("u", i); + auto v = Read("v", i); + if(!(u && v)) return Data(); + auto out = u; + for(size_t ind = 0; ind < out.N(); ind++) + { + if(u.IsFill(ind) || v.IsFill(ind)) + out.V(ind) = out.Fillval(); + else + out.V(ind) = square ? (u(ind) * u(ind) + v(ind) * v(ind)) : michlib::Hypot(u(ind), v(ind)); + } + return out; + } + + if(vname == "u" || vname == "v") + { + bool isu = vname == "u"; + for(size_t ix = 0; ix < data->Nx(); ix++) + for(size_t iy = 0; iy < data->Ny(); iy++) + { + if(data->IsOk(ix, iy, i)) + out.V(ix, iy) = isu ? data->Ur(ix, iy, i) : data->Vr(ix, iy, i); + else + out.V(ix, iy) = out.Fillval(); + } + } + + return out; +} diff --git a/src/data.cpp b/src/data.cpp index 655de5a..ba80817 100644 --- a/src/data.cpp +++ b/src/data.cpp @@ -33,6 +33,13 @@ MString Data::Init(const CLArgs& args) if(res.Exist()) return "Can't open source " + src + ":\n" + res; *this = Data(std::move(data)); } + else if(src == "BINFILE") + { + BINFILEData data; + auto res = data.Open(args); + if(res.Exist()) return "Can't open source " + src + ":\n" + res; + *this = Data(std::move(data)); + } else return "Unknown source: " + src; return ""; diff --git a/src/odm.cpp b/src/odm.cpp index e1e7699..7fbfaa6 100644 --- a/src/odm.cpp +++ b/src/odm.cpp @@ -6,7 +6,7 @@ inline void Usage(const MString& arg0) message("Keys are:"); message(" action. What the program should do. May be: info, tsc, uv. Default: info."); message(" Keys for action=info. Print some information about dataset."); - message(" source. Required. May be: NEMO, HYCOM, AVISO, AVISOLOCAL"); + message(" source. Required. May be: NEMO, HYCOM, AVISO, AVISOLOCAL, BINFILE"); message(" Keys for source=NEMO"); message(" dataset. Can be DT, NRT or NRT6. Default: DT"); message(" Keys for source=HYCOM"); @@ -14,7 +14,7 @@ inline void Usage(const MString& arg0) message(" Keys for source=AVISO"); message(" dataset. Can be DT, NRT, EckmanDT or EckmanNRT. Default: DT"); message(" Keys for action=tsc. Get temperature, salinity, chlorofill from dataset."); - message(" source. Required. May be: NEMO, HYCOM, AVISO, AVISOLOCAL"); + message(" source. Required. May be: NEMO, HYCOM, AVISO, AVISOLOCAL, BINFILE"); message(" var. Required. May be: U, U2, u, v, temp, ptemp, pdens, sal, chl, mld, ssh or w."); message(" time. Time moment or regular expression. If present, timeb and timee must be absent"); message(" timeb, timee. Time interval. If present, time must be absent"); @@ -29,8 +29,10 @@ inline void Usage(const MString& arg0) message(" Keys for source=AVISO"); message(" dataset. Can be DT, NRT, EckmanDT or EckmanNRT. Default: DT"); message(" layer and/or depth. Layer or depth of AVISO dataset. If depth is specified, layer is ignored. Both ignored for datasets DT and NRT. Default: layer=0"); + message(" Keys for source=BINFILE"); + message(" dataset. Path or DataID of interpolation file"); message(" Keys for action=uv. Get velocity field and its derivatives."); - message(" source. Required. May be: NEMO, HYCOM, AVISO, AVISOLOCAL"); + message(" source. Required. May be: NEMO, HYCOM, AVISO, AVISOLOCAL, BINFILE"); message(" time. Time moment or regular expression. If present, timeb and timee must be absent"); message(" timeb, timee. Time interval. If present, time must be absent"); message(" out. Output file for components of velocity field, divergency, rotor and Okubo-Weiss parameter. If absent, this data not calculated.");