michael_ul
/
odm
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: michael_ul:ce31ba091cffe047a789cb7c62703fd2b04f3cbe
Branches Tags
michael_ul:interpolate
michael_ul:lintest
michael_ul:master
..
compare: michael_ul:802dcb55cd670cbac6bc3328f053894b3abb6807
Branches Tags
michael_ul:interpolate
michael_ul:lintest
michael_ul:master

No commits in common. 'ce31ba091cffe047a789cb7c62703fd2b04f3cbe' and '802dcb55cd670cbac6bc3328f053894b3abb6807' have entirely different histories.
ce31ba091c ... 802dcb55cd

9 changed files with 88 additions and 373 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats
  1. 2
      CMakeLists.txt
  2. 118
      include/cache.h
  3. 4
      include/mirrorfuncs.h
  4. 66
      include/ncfilew.h
  5. 3
      include/zarr.h
  6. 3
      src/CMakeLists.txt
  7. 240
      src/cache.cpp
  8. 6
      src/mirrorfuncs.cpp
  9. 1
      src/zarr.cpp

2
CMakeLists.txt
Unescape View File

@ -58,7 +58,7 @@ endif()
add_compile_options(-Wno-deprecated-declarations)
CHECK_CXX_COMPILER_FLAG(-Wno-class-memaccess COMPILER_SUPPORTS_CLASSMEMACCESS)
if(COMPILER_SUPPORTS_CLASSMEMACCESS)
add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-Wno-class-memaccess>)
add_compile_options(-Wno-class-memaccess)
endif()
add_library(teos STATIC GSW-C/gsw_oceanographic_toolbox.c GSW-C/gsw_saar.c)

118
include/cache.h
Unescape View File

@ -1,16 +1,12 @@
#pragma once
#include "GPL.h"
#include "mirrorfuncs.h"
#include <functional>
#include <libpq-fe.h>
#include <optional>
#include <sqlite3.h>
#include <time.h>
#include <variant>
using michlib::GPL;
using michlib::int1;
using michlib::int4;
using michlib::int_cast;
using michlib::MString;
using michlib::pointer_cast;
@ -257,35 +253,12 @@ class SQLiteCache: public GenericCache
explicit operator bool() const { return db != nullptr; }
};
class PostgreSQLHelpers
{
static constexpr time_t postgresepoch = 946648800;
class PGResultRAIIDT
class PostgreSQLCache: public GenericCache
{
public:
// TODO: make static
void operator()(PGresult* res) { PQclear(res); }
};
static bool regdest;
protected:
PostgreSQLConnection conn;
// Convert Postgres binary representation of timestamp to Unix epoch seconds. Microseconds ignored
static time_t raw2epoch(time_t raw) { return Invert(raw) / 1000000 + postgresepoch; }
// Convert Unix epoch time to Postres binary representation
static time_t epoch2raw(time_t epoch) { return Invert((epoch - postgresepoch) * 1000000); }
class PGresultRAII: public std::unique_ptr<PGresult, PGResultRAIIDT>
{
public:
PGresultRAII() = default;
PGresultRAII(PGresult* res): std::unique_ptr<PGresult, PGResultRAIIDT>(res) {}
operator PGresult*() const { return get(); }
};
bool CheckCon() const
{
if(!*this) return false;
@ -297,27 +270,14 @@ class PostgreSQLHelpers
template<class D> static D Invert(D d)
{
using michlib::int1;
if(sizeof(D) <= 1) return d;
D out;
Invert(&d, &out, sizeof(D));
int1* pout = pointer_cast<int1*>(&out);
int1* pin = pointer_cast<int1*>(&d);
for(size_t i = 0; i < sizeof(D); i++) pout[sizeof(D) - i - 1] = pin[i];
return out;
}
static void Invert(const void* src, void* dst, size_t sz)
{
if(sz == 0) return;
const int1* pin = pointer_cast<const int1*>(src);
int1* pout = pointer_cast<int1*>(dst);
for(size_t i = 0; i < sz; i++) pout[sz - i - 1] = pin[i];
}
public:
explicit operator bool() const { return conn != nullptr; }
};
class PostgreSQLCache: public GenericCache, public PostgreSQLHelpers
{
static bool regdest;
public:
bool Init()
{
@ -325,20 +285,31 @@ class PostgreSQLCache: public GenericCache, public PostgreSQLHelpers
if(!regdest)
{
PGresultRAII res;
// Create table
res = PQexec(conn, "SET client_min_messages=WARNING;");
res = PQexec(conn, "CREATE TABLE IF NOT EXISTS cache(key TEXT PRIMARY KEY NOT NULL, value BYTEA, exptime TIMESTAMP(0) NOT NULL);");
{
auto* res = PQexec(conn, "SET client_min_messages=WARNING;");
PQclear(res);
}
{
auto* res = PQexec(conn, "CREATE TABLE IF NOT EXISTS cache(key TEXT PRIMARY KEY NOT NULL, value BYTEA, exptime TIMESTAMP(0) NOT NULL);");
if(PQresultStatus(res) != PGRES_COMMAND_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
}
PQclear(res);
}
{
auto* res = PQexec(conn, "SET client_min_messages=NOTICE;");
PQclear(res);
}
res = PQexec(conn, "SET client_min_messages=NOTICE;");
conn.AddDestructor([](PostgreSQLConnection::DBType conn) { PGresultRAII res = PQexec(conn, "DELETE FROM cache WHERE exptime<localtimestamp;"); });
conn.AddDestructor(
[](PostgreSQLConnection::DBType conn)
{
auto* res = PQexec(conn, "DELETE FROM cache WHERE exptime<localtimestamp;");
PQclear(res);
});
regdest = true;
}
@ -352,10 +323,10 @@ class PostgreSQLCache: public GenericCache, public PostgreSQLHelpers
auto interval = michlib::int_cast<michlib::int8>(ttl);
michlib::int8 rinterval = Invert(interval);
const char* params[] = {key.Buf(), value.Buf(), pointer_cast<const char*>(&rinterval)};
int plens[] = {int_cast<int>(key.Len()), int_cast<int>(value.Len()), sizeof(rinterval)};
int plens[] = {int_cast<int>(key.Len()), int_cast<int>(value.Len()), 8};
int pfor[] = {0, 1, 1};
PGresultRAII res = PQexecParams(conn,
auto* res = PQexecParams(conn,
"INSERT INTO cache(key,value,exptime) VALUES($1,$2,localtimestamp + ($3::bigint ||' seconds')::interval)"
"ON CONFLICT(key) DO UPDATE SET value=EXCLUDED.value, exptime=EXCLUDED.exptime;",
3, nullptr, params, plens, pfor, 1);
@ -363,8 +334,10 @@ class PostgreSQLCache: public GenericCache, public PostgreSQLHelpers
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
PQclear(res);
return false;
}
PQclear(res);
return true;
}
@ -376,21 +349,28 @@ class PostgreSQLCache: public GenericCache, public PostgreSQLHelpers
int plens[] = {int_cast<int>(key.Len())};
int pfor[] = {0};
PGresultRAII res = PQexecParams(conn, "SELECT value from cache WHERE key=$1::text AND exptime>localtimestamp;", 1, nullptr, params, plens, pfor, 1);
auto* res = PQexecParams(conn, "SELECT value from cache WHERE key=$1::text AND exptime>localtimestamp;", 1, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_TUPLES_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
PQclear(res);
return {"", false};
}
else if(PQntuples(res) == 0)
{
PQclear(res);
return {"", false};
}
MString val(PQgetvalue(res, 0, 0), PQgetlength(res, 0, 0));
PQclear(res);
return {std::move(val), true};
}
virtual ~PostgreSQLCache() override = default;
explicit operator bool() const { return conn != nullptr; }
};
inline GenericCache* CreateCache(const MString& cachedesc)
@ -419,29 +399,3 @@ inline GenericCache* CreateCache(const MString& cachedesc)
return nullptr;
}
class FileInfoCache: public PostgreSQLHelpers
{
public:
using DataType = std::optional<MString>;
using CallbackType = std::function<DataType(const MString&)>;
private:
static bool regdest;
CallbackType readfunc;
MString dir;
int4 dirid;
FileInfoCache() = delete;
CallbackType::result_type GetData(const MString& fname) const { return readfunc(dir + "/" + fname); }
void GetDirId();
public:
FileInfoCache(CallbackType&& readfunc_, const MString& dir_);
Error UpdateCache(bool force = false) const;
DataType GetInfo(const MString& name) const;
};

4
include/mirrorfuncs.h
Unescape View File

@ -7,9 +7,9 @@
#include <sys/types.h>
#include <vector>
using michlib::Error;
using michlib::MDateTime;
using michlib::RetVal;
using michlib::Error;
class DIRRAIIDT
{
@ -52,7 +52,7 @@ inline MString FileName(const MString& name)
bool MakePath(const MString& dname);
// Get local file list
RetVal<std::vector<struct FileInfo>> ReadLocalFileList(const MString& dir, const bool nofollow = true, const MString& path = "");
RetVal<std::vector<struct FileInfo>> ReadLocalFileList(const MString& dir, const MString& path = "");
// Download file to the local mirror
Error DownloadFile(const CURLRAII& chandle, const struct FileInfo& rinfo, const MString& root);

66
include/ncfilew.h
Unescape View File

@ -280,7 +280,9 @@ class NCFileWBase
template<class T>
requires requires(int nc, int vid, const T* d) {
{ NCTypeD<T, void>::put_var(nc, vid, d) } -> std::same_as<int>;
{
NCTypeD<T, void>::put_var(nc, vid, d)
} -> std::same_as<int>;
}
void WriteVar(const MString& vname, const T* data)
{
@ -295,7 +297,9 @@ class NCFileWBase
template<class T>
requires requires(int nc, int vid, const size_t* start, const size_t* count, const T* d) {
{ NCTypeD<T, void>::put_vara(nc, vid, start, count, d) } -> std::same_as<int>;
{
NCTypeD<T, void>::put_vara(nc, vid, start, count, d)
} -> std::same_as<int>;
}
void WriteVar(const MString& vname, size_t ind, const T* data)
{
@ -418,23 +422,23 @@ class NCFileW: public NCFileWBase
if constexpr(Is1DType(dtype))
{
const size_t c = data.N();
auto buf = std::make_unique<float[]>(c);
float buf[c];
for(size_t ix = 0; ix < c; ix++) buf[ix] = data.IsFill(ix) ? fill : op(ix);
if(tdep)
WriteVar(name, tind, buf.get());
WriteVar(name, tind, buf);
else
WriteVar(name, buf.get());
WriteVar(name, buf);
}
else
{
const size_t c[2] = {data.Ny(), data.Nx()};
auto buf = std::make_unique<float[]>(c[0] * c[1]);
float buf[c[0] * c[1]];
for(size_t iy = 0; iy < c[0]; iy++)
for(size_t ix = 0; ix < c[1]; ix++) buf[iy * c[1] + ix] = data.IsFill(ix, iy) ? fill : op(ix, iy);
if(tdep)
WriteVar(name, tind, buf.get());
WriteVar(name, tind, buf);
else
WriteVar(name, buf.get());
WriteVar(name, buf);
}
if(!*this) return "Can't write variable " + name + ": " + ErrMessage();
@ -445,9 +449,11 @@ class NCFileW: public NCFileWBase
template<class D> MString WriteVariable(const D& data, const MString& name, size_t tind)
{
if constexpr(Is1DType(DetType<D>()))
return WriteVariable(data, name, [&data = std::as_const(data)](size_t i) { return data(i); }, tind);
return WriteVariable(
data, name, [&data = std::as_const(data)](size_t i) { return data(i); }, tind);
else
return WriteVariable(data, name, [&data = std::as_const(data)](size_t i, size_t j) { return data(i, j); }, tind);
return WriteVariable(
data, name, [&data = std::as_const(data)](size_t i, size_t j) { return data(i, j); }, tind);
}
template<class D, class Op>
@ -484,63 +490,63 @@ class NCFileW: public NCFileWBase
else if constexpr(dtype == GPSET)
{
const size_t c = data.N();
auto bufx = std::make_unique<float[]>(c);
auto bufy = std::make_unique<float[]>(c);
float bufx[c];
float bufy[c];
for(size_t ix = 0; ix < c; ix++)
{
bufx[ix] = data.Lon(ix);
bufy[ix] = data.Lat(ix);
}
WriteVar("longitude", bufx.get());
WriteVar("latitude", bufy.get());
WriteVar("longitude", bufx);
WriteVar("latitude", bufy);
}
else if constexpr(dtype == RGRID)
{
const size_t cx = data.Nx(), cy = data.Ny();
auto bufx = std::make_unique<float[]>(cx);
auto bufy = std::make_unique<float[]>(cy);
float bufx[cx];
float bufy[cy];
for(size_t ix = 0; ix < cx; ix++) bufx[ix] = data.Ix2X(ix);
for(size_t iy = 0; iy < cy; iy++) bufy[iy] = data.Iy2Y(iy);
WriteVar("x", bufx.get());
WriteVar("y", bufy.get());
WriteVar("x", bufx);
WriteVar("y", bufy);
}
else if constexpr(dtype == GRGRID)
{
const size_t cx = data.Nx(), cy = data.Ny();
auto bufx = std::make_unique<float[]>(cx);
auto bufy = std::make_unique<float[]>(cy);
float bufx[cx];
float bufy[cy];
for(size_t ix = 0; ix < cx; ix++) bufx[ix] = data.Ix2Lon(ix);
for(size_t iy = 0; iy < cy; iy++) bufy[iy] = data.Iy2Lat(iy);
WriteVar("longitude", bufx.get());
WriteVar("latitude", bufy.get());
WriteVar("longitude", bufx);
WriteVar("latitude", bufy);
}
else if constexpr(dtype == GRID)
{
const size_t c[2] = {data.Ny(), data.Nx()};
auto bufx = std::make_unique<float[]>(c[0] * c[1]);
auto bufy = std::make_unique<float[]>(c[0] * c[1]);
float bufx[c[0] * c[1]];
float bufy[c[0] * c[1]];
for(size_t iy = 0; iy < c[0]; iy++)
for(size_t ix = 0; ix < c[1]; ix++)
{
bufx[iy * c[1] + ix] = data.X(ix, iy);
bufy[iy * c[1] + ix] = data.Y(ix, iy);
}
WriteVar("x", bufx.get());
WriteVar("y", bufy.get());
WriteVar("x", bufx);
WriteVar("y", bufy);
}
else if constexpr(dtype == GGRID)
{
const size_t c[2] = {data.Ny(), data.Nx()};
auto bufx = std::make_unique<float[]>(c[0] * c[1]);
auto bufy = std::make_unique<float[]>(c[0] * c[1]);
float bufx[c[0] * c[1]];
float bufy[c[0] * c[1]];
for(size_t iy = 0; iy < c[0]; iy++)
for(size_t ix = 0; ix < c[1]; ix++)
{
bufx[iy * c[1] + ix] = data.Lon(ix, iy);
bufy[iy * c[1] + ix] = data.Lat(ix, iy);
}
WriteVar("longitude", bufx.get());
WriteVar("latitude", bufy.get());
WriteVar("longitude", bufx);
WriteVar("latitude", bufy);
}
else
return "Unknown data type";

3
include/zarr.h
Unescape View File

@ -94,7 +94,6 @@ class ZarrFunctions: public ZarrTypes
std::unique_ptr<GenericCache> cache;
CURLRAII chandle;
MString url;
MString proxyurl;
std::vector<std::vector<size_t>> chunks;
@ -108,8 +107,6 @@ class ZarrFunctions: public ZarrTypes
{
auto oldprefix = michlib::GPL.UsePrefix("ZARR");
cache.reset(CreateCache(michlib::GPL.ParameterSValue("Cache", "")));
proxyurl = michlib::GPL.ParameterSValue("Proxy", "");
if(proxyurl.Exist()) curl_easy_setopt(chandle, CURLOPT_PROXY, proxyurl.Buf());
michlib::GPL.UsePrefix(oldprefix);
if(!cache)
{

3
src/CMakeLists.txt
Unescape View File

@ -4,7 +4,6 @@ set(DATALISTINC ${CMAKE_CURRENT_BINARY_DIR}/../include/datalist.h) # Include dat
find_package(PkgConfig REQUIRED)
find_library(netcdf netcdf REQUIRED)
find_library(udunits udunits2 REQUIRED)
find_package(OpenMP REQUIRED)
find_package(CURL REQUIRED)
find_package(LibXml2 REQUIRED)
@ -21,7 +20,7 @@ file(GLOB srcs CONFIGURE_DEPENDS *.cpp)
add_executable(${EXENAME} ${srcs} ${ACTIONLISTINC} ${SOURCELISTINC})
target_include_directories(${EXENAME} PRIVATE ../michlib/michlib ${CMAKE_CURRENT_BINARY_DIR}/../include)
target_link_libraries(${EXENAME} ${linker_options} ${netcdf} ${udunits} OpenMP::OpenMP_CXX CURL::libcurl ${JSONCPP_LINK_LIBRARIES} ${BLOSC_LINK_LIBRARIES} ${LIBPQ_LINK_LIBRARIES} LibXml2::LibXml2 SQLite::SQLite3 teos)
target_link_libraries(${EXENAME} ${linker_options} ${netcdf} OpenMP::OpenMP_CXX CURL::libcurl ${JSONCPP_LINK_LIBRARIES} ${BLOSC_LINK_LIBRARIES} ${LIBPQ_LINK_LIBRARIES} LibXml2::LibXml2 SQLite::SQLite3 teos)
set_target_properties(${EXENAME} PROPERTIES POSITION_INDEPENDENT_CODE ON)
install(TARGETS ${EXENAME})

240
src/cache.cpp
Unescape View File

@ -12,243 +12,3 @@ std::vector<PostgreSQLConnection::FuncType> PostgreSQLConnection::destructs = {}
bool SQLiteCache::regdest = false;
bool PostgreSQLCache::regdest = false;
bool FileInfoCache::regdest = false;
void FileInfoCache::GetDirId()
{
if(dirid != 0) return;
const char* params[] = {dir.Buf()};
int plens[] = {int_cast<int>(dir.Len())};
int pfor[] = {0};
PGresultRAII res = PQexecParams(conn, "SELECT id FROM dirs WHERE name=$1::text;", 1, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_TUPLES_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
return;
}
else if(PQntuples(res) == 0)
{
res = PQexecParams(conn,
"INSERT INTO dirs(name,id) VALUES ($1, (SELECT min(num.numid) FROM (SELECT generate_series(1, (SELECT COALESCE((SELECT max(id) FROM dirs), 1)) + "
"1, 1) AS numid) num LEFT JOIN dirs ON dirs.id=num.numid WHERE id IS NULL)) RETURNING id;",
1, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
}
if(PQntuples(res) == 0) return;
}
if(PQgetlength(res, 0, 0) == sizeof(dirid)) dirid = *pointer_cast<const decltype(dirid)*>(PQgetvalue(res, 0, 0));
michlib::message("Dirid: ", Invert(dirid));
}
FileInfoCache::FileInfoCache(FileInfoCache::CallbackType&& readfunc_, const MString& dir_): readfunc(std::move(readfunc_)), dir(dir_), dirid(0)
{
if(!conn) return;
if(!regdest)
{
// Create table
PGresultRAII res = PQexec(conn, "SET client_min_messages=WARNING;");
res = PQexec(conn, "BEGIN;"
"CREATE TABLE IF NOT EXISTS dirs(name TEXT PRIMARY KEY, id INTEGER UNIQUE NOT NULL CONSTRAINT id_is_positive CHECK(id>0));"
"CREATE TABLE IF NOT EXISTS files(name TEXT NOT NULL, size BIGINT NOT NULL CONSTRAINT size_is_positive CHECK(size>0), modtime TIMESTAMP(0) NOT NULL, "
"dirid INTEGER REFERENCES dirs(id) ON DELETE CASCADE, lastaccess TIMESTAMP(0) NOT NULL, data BYTEA NOT NULL, PRIMARY KEY(name,dirid));"
"COMMIT;");
if(PQresultStatus(res) != PGRES_COMMAND_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
}
res = PQexec(conn, "SET client_min_messages=NOTICE;");
conn.AddDestructor(
[](PostgreSQLConnection::DBType conn)
{
PGresultRAII res = PQexec(conn, "BEGIN;"
"DELETE FROM files WHERE lastaccess+'100 days'::interval<localtimestamp;"
"DELETE FROM dirs WHERE id NOT IN (SELECT dirid FROM files);"
"COMMIT;");
});
regdest = true;
}
GetDirId();
//UpdateCache();
}
Error FileInfoCache::UpdateCache(bool force) const
{
const static MString pref = "FileInfoCache::UpdateCache";
DIRRAII dhandle;
dhandle.reset(opendir(dir.Buf()));
if(!dhandle) return {pref, "Can't open directory " + dir};
int dfd = dirfd(dhandle);
errno = 0;
struct dirent* dent = readdir(dhandle);
if(errno != 0) return {pref, "Can't read directory " + dir};
struct stat st;
do {
if(dent->d_name[0] != '.')
{
int ret = fstatat(dfd, dent->d_name, &st, 0);
if(ret != 0) return {pref, "Can't stat " + dir + "/" + dent->d_name};
if(S_ISREG(st.st_mode)) // Regular file
{
const char* params[] = {dent->d_name, pointer_cast<const char*>(&dirid)};
int plens[] = {int_cast<int>(strlen(dent->d_name)), sizeof(dirid)};
int pfor[] = {0, 1};
bool querysucc = true;
time_t modtime;
size_t size;
PGresultRAII res = PQexecParams(conn, "SELECT size,modtime FROM files WHERE name=$1::text AND dirid=$2::integer;", 2, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
querysucc = false;
}
else if(PQntuples(res) == 0 || PQntuples(res) > 1)
querysucc = false;
if(querysucc)
{
size = *pointer_cast<const decltype(size)*>(PQgetvalue(res, 0, 0));
modtime = raw2epoch(*pointer_cast<const time_t*>(PQgetvalue(res, 0, 1)));
}
else
{
size = int_cast<size_t>(st.st_size);
modtime = st.st_mtim.tv_sec;
}
if(!querysucc || force || size != int_cast<size_t>(st.st_size) || modtime != st.st_mtim.tv_sec)
{
auto ret = GetData(dent->d_name);
// Remove entry
if(!ret && querysucc)
{
PGresultRAII dres = PQexecParams(conn, "DELETE FROM files WHERE name=$1::text AND dirid=$2::integer;", 2, nullptr, params, plens, pfor, 1);
if(PQresultStatus(dres) != PGRES_COMMAND_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(dres)));
michlib::errmessage(PQerrorMessage(conn));
}
}
else // Update or insert
{
auto sizei = Invert(size);
auto modtimei = epoch2raw(modtime);
const char* params[] = {dent->d_name, pointer_cast<const char*>(&sizei), pointer_cast<const char*>(&modtimei), pointer_cast<const char*>(&dirid), ret.value().Buf()};
int plens[] = {int_cast<int>(strlen(dent->d_name)), sizeof(sizei), sizeof(modtimei), sizeof(dirid), int_cast<int>(ret.value().Len())};
int pfor[] = {0, 1, 1, 1, 1};
PGresultRAII res = PQexecParams(conn,
"INSERT INTO files (name,size,modtime,dirid,lastaccess,data) VALUES($1::text, $2::bigint, $3::timestamp, $4::integer, localtimestamp, $5) "
"ON CONFLICT ON CONSTRAINT files_pkey DO UPDATE SET "
"size=EXCLUDED.size, modtime=EXCLUDED.modtime, lastaccess=EXCLUDED.lastaccess, data=EXCLUDED.data;",
5, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_COMMAND_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
}
} // Insert or update branch
} // Need data update
} // Regular file
} // if(dent->d_name[0] != '.')
dent = readdir(dhandle);
} while(dent != nullptr || errno != 0);
return Error();
}
FileInfoCache::DataType FileInfoCache::GetInfo(const MString& name) const
{
if(!*this) return GetData(name);
bool querysucc = true;
MString data;
time_t modtime;
size_t size;
{
const char* params[] = {name.Buf(), pointer_cast<const char*>(&dirid)};
int plens[] = {int_cast<int>(name.Len()), sizeof(dirid)};
int pfor[] = {0, 1};
PGresultRAII res =
PQexecParams(conn, "UPDATE files SET lastaccess=localtimestamp WHERE name=$1::text AND dirid=$2::integer RETURNING data,size,modtime;", 2, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
querysucc = false;
}
if(PQntuples(res) == 0 || PQntuples(res) > 1)
{
michlib::errmessage("Data for file ", dir + "/" + name, (PQntuples(res) == 0 ? " not found " : " duplicated "), "in cache");
querysucc = false;
}
if(querysucc)
{
data = MString(PQgetvalue(res, 0, 0), PQgetlength(res, 0, 0));
size = *pointer_cast<const decltype(size)*>(PQgetvalue(res, 0, 1));
modtime = raw2epoch(*pointer_cast<const time_t*>(PQgetvalue(res, 0, 2)));
}
}
{
struct stat st;
int ret = stat((dir + "/" + name).Buf(), &st);
if(ret != 0) return DataType();
if(querysucc && st.st_mtim.tv_sec == modtime && size == int_cast<size_t>(st.st_size)) return data;
modtime = st.st_mtim.tv_sec;
size = st.st_size;
}
auto ret = GetData(name);
if(ret)
{
auto sizei = Invert(size);
auto modtimei = epoch2raw(modtime);
const char* params[] = {name.Buf(), pointer_cast<const char*>(&sizei), pointer_cast<const char*>(&modtimei), pointer_cast<const char*>(&dirid), ret.value().Buf()};
int plens[] = {int_cast<int>(name.Len()), sizeof(sizei), sizeof(modtimei), sizeof(dirid), int_cast<int>(ret.value().Len())};
int pfor[] = {0, 1, 1, 1, 1};
PGresultRAII res = PQexecParams(conn,
"INSERT INTO files (name,size,modtime,dirid,lastaccess,data) VALUES($1::text, $2::bigint, $3::timestamp, $4::integer, localtimestamp, $5) "
"ON CONFLICT ON CONSTRAINT files_pkey DO UPDATE SET "
"size=EXCLUDED.size, modtime=EXCLUDED.modtime, lastaccess=EXCLUDED.lastaccess, data=EXCLUDED.data;",
5, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_COMMAND_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
}
}
return ret;
}

6
src/mirrorfuncs.cpp
Unescape View File

@ -27,7 +27,7 @@ bool MakePath(const MString& dname)
return true;
}
RetVal<std::vector<struct FileInfo>> ReadLocalFileList(const MString& dir, const bool nofollow, const MString& path)
RetVal<std::vector<struct FileInfo>> ReadLocalFileList(const MString& dir, const MString& path)
{
const static MString pref = "ReadLocalFileList";
@ -48,11 +48,11 @@ RetVal<std::vector<struct FileInfo>> ReadLocalFileList(const MString& dir, const
do {
if(dent->d_name[0] != '.')
{
int ret = fstatat(dfd, dent->d_name, &st, nofollow ? AT_SYMLINK_NOFOLLOW : 0);
int ret = fstatat(dfd, dent->d_name, &st, AT_SYMLINK_NOFOLLOW);
if(ret != 0) return {pref, "Can't stat " + path + "/" + dir + "/" + dent->d_name};
if(S_ISDIR(st.st_mode)) // Directory, recurse
{
auto list = ReadLocalFileList(dir + "/" + dent->d_name, nofollow, path + (path.Exist() ? "/" : "") + dent->d_name);
auto list = ReadLocalFileList(dir + "/" + dent->d_name, path + (path.Exist() ? "/" : "") + dent->d_name);
if(!list) return list;
out.insert(out.end(), list.Value().begin(), list.Value().end());
}

1
src/zarr.cpp
Unescape View File

@ -171,7 +171,6 @@ Error ZarrFunctions::GetChunk(const MString& var, const std::vector<size_t>& chu
{
michlib::message(str + " not found in cache, downloading");
CURLRAII myhandle; // TODO: remove this workaround of unknown bug
if(proxyurl.Exist()) curl_easy_setopt(myhandle, CURLOPT_PROXY, proxyurl.Buf());
//auto [out, res] = GetUrl(chandle, str);
auto [out, res] = GetUrl(myhandle, str);
if(res != CURLE_OK) return Error(pref, MString("can't download chunk: ") + chandle.Err());

Write Preview
Loading…
Cancel
Save