From 19f63956a5c4d683fa7ad379fa5a17a329a998d0 Mon Sep 17 00:00:00 2001 From: Michael Uleysky Date: Wed, 2 Oct 2024 12:49:21 +1000 Subject: [PATCH] Added class FileInfoCache - PostgreSQL-based cache for the information about files --- include/cache.h | 30 ++++++ src/cache.cpp | 240 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 270 insertions(+) diff --git a/include/cache.h b/include/cache.h index 497b18c..1bfcdb3 100644 --- a/include/cache.h +++ b/include/cache.h @@ -1,12 +1,16 @@ #pragma once #include "GPL.h" +#include "mirrorfuncs.h" #include #include +#include #include #include #include using michlib::GPL; +using michlib::int1; +using michlib::int4; using michlib::int_cast; using michlib::MString; using michlib::pointer_cast; @@ -415,3 +419,29 @@ inline GenericCache* CreateCache(const MString& cachedesc) return nullptr; } + +class FileInfoCache: public PostgreSQLHelpers +{ + public: + using DataType = std::optional; + using CallbackType = std::function; + + private: + static bool regdest; + CallbackType readfunc; + MString dir; + int4 dirid; + + FileInfoCache() = delete; + + CallbackType::result_type GetData(const MString& fname) const { return readfunc(dir + "/" + fname); } + + void GetDirId(); + + public: + FileInfoCache(CallbackType&& readfunc_, const MString& dir_); + + Error UpdateCache(bool force = false) const; + + DataType GetInfo(const MString& name) const; +}; diff --git a/src/cache.cpp b/src/cache.cpp index 15d52cf..e9c8a5b 100644 --- a/src/cache.cpp +++ b/src/cache.cpp @@ -12,3 +12,243 @@ std::vector PostgreSQLConnection::destructs = {} bool SQLiteCache::regdest = false; bool PostgreSQLCache::regdest = false; + +bool FileInfoCache::regdest = false; + +void FileInfoCache::GetDirId() +{ + if(dirid != 0) return; + const char* params[] = {dir.Buf()}; + int plens[] = {int_cast(dir.Len())}; + int pfor[] = {0}; + + PGresultRAII res = PQexecParams(conn, "SELECT id FROM dirs WHERE name=$1::text;", 1, nullptr, params, plens, pfor, 1); + if(PQresultStatus(res) != PGRES_TUPLES_OK) + { + michlib::errmessage(PQresStatus(PQresultStatus(res))); + michlib::errmessage(PQerrorMessage(conn)); + return; + } + else if(PQntuples(res) == 0) + { + res = PQexecParams(conn, + "INSERT INTO dirs(name,id) VALUES ($1, (SELECT min(num.numid) FROM (SELECT generate_series(1, (SELECT COALESCE((SELECT max(id) FROM dirs), 1)) + " + "1, 1) AS numid) num LEFT JOIN dirs ON dirs.id=num.numid WHERE id IS NULL)) RETURNING id;", + 1, nullptr, params, plens, pfor, 1); + + if(PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK) + { + michlib::errmessage(PQresStatus(PQresultStatus(res))); + michlib::errmessage(PQerrorMessage(conn)); + } + if(PQntuples(res) == 0) return; + } + + if(PQgetlength(res, 0, 0) == sizeof(dirid)) dirid = *pointer_cast(PQgetvalue(res, 0, 0)); + michlib::message("Dirid: ", Invert(dirid)); +} + +FileInfoCache::FileInfoCache(FileInfoCache::CallbackType&& readfunc_, const MString& dir_): readfunc(std::move(readfunc_)), dir(dir_), dirid(0) +{ + if(!conn) return; + + if(!regdest) + { + // Create table + PGresultRAII res = PQexec(conn, "SET client_min_messages=WARNING;"); + + res = PQexec(conn, "BEGIN;" + "CREATE TABLE IF NOT EXISTS dirs(name TEXT PRIMARY KEY, id INTEGER UNIQUE NOT NULL CONSTRAINT id_is_positive CHECK(id>0));" + "CREATE TABLE IF NOT EXISTS files(name TEXT NOT NULL, size BIGINT NOT NULL CONSTRAINT size_is_positive CHECK(size>0), modtime TIMESTAMP(0) NOT NULL, " + "dirid INTEGER REFERENCES dirs(id) ON DELETE CASCADE, lastaccess TIMESTAMP(0) NOT NULL, data BYTEA NOT NULL, PRIMARY KEY(name,dirid));" + "COMMIT;"); + if(PQresultStatus(res) != PGRES_COMMAND_OK) + { + michlib::errmessage(PQresStatus(PQresultStatus(res))); + michlib::errmessage(PQerrorMessage(conn)); + } + + res = PQexec(conn, "SET client_min_messages=NOTICE;"); + + conn.AddDestructor( + [](PostgreSQLConnection::DBType conn) + { + PGresultRAII res = PQexec(conn, "BEGIN;" + "DELETE FROM files WHERE lastaccess+'100 days'::intervald_name[0] != '.') + { + int ret = fstatat(dfd, dent->d_name, &st, 0); + if(ret != 0) return {pref, "Can't stat " + dir + "/" + dent->d_name}; + if(S_ISREG(st.st_mode)) // Regular file + { + const char* params[] = {dent->d_name, pointer_cast(&dirid)}; + int plens[] = {int_cast(strlen(dent->d_name)), sizeof(dirid)}; + int pfor[] = {0, 1}; + bool querysucc = true; + time_t modtime; + size_t size; + + PGresultRAII res = PQexecParams(conn, "SELECT size,modtime FROM files WHERE name=$1::text AND dirid=$2::integer;", 2, nullptr, params, plens, pfor, 1); + + if(PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK) + { + michlib::errmessage(PQresStatus(PQresultStatus(res))); + michlib::errmessage(PQerrorMessage(conn)); + querysucc = false; + } + else if(PQntuples(res) == 0 || PQntuples(res) > 1) + querysucc = false; + + if(querysucc) + { + size = *pointer_cast(PQgetvalue(res, 0, 0)); + modtime = raw2epoch(*pointer_cast(PQgetvalue(res, 0, 1))); + } + else + { + size = int_cast(st.st_size); + modtime = st.st_mtim.tv_sec; + } + + if(!querysucc || force || size != int_cast(st.st_size) || modtime != st.st_mtim.tv_sec) + { + auto ret = GetData(dent->d_name); + // Remove entry + if(!ret && querysucc) + { + PGresultRAII dres = PQexecParams(conn, "DELETE FROM files WHERE name=$1::text AND dirid=$2::integer;", 2, nullptr, params, plens, pfor, 1); + if(PQresultStatus(dres) != PGRES_COMMAND_OK) + { + michlib::errmessage(PQresStatus(PQresultStatus(dres))); + michlib::errmessage(PQerrorMessage(conn)); + } + } + else // Update or insert + { + auto sizei = Invert(size); + auto modtimei = epoch2raw(modtime); + + const char* params[] = {dent->d_name, pointer_cast(&sizei), pointer_cast(&modtimei), pointer_cast(&dirid), ret.value().Buf()}; + int plens[] = {int_cast(strlen(dent->d_name)), sizeof(sizei), sizeof(modtimei), sizeof(dirid), int_cast(ret.value().Len())}; + int pfor[] = {0, 1, 1, 1, 1}; + + PGresultRAII res = PQexecParams(conn, + "INSERT INTO files (name,size,modtime,dirid,lastaccess,data) VALUES($1::text, $2::bigint, $3::timestamp, $4::integer, localtimestamp, $5) " + "ON CONFLICT ON CONSTRAINT files_pkey DO UPDATE SET " + "size=EXCLUDED.size, modtime=EXCLUDED.modtime, lastaccess=EXCLUDED.lastaccess, data=EXCLUDED.data;", + 5, nullptr, params, plens, pfor, 1); + if(PQresultStatus(res) != PGRES_COMMAND_OK) + { + michlib::errmessage(PQresStatus(PQresultStatus(res))); + michlib::errmessage(PQerrorMessage(conn)); + } + } // Insert or update branch + } // Need data update + } // Regular file + } // if(dent->d_name[0] != '.') + dent = readdir(dhandle); + } while(dent != nullptr || errno != 0); + + return Error(); +} + +FileInfoCache::DataType FileInfoCache::GetInfo(const MString& name) const +{ + if(!*this) return GetData(name); + + bool querysucc = true; + MString data; + time_t modtime; + size_t size; + + { + const char* params[] = {name.Buf(), pointer_cast(&dirid)}; + int plens[] = {int_cast(name.Len()), sizeof(dirid)}; + int pfor[] = {0, 1}; + + PGresultRAII res = + PQexecParams(conn, "UPDATE files SET lastaccess=localtimestamp WHERE name=$1::text AND dirid=$2::integer RETURNING data,size,modtime;", 2, nullptr, params, plens, pfor, 1); + if(PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK) + { + michlib::errmessage(PQresStatus(PQresultStatus(res))); + michlib::errmessage(PQerrorMessage(conn)); + querysucc = false; + } + + if(PQntuples(res) == 0 || PQntuples(res) > 1) + { + michlib::errmessage("Data for file ", dir + "/" + name, (PQntuples(res) == 0 ? " not found " : " duplicated "), "in cache"); + querysucc = false; + } + + if(querysucc) + { + data = MString(PQgetvalue(res, 0, 0), PQgetlength(res, 0, 0)); + size = *pointer_cast(PQgetvalue(res, 0, 1)); + modtime = raw2epoch(*pointer_cast(PQgetvalue(res, 0, 2))); + } + } + + { + struct stat st; + + int ret = stat((dir + "/" + name).Buf(), &st); + if(ret != 0) return DataType(); + + if(querysucc && st.st_mtim.tv_sec == modtime && size == int_cast(st.st_size)) return data; + modtime = st.st_mtim.tv_sec; + size = st.st_size; + } + + auto ret = GetData(name); + if(ret) + { + auto sizei = Invert(size); + auto modtimei = epoch2raw(modtime); + + const char* params[] = {name.Buf(), pointer_cast(&sizei), pointer_cast(&modtimei), pointer_cast(&dirid), ret.value().Buf()}; + int plens[] = {int_cast(name.Len()), sizeof(sizei), sizeof(modtimei), sizeof(dirid), int_cast(ret.value().Len())}; + int pfor[] = {0, 1, 1, 1, 1}; + + PGresultRAII res = PQexecParams(conn, + "INSERT INTO files (name,size,modtime,dirid,lastaccess,data) VALUES($1::text, $2::bigint, $3::timestamp, $4::integer, localtimestamp, $5) " + "ON CONFLICT ON CONSTRAINT files_pkey DO UPDATE SET " + "size=EXCLUDED.size, modtime=EXCLUDED.modtime, lastaccess=EXCLUDED.lastaccess, data=EXCLUDED.data;", + 5, nullptr, params, plens, pfor, 1); + if(PQresultStatus(res) != PGRES_COMMAND_OK) + { + michlib::errmessage(PQresStatus(PQresultStatus(res))); + michlib::errmessage(PQerrorMessage(conn)); + } + } + + return ret; +}