From a12186a1ed6dc31d0ae146db356edcbad35dacef Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Thu, 25 Aug 2016 23:15:31 +1000
Subject: [PATCH 01/33] * draft for source library api

---
 src/sources/source.h | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 src/sources/source.h

diff --git a/src/sources/source.h b/src/sources/source.h
new file mode 100644
index 0000000..ad889cf
--- /dev/null
+++ b/src/sources/source.h
@@ -0,0 +1,20 @@
+/* Copyright 2016 Alex 'AdUser' Z (ad_user@runbox.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <stdbool.h>
+
+#define INIT_MAX 256
+
+typedef struct _config cfg_t;
+
+extern cfg_t *create(const char *init);
+extern bool   config(cfg_t *c, const char *key, const char *value);
+extern bool    ready(cfg_t *c);
+extern char   *error(cfg_t *c);
+extern bool    start(cfg_t *c);
+extern bool     stop(cfg_t *c);
+extern bool     next(cfg_t *c, char *buf, size_t bufsize, bool reset);
+extern void  destroy(cfg_t *c);

From c26b527fb34c5ea37d716b4acbc149bd023a16b2 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 17:47:09 +1000
Subject: [PATCH 02/33] * src/sources/source.h : more includes

---
 src/sources/source.h | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/sources/source.h b/src/sources/source.h
index ad889cf..bc65dbc 100644
--- a/src/sources/source.h
+++ b/src/sources/source.h
@@ -4,7 +4,14 @@
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */
+#include <assert.h>
 #include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "../strlcpy.h"
 
 #define INIT_MAX 256
 

From 9ad8fde396dd7cdd480b86ed9c9ac5b0e550ca99 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 17:47:28 +1000
Subject: [PATCH 03/33] + src/sources/files.c : draft

---
 src/sources/files.c | 148 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 148 insertions(+)
 create mode 100644 src/sources/files.c

diff --git a/src/sources/files.c b/src/sources/files.c
new file mode 100644
index 0000000..0d61420
--- /dev/null
+++ b/src/sources/files.c
@@ -0,0 +1,148 @@
+/* Copyright 2016 Alex 'AdUser' Z (ad_user@runbox.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include "source.h"
+
+#include <limits.h>
+#include <sys/file.h>
+#include <sys/stat.h>
+
+#include <glob.h>
+
+typedef struct f2b_file_t {
+  struct f2b_file_t *next;
+  bool opened;
+  char path[PATH_MAX];
+  FILE *fd;
+  struct stat st;
+} f2b_file_t;
+
+struct _config {
+  char error[256];
+  f2b_file_t *files;
+};
+
+static bool
+file_open(f2b_file_t *file, const char *path) {
+  struct stat st;
+  char buf[PATH_MAX] = "";
+
+  assert(file != NULL);
+  assert(path != NULL || file->path[0] != '\0');
+
+  strlcpy(buf, path ? path : file->path, sizeof(buf));
+
+  memset(file, 0x0, sizeof(f2b_file_t));
+
+  if (stat(buf, &st) != 0)
+    return false;
+
+  if (!(S_ISREG(st.st_mode) || S_ISFIFO(st.st_mode)))
+    return false;
+
+  if ((file->fd = fopen(buf, "r")) == NULL)
+    return false;
+
+  if (S_ISREG(st.st_mode) && fseek(file->fd, 0, SEEK_END) < 0)
+    return false;
+
+  memcpy(&file->st, &st, sizeof(st));
+  strlcpy(file->path, buf, sizeof(file->path));
+  file->opened = true;
+
+  return true;
+}
+
+static void
+file_close(f2b_file_t *file) {
+  assert(file != NULL);
+
+  if (file->fd)
+    fclose(file->fd);
+
+  file->opened = false;
+  file->fd = NULL;
+}
+
+static bool
+file_rotated(const f2b_file_t *file) {
+  struct stat st;
+
+  assert(file != NULL);
+
+  if (!file->opened)
+    return true;
+
+  if (stat(file->path, &st) != 0)
+    return true;
+
+  if (file->st.st_dev  != st.st_dev ||
+      file->st.st_ino  != st.st_ino ||
+      file->st.st_size  > st.st_size)
+    return true;
+
+  return false;
+}
+
+static bool
+file_getline(const f2b_file_t *file, char *buf, size_t bufsize) {
+  assert(file != NULL);
+  assert(buf != NULL);
+
+  if (feof(file->fd))
+    clearerr(file->fd);
+  /* fread()+EOF set is implementation defined */
+  if (fgets(buf, bufsize, file->fd) != NULL)
+    return true;
+
+  return false;
+}
+
+static f2b_file_t *
+list_append(f2b_file_t *list, f2b_file_t *file) {
+  assert(file != NULL);
+
+  if (list != NULL)
+    return file->next = list;
+  return file;
+}
+
+static f2b_file_t *
+list_from_glob(const char *pattern) {
+  f2b_file_t *file = NULL;
+  f2b_file_t *files = NULL;
+  glob_t globbuf;
+
+  assert(pattern != NULL);
+
+  if (glob(pattern, GLOB_MARK | GLOB_NOESCAPE, NULL, &globbuf) != 0)
+    return NULL;
+
+  for (size_t i = 0; i < globbuf.gl_pathc; i++) {
+    if ((file = calloc(1, sizeof(f2b_file_t))) == NULL)
+      continue;
+    if (file_open(file, globbuf.gl_pathv[i]) == false) {
+      /* f2b_log_msg(log_error, "can't open file: %s: %s", globbuf.gl_pathv[i], strerror(errno)); */
+      free(file);
+      continue;
+    }
+    files = list_append(files, file);
+  }
+
+  globfree(&globbuf);
+  return files;
+}
+
+static void
+list_destroy(f2b_file_t *list) {
+  f2b_file_t *next = NULL;
+
+  for (; list != NULL; list = next) {
+    next = list->next;
+    file_close(list);
+    free(list);
+  }
+}

From cba1cc59ecadf5daec4123b05cc696d53c482810 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 17:47:54 +1000
Subject: [PATCH 04/33] + src/sources/CMakeLists.txt

---
 src/CMakeLists.txt         |  1 +
 src/sources/CMakeLists.txt | 10 ++++++++++
 2 files changed, 11 insertions(+)
 create mode 100644 src/sources/CMakeLists.txt

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 56bc59d..61074fe 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -33,3 +33,4 @@ install(TARGETS "f2b-filter-test" "f2b-backend-test" RUNTIME DESTINATION ${CMAKE
 
 add_subdirectory("backends")
 add_subdirectory("filters")
+add_subdirectory("sources")
diff --git a/src/sources/CMakeLists.txt b/src/sources/CMakeLists.txt
new file mode 100644
index 0000000..f2d70c5
--- /dev/null
+++ b/src/sources/CMakeLists.txt
@@ -0,0 +1,10 @@
+set(CMAKE_INCLUDE_CURRENT_DIR ON)
+set(SOURCES "")
+
+add_library("f2b_source_files" MODULE "files.c" "../strlcpy.c")
+list(APPEND SOURCES "f2b_source_files")
+
+message(STATUS "- Sources  : ${SOURCES}")
+
+install(TARGETS ${SOURCES}
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})

From 3201c949c57efaecb1d68e204f27b920231cd62f Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 17:59:17 +1000
Subject: [PATCH 05/33] * src/source/files.c : inline list_append()

---
 src/sources/files.c | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/src/sources/files.c b/src/sources/files.c
index 0d61420..9a5b35d 100644
--- a/src/sources/files.c
+++ b/src/sources/files.c
@@ -101,15 +101,6 @@ file_getline(const f2b_file_t *file, char *buf, size_t bufsize) {
   return false;
 }
 
-static f2b_file_t *
-list_append(f2b_file_t *list, f2b_file_t *file) {
-  assert(file != NULL);
-
-  if (list != NULL)
-    return file->next = list;
-  return file;
-}
-
 static f2b_file_t *
 list_from_glob(const char *pattern) {
   f2b_file_t *file = NULL;
@@ -129,7 +120,12 @@ list_from_glob(const char *pattern) {
       free(file);
       continue;
     }
-    files = list_append(files, file);
+    if (files != NULL) {
+      file->next = files;
+      files = file;
+    } else {
+      files = file;
+    }
   }
 
   globfree(&globbuf);

From b424b7fdfeeb77742c75323c754475c7e496db83 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 17:59:43 +1000
Subject: [PATCH 06/33] * src/source/files.c : list_destroy() -> stop()

---
 src/sources/files.c | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/sources/files.c b/src/sources/files.c
index 9a5b35d..d2cbf0b 100644
--- a/src/sources/files.c
+++ b/src/sources/files.c
@@ -132,13 +132,15 @@ list_from_glob(const char *pattern) {
   return files;
 }
 
-static void
-list_destroy(f2b_file_t *list) {
+bool
+stop(cfg_t *cfg) {
   f2b_file_t *next = NULL;
 
-  for (; list != NULL; list = next) {
-    next = list->next;
-    file_close(list);
-    free(list);
+  for (; cfg->files != NULL; cfg->files = next) {
+    next = cfg->files->next;
+    file_close(cfg->files);
+    free(cfg->files);
   }
+
+  return true;
 }

From 6f826290423a638e04128ddcbf15c75959395d0d Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 17:59:59 +1000
Subject: [PATCH 07/33] * src/source/files.c : changes in config struct

---
 src/sources/files.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/sources/files.c b/src/sources/files.c
index d2cbf0b..cca929e 100644
--- a/src/sources/files.c
+++ b/src/sources/files.c
@@ -21,8 +21,10 @@ typedef struct f2b_file_t {
 } f2b_file_t;
 
 struct _config {
+  char glob[256];
   char error[256];
   f2b_file_t *files;
+  f2b_file_t *current;
 };
 
 static bool

From 6982cbc8062049a5a4330fc5b4cb4d84937765d7 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 20:35:30 +1000
Subject: [PATCH 08/33] * src/sources/source.h : more includes

---
 src/sources/source.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/sources/source.h b/src/sources/source.h
index bc65dbc..125bfe7 100644
--- a/src/sources/source.h
+++ b/src/sources/source.h
@@ -5,6 +5,7 @@
  * published by the Free Software Foundation.
  */
 #include <assert.h>
+#include <errno.h>
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>

From b7d9b4988bbbce352f8ced2db3ee35f144ef9e21 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 20:35:39 +1000
Subject: [PATCH 09/33] * src/sources/source.h : tune api

---
 src/sources/source.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/sources/source.h b/src/sources/source.h
index 125bfe7..f344757 100644
--- a/src/sources/source.h
+++ b/src/sources/source.h
@@ -22,7 +22,8 @@ extern cfg_t *create(const char *init);
 extern bool   config(cfg_t *c, const char *key, const char *value);
 extern bool    ready(cfg_t *c);
 extern char   *error(cfg_t *c);
+extern void  errorcb(cfg_t *c, void (*cb)(char *errstr));
 extern bool    start(cfg_t *c);
-extern bool     stop(cfg_t *c);
 extern bool     next(cfg_t *c, char *buf, size_t bufsize, bool reset);
+extern bool     stop(cfg_t *c);
 extern void  destroy(cfg_t *c);

From b80a6b323e5b88479adf2c0c45c72d79e8144075 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 20:37:04 +1000
Subject: [PATCH 10/33] * src/sources/files.c : implement
 {create,config,ready,error,errorcb,start,destroy}()

---
 src/sources/files.c | 84 ++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 72 insertions(+), 12 deletions(-)

diff --git a/src/sources/files.c b/src/sources/files.c
index cca929e..777e672 100644
--- a/src/sources/files.c
+++ b/src/sources/files.c
@@ -21,8 +21,9 @@ typedef struct f2b_file_t {
 } f2b_file_t;
 
 struct _config {
-  char glob[256];
+  char path[256];
   char error[256];
+  void (*errcb)(char *errstr);
   f2b_file_t *files;
   f2b_file_t *current;
 };
@@ -103,35 +104,80 @@ file_getline(const f2b_file_t *file, char *buf, size_t bufsize) {
   return false;
 }
 
-static f2b_file_t *
-list_from_glob(const char *pattern) {
+cfg_t *
+create(const char *init) {
+  cfg_t *cfg = NULL;
+  assert(init != NULL);
+  if ((cfg = calloc(1, sizeof(cfg_t))) == NULL)
+    return NULL;
+  strlcpy(cfg->path, init, sizeof(cfg->path));
+  return cfg;
+}
+
+bool
+config(cfg_t *cfg, const char *key, const char *value) {
+  assert(cfg != NULL);
+  assert(key   != NULL);
+  assert(value != NULL);
+  /* no options */
+  return false;
+}
+
+bool
+ready(cfg_t *cfg) {
+  assert(cfg != NULL);
+  if (cfg->files)
+    return true;
+  return false;
+}
+
+char *
+error(cfg_t *cfg) {
+  assert(cfg != NULL);
+
+  return cfg->error;
+}
+
+void
+errorcb(cfg_t *cfg, void (*cb)(char *errstr)) {
+  assert(cfg != NULL);
+  assert(cb  != NULL);
+
+  cfg->errcb = cb;
+}
+
+bool
+start(cfg_t *cfg) {
   f2b_file_t *file = NULL;
-  f2b_file_t *files = NULL;
   glob_t globbuf;
 
-  assert(pattern != NULL);
+  assert(cfg != NULL);
 
-  if (glob(pattern, GLOB_MARK | GLOB_NOESCAPE, NULL, &globbuf) != 0)
+  if (glob(cfg->path, GLOB_MARK | GLOB_NOESCAPE, NULL, &globbuf) != 0)
     return NULL;
 
   for (size_t i = 0; i < globbuf.gl_pathc; i++) {
     if ((file = calloc(1, sizeof(f2b_file_t))) == NULL)
       continue;
     if (file_open(file, globbuf.gl_pathv[i]) == false) {
-      /* f2b_log_msg(log_error, "can't open file: %s: %s", globbuf.gl_pathv[i], strerror(errno)); */
+      if (cfg->errcb) {
+        snprintf(cfg->error, sizeof(cfg->error), "can't open file: %s -- %s",
+          globbuf.gl_pathv[i], strerror(errno));
+        cfg->errcb(cfg->error);
+      }
       free(file);
       continue;
     }
-    if (files != NULL) {
-      file->next = files;
-      files = file;
+    if (cfg->files == NULL) {
+      cfg->files = file;
     } else {
-      files = file;
+      file->next = cfg->files;
+      cfg->files = file;
     }
   }
 
   globfree(&globbuf);
-  return files;
+  return true;
 }
 
 bool
@@ -146,3 +192,17 @@ stop(cfg_t *cfg) {
 
   return true;
 }
+
+bool
+next(cfg_t *cfg, char *buf, size_t bufsize, bool reset) {
+  assert(cfg != NULL);
+  /* TODO */
+  return false;
+}
+
+void
+destroy(cfg_t *cfg) {
+  assert(cfg != NULL);
+
+  free(cfg);
+}

From ff807bbde4523e96d12f518a9dd9db89b08b612c Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 20:50:35 +1000
Subject: [PATCH 11/33] * src/sources/files.c : implement next()

---
 src/sources/files.c | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/src/sources/files.c b/src/sources/files.c
index 777e672..33e49f9 100644
--- a/src/sources/files.c
+++ b/src/sources/files.c
@@ -196,7 +196,26 @@ stop(cfg_t *cfg) {
 bool
 next(cfg_t *cfg, char *buf, size_t bufsize, bool reset) {
   assert(cfg != NULL);
-  /* TODO */
+  assert(buf != NULL);
+  assert(bufsize > 0);
+
+  if (reset || cfg->current == NULL)
+    cfg->current = cfg->files;
+
+  for (f2b_file_t *file = cfg->current; file != NULL; file = file->next) {
+    if (file_rotated(file))
+      file_close(file);
+    if (!file->opened && !file_open(file, NULL)) {
+      if (cfg->errcb) {
+        snprintf(cfg->error, sizeof(cfg->error), "can't open file -- %s", file->path);
+        cfg->errcb(cfg->error);
+      }
+      continue;
+    }
+    if (file_getline(file, buf, bufsize))
+      return true;
+  }
+
   return false;
 }
 

From 91e2fd0aa6684ee03d67ba9d5ce2b3443c536e37 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 21:30:25 +1000
Subject: [PATCH 12/33] - t/t_logfile.c

---
 t/CMakeLists.txt |  2 --
 t/t_logfile.c    | 58 ------------------------------------------------
 2 files changed, 60 deletions(-)
 delete mode 100644 t/t_logfile.c

diff --git a/t/CMakeLists.txt b/t/CMakeLists.txt
index 6fdee34..629144f 100644
--- a/t/CMakeLists.txt
+++ b/t/CMakeLists.txt
@@ -3,13 +3,11 @@ enable_testing()
 set(SRC_DIR "../src")
 
 add_executable("t_cmsg"         "t_cmsg.c"    "${SRC_DIR}/strlcpy.c" "${SRC_DIR}/cmsg.c")
-add_executable("t_logfile"      "t_logfile.c" "${SRC_DIR}/strlcpy.c" "${SRC_DIR}/logfile.c")
 add_executable("t_matches"      "t_matches.c" "${SRC_DIR}/strlcpy.c" "${SRC_DIR}/matches.c")
 add_executable("t_ipaddr"       "t_ipaddr.c"  "${SRC_DIR}/strlcpy.c" "${SRC_DIR}/matches.c" "${SRC_DIR}/ipaddr.c")
 add_executable("t_config_param" "t_config_param.c" "${SRC_DIR}/strlcpy.c" "${SRC_DIR}/config.c" "${SRC_DIR}/log.c")
 
 add_test("tests/f2b_cmsg_*"    "t_cmsg")
-add_test("tests/f2b_logfile_*" "t_logfile")
 add_test("tests/f2b_matches_*" "t_matches")
 add_test("tests/f2b_ipaddr_*"  "t_ipaddr")
 add_test("tests/f2b_config_param*" "t_config_param")
diff --git a/t/t_logfile.c b/t/t_logfile.c
deleted file mode 100644
index 5885889..0000000
--- a/t/t_logfile.c
+++ /dev/null
@@ -1,58 +0,0 @@
-#include "../src/common.h"
-#include "../src/logfile.h"
-
-int main() {
-  f2b_logfile_t file;
-  char filename[] = "/tmp/f2b-test.XXXXXX";
-  int fd = 0;
-  int ret;
-  bool res;
-  FILE *wfd = NULL;
-  char buf[2048];
-
-  UNUSED(res);
-  UNUSED(ret);
-
-  fd = mkstemp(filename);
-  assert(fd > 0);
-  wfd = fdopen(fd, "a");
-  assert(wfd != NULL);
-  ret = fputs("test1\n", wfd);
-  assert(ret > 0);
-  ret = fflush(wfd);
-  assert(ret == 0);
-
-  res = f2b_logfile_open(&file, filename);
-  assert(res == true);
-  res = f2b_logfile_getline(&file, buf, sizeof(buf));
-  assert(res == false);
-
-  ret = fputs("test2\n", wfd);
-  assert(ret > 0);
-  ret = fflush(wfd);
-  assert(ret == 0);
-
-  res = f2b_logfile_getline(&file, buf, sizeof(buf));
-  assert(res == true);
-  ret = strncmp(buf, "test2\n", sizeof(buf));
-  assert(ret == 0);
-
-  res = f2b_logfile_getline(&file, buf, sizeof(buf));
-  assert(res == false);
-
-  fclose(wfd);
-  close(fd);
-  unlink(filename);
-
-  wfd = fopen(filename, "a");
-  assert(wfd != NULL);
-
-  res = f2b_logfile_rotated(&file);
-  assert(res == true);
-
-  f2b_logfile_close(&file);
-  fclose(wfd);
-  unlink(filename);
-
-  exit(0);
-}

From fa0fde9b529d7ac00a0020b400ab87c7017188f7 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 21:38:03 +1000
Subject: [PATCH 13/33] * src/sources/source.h : tune api

---
 src/sources/files.c  | 2 +-
 src/sources/source.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/sources/files.c b/src/sources/files.c
index 33e49f9..ac8c2c3 100644
--- a/src/sources/files.c
+++ b/src/sources/files.c
@@ -139,7 +139,7 @@ error(cfg_t *cfg) {
 }
 
 void
-errorcb(cfg_t *cfg, void (*cb)(char *errstr)) {
+errcb(cfg_t *cfg, void (*cb)(char *errstr)) {
   assert(cfg != NULL);
   assert(cb  != NULL);
 
diff --git a/src/sources/source.h b/src/sources/source.h
index f344757..79b464e 100644
--- a/src/sources/source.h
+++ b/src/sources/source.h
@@ -22,7 +22,7 @@ extern cfg_t *create(const char *init);
 extern bool   config(cfg_t *c, const char *key, const char *value);
 extern bool    ready(cfg_t *c);
 extern char   *error(cfg_t *c);
-extern void  errorcb(cfg_t *c, void (*cb)(char *errstr));
+extern void    errcb(cfg_t *c, void (*cb)(char *errstr));
 extern bool    start(cfg_t *c);
 extern bool     next(cfg_t *c, char *buf, size_t bufsize, bool reset);
 extern bool     stop(cfg_t *c);

From f109d7a080d411118b0d17954eab1681832522b0 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 21:41:34 +1000
Subject: [PATCH 14/33] * filter.h : reorganize definitions

---
 src/filter.h | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/filter.h b/src/filter.h
index 5fa0671..9eaf2fe 100644
--- a/src/filter.h
+++ b/src/filter.h
@@ -23,12 +23,11 @@ typedef struct f2b_filter_t {
   void  (*destroy) (void *cfg);
 } f2b_filter_t;
 
-f2b_filter_t * f2b_filter_create (f2b_config_section_t *config, const char *id);
-void           f2b_filter_destroy(f2b_filter_t *b);
-
-bool f2b_filter_append(f2b_filter_t *b, const char *pattern);
-bool f2b_filter_match(f2b_filter_t *b, const char *line, char *buf, size_t buf_size);
-const char * f2b_filter_error(f2b_filter_t *b);
-void f2b_filter_stats (f2b_filter_t *b, char *res, size_t ressize);
+f2b_filter_t * f2b_filter_create  (f2b_config_section_t *config, const char *id);
+const char *   f2b_filter_error   (f2b_filter_t *f);
+bool           f2b_filter_append  (f2b_filter_t *f, const char *pattern);
+bool           f2b_filter_match   (f2b_filter_t *f, const char *line, char *buf, size_t buf_size);
+void           f2b_filter_stats   (f2b_filter_t *f, char *res, size_t ressize);
+void           f2b_filter_destroy (f2b_filter_t *f);
 
 #endif /* F2B_FILTER_H_ */

From b78e45ed120acb429b90ddfeaa12fa722ecdfae2 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 22:02:28 +1000
Subject: [PATCH 15/33] + src/source.[ch]

---
 src/CMakeLists.txt |   2 +-
 src/source.c       | 113 +++++++++++++++++++++++++++++++++++++++++++++
 src/source.h       |  32 +++++++++++++
 3 files changed, 146 insertions(+), 1 deletion(-)
 create mode 100644 src/source.c
 create mode 100644 src/source.h

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 61074fe..9bc1a9f 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -1,6 +1,6 @@
 set(CMAKE_INCLUDE_CURRENT_DIR ON)
 
-set(SOURCES "daemon.c" "strlcpy.c" "logfile.c" "log.c" "matches.c" "ipaddr.c" "filelist.c" "filter.c" "config.c" "jail.c" "backend.c")
+set(SOURCES "daemon.c" "strlcpy.c" "logfile.c" "log.c" "matches.c" "source.c" "ipaddr.c" "filelist.c" "filter.c" "config.c" "jail.c" "backend.c")
 
 if (WITH_CSOCKET)
   list(APPEND SOURCES "csocket.c" "cmsg.c")
diff --git a/src/source.c b/src/source.c
new file mode 100644
index 0000000..7c11f03
--- /dev/null
+++ b/src/source.c
@@ -0,0 +1,113 @@
+/* Copyright 2016 Alex 'AdUser' Z (ad_user@runbox.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include "common.h"
+#include "source.h"
+
+#define SOURCE_LIBRARY_PARAM "load"
+
+f2b_source_t *
+f2b_source_create(f2b_config_section_t *config, const char *init, void (*errcb)(const char *)) {
+  f2b_config_param_t *param = NULL;
+  f2b_source_t *source = NULL;
+  int flags = RTLD_NOW | RTLD_LOCAL;
+  const char *dlerr = NULL;
+
+  assert(config != NULL);
+  assert(config->type == t_source);
+
+  param = f2b_config_param_find(config->param, SOURCE_LIBRARY_PARAM);
+  if (!param) {
+    f2b_log_msg(log_error, "can't find '%s' param in source config", SOURCE_LIBRARY_PARAM);
+    return NULL;
+  }
+
+  if ((source = calloc(1, sizeof(f2b_source_t))) == NULL)
+    return NULL;
+
+  if ((source->h = dlopen(param->value, flags)) == NULL)
+     goto cleanup;
+  if ((*(void **) (&source->create)  = dlsym(source->h, "create"))  == NULL)
+    goto cleanup;
+  if ((*(void **) (&source->config)  = dlsym(source->h, "config"))  == NULL)
+    goto cleanup;
+  if ((*(void **) (&source->ready)   = dlsym(source->h, "ready"))   == NULL)
+    goto cleanup;
+  if ((*(void **) (&source->error)   = dlsym(source->h, "error"))   == NULL)
+    goto cleanup;
+  if ((*(void **) (&source->errcb)   = dlsym(source->h, "errcb"))   == NULL)
+    goto cleanup;
+  if ((*(void **) (&source->start)   = dlsym(source->h, "start"))   == NULL)
+    goto cleanup;
+  if ((*(void **) (&source->next)    = dlsym(source->h, "next"))    == NULL)
+    goto cleanup;
+  if ((*(void **) (&source->stop)    = dlsym(source->h, "stop"))    == NULL)
+    goto cleanup;
+  if ((*(void **) (&source->destroy) = dlsym(source->h, "destroy")) == NULL)
+    goto cleanup;
+
+  if ((source->cfg = source->create(init)) == NULL) {
+    f2b_log_msg(log_error, "source create config failed");
+    goto cleanup;
+  }
+
+  /* try init */
+  for (param = config->param; param != NULL; param = param->next) {
+    if (strcmp(param->name, SOURCE_LIBRARY_PARAM) == 0)
+      continue;
+    if (source->config(source->cfg, param->name, param->value))
+      continue;
+    f2b_log_msg(log_warn, "param pair not accepted by source '%s': %s=%s",
+      config->name, param->name, param->value);
+  }
+
+  if (errcb)
+    source->errcb(source->cfg, errcb);
+
+  if (source->ready(source->cfg))
+    return source;
+
+  /* still not ready */
+  f2b_log_msg(log_error, "source '%s' not fully configured", config->name);
+
+  cleanup:
+  dlerr = dlerror();
+  if (dlerr)
+    f2b_log_msg(log_error, "source load error: %s", dlerr);
+  if (source->h) {
+    if (source->cfg && source->destroy)
+      source->destroy(source->cfg);
+    dlclose(source->h);
+  }
+  free(source);
+  return NULL;
+}
+
+void
+f2b_source_destroy(f2b_source_t *source) {
+  assert(source != NULL);
+  source->destroy(source->cfg);
+  dlclose(source->h);
+  free(source);
+}
+
+bool
+f2b_source_next(f2b_source_t *source, char *buf, size_t bufsize, bool reset) {
+  assert(source != NULL);
+  return source->next(source->cfg, buf, bufsize, reset);
+}
+
+#define SOURCE_CMD_ARG0(CMD, RETURNS) \
+RETURNS \
+f2b_source_ ## CMD(f2b_source_t *source) { \
+  assert(source != NULL); \
+  return source->CMD(source->cfg); \
+}
+
+SOURCE_CMD_ARG0(error, const char *)
+SOURCE_CMD_ARG0(start, bool)
+SOURCE_CMD_ARG0(stop,  bool)
+SOURCE_CMD_ARG0(ready, bool)
diff --git a/src/source.h b/src/source.h
new file mode 100644
index 0000000..2766f68
--- /dev/null
+++ b/src/source.h
@@ -0,0 +1,32 @@
+/* Copyright 2016 Alex 'AdUser' Z (ad_user@runbox.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef F2B_SOURCE_H_
+#define F2B_SOURCE_H_
+
+#include "config.h"
+#include "log.h"
+
+typedef struct f2b_source_t {
+  void *h;
+  void *cfg;
+  void *(*create)  (const char *init);
+  bool  (*config)  (void *cfg, const char *key, const char *value);
+  bool  (*ready)   (void *cfg);
+  char *(*error)   (void *cfg);
+  void  (*errcb)   (void *cfg, void (*cb)(const char *errstr));
+  bool  (*start)   (void *cfg);
+  bool  (*next)    (void *cfg, char *buf, size_t bufsize, bool reset);
+  bool  (*stop)    (void *cfg);
+  void  (*destroy) (void *cfg);
+} f2b_source_t;
+
+f2b_source_t * f2b_source_create  (f2b_config_section_t *config, const char *init, void (*errcb)(const char *));
+void           f2b_source_destroy (f2b_source_t *s);
+bool           f2b_source_next    (f2b_source_t *s, char *buf, size_t bufsize, bool reset);
+const char *   f2b_source_error   (f2b_source_t *s);
+
+#endif /* F2B_SOURCE_H_ */

From 82335db1c7323af11edef8c7c737fbbc827df69c Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 22:27:41 +1000
Subject: [PATCH 16/33] * src/config.[ch] : add support of 'source' section
 type

---
 src/config.c | 8 ++++++++
 src/config.h | 2 ++
 2 files changed, 10 insertions(+)

diff --git a/src/config.c b/src/config.c
index b3a521c..1aacbf9 100644
--- a/src/config.c
+++ b/src/config.c
@@ -132,6 +132,13 @@ f2b_config_section_create(const char *src) {
     return section;
   }
 
+  name = "source:";
+  if (strncmp(line, name, strlen(name)) == 0) {
+    section->type = t_source;
+    strlcpy(section->name, line + strlen(name), sizeof(section->name));
+    return section;
+  }
+
   name = "backend:";
   if (strncmp(line, name, strlen(name)) == 0) {
     section->type = t_backend;
@@ -305,6 +312,7 @@ f2b_config_section_append(f2b_config_t *config, f2b_config_section_t *section) {
   switch (section->type) {
     case t_main:     s = &config->main;     break;
     case t_defaults: s = &config->defaults; break;
+    case t_source:   s = &config->sources;  break;
     case t_filter:   s = &config->filters;  break;
     case t_backend:  s = &config->backends; break;
     case t_jail:     s = &config->jails;    break;
diff --git a/src/config.h b/src/config.h
index 249f3c5..e7fe646 100644
--- a/src/config.h
+++ b/src/config.h
@@ -16,6 +16,7 @@ typedef enum f2b_section_type {
   t_unknown = 0,
   t_main,
   t_defaults,
+  t_source,
   t_filter,
   t_backend,
   t_jail,
@@ -38,6 +39,7 @@ typedef struct f2b_config_section_t {
 typedef struct f2b_config_t {
   f2b_config_section_t *main;
   f2b_config_section_t *defaults;
+  f2b_config_section_t *sources;
   f2b_config_section_t *filters;
   f2b_config_section_t *backends;
   f2b_config_section_t *jails;

From 0ad3e37f2ea2df04fcb2535e6fdea7a8765c0ed6 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 22:28:26 +1000
Subject: [PATCH 17/33] * src/log.[ch] : add specific log callback for use in
 sources

---
 src/log.c | 4 ++++
 src/log.h | 1 +
 2 files changed, 5 insertions(+)

diff --git a/src/log.c b/src/log.c
index 7e7411a..16007ec 100644
--- a/src/log.c
+++ b/src/log.c
@@ -65,6 +65,10 @@ void f2b_log_msg(log_msgtype_t l, const char *fmt, ...) {
   return;
 }
 
+void f2b_log_error_cb(const char *errstr) {
+  f2b_log_msg(log_error, "%s", errstr);
+}
+
 void f2b_log_set_level(const char *level) {
   if (strcmp(level, "debug") == 0) { minlevel = log_debug; return; }
   if (strcmp(level, "info")  == 0) { minlevel = log_info;  return; }
diff --git a/src/log.h b/src/log.h
index 7158410..a727b6e 100644
--- a/src/log.h
+++ b/src/log.h
@@ -20,6 +20,7 @@ typedef enum {
 
 void f2b_log_msg(log_msgtype_t l, const char *fmt, ...)
   __attribute__ ((format (printf, 2, 3)));
+void f2b_log_error_cb(const char *errstr);
 void f2b_log_set_level(const char *level);
 void f2b_log_to_file  (const char *path);
 void f2b_log_to_stderr();

From e8435e3e2bd5641e07a7b3af1cc535cf4e72fb4a Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 22:28:58 +1000
Subject: [PATCH 18/33] * src/source.h : export f2b_source_{start,stop}()

---
 src/source.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/source.h b/src/source.h
index 2766f68..64b8040 100644
--- a/src/source.h
+++ b/src/source.h
@@ -26,7 +26,9 @@ typedef struct f2b_source_t {
 
 f2b_source_t * f2b_source_create  (f2b_config_section_t *config, const char *init, void (*errcb)(const char *));
 void           f2b_source_destroy (f2b_source_t *s);
+bool           f2b_source_start   (f2b_source_t *s);
 bool           f2b_source_next    (f2b_source_t *s, char *buf, size_t bufsize, bool reset);
+bool           f2b_source_stop    (f2b_source_t *s);
 const char *   f2b_source_error   (f2b_source_t *s);
 
 #endif /* F2B_SOURCE_H_ */

From 4e08640917ba66bb47507b55643b040c2cf10eb2 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 22:31:21 +1000
Subject: [PATCH 19/33] * src/jail.h : chg jail definition

---
 src/jail.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/jail.h b/src/jail.h
index 426fbbd..24a05d9 100644
--- a/src/jail.h
+++ b/src/jail.h
@@ -11,6 +11,7 @@
 #include "logfile.h"
 #include "ipaddr.h"
 #include "config.h"
+#include "source.h"
 #include "filter.h"
 #include "backend.h"
 #include "filelist.h"
@@ -35,9 +36,10 @@ typedef struct f2b_jail_t {
   char source_name[CONFIG_KEY_MAX];
   char source_init[CONFIG_VAL_MAX];
   f2b_logfile_t *logfiles;
+  f2b_source_t  *source;
   f2b_filter_t  *filter;
-  f2b_ipaddr_t  *ipaddrs;
   f2b_backend_t *backend;
+  f2b_ipaddr_t  *ipaddrs;
 } f2b_jail_t;
 
 void f2b_jail_parse_compound_value(const char *value, char *name, char *init);

From 46d47b3ce3f85c19882303a18f1531ecabe643d1 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 22:32:30 +1000
Subject: [PATCH 20/33] * src/jail.c : f2b_jail_init() : use source

---
 src/jail.c | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/src/jail.c b/src/jail.c
index 5f0b841..b30ab30 100644
--- a/src/jail.c
+++ b/src/jail.c
@@ -308,6 +308,7 @@ f2b_jail_process(f2b_jail_t *jail) {
 
 bool
 f2b_jail_init(f2b_jail_t *jail, f2b_config_t *config) {
+  f2b_config_section_t * s_section = NULL;
   f2b_config_section_t * b_section = NULL;
   f2b_config_section_t * f_section = NULL;
 
@@ -319,13 +320,8 @@ f2b_jail_init(f2b_jail_t *jail, f2b_config_t *config) {
     f2b_log_msg(log_error, "jail '%s': missing 'source' parameter", jail->name);
     return false;
   }
-  /* TODO: temp stub */
-  if (strcmp(jail->source_name, "files") != 0) {
-    f2b_log_msg(log_error, "jail '%s': 'source' supports only 'files' for now", jail->name);
-    return false;
-  }
-  if (jail->source_init[0] == '\0') {
-    f2b_log_msg(log_error, "jail '%s': 'source' requires file or files pattern", jail->name);
+  if ((s_section = f2b_config_section_find(config->sources, jail->source_name)) == NULL) {
+    f2b_log_msg(log_error, "jail '%s': no source with name '%s'", jail->name, jail->source_name);
     return false;
   }
 
@@ -350,8 +346,13 @@ f2b_jail_init(f2b_jail_t *jail, f2b_config_t *config) {
   }
 
   /* init all */
-  if ((jail->logfiles = f2b_filelist_from_glob(jail->source_init)) == NULL) {
-    f2b_log_msg(log_error, "jail '%s': no files matching '%s' pattern", jail->name, jail->source_init);
+  if ((jail->source = f2b_source_create(s_section, jail->source_init, f2b_log_error_cb)) == NULL) {
+    f2b_log_msg(log_error, "jail '%s': no regexps loaded from '%s'", jail->name, jail->source_init);
+    goto cleanup;
+  }
+  if (!f2b_source_start(jail->source)) {
+    f2b_log_msg(log_warn, "jail '%s': source action 'start' failed -- %s",
+      jail->name, f2b_source_error(jail->source));
     goto cleanup;
   }
 
@@ -375,8 +376,8 @@ f2b_jail_init(f2b_jail_t *jail, f2b_config_t *config) {
   return true;
 
   cleanup:
-  if (jail->logfiles)
-    f2b_filelist_destroy(jail->logfiles);
+  if (jail->source)
+    f2b_source_destroy(jail->source);
   if (jail->filter)
     f2b_filter_destroy(jail->filter);
   if (jail->backend)

From 96dfab824b4211026c964cedac542b35b67e99ed Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 22:33:07 +1000
Subject: [PATCH 21/33] * src/jail.c : f2b_jail_stop() : destroy source

---
 src/jail.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/jail.c b/src/jail.c
index b30ab30..206ee74 100644
--- a/src/jail.c
+++ b/src/jail.c
@@ -393,7 +393,13 @@ f2b_jail_stop(f2b_jail_t *jail) {
 
   f2b_log_msg(log_info, "jail '%s': gracefull shutdown", jail->name);
 
-  f2b_filelist_destroy(jail->logfiles);
+  if (!f2b_source_stop(jail->source)) {
+    f2b_log_msg(log_error, "jail '%s': action 'stop' for source failed: %s",
+      jail->name, f2b_source_error(jail->source));
+    errors = true;
+  }
+
+  f2b_source_destroy(jail->source);
   f2b_filter_destroy(jail->filter);
 
   for (f2b_ipaddr_t *addr = jail->ipaddrs; addr != NULL; addr = addr->next) {
@@ -406,7 +412,7 @@ f2b_jail_stop(f2b_jail_t *jail) {
   f2b_addrlist_destroy(jail->ipaddrs);
 
   if (!f2b_backend_stop(jail->backend)) {
-    f2b_log_msg(log_error, "jail '%s': action 'stop' failed: %s",
+    f2b_log_msg(log_error, "jail '%s': action 'stop' for backend failed: %s",
       jail->name, f2b_backend_error(jail->backend));
     errors = true;
   }

From 0024327726ed0aab305c7ade2e6d631bf0b68c8f Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 22:33:45 +1000
Subject: [PATCH 22/33] * src/jail.c : f2b_jail_init() : fail on backend
 failure

---
 src/jail.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/jail.c b/src/jail.c
index 206ee74..a259c05 100644
--- a/src/jail.c
+++ b/src/jail.c
@@ -369,6 +369,7 @@ f2b_jail_init(f2b_jail_t *jail, f2b_config_t *config) {
   if (!f2b_backend_start(jail->backend)) {
     f2b_log_msg(log_warn, "jail '%s': backend action 'start' failed -- %s",
       jail->name, f2b_backend_error(jail->backend));
+    goto cleanup;
   }
 
   f2b_log_msg(log_info, "jail '%s': started", jail->name);

From 960d4a4fcedc682411bc53d01c06e94a2b16c502 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 22:34:02 +1000
Subject: [PATCH 23/33] * src/jail.c : f2b_jail_process() : use source

---
 src/jail.c | 87 ++++++++++++++++++++++++------------------------------
 1 file changed, 39 insertions(+), 48 deletions(-)

diff --git a/src/jail.c b/src/jail.c
index a259c05..60675b1 100644
--- a/src/jail.c
+++ b/src/jail.c
@@ -202,16 +202,16 @@ f2b_jail_find(f2b_jail_t *list, const char *name) {
 
 size_t
 f2b_jail_process(f2b_jail_t *jail) {
-  f2b_logfile_t *file = NULL;
   f2b_ipaddr_t  *prev = NULL;
   f2b_ipaddr_t  *addr = NULL;
   size_t processed = 0;
-  char logline[LOGLINE_MAX] = "";
+  char line[LOGLINE_MAX] = "";
   char matchbuf[IPADDR_MAX] = "";
   time_t now  = time(NULL);
   time_t findtime = 0;
   time_t expiretime = 0;
   bool remove = false;
+  bool reset = true; /* source reset */
 
   assert(jail != NULL);
 
@@ -219,55 +219,46 @@ f2b_jail_process(f2b_jail_t *jail) {
 
   f2b_backend_ping(jail->backend);
 
-  for (file = jail->logfiles; file != NULL; file = file->next) {
-    if (f2b_logfile_rotated(file)) {
-      f2b_log_msg(log_info, "jail '%s': file changed -- %s", jail->name, file->path);
-      f2b_logfile_close(file);
-    }
-    if (!file->opened && !f2b_logfile_open(file, NULL)) {
-      f2b_log_msg(log_error, "jail '%s': can't open file -- %s", jail->name, file->path);
+  while (f2b_source_next(jail->source, line, sizeof(line), reset)) {
+    reset = false;
+    if (!f2b_filter_match(jail->filter, line, matchbuf, sizeof(matchbuf)))
       continue;
-    }
-    while (f2b_logfile_getline(file, logline, sizeof(logline))) {
-      if (!f2b_filter_match(jail->filter, logline, matchbuf, sizeof(matchbuf)))
-        continue;
-      /* some regex matches the line */
-      jail->matchcount++;
-      addr = f2b_addrlist_lookup(jail->ipaddrs, matchbuf);
-      if (!addr) {
-        /* new ip */
-        addr = f2b_ipaddr_create(matchbuf, jail->maxretry);
-        addr->lastseen = now;
-        f2b_matches_append(&addr->matches, now);
-        jail->ipaddrs = f2b_addrlist_append(jail->ipaddrs, addr);
-        f2b_log_msg(log_info, "jail '%s': new ip found -- %s", jail->name, matchbuf);
-        continue;
-      }
-      /* this ip was seen before */
+    /* some regex matches the line */
+    jail->matchcount++;
+    addr = f2b_addrlist_lookup(jail->ipaddrs, matchbuf);
+    if (!addr) {
+      /* new ip */
+      addr = f2b_ipaddr_create(matchbuf, jail->maxretry);
       addr->lastseen = now;
-      if (addr->banned) {
-        if (addr->banned_at != now)
-          f2b_log_msg(log_warn, "jail '%s': ip %s was already banned", jail->name, matchbuf);
-        continue;
-      }
-      if (jail->incr_findtime > 0 && addr->matches.hits > jail->maxretry) {
-        findtime = now - jail->findtime;
-        findtime -= (int) ((addr->matches.hits - jail->maxretry) *
-                           (jail->findtime * jail->incr_findtime));
-      } else {
-        findtime = now - jail->findtime;
-      }
-      f2b_matches_expire(&addr->matches, findtime);
       f2b_matches_append(&addr->matches, now);
-      if (addr->matches.used < jail->maxretry) {
-        f2b_log_msg(log_info, "jail '%s': new ip match -- %s (%zu/%zu)",
-          jail->name, matchbuf, addr->matches.used, addr->matches.max);
-        continue;
-      }
-      /* limit reached, ban ip */
-      f2b_jail_ban(jail, addr);
-     } /* while(lines) */
-  } /* for(files) */
+      jail->ipaddrs = f2b_addrlist_append(jail->ipaddrs, addr);
+      f2b_log_msg(log_info, "jail '%s': new ip found -- %s", jail->name, matchbuf);
+      continue;
+    }
+    /* this ip was seen before */
+    addr->lastseen = now;
+    if (addr->banned) {
+      if (addr->banned_at != now)
+        f2b_log_msg(log_warn, "jail '%s': ip %s was already banned", jail->name, matchbuf);
+      continue;
+    }
+    if (jail->incr_findtime > 0 && addr->matches.hits > jail->maxretry) {
+      findtime = now - jail->findtime;
+      findtime -= (int) ((addr->matches.hits - jail->maxretry) *
+                         (jail->findtime * jail->incr_findtime));
+    } else {
+      findtime = now - jail->findtime;
+    }
+    f2b_matches_expire(&addr->matches, findtime);
+    f2b_matches_append(&addr->matches, now);
+    if (addr->matches.used < jail->maxretry) {
+      f2b_log_msg(log_info, "jail '%s': new ip match -- %s (%zu/%zu)",
+        jail->name, matchbuf, addr->matches.used, addr->matches.max);
+      continue;
+    }
+    /* limit reached, ban ip */
+    f2b_jail_ban(jail, addr);
+  }  /* while(1) */
 
   for (addr = jail->ipaddrs, prev = NULL; addr != NULL; ) {
     remove = false;

From 8edf2347134838a8f381abbecc73c74320140e8b Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 22:34:58 +1000
Subject: [PATCH 24/33] * src/jail.h : jail definition cleanup

---
 src/jail.h | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/jail.h b/src/jail.h
index 24a05d9..1820bcc 100644
--- a/src/jail.h
+++ b/src/jail.h
@@ -8,13 +8,11 @@
 #define F2B_JAIL_H_
 
 #include "log.h"
-#include "logfile.h"
 #include "ipaddr.h"
 #include "config.h"
 #include "source.h"
 #include "filter.h"
 #include "backend.h"
-#include "filelist.h"
 
 typedef struct f2b_jail_t {
   struct f2b_jail_t *next;
@@ -35,7 +33,6 @@ typedef struct f2b_jail_t {
   char filter_init[CONFIG_VAL_MAX];
   char source_name[CONFIG_KEY_MAX];
   char source_init[CONFIG_VAL_MAX];
-  f2b_logfile_t *logfiles;
   f2b_source_t  *source;
   f2b_filter_t  *filter;
   f2b_backend_t *backend;

From aea49c97f7b4e9f4b89ef9337f8d8a4bc3b980af Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 22:35:17 +1000
Subject: [PATCH 25/33] - src/filelist.[ch]

---
 src/filelist.c | 58 --------------------------------------------------
 src/filelist.h | 19 -----------------
 2 files changed, 77 deletions(-)
 delete mode 100644 src/filelist.c
 delete mode 100644 src/filelist.h

diff --git a/src/filelist.c b/src/filelist.c
deleted file mode 100644
index 50d01bc..0000000
--- a/src/filelist.c
+++ /dev/null
@@ -1,58 +0,0 @@
-/* Copyright 2016 Alex 'AdUser' Z (ad_user@runbox.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-#include <glob.h>
-
-#include "common.h"
-#include "logfile.h"
-#include "filelist.h"
-#include "log.h"
-
-f2b_logfile_t *
-f2b_filelist_append(f2b_logfile_t *list, f2b_logfile_t *file) {
-  assert(file != NULL);
-
-  if (list != NULL)
-    return file->next = list;
-  return file;
-}
-
-f2b_logfile_t *
-f2b_filelist_from_glob(const char *pattern) {
-  f2b_logfile_t *file = NULL;
-  f2b_logfile_t *files = NULL;
-  glob_t globbuf;
-
-  assert(pattern != NULL);
-
-  if (glob(pattern, GLOB_MARK | GLOB_NOESCAPE, NULL, &globbuf) != 0)
-    return NULL;
-
-  for (size_t i = 0; i < globbuf.gl_pathc; i++) {
-    if ((file = calloc(1, sizeof(f2b_logfile_t))) == NULL)
-      continue;
-    if (f2b_logfile_open(file, globbuf.gl_pathv[i]) == false) {
-      f2b_log_msg(log_error, "can't open file: %s: %s", globbuf.gl_pathv[i], strerror(errno));
-      free(file);
-      continue;
-    }
-    files = f2b_filelist_append(files, file);
-  }
-
-  globfree(&globbuf);
-  return files;
-}
-
-void
-f2b_filelist_destroy(f2b_logfile_t *list) {
-  f2b_logfile_t *next = NULL;
-
-  for (; list != NULL; list = next) {
-    next = list->next;
-    f2b_logfile_close(list);
-    free(list);
-  }
-}
diff --git a/src/filelist.h b/src/filelist.h
deleted file mode 100644
index 3eefe3b..0000000
--- a/src/filelist.h
+++ /dev/null
@@ -1,19 +0,0 @@
-/* Copyright 2016 Alex 'AdUser' Z (ad_user@runbox.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-#ifndef F2B_FILELIST_H_
-#define F2B_FILELIST_H_
-
-f2b_logfile_t *
-f2b_filelist_from_glob(const char *pattern);
-
-f2b_logfile_t *
-f2b_filelist_append(f2b_logfile_t *list, f2b_logfile_t *file);
-
-void
-f2b_filelist_destroy(f2b_logfile_t *list);
-
-#endif /* F2B_FILELIST_H_ */

From 0dc3b72fbba0adc7d435c1ff377650d93e62fcb6 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 22:35:30 +1000
Subject: [PATCH 26/33] - src/logfile.[ch]

---
 src/logfile.c | 84 ---------------------------------------------------
 src/logfile.h | 23 --------------
 2 files changed, 107 deletions(-)
 delete mode 100644 src/logfile.c
 delete mode 100644 src/logfile.h

diff --git a/src/logfile.c b/src/logfile.c
deleted file mode 100644
index bd2fd2f..0000000
--- a/src/logfile.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/* Copyright 2016 Alex 'AdUser' Z (ad_user@runbox.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-#include "common.h"
-#include "logfile.h"
-
-bool
-f2b_logfile_open(f2b_logfile_t *file, const char *path) {
-  struct stat st;
-  char buf[PATH_MAX] = "";
-
-  assert(file != NULL);
-  assert(path != NULL || file->path[0] != '\0');
-
-  strlcpy(buf, path ? path : file->path, sizeof(buf));
-
-  memset(file, 0x0, sizeof(f2b_logfile_t));
-
-  if (stat(buf, &st) != 0)
-    return false;
-
-  if (!(S_ISREG(st.st_mode) || S_ISFIFO(st.st_mode)))
-    return false;
-
-  if ((file->fd = fopen(buf, "r")) == NULL)
-    return false;
-
-  if (S_ISREG(st.st_mode) && fseek(file->fd, 0, SEEK_END) < 0)
-    return false;
-
-  memcpy(&file->st, &st, sizeof(st));
-  strlcpy(file->path, buf, sizeof(file->path));
-  file->opened = true;
-
-  return true;
-}
-
-void
-f2b_logfile_close(f2b_logfile_t *file) {
-  assert(file != NULL);
-
-  if (file->fd)
-    fclose(file->fd);
-
-  file->opened = false;
-  file->fd = NULL;
-}
-
-bool
-f2b_logfile_rotated(const f2b_logfile_t *file) {
-  struct stat st;
-
-  assert(file != NULL);
-
-  if (!file->opened)
-    return true;
-
-  if (stat(file->path, &st) != 0)
-    return true;
-
-  if (file->st.st_dev  != st.st_dev ||
-      file->st.st_ino  != st.st_ino ||
-      file->st.st_size  > st.st_size)
-    return true;
-
-  return false;
-}
-
-bool
-f2b_logfile_getline(const f2b_logfile_t *file, char *buf, size_t bufsize) {
-  assert(file != NULL);
-  assert(buf != NULL);
-
-  if (feof(file->fd))
-    clearerr(file->fd);
-  /* fread()+EOF set is implementation defined */
-  if (fgets(buf, bufsize, file->fd) != NULL)
-    return true;
-
-  return false;
-}
diff --git a/src/logfile.h b/src/logfile.h
deleted file mode 100644
index ff6c475..0000000
--- a/src/logfile.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/* Copyright 2016 Alex 'AdUser' Z (ad_user@runbox.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-#ifndef F2B_LOGFILE_H_
-#define F2B_LOGFILE_H_
-
-typedef struct f2b_logfile_t {
-  struct f2b_logfile_t *next;
-  bool opened;
-  char path[PATH_MAX];
-  FILE *fd;
-  struct stat st;
-} f2b_logfile_t;
-
-bool f2b_logfile_open(f2b_logfile_t *file, const char *path);
-void f2b_logfile_close(f2b_logfile_t *file);
-bool f2b_logfile_rotated(const f2b_logfile_t *file);
-bool f2b_logfile_getline(const f2b_logfile_t *file, char *buf, size_t bufsize);
-
-#endif /* F2B_LOGFILE_H_ */

From d3cce0b91cb598f2dbdc727ab9f1838f0f0a9ddd Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 22:37:57 +1000
Subject: [PATCH 27/33] * cleanup

---
 src/CMakeLists.txt | 2 +-
 src/daemon.c       | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 9bc1a9f..db3668b 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -1,6 +1,6 @@
 set(CMAKE_INCLUDE_CURRENT_DIR ON)
 
-set(SOURCES "daemon.c" "strlcpy.c" "logfile.c" "log.c" "matches.c" "source.c" "ipaddr.c" "filelist.c" "filter.c" "config.c" "jail.c" "backend.c")
+set(SOURCES "daemon.c" "strlcpy.c" "config.c" "log.c" "matches.c" "ipaddr.c" "source.c" "filter.c" "backend.c" "jail.c")
 
 if (WITH_CSOCKET)
   list(APPEND SOURCES "csocket.c" "cmsg.c")
diff --git a/src/daemon.c b/src/daemon.c
index c512a46..d82529b 100644
--- a/src/daemon.c
+++ b/src/daemon.c
@@ -5,7 +5,6 @@
  * published by the Free Software Foundation.
  */
 #include "common.h"
-#include "logfile.h"
 #include "ipaddr.h"
 #include "config.h"
 #include "jail.h"

From fdf2c12cd3358a600ff6ad702a90511c36c5a8c8 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 22:39:57 +1000
Subject: [PATCH 28/33] + configs/conf-available/05-source-files.conf

---
 configs/conf-available/05-source-files.conf | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 configs/conf-available/05-source-files.conf

diff --git a/configs/conf-available/05-source-files.conf b/configs/conf-available/05-source-files.conf
new file mode 100644
index 0000000..4df3717
--- /dev/null
+++ b/configs/conf-available/05-source-files.conf
@@ -0,0 +1,2 @@
+[source:files]
+load = libf2b_source_files.so

From 23c4545c2f69ad96531c0037a857655cfa114650 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 23:10:54 +1000
Subject: [PATCH 29/33] * src/sources/files.c : fix ready()

---
 src/sources/files.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/sources/files.c b/src/sources/files.c
index ac8c2c3..7e9e0f0 100644
--- a/src/sources/files.c
+++ b/src/sources/files.c
@@ -126,9 +126,9 @@ config(cfg_t *cfg, const char *key, const char *value) {
 bool
 ready(cfg_t *cfg) {
   assert(cfg != NULL);
-  if (cfg->files)
-    return true;
-  return false;
+  if (cfg->path[0] == '\0')
+    return false;
+  return true;
 }
 
 char *

From 7a3b6d3541881c4d927868afed2f441ece2151df Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 23:11:20 +1000
Subject: [PATCH 30/33] * src/filter-test.c : name a test filter

---
 src/filter-test.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/filter-test.c b/src/filter-test.c
index c869702..c6ab045 100644
--- a/src/filter-test.c
+++ b/src/filter-test.c
@@ -17,7 +17,7 @@ void usage() {
 
 int main(int argc, char *argv[]) {
   f2b_config_param_t  param = { .name = "load", .value = "", .next = 0x0 };
-  f2b_config_section_t config = { .type = t_filter, .param = 0x0, .next = 0x0 };
+  f2b_config_section_t config = { .name = "test", .type = t_filter, .param = 0x0, .next = 0x0 };
   f2b_filter_t *filter = NULL;
   char match[IPADDR_MAX] = "";
   char line[LOGLINE_MAX] = "";

From 3e4970c3b5400acb18aa05e53e89001a2e2a82d0 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 23:11:35 +1000
Subject: [PATCH 31/33] + src/source-test.c

---
 src/source-test.c | 48 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 src/source-test.c

diff --git a/src/source-test.c b/src/source-test.c
new file mode 100644
index 0000000..4d67228
--- /dev/null
+++ b/src/source-test.c
@@ -0,0 +1,48 @@
+/* Copyright 2016 Alex 'AdUser' Z (ad_user@runbox.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include "common.h"
+#include "log.h"
+#include "config.h"
+#include "source.h"
+
+void usage() {
+  fprintf(stderr, "Usage: source-test <library.so> <init string>\n");
+  exit(EXIT_FAILURE);
+}
+
+int main(int argc, char *argv[]) {
+  f2b_config_param_t  param = { .name = "load", .value = "", .next = 0x0 };
+  f2b_config_section_t config = { .name = "test", .type = t_source, .param = 0x0, .next = 0x0 };
+  f2b_source_t *source = NULL;
+  char buf[1024] = "";
+  bool reset;
+
+  if (argc < 2)
+    usage();
+
+  config.param = &param;
+  snprintf(param.value, sizeof(param.value), "%s", argv[1]);
+
+  if ((source = f2b_source_create(&config, argv[2], f2b_log_error_cb)) == false)
+    usage();
+
+  if (f2b_source_start(source) == false) {
+    f2b_log_msg(log_error, "source start error: %s", f2b_source_error(source));
+    exit(EXIT_FAILURE);
+  }
+
+  while (1) {
+    reset = true;
+    while (f2b_source_next(source, buf, sizeof(buf), reset)) {
+      reset = false;
+      fputs(buf, stderr);
+    }
+    sleep(1);
+  }
+
+  return EXIT_SUCCESS;
+}

From 19de6bfa95d786964f5b070e3cf1548108251946 Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 23:12:11 +1000
Subject: [PATCH 32/33] * add f2b-source-test to build

---
 src/CMakeLists.txt | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index db3668b..125f314 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -17,19 +17,24 @@ if (WITH_CLIENT)
   install(TARGETS "f2bc" RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
 endif ()
 
-set(SOURCES "strlcpy.c" "backend-test.c" "log.c" "config.c" "backend.c")
-add_executable("f2b-backend-test" ${SOURCES})
+set(SOURCES "strlcpy.c" "source-test.c" "log.c" "config.c" "source.c")
+add_executable("f2b-source-test" ${SOURCES})
 
 set(SOURCES "strlcpy.c" "filter-test.c" "log.c" "config.c" "filter.c")
 add_executable("f2b-filter-test" ${SOURCES})
 
+set(SOURCES "strlcpy.c" "backend-test.c" "log.c" "config.c" "backend.c")
+add_executable("f2b-backend-test" ${SOURCES})
+
 if (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
   target_link_libraries(f2b "dl")
+  target_link_libraries("f2b-source-test"  "dl")
+  target_link_libraries("f2b-filter-test"  "dl")
   target_link_libraries("f2b-backend-test" "dl")
-  target_link_libraries("f2b-filter-test" "dl")
 endif ()
 
-install(TARGETS "f2b-filter-test" "f2b-backend-test" RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+install(TARGETS "f2b-source-test" "f2b-filter-test" "f2b-backend-test"
+  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
 
 add_subdirectory("backends")
 add_subdirectory("filters")

From b89d059bf622dcead51d36f63a695cac4564b04a Mon Sep 17 00:00:00 2001
From: Alex 'AdUser' Z <ad_user@runbox.com>
Date: Tue, 13 Sep 2016 23:15:22 +1000
Subject: [PATCH 33/33] * update readme

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 45a0a6f..3192bc2 100644
--- a/README.md
+++ b/README.md
@@ -7,8 +7,8 @@ Features:
 
 * written in pure C
 * small memory footprint
-* minimum dependencies (req: libc, libdl; optional: pcre)
-* pluggable filters / backends (you may write custom one)
+* minimum dependencies (req: libc, libdl; optional: pcre, redis)
+* pluggable sources / filters / backends (you may write custom one)
 
 Similar software: