Browse Source

* global rename : imdb -> simdb

master
Alex 'AdUser' Z 8 years ago
parent
commit
934e9d546e
  1. 26
      src/CMakeLists.txt
  2. 2
      src/common.h
  3. 166
      src/database.c
  4. 74
      src/database.h
  5. 6
      src/sample.c
  6. 3
      src/sample.h
  7. 54
      src/simdb-1to2.c
  8. 50
      src/simdb-query.c
  9. 24
      src/simdb-write.c
  10. 39
      tests/database.c

26
src/CMakeLists.txt

@ -1,20 +1,20 @@
set(LIB_SOURCES "database.c" "bitmap.c") set(LIB_SOURCES "database.c" "bitmap.c")
add_library("imgdup" SHARED ${LIB_SOURCES}) add_library("simdb" SHARED ${LIB_SOURCES})
set_property(TARGET "imgdup" PROPERTY SONAME ${VERSION}) set_property(TARGET "simdb" PROPERTY SONAME ${VERSION})
install(TARGETS "imgdup" LIBRARY DESTINATION "lib") install(TARGETS "simdb" LIBRARY DESTINATION "lib")
add_executable("imdb-1to2" "imdb-1to2.c") add_executable("simdb-1to2" "simdb-1to2.c")
install(TARGETS "imdb-1to2" RUNTIME DESTINATION "bin") install(TARGETS "simdb-1to2" RUNTIME DESTINATION "bin")
add_executable("imdb-query" "imdb-query.c") add_executable("simdb-query" "simdb-query.c")
target_link_libraries("imdb-query" "imgdup") target_link_libraries("simdb-query" "simdb")
install(TARGETS "imdb-query" RUNTIME DESTINATION "bin") install(TARGETS "simdb-query" RUNTIME DESTINATION "bin")
find_package(ImageMagick COMPONENTS MagickCore MagickWand) find_package(ImageMagick COMPONENTS MagickCore MagickWand)
add_executable("imdb-write" "imdb-write.c" "sample.c") add_executable("simdb-write" "simdb-write.c" "sample.c")
target_link_libraries("imdb-write" "imgdup") target_link_libraries("simdb-write" "simdb")
target_link_libraries("imdb-write" ${ImageMagick_MagickCore_LIBRARY}) target_link_libraries("simdb-write" ${ImageMagick_MagickCore_LIBRARY})
target_link_libraries("imdb-write" ${ImageMagick_MagickWand_LIBRARY}) target_link_libraries("simdb-write" ${ImageMagick_MagickWand_LIBRARY})
install(TARGETS "imdb-write" RUNTIME DESTINATION "bin") install(TARGETS "simdb-write" RUNTIME DESTINATION "bin")

2
src/common.h

@ -27,5 +27,5 @@
extern int image_add(uint64_t, char *path); extern int image_add(uint64_t, char *path);
extern int image_del(uint64_t); extern int image_del(uint64_t);
extern int image_exists(uint64_t); extern int image_exists(uint64_t);
extern int image_search(uint64_t, imdb_rec_t **data, size_t limit); extern int image_search(uint64_t, simdb_rec_t **data, size_t limit);
*/ */

166
src/database.c

@ -22,32 +22,32 @@
errno = 0; \ errno = 0; \
memset((buf), 0x0, (len)); \ memset((buf), 0x0, (len)); \
bytes = pread((db)->fd, (buf), (len), (off)); \ bytes = pread((db)->fd, (buf), (len), (off)); \
if (errno) { return IMDB_ERR_SYSTEM; } if (errno) { return SIMDB_ERR_SYSTEM; }
#define DB_WRITE(db, buf, len, off) \ #define DB_WRITE(db, buf, len, off) \
errno = 0; \ errno = 0; \
bytes = pwrite((db)->fd, (buf), (len), (off)); \ bytes = pwrite((db)->fd, (buf), (len), (off)); \
if (errno) { return IMDB_ERR_SYSTEM; } if (errno) { return SIMDB_ERR_SYSTEM; }
const char *imdb_hdr_fmt = "IMDB v%02u, CAPS: %s;"; const char *simdb_hdr_fmt = "IMDB v%02u, CAPS: %s;";
bool bool
imdb_create(const char *path) { simdb_create(const char *path) {
ssize_t bytes = 0; ssize_t bytes = 0;
unsigned char buf[IMDB_REC_LEN]; unsigned char buf[SIMDB_REC_LEN];
const char *caps = "M-R"; const char *caps = "M-R";
bool result = false; bool result = false;
int fd = -1; int fd = -1;
memset(buf, 0x0, sizeof(char) * IMDB_REC_LEN); memset(buf, 0x0, sizeof(char) * SIMDB_REC_LEN);
if ((fd = creat(path, 0644)) < 0) if ((fd = creat(path, 0644)) < 0)
return result; return result;
snprintf((char *) buf, IMDB_REC_LEN, imdb_hdr_fmt, IMDB_VERSION, caps); snprintf((char *) buf, SIMDB_REC_LEN, simdb_hdr_fmt, SIMDB_VERSION, caps);
bytes = pwrite(fd, buf, IMDB_REC_LEN, 0); bytes = pwrite(fd, buf, SIMDB_REC_LEN, 0);
if (bytes == IMDB_REC_LEN) if (bytes == SIMDB_REC_LEN)
result = true; /* success */ result = true; /* success */
close(fd); close(fd);
@ -55,12 +55,12 @@ imdb_create(const char *path) {
return result; return result;
} }
imdb_db_t * simdb_t *
imdb_open(const char *path, int mode, int *error) { simdb_open(const char *path, int mode, int *error) {
imdb_db_t *db = NULL; simdb_t *db = NULL;
ssize_t bytes = 0; ssize_t bytes = 0;
struct stat st; struct stat st;
char buf[IMDB_REC_LEN] = "\0"; char buf[SIMDB_REC_LEN] = "\0";
int flags = 0, fd = -1; int flags = 0, fd = -1;
char *p; char *p;
@ -68,31 +68,31 @@ imdb_open(const char *path, int mode, int *error) {
assert(error != NULL); assert(error != NULL);
if (stat(path, &st) < 0) { if (stat(path, &st) < 0) {
*error = IMDB_ERR_SYSTEM; *error = SIMDB_ERR_SYSTEM;
return NULL; return NULL;
} }
if ((fd = open(path, (mode & IMDB_FLAG_WRITE) ? O_RDWR : O_RDONLY)) < 0) { if ((fd = open(path, (mode & SIMDB_FLAG_WRITE) ? O_RDWR : O_RDONLY)) < 0) {
*error = IMDB_ERR_SYSTEM; *error = SIMDB_ERR_SYSTEM;
return NULL; return NULL;
} }
errno = 0; errno = 0;
bytes = pread(fd, buf, IMDB_REC_LEN, 0); bytes = pread(fd, buf, SIMDB_REC_LEN, 0);
if (bytes < IMDB_REC_LEN) { if (bytes < SIMDB_REC_LEN) {
*error = errno ? IMDB_ERR_SYSTEM : IMDB_ERR_CORRUPTDB; *error = errno ? SIMDB_ERR_SYSTEM : SIMDB_ERR_CORRUPTDB;
return NULL; return NULL;
} }
p = buf + 0; p = buf + 0;
if (memcmp("IMDB", p, 4) != 0) { if (memcmp("IMDB", p, 4) != 0) {
*error = IMDB_ERR_CORRUPTDB; *error = SIMDB_ERR_CORRUPTDB;
return NULL; return NULL;
} }
p = buf + 6; p = buf + 6;
if (atoi(p) != IMDB_VERSION) { if (atoi(p) != SIMDB_VERSION) {
*error = IMDB_ERR_WRONGVERS; *error = SIMDB_ERR_WRONGVERS;
return NULL; return NULL;
} }
@ -102,22 +102,22 @@ imdb_open(const char *path, int mode, int *error) {
return NULL; return NULL;
} }
if (mode & IMDB_FLAG_WRITE) if (mode & SIMDB_FLAG_WRITE)
flags |= IMDB_FLAG_WRITE; flags |= SIMDB_FLAG_WRITE;
/* all seems to be ok */ /* all seems to be ok */
if ((db = calloc(1, sizeof(imdb_db_t))) == NULL) { if ((db = calloc(1, sizeof(simdb_t))) == NULL) {
*error = IMDB_ERR_OOM; *error = SIMDB_ERR_OOM;
return NULL; return NULL;
} }
p = buf + 16; p = buf + 16;
for (size_t i = 0; i < 8 && *p != '\0'; p++) { for (size_t i = 0; i < 8 && *p != '\0'; p++) {
switch (*p) { switch (*p) {
case 'M' : flags |= IMDB_CAP_BITMAP; break; case 'M' : flags |= SIMDB_CAP_BITMAP; break;
case 'C' : flags |= IMDB_CAP_COLORS; break; case 'C' : flags |= SIMDB_CAP_COLORS; break;
case 'R' : flags |= IMDB_CAP_RATIO; break; case 'R' : flags |= SIMDB_CAP_RATIO; break;
case ';' : i = 9; /* end of flags */ break; case ';' : i = 9; /* end of flags */ break;
default: /* ignore */ break; default: /* ignore */ break;
} }
@ -132,7 +132,7 @@ imdb_open(const char *path, int mode, int *error) {
} }
void void
imdb_close(imdb_db_t *db) simdb_close(simdb_t *db)
{ {
assert(db != NULL); assert(db != NULL);
@ -141,26 +141,26 @@ imdb_close(imdb_db_t *db)
} }
const char * const char *
imdb_error(int error) { simdb_error(int error) {
if (error == IMDB_SUCCESS) { if (error == SIMDB_SUCCESS) {
return "success"; return "success";
} else if (error == IMDB_ERR_SYSTEM) { } else if (error == SIMDB_ERR_SYSTEM) {
return strerror(errno); return strerror(errno);
} else if (error == IMDB_ERR_OOM) { } else if (error == SIMDB_ERR_OOM) {
return "can't allocate memory"; return "can't allocate memory";
} else if (error == IMDB_ERR_CORRUPTDB) { } else if (error == SIMDB_ERR_CORRUPTDB) {
return "database corrupted"; return "database corrupted";
} else if (error == IMDB_ERR_WRONGVERS) { } else if (error == SIMDB_ERR_WRONGVERS) {
return "database version differs from library version"; return "database version differs from library version";
} else if (error == IMDB_ERR_READONLY) { } else if (error == SIMDB_ERR_READONLY) {
return "database opened in read-only mode"; return "database opened in read-only mode";
} else if (error == IMDB_ERR_NXRECORD) { } else if (error == SIMDB_ERR_NXRECORD) {
return "no such record in database"; return "no such record in database";
} }
return "unknown error"; return "unknown error";
} }
int imdb_read_rec(imdb_db_t *db, imdb_rec_t *rec) int simdb_read_rec(simdb_t *db, simdb_rec_t *rec)
{ {
ssize_t bytes = 0; ssize_t bytes = 0;
@ -168,10 +168,10 @@ int imdb_read_rec(imdb_db_t *db, imdb_rec_t *rec)
assert(rec != NULL); assert(rec != NULL);
assert(rec->num > 0); assert(rec->num > 0);
DB_READ(db, rec->data, IMDB_REC_LEN, rec->num * IMDB_REC_LEN); DB_READ(db, rec->data, SIMDB_REC_LEN, rec->num * SIMDB_REC_LEN);
if (bytes != IMDB_REC_LEN) if (bytes != SIMDB_REC_LEN)
return errno ? IMDB_ERR_SYSTEM : IMDB_ERR_NXRECORD; return errno ? SIMDB_ERR_SYSTEM : SIMDB_ERR_NXRECORD;
if (rec->data[0] != 0xFF) if (rec->data[0] != 0xFF)
return 0; return 0;
@ -179,7 +179,7 @@ int imdb_read_rec(imdb_db_t *db, imdb_rec_t *rec)
return 1; return 1;
} }
int imdb_write_rec(imdb_db_t *db, imdb_rec_t *rec) int simdb_write_rec(simdb_t *db, simdb_rec_t *rec)
{ {
ssize_t bytes = 0; ssize_t bytes = 0;
@ -187,18 +187,18 @@ int imdb_write_rec(imdb_db_t *db, imdb_rec_t *rec)
assert(rec != NULL); assert(rec != NULL);
assert(rec->num > 0); assert(rec->num > 0);
if (!(db->flags & IMDB_FLAG_WRITE)) if (!(db->flags & SIMDB_FLAG_WRITE))
return IMDB_ERR_READONLY; return SIMDB_ERR_READONLY;
DB_WRITE(db, rec->data, IMDB_REC_LEN, rec->num * IMDB_REC_LEN); DB_WRITE(db, rec->data, SIMDB_REC_LEN, rec->num * SIMDB_REC_LEN);
if (bytes != IMDB_REC_LEN) if (bytes != SIMDB_REC_LEN)
return IMDB_ERR_SYSTEM; return SIMDB_ERR_SYSTEM;
return 1; return 1;
} }
int imdb_read_blk(imdb_db_t *db, imdb_block_t *blk) int simdb_read_blk(simdb_t *db, simdb_block_t *blk)
{ {
ssize_t bytes = 0; ssize_t bytes = 0;
@ -208,15 +208,15 @@ int imdb_read_blk(imdb_db_t *db, imdb_block_t *blk)
assert(blk->records > 0); assert(blk->records > 0);
FREE(blk->data); FREE(blk->data);
CALLOC(blk->data, blk->records, IMDB_REC_LEN); CALLOC(blk->data, blk->records, SIMDB_REC_LEN);
DB_READ(db, blk->data, blk->records * IMDB_REC_LEN, blk->start * IMDB_REC_LEN); DB_READ(db, blk->data, blk->records * SIMDB_REC_LEN, blk->start * SIMDB_REC_LEN);
blk->records = bytes / IMDB_REC_LEN; blk->records = bytes / SIMDB_REC_LEN;
return blk->records; return blk->records;
} }
uint64_t uint64_t
imdb_records_count(imdb_db_t * const db) { simdb_records_count(simdb_t * const db) {
struct stat st; struct stat st;
off_t size = 0; off_t size = 0;
@ -225,7 +225,7 @@ imdb_records_count(imdb_db_t * const db) {
fstat(db->fd, &st); fstat(db->fd, &st);
size = st.st_size; size = st.st_size;
return size / IMDB_REC_LEN; return size / SIMDB_REC_LEN;
} }
float float
@ -235,17 +235,17 @@ ratio_from_rec_data(unsigned char * const data) {
iw = *((uint16_t *)(data + REC_OFF_IW)); iw = *((uint16_t *)(data + REC_OFF_IW));
ih = *((uint16_t *)(data + REC_OFF_IH)); ih = *((uint16_t *)(data + REC_OFF_IH));
return (iw > 0 && ih > 0) ? ((float) iw / ih) : 0.0; return (iw > 0 && ih > 0) ? ((float) iw / ih) : 0.0;
} }
int int
imdb_search(imdb_db_t * const db, simdb_search(simdb_t * const db,
imdb_rec_t * const sample, simdb_rec_t * const sample,
imdb_search_t * const search, simdb_search_t * const search,
imdb_match_t **matches) simdb_match_t **matches)
{ {
imdb_block_t blk; simdb_block_t blk;
imdb_match_t match; simdb_match_t match;
const int blk_size = 4096; const int blk_size = 4096;
uint64_t found = 0; uint64_t found = 0;
unsigned int i = 0; unsigned int i = 0;
@ -261,12 +261,12 @@ imdb_search(imdb_db_t * const db,
assert(search->maxdiff_ratio >= 0.0 && search->maxdiff_ratio <= 1.0); assert(search->maxdiff_ratio >= 0.0 && search->maxdiff_ratio <= 1.0);
assert(search->maxdiff_bitmap >= 0.0 && search->maxdiff_bitmap <= 1.0); assert(search->maxdiff_bitmap >= 0.0 && search->maxdiff_bitmap <= 1.0);
memset(&blk, 0x0, sizeof(imdb_block_t)); memset(&blk, 0x0, sizeof(simdb_block_t));
memset(&match, 0x0, sizeof(imdb_match_t)); memset(&match, 0x0, sizeof(simdb_match_t));
blk.start = 1; blk.start = 1;
blk.records = blk_size; blk.records = blk_size;
if ((ret = imdb_read_rec(db, sample)) < 1) if ((ret = simdb_read_rec(db, sample)) < 1)
return ret; return ret;
if (search->limit == 0) if (search->limit == 0)
@ -275,11 +275,11 @@ imdb_search(imdb_db_t * const db,
if (search->maxdiff_ratio > 0.0) if (search->maxdiff_ratio > 0.0)
ratio_s = ratio_from_rec_data(sample->data); ratio_s = ratio_from_rec_data(sample->data);
CALLOC(*matches, search->limit, sizeof(imdb_match_t)); CALLOC(*matches, search->limit, sizeof(simdb_match_t));
while (imdb_read_blk(db, &blk) > 0) { while (simdb_read_blk(db, &blk) > 0) {
p = blk.data; p = blk.data;
for (i = 0; i < blk.records; i++, p += IMDB_REC_LEN) { for (i = 0; i < blk.records; i++, p += SIMDB_REC_LEN) {
if (*(p + REC_OFF_RU) == 0x0) if (*(p + REC_OFF_RU) == 0x0)
continue; /* record missing */ continue; /* record missing */
@ -304,7 +304,7 @@ imdb_search(imdb_db_t * const db,
/* create match record */ /* create match record */
match.num = blk.start + i; match.num = blk.start + i;
memcpy(&(*matches)[found], &match, sizeof(imdb_match_t)); memcpy(&(*matches)[found], &match, sizeof(simdb_match_t));
found++; found++;
if (found >= search->limit) if (found >= search->limit)
break; break;
@ -319,26 +319,26 @@ imdb_search(imdb_db_t * const db,
} }
uint64_t uint64_t
imdb_usage_map(imdb_db_t * const db, simdb_usage_map(simdb_t * const db,
char ** const map) { char ** const map) {
const int blk_size = 4096; const int blk_size = 4096;
imdb_block_t blk; simdb_block_t blk;
uint64_t records; uint64_t records;
unsigned char *r; /* mnemonics : block, record */ unsigned char *r; /* mnemonics : block, record */
char *m = NULL; /* mnemonics : map */ char *m = NULL; /* mnemonics : map */
memset(&blk, 0x0, sizeof(imdb_block_t)); memset(&blk, 0x0, sizeof(simdb_block_t));
records = imdb_records_count(db); records = simdb_records_count(db);
CALLOC(*map, records + 1, sizeof(char)); CALLOC(*map, records + 1, sizeof(char));
m = *map; m = *map;
blk.start = 1; blk.start = 1;
blk.records = blk_size; blk.records = blk_size;
while (imdb_read_blk(db, &blk) > 0) { while (simdb_read_blk(db, &blk) > 0) {
r = blk.data; r = blk.data;
for (unsigned int i = 0; i < blk.records; i++, m++, r += IMDB_REC_LEN) { for (unsigned int i = 0; i < blk.records; i++, m++, r += SIMDB_REC_LEN) {
*m = (r[REC_OFF_RU] == 0xFF) ? CHAR_USED : CHAR_NONE; *m = (r[REC_OFF_RU] == 0xFF) ? CHAR_USED : CHAR_NONE;
} }
blk.start += blk_size; blk.start += blk_size;
@ -349,24 +349,24 @@ imdb_usage_map(imdb_db_t * const db,
} }
uint16_t uint16_t
imdb_usage_slice(imdb_db_t * const db, simdb_usage_slice(simdb_t * const db,
char ** const map, char ** const map,
uint64_t offset, uint64_t offset,
uint16_t limit) { uint16_t limit) {
imdb_block_t blk; simdb_block_t blk;
unsigned char *r; /* mnemonics : block, record */ unsigned char *r; /* mnemonics : block, record */
char *m = NULL; /* mnemonics : map */ char *m = NULL; /* mnemonics : map */
memset(&blk, 0x0, sizeof(imdb_block_t)); memset(&blk, 0x0, sizeof(simdb_block_t));
CALLOC(*map, limit + 1, sizeof(char)); CALLOC(*map, limit + 1, sizeof(char));
m = *map; m = *map;
blk.start = offset; blk.start = offset;
blk.records = limit; blk.records = limit;
limit = imdb_read_blk(db, &blk); limit = simdb_read_blk(db, &blk);
r = blk.data; r = blk.data;
for (uint16_t i = 0; i < blk.records; i++, m++, r += IMDB_REC_LEN) { for (uint16_t i = 0; i < blk.records; i++, m++, r += SIMDB_REC_LEN) {
*m = (r[REC_OFF_RU] == 0xFF) ? CHAR_USED : CHAR_NONE; *m = (r[REC_OFF_RU] == 0xFF) ? CHAR_USED : CHAR_NONE;
} }

74
src/database.h

@ -1,32 +1,32 @@
#ifndef HAS_DATABASE_H #ifndef HAS_DATABASE_H
#define HAS_DATABASE_H 1 #define HAS_DATABASE_H 1
#define IMDB_REC_LEN 48 #define SIMDB_REC_LEN 48
#define IMDB_VERSION 2 #define SIMDB_VERSION 2
/* runtime flags */ /* runtime flags */
#define IMDB_FLAG_WRITE 1 << (0 + 0) #define SIMDB_FLAG_WRITE 1 << (0 + 0)
/* database capabilities */ /* database capabilities */
#define IMDB_CAP_BITMAP 1 << (8 + 0) #define SIMDB_CAP_BITMAP 1 << (8 + 0)
#define IMDB_CAP_COLORS 1 << (8 + 1) #define SIMDB_CAP_COLORS 1 << (8 + 1)
#define IMDB_CAP_RATIO 1 << (8 + 2) #define SIMDB_CAP_RATIO 1 << (8 + 2)
/* 3 used, 5 reserved */ /* 3 used, 5 reserved */
#define IMDB_SUCCESS 0 #define SIMDB_SUCCESS 0
/* database errors */ /* database errors */
#define IMDB_ERR_SYSTEM -1 /* see errno for details */ #define SIMDB_ERR_SYSTEM -1 /* see errno for details */
#define IMDB_ERR_OOM -2 /* can't allocate memory */ #define SIMDB_ERR_OOM -2 /* can't allocate memory */
#define IMDB_ERR_CORRUPTDB -3 /* empty or currupted database */ #define SIMDB_ERR_CORRUPTDB -3 /* empty or currupted database */
#define IMDB_ERR_WRONGVERS -4 /* database version mismatch */ #define SIMDB_ERR_WRONGVERS -4 /* database version mismatch */
#define IMDB_ERR_NXRECORD -5 /* no such record in database */ #define SIMDB_ERR_NXRECORD -5 /* no such record in database */
#define IMDB_ERR_READONLY -6 /* database opened in read-only mode */ #define SIMDB_ERR_READONLY -6 /* database opened in read-only mode */
typedef struct { typedef struct {
int fd; int fd;
int flags; int flags;
char path[PATH_MAX]; char path[PATH_MAX];
} imdb_db_t; } simdb_t;
/** /**
Database header format - fixed length, 48 bytes Database header format - fixed length, 48 bytes
@ -70,15 +70,15 @@ typedef struct {
uint64_t start; uint64_t start;
size_t records; size_t records;
unsigned char *data; unsigned char *data;
} imdb_block_t; } simdb_block_t;
/** /**
* database record * database record
*/ */
typedef struct { typedef struct {
uint64_t num; uint64_t num;
unsigned char data[IMDB_REC_LEN]; unsigned char data[SIMDB_REC_LEN];
} imdb_rec_t; } simdb_rec_t;
/** /**
* search parameters * search parameters
@ -88,7 +88,7 @@ typedef struct {
uint8_t limit; /**< max results */ uint8_t limit; /**< max results */
float maxdiff_bitmap; /**< max difference of luma bitmaps */ float maxdiff_bitmap; /**< max difference of luma bitmaps */
float maxdiff_ratio; /**< max difference of ratios, default - 7% */ float maxdiff_ratio; /**< max difference of ratios, default - 7% */
} imdb_search_t; } simdb_search_t;
/** /**
* search matches * search matches
@ -97,16 +97,16 @@ typedef struct {
uint64_t num; /**< record id */ uint64_t num; /**< record id */
float diff_ratio; /**< difference of ratio */ float diff_ratio; /**< difference of ratio */
float diff_bitmap; /**< difference of bitmap */ float diff_bitmap; /**< difference of bitmap */
} imdb_match_t; } simdb_match_t;
/** /**
* @returns 1 on success, 0 if record missing and <0 on error * @returns 1 on success, 0 if record missing and <0 on error
*/ */
int imdb_read_rec (imdb_db_t *db, imdb_rec_t *rec); int simdb_read_rec (simdb_t *db, simdb_rec_t *rec);
int imdb_write_rec(imdb_db_t *db, imdb_rec_t *rec); int simdb_write_rec(simdb_t *db, simdb_rec_t *rec);
int imdb_read_blk (imdb_db_t *db, imdb_block_t *blk); int simdb_read_blk (simdb_t *db, simdb_block_t *blk);
int imdb_write_blk(imdb_db_t *db, imdb_block_t *blk); int simdb_write_blk(simdb_t *db, simdb_block_t *blk);
/** /**
* @brief Creates empty database at given path * @brief Creates empty database at given path
@ -115,29 +115,29 @@ int imdb_write_blk(imdb_db_t *db, imdb_block_t *blk);
* @note See errno value for details * @note See errno value for details
* @todo 2nd arg: caps * @todo 2nd arg: caps
*/ */
bool imdb_create(const char *path); bool simdb_create(const char *path);
/** /**
* @brief Open database at given path * @brief Open database at given path
* @param path Path to database * @param path Path to database
* @param mode Database open modes. See IMDB_FLAG_* defines above * @param mode Database open modes. See SIMDB_FLAG_* defines above
* @param error Pointer to error code storage * @param error Pointer to error code storage
* @returns Pointer to database handle on success, NULL on error * @returns Pointer to database handle on success, NULL on error
* @note use @a imdb_error() to get error description * @note use @a simdb_error() to get error description
*/ */
imdb_db_t * imdb_open(const char *path, int mode, int *error); simdb_t * simdb_open(const char *path, int mode, int *error);
/** /**
* @brief Close database and free associated resources * @brief Close database and free associated resources
* @param db Database handle * @param db Database handle
*/ */
void imdb_close(imdb_db_t *db); void simdb_close(simdb_t *db);
/** /**
* @brief Get error desctiption by error code * @brief Get error desctiption by error code
* @param error Error code, see IMDB_ERR_* defines above * @param error Error code, see SIMDB_ERR_* defines above
*/ */
const char * imdb_error(int code); const char * simdb_error(int code);
float float
ratio_from_rec_data(unsigned char * const data); ratio_from_rec_data(unsigned char * const data);
@ -146,23 +146,23 @@ ratio_from_rec_data(unsigned char * const data);
@returns: >0 if found some matches, 0 if nothing found, <0 on error @returns: >0 if found some matches, 0 if nothing found, <0 on error
*/ */
int int
imdb_search(imdb_db_t * const db, simdb_search(simdb_t * const db,
imdb_rec_t * const sample, simdb_rec_t * const sample,
imdb_search_t * const search, simdb_search_t * const search,
imdb_match_t ** matches); simdb_match_t ** matches);
/** /**
@returns: number of records in database @returns: number of records in database
*/ */
uint64_t uint64_t
imdb_records_count(imdb_db_t * const db); simdb_records_count(simdb_t * const db);
/** /**
@brief fills buffer 'map' according to records existense in database @brief fills buffer 'map' according to records existense in database
@returns records processed (and also buffer size) @returns records processed (and also buffer size)
*/ */
uint64_t uint64_t
imdb_usage_map(imdb_db_t * const db, simdb_usage_map(simdb_t * const db,
char ** const map); char ** const map);
/** /**
@ -172,7 +172,7 @@ imdb_usage_map(imdb_db_t * const db,
@returns records processed (and also buffer size) @returns records processed (and also buffer size)
*/ */
uint16_t uint16_t
imdb_usage_slice(imdb_db_t * const db, simdb_usage_slice(simdb_t * const db,
char ** const map, char ** const map,
uint64_t offset, uint64_t offset,
uint16_t limit); uint16_t limit);

6
src/sample.c

@ -6,8 +6,8 @@
#include <wand/magick_wand.h> #include <wand/magick_wand.h>
int int
imdb_sample(imdb_rec_t * const rec, simdb_sample(simdb_rec_t * const rec,
const char * const source) const char * const source)
{ {
MagickWand *wand = NULL; MagickWand *wand = NULL;
MagickPassFail status = MagickPass; MagickPassFail status = MagickPass;
@ -83,7 +83,7 @@ imdb_sample(imdb_rec_t * const rec,
if (status == MagickPass) { if (status == MagickPass) {
assert(buf_size == BITMAP_SIZE); assert(buf_size == BITMAP_SIZE);
memset(rec->data, 0x0, IMDB_REC_LEN); memset(rec->data, 0x0, SIMDB_REC_LEN);
rec->data[REC_OFF_RU] = 0xFF; rec->data[REC_OFF_RU] = 0xFF;
memcpy(&rec->data[REC_OFF_IW], &w, sizeof(uint16_t)); memcpy(&rec->data[REC_OFF_IW], &w, sizeof(uint16_t));
memcpy(&rec->data[REC_OFF_IH], &h, sizeof(uint16_t)); memcpy(&rec->data[REC_OFF_IH], &h, sizeof(uint16_t));

3
src/sample.h

@ -2,7 +2,6 @@
#define HAS_SAMPLER_H 1 #define HAS_SAMPLER_H 1
int int
imdb_sample(imdb_rec_t * const rec, simdb_sample(simdb_rec_t * const rec, const char * const source);
const char * const source);
#endif /* HAS_SAMPLER_H */ #endif /* HAS_SAMPLER_H */

54
src/imdb-1to2.c → src/simdb-1to2.c

@ -7,28 +7,28 @@
#include <sys/types.h> #include <sys/types.h>
#include <unistd.h> #include <unistd.h>
#define IMDB_REC_LEN 48 #define SIMDB_REC_LEN 48
#define BLK_SIZE 1000 #define BLK_SIZE 1000
void usage(const char *message) { void usage(const char *message) {
if (message) if (message)
printf("error: %s\n", message); printf("error: %s\n", message);
printf("Usage: imdb_convert_1to2 <infile> <outfile>\n"); printf("Usage: simdb_convert_1to2 <infile> <outfile>\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
int main(int argc, char **argv) { int main(int argc, char **argv) {
int in, out; int in, out;
long unsigned int imdb_rec_total, rec_first, rec_last, records; long unsigned int simdb_rec_total, rec_first, rec_last, records;
unsigned char in_buf[IMDB_REC_LEN * BLK_SIZE]; unsigned char in_buf[SIMDB_REC_LEN * BLK_SIZE];
unsigned char out_buf[IMDB_REC_LEN * BLK_SIZE]; unsigned char out_buf[SIMDB_REC_LEN * BLK_SIZE];
unsigned char header[IMDB_REC_LEN]; unsigned char header[SIMDB_REC_LEN];
unsigned char *src, *dst; unsigned char *src, *dst;
struct stat st; struct stat st;
ssize_t bytes; ssize_t bytes;
memset(header, 0x0, IMDB_REC_LEN); memset(header, 0x0, SIMDB_REC_LEN);
snprintf((char *) header, IMDB_REC_LEN, "IMDB v%02u, CAPS: %s;", 2, "M-R"); snprintf((char *) header, SIMDB_REC_LEN, "IMDB v%02u, CAPS: %s;", 2, "M-R");
if (argc < 2) if (argc < 2)
usage(NULL); usage(NULL);
@ -43,54 +43,54 @@ int main(int argc, char **argv) {
if (fstat(in, &st) != 0) if (fstat(in, &st) != 0)
usage(strerror(errno)); usage(strerror(errno));
if ((st.st_size % IMDB_REC_LEN) != 0) if ((st.st_size % SIMDB_REC_LEN) != 0)
usage("database size expected to be multiples to 48"); usage("database size expected to be multiples to 48");
if (lseek(in, 0, SEEK_SET) < 0) if (lseek(in, 0, SEEK_SET) < 0)
usage(strerror(errno)); usage(strerror(errno));
if (read(in, in_buf, IMDB_REC_LEN) != IMDB_REC_LEN) if (read(in, in_buf, SIMDB_REC_LEN) != SIMDB_REC_LEN)
usage("can't read header of database"); usage("can't read header of database");
if (memcmp(in_buf, "DB of image fingerprints (ver 1)", 32) != 0) if (memcmp(in_buf, "DB of image fingerprints (ver 1)", 32) != 0)
usage("wrong database header / version mismatch"); usage("wrong database header / version mismatch");
imdb_rec_total = (st.st_size / IMDB_REC_LEN); simdb_rec_total = (st.st_size / SIMDB_REC_LEN);
printf("Processing %lu records\n", imdb_rec_total - 1); printf("Processing %lu records\n", simdb_rec_total - 1);
if (lseek(in, IMDB_REC_LEN, SEEK_SET) < 0) if (lseek(in, SIMDB_REC_LEN, SEEK_SET) < 0)
usage(strerror(errno)); usage(strerror(errno));
if (lseek(out, 0, SEEK_SET) < 0) if (lseek(out, 0, SEEK_SET) < 0)
usage(strerror(errno)); usage(strerror(errno));
if (write(out, header, IMDB_REC_LEN) != IMDB_REC_LEN) if (write(out, header, SIMDB_REC_LEN) != SIMDB_REC_LEN)
usage(strerror(errno)); usage(strerror(errno));
for (unsigned int block = 0; block <= (imdb_rec_total / BLK_SIZE); block++) { for (unsigned int block = 0; block <= (simdb_rec_total / BLK_SIZE); block++) {
rec_first = rec_last = (block * BLK_SIZE) + 1; rec_first = rec_last = (block * BLK_SIZE) + 1;
records = ((rec_first + BLK_SIZE) > imdb_rec_total) ? imdb_rec_total - rec_first : BLK_SIZE; records = ((rec_first + BLK_SIZE) > simdb_rec_total) ? simdb_rec_total - rec_first : BLK_SIZE;
rec_last = rec_first + records; rec_last = rec_first + records;
printf("* block %u, %4lu records [%lu, %lu]\n", printf("* block %u, %4lu records [%lu, %lu]\n",
block + 1, records, rec_first, rec_last - 1); block + 1, records, rec_first, rec_last - 1);
lseek(in, rec_first * IMDB_REC_LEN, SEEK_SET); lseek(in, rec_first * SIMDB_REC_LEN, SEEK_SET);
lseek(out, rec_first * IMDB_REC_LEN, SEEK_SET); lseek(out, rec_first * SIMDB_REC_LEN, SEEK_SET);
bytes = read(in, in_buf, IMDB_REC_LEN * records); bytes = read(in, in_buf, SIMDB_REC_LEN * records);
if (bytes != (ssize_t) records * IMDB_REC_LEN) { if (bytes != (ssize_t) records * SIMDB_REC_LEN) {
printf("Read size mismatch, expected %lu, got: %zi\n", printf("Read size mismatch, expected %lu, got: %zi\n",
IMDB_REC_LEN * records, bytes); SIMDB_REC_LEN * records, bytes);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
memset(out_buf, 0x0, records * IMDB_REC_LEN); memset(out_buf, 0x0, records * SIMDB_REC_LEN);
for (unsigned int i = 0; i < records; i++) { for (unsigned int i = 0; i < records; i++) {
src = &in_buf[i * IMDB_REC_LEN]; src = &in_buf[i * SIMDB_REC_LEN];
dst = &out_buf[i * IMDB_REC_LEN]; dst = &out_buf[i * SIMDB_REC_LEN];
memcpy(dst + 0, src + 0, sizeof(char) * 1); // usage flag memcpy(dst + 0, src + 0, sizeof(char) * 1); // usage flag
memcpy(dst + 16, src + 2, sizeof(char) * 32); // image bitmap memcpy(dst + 16, src + 2, sizeof(char) * 32); // image bitmap
} }
bytes = write(out, out_buf, IMDB_REC_LEN * records); bytes = write(out, out_buf, SIMDB_REC_LEN * records);
if (bytes != (ssize_t) records * IMDB_REC_LEN) { if (bytes != (ssize_t) records * SIMDB_REC_LEN) {
printf("Write size mismatch, expected %lu, got: %zi\n", printf("Write size mismatch, expected %lu, got: %zi\n",
IMDB_REC_LEN * records, bytes); SIMDB_REC_LEN * records, bytes);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} }

50
src/imdb-query.c → src/simdb-query.c

@ -23,7 +23,7 @@
void usage(int exitcode) { void usage(int exitcode) {
fprintf(stderr, fprintf(stderr,
"Usage: imdb-query <opts>\n" "Usage: simdb-query <opts>\n"
" -b <path> Path to database\n" " -b <path> Path to database\n"
" -t <int> Maximum difference pct (0 - 50, default: 10%%)\n" " -t <int> Maximum difference pct (0 - 50, default: 10%%)\n"
); );
@ -40,21 +40,21 @@ void usage(int exitcode) {
exit(exitcode); exit(exitcode);
} }
int search_similar(imdb_db_t *db, uint64_t number, float maxdiff) int search_similar(simdb_t *db, uint64_t number, float maxdiff)
{ {
int ret = 0, i = 0; int ret = 0, i = 0;
imdb_rec_t sample; simdb_rec_t sample;
imdb_match_t *matches = NULL; simdb_match_t *matches = NULL;
imdb_search_t search; simdb_search_t search;
memset(&sample, 0x0, sizeof(imdb_rec_t)); memset(&sample, 0x0, sizeof(simdb_rec_t));
memset(&search, 0x0, sizeof(imdb_search_t)); memset(&search, 0x0, sizeof(simdb_search_t));
search.maxdiff_ratio = 0.2; /* 20% */ search.maxdiff_ratio = 0.2; /* 20% */
search.maxdiff_bitmap = maxdiff; search.maxdiff_bitmap = maxdiff;
sample.num = number; sample.num = number;
if ((ret = imdb_search(db, &sample, &search, &matches)) < 0) { if ((ret = simdb_search(db, &sample, &search, &matches)) < 0) {
fprintf(stderr, "%s\n", imdb_error(ret)); fprintf(stderr, "%s\n", simdb_error(ret));
return 1; return 1;
} }
@ -70,7 +70,7 @@ int search_similar(imdb_db_t *db, uint64_t number, float maxdiff)
return 0; return 0;
} }
int db_usage_map(imdb_db_t *db, unsigned short int cols) int db_usage_map(simdb_t *db, unsigned short int cols)
{ {
char *map = NULL; char *map = NULL;
char *m = NULL; char *m = NULL;
@ -80,7 +80,7 @@ int db_usage_map(imdb_db_t *db, unsigned short int cols)
memset(row, 0x0, sizeof(char) * 256); memset(row, 0x0, sizeof(char) * 256);
if ((records = imdb_usage_map(db, &map)) == 0) { if ((records = simdb_usage_map(db, &map)) == 0) {
fprintf(stderr, "database usage: can't get database map\n"); fprintf(stderr, "database usage: can't get database map\n");
FREE(map); FREE(map);
return 1; return 1;
@ -107,26 +107,26 @@ int db_usage_map(imdb_db_t *db, unsigned short int cols)
return 0; return 0;
} }
int db_usage_slice(imdb_db_t *db, uint64_t offset, uint16_t limit) int db_usage_slice(simdb_t *db, uint64_t offset, uint16_t limit)
{ {
char *map = NULL; char *map = NULL;
limit = imdb_usage_slice(db, &map, offset, limit); limit = simdb_usage_slice(db, &map, offset, limit);
puts(map); puts(map);
FREE(map); FREE(map);
return 0; return 0;
} }
int rec_bitmap(imdb_db_t *db, uint64_t number) int rec_bitmap(simdb_t *db, uint64_t number)
{ {
imdb_rec_t rec; simdb_rec_t rec;
assert(db != NULL); assert(db != NULL);
memset(&rec, 0x0, sizeof(imdb_rec_t)); memset(&rec, 0x0, sizeof(simdb_rec_t));
rec.num = number; rec.num = number;
if (imdb_read_rec(db, &rec) < 1) { if (simdb_read_rec(db, &rec) < 1) {
fprintf(stderr, "bitmap: %s\n", "sample not found"); fprintf(stderr, "bitmap: %s\n", "sample not found");
return 1; return 1;
} }
@ -136,26 +136,26 @@ int rec_bitmap(imdb_db_t *db, uint64_t number)
return 0; return 0;
} }
int rec_diff(imdb_db_t *db, uint64_t a, uint64_t b, unsigned short int showmap) int rec_diff(simdb_t *db, uint64_t a, uint64_t b, unsigned short int showmap)
{ {
imdb_rec_t rec; simdb_rec_t rec;
float diff = 0.0; float diff = 0.0;
unsigned char one[BITMAP_SIZE]; unsigned char one[BITMAP_SIZE];
unsigned char two[BITMAP_SIZE]; unsigned char two[BITMAP_SIZE];
unsigned char map[BITMAP_SIZE]; unsigned char map[BITMAP_SIZE];
assert(db != NULL); assert(db != NULL);
memset(&rec, 0x0, sizeof(imdb_rec_t)); memset(&rec, 0x0, sizeof(simdb_rec_t));
rec.num = a; rec.num = a;
if (imdb_read_rec(db, &rec) < 1) { if (simdb_read_rec(db, &rec) < 1) {
fprintf(stderr, "record diff: first sample not exists\n"); fprintf(stderr, "record diff: first sample not exists\n");
return 1; return 1;
} }
memcpy(one, &rec.data[REC_OFF_BM], BITMAP_SIZE); memcpy(one, &rec.data[REC_OFF_BM], BITMAP_SIZE);
rec.num = b; rec.num = b;
if (imdb_read_rec(db, &rec) < 1) { if (simdb_read_rec(db, &rec) < 1) {
fprintf(stderr, "record diff: second sample not exists\n"); fprintf(stderr, "record diff: second sample not exists\n");
return 1; return 1;
} }
@ -179,7 +179,7 @@ int main(int argc, char **argv)
const char *db_path = NULL; const char *db_path = NULL;
float maxdiff = 0.10; float maxdiff = 0.10;
unsigned short int cols = 64, map = 0, ret = 0; unsigned short int cols = 64, map = 0, ret = 0;
imdb_db_t *db = NULL; simdb_t *db = NULL;
uint64_t a = 0, b = 0; uint64_t a = 0, b = 0;
char *c = NULL; char *c = NULL;
int err; int err;
@ -242,7 +242,7 @@ int main(int argc, char **argv)
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if ((db = imdb_open(db_path, 0, &err)) == NULL) { if ((db = simdb_open(db_path, 0, &err)) == NULL) {
fprintf(stderr, "database open: %d\n", err); fprintf(stderr, "database open: %d\n", err);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
@ -280,7 +280,7 @@ int main(int argc, char **argv)
break; break;
} }
imdb_close(db); simdb_close(db);
return ret; return ret;
} }

24
src/imdb-write.c → src/simdb-write.c

@ -7,7 +7,7 @@
void usage(int exitcode) { void usage(int exitcode) {
fprintf(stderr, fprintf(stderr,
"Usage: imdb-write <opts>\n" "Usage: simdb-write <opts>\n"
" -b <path> Path to database\n" " -b <path> Path to database\n"
); );
fprintf(stderr, fprintf(stderr,
@ -25,11 +25,11 @@ int main(int argc, char **argv)
const char *db_path = NULL; const char *db_path = NULL;
const char *sample = NULL; const char *sample = NULL;
const char *c = NULL; const char *c = NULL;
imdb_db_t *db = NULL; simdb_t *db = NULL;
imdb_rec_t rec; simdb_rec_t rec;
int err; int err;
memset(&rec, 0x0, sizeof(imdb_rec_t)); memset(&rec, 0x0, sizeof(simdb_rec_t));
if (argc < 3) if (argc < 3)
usage(EXIT_FAILURE); usage(EXIT_FAILURE);
@ -68,12 +68,12 @@ int main(int argc, char **argv)
} }
if (mode == init) { if (mode == init) {
if (!imdb_create(db_path)) { if (!simdb_create(db_path)) {
fprintf(stderr, "database init: %s\n", strerror(errno)); fprintf(stderr, "database init: %s\n", strerror(errno));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} }
if ((db = imdb_open(db_path, 1, &err)) == NULL) { if ((db = simdb_open(db_path, 1, &err)) == NULL) {
fprintf(stderr, "can't open database: %d\n", err); fprintf(stderr, "can't open database: %d\n", err);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
@ -82,18 +82,18 @@ int main(int argc, char **argv)
case add : case add :
if (rec.num == 0 || sample == NULL) if (rec.num == 0 || sample == NULL)
usage(EXIT_FAILURE); usage(EXIT_FAILURE);
if (imdb_sample(&rec, sample) != 0) { if (simdb_sample(&rec, sample) != 0) {
fprintf(stderr, "sampler failure\n"); fprintf(stderr, "sampler failure\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if ((err = imdb_write_rec(db, &rec)) < 1) { if ((err = simdb_write_rec(db, &rec)) < 1) {
fprintf(stderr, "%s\n", imdb_error(err)); fprintf(stderr, "%s\n", simdb_error(err));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
break; break;
case del : case del :
if ((err = imdb_write_rec(db, &rec)) < 1) { if ((err = simdb_write_rec(db, &rec)) < 1) {
fprintf(stderr, "%s\n", imdb_error(err)); fprintf(stderr, "%s\n", simdb_error(err));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
break; break;
@ -104,7 +104,7 @@ int main(int argc, char **argv)
usage(EXIT_FAILURE); usage(EXIT_FAILURE);
break; break;
} }
imdb_close(db); simdb_close(db);
exit(EXIT_SUCCESS); exit(EXIT_SUCCESS);
} }

39
tests/database.c

@ -1,53 +1,52 @@
#include "../src/common.h" #include "../src/common.h"
#include "../src/database.h" #include "../src/database.h"
int main() int main() {
{ simdb_t *db;
imdb_db_t *db; simdb_rec_t rec[2];
imdb_rec_t rec[2]; simdb_block_t blk;
imdb_block_t blk;
char *path = "test.db"; char *path = "test.db";
int mode = 0, err = 0, num; int mode = 0, err = 0, num;
bool ret; bool ret;
unlink(path); unlink(path);
db = imdb_open(path, mode, &err); db = simdb_open(path, mode, &err);
assert(db == NULL); assert(db == NULL);
assert(err == -1); /* no such file */ assert(err == -1); /* no such file */
ret = imdb_create(path); ret = simdb_create(path);
assert(ret == true); assert(ret == true);
db = imdb_open(path, mode, &err); db = simdb_open(path, mode, &err);
assert(db != NULL); assert(db != NULL);
rec[0].num = 1; rec[0].num = 1;
err = imdb_read_rec(db, rec); err = simdb_read_rec(db, rec);
assert(err == IMDB_ERR_NXRECORD); assert(err == SIMDB_ERR_NXRECORD);
memset(rec[0].data, 0xAA, IMDB_REC_LEN); memset(rec[0].data, 0xAA, SIMDB_REC_LEN);
memset(rec[0].data, 0xFF, 1); /* record is used */ memset(rec[0].data, 0xFF, 1); /* record is used */
num = imdb_write_rec(db, rec); num = simdb_write_rec(db, rec);
assert(num == IMDB_ERR_READONLY); /* database open in read-only mode */ assert(num == SIMDB_ERR_READONLY); /* database open in read-only mode */
imdb_close(db); simdb_close(db);
mode |= IMDB_FLAG_WRITE; mode |= SIMDB_FLAG_WRITE;
db = imdb_open(path, mode, &err); db = simdb_open(path, mode, &err);
assert(db != NULL); assert(db != NULL);
num = imdb_write_rec(db, rec); num = simdb_write_rec(db, rec);
assert(num == 1); /* success */ assert(num == 1); /* success */
num = imdb_read_rec(db, rec); num = simdb_read_rec(db, rec);
assert(num == 1); assert(num == 1);
blk.start = 1; blk.start = 1;
blk.records = 2; blk.records = 2;
blk.data = NULL; blk.data = NULL;
num = imdb_read_blk(db, &blk); num = simdb_read_blk(db, &blk);
assert(num == 1); assert(num == 1);
assert(blk.records == 1); assert(blk.records == 1);
assert(blk.data != NULL); assert(blk.data != NULL);
@ -55,7 +54,7 @@ int main()
rec[0].num = 1; rec[0].num = 1;
rec[1].num = 3; rec[1].num = 3;
imdb_close(db); simdb_close(db);
return 0; return 0;
} }

Loading…
Cancel
Save