From 17f23023e834656601355c45d4de06ae80b08a8c Mon Sep 17 00:00:00 2001 From: Alex 'AdUser' Z Date: Mon, 9 Jan 2017 17:17:47 +1000 Subject: [PATCH] * use simdb_read() instead simdb_{record,block}_read() --- src/database.c | 121 ++++++++++++++++++++++++---------------------- src/simdb-query.c | 65 ++++++++++++------------- src/simdb.h | 18 ++----- 3 files changed, 99 insertions(+), 105 deletions(-) diff --git a/src/database.c b/src/database.c index 09ce58e..2df4327 100644 --- a/src/database.c +++ b/src/database.c @@ -289,7 +289,7 @@ int simdb_block_read(simdb_t *db, simdb_block_t *blk) { return blk->records; } -uint64_t +int simdb_records_count(simdb_t * const db) { struct stat st; off_t size = 0; @@ -313,54 +313,55 @@ simdb_record_ratio(simdb_urec_t *r) { } int -simdb_search(simdb_t * const db, - simdb_rec_t * const sample, +simdb_search(simdb_t * const db, int num, simdb_search_t * const search, simdb_match_t **matches) { - simdb_block_t blk; simdb_match_t match; - simdb_urec_t *rec; - const int blk_size = 4096; + simdb_urec_t *data = NULL; + simdb_urec_t *rec, sample; + const int blksize = 4096; uint64_t found = 0; - unsigned int i = 0; float ratio_s = 0.0; /* source */ float ratio_t = 0.0; /* tested */ int ret = 0; assert(db != NULL); - assert(sample != NULL); assert(search != NULL); assert(matches != NULL); assert(search->maxdiff_ratio >= 0.0 && search->maxdiff_ratio <= 1.0); assert(search->maxdiff_bitmap >= 0.0 && search->maxdiff_bitmap <= 1.0); - memset(&blk, 0x0, sizeof(simdb_block_t)); memset(&match, 0x0, sizeof(simdb_match_t)); - blk.start = 1; - blk.records = blk_size; - if ((ret = simdb_record_read(db, sample)) < 1) + if ((ret = simdb_read(db, num, 1, &rec)) < 1) return ret; + memcpy(&sample, rec, sizeof(sample)); + FREE(rec); + if (search->limit == 0) search->limit = -1; /* unsigned -> max */ if (search->maxdiff_ratio > 0.0) - ratio_s = simdb_record_ratio((simdb_urec_t *) sample->data); + ratio_s = simdb_record_ratio(&sample); CALLOC(*matches, search->limit, sizeof(simdb_match_t)); - while (simdb_block_read(db, &blk) > 0) { - rec = (simdb_urec_t *) blk.data; - for (i = 0; i < blk.records; i++, rec++) { - if (rec->used == 0x0) + for (num = 1; ; num += blksize) { + ret = simdb_read(db, num, blksize, &data); + if (ret < 0) + return ret; + rec = data; + for (int i = 0; i < ret; i++, rec++) { + if (!rec->used) continue; /* record missing */ match.diff_ratio = 0.0; match.diff_bitmap = 0.0; /* - compare ratio - cheap */ + /* TODO: check caps */ if (ratio_s > 0.0 && (ratio_t = simdb_record_ratio(rec)) > 0.0) { match.diff_ratio = ratio_s - ratio_t; match.diff_ratio *= (ratio_s > ratio_t) ? 1.0 : -1.0; @@ -371,77 +372,83 @@ simdb_search(simdb_t * const db, } /* - compare bitmap - more expensive */ - match.diff_bitmap = simdb_bitmap_compare(rec->bitmap, ((simdb_urec_t *) sample)->bitmap) / SIMDB_BITMAP_BITS; + match.diff_bitmap = simdb_bitmap_compare(rec->bitmap, sample.bitmap) / SIMDB_BITMAP_BITS; if (match.diff_bitmap > search->maxdiff_bitmap) continue; /* create match record */ - match.num = blk.start + i; + match.num = num + i; memcpy(&(*matches)[found], &match, sizeof(simdb_match_t)); found++; if (found >= search->limit) break; } - FREE(blk.data); + FREE(data); if (found >= search->limit) break; - blk.start += blk_size; } return found; } -uint64_t -simdb_usage_map(simdb_t * const db, - char ** const map) { - const int blk_size = 4096; - simdb_block_t blk; - uint64_t records; - simdb_urec_t *r; /* mnemonics : block, record */ - char *m = NULL; /* mnemonics : map */ +int +simdb_usage_map(simdb_t * const db, char ** const map) { + const int blksize = 4096; + simdb_urec_t *data = NULL; + simdb_urec_t *r; /* mnemonics : record pointer */ + char *m = NULL; /* mnemonics : map pointer */ + int ret = 0, records = 0; - memset(&blk, 0x0, sizeof(simdb_block_t)); + assert(db != NULL); + assert(map != NULL); records = simdb_records_count(db); - CALLOC(*map, records + 1, sizeof(char)); - - m = *map; - blk.start = 1; - blk.records = blk_size; - - while (simdb_block_read(db, &blk) > 0) { - r = (simdb_urec_t *) blk.data; - for (unsigned int i = 0; i < blk.records; i++, m++, r++) { + CALLOC(m, records + 1, sizeof(char)); + *map = m; + + for (int num = 1; ; num += blksize) { + ret = simdb_read(db, num, blksize, &data); + if (ret < 0) { + FREE(*map); + return ret; + } + if (ret == 0) + break; + r = data; + for (int i = 0; i < ret; i++, m++, r++) { *m = (r->used == 0xFF) ? CHAR_USED : CHAR_NONE; } - blk.start += blk_size; + FREE(data); } - FREE(blk.data); return records; } -uint16_t -simdb_usage_slice(simdb_t * const db, - char ** const map, - uint64_t offset, - uint16_t limit) { - simdb_block_t blk; - simdb_urec_t *r; /* mnemonics : block, record */ - char *m = NULL; /* mnemonics : map */ +int +simdb_usage_slice(simdb_t * const db, char ** const map, int offset, int limit) { + simdb_urec_t *data = NULL; + simdb_urec_t *r; /* mnemonics : record pointer */ + char *m = NULL; /* mnemonics : map pointer */ + int ret = 0; + + assert(db != NULL); + assert(map != NULL); + + if (offset < 1 || limit < 1) + return SIMDB_ERR_USAGE; - memset(&blk, 0x0, sizeof(simdb_block_t)); - CALLOC(*map, limit + 1, sizeof(char)); + ret = simdb_read(db, offset, limit, &data); + if (ret <= 0) + return ret; + CALLOC(m, limit + 1, sizeof(char)); m = *map; - blk.start = offset; - blk.records = limit; - limit = simdb_block_read(db, &blk); - r = (simdb_urec_t *) blk.data; - for (uint16_t i = 0; i < blk.records; i++, m++, r++) { + r = data; + for (int i = 0; i < limit; i++, m++, r++) { *m = (r->used == 0xFF) ? CHAR_USED : CHAR_NONE; } + FREE(data); - return limit; + return ret; } diff --git a/src/simdb-query.c b/src/simdb-query.c index 0037ff3..9d1610c 100644 --- a/src/simdb-query.c +++ b/src/simdb-query.c @@ -16,8 +16,9 @@ #include "common.h" #include "bitmap.h" -#include "simdb.h" #include "record.h" +#include "io.h" +#include "simdb.h" #include #include @@ -41,20 +42,16 @@ void usage(int exitcode) { exit(exitcode); } -int search_similar(simdb_t *db, uint64_t number, float maxdiff) -{ +int search_similar(simdb_t *db, int num, float maxdiff) { int ret = 0, i = 0; - simdb_rec_t sample; simdb_match_t *matches = NULL; simdb_search_t search; - memset(&sample, 0x0, sizeof(simdb_rec_t)); memset(&search, 0x0, sizeof(simdb_search_t)); search.maxdiff_ratio = 0.2; /* 20% */ search.maxdiff_bitmap = maxdiff; - sample.num = number; - if ((ret = simdb_search(db, &sample, &search, &matches)) < 0) { + if ((ret = simdb_search(db, num, &search, &matches)) < 0) { fprintf(stderr, "%s\n", simdb_error(ret)); return 1; } @@ -119,64 +116,62 @@ int db_usage_slice(simdb_t *db, uint64_t offset, uint16_t limit) return 0; } -int rec_bitmap(simdb_t *db, uint64_t number) -{ - simdb_rec_t rec; +int rec_bitmap(simdb_t *db, int num) { simdb_urec_t *r; + int ret; assert(db != NULL); - memset(&rec, 0x0, sizeof(simdb_rec_t)); - rec.num = number; - if (simdb_record_read(db, &rec) < 1) { + if ((ret = simdb_read(db, num, 1, &r)) < 0) { + fprintf(stderr, "read failed: %s\n", simdb_error(ret)); + return 1; + } else if (ret == 0) { fprintf(stderr, "bitmap: %s\n", "sample not found"); return 1; } - r = (simdb_urec_t *) rec.data; simdb_bitmap_print(r->bitmap); + free(r); return 0; } -int rec_diff(simdb_t *db, uint64_t a, uint64_t b, unsigned short int showmap) -{ - simdb_rec_t rec; - simdb_urec_t *r; - float diff = 0.0; - unsigned char one[SIMDB_BITMAP_SIZE]; - unsigned char two[SIMDB_BITMAP_SIZE]; +int rec_diff(simdb_t *db, int a, int b, unsigned short int showmap) { unsigned char map[SIMDB_BITMAP_SIZE]; + simdb_urec_t *one, *two; + float diff = 0.0; + int ret; assert(db != NULL); - memset(&rec, 0x0, sizeof(rec)); - memset(one, 0x0, sizeof(one)); - memset(two, 0x0, sizeof(two)); - rec.num = a; - if (simdb_record_read(db, &rec) < 1) { + if ((ret = simdb_read(db, a, 1, &one)) < 0) { + fprintf(stderr, "read error: %s\n", simdb_error(ret)); + return 1; + } else if (ret == 0 || (ret > 0 && !one->used)) { fprintf(stderr, "record diff: first sample not exists\n"); + if (ret > 0) free(one); return 1; } - r = (simdb_urec_t *) rec.data; - memcpy(one, r->bitmap, sizeof(one)); - rec.num = b; - if (simdb_record_read(db, &rec) < 1) { - fprintf(stderr, "record diff: second sample not exists\n"); + if ((ret = simdb_read(db, b, 1, &two)) < 0) { + fprintf(stderr, "read error: %s\n", simdb_error(ret)); + return 1; + } else if (ret == 0 || (ret > 0 && !two->used)) { + fprintf(stderr, "record diff: first sample not exists\n"); + if (ret > 0) free(two); return 1; } - r = (simdb_urec_t *) rec.data; - memcpy(two, r->bitmap, sizeof(two)); if (showmap) { - simdb_bitmap_diffmap(one, two, map); + simdb_bitmap_diffmap(one->bitmap, two->bitmap, map); simdb_bitmap_print(map); return 0; } - diff = (float) simdb_bitmap_compare(one, two); + diff = (float) simdb_bitmap_compare(one->bitmap, two->bitmap); printf("%.2f%%\n", (diff / SIMDB_BITMAP_BITS) * 100); + free(one); + free(two); return 0; } diff --git a/src/simdb.h b/src/simdb.h index 06aabfe..ba82243 100644 --- a/src/simdb.h +++ b/src/simdb.h @@ -110,15 +110,14 @@ const char * simdb_error(int code); /** * @brief Search compare given record in database to other images * @param db Database handle - * @param sample Record sample + * @param num Record sample number * @param search Search parameters * @param matches Pointer to storage for found matches * @retval >0 if found some matches * @retval 0 if nothing found * @retval <0 on error */ -int simdb_search(simdb_t * const db, - simdb_rec_t * const sample, +int simdb_search(simdb_t * const db, int num, simdb_search_t * const search, simdb_match_t ** matches); @@ -162,16 +161,13 @@ int simdb_block_read(simdb_t *db, simdb_block_t *blk); /** * @brief Get database capacity */ -uint64_t -simdb_records_count(simdb_t * const db); +int simdb_records_count(simdb_t * const db); /** @brief Fills buffer 'map' according to records existense in database @returns records processed (and also buffer size) */ -uint64_t -simdb_usage_map(simdb_t * const db, - char ** const map); +int simdb_usage_map(simdb_t * const db, char ** const map); /** * @brief Fills buffer 'map' according to records existense in given range @@ -181,10 +177,6 @@ simdb_usage_map(simdb_t * const db, * @param limit Slice size * @returns Records processed (and also buffer size) */ -uint16_t -simdb_usage_slice(simdb_t * const db, - char ** const map, - uint64_t offset, - uint16_t limit); +int simdb_usage_slice(simdb_t * const db, char ** const map, int offset, int limit); #endif