Menu
▾
▴
bitcollider-commits
|
From: Robert K. <may...@us...> - 2001-10-12 19:59:54
|
Update of /cvsroot/bitcollider/bitcollider/lib
In directory usw-pr-cvs1:/tmp/cvs-serv28703/lib
Modified Files:
cache.c main.c
Log Message:
Next set of improvements to the cache support. The cache now has a size
limit (by default 8192 entries). If the cache grows past 8192, then
remove 10% of the least recently used entries. Also I've added support
for making sure the database gets cleaned up properly at exit.
Index: cache.c
===================================================================
RCS file: /cvsroot/bitcollider/bitcollider/lib/cache.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -r1.1 -r1.2
*** cache.c 2001/10/10 23:25:48 1.1
--- cache.c 2001/10/12 19:59:51 1.2
***************
*** 8,14 ****
--- 8,17 ----
#include <stdlib.h>
#include <string.h>
+ #include <time.h>
#include "cache.h"
#define DB printf("%s:%d\n", __FILE__, __LINE__);
+ #define COUNT_KEY "??count??"
+ #define MAX_COUNT_KEY "??maxcount??"
#if USE_BDB
***************
*** 27,43 ****
int ret;
ret = db_create(&info->dbp, NULL, 0);
if (ret != 0)
{
! info->errorString = strdup(db_strerror(ret));
return false;
}
ret = info->dbp->open(info->dbp, fileName, NULL, DB_BTREE, DB_CREATE, 0664);
if (ret != 0)
{
! info->errorString = strdup(db_strerror(ret));
return false;
}
return true;
}
--- 30,74 ----
int ret;
+ strcpy(info->fileName, fileName);
ret = db_create(&info->dbp, NULL, 0);
if (ret != 0)
{
! strcpy(info->errorString, db_strerror(ret));
return false;
}
+ ret = info->dbp->open(info->dbp, fileName, NULL, DB_BTREE, 0, 0664);
+ if (ret != 0)
+ {
+ strcpy(info->errorString, db_strerror(ret));
+ return false;
+ }
+
+ info->maxCount = get_max_count(info);
+
+ return true;
+ }
+
+ b_bool create_cache(cache_info *info, const char *fileName, int maxCount)
+ {
+ int ret;
+
+ strcpy(info->fileName, fileName);
+ ret = db_create(&info->dbp, NULL, 0);
+ if (ret != 0)
+ {
+ strcpy(info->errorString, db_strerror(ret));
+ return false;
+ }
ret = info->dbp->open(info->dbp, fileName, NULL, DB_BTREE, DB_CREATE, 0664);
if (ret != 0)
{
! strcpy(info->errorString, db_strerror(ret));
return false;
}
+ info->maxCount = maxCount;
+ set_max_count(info, info->maxCount);
+ set_count(info, 0);
+
return true;
}
***************
*** 46,52 ****
{
info->dbp->close(info->dbp, 0);
- if (info->errorString)
- free(info->errorString);
-
free(info);
}
--- 77,80 ----
***************
*** 67,73 ****
if (ret == DB_NOTFOUND)
{
! if (info->errorString)
! free(info->errorString);
! info->errorString = NULL;
return false;
}
--- 95,99 ----
if (ret == DB_NOTFOUND)
{
! info->errorString[0] = 0;
return false;
}
***************
*** 75,79 ****
if (ret != 0)
{
! info->errorString = strdup(db_strerror(ret));
return false;
}
--- 101,105 ----
if (ret != 0)
{
! strcpy(info->errorString, db_strerror(ret));
return false;
}
***************
*** 86,93 ****
if (token)
{
! entry->lastModDate = atoi(token);
token = strtok(NULL, "*");
if (token)
! strcpy(entry->bitprint, token);
}
else
--- 112,124 ----
if (token)
{
! entry->lastUsedDate = atoi(token);
token = strtok(NULL, "*");
if (token)
! {
! entry->lastModDate = atoi(token);
! token = strtok(NULL, "*");
! if (token)
! strcpy(entry->bitprint, token);
! }
}
else
***************
*** 102,110 ****
}
! b_bool add_cache_entry(cache_info *info, cache_entry *entry)
{
! DBT key, data;
! int ret;
! char strData[100];
memset(&key, 0, sizeof(DBT));
--- 133,142 ----
}
! b_bool add_cache_entry(cache_info *info, cache_entry *entry, b_bool update)
{
! DBT key, data;
! int ret;
! char strData[100];
! time_t t;
memset(&key, 0, sizeof(DBT));
***************
*** 114,127 ****
key.size = strlen(entry->fileName);
! sprintf(strData, "%ld*%s", (long)entry->lastModDate, entry->bitprint);
data.data = strData;
data.size = strlen(strData);
! ret = info->dbp->put(info->dbp, NULL, &key, &data, DB_NOOVERWRITE);
if (ret != 0)
{
! info->errorString = strdup(db_strerror(ret));
return false;
}
return true;
}
--- 146,177 ----
key.size = strlen(entry->fileName);
! time(&t);
! sprintf(strData, "%ld*%ld*%s",
! t, (long)entry->lastModDate, entry->bitprint);
data.data = strData;
data.size = strlen(strData);
! ret = info->dbp->put(info->dbp, NULL, &key, &data,
! update ? 0 : DB_NOOVERWRITE);
if (ret != 0)
{
! strcpy(info->errorString, db_strerror(ret));
! printf("insert failed: %s\n", info->errorString);
return false;
}
+
+ if (!update)
+ {
+ int count;
+
+ count = get_count(info) + 1;
+ if (count > info->maxCount)
+ {
+ /* remove_excessive will update the record count */
+ remove_excessive_entries(info, count);
+ }
+ else
+ set_count(info, count);
+ }
return true;
}
***************
*** 139,147 ****
ret = info->dbp->del(info->dbp, NULL, &key, 0);
if (ret != 0)
{
! info->errorString = strdup(db_strerror(ret));
return false;
}
return true;
}
#endif
--- 189,422 ----
ret = info->dbp->del(info->dbp, NULL, &key, 0);
if (ret != 0)
+ {
+ strcpy(info->errorString, db_strerror(ret));
+ return false;
+ }
+ return true;
+ }
+
+ int get_count(cache_info *info)
+ {
+ DBT key, data;
+ int ret;
+
+ memset(&key, 0, sizeof(DBT));
+ memset(&data, 0, sizeof(DBT));
+
+ key.data = COUNT_KEY;
+ key.size = sizeof(COUNT_KEY);
+
+ ret = info->dbp->get(info->dbp, NULL, &key, &data, 0);
+ if (ret == DB_NOTFOUND)
+ {
+ info->errorString[0] = 0;
+ return 0;
+ }
+
+ if (ret != 0)
{
! strcpy(info->errorString, db_strerror(ret));
! return -1;
! }
! ((char *)data.data)[data.size] = 0;
!
! return atoi(data.data);
! }
!
! b_bool set_count(cache_info *info, int count)
! {
! DBT key, data;
! int ret;
! char strData[10];
!
! memset(&key, 0, sizeof(DBT));
! memset(&data, 0, sizeof(DBT));
!
! key.data = COUNT_KEY;
! key.size = sizeof(COUNT_KEY);
!
! sprintf(strData, "%d", count);
! data.data = strData;
! data.size = strlen(strData);
!
! ret = info->dbp->put(info->dbp, NULL, &key, &data, 0);
! if (ret != 0)
! {
! strcpy(info->errorString, db_strerror(ret));
return false;
}
return true;
+ }
+
+ int get_max_count(cache_info *info)
+ {
+ DBT key, data;
+ int ret;
+
+ memset(&key, 0, sizeof(DBT));
+ memset(&data, 0, sizeof(DBT));
+
+ key.data = MAX_COUNT_KEY;
+ key.size = sizeof(MAX_COUNT_KEY);
+
+ ret = info->dbp->get(info->dbp, NULL, &key, &data, 0);
+ if (ret == DB_NOTFOUND)
+ {
+ info->errorString[0] = 0;
+ return 0;
+ }
+
+ if (ret != 0)
+ {
+ strcpy(info->errorString, db_strerror(ret));
+ return -1;
+ }
+ ((char *)data.data)[data.size] = 0;
+
+ return atoi(data.data);
+ }
+
+ b_bool set_max_count(cache_info *info, int count)
+ {
+ DBT key, data;
+ int ret;
+ char strData[10];
+
+ memset(&key, 0, sizeof(DBT));
+ memset(&data, 0, sizeof(DBT));
+
+ key.data = MAX_COUNT_KEY;
+ key.size = sizeof(MAX_COUNT_KEY);
+
+ sprintf(strData, "%d", count);
+ data.data = strData;
+ data.size = strlen(strData);
+
+ ret = info->dbp->put(info->dbp, NULL, &key, &data, 0);
+ if (ret != 0)
+ {
+ strcpy(info->errorString, db_strerror(ret));
+ return false;
+ }
+ return true;
+ }
+
+ static int compare_int(const void *a, const void *b)
+ {
+ if (*(unsigned int *)a < *(unsigned int *)b)
+ return -1;
+ if (*(unsigned int *)a > *(unsigned int *)b)
+ return 1;
+ return 0;
+ }
+
+ b_bool remove_excessive_entries(cache_info *info, int curCount)
+ {
+ DBT key, data;
+ DBC *dbcp;
+ int ret, i, newCount;
+ time_t cutoff, *timeCache;
+ int numEntries;
+
+ numEntries = info->maxCount / 10;
+ if (curCount < 0)
+ curCount = get_count(info);
+
+ if (numEntries < 1 || numEntries > curCount)
+ return false;
+
+ ret = info->dbp->cursor(info->dbp, NULL, &dbcp, 0);
+ if (ret != 0)
+ {
+ strcpy(info->errorString, db_strerror(ret));
+ return false;
+ }
+
+ memset(&key, 0, sizeof(key));
+ memset(&data, 0, sizeof(data));
+
+ timeCache = malloc(sizeof(int) * curCount);
+ for(i = 0; i < curCount + 1; i++)
+ {
+ ret = dbcp->c_get(dbcp, &key, &data, DB_NEXT);
+ if (ret != 0)
+ break;
+
+ if (((char *)key.data)[0] == '?')
+ {
+ i--;
+ continue;
+ }
+
+ timeCache[i] = atoi(data.data);
+ }
+ dbcp->c_close(dbcp);
+
+ qsort(timeCache, curCount, sizeof(int), compare_int);
+ cutoff = timeCache[numEntries];
+ free(timeCache);
+
+ ret = info->dbp->cursor(info->dbp, NULL, &dbcp, 0);
+ if (ret != 0)
+ {
+ strcpy(info->errorString, db_strerror(ret));
+ return false;
+ }
+
+ newCount = curCount;
+ for(i = 0; i < curCount && newCount > 0; i++)
+ {
+ ret = dbcp->c_get(dbcp, &key, &data, DB_NEXT);
+ if (ret != 0)
+ break;
+
+ if (((char *)key.data)[0] == '?')
+ {
+ i--;
+ continue;
+ }
+
+ if (atoi(data.data) <= cutoff)
+ {
+ ret = info->dbp->del(info->dbp, NULL, &key, 0);
+ if (ret == 0)
+ newCount--;
+ }
+ }
+ dbcp->c_close(dbcp);
+ set_count(info, newCount);
+
+ return true;
+ }
+ #endif
+
+ #if 0
+ int main(void)
+ {
+ int i;
+ unsigned rand;
+ cache_info *info;
+ cache_entry entry;
+ time_t t;
+
+ time(&t);
+ srandom(t);
+
+ info = init_cache();
+ open_cache(info, "test.db");
+
+ for(i = 0; i < 34873; i++)
+ {
+ sprintf(entry.fileName, "%u%u%u", random(), random(), random());
+ entry.lastUsedDate = random() % t;
+ entry.lastModDate = random() % t;
+ sprintf(entry.bitprint, "foo");
+
+ printf("\nInsert entry %d\n", i);
+ add_cache_entry(info, &entry, 0);
+ }
+ close_cache(info);
+
+ return 1;
}
#endif
Index: main.c
===================================================================
RCS file: /cvsroot/bitcollider/bitcollider/lib/main.c,v
retrieving revision 1.34
retrieving revision 1.35
diff -C2 -r1.34 -r1.35
*** main.c 2001/10/10 23:25:48 1.34
--- main.c 2001/10/12 19:59:51 1.35
***************
*** 170,173 ****
--- 170,174 ----
#if USE_BDB
+ bc->maxCacheSize = DEFAULT_MAX_CACHE_SIZE;
if (cacheFile[0] != 0)
{
***************
*** 177,185 ****
if (!open_cache(bc->cache, cacheFile))
{
! if (printDebugInfo)
! fprintf(stderr, "Failed to open cache file: %s.\n\n",
! cacheFile);
! close_cache(bc->cache);
! bc->cache = NULL;
}
}
--- 178,193 ----
if (!open_cache(bc->cache, cacheFile))
{
! if (printDebugInfo)
! fprintf(stderr, "Failed to open cache file: %s. "
! "Creating a new one.\n\n", cacheFile);
! unlink(cacheFile);
! if (!create_cache(bc->cache, cacheFile, bc->maxCacheSize))
! {
! if (printDebugInfo)
! fprintf(stderr, "Failed to create cache file: %s.\n\n",
! cacheFile);
! close_cache(bc->cache);
! bc->cache = NULL;
! }
}
}
***************
*** 295,300 ****
#if USE_BDB
/* Check to see if this bitprint has already been bitprinted */
! strcpy(centry.fileName, fileName);
! if (get_cache_entry(submission->bc->cache, ¢ry))
{
if (stat(submission->fileName, &fileInfo) == 0)
--- 303,311 ----
#if USE_BDB
/* Check to see if this bitprint has already been bitprinted */
! if (realpath(fileName, centry.fileName) == NULL)
! strcpy(centry.fileName, fileName);
!
! if (submission->bc->cache &&
! get_cache_entry(submission->bc->cache, ¢ry))
{
if (stat(submission->fileName, &fileInfo) == 0)
***************
*** 309,316 ****
}
}
- else
- {
- printf("Stat failed.\n");
- }
/* Its out of date or something else is wrong. Remove the entry
from the cache */
--- 320,323 ----
***************
*** 444,454 ****
#if USE_BDB
- strcpy(centry.fileName, fileName);
- strcpy(centry.bitprint, bitprint);
-
if (stat(submission->fileName, &fileInfo) == 0)
{
centry.lastModDate = fileInfo.st_mtime;
! if (!add_cache_entry(submission->bc->cache, ¢ry))
{
printf("Failed to insert bitprint into cache.\n");
--- 451,459 ----
#if USE_BDB
if (stat(submission->fileName, &fileInfo) == 0)
{
centry.lastModDate = fileInfo.st_mtime;
! if (submission->bc->cache &&
! !add_cache_entry(submission->bc->cache, ¢ry, false))
{
printf("Failed to insert bitprint into cache.\n");
***************
*** 492,495 ****
--- 497,520 ----
bc->exitNow = exitNow;
}
+
+ void clear_bitprint_cache(Bitcollider *bc)
+ {
+ #if USE_BDB
+ if (bc->cache)
+ {
+ int maxCount;
+ char fileName[MAX_PATH];
+
+ strcpy(fileName, bc->cache->fileName);
+ maxCount = get_max_count(bc->cache);
+ close_cache(bc->cache);
+ unlink(fileName);
+
+ bc->cache = init_cache();
+ create_cache(bc->cache, fileName, maxCount);
+ }
+ #endif
+ }
+
int get_num_bitprints(BitcolliderSubmission *sub)
|
×
Want the latest updates on software, tech news, and AI?
Get latest updates about software, tech news, and AI from SourceForge directly in your inbox once a month.
Thanks for helping keep SourceForge clean.
X