[oprof-cvs] CVS: oprofile-tests/db ChangeLog,1.2,1.3 db-debug.c,1.2,1.3 db-insert.c,1.2,1.3 db-manage.c,1.5,1.6 db-test.c,1.5,1.6 db-travel.c,1.1,1.2 db.h,1.3,1.4

[Philippe E. <ph...@us...>]

Update of /cvsroot/oprofile/oprofile-tests/db
In directory usw-pr-cvs1:/tmp/cvs-serv29563

Modified Files:
	ChangeLog db-debug.c db-insert.c db-manage.c db-test.c 
	db-travel.c db.h 
Log Message:

this finish implementation of db (I hope) adding file locking
and some trickery to allow use it w/o knowledge of the exact
file format.

Next step is to (rtfm cvs) create an oprofile branch to use it

Phil

Index: ChangeLog
===================================================================
RCS file: /cvsroot/oprofile/oprofile-tests/db/ChangeLog,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- ChangeLog	16 Apr 2002 02:16:51 -0000	1.2
+++ ChangeLog	17 Apr 2002 04:09:01 -0000	1.3
@@ -1,3 +1,14 @@
+2002-04-17  Philippe Elie  <ph...@wa...>
+
+	* db-debug.c:
+	* db-insert.c:
+	* db-manage.c:
+	* dbtest.c:
+	* db-travel.c:
+	* db.h: add file locking and necessary stuff to allow
+	  plugging db in an imbedded file format like oprofile
+	  samples files.
+
 2002-04-16  Philippe Elie  <ph...@wa...>
 
 	* db-manage.c: silly bug fix, db works now with

Index: db-debug.c
===================================================================
RCS file: /cvsroot/oprofile/oprofile-tests/db/db-debug.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- db-debug.c	15 Apr 2002 04:20:32 -0000	1.2
+++ db-debug.c	17 Apr 2002 04:09:01 -0000	1.3
@@ -24,14 +24,14 @@
 
 void display_tree(const root_t * root)
 {
-	do_display_tree(root, root->root);
+	do_display_tree(root, root->descr->root_idx);
 }
 
 static void do_raw_display_tree(const root_t * root, int page_idx)
 {
 	unsigned int i;
-	printf("root %d\n", root->root);
-	for (i = 0 ; i < root->current_size ; ++i) {
+	printf("root %d\n", root->descr->root_idx);
+	for (i = 0 ; i < root->descr->current_size ; ++i) {
 		page_t * page;
 		int j;
 
@@ -49,7 +49,7 @@
 
 void raw_display_tree(const root_t * root)
 {
-	do_raw_display_tree(root, root->root);
+	do_raw_display_tree(root, root->descr->root_idx);
 }
 
 static int do_check_page_pointer(const root_t * root, unsigned int page_idx,
@@ -62,9 +62,10 @@
 	if (page_idx == nil_page)
 		return 0;
 
-	if (page_idx >= root->current_size) {
+	if (page_idx >= root->descr->current_size) {
 		printf("%s:%d invalid page number, max is %d page_nr is %d\n",
-		       __FILE__, __LINE__, root->current_size, page_idx);
+		       __FILE__, __LINE__, root->descr->current_size,
+		       page_idx);
 		return 1;
 	}
 
@@ -104,14 +105,15 @@
 	int ret;
 	int * viewed_page;
 
-	if (root->current_size > root->size) {
+	if (root->descr->current_size > root->descr->size) {
 		printf("%s:%d invalid current size %d, %d\n",
-		       __FILE__, __LINE__, root->current_size, root->size);
+		       __FILE__, __LINE__,
+		       root->descr->current_size, root->descr->size);
 	}
 
-	viewed_page = calloc(root->current_size, sizeof(int));
+	viewed_page = calloc(root->descr->current_size, sizeof(int));
 
-	ret = do_check_page_pointer(root, root->root, viewed_page);
+	ret = do_check_page_pointer(root, root->descr->root_idx, viewed_page);
 
 	free(viewed_page);
 
@@ -147,7 +149,7 @@
 {
 	int ret = check_page_pointer(root);
 	if (!ret)
-		ret = do_check_tree(root, root->root, 0u);
+		ret = do_check_tree(root, root->descr->root_idx, 0u);
 
 	return ret;
 }

Index: db-insert.c
===================================================================
RCS file: /cvsroot/oprofile/oprofile-tests/db/db-insert.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- db-insert.c	15 Apr 2002 02:50:16 -0000	1.2
+++ db-insert.c	17 Apr 2002 04:09:01 -0000	1.3
@@ -1,7 +1,24 @@
+#include <sys/file.h>
 #include <assert.h>
 
 #include "db.h"
 
+static void lock_db(root_t * root)
+{
+	if (root->is_locked == 0) {
+		flock(root->fd, LOCK_EX);
+		root->is_locked = 1;
+	}
+}
+
+static void unlock_db(root_t * root)
+{
+	if (root->is_locked) {
+		flock(root->fd, LOCK_UN);
+		root->is_locked = 0;
+	}
+}
+
 static void copy_item(value_t * dest, const value_t * src,
 		      unsigned int nr_item)
 {
@@ -182,6 +199,7 @@
 	}
 
 	if (need_reorg) {
+		lock_db(root);
 		need_reorg = do_reorg(root, page_idx, left, excess_elt, value);
 	}
 
@@ -230,6 +248,7 @@
 	}
 
 	if (need_reorg) {
+		lock_db(root);
 		need_reorg = do_reorg(root, page_idx, pos, excess_elt, value);
 	}
 
@@ -247,34 +266,39 @@
 	value.info = info;
 	value.child_page = nil_page;
 
-	if (!root->base_area) {
+	if (root->descr->root_idx == nil_page) {
 		/* create the root. */
 		page_t * page;
 
-		root->root = add_page(root);
+		/* we don't need to lock_db() here */
 
-		page = page_nr_to_page_ptr(root, root->root);
+		root->descr->root_idx = add_page(root);
+
+		page = page_nr_to_page_ptr(root, root->descr->root_idx);
 
 		page->page_table[0] = value;
 		page->count = 1;
 		return;
 	}
 
-	need_reorg = do_insert(root, root->root, &excess_elt, &value);
+	need_reorg = do_insert(root, root->descr->root_idx, &excess_elt,
+			       &value);
 	if (need_reorg) {
 		/* increase the level of tree. */
 		page_t * new_page;
 		page_idx_t old_root;
 
-		old_root = root->root;
-		root->root = add_page(root);
+		old_root = root->descr->root_idx;
+		root->descr->root_idx = add_page(root);
 
 		/* page pointer can be invalidated by add_page, reload it */
-		new_page = page_nr_to_page_ptr(root, root->root);
+		new_page = page_nr_to_page_ptr(root, root->descr->root_idx);
 
 		new_page->page_table[0] = excess_elt;
 		new_page->count = 1;
 		new_page->p0 = old_root;
 	}
+
+	unlock_db(root);
 }
 

Index: db-manage.c
===================================================================
RCS file: /cvsroot/oprofile/oprofile-tests/db/db-manage.c,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- db-manage.c	16 Apr 2002 02:16:51 -0000	1.5
+++ db-manage.c	17 Apr 2002 04:09:01 -0000	1.6
@@ -1,80 +1,164 @@
 #define _GNU_SOURCE
 
 #include <stdlib.h>
+#include <sys/fcntl.h>
 #include <sys/mman.h>
+#include <unistd.h>
 #include <errno.h>
 #include <string.h>
 #include <stdio.h>
 
 #include "db.h"
 
-#define USE_MMAP
+static __inline db_descr * to_descr(root_t * root)
+{
+	return (db_descr *)(((char *)root->base_memory) + root->offset_descr);
+}
 
-page_idx_t add_page(root_t * root)
+static __inline page_t * to_page(root_t * root)
 {
-	if (root->current_size >= root->size) {
-		unsigned int old_size = root->size;
-		unsigned int pos;
+	return (page_t *)(((char *)root->base_memory) + root->offset_page);
+}
 
-#ifndef USE_MMAP
-		root->size = root->size ? root->size * 2 : 1;
-		root->base_area =
-			realloc(root->base_area, (root->size * sizeof(page_t)));
-//		printf("%p\n", root->base_area);
+static void init_page(root_t * root, unsigned int from, unsigned to)
+{
+	/* FIXME: nil_page is not currently a zero value so we need
+	 * to initialize it explicitely: perhaps we can consider to
+	 * not use the zero page and to use zero as nil_page value
+	 * avoiding to touch memory of the mmaped file */
+	for ( ; from != to ; ++from) {
+		unsigned int count;
+		page_t * page;
 
-		memset(&root->base_area[old_size], '\0',
-		       (old_size == 0 ? 1 : old_size) * sizeof(page_t));
-#else
-		root->size = root->size ? root->size * 2 : 1;
-		if (root->base_area) {
-			root->base_area = mremap(root->base_area,
-						 old_size * sizeof(page_t),
-						 root->size * sizeof(page_t),
-						 MREMAP_MAYMOVE);
-		} else {
-			root->base_area = mmap(0, root->size * sizeof(page_t),
-					       PROT_READ | PROT_WRITE,
-					       MAP_PRIVATE | MAP_ANON, -1, 0);
+		/* we can't use page_nr_to_page_ptr here because
+		 * page_nr_to_page_ptr can trigger an assertion ! */
+		page = &root->page_base[from];
+		page->p0 = nil_page;
+		for (count = 0 ; count < MAX_PAGE ; ++count) {
+			page->page_table[count].child_page = nil_page;
 		}
-		if (root->base_area == MAP_FAILED) {
-			printf("%s\n", strerror(errno));
-			exit(1);
+	}
+}
+
+page_idx_t add_page(root_t * root)
+{
+	if (root->descr->current_size >= root->descr->size) {
+		unsigned int old_size = root->descr->size;
+		unsigned int new_file_size;
+
+		root->descr->size *= 2;
+
+		new_file_size = root->descr->size * sizeof(page_t);
+		new_file_size += root->offset_page;
+
+		if (ftruncate(root->fd, new_file_size)) {
+			fprintf(stderr, "unable to resize file to %d "
+				"length, cause : %s\n",
+				new_file_size, strerror(errno));
+			exit(EXIT_FAILURE);
 		}
 
-		memset(&root->base_area[old_size], '\0',
-		       (old_size == 0 ? 1 : old_size) * sizeof(page_t));
-#endif
-		for (pos = old_size ; pos < root->size ; ++pos) {
-			unsigned int count;
-			page_t * page;
+		root->base_memory = mremap(root->base_memory,
+			 (old_size * sizeof(page_t)) + root->offset_page,
+			 new_file_size, MREMAP_MAYMOVE);
 
-			/* we can't use page_nr_to_page_ptr here because
-			 * page_nr_to_page_ptr can trigger an assertion ! */
-			page = &root->base_area[pos];
-//			page = page_nr_to_page_ptr(root, pos);
-			page->p0 = nil_page;
-			for (count = 0 ; count < MAX_PAGE ; ++count) {
-				page->page_table[count].child_page = nil_page;
-			}
+		if (root->base_memory == MAP_FAILED) {
+			fprintf(stderr, "add_page() mremap failure "
+				"cause: %s\n", strerror(errno));
+			exit(EXIT_FAILURE);
 		}
+
+		root->descr = to_descr(root);
+		root->page_base = to_page(root);
+
+		init_page(root, old_size, root->descr->size);
 	}
 
-	return (page_idx_t)root->current_size++;
+	return (page_idx_t)root->descr->current_size++;
 }
 
-/* TODO: this as nothing todo here ? */
-void init_root(root_t * root)
+/* the default number of page, calculated to fit in 4096 bytes */
+#define DEFAULT_PAGE_NR(offset_page)			\
+	(4096 - offset_page) / sizeof(page_t) ?		\
+	(4096 - offset_page) / sizeof(page_t) : 1 
+
+void db_open(root_t * root, const char * filename, unsigned int offset_page,
+	     unsigned int offset_descr)
 {
+	struct stat stat_buf;
+	unsigned int nr_page;
+
 	memset(root, '\0', sizeof(root_t));
+
+	root->offset_page = offset_page;
+	root->offset_descr = offset_descr;
+
+	root->fd = open(filename, O_RDWR | O_CREAT, 0644);
+	if (root->fd < 0) {
+		fprintf(stderr, "db_open() fail to open %s case: %s\n",
+			filename, strerror(errno));
+		exit(EXIT_FAILURE);
+	}
+
+	/* if the length of file is zero we have created it so we must grow
+	 * it. Can we grow it lazilly or must the lazilly things handled
+	 * by caller ? */
+	if (fstat(root->fd, &stat_buf)) {
+		fprintf(stderr, "unable to stat %s cause %s\n",
+			filename, strerror(errno));
+		exit(EXIT_FAILURE);
+	}
+
+	if (stat_buf.st_size == 0) {
+		unsigned int file_size;
+
+		nr_page = DEFAULT_PAGE_NR(root->offset_page);
+
+		file_size = offset_page + (nr_page * sizeof(page_t));
+		if (ftruncate(root->fd, file_size)) {
+			fprintf(stderr, "unable to resize file %s to %d "
+				"length, cause : %s\n",
+				filename, file_size, strerror(errno));
+		}
+	} else {
+		nr_page = (stat_buf.st_size - offset_page) / sizeof(page_t);
+
+		if (nr_page != root->descr->size) {
+			fprintf(stderr, "nr_page != root->descr->size\n");
+			exit(EXIT_FAILURE);
+		}
+	}
+
+	root->base_memory = 
+		mmap(0, (nr_page * sizeof(page_t)) + root->offset_page,
+		     PROT_READ | PROT_WRITE, MAP_SHARED, root->fd, 0);
+
+	if (root->base_memory == MAP_FAILED) {
+		fprintf(stderr, "add_page() mmap failure "
+			"cause: %s\n", strerror(errno));
+		exit(EXIT_FAILURE);
+	}
+
+
+	root->descr = to_descr(root);
+	root->page_base = to_page(root);
+
+	if (stat_buf.st_size == 0) {
+		root->descr->size = nr_page;
+		root->descr->current_size = 0;
+		root->descr->root_idx = nil_page;
+
+		init_page(root, 0, root->descr->size);
+	}
 }
 
-void delete_root(root_t * root)
+void db_close(root_t * root)
 {
-	if (root->base_area)
-#ifndef USE_MMAP
-		free(root->base_area);
-#else
-		munmap(root->base_area, root->size * sizeof(page_t));
-#endif
+	if (root->page_base)
+		munmap(root->base_memory,
+		     (root->descr->size * sizeof(page_t)) + root->offset_page);
+
+	if (root->fd != -1)
+		close(root->fd);
 }
 

Index: db-test.c
===================================================================
RCS file: /cvsroot/oprofile/oprofile-tests/db/db-test.c,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- db-test.c	16 Apr 2002 02:16:51 -0000	1.5
+++ db-test.c	17 Apr 2002 04:09:01 -0000	1.6
@@ -4,6 +4,8 @@
 
 #include "db.h"
 
+#define TEST_FILENAME "test-db.dat"
+
 static int nr_error;
 
 static double user_time()
@@ -22,13 +24,15 @@
 	double begin, end;
 	root_t root;
 
-	init_root(&root);
+	db_open(&root, TEST_FILENAME, 128, 0);
 	begin = user_time();
 	for (i = 0 ; i < nr_item ; ++i) {
 		insert(&root, (random() % nr_unique_item) + 1, 1);
 	}
 	end = user_time();
-	delete_root(&root);
+	db_close(&root);
+
+	remove(TEST_FILENAME);
 
 	fprintf(stderr, "nr item: %d, unique item: %d, elapsed: %f\n",
 		nr_item, nr_unique_item, end - begin);
@@ -51,7 +55,8 @@
 	root_t root;
 	int ret;
 
-	init_root(&root);
+	db_open(&root, TEST_FILENAME, 128, 0);
+
 
 	for (i = 0 ; i < nr_item ; ++i) {
 		insert(&root, (random() % nr_unique_item) + 1, 1);
@@ -59,7 +64,9 @@
 
 	ret = check_tree(&root);
 
-	delete_root(&root);
+	db_close(&root);
+
+	remove(TEST_FILENAME);
 
 	return ret;
 }
@@ -106,7 +113,7 @@
 	unsigned int first_key, last_key;
 	unsigned int old_nr_error = nr_error;
 
-	init_root(&root);
+	db_open(&root, TEST_FILENAME, 128, 0);
 
 	for (i = 0 ; i < nr_item ; ++i) {
 		insert(&root, (random() % nr_unique_item) + 1, 1);
@@ -133,7 +140,9 @@
 	
 	travel(&root, range_first, range_last, call_back, 0);
 
-	delete_root(&root);
+	db_close(&root);
+
+	remove(TEST_FILENAME);
 
 	return old_nr_error == nr_error ? 0 : 1;
 }

Index: db-travel.c
===================================================================
RCS file: /cvsroot/oprofile/oprofile-tests/db/db-travel.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- db-travel.c	15 Apr 2002 02:50:16 -0000	1.1
+++ db-travel.c	17 Apr 2002 04:09:01 -0000	1.2
@@ -45,5 +45,5 @@
 void travel(const root_t * root, unsigned int first, unsigned int last,
 	    travel_callback callback, void * data)
 {
-	do_travel(root, root->root, first, last, callback, data);
+	do_travel(root, root->descr->root_idx, first, last, callback, data);
 }

Index: db.h
===================================================================
RCS file: /cvsroot/oprofile/oprofile-tests/db/db.h,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- db.h	15 Apr 2002 19:13:00 -0000	1.3
+++ db.h	17 Apr 2002 04:09:01 -0000	1.4
@@ -1,6 +1,11 @@
-
-/** a tiny data base struct designed for oprofile
- * allow only an insert operation
+/**
+ * \file db.h
+ * Copyright 2002 OProfile authors
+ * Read the file COPYING
+ * this file contains various definitions and iterface for management
+ * of in-memory Btree.
+ *
+ * \author Philippe Elie <ph...@wa...>
  */
 
 #ifndef DB_H
@@ -8,6 +13,10 @@
 
 #include <assert.h>
 
+#ifndef fd_t
+#define fd_t int
+#endif
+
 /* must be in [2-a sensible value] */
 /* in function of MIN_PAGE two different search algo are selected at compile
  * time, a straight liner search or a dicho search, the best perf for the
@@ -34,24 +43,58 @@
 typedef struct {
 	unsigned int count;		/*< nr entry used in page_table */
 	page_idx_t p0;			/*< left page index */
-	value_t page_table[MAX_PAGE];	/*< key, data and child pointer */
+	value_t page_table[MAX_PAGE];	/*< key, data and child index */
 } page_t;
 
-/** a "database", for oprofile this a part of the samples files header */
+/** the minimal information which must be stored in the file to reload
+ * properly the data base */
 typedef struct {
-	page_t * base_area;		/*< base memory area */
-	page_idx_t root;		/*< the root page index */
-	unsigned int size;		/*< nr page */
+	unsigned int size;		/*< in page nr */
 	unsigned int current_size;	/*< nr used page */
-	// TODO we need a lock ? or do you want to lock the file itself ?
-	// perhaps a semaphore is more efficient ?
+	page_idx_t root_idx;		/*< the root page index */
+} db_descr;
+
+/** a "database". this is an in memory only description.
+ *
+ * The mmaped file must contain a db_descr struct which manage the current
+ * state of memory allocation (base pointer, root_idx etc.) the point here
+ * is to allow using this library w/o knowning the offset inside the mapped
+ * file of management data or the page base memory offset. A typical is use is:
+ *
+ * struct header { int etc; ... struct db_descr descr; .... };
+ * db_open(&root, filename, offsetof(header, descr), sizeof(header));
+ * so on this library have no dependency on the header type.
+ */
+typedef struct {
+	page_t * page_base;		/*< base memory area of the page */
+	fd_t fd;			/*< file descriptor of the maped mem */
+	void * base_memory;		/*< base memory of the maped memory */
+	db_descr * descr;		/*< the current state of database */
+	unsigned int offset_descr;	/*< from base_memory to descr */
+	unsigned int offset_page;	/*< from base_memory to page_base */
+	unsigned int is_locked;		/*< is fd already locked */
 } root_t;
 
 /* db-manage.c */
-void init_root(root_t * root);
-void delete_root(root_t * root);
+/** 
+ * \param root the data base object to setup 
+ * \param root_idx_ptr an external pointer to put the root index, can be null
+ * \param filename the filename where go the maped memory
+ * \param offset_page offset between the mapped memory and the data base page
+ * area.
+ *
+ * parameter root_idx_ptr and offset allow to use a data base imbeded in
+ * a file containing an header such as opd_header. db_open always preallocate
+ * a few number of page
+ */
+void db_open(root_t * root, const char * filename, unsigned int offset_page,
+	     unsigned int offset_descr);
+/**
+ * \param root the data base to close
+ */
+void db_close(root_t * root);
 
-/** add a page returning his index. Take care all page pointer are
+/** add a page returning its index. Take care all page pointer can be
  * invalidated by this call ! */
 page_idx_t add_page(root_t * root);
 
@@ -85,7 +128,7 @@
 					     page_idx_t page_nr)
 {
 	assert(page_nr < root->current_size);
-	return &root->base_area[page_nr];
+	return &root->page_base[page_nr];
 }
 
 #endif /* !DB_H */