Menu
▾
▴
[mdb-dev] MDB files that need decoding - that otherwise cause segfaults or errors
|
From: Ross J. <ros...@ho...> - 2005-12-11 02:36:24
|
Hi,
I have a large 27 Mb MDB file (Access 97 - Jet 3) that was seg faulting
with most utils in mdbtools-0.5 and producing a 'Does not appear to be
an Access database file' message from mdbtools-0.6pre1. Searching the
mdb-tools-dev list archives I found David Mansfield's patch (message
dated 2004-06-13), which, although the attached patch seemed a little
corrupted at the end, worked perfectly for my MDB file once the missing
bit was fixed. I'm assuming the attachment was corrupted when it was
processed for the web archive page.
Parts of this patch appear to have been incorporated into 0.6pre1, but
the essential RC4 decoding parts appear to have been left out. Given
that many people still seem to be having the same problem, is there some
reason, perhaps legal, that the RC4 decoding can't be included?
David's patch (reconstructed relative to the current mdb-
tools-0.5.tar.gz distribution) is included again below, with some very
minor header prototype additions to build it error free on my Fedora
Core 3 Linux system (these additions aren't conditional but should not
affect other systems).
One minor change to David's patch is the inclusion of the libm.a library
to the LIBS variable in Makefile.in rather than Makefile.am. Makefile.am
is the correct place to add it, but on my system 'make' was attempting
to run the auto tools and was failing. Adding it to Makefile.in simply
required running the top level ./configure.
Many many thanks to the MDB tools developers for a great tool and to
David Mansfield for his essential patch.
To apply the patch to mdb-tools-0.5.tar.gz, save it to a file 'mdb-
tools-0.5-patch' making sure not to fold any long lines, then:
tar zxvf mdb-tools-0.5.tar.gz
cd mdb-tools-0.5
patch -p1 < ../mdb-tools-0.5-patch
To build mdb-tools-0.5:
./configure
make
make install
(i.e. as usual)
------------------------------cut here------------------------------
diff -rwc mdbtools-0.5-orig/include/mdbtools.h mdbtools-0.5/include/mdbtools.h
*** mdbtools-0.5-orig/include/mdbtools.h 2003-01-13 09:59:42.000000000 +1100
--- mdbtools-0.5/include/mdbtools.h 2005-12-11 12:07:37.000000000 +1100
***************
*** 327,339 ****
extern int mdb_is_fixed_col(MdbColumn *col);
extern char *mdb_col_to_string(MdbHandle *mdb, int start, int datatype, int size);
extern int mdb_find_end_of_row(MdbHandle *mdb, int row);
!
/* dump.c */
extern void buffer_dump(const unsigned char* buf, int start, int end);
/* backend.c */
extern char *mdb_get_coltype_string(MdbBackend *backend, int col_type);
extern void mdb_init_backends();
extern void mdb_register_backend(MdbBackend *backend, char *backend_name);
extern int mdb_set_default_backend(MdbHandle *mdb, char *backend_name);
--- 327,340 ----
extern int mdb_is_fixed_col(MdbColumn *col);
extern char *mdb_col_to_string(MdbHandle *mdb, int start, int datatype, int size);
extern int mdb_find_end_of_row(MdbHandle *mdb, int row);
! extern void mdb_set_date_fmt(const char *);
/* dump.c */
extern void buffer_dump(const unsigned char* buf, int start, int end);
/* backend.c */
extern char *mdb_get_coltype_string(MdbBackend *backend, int col_type);
+ int mdb_coltype_takes_length(MdbBackend *backend, int col_type);
extern void mdb_init_backends();
extern void mdb_register_backend(MdbBackend *backend, char *backend_name);
extern int mdb_set_default_backend(MdbHandle *mdb, char *backend_name);
diff -rwc mdbtools-0.5-orig/src/gmdb2/gmdb.h mdbtools-0.5/src/gmdb2/gmdb.h
*** mdbtools-0.5-orig/src/gmdb2/gmdb.h 2003-01-13 09:59:42.000000000 +1100
--- mdbtools-0.5/src/gmdb2/gmdb.h 2005-12-11 12:07:37.000000000 +1100
***************
*** 24,29 ****
--- 24,30 ----
GtkWidget *gmdb_table_def_new(MdbCatalogEntry *entry);
GtkWidget *gmdb_table_export_new(MdbCatalogEntry *entry);
void gmdb_table_export(MdbCatalogEntry *entry) ;
+ void gmdb_table_export_populate_dialog(GladeXML *xml);
void gmdb_file_select_cb(GtkWidget *w, gpointer data);
void gmdb_file_open_cb(GtkWidget *w, gpointer data);
***************
*** 32,37 ****
--- 33,40 ----
void gmdb_file_open_recent(gchar *menuname);
void gmdb_sql_new_window_cb(GtkWidget *w, gpointer data);
+ void gmdb_sql_save_as_cb(GtkWidget *w, GladeXML *xml);
+ void gmdb_sql_save_query(GladeXML *xml, gchar *file_path);
void gmdb_table_populate(MdbHandle *mdb);
void gmdb_form_populate(MdbHandle *mdb);
diff -rwc mdbtools-0.5-orig/src/libmdb/backend.c mdbtools-0.5/src/libmdb/backend.c
*** mdbtools-0.5-orig/src/libmdb/backend.c 2002-12-11 10:35:25.000000000 +1100
--- mdbtools-0.5/src/libmdb/backend.c 2005-12-11 12:07:37.000000000 +1100
***************
*** 53,59 ****
"NUMBER",
"FLOAT",
"FLOAT",
! "DATE",
"Oracle_Unknown 0x09",
"VARCHAR2",
"BLOB",
--- 53,59 ----
"NUMBER",
"FLOAT",
"FLOAT",
! "-DATE",
"Oracle_Unknown 0x09",
"VARCHAR2",
"BLOB",
***************
*** 112,126 ****
char *mdb_get_coltype_string(MdbBackend *backend, int col_type)
{
! char buf[100];
if (col_type > 0x10) {
// return NULL;
sprintf(buf,"type %04x", col_type);
return buf;
} else {
! return backend->types_table[col_type];
}
}
/*
** mdb_init_backends() initializes the mdb_backends hash and loads the builtin
** backends
--- 112,139 ----
char *mdb_get_coltype_string(MdbBackend *backend, int col_type)
{
! static char buf[100];
if (col_type > 0x10) {
// return NULL;
sprintf(buf,"type %04x", col_type);
return buf;
} else {
! char *str = backend->types_table[col_type];
! if (*str == "-")
! str++;
! return str;
}
}
+
+ int mdb_coltype_takes_length(MdbBackend *backend, int col_type)
+ {
+ if (col_type > 0x10) {
+ return 1;
+ } else {
+ return (*backend->types_table[col_type] != "-");
+ }
+ }
+
/*
** mdb_init_backends() initializes the mdb_backends hash and loads the builtin
** backends
diff -rwc mdbtools-0.5-orig/src/libmdb/data.c mdbtools-0.5/src/libmdb/data.c
*** mdbtools-0.5-orig/src/libmdb/data.c 2003-01-13 09:59:43.000000000 +1100
--- mdbtools-0.5/src/libmdb/data.c 2005-12-11 12:07:37.000000000 +1100
***************
*** 22,33 ****
--- 22,41 ----
#include "math.h"
#define MDB_DEBUG_OLE 1
+ #define OFFSET_MASK 0x1fff
char *mdb_money_to_string(MdbHandle *mdb, int start, char *s);
static int _mdb_attempt_bind(MdbHandle *mdb,
MdbColumn *col, unsigned char isnull, int offset, int len);
char *mdb_num_to_string(MdbHandle *mdb, int start, int datatype, int prec, int scale);
+ static char date_fmt[64] = "%x %X";
+
+ void mdb_set_date_fmt(const char * fmt)
+ {
+ date_fmt[63] = 0;
+ strncpy(date_fmt, fmt, 63);
+ }
void mdb_bind_column(MdbTableDef *table, int col_num, void *bind_ptr)
{
***************
*** 52,70 ****
MdbFormatConstants *fmt = mdb->fmt;
int row_end;
! /* Search the previous "row start" values for the first non-deleted one.
* If we don't find one, then the end of the page is the correct value.
*/
#if 1
if (row==0) {
row_end = fmt->pg_size - 1;
} else {
! row_end = (mdb_get_int16(mdb, ((fmt->row_count_offset + 2) + (row - 1) * 2)) & 0x0FFF) - 1;
}
return row_end;
#else
for (i = row - 1; i >= 0; i--) {
row_start = mdb_get_int16(mdb, ((fmt->row_count_offset + 2) + i * 2));
if (!(row_start & 0x8000)) {
break;
}
--- 60,80 ----
MdbFormatConstants *fmt = mdb->fmt;
int row_end;
! /* Search the previous "row start" values for the first non-"lookupflag" one.
* If we don't find one, then the end of the page is the correct value.
*/
#if 1
+ int i, row_start;
if (row==0) {
row_end = fmt->pg_size - 1;
} else {
! row_end = (mdb_get_int16(mdb, ((fmt->row_count_offset + 2) + (row - 1) * 2)) & OFFSET_MASK) - 1;
}
return row_end;
#else
for (i = row - 1; i >= 0; i--) {
row_start = mdb_get_int16(mdb, ((fmt->row_count_offset + 2) + i * 2));
+ /* if lookupflag is not set, it's good (deleteflag is ok) */
if (!(row_start & 0x8000)) {
break;
}
***************
*** 74,80 ****
if (i == -1) {
row_end = fmt->pg_size - 1;
} else {
! row_end = row_start - 1;
}
return row_end;
#endif
--- 84,90 ----
if (i == -1) {
row_end = fmt->pg_size - 1;
} else {
! row_end = (row_start & OFFSET_MASK) - 1;
}
return row_end;
#endif
***************
*** 177,183 ****
delflag = lookupflag = 0;
if (row_start & 0x8000) lookupflag++;
if (row_start & 0x4000) delflag++;
! row_start &= 0x0FFF; /* remove flags */
#if MDB_DEBUG
fprintf(stdout,"Row %d bytes %d to %d %s %s\n",
row, row_start, row_end,
--- 187,193 ----
delflag = lookupflag = 0;
if (row_start & 0x8000) lookupflag++;
if (row_start & 0x4000) delflag++;
! row_start &= OFFSET_MASK; /* remove flags */
#if MDB_DEBUG
fprintf(stdout,"Row %d bytes %d to %d %s %s\n",
row, row_start, row_end,
***************
*** 253,259 ****
}
/* if fixed columns add up to more than 256, we need a jump */
! if (col_start >= 256) {
num_of_jumps++;
jumps_used++;
row_start = row_start + col_start - (col_start % 256);
--- 263,269 ----
}
/* if fixed columns add up to more than 256, we need a jump */
! if (IS_JET3(mdb) && col_start >= 256) {
num_of_jumps++;
jumps_used++;
row_start = row_start + col_start - (col_start % 256);
***************
*** 394,417 ****
{
MdbCatalogEntry *entry = table->entry;
MdbHandle *mdb = entry->mdb;
! guint32 pgnum, i, j, bitn, map_pg;
unsigned char map_byte;
! pgnum = 0;
//printf("map size %ld\n", table->map_sz);
for (i=1;i<table->map_sz-1;i+=4) {
map_pg = _mdb_get_int32(table->usage_map, i);
//printf("loop %d pg %ld %02x%02x%02x%02x\n",i, map_pg,table->usage_map[i],table->usage_map[i+1],table->usage_map[i+2],table->usage_map[i+3]);
! if (!map_pg) continue;
if(mdb_read_alt_pg(mdb, map_pg) != mdb->fmt->pg_size) {
fprintf(stderr, "Oops! didn't get a full page at %d\n", map_pg);
exit(1);
}
//printf("reading page %ld\n",map_pg);
! for (j=4;j<mdb->fmt->pg_size;j++) {
! for (bitn=0;bitn<8;bitn++) {
if (mdb->alt_pg_buf[j] & 1 << bitn && pgnum > table->cur_phys_pg) {
table->cur_phys_pg = pgnum;
if (!mdb_read_pg(mdb, pgnum)) {
--- 404,444 ----
{
MdbCatalogEntry *entry = table->entry;
MdbHandle *mdb = entry->mdb;
! guint32 pgnum, i, j, bitn, map_pg, map_ind, offset, bit_offset;
unsigned char map_byte;
! /*
! * table->cur_phys_pg will tell us where to (re)start the scan
! * for the next data page. each usage_map entry points to a
! * 0x05 page which bitmaps (mdb->fmt->pg_size - 4) * 8 pages.
! *
! * map_ind gives us the starting usage_map entry
! * offset gives us a page offset into the bitmap (must be converted
! * to bytes and bits).
! */
! pgnum = table->cur_phys_pg + 1;
! map_ind = pgnum / ((mdb->fmt->pg_size - 4) * 8);
! offset = pgnum % ((mdb->fmt->pg_size - 4) * 8);
! bit_offset = offset % 8;
//printf("map size %ld\n", table->map_sz);
for (i=1;i<table->map_sz-1;i+=4) {
map_pg = _mdb_get_int32(table->usage_map, i);
//printf("loop %d pg %ld %02x%02x%02x%02x\n",i, map_pg,table->usage_map[i],table->usage_map[i+1],table->usage_map[i+2],table->usage_map[i+3]);
! /* if the usage_map entry is empty, skip it, but advance the page counter */
! if (!map_pg) {pgnum += (mdb->fmt->pg_size - 4) * 8; continue;}
if(mdb_read_alt_pg(mdb, map_pg) != mdb->fmt->pg_size) {
fprintf(stderr, "Oops! didn't get a full page at %d\n", map_pg);
exit(1);
}
//printf("reading page %ld\n",map_pg);
!
! /* first time through, start j, bitn based on the starting offset
! * after that, reset offset to 0 to start at the beginning of the page/byte
! */
! for (j=4 + offset / 8;j<mdb->fmt->pg_size;j++) {
! for (bitn=bit_offset;bitn<8;bitn++) {
if (mdb->alt_pg_buf[j] & 1 << bitn && pgnum > table->cur_phys_pg) {
table->cur_phys_pg = pgnum;
if (!mdb_read_pg(mdb, pgnum)) {
***************
*** 423,429 ****
--- 450,459 ----
}
pgnum++;
}
+ bit_offset = 0;
}
+
+ offset = 0;
}
/* didn't find anything */
//printf("returning 0\n");
***************
*** 476,482 ****
if (!table->cur_pg_num) {
table->cur_pg_num=1;
table->cur_row=0;
! mdb_read_next_dpg(table);
}
do {
--- 506,513 ----
if (!table->cur_pg_num) {
table->cur_pg_num=1;
table->cur_row=0;
! if (!mdb_read_next_dpg(table))
! return 0;
}
do {
***************
*** 784,796 ****
*/
return text;
}
char *mdb_col_to_string(MdbHandle *mdb, int start, int datatype, int size)
{
/* FIX ME -- not thread safe */
static char text[MDB_BIND_SIZE];
char tmpbuf[10];
time_t t;
! int i,j;
switch (datatype) {
case MDB_BOOL:
--- 815,853 ----
*/
return text;
}
+
+ static int trim_trailing_zeros(char * buff, int n)
+ {
+ char * p = buff + n - 1;
+ while (p >= buff && *p == "0")
+ *p-- = "\0";
+ if (*p == ".")
+ *p = "\0";
+ }
+
+
+ /*
+ * how much precision should we ask for from printf and friends?
+ * pass FLT_DIG for floats, DBL_DIG for doubles
+ */
+ int max_dec_places(double f, int dig_prec)
+ {
+ int ret = dig_prec - (int)ceil(log10(fabs(f)));
+ /* always at least one digit of precision so the trim function works */
+ if (ret < 1)
+ ret = 1;
+ return ret;
+ }
+
char *mdb_col_to_string(MdbHandle *mdb, int start, int datatype, int size)
{
/* FIX ME -- not thread safe */
static char text[MDB_BIND_SIZE];
char tmpbuf[10];
time_t t;
! int i,j,n;
! float tf;
! double td;
switch (datatype) {
case MDB_BOOL:
***************
*** 810,820 ****
return text;
break;
case MDB_FLOAT:
! sprintf(text,"%f",mdb_get_single(mdb, start));
return text;
break;
case MDB_DOUBLE:
! sprintf(text,"%f",mdb_get_double(mdb, start));
return text;
break;
case MDB_TEXT:
--- 867,881 ----
return text;
break;
case MDB_FLOAT:
! tf = mdb_get_single(mdb, start);
! n = sprintf(text,"%.*f", max_dec_places(tf, FLT_DIG), tf);
! trim_trailing_zeros(text, n);
return text;
break;
case MDB_DOUBLE:
! td = mdb_get_double(mdb, start);
! n = sprintf(text,"%.*f", max_dec_places(td, DBL_DIG), td);
! trim_trailing_zeros(text, n);
return text;
break;
case MDB_TEXT:
***************
*** 846,852 ****
break;
case MDB_SDATETIME:
t = (long int)((mdb_get_double(mdb, start) - 25569.0) * 86400.0);
! strftime(text, MDB_BIND_SIZE, "%x %X",
(struct tm*)gmtime(&t));
return text;
--- 907,913 ----
break;
case MDB_SDATETIME:
t = (long int)((mdb_get_double(mdb, start) - 25569.0) * 86400.0);
! strftime(text, MDB_BIND_SIZE, date_fmt,
(struct tm*)gmtime(&t));
return text;
diff -rwc mdbtools-0.5-orig/src/libmdb/file.c mdbtools-0.5/src/libmdb/file.c
*** mdbtools-0.5-orig/src/libmdb/file.c 2003-01-13 09:59:43.000000000 +1100
--- mdbtools-0.5/src/libmdb/file.c 2005-12-11 12:07:37.000000000 +1100
***************
*** 26,31 ****
--- 26,95 ----
2048, 0x08, 12, 25, 27, 31, 35, 36, 43, 8, 13, 16, 1, 18
};
+ typedef struct _RC4_KEY
+ {
+ unsigned char state[256];
+ unsigned char x;
+ unsigned char y;
+ } RC4_KEY;
+
+ #define swap_byte(x,y) t = *(x); *(x) = *(y); *(y) = t
+
+ void RC4_set_key(RC4_KEY *key, int key_data_len, unsigned char *key_data_ptr)
+ {
+ int i;
+ unsigned char t;
+ unsigned char swapByte;
+ unsigned char index1;
+ unsigned char index2;
+ unsigned char* state;
+ short counter;
+
+ state = &key->state[0];
+ for(counter = 0; counter < 256; counter++)
+ state[counter] = counter;
+ key->x = 0;
+ key->y = 0;
+ index1 = 0;
+ index2 = 0;
+ for(counter = 0; counter < 256; counter++)
+ {
+ index2 = (key_data_ptr[index1] + state[counter] + index2) % 256;
+ swap_byte(&state[counter], &state[index2]);
+ index1 = (index1 + 1) % key_data_len;
+ }
+ }
+
+ /*
+ * this algorithm does "encrypt in place" instead of inbuff/outbuff
+ * note also: encryption and decryption use same routine
+ * implementation supplied by (Adam Back) at <adam at cypherspace dot org>
+ */
+
+ void RC4(RC4_KEY *key, int buffer_len, unsigned char * buff)
+ {
+ unsigned char t;
+ unsigned char x;
+ unsigned char y;
+ unsigned char* state;
+ unsigned char xorIndex;
+ short counter;
+
+ x = key->x;
+ y = key->y;
+ state = &key->state[0];
+ for(counter = 0; counter < buffer_len; counter++)
+ {
+ x = (x + 1) % 256;
+ y = (state[x] + y) % 256;
+ swap_byte(&state[x], &state[y]);
+ xorIndex = (state[x] + state[y]) % 256;
+ buff[counter] ^= state[xorIndex];
+ }
+ key->x = x;
+ key->y = y;
+ }
+
static size_t _mdb_read_pg(MdbHandle *mdb, unsigned char *pg_buf, unsigned long pg);
static int mdb_find_file(char *file_name, char *file_path, int bufsize)
{
***************
*** 118,125 ****
/* get the db encryption key and xor it back to clear text */
f->db_key = mdb_get_int32(mdb, 0x3e);
- f->db_key ^= 0xe15e01b9;
/* get the db password located at 0x42 bytes into the file */
for (pos=0;pos<14;pos++) {
--- 182,194 ----
/* get the db encryption key and xor it back to clear text */
f->db_key = mdb_get_int32(mdb, 0x3e);
+ /* I don"t know if this value is valid for some versions?
+ * it doesn"t seem to be valid for the databases I have
+ *
+ * f->db_key ^= 0xe15e01b9;
+ */
+ f->db_key ^= 0x4ebc8afb;
/* get the db password located at 0x42 bytes into the file */
for (pos=0;pos<14;pos++) {
***************
*** 209,214 ****
--- 278,296 ----
/* fprintf(stderr,"EOF reached %d bytes returned.\n",len, mdb->fmt->pg_size); */
return 0;
}
+
+ /*
+ * unencrypt the page if necessary.
+ * it might make sense to cache the unencrypted data blocks?
+ */
+ if (pg != 0 && mdb->f->db_key != 0)
+ {
+ RC4_KEY rc4_key;
+ unsigned int tmp_key = mdb->f->db_key ^ pg;
+ RC4_set_key(&rc4_key, 4, (unsigned char *)&tmp_key);
+ RC4(&rc4_key, mdb->fmt->pg_size, pg_buf);
+ }
+
return len;
}
void mdb_swap_pgbuf(MdbHandle *mdb)
***************
*** 332,338 ****
#endif
double d;
! if (offset <0 || offset+4 > mdb->fmt->pg_size) return -1;
memcpy(&d, &mdb->pg_buf[offset], 8);
--- 414,420 ----
#endif
double d;
! if (offset <0 || offset+8 > mdb->fmt->pg_size) return -1;
memcpy(&d, &mdb->pg_buf[offset], 8);
diff -rwc mdbtools-0.5-orig/src/libmdb/Makefile.am mdbtools-0.5/src/libmdb/Makefile.am
*** mdbtools-0.5-orig/src/libmdb/Makefile.am 2003-01-06 01:58:33.000000000 +1100
--- mdbtools-0.5/src/libmdb/Makefile.am 2005-12-11 12:07:37.000000000 +1100
***************
*** 1,4 ****
lib_LTLIBRARIES = libmdb.la
libmdb_la_SOURCES= catalog.c mem.c file.c kkd.c table.c data.c dump.c backend.c money.c sargs.c index.c like.c write.c stats.c
INCLUDES = -I$(top_srcdir)/include $(GLIB_CFLAGS)
! LIBS = $(GLIB_LIBS)
--- 1,4 ----
lib_LTLIBRARIES = libmdb.la
libmdb_la_SOURCES= catalog.c mem.c file.c kkd.c table.c data.c dump.c backend.c money.c sargs.c index.c like.c write.c stats.c
INCLUDES = -I$(top_srcdir)/include $(GLIB_CFLAGS)
! LIBS = $(GLIB_LIBS) -lm
diff -rwc mdbtools-0.5-orig/src/libmdb/Makefile.in mdbtools-0.5/src/libmdb/Makefile.in
*** mdbtools-0.5-orig/src/libmdb/Makefile.in 2003-01-19 12:55:41.000000000 +1100
--- mdbtools-0.5/src/libmdb/Makefile.in 2005-12-11 12:08:15.000000000 +1100
***************
*** 104,110 ****
lib_LTLIBRARIES = libmdb.la
libmdb_la_SOURCES = catalog.c mem.c file.c kkd.c table.c data.c dump.c backend.c money.c sargs.c index.c like.c write.c stats.c
INCLUDES = -I$(top_srcdir)/include $(GLIB_CFLAGS)
! LIBS = $(GLIB_LIBS)
subdir = src/libmdb
mkinstalldirs = $(SHELL) $(top_srcdir)/mkinstalldirs
CONFIG_CLEAN_FILES =
--- 104,110 ----
lib_LTLIBRARIES = libmdb.la
libmdb_la_SOURCES = catalog.c mem.c file.c kkd.c table.c data.c dump.c backend.c money.c sargs.c index.c like.c write.c stats.c
INCLUDES = -I$(top_srcdir)/include $(GLIB_CFLAGS)
! LIBS = $(GLIB_LIBS) -lm
subdir = src/libmdb
mkinstalldirs = $(SHELL) $(top_srcdir)/mkinstalldirs
CONFIG_CLEAN_FILES =
------------------------------cut here------------------------------
Regards.
Ross Johnson
|
