[Mhash-dev] Mhash (md5) performance on huge lists

[Sebastian H. <seb...@ya...>]

Hi,

Iam playing at the moment a little with has pre-processing. I started off using a obscenely slow bash script, and then went on using a perl one-liner. 
Now I though I could increase performance even more using a C implementation. 

My test setup looks like this

perl:
cat "$1" | perl -MDigest::MD5=md5_hex -lpe '$_ = md5_hex $_' >"$1".md5

c:
cat "$1" | ./mhash_wrapper.md5 >"$1".md5

whereas I tested 2 types of C programs to do the job. (The c sources are attached below)

The result was strange.

Perl: 33 seconds
C using fread (single-character-input): 78 seconds
C using fgets (line-input): 128 seconds

the password list has 18584390 passwords. 

Now for my understanding. How can I improve the performance of my kiddie-c-program using mhash - or can I use some cpu-depending compiler switches (sse/mmx) to increase performance?
The mhash is not compiled statically - does this have a influcence

C Number 1 using fgets

#include <mhash.h> //mhash
#include <stdio.h>
#include <stdlib.h> //exit
#include <unistd.h> //getopt
#include <ctype.h> //getopt



int main(int argc, char **argv)
 {

        int i;
        int len;
        MHASH td;
        unsigned char buffer;
        unsigned char *hash;
        char line[4096];

//        td = mhash_init(MHASH_WHIRLPOOL);
//        td = mhash_init(MHASH_RIPEMD160);
        td = mhash_init(MHASH_MD5);

        if (td == MHASH_FAILED) exit(1);

        while (fgets(line, sizeof line, stdin) != NULL) { // read a line each time
                len = strlen(line);
                char *p = strrchr(line, '\n');

                    if (p != NULL)
                        mhash(td, line, len - 1);  // strip the new line
                    else
                        mhash(td, line, len);

                hash = mhash_end(td);
//                for (i = 0; i < mhash_get_block_size(MHASH_WHIRLPOOL); i++) { printf("%.2x", hash[i]); }
//                for (i = 0; i < mhash_get_block_size(MHASH_RIPEMD160); i++) { printf("%.2x", hash[i]); }
                for (i = 0; i < mhash_get_block_size(MHASH_MD5); i++) { printf("%.2x", hash[i]); }

                printf("\n");
                td = mhash_init(MHASH_WHIRLPOOL);
                free(hash);
}




        exit(0);
 }




and C number 2 using fread

#include <mhash.h> //mhash
#include <stdio.h>
#include <stdlib.h> //exit
#include <unistd.h> //getopt


int main(void)
 {

        int i;
        MHASH td;
        unsigned char buffer;
        unsigned char *hash;

        td = mhash_init(MHASH_MD5);

        if (td == MHASH_FAILED) exit(1);

        while (fread(&buffer, 1, 1, stdin) == 1) { // read from stdin until receive EOF

        if ( buffer != '\n' ) { mhash(td, &buffer, 1); } //dont calculate line break
        if ( buffer == '\n' ) {  //received line break
        hash = mhash_end(td);

        for (i = 0; i < mhash_get_block_size(MHASH_MD5); i++) { printf("%.2x", hash[i]); }
        printf("\n");
        td = mhash_init(MHASH_MD5);
}
}



        exit(0);
 }