[popfile-commit] engine/tests TestBayes.tst,1.27,1.28 TestHTML.script,1.17,1.18 TestHTML.tst,1.19,1.20 TestHTTP.tst,1.3,1.4 TestPOP3.tst,1.13,1.14

[<jgr...@us...>]

Update of /cvsroot/popfile/engine/tests
In directory sc8-pr-cvs1:/tmp/cvs-serv15349/tests

Modified Files:
	TestBayes.tst TestHTML.script TestHTML.tst TestHTTP.tst 
	TestPOP3.tst 
Log Message:
BROUGHT TEST SUITE TO 100% AFTER BERKELEYDB CHANGE

HTML.pm:

Small modification to the way that showing an individual bucket works 
so that when it is cleared we show the same page with a zero word 
count and no words.

Bayes.pm:

Mechanism for marking a bucket deleted even though the files in
its directory have not all disappeared.  This is needed because if
a user deletes a bucket while a download is occurring POPFile will
have a connection open to the db which will prevent it from getting
deleted.  Now have a helper delete_bucket_files__ that will do the
delete and add the special delete file as needed.

clear_bucket can't rely on being able to remove the table.db for
the same reasons as the above, so change the way in which it operates.

TestBayes.tst:

Tests to cover upgrading of corpus tables, fork handling and deal
with the fact that the corpus is no longer in flat files.  

TestHTML.tst:

Have to call prefork/forked/postfork when forking.  Test the new
force_fork UI elements.  

TestPOP3.tst:

Have to call prefork/forked/postfork when forking.

TestProxy.tst:

Add code to test the forking/non-forking version of the proxy code.















Index: TestBayes.tst
===================================================================
RCS file: /cvsroot/popfile/engine/tests/TestBayes.tst,v
retrieving revision 1.27
retrieving revision 1.28
diff -C2 -d -r1.27 -r1.28
*** TestBayes.tst	9 Sep 2003 00:28:20 -0000	1.27
--- TestBayes.tst	15 Sep 2003 14:55:37 -0000	1.28
***************
*** 62,66 ****
--- 62,72 ----
  
  $b->initialize();
+ 
+ # Hide the upgrading messages
+ 
+ open STDOUT, ">temp.tmp";
  test_assert( $b->start() );
+ close STDOUT;
+ unlink 'temp.tmp';
  
  # Test the unclassifier_probability parameter
***************
*** 90,99 ****
  test_assert_equal( $b->get_bucket_word_count($buckets[2]), 12114 );
  
! # get_bucket_word_list
  
! my @words = $b->get_bucket_word_list( 'personal' );
  test_assert_equal( $#words, 1 );
! test_assert_equal( $words[0], '|bar 2||baz 100|' );
! test_assert_equal( $words[1], '|foo 1|' );
  
  # get_word_count
--- 96,114 ----
  test_assert_equal( $b->get_bucket_word_count($buckets[2]), 12114 );
  
! # get_bucket_word_list and prefixes
  
! my @words = $b->get_bucket_word_prefixes( 'personal' );
  test_assert_equal( $#words, 1 );
! test_assert_equal( $words[0], 'b' );
! test_assert_equal( $words[1], 'f' );
! 
! @words = $b->get_bucket_word_list( 'personal', 'b' );
! test_assert_equal( $#words, 1 );
! test_assert_equal( $words[0], 'bar' );
! test_assert_equal( $words[1], 'baz' );
! 
! @words = $b->get_bucket_word_list( 'personal', 'f' );
! test_assert_equal( $#words, 0 );
! test_assert_equal( $words[0], 'foo' );
  
  # get_word_count
***************
*** 190,199 ****
  
  test_assert( -e 'corpus/zebra' );
! test_assert( -e 'corpus/zebra/table' );
! 
! open FILE, "<corpus/zebra/table";
! my $line = <FILE>;
! test_assert_regexp( $line, '__CORPUS__ __VERSION__ 1' );
! close FILE;
  
  # rename_bucket
--- 205,209 ----
  
  test_assert( -e 'corpus/zebra' );
! test_assert( -e 'corpus/zebra/table.db' );
  
  # rename_bucket
***************
*** 219,230 ****
  
  test_assert( -e 'corpus/zeotrope' );
! test_assert( -e 'corpus/zeotrope/table' );
! 
! open FILE, "<corpus/zeotrope/table";
! $line = <FILE>;
! test_assert_regexp( $line, '__CORPUS__ __VERSION__ 1' );
! $line = <FILE>;
! test_assert( !defined( $line ) );
! close FILE;
  
  # add_message_to_bucket
--- 229,233 ----
  
  test_assert( -e 'corpus/zeotrope' );
! test_assert( -e 'corpus/zeotrope/table.db' );
  
  # add_message_to_bucket
***************
*** 242,267 ****
  test_assert( !$b->add_message_to_bucket( 'none', 'TestMailParse021.msg' ) );
  test_assert( $b->add_message_to_bucket( 'zeotrope', 'TestMailParse021.msg' ) );
! open FILE, "<corpus/zeotrope/table";
! $line = <FILE>;
! test_assert_regexp( $line, '__CORPUS__ __VERSION__ 1' );
! while ( <FILE> ) {
!     if ( /(.+) (\d+)/ ) {
!         test_assert_equal( $words{$1}, $2, "zeotrope: $1 $2 $words{$1}" );
!     }
  }
- close FILE;
- `cp corpus/zeotrope/table temp.wrd`;
  
  test_assert( $b->add_message_to_bucket( 'zeotrope', 'TestMailParse021.msg' ) );
  
! open FILE, "<corpus/zeotrope/table";
! $line = <FILE>;
! test_assert_regexp( $line, '__CORPUS__ __VERSION__ 1' );
! while ( <FILE> ) {
!     if ( /(.+) (\d+)/ ) {
!         test_assert_equal( ($words{$1}*2), $2, "zeotrope: $1 $2 $words{$1}" );
!     }
  }
- close FILE;
  
  # remove_message_from_bucket
--- 245,258 ----
  test_assert( !$b->add_message_to_bucket( 'none', 'TestMailParse021.msg' ) );
  test_assert( $b->add_message_to_bucket( 'zeotrope', 'TestMailParse021.msg' ) );
! 
! foreach my $word (keys %words) {
!     test_assert_equal( $b->get_base_value_( 'zeotrope', $word ), $words{$word}, "zeotrope: $word $words{$word}" );
  }
  
  test_assert( $b->add_message_to_bucket( 'zeotrope', 'TestMailParse021.msg' ) );
  
! foreach my $word (keys %words) {
!     test_assert_equal( $b->get_base_value_( 'zeotrope', $word ), $words{$word}*2, "zeotrope: $word $words{$word}" );
  }
  
  # remove_message_from_bucket
***************
*** 271,294 ****
  test_assert( $b->remove_message_from_bucket( 'zeotrope', 'TestMailParse021.msg' ) );
  
! open FILE, "<corpus/zeotrope/table";
! $line = <FILE>;
! test_assert_regexp( $line, '__CORPUS__ __VERSION__ 1' );
! $line = <FILE>;
! test_assert( !defined( $line ) );
! close FILE;
  
  # add_messages_to_bucket
  
! test_assert( $b->add_message_to_bucket( 'zeotrope', [ 'TestMailParse021.msg', 'TestMailParse021.msg' ] ) );
  
! open FILE, "<corpus/zeotrope/table";
! $line = <FILE>;
! test_assert_regexp( $line, '__CORPUS__ __VERSION__ 1' );
! while ( <FILE> ) {
!     if ( /(.+) (\d+)/ ) {
!         test_assert_equal( ($words{$1}*2), $2, "zeotrope: $1 $2 $words{$1}" );
!     }
  }
! close FILE;
  
  # Test corrupting the corpus
--- 262,285 ----
  test_assert( $b->remove_message_from_bucket( 'zeotrope', 'TestMailParse021.msg' ) );
  
! test_assert_equal( $b->get_bucket_word_count( 'zeotrope' ), 0 );
  
  # add_messages_to_bucket
  
! test_assert( $b->add_messages_to_bucket( 'zeotrope', ( 'TestMailParse021.msg', 'TestMailParse021.msg' ) ) );
  
! foreach my $word (keys %words) {
!     test_assert_equal( $b->get_base_value_( 'zeotrope', $word ), $words{$word}*2, "zeotrope: $word $words{$word}" );
  }
! 
! # Make sure that fork functions keep the database open
! 
! $b->prefork();
! test_assert( !defined( $b->{matrix__}{zeotrope} ) );
! $b->forked();
! test_assert( defined( $b->{matrix__}{zeotrope} ) );
! $b->prefork();
! test_assert( !defined( $b->{matrix__}{zeotrope} ) );
! $b->postfork();
! test_assert( defined( $b->{matrix__}{zeotrope} ) );
  
  # Test corrupting the corpus
***************
*** 299,322 ****
  
  open STDERR, ">temp.tmp";
- test_assert( !$b->add_message_to_bucket( 'zeotrope', 'TestMailParse021.msg' ) );
- close STDERR;
- open FILE, "<temp.tmp";
- $line = <FILE>;
- test_assert_regexp( $line, 'Incompatible corpus version in zeotrope' );
- close FILE;
- 
- open STDERR, ">temp.tmp";
- test_assert( !$b->remove_message_from_bucket( 'zeotrope', 'TestMailParse021.msg' ) );
- close STDERR;
- open FILE, "<temp.tmp";
- $line = <FILE>;
- test_assert_regexp( $line, 'Incompatible corpus version in zeotrope' );
- close FILE;
- 
- open STDERR, ">temp.tmp";
  test_assert( !$b->load_bucket_( 'zeotrope' ) );
  close STDERR;
  open FILE, "<temp.tmp";
! $line = <FILE>;
  test_assert_regexp( $line, 'Incompatible corpus version in zeotrope' );
  close FILE;
--- 290,297 ----
  
  open STDERR, ">temp.tmp";
  test_assert( !$b->load_bucket_( 'zeotrope' ) );
  close STDERR;
  open FILE, "<temp.tmp";
! my $line = <FILE>;
  test_assert_regexp( $line, 'Incompatible corpus version in zeotrope' );
  close FILE;
***************
*** 404,408 ****
  
  $b->clear_bucket( 'zeotrope' );
! test_assert( !( -e 'corpus/zeotrope/table' ) );
  test_assert_equal( $b->get_bucket_word_count('zeotrope'), 0 );
  
--- 379,383 ----
  
  $b->clear_bucket( 'zeotrope' );
! test_assert( -e 'corpus/zeotrope/table.db' );
  test_assert_equal( $b->get_bucket_word_count('zeotrope'), 0 );
  
***************
*** 425,429 ****
  test_assert( !$b->delete_bucket( 'zebrazerba' ) );
  test_assert( $b->delete_bucket( 'zeotrope' ) );
! test_assert( !( -e 'corpus/zeotrope/table' ) );
  test_assert( !( -e 'corpus/zeotrope' ) );
  
--- 400,404 ----
  test_assert( !$b->delete_bucket( 'zebrazerba' ) );
  test_assert( $b->delete_bucket( 'zeotrope' ) );
! test_assert( !( -e 'corpus/zeotrope/table.db' ) );
  test_assert( !( -e 'corpus/zeotrope' ) );
  
***************
*** 438,447 ****
  test_assert_equal( $b->get_value_( 'personal', 'foo' ), log(1/103) );
  test_assert_equal( $b->get_sort_value_( 'personal', 'foo' ), log(1/103) );
- $b->{total__}{personal} = 100;
  $b->set_value_( 'personal', 'foo', 100 );
  test_assert_equal( $b->get_value_( 'personal', 'foo' ), 0 );
  test_assert_equal( $b->get_sort_value_( 'personal', 'foo' ), $b->{not_likely__} );
- $b->{total__}{personal} = 1000;
  $b->set_value_( 'personal', 'foo', 100 );
  test_assert_equal( $b->get_value_( 'personal', 'foo' ), -log(10) );
  test_assert_equal( $b->get_sort_value_( 'personal', 'foo' ), -log(10) );
--- 413,423 ----
  test_assert_equal( $b->get_value_( 'personal', 'foo' ), log(1/103) );
  test_assert_equal( $b->get_sort_value_( 'personal', 'foo' ), log(1/103) );
  $b->set_value_( 'personal', 'foo', 100 );
+ $b->{matrix__}{personal}{__POPFILE__TOTAL__} = 100;
  test_assert_equal( $b->get_value_( 'personal', 'foo' ), 0 );
  test_assert_equal( $b->get_sort_value_( 'personal', 'foo' ), $b->{not_likely__} );
  $b->set_value_( 'personal', 'foo', 100 );
+ $b->{matrix__}{personal}{__POPFILE__TOTAL__} = 1000;
+ test_assert_equal( $b->get_base_value_( 'personal', 'foo' ), 100 );
  test_assert_equal( $b->get_value_( 'personal', 'foo' ), -log(10) );
  test_assert_equal( $b->get_sort_value_( 'personal', 'foo' ), -log(10) );

Index: TestHTML.script
===================================================================
RCS file: /cvsroot/popfile/engine/tests/TestHTML.script,v
retrieving revision 1.17
retrieving revision 1.18
diff -C2 -d -r1.17 -r1.18
*** TestHTML.script	9 Sep 2003 00:28:20 -0000	1.17
--- TestHTML.script	15 Sep 2003 14:55:37 -0000	1.18
***************
*** 650,653 ****
--- 650,663 ----
  INPUTIS   pop3_separator ;
  
+ # Check fork for POP3
+ 
+ INPUTIS   pop3_force_fork 1
+ CONFIGIS  pop3_force_fork 0
+ SETSUBMIT pop3_force_fork 0
+ INPUTIS   pop3_force_fork 1
+ CONFIGIS  pop3_force_fork 0
+ SETSUBMIT pop3_force_fork 1
+ INPUTIS   pop3_force_fork 0
+ 
  # Check Subject Line Modification
  
***************
*** 964,968 ****
  
  CLICK clearbucket
! MATCH <td width="1%">&nbsp;</td><td align="right">0</td><td width="1%">&nbsp;</td>
  
  # Test removal of history files that are out of date (old and new style)
--- 974,978 ----
  
  CLICK clearbucket
! MATCH (0 unique)</td>
  
  # Test removal of history files that are out of date (old and new style)

Index: TestHTML.tst
===================================================================
RCS file: /cvsroot/popfile/engine/tests/TestHTML.tst,v
retrieving revision 1.19
retrieving revision 1.20
diff -C2 -d -r1.19 -r1.20
*** TestHTML.tst	8 Sep 2003 18:03:58 -0000	1.19
--- TestHTML.tst	15 Sep 2003 14:55:37 -0000	1.20
***************
*** 141,144 ****
--- 141,148 ----
  my $b = new Classifier::Bayes;
  
+ $b->configuration( $c );
+ $b->mq( $mq );
+ $b->logger( $l );
+ 
  $c->configuration( $c );
  $c->mq( $mq );
***************
*** 156,181 ****
  $mq->logger( $l );
  
- $b->configuration( $c );
- $b->mq( $mq );
- $b->logger( $l );
- 
- $b->initialize();
- test_assert( $b->start() );
- 
  my $p = new Proxy::POP3;
  
  $p->configuration( $c );
  $p->mq( $mq );
- $p->logger( $l );
  $p->classifier( $b );
  $p->version( 'vtest.suite.ver' );
  $p->initialize();
  
  our $h = new UI::HTML;
  
  $h->configuration( $c );
  $h->mq( $mq );
  $h->logger( $l );
- $h->classifier( $b );
  $h->initialize();
  $h->version( 'vtest.suite.ver' );
--- 160,184 ----
  $mq->logger( $l );
  
  my $p = new Proxy::POP3;
  
  $p->configuration( $c );
  $p->mq( $mq );
  $p->classifier( $b );
+ $p->logger( $l );
  $p->version( 'vtest.suite.ver' );
  $p->initialize();
  
+ $b->initialize();
+ open STDOUT, ">temp.tmp";
+ test_assert( $b->start() );
+ close STDOUT;
+ unlink 'temp.tmp';
+ 
  our $h = new UI::HTML;
  
+ $h->classifier( $b );
  $h->configuration( $c );
  $h->mq( $mq );
  $h->logger( $l );
  $h->initialize();
  $h->version( 'vtest.suite.ver' );
***************
*** 192,202 ****
  test_assert_equal( $h->url_encode_( '[foo]' ), '%5bfoo%5d' );
  
! our $port = 9000 + int(rand(1000));
  pipe my $dreader, my $dwriter;
  pipe my $ureader, my $uwriter;
  my $pid = fork();
  
  if ( $pid == 0 ) {
  
      # CHILD THAT WILL RUN THE HTML INTERFACE
  
--- 195,208 ----
  test_assert_equal( $h->url_encode_( '[foo]' ), '%5bfoo%5d' );
  
! our $port = 9001 + int(rand(1000));
  pipe my $dreader, my $dwriter;
  pipe my $ureader, my $uwriter;
+ $b->prefork();
  my $pid = fork();
  
  if ( $pid == 0 ) {
  
+     $b->forked();
+ 
      # CHILD THAT WILL RUN THE HTML INTERFACE
  
***************
*** 306,309 ****
--- 312,317 ----
      exit(0);
  } else {
+ 
+     $b->postfork();
  
      # PARENT THAT WILL SEND COMMANDS TO THE WEB INTERFACE

Index: TestHTTP.tst
===================================================================
RCS file: /cvsroot/popfile/engine/tests/TestHTTP.tst,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** TestHTTP.tst	8 Sep 2003 18:03:58 -0000	1.3
--- TestHTTP.tst	15 Sep 2003 14:55:37 -0000	1.4
***************
*** 253,256 ****
--- 253,258 ----
      }
  
+     $h->stop();
+ 
      exit(0);
  } else {

Index: TestPOP3.tst
===================================================================
RCS file: /cvsroot/popfile/engine/tests/TestPOP3.tst,v
retrieving revision 1.13
retrieving revision 1.14
diff -C2 -d -r1.13 -r1.14
*** TestPOP3.tst	28 Aug 2003 17:54:55 -0000	1.13
--- TestPOP3.tst	15 Sep 2003 14:55:37 -0000	1.14
***************
*** 294,301 ****
--- 294,304 ----
  mkdir( 'messages' );
  
+ $b->prefork();
  my $pid = fork();
  
  if ( $pid == 0 ) {
  
+     $b->forked();
+ 
      # CHILD THAT WILL RUN THE POP3 SERVER
  
***************
*** 320,323 ****
--- 323,328 ----
  } else {
  
+     $b->postfork();
+ 
      my $port = 9000 + int(rand(1000));
  
***************
*** 340,347 ****
--- 345,355 ----
      pipe my $dreader, my $dwriter;
      pipe my $ureader, my $uwriter;
+     $b->prefork();
      my $pid2 = fork();
  
      if ( $pid2 == 0 ) {
  
+         $b->forked();
+ 
          # CHILD THAT WILL RUN THE POP3 PROXY
  
***************
*** 414,417 ****
--- 422,427 ----
          exit(0);
      } else {
+ 
+         $b->postfork();
  
          # PARENT THAT WILL SEND COMMAND TO THE PROXY