[Openinteract-commits] CVS: OpenInteract/pkg/full_text/OpenInteract FullText.pm,1.9,1.10

[Chris W. <la...@us...>]

Update of /cvsroot/openinteract/OpenInteract/pkg/full_text/OpenInteract
In directory usw-pr-cvs1:/tmp/cvs-serv5126/OpenInteract

Modified Files:
	FullText.pm 
Log Message:
updated OI/FullText.pm to handle scalar refs and filehandles;
indexable text is passed back as a scalar ref now, and you can
override the fields to index by passing in 'fulltext_field' to the
save() call


Index: FullText.pm
===================================================================
RCS file: /cvsroot/openinteract/OpenInteract/pkg/full_text/OpenInteract/FullText.pm,v
retrieving revision 1.9
retrieving revision 1.10
diff -C2 -d -r1.9 -r1.10
*** FullText.pm	2001/10/01 17:13:51	1.9
--- FullText.pm	2001/10/29 13:07:32	1.10
***************
*** 145,149 ****
  # to the subclass and send it on up the line 
  
! sub ruleset_add {
      my ( $class, $rs_table ) = @_;
      my $obj_class = ref $class || $class;
--- 145,149 ----
  # to the subclass and send it on up the line 
  
! sub ruleset_factory {
      my ( $class, $rs_table ) = @_;
      my $obj_class = ref $class || $class;
***************
*** 163,168 ****
      my $R = OpenInteract::Request->instance;
      $R->DEBUG && $R->scrib( 1, "Trying to index ", ref $self, " (", $self->id, ")" );
!     my $indexable = $self->_indexable_object_text;
!     $R->DEBUG && $R->scrib( 2, "Indexable text: ", $indexable );
      my $wc = $self->_tokenize( $indexable );
      $R->DEBUG && $R->scrib( 2, "Found the following tokens:", Dumper( $wc ) );
--- 163,168 ----
      my $R = OpenInteract::Request->instance;
      $R->DEBUG && $R->scrib( 1, "Trying to index ", ref $self, " (", $self->id, ")" );
!     my $indexable = $self->_indexable_object_text( $p );
!     $R->DEBUG && $R->scrib( 2, "Indexable text: ", $$indexable );
      my $wc = $self->_tokenize( $indexable );
      $R->DEBUG && $R->scrib( 2, "Found the following tokens:", Dumper( $wc ) );
***************
*** 201,215 ****
  # Get the fields that should be indexed and join the values together
  # with a space (easy), since we're just going to index all the text as
! # one big field
  
  sub _indexable_object_text {
!     my ( $self ) = @_;
      my $R = OpenInteract::Request->instance;
!     my $field_list = $self->CONFIG->{fulltext_field};
      unless ( ref $field_list eq 'ARRAY' ) {
          $R->scrib( 0, "Cannot index object text -- no fields presented in config file." );
          return undef;
      }
!     return join ' ', map { $self->{$_} } @{ $field_list };
  }
  
--- 201,231 ----
  # Get the fields that should be indexed and join the values together
  # with a space (easy), since we're just going to index all the text as
! # one big field. Returns a scalar REF.
  
  sub _indexable_object_text {
!     my ( $self, $p ) = @_;
!     $p ||= {};
      my $R = OpenInteract::Request->instance;
! 
!     my $field_list = $p->{fulltext_field} || $self->CONFIG->{fulltext_field};
      unless ( ref $field_list eq 'ARRAY' ) {
          $R->scrib( 0, "Cannot index object text -- no fields presented in config file." );
          return undef;
      }
!     my ( $indexable );
!     foreach my $field ( @{ $field_list } ) {
!         if ( ! ref $self->{ $field } ) {
!             $indexable = join( ' ', $indexable, $self->{ $field } );
!         }
!         elsif ( ref $self->{ $field } eq 'SCALAR' ) {
!             $indexable = join( ' ', $indexable, $$self->{ $field } );
!         }
! 
!         else {
!             my $fh = $self->{ $field };
!             $indexable = join( ' ', $indexable, <$fh> );
!         }
!     }
!     return \$indexable;
  }
  
***************
*** 217,228 ****
  # Break up the text into tokens -- stemmed using Lingua::Stem and
  # counted for occurrences. Remove the words that are too long, too
! # short and those that are found in our STOPWORDS listing.
  
  sub _tokenize {
!     my ( $self, $text ) = @_;
!     $text =~ tr/A-Z/a-z/;  # lowercase
      my %words = ();
!     map { $words{ $_ }++ } map { Lingua::Stem::stem( $_ )->[0] } ( $text =~ /\w+/g );
!     map { delete $words{ $_ } }  
          grep { length $_ < MIN_WORD_LENGTH || length $_ > MAX_WORD_LENGTH }
          keys %words;
--- 233,245 ----
  # Break up the text into tokens -- stemmed using Lingua::Stem and
  # counted for occurrences. Remove the words that are too long, too
! # short and those that are found in our STOPWORDS listing. Takes a
! # scalar REF as an argument.
  
  sub _tokenize {
!     my ( $self, $text_ref ) = @_;
!     $$text_ref =~ tr/A-Z/a-z/;  # lowercase
      my %words = ();
!     map { $words{ $_ }++ } map { Lingua::Stem::stem( $_ )->[0] } ( $$text_ref =~ /\w+/g );
!     map { delete $words{ $_ } }
          grep { length $_ < MIN_WORD_LENGTH || length $_ > MAX_WORD_LENGTH }
          keys %words;
***************
*** 522,529 ****
  (From: http://ollie.dcccd.edu/library/Module2/Books/concepts.htm)
  
! We use the L<Lingua::Stem> module for this, which implements the
! I<Porter algorithm> for stemming, as do most implementations,
! apparently. (This is something that I<OpenInteract::FullText> treats as a
! black box itself :)
  
  Parameters:
--- 539,546 ----
  (From: http://ollie.dcccd.edu/library/Module2/Books/concepts.htm)
  
! We use the L<Lingua::Stem|Lingua::Stem> module for this, which
! implements the I<Porter algorithm> for stemming, as do most
! implementations, apparently. (This is something that this class treats
! as a black box itself :)
  
  Parameters: