Thread: [Openinteract-commits] CVS: OpenInteract2/lib/OpenInteract2/I18N Initializer.pm,1.11,1.12

SourceForge Headquarters 225 Broadway Suite 1600 San Diego, CA 92101 +1 (858) 422-6466

Update of /cvsroot/openinteract/OpenInteract2/lib/OpenInteract2/I18N
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv4782/lib/OpenInteract2/I18N

Modified Files:
	Initializer.pm 
Log Message:
OIN-143: be able to read localization messages from PO/MO files just like our custom .msg files

Index: Initializer.pm
===================================================================
RCS file: /cvsroot/openinteract/OpenInteract2/lib/OpenInteract2/I18N/Initializer.pm,v
retrieving revision 1.11
retrieving revision 1.12
diff -C2 -d -r1.11 -r1.12
*** Initializer.pm	2 Feb 2005 13:10:42 -0000	1.11
--- Initializer.pm	12 Mar 2005 18:37:18 -0000	1.12
***************
*** 22,26 ****
  }

! # This may be really naive...
  # should match:  en | en-US | en_US

--- 22,26 ----
  }

! # This may be really naive, but it should work for po/mo/msg files
  # should match:  en | en-US | en_US

***************
*** 32,43 ****
          $log->debug( "Pulled language '$lang' from file '$filename'" );
      return $lang;
- 
  }

  sub add_message_files {
      my ( $self, @files ) = @_;
!     return unless ( scalar @files );
      $log ||= get_logger( LOG_INIT );
!     $log->info( "Adding message files: ", join( ', ', @files ) );
      push @{ $self->{_files} }, @files;
      return $self->{_files};
--- 32,56 ----
          $log->debug( "Pulled language '$lang' from file '$filename'" );
      return $lang;
  }

+ 
  sub add_message_files {
      my ( $self, @files ) = @_;
!     return $self->{_files} unless ( scalar @files );
      $log ||= get_logger( LOG_INIT );
! 
!     # ensure all files have an identifiable language
!     foreach my $msg_file ( @files ) {
!         my $lang = $self->is_valid_message_file( $msg_file );
!         unless ( $lang ) {
!             $log->error( "File '$msg_file' does not have identifiable language" );
!             oi_error "Cannot identify language from message file ",
!                      "'$msg_file'. It must end with a language code ",
!                      "before the file extension. For example: ",
!                      "'myapp-en.msg', 'MyReallyBigApp-es-MX.dat'";
!         }
!     }
!     $log->is_debug &&
!         $log->debug( "Adding message files: ", join( ', ', @files ) );
      push @{ $self->{_files} }, @files;
      return $self->{_files};
***************
*** 50,57 ****
      opendir( MSGDIR, $msg_dir )
          || oi_error "Cannot read from global message directory '$msg_dir': $!";
!     my @msg_files = grep /\.msg$/, readdir( MSGDIR );
      closedir( MSGDIR );
      my @full_msg_files = map { catfile( $msg_dir, $_ ) } @msg_files ;
!     $log->info( "Found global message files: ", join( ', ', @full_msg_files ) );
      $self->add_message_files( @full_msg_files );
      return \@full_msg_files;
--- 63,72 ----
      opendir( MSGDIR, $msg_dir )
          || oi_error "Cannot read from global message directory '$msg_dir': $!";
!     my @msg_files = grep /\.(msg|mo|po)$/, readdir( MSGDIR );
      closedir( MSGDIR );
      my @full_msg_files = map { catfile( $msg_dir, $_ ) } @msg_files ;
!     $log->is_debug &&
!         $log->debug( "Found global message files: ",
!                      join( ', ', @full_msg_files ) );
      $self->add_message_files( @full_msg_files );
      return \@full_msg_files;
***************
*** 61,110 ****
      my ( $self ) = @_;
      $log ||= get_logger( LOG_INIT );

!     my %lang_msg = ();
!     my %key_from = ();
! 
!     # Some initialization...

!     $TEMPLATE ||= Template->new();
!     $BASE_CLASS = $self->_get_lang_template();

! MSGFILE:
      foreach my $msg_file ( @{ $self->{_files} } ) {
!         my $lang = $self->is_valid_message_file( $msg_file ); 
!         unless ( $lang ) {
!             oi_error "Cannot identify language from message file ",
!                      "'$msg_file'. It must end with a language code ",
!                      "before the file extension. For example: ",
!                      "'myapp-en.msg', 'MyReallyBigApp-es-MX.dat'";
          }
!         $log->is_info &&
!             $log->info( "Reading messages from file '$msg_file'" );
! 
!         # This may throw an exception, let it bubble up...
!         my $messages = $self->_read_messages( $msg_file );
!  
!         $lang_msg{ $lang } ||= {};
!         foreach my $msg_key ( keys %{ $messages } ) {
!             if ( $lang_msg{ $lang }->{ $msg_key } ) {
!                 $log->error( "DUPLICATE MESSAGE KEY FOUND. Key '$msg_key' ",
!                              "from '$msg_file' was already found in message ",
!                              "file '$key_from{ $msg_key }' read in earlier. ",
!                              "Existing key will not be overwritten which ",
!                              "may cause odd application behavior." );
!             }
!             else {
!                 $lang_msg{ $lang }->{ $msg_key } = $messages->{ $msg_key };
!                 $key_from{ $msg_key } = $msg_file;
!             }
          }
      }

!     # Now all messages are read in, generate the classes

      my @generated_classes = ();
!     foreach my $lang ( keys %lang_msg ) {
          my $generated_class =
!             $self->_generate_language_class( $lang, $lang_msg{ $lang } );
          push @generated_classes, $generated_class;
      }
--- 76,118 ----
      my ( $self ) = @_;
      $log ||= get_logger( LOG_INIT );
+     unless ( ref $self->{_files} eq 'ARRAY' and @{ $self->{_files} } ) {
+         $log->warn( "Asked to generate localization classes but no ",
+                     "localization message files assigned; weird..." );
+         return [];
+     }

!     $self->_check_for_gettext_files();

!     # %all_messages is:
!     #   ->lang->lang_key         = msg
!     #   ->lang->SOURCE->lang_key = file with key
!     # ...see _assign_file_messages() for where it gets filled

!     my %all_messages = ();
      foreach my $msg_file ( @{ $self->{_files} } ) {
!         $log->is_debug &&
!             $log->debug( "Reading messages from file '$msg_file'" );
!         my %file_messages = ();
!         if ( $msg_file =~ /\.msg$/ ) {
!             %file_messages = $self->_read_msg_file( $msg_file );
          }
!         elsif ( $msg_file =~ /\.(mo|po)$/ ) {
!             %file_messages = $self->_read_gettext_file( $msg_file );
          }
+         $self->_assign_file_messages( $msg_file, \%all_messages, \%file_messages );
      }

!     # ...message key sources aren't needed anymore so delete
! 
!     while ( my ( $lang, $lang_msg ) = each %all_messages ) {
!         delete $lang_msg->{SOURCE};
!     }
! 
!     # Now all messages are read in and merged, generate the classes

      my @generated_classes = ();
!     foreach my $lang ( keys %all_messages ) {
          my $generated_class =
!             $self->_generate_language_class( $lang, $all_messages{ $lang } );
          push @generated_classes, $generated_class;
      }
***************
*** 116,135 ****
  # private methods below here

! sub _read_messages {
!     my ( $self, $msg_file ) = @_;
!     $log ||= get_logger( LOG_INIT );

!     $log->is_debug &&
!         $log->debug( "Reading messages from file '$msg_file'" );
!     open( MSG, '<', $msg_file )
!         || oi_error "Cannot read messages from '$msg_file': $!";

      my %messages = ();
      my ( $current_key, $current_msg, $readmore );
      while ( <MSG> ) {
          chomp;

          # Skip comments and blanks unless we're in a readmore block
- 
          next if ( ! $readmore and /^\s*\#/ );
          next if ( ! $readmore and /^\s*$/ );
--- 124,170 ----
  # private methods below here

! sub _check_for_gettext_files {
!     my ( $self ) = @_;
!     my $has_gettext = grep /\.(mo|po)$/, @{ $self->{_files} };
!     if ( $has_gettext ) {
!         eval "require Locale::Maketext::Lexicon::Gettext";
!         if ( $@ ) {
!             oi_error "Locale::Maketext::Lexicon is required to parse mo/po ",
!                      "localization files. Please install it.";
!         }
!     }
! }

! # merge messages from a file into all messages

+ sub _assign_file_messages {
+     my ( $self, $file, $all_messages, $file_messages ) = @_;
+     my $lang = $self->is_valid_message_file( $file );
+     while ( my ( $key, $value ) = each %{ $file_messages } ) {
+         if ( $all_messages->{ $lang }{ $key } ) {
+             my $source = $all_messages->{ $lang }{SOURCE}{ $key };
+             $log->warn(
+                 "DUPLICATE MESSAGE KEY FOUND. Key '$key' from ",
+                 "'$file' was already found in message file ",
+                 "'$source' read in earlier. Existing key WILL NOT BE ",
+                 "OVERWRITTEN, which may cause odd application behavior." );
+         }
+         else {
+             $all_messages->{ $lang }{ $key }         = $value;
+             $all_messages->{ $lang }{SOURCE}{ $key } = $file;
+         }
+     }
+ }
+ 
+ sub _read_msg_file {
+     my ( $self, $msg_file ) = @_;
      my %messages = ();
      my ( $current_key, $current_msg, $readmore );
+     open( MSG, '<', $msg_file )
+         || oi_error "Cannot read messages from '$msg_file': $!";
      while ( <MSG> ) {
          chomp;

          # Skip comments and blanks unless we're in a readmore block
          next if ( ! $readmore and /^\s*\#/ );
          next if ( ! $readmore and /^\s*$/ );
***************
*** 137,145 ****
          my $line = $_;
          my $this_readmore = $line =~ s|\\\s*$||;
-         if ( $readmore ) {
- 
-             # lop off spaces at the beginning of continued lines so
-             # they're more easily distinguished

              $line =~ s/^\s+//;
              $current_msg .= $line;
--- 172,179 ----
          my $line = $_;
          my $this_readmore = $line =~ s|\\\s*$||;

+         # lop off spaces at the beginning of continued lines so
+         # they're more easily distinguished
+         if ( $readmore ) {
              $line =~ s/^\s+//;
              $current_msg .= $line;
***************
*** 161,170 ****
      }
      close( MSG );
!     $log->is_debug &&
!         $log->debug( "Set '$current_key' = '$current_msg'" );
      $messages{ $current_key } = $current_msg;
!     return \%messages;
  }

  sub _generate_language_class {
      my ( $self, $lang, $messages ) = @_;
--- 195,215 ----
      }
      close( MSG );
!     $log->is_debug && $log->debug( "Set '$current_key' = '$current_msg'" );
      $messages{ $current_key } = $current_msg;
!     return %messages;
! }
! 
! 
! sub _read_gettext_file {
!     my ( $self, $gettext_file ) = @_;
!     open( GETTEXT, '<', $gettext_file )
!         || oi_error "Failed to open gettext file: $!";
!     my @content = <GETTEXT>;
!     close( GETTEXT );
!     my $msg = Locale::Maketext::Lexicon::Gettext->parse( @content );
!     return %{ $msg }
  }

+ 
  sub _generate_language_class {
      my ( $self, $lang, $messages ) = @_;
***************
*** 206,209 ****
--- 251,258 ----
                       "'$lang' with base class '$base_class'" );
      my ( $gen_class );
+ 
+     $TEMPLATE   ||= Template->new();
+     $BASE_CLASS ||= $self->_get_lang_template();
+ 
      $TEMPLATE->process( \$BASE_CLASS, \%params, \$gen_class )
          || oi_error "Failed to process maketext subclass template: ",
***************
*** 269,272 ****
--- 318,350 ----
  variable L<%Lexicon> which C<L::M> uses to work its magic.

+ The message files may be in one of three formats:
+ 
+ =over 4
+ 
+ =item *
+ 
+ B<.msg> - Custom key/value pair format with interpolated variables
+ indicated by C<[_1]> , C<[_2]>, etc. as supported by
+ L<Locale::Maketext>. End-of-line continuations ('\') are also
+ supported -- see L<OpenInteract2::Manual::I18N> for formatting
+ details.
+ 
+ =item *
+ 
+ B<.po> - Plaintext format used by gettext. This format is documented
+ elsewhere (see L<SEE ALSO>) and parsed by L<Locale::Maketext::Lexicon>.
+ 
+ =item *
+ 
+ B<.mo> - Compiled gettext files. This format is documented elsewhere
+ (see L<SEE ALSO>) and parsed by L<Locale::Maketext::Lexicon>.
+ 
+ =back
+ 
+ Message files can also be mixed and matched, even within a package. So
+ if you've got one translator who likes to use gettext tools you can
+ include them alongside people who are fine with the OI2 message
+ format.
+ 
  =head1 CLASS METHODS

***************
*** 285,300 ****

    myapp-en.msg
!   myotherapp-en-MX.dat
!   messages_en-HK.msg

  The following are not:

   english-messages.msg
!  messages-en-part2.msg
!  messagesen.msg

  Currently we assume the base language identifier is two characters
  (e.g., 'en', 'jp', 'ru') and the extension (e.g., 'US', 'CA', 'MX')
! can by any number of characters. This may be wildly naive.

  =head1 OBJECT METHODS
--- 363,379 ----

    myapp-en.msg
!   myotherapp-en_MX.po
!   messages_en-HK.mo

  The following are not:

   english-messages.msg
!  messages-en-part2.po
!  messagesen.mo

  Currently we assume the base language identifier is two characters
  (e.g., 'en', 'jp', 'ru') and the extension (e.g., 'US', 'CA', 'MX')
! can by any number of characters. This may be wildly naive and may
! change.

  =head1 OBJECT METHODS
***************
*** 305,308 ****
--- 384,390 ----
  files to process. It does not process these files until C<run()>.

+ If any file in C<@fully_qualified_files> does not have a valid
+ language we throw an exception.
+ 
  B<locate_global_message_files()>

***************
*** 317,324 ****

  Reads messages from all files added via C<add_message_files()> and
! generates language-specific subclasses for all messages found. (Once
  the subclasses are created the system does not know from where the
  messages come since all messages are flattened into a per-language
! data structure.) So the following:

   file: msg-en.msg
--- 399,406 ----

  Reads messages from all files added via C<add_message_files()> and
! generates language-specific subclasses for all messages found. Once
  the subclasses are created the system does not know from where the
  messages come since all messages are flattened into a per-language
! data structure. So the following:

   file: msg-en.msg
***************
*** 328,332 ****
     foo.label.main

!  file: other_msg-en.msg
   keys:
     baz.title
--- 410,414 ----
     foo.label.main

!  file: other_msg-en.mo
   keys:
     baz.title
***************
*** 334,338 ****
     baz.conclusion

!  file: another_msg-en.msg
     bar.title
     bar.intro
--- 416,420 ----
     baz.conclusion

!  file: another_msg-en.po
     bar.title
     bar.intro
***************
*** 362,367 ****
  =item *

! Cannot discern a language from the given filename. See
! C<is_valid_message_file()> for more.

  =item *
--- 444,449 ----
  =item *

! If you have any gettext files (mo/po extension) and do not have
! L<Locale::Maketext::Lexicon>.

  =item *
***************
*** 377,383 ****
  Note that a duplicate key (that is, a key defined in multiple message
  files) will not generate an exception. Instead it will generate a
! logging message with an 'error' level.

! See more about the format used for the message files in
  L<OpenInteract2::Manual::I18N>.

--- 459,465 ----
  Note that a duplicate key (that is, a key defined in multiple message
  files) will not generate an exception. Instead it will generate a
! logging message with an 'warn' level.

! See more about the format used for the custom message files in
  L<OpenInteract2::Manual::I18N>.

***************
*** 386,389 ****
--- 468,477 ----
  =head1 SEE ALSO

+ L<Locale::Maketext>
+ 
+ L<Locale::Maketext::Lexicon>
+ 
+ gettext: L<http://www.gnu.org/software/gettext/>
+ 
  L<OpenInteract2::I18N>

***************
*** 392,397 ****
  L<OpenInteract2::Setup::ReadLocalizedMessages>

- L<Locale::Maketext>
- 
  =head1 COPYRIGHT

--- 480,483 ----
***************
*** 404,405 ****
--- 490,492 ----

  Chris Winters E<lt>ch...@cw...E<gt>
+ 

Thread: [Openinteract-commits] CVS: OpenInteract2/lib/OpenInteract2/I18N Initializer.pm,1.11,1.12

openinteract-commits