From: Chris W. <la...@us...> - 2005-04-02 23:42:31
|
Update of /cvsroot/openinteract/OpenInteract2/lib/OpenInteract2 In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv30168/lib/OpenInteract2 Modified Files: Package.pm SQLInstall.pm Log Message: OIN-138: allow use of '.csv' files when defining installation data (security settings, initial data, etc.); use base_user as example Index: Package.pm =================================================================== RCS file: /cvsroot/openinteract/OpenInteract2/lib/OpenInteract2/Package.pm,v retrieving revision 1.58 retrieving revision 1.59 diff -C2 -d -r1.58 -r1.59 *** Package.pm 25 Mar 2005 16:46:35 -0000 1.58 --- Package.pm 2 Apr 2005 23:42:21 -0000 1.59 *************** *** 559,563 **** } ! my @data_files = grep /^data\/.*\.dat$/, @{ $pkg_files }; push @status, $self->_check_data_files( \@data_files ); --- 559,563 ---- } ! my @data_files = grep /^data\//, @{ $pkg_files }; push @status, $self->_check_data_files( \@data_files ); *************** *** 1165,1178 **** foreach my $data_file ( sort @{ $files } ) { my $s = { action => 'Check data file', filename => $data_file }; ! eval { OpenInteract2::Util->read_file_perl( $data_file ) }; ! if ( $@ ) { ! $s->{is_ok} = 'no'; ! $s->{message} = "Not a valid Perl structure: $@" } ! else { ! $s->{is_ok} = 'yes'; ! $s->{message} = "File is a valid Perl data structure"; } - push @status, $s; } return @status; --- 1165,1199 ---- foreach my $data_file ( sort @{ $files } ) { my $s = { action => 'Check data file', filename => $data_file }; ! my ( $ok, $msg ); ! if ( $data_file =~ /\.dat$/ ) { ! eval { OpenInteract2::Util->read_file_perl( $data_file ) }; ! if ( $@ ) { ! $ok = 'no'; ! $msg = "Not a valid Perl structure: $@"; ! } ! else { ! $ok = 'yes'; ! $msg = "File is a valid Perl data structure"; ! } } ! elsif ( $data_file =~ /\.csv$/ ) { ! require OpenInteract2::SQLInstall; ! my $struct = eval { ! OpenInteract2::SQLInstall->translate_csv_data_file( $data_file ) ! }; ! if ( $@ ) { ! $ok = 'no'; ! $msg = "Not a valid CSV data file: $@"; ! } ! else { ! $ok = 'yes'; ! $msg = "File is a valid CSV data file"; ! } ! ! } ! if ( $ok and $msg ) { ! $s->{is_ok} = $ok; $s->{message} = $msg; ! push @status, $s; } } return @status; Index: SQLInstall.pm =================================================================== RCS file: /cvsroot/openinteract/OpenInteract2/lib/OpenInteract2/SQLInstall.pm,v retrieving revision 1.30 retrieving revision 1.31 diff -C2 -d -r1.30 -r1.31 *** SQLInstall.pm 17 Mar 2005 14:57:58 -0000 1.30 --- SQLInstall.pm 2 Apr 2005 23:42:21 -0000 1.31 *************** *** 356,372 **** my $relative_file = "$DATA_DIR/$data_file"; my $full_file = $pkg->find_file( $relative_file ); ! my $data_text = $pkg->read_file( $relative_file ); ! my ( $data_struct ); ! { ! no strict 'vars'; ! $data_struct = eval $data_text; ! if ( $@ ) { ! $self->_set_state( $full_file, ! undef, ! "Invalid Perl data structure: $@", ! undef ); ! next DATAFILE; ! } ! } $log->is_debug && $log->debug( "Data structure read from '$full_file': ", --- 356,361 ---- my $relative_file = "$DATA_DIR/$data_file"; my $full_file = $pkg->find_file( $relative_file ); ! my $data_struct = $self->_translate_file_to_struct( $full_file ); ! next DATAFILE unless ( defined $data_struct and ref $data_struct eq 'ARRAY' ); $log->is_debug && $log->debug( "Data structure read from '$full_file': ", *************** *** 456,459 **** --- 445,518 ---- } + sub _translate_file_to_struct { + my ( $self, $filename ) = @_; + my ( $data_struct ); + if ( $filename =~ /\.dat$/ ) { + no strict 'vars'; + my $data = OpenInteract2::Util->read_file( $filename ); + $data_struct = eval $data; + if ( $@ ) { + $self->_set_state( $filename, + undef, + "Invalid Perl data structure: $@", + undef ); + $data_struct = undef; + } + } + elsif ( $filename =~ /\.csv$/ ) { + $data_struct = eval { $self->translate_csv_data_file( $filename ) }; + if ( $@ ) { + $self->_set_state( $filename, undef, "$@", undef ); + $data_struct = undef; + } + } + return $data_struct; + } + + sub translate_csv_data_file { + my ( $self, $filename ) = @_; + my $data_struct = []; + my $data = OpenInteract2::Util->read_file( $filename ); + my ( $meta_list, $labels, @records ) = split /[\r\n]+/, $data; + $meta_list =~ s/^\s+//; $meta_list =~ s/\s+$//; + my %meta = (); + foreach my $pair ( split /\s*;\s*/, $meta_list ) { + my ( $key, $value ) = split /\s*=\s*/, $pair, 2; + if ( $key =~ /^transform_(default|now)$/ ) { + $value = [ split( /\s*,\s*/, $value ) ]; + } + $meta{ $key } = $value; + } + my $delimiter = $meta{delimiter}; + if ( $delimiter ) { + $labels =~ s/^\s+//; $labels =~ s/\s+$//; + $delimiter =~ s/\|/\\|/; + my @record_labels = split /\s*$delimiter\s*/, $labels; + my $num_labels = scalar @record_labels; + $meta{field_order} = \@record_labels; + push @{ $data_struct }, \%meta; + + my $count = 1; + foreach my $rec ( @records ) { + $rec =~ s/^\s+//; $rec =~ s/\s+$//; + my @fields = split /\s*$delimiter\s*/, $rec; + my $num_fields = scalar @fields; + if ( $num_labels == $num_fields ) { + push @{ $data_struct }, \@fields; + } + else { + oi_error "Record $count has a different number of fields ", + "($num_fields) than specified in the labels ", + "($num_labels)"; + } + $count++; + } + } + else { + oi_error "You must set the 'delimiter' to split fields/records"; + } + return $data_struct; + } + sub transform_data { *************** *** 1349,1352 **** --- 1408,1413 ---- =head1 DEVELOPERS: IMPORTING DATA + =head2 Import data formats + We need to be able to pass data from one database to another and be very flexible as to how we do it. The various data file formats have *************** *** 1354,1361 **** up some more. ! The data file discussed below is a Perl data structure. This does against the general OI2 bias against using data structures for humans to edit, but since this is generally a write-once operation it is not ! as important that it be human-readable. To begin, there are two elements to a data file. The first element --- 1415,1448 ---- up some more. ! The data file discussed below is in one of two formats, either a perl ! data structure or a text file with delimited data. (The former goes against the general OI2 bias against using data structures for humans to edit, but since this is generally a write-once operation it is not ! as important that it be human-readable.) ! ! Both files are translated into the same data structure when they're ! read in so later parts of the process don't know the difference. ! ! Here's an example of a perl data structure: ! ! $var = [ { import_type => 'object', ! spops_class => 'OpenInteract2::Group', ! field_order => [ qw/ group_id name / ] }, ! [ 1, 'admin' ], ! [ 2, 'public' ], ! [ 3, 'site admin' ], ! ]; ! ! And here's an example of the same file in delimited format: ! ! import_type = object; spops_class = OpenInteract2::Group; delimiter = | ! group_id | name ! 1 | admin ! 2 | public ! 3 | site admin ! ! The first line has metadata about the data to import, the second has ! the delimited field labels, and every line thereafter is a delimited ! set of record data. To begin, there are two elements to a data file. The first element *************** *** 1385,1389 **** deleting data as well as inserting. ! =head2 Object Processing The first item in the list describes the class you want to use to --- 1472,1476 ---- deleting data as well as inserting. ! =head2 Object Processing: What's in the metadata? The first item in the list describes the class you want to use to *************** *** 1399,1402 **** --- 1486,1497 ---- ]; + And the same thing in delimited format: + + import_type = object; spops_class = OpenInteract2::Group; delimiter = | + group_id | name + 1 | admin + 2 | public + 3 | site admin + Here is a slightly abbreviated form of what steps would look like if they were done in code: *************** *** 1432,1435 **** --- 1527,1542 ---- ]; + In delimited format: + + import_type = object; spops_class = OpenInteract2::Security; transform_default => scope_id; delimiter = | + class | object_id | scope | scope_id | security_level + OpenInteract2::Group | 1 | w | world | 1 + OpenInteract2::Group | 2 | w | world | 4 + OpenInteract2::Group | 2 | g | site_admin_group | 8 + OpenInteract2::Group | 3 | w | world | 4 + OpenInteract2::Group | 3 | g | site_admin_group | 8 + OpenInteract2::Action::Group | 0 | w | world | 4 + OpenInteract2::Action::Group | 0 | g | site_admin_group | 8 + So these steps would look like: *************** *** 1471,1474 **** --- 1578,1587 ---- ]; + And in delimited format: + + import_type = dbdata; datasource_pointer = group; sql_table = sys_group_user; delimiter = | + group_id | user_id + 1 | 1 + So we specify the import type ('dbdata', which corresponds to L<SPOPS::Import::DBI::Data>), the table to operate on *************** *** 1526,1529 **** --- 1639,1646 ---- ]; + There's currently no way to map a nested data structure like + 'field_type' into delimited format, so you must use the serialized + perl data structure.. + Additionally you can create Perl code to do this for you. *************** *** 1546,1549 **** --- 1663,1669 ---- } ]; + It doesn't make any sense to represent this as a delimited file so you + must use the serialized perl data structure. + (See L<SQL Processing: Inserting Data> for how to declare a datasource using 'datasource_pointer'.) *************** *** 1599,1602 **** --- 1719,1725 ---- } ]; + It doesn't make any sense to represent this as a delimited file so you + must use the serialized perl data structure. + (See L<SQL Processing: Inserting Data> for how to declare a datasource using 'datasource_pointer'.) |