From: <pau...@us...> - 2007-05-22 23:14:57
|
Revision: 962 http://svn.sourceforge.net/everydevel/?rev=962&view=rev Author: paul_the_nomad Date: 2007-05-22 16:14:54 -0700 (Tue, 22 May 2007) Log Message: ----------- Export nodeball and further checking methods for nodeball development. Modified Paths: -------------- trunk/ebase/lib/Everything/Storage/Nodeball.pm trunk/ebase/lib/Everything/Storage/Test/Nodeball.pm trunk/ebase/lib/Everything/XML/Node.pm Property Changed: ---------------- trunk/ebase/ Property changes on: trunk/ebase ___________________________________________________________________ Name: svk:merge - 16c2b9cb-492b-4d64-9535-64d4e875048d:/wip/ebase:995 a6810612-c0f9-0310-9d3e-a9e4af8c5745:/ebase/offline:17930 + 16c2b9cb-492b-4d64-9535-64d4e875048d:/wip/ebase:996 a6810612-c0f9-0310-9d3e-a9e4af8c5745:/ebase/offline:17930 Modified: trunk/ebase/lib/Everything/Storage/Nodeball.pm =================================================================== --- trunk/ebase/lib/Everything/Storage/Nodeball.pm 2007-05-22 23:14:25 UTC (rev 961) +++ trunk/ebase/lib/Everything/Storage/Nodeball.pm 2007-05-22 23:14:54 UTC (rev 962) @@ -1,3 +1,10 @@ + +=head1 Everything::Storage::Nodeball + +A module that manages the import and export of nodeballs to/from a nodebase. + +=cut + package Everything::Storage::Nodeball; { @@ -116,6 +123,8 @@ use Carp; use IO::File; +use File::Path (); +use File::Temp (); use Everything::XML qw/readTag xmlfile2node xml2node fixNodes/; use Everything::XML::Node; use Everything::NodeBase; @@ -125,8 +134,10 @@ =head2 C<set_nodeball> -Sets a nodeball file or directory +Sets the nodeball attribute. The argument may be a file or directory. If it is a file, the file is expanded and the nodeball_dir attribute is set to the directory of the expanded nodeball. +If the argument is a directory, the nodeball_dir is set to it. + =cut sub set_nodeball { @@ -145,13 +156,19 @@ return; } +=head2 C<get_nodeball> + +If the file attribute is set returns its value. Otherwise returns the value of nodeball_dir. + +=cut + sub get_nodeball { my ($self) = @_; return $self->get_file || $self->get_nodeball_dir; } -=head2 C<expandNodeball> +=head2 C<expand_nodeball> Take a tar-gziped nodeball and expand it to a dir in /tmp return the directory @@ -467,11 +484,9 @@ =head2 C<install_xml_nodes> -This is a method. - It installs nodes stored as XML in the nodeballs. -Takes two optional arguments. The first is the path to the nodeball directory. The second is a regular expression of node paths to avoid. +Takes an optional argument of a call back that examines each node. The call back should return true if it's a node we want or false otherwise. Returns undef. @@ -485,14 +500,58 @@ my $iterator = $self->make_node_iterator($select_cb); while ( my $xmlnode = $iterator->() ) { - xml2node( $xmlnode->get_raw_xml ); + $self->install_xml_node( $xmlnode ); } return; } +=head2 C<install_xml_node> +It installs a node stored as XML into the the current nodebase. + +It takes on argument which is the Everything::XML::Node object to be +installed. + + +=cut + +sub install_xml_node { + + my ( $self, $xmlnode ) = @_; + xml2node( $xmlnode->get_raw_xml ); + +} + + +=head2 C<install_nodeball_description> + +It installs a node representing the current nodeball, as XML, into the +the current nodebase. + +Currently, this means reading from the ME file. + + +=cut + +sub install_nodeball_description { + + my ( $self ) = @_; + + my $dir = $self->get_nodeball_dir; + my $mefile = File::Spec->catfile( $dir, 'ME' ); + my $fh = IO::File->new ( $mefile ); + local $/; + my $xml = <$fh>; + $fh->close; + my $xmlnode = Everything::XML::Node->new; + $xmlnode->parse_xml( $xml ); + $self->install_xml_node( $xmlnode ); + +} + + =head2 C<install_xml_nodetype_nodes> This is a method. @@ -599,10 +658,40 @@ #nodeballs are not installed... but we don't } -=cut +sub export_nodeball_to_directory { + my ( $self, $nodeball_name, $dir ) = @_; + my $nodeball = $self->get_nodebase->getNode( $nodeball_name, 'nodeball'); + croak "No nodeball, $nodeball_name" unless $nodeball; + + ###setup directory for export + $self->set_nodeball_dir( $dir || $self->get_temp_dir ); + $self->write_node_to_nodeball( $nodeball, 'ME' ); # create ME file + my $group = $nodeball->selectNodegroupFlat; + foreach ( @$group ) { + + if ( $$_{type}{title} eq 'dbtable' ) { + $self->write_sql_table_to_nodeball( $$_{title} ); + } + + $self->write_node_to_nodeball( $_ ); + + } + +} + +sub export_nodeball_to_file { + + my ( $self, $nodeball_name, $filename ) = @_; + + $self->export_nodeball_to_directory( $nodeball_name ); + $self->create_nodeball_file( $nodeball_name, undef, $filename ); + + +} + =head2 C<installModules> Copy any perl modules that exist in this nodeball to the appropriate @@ -893,6 +982,123 @@ return \%tables; } +=head2 C<write_node_to_nodeball> + + Writes a node to a nodeball turning it into XML in the process. Takes one argument which should be the Everything::Node object to be written to the nodeball. Takes an optional second argument which is the path (under the nodeball directory) to which the node should be written. + +=cut + +sub write_node_to_nodeball { + my ( $self, $node, $filepath ) = @_; + + + my $volume; + my $save_title; + my $save_dir; + if ( ! $filepath ) { + $save_title = $$node{title}; + $save_dir = $$node{type}{title}; + $save_dir =~ tr/ /_/; + $save_dir = File::Spec->catfile ('nodes', $save_dir); + $save_title =~ tr/ /_/; + $save_title =~ s/:+/-/; + $save_title .= '.xml'; + $filepath = File::Spec->catfile( $save_dir, $save_title ); + } else { + ( $volume, $save_dir, $save_title ) = File::Spec->splitpath( $filepath ); + } + + my $save_path = File::Spec->catfile( $self->get_nodeball_dir, $save_dir ); + File::Path::mkpath( $save_path ) unless -d $save_path; + $save_path = File::Spec->catfile( $save_path, $save_title ); + my $xml = Everything::XML::Node->new( nodebase => $self->get_nodebase, node => $node )->toXML; + my $fh = IO::File->new( $save_path, 'w' ) || croak "Can't open $save_path for writing, $!"; + print $fh $xml; + $fh->close; + +} + +=head2 C<write_sql_table_to_nodeball> + + Writes a sql create statement to a nodeball. Takes one argument which is the table name. + +=cut + +sub write_sql_table_to_nodeball { + my ( $self, $table_name ) = @_; + + my $nb = $self->get_nodebase; + + $nb->{storage} =~ /DB::(\w+)/; + my $storage_type = $1; + my $dir = $self->get_nodeball_dir; + $dir = File::Spec->catfile( $dir, 'tables' ); + mkdir $dir unless -d $dir; + if ( $storage_type eq 'Pg' ) { + $dir = File::Spec->catfile( $dir, 'Pg' ); + } + elsif ( $storage_type eq 'sqlite' ) { + $dir = File::Spec->catfile( $dir, 'SQLite' ); + } + elsif ( $storage_type eq 'mysql' ) { + $dir = File::Spec->catfile( $dir, 'mysql' ); + } + mkdir $dir unless -d $dir; + + my $sql = $nb->{storage}->get_create_table($table_name); + my $file_name = File::Spec->catfile( $dir, "$table_name.sql" ); + + my $fh = IO::File->new( $file_name, 'w' ) + or croak "Can't open $file_name for writing, $!"; + print $fh $sql; + $fh->close; + +} + + +=head2 C<create_nodeball_file> + +Tar-gzips a directory it -- as a nodeball + +=over 4 + +=item * NODEBALL + +The nodeball object we are exporting + +=item * dir + +directory of stuff (optional if nodeball_dir is set). + +=back + +=cut + +sub create_nodeball_file +{ + my ( $self, $NODEBALL, $dir, $filename ) = @_; + + $dir ||= $self->get_nodeball_dir; + my $nodeball = $self->get_nodebase->getNode( $NODEBALL, 'nodeball' ); + my $VARS = $nodeball->getVars(); + my $version = $$VARS{version}; + + if ( ! $filename) { + + $filename = $$NODEBALL{title}; + $filename =~ tr/ /_/; + $filename .= "-$version" if $version; + $filename .= ".nbz"; + } + + use Cwd; + my $cwd = getcwd(); + $cwd .= '/' . $filename; + + `tar -cvzf $cwd -C $dir .`; + +} + =head2 C<buildNodeballMembers> Builds a hash of node_id-E<gt>nodeball that it belongs to. The nodeball(s) @@ -930,6 +1136,65 @@ return \%nbmembers; } +=head2 C<check_nodeball_integrity> + +Checks the internal structure of a nodeball. Return undef if everything is OK. + +Otherwise, it returns a list of two array refs of hash refs. The hash refs have two keys 'title' and 'type'. The first hashref lists the nodes present in the nodeball but not listed in the ME file. The second lists those listed in the ME file, but not present in the nodeball. + +=cut + +sub check_nodeball_integrity { + my $self = shift; + local $/; + my $fh = IO::File->new( File::Spec->catfile ( $self->get_nodeball_dir, 'ME' ) ); + my $xml = <$fh>; + my $me = Everything::XML::Node->new; + $me->parse_xml( $xml ); + my @members = @{ $me->get_group_members || [] }; + + my $iterator = $self->make_node_iterator; + my @nodes; + while ( my $xmlnode = $iterator->() ) { + push @nodes, $xmlnode; + } + + my ( @not_in_me, @not_in_nodeball ); + + foreach my $member (@members) { + my ( $member_type ) = split /,/, $member->get_type_nodetype; + my $found_xmlnode = 0; + XMLNODE: + foreach my $xmlnode ( @nodes ) { + + if ( ($xmlnode->get_title eq $member->get_name) && ( $xmlnode->get_nodetype eq $member_type)) { + $found_xmlnode++; + last XMLNODE + } + } + push @not_in_nodeball, { title => $member->get_name, type=> $member_type } unless $found_xmlnode; + } + + + foreach my $xmlnode (@nodes) { + my $found_xmlnode = 0; + GROUPMEMBER: + foreach my $member ( @members ) { + my ( $member_type ) = split /,/, $member->get_type_nodetype; + if ( ($xmlnode->get_title eq $member->get_name) && ( $xmlnode->get_nodetype eq $member_type ) ) { + $found_xmlnode++; + last GROUPMEMBER + } + + } + push @not_in_me, { title => $xmlnode->get_title, type=> $xmlnode->get_nodetype } unless $found_xmlnode; + + } + + return if ( ! @not_in_nodeball && ! @not_in_me ); + return \@not_in_me, \@not_in_nodeball; +} + package Everything::Storage::Nodeball::SQLParser; use base 'SQL::Parser'; Modified: trunk/ebase/lib/Everything/Storage/Test/Nodeball.pm =================================================================== --- trunk/ebase/lib/Everything/Storage/Test/Nodeball.pm 2007-05-22 23:14:25 UTC (rev 961) +++ trunk/ebase/lib/Everything/Storage/Test/Nodeball.pm 2007-05-22 23:14:54 UTC (rev 962) @@ -101,7 +101,8 @@ my $mock = Test::MockObject->new; my $fh = File::Temp->new( UNLINK => 1 ); - $mock->fake_module( 'IO::File', 'new', sub { $fh } ); + local *IO::File::new; + *IO::File::new = sub { $fh }; $mock->set_always( readline => 'some xml' ); my @readTag_r = qw/version author description title/; my @readTag_a; @@ -125,7 +126,8 @@ # test exception throwing. my $dir = "/some/path"; - $mock->fake_module( 'IO::File', 'new', sub { } ); + local *IO::File::new; + *IO::File::new = sub { }; throws_ok { $instance->nodeball_vars($dir) } 'Everything::Exception::CorruptNodeball', "...if can't open ME file throws an error"; @@ -468,9 +470,28 @@ ok( !-e $tempdir, '..temp directory shouldn\'t exist.' ); } -sub test_update_nodeball : Test(6) { +sub test_install_xml_node : Test(1) { my $self = shift; + my $instance = $self->{instance}; + my $mock = Test::MockObject->new; + $mock->set_always( get_raw_xml => 'some xml' ); + + my @xml2node_args = (); + no strict 'refs'; + local *{ $self->{class} . '::xml2node' }; + *{ $self->{class} . '::xml2node' } = sub { push @xml2node_args, $_[0] }; + use strict 'refs'; + + $instance->install_xml_node( $mock ); + is_deeply( \@xml2node_args, [ 'some xml' ], '...calls xml2node with with xml.'); + + +} + +sub test_update_nodebase_from_nodeball : Test(6) { + my $self = shift; + can_ok( $self->{class}, 'update_nodeball' ) || return 'update_nodeball not implemented.'; my $instance = $self->{instance}; @@ -532,8 +553,7 @@ ); $rv = $test_code->( [qw/mail node/], $dir, $mock ); - use Data::Dumper; - print Dumper $rv; + is_deeply( $rv, undef, '...returns undef if tables are the same.' ); rmtree $dir; @@ -577,6 +597,13 @@ local $TODO = "Methods to export a nodeball stored in a nodebase."; can_ok( $self->{class}, 'export_nodeball' ); + + my @toXMLReturns = ('me file contents', 'data'); + + local *Everything::XML::Node; + *Everything::XML::Node::toXML = sub { shift @toXMLReturns }; + + ### calls update_nodeball_from_nodebase; ok( undef, '.... read nodeball data.' ); ok( undef, '....create ME file and put nodeball data into it.' ); @@ -642,6 +669,255 @@ } +sub test_write_sql_table_to_nodeball : Test(3) { + + my $self = shift; + + return unless can_ok( $self->{class}, 'write_sql_table_to_nodeball' ); + + my $instance = $self->{instance}; + my $mock = $self->{mock}; + + use Everything::DB::sqlite; + $mock->{storage} = Everything::DB::sqlite->new; + + my @get_create_table_args; + local *Everything::DB::sqlite::get_create_table; + *Everything::DB::sqlite::get_create_table = + sub { push @get_create_table_args, $_[1]; return 'create statement' }; + + my $tempdir = get_temp_dir(); + mkdir $tempdir; + $instance->set_nodebase($mock); + $instance->set_nodeball_dir($tempdir); + + my $rv = $instance->write_sql_table_to_nodeball('atable'); + + is( $get_create_table_args[0], + "atable", '...asks for table passed as argument.' ); + my $file = + File::Spec->catfile( $instance->get_nodeball_dir, 'tables', 'SQLite', + 'atable.sql' ); + my $fh = IO::File->new($file) || die "Can't open $file, $!"; + local $/; + my $sql = <$fh>; + close $fh; + is( + $sql, + 'create statement', + '...writes the create statement to an appropriately named file' + ); + +} + + +sub test_write_node_to_nodeball :Test(3) { + + my $self = shift; + my $instance = $self->{instance}; + my $mock = $self->{mock}; + + return unless can_ok( $self->{class}, 'write_node_to_nodeball'); + + local *Everything::XML::Node::new; + *Everything::XML::Node::new = sub { $mock }; + $mock->set_always( 'toXML' => 'some xml' ); + + $mock->{ title } = 'a node title'; + $mock->{ type } = { title => 'a node type title' }; + + $instance->set_nodeball_dir( get_temp_dir() ); + + ## a node object is passed as the argument + my $rv = $instance->write_node_to_nodeball( $mock ); + + ( my $title = $$mock{title} ) =~ s/\s/_/g; + my $dir = $$mock{type}{title}; + $dir =~ s/\s/_/g; + $title .= '.xml'; + my $file = File::Spec->catfile( $instance->get_nodeball_dir , 'nodes', $dir, $title ); + my $fh = IO::File->new( $file ) || die "Can't open file, $file, $!"; + local $/; + my $sql = <$fh>; + close $fh; + + is ( $sql, 'some xml', '...writes the XML to the selected file.'); + + ### Now with our own filepath + $rv = $instance->write_node_to_nodeball( $mock, 'filepath' ); + $fh = IO::File->new( File::Spec->catfile( $instance->get_nodeball_dir , 'filepath' )); + + $sql = <$fh>; + close $fh; + is ( $sql, 'some xml', '...writes the XML to the filename of our choosing.'); + +} + +sub test_create_nodeball : Test(2) { + return "Uses untestable backticks"; + my $self = shift; + can_ok( $self->{class}, 'createNodeball' ) + || return 'createNodeball not implemented.'; + my $instance = $self->{instance}; + my $mock = $self->{mock}; + $mock->{title} = 'a nodeball'; + my $test_code = \&{ $self->{class} . '::createNodeball' }; + + my $tmpdir = get_temp_dir(); + no strict 'refs'; + local *{ $self->{class} . '::getcwd' }; + *{ $self->{class} . '::getcwd' } = sub { File::Spec->tmpdir }; + use strict 'refs'; + + $mock->set_always( 'getVars', { a => 1, b => 2 } ); + + mkdir $tmpdir; + my $printed; + my $in = 'y'; + + { + local *STDOUT; # stop the noise; + $test_code->( $tmpdir, $mock ); + } + ok( + -e File::Spec->tmpdir . "/a_nodeball.nbz", + '..nodeball file should be created.' + ); + $self->{nodeball_file} = File::Spec->tmpdir . "/a_nodeball.nbz"; + rmdir $tmpdir; +} + +sub test_update_nodeball_from_nodebase :Test(6) { + local $TODO = 'Unimplemented.'; + my $self = shift; + + #### does almost everything by called write_node_to_nodeball + + can_ok( $self->{class}, 'update_nodeball_from_nodebase' ) || return "Unimplemented"; + + local *Everything::XML::Node; + *Everything::XML::Node::toXML = sub { 'some xml' }; + + ok( undef, '...create new ME file.'); + + ok( undef, '...replace new ME file with old one.'); + + ok( undef, '...remove dbtables not in new ME file.'); + + ok( undef, '...remove nodes not in new ME file.'); + + ok( undef, '...run through nodeball members with modified dates greater than createtime (of nodeball) and save them.'); + +} + +sub test_check_nodeball_against_nodebase :Test(1) { + local $TODO = "Unimplemented"; + + ok( undef, '...runs through each node in the nodeball and checks for type, attributes and values against the node stored in the nodebase.'); + + +} + +sub test_check_nodebase_against_nodeball :Test(1) { + local $TODO = "Unimplemented"; + + ok( undef, '...runs through each node in the nodebase and checks for type, attributes and values against the xmlnode stored in the nodeball.'); + + +} + +sub test_check_nodeball_integrity :Test(4) { + + my $self = shift; + my $instance = $self->{instance}; + + my $dir = get_temp_dir(); + mkdir $dir; + + $instance->set_nodeball_dir( $dir ); + + my $mefile = File::Spec->catfile ($dir, 'ME'); + my $fh = IO::File->new; + $fh->open( $mefile, 'w' ) || die "Can't open $mefile, $!"; + print $fh <<HERE; +<NODE export_version="0.5" nodetype="nodeball" title="core system"> + <group> + <member name="group_node" type="noderef" type_nodetype="theme,nodetype">default theme</member> + <member name="group_node" type="noderef" type_nodetype="superdoc,nodetype">Create a new user</member> + <member name="group_node" type="noderef" type_nodetype="superdoc,nodetype">Duplicates Found</member> +</group> +</NODE> +HERE + $fh->close; + my $one = Everything::XML::Node->new; + $one->set_title('Create a new user'); + $one->set_nodetype( 'htmlcode' ); + + my $two = Everything::XML::Node->new; + $two->set_title('thingo'); + $two->set_nodetype( 'thingotype' ); + + my $three = Everything::XML::Node->new; + $three->set_title('default theme'); + $three->set_nodetype( 'theme' ); + + my @xmlnodes = ( $one, $two, $three ); + no strict 'refs'; + local *{ $self->{class} . '::make_node_iterator' }; + *{ $self->{class} . '::make_node_iterator' } = sub { sub { shift @xmlnodes }}; + use strict 'refs'; + + my ($not_in_ME, $not_in_nodeball) = $instance->check_nodeball_integrity; + use Data::Dumper; diag Dumper $not_in_ME, $not_in_nodeball; + my @sorted = sort { $a->{title} cmp $b->{title} } @$not_in_ME; + is($sorted[0]->{title}, 'Create a new user', '...not in ME when titles same but types are different.'); + is($sorted[1]->{title}, 'thingo', '...not in ME when title not presnet.'); + + @sorted = sort { $a->{title} cmp $b->{title} } @$not_in_nodeball; + is($sorted[0]->{title}, 'Create a new user', '...not in nodeball when titles same but types are different.'); + is($sorted[1]->{title}, 'Duplicates Found', '...not in nodeball when title not presnet.'); + + +} + +sub test_check_nodeball_presence :Test(4) { + local $TODO = "Unimplemented."; + + my $self = shift; + my $instance = $self->{instance}; + return "unimplemented"; + my $dir = get_temp_dir(); + mkdir $dir; + my $mock = Test::MockObject->new; + $instance->set_nodebase( $mock ); + $mock->set_always( selectNodegroupFlat => [ { title => "Duplicate Found", type => { title => 'superdoc' }}, { title => "Create a new user", type => { title => 'htmlcode' }}, { title => "thingo", type => { title => 'thingtype' }} ] ); + + $instance->set_nodeball_dir( $dir ); + + my $mefile = File::Spec->catfile ($dir, 'ME'); + my $fh = IO::File->new( $mefile, 'w' ) || die "Can't open $mefile, $!"; + print $fh <<HERE; +<NODE export_version="0.5" nodetype="nodeball" title="core system"> + <group> + <member name="group_node" type="noderef" type_nodetype="theme,nodetype">default theme</member> + <member name="group_node" type="noderef" type_nodetype="superdoc,nodetype">Create a new user</member> + <member name="group_node" type="noderef" type_nodetype="superdoc,nodetype">Duplicates Found</member> +</group> +</NODE> +HERE + + my ($not_in_nodebase, $not_in_nodeball) = $instance->check_nodeball_integrity; + my @sorted = sort { $a->get_title cmp $b->get_title } @{ $not_in_nodebase || [] }; + is($sorted[0]->get_title, 'Create a new user', '...not in nodebase when titles same but types are different.'); + is($sorted[1]->get_title, 'default theme', '...not in nodebase when title not presnet.'); + + @sorted = sort { $a->{title} cmp $b->{title} } @{ $not_in_nodeball || [] }; + is($sorted[0]->get_title, 'Create a new user', '...not in nodeball when titles same but types are different.'); + is($sorted[1]->get_title, 'default theme', '...not in nodeball when title not presnet.'); + + +} + sub parse_sql_file_returns { ( Modified: trunk/ebase/lib/Everything/XML/Node.pm =================================================================== --- trunk/ebase/lib/Everything/XML/Node.pm 2007-05-22 23:14:25 UTC (rev 961) +++ trunk/ebase/lib/Everything/XML/Node.pm 2007-05-22 23:14:54 UTC (rev 962) @@ -267,7 +267,7 @@ unless ( $REF->isOfType( $type, 1 ) ) { - Everything::logErrors( "Field '$fieldname' needs a node of type " + Everything::logErrors( "$doc Field '$fieldname' needs a node of type " . "'$type',\nbut it is pointing to a node of type " . "'$REF->{type}{title}'!" ); } This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |