[Webfetch-commits] SF.net SVN: webfetch:[61] branches/v0.13/lib/WebFetch/Output/TWiki.pm

[<ik...@us...>]

Revision: 61
          http://webfetch.svn.sourceforge.net/webfetch/?rev=61&view=rev
Author:   ikluft
Date:     2009-09-14 23:19:20 +0000 (Mon, 14 Sep 2009)

Log Message:
-----------
first cut at multiple records per topic

Modified Paths:
--------------
    branches/v0.13/lib/WebFetch/Output/TWiki.pm

Modified: branches/v0.13/lib/WebFetch/Output/TWiki.pm
===================================================================
--- branches/v0.13/lib/WebFetch/Output/TWiki.pm	2009-09-14 23:17:37 UTC (rev 60)
+++ branches/v0.13/lib/WebFetch/Output/TWiki.pm	2009-09-14 23:19:20 UTC (rev 61)
@@ -50,10 +50,10 @@
 		description => "WebFetch::Output::TWiki returned errors from "
 			."saving one or more entries",
 	},
-	"WebFetch::Output::TWiki::Exception::NoIdentifier" => {
+	"WebFetch::Output::TWiki::Exception::FieldNotSpecified" => {
 		isa => "WebFetch::Exception",
-		alias => "throw_no_id_field",
-		description => "no identifier field could be found",
+		alias => "throw_field_not_specified",
+		description => "a required field was not defined or found",
 	},
 );
 
@@ -172,27 +172,18 @@
 	foreach $line ( split /\r*\n+/s, $config ) {
 		if ( $line =~ /^\|\s*(.*)\s*\|\s*$/ ) {
 			my @entries = split /\s*\|\s*/, $1;
+			WebFetch::debug "read entries: ".join( ', ', @entries );
 
-			# check first line for field headings
-			if ( ! @{$self->{twiki_config_all}}) {
-				if ( $entries[0] =~ /\*\w+\*/ ) {
-					# save table headings as field names
-					my $field;
-					foreach $field ( @entries ) {
-						if ( $field =~ /\*(\w*)\*/ )
-						{
-							push @fnames, lc($1);
-						} else {
-							my $tmp = lc($field);
-							$tmp =~ s/\W//g;
-							push @fnames, $tmp;
-						}
-					}
-					next;
-				} else {
-					# use default field names
-					@fnames = @default_field_names;
+			# first line contains field headings
+			if ( ! @fnames) {
+				# save table headings as field names
+				my $field;
+				foreach $field ( @entries ) {
+					my $tmp = lc($field);
+					$tmp =~ s/\W//g;
+					push @fnames, $tmp;
 				}
+				next;
 			}
 			WebFetch::debug "field names: ".join " ", @fnames;
 
@@ -239,29 +230,67 @@
 
 	# get config variables
 	$config = $self->{twiki_config};
-	foreach $name ( qw( key web parent prefix template form )) {
-		if ( !exists $self->{twiki_config}{$name}) {
-			throw_twiki_config_missing( "missing config parameter "
-				.$name );
+
+	# parse options
+	my ( $option );
+	$self->{twiki_options} = {};
+	foreach $option ( split /\s+/, $self->{twiki_config}{options}) {
+		if ( $option =~ /^([^=]+)=(.*)/ ) {
+			$self->{twiki_options}{$1} = $2;
+		} else {
+			$self->{twiki_options}{$option} = 1;
 		}
 	}
 
-	# get text of template topic
-	my $template = TWiki::Func::readTopic( $config->{web},
-		$config->{template});
-	
 	# determine unique identifier field
-	my $id_field = $self->wk2fnum( "id" );
+	my $id_field;
+	if ( exists $self->{twiki_options}{id_field}) {
+		$id_field = $self->{twiki_options}{id_field};
+	}
 	if ( ! defined $id_field ) {
-		$id_field = $self->wk2fnum( "url" );
+		$id_field = $self->wk2fname( "id" );
 	}
 	if ( ! defined $id_field ) {
-		$id_field = $self->wk2fnum( "title" );
+		$id_field = $self->wk2fname( "url" );
 	}
 	if ( ! defined $id_field ) {
-		throw_no_id_field "no usable identifier field was found";
+		$id_field = $self->wk2fname( "title" );
 	}
-	
+	if ( ! defined $id_field ) {
+		throw_field_not_specified "identifier field not specified";
+	}
+	$self->{id_field} = $id_field;
+
+	# determine from options whether each item is making metadata or topics
+	if ( exists $self->{twiki_options}{separate_topics}) {
+		$self->write_to_twiki_topics;
+	} else {
+		$self->write_to_twiki_metadata;
+	}
+}
+
+=head2 write_to_twiki_topics
+
+=cut
+
+sub write_to_twiki_topics
+{
+	my $self = shift;
+
+	# get config variables
+	my $config = $self->{twiki_config};
+	my $name;
+	foreach $name ( qw( key web parent prefix template form )) {
+		if ( !exists $self->{twiki_config}{$name}) {
+			throw_twiki_config_missing( "missing config parameter "
+				.$name );
+		}
+	}
+
+	# get text of template topic
+	my ($meta, $template ) = TWiki::Func::readTopic( $config->{web},
+		$config->{template});
+
 	# open DB file for tracking unique IDs of articles already processed
 	my %id_index;
 	tie %id_index, 'DB_File',
@@ -280,14 +309,15 @@
 	# create topics with metadata from each WebFetch data record
 	my $entry;
 	my @oopses;
+	my $id_field = $self->{id_field};
 	$self->data->reset_pos;
 	while ( $entry = $self->data->next_record ) {
 
 		# check that this entry hasn't already been forwarded to TWiki
-		if ( exists $id_index{$entry->bynum( $id_field )}) {
+		if ( exists $id_index{$entry->byname( $id_field )}) {
 			next;
 		}
-		$id_index{$entry->bynum( $id_field )} = time;
+		$id_index{$entry->byname( $id_field )} = time;
 
 		# select topic name
 		my $topicname = sprintf $tnum_format, $tnum_counter;
@@ -298,11 +328,10 @@
 		$tnum_counter++;
 		$topics{$topicname} = 1;
 		my $text = $template;
-		WebFetch::debug "write_to_twiki: writing $topicname";
+		WebFetch::debug "write_to_twiki_topics: writing $topicname";
 
 		# create topic metadata
-		my $meta = TWiki::Meta->new ( $self->{twiki_obj},
-			$config->{web}, $topicname );
+		#my $meta = TWiki::Meta->new ( $self->{twiki_obj}, $config->{web}, $topicname );
 		$meta->put( "TOPICPARENT",
 			{ name => $config->{parent}});
 		$meta->put( "FORM", { name => $config->{form}});
@@ -338,7 +367,8 @@
 		my $oopsurl = TWiki::Func::saveTopic( $config->{web},
 			$topicname, $meta, $text );
 		if ( $oopsurl ) {
-			WebFetch::debug "write_to_twiki: $topicname - $oopsurl";
+			WebFetch::debug "write_to_twiki_topics: "
+				."$topicname - $oopsurl";
 			push @oopses, $entry->title." -> "
 				.$topicname." ".$oopsurl;
 		}
@@ -346,10 +376,105 @@
 
 	# check for errors
 	if ( @oopses ) {
-		throw_twiki_oops( "These saves failed:\n".join "\n", @oopses );
+		throw_twiki_oops( "TWiki saves failed:\n".join "\n", @oopses );
 	}
 }
 
+=head2 write_to_twiki_metadata
+
+=cut
+
+sub write_to_twiki_metadata
+{
+	my $self = shift;
+
+	# get config variables
+	my $config = $self->{twiki_config};
+	my $name;
+	foreach $name ( qw( key web parent )) {
+		if ( !exists $self->{twiki_config}{$name}) {
+			throw_twiki_config_missing( "missing config parameter "
+				.$name );
+		}
+	}
+
+	# determine metadata title field
+	my $title_field;
+	if ( exists $self->{twiki_options}{title_field}) {
+		$title_field = $self->{twiki_options}{title_field};
+	}
+	if ( ! defined $title_field ) {
+		$title_field = $self->wk2fname( "title" );
+	}
+	if ( ! defined $title_field ) {
+		throw_field_not_specified "title field not specified";
+	}
+
+	# determine metadata value field
+	my $value_field;
+	if ( exists $self->{twiki_options}{value_field}) {
+		$value_field = $self->{twiki_options}{value_field};
+	}
+	if ( ! defined $value_field ) {
+		$value_field = $self->wk2fname( "summary" );
+	}
+	if ( ! defined $value_field ) {
+		throw_field_not_specified "value field not specified";
+	}
+
+	# open DB file for tracking unique IDs of articles already processed
+	my %id_index;
+	tie %id_index, 'DB_File',
+		$self->{dir}."/".$config->{key}."_id_index.db",
+		&DB_File::O_CREAT|&DB_File::O_RDWR, 0640;
+
+	# get text of topic
+	my ($meta, $text) = TWiki::Func::readTopic( $config->{web},
+		$config->{parent});
+	
+	# start metadata line counter
+	my $mnum_counter = 0;
+	my $mnum_format = "line-%07d";
+
+	# create metadata lines for each entry
+	my $entry;
+	my @oopses;
+	my $id_field = $self->{id_field};
+	$self->data->reset_pos;
+	while ( $entry = $self->data->next_record ) {
+		# check that this entry hasn't already been forwarded to TWiki
+		if ( exists $id_index{$entry->byname( $id_field )}) {
+			next;
+		}
+		$id_index{$entry->byname( $id_field )} = time;
+
+		# select metadata field name
+		my ( $value, $metaname );
+		$value = $meta->get( "FIELD",
+			$metaname = sprintf( $mnum_format, $mnum_counter ));
+		while ( defined $value ) {
+			$value = $meta->get( "FIELD",
+				$metaname = sprintf( $mnum_format,
+					++$mnum_counter ));
+		}
+
+		# write the value
+		$meta->putKeyed( "FIELD", {
+			name => $metaname,
+			title => $entry->byname( $title_field ),
+			value => $entry->byname( $value_field ),
+			});
+	}
+
+	# save the topic
+	my $oopsurl = TWiki::Func::saveTopic( $config->{web},
+		$config->{parent}, $meta, $text );
+	if ( $oopsurl ) {
+		throw_twiki_oops "TWiki saves failed: "
+			.$config->{parent}." ".$oopsurl;
+	}
+}
+
 =head2 fmt_handler_twiki
 
 =cut
@@ -366,7 +491,7 @@
 	# write to TWiki topic
 	$self->write_to_twiki;
 
-	# no savables for WebFetch::save - mark it OK
+	# no savables - mark it OK so WebFetch::save won't call it an error
 	$self->no_savables_ok;
         1;
 }


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.