[SimBot-commits] CVS: simbot/tools create_wx_station_db.pl,NONE,1.1
Status: Abandoned
Brought to you by:
kstange
|
From: Pete P. <fou...@us...> - 2005-05-07 02:36:14
|
Update of /cvsroot/simbot/simbot/tools In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv12433/tools Added Files: create_wx_station_db.pl Log Message: create_wx_station_db.pl creates data/weather, which contains station names and locations weather.pl now uses this file instead of caches/weather --- NEW FILE: create_wx_station_db.pl --- #!/usr/bin/perl # SimBot Weather Station Names Importer # # DESCRIPTION: # Imports the list of METAR stations from NOAA's web site. # # USAGE: # While inside SimBot's directory, run tools/create_wx_station_db.pl # # COPYRIGHT: # Copyright (C) 2005, Pete Pearson # # This program is free software; you can redistribute and/or modify it # under the terms of version 2 of the GNU General Public License as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA use Compress::Zlib; use LWP::UserAgent; use DBI; use XML::Simple; use warnings; use strict; # turn off IO buffering on STDOUT $| = 1; # let's create our database my $dbh = DBI->connect('dbi:SQLite:dbname=data/weather','','', { RaiseError => 1, AutoCommit => 0 }) or die; # let's create the table. If this fails, we don't care, as it # probably already exists { local $dbh->{RaiseError}; # let's not die on errors local $dbh->{PrintError}; # and let's be quiet $dbh->do(<<EOT); CREATE TABLE stations ( id STRING UNIQUE, name STRING, state STRING, country STRING, latitude REAL, longitude REAL, url STRING ); EOT } # we'll create the indices when we are done, it's supposedly faster # Set up our user agent my $ua = LWP::UserAgent->new; $ua->default_header('Accept-Encoding' => 'gzip, deflate'); # Now with the boring stuff down, get the rather large METAR list print "Downloading METAR station list... "; my $response = $ua->get('http://weather.noaa.gov/data/nsd_cccc.gz'); if($response->is_error) { print STDERR "Failed!\n " . $response->code . ' ' . $response->message . "\n"; } else { print "Done!\nReading it in"; my $content = $response->decoded_content; my $cur_line; my $line_count = 0; my $update_station_query = $dbh->prepare( 'INSERT OR REPLACE INTO stations (id, name, state, country, latitude, longitude)' . ' VALUES (?,?,?,?,?,?)'); while($content) { if(++$line_count % 300 == 0) { print '.'; } ($cur_line, $content) = split(/\n/, $content, 2); my ($station, undef, undef, $name, $state, $country, undef, $lat_dms, $long_dms) = split(/;/, $cur_line, 10); my ($long_deg); my ($lat_deg, $minutes, $seconds, $dir) = $lat_dms =~ m/(\d+)-(\d+)(?:-(\d+))?([NS])/; $lat_deg = &dms_to_degrees($lat_deg, $minutes, $seconds, $dir); ($long_deg, $minutes, $seconds, $dir) = $long_dms =~ m/(\d+)-(\d+)(?:-(\d+))?([EW])/; $long_deg = &dms_to_degrees($long_deg, $minutes, $seconds, $dir); { no warnings qw( uninitialized ); $update_station_query->execute( $station, $name, ($state ? $state : undef), $country, $lat_deg, $long_deg, ); } } print "\nDone! Read $line_count lines\n" } # now let's get the XML data file. # this only has US stations, and generally lacks lat/long. print "Downloading XML station list... "; $response = $ua->get('http://www.nws.noaa.gov/data/current_obs/index.xml'); if($response->is_error) { print STDERR "Failed!\n " . $response->code . ' ' . $response->message . "\n"; } else { print "Done!\nReading it in"; my $xml; if (!eval { $xml = XMLin($response->decoded_content, SuppressEmpty => 1); }) { print STDERR " Failed!\n$@\n"; } else { my $update_station_query = $dbh->prepare( 'UPDATE stations SET url = ? WHERE id = ?'); my $line_count = 0; foreach my $cur_station (@{$xml->{'station'}}) { if(++$line_count % 300 == 0) { print '.'; } no warnings qw( uninitialized ); $update_station_query->execute( $cur_station->{'xml_url'}, $cur_station->{'station_id'} ); } print "\nDone! Read $line_count lines\n"; } } { local $dbh->{RaiseError}; # let's not die on errors local $dbh->{PrintError}; # and let's be quiet $dbh->do(<<EOT); CREATE UNIQUE INDEX stationid ON stations (id); CREATE INDEX latlong ON stations (latitude, longitude); EOT } $dbh->commit; $dbh->disconnect; sub dms_to_degrees { my ($degrees, $minutes, $seconds, $dir) = @_; if(defined $minutes) { $degrees += $minutes * 0.0166666667; } if(defined $seconds) { $degrees += $seconds * 0.000277777778; } if(defined $dir && $dir =~ m/[SW]/) { $degrees = $degrees * -1; } return $degrees; } |