Menu
▾
▴
[Ngetsuite-cvs] ngetsuite/ngetsuite articles.rb,1.2,1.3 config.rb,1.3,1.4 core.rb,1.29,1.30 db.rb,1.22,1.23 downloader.rb,1.21,1.22 downloadmanager.rb,1.19,1.20 group.rb,1.30,1.31 ngetcache.rb,1.6,1.7 queue.rb,1.20,1.21 utils.rb,1.11,1.12 web.rb,1.4,1.5
Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv11482/ngetsuite Modified Files: articles.rb config.rb core.rb db.rb downloader.rb downloadmanager.rb group.rb ngetcache.rb queue.rb utils.rb web.rb Log Message: config: defaults are in config.rb (no need to .fetch), added 'daemon_debug', 'daemon_host', 'dbhost' ; changed 'autostart_httpd' to 'httpd_autostart', 'extern_nfo_update' to 'extern_nfos_update', 'port' to 'daemon_port' ; reqesting a config keyword inexisting raises an exception group: somewhat gives a success status, fixed shortname, *sexy* headers update global: removed some global variables ($dbh->Db.dbh..) Index: articles.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/articles.rb,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** articles.rb 4 Feb 2004 15:40:22 -0000 1.2 --- articles.rb 6 Feb 2004 23:46:59 -0000 1.3 *************** *** 167,171 **** def to_s ! @_mid + ": " + @_nbactual.to_s + "/" + @_nbexpected.to_s + " [" + @_rlsmap + ']' end --- 167,171 ---- def to_s ! "#{@_mid}: #{@_nbactual}/#{@_nbexpected} [#{@_rlsmap}]" end Index: config.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/config.rb,v retrieving revision 1.3 retrieving revision 1.4 diff -C2 -d -r1.3 -r1.4 *** config.rb 2 Feb 2004 02:41:34 -0000 1.3 --- config.rb 6 Feb 2004 23:46:59 -0000 1.4 *************** *** 4,30 **** # just treat it like a hash class Config < Hash - # create a new config hash from @file def initialize(file) super(false) @file = file ! if(File.exist?("#{@file}")) ! IO.foreach("#{@file}") do |line| ! next if(line =~ /^\s*(#|$)/) ! if(line =~ /^\s*(\S+)\s*=\s*(.*)$/) ! self[$1] = eval($2) if($2) ! end end end end ! # write current configuration to @file ! def save ! File.open("#{@file}", 'w') do |file| ! self.each do |key, value| file.puts "#{key} = #{value}" end end end end --- 4,66 ---- # just treat it like a hash class Config < Hash # create a new config hash from @file def initialize(file) super(false) @file = file ! raise RuntimeError.new("unable to read #{@file}") unless File.readable?("#{@file}") ! # default values ! " ! dbhost = 'localhost' ! dbuser = 'nget' ! dbpass = 'mypass' ! dbname = 'ngetcache' ! nget = '/home/nget/bin/nget' ! yydecode = '/usr/bin/yydecode' ! cachedir = ENV['HOME'] + File::Separator + '.nget4' ! basedir = ENV['HOME'] + File::Separator + 'ngot' ! tempdir = ENV['TMP'] or ENV['TEMP'] or '/tmp' ! max_dls = 3 ! daemon_port = 15345 ! daemon_host = 'localhost' ! daemon_log = ENV['HOME'] + File::Separator + 'ngot' + File::Separator + 'ngetsuite-daemon.log' ! resume_downloads = false ! daemon_autostart = true ! debug = false ! daemon_debug = false ! extern_headers_update = '' ! extern_nfos_fetch = '' ! httpd_port = 2000 ! httpd_autostart = true ! ".each("\n") { |l| self[$1] = eval($2) if l=~ /^(\S+)\s*=\s*(.*)$/ } ! IO.foreach(@file) do |line| ! next if(line =~ /^\s*(#|$)/) ! if(line =~ /^\s*(\S+)\s*=\s*(.*)$/) ! self[$1] = eval($2) if($2) end end end ! # write current configuration to @file ! def save(filename) ! if FileTest.exists? filename ! puts "Cannot overwrite an existing file" ! return ! end ! File.open(filename, 'w') do |file| ! self.keys.sort.each do |key| ! value = self[key] ! value = "'"+value+"'" if value.class == String file.puts "#{key} = #{value}" end end end + + def [](key) + begin + fetch(key) + rescue IndexError + raise RuntimeError.new("Config file key '#{key}' not found (found #{self.keys.map{|k| "'#{k}'"}.sort.join(', ')})") + end + end end Index: core.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/core.rb,v retrieving revision 1.29 retrieving revision 1.30 diff -C2 -d -r1.29 -r1.30 *** core.rb 4 Feb 2004 05:56:50 -0000 1.29 --- core.rb 6 Feb 2004 23:46:59 -0000 1.30 *************** *** 1,8 **** ! require 'drb' ! require 'getoptlong' require 'ngetsuite/utils' require 'ngetsuite/queue' require 'ngetsuite/group' require 'ngetsuite/nfo' module NgetSuite --- 1,9 ---- ! # todo: move requires in the blocks needing them (move exception handling as well) require 'ngetsuite/utils' require 'ngetsuite/queue' require 'ngetsuite/group' require 'ngetsuite/nfo' + require 'dbi' + require 'drb' module NgetSuite *************** *** 53,60 **** def get_manager DRb.start_service() ! port = $config.fetch('port', 12345) ! manager = DRbObject.new(nil, 'druby://localhost:' + port.to_s) if !manager ! raise ManagerNotFoundException.new("localhost:#{port}") end manager --- 54,61 ---- def get_manager DRb.start_service() ! hp = "#{$config['daemon_host']}:#{$config['daemon_port']}" ! manager = DRbObject.new(nil, "druby://#{hp}") if !manager ! raise ManagerNotFoundException.new(hp) end manager *************** *** 77,82 **** def delete_group(groupid) ! name = Group.getname(groupid) ! $dbh.do("delete from `groups` where `fullname` = \"#{name}\"") end --- 78,82 ---- def delete_group(groupid) ! Group.delete(groupid) end *************** *** 84,90 **** if !groupid puts "Update all groups" ! groups = $dbh.select_all 'select `fullname` from `groups` ! where active="yes"' ! groups.each do |group| name = group.to_s puts "Updating #{name}" --- 84,88 ---- if !groupid puts "Update all groups" ! Group.getallactive.each do |group| name = group.to_s puts "Updating #{name}" *************** *** 156,176 **** def printusage print "Usage: #{$0} command [options]\n\n" ! i = 0 ! puts "List of options:" ! $args.each { |item| ! # Uglllllllly ! if i == 3 puts "\nList of actions:" end ! full, short, type = item.values_at(0, 1, 2) ! left = "\t#{full}, #{short}" ! if type != GetoptLong::NO_ARGUMENT ! left += " #{item[3]}" desc = item[4] else desc = item[3] end ! printf("%-45.45s\t%s\n", left, desc) ! i += 1 } end --- 154,171 ---- def printusage print "Usage: #{$0} command [options]\n\n" ! puts 'List of options:' ! $args.each_index { |i| ! unless item = $args[i] puts "\nList of actions:" + next end ! arg = "\t#{item[0]}, #{item[1]}" ! if item.length == 5 ! arg += " #{item[3]}" desc = item[4] else desc = item[3] end ! printf("%-45.45s\t%s\n", arg, desc) } end *************** *** 183,186 **** --- 178,187 ---- begin case opt + when 'debug' + $config['debug']=true + when 'no-debug' + $config['no-debug']=false + when 'write-config' + $config.save(arg) when 'show-groups' Group.show *************** *** 291,294 **** --- 292,298 ---- rescue NoSuchNfoException => e puts e.message + rescue Exception => e + puts e.message + print e.backtrace.join("\n") rescue ArgumentError => e puts e.message *************** *** 302,308 **** puts "Problem connecting to the manager" puts e - rescue Exception => e - puts e.message - print e.backtrace.join("\n") ensure # Nothing --- 306,309 ---- Index: db.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/db.rb,v retrieving revision 1.22 retrieving revision 1.23 diff -C2 -d -r1.22 -r1.23 *** db.rb 3 Feb 2004 21:31:32 -0000 1.22 --- db.rb 6 Feb 2004 23:46:59 -0000 1.23 *************** *** 1,6 **** module NgetSuite module Db @@db_handler = nil ! ## Exceptions class TableNotFoundException < Exception --- 1,7 ---- + require 'dbi' module NgetSuite module Db @@db_handler = nil ! ## Exceptions class TableNotFoundException < Exception *************** *** 132,137 **** def Db.dbh unless @@db_handler and @@db_handler.connected? ! @@db_handler = DBI.connect("DBI:Mysql:#{$config['dbname']}", ! $config['dbuser'], $config['dbpass']) end --- 133,137 ---- def Db.dbh unless @@db_handler and @@db_handler.connected? ! @@db_handler = DBI.connect("DBI:Mysql:#{$config['dbname']}:#{$config['dbhost']}", $config['dbuser'], $config['dbpass']) end *************** *** 215,219 **** req += " from #{table} where " + ids_constraint ! debug 'DB: '+ req sth = Db.dbh.execute req hash = sth.fetch_hash --- 215,219 ---- req += " from #{table} where " + ids_constraint ! # debug 'DB: '+ req sth = Db.dbh.execute req hash = sth.fetch_hash *************** *** 283,287 **** req += ')' ! debug 'DB: ' + req exec_request(req, values) end --- 283,287 ---- req += ')' ! # debug 'DB: ' + req exec_request(req, values) end Index: downloader.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloader.rb,v retrieving revision 1.21 retrieving revision 1.22 diff -C2 -d -r1.21 -r1.22 *** downloader.rb 4 Feb 2004 15:21:48 -0000 1.21 --- downloader.rb 6 Feb 2004 23:46:59 -0000 1.22 *************** *** 1,2 **** --- 1,3 ---- + require 'ngetsuite/db' require 'ngetsuite/queue_file' require 'ngetsuite/utils' *************** *** 37,52 **** @status = 'PENDING' @nbfiles = 0 end def update_status(line) line.chomp! case line when /^already have article (<.*>) \(.*\/([^\/]*)\)$/ ! debug yellow('file already there') + ' : ' + green($1) + ' in ' + blue($2) @nbfiles += 1 when /^Retrieving: (\d+).*\t(\d+)l.*(<.*>)$/ parts, nblines, mid = $1, $2, $3 @nbfiles += 1 ! $dbh.do "update queue set `nbfiles`='#{@nbfiles}' where `id`='#{@id}'" @curfile = QueueFile.new(@id, mid, false, parts, nblines, @nbfiles) --- 38,58 ---- @status = 'PENDING' @nbfiles = 0 + @finish_status = 'FAILED' end + def ldebug(str) + debug "(#{white @id})" + str + end + def update_status(line) line.chomp! case line when /^already have article (<.*>) \(.*\/([^\/]*)\)$/ ! ldebug "already have #{green $1} in #{blue $2}" @nbfiles += 1 when /^Retrieving: (\d+).*\t(\d+)l.*(<.*>)$/ parts, nblines, mid = $1, $2, $3 @nbfiles += 1 ! Db.dbh.do "update queue set `nbfiles`='#{@nbfiles}' where `id`='#{@id}'" @curfile = QueueFile.new(@id, mid, false, parts, nblines, @nbfiles) *************** *** 55,59 **** else @curfile.update_from_db ! debug (Utils.red "already in database") end when /^(\d+) \((\d+)\/(\d+)\): (\d+)\/(\d+)L (\d+)\/(\d+)B (\d+)% (\d+)B\/s (\w+) (\d+)\/(\d+) (\w+)$/ --- 61,65 ---- else @curfile.update_from_db ! ldebug (Utils.red "already in database") end when /^(\d+) \((\d+)\/(\d+)\): (\d+)\/(\d+)L (\d+)\/(\d+)B (\d+)% (\d+)B\/s (\w+) (\d+)\/(\d+) (\w+)$/ *************** *** 62,69 **** speed = $9 filenb = $12 ! debug "#{yellow $5+'L'}(#{yellow $7+'B'}) at #{red $9+'B/s'} [#{blue $2}/#{blue $3}] [#{green $11}/#{green $12}]" @curfile.update(part, speed, filenb, size) when /\d+ \((\d+)\/\d+\).*\d+\/(\d+)B.* (\d+)B\/s \w+ (\d+)\/(\d+) \w+$/ ! debug (Utils.red "the previous regexp should have been matched instead of this one") part = $1 size = $2 --- 68,75 ---- speed = $9 filenb = $12 ! ldebug "#{yellow $5+'L'}(#{yellow $7+'B'}) at #{red $9+'B/s'} [#{blue $2}/#{blue $3}] [#{green $11}/#{green $12}]" @curfile.update(part, speed, filenb, size) when /\d+ \((\d+)\/\d+\).*\d+\/(\d+)B.* (\d+)B\/s \w+ (\d+)\/(\d+) \w+$/ ! ldebug (Utils.red "the previous regexp should have been matched instead of this one") part = $1 size = $2 *************** *** 71,100 **** filenb = $4 @curfile.update(part, speed, filenb, size) ! when /^autopar.*seems like a par2 \((.*)\)/ ! debug 'autopar: found a par2 for ' + blue($1) ! when /^uu_msg\(\d+\):Loaded from .*: '.*' \(.*\): (.*) part \d+.*end (.*)$/ ! debug (Utils.yellow "filename : #{$1}, encoding : #{$2}") @curfile._filename = $1 @curfile.encoding = $2 when /^decoded ok/ ! debug 'file decoded' @curfile.finish(true) log "Download finished for file #{@curfile._filename}, queue #{@value}" when /^Connecting to (.*)$/ ! debug blue("Connection to #{$1}..") when />> 200/ ! debug blue('Connected') ! when />> (480|381|281)/, /<< (GROUP|AUTHINFO)/ # nothing ! when /^download error occured, keeping temp file$/ # same as below when /^couldn't get (<.*>) from anywhere$/ ! debug red("Couldn't download mid ") + green($1) when /^ERRORS: (.*)$/ ! debug red("Finished, got errors for #{$1}") else #TODO : handle decoding failures (=> @curfile.finish(false)) ! debug "unhandled msg : #{blue line}" end #debug (Utils.red "end update_status") --- 77,118 ---- filenb = $4 @curfile.update(part, speed, filenb, size) ! when /^autopar.*seems like a (par2?) \((.*)\)/ ! ldebug "autopar: found a #{$1} file for " + blue($2) ! when /^already have (.*)$/ ! ldebug "already have the file '#{blue $1}'" ! when /^autopar: .* matches local (par2?)set \((.*)\)$/ ! ldebug "#{yellow 'autopar'}: got a #{$1} for the set #{blue $2}" when /^uu_msg\(\d+\):Loaded from .*: '.*' \(.*\): (.*) part \d+.*end (.*)$/ ! ldebug "decoding filename : #{blue $1}, encoding : #{yellow $2}" @curfile._filename = $1 @curfile.encoding = $2 + when /^uu_msg\(\d+\):Loaded from (.*): '.*' \(.*\): (.*) part (\d+).*$/ + file, dfile, nb = $1, $2, $3 + ldebug "loaded #{green file.sub(/^.*\/([^\/]*)$/, '\1')} (n° #{yellow nb} for #{blue dfile})" when /^decoded ok/ ! ldebug 'file decoded' @curfile.finish(true) + @finish_status = 'FINISHED' log "Download finished for file #{@curfile._filename}, queue #{@value}" when /^Connecting to (.*)$/ ! ldebug "Connection to #{$1}.." when />> 200/ ! ldebug 'Connected' ! when />> 400/ ! ldebug 'Disconnected' ! when /<< QUIT/ ! ldebug 'Quitted' ! when />> \d\d\d/, /<< (GROUP|AUTHINFO)/ # nothing ! when 'download error occured, keeping temp files.' # same as below when /^couldn't get (<.*>) from anywhere$/ ! ldebug red("Couldn't download mid ") + green($1) when /^ERRORS: (.*)$/ ! ldebug red("Finished, got errors (#{$1})") ! @finish_status = 'FAILED' else #TODO : handle decoding failures (=> @curfile.finish(false)) ! ldebug "unhandled msg : #{blue line}" end #debug (Utils.red "end update_status") *************** *** 106,112 **** @start_time = Time.new @status = 'PROCESSING' ! $dbh.do 'lock tables queue write' ! $dbh.do "update queue set `status` = 'PROCESSING', `date_started`= NOW() where `id` = '#{@id}'" ! $dbh.do 'unlock tables' @io = IO.popen("#{$nget} -q -G #{@group} -r '#{@value}'") @t = Thread.new(@io) { |io| --- 124,130 ---- @start_time = Time.new @status = 'PROCESSING' ! Db.dbh.do 'lock tables queue write' ! Db.dbh.do "update queue set `status` = 'PROCESSING', `date_started`= NOW() where `id` = '#{@id}'" ! Db.dbh.do 'unlock tables' @io = IO.popen("#{$nget} -q -G #{@group} -r '#{@value}'") @t = Thread.new(@io) { |io| *************** *** 116,120 **** } rescue IOError ! puts 'nget got killed' rescue Exception => e --- 134,138 ---- } rescue IOError ! puts 'nget got killed!' rescue Exception => e *************** *** 130,133 **** --- 148,152 ---- end } + ldebug 'Downloader started for ' + green(@value) log "Download started for #{@value}, group #{@group} to #{@dest}" end *************** *** 139,162 **** def stop # oh no they killed kenny ! ! debug "#{red 'dlder.stop'}" Process.kill('SIGTERM', @io.pid) @io.close @t.exit ! $dbh.do 'lock tables queue write' ! $dbh.do "update `queue` set `status` = 'STOPPED', `date_finished`=NOW() where `id` = '#{@id}'" ! $dbh.do 'unlock tables' ! log "Download stopped for queue #{@value}, group #{@group}" end ! def finish(success) ! @io.close @t.exit time = Time.now.to_s ! $dbh.do 'lock tables queue write' ! newstatus = success ? 'FINISHED' : 'FAILED' ! $dbh.do "update queue set `status` = '#{newstatus}', `date_finished`=NOW() where `id` = '#{@id}'" ! $dbh.do 'unlock tables' ! log "Download #{newstatus.downcase} for queue #{@value}, group #{@group}" end --- 158,188 ---- def stop # oh no they killed kenny ! ! ldebug red('dlder.stop') ! Db.dbh.do 'lock tables queue write' ! Db.dbh.do "update `queue` set `status` = 'STOPPED', `date_finished`=NOW() where `id` = '#{@id}'" ! Db.dbh.do 'unlock tables' ! Process.kill('SIGTERM', @io.pid) @io.close @t.exit ! ldebug "Download stopped for #{green @value}, group #{@group}" ! log "Download stopped for #{@value}, group #{@group}" end ! def finish(success=true) ! begin ! @io.close ! rescue ! end @t.exit time = Time.now.to_s ! Db.dbh.do 'lock tables queue write' ! # newstatus = success ? 'FINISHED' : 'FAILED' ! newstatus = @finish_status ! Db.dbh.do "update queue set `status` = '#{newstatus}', `date_finished`=NOW() where `id` = '#{@id}'" ! Db.dbh.do 'unlock tables' ! ldebug "Download #{newstatus.downcase} for #{green @value}" ! log "Download #{newstatus.downcase} for #{@value}, group #{@group}" end Index: downloadmanager.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloadmanager.rb,v retrieving revision 1.19 retrieving revision 1.20 diff -C2 -d -r1.19 -r1.20 *** downloadmanager.rb 4 Feb 2004 15:21:48 -0000 1.19 --- downloadmanager.rb 6 Feb 2004 23:46:59 -0000 1.20 *************** *** 1,2 **** --- 1,3 ---- + require 'ngetsuite/db' require 'ngetsuite/downloader' require 'ngetsuite/utils' *************** *** 20,24 **** @poll_interval = 30 @started = false - start if $config.fetch('autostart', false) end --- 21,24 ---- *************** *** 32,36 **** @mainThread = Thread.new { begin ! while @started do log "current : #{@current_dlers}, max : #{@max_dlers}" if @lastlog_current != @current_dlers or @lastlog_max != @max_dlers @lastlog_current, @lastlog_max = @current_dlers, @max_dlers --- 32,37 ---- @mainThread = Thread.new { begin ! loop do ! break unless @started log "current : #{@current_dlers}, max : #{@max_dlers}" if @lastlog_current != @current_dlers or @lastlog_max != @max_dlers @lastlog_current, @lastlog_max = @current_dlers, @max_dlers *************** *** 38,42 **** --- 39,45 ---- start_new_downloader end + break unless @started check_finished_dls + break unless @started sleep(@poll_interval) end *************** *** 56,61 **** def start_new_downloader ! row = $dbh.select_one("select * from queue where `status`='PENDING' order by `priority` desc limit 1") ! row = $dbh.select_one("select * from queue where `status`='STOPPED' order by `priority` desc limit 1") if not row and $config.fetch('resume_downloads', false) return false if row == nil --- 59,64 ---- def start_new_downloader ! row = Db.dbh.select_one("select * from queue where `status`='PENDING' order by `priority` desc limit 1") ! row = Db.dbh.select_one("select * from queue where `status`='STOPPED' order by `priority` desc limit 1") if not row and $config['resume_downloads'] return false if row == nil *************** *** 63,68 **** @current_dlers += 1 id, group, type, value, subdir, prio = row.values_at(0, 1, 2, 3, 4, 5) - debug(green("New downloader starting queue #{id}")) - log "found #{value} for group #{group}\n" dler = Downloader.new(id, type, value, group, @basedir + File::Separator + subdir, prio) --- 66,69 ---- *************** *** 89,97 **** @started = false @dlers.each { |i, d| ! d.stop } @mainThread.run if @mainThread.alive? 10.times do |i| ! sleep 0.1 break unless @mainThread.alive? if i == 9 --- 90,100 ---- @started = false @dlers.each { |i, d| ! Thread.new { d.stop } } + @dlers = {} + @current_dlers = 0 @mainThread.run if @mainThread.alive? 10.times do |i| ! sleep 0.2 break unless @mainThread.alive? if i == 9 Index: group.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/group.rb,v retrieving revision 1.30 retrieving revision 1.31 diff -C2 -d -r1.30 -r1.31 *** group.rb 4 Feb 2004 15:21:48 -0000 1.30 --- group.rb 6 Feb 2004 23:46:59 -0000 1.31 *************** *** 30,33 **** --- 30,37 ---- Db.autotable(Group, 'groups') + def Group.delete(id) + Db.dbh.do "delete from `groups` where `fullname` = '#{Group.getname(id)}'" + end + def Group.getname(id) begin *************** *** 61,64 **** --- 65,72 ---- end + def Group.getallactive + Db.dbh.select_all "select `fullname` from `groups` where active='yes'" + end + def Group.getgroup(id) Group.new(Group.getname(id), true) *************** *** 93,98 **** @_description = description @_date_added = Time.now ! @nget = $config.fetch('nget', '/usr/bin/nget') ! @yydecode = $config.fetch('yydecode', '/usr/bin/yydecode') @cachedir = $config['cachedir'] update_from_db if fetch_from_db --- 101,106 ---- @_description = description @_date_added = Time.now ! @nget = $config['nget'] ! @yydecode = $config['yydecode'] @cachedir = $config['cachedir'] update_from_db if fetch_from_db *************** *** 105,109 **** def _shortname if (@_shortname == nil) ! @_shortname = @_fullname.split('.').map {|item| item[0] }.join('') end --- 113,117 ---- def _shortname if (@_shortname == nil) ! @_shortname = @_fullname.to_s.split('.').map {|item| item[0].chr }.join('') end *************** *** 127,174 **** end - def announceupdate - #Db.dbh.execute "insert into `articles` (`group`, `action`) values ('#{@_fullname}', 'updating')" - end - def announcedupdate - #'updating' == Db.dbh.execute "select `action` from `articles` where `group` == '#{@_fullname}'" - end - def finishupdate - #Db.dbh.execute "delete from `articles` where `group`='#{@_fullname}', `action`='updating'" - end - def update ! puts "Warning: A headers update seems to be running !" if announcedupdate ! updatecache unless $config.fetch('extern_headers_update', '').split(',').include?(@_shortname) clear_articles ! clear_nfos clear_releases - announceupdate - mknewscache - fetchnfos unless $config.fetch('extern_nfo_fetch', '').split(',').include?(@_shortname) recognizereleases @_date_updated = Time.now sync_to_db - finishupdate end def clear_articles - debug "Beginning clear_articles at #{Time.now.strftime('%H:%M:%S')}" nb_rows = Db.dbh.execute "delete from `articles` where `group` = '#{@_fullname}'" ! debug "End clear_articles at #{Time.now.strftime('%H:%M:%S')}" end def clear_nfos - debug "Beginning clear_nfos at #{Time.now.strftime('%H:%M:%S')}" nb_rows = Db.dbh.do "delete from `nfos` where `group` = '#{@_fullname}'" debug "#{nb_rows} nfos deleted" - debug "End clear_nfos at #{Time.now.strftime('%H:%M:%S')}" - end def clear_releases - debug "Beginning clear_releases at #{Time.now.strftime('%H:%M:%S')}" nb_rows = Db.dbh.do "delete from `releases` where `group` = '#{@_fullname}'" debug "#{nb_rows} releases deleted" - debug "End clear_releases at #{Time.now.strftime('%H:%M:%S')}" end --- 135,166 ---- end def update ! updatengetcache unless $config['extern_headers_update'].split(',').include?(@_shortname) clear_articles ! parsengetcache ! clear_releases recognizereleases + + clear_nfos + fetchnfos unless $config['extern_nfos_fetch'].split(',').include?(@_shortname) + @_date_updated = Time.now sync_to_db end def clear_articles nb_rows = Db.dbh.execute "delete from `articles` where `group` = '#{@_fullname}'" ! debug "#{nb_rows} articles deleted" end def clear_nfos nb_rows = Db.dbh.do "delete from `nfos` where `group` = '#{@_fullname}'" debug "#{nb_rows} nfos deleted" end def clear_releases nb_rows = Db.dbh.do "delete from `releases` where `group` = '#{@_fullname}'" debug "#{nb_rows} releases deleted" end *************** *** 181,201 **** end ! def updatecache ! debug "Beginning updatecache at #{Time.now.strftime('%H:%M:%S')}" ! # to avoid ngetsuite poisoning n = rand(900000)+100000 ! `nice -n19 #{@nget} -q -g #{@_fullname} -c -r '^#{n}Dummy'` ! debug "End updatecache at #{Time.now.strftime('%H:%M:%S')}" end ! def mknewscache ! debug "Beginning mknewscache at #{Time.now.strftime('%H:%M:%S')}" ! sth = Db.dbh.prepare 'insert into articles(`group`, `mid`, `nbparts`, `size`, `subject`, `from`, `time`) values (?, ?, ?, ?, ?, ?, FROM_UNIXTIME(?))' ! filecallback = Proc.new{ |f| if f.req >= 1 partcnt = 0 size = 0 a = nil ! f.parts.each{ |p| unless p.articles.empty? a = p.articles[0] --- 173,203 ---- end ! def updatengetcache ! t = Time.now ! debug "Beginning updatengetcache at #{t.strftime('%H:%M:%S')}" n = rand(900000)+100000 ! $stdout.sync = true if $stdout.isatty ! begin ! IO.popen("nice -n19 #{@nget} -g #{@_fullname} -cr '^#{n}Dummy'").each("\r") { |line| ! print "progress#{red $1} speed#{blue $2} ETA#{green $3} \r" if line =~ /Retrieving headers \d+-\d+ : \d+\/\d+\/\d+(\s+\d+%)(\s+\d+B\/s)(.*s)/ and $stdout.isatty ! } ! rescue ! puts "\nInterrupted" ! end ! print "\n" if $stdout.isatty ! $stdout.sync = false if $stdout.isatty ! debug "Updatengetcache finished in #{Time.now-t}s" end ! def parsengetcache ! t = Time.now ! debug "Beginning parsengetcache at #{t.strftime('%H:%M:%S')}" ! sth = Db.dbh.prepare 'insert into articles (`group`, `mid`, `nbparts`, `size`, `subject`, `from`, `time`) values (?, ?, ?, ?, ?, ?, FROM_UNIXTIME(?))' ! filecallback = Proc.new { |f| if f.req >= 1 partcnt = 0 size = 0 a = nil ! f.parts.each { |p| unless p.articles.empty? a = p.articles[0] *************** *** 206,209 **** --- 208,212 ---- end end + # free mem p.articles = nil } *************** *** 213,229 **** NgetCache.new(@_fullname, @cachedir, nil, filecallback) ! debug "End mknewscache at #{Time.now.strftime('%H:%M:%S')}" end def fetchnfos # fetches the nfos ! `nice -n19 #{@nget} --path #{$tempdir} -G #{@_fullname} -K -L 150 -r '\\.nfo|\\.txt'` ! # but mark them as if we didn't download them ! `#{@nget} --path #{$tempdir} -qq -G #{@_fullname} -U -r '\.nfo|\.txt'` sth = Db.dbh.prepare 'insert ignore into nfos (`group`, `mid`, `text`) values (?, ?, ?)' ! ! pwd = Dir.pwd ! Dir.chdir($tempdir) Dir['ngettemp-*'].each { |file| File.open(file).each { |line| --- 216,241 ---- NgetCache.new(@_fullname, @cachedir, nil, filecallback) ! debug "Parsengetcache done in #{Time.now-t}s" end def fetchnfos + t = Time.now + debug "Beginning fetchnfos at #{t.strftime('%H:%M:%S')}" + tempdir = $config['tempdir'] + File::Separator + "ngetupdate.#{@_shortname}.#{Process.pid}" + if FileTest.exists? tempdir + puts "Error: The temporary directory #{tempdir} already exists !" + return + end + Dir.mkdir tempdir + pwd = Dir.pwd + begin # fetches the nfos ! `nice -n19 #{@nget} --path #{tempdir} -G #{@_fullname} -K -L 150 -r '\\.nfo|\\.txt'` ! # but mark them for nget as if we didn't download them ! `#{@nget} --path #{tempdir} -qq -G #{@_fullname} -U -r '\\.nfo|\\.txt'` ! debug "Fetchnfos: files downloaded in #{Time.now-t}s" sth = Db.dbh.prepare 'insert ignore into nfos (`group`, `mid`, `text`) values (?, ?, ?)' ! Dir.chdir(tempdir) Dir['ngettemp-*'].each { |file| File.open(file).each { |line| *************** *** 231,235 **** if $1 != nil mid = $1 - debug "#{file}: #{mid}" text = `#{@yydecode} #{file} -o -`.chomp sth.execute(@_fullname, mid, text) if text and text.length < 15000 --- 243,246 ---- *************** *** 244,272 **** File.delete file } ! ! Dir.chdir(pwd) end def recognizereleases articles = NgetSuite::ArticleList.new("where `group`='#{@_fullname}' order by `subject`") ! @rls = nil articles.each { |art| ! @rls = NgetSuite::Release.new(art._group, art._mid) if !@rls ! ! if @rls.add?(art) == false then # New release! Commit the current one to the db ! @rls.insert_to_db ! # And make another ! @rls = NgetSuite::Release.new(art._group, art._mid) ! @rls.add?(art) end } ! @rls.insert_to_db ! @rls = nil end end # class - end # module --- 255,288 ---- File.delete file } ! ensure ! Dir.chdir(pwd) ! `rm -rf #{tempdir}` ! end ! debug "Fetchnfos finished in #{Time.now-t}s" end def recognizereleases + t = Time.now + debug "Recognizing releases at #{t.strftime('%H:%M:%S')}" articles = NgetSuite::ArticleList.new("where `group`='#{@_fullname}' order by `subject`") ! rls = nil ! nb_releases = 0 articles.each { |art| ! rls = NgetSuite::Release.new(art._group, art._mid) if !rls ! if rls.add?(art) == false then # New release! Commit the current one to the db ! rls.insert_to_db ! nb_releases += 1 ! # And make another ! rls = NgetSuite::Release.new(art._group, art._mid) ! rls.add? art end } ! rls.insert_to_db ! debug "Inserted #{nb_releases} in #{Time.now-t}s" end end # class end # module Index: ngetcache.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/ngetcache.rb,v retrieving revision 1.6 retrieving revision 1.7 diff -C2 -d -r1.6 -r1.7 *** ngetcache.rb 4 Feb 2004 07:13:05 -0000 1.6 --- ngetcache.rb 6 Feb 2004 23:46:59 -0000 1.7 *************** *** 11,15 **** require 'zlib' ! attr_reader :servers, :files, :sane, :lineno CacheServer = Struct.new('CacheServer', 'serverid', 'high', 'low', 'num') --- 11,16 ---- require 'zlib' ! attr_reader :servers, :files, :sane, :lineno, :groupname ! attr_accessor :writeoldformat CacheServer = Struct.new('CacheServer', 'serverid', 'high', 'low', 'num') *************** *** 24,27 **** --- 25,30 ---- @filecallback = filecallback @sane = true + @groupname = groupname + @path = path @fd = Zlib::GzipReader.open(path + File::Separator + groupname + ',cache.gz') *************** *** 35,38 **** --- 38,42 ---- @fd.close end + print "\n" if $stdout.isatty # cache_check if sanitycheck and @sane end *************** *** 123,127 **** end ! def save_tofile(groupname, path) @fd = Zlib::GzipWriter.open(path + File::Separator + groupname + ',cache.newformat.gz') write_all --- 127,131 ---- end ! def save_tofile(groupname=@groupname, path=@path) @fd = Zlib::GzipWriter.open(path + File::Separator + groupname + ',cache.newformat.gz') write_all *************** *** 145,149 **** def write_file(f) ! @fd.puts [f.req, f.flags, f.subject, f.author, f.partoff, f.tailoff].join("\t") f.references.each { |r| write_reference(r) --- 149,153 ---- def write_file(f) ! @fd.puts [f.req, f.flags, (f.fileid if @writeoldformat), f.subject, f.author, f.partoff, f.tailoff].compact.join("\t") f.references.each { |r| write_reference(r) Index: queue.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/queue.rb,v retrieving revision 1.20 retrieving revision 1.21 diff -C2 -d -r1.20 -r1.21 *** queue.rb 4 Feb 2004 01:45:39 -0000 1.20 --- queue.rb 6 Feb 2004 23:46:59 -0000 1.21 *************** *** 1,4 **** --- 1,5 ---- require 'ngetsuite/utils' require 'ngetsuite/db' + require 'ngetsuite/queue_file' module NgetSuite Index: utils.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/utils.rb,v retrieving revision 1.11 retrieving revision 1.12 diff -C2 -d -r1.11 -r1.12 *** utils.rb 4 Feb 2004 05:56:50 -0000 1.11 --- utils.rb 6 Feb 2004 23:46:59 -0000 1.12 *************** *** 24,28 **** def Utils.debug(message=nil) ! puts "D: #{message}" if($config.fetch('debug', false) && message) end --- 24,28 ---- def Utils.debug(message=nil) ! puts "#{green 'D'}: #{message}" if $config['debug'] and message end *************** *** 31,38 **** end def Utils.red msg color(msg,RED_FG,1) end - def red msg Utils.red msg --- 31,44 ---- end + def Utils.white msg + color(msg,WHITE_FG,1) + end + def white msg + Utils.white msg + end + def Utils.red msg color(msg,RED_FG,1) end def red msg Utils.red msg *************** *** 42,46 **** color(msg,BLUE_FG,1) end - def blue msg Utils.blue msg --- 48,51 ---- *************** *** 50,54 **** color(msg,GREEN_FG,1) end - def green msg Utils.green msg --- 55,58 ---- *************** *** 58,62 **** color(msg,YELLOW_FG,1) end - def yellow msg Utils.yellow msg --- 62,65 ---- Index: web.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/web.rb,v retrieving revision 1.4 retrieving revision 1.5 diff -C2 -d -r1.4 -r1.5 *** web.rb 4 Feb 2004 03:07:57 -0000 1.4 --- web.rb 6 Feb 2004 23:46:59 -0000 1.5 *************** *** 1,3 **** - require 'dbi' require 'ngetsuite/utils' require 'ngetsuite/queue' --- 1,2 ---- |
