Menu
▾
▴
[Ngetsuite-cvs] ngetsuite/ngetsuite core.rb,1.34,1.35 downloader.rb,1.24,1.25 downloadmanager.rb,1.23,1.24 group.rb,1.38,1.39 queue_file.rb,1.16,1.17
|
From: <jj...@us...> - 2004-02-25 22:44:46
|
Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv20904/ngetsuite Modified Files: core.rb downloader.rb downloadmanager.rb group.rb queue_file.rb Log Message: queue_file should work now the daemon puts the queues processing into PENDING if it is killed Index: core.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/core.rb,v retrieving revision 1.34 retrieving revision 1.35 diff -C2 -d -r1.34 -r1.35 *** core.rb 20 Feb 2004 16:44:39 -0000 1.34 --- core.rb 25 Feb 2004 22:37:10 -0000 1.35 *************** *** 138,142 **** def set_period(queueindex, newperiod) q = Queue.getqueue(queueindex) ! q._period = period q.sync_to_db end --- 138,142 ---- def set_period(queueindex, newperiod) q = Queue.getqueue(queueindex) ! q._period = newperiod q.sync_to_db end Index: downloader.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloader.rb,v retrieving revision 1.24 retrieving revision 1.25 diff -C2 -d -r1.24 -r1.25 *** downloader.rb 23 Feb 2004 17:08:08 -0000 1.24 --- downloader.rb 25 Feb 2004 22:37:10 -0000 1.25 *************** *** 49,60 **** line.chomp! case line when /^already have article (<.*>) \(.*\/([^\/]*)\)$/ ldebug "already have #{green $1} in #{blue $2}" @nbfiles += 1 ! when /^Retrieving: (\d+).*\t(\d+)l.*(<.*>)$/ ! parts, nblines, mid = $1, $2, $3 artindex = Db.dbh.select_one("select `artindex` from `articles` where `mid`='#{mid}'") unless artindex ! puts "No such article: #{mid}" artindex = [nil] end --- 49,87 ---- line.chomp! case line + when /^(\d+) \((\d+)\/(\d+)\): (\d+)\/(\d+)L (\d+)\/(\d+)B (\d+)% (\d+)B\/s (\w+) (\d+)\/(\d+) (\w+)$/ + part, size, speed, filenb = $2, $7, $9, $12 + ldebug "#{yellow $5+'L'}(#{yellow $7+'B'}) at #{red $9+'B/s'} [#{blue $2}/#{blue $3}] [#{green $11}/#{green $12}]" + @curfile.update(part, speed, filenb, size) + + when /^uu_msg\(\d+\):Loaded from (.*): '.*' \((.*)\): (.*) part (\d+)\s+begin\s+(yEnc|UUdata|Base64|XXdata|BinHex|plaintext|qptext|unknown)$/ + artfile, base, filename, nb, enc = $1, $2, $3, $4, $5 + ldebug "decoding filename : #{blue filename}, encoding : #{yellow enc}" + artfile.sub!(/^.*\/([^\/]*)$/, '\1') + ldebug "loaded #{green artfile}" + @curfile._filename = filename + @curfile.encoding = enc + + when /^uu_msg\(\d+\):Loaded from (.*): '.*' \(.*\):.*part (\d+)\s+end\s+(yEnc|UUdata|Base64|XXdata|BinHex|plaintext|qptext|unknown)$/ + artfile, nb = $1, $3 + # basename + artfile.sub!(/^.*\/([^\/]*)$/, '\1') + ldebug "loaded #{green artfile} (last)" + + when /^uu_msg\(\d+\):Loaded from (.*): '.*' \(.*\):.*part (\d+)\s+(yEnc|UUdata|Base64|XXdata|BinHex|plaintext|qptext|unknown)$/ + artfile, nb = $1, $3 + # basename + artfile.sub!(/^.*\/([^\/]*)$/, '\1') + ldebug "loaded #{green artfile}" + when /^already have article (<.*>) \(.*\/([^\/]*)\)$/ ldebug "already have #{green $1} in #{blue $2}" @nbfiles += 1 ! ! when /^Retrieving: (\d+).*\t(\d+)l\t(\d+T\d+)\t(.*)\t(.*)\t(<.*>)$/ ! parts, nblines, date, subject, from, mid = $1, $2, $3, $4, $5, $6 ! ldebug "Retrieving #{blue subject}" artindex = Db.dbh.select_one("select `artindex` from `articles` where `mid`='#{mid}'") unless artindex ! ldebug "#{red 'BUG'}: No such article in db.articles: #{mid}" artindex = [nil] end *************** *** 64,100 **** Db.dbh.do "update `queue` set `nbfiles`='#{@nbfiles}' where `queueindex`='#{@queueindex}'" @curfile = QueueFile.new(@queueindex, artindex, false, parts, nblines, @nbfiles) if not QueueFile.already_in_db(@queueindex, artindex) @curfile.insert_to_db else @curfile.update_from_db - ldebug (red "Article already in database") end ! when /^(\d+) \((\d+)\/(\d+)\): (\d+)\/(\d+)L (\d+)\/(\d+)B (\d+)% (\d+)B\/s (\w+) (\d+)\/(\d+) (\w+)$/ ! part, size, speed, filenb = $2, $7, $9, $12 ! ldebug "#{yellow $5+'L'}(#{yellow $7+'B'}) at #{red $9+'B/s'} [#{blue $2}/#{blue $3}] [#{green $11}/#{green $12}]" ! @curfile.update(part, speed, filenb, size) ! when /\d+ \((\d+)\/\d+\).*\d+\/(\d+)B.* (\d+)B\/s \w+ (\d+)\/(\d+) \w+$/ ! ldebug (red "ERR the previous regexp should have been matched instead of this one") ! part, size, speed, filenb = $1, $2, $3, $4 ! @curfile.update(part, speed, filenb, size) ! when /^autopar.*seems like a (par2?) \((.*)\)/ ! ldebug "#{yellow 'autopar'}: found a #{$1} file for " + blue($2) when /^already have (.*)$/ ! ldebug "already have the file '#{blue $1}'" when /^autopar: .* matches local (par2?)set \((.*)\)$/ ! ldebug "#{yellow 'autopar'}: got a #{$1} for the set #{blue $2}" ! when /^uu_msg\(\d+\):Loaded from .*: '.*' \(.*\): (.*) part \d+.*end (.*)$/ ! ldebug "decoding filename : #{blue $1}, encoding : #{yellow $2}" ! @curfile._filename = $1 ! @curfile.encoding = $2 ! when /^uu_msg\(\d+\):Loaded from (.*): '.*' \(.*\): (.*) part (\d+).*$/ ! file, dfile, nb = $1, $2, $3 ! ldebug "loaded #{green file.sub(/^.*\/([^\/]*)$/, '\1')} (n° #{yellow nb} for #{blue dfile})" when /^decoded ok/ ldebug 'file decoded' ! @curfile.finish(true) @finish_status = 'FINISHED' log "Download finished for file #{@curfile._filename}, queue #{@value}" when /^Connecting to (.*)$/ ldebug "Connection to #{$1}.." --- 91,140 ---- Db.dbh.do "update `queue` set `nbfiles`='#{@nbfiles}' where `queueindex`='#{@queueindex}'" + ldebug "#{red 'BUG'}: Old queue_file present while creating new" if @curfile + @curfile.finish_err if @curfile @curfile = QueueFile.new(@queueindex, artindex, false, parts, nblines, @nbfiles) if not QueueFile.already_in_db(@queueindex, artindex) @curfile.insert_to_db else + ldebug 'Queue_file already in db' @curfile.update_from_db end ! when /^already have (.*)$/ ! ldebug "already have '#{blue $1}'" ! ! # What is autopar ? ! when /^autopar: (.*) seems like a (par2?) \((.*)\)/ ! subject, ft, base = $1, $2, $3 ! ldebug "#{yellow 'autopar'}: found a #{ft} file: " + blue(base.length > 4 ? base : subject) ! when /^autopar: .* matches local (par2?)set \((.*)\)$/ ! ldebug "#{yellow 'autopar'}: got a #{$1}, set #{green $2}" ! when /^decoded ok/ ldebug 'file decoded' ! @curfile.finish_success @finish_status = 'FINISHED' log "Download finished for file #{@curfile._filename}, queue #{@value}" + @curfile = nil + + when 'download error occured, keeping temp files.' + ldebug "error while downloading #{@curfile._filename}" + @curfile.finish_err + @finish_status = 'FAILED' + log "Download failed for #{@curfile._filename}, queue #{@value}" + @curfile = nil + + # same as below + when /^couldn't get (<.*>) from anywhere$/ + ldebug "#{red 'Couldn\'t download mid '} #{$1}" + + when / decoding errors occured, keeping temp files\.$/ + ldebug "error while decoding #{@curfile._filename}" + @curfile.finish_err_dec + @finish_status = 'FAILED' + log "Download finished, decoding failed for #{@curfile._filename}, queue #{@value}" + @curfile = nil + when /^Connecting to (.*)$/ ldebug "Connection to #{$1}.." *************** *** 109,117 **** ldebug 'unequal line count' when />> \d\d\d/, /<< (GROUP|AUTHINFO)/ ! # nothing ! when 'download error occured, keeping temp files.' ! # same as below ! when /^couldn't get (<.*>) from anywhere$/ ! ldebug red("Couldn't download mid ") + green($1) when /^(?:OK:(.*))?(?:WARNINGS:(.*))?(?:ERRORS:(.*))?$/ return unless $1 or $2 or $3 --- 149,153 ---- ldebug 'unequal line count' when />> \d\d\d/, /<< (GROUP|AUTHINFO)/ ! when /^(?:OK:(.*))?(?:WARNINGS:(.*))?(?:ERRORS:(.*))?$/ return unless $1 or $2 or $3 *************** *** 147,152 **** puts 'Error message: ' + e.errstr parent.stop ! rescue Object parent.stop end } --- 183,190 ---- puts 'Error message: ' + e.errstr parent.stop ! rescue Object => e ! puts 'Exception while parsing nget output: '+e.class.to_s + "\n" + e.backtrace.join("\n") parent.stop + raise end } *************** *** 162,167 **** # oh no they killed kenny ! ldebug red('dlder.stop') ! @finish_status = command ? 'STOPPED' : 'FAILED' ! newstat = "`status`='#{finish_status}',`date_finished`=NOW()" newstat += ",`period`='0'" if command Db.dbh.do 'lock tables queue write' --- 200,205 ---- # oh no they killed kenny ! ldebug red('dlder.stop') ! @finish_status = (command ? 'STOPPED' : 'PENDING') ! newstat = "`status`='#{@finish_status}',`date_finished`=NOW()" newstat += ",`period`='0'" if command Db.dbh.do 'lock tables queue write' Index: downloadmanager.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloadmanager.rb,v retrieving revision 1.23 retrieving revision 1.24 diff -C2 -d -r1.23 -r1.24 *** downloadmanager.rb 20 Feb 2004 16:44:39 -0000 1.23 --- downloadmanager.rb 25 Feb 2004 22:37:10 -0000 1.24 *************** *** 61,67 **** def start_new_downloader row = Db.dbh.select_one("select * from `queue` where `status`='PENDING' order by `priority` desc limit 1") ! row = Db.dbh.select_one("select * from `queue` where (`status`='FINISHED' and (`period` != 0) and ((NOW()-`date_finished`) > `period`)) order by `priority` desc limit 1") if not row and $config['resume_downloads'] ! row = Db.dbh.select_one("select * from `queue` where (`status`='STOPPED' and (`period` != 0) and ((NOW()-`date_finished`) > `period`)) order by `priority` desc limit 1") if not row and $config['resume_downloads'] ! row = Db.dbh.select_one("select * from `queue` where (`status`='FAILED' and ((NOW()-`date_finished`) > '7200')) order by `priority` desc limit 1") if not row and $config['resume_downloads'] return false if row == nil --- 61,67 ---- def start_new_downloader row = Db.dbh.select_one("select * from `queue` where `status`='PENDING' order by `priority` desc limit 1") ! row = Db.dbh.select_one("select * from `queue` where (`status`='FINISHED' and (`period`!='0') and ((NOW()-`date_finished`) > `period`)) order by `priority` desc limit 1") if not row and $config['resume_downloads'] ! row = Db.dbh.select_one("select * from `queue` where (`status`='STOPPED' and (`period`!='0') and ((NOW()-`date_finished`) > `period`)) order by `priority` desc limit 1") if not row and $config['resume_downloads'] ! row = Db.dbh.select_one("select * from `queue` where (`status`='FAILED' and (`period`!='0') and ((NOW()-`date_finished`) > 'period')) order by `priority` desc limit 1") if not row and $config['resume_downloads'] return false if row == nil Index: group.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/group.rb,v retrieving revision 1.38 retrieving revision 1.39 diff -C2 -d -r1.38 -r1.39 *** group.rb 23 Feb 2004 17:08:08 -0000 1.38 --- group.rb 25 Feb 2004 22:37:10 -0000 1.39 *************** *** 188,193 **** lasthigh = @_cache_high filecallback = Proc.new { |f| ! if f.req >= 1 ! # and f.parts.articles.artnum.each > lasthigh partcnt = 0 size = 0 --- 188,197 ---- lasthigh = @_cache_high filecallback = Proc.new { |f| ! if f.req >= 1 and not f.parts.empty? and not f.parts[0].articles.empty? ! p = f.parts[0] ! artidx = p.articles[0].articlenum ! artmid = p.mid ! artdate = p.date ! partcnt = 0 size = 0 *************** *** 195,206 **** f.parts.each { |p| next if p.articles.empty? ! a = p.articles[0] ! size += a.bytes.to_i partcnt += 1 if partcnt >= f.req ! sth.execute(a.articlenum, p.mid, f.req, size, f.subject, f.author, p.date) end - # free mem - p.articles = nil } f.parts = nil --- 199,207 ---- f.parts.each { |p| next if p.articles.empty? ! size += p.articles[0].bytes.to_i partcnt += 1 if partcnt >= f.req ! sth.execute(artidx, artmid, f.req, size, f.subject, f.author, artdate) end } f.parts = nil Index: queue_file.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/queue_file.rb,v retrieving revision 1.16 retrieving revision 1.17 diff -C2 -d -r1.16 -r1.17 *** queue_file.rb 20 Feb 2004 16:44:39 -0000 1.16 --- queue_file.rb 25 Feb 2004 22:37:17 -0000 1.17 *************** *** 41,45 **** def update(part, speed, filenb, size) - #puts "begin update : part = #{part}, speed = #{speed}, filenb = #{filenb}, size = #{size}" @part = part @_filenumber = filenb --- 41,44 ---- *************** *** 57,69 **** @_avgspeed = 0 if (@_avgspeed == nil) @_filename = 'not found' if (@_filename == nil) - @_date_finished = Db.now ! ! if res ! @_status = 'FINISHED' ! else ! @_status = 'ERROR_DECODING' ! end ! if @encoding == 'yEnc' @_size = @_size / YENC_OVERHEAD --- 56,61 ---- @_avgspeed = 0 if (@_avgspeed == nil) @_filename = 'not found' if (@_filename == nil) @_date_finished = Db.now ! @_status = res if @encoding == 'yEnc' @_size = @_size / YENC_OVERHEAD *************** *** 75,78 **** --- 67,82 ---- end + def finish_success + finish('FINISHED') + end + + def finish_err_dec + finish('ERROR_DECODING') + end + + def finish_err + finish('ERROR_RETRIEVING') + end + def to_s "-- #{@_filename} : [#{@part}/#{@dlparts}], speed = #{@_avgspeed}" |
