Menu
▾
▴
[Ngetsuite-cvs] ngetsuite/ngetsuite articles.rb,1.10,1.11 core.rb,1.40,1.41 downloader.rb,1.32,1.33 group.rb,1.46,1.47 nfo.rb,1.6,1.7 ngetcache.rb,1.10,1.11 queue.rb,1.26,1.27
|
From: Johnjohn <jj...@us...> - 2004-06-19 13:27:59
|
Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv16030/ngetsuite Modified Files: articles.rb core.rb downloader.rb group.rb nfo.rb ngetcache.rb queue.rb Log Message: misc updates, better command-line handling of nfos/releases Index: downloader.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloader.rb,v retrieving revision 1.32 retrieving revision 1.33 diff -C2 -d -r1.32 -r1.33 *** downloader.rb 25 May 2004 22:41:59 -0000 1.32 --- downloader.rb 19 Jun 2004 13:27:49 -0000 1.33 *************** *** 151,155 **** when /^doarticle \d+: \d+!=\d+ \|\| \d+!=\d+$/ when /^unequal line count (\d+) should equal (\d+)$/ ! ldebug "unequal line count (#{$1} != #{$2})", false when />> \d\d\d/, /<< (GROUP|AUTHINFO)/ --- 151,155 ---- when /^doarticle \d+: \d+!=\d+ \|\| \d+!=\d+$/ when /^unequal line count (\d+) should equal (\d+)$/ ! # ldebug "unequal line count (#{$1} != #{$2})", false when />> \d\d\d/, /<< (GROUP|AUTHINFO)/ Index: group.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/group.rb,v retrieving revision 1.46 retrieving revision 1.47 diff -C2 -d -r1.46 -r1.47 *** group.rb 14 Jun 2004 05:31:28 -0000 1.46 --- group.rb 19 Jun 2004 13:27:49 -0000 1.47 *************** *** 167,180 **** debug "Beginning updatengetcache at #{t.strftime('%H:%M:%S')}" ! n = rand(900000)+100000 ! cmdline = "#{$config['nget']} -g #{@_fullname} -cr '^#{n}Dummy' 2>&1" cmdline = "nice -n#{$config['nice_nget_update']} -- #{cmdline}" if $config['nice_nget_update'] != 0 begin ! IO.popen(cmdline).each("\r") { |line| ! debug("progress#{yellow $1} speed#{blue $2} ETA #{$3}", false) if line =~ /Retrieving headers \d+-\d+ : \d+\/\d+\/\d+(\s+\d+%)(\s+\d+B\/s)(.*s)/ } ! rescue ! print "\nInterrupted" end --- 167,192 ---- debug "Beginning updatengetcache at #{t.strftime('%H:%M:%S')}" ! cmdline = "#{$config['nget']} -g #{@_fullname} 2>&1" cmdline = "nice -n#{$config['nice_nget_update']} -- #{cmdline}" if $config['nice_nget_update'] != 0 begin ! line = '' ! IO.popen(cmdline).each_byte { |char| ! char = char.chr ! if char != "\r" and char != "\n" ! line << char ! next ! end ! if line =~ /^Retrieving headers \d+-\d+ : \d+\/\d+\/\d+(\s+\d+%)\s+(\d+B\/s)(.*s)/ ! if $1 == ' 100%' ! debug("finished, saving cache..", false) ! else ! debug("#{$2} - #{$1} (#{$3} left)", false) ! end ! end ! line = '' } ! rescue Object=>e ! print "\nInterrupted: #{e.inspect}" end *************** *** 187,191 **** nb_rows = Db.dbh.do "delete from `articles` where `groupindex`='#{@_groupindex}'" ! debug "#{nb_rows} articles deleted" cache = NgetCache.new(@_fullname, $config['cachedir']) --- 199,203 ---- nb_rows = Db.dbh.do "delete from `articles` where `groupindex`='#{@_groupindex}'" ! debug "#{nb_rows} files deleted" cache = NgetCache.new(@_fullname, $config['cachedir']) *************** *** 217,221 **** @_cache_low,@_cache_high = cache.servers[0].low, cache.servers[0].high nb_rows = Db.dbh.select_one("select count(*) from `articles` where `groupindex`='#{@_groupindex}'")[0] ! debug "#{nb_rows} articles inserted in #{Utils.cooltime(Time.now-t)}" end --- 229,233 ---- @_cache_low,@_cache_high = cache.servers[0].low, cache.servers[0].high nb_rows = Db.dbh.select_one("select count(*) from `articles` where `groupindex`='#{@_groupindex}'")[0] ! debug "#{nb_rows} files inserted in #{Utils.cooltime(Time.now-t)}" end *************** *** 224,230 **** debug "Beginning fetchnfos at #{t.strftime('%H:%M:%S')}" - nb_rows = Db.dbh.do "delete from `nfos` where `groupindex`='#{@_groupindex}'" - debug "#{nb_rows} nfos deleted" - tempdir = $config['tempdir'] + File::Separator + "ngetupdate.#{@_shortname}.#{Process.pid}" if FileTest.exists? tempdir --- 236,239 ---- *************** *** 232,235 **** --- 241,247 ---- return end + nb_rows = Db.dbh.do "delete from `nfos` where `groupindex`='#{@_groupindex}'" + debug "#{nb_rows} nfos deleted" + Dir.mkdir tempdir pwd = Dir.pwd *************** *** 276,285 **** --- 288,308 ---- debug "#{nb_rows} releases deleted" + total = Db.dbh.select_one "select count(*) from `articles` where `groupindex`='#{@_groupindex}'" articles = ArticleList.new("where `groupindex`='#{@_groupindex}' order by `subject`") + + cur = 0 + threadaff = Thread.new { + loop do + sleep 0.4 + debug("#{cur}/#{total}", false) + end + } nb_releases = Release.mkreleases(true) articles.each { |art| + cur += 1 nb_releases += Release.mkreleases(art) } nb_releases += Release.mkreleases(false) + threadaff.kill debug "#{nb_releases} releases inserted in #{Utils.cooltime(Time.now-t)}" Index: core.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/core.rb,v retrieving revision 1.40 retrieving revision 1.41 diff -C2 -d -r1.40 -r1.41 *** core.rb 14 Jun 2004 05:31:28 -0000 1.40 --- core.rb 19 Jun 2004 13:27:49 -0000 1.41 *************** *** 104,109 **** end ! def show_nfo(nb_nfo, groupid = '') ! puts Nfo.get(Integer(nb_nfo), groupid) end --- 104,109 ---- end ! def show_nfo(groupid, nb_nfo) ! puts Nfo.get(groupid, nb_nfo.to_i) end Index: nfo.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/nfo.rb,v retrieving revision 1.6 retrieving revision 1.7 diff -C2 -d -r1.6 -r1.7 *** nfo.rb 23 May 2004 07:28:48 -0000 1.6 --- nfo.rb 19 Jun 2004 13:27:49 -0000 1.7 *************** *** 10,22 **** Db.autotable(Nfo, 'nfos') ! def Nfo.get(n, groupid = '') ! req = 'select nfos.artindex,nfos.groupindex from nfos,articles where nfos.artindex=articles.artindex and nfos.groupindex=articles.groupindex' ! ! if (groupid != '') ! groupindex = Group.getindex(groupid) ! req += " and nfos.groupindex='#{groupindex}'" end ! ! req += " order by articles.time asc limit #{n},1" row = Db.dbh.select_one req --- 10,20 ---- Db.autotable(Nfo, 'nfos') ! def Nfo.get(groupid, n) ! groupindex = Group.getindex(groupid) ! if (n > 1000) ! return Nfo.new(n, groupindex, true) end ! ! req = "select nfos.artindex,nfos.groupindex from nfos,articles where nfos.artindex=articles.artindex and nfos.groupindex=articles.groupindex and nfos.groupindex='#{groupindex}' order by articles.time asc limit #{n},1" row = Db.dbh.select_one req Index: queue.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/queue.rb,v retrieving revision 1.26 retrieving revision 1.27 diff -C2 -d -r1.26 -r1.27 *** queue.rb 30 Apr 2004 12:15:58 -0000 1.26 --- queue.rb 19 Jun 2004 13:27:49 -0000 1.27 *************** *** 87,91 **** fs= status ? "`status`='#{status}'" : nil # DEPLIAGE ! filter = ([fg, fs].compact.join(' and ').map{|f| 'where '+f}[0] or '') QueueList.new(filter).each { |queue| puts queue --- 87,91 ---- fs= status ? "`status`='#{status}'" : nil # DEPLIAGE ! filter = ([fg, fs].compact.join(' and ').map{|f| 'where '+f}[0] or '') + ' order by queueindex' QueueList.new(filter).each { |queue| puts queue Index: ngetcache.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/ngetcache.rb,v retrieving revision 1.10 retrieving revision 1.11 diff -C2 -d -r1.10 -r1.11 *** ngetcache.rb 21 May 2004 08:42:20 -0000 1.10 --- ngetcache.rb 19 Jun 2004 13:27:49 -0000 1.11 *************** *** 143,148 **** return unless readline ! raise BadNgetCache.new('invalid part description') unless (@linesplit = @line.split("\t")).length == 3 ! part = CachePart.new(@linesplit[0], @linesplit[1], @linesplit[2], Array.new) # articles giving this part --- 143,149 ---- return unless readline ! raise BadNgetCache.new('invalid part description') unless @line =~ /^(-?\d+)\t(\d+)\t([^\t]*)$/ ! ! part = CachePart.new($1, $2, $3, Array.new) # articles giving this part Index: articles.rb =================================================================== RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/articles.rb,v retrieving revision 1.10 retrieving revision 1.11 diff -C2 -d -r1.10 -r1.11 *** articles.rb 14 Jun 2004 05:31:28 -0000 1.10 --- articles.rb 19 Jun 2004 13:27:49 -0000 1.11 *************** *** 48,52 **** elsif subject =~ /[^\d](\d{1,3})\/(\d{1,3})[^\d]/ return $1.to_i, $2.to_i, $2.length ! elsif subject =~ /(\d{1,3}) *of *(\d{1,3})/ return $1.to_i, $2.to_i, $2.length elsif subject =~ /(\d{1,3}) *sur *(\d{1,3})/ --- 48,52 ---- elsif subject =~ /[^\d](\d{1,3})\/(\d{1,3})[^\d]/ return $1.to_i, $2.to_i, $2.length ! elsif subject =~ /(\d{1,3}) *of? *(\d{1,3})/ return $1.to_i, $2.to_i, $2.length elsif subject =~ /(\d{1,3}) *sur *(\d{1,3})/ *************** *** 207,211 **** def Release.list(groupid = '') ! req = 'select articles.subject,articles.groupindex,releases.rlsmap,releases.nbactual,releases.nbexpected from releases,articles where articles.artindex = releases.artindex and articles.groupindex = releases.groupindex' --- 207,211 ---- def Release.list(groupid = '') ! req = 'select articles.subject,releases.rlsmap,releases.nbactual,releases.nbexpected,releases.artindex,releases.groupindex from releases,articles where articles.artindex = releases.artindex and articles.groupindex = releases.groupindex' *************** *** 215,228 **** end ! req += ' order by articles.time asc' sth = Db.dbh.execute req i = 0 ! subj, grpidx, rlsmap, nbact, nbtot = nil ! #rls = nil sth.fetch do |row| ! subj, grpidx, rlsmap, nbact, nbtot = *row ! puts "[#{Utils.blue i}] #{Utils.yellow nbact}/#{Utils.yellow nbtot} [#{Utils.green rlsmap if rlsmap}] \"#{subj}\"" i += 1 end --- 215,228 ---- end ! req += ' order by releases.time asc' sth = Db.dbh.execute req i = 0 ! subj, rlsmap, nbact, nbtot, rlsidx, grpidx, nfo_list = nil sth.fetch do |row| ! subj, rlsmap, nbact, nbtot, rlsidx, grpidx = *row ! nfo_list = Db.dbh.select_one("select nfos.artindex from nfos,articles where articles.groupindex='#{grpidx}' and nfos.groupindex='#{grpidx}' and nfos.artindex=articles.artindex and articles.rlsartindex='#{rlsidx}'") ! puts "[#{Utils.blue i}] #{Utils.yellow nbact}/#{Utils.yellow nbtot} [#{Utils.green rlsmap if rlsmap}] \"#{subj}\"#{' nfo n°'+Utils.red(nfo_list[0]) if nfo_list}" i += 1 end |
