[Ngetsuite-cvs] ngetsuite/ngetsuite articles.rb,1.5,1.6 core.rb,1.33,1.34 db.rb,1.23,1.24 downloader.rb,1.22,1.23 downloadmanager.rb,1.22,1.23 group.rb,1.36,1.37 nfo.rb,1.4,1.5 ngetcache.rb,1.7,1.8 queue.rb,1.23,1.24 queue_file.rb,1.15,1.16 utils.rb,1.12,1.13 web.rb,1.5,1.6

[<jj...@us...>]

Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv20763/ngetsuite

Modified Files:
	articles.rb core.rb db.rb downloader.rb downloadmanager.rb 
	group.rb nfo.rb ngetcache.rb queue.rb queue_file.rb utils.rb 
	web.rb 
Log Message:
db structure adaptation


Index: articles.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/articles.rb,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** articles.rb	17 Feb 2004 14:56:31 -0000	1.5
--- articles.rb	20 Feb 2004 16:44:39 -0000	1.6
***************
*** 16,23 ****
  	Db.autotable(Article, 'articles')
  
! 	def initialize(group, mid, fetch_from_db = false, nbparts = 0, rlsnb = 0,
! 			size = 0, subject = nil, from = nil, time = nil, release = nil)
! 		@_mid = mid
! 		@_group = group
  		@_nbparts = nbparts
  		@_size = size
--- 16,22 ----
  	Db.autotable(Article, 'articles')
  
! 	def initialize(artindex, groupindex, fetch_from_db = false, nbparts = 0, rlsnb = 0, size = 0, subject = nil, from = nil, time = nil, rlsartindex = nil)
! 		@_artindex = artindex
! 		@_groupindex = groupindex
  		@_nbparts = nbparts
  		@_size = size
***************
*** 25,29 ****
  		@_from = from
  		@_time = time
! 		@_release = release
  		update_from_db if fetch_from_db
  	end
--- 24,28 ----
  		@_from = from
  		@_time = time
! 		@_rlsartindex = rlsartindex
  		update_from_db if fetch_from_db
  	end
***************
*** 40,52 ****
  	Db.autotable(Release, 'releases')
  	
! 	def initialize(group, mid, fetch_from_db = false )
  		@files = Array.new
! 		@_mid = mid
! 		@_group = group
  		@_size = 0
  		@_nbexpected = 0
  		@_nbactual = 0
  		@lastnb = 0
! 		@_time = Time.gm("1980")
  		update_from_db if fetch_from_db
  	end
--- 39,51 ----
  	Db.autotable(Release, 'releases')
  	
! 	def initialize(groupindex, artindex, fetch_from_db = false )
  		@files = Array.new
! 		@_artindex = artindex
! 		@_groupindex = groupindex
  		@_size = 0
  		@_nbexpected = 0
  		@_nbactual = 0
  		@lastnb = 0
! 		@_time = Time.gm('1980')
  		update_from_db if fetch_from_db
  	end
***************
*** 54,58 ****
  	def pushart(art, pos)
  		art._rlsnb = pos
! 		art._release = @_mid
  		@_size += art._size
  		@_nbactual += 1
--- 53,57 ----
  	def pushart(art, pos)
  		art._rlsnb = pos
! 		art._rlsartindex = @_artindex
  		@_size += art._size
  		@_nbactual += 1
***************
*** 164,172 ****
  
  	def Release.list(groupid = '')
! 		req = 'select articles.subject,articles.group,releases.rlsmap,releases.nbactual,releases.nbexpected from releases,articles where articles.mid = releases.mid'
  		
  		if (groupid != '')
! 			name = Group.getname(groupid)
! 			req += " and articles.group = '#{name}'"
  		end
  
--- 163,172 ----
  
  	def Release.list(groupid = '')
! 		req = 'select articles.subject,articles.groupindex,releases.rlsmap,releases.nbactual,releases.nbexpected
! 		from releases,articles where articles.artindex = releases.artindex and articles.groupindex = releases.groupindex'
  		
  		if (groupid != '')
! 			groupindex = Group.getindex(groupid)
! 			req += " and releases.groupindex='#{groupindex}'"
  		end
  
***************
*** 176,197 ****
  
  		i = 0
! 		subj, grp, rlsmap, nbact, nbtot = nil
  		#rls = nil
  		sth.fetch do |row|
! 			#rls = Release.new(name, row[2].to_s, true)
! 			#puts "[#{i}] #{rls} (#{row[0]})"
! 			subj, grp, rlsmap, nbact, nbtot = *row
  			puts "[#{Utils.blue i}] #{Utils.yellow nbact}/#{Utils.yellow nbtot} [#{Utils.green rlsmap}] \"#{subj}\""
  			i += 1
  		end
  		sth.finish
! 		puts "No release found." if i == 0
  	end	
  	
  	def to_s
! 		"#{Utils.green @_mid}: #{@_nbactual}/#{@_nbexpected} [#{Utils.yellow @_rlsmap}]"
  	end
- 	
  end
- 
  end
--- 176,193 ----
  
  		i = 0
! 		subj, grpidx, rlsmap, nbact, nbtot = nil
  		#rls = nil
  		sth.fetch do |row|
! 			subj, grpidx, rlsmap, nbact, nbtot = *row
  			puts "[#{Utils.blue i}] #{Utils.yellow nbact}/#{Utils.yellow nbtot} [#{Utils.green rlsmap}] \"#{subj}\""
  			i += 1
  		end
  		sth.finish
! 		puts 'No release found.' if i == 0
  	end	
  	
  	def to_s
! 		"#{Utils.green @_artindex}: #{@_nbactual}/#{@_nbexpected} [#{Utils.yellow @_rlsmap}]"
  	end
  end
  end

Index: core.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/core.rb,v
retrieving revision 1.33
retrieving revision 1.34
diff -C2 -d -r1.33 -r1.34
*** core.rb	9 Feb 2004 21:34:49 -0000	1.33
--- core.rb	20 Feb 2004 16:44:39 -0000	1.34
***************
*** 63,88 ****
  	end
  	
! 	def search_group(groupid, filter)	
! 		g = Group.getgroup(groupid)
! 		g.search(filter)
  	end
  
! 	def search_releases(groupid, filter)	
! 		g = Group.getgroup(groupid)
! 		g.search_releases(filter)
  	end
  
  	def add_group(groupname, shortname = nil, dir = nil, desc = nil)
! 		g = Group.new(groupname, false, shortname, dir, true, desc)
  		g.insert_to_db
! 		begin
! 			Dir.mkdir(g._directory)
! 		rescue Errno::EEXIST => e
! 			debug g._directory + ' already exists'
! 		end
  	end
  
! 	def delete_group(groupid)
! 		Group.delete(groupid)
  	end
  
--- 63,82 ----
  	end
  	
! 	def search_group(groupindex, filter)	
! 		Group.getgroup(groupindex).search(filter)
  	end
  
! 	def search_releases(groupindex, filter)	
! 		Group.getgroup(groupindex).search_releases(filter)
  	end
  
  	def add_group(groupname, shortname = nil, dir = nil, desc = nil)
! 		g = Group.new(nil, false, groupname, shortname, dir, true, desc)
  		g.insert_to_db
! 		Dir.mkdir(g._directory) unless FileTest.exists? g._directory
  	end
  
! 	def delete_group(id)
! 		Group.delete(Groupe.getindex(id))
  	end
  
***************
*** 91,102 ****
  			puts "Update all groups"
  			Group.getallactive.each do |group|
! 				name = group.to_s
! 				puts "Updating #{name}"
! 				g = Group.new(name,true)
  				g.update
  			end
  		else
! 			g = Group.getgroup(groupid)
! 			g.update
  		end
  	end
--- 85,94 ----
  			puts "Update all groups"
  			Group.getallactive.each do |group|
! 				g = Group.new(group[0], true)
! 				puts "Updating #{g._fullname}"
  				g.update
  			end
  		else
! 			Group.getgroup(groupid).update
  		end
  	end
***************
*** 107,112 ****
  
  	def show_nfo(nb_nfo, groupid = '')
! 		n = Integer(nb_nfo)
! 		puts Nfo.get(n, groupid)
  	end
  
--- 99,103 ----
  
  	def show_nfo(nb_nfo, groupid = '')
! 		puts Nfo.get(Integer(nb_nfo), groupid)
  	end
  
***************
*** 115,119 ****
  	end
  	
! 	def queue_article(groupid, article_id, prio = '5')
  		begin
  			n = Integer(prio)
--- 106,110 ----
  	end
  	
! 	def queue_article(groupid, articleindex, prio = '5')
  		begin
  			n = Integer(prio)
***************
*** 130,137 ****
  			n = 5
  		end
! 		
! 		name = Group.getname(groupid)
! 		g = Group.new(name)
! 		g.queue_regexp(regexp, n, period, subpath)
  	end
  
--- 121,125 ----
  			n = 5
  		end
! 		Group.getgroup(groupid).queue_regexp(regexp, n, period, subpath)
  	end
  
***************
*** 145,154 ****
  
  	def show_status
! 		m = get_manager
! 		puts m.status
  	end
  
! 	def set_period(queueid, period)
! 		q = Queue.getqueue(queueid)
  		q._period = period
  		q.sync_to_db
--- 133,141 ----
  
  	def show_status
! 		puts get_manager.status
  	end
  
! 	def set_period(queueindex, newperiod)
! 		q = Queue.getqueue(queueindex)
  		q._period = period
  		q.sync_to_db
***************
*** 197,201 ****
  					$config['debug']=true
  				when 'no-debug'
! 					$config['no-debug']=false
  				when 'write-config'
  					$config.save(arg)
--- 184,188 ----
  					$config['debug']=true
  				when 'no-debug'
! 					$config['debug']=false
  				when 'write-config'
  					$config.save(arg)

Index: db.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/db.rb,v
retrieving revision 1.23
retrieving revision 1.24
diff -C2 -d -r1.23 -r1.24
*** db.rb	6 Feb 2004 23:46:59 -0000	1.23
--- db.rb	20 Feb 2004 16:44:39 -0000	1.24
***************
*** 98,101 ****
--- 98,102 ----
  	end
  
+ 	# The class having a list must have a constructor whose required arguments are the PRImary keys *in the same order*
  	class DbList
  		include Db
***************
*** 138,141 ****
--- 139,147 ----
  		return @@db_handler
  	end
+ 	
+ 	def Db.now
+ 		#Db.dbh.select_one('select NOW()')[0]
+ 		return Time.now
+ 	end
  
  	## Database registry

Index: downloader.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloader.rb,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -d -r1.22 -r1.23
*** downloader.rb	6 Feb 2004 23:46:59 -0000	1.22
--- downloader.rb	20 Feb 2004 16:44:39 -0000	1.23
***************
*** 9,17 ****
  	include Utils
  	
! 	# the id as defined in the database
! 	attr_reader :id
  	attr_reader :start_time
  	# the status
  	attr_reader :status
  	# single or regexp
  	attr_reader :type	
--- 9,18 ----
  	include Utils
  	
! 	# the queueindex as defined in the database
! 	attr_reader :queueindex
  	attr_reader :start_time
  	# the status
  	attr_reader :status
+ 	attr_accessor :finish_status
  	# single or regexp
  	attr_reader :type	
***************
*** 29,34 ****
  	attr_reader :nbfiles
  	
! 	def initialize(id, type, value, group, dest, prio)
! 		@id = id
  		@type = type
  		@value = value
--- 30,35 ----
  	attr_reader :nbfiles
  	
! 	def initialize(queueindex, type, value, group, dest, prio)
! 		@queueindex = queueindex
  		@type = type
  		@value = value
***************
*** 42,46 ****
  
  	def ldebug(str)
! 		debug "(#{white @id})" + str
  	end
  	
--- 43,47 ----
  
  	def ldebug(str)
! 		debug "(#{white @queueindex})" + str
  	end
  	
***************
*** 53,79 ****
  		when /^Retrieving: (\d+).*\t(\d+)l.*(<.*>)$/
  			parts, nblines, mid = $1, $2, $3
  			@nbfiles += 1
! 			Db.dbh.do "update queue set `nbfiles`='#{@nbfiles}' where `id`='#{@id}'"
  
! 			@curfile = QueueFile.new(@id, mid, false, parts, nblines, @nbfiles)
! 			if not QueueFile.already_in_db(@id, $3)
  				@curfile.insert_to_db
  			else
  				@curfile.update_from_db
! 				ldebug (Utils.red "already in database")
  			end
  		when /^(\d+) \((\d+)\/(\d+)\): (\d+)\/(\d+)L (\d+)\/(\d+)B (\d+)% (\d+)B\/s (\w+) (\d+)\/(\d+) (\w+)$/
! 			part = $2
! 			size = $7
! 			speed = $9
! 			filenb = $12
  			ldebug "#{yellow $5+'L'}(#{yellow $7+'B'}) at #{red $9+'B/s'} [#{blue $2}/#{blue $3}] [#{green $11}/#{green $12}]"
  			@curfile.update(part, speed, filenb, size)
  		when /\d+ \((\d+)\/\d+\).*\d+\/(\d+)B.* (\d+)B\/s \w+ (\d+)\/(\d+) \w+$/
! 			ldebug (Utils.red "the previous regexp should have been matched instead of this one")
! 			part = $1
! 			size = $2
! 			speed = $3
! 			filenb = $4
  			@curfile.update(part, speed, filenb, size)
  		when /^autopar.*seems like a (par2?) \((.*)\)/
--- 54,81 ----
  		when /^Retrieving: (\d+).*\t(\d+)l.*(<.*>)$/
  			parts, nblines, mid = $1, $2, $3
+ 			artindex = Db.dbh.select_one("select `artindex` from `articles` where `mid`='#{mid}'")
+ 			unless artindex
+ 				puts "No such article: #{mid}"
+ 				artindex = [nil]
+ 			end
+ 			artindex = artindex[0]
+ 			
  			@nbfiles += 1
! 			Db.dbh.do "update `queue` set `nbfiles`='#{@nbfiles}' where `queueindex`='#{@queueindex}'"
  
! 			@curfile = QueueFile.new(@queueindex, artindex, false, parts, nblines, @nbfiles)
! 			if not QueueFile.already_in_db(@queueindex, artindex)
  				@curfile.insert_to_db
  			else
  				@curfile.update_from_db
! 				ldebug (red "Article already in database")
  			end
  		when /^(\d+) \((\d+)\/(\d+)\): (\d+)\/(\d+)L (\d+)\/(\d+)B (\d+)% (\d+)B\/s (\w+) (\d+)\/(\d+) (\w+)$/
! 			part, size, speed, filenb = $2, $7, $9, $12
  			ldebug "#{yellow $5+'L'}(#{yellow $7+'B'}) at #{red $9+'B/s'} [#{blue $2}/#{blue $3}] [#{green $11}/#{green $12}]"
  			@curfile.update(part, speed, filenb, size)
  		when /\d+ \((\d+)\/\d+\).*\d+\/(\d+)B.* (\d+)B\/s \w+ (\d+)\/(\d+) \w+$/
! 			ldebug (red "ERR the previous regexp should have been matched instead of this one")
! 			part, size, speed, filenb = $1, $2, $3, $4
  			@curfile.update(part, speed, filenb, size)
  		when /^autopar.*seems like a (par2?) \((.*)\)/
***************
*** 93,97 ****
  			ldebug 'file decoded'
  			@curfile.finish(true)
! 			@finish_status = 'FINISHED'
  			log "Download finished for file #{@curfile._filename}, queue #{@value}"
  		when /^Connecting to (.*)$/
--- 95,99 ----
  			ldebug 'file decoded'
  			@curfile.finish(true)
! 			@finish_status = 'FINISHED' 
  			log "Download finished for file #{@curfile._filename}, queue #{@value}"
  		when /^Connecting to (.*)$/
***************
*** 103,106 ****
--- 105,111 ----
  		when /<< QUIT/
  			ldebug 'Quitted'
+ 		when /^doarticle \d+: \d+!=\d+ \|\| \d+!=\d+$/
+ 		when /^unequal line count \d+ should equal \d+$/
+ 			ldebug 'unequal line count'
  		when />> \d\d\d/, /<< (GROUP|AUTHINFO)/
  			# nothing
***************
*** 109,120 ****
  		when /^couldn't get (<.*>) from anywhere$/
  			ldebug red("Couldn't download mid ") + green($1)
! 		when /^ERRORS: (.*)$/
! 			ldebug red("Finished, got errors (#{$1})")
! 			@finish_status = 'FAILED'
  		else
! 		#TODO : handle decoding failures (=> @curfile.finish(false))
! 			ldebug "unhandled msg : #{blue line}"
  		end
- 		#debug (Utils.red "end update_status")
  	end
  
--- 114,125 ----
  		when /^couldn't get (<.*>) from anywhere$/
  			ldebug red("Couldn't download mid ") + green($1)
! 		when /^(?:OK:(.*))?(?:WARNINGS:(.*))?(?:ERRORS:(.*))?$/
! 			return unless $1 or $2 or $3
! 			ok,warn,errs = $1, $2, $3
! 			@finish_status = (errs ? 'FAILED' : 'FINISHED')
! 			ldebug "Finished: ok#{green ok} warnings#{blue warn} errors#{red errs}"
  		else
! 			ldebug blue('unhandled msg: ') + line
  		end
  	end
  
***************
*** 122,132 ****
  		File.makedirs(@dest)
  		Dir.chdir(@dest)
  		@start_time = Time.new
  		@status = 'PROCESSING'
  		Db.dbh.do 'lock tables queue write'
! 		Db.dbh.do "update queue set `status` = 'PROCESSING', `date_started`= NOW()  where `id` = '#{@id}'"
  		Db.dbh.do 'unlock tables'
! 		@io = IO.popen("#{$nget} -q -G #{@group} -r '#{@value}'")
! 		@t = Thread.new(@io) { |io|
  			begin
  				io.each { |line|
--- 127,138 ----
  		File.makedirs(@dest)
  		Dir.chdir(@dest)
+ 		# unused
  		@start_time = Time.new
  		@status = 'PROCESSING'
  		Db.dbh.do 'lock tables queue write'
! 		Db.dbh.do "update queue set `status`='PROCESSING', `date_started`=NOW() where `queueindex`='#{@queueindex}'"
  		Db.dbh.do 'unlock tables'
! 		@io = IO.popen("#{$config['nget']} -q -I -G #{@group} -r '#{@value}' 2>&1")
! 		@t = Thread.new(@io, self){ |io, parent|
  			begin
  				io.each { |line|
***************
*** 135,149 ****
  			rescue IOError
  				puts 'nget got killed!'
! 				
! 			rescue Exception => e
! 				puts "Exception : #{e.class} => " + e.message
! 				print e.backtrace.join("\n")
! 			rescue Error => e
! 				puts 'Error : ' + e
! 				print e.backtrace.join("\n")
  			rescue DBI::DatabaseError => e
! 				puts 'An error occurred'
  				puts 'Error code: ' + e.err
  				puts 'Error message: ' + e.errstr
  			end
  		}
--- 141,152 ----
  			rescue IOError
  				puts 'nget got killed!'
! 				parent.stop
  			rescue DBI::DatabaseError => e
! 				puts 'A DB error occurred'
  				puts 'Error code: ' + e.err
  				puts 'Error message: ' + e.errstr
+ 				parent.stop
+ 			rescue Object
+ 				parent.stop
  			end
  		}
***************
*** 156,164 ****
  	end
  
! 	def stop
  		# oh no they killed kenny !
  		ldebug red('dlder.stop')
  		Db.dbh.do 'lock tables queue write'
! 		Db.dbh.do "update `queue` set `status` = 'STOPPED', `date_finished`=NOW() where `id` = '#{@id}'"
  		Db.dbh.do 'unlock tables'
  		
--- 159,170 ----
  	end
  
! 	def stop(command=false)
  		# oh no they killed kenny !
  		ldebug red('dlder.stop')
+ 		@finish_status = command ? 'STOPPED' : 'FAILED'
+ 		newstat = "`status`='#{finish_status}',`date_finished`=NOW()"
+ 		newstat += ",`period`='0'" if command
  		Db.dbh.do 'lock tables queue write'
! 		Db.dbh.do "update `queue` set #{newstat} where `queueindex`='#{@queueindex}'"
  		Db.dbh.do 'unlock tables'
  		
***************
*** 181,185 ****
  #		newstatus = success ? 'FINISHED' : 'FAILED'
  		newstatus = @finish_status
! 		Db.dbh.do "update queue set `status` = '#{newstatus}', `date_finished`=NOW() where `id` = '#{@id}'"
  		Db.dbh.do 'unlock tables'
  		ldebug "Download #{newstatus.downcase} for #{green @value}"		
--- 187,191 ----
  #		newstatus = success ? 'FINISHED' : 'FAILED'
  		newstatus = @finish_status
! 		Db.dbh.do "update queue set `status`='#{newstatus}', `date_finished`=NOW() where `queueindex`='#{@queueindex}'"
  		Db.dbh.do 'unlock tables'
  		ldebug "Download #{newstatus.downcase} for #{green @value}"		
***************
*** 188,192 ****
  
  	def status
! 		"[#{@id}] -- #{@value} (#{@status})"
  	end
  end
--- 194,198 ----
  
  	def status
! 		"[#{@queueindex}] -- #{@value} (#{@status})"
  	end
  end

Index: downloadmanager.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloadmanager.rb,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -d -r1.22 -r1.23
*** downloadmanager.rb	9 Feb 2004 21:34:49 -0000	1.22
--- downloadmanager.rb	20 Feb 2004 16:44:39 -0000	1.23
***************
*** 1,4 ****
--- 1,5 ----
  require 'ngetsuite/db'
  require 'ngetsuite/downloader'
+ require 'ngetsuite/group'
  require 'ngetsuite/utils'
  
***************
*** 59,73 ****
  		
  	def start_new_downloader
! 		row = Db.dbh.select_one("select * from queue where `status`='PENDING' order by `priority` desc limit 1")
! 		row = Db.dbh.select_one("select * from queue where (`status`='STOPPED' and (`period` != 0) and ((NOW()-`date_finished`) > `period`)) order by `priority` desc limit 1") if not row and $config['resume_downloads']
! 		row = Db.dbh.select_one("select * from queue where (`status`='FAILED' and (`period` != 0) and ((NOW()-`date_finished`) > `period`)) order by `priority` desc limit 1") if not row and $config['resume_downloads']
  		
  		return false if row == nil
  
  		@current_dlers += 1
! 		id, group, type, value, subdir, prio = row.values_at(0, 1, 2, 3, 4, 5)
  		log "found /#{value}/ for group #{group}\n"
! 		dler = Downloader.new(id, type, value, group, @basedir + File::Separator + subdir, prio)
! 		@dlers[id] = dler
  		dler.start
  		
--- 60,76 ----
  		
  	def start_new_downloader
! 		row = Db.dbh.select_one("select * from `queue` where `status`='PENDING' order by `priority` desc limit 1")
! 		row = Db.dbh.select_one("select * from `queue` where (`status`='FINISHED' and (`period` != 0) and ((NOW()-`date_finished`) > `period`)) order by `priority` desc limit 1") if not row and $config['resume_downloads']
! 		row = Db.dbh.select_one("select * from `queue` where (`status`='STOPPED' and (`period` != 0) and ((NOW()-`date_finished`) > `period`)) order by `priority` desc limit 1") if not row and $config['resume_downloads']
! 		row = Db.dbh.select_one("select * from `queue` where (`status`='FAILED' and ((NOW()-`date_finished`) > '7200')) order by `priority` desc limit 1") if not row and $config['resume_downloads']
  		
  		return false if row == nil
  
  		@current_dlers += 1
! 		queueindex, groupindex, type, value, subdir, prio = row.values_at(0, 1, 2, 3, 4, 5)
! 		group = Group.getname(groupindex)
  		log "found /#{value}/ for group #{group}\n"
! 		dler = Downloader.new(queueindex, type, value, group, @basedir + File::Separator + subdir, prio)
! 		@dlers[queueindex] = dler
  		dler.start
  		
***************
*** 79,83 ****
  		if @dlers.has_key? id
  			dler = @dlers.delete(id)
! 			dler.stop
  			@current_dlers -= 1
  		else
--- 82,86 ----
  		if @dlers.has_key? id
  			dler = @dlers.delete(id)
! 			dler.stop(true)
  			@current_dlers -= 1
  		else
***************
*** 121,125 ****
  			"Maximum of parallel downloads is #{@max_dlers}\n" +
  			"There are currently #{@current_dlers} downloads running\n"
! 			@dlers.each {|id, dler| res += dler.status + '\n'}
  		else
  			res = 'Stopped'
--- 124,128 ----
  			"Maximum of parallel downloads is #{@max_dlers}\n" +
  			"There are currently #{@current_dlers} downloads running\n"
! 			@dlers.each {|id, dler| res += dler.status + "\n"}
  		else
  			res = 'Stopped'

Index: group.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/group.rb,v
retrieving revision 1.36
retrieving revision 1.37
diff -C2 -d -r1.36 -r1.37
*** group.rb	9 Feb 2004 21:34:49 -0000	1.36
--- group.rb	20 Feb 2004 16:44:39 -0000	1.37
***************
*** 7,18 ****
  
  class NoSuchGroupException < Exception
! 	attr_reader :groupid
  
! 	def initialize(groupid)
! 		@groupid = groupid
  	end
  
  	def message
! 		"No such group : #{groupid}"
  	end
  end
--- 7,18 ----
  
  class NoSuchGroupException < Exception
! 	attr_reader :groupindex
  
! 	def initialize(groupindex)
! 		@groupindex = groupindex
  	end
  
  	def message
! 		"No such group : #{groupindex}"
  	end
  end
***************
*** 30,74 ****
  	Db.autotable(Group, 'groups')
  
! 	def Group.delete(id)
! 		Db.dbh.do "delete from `groups` where `fullname` = '#{Group.getname(id)}'"
  	end
  	
! 	def Group.getname(id)
  		begin
! 			n = Integer(id)
! 			groups = Db.dbh.select_all 'select fullname from `groups` order by date_added'
! 			if ((n < groups.size) && (n>=0))
! 				return groups[n].to_s
! 			else
! 				raise NoSuchGroupException.new(id)
! 			end
  		rescue ArgumentError
- 			# id is not an integer, it must be the abbreviated name or the full
- 			# name
- 			groups = Db.dbh.select_all "select fullname from `groups` where `shortname` = '#{id}'"
- 			case groups.size
- 				when 0
- 					# id is not a shortname, maybe it is a fullname
- 					fullnames = Db.dbh.select_all "select fullname from `groups` where `fullname` = '#{id}'"
- 					if (fullnames.size == 1)
- 						# Got it
- 						return id
- 					else
- 						raise NoSuchGroupException.new(id)
- 					end
- 				when 1
- 					return groups.to_s
- 				else
- 					raise NoSuchGroupException.new(id)
- 			end
  		end
  	end
  
  	def Group.getallactive
! 		Db.dbh.select_all "select `fullname` from `groups` where active='yes'"
  	end
  	
  	def Group.getgroup(id)
! 		Group.new(Group.getname(id), true)
  	end
  
--- 30,68 ----
  	Db.autotable(Group, 'groups')
  
! 	def Group.delete(index)
! 		Db.dbh.do "delete from `groups` where `groupindex`='#{index}'"
  	end
  	
! 	def Group.getindex(id)
! 		id = id.to_s
  		begin
! 			index = Integer(id)
! 			groups = Db.dbh.select_one("select `groupindex` from `groups` where `groupindex`='#{index}'")
! 			return groups[0] if groups and groups.length == 1
  		rescue ArgumentError
  		end
+ 			
+ 		# id is not an integer, it must be a shortname
+ 		groups = Db.dbh.select_all("select `groupindex` from `groups` where `shortname`='#{id}'")
+ 		return groups[0] if groups and groups.length == 1
+ 		
+ 		# or a fullname
+ 		groups = Db.dbh.select_all("select `groupindex` from `groups` where `fullname`='#{id}'")
+ 		return groups[0] if groups and groups.length == 1
+ 		
+ 		# none of these => error
+ 		raise NoSuchGroupException.new(id)
+ 	end
+ 	
+ 	def Group.getname(id)
+ 		Db.dbh.select_one("select `fullname` from `groups` where `groupindex`='#{Group.getindex(id)}'")[0]
  	end
  
  	def Group.getallactive
! 		Db.dbh.select_all "select `groupindex` from `groups` where `active`='yes'"
  	end
  	
  	def Group.getgroup(id)
! 		Group.new(Group.getindex(id), true)
  	end
  
***************
*** 78,88 ****
  		
  		req = ''
! 		req = " where `active` = 'yes' " if onlyactive
  		req += 'order by `date_added`'
  		
  		GroupList.new(req).each { |group|
  			name = Utils.red group._fullname
  			sname = Utils.blue group._shortname
! 			puts "[#{name}] (#{sname})"
  			labels.each_index { |i|
  				label, value = Utils.green(labels[i]),
--- 72,83 ----
  		
  		req = ''
! 		req = " where `active`='yes' " if onlyactive
  		req += 'order by `date_added`'
  		
  		GroupList.new(req).each { |group|
+ 			idx = Utils.green group._groupindex
  			name = Utils.red group._fullname
  			sname = Utils.blue group._shortname
! 			puts "<#{idx}> [#{name}] (#{sname})"
  			labels.each_index { |i|
  				label, value = Utils.green(labels[i]),
***************
*** 94,98 ****
  	end
  
! 	def initialize(fullname, fetch_from_db = false, shortname = nil, directory = nil, active = true, description = nil)
  		@_fullname = fullname
  		@_shortname = shortname
--- 89,94 ----
  	end
  
! 	def initialize(index=nil, fetch_from_db=false, fullname=nil, shortname=nil, directory=nil, active=true, description=nil)
! 		@_groupindex = index
  		@_fullname = fullname
  		@_shortname = shortname
***************
*** 101,107 ****
  		@_description = description
  		@_date_added = Time.now
- 		@nget = $config['nget']
- 		@yydecode = $config['yydecode']
- 		@cachedir = $config['cachedir']
  		update_from_db if fetch_from_db
  	end
--- 97,100 ----
***************
*** 112,120 ****
  
  	def _shortname
! 		if (@_shortname == nil)
! 			@_shortname = @_fullname.to_s.split('.').map {|item| item[0].chr }.join('')
! 		end
! 
! 		return @_shortname
  	end
  
--- 105,110 ----
  
  	def _shortname
! 		return @_shortname if @_shortname
! 		@_shortname = @_fullname.to_s.split('.').map {|item| item[0].chr }.join('')
  	end
  
***************
*** 128,132 ****
  	
  	def search_releases(filter)
! 		sth = Db.dbh.execute('select articles.subject,releases.rlsmap,releases.nbactual,releases.nbexpected from releases,articles where articles.mid = releases.mid and articles.group = ? and articles.subject RLIKE ? order by articles.time desc', @_fullname, filter)
  		i = 0
  		subj, rlsmap, nbact, nbtot = nil
--- 118,122 ----
  	
  	def search_releases(filter)
! 		sth = Db.dbh.execute('select articles.subject,releases.rlsmap,releases.nbactual,releases.nbexpected from releases,articles where articles.artindex = releases.artindex and articles.groupindex = ? and articles.subject RLIKE ? order by articles.time desc', @_groupindex, filter)
  		i = 0
  		subj, rlsmap, nbact, nbtot = nil
***************
*** 141,147 ****
  
  	def search(filter)
! 		sth = Db.dbh.execute('select * from `articles` where `group` = ? and `subject` RLIKE ? order by `time` desc', @_fullname, filter)
! 		sth.fetch_hash do |row|
! 			puts row['subject']
  		end
  		sth.finish
--- 131,137 ----
  
  	def search(filter)
! 		sth = Db.dbh.execute("select `subject` from `articles` where `groupindex`='#{@_groupindex}' and `subject` RLIKE ? order by `time` desc", filter)
! 		sth.fetch do |row|
! 			puts row[0]
  		end
  		sth.finish
***************
*** 150,185 ****
  	def update
  		updatengetcache unless $config['extern_headers_update'].split(',').include?(@_shortname)
- 		clear_articles
  		parsengetcache
- 
- 		clear_releases
  		recognizereleases
- 		
- 		clear_nfos
  		fetchnfos unless $config['extern_nfos_fetch'].split(',').include?(@_shortname)
- 		
  		@_date_updated = Time.now
  		sync_to_db
  	end
  
! 	def clear_articles
! 		nb_rows = Db.dbh.do "delete from `articles` where `group` = '#{@_fullname}'"
! 		debug "#{nb_rows} articles deleted"
! 	end
! 
! 	def clear_nfos
! 		nb_rows = Db.dbh.do "delete from `nfos` where `group` = '#{@_fullname}'"
! 		debug "#{nb_rows} nfos deleted"
! 	end
! 
! 	def clear_releases
! 		nb_rows = Db.dbh.do "delete from `releases` where `group` = '#{@_fullname}'"
! 		debug "#{nb_rows} releases deleted"
! 
! 	end
! 	
! 	def queue_regexp(regexp, prio, period=0, subpath='')
! 		q = RegexpQueue.new(nil, false, @_fullname, regexp, prio, period, subpath)
! 		q._date_added = Time.now
  		q._status = NgetSuite::PENDING
  		q.insert_to_db
--- 140,153 ----
  	def update
  		updatengetcache unless $config['extern_headers_update'].split(',').include?(@_shortname)
  		parsengetcache
  		recognizereleases
  		fetchnfos unless $config['extern_nfos_fetch'].split(',').include?(@_shortname)
  		@_date_updated = Time.now
  		sync_to_db
  	end
  
! 	def queue_regexp(regexp, prio=nil, period=nil, subpath=nil)
! 		q = RegexpQueue.new(nil, false, @_groupindex, regexp, prio, period, subpath)
! 		q._date_added = Db.now
  		q._status = NgetSuite::PENDING
  		q.insert_to_db
***************
*** 189,203 ****
  		t = Time.now
  		debug "Beginning updatengetcache at #{t.strftime('%H:%M:%S')}"
  		n = rand(900000)+100000
  		$stdout.sync = true if $stdout.isatty
  		begin
! 		IO.popen("nice -n19 #{@nget} -g #{@_fullname} -cr '^#{n}Dummy'").each("\r") { |line|
  			print "progress#{red $1} speed#{blue $2} ETA#{green $3}            \r" if line =~ /Retrieving headers \d+-\d+ : \d+\/\d+\/\d+(\s+\d+%)(\s+\d+B\/s)(.*s)/ and $stdout.isatty
! 		}
  		rescue
! 			puts "\nInterrupted"
  		end
  		print "\n" if $stdout.isatty
  		$stdout.sync = false if $stdout.isatty
  		debug "Updatengetcache finished in #{Time.now-t}s"
  	end
--- 157,175 ----
  		t = Time.now
  		debug "Beginning updatengetcache at #{t.strftime('%H:%M:%S')}"
+ 		
  		n = rand(900000)+100000
  		$stdout.sync = true if $stdout.isatty
+ 		
  		begin
! 		IO.popen("nice -n19 #{$config['nget']} -g #{@_fullname} -cr '^#{n}Dummy' 2>&1").each("\r") do |line|
  			print "progress#{red $1} speed#{blue $2} ETA#{green $3}            \r" if line =~ /Retrieving headers \d+-\d+ : \d+\/\d+\/\d+(\s+\d+%)(\s+\d+B\/s)(.*s)/ and $stdout.isatty
! 		end
  		rescue
! 			print "\nInterrupted"
  		end
+ 		
  		print "\n" if $stdout.isatty
  		$stdout.sync = false if $stdout.isatty
+ 		
  		debug "Updatengetcache finished in #{Time.now-t}s"
  	end
***************
*** 206,223 ****
  		t = Time.now
  		debug "Beginning parsengetcache at #{t.strftime('%H:%M:%S')}"
! 		sth = Db.dbh.prepare 'insert into articles (`group`, `mid`, `nbparts`, `size`, `subject`, `from`, `time`) values (?, ?, ?, ?, ?, ?, FROM_UNIXTIME(?))'
  		filecallback = Proc.new { |f|
  			if f.req >= 1
  			partcnt = 0
  			size = 0
  			a = nil
  			f.parts.each { |p|
! 				unless p.articles.empty?
  				a = p.articles[0]
! 				size += a.bytes / ( a.bytes < 100*a.lines ? 1.39 : 1.0334 )
  				partcnt += 1
  				if partcnt >= f.req
! 					sth.execute(@_fullname, p.mid, f.req, size, f.subject, f.author, p.date)
! 				end
  				end
  				# free mem
--- 178,203 ----
  		t = Time.now
  		debug "Beginning parsengetcache at #{t.strftime('%H:%M:%S')}"
! 		
! 		nb_rows = Db.dbh.do "delete from `articles` where `groupindex`='#{@_groupindex}'"
! 		# and `artindex` < '#{@_cache_low}'
! 		debug "#{nb_rows} articles deleted"
! 		
! 		cache = NgetCache.new(@_fullname, $config['cachedir'])
! 
! 		sth = Db.dbh.prepare "insert into articles (`groupindex`, `artindex`, `mid`, `nbparts`, `size`, `subject`, `from`, `time`) values ('#{@_groupindex}', ?, ?, ?, ?, ?, ?, FROM_UNIXTIME(?))"
! 		lasthigh = @_cache_high
  		filecallback = Proc.new { |f|
  			if f.req >= 1
+ 			# and f.parts.articles.artnum.each > lasthigh
  			partcnt = 0
  			size = 0
  			a = nil
  			f.parts.each { |p|
! 				next if p.articles.empty?
  				a = p.articles[0]
! 				size += a.bytes.to_i
  				partcnt += 1
  				if partcnt >= f.req
! 					sth.execute(a.articlenum, p.mid, f.req, size, f.subject, f.author, p.date)
  				end
  				# free mem
***************
*** 227,232 ****
  			end
  		}
! 			
! 		NgetCache.new(@_fullname, @cachedir, nil, filecallback)
  		debug "Parsengetcache done in #{Time.now-t}s"
  	end
--- 207,212 ----
  			end
  		}
! 		cache.parse(filecallback)
! 		@_cache_low,@_cache_high = cache.servers[0].low, cache.servers[0].high
  		debug "Parsengetcache done in #{Time.now-t}s"
  	end
***************
*** 235,238 ****
--- 215,222 ----
  		t = Time.now
  		debug "Beginning fetchnfos at #{t.strftime('%H:%M:%S')}"
+ 		
+ 		nb_rows = Db.dbh.do "delete from `nfos` where `groupindex`='#{@_groupindex}'"
+ 		debug "#{nb_rows} nfos deleted"
+ 		
  		tempdir = $config['tempdir'] + File::Separator + "ngetupdate.#{@_shortname}.#{Process.pid}"
  		if FileTest.exists? tempdir
***************
*** 243,255 ****
  		pwd = Dir.pwd
  		Dir.chdir(tempdir)
  		begin
! 		# fetches the nfos
! 		`nice -n19 #{@nget} --path #{tempdir} -G #{@_fullname} -K -L 150 -r '\\.nfo|\\.txt'`
  		# but mark them for nget as if we didn't download them
! 		`#{@nget} --path #{tempdir} -qq -G #{@_fullname} -U -r '\\.nfo|\\.txt'`
  		debug "Fetchnfos: #{Dir['ngettemp-*'].length} files downloaded in #{Time.now-t}s"
  
  		nb_nfos = 0
! 		sth = Db.dbh.prepare 'insert ignore into nfos (`group`, `mid`, `text`) values (?, ?, ?)'
  		Dir['ngettemp-*'].each { |file|
  			File.open(file).each { |line|
--- 227,240 ----
  		pwd = Dir.pwd
  		Dir.chdir(tempdir)
+ 		
  		begin
! 		# fetches the nfos, w/o decoding
! 		`nice -n19 #{$config['nget']} --path #{tempdir} -G #{@_fullname} -K -L 150 -r '\\.nfo|\\.txt' 2>&1`
  		# but mark them for nget as if we didn't download them
! 		`#{$config['nget']} --path #{tempdir} -qq -G #{@_fullname} -U -r '\\.nfo|\\.txt' 2>&1`
  		debug "Fetchnfos: #{Dir['ngettemp-*'].length} files downloaded in #{Time.now-t}s"
  
  		nb_nfos = 0
! 		sth = Db.dbh.prepare "insert ignore into nfos (`groupindex`, `artindex`, `text`) values (#{@_groupindex}, ?, ?)"
  		Dir['ngettemp-*'].each { |file|
  			File.open(file).each { |line|
***************
*** 257,270 ****
  				if $1 != nil
  					mid = $1
! 					text = `#{@yydecode} #{file} -o -`.chomp
! 					if text and text.length < 15000
! 						sth.execute(@_fullname, mid, text)
  						nb_nfos += 1
  					end
- 					# Does not fucking work
- 					#if text and (text.length < 15000)
- 					#	n = Nfo.new(mid, false, @_fullname, text)
- 					#	n.insert_to_db
- 					#end
  					break
  				end
--- 242,251 ----
  				if $1 != nil
  					mid = $1
! 					text = `#{$config['yydecode']} #{file} -o - 2>/dev/null `.chomp
! 					artindex = Db.dbh.select_one("select `artindex` from `articles` where `mid`='#{mid}'")
! 					if text and text.length < 15000 and text.length > 800 and artindex
! 						sth.execute(artindex[0], text)
  						nb_nfos += 1
  					end
  					break
  				end
***************
*** 276,279 ****
--- 257,261 ----
  			`rm -rf #{tempdir}`
  		end
+ 		
  		debug "Indexed #{nb_nfos} nfos in #{Time.now-t}s"
  	end
***************
*** 282,290 ****
  		t = Time.now
  		debug "Recognizing releases at #{t.strftime('%H:%M:%S')}"
! 		articles = NgetSuite::ArticleList.new("where `group`='#{@_fullname}' order by `subject`")
  		rls = nil
  		nb_releases = 0
  		articles.each { |art|
! 			rls = NgetSuite::Release.new(art._group, art._mid) if !rls
  			if rls.add?(art) == false then
  				# New release! Commit the current one to the db
--- 264,276 ----
  		t = Time.now
  		debug "Recognizing releases at #{t.strftime('%H:%M:%S')}"
! 		
! 		nb_rows = Db.dbh.do "delete from `releases` where `groupindex`='#{@_groupindex}'"
! 		debug "#{nb_rows} releases deleted"
! 	
! 		articles = ArticleList.new("where `groupindex`='#{@_groupindex}' order by `subject`")
  		rls = nil
  		nb_releases = 0
  		articles.each { |art|
! 			rls = Release.new(art._groupindex, art._artindex) if !rls
  			if rls.add?(art) == false then
  				# New release! Commit the current one to the db
***************
*** 293,302 ****
  
  				# And make another
! 				rls = NgetSuite::Release.new(art._group, art._mid)
  				rls.add? art
  			end
  		}
- 
  		rls.insert_to_db
  		debug "Inserted #{nb_releases} releases in #{Time.now-t}s"
  	end
--- 279,288 ----
  
  				# And make another
! 				rls = NgetSuite::Release.new(art._groupindex, art._artindex)
  				rls.add? art
  			end
  		}
  		rls.insert_to_db
+ 
  		debug "Inserted #{nb_releases} releases in #{Time.now-t}s"
  	end

Index: nfo.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/nfo.rb,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** nfo.rb	2 Feb 2004 02:41:57 -0000	1.4
--- nfo.rb	20 Feb 2004 16:44:39 -0000	1.5
***************
*** 11,19 ****
  
  	def Nfo.get(n, groupid = '')
! 		req = 'select articles.mid from nfos,articles where articles.mid = nfos.mid'
  	
  		if (groupid != '')
! 			name = Group.getname(groupid)
! 			req += " and nfos.group = '#{name}'"
  		end
  	
--- 11,19 ----
  
  	def Nfo.get(n, groupid = '')
! 		req = 'select nfos.artindex,nfos.groupindex from nfos,articles where nfos.artindex=articles.artindex and nfos.groupindex=articles.groupindex'
  	
  		if (groupid != '')
! 			name = Group.getindex(groupid)
! 			req += " and nfos.groupindex='#{groupindex}'"
  		end
  	
***************
*** 25,38 ****
  		end
  
! 		return Nfo.new(row[0], true)
  	end
  
  	def Nfo.list(groupid = '')
! 		req = 'select articles.subject,articles.group,articles.mid from
! 			nfos,articles where articles.mid = nfos.mid'
  		
  		if (groupid != '')
! 			name = Group.getname(groupid)
! 			req += " and articles.group = '#{name}'"
  		end
  
--- 25,37 ----
  		end
  
! 		return Nfo.new(row[0], row[1], true)
  	end
  
  	def Nfo.list(groupid = '')
! 		req = 'select articles.subject,nfos.groupindex,nfos.artindex from nfos,articles where articles.artindex=nfos.artindex and articles.groupindex=nfos.groupindex'
  		
  		if (groupid != '')
! 			groupindex = Group.getindex(groupid)
! 			req += " and nfos.groupindex='#{groupindex}'"
  		end
  
***************
*** 43,51 ****
  		i = 0
  		sth.fetch do |row|
! 			puts " mid : #{row[2]}"
! 			nfo = Nfo.new(row[2].to_s, true)
! 			#puts "[#{i}] (#{row[1]}) -- #{row[0]}"
! 			print "[#{i}] "
! 			puts nfo._group
  			i += 1
  		end
--- 42,46 ----
  		i = 0
  		sth.fetch do |row|
! 			puts "[#{i}] (#{puts Group.getgroup(row[1])._shortname}) #{row[0]}"
  			i += 1
  		end
***************
*** 54,60 ****
  	end	
  	
! 	def initialize(mid, fetch_from_db = false, group = nil, text = 'EMPTY')
! 		@_mid = mid
! 		@_group = group
  		@_text = text
  		update_from_db if fetch_from_db
--- 49,55 ----
  	end	
  	
! 	def initialize(artindex, groupindex, fetch_from_db=false, text='EMPTY')
! 		@_artindex = artindex
! 		@_groupindex = groupindex
  		@_text = text
  		update_from_db if fetch_from_db

Index: ngetcache.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/ngetcache.rb,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** ngetcache.rb	6 Feb 2004 23:46:59 -0000	1.7
--- ngetcache.rb	20 Feb 2004 16:44:39 -0000	1.8
***************
*** 12,16 ****
  
  	attr_reader :servers, :files, :sane, :lineno, :groupname
! 	attr_accessor :writeoldformat
  
  	CacheServer = Struct.new('CacheServer', 'serverid', 'high', 'low', 'num')
--- 12,16 ----
  
  	attr_reader :servers, :files, :sane, :lineno, :groupname
! 	attr_accessor :writeoldformat, :printadvance
  
  	CacheServer = Struct.new('CacheServer', 'serverid', 'high', 'low', 'num')
***************
*** 19,35 ****
  	CacheArticle = Struct.new('CacheArticle', 'serverid', 'articlenum', 'bytes', 'lines')
  
! 	# TODO ajouter une option pour ne parser que les nouveaux articles ? (server.index > lasttime.server.high)
! 	def initialize(groupname, path='.', limit=nil, filecallback=nil)
  		@servers = Array.new
  		@files = Array.new
- 		@filecallback = filecallback
  		@sane = true
  		@groupname = groupname
  		@path = path
  
! 		@fd = Zlib::GzipReader.open(path + File::Separator + groupname + ',cache.gz')
  		@lineno = 0
  		begin
! 			read_all(limit)
  		rescue BadNgetCache => e
  			@sane = false
--- 19,53 ----
  	CacheArticle = Struct.new('CacheArticle', 'serverid', 'articlenum', 'bytes', 'lines')
  
! 	def initialize(groupname, path='.')
  		@servers = Array.new
  		@files = Array.new
  		@sane = true
  		@groupname = groupname
  		@path = path
+ 		@printadvance = $stdout.isatty and false
+ 		
+ 		# parse the file head
+ 		@fd = Zlib::GzipReader.open(@path + File::Separator + @groupname + ',cache.gz')
+ 		@lineno = 0
+ 		begin
+ 			read_head
+ 		rescue BadNgetCache => e
+ 			@sane = false
+ 			puts "The cache is a invalid file (#{lineno}: #{e.msg})"
+ 		ensure
+ 			@fd.close
+ 		end
+ 	end
  
! 	def parse(filecallback=nil, limit=nil)
! 		@limit = limit
! 		@filecallback = filecallback
! 		
! 		@fd = Zlib::GzipReader.open(@path + File::Separator + @groupname + ',cache.gz')
  		@lineno = 0
+ 		read_head
+ 
  		begin
! 			read_body
  		rescue BadNgetCache => e
  			@sane = false
***************
*** 38,57 ****
  			@fd.close
  		end
- 		print "\n" if $stdout.isatty
- #		cache_check if sanitycheck and @sane
  	end
  
  	def readline
  		raise BadNgetCache.new('unattended EOF') unless @line = @fd.gets
! 		@line.chomp!
  		@lineno += 1
! 		return true if @line != '.'
  	end
  	
! 	def read_all(limit)
  		readline
  		
! 		raise BadNgetCache.new('not a ngetcache file') unless @line =~ /^([^\t]*)\t(\d+) ([^\t]*)$/
! 		raise BadNgetCache.new('not a ngetcache file') unless $1 == 'NGET4' and $3 == '1'
  		@partscounttotal = $2.to_i
  		@partscountcurrent = 0
--- 56,73 ----
  			@fd.close
  		end
  	end
  
  	def readline
  		raise BadNgetCache.new('unattended EOF') unless @line = @fd.gets
! 		@line.chop!
  		@lineno += 1
! 		return @line != '.'
  	end
  	
! 	def read_head
  		readline
  		
! 		raise BadNgetCache.new('invalid file format') unless @line =~ /^([^\t]*)\t(\d+) ([^\t]*)$/
! 		raise BadNgetCache.new('invalid file format or version number') unless $1 == 'NGET4' and $3 == '1'
  		@partscounttotal = $2.to_i
  		@partscountcurrent = 0
***************
*** 61,65 ****
  			@servers << server
  		end
! 		
  		# files list
  		while (file = read_file)
--- 77,83 ----
  			@servers << server
  		end
! 	end
! 
! 	def read_body
  		# files list
  		while (file = read_file)
***************
*** 69,76 ****
  				@files << file
  			end
! 			if limit
! 				break if (limit -= 1) < 0
! 			end
  		end
  	end
  	
--- 87,93 ----
  				@files << file
  			end
! 			break if @limit and ((@limit -= 1) < 0)
  		end
+ 		print "\rFinished                           \n" if @printadvance
  	end
  	
***************
*** 109,114 ****
  		return unless readline
  		
! 		raise BadNgetCache.new('invalid part description') unless @line =~ /^(-?\d+)\t(\d+)\t([^\t]*)$/
! 		part = CachePart.new($1.to_i, $2.to_i, $3, Array.new)
  		
  		# articles giving this part
--- 126,131 ----
  		return unless readline
  		
! 		raise BadNgetCache.new('invalid part description') unless (@linesplit = @line.split("\t")).length == 3
! 		part = CachePart.new(@linesplit[0], @linesplit[1], @linesplit[2], Array.new)
  		
  		# articles giving this part
***************
*** 117,121 ****
  		end
  		@partscountcurrent += 1
! 		print "#{@partscountcurrent}/#{@partscounttotal}\r" if $stdout.isatty
  		part
  	end
--- 134,138 ----
  		end
  		@partscountcurrent += 1
! 		print "#{@partscountcurrent}/#{@partscounttotal}\r" if @printadvance and @partscountcurrent % 20 == 0
  		part
  	end
***************
*** 123,132 ****
  	def read_article
  		return unless readline
! 		raise BadNgetCache.new('invalid article description') unless @line =~ /^(\d+)\t(\d+)\t(\d+)\t(\d+)$/
! 		CacheArticle.new($1.to_i, $2.to_i, $3.to_i, $4.to_i)
  	end
  	
! 	def save_tofile(groupname=@groupname, path=@path)
! 		@fd = Zlib::GzipWriter.open(path + File::Separator + groupname + ',cache.newformat.gz')
  		write_all
  		@fd.close
--- 140,149 ----
  	def read_article
  		return unless readline
! 		raise BadNgetCache.new('invalid article description') unless (@linesplit = @line.split("\t")).length == 4
! 		CacheArticle.new(*@linesplit)
  	end
  	
! 	def save_tofile(groupname=@groupname, path=@path, extension='cache.newformat.gz')
! 		@fd = Zlib::GzipWriter.open(path + File::Separator + groupname + extension)
  		write_all
  		@fd.close

Index: queue.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/queue.rb,v
retrieving revision 1.23
retrieving revision 1.24
diff -C2 -d -r1.23 -r1.24
*** queue.rb	8 Feb 2004 12:55:48 -0000	1.23
--- queue.rb	20 Feb 2004 16:44:39 -0000	1.24
***************
*** 50,109 ****
  	Db.autotable(Queue, 'queue')
  
! 	def Queue.getqueue(id)
! 		row = Db.dbh.select_one('select `type` from queue where `id` = ' + id.to_s)
! 
! 		if row == nil
! 			raise NoSuchQueueException.new(id)
! 		end
  		
! 		type = row[0]
! 
! 		if type == SINGLE
! 			SingleQueue.new(id, true)
! 		elsif type == REGEXP
! 			RegexpQueue.new(id, true)
  		else
! 			raise UnknownQueueTypeException.new(type)
  		end
  	end
  
! 	def Queue.nextid
! 		row = Db.dbh.select_one('select id from queue
! 			where `status`  = \'PENDING\' or 
! 			((`status` = \'FINISHED\') and (`period` != 0) and ((NOW()-`date_finished`) > `period`))
! 			order by `priority` desc limit 1')
! 		if (row == nil)
! 			return -1
! 		end
! 		row[0]
  	end
  
  	def Queue.nextqueue
! 		id = nextid
! 		if (id == -1)
! 			return nil
! 		end
! 		getqueue(id)
  	end
  
! 	def Queue.delete(queueid, fullclean = false)
! 		q = Queue.getqueue(queueid)
  		q.delete_from_db
! 		QueueFile.clean(queueid) if fullclean
  	end
  
  	def Queue.show(groupid = nil, status = nil)
! 		filter = ""
! 		if groupid != nil
! 			name = Group.getname(groupid)
! 			filter += " where `group` = '#{name}' "
! 		end
! 		if status != nil
! 			if groupid != nil
! 				filter += ' and '
! 			end
! 			filter += " `status` = '#{status}'"
! 		end
! 		
  		QueueList.new(filter).each { |queue|
  			puts queue
--- 50,91 ----
  	Db.autotable(Queue, 'queue')
  
! 	def Queue.getqueue(queueindex)
! 		return nil if queueindex == -1
! 		row = Db.dbh.select_one("select `type` from queue where `queueindex`='#{queueindex}'")
! 		raise NoSuchQueueException.new(queueindex) unless row
  		
! 		case row[0]
! 		when SINGLE
! 			SingleQueue.new(queueindex, true)
! 		when REGEXP
! 			RegexpQueue.new(queueindex, true)
  		else
! 			raise UnknownQueueTypeException.new(row[0])
  		end
  	end
  
! 	def Queue.nextindex
! 		row = Db.dbh.select_one("select `queueindex` from `queue`
! 			where `status`='PENDING' or 
! 			((`status`='FINISHED') and (`period`!=0) and ((NOW()-`date_finished`)>`period`))
! 			order by `priority` desc limit 1")
! 		return row ? row[0] : -1
  	end
  
  	def Queue.nextqueue
! 		Queue.getqueue Queue.nextindex
  	end
  
! 	def Queue.delete(queueindex, fullclean = false)
! 		q = Queue.getqueue(queueindex)
  		q.delete_from_db
! 		QueueFile.clean(queueindex) if fullclean
  	end
  
  	def Queue.show(groupid = nil, status = nil)
! 		fg= groupid ? "`groupindex`='#{Groupe.getindex(groupid)}'" : nil
! 		fs= status ? "`status`='#{status}'" : nil
! 		# DEPLIAGE
! 		filter = ([fg, fs].compact.join(' and ').map{|f| 'where '+f}[0] or '')
  		QueueList.new(filter).each { |queue|
  			puts queue
***************
*** 111,125 ****
  	end
  
! 	def initialize(id, fetch_from_db = false, group='', value='', priority=1, period=0, subpath = '')
! 		@_id = id
! 		@_group = group
  		@_value = value
  		@_priority = priority
  		@_period = period
! 		@_subdir = group
! 		if group != ''
! 			@_subdir = Group.getgroup(group)._directory.sub(/#{$config['basedir']}(#{File::Separator})*/, '')
! 			if subpath != ''
! 				@_subdir += File::Separator + subpath
  				dir = $config['basedir'] + File::Separator + @_subdir
  				if FileTest.exists? dir
--- 93,109 ----
  	end
  
! 	def initialize(queueindex=nil, fetch_from_db=false, groupindex=nil, value=nil, priority=nil, period=nil, subdir=nil)
! 		@_queueindex = queueindex
! 		@_groupindex = groupindex
  		@_value = value
  		@_priority = priority
  		@_period = period
! 
! 		# fine-tune subdir
! 		@_subdir = nil
! 		if groupindex and not fetch_from_db
! 			@_subdir = Group.getgroup(groupindex)._directory.sub(/#{$config['basedir']}(#{File::Separator})*/, '')
! 			if subdir
! 				@_subdir += File::Separator + subdir
  				dir = $config['basedir'] + File::Separator + @_subdir
  				if FileTest.exists? dir
***************
*** 134,156 ****
  
  	def finish
! 		@status = FINISHED
! 		
! 		# TODO : sets @date_finished and use only commit_to_db
! 		dbh.do('update `queue` set `date_finished` = NOW() where `id` = ' + @id)
  		sync_to_db
  	end
  
  	def to_s
! 		s =	"#{yellow '<'}#{blue @_id}#{yellow '>'} [#{red _group}] #{yellow _value}, prio : #{_priority}, dest : #{green _subdir}, status : #{_status}"
! 		case @status
  			when PENDING
! 				s += ", added : #{_date_added}"
  			when PROCESSING
! 				s += ", started : #{_date_started}"
! 			when STOPPED
! 				s += ", finished : #{_date_finished}"
  		end
  		if @_period != 0
! 			s += " (period = #{@_period}s)"
  		end
  		s
--- 118,142 ----
  
  	def finish
! 		@_status = FINISHED
! 		@_date_finished = Db.now
  		sync_to_db
  	end
  
+ 	def group
+ 		Group.getname(@_groupindex)
+ 	end
+ 	
  	def to_s
! 		s = "#{yellow '<'}#{blue @_queueindex}#{yellow '>'} [#{red group}] #{yellow @_value}, prio : #{@_priority}, dest : #{green @_subdir}, status : #{@_status}"
! 		case @_status
  			when PENDING
! 				s += ", added #{@_date_added.strftime('%d/%m %H:%M:%S')}"
  			when PROCESSING
! 				s += ", started #{@_date_started.strftime('%d/%m %H:%M:%S')}"
! 			else
! 				s += ", #{@_status.downcase} since #{@_date_finished.strftime('%d/%m %H:%M:%S')}"
  		end
  		if @_period != 0
! 			s += " (period #{cooltime @_period})"
  		end
  		s
***************
*** 158,163 ****
  
  	def to_html
! 		"<tr class='queue'><td class='id'>#{@id}</td><td class='group'>#{@group}</td>
! 		     <td class='prio'>#{@priority}</td><td class='status'>#{@status}</td></tr>"
  	end	
  end
--- 144,148 ----
  
  	def to_html
! 		"<tr class='queue'><td class='id'>#{@_queueindex}</td><td class='group'>#{group}</td><td class='prio'>#{@_priority}</td><td class='status'>#{@_status}</td></tr>"
  	end	
  end
***************
*** 188,191 ****
  	end
  end
- 
  end
--- 173,175 ----

Index: queue_file.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/queue_file.rb,v
retrieving revision 1.15
retrieving revision 1.16
diff -C2 -d -r1.15 -r1.16
*** queue_file.rb	7 Feb 2004 16:21:20 -0000	1.15
--- queue_file.rb	20 Feb 2004 16:44:39 -0000	1.16
***************
*** 14,20 ****
  	attr_accessor :encoding
  	
! 	def QueueFile.already_in_db(queueid, mid)
  		row = Db.dbh.select_one "select * from `queue_files` where
! 			`queue_id` = '#{queueid}' and `mid` = '#{mid}'"
  		return row != nil
  	end
--- 14,21 ----
  	attr_accessor :encoding
  	
! 	def QueueFile.already_in_db(queueindex, artindex)
! 		return nil unless artindex
  		row = Db.dbh.select_one "select * from `queue_files` where
! 			`queueindex`='#{queueindex}' and `artindex`='#{artindex}'"
  		return row != nil
  	end
***************
*** 22,39 ****
  	# Removes the queuefiles associated to queueid from the 
  	# database and returns the number of rows deleted
! 	def QueueFile.clean(queueid)
! 		rows = Db.dbh.do "delete from `queue_files` where `queue_id` = '#{queueid}'"
  		return rows
  	end
  
! 	def initialize(queueid, mid, fetch_from_db = false, parts = 0, nblines = 0, filenumber = 0, size = 0, filename = 'unknown')
! 		@_queue_id = queueid
  		@dlparts = Integer(parts)
  		@nblines = nblines
! 		@_mid = mid
  		@_filenumber = filenumber
  		@_size = size
  		@_filename = filename
! 		@_date_started = Time.now
  		update_from_db if fetch_from_db
  	end
--- 23,40 ----
  	# Removes the queuefiles associated to queueid from the 
  	# database and returns the number of rows deleted
! 	def QueueFile.clean(queueindex)
! 		rows = Db.dbh.do "delete from `queue_files` where `queueindex`='#{queueindex}'"
  		return rows
  	end
  
! 	def initialize(queueindex, artindex, fetch_from_db = false, parts = 0, nblines = 0, filenumber = 0, size = 0, filename = 'unknown')
! 		@_queueindex = queueindex
  		@dlparts = Integer(parts)
  		@nblines = nblines
! 		@_artindex = artindex
  		@_filenumber = filenumber
  		@_size = size
  		@_filename = filename
! 		@_date_started = Db.now
  		update_from_db if fetch_from_db
  	end
***************
*** 57,61 ****
  		@_filename = 'not found' if (@_filename == nil)
  
! 		@_date_finished = Time.now
  
  		if res
--- 58,62 ----
  		@_filename = 'not found' if (@_filename == nil)
  
! 		@_date_finished = Db.now
  
  		if res

Index: utils.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/utils.rb,v
retrieving revision 1.12
retrieving revision 1.13
diff -C2 -d -r1.12 -r1.13
*** utils.rb	6 Feb 2004 23:46:59 -0000	1.12
--- utils.rb	20 Feb 2004 16:44:39 -0000	1.13
***************
*** 89,111 ****
  	end
  
! 	def Utils.pretty_size(asize)
! 		if asize != nil
! 			size = asize.to_i
! 			if size > 4096
! 				ksize = size / 1024
! 				if ksize > 4096
! 					msize = ksize / 1024
! 					"#{msize}mb"
! 				else
! 					"#{ksize}kb"
! 				end
! 			else
! 				"#{size}b"
! 			end
! 		else
! 			"-1"
! 		end
  	end
- 
  	def pretty_size(asize)
  		Utils.pretty_size(asize)
--- 89,113 ----
  	end
  
! 	def Utils.spreadbase(value, base, delimiters)
! 		return '-1' unless value
! 		last = delimiters.pop
! 		arr = delimiters.map { |d|
! 			v = value % base
! 			value /= base
! 			"#{v}#{d}"
! 		} << "#{value}#{last}"
! 		return arr.reverse[0..1].join(' ')
! 	end
! 	
! 	def Utils.cooltime(seconds)
! 		Utils.spreadbase(seconds.to_i, 60, ['s', 'm', 'h'])
! 	end
! 	def cooltime(seconds)
! 		Utils.cooltime(seconds)
! 	end
! 	
! 	def Utils.pretty_size(size)
! 		Utils.spreadbase(size.to_i, 1024, ['b', 'kb', 'Mb', 'Gb'])
  	end
  	def pretty_size(asize)
  		Utils.pretty_size(asize)

Index: web.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/web.rb,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** web.rb	6 Feb 2004 23:46:59 -0000	1.5
--- web.rb	20 Feb 2004 16:44:39 -0000	1.6
***************
*** 7,11 ****
  class Web
  	include Utils
! 	attr_reader :group, :rlsmid, :offset, :articles_by_page, :filter, :date
  
  	def initialize(servlet_request)
--- 7,11 ----
  class Web
  	include Utils
! 	attr_reader :group, :rlsartind, :offset, :articles_by_page, :filter, :date
  
  	def initialize(servlet_request)
***************
*** 23,27 ****
  		@filter   = @query['filter'  ].to_s   if @query.has_key? 'filter'
  		@date     = @query['date'    ].to_s   if @query.has_key? 'date'
! 		@rlsmid   = @query['rlsmid'  ].to_s   if @query.has_key? 'rlsmid'
  		@group = NgetSuite::Group.getgroup(@query['group'].to_s) if @query.has_key? 'group'
  
--- 23,27 ----
  		@filter   = @query['filter'  ].to_s   if @query.has_key? 'filter'
  		@date     = @query['date'    ].to_s   if @query.has_key? 'date'
! 		@rlsartind= @query['rlsartind'].to_s  if @query.has_key? 'rlsartind'
  		@group = NgetSuite::Group.getgroup(@query['group'].to_s) if @query.has_key? 'group'