[Ngetsuite-cvs] ngetsuite/ngetsuite downloader.rb,1.25,1.26 group.rb,1.39,1.40 ngetcache.rb,1.8,1.9 queue.rb,1.24,1.25 utils.rb,1.15,1.16

[<jj...@us...>]

Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv10822/ngetsuite

Modified Files:
	downloader.rb group.rb ngetcache.rb queue.rb utils.rb 
Log Message:
even more sexier daemon, unified colors (blue: info, green: success, yellow: error in nget, red: error in ngetsuite)
added an arg for Utils.debug to allow \r instead of \n, used it everywhere


Index: downloader.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloader.rb,v
retrieving revision 1.25
retrieving revision 1.26
diff -C2 -d -r1.25 -r1.26
*** downloader.rb	25 Feb 2004 22:37:10 -0000	1.25
--- downloader.rb	27 Feb 2004 13:12:07 -0000	1.26
***************
*** 42,47 ****
  	end
  
! 	def ldebug(str)
! 		debug "(#{white @queueindex})" + str
  	end
  	
--- 42,47 ----
  	end
  
! 	def ldebug(str, linefeed=true)
! 		debug "(#{white @queueindex})" + str, linefeed
  	end
  	
***************
*** 51,62 ****
  		when /^(\d+) \((\d+)\/(\d+)\): (\d+)\/(\d+)L (\d+)\/(\d+)B (\d+)% (\d+)B\/s (\w+) (\d+)\/(\d+) (\w+)$/
  			part, size, speed, filenb = $2, $7, $9, $12
! 			ldebug "#{yellow $5+'L'}(#{yellow $7+'B'}) at #{red $9+'B/s'} [#{blue $2}/#{blue $3}] [#{green $11}/#{green $12}]"
  			@curfile.update(part, speed, filenb, size)
  			
! 		when /^uu_msg\(\d+\):Loaded from (.*): '.*' \((.*)\): (.*) part (\d+)\s+begin\s+(yEnc|UUdata|Base64|XXdata|BinHex|plaintext|qptext|unknown)$/
  			artfile, base, filename, nb, enc = $1, $2, $3, $4, $5
! 			ldebug "decoding filename : #{blue filename}, encoding : #{yellow enc}"
! 			artfile.sub!(/^.*\/([^\/]*)$/, '\1')
! 			ldebug "loaded #{green artfile}"
  			@curfile._filename = filename
  			@curfile.encoding = enc
--- 51,60 ----
  		when /^(\d+) \((\d+)\/(\d+)\): (\d+)\/(\d+)L (\d+)\/(\d+)B (\d+)% (\d+)B\/s (\w+) (\d+)\/(\d+) (\w+)$/
  			part, size, speed, filenb = $2, $7, $9, $12
! 			ldebug "#{green $5+'L'}(#{green $7+'B'}) at #{$9+'B/s'} [#{blue $2}/#{blue $3}] [#{yellow $11}/#{yellow $12}]", false
  			@curfile.update(part, speed, filenb, size)
  			
! 		when /^uu_msg\(\d+\):Loaded from (.*): '.*' \((.*)\): (.*) part (\d+)\s+begin\s+(?:end\s+)?(yEnc|UUdata|Base64|XXdata|BinHex|plaintext|qptext|unknown)$/
  			artfile, base, filename, nb, enc = $1, $2, $3, $4, $5
! 			ldebug "#{blue 'Decoding'} #{filename} (encoding : #{enc})", false
  			@curfile._filename = filename
  			@curfile.encoding = enc
***************
*** 66,70 ****
  			# basename
  			artfile.sub!(/^.*\/([^\/]*)$/, '\1')
! 			ldebug "loaded #{green artfile} (last)"
  		
  		when /^uu_msg\(\d+\):Loaded from (.*): '.*' \(.*\):.*part (\d+)\s+(yEnc|UUdata|Base64|XXdata|BinHex|plaintext|qptext|unknown)$/
--- 64,68 ----
  			# basename
  			artfile.sub!(/^.*\/([^\/]*)$/, '\1')
! 			ldebug "#{blue 'Loaded'} #{artfile} (last)", false
  		
  		when /^uu_msg\(\d+\):Loaded from (.*): '.*' \(.*\):.*part (\d+)\s+(yEnc|UUdata|Base64|XXdata|BinHex|plaintext|qptext|unknown)$/
***************
*** 72,84 ****
  			# basename
  			artfile.sub!(/^.*\/([^\/]*)$/, '\1')
! 			ldebug "loaded #{green artfile}"
  
- 		when /^already have article (<.*>) \(.*\/([^\/]*)\)$/
- 			ldebug "already have #{green $1} in #{blue $2}"
- 			@nbfiles += 1
- 			
  		when /^Retrieving: (\d+).*\t(\d+)l\t(\d+T\d+)\t(.*)\t(.*)\t(<.*>)$/
  			parts, nblines, date, subject, from, mid = $1, $2, $3, $4, $5, $6
! 			ldebug "Retrieving #{blue subject}"
  			artindex = Db.dbh.select_one("select `artindex` from `articles` where `mid`='#{mid}'")
  			unless artindex
--- 70,78 ----
  			# basename
  			artfile.sub!(/^.*\/([^\/]*)$/, '\1')
! 			ldebug "#{blue 'Loaded'} #{artfile}", false
  
  		when /^Retrieving: (\d+).*\t(\d+)l\t(\d+T\d+)\t(.*)\t(.*)\t(<.*>)$/
  			parts, nblines, date, subject, from, mid = $1, $2, $3, $4, $5, $6
! 			ldebug "#{blue 'Retrieving'} #{subject}"
  			artindex = Db.dbh.select_one("select `artindex` from `articles` where `mid`='#{mid}'")
  			unless artindex
***************
*** 100,117 ****
  				@curfile.update_from_db
  			end
! 			
  		when /^already have (.*)$/
! 			ldebug "already have '#{blue $1}'"
  			
  		# What is autopar ?
  		when /^autopar: (.*) seems like a (par2?) \((.*)\)/
  			subject, ft, base = $1, $2, $3
! 			ldebug "#{yellow 'autopar'}: found a #{ft} file: " + blue(base.length > 4 ? base : subject)
  			
  		when /^autopar: .* matches local (par2?)set \((.*)\)$/
! 			ldebug "#{yellow 'autopar'}: got a #{$1}, set #{green $2}"
  
  		when /^decoded ok/
! 			ldebug 'file decoded'
  			@curfile.finish_success
  			@finish_status = 'FINISHED' 
--- 94,115 ----
  				@curfile.update_from_db
  			end
! 		
! 		when /^already have article (<.*>) \(.*\/([^\/]*)\)$/
! 			ldebug "#{blue 'found'} #{$1} in #{$2}", false
! 			@nbfiles += 1
! 		
  		when /^already have (.*)$/
! 			ldebug "#{green 'Already have'} '#{$1}'"
  			
  		# What is autopar ?
  		when /^autopar: (.*) seems like a (par2?) \((.*)\)/
  			subject, ft, base = $1, $2, $3
! #			ldebug "#{blue 'autopar'}: found a #{ft} file: " + blue(base.length > 4 ? base : subject)
  			
  		when /^autopar: .* matches local (par2?)set \((.*)\)$/
! #			ldebug "#{blue 'autopar'}: got a #{$1}, set #{blue $2}"
  
  		when /^decoded ok/
! 			ldebug "#{green 'Decoded'} #{@curfile._filename}"
  			@curfile.finish_success
  			@finish_status = 'FINISHED' 
***************
*** 120,135 ****
  
  		when 'download error occured, keeping temp files.'
! 			ldebug "error while downloading #{@curfile._filename}"
  			@curfile.finish_err
  			@finish_status = 'FAILED'
! 			log "Download failed for #{@curfile._filename}, queue #{@value}"
  			@curfile = nil
  
  			# same as below
  		when /^couldn't get (<.*>) from anywhere$/
! 			ldebug "#{red 'Couldn\'t download mid '} #{$1}"
  		
  		when / decoding errors occured, keeping temp files\.$/
! 			ldebug "error while decoding #{@curfile._filename}"
  			@curfile.finish_err_dec
  			@finish_status = 'FAILED'
--- 118,135 ----
  
  		when 'download error occured, keeping temp files.'
! 			ldebug "#{yellow 'Error'} while downloading"
  			@curfile.finish_err
  			@finish_status = 'FAILED'
! 			# we do not know the file name
! #			log "Download failed for #{@curfile._filename}, queue #{@value}"
  			@curfile = nil
  
  			# same as below
  		when /^couldn't get (<.*>) from anywhere$/
! 			ldebug "#{yellow 'Error'}: couldn't get #{$1}"
! 			@finish_status = 'FAILED'
  		
  		when / decoding errors occured, keeping temp files\.$/
! 			ldebug "#{yellow 'Error'} while decoding #{@curfile._filename}"
  			@curfile.finish_err_dec
  			@finish_status = 'FAILED'
***************
*** 138,151 ****
  
  		when /^Connecting to (.*)$/
! 			ldebug "Connection to #{$1}.."
  		when />> 200/
! 			ldebug 'Connected'
  		when />> 400/
! 			ldebug 'Disconnected'
  		when /<< QUIT/
! 			ldebug 'Quitted'
  		when /^doarticle \d+: \d+!=\d+ \|\| \d+!=\d+$/
! 		when /^unequal line count \d+ should equal \d+$/
! 			ldebug 'unequal line count'
  		when />> \d\d\d/, /<< (GROUP|AUTHINFO)/
  		
--- 138,161 ----
  
  		when /^Connecting to (.*)$/
! 			ldebug "Connection to #{$1}..", false
  		when />> 200/
! 			ldebug green('Connected')
  		when />> 400/
! 			ldebug red('Disconnected')
  		when /<< QUIT/
! 			ldebug blue('Quitted')
! 		
! 		when 'ignored error (nntp_getline: connection closed unexpectedly)'
! 			ldebug "#{red 'Lost connection'}. Reconnecting..."
! 		
! 		when /close\(fast\)$/
! 		when /^error, will try .* again \(server says byebye: (.*)\)/
! 			ldebug red("Disconnected (#{$1})") + '. Reconnecting...', false
! 		when /^nntp_doarticle: try again./
! 		when /^make_connection/
! 		
  		when /^doarticle \d+: \d+!=\d+ \|\| \d+!=\d+$/
! 		when /^unequal line count (\d+) should equal (\d+)$/
! 			ldebug "unequal line count (#{$1} != #{$2})"
  		when />> \d\d\d/, /<< (GROUP|AUTHINFO)/
  		
***************
*** 154,160 ****
  			ok,warn,errs = $1, $2, $3
  			@finish_status = (errs ? 'FAILED' : 'FINISHED')
! 			ldebug "Finished: ok#{green ok} warnings#{blue warn} errors#{red errs}"
  		else
! 			ldebug blue('unhandled msg: ') + line
  		end
  	end
--- 164,170 ----
  			ok,warn,errs = $1, $2, $3
  			@finish_status = (errs ? 'FAILED' : 'FINISHED')
! 			ldebug "Finished:#{' ok'+green(ok) if ok}#{' warnings'+yellow(warn) if warn}#{' errors'+red(errs) if errs}"
  		else
! 			ldebug blue('unhandled: ') + line
  		end
  	end

Index: group.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/group.rb,v
retrieving revision 1.39
retrieving revision 1.40
diff -C2 -d -r1.39 -r1.40
*** group.rb	25 Feb 2004 22:37:10 -0000	1.39
--- group.rb	27 Feb 2004 13:12:07 -0000	1.40
***************
*** 159,175 ****
  		
  		n = rand(900000)+100000
! 		$stdout.sync = true if $stdout.isatty
  		
  		begin
! 		IO.popen("nice -n#{$config['nice_nget_update']} -- #{$config['nget']} -g #{@_fullname} -cr '^#{n}Dummy' 2>&1").each("\r") do |line|
! 			print "progress#{yellow $1} speed#{blue $2} ETA #{$3}            \r" if line =~ /Retrieving headers \d+-\d+ : \d+\/\d+\/\d+(\s+\d+%)(\s+\d+B\/s)(.*s)/ and $stdout.isatty
! 		end
  		rescue
  			print "\nInterrupted"
  		end
  		
- 		print "\n" if $stdout.isatty
- 		$stdout.sync = false if $stdout.isatty
- 		
  		debug "Updatengetcache finished in #{Utils.cooltime(Time.now-t)}"
  	end
--- 159,172 ----
  		
  		n = rand(900000)+100000
! 		cmdline = "nice -n#{$config['nice_nget_update']} -- #{$config['nget']} -g #{@_fullname} -cr '^#{n}Dummy' 2>&1"
  		
  		begin
! 			IO.popen(cmdline).each("\r") { |line|
! 				debug("progress#{yellow $1} speed#{blue $2} ETA #{$3}", false) if line =~ /Retrieving headers \d+-\d+ : \d+\/\d+\/\d+(\s+\d+%)(\s+\d+B\/s)(.*s)/
! 			}
  		rescue
  			print "\nInterrupted"
  		end
  		
  		debug "Updatengetcache finished in #{Utils.cooltime(Time.now-t)}"
  	end
***************
*** 180,184 ****
  		
  		nb_rows = Db.dbh.do "delete from `articles` where `groupindex`='#{@_groupindex}'"
- 		# and `artindex` < '#{@_cache_low}'
  		debug "#{nb_rows} articles deleted"
  		
--- 177,180 ----
***************
*** 210,214 ****
  		cache.parse(filecallback)
  		@_cache_low,@_cache_high = cache.servers[0].low, cache.servers[0].high
! 		debug "Parsengetcache done in #{Utils.cooltime(Time.now-t)}"
  	end
  
--- 206,211 ----
  		cache.parse(filecallback)
  		@_cache_low,@_cache_high = cache.servers[0].low, cache.servers[0].high
! 		nb_rows = Db.dbh.select_one("select count(*) from `articles` where `groupindex`='#{@_groupindex}'")[0]
! 		debug "#{nb_rows} articles inserted in #{Utils.cooltime(Time.now-t)}"
  	end
  
***************
*** 259,263 ****
  		end
  		
! 		debug "Indexed #{nb_nfos} nfos in #{Utils.cooltime(Time.now-t)}"
  	end
  
--- 256,260 ----
  		end
  		
! 		debug "#{nb_nfos} nfos indexed in #{Utils.cooltime(Time.now-t)}"
  	end
  
***************
*** 286,290 ****
  		rls.insert_to_db
  
! 		debug "Inserted #{nb_releases} releases in #{Utils.cooltime(Time.now-t)}"
  	end
  	
--- 283,287 ----
  		rls.insert_to_db
  
! 		debug "#{nb_releases} releases inserted in #{Utils.cooltime(Time.now-t)}"
  	end
  	

Index: ngetcache.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/ngetcache.rb,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** ngetcache.rb	20 Feb 2004 16:44:39 -0000	1.8
--- ngetcache.rb	27 Feb 2004 13:12:07 -0000	1.9
***************
*** 10,16 ****
  class NgetCache
  	require 'zlib'
  
  	attr_reader :servers, :files, :sane, :lineno, :groupname
! 	attr_accessor :writeoldformat, :printadvance
  
  	CacheServer = Struct.new('CacheServer', 'serverid', 'high', 'low', 'num')
--- 10,28 ----
  class NgetCache
  	require 'zlib'
+ 	begin
+ 		require 'ngetsuite/utils'
+ 	rescue LoadError => e
+ 		module NgetSuite
+ 			class Utils
+ 				def Utils.debug(arg, booh)
+ 					print "\r"+' '*80+"\r"+arg
+ 					$stdout.flush
+ 				end
+ 			end
+ 		end
+ 	end
  
  	attr_reader :servers, :files, :sane, :lineno, :groupname
! 	attr_accessor :writeoldformat
  
  	CacheServer = Struct.new('CacheServer', 'serverid', 'high', 'low', 'num')
***************
*** 25,29 ****
  		@groupname = groupname
  		@path = path
- 		@printadvance = $stdout.isatty and false
  		
  		# parse the file head
--- 37,40 ----
***************
*** 89,93 ****
  			break if @limit and ((@limit -= 1) < 0)
  		end
- 		print "\rFinished                           \n" if @printadvance
  	end
  	
--- 100,103 ----
***************
*** 134,138 ****
  		end
  		@partscountcurrent += 1
! 		print "#{@partscountcurrent}/#{@partscounttotal}\r" if @printadvance and @partscountcurrent % 20 == 0
  		part
  	end
--- 144,148 ----
  		end
  		@partscountcurrent += 1
! 		NgetSuite::Utils.debug "#{@partscountcurrent}/#{@partscounttotal}", false if @partscountcurrent % 20 == 0
  		part
  	end

Index: queue.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/queue.rb,v
retrieving revision 1.24
retrieving revision 1.25
diff -C2 -d -r1.24 -r1.25
*** queue.rb	20 Feb 2004 16:44:39 -0000	1.24
--- queue.rb	27 Feb 2004 13:12:07 -0000	1.25
***************
*** 135,139 ****
  				s += ", started #{@_date_started.strftime('%d/%m %H:%M:%S')}"
  			else
! 				s += ", #{@_status.downcase} since #{@_date_finished.strftime('%d/%m %H:%M:%S')}"
  		end
  		if @_period != 0
--- 135,140 ----
  				s += ", started #{@_date_started.strftime('%d/%m %H:%M:%S')}"
  			else
! 				s += ", #{@_status.downcase}"
! 				s += " since #{@_date_finished.strftime('%d/%m %H:%M:%S')}" if @_date_finished
  		end
  		if @_period != 0

Index: utils.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/utils.rb,v
retrieving revision 1.15
retrieving revision 1.16
diff -C2 -d -r1.15 -r1.16
*** utils.rb	23 Feb 2004 17:08:08 -0000	1.15
--- utils.rb	27 Feb 2004 13:12:07 -0000	1.16
***************
*** 18,28 ****
  	WHITE_BG   = 47
  	
  	# print +message+ if debugging is enabled
! 	def debug(message=nil)
! 		Utils.debug message
  	end
  
! 	def Utils.debug(message=nil)
!   		puts "#{green 'D'}: #{message}" if $config['debug'] and message
  	end
  
--- 18,42 ----
  	WHITE_BG   = 47
  	
+ 	@@clearlastdebugline = false
+ 	
  	# print +message+ if debugging is enabled
! 	def debug(message=nil, linefeed=true)
! 		Utils.debug message, linefeed
  	end
  
! 	def Utils.debug(message=nil, linefeed=true)
! 		return unless $config['debug'] and message
! 		return if not linefeed and not $stdout.isatty
! 		msg = "#{green 'D'}: #{message}"
! 		# TODO: $stdout.tty.length
! 		msg = ' ' * 80 + "\r" + msg if @@clearlastdebugline
! 		if linefeed or not $stdout.isatty
! 			@@clearlastdebugline = false
! 			puts msg
! 		else
! 			@@clearlastdebugline = true
! 			print msg + "\r"
! 			$stdout.flush
! 		end
  	end