[Ngetsuite-cvs] ngetsuite/ngetsuite config.rb,1.2,1.3 core.rb,1.24,1.25 db.rb,1.20,1.21 downloader.rb,1.15,1.16 downloadmanager.rb,1.12,1.13 group.rb,1.20,1.21 nfo.rb,1.3,1.4 queue.rb,1.18,1.19 utils.rb,1.8,1.9

[<jj...@pr...>]

Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv32633/ngetsuite

Modified Files:
	config.rb core.rb db.rb downloader.rb downloadmanager.rb 
	group.rb nfo.rb queue.rb utils.rb 
Log Message:
new config file format, rhs are now evaluated (strings must be quoted)
new config option to disable the headers update by the program (extern_headers_update)
the working directory is changed to where the main script is located at its startup (to allow the invocation by cron)
misc bugfixes


Index: config.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/config.rb,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** config.rb	23 Oct 2003 21:44:29 -0000	1.2
--- config.rb	2 Feb 2004 02:41:34 -0000	1.3
***************
*** 13,17 ****
            next if(line =~ /^\s*(#|$)/)
            if(line =~ /^\s*(\S+)\s*=\s*(.*)$/)
!             self[$1] = $2 if($2)
            end
          end
--- 13,17 ----
            next if(line =~ /^\s*(#|$)/)
            if(line =~ /^\s*(\S+)\s*=\s*(.*)$/)
!             self[$1] = eval($2) if($2)
            end
          end

Index: core.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/core.rb,v
retrieving revision 1.24
retrieving revision 1.25
diff -C2 -d -r1.24 -r1.25
*** core.rb	14 Jan 2004 22:01:43 -0000	1.24
--- core.rb	2 Feb 2004 02:41:57 -0000	1.25
***************
*** 73,77 ****
  			groups = $dbh.select_all 'select `fullname` from `groups`
  				where active="yes"'
- 			puts "kikoo"
  			groups.each do |group|
  				name = group.to_s
--- 73,76 ----

Index: db.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/db.rb,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** db.rb	28 Dec 2003 20:04:03 -0000	1.20
--- db.rb	2 Feb 2004 02:41:57 -0000	1.21
***************
*** 215,219 ****
  		req += " from #{table} where " + ids_constraint
  		
! 		puts req
  		sth = Db.dbh.execute req
  		hash = sth.fetch_hash
--- 215,219 ----
  		req += " from #{table} where " + ids_constraint
  		
! 		debug 'DB: '+ req
  		sth = Db.dbh.execute req
  		hash = sth.fetch_hash
***************
*** 278,282 ****
  		req += ')'
  		
! 		puts req
  		exec_request req
  	end
--- 278,282 ----
  		req += ')'
  		
! 		debug 'DB: ' + req
  		exec_request req
  	end

Index: downloader.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloader.rb,v
retrieving revision 1.15
retrieving revision 1.16
diff -C2 -d -r1.15 -r1.16
*** downloader.rb	27 Dec 2003 20:01:54 -0000	1.15
--- downloader.rb	2 Feb 2004 02:41:57 -0000	1.16
***************
*** 131,135 ****
  		@io.close
  		@t.exit
! 		time = `date`.chomp
  		$dbh.do 'lock tables queue write'
  		$dbh.do "update queue set `status` = 'FINISHED', `date_finished`=NOW() where `id` = '#{@id}'"
--- 131,135 ----
  		@io.close
  		@t.exit
! 		time = Time.now.to_s
  		$dbh.do 'lock tables queue write'
  		$dbh.do "update queue set `status` = 'FINISHED', `date_finished`=NOW() where `id` = '#{@id}'"

Index: downloadmanager.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloadmanager.rb,v
retrieving revision 1.12
retrieving revision 1.13
diff -C2 -d -r1.12 -r1.13
*** downloadmanager.rb	28 Dec 2003 20:04:04 -0000	1.12
--- downloadmanager.rb	2 Feb 2004 02:41:57 -0000	1.13
***************
*** 29,33 ****
  				begin
  					loop do
! 						log "current : #{@current_dlers}, max : #{@max_dlers}"
  						if (@current_dlers < @max_dlers)
  							start_new_downloader
--- 29,33 ----
  				begin
  					loop do
! #						log "current : #{@current_dlers}, max : #{@max_dlers}"
  						if (@current_dlers < @max_dlers)
  							start_new_downloader

Index: group.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/group.rb,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** group.rb	27 Dec 2003 18:13:36 -0000	1.20
--- group.rb	2 Feb 2004 02:41:57 -0000	1.21
***************
*** 65,72 ****
  
  	def Group.show(onlyactive = false)
! 		labels = Array["Added"     , "Last update" , "Directory", "Active"]
! 		keys   = Array["date_added", "date_updated", "directory", "active"]
  		
! 		req = ""
  		req = " where `active` = 'yes' " if onlyactive
  		req += 'order by `date_added`'
--- 65,72 ----
  
  	def Group.show(onlyactive = false)
! 		labels = Array['Added'     , 'Last update' , 'Directory', 'Active']
! 		keys   = Array['date_added', 'date_updated', 'directory', 'active']
  		
! 		req = ''
  		req = " where `active` = 'yes' " if onlyactive
  		req += 'order by `date_added`'
***************
*** 120,125 ****
  	
  	def search(filter)
! 		sth = Db.dbh.execute("select * from `articles` where `group` = ? and
! 			`subject` LIKE ? order by `time` desc", @_fullname, "%"+filter+"%")
  		sth.fetch_hash do |row|
  			puts row['subject']
--- 120,124 ----
  	
  	def search(filter)
! 		sth = Db.dbh.execute('select * from `articles` where `group` = ? and `subject` LIKE ? order by `time` desc', @_fullname, '%'+filter+'%')
  		sth.fetch_hash do |row|
  			puts row['subject']
***************
*** 129,135 ****
  
  	def update
  		clear_articles
  		clear_nfos
- 		updatecache
  		mknewscache
  		fetchnfos
--- 128,134 ----
  
  	def update
+ 		updatecache unless $config['extern_headers_update'].split(',').include?(@_shortname)
  		clear_articles
  		clear_nfos
  		mknewscache
  		fetchnfos
***************
*** 139,152 ****
  
  	def clear_articles
! 		debug "Beginning clear_articles at #{`date`.chomp}"
  		nb_rows = Db.dbh.execute "delete from `articles` where `group` = '#{@_fullname}'"
! 		debug "End clear_articles at #{`date`.chomp}"
  	end
  
  	def clear_nfos
! 		debug "Beginning clear_nfos at #{`date`.chomp}"
  		nb_rows = Db.dbh.do "delete from `nfos` where `group` = '#{@_fullname}'"
! 		puts "#{nb_rows} nfos deleted"
! 		debug "End clear_nfos at #{`date`.chomp}"
  
  	end
--- 138,151 ----
  
  	def clear_articles
! 		debug "Beginning clear_articles at #{Time.now.strftime('%H:%M:%S')}"
  		nb_rows = Db.dbh.execute "delete from `articles` where `group` = '#{@_fullname}'"
! 		debug "End clear_articles at #{Time.now.strftime('%H:%M:%S')}"
  	end
  
  	def clear_nfos
! 		debug "Beginning clear_nfos at #{Time.now.strftime('%H:%M:%S')}"
  		nb_rows = Db.dbh.do "delete from `nfos` where `group` = '#{@_fullname}'"
! 		debug "#{nb_rows} nfos deleted"
! 		debug "End clear_nfos at #{Time.now.strftime('%H:%M:%S')}"
  
  	end
***************
*** 160,195 ****
  
  	def updatecache
! 		debug "Beginning updatecache at #{`date`.chomp}"
! 		`nice -n19 #{@nget} -q -g #{@_fullname} -l 99999 -r 'DummyString'`
! 		debug "End updatecache at #{`date`.chomp}"
  	end
  
  	def mknewscache
! 		debug "Beginning mknewscache at #{`date`.chomp}"
  		sth = Db.dbh.prepare 'insert into articles(`group`, `mid`, `nbparts`, `size`,
  			`subject`, `from`, `time`, `rls_subject`) values (?, ?, ?, ?, ?, ?, FROM_UNIXTIME(?), ?)'
  		debug 'Parsing cache'
  		cachefile = Zlib::GzipReader.open("#{@cachedir}/#{@_fullname},cache.gz")
  		while line = cachefile.gets
  			next if line !~ /^(\d+)\t0\t/ or $1.to_i < 1
  			parts, subject, from = line.split(/\t/).values_at(0, 3, 4)
  			parts = parts.to_i
! 			next if cachefile.gets != ".\n"
  			
  			partcnt = 0
- 			line = cachefile.gets
- 			
  			size = 0
  	 		while line != ".\n"
! 				time, mid = /^\d+\t(.*)\t(.*)$/.match(line).values_at(1,2)
! 				while line = cachefile.gets
! 	  				if line == ".\n"
! 						line = cachefile.gets
! 						break
! 	  				else
! 						serverid, bytes, lines = line.split(/\t/).values_at(1,2,3).collect {|x| x.to_i}
! 	  				end
  				end
! 				lines = lines<1 ? 1 : lines
  				linelength = (bytes / lines).to_i
  				size = size.to_i + (bytes / ( linelength < 100 ? 1.39 : 1.0334 ))
--- 159,194 ----
  
  	def updatecache
! 		debug "Beginning updatecache at #{Time.now.strftime('%H:%M:%S')}"
! 		# to avoid ngetsuite poisoning
! 		n = rand(900000)+100000
! 		`nice -n19 #{@nget} -q -g #{@_fullname} -c -r '^#{n}Dummy'`
! 		debug "End updatecache at #{Time.now.strftime('%H:%M:%S')}"
  	end
  
  	def mknewscache
! 		debug "Beginning mknewscache at #{Time.now.strftime('%H:%M:%S')}"
  		sth = Db.dbh.prepare 'insert into articles(`group`, `mid`, `nbparts`, `size`,
  			`subject`, `from`, `time`, `rls_subject`) values (?, ?, ?, ?, ?, ?, FROM_UNIXTIME(?), ?)'
  		debug 'Parsing cache'
  		cachefile = Zlib::GzipReader.open("#{@cachedir}/#{@_fullname},cache.gz")
+ 		while (cachefile.gets != ".\n")
+ 		end
  		while line = cachefile.gets
  			next if line !~ /^(\d+)\t0\t/ or $1.to_i < 1
  			parts, subject, from = line.split(/\t/).values_at(0, 3, 4)
  			parts = parts.to_i
! 			while (cachefile.gets != ".\n")
! 			end
  			
  			partcnt = 0
  			size = 0
+ 			line = cachefile.gets
  	 		while line != ".\n"
! 				line =~ /^\d+\t(.*)\t(.*)$/
! 				time, mid = $1, $2
! 				while (line = cachefile.gets) != ".\n"
! 					serverid, bytes, lines = line.split(/\t/).values_at(1,2,3).collect {|x| x.to_i}
  				end
! 				lines = 1 if lines < 1
  				linelength = (bytes / lines).to_i
  				size = size.to_i + (bytes / ( linelength < 100 ? 1.39 : 1.0334 ))
***************
*** 205,214 ****
  		end
  		cachefile.close
! #		File.delete("#{$tempdir}/cache")
! 		debug "End mknewscache at #{`date`.chomp}"
  	end
  
  	def fetchnfos
  		`nice -n19 #{@nget} --path #{$tempdir} -G #{@_fullname} -K -L 150 -r '\.nfo|\.txt'`
  		`#{@nget} --path #{$tempdir} -qq -G #{@_fullname} -U -r '\.nfo|\.txt'`
  
--- 204,214 ----
  		end
  		cachefile.close
! 		debug "End mknewscache at #{Time.now.strftime('%H:%M:%S')}"
  	end
  
  	def fetchnfos
+ 		# fetches the nfos
  		`nice -n19 #{@nget} --path #{$tempdir} -G #{@_fullname} -K -L 150 -r '\.nfo|\.txt'`
+ 		# but mark them as if we didn't download them
  		`#{@nget} --path #{$tempdir} -qq -G #{@_fullname} -U -r '\.nfo|\.txt'`
  
***************
*** 217,221 ****
  		pwd = Dir.pwd
  		Dir.chdir($tempdir)
! 		Dir["ngettemp-*"].each { |file|
  			File.open(file).each { |line|
  				line =~ /^Message-ID: (.*)$/
--- 217,221 ----
  		pwd = Dir.pwd
  		Dir.chdir($tempdir)
! 		Dir['ngettemp-*'].each { |file|
  			File.open(file).each { |line|
  				line =~ /^Message-ID: (.*)$/

Index: nfo.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/nfo.rb,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** nfo.rb	6 Nov 2003 16:02:17 -0000	1.3
--- nfo.rb	2 Feb 2004 02:41:57 -0000	1.4
***************
*** 51,54 ****
--- 51,55 ----
  		end
  		sth.finish
+ 		puts "No nfos found." if i == 0
  	end	
  	

Index: queue.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/queue.rb,v
retrieving revision 1.18
retrieving revision 1.19
diff -C2 -d -r1.18 -r1.19
*** queue.rb	27 Dec 2003 18:13:36 -0000	1.18
--- queue.rb	2 Feb 2004 02:41:57 -0000	1.19
***************
*** 136,141 ****
  				s += ", finished : #{_date_finished}"
  		end
! 		if @period != 0
! 			s += " (period = #{_period}s)"
  		end
  		s
--- 136,141 ----
  				s += ", finished : #{_date_finished}"
  		end
! 		if @_period != 0
! 			s += " (period = #{@_period}s)"
  		end
  		s

Index: utils.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/utils.rb,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** utils.rb	3 Nov 2003 14:58:06 -0000	1.8
--- utils.rb	2 Feb 2004 02:41:57 -0000	1.9
***************
*** 24,28 ****
  
  	def Utils.debug(message=nil)
!   		puts "DEBUG: #{message}" if($config["debug"] && message)
  	end
  
--- 24,28 ----
  
  	def Utils.debug(message=nil)
!   		puts "DEBUG: #{message}" if($config['debug'] && message)
  	end