ngetsuite-cvs Mailing List for ngetsuite (Page 4)

Brought to you by: blackmor, jjie, ysc

ngetsuite-cvs — cvs commits

You can subscribe to this list here.

2003	_Jan	_Feb	_Mar	_Apr	_May	_Jun	_Jul	_Aug	_Sep	_Oct	_Nov	_Dec (7)
2004	_Jan (4)	_Feb (56)	_Mar (10)	_Apr (1)	_May (6)	_Jun (5)	_Jul (2)	_Aug	_Sep (22)	_Oct (7)	_Nov (1)	_Dec
2005	_Jan	_Feb	_Mar	_Apr	_May	_Jun	_Jul	_Aug	_Sep	_Oct	_Nov	_Dec (4)
2006	_Jan	_Feb	_Mar	_Apr (1)	_May	_Jun	_Jul	_Aug	_Sep	_Oct	_Nov	_Dec

Flat | Threaded

<< < 1 2 3 4 5 6 > >> (Page 4 of 6)

[Ngetsuite-cvs] ngetsuite/ngetsuite core.rb,1.32,1.33 downloadmanager.rb,1.21,1.22 group.rb,1.35,1.36

From: <jj...@us...> - 2004-02-10 01:20:47

Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv18893/ngetsuite

Modified Files:
	core.rb downloadmanager.rb group.rb 
Log Message:
added -G to search within releases, the daemon now retries the download of failed queues with a period defined (with 'resume_downloads')


Index: core.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/core.rb,v
retrieving revision 1.32
retrieving revision 1.33
diff -C2 -d -r1.32 -r1.33
*** core.rb	8 Feb 2004 12:55:48 -0000	1.32
--- core.rb	9 Feb 2004 21:34:49 -0000	1.33
***************
*** 68,71 ****
--- 68,76 ----
  	end
  
+ 	def search_releases(groupid, filter)	
+ 		g = Group.getgroup(groupid)
+ 		g.search_releases(filter)
+ 	end
+ 
  	def add_group(groupname, shortname = nil, dir = nil, desc = nil)
  		g = Group.new(groupname, false, shortname, dir, true, desc)
***************
*** 253,256 ****
--- 258,264 ----
  						search_group(*args)
  					end
+ 				when 'search-releases'
+ 					args = arg.split(',')
+ 					search_releases(*args)
  				when 'set-period'
  					args = arg.split(',')

Index: downloadmanager.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloadmanager.rb,v
retrieving revision 1.21
retrieving revision 1.22
diff -C2 -d -r1.21 -r1.22
*** downloadmanager.rb	7 Feb 2004 16:21:20 -0000	1.21
--- downloadmanager.rb	9 Feb 2004 21:34:49 -0000	1.22
***************
*** 60,64 ****
  	def start_new_downloader
  		row = Db.dbh.select_one("select * from queue where `status`='PENDING' order by `priority` desc limit 1")
! 		row = Db.dbh.select_one("select * from queue where `status`='STOPPED' order by `priority` desc limit 1") if not row and $config['resume_downloads']
  		
  		return false if row == nil
--- 60,65 ----
  	def start_new_downloader
  		row = Db.dbh.select_one("select * from queue where `status`='PENDING' order by `priority` desc limit 1")
! 		row = Db.dbh.select_one("select * from queue where (`status`='STOPPED' and (`period` != 0) and ((NOW()-`date_finished`) > `period`)) order by `priority` desc limit 1") if not row and $config['resume_downloads']
! 		row = Db.dbh.select_one("select * from queue where (`status`='FAILED' and (`period` != 0) and ((NOW()-`date_finished`) > `period`)) order by `priority` desc limit 1") if not row and $config['resume_downloads']
  		
  		return false if row == nil

Index: group.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/group.rb,v
retrieving revision 1.35
retrieving revision 1.36
diff -C2 -d -r1.35 -r1.36
*** group.rb	9 Feb 2004 15:32:58 -0000	1.35
--- group.rb	9 Feb 2004 21:34:49 -0000	1.36
***************
*** 127,130 ****
--- 127,143 ----
  	end
  	
+ 	def search_releases(filter)
+ 		sth = Db.dbh.execute('select articles.subject,releases.rlsmap,releases.nbactual,releases.nbexpected from releases,articles where articles.mid = releases.mid and articles.group = ? and articles.subject RLIKE ? order by articles.time desc', @_fullname, filter)
+ 		i = 0
+ 		subj, rlsmap, nbact, nbtot = nil
+ 		sth.fetch do |row|
+ 			subj, rlsmap, nbact, nbtot = *row
+ 			puts "#{yellow nbact}/#{yellow nbtot} [#{green rlsmap}] \"#{subj}\""
+ 			i += 1
+ 		end
+ 		sth.finish
+ 		puts "#{green i} matche#{i>1 ? 's' : ''} found"
+ 	end
+ 
  	def search(filter)
  		sth = Db.dbh.execute('select * from `articles` where `group` = ? and `subject` RLIKE ? order by `time` desc', @_fullname, filter)

[Ngetsuite-cvs] ngetsuite/ngetsuite core.rb,1.31,1.32 group.rb,1.33,1.34 queue.rb,1.22,1.23

From: <jj...@us...> - 2004-02-08 12:59:05

Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv7965/ngetsuite

Modified Files:
	core.rb group.rb queue.rb 
Log Message:
added -p to change the period of queues, added an arg to -r to allow to download in a subdir of basedir/groupdir, fixed the initial value of the period


Index: core.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/core.rb,v
retrieving revision 1.31
retrieving revision 1.32
diff -C2 -d -r1.31 -r1.32
*** core.rb	7 Feb 2004 16:21:20 -0000	1.31
--- core.rb	8 Feb 2004 12:55:48 -0000	1.32
***************
*** 119,123 ****
  	end
  
! 	def queue_regexp(groupid, regexp, prio = '5', period = '0')
  		begin
  			n = Integer(prio)
--- 119,123 ----
  	end
  
! 	def queue_regexp(groupid, regexp, prio='5', period='0', subpath='')
  		begin
  			n = Integer(prio)
***************
*** 128,132 ****
  		name = Group.getname(groupid)
  		g = Group.new(name)
! 		g.queue_regexp(regexp, n)
  	end
  
--- 128,132 ----
  		name = Group.getname(groupid)
  		g = Group.new(name)
! 		g.queue_regexp(regexp, n, period, subpath)
  	end
  
***************
*** 144,147 ****
--- 144,153 ----
  	end
  
+ 	def set_period(queueid, period)
+ 		q = Queue.getqueue(queueid)
+ 		q._period = period
+ 		q.sync_to_db
+ 	end
+ 
  	def set_priority(queueid, prio)
  		q = Queue.getqueue(queueid)
***************
*** 228,232 ****
  					args = arg.split(',')
  					size = args.size
! 					if ((size >= 2) && (size <= 4))
  						queue_regexp(*args)
  					else
--- 234,238 ----
  					args = arg.split(',')
  					size = args.size
! 					if ((size >= 2) && (size <= 5))
  						queue_regexp(*args)
  					else
***************
*** 247,250 ****
--- 253,263 ----
  						search_group(*args)
  					end
+ 				when 'set-period'
+ 					args = arg.split(',')
+ 					if (args.size == 2)
+ 						set_period(*args)
+ 					else
+ 						raise ArgumentError.new("Incorrect args : #{arg}")
+ 					end
  				when 'set-priority'
  					args = arg.split(',')

Index: group.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/group.rb,v
retrieving revision 1.33
retrieving revision 1.34
diff -C2 -d -r1.33 -r1.34
*** group.rb	7 Feb 2004 16:21:20 -0000	1.33
--- group.rb	8 Feb 2004 12:55:48 -0000	1.34
***************
*** 166,171 ****
  	end
  	
! 	def queue_regexp(regexp, prio)
! 		q = RegexpQueue.new(nil, false, @_fullname, regexp, prio)
  		q._date_added = Time.now
  		q._status = NgetSuite::PENDING
--- 166,171 ----
  	end
  	
! 	def queue_regexp(regexp, prio, period=0, subpath='')
! 		q = RegexpQueue.new(nil, false, @_fullname, regexp, prio, period, subpath)
  		q._date_added = Time.now
  		q._status = NgetSuite::PENDING
***************
*** 229,232 ****
--- 229,233 ----
  		Dir.mkdir tempdir
  		pwd = Dir.pwd
+ 		Dir.chdir(tempdir)
  		begin
  		# fetches the nfos
***************
*** 234,241 ****
  		# but mark them for nget as if we didn't download them
  		`#{@nget} --path #{tempdir} -qq -G #{@_fullname} -U -r '\\.nfo|\\.txt'`
! 		debug "Fetchnfos: files downloaded in #{Time.now-t}s"
  
  		sth = Db.dbh.prepare 'insert ignore into nfos (`group`, `mid`, `text`) values (?, ?, ?)'
- 		Dir.chdir(tempdir)
  		Dir['ngettemp-*'].each { |file|
  			File.open(file).each { |line|
--- 235,242 ----
  		# but mark them for nget as if we didn't download them
  		`#{@nget} --path #{tempdir} -qq -G #{@_fullname} -U -r '\\.nfo|\\.txt'`
! 		debug "Fetchnfos: #{Dir['ngettemp-*'].length} files downloaded in #{Time.now-t}s"
  
+ 		nb_nfos = 0
  		sth = Db.dbh.prepare 'insert ignore into nfos (`group`, `mid`, `text`) values (?, ?, ?)'
  		Dir['ngettemp-*'].each { |file|
  			File.open(file).each { |line|
***************
*** 244,248 ****
  					mid = $1
  					text = `#{@yydecode} #{file} -o -`.chomp
! 					sth.execute(@_fullname, mid, text) if text and text.length < 15000
  					# Does not fucking work
  					#if text and (text.length < 15000)
--- 245,252 ----
  					mid = $1
  					text = `#{@yydecode} #{file} -o -`.chomp
! 					if text and text.length < 15000
! 						sth.execute(@_fullname, mid, text)
! 						nb_nfos += 1
! 					end
  					# Does not fucking work
  					#if text and (text.length < 15000)
***************
*** 259,263 ****
  			`rm -rf #{tempdir}`
  		end
! 		debug "Fetchnfos finished in #{Time.now-t}s"
  	end
  
--- 263,267 ----
  			`rm -rf #{tempdir}`
  		end
! 		debug "Indexed #{nb_nfos} nfos in #{Time.now-t}s"
  	end
  

Index: queue.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/queue.rb,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -d -r1.22 -r1.23
*** queue.rb	7 Feb 2004 16:21:20 -0000	1.22
--- queue.rb	8 Feb 2004 12:55:48 -0000	1.23
***************
*** 2,5 ****
--- 2,6 ----
  require 'ngetsuite/db'
  require 'ngetsuite/queue_file'
+ require 'ftools'
  
  module NgetSuite	
***************
*** 110,114 ****
  	end
  
! 	def initialize(id, fetch_from_db = false, group = nil, value = nil, priority = 1, period = 0)
  		@_id = id
  		@_group = group
--- 111,115 ----
  	end
  
! 	def initialize(id, fetch_from_db = false, group='', value='', priority=1, period=0, subpath = '')
  		@_id = id
  		@_group = group
***************
*** 117,122 ****
  		@_period = period
  		@_subdir = group
! 		if group
! 			@_subdir = Group.getgroup(group)._directory.sub(/#{$config['basedir']}\/*/, '')
  		end
  		update_from_db if fetch_from_db
--- 118,132 ----
  		@_period = period
  		@_subdir = group
! 		if group != ''
! 			@_subdir = Group.getgroup(group)._directory.sub(/#{$config['basedir']}(#{File::Separator})*/, '')
! 			if subpath != ''
! 				@_subdir += File::Separator + subpath
! 				dir = $config['basedir'] + File::Separator + @_subdir
! 				if FileTest.exists? dir
! 					puts "The download target #{dir} already exists"
! 				else
! 					File.makedirs dir
! 				end
! 			end
  		end
  		update_from_db if fetch_from_db
***************
*** 156,162 ****
  	public_class_method :new
  
! 	def initialize(id, fetch_from_db = false, group = nil, value = nil, priority = 1, period = 0)
  		@_type = REGEXP
! 		super(id, fetch_from_db, group, value, priority, period)
  	end
  
--- 166,172 ----
  	public_class_method :new
  
! 	def initialize(*args)
  		@_type = REGEXP
! 		super(*args)
  	end
  
***************
*** 169,175 ****
  	public_class_method :new
  
! 	def initialize(id, fetch_from_db = false, group = nil, value = nil, priority = 1, period = 0)
  		@_type = SINGLE
! 		super(id, fetch_from_db, group, value, priority, period)
  	end
  
--- 179,185 ----
  	public_class_method :new
  
! 	def initialize(*args)
  		@_type = SINGLE
! 		super(*args)
  	end

[Ngetsuite-cvs] ngetsuite ngetsuite.rb,1.17,1.18

From: <jj...@us...> - 2004-02-08 12:59:05

Update of /cvsroot/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv7965

Modified Files:
	ngetsuite.rb 
Log Message:
added -p to change the period of queues, added an arg to -r to allow to download in a subdir of basedir/groupdir, fixed the initial value of the period


Index: ngetsuite.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite.rb,v
retrieving revision 1.17
retrieving revision 1.18
diff -C2 -d -r1.17 -r1.18
*** ngetsuite.rb	7 Feb 2004 16:21:20 -0000	1.17
--- ngetsuite.rb	8 Feb 2004 12:55:47 -0000	1.18
***************
*** 23,27 ****
  	[ '--show-nfo', '-n', GetoptLong::REQUIRED_ARGUMENT, 'nb_nfo[,groupid]', 'Prints the specified nfo' ],
  	[ '--queue-article', '-q', GetoptLong::REQUIRED_ARGUMENT, 'groupid,articleid', 'Adds an article to the download queue'],
! 	[ '--queue-regexp', '-r', GetoptLong::REQUIRED_ARGUMENT, 'gid,regexp[,prio,period]', 'Adds a regexp to the download queue' ],
  	[ '--show-queue', '-t', GetoptLong::OPTIONAL_ARGUMENT, '[groupid]', 'Shows the current download queue'],
  	[ '--start-manager', '-m', GetoptLong::NO_ARGUMENT, 'Launches the download manager'],
--- 23,27 ----
  	[ '--show-nfo', '-n', GetoptLong::REQUIRED_ARGUMENT, 'nb_nfo[,groupid]', 'Prints the specified nfo' ],
  	[ '--queue-article', '-q', GetoptLong::REQUIRED_ARGUMENT, 'groupid,articleid', 'Adds an article to the download queue'],
! 	[ '--queue-regexp', '-r', GetoptLong::REQUIRED_ARGUMENT, 'gid,regexp[,prio,period,dir]', 'Adds a regexp to the download queue' ],
  	[ '--show-queue', '-t', GetoptLong::OPTIONAL_ARGUMENT, '[groupid]', 'Shows the current download queue'],
  	[ '--start-manager', '-m', GetoptLong::NO_ARGUMENT, 'Launches the download manager'],
***************
*** 29,32 ****
--- 29,33 ----
  	[ '--search', '-g', GetoptLong::REQUIRED_ARGUMENT, 'groupid,filter', 'Searches the headers matching a filter for a group' ],
  	[ '--set-priority', '-f', GetoptLong::REQUIRED_ARGUMENT, 'queueid,prio', 'Changes the priority of the specified queue' ],
+ 	[ '--set-period', '-p', GetoptLong::REQUIRED_ARGUMENT, 'queueid,period', 'Changes the period of the specified queue' ],
  	[ '--set-status', '-i', GetoptLong::REQUIRED_ARGUMENT, 'queueid,status', 'Changes the status of the specified queue'],
  	[ '--delete-queue', '-x', GetoptLong::REQUIRED_ARGUMENT, 'queueid', 'Delete the specified queue'],

[Ngetsuite-cvs] ngetsuite/ngetsuite articles.rb,1.3,1.4 core.rb,1.30,1.31 downloadmanager.rb,1.20,1.21 group.rb,1.32,1.33 queue.rb,1.21,1.22 queue_file.rb,1.14,1.15

From: <jj...@us...> - 2004-02-07 16:24:13

Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv18697/ngetsuite

Modified Files:
	articles.rb core.rb downloadmanager.rb group.rb queue.rb 
	queue_file.rb 
Log Message:
typos
fixed the download directory to the directory of the group instead of the group name
added -L option to list the releases (beta)


Index: articles.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/articles.rb,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** articles.rb	6 Feb 2004 23:46:59 -0000	1.3
--- articles.rb	7 Feb 2004 16:21:20 -0000	1.4
***************
*** 4,8 ****
  module NgetSuite
  
- 
  class ArticleList < Db::DbList
  	def getclass
--- 4,7 ----
***************
*** 13,17 ****
  
  class Article
- 	include Utils
  	include Db
  
--- 12,15 ----
***************
*** 38,42 ****
  
  class Release
- 	include Utils
  	include Db
  	
--- 36,39 ----
***************
*** 72,78 ****
  	# if guessed from the subject (0 if not)
  	def Release.filenb_from_subject(subject)
! 		subject =~ /[\[\(]?(\d+)([\/]|\s*of\s*)(\d+)[\]\)]?/
  
! 		return [$1.to_i, $3.to_i]
  	end
  	
--- 69,75 ----
  	# if guessed from the subject (0 if not)
  	def Release.filenb_from_subject(subject)
! 		subject =~ /[\[\(]?(\d+)(?:[\/]|\s*of\s*)(\d+)[\]\)]?/
  
! 		return [$1.to_i, $2.to_i]
  	end
  	
***************
*** 166,171 ****
  	end
  
  	def to_s
! 		"#{@_mid}: #{@_nbactual}/#{@_nbexpected} [#{@_rlsmap}]"
  	end
  	
--- 163,194 ----
  	end
  
+ 	def Release.list(groupid = '')
+ 		req = 'select articles.subject,articles.group,releases.rlsmap,releases.nbactual,releases.nbexpected from releases,articles where articles.mid = releases.mid'
+ 		
+ 		if (groupid != '')
+ 			name = Group.getname(groupid)
+ 			req += " and articles.group = '#{name}'"
+ 		end
+ 
+ 		req += ' order by articles.time asc'
+ 
+ 		sth = Db.dbh.execute req
+ 
+ 		i = 0
+ 		subj, grp, rlsmap, nbact, nbtot = nil
+ 		#rls = nil
+ 		sth.fetch do |row|
+ 			#rls = Release.new(name, row[2].to_s, true)
+ 			#puts "[#{i}] #{rls} (#{row[0]})"
+ 			subj, grp, rlsmap, nbact, nbtot = *row
+ 			puts "[#{Utils.blue i}] #{Utils.yellow nbact}/#{Utils.yellow nbtot} [#{Utils.green rlsmap}] \"#{subj}\""
+ 			i += 1
+ 		end
+ 		sth.finish
+ 		puts "No release found." if i == 0
+ 	end	
+ 	
  	def to_s
! 		"#{Utils.green @_mid}: #{@_nbactual}/#{@_nbexpected} [#{Utils.yellow @_rlsmap}]"
  	end
  	

Index: core.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/core.rb,v
retrieving revision 1.30
retrieving revision 1.31
diff -C2 -d -r1.30 -r1.31
*** core.rb	6 Feb 2004 23:46:59 -0000	1.30
--- core.rb	7 Feb 2004 16:21:20 -0000	1.31
***************
*** 4,7 ****
--- 4,8 ----
  require 'ngetsuite/group'
  require 'ngetsuite/nfo'
+ require 'ngetsuite/articles'
  require 'dbi'
  require 'drb'
***************
*** 105,108 ****
--- 106,113 ----
  	end
  
+ 	def list_releases(groupid = '')
+ 		Release.list(groupid)
+ 	end
+ 	
  	def queue_article(groupid, article_id, prio = '5')
  		begin
***************
*** 202,205 ****
--- 207,212 ----
  				when 'list-nfos'
  					list_nfos(arg)
+ 				when 'list-releases'
+ 					list_releases(arg)
  				when 'show-nfo'
  					args = arg.split(',')

Index: downloadmanager.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloadmanager.rb,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** downloadmanager.rb	6 Feb 2004 23:46:59 -0000	1.20
--- downloadmanager.rb	7 Feb 2004 16:21:20 -0000	1.21
***************
*** 66,70 ****
  		@current_dlers += 1
  		id, group, type, value, subdir, prio = row.values_at(0, 1, 2, 3, 4, 5)
! 		log "found #{value} for group #{group}\n"
  		dler = Downloader.new(id, type, value, group, @basedir + File::Separator + subdir, prio)
  		@dlers[id] = dler
--- 66,70 ----
  		@current_dlers += 1
  		id, group, type, value, subdir, prio = row.values_at(0, 1, 2, 3, 4, 5)
! 		log "found /#{value}/ for group #{group}\n"
  		dler = Downloader.new(id, type, value, group, @basedir + File::Separator + subdir, prio)
  		@dlers[id] = dler

Index: group.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/group.rb,v
retrieving revision 1.32
retrieving revision 1.33
diff -C2 -d -r1.32 -r1.33
*** group.rb	7 Feb 2004 00:21:15 -0000	1.32
--- group.rb	7 Feb 2004 16:21:20 -0000	1.33
***************
*** 282,286 ****
  
  		rls.insert_to_db
! 		debug "Inserted #{nb_releases} in #{Time.now-t}s"
  	end
  	
--- 282,286 ----
  
  		rls.insert_to_db
! 		debug "Inserted #{nb_releases} releases in #{Time.now-t}s"
  	end
  	

Index: queue.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/queue.rb,v
retrieving revision 1.21
retrieving revision 1.22
diff -C2 -d -r1.21 -r1.22
*** queue.rb	6 Feb 2004 23:46:59 -0000	1.21
--- queue.rb	7 Feb 2004 16:21:20 -0000	1.22
***************
*** 117,120 ****
--- 117,123 ----
  		@_period = period
  		@_subdir = group
+ 		if group
+ 			@_subdir = Group.getgroup(group)._directory.sub(/#{$config['basedir']}\/*/, '')
+ 		end
  		update_from_db if fetch_from_db
  	end

Index: queue_file.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/queue_file.rb,v
retrieving revision 1.14
retrieving revision 1.15
diff -C2 -d -r1.14 -r1.15
*** queue_file.rb	2 Feb 2004 17:44:19 -0000	1.14
--- queue_file.rb	7 Feb 2004 16:21:20 -0000	1.15
***************
*** 23,28 ****
  	# database and returns the number of rows deleted
  	def QueueFile.clean(queueid)
! 		rows = Dbh.dbh.do "delete * from `queue_files`
! 			where `queue_id` = '#{queueid}'"
  		return rows
  	end
--- 23,27 ----
  	# database and returns the number of rows deleted
  	def QueueFile.clean(queueid)
! 		rows = Db.dbh.do "delete from `queue_files` where `queue_id` = '#{queueid}'"
  		return rows
  	end

[Ngetsuite-cvs] ngetsuite ngetsuite.rb,1.16,1.17

From: <jj...@us...> - 2004-02-07 16:24:13

Update of /cvsroot/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv18697

Modified Files:
	ngetsuite.rb 
Log Message:
typos
fixed the download directory to the directory of the group instead of the group name
added -L option to list the releases (beta)


Index: ngetsuite.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite.rb,v
retrieving revision 1.16
retrieving revision 1.17
diff -C2 -d -r1.16 -r1.17
*** ngetsuite.rb	6 Feb 2004 23:46:59 -0000	1.16
--- ngetsuite.rb	7 Feb 2004 16:21:20 -0000	1.17
***************
*** 20,23 ****
--- 20,24 ----
  	[ '--update-all-groups', '-v', GetoptLong::NO_ARGUMENT, 'Update headers for all groups'],
  	[ '--list-nfos', '-l', GetoptLong::OPTIONAL_ARGUMENT, '[groupid]', 'Lists already fetched nfos' ],
+ 	[ '--list-releases', '-L', GetoptLong::OPTIONAL_ARGUMENT, '[groupid]', 'Lists detected releases' ],
  	[ '--show-nfo', '-n', GetoptLong::REQUIRED_ARGUMENT, 'nb_nfo[,groupid]', 'Prints the specified nfo' ],
  	[ '--queue-article', '-q', GetoptLong::REQUIRED_ARGUMENT, 'groupid,articleid', 'Adds an article to the download queue'],

[Ngetsuite-cvs] ngetsuite/ngetsuite group.rb,1.31,1.32

From: <bla...@us...> - 2004-02-07 00:24:08

Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv21535

Modified Files:
	group.rb 
Log Message:
trivial fix

Index: group.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/group.rb,v
retrieving revision 1.31
retrieving revision 1.32
diff -C2 -d -r1.31 -r1.32
*** group.rb	6 Feb 2004 23:46:59 -0000	1.31
--- group.rb	7 Feb 2004 00:21:15 -0000	1.32
***************
*** 151,155 ****
  
  	def clear_articles
! 		nb_rows = Db.dbh.execute "delete from `articles` where `group` = '#{@_fullname}'"
  		debug "#{nb_rows} articles deleted"
  	end
--- 151,155 ----
  
  	def clear_articles
! 		nb_rows = Db.dbh.do "delete from `articles` where `group` = '#{@_fullname}'"
  		debug "#{nb_rows} articles deleted"
  	end

[Ngetsuite-cvs] ngetsuite/ngetsuite articles.rb,1.2,1.3 config.rb,1.3,1.4 core.rb,1.29,1.30 db.rb,1.22,1.23 downloader.rb,1.21,1.22 downloadmanager.rb,1.19,1.20 group.rb,1.30,1.31 ngetcache.rb,1.6,1.7 queue.rb,1.20,1.21 utils.rb,1.11,1.12 web.rb,1.4,1.5

From: <jj...@us...> - 2004-02-06 23:50:02

Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv11482/ngetsuite

Modified Files:
	articles.rb config.rb core.rb db.rb downloader.rb 
	downloadmanager.rb group.rb ngetcache.rb queue.rb utils.rb 
	web.rb 
Log Message:
config: defaults are in config.rb (no need to .fetch), added 'daemon_debug', 'daemon_host', 'dbhost' ; changed 'autostart_httpd' to 'httpd_autostart', 'extern_nfo_update' to 'extern_nfos_update', 'port' to 'daemon_port' ; reqesting a config keyword inexisting raises an exception
group: somewhat gives a success status, fixed shortname, *sexy* headers update
global: removed some global variables ($dbh->Db.dbh..)


Index: articles.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/articles.rb,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** articles.rb	4 Feb 2004 15:40:22 -0000	1.2
--- articles.rb	6 Feb 2004 23:46:59 -0000	1.3
***************
*** 167,171 ****
  
  	def to_s
! 		@_mid + ": " + @_nbactual.to_s + "/" + @_nbexpected.to_s + " [" + @_rlsmap + ']'
  	end
  	
--- 167,171 ----
  
  	def to_s
! 		"#{@_mid}: #{@_nbactual}/#{@_nbexpected} [#{@_rlsmap}]"
  	end
  	

Index: config.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/config.rb,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** config.rb	2 Feb 2004 02:41:34 -0000	1.3
--- config.rb	6 Feb 2004 23:46:59 -0000	1.4
***************
*** 4,30 ****
    # just treat it like a hash
    class Config < Hash
- 
      # create a new config hash from @file
      def initialize(file)
        super(false)
        @file = file
!       if(File.exist?("#{@file}"))
!         IO.foreach("#{@file}") do |line|
!           next if(line =~ /^\s*(#|$)/)
!           if(line =~ /^\s*(\S+)\s*=\s*(.*)$/)
!             self[$1] = eval($2) if($2)
!           end
          end
        end
      end
! 
      # write current configuration to @file
!     def save
!       File.open("#{@file}", 'w') do |file|
!         self.each do |key, value|
            file.puts "#{key} = #{value}"
          end
        end
      end
    end
  
--- 4,66 ----
    # just treat it like a hash
    class Config < Hash
      # create a new config hash from @file
      def initialize(file)
        super(false)
        @file = file
!       raise RuntimeError.new("unable to read #{@file}") unless File.readable?("#{@file}")
!       # default values
! "
! dbhost = 'localhost'
! dbuser = 'nget'
! dbpass = 'mypass'
! dbname = 'ngetcache'
! nget = '/home/nget/bin/nget'
! yydecode = '/usr/bin/yydecode'
! cachedir = ENV['HOME'] + File::Separator + '.nget4'
! basedir = ENV['HOME'] + File::Separator + 'ngot'
! tempdir = ENV['TMP'] or ENV['TEMP'] or '/tmp'
! max_dls = 3
! daemon_port = 15345
! daemon_host = 'localhost'
! daemon_log = ENV['HOME'] + File::Separator + 'ngot' + File::Separator + 'ngetsuite-daemon.log'
! resume_downloads = false
! daemon_autostart = true
! debug = false
! daemon_debug = false
! extern_headers_update = ''
! extern_nfos_fetch = ''
! httpd_port = 2000
! httpd_autostart = true
! ".each("\n") { |l| self[$1] = eval($2) if l=~ /^(\S+)\s*=\s*(.*)$/ }
!       IO.foreach(@file) do |line|
!         next if(line =~ /^\s*(#|$)/)
!         if(line =~ /^\s*(\S+)\s*=\s*(.*)$/)
!           self[$1] = eval($2) if($2)
          end
        end
      end
!     
      # write current configuration to @file
!     def save(filename)
!       if FileTest.exists? filename
!         puts "Cannot overwrite an existing file"
!         return
!       end
!       File.open(filename, 'w') do |file|
!         self.keys.sort.each do |key|
! 	  value = self[key]
! 	  value = "'"+value+"'" if value.class == String
            file.puts "#{key} = #{value}"
          end
        end
      end
+ 
+     def [](key)
+       begin
+         fetch(key)
+       rescue IndexError
+         raise RuntimeError.new("Config file key '#{key}' not found (found #{self.keys.map{|k| "'#{k}'"}.sort.join(', ')})")
+       end
+     end
    end
  

Index: core.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/core.rb,v
retrieving revision 1.29
retrieving revision 1.30
diff -C2 -d -r1.29 -r1.30
*** core.rb	4 Feb 2004 05:56:50 -0000	1.29
--- core.rb	6 Feb 2004 23:46:59 -0000	1.30
***************
*** 1,8 ****
! require 'drb'
! require 'getoptlong'
  require 'ngetsuite/utils'
  require 'ngetsuite/queue'
  require 'ngetsuite/group'
  require 'ngetsuite/nfo'
  
  module NgetSuite
--- 1,9 ----
! # todo: move requires in the blocks needing them (move exception handling as well)
  require 'ngetsuite/utils'
  require 'ngetsuite/queue'
  require 'ngetsuite/group'
  require 'ngetsuite/nfo'
+ require 'dbi'
+ require 'drb'
  
  module NgetSuite
***************
*** 53,60 ****
  	def get_manager
  		DRb.start_service()
! 		port = $config.fetch('port', 12345)
! 		manager = DRbObject.new(nil, 'druby://localhost:' + port.to_s)
  		if !manager
! 			raise ManagerNotFoundException.new("localhost:#{port}")
  		end
  		manager
--- 54,61 ----
  	def get_manager
  		DRb.start_service()
! 		hp = "#{$config['daemon_host']}:#{$config['daemon_port']}"
! 		manager = DRbObject.new(nil, "druby://#{hp}")
  		if !manager
! 			raise ManagerNotFoundException.new(hp)
  		end
  		manager
***************
*** 77,82 ****
  
  	def delete_group(groupid)
! 		name = Group.getname(groupid)
! 		$dbh.do("delete from `groups` where `fullname` = \"#{name}\"")
  	end
  
--- 78,82 ----
  
  	def delete_group(groupid)
! 		Group.delete(groupid)
  	end
  
***************
*** 84,90 ****
  		if !groupid
  			puts "Update all groups"
! 			groups = $dbh.select_all 'select `fullname` from `groups`
! 				where active="yes"'
! 			groups.each do |group|
  				name = group.to_s
  				puts "Updating #{name}"
--- 84,88 ----
  		if !groupid
  			puts "Update all groups"
! 			Group.getallactive.each do |group|
  				name = group.to_s
  				puts "Updating #{name}"
***************
*** 156,176 ****
  	def printusage
  		print "Usage: #{$0} command [options]\n\n"
! 		i = 0
! 		puts "List of options:"
! 		$args.each { |item|
! 			# Uglllllllly
! 			if i == 3
  				puts "\nList of actions:"
  			end
! 			full, short, type = item.values_at(0, 1, 2)
! 			left = "\t#{full}, #{short}"
! 			if type != GetoptLong::NO_ARGUMENT
! 				left += " #{item[3]}"
  				desc = item[4]
  			else
  				desc = item[3]
  			end
! 			printf("%-45.45s\t%s\n", left, desc)
! 			i += 1
  		}
  	end
--- 154,171 ----
  	def printusage
  		print "Usage: #{$0} command [options]\n\n"
! 		puts 'List of options:'
! 		$args.each_index { |i|
! 			unless item = $args[i]
  				puts "\nList of actions:"
+ 				next
  			end
! 			arg = "\t#{item[0]}, #{item[1]}"
! 			if item.length == 5
! 				arg += " #{item[3]}"
  				desc = item[4]
  			else
  				desc = item[3]
  			end
! 			printf("%-45.45s\t%s\n", arg, desc)
  		}
  	end
***************
*** 183,186 ****
--- 178,187 ----
  			begin
  				case opt
+ 				when 'debug'
+ 					$config['debug']=true
+ 				when 'no-debug'
+ 					$config['no-debug']=false
+ 				when 'write-config'
+ 					$config.save(arg)
  				when 'show-groups'
  					Group.show
***************
*** 291,294 ****
--- 292,298 ----
  			rescue NoSuchNfoException => e
  				puts e.message
+ 			rescue Exception => e
+ 				puts e.message
+ 				print e.backtrace.join("\n")
  			rescue ArgumentError => e
  				puts e.message
***************
*** 302,308 ****
  				puts "Problem connecting to the manager"
  				puts e
- 			rescue Exception => e
- 				puts e.message
- 				print e.backtrace.join("\n")
  			ensure
  				# Nothing
--- 306,309 ----

Index: db.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/db.rb,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -d -r1.22 -r1.23
*** db.rb	3 Feb 2004 21:31:32 -0000	1.22
--- db.rb	6 Feb 2004 23:46:59 -0000	1.23
***************
*** 1,6 ****
  module NgetSuite
  module Db
  	@@db_handler = nil
! 		
  	## Exceptions
  	class TableNotFoundException < Exception
--- 1,7 ----
+ require 'dbi'
  module NgetSuite
  module Db
  	@@db_handler = nil
! 
  	## Exceptions
  	class TableNotFoundException < Exception
***************
*** 132,137 ****
  	def Db.dbh
  		unless @@db_handler and @@db_handler.connected?
! 			@@db_handler = DBI.connect("DBI:Mysql:#{$config['dbname']}",
! 				$config['dbuser'], $config['dbpass'])
  		end
  
--- 133,137 ----
  	def Db.dbh
  		unless @@db_handler and @@db_handler.connected?
! 			@@db_handler = DBI.connect("DBI:Mysql:#{$config['dbname']}:#{$config['dbhost']}", $config['dbuser'], $config['dbpass'])
  		end
  
***************
*** 215,219 ****
  		req += " from #{table} where " + ids_constraint
  		
! 		debug 'DB: '+ req
  		sth = Db.dbh.execute req
  		hash = sth.fetch_hash
--- 215,219 ----
  		req += " from #{table} where " + ids_constraint
  		
! #		debug 'DB: '+ req
  		sth = Db.dbh.execute req
  		hash = sth.fetch_hash
***************
*** 283,287 ****
  		req += ')'
  		
! 		debug 'DB: ' + req
  		exec_request(req, values)
  	end
--- 283,287 ----
  		req += ')'
  		
! #		debug 'DB: ' + req
  		exec_request(req, values)
  	end

Index: downloader.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloader.rb,v
retrieving revision 1.21
retrieving revision 1.22
diff -C2 -d -r1.21 -r1.22
*** downloader.rb	4 Feb 2004 15:21:48 -0000	1.21
--- downloader.rb	6 Feb 2004 23:46:59 -0000	1.22
***************
*** 1,2 ****
--- 1,3 ----
+ require 'ngetsuite/db'
  require 'ngetsuite/queue_file'
  require 'ngetsuite/utils'
***************
*** 37,52 ****
  		@status = 'PENDING'
  		@nbfiles = 0
  	end
  
  	def update_status(line)
  		line.chomp!
  		case line
  		when /^already have article (<.*>) \(.*\/([^\/]*)\)$/
! 			debug yellow('file already there') + ' : ' + green($1) + ' in ' + blue($2)
  			@nbfiles += 1
  		when /^Retrieving: (\d+).*\t(\d+)l.*(<.*>)$/
  			parts, nblines, mid = $1, $2, $3
  			@nbfiles += 1
! 			$dbh.do "update queue set `nbfiles`='#{@nbfiles}' where `id`='#{@id}'"
  
  			@curfile = QueueFile.new(@id, mid, false, parts, nblines, @nbfiles)
--- 38,58 ----
  		@status = 'PENDING'
  		@nbfiles = 0
+ 		@finish_status = 'FAILED'
  	end
  
+ 	def ldebug(str)
+ 		debug "(#{white @id})" + str
+ 	end
+ 	
  	def update_status(line)
  		line.chomp!
  		case line
  		when /^already have article (<.*>) \(.*\/([^\/]*)\)$/
! 			ldebug "already have #{green $1} in #{blue $2}"
  			@nbfiles += 1
  		when /^Retrieving: (\d+).*\t(\d+)l.*(<.*>)$/
  			parts, nblines, mid = $1, $2, $3
  			@nbfiles += 1
! 			Db.dbh.do "update queue set `nbfiles`='#{@nbfiles}' where `id`='#{@id}'"
  
  			@curfile = QueueFile.new(@id, mid, false, parts, nblines, @nbfiles)
***************
*** 55,59 ****
  			else
  				@curfile.update_from_db
! 				debug (Utils.red "already in database")
  			end
  		when /^(\d+) \((\d+)\/(\d+)\): (\d+)\/(\d+)L (\d+)\/(\d+)B (\d+)% (\d+)B\/s (\w+) (\d+)\/(\d+) (\w+)$/
--- 61,65 ----
  			else
  				@curfile.update_from_db
! 				ldebug (Utils.red "already in database")
  			end
  		when /^(\d+) \((\d+)\/(\d+)\): (\d+)\/(\d+)L (\d+)\/(\d+)B (\d+)% (\d+)B\/s (\w+) (\d+)\/(\d+) (\w+)$/
***************
*** 62,69 ****
  			speed = $9
  			filenb = $12
! 			debug "#{yellow $5+'L'}(#{yellow $7+'B'}) at #{red $9+'B/s'} [#{blue $2}/#{blue $3}] [#{green $11}/#{green $12}]"
  			@curfile.update(part, speed, filenb, size)
  		when /\d+ \((\d+)\/\d+\).*\d+\/(\d+)B.* (\d+)B\/s \w+ (\d+)\/(\d+) \w+$/
! 			debug (Utils.red "the previous regexp should have been matched instead of this one")
  			part = $1
  			size = $2
--- 68,75 ----
  			speed = $9
  			filenb = $12
! 			ldebug "#{yellow $5+'L'}(#{yellow $7+'B'}) at #{red $9+'B/s'} [#{blue $2}/#{blue $3}] [#{green $11}/#{green $12}]"
  			@curfile.update(part, speed, filenb, size)
  		when /\d+ \((\d+)\/\d+\).*\d+\/(\d+)B.* (\d+)B\/s \w+ (\d+)\/(\d+) \w+$/
! 			ldebug (Utils.red "the previous regexp should have been matched instead of this one")
  			part = $1
  			size = $2
***************
*** 71,100 ****
  			filenb = $4
  			@curfile.update(part, speed, filenb, size)
! 		when /^autopar.*seems like a par2 \((.*)\)/
! 			debug 'autopar: found a par2 for ' + blue($1)
! 			
  		when /^uu_msg\(\d+\):Loaded from .*: '.*' \(.*\): (.*) part \d+.*end (.*)$/
! 			debug (Utils.yellow "filename : #{$1}, encoding : #{$2}")
  			@curfile._filename = $1
  			@curfile.encoding = $2
  		when /^decoded ok/
! 			debug 'file decoded'
  			@curfile.finish(true)
  			log "Download finished for file #{@curfile._filename}, queue #{@value}"
  		when /^Connecting to (.*)$/
! 			debug blue("Connection to #{$1}..")
  		when />> 200/
! 			debug blue('Connected')
! 		when />> (480|381|281)/, /<< (GROUP|AUTHINFO)/
  			# nothing
! 		when /^download error occured, keeping temp file$/
  			# same as below
  		when /^couldn't get (<.*>) from anywhere$/
! 			debug red("Couldn't download mid ") + green($1)
  		when /^ERRORS: (.*)$/
! 			debug red("Finished, got errors for #{$1}")
  		else
  		#TODO : handle decoding failures (=> @curfile.finish(false))
! 			debug "unhandled msg : #{blue line}"
  		end
  		#debug (Utils.red "end update_status")
--- 77,118 ----
  			filenb = $4
  			@curfile.update(part, speed, filenb, size)
! 		when /^autopar.*seems like a (par2?) \((.*)\)/
! 			ldebug "autopar: found a #{$1} file for " + blue($2)
! 		when /^already have (.*)$/
! 			ldebug "already have the file '#{blue $1}'"
! 		when /^autopar: .* matches local (par2?)set \((.*)\)$/
! 			ldebug "#{yellow 'autopar'}: got a #{$1} for the set #{blue $2}"
  		when /^uu_msg\(\d+\):Loaded from .*: '.*' \(.*\): (.*) part \d+.*end (.*)$/
! 			ldebug "decoding filename : #{blue $1}, encoding : #{yellow $2}"
  			@curfile._filename = $1
  			@curfile.encoding = $2
+ 		when /^uu_msg\(\d+\):Loaded from (.*): '.*' \(.*\): (.*) part (\d+).*$/
+ 			file, dfile, nb = $1, $2, $3
+ 			ldebug "loaded #{green file.sub(/^.*\/([^\/]*)$/, '\1')} (n° #{yellow nb} for #{blue dfile})"
  		when /^decoded ok/
! 			ldebug 'file decoded'
  			@curfile.finish(true)
+ 			@finish_status = 'FINISHED'
  			log "Download finished for file #{@curfile._filename}, queue #{@value}"
  		when /^Connecting to (.*)$/
! 			ldebug "Connection to #{$1}.."
  		when />> 200/
! 			ldebug 'Connected'
! 		when />> 400/
! 			ldebug 'Disconnected'
! 		when /<< QUIT/
! 			ldebug 'Quitted'
! 		when />> \d\d\d/, /<< (GROUP|AUTHINFO)/
  			# nothing
! 		when 'download error occured, keeping temp files.'
  			# same as below
  		when /^couldn't get (<.*>) from anywhere$/
! 			ldebug red("Couldn't download mid ") + green($1)
  		when /^ERRORS: (.*)$/
! 			ldebug red("Finished, got errors (#{$1})")
! 			@finish_status = 'FAILED'
  		else
  		#TODO : handle decoding failures (=> @curfile.finish(false))
! 			ldebug "unhandled msg : #{blue line}"
  		end
  		#debug (Utils.red "end update_status")
***************
*** 106,112 ****
  		@start_time = Time.new
  		@status = 'PROCESSING'
! 		$dbh.do 'lock tables queue write'
! 		$dbh.do "update queue set `status` = 'PROCESSING', `date_started`= NOW()  where `id` = '#{@id}'"
! 		$dbh.do 'unlock tables'
  		@io = IO.popen("#{$nget} -q -G #{@group} -r '#{@value}'")
  		@t = Thread.new(@io) { |io|
--- 124,130 ----
  		@start_time = Time.new
  		@status = 'PROCESSING'
! 		Db.dbh.do 'lock tables queue write'
! 		Db.dbh.do "update queue set `status` = 'PROCESSING', `date_started`= NOW()  where `id` = '#{@id}'"
! 		Db.dbh.do 'unlock tables'
  		@io = IO.popen("#{$nget} -q -G #{@group} -r '#{@value}'")
  		@t = Thread.new(@io) { |io|
***************
*** 116,120 ****
  				}
  			rescue IOError
! 				puts 'nget got killed'
  				
  			rescue Exception => e
--- 134,138 ----
  				}
  			rescue IOError
! 				puts 'nget got killed!'
  				
  			rescue Exception => e
***************
*** 130,133 ****
--- 148,152 ----
  			end
  		}
+ 		ldebug 'Downloader started for ' + green(@value)
  		log "Download started for #{@value}, group #{@group} to #{@dest}"
  	end
***************
*** 139,162 ****
  	def stop
  		# oh no they killed kenny !
! 		debug "#{red 'dlder.stop'}"
  		Process.kill('SIGTERM', @io.pid)
  		@io.close
  		@t.exit
  		
! 		$dbh.do 'lock tables queue write'
! 		$dbh.do "update `queue` set `status` = 'STOPPED', `date_finished`=NOW() where `id` = '#{@id}'"
! 		$dbh.do 'unlock tables'
! 		log "Download stopped for queue #{@value}, group #{@group}"	
  	end
  
! 	def finish(success)
! 		@io.close
  		@t.exit
  		time = Time.now.to_s
! 		$dbh.do 'lock tables queue write'
! 		newstatus = success ? 'FINISHED' : 'FAILED'
! 		$dbh.do "update queue set `status` = '#{newstatus}', `date_finished`=NOW() where `id` = '#{@id}'"
! 		$dbh.do 'unlock tables'
! 		log "Download #{newstatus.downcase} for queue #{@value}, group #{@group}"		
  	end
  
--- 158,188 ----
  	def stop
  		# oh no they killed kenny !
! 		ldebug red('dlder.stop')
! 		Db.dbh.do 'lock tables queue write'
! 		Db.dbh.do "update `queue` set `status` = 'STOPPED', `date_finished`=NOW() where `id` = '#{@id}'"
! 		Db.dbh.do 'unlock tables'
! 		
  		Process.kill('SIGTERM', @io.pid)
  		@io.close
  		@t.exit
  		
! 		ldebug "Download stopped for #{green @value}, group #{@group}"	
! 		log "Download stopped for #{@value}, group #{@group}"	
  	end
  
! 	def finish(success=true)
! 		begin
! 			@io.close
! 		rescue
! 		end
  		@t.exit
  		time = Time.now.to_s
! 		Db.dbh.do 'lock tables queue write'
! #		newstatus = success ? 'FINISHED' : 'FAILED'
! 		newstatus = @finish_status
! 		Db.dbh.do "update queue set `status` = '#{newstatus}', `date_finished`=NOW() where `id` = '#{@id}'"
! 		Db.dbh.do 'unlock tables'
! 		ldebug "Download #{newstatus.downcase} for #{green @value}"		
! 		log "Download #{newstatus.downcase} for #{@value}, group #{@group}"		
  	end
  

Index: downloadmanager.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloadmanager.rb,v
retrieving revision 1.19
retrieving revision 1.20
diff -C2 -d -r1.19 -r1.20
*** downloadmanager.rb	4 Feb 2004 15:21:48 -0000	1.19
--- downloadmanager.rb	6 Feb 2004 23:46:59 -0000	1.20
***************
*** 1,2 ****
--- 1,3 ----
+ require 'ngetsuite/db'
  require 'ngetsuite/downloader'
  require 'ngetsuite/utils'
***************
*** 20,24 ****
  		@poll_interval = 30
  		@started = false
- 		start if $config.fetch('autostart', false)
  	end
  
--- 21,24 ----
***************
*** 32,36 ****
  		@mainThread = Thread.new {
  		begin
! 			while @started do
  				log "current : #{@current_dlers}, max : #{@max_dlers}" if @lastlog_current != @current_dlers or @lastlog_max != @max_dlers
  				@lastlog_current, @lastlog_max = @current_dlers, @max_dlers
--- 32,37 ----
  		@mainThread = Thread.new {
  		begin
! 			loop do
! 				break unless @started
  				log "current : #{@current_dlers}, max : #{@max_dlers}" if @lastlog_current != @current_dlers or @lastlog_max != @max_dlers
  				@lastlog_current, @lastlog_max = @current_dlers, @max_dlers
***************
*** 38,42 ****
--- 39,45 ----
  					start_new_downloader
  				end
+ 				break unless @started
  				check_finished_dls
+ 				break unless @started
  				sleep(@poll_interval)
  			end
***************
*** 56,61 ****
  		
  	def start_new_downloader
! 		row = $dbh.select_one("select * from queue where `status`='PENDING' order by `priority` desc limit 1")
! 		row = $dbh.select_one("select * from queue where `status`='STOPPED' order by `priority` desc limit 1") if not row and $config.fetch('resume_downloads', false)
  		
  		return false if row == nil
--- 59,64 ----
  		
  	def start_new_downloader
! 		row = Db.dbh.select_one("select * from queue where `status`='PENDING' order by `priority` desc limit 1")
! 		row = Db.dbh.select_one("select * from queue where `status`='STOPPED' order by `priority` desc limit 1") if not row and $config['resume_downloads']
  		
  		return false if row == nil
***************
*** 63,68 ****
  		@current_dlers += 1
  		id, group, type, value, subdir, prio = row.values_at(0, 1, 2, 3, 4, 5)
- 		debug(green("New downloader starting queue #{id}"))
- 		
  		log "found #{value} for group #{group}\n"
  		dler = Downloader.new(id, type, value, group, @basedir + File::Separator + subdir, prio)
--- 66,69 ----
***************
*** 89,97 ****
  			@started = false
  			@dlers.each { |i, d|
! 				d.stop
  			}
  			@mainThread.run if @mainThread.alive?
  			10.times do |i|
! 				sleep 0.1
  				break unless @mainThread.alive?
  				if i == 9
--- 90,100 ----
  			@started = false
  			@dlers.each { |i, d|
! 				Thread.new { d.stop }
  			}
+ 			@dlers = {}
+ 			@current_dlers = 0
  			@mainThread.run if @mainThread.alive?
  			10.times do |i|
! 				sleep 0.2
  				break unless @mainThread.alive?
  				if i == 9

Index: group.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/group.rb,v
retrieving revision 1.30
retrieving revision 1.31
diff -C2 -d -r1.30 -r1.31
*** group.rb	4 Feb 2004 15:21:48 -0000	1.30
--- group.rb	6 Feb 2004 23:46:59 -0000	1.31
***************
*** 30,33 ****
--- 30,37 ----
  	Db.autotable(Group, 'groups')
  
+ 	def Group.delete(id)
+ 		Db.dbh.do "delete from `groups` where `fullname` = '#{Group.getname(id)}'"
+ 	end
+ 	
  	def Group.getname(id)
  		begin
***************
*** 61,64 ****
--- 65,72 ----
  	end
  
+ 	def Group.getallactive
+ 		Db.dbh.select_all "select `fullname` from `groups` where active='yes'"
+ 	end
+ 	
  	def Group.getgroup(id)
  		Group.new(Group.getname(id), true)
***************
*** 93,98 ****
  		@_description = description
  		@_date_added = Time.now
! 		@nget = $config.fetch('nget', '/usr/bin/nget')
! 		@yydecode = $config.fetch('yydecode', '/usr/bin/yydecode')
  		@cachedir = $config['cachedir']
  		update_from_db if fetch_from_db
--- 101,106 ----
  		@_description = description
  		@_date_added = Time.now
! 		@nget = $config['nget']
! 		@yydecode = $config['yydecode']
  		@cachedir = $config['cachedir']
  		update_from_db if fetch_from_db
***************
*** 105,109 ****
  	def _shortname
  		if (@_shortname == nil)
! 			@_shortname = @_fullname.split('.').map {|item| item[0] }.join('')
  		end
  
--- 113,117 ----
  	def _shortname
  		if (@_shortname == nil)
! 			@_shortname = @_fullname.to_s.split('.').map {|item| item[0].chr }.join('')
  		end
  
***************
*** 127,174 ****
  	end
  
- 	def announceupdate
- 		#Db.dbh.execute "insert into `articles` (`group`, `action`) values ('#{@_fullname}', 'updating')"
- 	end
- 	def announcedupdate
- 		#'updating' == Db.dbh.execute "select `action` from `articles` where `group` == '#{@_fullname}'"
- 	end
- 	def finishupdate
- 		#Db.dbh.execute "delete from `articles` where `group`='#{@_fullname}', `action`='updating'"
- 	end
- 	
  	def update
! 		puts "Warning: A headers update seems to be running !" if announcedupdate
! 		updatecache unless $config.fetch('extern_headers_update', '').split(',').include?(@_shortname)
  		clear_articles
! 		clear_nfos
  		clear_releases
- 		announceupdate
- 		mknewscache
- 		fetchnfos unless $config.fetch('extern_nfo_fetch', '').split(',').include?(@_shortname)
  		recognizereleases
  		@_date_updated = Time.now
  		sync_to_db
- 		finishupdate
  	end
  
  	def clear_articles
- 		debug "Beginning clear_articles at #{Time.now.strftime('%H:%M:%S')}"
  		nb_rows = Db.dbh.execute "delete from `articles` where `group` = '#{@_fullname}'"
! 		debug "End clear_articles at #{Time.now.strftime('%H:%M:%S')}"
  	end
  
  	def clear_nfos
- 		debug "Beginning clear_nfos at #{Time.now.strftime('%H:%M:%S')}"
  		nb_rows = Db.dbh.do "delete from `nfos` where `group` = '#{@_fullname}'"
  		debug "#{nb_rows} nfos deleted"
- 		debug "End clear_nfos at #{Time.now.strftime('%H:%M:%S')}"
- 
  	end
  
  	def clear_releases
- 		debug "Beginning clear_releases at #{Time.now.strftime('%H:%M:%S')}"
  		nb_rows = Db.dbh.do "delete from `releases` where `group` = '#{@_fullname}'"
  		debug "#{nb_rows} releases deleted"
- 		debug "End clear_releases at #{Time.now.strftime('%H:%M:%S')}"
  
  	end
--- 135,166 ----
  	end
  
  	def update
! 		updatengetcache unless $config['extern_headers_update'].split(',').include?(@_shortname)
  		clear_articles
! 		parsengetcache
! 
  		clear_releases
  		recognizereleases
+ 		
+ 		clear_nfos
+ 		fetchnfos unless $config['extern_nfos_fetch'].split(',').include?(@_shortname)
+ 		
  		@_date_updated = Time.now
  		sync_to_db
  	end
  
  	def clear_articles
  		nb_rows = Db.dbh.execute "delete from `articles` where `group` = '#{@_fullname}'"
! 		debug "#{nb_rows} articles deleted"
  	end
  
  	def clear_nfos
  		nb_rows = Db.dbh.do "delete from `nfos` where `group` = '#{@_fullname}'"
  		debug "#{nb_rows} nfos deleted"
  	end
  
  	def clear_releases
  		nb_rows = Db.dbh.do "delete from `releases` where `group` = '#{@_fullname}'"
  		debug "#{nb_rows} releases deleted"
  
  	end
***************
*** 181,201 ****
  	end
  
! 	def updatecache
! 		debug "Beginning updatecache at #{Time.now.strftime('%H:%M:%S')}"
! 		# to avoid ngetsuite poisoning
  		n = rand(900000)+100000
! 		`nice -n19 #{@nget} -q -g #{@_fullname} -c -r '^#{n}Dummy'`
! 		debug "End updatecache at #{Time.now.strftime('%H:%M:%S')}"
  	end
  
! 	def mknewscache
! 		debug "Beginning mknewscache at #{Time.now.strftime('%H:%M:%S')}"
! 		sth = Db.dbh.prepare 'insert into articles(`group`, `mid`, `nbparts`, `size`, `subject`, `from`, `time`) values (?, ?, ?, ?, ?, ?, FROM_UNIXTIME(?))'
! 		filecallback = Proc.new{ |f|
  			if f.req >= 1
  			partcnt = 0
  			size = 0
  			a = nil
! 			f.parts.each{ |p|
  				unless p.articles.empty?
  				a = p.articles[0]
--- 173,203 ----
  	end
  
! 	def updatengetcache
! 		t = Time.now
! 		debug "Beginning updatengetcache at #{t.strftime('%H:%M:%S')}"
  		n = rand(900000)+100000
! 		$stdout.sync = true if $stdout.isatty
! 		begin
! 		IO.popen("nice -n19 #{@nget} -g #{@_fullname} -cr '^#{n}Dummy'").each("\r") { |line|
! 			print "progress#{red $1} speed#{blue $2} ETA#{green $3}            \r" if line =~ /Retrieving headers \d+-\d+ : \d+\/\d+\/\d+(\s+\d+%)(\s+\d+B\/s)(.*s)/ and $stdout.isatty
! 		}
! 		rescue
! 			puts "\nInterrupted"
! 		end
! 		print "\n" if $stdout.isatty
! 		$stdout.sync = false if $stdout.isatty
! 		debug "Updatengetcache finished in #{Time.now-t}s"
  	end
  
! 	def parsengetcache
! 		t = Time.now
! 		debug "Beginning parsengetcache at #{t.strftime('%H:%M:%S')}"
! 		sth = Db.dbh.prepare 'insert into articles (`group`, `mid`, `nbparts`, `size`, `subject`, `from`, `time`) values (?, ?, ?, ?, ?, ?, FROM_UNIXTIME(?))'
! 		filecallback = Proc.new { |f|
  			if f.req >= 1
  			partcnt = 0
  			size = 0
  			a = nil
! 			f.parts.each { |p|
  				unless p.articles.empty?
  				a = p.articles[0]
***************
*** 206,209 ****
--- 208,212 ----
  				end
  				end
+ 				# free mem
  				p.articles = nil
  			}
***************
*** 213,229 ****
  			
  		NgetCache.new(@_fullname, @cachedir, nil, filecallback)
! 		debug "End mknewscache at #{Time.now.strftime('%H:%M:%S')}"
  	end
  
  	def fetchnfos
  		# fetches the nfos
! 		`nice -n19 #{@nget} --path #{$tempdir} -G #{@_fullname} -K -L 150 -r '\\.nfo|\\.txt'`
! 		# but mark them as if we didn't download them
! 		`#{@nget} --path #{$tempdir} -qq -G #{@_fullname} -U -r '\.nfo|\.txt'`
  
  		sth = Db.dbh.prepare 'insert ignore into nfos (`group`, `mid`, `text`) values (?, ?, ?)'
! 
! 		pwd = Dir.pwd
! 		Dir.chdir($tempdir)
  		Dir['ngettemp-*'].each { |file|
  			File.open(file).each { |line|
--- 216,241 ----
  			
  		NgetCache.new(@_fullname, @cachedir, nil, filecallback)
! 		debug "Parsengetcache done in #{Time.now-t}s"
  	end
  
  	def fetchnfos
+ 		t = Time.now
+ 		debug "Beginning fetchnfos at #{t.strftime('%H:%M:%S')}"
+ 		tempdir = $config['tempdir'] + File::Separator + "ngetupdate.#{@_shortname}.#{Process.pid}"
+ 		if FileTest.exists? tempdir
+ 			puts "Error: The temporary directory #{tempdir} already exists !"
+ 			return
+ 		end
+ 		Dir.mkdir tempdir
+ 		pwd = Dir.pwd
+ 		begin
  		# fetches the nfos
! 		`nice -n19 #{@nget} --path #{tempdir} -G #{@_fullname} -K -L 150 -r '\\.nfo|\\.txt'`
! 		# but mark them for nget as if we didn't download them
! 		`#{@nget} --path #{tempdir} -qq -G #{@_fullname} -U -r '\\.nfo|\\.txt'`
! 		debug "Fetchnfos: files downloaded in #{Time.now-t}s"
  
  		sth = Db.dbh.prepare 'insert ignore into nfos (`group`, `mid`, `text`) values (?, ?, ?)'
! 		Dir.chdir(tempdir)
  		Dir['ngettemp-*'].each { |file|
  			File.open(file).each { |line|
***************
*** 231,235 ****
  				if $1 != nil
  					mid = $1
- 					debug "#{file}: #{mid}"
  					text = `#{@yydecode} #{file} -o -`.chomp
  					sth.execute(@_fullname, mid, text) if text and text.length < 15000
--- 243,246 ----
***************
*** 244,272 ****
  			File.delete file
  		}
! 
! 		Dir.chdir(pwd)
  	end
  
  	def recognizereleases
  		articles = NgetSuite::ArticleList.new("where `group`='#{@_fullname}' order by `subject`")
! 		@rls = nil
  		articles.each { |art|
! 			@rls = NgetSuite::Release.new(art._group, art._mid) if !@rls
! 
! 			if @rls.add?(art) == false then
  				# New release! Commit the current one to the db
! 				@rls.insert_to_db
! 				
  				# And make another
! 				@rls = NgetSuite::Release.new(art._group, art._mid)
! 				@rls.add?(art)
  			end
  		}
  
! 		@rls.insert_to_db
! 		@rls = nil
  	end
  	
  end # class
- 	
  end # module
--- 255,288 ----
  			File.delete file
  		}
! 		ensure
! 			Dir.chdir(pwd)
! 			`rm -rf #{tempdir}`
! 		end
! 		debug "Fetchnfos finished in #{Time.now-t}s"
  	end
  
  	def recognizereleases
+ 		t = Time.now
+ 		debug "Recognizing releases at #{t.strftime('%H:%M:%S')}"
  		articles = NgetSuite::ArticleList.new("where `group`='#{@_fullname}' order by `subject`")
! 		rls = nil
! 		nb_releases = 0
  		articles.each { |art|
! 			rls = NgetSuite::Release.new(art._group, art._mid) if !rls
! 			if rls.add?(art) == false then
  				# New release! Commit the current one to the db
! 				rls.insert_to_db
! 				nb_releases += 1
! 
  				# And make another
! 				rls = NgetSuite::Release.new(art._group, art._mid)
! 				rls.add? art
  			end
  		}
  
! 		rls.insert_to_db
! 		debug "Inserted #{nb_releases} in #{Time.now-t}s"
  	end
  	
  end # class
  end # module

Index: ngetcache.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/ngetcache.rb,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** ngetcache.rb	4 Feb 2004 07:13:05 -0000	1.6
--- ngetcache.rb	6 Feb 2004 23:46:59 -0000	1.7
***************
*** 11,15 ****
  	require 'zlib'
  
! 	attr_reader :servers, :files, :sane, :lineno
  
  	CacheServer = Struct.new('CacheServer', 'serverid', 'high', 'low', 'num')
--- 11,16 ----
  	require 'zlib'
  
! 	attr_reader :servers, :files, :sane, :lineno, :groupname
! 	attr_accessor :writeoldformat
  
  	CacheServer = Struct.new('CacheServer', 'serverid', 'high', 'low', 'num')
***************
*** 24,27 ****
--- 25,30 ----
  		@filecallback = filecallback
  		@sane = true
+ 		@groupname = groupname
+ 		@path = path
  
  		@fd = Zlib::GzipReader.open(path + File::Separator + groupname + ',cache.gz')
***************
*** 35,38 ****
--- 38,42 ----
  			@fd.close
  		end
+ 		print "\n" if $stdout.isatty
  #		cache_check if sanitycheck and @sane
  	end
***************
*** 123,127 ****
  	end
  	
! 	def save_tofile(groupname, path)
  		@fd = Zlib::GzipWriter.open(path + File::Separator + groupname + ',cache.newformat.gz')
  		write_all
--- 127,131 ----
  	end
  	
! 	def save_tofile(groupname=@groupname, path=@path)
  		@fd = Zlib::GzipWriter.open(path + File::Separator + groupname + ',cache.newformat.gz')
  		write_all
***************
*** 145,149 ****
  	
  	def write_file(f)
! 		@fd.puts [f.req, f.flags, f.subject, f.author, f.partoff, f.tailoff].join("\t")
  		f.references.each { |r|
  			write_reference(r)
--- 149,153 ----
  	
  	def write_file(f)
! 		@fd.puts [f.req, f.flags, (f.fileid if @writeoldformat), f.subject, f.author, f.partoff, f.tailoff].compact.join("\t")
  		f.references.each { |r|
  			write_reference(r)

Index: queue.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/queue.rb,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** queue.rb	4 Feb 2004 01:45:39 -0000	1.20
--- queue.rb	6 Feb 2004 23:46:59 -0000	1.21
***************
*** 1,4 ****
--- 1,5 ----
  require 'ngetsuite/utils'
  require 'ngetsuite/db'
+ require 'ngetsuite/queue_file'
  
  module NgetSuite	

Index: utils.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/utils.rb,v
retrieving revision 1.11
retrieving revision 1.12
diff -C2 -d -r1.11 -r1.12
*** utils.rb	4 Feb 2004 05:56:50 -0000	1.11
--- utils.rb	6 Feb 2004 23:46:59 -0000	1.12
***************
*** 24,28 ****
  
  	def Utils.debug(message=nil)
!   		puts "D: #{message}" if($config.fetch('debug', false) && message)
  	end
  
--- 24,28 ----
  
  	def Utils.debug(message=nil)
!   		puts "#{green 'D'}: #{message}" if $config['debug'] and message
  	end
  
***************
*** 31,38 ****
  	end
  
  	def Utils.red msg
  		color(msg,RED_FG,1)
  	end
- 
  	def red msg
  		Utils.red msg
--- 31,44 ----
  	end
  
+ 	def Utils.white msg
+ 		color(msg,WHITE_FG,1)
+ 	end
+ 	def white msg
+ 		Utils.white msg
+ 	end
+ 
  	def Utils.red msg
  		color(msg,RED_FG,1)
  	end
  	def red msg
  		Utils.red msg
***************
*** 42,46 ****
  		color(msg,BLUE_FG,1)
  	end
- 
  	def blue msg
  		Utils.blue msg
--- 48,51 ----
***************
*** 50,54 ****
  		color(msg,GREEN_FG,1)
  	end
- 
  	def green msg
  		Utils.green msg
--- 55,58 ----
***************
*** 58,62 ****
  		color(msg,YELLOW_FG,1)
  	end
- 
  	def yellow msg
  		Utils.yellow msg
--- 62,65 ----

Index: web.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/web.rb,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** web.rb	4 Feb 2004 03:07:57 -0000	1.4
--- web.rb	6 Feb 2004 23:46:59 -0000	1.5
***************
*** 1,3 ****
- require 'dbi'
  require 'ngetsuite/utils'
  require 'ngetsuite/queue'
--- 1,2 ----

[Ngetsuite-cvs] ngetsuite HOWTO,1.3,1.4 daemon.rb,1.11,1.12 ngetsuite.rb,1.15,1.16 resetdb.rb,1.1,1.2 sampleconfig,1.9,1.10 testngetcache.rb,1.2,1.3

From: <jj...@us...> - 2004-02-06 23:50:01

Update of /cvsroot/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv11482

Modified Files:
	HOWTO daemon.rb ngetsuite.rb resetdb.rb sampleconfig 
	testngetcache.rb 
Log Message:
config: defaults are in config.rb (no need to .fetch), added 'daemon_debug', 'daemon_host', 'dbhost' ; changed 'autostart_httpd' to 'httpd_autostart', 'extern_nfo_update' to 'extern_nfos_update', 'port' to 'daemon_port' ; reqesting a config keyword inexisting raises an exception
group: somewhat gives a success status, fixed shortname, *sexy* headers update
global: removed some global variables ($dbh->Db.dbh..)


Index: HOWTO
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/HOWTO,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** HOWTO	4 Feb 2004 07:13:04 -0000	1.3
--- HOWTO	6 Feb 2004 23:46:58 -0000	1.4
***************
*** 5,8 ****
--- 5,10 ----
  Set up the config file
  	cp sampleconfig ~/.ngetsuite
+ 		or
+ 	./ngetsuite.rb -C ~/.ngetsuite
  	vim ~/.ngetsuite
  the most important is the login/pass for the database.

Index: daemon.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/daemon.rb,v
retrieving revision 1.11
retrieving revision 1.12
diff -C2 -d -r1.11 -r1.12
*** daemon.rb	3 Feb 2004 21:29:38 -0000	1.11
--- daemon.rb	6 Feb 2004 23:46:59 -0000	1.12
***************
*** 2,8 ****
  
  require 'drb'
! require 'dbi'
  require 'ngetsuite/config'
  
  homedir = ENV['HOME'] + File::Separator
  conf_file = homedir + '.ngetsuite'
--- 2,10 ----
  
  require 'drb'
! require 'ngetsuite/utils'
  require 'ngetsuite/config'
+ include NgetSuite::Utils
  
+ # TODO handle_opts -d -c
  homedir = ENV['HOME'] + File::Separator
  conf_file = homedir + '.ngetsuite'
***************
*** 11,53 ****
  require 'ngetsuite/downloadmanager'
  
! dbuser = $config.fetch('dbuser', 'nget')
! dbname = $config.fetch('dbname', 'ngetcache')
! 
! if not $config.include? 'dbpass'
! 	puts 'Missing required value in config file : dbpass'
! 	exit 1
! end
  
! $dbh = DBI.connect("DBI:Mysql:#{dbname}", dbuser, $config['dbpass'])
  
! $nget = $config.fetch('nget', '/usr/bin/nget')
! basedir = $config.fetch('basedir', homedir + 'ngetsuite')
! max_dls = $config.fetch('max_dls', 2)
! port = $config.fetch('port', 12345)
! logfile = $config.fetch('daemon_log', basedir + 'daemon.log')
  
  $logger = File.new(logfile, 'a')
- $debug = $config.fetch('debug', false)
  
  def log(msg=nil)
! 	$logger.puts "[#{Time.now.localtime}] #{msg}" if (msg)
  	$logger.flush
  end
  
! puts "Starting daemon with log file '#{logfile}'"
! $manager = NgetSuite::DownloadManager.new(max_dls, basedir)
  
  drburl = 'druby://localhost:' + port.to_s
! DRb.start_service(drburl, $manager)
  
! $thingsrunning = [$manager]
! trap('INT') { $thingsrunning.each{ |t| t.shutdown }; DRb.stop_service }
! trap('TERM') { $thingsrunning.each{ |t| t.shutdown }; DRb.stop_service }
  
! if $config.fetch('autostart_httpd', false) or $config.fetch('httpd_autostart', false)
  	require 'webrick'
  	include WEBrick
  	
! 	port = $config.fetch('httpd_port', 2000)
  	puts "Starting web server on port #{port}"
  	log 'Web server start'
--- 13,51 ----
  require 'ngetsuite/downloadmanager'
  
! dbuser = $config['dbuser']
! dbname = $config['dbname']
  
! $config['debug'] = true if $config['daemon_debug']
  
! $nget = $config['nget']
! basedir = $config['basedir']
! max_dls = $config['max_dls']
! port = $config['daemon_port']
! logfile = $config['daemon_log']
  
  $logger = File.new(logfile, 'a')
  
  def log(msg=nil)
! 	$logger.puts "[#{Time.now.localtime.strftime('%d/%m %H:%M:%S')}] #{msg}" if (msg)
  	$logger.flush
  end
  
! puts "Starting daemon with log file '#{green logfile}'"
! manager = NgetSuite::DownloadManager.new(max_dls, basedir)
! 
! manager.start if $config['daemon_autostart']
  
  drburl = 'druby://localhost:' + port.to_s
! DRb.start_service(drburl, manager)
  
! $thingsrunning = [manager]
! trap('INT') { debug red('Ok, ok, i die') ; $thingsrunning.each{ |t| t.shutdown }; DRb.stop_service }
! trap('TERM') { debug red('Ok, ok, i die') ; $thingsrunning.each{ |t| t.shutdown }; DRb.stop_service }
  
! if $config['httpd_autostart']
  	require 'webrick'
  	include WEBrick
  	
! 	port = $config['httpd_port']
  	puts "Starting web server on port #{port}"
  	log 'Web server start'

Index: ngetsuite.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite.rb,v
retrieving revision 1.15
retrieving revision 1.16
diff -C2 -d -r1.15 -r1.16
*** ngetsuite.rb	4 Feb 2004 01:45:38 -0000	1.15
--- ngetsuite.rb	6 Feb 2004 23:46:59 -0000	1.16
***************
*** 1,22 ****
  #!/usr/bin/env ruby
  
! Dir.chdir(__FILE__.sub(/\/[^\/]*$/, ''))
! 
! $opts = Hash.new
! $opts['config-file'] = ENV['HOME'] + File::Separator + '.ngetsuite' unless $opts['config-file']
! 
! require 'ngetsuite/config'
! $config = NgetSuite::Config.new($opts['config-file'])
  
  require 'getoptlong'
- require 'dbi'
- require 'ngetsuite/core'
- 
  
- # reste -p, -w, -y
  $args = Array[
  	[ '--debug', '-d', GetoptLong::NO_ARGUMENT, 'Enable debug mode'],
  	[ '--help', '-h', GetoptLong::NO_ARGUMENT, 'Shows this message'],
  	[ '--config-file', '-c', GetoptLong::REQUIRED_ARGUMENT, 'file', 'Use another config file (def: $HOME/.ngetsuite)' ],
  	[ '--show-groups', '-s', GetoptLong::NO_ARGUMENT, 'List all known groups'],
  	[ '--add-group', '-a', GetoptLong::REQUIRED_ARGUMENT, 'fullname[,alias,dir,desc]', 'Add a group'],
--- 1,17 ----
  #!/usr/bin/env ruby
  
! wd = __FILE__.sub(/\/[^\/]*$/, '')
! Dir.chdir(wd) if FileTest.directory? wd
  
  require 'getoptlong'
  
  $args = Array[
  	[ '--debug', '-d', GetoptLong::NO_ARGUMENT, 'Enable debug mode'],
+ 	[ '--no-debug', '-D', GetoptLong::NO_ARGUMENT, 'Disable debug mode'],
  	[ '--help', '-h', GetoptLong::NO_ARGUMENT, 'Shows this message'],
  	[ '--config-file', '-c', GetoptLong::REQUIRED_ARGUMENT, 'file', 'Use another config file (def: $HOME/.ngetsuite)' ],
+ 	['--write-config', '-C', GetoptLong::REQUIRED_ARGUMENT, 'file', 'Write a defaut config file'],
+ 	# end of options, begin of actions
+ 	nil,
  	[ '--show-groups', '-s', GetoptLong::NO_ARGUMENT, 'List all known groups'],
  	[ '--add-group', '-a', GetoptLong::REQUIRED_ARGUMENT, 'fullname[,alias,dir,desc]', 'Add a group'],
***************
*** 27,32 ****
  	[ '--show-nfo', '-n', GetoptLong::REQUIRED_ARGUMENT, 'nb_nfo[,groupid]', 'Prints the specified nfo' ],
  	[ '--queue-article', '-q', GetoptLong::REQUIRED_ARGUMENT, 'groupid,articleid', 'Adds an article to the download queue'],
! 	[ '--queue-regexp', '-r', GetoptLong::REQUIRED_ARGUMENT, 'groupid,regexp[,prio,period]', 'Adds a regexp to the download queue' ],
! 	[ '--show-queue', '-t', GetoptLong::OPTIONAL_ARGUMENT, 'groupid', 'Shows the current download queue'],
  	[ '--start-manager', '-m', GetoptLong::NO_ARGUMENT, 'Launches the download manager'],
  	[ '--show-status', '-z', GetoptLong::NO_ARGUMENT, 'Prints the manager status'],
--- 22,27 ----
  	[ '--show-nfo', '-n', GetoptLong::REQUIRED_ARGUMENT, 'nb_nfo[,groupid]', 'Prints the specified nfo' ],
  	[ '--queue-article', '-q', GetoptLong::REQUIRED_ARGUMENT, 'groupid,articleid', 'Adds an article to the download queue'],
! 	[ '--queue-regexp', '-r', GetoptLong::REQUIRED_ARGUMENT, 'gid,regexp[,prio,period]', 'Adds a regexp to the download queue' ],
! 	[ '--show-queue', '-t', GetoptLong::OPTIONAL_ARGUMENT, '[groupid]', 'Shows the current download queue'],
  	[ '--start-manager', '-m', GetoptLong::NO_ARGUMENT, 'Launches the download manager'],
  	[ '--show-status', '-z', GetoptLong::NO_ARGUMENT, 'Prints the manager status'],
***************
*** 34,38 ****
  	[ '--set-priority', '-f', GetoptLong::REQUIRED_ARGUMENT, 'queueid,prio', 'Changes the priority of the specified queue' ],
  	[ '--set-status', '-i', GetoptLong::REQUIRED_ARGUMENT, 'queueid,status', 'Changes the status of the specified queue'],
! 	[ '--delete-queue', '-x', GetoptLong::REQUIRED_ARGUMENT, 'queueid', 'Changes the status of the specified queue'],
  	[ '--set-limit', '-j', GetoptLong::REQUIRED_ARGUMENT, 'limit', 'Sets the maximum limit of parallel downloads'],
  	[ '--set-poll-interval', '-o', GetoptLong::REQUIRED_ARGUMENT, 'intervall', 'Sets the polling intervall of the download manager'],
--- 29,33 ----
  	[ '--set-priority', '-f', GetoptLong::REQUIRED_ARGUMENT, 'queueid,prio', 'Changes the priority of the specified queue' ],
  	[ '--set-status', '-i', GetoptLong::REQUIRED_ARGUMENT, 'queueid,status', 'Changes the status of the specified queue'],
! 	[ '--delete-queue', '-x', GetoptLong::REQUIRED_ARGUMENT, 'queueid', 'Delete the specified queue'],
  	[ '--set-limit', '-j', GetoptLong::REQUIRED_ARGUMENT, 'limit', 'Sets the maximum limit of parallel downloads'],
  	[ '--set-poll-interval', '-o', GetoptLong::REQUIRED_ARGUMENT, 'intervall', 'Sets the polling intervall of the download manager'],
***************
*** 41,67 ****
  ]
  
! opts = GetoptLong.new(*($args.collect { |c| c[0..2] }))
  
  opts.each {|opt, arg|
  	$opts[opt.sub(/^-+/, '')] = arg
  }
  
  
! $tempdir = "/tmp/ngetupdate.#{Process.pid}"
! Dir.mkdir $tempdir
! 
! begin
! 	Dir.mkdir $config['basedir']
! rescue Errno::EEXIST
! #	puts "#{$config['basedir']} already exists"
! end
! 
! $dbh = DBI.connect("DBI:Mysql:#{$config['dbname']}", $config['dbuser'], $config['dbpass'])
  
! begin
! 	core = NgetSuite::Core.new()
! 	core.handle_opts() if core
! ensure
! 	$dbh.disconnect if $dbh
! 	`rm -rf #{$tempdir}`
! end
--- 36,52 ----
  ]
  
! opts = GetoptLong.new(*($args.compact.collect { |c| c[0..2] }))
  
+ $opts = Hash.new
  opts.each {|opt, arg|
  	$opts[opt.sub(/^-+/, '')] = arg
  }
  
+ require 'ngetsuite/config'
+ $config = NgetSuite::Config.new($opts.fetch('config-file', ENV['HOME'] + File::Separator + '.ngetsuite'))
  
! basedir = $config['basedir']
! Dir.mkdir basedir unless FileTest.directory? basedir
  
! require 'ngetsuite/core'
! NgetSuite::Core.new().handle_opts()

Index: resetdb.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/resetdb.rb,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** resetdb.rb	4 Feb 2004 05:56:49 -0000	1.1
--- resetdb.rb	6 Feb 2004 23:46:59 -0000	1.2
***************
*** 6,13 ****
  require 'dbi'
  
! sql = File.open('ngetcache.sql')
  dbh = DBI.connect("DBI:Mysql:#{config['dbname']}", config['dbuser'], config['dbpass'])
  cmd = ''
! sql.each { |l|
  	next if l =~ /^(#|\s*$)/
  	if l=~/CREATE TABLE (`.*`)/
--- 6,13 ----
  require 'dbi'
  
! sqlfile = File.open('ngetcache.sql')
  dbh = DBI.connect("DBI:Mysql:#{config['dbname']}", config['dbuser'], config['dbpass'])
  cmd = ''
! sqlfile.each { |l|
  	next if l =~ /^(#|\s*$)/
  	if l=~/CREATE TABLE (`.*`)/
***************
*** 21,25 ****
  	end
  }	
! dbh.disconnect if $dbh
! sql.close
  puts 'done'
--- 21,25 ----
  	end
  }	
! dbh.disconnect
! sqlfile.close
  puts 'done'

Index: sampleconfig
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/sampleconfig,v
retrieving revision 1.9
retrieving revision 1.10
diff -C2 -d -r1.9 -r1.10
*** sampleconfig	4 Feb 2004 07:13:04 -0000	1.9
--- sampleconfig	6 Feb 2004 23:46:59 -0000	1.10
***************
*** 1,5 ****
--- 1,10 ----
+ # file to copy to ~/.ngetsuite after customisation
+ # for windows users, use '\' in pathes instead of '/', and use -c to specify this file path
+ # ngetsuite.rb --write-config gives this file without the comments
+ 
  #########################
  #  Database parameters
  #
+ dbhost = 'localhost'
  dbuser = 'nget'
  dbpass = 'cache4pdv'
***************
*** 21,24 ****
--- 26,31 ----
  # where to download unless otherwise specified
  basedir = '/home/nget/testdl'
+ # ngetsuite will create subdirectories here
+ tempdir = '/tmp'
  
  
***************
*** 29,38 ****
  max_dls = 3
  # the port for communication with the daemon
! port = 12345
  daemon_log = '/tmp/nget-daemon.log'
  # should we automatically restart downloads in state STOPPED ?
  resume_downloads = false
  # should the daemon look for eligible queues on startup or wait for -m
! autostart = true
  
  
--- 36,47 ----
  max_dls = 3
  # the port for communication with the daemon
! daemon_port = 12345
  daemon_log = '/tmp/nget-daemon.log'
  # should we automatically restart downloads in state STOPPED ?
  resume_downloads = false
  # should the daemon look for eligible queues on startup or wait for -m
! daemon_autostart = true
! # you can view informative messages with this on
! daemon_debug = false
  
  
***************
*** 43,46 ****
--- 52,56 ----
  # coma-separated list of the shortnames of the groups for which the program will not try to update the headers (but will still parse them and dl nfos)
  extern_headers_update = ''
+ extern_nfos_fetch = ''
  
  

Index: testngetcache.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/testngetcache.rb,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** testngetcache.rb	4 Feb 2004 03:07:56 -0000	1.2
--- testngetcache.rb	6 Feb 2004 23:46:59 -0000	1.3
***************
*** 7,17 ****
  end
  puts 'Testing cache for ' + ARGV[0]
! cache = NgetCache.new(ARGV[0], NgetSuite::Config.new(ENV['HOME']+'/'+'.ngetsuite')['cachedir'])
  
- cache.files.sort! { |x,y|
- 	x.subject <=> y.subject
- }.each { |file|
- 	puts file.subject
- }
  
  puts 'Done.'
--- 7,15 ----
  end
  puts 'Testing cache for ' + ARGV[0]
! cache = NgetCache.new(ARGV[0], NgetSuite::Config.new(ENV['HOME']+'/'+'.ngetsuite')['cachedir'], 100)
! puts 'read'
! cache.writeoldformat = true
! cache.save_tofile
  
  
  puts 'Done.'

[Ngetsuite-cvs] ngetsuite/web articles.rhtml,1.5,1.6 queue.rhtml,1.9,1.10 test.rhtml,1.7,1.8

From: <jj...@us...> - 2004-02-06 23:50:01

Update of /cvsroot/ngetsuite/ngetsuite/web
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv11482/web

Modified Files:
	articles.rhtml queue.rhtml test.rhtml 
Log Message:
config: defaults are in config.rb (no need to .fetch), added 'daemon_debug', 'daemon_host', 'dbhost' ; changed 'autostart_httpd' to 'httpd_autostart', 'extern_nfo_update' to 'extern_nfos_update', 'port' to 'daemon_port' ; reqesting a config keyword inexisting raises an exception
group: somewhat gives a success status, fixed shortname, *sexy* headers update
global: removed some global variables ($dbh->Db.dbh..)


Index: articles.rhtml
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/web/articles.rhtml,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** articles.rhtml	9 Nov 2003 01:25:33 -0000	1.5
--- articles.rhtml	6 Feb 2004 23:46:59 -0000	1.6
***************
*** 2,5 ****
--- 2,6 ----
  	require 'cgi'	
  	require 'ngetsuite/config'
+ 	require 'ngetsuite/db'
  
  	conf = "/home/nget/.ngetsuite"
***************
*** 8,15 ****
  	require 'ngetsuite/queue'
  	require 'ngetsuite/group'
- 	require 'dbi'
  
  	cgi = CGI.new()
- 	$dbh = DBI.connect("DBI:Mysql:#{$config['dbname']}", $config['dbuser'], $config['dbpass'])
  
  	$config['debug'] = false
--- 9,14 ----
***************
*** 52,56 ****
  <ul>
  <%
! 	sth = $dbh.execute "select `fullname`, `shortname` from `groups`"
  	sth.fetch do |row|
  		puts "<li><a href=?group=#{row[0]}>#{row[0]} (#{row[1]})</a>"
--- 51,55 ----
  <ul>
  <%
! 	sth = Db.dbh.execute "select `fullname`, `shortname` from `groups`"
  	sth.fetch do |row|
  		puts "<li><a href=?group=#{row[0]}>#{row[0]} (#{row[1]})</a>"
***************
*** 82,86 ****
  	req += " order by `time` desc limit #{offset},#{articles_by_page}"
  
! 	sth = $dbh.execute req
  	sth.fetch do |row|
  		puts "<input type='hidden' name='mid' value='#{row[0]}'>"
--- 81,85 ----
  	req += " order by `time` desc limit #{offset},#{articles_by_page}"
  
! 	sth = Db.dbh.execute req
  	sth.fetch do |row|
  		puts "<input type='hidden' name='mid' value='#{row[0]}'>"
***************
*** 99,101 ****
  </body>
  </html>
! <% $dbh.disconnect %>
--- 98,100 ----
  </body>
  </html>
! <% Db.dbh.disconnect %>

Index: queue.rhtml
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/web/queue.rhtml,v
retrieving revision 1.9
retrieving revision 1.10
diff -C2 -d -r1.9 -r1.10
*** queue.rhtml	1 Nov 2003 21:09:26 -0000	1.9
--- queue.rhtml	6 Feb 2004 23:46:59 -0000	1.10
***************
*** 2,13 ****
  	require 'cgi'	
  	require 'ngetsuite/config'
  
  	conf = "/home/nget/.ngetsuite"
  	$config = NgetSuite::Config.new(conf)	
- 	$dbh = DBI.connect("DBI:Mysql:#{$config['dbname']}", $config['dbuser'], $config['dbpass'])
  
  	require 'ngetsuite/queue'
  	require 'ngetsuite/group'
- 	require 'dbi'
  
  	cgi = CGI.new()
--- 2,12 ----
  	require 'cgi'	
  	require 'ngetsuite/config'
+ 	require 'ngetsuite/db'
  
  	conf = "/home/nget/.ngetsuite"
  	$config = NgetSuite::Config.new(conf)	
  
  	require 'ngetsuite/queue'
  	require 'ngetsuite/group'
  
  	cgi = CGI.new()
***************
*** 79,83 ****
  	req += ' order by `priority` desc'
  
! 	sth = $dbh.execute req
  	sth.fetch do |row|
  		q = NgetSuite::Queue.getqueue(row[0])
--- 78,82 ----
  	req += ' order by `priority` desc'
  
! 	sth = Db.dbh.execute req
  	sth.fetch do |row|
  		q = NgetSuite::Queue.getqueue(row[0])
***************
*** 115,117 ****
  </body>
  </html>
! <% $dbh.disconnect %>
--- 114,116 ----
  </body>
  </html>
! <% Db.dbh.disconnect %>

Index: test.rhtml
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/web/test.rhtml,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** test.rhtml	9 Nov 2003 01:25:33 -0000	1.7
--- test.rhtml	6 Feb 2004 23:46:59 -0000	1.8
***************
*** 57,61 ****
  <body>
  <%
- 	require 'dbi'
  	require 'ngetsuite/config'
  	require 'ngetsuite/utils'
--- 57,60 ----

[Ngetsuite-cvs] ngetsuite/web group.rhtml,1.2,1.3

From: <ys...@us...> - 2004-02-04 15:42:47

Update of /cvsroot/ngetsuite/ngetsuite/web
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv12272/web

Modified Files:
	group.rhtml 
Log Message:
orphan articles link + fixed bug that may crash update


Index: group.rhtml
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/web/group.rhtml,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** group.rhtml	4 Feb 2004 03:07:57 -0000	1.2
--- group.rhtml	4 Feb 2004 15:40:22 -0000	1.3
***************
*** 82,86 ****
  	<a href="group.rhtml?group=<%= web.group._fullname %>&date=<%=iso%>"><%=display%></a>&nbsp;|&nbsp;
  <% } %>
! <a href="group.rhtml?date=any">any</a>
  
  <div class="group"><%= web.date_str %></div>
--- 82,86 ----
  	<a href="group.rhtml?group=<%= web.group._fullname %>&date=<%=iso%>"><%=display%></a>&nbsp;|&nbsp;
  <% } %>
! <a href="group.rhtml?group=<%= web.group._fullname %>&date=any">any</a>
  
  <div class="group"><%= web.date_str %></div>
***************
*** 104,107 ****
--- 104,130 ----
  	</tr>
  	<% } %>
+ 
+ 	<tr>
+ 		<td colspan="2">
+ <%
+ 	# Count "orphan" articles
+ 
+ 	orphanq = "select count(*), sum(`size`) from `articles` "
+ 	orphanq += "where `release`='' "
+ 
+ 	orphanq += "and DATE_FORMAT(`time`, '%Y-%m-%d')='#{web.date}' " if web.date != "any"
+ 	orphanq += "group by `release`"
+ 	
+ 	@orphsth = NgetSuite::Db.dbh.execute orphanq
+ 
+ 	orphans = String.new
+ 
+ 	@orphsth.fetch { |row|
+ 		orphans = "#{row[0]} \"orphans\", taking up #{NgetSuite::Utils.pretty_size(row[1])}"
+ 	}
+ %>
+ 			<a href="orphans.rhtml?group=<%= web.group %>&date=<%= web.date %>"><%= orphans %></a>
+ 		</td>
+ 	</tr>
  </table>

[Ngetsuite-cvs] ngetsuite/ngetsuite articles.rb,1.1,1.2

From: <ys...@us...> - 2004-02-04 15:42:46

Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv12272/ngetsuite

Modified Files:
	articles.rb 
Log Message:
orphan articles link + fixed bug that may crash update


Index: articles.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/articles.rb,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** articles.rb	4 Feb 2004 03:07:57 -0000	1.1
--- articles.rb	4 Feb 2004 15:40:22 -0000	1.2
***************
*** 50,53 ****
--- 50,54 ----
  		@_nbexpected = 0
  		@_nbactual = 0
+ 		@lastnb = 0
  		@_time = Time.gm("1980")
  		update_from_db if fetch_from_db

[Ngetsuite-cvs] ngetsuite/ngetsuite downloader.rb,1.20,1.21 downloadmanager.rb,1.18,1.19 group.rb,1.29,1.30

From: <jj...@us...> - 2004-02-04 15:24:14

Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv7992/ngetsuite

Modified Files:
	downloader.rb downloadmanager.rb group.rb 
Log Message:
better nget output parsing


Index: downloader.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloader.rb,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** downloader.rb	4 Feb 2004 07:13:05 -0000	1.20
--- downloader.rb	4 Feb 2004 15:21:48 -0000	1.21
***************
*** 42,47 ****
  		line.chomp!
  		case line
! 		when /^already have (.*)$/
! 			debug 'file already there : ' + $1
  			@nbfiles += 1
  		when /^Retrieving: (\d+).*\t(\d+)l.*(<.*>)$/
--- 42,47 ----
  		line.chomp!
  		case line
! 		when /^already have article (<.*>) \(.*\/([^\/]*)\)$/
! 			debug yellow('file already there') + ' : ' + green($1) + ' in ' + blue($2)
  			@nbfiles += 1
  		when /^Retrieving: (\d+).*\t(\d+)l.*(<.*>)$/
***************
*** 71,76 ****
  			filenb = $4
  			@curfile.update(part, speed, filenb, size)
! 		when /^autopar/
! 			debug 'autopar: ' + blue(line)
  			
  		when /^uu_msg\(\d+\):Loaded from .*: '.*' \(.*\): (.*) part \d+.*end (.*)$/
--- 71,76 ----
  			filenb = $4
  			@curfile.update(part, speed, filenb, size)
! 		when /^autopar.*seems like a par2 \((.*)\)/
! 			debug 'autopar: found a par2 for ' + blue($1)
  			
  		when /^uu_msg\(\d+\):Loaded from .*: '.*' \(.*\): (.*) part \d+.*end (.*)$/
***************
*** 87,90 ****
--- 87,97 ----
  			debug blue('Connected')
  		when />> (480|381|281)/, /<< (GROUP|AUTHINFO)/
+ 			# nothing
+ 		when /^download error occured, keeping temp file$/
+ 			# same as below
+ 		when /^couldn't get (<.*>) from anywhere$/
+ 			debug red("Couldn't download mid ") + green($1)
+ 		when /^ERRORS: (.*)$/
+ 			debug red("Finished, got errors for #{$1}")
  		else
  		#TODO : handle decoding failures (=> @curfile.finish(false))

Index: downloadmanager.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloadmanager.rb,v
retrieving revision 1.18
retrieving revision 1.19
diff -C2 -d -r1.18 -r1.19
*** downloadmanager.rb	4 Feb 2004 12:22:46 -0000	1.18
--- downloadmanager.rb	4 Feb 2004 15:21:48 -0000	1.19
***************
*** 91,95 ****
  				d.stop
  			}
! 			@mainThread.run
  			10.times do |i|
  				sleep 0.1
--- 91,95 ----
  				d.stop
  			}
! 			@mainThread.run if @mainThread.alive?
  			10.times do |i|
  				sleep 0.1
***************
*** 106,110 ****
  		@dlers.each {|id, dler|
  			next unless dler.finished?
- 			log "Finished download of queue #{dler.value}"
  			dler.finish(true)
  			@dlers.delete(id)
--- 106,109 ----

Index: group.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/group.rb,v
retrieving revision 1.29
retrieving revision 1.30
diff -C2 -d -r1.29 -r1.30
*** group.rb	4 Feb 2004 12:22:46 -0000	1.29
--- group.rb	4 Feb 2004 15:21:48 -0000	1.30
***************
*** 218,222 ****
  	def fetchnfos
  		# fetches the nfos
! 		`nice -n19 #{@nget} --path #{$tempdir} -G #{@_fullname} -K -L 150 -r '\.nfo|\.txt'`
  		# but mark them as if we didn't download them
  		`#{@nget} --path #{$tempdir} -qq -G #{@_fullname} -U -r '\.nfo|\.txt'`
--- 218,222 ----
  	def fetchnfos
  		# fetches the nfos
! 		`nice -n19 #{@nget} --path #{$tempdir} -G #{@_fullname} -K -L 150 -r '\\.nfo|\\.txt'`
  		# but mark them as if we didn't download them
  		`#{@nget} --path #{$tempdir} -qq -G #{@_fullname} -U -r '\.nfo|\.txt'`

[Ngetsuite-cvs] ngetsuite/ngetsuite downloadmanager.rb,1.17,1.18 group.rb,1.28,1.29

From: <jj...@us...> - 2004-02-04 12:25:08

Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv3743/ngetsuite

Modified Files:
	downloadmanager.rb group.rb 
Log Message:
typo in dlmanager.check_finished; put default value for extern_*_update to avoid nil.split


Index: downloadmanager.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloadmanager.rb,v
retrieving revision 1.17
retrieving revision 1.18
diff -C2 -d -r1.17 -r1.18
*** downloadmanager.rb	4 Feb 2004 07:13:05 -0000	1.17
--- downloadmanager.rb	4 Feb 2004 12:22:46 -0000	1.18
***************
*** 106,112 ****
  		@dlers.each {|id, dler|
  			next unless dler.finished?
  			dler.finish(true)
  			@dlers.delete(id)
- 			log "Finished download of queue #{dlers.value}"
  			@current_dlers -= 1
  		}
--- 106,112 ----
  		@dlers.each {|id, dler|
  			next unless dler.finished?
+ 			log "Finished download of queue #{dler.value}"
  			dler.finish(true)
  			@dlers.delete(id)
  			@current_dlers -= 1
  		}

Index: group.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/group.rb,v
retrieving revision 1.28
retrieving revision 1.29
diff -C2 -d -r1.28 -r1.29
*** group.rb	4 Feb 2004 03:24:43 -0000	1.28
--- group.rb	4 Feb 2004 12:22:46 -0000	1.29
***************
*** 139,143 ****
  	def update
  		puts "Warning: A headers update seems to be running !" if announcedupdate
! 		updatecache unless $config['extern_headers_update'].split(',').include?(@_shortname)
  		clear_articles
  		clear_nfos
--- 139,143 ----
  	def update
  		puts "Warning: A headers update seems to be running !" if announcedupdate
! 		updatecache unless $config.fetch('extern_headers_update', '').split(',').include?(@_shortname)
  		clear_articles
  		clear_nfos
***************
*** 145,149 ****
  		announceupdate
  		mknewscache
! 		fetchnfos unless $config['extern_nfo_fetch'].split(',').include?(@_shortname)
  		recognizereleases
  		@_date_updated = Time.now
--- 145,149 ----
  		announceupdate
  		mknewscache
! 		fetchnfos unless $config.fetch('extern_nfo_fetch', '').split(',').include?(@_shortname)
  		recognizereleases
  		@_date_updated = Time.now

[Ngetsuite-cvs] ngetsuite/ngetsuite core.rb,1.26,1.27 group.rb,1.25,1.26 ngetcache.rb,1.3,1.4 queue.rb,1.19,1.20

From: <jj...@us...> - 2004-02-04 11:37:33

Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv21764/ngetsuite

Modified Files:
	core.rb group.rb ngetcache.rb queue.rb 
Log Message:
added --delete-queue (-x), changed callback when parsing nget cache


Index: core.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/core.rb,v
retrieving revision 1.26
retrieving revision 1.27
diff -C2 -d -r1.26 -r1.27
*** core.rb	2 Feb 2004 17:44:19 -0000	1.26
--- core.rb	4 Feb 2004 01:45:38 -0000	1.27
***************
*** 212,215 ****
--- 212,217 ----
  						raise ArgumentError.new("Incorrect args : #{arg}")
  					end
+ 				when 'delete-queue'
+ 					Queue.delete(arg, false)
  				when 'show-queue'
  					args = arg.split(',')

Index: group.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/group.rb,v
retrieving revision 1.25
retrieving revision 1.26
diff -C2 -d -r1.25 -r1.26
*** group.rb	3 Feb 2004 16:33:28 -0000	1.25
--- group.rb	4 Feb 2004 01:45:39 -0000	1.26
***************
*** 182,207 ****
  		sth = Db.dbh.prepare 'insert into articles(`group`, `mid`, `nbparts`, `size`, `subject`, `from`, `time`, `rls_subject`) values (?, ?, ?, ?, ?, ?, FROM_UNIXTIME(?), ?)'
  		filecallback = Proc.new{ |f|
! 			if f.req < 1
! 				@cachecb_invalid = true
! 			else
! 				@cachecb_invalid = false
! 			end
! 			@cachecb_file = f
! 			@cachecb_rsubj = f.subject.gsub(/^(.* )(- .*)?".*$/, '\1').gsub(/\d{1,3}(\/| of )\d{1,3}/, '#\1#').gsub(/ \d\d? /, ' # ')
! 			@cachecb_partcnt = 0
! 			@cachecb_size = 0
! 		}
! 		partcallback = Proc.new{ |p|
! 			unless @cachecb_invalid or p.articles.empty?
  				a = p.articles[0]
! 				@cachecb_size += a.bytes / ( a.bytes < 100*a.lines ? 1.39 : 1.0334 )
! 				@cachecb_partcnt += 1
! 				if @cachecb_partcnt >= @cachecb_file.req
! 					sth.execute(@_fullname, p.mid, @cachecb_file.req, @cachecb_size, @cachecb_file.subject, @cachecb_file.author, p.date, @cachecb_rsubj)
  				end
  			end
  		}
  			
! 		NgetCache.new(@_fullname, @cachedir, nil, filecallback, partcallback)
  		debug "End mknewscache at #{Time.now.strftime('%H:%M:%S')}"
  	end
--- 182,206 ----
  		sth = Db.dbh.prepare 'insert into articles(`group`, `mid`, `nbparts`, `size`, `subject`, `from`, `time`, `rls_subject`) values (?, ?, ?, ?, ?, ?, FROM_UNIXTIME(?), ?)'
  		filecallback = Proc.new{ |f|
! 			if f.req >= 1
! 			rsubj = f.subject.gsub(/^(.* )(- .*)?".*$/, '\1').gsub(/\d{1,3}(\/| of )\d{1,3}/, '#\1#').gsub(/ \d\d? /, ' # ')
! 			partcnt = 0
! 			size = 0
! 			a = nil
! 			f.parts.each{ |p|
! 				unless p.articles.empty?
  				a = p.articles[0]
! 				size += a.bytes / ( a.bytes < 100*a.lines ? 1.39 : 1.0334 )
! 				partcnt += 1
! 				if partcnt >= f.req
! 					sth.execute(@_fullname, p.mid, f.req, size, f.subject, f.author, p.date, rsubj)
  				end
+ 				end
+ 				p.articles = nil
+ 			}
+ 			f.parts = nil
  			end
  		}
  			
! 		NgetCache.new(@_fullname, @cachedir, nil, filecallback)
  		debug "End mknewscache at #{Time.now.strftime('%H:%M:%S')}"
  	end

Index: ngetcache.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/ngetcache.rb,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** ngetcache.rb	3 Feb 2004 16:33:28 -0000	1.3
--- ngetcache.rb	4 Feb 2004 01:45:39 -0000	1.4
***************
*** 18,25 ****
  	CacheArticle = Struct.new('CacheArticle', 'serverid', 'articlenum', 'bytes', 'lines')
  
! 	def initialize(groupname, path='.', limit=nil, filecallback=nil, partcallback=nil, articlecallback=nil)
  		@servers = Array.new
  		@files = Array.new
! 		@filecallback, @partcallback, @articlecallback = filecallback, partcallback, articlecallback
  		@sane = true
  
--- 18,25 ----
  	CacheArticle = Struct.new('CacheArticle', 'serverid', 'articlenum', 'bytes', 'lines')
  
! 	def initialize(groupname, path='.', limit=nil, filecallback=nil)
  		@servers = Array.new
  		@files = Array.new
! 		@filecallback = filecallback
  		@sane = true
  
***************
*** 58,62 ****
  		# files list
  		while (file = read_file)
! 			@files << file unless @filecallback
  			if limit
  				break if (limit -= 1) < 0
--- 58,66 ----
  		# files list
  		while (file = read_file)
! 			if @filecallback
! 				@filecallback.call(file)
! 			else
! 				@files << file
! 			end
  			if limit
  				break if (limit -= 1) < 0
***************
*** 84,89 ****
  		end
  		
- 		@filecallback.call(file) if @filecallback
- 		
  		# parts of the file
  		while (part = read_part)
--- 88,91 ----
***************
*** 108,112 ****
  			part.articles << article
  		end
- 		@partcallback.call(part) if @partcallback
  		part
  	end
--- 110,113 ----
***************
*** 115,121 ****
  		return unless readline
  		raise BadNgetCache.new('invalid article description') unless @line =~ /^(\d+)\t(\d+)\t(\d+)\t(\d+)$/
! 		article = CacheArticle.new($1.to_i, $2.to_i, $3.to_i, $4.to_i)
! 		@articlecallback.call(article) if @articlecallback
! 		article
  	end
  	
--- 116,120 ----
  		return unless readline
  		raise BadNgetCache.new('invalid article description') unless @line =~ /^(\d+)\t(\d+)\t(\d+)\t(\d+)$/
! 		CacheArticle.new($1.to_i, $2.to_i, $3.to_i, $4.to_i)
  	end
  	

Index: queue.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/queue.rb,v
retrieving revision 1.19
retrieving revision 1.20
diff -C2 -d -r1.19 -r1.20
*** queue.rb	2 Feb 2004 02:41:57 -0000	1.19
--- queue.rb	4 Feb 2004 01:45:39 -0000	1.20
***************
*** 7,10 ****
--- 7,11 ----
  	PROCESSING = 'PROCESSING'
  	STOPPED = 'STOPPED'
+ 	FAILED = 'FAILED'
  
  	SINGLE = 'SINGLE'

[Ngetsuite-cvs] ngetsuite daemon.rb,1.10,1.11 ngetcache.sql,1.8,1.9

From: <jj...@us...> - 2004-02-04 08:50:58

Update of /cvsroot/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv16067

Modified Files:
	daemon.rb ngetcache.sql 
Log Message:
daemon and manager handles shutdown, added 'FAILED' status for downloaded file


Index: daemon.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/daemon.rb,v
retrieving revision 1.10
retrieving revision 1.11
diff -C2 -d -r1.10 -r1.11
*** daemon.rb	2 Feb 2004 17:44:18 -0000	1.10
--- daemon.rb	3 Feb 2004 21:29:38 -0000	1.11
***************
*** 15,19 ****
  
  if not $config.include? 'dbpass'
! 	puts "Missing required value in config file : dbpass"
  	exit 1
  end
--- 15,19 ----
  
  if not $config.include? 'dbpass'
! 	puts 'Missing required value in config file : dbpass'
  	exit 1
  end
***************
*** 35,53 ****
  end
  
  $manager = NgetSuite::DownloadManager.new(max_dls, basedir)
  
! DRb.start_service('druby://localhost:' + port.to_s, $manager)
  
! if $config.fetch('autostart_httpd', false)
  	require 'webrick'
  	include WEBrick
  	s = HTTPServer.new(
! 	  :Port            => ($config.fetch('httpd_port', 2000)),
! 	  :DocumentRoot    => Dir::pwd + "/web"
!     )
! 	trap("INT"){ s.shutdown }
! 	httpThread = Thread.new { s.start }
  	httpThread.join
  end
! 
! DRb.thread.join
--- 35,62 ----
  end
  
+ puts "Starting daemon with log file '#{logfile}'"
  $manager = NgetSuite::DownloadManager.new(max_dls, basedir)
  
! drburl = 'druby://localhost:' + port.to_s
! DRb.start_service(drburl, $manager)
  
! $thingsrunning = [$manager]
! trap('INT') { $thingsrunning.each{ |t| t.shutdown }; DRb.stop_service }
! trap('TERM') { $thingsrunning.each{ |t| t.shutdown }; DRb.stop_service }
! 
! if $config.fetch('autostart_httpd', false) or $config.fetch('httpd_autostart', false)
  	require 'webrick'
  	include WEBrick
+ 	
+ 	port = $config.fetch('httpd_port', 2000)
+ 	puts "Starting web server on port #{port}"
+ 	log 'Web server start'
  	s = HTTPServer.new(
! 		:Port            => port,
! 		:DocumentRoot    => Dir::pwd + File::Separator + 'web'
! 	)
! 	$thingsrunning << s
! 	httpThread = Thread.new { s.start ; log 'Web server down' }
  	httpThread.join
  end
! DRb.thread.join if DRb.thread

Index: ngetcache.sql
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetcache.sql,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** ngetcache.sql	27 Oct 2003 23:06:12 -0000	1.8
--- ngetcache.sql	3 Feb 2004 21:29:38 -0000	1.9
***************
*** 77,81 ****
    `date_started` datetime default NULL,
    `date_finished` datetime default NULL,
!   `status` set('PENDING','PROCESSING','FINISHED','STOPPED') NOT NULL default 'PENDING',
    `nbfiles` smallint(6) NOT NULL default '0',
    `finalword` varchar(64) default NULL,
--- 77,81 ----
    `date_started` datetime default NULL,
    `date_finished` datetime default NULL,
!   `status` set('PENDING','PROCESSING','FINISHED','STOPPED', 'FAILED') NOT NULL default 'PENDING',
    `nbfiles` smallint(6) NOT NULL default '0',
    `finalword` varchar(64) default NULL,

[Ngetsuite-cvs] ngetsuite/ngetsuite downloader.rb,1.17,1.18 downloadmanager.rb,1.14,1.15

From: <jj...@us...> - 2004-02-04 08:45:06

Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv16067/ngetsuite

Modified Files:
	downloader.rb downloadmanager.rb 
Log Message:
daemon and manager handles shutdown, added 'FAILED' status for downloaded file


Index: downloader.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloader.rb,v
retrieving revision 1.17
retrieving revision 1.18
diff -C2 -d -r1.17 -r1.18
*** downloader.rb	2 Feb 2004 17:44:19 -0000	1.17
--- downloader.rb	3 Feb 2004 21:29:38 -0000	1.18
***************
*** 90,98 ****
  				}
  			rescue Exception => e
! 				puts "Exception : " + e
! 				puts " => " + e.message
  				print e.backtrace.join("\n")
  			rescue Error => e
! 				puts "Error : " + e
  				print e.backtrace.join("\n")
  			rescue DBI::DatabaseError => e
--- 90,97 ----
  				}
  			rescue Exception => e
! 				puts "Exception : #{e} => " + e.message
  				print e.backtrace.join("\n")
  			rescue Error => e
! 				puts 'Error : ' + e
  				print e.backtrace.join("\n")
  			rescue DBI::DatabaseError => e
***************
*** 111,115 ****
  	def stop
  		# oh no they killed kenny !
! 		Process.kill("SIGTERM", @io.pid)
  		@io.close
  		@t.exit
--- 110,114 ----
  	def stop
  		# oh no they killed kenny !
! 		Process.kill('SIGTERM', @io.pid)
  		@io.close
  		@t.exit
***************
*** 126,132 ****
  		time = Time.now.to_s
  		$dbh.do 'lock tables queue write'
! 		$dbh.do "update queue set `status` = 'FINISHED', `date_finished`=NOW() where `id` = '#{@id}'"
  		$dbh.do 'unlock tables'
! 		log "Download finished for queue #{@value}, group #{@group}"		
  	end
  
--- 125,132 ----
  		time = Time.now.to_s
  		$dbh.do 'lock tables queue write'
! 		newstatus = success ? 'FINISHED' : 'FAILED'
! 		$dbh.do "update queue set `status` = '#{newstatus}', `date_finished`=NOW() where `id` = '#{@id}'"
  		$dbh.do 'unlock tables'
! 		log "Download #{newstatus.downcase} for queue #{@value}, group #{@group}"		
  	end
  

Index: downloadmanager.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloadmanager.rb,v
retrieving revision 1.14
retrieving revision 1.15
diff -C2 -d -r1.14 -r1.15
*** downloadmanager.rb	2 Feb 2004 17:44:19 -0000	1.14
--- downloadmanager.rb	3 Feb 2004 21:29:38 -0000	1.15
***************
*** 24,65 ****
  
  	def start
! 		if (@started == false)
! 			log "Starting manager"
! 			@started = true
! 			@mainThread = Thread.new {
! 				begin
! 					loop do
! #						log "current : #{@current_dlers}, max : #{@max_dlers}"
! 						if (@current_dlers < @max_dlers)
! 							start_new_downloader
! 						end
! 						check_finished_dls
! 						sleep(@poll_interval)
! 					end
! 				rescue Exception => e
! 					puts "Exception : " + e
! 					puts " => " + e.message
! 					print e.backtrace.join("\n")
! 				rescue Error => e
! 					puts "Error : " + e
! 					print e.backtrace.join("\n")
! 				rescue DBI::DatabaseError => e
! 					puts 'An error occurred'
! 					puts 'Error code: ' + e.err
! 					puts 'Error message: ' + e.errstr
! 				end
! 			}
! 		else
  			debug 'Already started'
  		end
  	end
  		
  	def start_new_downloader
! 		row = $dbh.select_one('select * from queue where `status`  =
! 			\'PENDING\' order by `priority` desc limit 1')
  		
! 		if row == nil
! 			return false
! 		end
  
  		@current_dlers += 1
--- 24,62 ----
  
  	def start
! 		unless @started == false
  			debug 'Already started'
+ 			return
  		end
+ 		@started = true
+ 		log 'Starting manager'
+ 		@mainThread = Thread.new {
+ 		begin
+ 			while @started do
+ 				log "current : #{@current_dlers}, max : #{@max_dlers}" if @lastlog_current != @current_dlers or @lastlog_max != @max_dlers
+ 				@lastlog_current, @lastlog_max = @current_dlers, @max_dlers
+ 				if (@current_dlers < @max_dlers)
+ 					start_new_downloader
+ 				end
+ 				check_finished_dls
+ 				sleep(@poll_interval)
+ 			end
+ 		rescue Exception => e
+ 			puts "Exception : #{e} => #{e.message}"
+ 			print e.backtrace.join("\n")
+ 		rescue Error => e
+ 			puts 'Error : ' + e
+ 			print e.backtrace.join("\n")
+ 		rescue DBI::DatabaseError => e
+ 			puts 'An error occurred'
+ 			puts 'Error code: ' + e.err
+ 			puts 'Error message: ' + e.errstr
+ 		end
+ 		}
  	end
  		
  	def start_new_downloader
! 		row = $dbh.select_one("select * from queue where `status`='PENDING' order by `priority` desc limit 1")
  		
! 		return false if row == nil
  
  		@current_dlers += 1
***************
*** 67,72 ****
  		
  		log "found #{value} for group #{group}\n"
! 		dler = Downloader.new(id, type, value,
! 			group, @basedir + '/' + subdir, prio)
  		@dlers[id] = dler
  		dler.start
--- 64,68 ----
  		
  		log "found #{value} for group #{group}\n"
! 		dler = Downloader.new(id, type, value, group, @basedir + File::Separator + subdir, prio)
  		@dlers[id] = dler
  		dler.start
***************
*** 86,96 ****
  	end
  
  	def check_finished_dls
  		@dlers.each {|id, dler|
! 			if dler.finished?
! 				dler.finish(true)
! 				@dlers.delete(id)
! 				@current_dlers -= 1
! 			end
  		}
  	end
--- 82,111 ----
  	end
  
+ 	def shutdown
+ 		log 'Shutdown'
+ 		if @started
+ 			@started = false
+ 			@dlers.each { |d|
+ 				d.stop
+ 			}
+ 			@mainThread.run
+ 			10.times do |i|
+ 				sleep 0.1
+ 				break unless @mainThread.alive?
+ 				if i == 9
+ 					log 'Killing manager thread'
+ 					@mainThread.kill
+ 				end
+ 			end
+ 		end
+ 	end
+ 	
  	def check_finished_dls
  		@dlers.each {|id, dler|
! 			next unless dler.finished?
! 			dler.finish(true)
! 			@dlers.delete(id)
! 			log "Finished download of queue #{dlers.value}"
! 			@current_dlers -= 1
  		}
  	end

[Ngetsuite-cvs] ngetsuite HOWTO,1.2,1.3 sampleconfig,1.8,1.9

From: <jj...@us...> - 2004-02-04 07:15:26

Update of /cvsroot/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv10708

Modified Files:
	HOWTO sampleconfig 
Log Message:
minor updates, howto improved, sampleconfig commented, added a config option to start STOPPED queues automatically


Index: HOWTO
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/HOWTO,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** HOWTO	20 Jan 2004 23:15:54 -0000	1.2
--- HOWTO	4 Feb 2004 07:13:04 -0000	1.3
***************
*** 1,17 ****
  How to use ngetsuite :
  
  I) Launching the daemon and the web server
  ------------------------------------------
! Before everything, start the daemon and optionnaly the httpd :
! 	./daemon.rb >/dev/null 2>&1 &
! 	./testhttpd.rb >testhttpd.log 2>&1 &
  	disown -a
  
- you can show the groups watched with
- 	./ngetsuite.rb -s
- 
  II) Add some groups
  -------------------
! Add one or more groups with :
  	./ngetsuite.rb -a alt.binaries.pictures.linux,linux-pics,/home/me/download/abpl
  
--- 1,30 ----
  How to use ngetsuite :
  
+ 0) Before the first run
+ -----------------------
+ Set up the config file
+ 	cp sampleconfig ~/.ngetsuite
+ 	vim ~/.ngetsuite
+ the most important is the login/pass for the database.
+ 
+ Then set up the database
+ 	log in mysql as root, add the database and the above user with drop, create, insert, delete, update privileges.
+ 	./resetdb.rb should set up the tables correctly inside the database
+ 
+ You can now proceed to the next step.
+ 
+ 
  I) Launching the daemon and the web server
  ------------------------------------------
! Before everything, start the daemon and optionnaly the httpd (see configfile)
! 	./daemon.rb &
  	disown -a
  
  II) Add some groups
  -------------------
! You can list the newsgroups watched with
! 	./ngetsuite.rb -s
! 
! Add one or more groups with -a, like :
  	./ngetsuite.rb -a alt.binaries.pictures.linux,linux-pics,/home/me/download/abpl
  
***************
*** 20,29 ****
  letter of each component of the full name (alt.binaries.pictures.linux => abpl).
  
! You can get the full list of registered newsgroup with the command :
  	./ngetsuite.rb -s
  
  III) Update the headers from the news server
  --------------------------------------------
! you must update the news reader cache with for example a crontab like this :
  
  10 5 * * * ~/ngetupdate.sh
--- 33,42 ----
  letter of each component of the full name (alt.binaries.pictures.linux => abpl).
  
! You can verify the operation with -s again :
  	./ngetsuite.rb -s
  
  III) Update the headers from the news server
  --------------------------------------------
! you must regularly update the news reader cache with for example a crontab like this :
  
  10 5 * * * ~/ngetupdate.sh
***************
*** 35,48 ****
  
  This will fetch the headers from the news server, parse the cache file and
! insert the headers into the mysql database.
  
! you can void the output if you don't want mails from cron.
  
- Alternatively, you can update a specific group with :
- 	./ngetsuite.rb -u groupid
  
  IV) Add a regexp to the download queue
  --------------------------------------
! You must add a queue to download with :
  	./ngetsuite.rb -r groupid,regexp,prio
  
--- 48,65 ----
  
  This will fetch the headers from the news server, parse the cache file and
! insert the headers into the mysql database, and fetch and save the nfos as
! needed.
  
! You can append '>/dev/null 2>&1' to the output if you don't want mails from cron.
! 
! Alternatively, you can update a specific group on demand with -u, like :
! 	./ngetsuite.rb -u linux-pics
  
  
  IV) Add a regexp to the download queue
  --------------------------------------
! The daemon will automatically retrieve the articles and build the binarie
! matching some regexp (download queue).
! You add a queue with -r :
  	./ngetsuite.rb -r groupid,regexp,prio
  
***************
*** 55,62 ****
  regexp.
  
! When the nget process has finished, the queue will be marked as 'FINISHED'. If
! you want to start processing this queue again (because of a network failure),
! you just have to change the status of the queue to 'PENDING' with :
! 	./ngetsuite.rb -i queueid,PENDING
  
  You can get the list of queued items with :
--- 72,86 ----
  regexp.
  
! List the queues with -t :
! 	./ngetsuite.rb -t
! or	./ngetsuite.rb -t linux-pics
! 
! When the nget process has finished, the queue will be marked as 'FINISHED'.
! You can remove theses queues with -x (get the index with -t) :
! 	./ngetsuite.rb -x 23
! 
! If you want to start processing this queue again (because of a network failure,
! repost..), you just have to change the status of the queue to 'PENDING' with :
! 	./ngetsuite.rb -i 23,PENDING
  
  You can get the list of queued items with :

Index: sampleconfig
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/sampleconfig,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** sampleconfig	2 Feb 2004 17:44:18 -0000	1.8
--- sampleconfig	4 Feb 2004 07:13:04 -0000	1.9
***************
*** 1,20 ****
  dbuser = 'nget'
  dbpass = 'cache4pdv'
  dbname = 'ngetcache'
  
  nget = '/home/nget/bin/nget'
  yydecode = '/usr/bin/yydecode'
  
  cachedir = '/home/nget/.nget4'
  basedir = '/home/nget/testdl'
  
  max_dls = 3
  port = 12345
  daemon_log = '/tmp/nget-daemon.log'
  
  debug = false
  extern_headers_update = ''
  
  httpd_port = 2000
! autostart = true
! autostart_httpd = true
--- 1,53 ----
+ #########################
+ #  Database parameters
+ #
  dbuser = 'nget'
  dbpass = 'cache4pdv'
  dbname = 'ngetcache'
  
+ 
+ ###################
+ #  Program paths
+ #
  nget = '/home/nget/bin/nget'
  yydecode = '/usr/bin/yydecode'
  
+ 
+ ##################
+ #  Directories
+ #
+ # where nget stores its cache
  cachedir = '/home/nget/.nget4'
+ # where to download unless otherwise specified
  basedir = '/home/nget/testdl'
  
+ 
+ ##########################
+ #  Daemon configuration
+ #
+ # maximum number of nget instances in parallel
  max_dls = 3
+ # the port for communication with the daemon
  port = 12345
  daemon_log = '/tmp/nget-daemon.log'
+ # should we automatically restart downloads in state STOPPED ?
+ resume_downloads = false
+ # should the daemon look for eligible queues on startup or wait for -m
+ autostart = true
+ 
  
+ #####################
+ #  Misc
+ #
  debug = false
+ # coma-separated list of the shortnames of the groups for which the program will not try to update the headers (but will still parse them and dl nfos)
  extern_headers_update = ''
  
+ 
+ #####################################
+ # Embedded web server configuration
+ #
  httpd_port = 2000
! # should the daemon start the httpd
! httpd_autostart = true
!

[Ngetsuite-cvs] ngetsuite/ngetsuite downloader.rb,1.19,1.20 downloadmanager.rb,1.16,1.17 ngetcache.rb,1.5,1.6

From: <jj...@us...> - 2004-02-04 07:15:26

Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv10708/ngetsuite

Modified Files:
	downloader.rb downloadmanager.rb ngetcache.rb 
Log Message:
minor updates, howto improved, sampleconfig commented, added a config option to start STOPPED queues automatically


Index: downloader.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloader.rb,v
retrieving revision 1.19
retrieving revision 1.20
diff -C2 -d -r1.19 -r1.20
*** downloader.rb	4 Feb 2004 05:56:50 -0000	1.19
--- downloader.rb	4 Feb 2004 07:13:05 -0000	1.20
***************
*** 72,76 ****
  			@curfile.update(part, speed, filenb, size)
  		when /^autopar/
! 			debug 'autopar: ' + blue(line.gsub(/\t/, red('\\t')))
  			
  		when /^uu_msg\(\d+\):Loaded from .*: '.*' \(.*\): (.*) part \d+.*end (.*)$/
--- 72,76 ----
  			@curfile.update(part, speed, filenb, size)
  		when /^autopar/
! 			debug 'autopar: ' + blue(line)
  			
  		when /^uu_msg\(\d+\):Loaded from .*: '.*' \(.*\): (.*) part \d+.*end (.*)$/
***************
*** 82,87 ****
  			@curfile.finish(true)
  			log "Download finished for file #{@curfile._filename}, queue #{@value}"
! 		#TODO : handle decoding failures (=> @curfile.finish(false))
  		else
  			debug "unhandled msg : #{blue line}"
  		end
--- 82,92 ----
  			@curfile.finish(true)
  			log "Download finished for file #{@curfile._filename}, queue #{@value}"
! 		when /^Connecting to (.*)$/
! 			debug blue("Connection to #{$1}..")
! 		when />> 200/
! 			debug blue('Connected')
! 		when />> (480|381|281)/, /<< (GROUP|AUTHINFO)/
  		else
+ 		#TODO : handle decoding failures (=> @curfile.finish(false))
  			debug "unhandled msg : #{blue line}"
  		end

Index: downloadmanager.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloadmanager.rb,v
retrieving revision 1.16
retrieving revision 1.17
diff -C2 -d -r1.16 -r1.17
*** downloadmanager.rb	4 Feb 2004 05:56:50 -0000	1.16
--- downloadmanager.rb	4 Feb 2004 07:13:05 -0000	1.17
***************
*** 57,66 ****
  	def start_new_downloader
  		row = $dbh.select_one("select * from queue where `status`='PENDING' order by `priority` desc limit 1")
  		
  		return false if row == nil
- 		debug(green("New downloader starting"))
  
  		@current_dlers += 1
  		id, group, type, value, subdir, prio = row.values_at(0, 1, 2, 3, 4, 5)
  		
  		log "found #{value} for group #{group}\n"
--- 57,67 ----
  	def start_new_downloader
  		row = $dbh.select_one("select * from queue where `status`='PENDING' order by `priority` desc limit 1")
+ 		row = $dbh.select_one("select * from queue where `status`='STOPPED' order by `priority` desc limit 1") if not row and $config.fetch('resume_downloads', false)
  		
  		return false if row == nil
  
  		@current_dlers += 1
  		id, group, type, value, subdir, prio = row.values_at(0, 1, 2, 3, 4, 5)
+ 		debug(green("New downloader starting queue #{id}"))
  		
  		log "found #{value} for group #{group}\n"

Index: ngetcache.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/ngetcache.rb,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** ngetcache.rb	4 Feb 2004 05:56:50 -0000	1.5
--- ngetcache.rb	4 Feb 2004 07:13:05 -0000	1.6
***************
*** 18,21 ****
--- 18,22 ----
  	CacheArticle = Struct.new('CacheArticle', 'serverid', 'articlenum', 'bytes', 'lines')
  
+ 	# TODO ajouter une option pour ne parser que les nouveaux articles ? (server.index > lasttime.server.high)
  	def initialize(groupname, path='.', limit=nil, filecallback=nil)
  		@servers = Array.new

[Ngetsuite-cvs] ngetsuite/web/cgi-bin 500.rbx,1.2,NONE

From: <jj...@us...> - 2004-02-04 06:50:22

Update of /cvsroot/ngetsuite/ngetsuite/web/cgi-bin
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv7305/cgi-bin

Removed Files:
	500.rbx 
Log Message:
ménache


--- 500.rbx DELETED ---

[Ngetsuite-cvs] ngetsuite/web README,1.2,NONE README.Debian,1.1,NONE README.gentoo,1.1,NONE htaccess.example,1.1,NONE

From: <jj...@us...> - 2004-02-04 06:50:22

Update of /cvsroot/ngetsuite/ngetsuite/web
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv7305

Removed Files:
	README README.Debian README.gentoo htaccess.example 
Log Message:
ménache


--- README DELETED ---

--- README.Debian DELETED ---

--- README.gentoo DELETED ---

--- htaccess.example DELETED ---

[Ngetsuite-cvs] ngetsuite testhttpd.rb,1.3,NONE

From: <jj...@us...> - 2004-02-04 06:12:15

Update of /cvsroot/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv787

Removed Files:
	testhttpd.rb 
Log Message:
see you in hell


--- testhttpd.rb DELETED ---

[Ngetsuite-cvs] ngetsuite resetdb.rb,NONE,1.1

From: <jj...@us...> - 2004-02-04 05:59:13

Update of /cvsroot/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv31139

Added Files:
	resetdb.rb 
Log Message:
downloader shutdown bugfix, misc bugfix


--- NEW FILE: resetdb.rb ---
#!/usr/bin/env ruby
conffile = ENV['HOME'] + File::Separator + '.ngetsuite'
require 'ngetsuite/config'
config = NgetSuite::Config.new(conffile)

require 'dbi'

sql = File.open('ngetcache.sql')
dbh = DBI.connect("DBI:Mysql:#{config['dbname']}", config['dbuser'], config['dbpass'])
cmd = ''
sql.each { |l|
	next if l =~ /^(#|\s*$)/
	if l=~/CREATE TABLE (`.*`)/
		puts "Resetting #{$1}"
		dbh.do "drop table if exists #{$1}"
	end
	cmd += l
	if l =~ /;/
		dbh.do cmd
		cmd = ''
	end
}	
dbh.disconnect if $dbh
sql.close
puts 'done'

[Ngetsuite-cvs] ngetsuite/ngetsuite core.rb,1.28,1.29 downloader.rb,1.18,1.19 downloadmanager.rb,1.15,1.16 ngetcache.rb,1.4,1.5 utils.rb,1.10,1.11

From: <jj...@us...> - 2004-02-04 05:59:13

Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv31139/ngetsuite

Modified Files:
	core.rb downloader.rb downloadmanager.rb ngetcache.rb utils.rb 
Log Message:
downloader shutdown bugfix, misc bugfix


Index: core.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/core.rb,v
retrieving revision 1.28
retrieving revision 1.29
diff -C2 -d -r1.28 -r1.29
*** core.rb	4 Feb 2004 03:07:57 -0000	1.28
--- core.rb	4 Feb 2004 05:56:50 -0000	1.29
***************
*** 226,230 ****
  					end
  				when 'delete-queue'
! 					Queue.delete(arg, false)
  				when 'show-queue'
  					args = arg.split(',')
--- 226,230 ----
  					end
  				when 'delete-queue'
! 					Queue.delete(arg, true)
  				when 'show-queue'
  					args = arg.split(',')

Index: downloader.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloader.rb,v
retrieving revision 1.18
retrieving revision 1.19
diff -C2 -d -r1.18 -r1.19
*** downloader.rb	3 Feb 2004 21:29:38 -0000	1.18
--- downloader.rb	4 Feb 2004 05:56:50 -0000	1.19
***************
*** 40,47 ****
  
  	def update_status(line)
! 		#debug(Utils.red("begin update_status"))
! 		if line =~ /^already have .*\n$/
  			@nbfiles += 1
! 		elsif line =~ /^Retrieving: (\d+).*\t(\d+)l.*(<.*>)\n$/
  			parts, nblines, mid = $1, $2, $3
  			@nbfiles += 1
--- 40,49 ----
  
  	def update_status(line)
! 		line.chomp!
! 		case line
! 		when /^already have (.*)$/
! 			debug 'file already there : ' + $1
  			@nbfiles += 1
! 		when /^Retrieving: (\d+).*\t(\d+)l.*(<.*>)$/
  			parts, nblines, mid = $1, $2, $3
  			@nbfiles += 1
***************
*** 55,59 ****
  				debug (Utils.red "already in database")
  			end
! 		elsif line =~ /\d+ \((\d+)\/\d+\).*\d+\/(\d+)B.* (\d+)B\/s \w+ (\d+)\/(\d+) \w+\n$/
  			part = $1
  			size = $2
--- 57,69 ----
  				debug (Utils.red "already in database")
  			end
! 		when /^(\d+) \((\d+)\/(\d+)\): (\d+)\/(\d+)L (\d+)\/(\d+)B (\d+)% (\d+)B\/s (\w+) (\d+)\/(\d+) (\w+)$/
! 			part = $2
! 			size = $7
! 			speed = $9
! 			filenb = $12
! 			debug "#{yellow $5+'L'}(#{yellow $7+'B'}) at #{red $9+'B/s'} [#{blue $2}/#{blue $3}] [#{green $11}/#{green $12}]"
! 			@curfile.update(part, speed, filenb, size)
! 		when /\d+ \((\d+)\/\d+\).*\d+\/(\d+)B.* (\d+)B\/s \w+ (\d+)\/(\d+) \w+$/
! 			debug (Utils.red "the previous regexp should have been matched instead of this one")
  			part = $1
  			size = $2
***************
*** 61,72 ****
  			filenb = $4
  			@curfile.update(part, speed, filenb, size)
! 		elsif line =~ /^uu_msg\(\d+\):Loaded from .*: '.*' \(.*\): (.*) part \d+.*end (.*)\n$/
  			debug (Utils.yellow "filename : #{$1}, encoding : #{$2}")
  			@curfile._filename = $1
  			@curfile.encoding = $2
! 		elsif line =~ /^decoded ok/
  			@curfile.finish(true)
  			log "Download finished for file #{@curfile._filename}, queue #{@value}"
  		#TODO : handle decoding failures (=> @curfile.finish(false))
  		end
  		#debug (Utils.red "end update_status")
--- 71,88 ----
  			filenb = $4
  			@curfile.update(part, speed, filenb, size)
! 		when /^autopar/
! 			debug 'autopar: ' + blue(line.gsub(/\t/, red('\\t')))
! 			
! 		when /^uu_msg\(\d+\):Loaded from .*: '.*' \(.*\): (.*) part \d+.*end (.*)$/
  			debug (Utils.yellow "filename : #{$1}, encoding : #{$2}")
  			@curfile._filename = $1
  			@curfile.encoding = $2
! 		when /^decoded ok/
! 			debug 'file decoded'
  			@curfile.finish(true)
  			log "Download finished for file #{@curfile._filename}, queue #{@value}"
  		#TODO : handle decoding failures (=> @curfile.finish(false))
+ 		else
+ 			debug "unhandled msg : #{blue line}"
  		end
  		#debug (Utils.red "end update_status")
***************
*** 78,82 ****
  		@start_time = Time.new
  		@status = 'PROCESSING'
- 		puts "before locking"
  		$dbh.do 'lock tables queue write'
  		$dbh.do "update queue set `status` = 'PROCESSING', `date_started`= NOW()  where `id` = '#{@id}'"
--- 94,97 ----
***************
*** 86,94 ****
  			begin
  				io.each { |line|
- 					debug(Utils.blue(line))
  					update_status line
  				}
  			rescue Exception => e
! 				puts "Exception : #{e} => " + e.message
  				print e.backtrace.join("\n")
  			rescue Error => e
--- 101,111 ----
  			begin
  				io.each { |line|
  					update_status line
  				}
+ 			rescue IOError
+ 				puts 'nget got killed'
+ 				
  			rescue Exception => e
! 				puts "Exception : #{e.class} => " + e.message
  				print e.backtrace.join("\n")
  			rescue Error => e
***************
*** 110,113 ****
--- 127,131 ----
  	def stop
  		# oh no they killed kenny !
+ 		debug "#{red 'dlder.stop'}"
  		Process.kill('SIGTERM', @io.pid)
  		@io.close
***************
*** 115,119 ****
  		
  		$dbh.do 'lock tables queue write'
! 		$dbh.do "update queue set `status` = 'STOPPED', `date_finished`=NOW() where `id` = '#{@id}'"
  		$dbh.do 'unlock tables'
  		log "Download stopped for queue #{@value}, group #{@group}"	
--- 133,137 ----
  		
  		$dbh.do 'lock tables queue write'
! 		$dbh.do "update `queue` set `status` = 'STOPPED', `date_finished`=NOW() where `id` = '#{@id}'"
  		$dbh.do 'unlock tables'
  		log "Download stopped for queue #{@value}, group #{@group}"	

Index: downloadmanager.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/downloadmanager.rb,v
retrieving revision 1.15
retrieving revision 1.16
diff -C2 -d -r1.15 -r1.16
*** downloadmanager.rb	3 Feb 2004 21:29:38 -0000	1.15
--- downloadmanager.rb	4 Feb 2004 05:56:50 -0000	1.16
***************
*** 59,62 ****
--- 59,63 ----
  		
  		return false if row == nil
+ 		debug(green("New downloader starting"))
  
  		@current_dlers += 1
***************
*** 86,90 ****
  		if @started
  			@started = false
! 			@dlers.each { |d|
  				d.stop
  			}
--- 87,91 ----
  		if @started
  			@started = false
! 			@dlers.each { |i, d|
  				d.stop
  			}

Index: ngetcache.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/ngetcache.rb,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** ngetcache.rb	4 Feb 2004 01:45:39 -0000	1.4
--- ngetcache.rb	4 Feb 2004 05:56:50 -0000	1.5
***************
*** 50,53 ****
--- 50,54 ----
  		raise BadNgetCache.new('not a ngetcache file') unless $1 == 'NGET4' and $3 == '1'
  		@partscounttotal = $2.to_i
+ 		@partscountcurrent = 0
  		
  		# servers description
***************
*** 110,113 ****
--- 111,116 ----
  			part.articles << article
  		end
+ 		@partscountcurrent += 1
+ 		print "#{@partscountcurrent}/#{@partscounttotal}\r" if $stdout.isatty
  		part
  	end

Index: utils.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/utils.rb,v
retrieving revision 1.10
retrieving revision 1.11
diff -C2 -d -r1.10 -r1.11
*** utils.rb	2 Feb 2004 17:44:19 -0000	1.10
--- utils.rb	4 Feb 2004 05:56:50 -0000	1.11
***************
*** 24,28 ****
  
  	def Utils.debug(message=nil)
!   		puts "DEBUG: #{message}" if($config.fetch('debug', false) && message)
  	end
  
--- 24,28 ----
  
  	def Utils.debug(message=nil)
!   		puts "D: #{message}" if($config.fetch('debug', false) && message)
  	end

[Ngetsuite-cvs] ngetsuite/ngetsuite group.rb,1.27,1.28

From: <ys...@us...> - 2004-02-04 03:27:02

Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv6150/ngetsuite

Modified Files:
	group.rb 
Log Message:
stupid


Index: group.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/group.rb,v
retrieving revision 1.27
retrieving revision 1.28
diff -C2 -d -r1.27 -r1.28
*** group.rb	4 Feb 2004 03:07:57 -0000	1.27
--- group.rb	4 Feb 2004 03:24:43 -0000	1.28
***************
*** 191,195 ****
  	def mknewscache
  		debug "Beginning mknewscache at #{Time.now.strftime('%H:%M:%S')}"
! 		sth = Db.dbh.prepare 'insert into articles(`group`, `mid`, `nbparts`, `size`, `subject`, `from`, `time`, `rls_subject`) values (?, ?, ?, ?, ?, ?, FROM_UNIXTIME(?))'
  		filecallback = Proc.new{ |f|
  			if f.req >= 1
--- 191,195 ----
  	def mknewscache
  		debug "Beginning mknewscache at #{Time.now.strftime('%H:%M:%S')}"
! 		sth = Db.dbh.prepare 'insert into articles(`group`, `mid`, `nbparts`, `size`, `subject`, `from`, `time`) values (?, ?, ?, ?, ?, ?, FROM_UNIXTIME(?))'
  		filecallback = Proc.new{ |f|
  			if f.req >= 1

[Ngetsuite-cvs] ngetsuite/web release.rhtml,NONE,1.1 dump.rhtml,1.2,1.3 group.rhtml,1.1,1.2 ngetsuite.css,1.9,1.10

From: <ys...@us...> - 2004-02-04 03:10:27

Update of /cvsroot/ngetsuite/ngetsuite/web
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv3058/web

Modified Files:
	dump.rhtml group.rhtml ngetsuite.css 
Added Files:
	release.rhtml 
Log Message:
Now recognizing releases!
Web interface improved!
J'ai toujours pas de stage!

Make sure you've updated your dbs.



--- NEW FILE: release.rhtml ---
<%

require 'ngetsuite/db'
require 'ngetsuite/utils'
require 'ngetsuite/web'
require 'ngetsuite/articles'

web = NgetSuite::Web.new(servlet_request)

web.handle_args
web.update_queue


class ReleaseArticles
	attr_reader :articles, :nfos

	def initialize(group,mid)
		@articles = Array.new
		@nfos = Array.new

		@rls = NgetSuite::Release.new(group,mid,true)

		artq = "select * from `articles` a left join `nfos` n using (`mid`) "
		artq += "where a.`group`='#{group}' and `release`='#{mid}'"
		artq += "order by `subject`"

		puts artq
		@sth = NgetSuite::Db.dbh.execute artq

		@sth.fetch_hash { |row|
			@nfos.push [row["mid"], row["subject"], row["text"]] if row["text"]
			@articles.push row
		}
	end
end

%>
<?xml version="1.0" encoding="ISO-8859-1"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
<head>
<link href="ngetsuite.css" rel="stylesheet" type="text/css">
<title>Queue editor</title>
</head>
<body>

<a href="queue.rhtml?group=<%= web.group._fullname %>">return to queue list</a>

<% rls = ReleaseArticles.new(web.group._fullname, web.rlsmid) %>

<a name="top"></a>
<h1>Release details</h1>

<ul>
	<% if !rls.nfos.empty? %>
		<li><a href="#nfos"><%= rls.nfos.size %> viewable text file(s)</a></li>
	<% end %>
	<li><a href="#artlist">View articles list</a></li>
	<li><a href="#actions">Search/queue</a></li>
</ul>


<a name="nfos"></a>
<% rls.nfos.each { |nfo| %>
	<div class="group">Contents of: <i><%= nfo[1] %></i></div>
	<pre>
<%= nfo[2] %>
	</pre>
	<p></p>
<% } %>

<a name="artlist"></a>
<div class="group">Articles list</div>

<table class='articles'>
	<tr class='headers'>
	    <th>Date/Size</th><th>Subject</th>
	</tr>

	<% rls.articles.each { |art| %>
	<tr>
		<td><i><%= NgetSuite::Utils.pretty_date(art["time"]) %></i><br>
		<b><%= NgetSuite::Utils.pretty_size(art["size"]) %><b></td>
		<td><%= art["subject"] %></td>
	</tr>
	<tr>
		<td class="linesep" colspan="2"></td>
	</tr>
	<% } %>
</table>

</body>
</html>
<% NgetSuite::Db.dbh.disconnect %>


Index: dump.rhtml
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/web/dump.rhtml,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** dump.rhtml	2 Feb 2004 02:41:57 -0000	1.2
--- dump.rhtml	4 Feb 2004 03:07:57 -0000	1.3
***************
*** 36,39 ****
--- 36,55 ----
  	end
  	
+ 	# Returns the common part between this subject and
+ 	# the reference subject. We work with arrays of words
+ 	def compare_subjects(subject)
+ 		common = Array.new
+ 		i = 0
+ 		
+ 		while @refsubject[i] != nil and
+ 		  subject[i] != nil and
+ 		  @refsubject[i] == subject[i]
+ 			common.push subject[i]
+ 			i += 1
+ 		end
+ 		
+ 		return common
+ 	end
+ 	
  
  	# Magic function to tell if an article belongs to a release
***************
*** 43,66 ****
  		fn = Release.filenb_from_subject(subject)
  		
! 		# If the release is still empty, everything matches
! 		# And this file will act as a reference for the others
! 		if @files.empty? then
! 			@refsubject = subject
  			@author = author
! 			@nbexpected = fn[1] if fn[1]
! 			@lastnb = fn[0] if fn[0]
  			@files.push [mid,subject,fn[0]]
  			return true
  		end
  
! 		return false if author != @author
! 		return false if fn[1] != @nbexpected
! 		return false if fn[0] <= @lastnb
  
! 		if 1		# TODO Compare subject with the reference
! 			@files.push [mid,subject,fn[0]]
! 			return true
! 		end
  		
  	end
  
--- 59,116 ----
  		fn = Release.filenb_from_subject(subject)
  		
! 		if !@files.empty?
! 			# Deal with trivial cases
! 			# if the file sequence number is present and
! 			# seems correct, we can add the article without
! 			# further investigation
! 			return false if author != @author
! 			if fn[1] > 0 and fn[0] > 0
! 				puts fn[0].to_s+"/"+fn[1].to_s
! 				return false if fn[1] != @nbexpected
! 				return false if fn[0] <= @lastnb
! 				@files.push [mid,subject,fn[0]]
! 				return true
! 			end
! 		else
! 			# First article: add it,
! 			# and store the subject of words
! 			@refsubject = subject.split(' ')
  			@author = author
! 			@nbexpected = fn[1] if fn[1] > 0
! 			@lastnb = fn[0] if fn[0] > 0
  			@files.push [mid,subject,fn[0]]
  			return true
  		end
  
! 		puts "prout!"
  
! 		# If we're here, the file sequence number
! 		# ("[xx/xx]"-like thing) was not found in the subject
! 		# We have to compare it with the reference subject
! 		# This is ugly and *a little* error prone, but hell...
  		
+ 		if @files.size == 1
+ 			# Second article: update the reference by storing
+ 			# only the common words between the 2 subjects,
+ 			# and add the article if there are any common words
+ 			@refsubject = compare_subjects(subject.split(' '))
+ 			if @refsubject.size > 0
+ 				@files.push [mid,subject,fn[0]]
+ 				return true
+ 			else
+ 				return false
+ 			end
+ 		else
+ 			# Already 2 or more articles in the release
+ 			# Add the article if the common words between
+ 			# its subject and the reference *is* the reference
+ 			
+ 			if compare_subjects(subject.split(' ')) == @refsubject
+ 				@files.push [mid,subject,fn[0]]
+ 				return true
+ 			else
+ 				return false
+ 			end
+ 		end
  	end
  

Index: group.rhtml
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/web/group.rhtml,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** group.rhtml	9 Nov 2003 01:25:33 -0000	1.1
--- group.rhtml	4 Feb 2004 03:07:57 -0000	1.2
***************
*** 10,13 ****
--- 10,70 ----
  web.update_queue
  
+ class GroupDates
+ 	def initialize(group)
+ 		datesq = "select DATE_FORMAT(`time`,'%Y-%m-%d') as isodate, DATE_FORMAT(`time`, '%d/%m') from `articles` where `group`='#{group}' group by isodate order by isodate desc"
+ 		@sth = NgetSuite::Db.dbh.execute datesq
+ 	end
+ 
+ 	def each
+ 		return unless @sth
+ 		@sth.fetch { |row|
+ 			yield row[0], row[1]
+ 		}
+ 	end
+ end
+ 
+ class Releases
+ 	def initialize(group,date)
+ 		rlsq = "select r.*,a.`subject` "
+ #		rlsq += "(select `subject` from `articles` a where r.mid=a.mid) "
+ 		rlsq += "from `releases` r left join `articles` a on r.mid=a.mid "
+ 		rlsq += "where r.`group`='#{group}' "
+ 		if date != "any"
+ 			rlsq += "and DATE_FORMAT(r.`time`, '%Y-%m-%d')='#{date}' "
+ 		end
+ 		rlsq += "order by `completeness` desc, `nbactual` desc, r.`time` desc"
+ 		puts rlsq
+ 		@sth = NgetSuite::Db.dbh.execute rlsq
+ 	end
+ 
+ 	def each
+ 		return unless @sth
+ 		@sth.fetch_hash { |row|
+ 			row["graphmap"] = ""
+ 			if row["rlsmap"] != ""
+ 				for i in 0..row["rlsmap"].size-1
+ 					if row["rlsmap"][i] == "#"[0] then
+ 						row["graphmap"] += "<font class=\"greenbox\">#{i+1}</font>"
+ 					else
+ 						row["graphmap"] += "<font class=\"redbox\">#{i+1}</font>"
+ 					end
+ 					
+ 					row["graphmap"] += "<br>" if i % 25 == 24
+ 						
+ 				end
+ 				row["graphmap"] += "<br>"
+ 			else
+ 				for i in 1..row["nbactual"]
+ 					row["graphmap"] += "<font class=\"cyanbox\">#{i}</font>"
+ 					row["graphmap"] += "<br>" if i % 25 == 24
+ 				end
+ 
+ 				row["graphmap"] += "<br>"
+ 			end
+ 			yield row
+ 		}
+ 	end
+ end
+ 
  %>
  <?xml version="1.0" encoding="ISO-8859-1"?>
***************
*** 17,78 ****
  <link href="ngetsuite.css" rel="stylesheet" type="text/css">
  <title>Queue editor</title>
- <script language="JavaScript">
- <!--
- function CA(frm){
-   for (var i=0;i<frm.elements.length;i++) {
-     var e = frm.elements[i];
-     if ((e.name == 'selected')) {
-       e.checked = frm.markall.checked;
-     }
-   }
- }
- -->
- </script>
  </head>
  <body>
  
! <%= web.prev_page %>
! <%= web.next_page %>
! 	<table class='articles'>
! 		<tr class='headers'>
! 			<th><input type='checkbox' name="markall" onClick="CA(document.articles);"></th>
! 		    <th>Date</th><th>Subject</th><th>Size</th>
! 		</tr>
! <%
! if web.group != nil
! 	req = "select articles.mid, DATE_FORMAT(`time`, '%e/%c %H:%i'), `subject`,
! 		`size`,`from`, nfos.mid
! 		from `articles` LEFT JOIN `nfos` USING (`mid`)
! 		where articles.group = '#{web.group._fullname}' "
! 	if web.filter != nil
! 		req += " and LOWER(`subject`) LIKE  '%#{web.filter.downcase}%' "
! 	end
! 		
! 	req += " order by `time` desc limit #{web.offset},#{web.articles_by_page} "	
! 	puts req
! 	sth = NgetSuite::Db.dbh.execute req
! 	sth.fetch do |row| %>
! 		<input type='hidden' name='mid' value='<%= row[0] %>'>
! 		<tr class='article'>
! 		<td class='selection'><input type='checkbox' name='selected' value='<%= row[0] %>'></td>
! 		<td class='time'><%= row[1] %></td>
! 		<td class='subject'>
! 			<%= 
! 				if row[5].to_s != ''
! 					"<a href=\"nfo.rhtml?group=#{web.group._fullname}&mid=#{row[5]}\" target=\"right\">#{row[2]}</a>"
! 				else
! 					row[2]
! 				end
! 			%>
! 		</td>
! 		<td class='size'><%= NgetSuite::Utils.pretty_size row[3] %></td>
! 		</tr>
! <%	end
! 	sth.finish
! end
! %>
! 	</table>
! <%= web.prev_page %>
! <%= web.next_page %>
  
  </body>
--- 74,108 ----
  <link href="ngetsuite.css" rel="stylesheet" type="text/css">
  <title>Queue editor</title>
  </head>
  <body>
  
! <% dates = GroupDates.new(web.group._fullname)
! 
! dates.each { |iso,display| %>
! 	<a href="group.rhtml?group=<%= web.group._fullname %>&date=<%=iso%>"><%=display%></a>&nbsp;|&nbsp;
! <% } %>
! <a href="group.rhtml?date=any">any</a>
! 
! <div class="group"><%= web.date_str %></div>
! 
! <% releases = Releases.new(web.group._fullname, web.date) %>
! 
! <table class='articles'>
! 	<tr class='headers'>
! 	    <th>Date/Size</th><th>Subject</th>
! 	</tr>
! 
! 	<% releases.each { |rls| %>
! 	<tr>
! 		<td><i><%= NgetSuite::Utils.pretty_date(rls["time"]).split("&nbsp;")[1] %></i><br>
! 		<b><%= NgetSuite::Utils.pretty_size(rls["size"]) %><b></td>
! 		<td><a class="rlslink" target="right" href="release.rhtml?group=<%= web.group._fullname %>&rlsmid=<%= WEBrick::HTTPUtils.escape(rls["mid"]) %>"><%= rls["subject"] %></a><br>
! 		<%= rls["graphmap"] %>
! 	</tr>
! 	<tr>
! 		<td class="linesep" colspan="2"></td>
! 	</tr>
! 	<% } %>
! </table>
  
  </body>

Index: ngetsuite.css
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/web/ngetsuite.css,v
retrieving revision 1.9
retrieving revision 1.10
diff -C2 -d -r1.9 -r1.10
*** ngetsuite.css	9 Nov 2003 01:25:33 -0000	1.9
--- ngetsuite.css	4 Feb 2004 03:07:57 -0000	1.10
***************
*** 30,33 ****
--- 30,39 ----
  }
  
+ td.linesep {
+ 	background-color: #707070;
+ 	height: 1px;
+ 	padding: 0px;
+ }
+ 
  td.grouptab {
  	border: 1px solid black;
***************
*** 51,54 ****
--- 57,61 ----
  	padding: 0px;
  	font-size: 10pt;
+ 	width: 100%;
  }
  
***************
*** 210,211 ****
--- 217,252 ----
  	display: none;
  }
+ 
+ font.greenbox {
+ 	padding: 1px;
+ 	border: 1px solid #000000;
+ 	background-color: #00FF00;
+ 	font-size: 5pt;
+ }
+ 
+ font.redbox {
+ 	padding: 1px;
+ 	border: 1px solid #000000;
+ 	background-color: #FF0000;
+ 	font-size: 5pt;
+ }
+ 
+ font.cyanbox {
+ 	padding: 1px;
+ 	border: 1px solid #000000;
+ 	background-color: #44DDFF;
+ 	font-size: 5pt;
+ }
+ 
+ a.rlslink {
+ 	font-size: 8pt;
+ 	color: #000000;
+ }
+ 
+ a.rlslink:hover {
+ 	color: #FF0000;
+ }
+ 
+ a.rlslink:visited {
+ 	color: #909090;
+ }

Flat | Threaded

<< < 1 2 3 4 5 6 > >> (Page 4 of 6)

2003	Jan	Feb	Mar	Apr	May	Jun	Jul	Aug	Sep	Oct	Nov	Dec (7)
2004	Jan (4)	Feb (56)	Mar (10)	Apr (1)	May (6)	Jun (5)	Jul (2)	Aug	Sep (22)	Oct (7)	Nov (1)	Dec
2005	Jan	Feb	Mar	Apr	May	Jun	Jul	Aug	Sep	Oct	Nov	Dec (4)
2006	Jan	Feb	Mar	Apr (1)	May	Jun	Jul	Aug	Sep	Oct	Nov	Dec