[Ngetsuite-cvs] ngetsuite/ngetsuite ngetcache.rb,NONE,1.1 group.rb,1.22,1.23

[<jj...@us...>]

Update of /cvsroot/ngetsuite/ngetsuite/ngetsuite
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv7561/ngetsuite

Modified Files:
	group.rb 
Added Files:
	ngetcache.rb 
Log Message:
should work


--- NEW FILE: ngetcache.rb ---
#!/usr/bin/ruby

class NgetCache
	require 'zlib'

	attr_reader :servers, :files, :sane

	class BadNgetCache < RuntimeError
	end
	CacheServer = Struct.new('CacheServer', 'serverid', 'high', 'low', 'num')
	CacheFile = Struct.new('CacheFile', 'req', 'flags', 'fileid', 'subject', 'author', 'partoff', 'tailoff', 'references', 'parts')
	CachePart = Struct.new('CachePart', 'partnum', 'date', 'mid', 'articles')
	CacheArticle = Struct.new('CacheArticle', 'serverid', 'articlenum', 'bytes', 'lines')

	def initialize(groupname, path='.', sanitycheck = true, limit=nil)
		@servers = Array.new
		@files = Array.new
		@sane = true

		@fd = Zlib::GzipReader.open(path + File::Separator + groupname + ',cache.gz')
		begin
			read_all(limit)
		rescue BadNgetCache
			@sane = false
		ensure
			@fd.close
		end
		cache_check if sanitycheck and @sane
	end

	def readline
		raise BadNgetCache unless @line = @fd.gets
		@line.chomp!
		return true if @line != '.'
	end
	
	def read_all(limit)
		readline
		
		raise BadNgetCache unless @line =~ /^([^\t]*)\t(\d+) ([^\t]*)$/
		raise BadNgetCache unless $1 == 'NGET4' and $3 == '1'
		@partscounttotal = $2.to_i
		
		# servers description
		while (server = read_server)
			@servers << server
		end
		
		# files list
		while (file = read_file)
			@files << file
			if limit
				break if (limit -= 1) < 0
			end
		end
	end
	
	def read_server
		return unless readline
		
		raise BadNgetCache unless @line =~ /^(\d+)\t(\d+)\t(\d+)\t(\d+)$/
		CacheServer.new($1.to_i, $2.to_i, $3.to_i, $4.to_i)
	end
	
	def read_file
		return unless @line = @fd.gets
		
		raise BadNgetCache unless @line =~ /^(-?\d+)\t(\d+)(?:\t(\d+))?\t([^\t]*)\t([^\t]*)\t(-?\d+)\t(-?\d+)$/
		file = CacheFile.new($1.to_i, $2.to_i, ($3.to_i if $3), $4, $5, $6.to_i, $7.to_i, Array.new, Array.new)
	
		# references
		while (ref = read_reference)
			file.references << ref
		end
		
		# parts of the file
		while (part = read_part)
			file.parts << part
		end
		file
	end
	
	def read_reference
		return unless readline
		@line.chomp.sub(/^\./, '')
	end
	
	def read_part
		return unless readline
		
		raise BadNgetCache unless @line =~ /^(-?\d+)\t(\d+)\t([^\t]*)$/
		part = CachePart.new($1.to_i, $2.to_i, $3, Array.new)
		
		# articles giving this part
		while (article = read_article)
			part.articles << article
		end
		part
	end
	
	def read_article
		return unless readline
		raise BadNgetCache unless @line =~ /^(\d+)\t(\d+)\t(\d+)\t(\d+)$/
		CacheArticle.new($1.to_i, $2.to_i, $3.to_i, $4.to_i)
	end
	
	def save_tofile(groupname, path)
		@fd = Zlib::GzipWriter.open(path + File::Separator + groupname + ',cache.newformat.gz')
		write_all
		@fd.close
	end
	
	def write_all
		@fd.puts "NGET4\t#{@partscounttotal} 1"
		@servers.each { |s|
			write_server s
		}
		@fd.puts '.'
		@files.each { |f|
			write_file f
		}
	end
	
	def write_server(s)
		@fd.puts [s.serverid, s.high, s.low, s.num].join("\t")
	end
	
	def write_file(f)
		@fd.puts [f.req, f.flags, f.subject, f.author, f.partoff, f.tailoff].join("\t")
		f.references.each { |r|
			write_reference(r)
		}
		@fd.puts '.'
		f.parts.each { |p|
			write_part(p)
		}
		@fd.puts '.'
	end
	
	def write_reference(r)
		ref = r
		ref = '.'+r if r[0] == '.'
		@fd.puts ref
	end
	
	def write_part(p)
		@fd.puts [p.partnum, p.date, p.mid].join("\t")
		p.articles.each { |a|
			write_article(a)
		}
		@fd.puts '.'
	end
	
	def write_article(a)
		@fd.puts [a.serverid, a.articlenum, a.bytes, a.lines].join("\t")
	end

	def cache_check
		partscount = 0
		@files.each { |f| partscount += f.parts.length }
		@sane = false if partscount != @totalpartscount
		@files.each { |f| f.parts.each { |p| p.articles.each { |a|
			@sane = false unless @servers.map{ |s| s.serverid }.include? a.serverid
		}}}
		puts "INSANE" unless @sane
	end
end

Index: group.rb
===================================================================
RCS file: /cvsroot/ngetsuite/ngetsuite/ngetsuite/group.rb,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -d -r1.22 -r1.23
*** group.rb	2 Feb 2004 17:44:19 -0000	1.22
--- group.rb	3 Feb 2004 01:31:49 -0000	1.23
***************
*** 1,5 ****
  require 'ngetsuite/utils'
  require 'ngetsuite/queue'
! require 'zlib'
  
  module NgetSuite
--- 1,5 ----
  require 'ngetsuite/utils'
  require 'ngetsuite/queue'
! require 'ngetsuite/ngetcache'
  
  module NgetSuite
***************
*** 27,31 ****
  	include Db
  
! 	Db.autotable(Group, "groups")
  
  	def Group.getname(id)
--- 27,31 ----
  	include Db
  
! 	Db.autotable(Group, 'groups')
  
  	def Group.getname(id)
***************
*** 104,109 ****
  	def _shortname
  		if (@_shortname == nil)
! 			@_shortname = ''
! 			@_fullname.split('.').each {|item| @_shortname << item[0] }
  		end
  
--- 104,108 ----
  	def _shortname
  		if (@_shortname == nil)
! 			@_shortname = @_fullname.split('.').map {|item| item[0] }.join('')
  		end
  
***************
*** 127,138 ****
--- 126,150 ----
  	end
  
+ 	def announceupdate
+ 		#Db.dbh.execute "insert into `articles` (`group`, `action`) values ('#{@_fullname}', 'updating')"
+ 	end
+ 	def announcedupdate
+ 		#'updating' == Db.dbh.execute "select `action` from `articles` where `group` == '#{@_fullname}'"
+ 	end
+ 	def finishupdate
+ 		#Db.dbh.execute "delete from `articles` where `group`='#{@_fullname}', `action`='updating'"
+ 	end
+ 	
  	def update
+ 		puts "Warning: A headers update seems to be running !" if announcedupdate
  		updatecache unless $config['extern_headers_update'].split(',').include?(@_shortname)
  		clear_articles
  		clear_nfos
+ 		announceupdate
  		mknewscache
  		fetchnfos
  		@_date_updated = Time.now
  		sync_to_db
+ 		finishupdate
  	end
  
***************
*** 168,175 ****
  	def mknewscache
  		debug "Beginning mknewscache at #{Time.now.strftime('%H:%M:%S')}"
! 		sth = Db.dbh.prepare 'insert into articles(`group`, `mid`, `nbparts`, `size`,
! 			`subject`, `from`, `time`, `rls_subject`) values (?, ?, ?, ?, ?, ?, FROM_UNIXTIME(?), ?)'
! 		debug 'Parsing cache'
! 		cachefile = Zlib::GzipReader.open("#{@cachedir}/#{@_fullname},cache.gz")
  		while (cachefile.gets != ".\n")
  		end
--- 180,201 ----
  	def mknewscache
  		debug "Beginning mknewscache at #{Time.now.strftime('%H:%M:%S')}"
! 		cache = NgetCache.new(@_fullname, @cachedir)
! 		puts "The cache file for #{@_fullname} seems corrupted !" if not cache.sane
! 		sth = Db.dbh.prepare 'insert into articles(`group`, `mid`, `nbparts`, `size`, `subject`, `from`, `time`, `rls_subject`) values (?, ?, ?, ?, ?, ?, FROM_UNIXTIME(?), ?)'
! 		cache.files.each { |f|
! 			next if f.req < 1
! 			partcnt, size, a = 0, 0, nil
! 			rsubj = f.subject.gsub(/^(.* )(- .*)?".*$/, '\1').gsub(/\d{1,3}(\/| of )\d{1,3}/, '#\1#').gsub(/ \d\d? /, ' # ')
! 			f.parts.each { |p|
! 				next if p.articles.empty?
! 				a = p.articles[0]
! 				size += a.bytes / ( a.bytes<100*a.lines ? 1.39 : 1.0334)
! 				partcnt += 1
! 				if partcnt >= f.req
! 					sth.execute(@_fullname, p.mid, f.req, size, f.subject, f.from, p.time, rsubj)
! 				end
! 			}
! 		}
! 		if false
  		while (cachefile.gets != ".\n")
  		end
***************
*** 203,207 ****
  	 		end
  		end
! 		cachefile.close
  		debug "End mknewscache at #{Time.now.strftime('%H:%M:%S')}"
  	end
--- 229,233 ----
  	 		end
  		end
! 		end
  		debug "End mknewscache at #{Time.now.strftime('%H:%M:%S')}"
  	end