From: <jh...@us...> - 2010-01-20 22:41:54
|
Revision: 190 http://etch.svn.sourceforge.net/etch/?rev=190&view=rev Author: jheiss Date: 2010-01-20 22:41:41 +0000 (Wed, 20 Jan 2010) Log Message: ----------- Store history logs as individual files rather than in RCS. RCS results in a very cumbersome UI for viewing the history logs, making it unlikely they'll be used. Individual files allows for easy viewing and inspection using standard Unix tools like ls, diff, grep, etc. History logs in the old RCS format are converted automatically to the new format. Add any exception message to the message that is reported to the etch server. Otherwise we just report a non-zero status with no message, which is difficult to debug. We were printing the exception message to stderr, but most users run etch in a cron job that sends stderr to /dev/null. Update the history log after performing a revert so that we capture that activity. Perform another bit of code cleanup in the revert action. Add nonrecursive_copy and nonrecursive_copy_and_rename methods to complement the recursive versions we had already. Modified Paths: -------------- trunk/client/etchclient.rb trunk/test/history.rb Modified: trunk/client/etchclient.rb =================================================================== --- trunk/client/etchclient.rb 2010-01-20 22:30:45 UTC (rev 189) +++ trunk/client/etchclient.rb 2010-01-20 22:41:41 UTC (rev 190) @@ -403,7 +403,9 @@ rescue Exception => e status = 1 $stderr.puts e.message + message << e.message $stderr.puts e.backtrace.join("\n") if @debug + message << e.backtrace.join("\n") if @debug end # begin/rescue end # catch @@ -540,8 +542,10 @@ # original file. if config.elements['/config/revert'] origpathbase = File.join(@origbase, file) + origpath = nil # Restore the original file if it is around + revert_occurred = false if File.exist?("#{origpathbase}.ORIG") origpath = "#{origpathbase}.ORIG" origdir = File.dirname(origpath) @@ -553,10 +557,7 @@ puts "Restoring #{origpath} to #{file}" recursive_copy_and_rename(origdir, origbase, file) if (!@dryrun) - - # Now remove the backed-up original so that future runs - # don't do anything - remove_file(origpath) if (!@dryrun) + revert_occurred = true elsif File.exist?("#{origpathbase}.TAR") origpath = "#{origpathbase}.TAR" filedir = File.dirname(file) @@ -566,19 +567,24 @@ puts "Restoring #{file} from #{origpath}" system("cd #{filedir} && tar xf #{origpath}") if (!@dryrun) - - # Now remove the backed-up original so that future runs - # don't do anything - remove_file(origpath) if (!@dryrun) + revert_occurred = true elsif File.exist?("#{origpathbase}.NOORIG") origpath = "#{origpathbase}.NOORIG" puts "Original #{file} didn't exist, restoring that state" # Remove anything we might have written out for this file remove_file(file) if (!@dryrun) + revert_occurred = true + end - # Now remove the backed-up original so that future runs - # don't do anything + # Update the history log + if revert_occurred + save_history(file) + end + + # Now remove the backed-up original so that future runs + # don't do anything + if origpath remove_file(origpath) if (!@dryrun) end @@ -1696,109 +1702,110 @@ origpath end - + # This subroutine maintains a revision history for the file in @historybase def save_history(file) - histpath = File.join(@historybase, "#{file}.HISTORY") + histdir = File.join(@historybase, "#{file}.HISTORY") + current = File.join(histdir, 'current') + + # Migrate old RCS history + if File.file?(histdir) && File.directory?(File.join(File.dirname(histdir), 'RCS')) + if !@dryrun + puts "Migrating old RCS history to new format" + rcsmax = nil + IO.popen("rlog #{histdir}") do |pipe| + pipe.each do |line| + if line =~ /^head: 1.(.*)/ + rcsmax = $1.to_i + break + end + end + end + if !rcsmax + raise "Failed to parse RCS history rlog output" + end + tmphistdir = tempdir(histdir) + 1.upto(rcsmax) do |rcsrev| + rcsrevcontents = `co -q -p1.#{rcsrev} #{histdir}` + # rcsrev-1 because RCS starts revisions at 1.1 and we start files + # at 0000 + File.open(File.join(tmphistdir, sprintf('%04d', rcsrev-1)), 'w') do |rcsrevfile| + rcsrevfile.write(rcsrevcontents) + end + end + FileUtils.copy(histdir, File.join(tmphistdir, 'current')) + File.delete(histdir) + File.rename(tmphistdir, histdir) + end + end # Make sure the directory tree for this file exists in the # directory we save history in. - histdir = File.dirname(histpath) - if !File.directory?(histdir) - puts "Making directory tree #{histdir}" + if !File.exist?(histdir) + puts "Making history directory #{histdir}" FileUtils.mkpath(histdir) if (!@dryrun) end - # Make sure the corresponding RCS directory exists as well. - histrcsdir = File.join(histdir, 'RCS') - if !File.directory?(histrcsdir) - puts "Making directory tree #{histrcsdir}" - FileUtils.mkpath(histrcsdir) if (!@dryrun) - end - - # If the history log doesn't exist and we didn't just create the - # original backup, that indicates that the original backup was made - # previously but the history log was not started at the same time. - # There are a variety of reasons why this might be the case (the - # original was saved by a previous version of etch that didn't have - # the history log feature, or the original was saved manually by - # someone) but whatever the reason is we want to use the original - # backup to start the history log before updating the history log - # with the current file. - if !File.exist?(histpath) && !@first_update[file] + + # If the history log doesn't exist and we didn't just create the original + # backup then that indicates that the original backup was made previously + # but the history log was not started at the same time. There are a + # variety of reasons why this might be the case (the most likely is that + # the original was saved manually by someone) but whatever the reason is + # we want to use the original backup to start the history log before + # updating the history log with the current file. + if !File.exist?(File.join(histdir, 'current')) && !@first_update[file] origpath = save_orig(file) if File.file?(origpath) && !File.symlink?(origpath) puts "Starting history log with saved original file: " + - "#{origpath} -> #{histpath}" - FileUtils.copy(origpath, histpath) if (!@dryrun) + "#{origpath} -> #{current}" + FileUtils.copy(origpath, current) if (!@dryrun) else puts "Starting history log with 'ls -ld' output for " + - "saved original file: #{origpath} -> #{histpath}" - system("ls -ld #{origpath} > #{histpath} 2>&1") if (!@dryrun) + "saved original file: #{origpath} -> #{current}" + system("ls -ld #{origpath} > #{current} 2>&1") if (!@dryrun) end - # Check the newly created history file into RCS - histbase = File.basename(histpath) - puts "Checking initial history log into RCS: #{histpath}" - if !@dryrun - # The -m flag shouldn't be needed, but it won't hurt - # anything and if something is out of sync and an RCS file - # already exists it will prevent ci from going interactive. - system( - "cd #{histdir} && " + - "ci -q -t-'Original of an etch modified file' " + - "-m'Update of an etch modified file' #{histbase} && " + - "co -q -r -kb #{histbase}") - end set_history_permissions(file) end - - # Copy current file - - # If the file already exists in RCS we need to check out a locked - # copy before updating it - histbase = File.basename(histpath) - rcsstatus = false - if !@dryrun - rcsstatus = system("cd #{histdir} && rlog -R #{histbase} > /dev/null 2>&1") - end - if rcsstatus - # set_history_permissions may set the checked-out file - # writeable, which normally causes co to abort. Thus the -f - # flag. - system("cd #{histdir} && co -q -l -f #{histbase}") if !@dryrun - end - + + # Make temporary copy of file + newcurrent = current+'.new' if File.file?(file) && !File.symlink?(file) - puts "Updating history log: #{file} -> #{histpath}" - FileUtils.copy(file, histpath) if (!@dryrun) + puts "Updating history log: #{file} -> #{current}" + remove_file(newcurrent) + FileUtils.copy(file, newcurrent) if (!@dryrun) else - puts "Updating history log with 'ls -ld' output: " + - "#{histpath}" - system("ls -ld #{file} > #{histpath} 2>&1") if (!@dryrun) + puts "Updating history log with 'ls -ld' output: #{file} -> #{current}" + system("ls -ld #{file} > #{newcurrent} 2>&1") if (!@dryrun) end - - # Check the history file into RCS - puts "Checking history log update into RCS: #{histpath}" - if !@dryrun - # We only need one of the -t or -m flags depending on whether - # the history log already exists or not, rather than try to - # keep track of which one we need just specify both and let RCS - # pick the one it needs. - system( - "cd #{histdir} && " + - "ci -q -t-'Original of an etch modified file' " + - "-m'Update of an etch modified file' #{histbase} && " + - "co -q -r -kb #{histbase}") + + # Roll current to next XXXX if current != XXXX + if File.exist?(current) + nextfile = '0000' + maxfile = Dir.entries(histdir).select{|e|e=~/^\d+/}.max + if maxfile + if compare_file_contents(File.join(histdir, maxfile), File.read(current)) + nextfile = nil + else + nextfile = sprintf('%04d', maxfile.to_i + 1) + end + end + if nextfile + File.rename(current, File.join(histdir, nextfile)) if (!@dryrun) + end end - + + # Move temporary copy to current + File.rename(current+'.new', current) if (!@dryrun) + set_history_permissions(file) end - + # Ensures that the history log file has appropriate permissions to avoid # leaking information. def set_history_permissions(file) origpath = File.join(@origbase, "#{file}.ORIG") - histpath = File.join(@historybase, "#{file}.HISTORY") - + histdir = File.join(@historybase, "#{file}.HISTORY") + # We set the permissions to the more restrictive of the original # file permissions and the current file permissions. origperms = 0777 @@ -1813,17 +1820,15 @@ # Mask off the file type fileperms = st.mode & 07777 end - histperms = origperms & fileperms - - File.chmod(histperms, histpath) if (!@dryrun) - - # Set the permissions on the RCS file too - histbase = File.basename(histpath) - histdir = File.dirname(histpath) - histrcsdir = "#{histdir}/RCS" - histrcspath = "#{histrcsdir}/#{histbase},v" - File.chmod(histperms, histrcspath) if (!@dryrun) + + if File.directory?(histdir) + Dir.foreach(histdir) do |histfile| + next if histfile == '.' + next if histfile == '..' + File.chmod(histperms, File.join(histdir, histfile)) if (!@dryrun) + end + end end def get_local_requests(file) @@ -2202,8 +2207,10 @@ # Note that cp -p will follow symlinks. GNU cp has a -d option to # prevent that, but Solaris cp does not, so we resort to cpio. # GNU cpio has a --quiet option, but Solaris cpio does not. Sigh. + # GNU find and cpio also have -print0/--null to handle filenames with + # spaces or special characters, but that's not standard either. system("cd #{sourcedir} && find #{sourcefile} | cpio -pdum #{destdir}") or - raise "Copy #{sourcedir}/#{sourcefile} to #{destdir} failed" + raise "Recursive copy #{sourcedir}/#{sourcefile} to #{destdir} failed" end def recursive_copy_and_rename(sourcedir, sourcefile, destname) tmpdir = tempdir(destname) @@ -2211,7 +2218,17 @@ File.rename(File.join(tmpdir, sourcefile), destname) Dir.delete(tmpdir) end - + def nonrecursive_copy(sourcedir, sourcefile, destdir) + system("cd #{sourcedir} && echo #{sourcefile} | cpio -pdum #{destdir}") or + raise "Non-recursive copy #{sourcedir}/#{sourcefile} to #{destdir} failed" + end + def nonrecursive_copy_and_rename(sourcedir, sourcefile, destname) + tmpdir = tempdir(destname) + nonrecursive_copy(sourcedir, sourcefile, tmpdir) + File.rename(File.join(tmpdir, sourcefile), destname) + Dir.delete(tmpdir) + end + def lock_file(file) lockpath = File.join(@lockbase, "#{file}.LOCK") Modified: trunk/test/history.rb =================================================================== --- trunk/test/history.rb 2010-01-20 22:30:45 UTC (rev 189) +++ trunk/test/history.rb 2010-01-20 22:41:41 UTC (rev 190) @@ -22,6 +22,9 @@ # Create a directory to use as a working directory for the client @testbase = tempdir #puts "Using #{@testbase} as client working directory" + + @origfile = File.join(@testbase, 'orig', "#{@targetfile}.ORIG") + @historydir = File.join(@testbase, 'history', "#{@targetfile}.HISTORY") end def test_history @@ -29,10 +32,6 @@ # Ensure original file is backed up and history log started # - origfile = File.join(@testbase, 'orig', "#{@targetfile}.ORIG") - historyfile = File.join(@testbase, 'history', "#{@targetfile}.HISTORY") - historydir = File.dirname(historyfile) - # Put some text into the original file so that we can make sure it was # properly backed up. origcontents = "This is the original text\n" @@ -63,15 +62,9 @@ #puts "Running initial history test" run_etch(@server, @testbase) - assert_equal(origcontents, get_file_contents(origfile), 'original backup of file') - system("cd #{historydir} && co -q -f -r1.1 #{historyfile}") - #system("ls -l #{historyfile}") - assert_equal(origcontents, get_file_contents(historyfile), 'history log started in rcs') - system("cd #{historydir} && co -q -f #{historyfile}") - #system("ls -l #{historyfile}") - assert_equal(sourcecontents, get_file_contents(historyfile), 'history log of file started and updated') - rcsexit = system("cd #{historydir} && rlog -h #{historyfile} | grep '^head: 1.2'") - assert(rcsexit, 'history log started and updated in rcs') + assert_equal(origcontents, get_file_contents(@origfile), 'original backup of file') + assert_equal(origcontents, get_file_contents(File.join(@historydir, '0000')), '0000 history file') + assert_equal(sourcecontents, get_file_contents(File.join(@historydir, 'current')), 'current history file') # # Ensure history log is updated and original file does not change @@ -86,10 +79,10 @@ #puts "Running update test" run_etch(@server, @testbase) - assert_equal(origcontents, get_file_contents(origfile), 'original backup of file unchanged') - assert_equal(updatedsourcecontents, get_file_contents(historyfile), 'history log of file updated') - rcsexit = system("cd #{historydir} && rlog -h #{historyfile} | grep '^head: 1.3'") - assert(rcsexit, 'history log updated in rcs') + assert_equal(origcontents, get_file_contents(@origfile), 'original backup of file unchanged') + assert_equal(origcontents, get_file_contents(File.join(@historydir, '0000')), '0000 history file') + assert_equal(sourcecontents, get_file_contents(File.join(@historydir, '0001')), '0001 history file') + assert_equal(updatedsourcecontents, get_file_contents(File.join(@historydir, 'current')), 'updated current history file') # # Test revert feature @@ -117,6 +110,11 @@ run_etch(@server, @testbase) assert_equal(origcontents, get_file_contents(@targetfile), 'original contents reverted') + assert(!File.exist?(@origfile), 'reverted original file') + assert_equal(origcontents, get_file_contents(File.join(@historydir, '0000')), '0000 history file') + assert_equal(sourcecontents, get_file_contents(File.join(@historydir, '0001')), '0001 history file') + assert_equal(updatedsourcecontents, get_file_contents(File.join(@historydir, '0002')), '0002 history file') + assert_equal(origcontents, get_file_contents(File.join(@historydir, 'current')), 'reverted current history file') # # Update the contents of a reverted file and make sure etch doesn't @@ -133,6 +131,8 @@ run_etch(@server, @testbase) assert_equal(updatedorigcontents, get_file_contents(@targetfile), 'Updated original contents unchanged') + assert(!File.exist?(@origfile), 'reverted original file') + assert_equal(origcontents, get_file_contents(File.join(@historydir, 'current')), 'Updated reverted current history file') end def test_history_setup @@ -150,7 +150,6 @@ # original contents. # - origfile = File.join(@testbase, 'orig', "#{@targetfile}.ORIG") origcontents = "This is the original text" FileUtils.mkdir_p("#{@repodir}/source/#{@targetfile}") @@ -180,7 +179,7 @@ #puts "Running history setup test" run_etch(@server, @testbase) - assert_equal(origcontents + "\n", get_file_contents(origfile), 'original backup of file via setup') + assert_equal(origcontents + "\n", get_file_contents(@origfile), 'original backup of file via setup') assert_equal(sourcecontents + origcontents + "\n", get_file_contents(@targetfile), 'contents using original backup of file via setup') end @@ -220,7 +219,6 @@ #puts "Running '#{testname}' test" run_etch(@server, @testbase) - origfile = File.join(@testbase, 'orig', "#{@targetfile}.ORIG") origcontents = "This is the original text for #{testname}" FileUtils.mkdir_p("#{@repodir}/source/#{@targetfile}") @@ -249,7 +247,7 @@ #puts "Running '#{testname}' test" run_etch(@server, @testbase) - assert_equal(origcontents + "\n", get_file_contents(origfile), testname) + assert_equal(origcontents + "\n", get_file_contents(@origfile), testname) end def test_history_link @@ -257,10 +255,6 @@ # Ensure original file is backed up when it is a link # - origfile = File.join(@testbase, 'orig', "#{@targetfile}.ORIG") - historyfile = File.join(@testbase, 'history', "#{@targetfile}.HISTORY") - historydir = File.dirname(historyfile) - # Generate another file to use as our link target @destfile = Tempfile.new('etchtest').path @@ -291,9 +285,8 @@ #puts "Running history link test" run_etch(@server, @testbase) - assert_equal(@destfile, File.readlink(origfile), 'original backup of link') - system("cd #{historydir} && co -q -f -r1.1 #{historyfile}") - assert_match("#{@targetfile} -> #{@destfile}", get_file_contents(historyfile), 'history backup of link') + assert_equal(@destfile, File.readlink(@origfile), 'original backup of link') + assert_match("#{@targetfile} -> #{@destfile}", get_file_contents(File.join(@historydir, '0000')), '0000 history file of link') end def test_history_directory @@ -301,10 +294,6 @@ # Ensure original file is backed up when it is a directory # - origfile = File.join(@testbase, 'orig', "#{@targetfile}.ORIG") - historyfile = File.join(@testbase, 'history', "#{@targetfile}.HISTORY") - historydir = File.dirname(historyfile) - # Make the original target a directory File.delete(@targetfile) Dir.mkdir(@targetfile) @@ -341,14 +330,14 @@ #puts "Running history directory test" run_etch(@server, @testbase) - assert(File.directory?(origfile), 'original backup of directory') + assert(File.directory?(@origfile), 'original backup of directory') # Verify that etch backed up the original directory properly - assert_equal(before_uid, File.stat(origfile).uid, 'original directory uid') - assert_equal(before_gid, File.stat(origfile).gid, 'original directory gid') - assert_equal(before_mode, File.stat(origfile).mode, 'original directory mode') + assert_equal(before_uid, File.stat(@origfile).uid, 'original directory uid') + assert_equal(before_gid, File.stat(@origfile).gid, 'original directory gid') + assert_equal(before_mode, File.stat(@origfile).mode, 'original directory mode') # Check that the history log looks reasonable, it should contain an # 'ls -ld' of the directory - assert_match(" #{@targetfile}", get_file_contents(historyfile), 'history backup of directory') + assert_match(" #{@targetfile}", get_file_contents(File.join(@historydir, '0000')), '0000 history file of directory') end def test_history_directory_contents @@ -358,10 +347,7 @@ # differently in that case # - #origfile = File.join(@testbase, 'orig', "#{@targetfile}.ORIG") - origfile = File.join(@testbase, 'orig', "#{@targetfile}.TAR") - historyfile = File.join(@testbase, 'history', "#{@targetfile}.HISTORY") - historydir = File.dirname(historyfile) + origtarfile = File.join(@testbase, 'orig', "#{@targetfile}.TAR") # Make the original target a directory File.delete(@targetfile) @@ -394,12 +380,78 @@ # In this case, because we converted a directory to something else the # original will be a tarball of the directory - assert(File.file?(origfile), 'original backup of directory converted to file') + assert(File.file?(origtarfile), 'original backup of directory converted to file') # The tarball should have two entries, the directory and the 'testfile' # we put inside it - assert_equal('2', `tar tf #{origfile} | wc -l`.chomp.strip, 'original backup of directory contents') + assert_equal('2', `tar tf #{origtarfile} | wc -l`.chomp.strip, 'original backup of directory contents') end - + + def test_history_conversion + # + # Test the conversion of old RCS history logs to the new format + # + + # Mock up an original file and RCS history log + mockorigcontents = "This is the original text\n" + FileUtils.mkdir_p(File.dirname(@origfile)) + File.open(@origfile, 'w') do |file| + file.write(mockorigcontents) + end + historyparent = File.dirname(@historydir) + FileUtils.mkdir_p(historyparent) + File.open(@historydir, 'w') do |file| + file.write(mockorigcontents) + end + histrcsdir = File.join(historyparent, 'RCS') + FileUtils.mkdir_p(histrcsdir) + histbase = File.basename(@historydir) + system( + "cd #{historyparent} && " + + "ci -q -t-'Original of an etch modified file' " + + "-m'Update of an etch modified file' #{histbase} && " + + "co -q -r -kb #{histbase}") + mocksourcecontents = "This is the contents in the RCS history log\n" + system("cd #{historyparent} && co -q -l #{histbase}") + File.open(@historydir, 'w') do |file| + file.write(mocksourcecontents) + end + system( + "cd #{historyparent} && " + + "ci -q -t-'Original of an etch modified file' " + + "-m'Update of an etch modified file' #{histbase} && " + + "co -q -r -kb #{histbase}") + File.open(@targetfile, 'w') do |file| + file.write(mocksourcecontents) + end + + FileUtils.mkdir_p("#{@repodir}/source/#{@targetfile}") + File.open("#{@repodir}/source/#{@targetfile}/config.xml", 'w') do |file| + file.puts <<-EOF + <config> + <file> + <warning_file/> + <source> + <plain>source</plain> + </source> + </file> + </config> + EOF + end + + sourcecontents = "This is a test\n" + File.open("#{@repodir}/source/#{@targetfile}/source", 'w') do |file| + file.write(sourcecontents) + end + + # Run etch + #puts "Running history conversion test" + run_etch(@server, @testbase) + + assert_equal(mockorigcontents, get_file_contents(File.join(@historydir, '0000')), 'RCS conv 0000 history file') + assert_equal(mocksourcecontents, get_file_contents(File.join(@historydir, '0001')), 'RCS conv 0001 history file') + assert_equal(sourcecontents, get_file_contents(File.join(@historydir, 'current')), 'RCS conv current history file') + end + def teardown remove_repository(@repodir) FileUtils.rm_rf(@testbase) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |