From: <chr...@us...> - 2009-05-03 04:17:55
|
Revision: 5385 http://jnode.svn.sourceforge.net/jnode/?rev=5385&view=rev Author: chrisboertien Date: 2009-05-03 04:17:51 +0000 (Sun, 03 May 2009) Log Message: ----------- Updates to command utilities and grep ADW - protected access to the filter mechansim - new optional callback for special files - directory filters to prevent recursing whole directory branches - directories beyond max level are no longer recursed - bunch of new javadoc IOUtils - methods to read all the lines of a file into a List GrepCommand - reworked to use ADW Signed-off-by: chrisboertien <chr...@gm...> Modified Paths: -------------- trunk/cli/src/commands/org/jnode/command/file/GrepCommand.java trunk/cli/src/commands/org/jnode/command/util/AbstractDirectoryWalker.java trunk/cli/src/commands/org/jnode/command/util/IOUtils.java Modified: trunk/cli/src/commands/org/jnode/command/file/GrepCommand.java =================================================================== --- trunk/cli/src/commands/org/jnode/command/file/GrepCommand.java 2009-05-02 11:03:59 UTC (rev 5384) +++ trunk/cli/src/commands/org/jnode/command/file/GrepCommand.java 2009-05-03 04:17:51 UTC (rev 5385) @@ -20,8 +20,9 @@ package org.jnode.command.file; +import java.io.BufferedReader; import java.io.File; -import java.io.FileReader; +import java.io.FileFilter; import java.io.InputStream; import java.io.IOException; import java.io.LineNumberReader; @@ -33,12 +34,13 @@ import java.util.regex.MatchResult; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; - -import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Deque; import java.util.List; +import org.apache.log4j.Logger; +import org.jnode.command.util.AbstractDirectoryWalker; +import org.jnode.command.util.AbstractDirectoryWalker.PathnamePatternFilter; +import org.jnode.command.util.IOUtils; import org.jnode.shell.AbstractCommand; import org.jnode.shell.syntax.Argument; import org.jnode.shell.syntax.FileArgument; @@ -57,7 +59,9 @@ */ public class GrepCommand extends AbstractCommand { + private static final Logger log = Logger.getLogger(GrepCommand.class); private static final boolean DEBUG = false; + private static final int BUFFER_SIZE = 8192; private static final String help_matcher_fixed = "Patterns are fixed strings, seperated by new lines. Any of " + "which is to be matched."; @@ -151,6 +155,7 @@ private static final String help_pattern_files = "File with patterns to match, one per line."; private static final String help_files = "The files to match against. If there are no files, or if any file is " + "'-' then match stdandard input."; + private static final String err_ex_walker = "Exception while walking."; private final StringArgument Patterns; private final FileArgument PatternFiles; @@ -215,8 +220,6 @@ private static final int PREFIX_FB = PREFIX_FILE | PREFIX_BYTE; private static final int PREFIX_LB = PREFIX_LINE | PREFIX_BYTE; - private static final Pattern multiGlob = Pattern.compile("[*]"); - private PrintWriter err; private PrintWriter out; private Reader in; @@ -309,7 +312,8 @@ FileArgument("files", Argument.MULTIPLE | Argument.EXISTING | FileArgument.HYPHEN_IS_SPECIAL, help_files); registerArguments(Patterns, PatternFiles, Files, NullTerm); - match = multiGlob.matcher(" "); + // Default matcher + match = Pattern.compile(".*").matcher(""); } /** @@ -344,10 +348,10 @@ name = file.getPath(); try { if (name.equals("-")) { - reader = new LineNumberReader(in); + reader = new LineNumberReader(in, BUFFER_SIZE); name = prefixLabel; } else { - reader = new LineNumberReader(new FileReader(file)); + reader = IOUtils.openLineReader(file, BUFFER_SIZE); } if (exitOnFirstMatch) { debug(" exitOnFirstMatch"); @@ -382,13 +386,7 @@ error("IOException greping file : " + file); e.printStackTrace(); } finally { - if (reader != null) { - try { - reader.close(); - } catch (IOException _) { - // ignore - } - } + IOUtils.close(reader); } } } catch (Exception e) { @@ -487,7 +485,6 @@ * Outputs a matched string, in the given file, at the given line and the given byte offset. Outputs * to stdout the match string, along with any set prefix options. */ - // TODO PREFIX_TAB private void printMatch(String line, String name, int lineCount, int byteCount) { String sLine; String sByte; @@ -623,19 +620,6 @@ return Pattern.compile(sb.toString(), flags); } - private Pattern glob2Pattern(String glob) { - debug("glob-1 > " + glob); - String s = glob; - glob = multiGlob.matcher(glob).replaceAll(".*"); - glob = glob.replace('?', '.'); - debug("glob-2 > " + glob); - try { - return Pattern.compile(glob); - } catch (PatternSyntaxException e) { - throw new InternalError("Invalid glob pattern {" + s + "," + glob + "}"); - } - } - /*********************************************************/ /************** Command Line Parsing *********************/ /*********************************************************/ @@ -720,12 +704,12 @@ } if ((prefix & (PREFIX_FILE | PREFIX_NOFILE)) == (PREFIX_FILE | PREFIX_NOFILE)) { - throw new InternalError("PREFIX_NOFILE && PREFIX_FILE"); + throw new AssertionError("PREFIX_NOFILE && PREFIX_FILE"); } } private void parsePatterns() { - LineNumberReader reader; + BufferedReader reader; String line; patterns = new ArrayList<Pattern>(); @@ -734,154 +718,133 @@ patterns.add(rewritePattern(s)); } catch (PatternSyntaxException e) { error("Invalid Pattern : " + s); + exit(2); } } for (File file : PatternFiles.getValues()) { reader = null; try { - reader = new LineNumberReader(new FileReader(file)); + reader = IOUtils.openBufferedReader(file, BUFFER_SIZE); while ((line = reader.readLine()) != null) { try { patterns.add(rewritePattern(line)); } catch (PatternSyntaxException e) { error("Invalid Pattern : " + line); + exit(2); } } } catch (IOException e) { debug("IOException while parsing pattern file : " + file); + error("Error reading file: " + file); + exit(2); } finally { - if (reader != null) { - try { - reader.close(); - } catch (IOException _) { - // ignore - } - } + IOUtils.close(reader); } } } + private class Walker extends AbstractDirectoryWalker { + @Override + public void handleFile(File file) { + files.add(file); + } + @Override + public void handleDir(File dir) { + // no-op + } + @Override + public void handleRestrictedFile(File file) { + // no-op + } + + private void doFile(File file) { + if (notFiltered(file)) { + files.add(file); + } + } + } + private void parseFiles() { String line; String name; - LineNumberReader reader; + BufferedReader reader; files = new ArrayList<File>(); + if (!Files.isSet()) { files.add(new File("-")); return; } - List<Pattern> excludes = new ArrayList<Pattern>(); - List<Pattern> includes = new ArrayList<Pattern>(); - List<String> excludeDirs = new ArrayList<String>(); + Walker walker = new Walker(); + for (String s : Include.getValues()) { - includes.add(glob2Pattern(s)); + walker.addFilter(new PathnamePatternFilter(s, false)); } - for (String s : ExcludeDir.getValues()) { - excludeDirs.add(s); - } - for (String s : Exclude.getValues()) { - excludes.add(glob2Pattern(s)); + walker.addFilter(new PathnamePatternFilter(s, true)); } for (File file : ExcludeFile.getValues()) { - reader = null; + reader = IOUtils.openBufferedReader(file, BUFFER_SIZE); + List<String> lines = null; try { - reader = new LineNumberReader(new FileReader(file)); - while ((line = reader.readLine()) != null) { - excludes.add(glob2Pattern(line)); - } - } catch (IOException _) { - debug("IOException while parsing exclude file: " + file); + lines = IOUtils.readLines(reader); } finally { - try { - if (reader != null) { - reader.close(); - } - } catch (IOException _) { - // ignore + IOUtils.close(reader); + } + if (lines != null) { + for (String s : lines) { + walker.addFilter(new PathnamePatternFilter(s, true)); } } } - Deque<File> stack = new ArrayDeque<File>(); + List<String> excludeDirs = new ArrayList<String>(); + + for (final String s : ExcludeDir.getValues()) { + walker.addDirectoryFilter(new FileFilter() { + @Override + public boolean accept(File file) { + return !(file.isDirectory() && file.getName().equals(s)); + } + }); + } + + List<File> dirs = new ArrayList<File>(); + for (File file : Files.getValues()) { - name = file.getName(); - if (name.equals("-")) { - files.add(file); - continue; - } if (file.isDirectory()) { - if (!excludeDir(name, excludeDirs) && recurse) { - stack.push(file); + if (recurse) { + dirs.add(file); } - } else if (includeFile(name, includes) && !excludeFile(name, excludes)) { - files.add(file); + } else if (file.isFile()) { + walker.doFile(file); + } else { + // skip special files } } - while (stack.size() > 0) { - File dir = stack.pop(); - if (!dir.isDirectory()) { - err.println("Stack has non-directory: " + dir); - continue; + try { + if (dirs.size() > 0) { + walker.walk(dirs); } - for (File file : dir.listFiles()) { - name = file.getName(); - if (file.isDirectory()) { - if (!excludeDir(name, excludeDirs)) { - stack.push(file); - } - } else if (includeFile(name, includes) && !excludeFile(name, excludes)) { - files.add(file); - } - } - } - } - - private boolean excludeFile(String name, List<Pattern> excludes) { - if (excludes.size() == 0) return false; - for (Pattern exclude : excludes) { - if (exclude.matcher(name).matches()) { - debug("Exclude: " + name); - return true; - } + } catch (IOException e) { + // technically, the walker shouldn't let this propogate unless something + // is really wrong. + error(err_ex_walker); + exit(2); } - return false; } - private boolean excludeDir(String name, List<String> excludeDirs) { - if (excludeDirs.size() == 0) return false; - for (String exclude : excludeDirs) { - if (name.equals(exclude)) { - debug("ExcludeDir: " + name); - return true; - } - } - return false; - } - - private boolean includeFile(String name, List<Pattern> includes) { - if (includes.size() == 0) return true; - for (Pattern include : includes) { - if (include.matcher(name).matches()) { - debug("Include: " + name); - return true; - } - } - return false; - } - private void error(String s) { if (!suppress) err.println(s); } private void debug(String s) { - if (debug) err.println(s); + if (debug) log.debug(s); } private void debugOptions() { Modified: trunk/cli/src/commands/org/jnode/command/util/AbstractDirectoryWalker.java =================================================================== --- trunk/cli/src/commands/org/jnode/command/util/AbstractDirectoryWalker.java 2009-05-02 11:03:59 UTC (rev 5384) +++ trunk/cli/src/commands/org/jnode/command/util/AbstractDirectoryWalker.java 2009-05-03 04:17:51 UTC (rev 5385) @@ -45,11 +45,11 @@ * DirectoryWalker. * * @author Alexander Kerner - * + * @author chris boertien */ public abstract class AbstractDirectoryWalker { - private class FileObject { + private static class FileObject { final File file; final Long depth; @@ -59,83 +59,150 @@ } } + /** + * A FileFilter that filters based on matching a pathname glob pattern + */ public static class PathnamePatternFilter implements FileFilter { private Matcher matcher; private boolean exclude; + /** + * Create the filter with the given pattern and orientation + * + * @param pattern the pathname glob pattern to match with + * @param exclude if true, reverse the sense of matching and + * exclude files that match. + */ public PathnamePatternFilter(String pattern, boolean exclude) { this.exclude = exclude; this.matcher = PathnamePattern.compilePosixShellPattern(pattern, 0).matcher(""); } + @Override public boolean accept(File file) { return matcher.reset(file.getName()).matches() ^ exclude; } } + /** + * A FileFilter that filters based on matching a regular expression. + */ public static class RegexPatternFilter implements FileFilter { private Matcher matcher; private boolean exclude; + /** + * Create the filter with the given pattern and orientation + * + * @param pattern the regular expression to match with + * @param exclude, if true, reverse the sense of matching and + * exclude files that match the pattern. + */ public RegexPatternFilter(String pattern, boolean exclude) { this.exclude = exclude; this.matcher = Pattern.compile(pattern).matcher(""); } + @Override public boolean accept(File file) { return matcher.reset(file.getName()).matches() ^ exclude; } } + /** + * A FileFilter that filters based on the file modification time. + */ public static class ModTimeFilter implements FileFilter { private long modTime; private boolean newer; + /** + * Create the filter with the given mod time and direction + * + * @param size the time point to filter on + * @param newer if true, accept if the file mtime is > time, false + * accepts if the file mtime is <= to time. + */ public ModTimeFilter(long time, boolean newer) { this.modTime = time; this.newer = newer; } + @Override public boolean accept(File file) { return file.lastModified() == modTime || ((file.lastModified() < modTime) ^ newer); } } + /** + * A FileFilter that filters based on the file size. + */ public static class SizeFilter implements FileFilter { private long size; private boolean greater; + /** + * Create the filter with the given size and direction. + * + * @param size the size point to filter on + * @param greater if true, accept if the file length is > size, false + * accepts if the file length is <= to size. + */ public SizeFilter(long size, boolean greater) { this.size = size; this.greater = greater; } + @Override public boolean accept(File file) { - return file.length() == size || ((file.length() < size) ^ greater); + return file.length() == size || ((file.length() <= size) ^ greater); } } private final Stack<FileObject> stack = new Stack<FileObject>(); private final Set<FileFilter> filters = new HashSet<FileFilter>(); - private volatile boolean cancelled = false; + private final Set<FileFilter> dirFilters = new HashSet<FileFilter>(); private volatile Long maxDepth = null; private volatile Long minDepth = null; + private volatile boolean cancelled = false; /** - * Main method to walk through directory hierarchy. + * Walk the directory heirarchies of the given directories. + * + * Before walking begins on each of the given directories, the + * extending class has a chance to do some initialization through + * {@code handleStartingDir}. Once walking has commenced, each file + * will be checked against the current set of constraints, and + * passed to the extending class for further processing if accepted. + * When walking is complete for that branch, the {@code lastAction} + * method is called, and the walker moves on to the next directory, + * or returns if there are no more directories to walk. + * + * If an IOException propogates beyond the {@code walk} method, there + * is currently no way to resume walking. The following reasons may + * cause this to happen. + * <ul> + * <li>Any of the supplied directories are null, or not a directory. + * <li>A SecurityException was triggered, and the caller has not overriden + * the {@code handleRestrictedDir} method. + * </ul> * * @param dirs Array of <code>File</code> to walk through. * @throws IOException if any IO error occurs. + * @throws NullPointerException if dirs is null, or contains no directories */ public synchronized void walk(final File... dirs) throws IOException { if (dirs == null || dirs.length == 0) { throw new NullPointerException("Directory to walk from must not be null"); } for (File dir : dirs) { + // perhaps this shouldn't fail like this, as it may + // be possible that this was simply due to a race condition + // with another process that has deleted the directory already if (dir == null || !dir.isDirectory()) throw new IOException("No such directory " + dir); @@ -148,6 +215,8 @@ handle(stack.pop()); } lastAction(cancelled); + // if this was cancelled, we need to clear the stack + stack.clear(); } } @@ -156,22 +225,27 @@ } private void handle(final FileObject file) throws IOException { - if ((minDepth != null && file.depth < minDepth) || - (maxDepth != null && file.depth > maxDepth)) { + if (minDepth != null && file.depth < minDepth) { // out of boundaries - } else if (notFiltered(file)) { + } else if (notFiltered(file.file)) { handleFileOrDir(file); } else { // filtered out } try { - handleChilds(file); + // Don't descend into directories beyond maxDepth + if (file.file.isDirectory() && (maxDepth == null || file.depth < maxDepth) && dirNotFiltered(file.file)) { + handleChilds(file); + } } catch (SecurityException e) { // Exception rises, when access to folder content was denied handleRestrictedFile(file.file); } } - + + /** + * Add a directories contents to the stack. + */ private void handleChilds(final FileObject file) throws IOException, SecurityException { final Stack<File> stack = new Stack<File>(); final File[] content = file.file.listFiles(); @@ -200,62 +274,142 @@ } } - + + /** + * Trigger the callbacks + */ private void handleFileOrDir(final FileObject file) throws IOException { if (file.file.isDirectory()) handleDir(file.file); else if (file.file.isFile()) handleFile(file.file); else { - // ignore unknown file type + handleSpecialFile(file.file); } } - - private boolean notFiltered(final FileObject file) { + + /** + * Process a file through a set of file filters. + * + * This may be called from extending classes in order to bypass + * the regaular walking procedure. + * + * As an example, if the caller simply wants to process specific + * files through the filter set, without setting up a full directory + * walk. + * + * @param file the file to check + * @return true if the file was accepted by all the filters, or if there + * were not filters. + */ + protected final boolean notFiltered(final File file) { if (!filters.isEmpty()) for (FileFilter filter : filters) - if (!filter.accept(file.file)) + if (!filter.accept(file)) return false; return true; } + + /** + * Stop recursing if this is false. + */ + private boolean dirNotFiltered(File file) { + if (!dirFilters.isEmpty()) { + for (FileFilter filter : dirFilters) { + if (!filter.accept(file)) { + return false; + } + } + } + return true; + } /** - * Abort walking. + * Stop walking the current directory heirarchy. + * + * This will not stop the walker altogether if there were multiple directories + * passed to {@code walk}. Instead, walking of the current directory heirarchy + * will stop, {@code lastAction(true)} is called, and the walker is reset with + * the next directory. If this it was the last directory, or there was only one + * then walk will return without error. */ public void stopWalking() { cancelled = true; } /** + * The minimum depth level (exclusive) at which to begin handling files. + * + * The initial directory has a depth level of 0. Therefore if you set the + * minimum depth to 0, the initial directory will not handled, but its + * contents will. + * + * A negative value will be seen as null. * * @param min starting depth at which actual action performing is started. */ public void setMinDepth(Long min) { - minDepth = min; + if (min >= 0) { + minDepth = min; + } } /** - * + * The maximum depth level (inclusive) at which to stop handling files. + * + * When the walker reaches this level, it will not recurse any deeper + * into the file heirarchy. If the maximjm depth is 0, the initial directory + * will be handled, but the walker will not query for its contents. + * + * A negative value will be seen as null. + * * @param max ending depth at which actual action performing is stopped. */ public void setMaxDepth(Long max) { - maxDepth = max; + if (max >= 0) { + maxDepth = max; + } } /** - * + * Add a FileFilter to this walker. + * + * Before the extended class is asked to handle a file or directory, it + * must be accepted by the set of filters supplied. If no filters are + * supplied, then every file and directory will be handled. + * * @param filter <code>FileFilter</code> to be added to this * DirectoryWalkers FilterSet. */ public synchronized void addFilter(FileFilter filter) { filters.add(filter); } - + /** + * Add a FileFilter to stop recursing of directories. + * + * Before recursing a directory, it must be accepted by the set of + * directory filters supplied. If no filters are supplied, then this + * will not prevent recursing of directories. + * + * @param filter <code>FileFilter</code> to be added + */ + public synchronized void addDirectoryFilter(FileFilter filter) { + dirFilters.add(filter); + } + + /** + * Handle a file or directory that triggered a SecurityException. + * * This method is called, when access to a file was denied.<br> * Default implementation will rise a <code>IOException</code> instead of * <code>SecurityException</code>. May be overridden by extending classes to * do something else. + * + * Because this method throws an IOException that will propogate beyond the + * walk method, if an application wishes to continue walking after encountering + * a SecurityException while accessing a file or directory, then it must override + * this and provide an implementation that does not throw an exception. * * @param file <code>File</code>-object, to which access was restricted. * @throws IOException in default implementation. @@ -265,9 +419,16 @@ } /** - * This method is called, when walking is about to start.<br> + * Handle the initial directory of a tree. + * + * This method is called, when walking is about to start. It gets called + * for each directory that was initially supplied to the walk method. + * * By default, it does nothing. May be overridden by extending classes to do * something else. + * + * Override this to do some initialization before starting to walk each of the + * given directory roots. * * @param file <code>File</code>-object, that represents starting dir. * @throws IOException if IO error occurs. @@ -286,19 +447,34 @@ } /** - * - * @param file <code>File</code>-object, that represents current directory. <br> - * Override this, to do some actions on this directory. + * Handle a directory. + * + * Override this to perform some operation on this directory. + * + * @param file <code>File</code>-object, that represents current directory. * @throws IOException if IO error occurs. */ public abstract void handleDir(final File file) throws IOException; /** - * - * @param file <code>File</code>-object, that represents current file. <br> - * Override this, to do some actions on this file. + * Handle a file. + * + * Override this to perform some operation on this file. + * + * @param file <code>File</code>-object, that represents current file. * @throws IOException if IO error occurs. */ public abstract void handleFile(final File file) throws IOException; - + + /** + * Handle a special file. + * + * Override this to perform some operation on this special file. + * + * @param <code>File</code> object that represents a special file. + * @throws IOException if an IO error occurs. + */ + public void handleSpecialFile(File file) throws IOException { + // do nothing by default + } } Modified: trunk/cli/src/commands/org/jnode/command/util/IOUtils.java =================================================================== --- trunk/cli/src/commands/org/jnode/command/util/IOUtils.java 2009-05-02 11:03:59 UTC (rev 5384) +++ trunk/cli/src/commands/org/jnode/command/util/IOUtils.java 2009-05-03 04:17:51 UTC (rev 5385) @@ -35,6 +35,8 @@ import java.io.OutputStream; import java.io.PrintWriter; import java.io.Reader; +import java.util.List; +import java.util.LinkedList; /** * Convenience IO methods. @@ -402,6 +404,7 @@ * @throws NullPointerException if in, out or str are null */ public static boolean promptYesOrNo(Reader in, PrintWriter out, String prompt, boolean defaultChoice) { + checkNull(in, out, prompt); String input; // put a cap on the loops so it doesn't become an infinite loop @@ -467,7 +470,63 @@ return input; } + public static List<String> readLines(Reader reader) { + return readLines(new BufferedReader(reader, BUFFER_SIZE), -1); + } + + public static List<String> readLines(BufferedReader reader) { + return readLines(reader, -1); + } + /** + * Read all of the lines from a Reader. + * + * Calling this is the same as readLines(new BufferedReader(reader), max) + * + * @param reader the source to read from + * @param max the max number of lines to read, if this is -1 Integer.MAX_VALUE is used + * @returns a list of strings, possibly empty, or null if there was an exception. + * @throws NullPointerException if reader is null + */ + public static List<String> readLines(Reader reader, int max) { + return readLines(new BufferedReader(reader, BUFFER_SIZE), max); + } + + /** + * Read all of the lines from a Reader. + * + * If there are no lines to read, an empty List will be returned. + * + * If there was an exception while reading, null will be returned. + * + * @param reader the source to read from + * @param max the max number of lines to read, if this is -1 Integer.MAX_VALUE is used + * @returns a list of strings, possibly empty, or null if there was an exception. + * @throws NullPointerException if reader is null + */ + public static List<String> readLines(BufferedReader reader, int max) { + List<String> ret = new LinkedList<String>(); + String line; + int count = 0; + + try { + if (max > 0) { + while ((count++ < max) && (line = reader.readLine()) != null) { + ret.add(line); + } + } else { + while ((line = reader.readLine()) != null) { + ret.add(line); + } + } + } catch (IOException e) { + return null; + } + + return ret; + } + + /** * Check for null objects in a list of objects. */ private static void checkNull(Object... objs) { This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |