Menu
▾
▴
[Rubyeclipse-commit] SF.net SVN: rubyeclipse: [1732] trunk/org.rubypeople.rdt.core/src/org/ rubypeople/rdt/internal/core
|
From: <caw...@us...> - 2006-12-22 19:25:27
|
Revision: 1732
http://svn.sourceforge.net/rubyeclipse/?rev=1732&view=rev
Author: cawilliams
Date: 2006-12-22 11:25:24 -0800 (Fri, 22 Dec 2006)
Log Message:
-----------
a quick hack to integrate code duplication checking from PMD into our build process. Eventually this will create warnings not spit out the offenders to the command line
Modified Paths:
--------------
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/AbstractRdtCompiler.java
Added Paths:
-----------
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/CPD.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/CPDListener.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/CPDNullListener.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/FileFinder.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/Language.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/Match.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/MatchAlgorithm.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/MatchCollector.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/PMD.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/RubyLanguage.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/RubyTokenizer.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/SourceCode.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/TokenEntry.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/Tokenizer.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/Tokens.java
Modified: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/AbstractRdtCompiler.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/AbstractRdtCompiler.java 2006-12-22 19:05:29 UTC (rev 1731)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/AbstractRdtCompiler.java 2006-12-22 19:25:24 UTC (rev 1732)
@@ -1,5 +1,6 @@
package org.rubypeople.rdt.internal.core.builder;
+import java.io.IOException;
import java.util.Iterator;
import java.util.List;
@@ -7,6 +8,10 @@
import org.eclipse.core.resources.IProject;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.IProgressMonitor;
+import org.rubypeople.rdt.internal.core.pmd.CPD;
+import org.rubypeople.rdt.internal.core.pmd.Match;
+import org.rubypeople.rdt.internal.core.pmd.PMD;
+import org.rubypeople.rdt.internal.core.pmd.TokenEntry;
import org.rubypeople.rdt.internal.core.symbols.SymbolIndex;
import org.rubypeople.rdt.internal.core.util.ListUtil;
@@ -46,10 +51,36 @@
monitor.worked(fileCount);
flushIndexEntries(symbolIndex);
monitor.worked(fileCount);
+ // FIXME Create warning markers for these duplicate code matches
+ // TODO Refactor out this stuff into a compiler, only visit files we've collected
+ try {
+ Iterator<Match> matches = CPD.findMatches(project);
+ StringBuffer buffer = new StringBuffer();
+ while (matches.hasNext()) {
+ Match match = matches.next();
+ renderOn(buffer, match);
+ }
+ System.out.println(buffer.toString());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
compileFiles(list, monitor);
monitor.done();
}
+
+ private void renderOn(StringBuffer rpt, Match match) {
+ rpt.append("Found a ").append(match.getLineCount()).append(" line (").append(match.getTokenCount()).append(" tokens) duplication in the following files: ").append(PMD.EOL);
+
+ TokenEntry mark;
+ for (Iterator occurrences = match.iterator(); occurrences.hasNext();) {
+ mark = (TokenEntry) occurrences.next();
+ rpt.append("Starting at line ").append(mark.getBeginLine()).append(" of ").append(mark.getTokenSrcID()).append(PMD.EOL);
+ }
+ rpt.append(PMD.EOL); // add a line to separate the source from the desc above
+ String source = match.getSourceCodeSlice();
+ rpt.append(source).append(PMD.EOL);
+ }
private void compileFiles(List list, IProgressMonitor monitor) throws CoreException {
for (Iterator iter = list.iterator(); iter.hasNext();) {
Added: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/CPD.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/CPD.java (rev 0)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/CPD.java 2006-12-22 19:25:24 UTC (rev 1732)
@@ -0,0 +1,101 @@
+package org.rubypeople.rdt.internal.core.pmd;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.eclipse.core.resources.IProject;
+
+public class CPD {
+
+ private Map<String, SourceCode> source = new HashMap<String, SourceCode>();
+ private int minimumTileSize;
+ private Language language;
+ private boolean skipDuplicates;
+ private MatchAlgorithm matchAlgorithm;
+ private Tokens tokens = new Tokens();
+ private CPDListener listener = new CPDNullListener();
+ private Set<String> current = new HashSet<String>();
+
+ private CPD(int minimumTileSize, Language language) {
+ this.minimumTileSize = minimumTileSize;
+ this.language = language;
+ }
+
+ public static Iterator<Match> findMatches(IProject project) throws IOException {
+ boolean skipDuplicateFiles = true;
+ int minimumTokens = 5;
+ Language language = new RubyLanguage();
+
+ CPD cpd = new CPD(minimumTokens, language);
+ if (skipDuplicateFiles) {
+ cpd.skipDuplicates();
+ }
+ cpd.addRecursively(project.getLocation().toOSString());
+ cpd.go();
+ return cpd.getMatches();
+ }
+
+ private void go() {
+ TokenEntry.clearImages();
+ matchAlgorithm = new MatchAlgorithm(source, tokens, minimumTileSize, listener);
+ matchAlgorithm.findMatches();
+ }
+
+ private void skipDuplicates() {
+ this.skipDuplicates = true;
+ }
+
+ private Iterator<Match> getMatches() {
+ return matchAlgorithm.matches();
+ }
+
+ private void addRecursively(String dir) throws IOException {
+ addDirectory(dir, true);
+ }
+
+ private void addDirectory(String dir, boolean recurse) throws IOException {
+ if (!(new File(dir)).exists()) {
+ throw new FileNotFoundException("Couldn't find directory " + dir);
+ }
+ FileFinder finder = new FileFinder();
+ // TODO - could use SourceFileSelector here
+ add(finder.findFilesFrom(dir, language.getFileFilter(), recurse));
+ }
+
+ private void add(List files) throws IOException {
+ for (Iterator i = files.iterator(); i.hasNext();) {
+ add(files.size(), (File) i.next());
+ }
+ }
+
+ private void add(int fileCount, File file) throws IOException {
+
+ if (skipDuplicates) {
+ // TODO refactor this thing into a separate class
+ String signature = file.getName() + '_' + file.length();
+ if (current.contains(signature)) {
+ System.out.println("Skipping " + file.getAbsolutePath() + " since it appears to be a duplicate file and --skip-duplicate-files is set");
+ return;
+ }
+ current.add(signature);
+ }
+
+ if (!file.getCanonicalPath().equals(file.getAbsolutePath())) {
+ System.out.println("Skipping " + file + " since it appears to be a symlink");
+ return;
+ }
+
+ listener.addedFile(fileCount, file);
+ SourceCode sourceCode = new SourceCode(new SourceCode.FileCodeLoader(file));
+ language.getTokenizer().tokenize(sourceCode, tokens);
+ source.put(sourceCode.getFileName(), sourceCode);
+ }
+
+}
Added: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/CPDListener.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/CPDListener.java (rev 0)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/CPDListener.java 2006-12-22 19:25:24 UTC (rev 1732)
@@ -0,0 +1,19 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+package org.rubypeople.rdt.internal.core.pmd;
+
+import java.io.File;
+
+public interface CPDListener {
+
+ public static final int INIT = 0;
+ public static final int HASH = 1;
+ public static final int MATCH = 2;
+ public static final int GROUPING = 3;
+ public static final int DONE = 4;
+
+ void addedFile(int fileCount, File file);
+
+ void phaseUpdate(int phase);
+}
Added: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/CPDNullListener.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/CPDNullListener.java (rev 0)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/CPDNullListener.java 2006-12-22 19:25:24 UTC (rev 1732)
@@ -0,0 +1,14 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+package org.rubypeople.rdt.internal.core.pmd;
+
+import java.io.File;
+
+public class CPDNullListener implements CPDListener {
+ public void addedFile(int fileCount, File file) {
+ }
+
+ public void phaseUpdate(int phase) {
+ }
+}
Added: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/FileFinder.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/FileFinder.java (rev 0)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/FileFinder.java 2006-12-22 19:25:24 UTC (rev 1732)
@@ -0,0 +1,42 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+package org.rubypeople.rdt.internal.core.pmd;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.util.ArrayList;
+import java.util.List;
+
+public class FileFinder {
+
+ private FilenameFilter filter;
+ private static final String FILE_SEP = System.getProperty("file.separator");
+
+ public List findFilesFrom(String dir, FilenameFilter filter, boolean recurse) {
+ this.filter = filter;
+ List files = new ArrayList();
+ scanDirectory(new File(dir), files, recurse);
+ return files;
+ }
+
+ /**
+ * Implements a tail recursive file scanner
+ */
+ private void scanDirectory(File dir, List list, boolean recurse) {
+ String[] candidates = dir.list(filter);
+ if (candidates == null) {
+ return;
+ }
+ for (int i = 0; i < candidates.length; i++) {
+ File tmp = new File(dir + FILE_SEP + candidates[i]);
+ if (tmp.isDirectory()) {
+ if (recurse) {
+ scanDirectory(tmp, list, true);
+ }
+ } else {
+ list.add(new File(dir + FILE_SEP + candidates[i]));
+ }
+ }
+ }
+}
Added: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/Language.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/Language.java (rev 0)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/Language.java 2006-12-22 19:25:24 UTC (rev 1732)
@@ -0,0 +1,11 @@
+package org.rubypeople.rdt.internal.core.pmd;
+
+import java.io.FilenameFilter;
+
+public interface Language {
+ String fileSeparator = System.getProperty("file.separator");
+
+ public Tokenizer getTokenizer();
+
+ public FilenameFilter getFileFilter();
+}
Added: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/Match.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/Match.java (rev 0)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/Match.java 2006-12-22 19:25:24 UTC (rev 1732)
@@ -0,0 +1,172 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+package org.rubypeople.rdt.internal.core.pmd;
+
+
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+public class Match implements Comparable {
+
+ private int tokenCount;
+ private int lineCount;
+ private Set<TokenEntry> markSet = new TreeSet<TokenEntry>();
+ private TokenEntry[] marks = new TokenEntry[2];
+ private String code;
+ private MatchCode mc;
+ private String label;
+
+ public static final Comparator MatchesComparator = new Comparator() {
+ public int compare(Object a, Object b) {
+ Match ma = (Match)a;
+ Match mb = (Match)b;
+ return mb.getMarkCount() - ma.getMarkCount();
+ }
+ };
+
+ public static final Comparator LinesComparator = new Comparator() {
+ public int compare(Object a, Object b) {
+ Match ma = (Match)a;
+ Match mb = (Match)b;
+
+ return mb.getLineCount() - ma.getLineCount();
+ }
+ };
+
+ public static final Comparator LabelComparator = new Comparator() {
+ public int compare(Object a, Object b) {
+ Match ma = (Match)a;
+ Match mb = (Match)b;
+ if (ma.getLabel() == null) return 1;
+ if (mb.getLabel() == null) return -1;
+ return mb.getLabel().compareTo(ma.getLabel());
+ }
+ };
+
+ public static final Comparator LengthComparator = new Comparator() {
+ public int compare(Object o1, Object o2) {
+ Match m1 = (Match) o1;
+ Match m2 = (Match) o2;
+ return m2.getLineCount() - m1.getLineCount();
+ }
+ };
+
+ public static class MatchCode {
+
+ private int first;
+ private int second;
+
+ public MatchCode() {
+ }
+
+ public MatchCode(TokenEntry m1, TokenEntry m2) {
+ first = m1.getIndex();
+ second = m2.getIndex();
+ }
+
+ public int hashCode() {
+ return first + 37 * second;
+ }
+
+ public boolean equals(Object other) {
+ MatchCode mc = (MatchCode) other;
+ return mc.first == first && mc.second == second;
+ }
+
+ public void setFirst(int first) {
+ this.first = first;
+ }
+
+ public void setSecond(int second) {
+ this.second = second;
+ }
+
+ }
+
+ public Match(int tokenCount, TokenEntry first, TokenEntry second) {
+ markSet.add(first);
+ markSet.add(second);
+ marks[0] = first;
+ marks[1] = second;
+ this.tokenCount = tokenCount;
+ }
+
+ public int getMarkCount() {
+ return markSet.size();
+ }
+
+ public void setLineCount(int lineCount) {
+ this.lineCount = lineCount;
+ }
+
+ public int getLineCount() {
+ return this.lineCount;
+ }
+
+ public int getTokenCount() {
+ return this.tokenCount;
+ }
+
+ public String getSourceCodeSlice() {
+ return this.code;
+ }
+
+ public void setSourceCodeSlice(String code) {
+ this.code = code;
+ }
+
+ public Iterator<TokenEntry> iterator() {
+ return markSet.iterator();
+ }
+
+ public int compareTo(Object o) {
+ Match other = (Match) o;
+ int diff = other.getTokenCount() - getTokenCount();
+ if (diff != 0) {
+ return diff;
+ }
+ return other.getFirstMark().getIndex() - getFirstMark().getIndex();
+ }
+
+ public TokenEntry getFirstMark() {
+ return marks[0];
+ }
+
+ public TokenEntry getSecondMark() {
+ return marks[1];
+ }
+
+ public String toString() {
+ return "Match: " + PMD.EOL + "tokenCount = " + tokenCount + PMD.EOL + "marks = " + markSet.size();
+ }
+
+ public Set<TokenEntry> getMarkSet() {
+ return markSet;
+ }
+
+ public MatchCode getMatchCode() {
+ if (mc == null) {
+ mc = new MatchCode(marks[0], marks[1]);
+ }
+ return mc;
+ }
+
+ public int getEndIndex() {
+ return marks[1].getIndex() + getTokenCount() - 1;
+ }
+
+ public void setMarkSet(Set<TokenEntry> markSet) {
+ this.markSet = markSet;
+ }
+
+ public void setLabel(String aLabel) {
+ label = aLabel;
+ }
+
+ public String getLabel() {
+ return label;
+ }
+}
\ No newline at end of file
Added: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/MatchAlgorithm.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/MatchAlgorithm.java (rev 0)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/MatchAlgorithm.java 2006-12-22 19:25:24 UTC (rev 1732)
@@ -0,0 +1,126 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+package org.rubypeople.rdt.internal.core.pmd;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+public class MatchAlgorithm {
+
+ private final static int MOD = 37;
+ private int lastHash;
+ private int lastMod = 1;
+
+ private List<Match> matches;
+ private Map source;
+ private Tokens tokens;
+ private List code;
+ private CPDListener cpdListener;
+ private int min;
+
+ public MatchAlgorithm(Map sourceCode, Tokens tokens, int min) {
+ this(sourceCode, tokens, min, new CPDNullListener());
+ }
+
+ public MatchAlgorithm(Map sourceCode, Tokens tokens, int min, CPDListener listener) {
+ this.source = sourceCode;
+ this.tokens = tokens;
+ this.code = tokens.getTokens();
+ this.min = min;
+ this.cpdListener = listener;
+ for (int i = 0; i < min; i++) {
+ lastMod *= MOD;
+ }
+ }
+
+ public void setListener(CPDListener listener) {
+ this.cpdListener = listener;
+ }
+
+ public Iterator<Match> matches() {
+ return matches.iterator();
+ }
+
+ public TokenEntry tokenAt(int offset, TokenEntry m) {
+ return (TokenEntry) code.get(offset + m.getIndex());
+ }
+
+ public int getMinimumTileSize() {
+ return this.min;
+ }
+
+ public void findMatches() {
+ cpdListener.phaseUpdate(CPDListener.HASH);
+ Map markGroups = hash();
+
+ cpdListener.phaseUpdate(CPDListener.MATCH);
+ MatchCollector matchCollector = new MatchCollector(this);
+ for (Iterator i = markGroups.values().iterator(); i.hasNext();) {
+ Object o = i.next();
+ if (o instanceof List) {
+ Collections.reverse((List) o);
+ matchCollector.collect((List) o);
+ }
+ i.remove();
+ }
+ cpdListener.phaseUpdate(CPDListener.GROUPING);
+ matches = matchCollector.getMatches();
+ matchCollector = null;
+ for (Iterator<Match> i = matches.iterator(); i.hasNext();) {
+ Match match = (Match) i.next();
+ for (Iterator<TokenEntry> occurrences = match.iterator(); occurrences.hasNext();) {
+ TokenEntry mark = (TokenEntry) occurrences.next();
+ match.setLineCount(tokens.getLineCount(mark, match));
+ if (!occurrences.hasNext()) {
+ int start = mark.getBeginLine();
+ int end = start + match.getLineCount() - 1;
+ SourceCode sourceCode = (SourceCode) source.get(mark.getTokenSrcID());
+ match.setSourceCodeSlice(sourceCode.getSlice(start, end));
+ }
+ }
+ }
+ cpdListener.phaseUpdate(CPDListener.DONE);
+ }
+
+ private Map hash() {
+ Map markGroups = new HashMap(tokens.size());
+ for (int i = code.size() - 1; i >= 0; i--) {
+ TokenEntry token = (TokenEntry) code.get(i);
+ if (token != TokenEntry.EOF) {
+ int last = tokenAt(min, token).getIdentifier();
+ lastHash = MOD * lastHash + token.getIdentifier() - lastMod * last;
+ token.setHashCode(lastHash);
+ Object o = markGroups.get(token);
+
+ // Note that this insertion method is worthwhile since the vast majority
+ // markGroup keys will have only one value.
+ if (o == null) {
+ markGroups.put(token, token);
+ } else if (o instanceof TokenEntry) {
+ List l = new ArrayList();
+ l.add(o);
+ l.add(token);
+ markGroups.put(token, l);
+ } else {
+ List l = (List) o;
+ l.add(token);
+ }
+ } else {
+ lastHash = 0;
+ for (int end = Math.max(0, i - min + 1); i > end; i--) {
+ token = (TokenEntry) code.get(i - 1);
+ lastHash = MOD * lastHash + token.getIdentifier();
+ if (token == TokenEntry.EOF) {
+ break;
+ }
+ }
+ }
+ }
+ return markGroups;
+ }
+}
Added: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/MatchCollector.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/MatchCollector.java (rev 0)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/MatchCollector.java 2006-12-22 19:25:24 UTC (rev 1732)
@@ -0,0 +1,165 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+package org.rubypeople.rdt.internal.core.pmd;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class MatchCollector {
+
+ private MatchAlgorithm ma;
+ private Map<Match.MatchCode, Match> startMap = new HashMap<Match.MatchCode, Match>();
+ private Map fileMap = new HashMap();
+
+ public MatchCollector(MatchAlgorithm ma) {
+ this.ma = ma;
+ }
+
+ public void collect(List marks) {
+ //first get a pairwise collection of all maximal matches
+ for (int i = 0; i < marks.size() - 1; i++) {
+ TokenEntry mark1 = (TokenEntry) marks.get(i);
+ for (int j = i + 1; j < marks.size(); j++) {
+ TokenEntry mark2 = (TokenEntry) marks.get(j);
+ int diff = mark1.getIndex() - mark2.getIndex();
+ if (-diff < ma.getMinimumTileSize()) {
+ continue;
+ }
+ if (hasPreviousDupe(mark1, mark2)) {
+ continue;
+ }
+
+ // "match too small" check
+ int dupes = countDuplicateTokens(mark1, mark2);
+ if (dupes < ma.getMinimumTileSize()) {
+ continue;
+ }
+ // is it still too close together
+ if (diff + dupes >= 1) {
+ continue;
+ }
+ determineMatch(mark1, mark2, dupes);
+ }
+ }
+ }
+
+ public List<Match> getMatches() {
+ List<Match> matchList = new ArrayList<Match>(startMap.values());
+ Collections.sort(matchList);
+ Set<Match.MatchCode> matchSet = new HashSet<Match.MatchCode>();
+ Match.MatchCode matchCode = new Match.MatchCode();
+ for (int i = matchList.size(); i > 1; i--) {
+ Match match1 = (Match) matchList.get(i - 1);
+ TokenEntry mark1 = (TokenEntry) match1.getMarkSet().iterator().next();
+ matchSet.clear();
+ matchSet.add(match1.getMatchCode());
+ for (int j = i - 1; j > 0; j--) {
+ Match match2 = (Match) matchList.get(j - 1);
+ if (match1.getTokenCount() != match2.getTokenCount()) {
+ break;
+ }
+ TokenEntry mark2 = null;
+ for (Iterator iter = match2.getMarkSet().iterator(); iter.hasNext();) {
+ mark2 = (TokenEntry) iter.next();
+ if (mark2 != mark1) {
+ break;
+ }
+ }
+ int dupes = countDuplicateTokens(mark1, mark2);
+ if (dupes < match1.getTokenCount()) {
+ break;
+ }
+ matchSet.add(match2.getMatchCode());
+ match1.getMarkSet().addAll(match2.getMarkSet());
+ matchList.remove(i - 2);
+ i--;
+ }
+ if (matchSet.size() == 1) {
+ continue;
+ }
+ //prune the mark set
+ Set pruned = match1.getMarkSet();
+ boolean done = false;
+ ArrayList a1 = new ArrayList(match1.getMarkSet());
+ Collections.sort(a1);
+ for (int outer = 0; outer < a1.size() - 1 && !done; outer++) {
+ TokenEntry cmark1 = (TokenEntry) a1.get(outer);
+ for (int inner = outer + 1; inner < a1.size() && !done; inner++) {
+ TokenEntry cmark2 = (TokenEntry) a1.get(inner);
+ matchCode.setFirst(cmark1.getIndex());
+ matchCode.setSecond(cmark2.getIndex());
+ if (!matchSet.contains(matchCode)) {
+ if (pruned.size() > 2) {
+ pruned.remove(cmark2);
+ }
+ if (pruned.size() == 2) {
+ done = true;
+ }
+ }
+ }
+ }
+ }
+ return matchList;
+ }
+
+ /**
+ * A greedy algorithm for determining non-overlapping matches
+ */
+ private void determineMatch(TokenEntry mark1, TokenEntry mark2, int dupes) {
+ Match match = new Match(dupes, mark1, mark2);
+ String fileKey = mark1.getTokenSrcID() + mark2.getTokenSrcID();
+ List pairMatches = (ArrayList) fileMap.get(fileKey);
+ if (pairMatches == null) {
+ pairMatches = new ArrayList();
+ fileMap.put(fileKey, pairMatches);
+ }
+ boolean add = true;
+ for (int i = 0; i < pairMatches.size(); i++) {
+ Match other = (Match) pairMatches.get(i);
+ if (other.getFirstMark().getIndex() + other.getTokenCount() - mark1.getIndex()
+ > 0) {
+ boolean ordered = other.getSecondMark().getIndex() - mark2.getIndex() < 0;
+ if ((ordered && (other.getEndIndex() - mark2.getIndex() > 0))
+ || (!ordered && (match.getEndIndex() - other.getSecondMark().getIndex()) > 0)) {
+ if (other.getTokenCount() >= match.getTokenCount()) {
+ add = false;
+ break;
+ } else {
+ pairMatches.remove(i);
+ startMap.remove(other.getMatchCode());
+ }
+ }
+ }
+ }
+ if (add) {
+ pairMatches.add(match);
+ startMap.put(match.getMatchCode(), match);
+ }
+ }
+
+ private boolean hasPreviousDupe(TokenEntry mark1, TokenEntry mark2) {
+ if (mark1.getIndex() == 0) {
+ return false;
+ }
+ return !matchEnded(ma.tokenAt(-1, mark1), ma.tokenAt(-1, mark2));
+ }
+
+ private int countDuplicateTokens(TokenEntry mark1, TokenEntry mark2) {
+ int index = 0;
+ while (!matchEnded(ma.tokenAt(index, mark1), ma.tokenAt(index, mark2))) {
+ index++;
+ }
+ return index;
+ }
+
+ private boolean matchEnded(TokenEntry token1, TokenEntry token2) {
+ return token1.getIdentifier() != token2.getIdentifier() || token1 == TokenEntry.EOF || token2 == TokenEntry.EOF;
+ }
+}
\ No newline at end of file
Added: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/PMD.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/PMD.java (rev 0)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/PMD.java 2006-12-22 19:25:24 UTC (rev 1732)
@@ -0,0 +1,5 @@
+package org.rubypeople.rdt.internal.core.pmd;
+
+public interface PMD {
+ public static final String EOL = System.getProperty("line.separator", "\n");
+}
Added: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/RubyLanguage.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/RubyLanguage.java (rev 0)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/RubyLanguage.java 2006-12-22 19:25:24 UTC (rev 1732)
@@ -0,0 +1,23 @@
+package org.rubypeople.rdt.internal.core.pmd;
+
+import java.io.File;
+import java.io.FilenameFilter;
+
+public class RubyLanguage implements Language {
+
+ public static class RubyFileOrDirectoryFilter implements FilenameFilter {
+ public boolean accept(File dir, String filename) {
+ return filename.endsWith("rb") || filename.endsWith("cgi") ||
+ filename.endsWith("class") ||
+ (new File(dir.getAbsolutePath() + fileSeparator + filename).isDirectory());
+ }
+ }
+
+ public Tokenizer getTokenizer() {
+ return new RubyTokenizer();
+ }
+
+ public FilenameFilter getFileFilter() {
+ return new RubyFileOrDirectoryFilter();
+ }
+}
\ No newline at end of file
Added: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/RubyTokenizer.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/RubyTokenizer.java (rev 0)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/RubyTokenizer.java 2006-12-22 19:25:24 UTC (rev 1732)
@@ -0,0 +1,139 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ * @authors: Zev Blut zb...@ub...
+ */
+package org.rubypeople.rdt.internal.core.pmd;
+
+import java.util.List;
+
+public class RubyTokenizer implements Tokenizer {
+ private boolean downcaseString = true;
+
+ public void tokenize(SourceCode tokens, Tokens tokenEntries) {
+ List code = tokens.getCode();
+ for (int i = 0; i < code.size(); i++) {
+ String currentLine = (String) code.get(i);
+ int loc = 0;
+ while (loc < currentLine.length()) {
+ StringBuffer token = new StringBuffer();
+ loc = getTokenFromLine(currentLine, token, loc);
+ if (token.length() > 0 && !isIgnorableString(token.toString())) {
+ if (downcaseString) {
+ token = new StringBuffer(token.toString().toLowerCase());
+ }
+ tokenEntries.add(new TokenEntry(token.toString(),
+ tokens.getFileName(),
+ i + 1));
+ }
+ }
+ }
+ tokenEntries.add(TokenEntry.getEOF());
+ }
+
+ private int getTokenFromLine(String line, StringBuffer token, int loc) {
+ for (int j = loc; j < line.length(); j++) {
+ char tok = line.charAt(j);
+ if (!Character.isWhitespace(tok) && !ignoreCharacter(tok)) {
+ if (isComment(tok)) {
+ if (token.length() > 0) {
+ return j;
+ } else {
+ return getCommentToken(line, token, loc);
+ }
+ } else if (isString(tok)) {
+ if (token.length() > 0) {
+ //if (loc == lin
+ return j; // we need to now parse the string as a seperate token.
+ } else {
+ // we are at the start of a string
+ return parseString(line, token, j, tok);
+ }
+ } else {
+ token.append(tok);
+ }
+ } else {
+ if (token.length() > 0) {
+ return j;
+ }
+ }
+ loc = j;
+ }
+ return loc + 1;
+ }
+
+ private int parseString(String line, StringBuffer token, int loc, char stringType) {
+ boolean escaped = false;
+ boolean done = false;
+ //System.out.println("Parsing String:" + stringType);
+ //System.out.println("Starting loc:" + loc);
+ // problem of strings that span multiple lines :-(
+ char tok = ' '; // this will be replaced.
+ while ((loc < line.length()) && !done) {
+ tok = line.charAt(loc);
+ if (escaped && tok == stringType) {
+ // System.out.println("Found an escaped string");
+ escaped = false;
+ } else if (tok == stringType && (token.length() > 0)) {
+ // we are done
+ // System.out.println("Found an end string");
+ done = true;
+ } else if (tok == '\\') {
+ // System.out.println("Found an escaped char");
+ escaped = true;
+ } else {
+ // System.out.println("Adding char:" + tok + ";loc:" + loc);
+ escaped = false;
+ }
+ //System.out.println("Adding char to String:" + token.toString());
+ token.append(tok);
+ loc++;
+ }
+ return loc + 1;
+ }
+
+ private boolean ignoreCharacter(char tok) {
+ boolean result = false;
+ switch (tok) {
+ case '{':
+ case '}':
+ case '(':
+ case ')':
+ case ';':
+ case ',':
+ result = true;
+ break;
+ default :
+ result = false;
+ }
+ return result;
+ }
+
+ private boolean isString(char tok) {
+ boolean result = false;
+ switch (tok) {
+ case '\'':
+ case '"':
+ result = true;
+ break;
+ default:
+ result = false;
+ }
+ return result;
+ }
+
+ private boolean isComment(char tok) {
+ return tok == '#';
+ }
+
+ private int getCommentToken(String line, StringBuffer token, int loc) {
+ while (loc < line.length()) {
+ token.append(line.charAt(loc));
+ loc++;
+ }
+ return loc;
+ }
+
+ private boolean isIgnorableString(String token) {
+ return "do".equals(token) || "end".equals(token);
+ }
+}
Added: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/SourceCode.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/SourceCode.java (rev 0)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/SourceCode.java 2006-12-22 19:25:24 UTC (rev 1732)
@@ -0,0 +1,135 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+package org.rubypeople.rdt.internal.core.pmd;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.LineNumberReader;
+import java.io.Reader;
+import java.io.StringReader;
+import java.lang.ref.SoftReference;
+import java.util.ArrayList;
+import java.util.List;
+
+public class SourceCode {
+
+ public static abstract class CodeLoader {
+ private SoftReference code;
+
+ public List getCode() {
+ List c = null;
+ if (code != null) {
+ c = (List) code.get();
+ }
+ if (c != null) {
+ return c;
+ }
+ this.code = new SoftReference(load());
+ return (List) code.get();
+ }
+
+ public abstract String getFileName();
+
+ protected abstract Reader getReader() throws Exception;
+
+ protected List load() {
+ LineNumberReader lnr = null;
+ try {
+ lnr = new LineNumberReader(getReader());
+ List lines = new ArrayList();
+ String currentLine;
+ while ((currentLine = lnr.readLine()) != null) {
+ lines.add(currentLine);
+ }
+ return lines;
+ } catch (Exception e) {
+ throw new RuntimeException("Problem while reading " + getFileName() + ":" + e.getMessage());
+ } finally {
+ try {
+ if (lnr != null)
+ lnr.close();
+ } catch (Exception e) {
+ throw new RuntimeException("Problem while reading " + getFileName() + ":" + e.getMessage());
+ }
+ }
+ }
+ }
+
+ public static class FileCodeLoader extends CodeLoader {
+ private File file;
+
+ public FileCodeLoader(File file) {
+ this.file = file;
+ }
+
+ public Reader getReader() throws Exception {
+ return new FileReader(file);
+ }
+
+ public String getFileName() {
+ return this.file.getAbsolutePath();
+ }
+ }
+
+ public static class StringCodeLoader extends CodeLoader {
+ public static final String DEFAULT_NAME = "CODE_LOADED_FROM_STRING";
+
+ private String source_code;
+
+ private String name;
+
+ public StringCodeLoader(String code) {
+ this(code, DEFAULT_NAME);
+ }
+
+ public StringCodeLoader(String code, String name) {
+ this.source_code = code;
+ this.name = name;
+ }
+
+ public Reader getReader() {
+ return new StringReader(source_code);
+ }
+
+ public String getFileName() {
+ return name;
+ }
+ }
+
+ private CodeLoader cl;
+
+ public SourceCode(CodeLoader cl) {
+ this.cl = cl;
+ }
+
+ public List getCode() {
+ return cl.getCode();
+ }
+
+ public StringBuffer getCodeBuffer() {
+ StringBuffer sb = new StringBuffer();
+ List lines = cl.getCode();
+ for (int i = 0; i < lines.size(); i++) {
+ sb.append((String) lines.get(i));
+ sb.append(PMD.EOL);
+ }
+ return sb;
+ }
+
+ public String getSlice(int startLine, int endLine) {
+ StringBuffer sb = new StringBuffer();
+ List lines = cl.getCode();
+ for (int i = startLine - 1; i < endLine && i < lines.size(); i++) {
+ if (sb.length() != 0) {
+ sb.append(PMD.EOL);
+ }
+ sb.append((String) lines.get(i));
+ }
+ return sb.toString();
+ }
+
+ public String getFileName() {
+ return cl.getFileName();
+ }
+}
\ No newline at end of file
Added: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/TokenEntry.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/TokenEntry.java (rev 0)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/TokenEntry.java 2006-12-22 19:25:24 UTC (rev 1732)
@@ -0,0 +1,85 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+package org.rubypeople.rdt.internal.core.pmd;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class TokenEntry implements Comparable {
+
+ public static final TokenEntry EOF = new TokenEntry();
+
+ private String tokenSrcID;
+ private int beginLine;
+ private int index;
+ private int identifier;
+ private int hashCode;
+
+ private final static Map Tokens = new HashMap();
+ private static int TokenCount = 0;
+
+ private TokenEntry() {
+ this.identifier = 0;
+ this.tokenSrcID = "EOFMarker";
+ }
+
+ public TokenEntry(String image, String tokenSrcID, int beginLine) {
+ Integer i = (Integer) Tokens.get(image);
+ if (i == null) {
+ i = new Integer(Tokens.size() + 1);
+ Tokens.put(image, i);
+ }
+ this.identifier = i.intValue();
+ this.tokenSrcID = tokenSrcID;
+ this.beginLine = beginLine;
+ this.index = TokenCount++;
+ }
+
+ public static TokenEntry getEOF() {
+ TokenCount++;
+ return EOF;
+ }
+
+ public static void clearImages() {
+ Tokens.clear();
+ TokenCount = 0;
+ }
+
+ public String getTokenSrcID() {
+ return tokenSrcID;
+ }
+
+ public int getBeginLine() {
+ return beginLine;
+ }
+
+ public int getIdentifier() {
+ return this.identifier;
+ }
+
+ public int getIndex() {
+ return this.index;
+ }
+
+ public int hashCode() {
+ return hashCode;
+ }
+
+ public void setHashCode(int hashCode) {
+ this.hashCode = hashCode;
+ }
+
+ public boolean equals(Object o) {
+ if (!(o instanceof TokenEntry)) {
+ return false;
+ }
+ TokenEntry other = (TokenEntry) o;
+ return other.hashCode == hashCode;
+ }
+
+ public int compareTo(Object o) {
+ TokenEntry other = (TokenEntry) o;
+ return getIndex() - other.getIndex();
+ }
+}
\ No newline at end of file
Added: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/Tokenizer.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/Tokenizer.java (rev 0)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/Tokenizer.java 2006-12-22 19:25:24 UTC (rev 1732)
@@ -0,0 +1,7 @@
+package org.rubypeople.rdt.internal.core.pmd;
+
+import java.io.IOException;
+
+public interface Tokenizer {
+ void tokenize(SourceCode tokens, Tokens tokenEntries) throws IOException;
+}
Added: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/Tokens.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/Tokens.java (rev 0)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/Tokens.java 2006-12-22 19:25:24 UTC (rev 1732)
@@ -0,0 +1,42 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+package org.rubypeople.rdt.internal.core.pmd;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+public class Tokens {
+
+ private List tokens = new ArrayList();
+
+ public void add(TokenEntry tokenEntry) {
+ this.tokens.add(tokenEntry);
+ }
+
+ public Iterator iterator() {
+ return tokens.iterator();
+ }
+
+ private TokenEntry get(int index) {
+ return (TokenEntry) tokens.get(index);
+ }
+
+ public int size() {
+ return tokens.size();
+ }
+
+ public int getLineCount(TokenEntry mark, Match match) {
+ TokenEntry endTok = get(mark.getIndex() + match.getTokenCount() - 1);
+ if (endTok == TokenEntry.EOF) {
+ endTok = get(mark.getIndex() + match.getTokenCount() - 2);
+ }
+ return endTok.getBeginLine() - mark.getBeginLine() + 1;
+ }
+
+ public List getTokens() {
+ return tokens;
+ }
+
+}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
