Menu
▾
▴
[Rubyeclipse-commit] SF.net SVN: rubyeclipse: [1743] trunk/org.rubypeople.rdt.core/src/org/ rubypeople/rdt/internal/core
|
From: <caw...@us...> - 2006-12-29 22:33:27
|
Revision: 1743
http://svn.sourceforge.net/rubyeclipse/?rev=1743&view=rev
Author: cawilliams
Date: 2006-12-29 14:33:18 -0800 (Fri, 29 Dec 2006)
Log Message:
-----------
make the duplicate code detection create markers, not spit out dupes to console. Still some ugliness to work out (like the offets are wrong, and that it should become a compiler that's only run on clean builds).
Modified Paths:
--------------
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/AbstractRdtCompiler.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/CleanRdtCompiler.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/IFileProvider.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/ProjectFileFinder.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/RubySourceFileCollectingVisitor.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/CPD.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/RubyTokenizer.java
trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/TokenEntry.java
Modified: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/AbstractRdtCompiler.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/AbstractRdtCompiler.java 2006-12-29 15:06:29 UTC (rev 1742)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/AbstractRdtCompiler.java 2006-12-29 22:33:18 UTC (rev 1743)
@@ -7,7 +7,10 @@
import org.eclipse.core.resources.IFile;
import org.eclipse.core.resources.IProject;
import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IPath;
import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.Path;
+import org.rubypeople.rdt.core.RubyCore;
import org.rubypeople.rdt.internal.core.pmd.CPD;
import org.rubypeople.rdt.internal.core.pmd.Match;
import org.rubypeople.rdt.internal.core.pmd.PMD;
@@ -32,7 +35,7 @@
protected abstract void removeMarkers(IMarkerManager markerManager);
protected abstract void flushIndexEntries(SymbolIndex symbolIndex);
- protected abstract List getFilesToCompile();
+ protected abstract List<IFile> getFilesToCompile();
protected abstract void analyzeFiles() throws CoreException;
protected static List compilers(MarkerManager markerManager) {
@@ -42,8 +45,8 @@
public void compile(IProgressMonitor monitor) throws CoreException {
analyzeFiles();
- List list = getFilesToCompile();
- int fileCount = list.size();
+ List<IFile> files = getFilesToCompile();
+ int fileCount = files.size();
monitor.beginTask("Building "+project.getName() + "...", fileCount * (compilers.size() + 3));
monitor.subTask("Removing Markers...");
@@ -52,37 +55,33 @@
monitor.subTask("Removing Search Indices...");
flushIndexEntries(symbolIndex);
monitor.worked(fileCount);
- // FIXME Create warning markers for these duplicate code matches
+
// TODO Refactor out this stuff into a compiler, only visit files we've collected
monitor.subTask("Finding duplicate code...");
try {
- Iterator<Match> matches = CPD.findMatches(project);
- StringBuffer buffer = new StringBuffer();
+ Iterator<Match> matches = CPD.findMatches(files);
while (matches.hasNext()) {
Match match = matches.next();
- renderOn(buffer, match);
+ addMarker(match);
}
- System.out.println(buffer.toString());
} catch (IOException e) {
e.printStackTrace();
}
monitor.worked(fileCount);
- compileFiles(list, monitor);
+ compileFiles(files, monitor);
monitor.done();
}
- private void renderOn(StringBuffer rpt, Match match) {
- rpt.append("Found a ").append(match.getLineCount()).append(" line (").append(match.getTokenCount()).append(" tokens) duplication in the following files: ").append(PMD.EOL);
-
- TokenEntry mark;
+ private void addMarker(Match match) {
+ StringBuffer message = new StringBuffer("Found a ");
+ message.append(match.getLineCount()).append(" line (").append(match.getTokenCount()).append(" tokens) duplication");
for (Iterator occurrences = match.iterator(); occurrences.hasNext();) {
- mark = (TokenEntry) occurrences.next();
- rpt.append("Starting at line ").append(mark.getBeginLine()).append(" of ").append(mark.getTokenSrcID()).append(PMD.EOL);
+ TokenEntry mark = (TokenEntry) occurrences.next();
+ // FIXME Make TokenEntry hold an IFile pointer to source file?
+ IFile file = RubyCore.getWorkspace().getRoot().getFileForLocation(Path.fromOSString(mark.getTokenSrcID()));
+ markerManager.addWarning(file, message.toString(), mark.getBeginLine(), mark.getStartOffset(), mark.getStartOffset() + match.getSourceCodeSlice().length());
}
- rpt.append(PMD.EOL); // add a line to separate the source from the desc above
- String source = match.getSourceCodeSlice();
- rpt.append(source).append(PMD.EOL);
}
private void compileFiles(List list, IProgressMonitor monitor) throws CoreException {
Modified: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/CleanRdtCompiler.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/CleanRdtCompiler.java 2006-12-29 15:06:29 UTC (rev 1742)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/CleanRdtCompiler.java 2006-12-29 22:33:18 UTC (rev 1743)
@@ -2,13 +2,14 @@
import java.util.List;
+import org.eclipse.core.resources.IFile;
import org.eclipse.core.resources.IProject;
import org.eclipse.core.runtime.CoreException;
import org.rubypeople.rdt.internal.core.symbols.SymbolIndex;
public class CleanRdtCompiler extends AbstractRdtCompiler {
- private List projectFiles;
+ private List<IFile> projectFiles;
public CleanRdtCompiler(IProject project, SymbolIndex symbolIndex) {
this(project, symbolIndex, new MarkerManager());
@@ -32,11 +33,11 @@
markerManager.removeProblemsAndTasksFor(project);
}
- protected List getFilesToClear() {
+ protected List<IFile> getFilesToClear() {
return projectFiles;
}
- protected List getFilesToCompile() {
+ protected List<IFile> getFilesToCompile() {
return projectFiles;
}
Modified: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/IFileProvider.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/IFileProvider.java 2006-12-29 15:06:29 UTC (rev 1742)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/IFileProvider.java 2006-12-29 22:33:18 UTC (rev 1743)
@@ -13,8 +13,9 @@
import java.util.List;
+import org.eclipse.core.resources.IFile;
import org.eclipse.core.runtime.CoreException;
interface IFileProvider {
- public List findFiles() throws CoreException;
+ public List<IFile> findFiles() throws CoreException;
}
\ No newline at end of file
Modified: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/ProjectFileFinder.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/ProjectFileFinder.java 2006-12-29 15:06:29 UTC (rev 1742)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/ProjectFileFinder.java 2006-12-29 22:33:18 UTC (rev 1743)
@@ -15,6 +15,7 @@
import java.util.ArrayList;
import java.util.List;
+import org.eclipse.core.resources.IFile;
import org.eclipse.core.resources.IProject;
import org.eclipse.core.resources.IResource;
import org.eclipse.core.runtime.CoreException;
@@ -26,13 +27,13 @@
this.project = project;
}
- public List findFiles() throws CoreException {
- List files = new ArrayList();
+ public List<IFile> findFiles() throws CoreException {
+ List<IFile> files = new ArrayList<IFile>();
addAllSourceFiles(files);
return files;
}
- protected void addAllSourceFiles(final List sourceFiles) throws CoreException {
+ protected void addAllSourceFiles(final List<IFile> sourceFiles) throws CoreException {
project.accept(new RubySourceFileCollectingVisitor(sourceFiles), IResource.NONE);
}
Modified: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/RubySourceFileCollectingVisitor.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/RubySourceFileCollectingVisitor.java 2006-12-29 15:06:29 UTC (rev 1742)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/builder/RubySourceFileCollectingVisitor.java 2006-12-29 22:33:18 UTC (rev 1743)
@@ -27,10 +27,10 @@
public final class RubySourceFileCollectingVisitor implements IResourceProxyVisitor {
private static final String RUBY_SOURCE_CONTENT_TYPE_ID = "org.rubypeople.rdt.core.rubySource";
- private final List files;
+ private final List<IFile> files;
private HashSet<String> visitedLinks;
- public RubySourceFileCollectingVisitor(List files) {
+ public RubySourceFileCollectingVisitor(List<IFile> files) {
this.files = files;
this.visitedLinks = new HashSet<String>();
}
@@ -41,7 +41,7 @@
case IResource.FILE:
if (org.rubypeople.rdt.internal.core.util.Util.isRubyLikeFileName(proxy.getName())) {
if (resource == null) resource = proxy.requestResource();
- files.add(resource);
+ files.add((IFile)resource);
return false;
}
// Check for Ruby Source content type
@@ -52,7 +52,7 @@
IContentType type = contentDescription.getContentType();
if (type != null)
if (type.getId().equals(RUBY_SOURCE_CONTENT_TYPE_ID))
- files.add(resource);
+ files.add(file);
}
return false;
case IResource.FOLDER:
Modified: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/CPD.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/CPD.java 2006-12-29 15:06:29 UTC (rev 1742)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/CPD.java 2006-12-29 22:33:18 UTC (rev 1743)
@@ -1,7 +1,6 @@
package org.rubypeople.rdt.internal.core.pmd;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
@@ -10,34 +9,26 @@
import java.util.Map;
import java.util.Set;
-import org.eclipse.core.resources.IProject;
+import org.eclipse.core.resources.IFile;
public class CPD {
private Map<String, SourceCode> source = new HashMap<String, SourceCode>();
private int minimumTileSize;
- private Language language;
- private boolean skipDuplicates;
+ private Language language = new RubyLanguage();
private MatchAlgorithm matchAlgorithm;
private Tokens tokens = new Tokens();
private CPDListener listener = new CPDNullListener();
private Set<String> current = new HashSet<String>();
- private CPD(int minimumTileSize, Language language) {
+ private CPD(int minimumTileSize) {
this.minimumTileSize = minimumTileSize;
- this.language = language;
}
- public static Iterator<Match> findMatches(IProject project) throws IOException {
- boolean skipDuplicateFiles = true;
- int minimumTokens = 5;
- Language language = new RubyLanguage();
-
- CPD cpd = new CPD(minimumTokens, language);
- if (skipDuplicateFiles) {
- cpd.skipDuplicates();
- }
- cpd.addRecursively(project.getLocation().toOSString());
+ public static Iterator<Match> findMatches(List<IFile> files) throws IOException {
+ int minimumTokens = 5; // TODO Make this configurable in a preference page
+ CPD cpd = new CPD(minimumTokens);
+ cpd.add(files);
cpd.go();
return cpd.getMatches();
}
@@ -47,53 +38,33 @@
matchAlgorithm = new MatchAlgorithm(source, tokens, minimumTileSize, listener);
matchAlgorithm.findMatches();
}
-
- private void skipDuplicates() {
- this.skipDuplicates = true;
- }
-
+
private Iterator<Match> getMatches() {
return matchAlgorithm.matches();
}
- private void addRecursively(String dir) throws IOException {
- addDirectory(dir, true);
+ private void add(List<IFile> files) throws IOException {
+ for (IFile file : files) {
+ add(files.size(), file);
+ }
}
- private void addDirectory(String dir, boolean recurse) throws IOException {
- if (!(new File(dir)).exists()) {
- throw new FileNotFoundException("Couldn't find directory " + dir);
+ private void add(int fileCount, IFile file) throws IOException {
+ File realFile = file.getLocation().toFile();
+ // TODO refactor this thing into a separate class
+ String signature = realFile.getName() + '_' + realFile.length();
+ if (current.contains(signature)) { // skip duplicates
+ return;
}
- FileFinder finder = new FileFinder();
- // TODO - could use SourceFileSelector here
- add(finder.findFilesFrom(dir, language.getFileFilter(), recurse));
- }
-
- private void add(List files) throws IOException {
- for (Iterator i = files.iterator(); i.hasNext();) {
- add(files.size(), (File) i.next());
- }
- }
-
- private void add(int fileCount, File file) throws IOException {
-
- if (skipDuplicates) {
- // TODO refactor this thing into a separate class
- String signature = file.getName() + '_' + file.length();
- if (current.contains(signature)) {
- System.out.println("Skipping " + file.getAbsolutePath() + " since it appears to be a duplicate file and --skip-duplicate-files is set");
- return;
- }
- current.add(signature);
- }
-
- if (!file.getCanonicalPath().equals(file.getAbsolutePath())) {
- System.out.println("Skipping " + file + " since it appears to be a symlink");
+ current.add(signature);
+
+ if (!realFile.getCanonicalPath().equals(realFile.getAbsolutePath())) { // skip symlinks
return;
}
- listener.addedFile(fileCount, file);
- SourceCode sourceCode = new SourceCode(new SourceCode.FileCodeLoader(file));
+ listener.addedFile(fileCount, realFile);
+// FIXME We need to compensate all our token offsets by the end-of-line characters!
+ SourceCode sourceCode = new SourceCode(new SourceCode.FileCodeLoader(realFile));
language.getTokenizer().tokenize(sourceCode, tokens);
source.put(sourceCode.getFileName(), sourceCode);
}
Modified: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/RubyTokenizer.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/RubyTokenizer.java 2006-12-29 15:06:29 UTC (rev 1742)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/RubyTokenizer.java 2006-12-29 22:33:18 UTC (rev 1743)
@@ -11,11 +11,14 @@
public void tokenize(SourceCode tokens, Tokens tokenEntries) {
List code = tokens.getCode();
+ int curLineOffset = 0;
for (int i = 0; i < code.size(); i++) {
String currentLine = (String) code.get(i);
int loc = 0;
+ int startOffset = 0;
while (loc < currentLine.length()) {
StringBuffer token = new StringBuffer();
+ startOffset = curLineOffset + loc;
loc = getTokenFromLine(currentLine, token, loc);
if (token.length() > 0 && !isIgnorableString(token.toString())) {
if (downcaseString) {
@@ -23,9 +26,10 @@
}
tokenEntries.add(new TokenEntry(token.toString(),
tokens.getFileName(),
- i + 1));
+ i + 1, startOffset, startOffset + token.length()));
}
}
+ curLineOffset += currentLine.length();
}
tokenEntries.add(TokenEntry.getEOF());
}
Modified: trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/TokenEntry.java
===================================================================
--- trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/TokenEntry.java 2006-12-29 15:06:29 UTC (rev 1742)
+++ trunk/org.rubypeople.rdt.core/src/org/rubypeople/rdt/internal/core/pmd/TokenEntry.java 2006-12-29 22:33:18 UTC (rev 1743)
@@ -15,6 +15,8 @@
private int index;
private int identifier;
private int hashCode;
+ private int startOffset;
+ private int endOffset;
private final static Map Tokens = new HashMap();
private static int TokenCount = 0;
@@ -24,7 +26,7 @@
this.tokenSrcID = "EOFMarker";
}
- public TokenEntry(String image, String tokenSrcID, int beginLine) {
+ public TokenEntry(String image, String tokenSrcID, int beginLine, int startOffset, int endOffset) {
Integer i = (Integer) Tokens.get(image);
if (i == null) {
i = new Integer(Tokens.size() + 1);
@@ -33,6 +35,8 @@
this.identifier = i.intValue();
this.tokenSrcID = tokenSrcID;
this.beginLine = beginLine;
+ this.startOffset = startOffset;
+ this.endOffset = endOffset;
this.index = TokenCount++;
}
@@ -46,6 +50,14 @@
TokenCount = 0;
}
+ public int getStartOffset() {
+ return startOffset;
+ }
+
+ public int getEndOffset() {
+ return endOffset;
+ }
+
public String getTokenSrcID() {
return tokenSrcID;
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
