archive-access-cvs

[Archive-access-cvs] SF.net SVN: archive-access:[2457] trunk/archive-access/projects/nutchwax/ imagesearch/src/java/org/archive/nutchwax/imagesearch/ImageSearcherBean. java

[<mi...@us...>]

Revision: 2457
          http://archive-access.svn.sourceforge.net/archive-access/?rev=2457&view=rev
Author:   miklosh
Date:     2008-07-17 10:18:46 +0000 (Thu, 17 Jul 2008)

Log Message:
-----------
Switched to using multiple valued Lucene fields.

Modified Paths:
--------------
    trunk/archive-access/projects/nutchwax/imagesearch/src/java/org/archive/nutchwax/imagesearch/ImageSearcherBean.java

Modified: trunk/archive-access/projects/nutchwax/imagesearch/src/java/org/archive/nutchwax/imagesearch/ImageSearcherBean.java
===================================================================
--- trunk/archive-access/projects/nutchwax/imagesearch/src/java/org/archive/nutchwax/imagesearch/ImageSearcherBean.java	2008-07-17 10:16:27 UTC (rev 2456)
+++ trunk/archive-access/projects/nutchwax/imagesearch/src/java/org/archive/nutchwax/imagesearch/ImageSearcherBean.java	2008-07-17 10:18:46 UTC (rev 2457)
@@ -172,7 +172,7 @@
         float docSim = 0.0f;
         float maxDist = Float.MAX_VALUE;
         float minScore = 0.0f;
-        
+
         long totalHits = 0;
 
         boolean more = spans.next();
@@ -195,8 +195,7 @@
                 docBoost = doc.getBoost();
 
                 // Get image positions
-                String posField = doc.getField(ImageSearch.IMAGE_POS_KEY).stringValue();
-                String[] positions = posField.split(":");
+                String[] positions = doc.getValues(ImageSearch.IMAGE_POS_KEY);
                 imagePositions = new int[positions.length];
                 numDocImages = positions.length;
                 for (int i = 0; i < numDocImages; i++) {
@@ -205,12 +204,10 @@
                 maxDist = (float)imagePositions[numDocImages-1];
 
                 // Get image ids
-                String idField = doc.getField(ImageSearch.IMAGE_IDS_KEY).stringValue();
-                imageIds = idField.split(":");
+                imageIds = doc.getValues(ImageSearch.IMAGE_IDS_KEY);
 
                 // Get image urls
-                String urlField = doc.getField(ImageSearch.IMAGE_URLS_KEY).stringValue();
-                imageUrls = urlField.split(" ");
+                imageUrls = doc.getValues(ImageSearch.IMAGE_URLS_KEY);
             }
 
             int pos = 0;


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Archive-access-cvs] SF.net SVN: archive-access:[2504] trunk/archive-access/projects/nutchwax/ imagesearch/src/java/org/archive/nutchwax/imagesearch/ImageSearcherBean. java

[<mi...@us...>]

Revision: 2504
          http://archive-access.svn.sourceforge.net/archive-access/?rev=2504&view=rev
Author:   miklosh
Date:     2008-07-26 15:47:56 +0000 (Sat, 26 Jul 2008)

Log Message:
-----------
Made modifications needed for using Hadoop 0.17.
Display image metadata in search results.

Modified Paths:
--------------
    trunk/archive-access/projects/nutchwax/imagesearch/src/java/org/archive/nutchwax/imagesearch/ImageSearcherBean.java

Modified: trunk/archive-access/projects/nutchwax/imagesearch/src/java/org/archive/nutchwax/imagesearch/ImageSearcherBean.java
===================================================================
--- trunk/archive-access/projects/nutchwax/imagesearch/src/java/org/archive/nutchwax/imagesearch/ImageSearcherBean.java	2008-07-26 15:46:55 UTC (rev 2503)
+++ trunk/archive-access/projects/nutchwax/imagesearch/src/java/org/archive/nutchwax/imagesearch/ImageSearcherBean.java	2008-07-26 15:47:56 UTC (rev 2504)
@@ -25,6 +25,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
@@ -40,13 +41,16 @@
 import org.apache.lucene.store.FSDirectory;
 import org.apache.nutch.indexer.FsDirectory;
 import org.apache.nutch.indexer.Indexer;
+import org.apache.nutch.metadata.Metadata;
+import org.apache.nutch.util.HadoopFSUtil;
 import org.apache.nutch.util.NutchConfiguration;
 
-public class ImageSearcherBean {
+public class ImageSearcherBean implements ImageLoader {
 
     public static final Log LOG = LogFactory.getLog(ImageSearcherBean.class);
     
     private IndexReader reader;
+    private ImageDataReader imageReader;
 
     private Path baseDir;
     private Configuration conf;
@@ -66,18 +70,9 @@
         Path indexesDir = new Path(baseDir, "indexes");
         if (this.fs.exists(indexesDir)) {
             Vector<Path> doneDirs = new Vector<Path>();
-            Path[] dirs = fs.listPaths(indexesDir, new PathFilter() {
-
-                public boolean accept(Path f) {
-                    try {
-                        if (fs.isDirectory(f)) {
-                            return true;
-                        }
-                    } catch (IOException ioe) {
-                    }
-                    return false;
-                }
-            });
+            FileStatus[] fstats = fs.listStatus(indexesDir, 
+                    HadoopFSUtil.getPassDirectoriesFilter(fs));
+            Path[] dirs = HadoopFSUtil.getPaths(fstats);
             for (Path dir : dirs) {
                 Path indexdone = new Path(dir, Indexer.DONE_NAME);
                 if (fs.isFile(indexdone)) {
@@ -95,6 +90,8 @@
             Path[] indexDir = {new Path(baseDir, "index")};
             init(indexDir);
         }
+        this.imageReader = new ImageDataReader(FileSystem.get(conf), 
+                new Path(baseDir, "segments").toString(), conf);
     }
 
     /** Init given a set of indexes or just one index. */
@@ -124,12 +121,19 @@
         if (reader != null) {
             reader.close();
         }
+        if (imageReader != null) {
+            imageReader.close();
+        }
     }
 
     public IndexReader getReader() {
         return reader;
     }
     
+    public ImageWritable getImage(String id) throws IOException {
+        return imageReader.getImage(id);
+    }
+    
     /**
      * Calculate the score for an image hit.
      * @param hit found hit
@@ -245,6 +249,9 @@
                     nextDist = imgIndex < numDocImages-1 ? 
                         Math.abs(imagePositions[imgIndex+1] - pos) + (end-pos) : Integer.MAX_VALUE;
                 }
+                if (imgIndex >= numDocImages) {
+                    continue;
+                }
                 // Check if this image is in the allowed proximity of the span
                 if (dist > distThreshold) {
                     if (LOG.isDebugEnabled()) {
@@ -261,6 +268,7 @@
                 ImageHit newHit = new ImageHit(imageIds[imgIndex], imageUrls[imgIndex], currentDoc);
                 newHit.docSim = docSim;
                 newHit.docScore = docBoost;
+                newHit.parentUrl = doc.get("url");
                 newHit.proximity = Math.min(1.0f, 1.0f-((float)dist/maxDist));
                 newHit.score = scoreHit(newHit, doc);
 
@@ -338,6 +346,10 @@
                 hits.getTotal() >= maxHits ? maxHits : (int)hits.getTotal());
         for (ImageHit hit : top) {
             System.out.println(hit.score + " " + hit.url + " " + hit.imageId);
+            ImageWritable imageData = isb.getImage(hit.imageId);
+            if (imageData != null) {
+                System.out.println("[ " + imageData.getMetadata() + "]");
+            }
         }
     }
 }


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Archive-access-cvs] SF.net SVN: archive-access:[2523] trunk/archive-access/projects/nutchwax/ imagesearch/src/java/org/archive/nutchwax/imagesearch/ImageSearcherBean. java

[<mi...@us...>]

Revision: 2523
          http://archive-access.svn.sourceforge.net/archive-access/?rev=2523&view=rev
Author:   miklosh
Date:     2008-08-01 17:17:44 +0000 (Fri, 01 Aug 2008)

Log Message:
-----------
Throw UnsupportedOperationException when searching against an index not suitable for image search.

Modified Paths:
--------------
    trunk/archive-access/projects/nutchwax/imagesearch/src/java/org/archive/nutchwax/imagesearch/ImageSearcherBean.java

Modified: trunk/archive-access/projects/nutchwax/imagesearch/src/java/org/archive/nutchwax/imagesearch/ImageSearcherBean.java
===================================================================
--- trunk/archive-access/projects/nutchwax/imagesearch/src/java/org/archive/nutchwax/imagesearch/ImageSearcherBean.java	2008-07-31 21:29:53 UTC (rev 2522)
+++ trunk/archive-access/projects/nutchwax/imagesearch/src/java/org/archive/nutchwax/imagesearch/ImageSearcherBean.java	2008-08-01 17:17:44 UTC (rev 2523)
@@ -188,7 +188,10 @@
                 currentDoc = spans.doc();
                 doc = reader.document(currentDoc);
                 // Skip document with no images
-                if ("0".equals(doc.getField(ImageSearch.HAS_IMAGE_KEY).stringValue())) {
+                String hasImagesValue = doc.getField(ImageSearch.HAS_IMAGE_KEY).stringValue();
+                if (hasImagesValue == null) {
+                    throw new UnsupportedOperationException("Index not suitable for image search.");
+                } else if ("0".equals(hasImagesValue)) {
                     while (more && spans.doc() == currentDoc) {
                         more = spans.next();                        
                     }


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.