[Jython-checkins] SF.net SVN: jython:[7063] trunk/jython

[<pj...@us...>]

Revision: 7063
          http://jython.svn.sourceforge.net/jython/?rev=7063&view=rev
Author:   pjenvey
Date:     2010-06-03 03:53:11 +0000 (Thu, 03 Jun 2010)

Log Message:
-----------
fix caching of packages with dir listings larger than the 64k DataOutputStream
UTF-8 string limit. changes the packagecache on disk format but in a backwards
compatible way
fixes #1595
patch from bpedman

Modified Paths:
--------------
    trunk/jython/NEWS
    trunk/jython/src/org/python/core/packagecache/CachedJarsPackageManager.java

Added Paths:
-----------
    trunk/jython/tests/java/org/python/core/packagecache/
    trunk/jython/tests/java/org/python/core/packagecache/CachedJarsOver64kTest.java
    trunk/jython/tests/java/org/python/core/packagecache/vim25-small.jar

Modified: trunk/jython/NEWS
===================================================================
--- trunk/jython/NEWS	2010-06-03 02:38:42 UTC (rev 7062)
+++ trunk/jython/NEWS	2010-06-03 03:53:11 UTC (rev 7063)
@@ -48,6 +48,7 @@
     - [ 1567 ] [Windows] Wildcard Parameter * gets expanded to filename
     - [ 1594 ] Glob patterns like *.txt processed incorrectly on startup
     - [ 1356 ] [Windows] test_subprocess test_communicate_pipe_buf fails
+    - [ 1595 ] [patch] CachedJarsPackageManager cannot write cache for packages in jar over 64k
     - Fix runtime issues during exitfuncs triggered via SystemRestart (such as
       during Django or Pylons development mode reloading)
     - Fix pickling of collections.defaultdict objects

Modified: trunk/jython/src/org/python/core/packagecache/CachedJarsPackageManager.java
===================================================================
--- trunk/jython/src/org/python/core/packagecache/CachedJarsPackageManager.java	2010-06-03 02:38:42 UTC (rev 7062)
+++ trunk/jython/src/org/python/core/packagecache/CachedJarsPackageManager.java	2010-06-03 03:53:11 UTC (rev 7063)
@@ -371,6 +371,14 @@
                 while (true) {
                     String packageName = istream.readUTF();
                     String classes = istream.readUTF();
+                    // XXX: Handle multiple chunks of classes and concatenate them
+                    // together. Multiple chunks were added in 2.5.2 (for #1595) in this
+                    // way to maintain compatibility with the pre 2.5.2 format. In the
+                    // future we should consider changing the cache format to prepend a
+                    // count of chunks to avoid this check
+                    if (packs.containsKey(packageName)) {
+                        classes = packs.get(packageName) + classes;
+                    }
                     packs.put(packageName, classes);
                 }
             } catch (EOFException eof) {
@@ -396,8 +404,12 @@
 
             for (Entry<String,String> kv : zipPackages.entrySet()) {
                 String classes = kv.getValue();
-                ostream.writeUTF(kv.getKey());
-                ostream.writeUTF(classes);
+                // Make sure each package is not larger than 64k
+                for (String part : splitString(classes, 65535)) {
+                    // For each chunk, write the package name followed by the classes. 
+                    ostream.writeUTF(kv.getKey());
+                    ostream.writeUTF(part);
+                }
             }
             ostream.close();
         } catch (IOException ioe) {
@@ -406,6 +418,35 @@
     }
 
     /**
+     * Split up a string into several chunks based on a certain size
+     * 
+     *  The writeCacheFile method will use the writeUTF method on a
+     *  DataOutputStream which only allows writing 64k chunks, so use 
+     *  this utility method to split it up
+     * 
+     * @param str - The string to split up into chunks
+     * @param maxLength - The max size a string should be
+     * @return - An array of strings, each of which will not be larger than maxLength
+     */
+    protected static String[] splitString(String str, int maxLength) {
+        if (str == null) {
+            return null;
+        }
+
+        int len = str.length();
+        if (len <= maxLength) {
+            return new String[] {str};
+        }
+
+        int chunkCount = (int) Math.ceil((float) len / maxLength);
+        String[] chunks = new String[chunkCount];
+        for (int i = 0; i < chunkCount; i++) {
+            chunks[i] = str.substring(i * maxLength, Math.min(i * maxLength + maxLength, len));
+        }
+        return chunks;
+    }
+
+    /**
      * Initializes cache. Eventually reads back cache index. Index persistent
      * storage is accessed through inOpenIndex().
      */

Added: trunk/jython/tests/java/org/python/core/packagecache/CachedJarsOver64kTest.java
===================================================================
--- trunk/jython/tests/java/org/python/core/packagecache/CachedJarsOver64kTest.java	                        (rev 0)
+++ trunk/jython/tests/java/org/python/core/packagecache/CachedJarsOver64kTest.java	2010-06-03 03:53:11 UTC (rev 7063)
@@ -0,0 +1,63 @@
+package org.python.core.packagecache;
+
+import java.io.File;
+
+import org.python.core.PyJavaPackage;
+import org.python.core.PyList;
+import org.python.core.packagecache.CachedJarsPackageManager;
+
+import junit.framework.TestCase;
+
+public class CachedJarsOver64kTest extends TestCase {
+
+    private TestCachePackageManager packageManager = null;
+
+    private File jarFile = null;
+
+    @Override
+    public void setUp() {
+        // Find the jar to use
+        packageManager = new TestCachePackageManager(new File(System
+                                                              .getProperty("java.io.tmpdir")));
+        File cwd = new File(System.getProperty("python.test.source.dir"),
+                            getClass().getPackage().getName().replace(".", "/"));
+        jarFile = new File(cwd, "vim25-small.jar");
+    }
+
+    public void testJarOver64k() {
+        assertTrue(jarFile.exists());
+        packageManager.addJarToPackages(jarFile, true);
+        assertFalse(packageManager.failed);
+    }
+
+    private class TestCachePackageManager extends CachedJarsPackageManager {
+
+        public boolean failed;
+
+        public TestCachePackageManager(File cachedir) {
+            if (useCacheDir(cachedir)){
+                initCache();
+            }
+        }
+
+        @Override
+        protected void warning(String msg){
+            failed = true;
+        }
+
+        @Override
+        public void addDirectory(File dir) {}
+        @Override
+        public void addJar(String jarfile, boolean cache) {}
+        @Override
+        public void addJarDir(String dir, boolean cache) {}
+        @Override
+        public PyList doDir(PyJavaPackage jpkg, boolean instantiate, boolean exclpkgs) {
+            return null;
+        }
+        @Override
+        public Class<?> findClass(String pkg, String name, String reason) { return null; }
+        @Override
+        public boolean packageExists(String pkg, String name) { return false; }
+    }
+}

Added: trunk/jython/tests/java/org/python/core/packagecache/vim25-small.jar
===================================================================
(Binary files differ)


Property changes on: trunk/jython/tests/java/org/python/core/packagecache/vim25-small.jar
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.