Menu
▾
▴
[Jython-checkins] SF.net SVN: jython:[7063] trunk/jython
|
From: <pj...@us...> - 2010-06-03 03:53:18
|
Revision: 7063
http://jython.svn.sourceforge.net/jython/?rev=7063&view=rev
Author: pjenvey
Date: 2010-06-03 03:53:11 +0000 (Thu, 03 Jun 2010)
Log Message:
-----------
fix caching of packages with dir listings larger than the 64k DataOutputStream
UTF-8 string limit. changes the packagecache on disk format but in a backwards
compatible way
fixes #1595
patch from bpedman
Modified Paths:
--------------
trunk/jython/NEWS
trunk/jython/src/org/python/core/packagecache/CachedJarsPackageManager.java
Added Paths:
-----------
trunk/jython/tests/java/org/python/core/packagecache/
trunk/jython/tests/java/org/python/core/packagecache/CachedJarsOver64kTest.java
trunk/jython/tests/java/org/python/core/packagecache/vim25-small.jar
Modified: trunk/jython/NEWS
===================================================================
--- trunk/jython/NEWS 2010-06-03 02:38:42 UTC (rev 7062)
+++ trunk/jython/NEWS 2010-06-03 03:53:11 UTC (rev 7063)
@@ -48,6 +48,7 @@
- [ 1567 ] [Windows] Wildcard Parameter * gets expanded to filename
- [ 1594 ] Glob patterns like *.txt processed incorrectly on startup
- [ 1356 ] [Windows] test_subprocess test_communicate_pipe_buf fails
+ - [ 1595 ] [patch] CachedJarsPackageManager cannot write cache for packages in jar over 64k
- Fix runtime issues during exitfuncs triggered via SystemRestart (such as
during Django or Pylons development mode reloading)
- Fix pickling of collections.defaultdict objects
Modified: trunk/jython/src/org/python/core/packagecache/CachedJarsPackageManager.java
===================================================================
--- trunk/jython/src/org/python/core/packagecache/CachedJarsPackageManager.java 2010-06-03 02:38:42 UTC (rev 7062)
+++ trunk/jython/src/org/python/core/packagecache/CachedJarsPackageManager.java 2010-06-03 03:53:11 UTC (rev 7063)
@@ -371,6 +371,14 @@
while (true) {
String packageName = istream.readUTF();
String classes = istream.readUTF();
+ // XXX: Handle multiple chunks of classes and concatenate them
+ // together. Multiple chunks were added in 2.5.2 (for #1595) in this
+ // way to maintain compatibility with the pre 2.5.2 format. In the
+ // future we should consider changing the cache format to prepend a
+ // count of chunks to avoid this check
+ if (packs.containsKey(packageName)) {
+ classes = packs.get(packageName) + classes;
+ }
packs.put(packageName, classes);
}
} catch (EOFException eof) {
@@ -396,8 +404,12 @@
for (Entry<String,String> kv : zipPackages.entrySet()) {
String classes = kv.getValue();
- ostream.writeUTF(kv.getKey());
- ostream.writeUTF(classes);
+ // Make sure each package is not larger than 64k
+ for (String part : splitString(classes, 65535)) {
+ // For each chunk, write the package name followed by the classes.
+ ostream.writeUTF(kv.getKey());
+ ostream.writeUTF(part);
+ }
}
ostream.close();
} catch (IOException ioe) {
@@ -406,6 +418,35 @@
}
/**
+ * Split up a string into several chunks based on a certain size
+ *
+ * The writeCacheFile method will use the writeUTF method on a
+ * DataOutputStream which only allows writing 64k chunks, so use
+ * this utility method to split it up
+ *
+ * @param str - The string to split up into chunks
+ * @param maxLength - The max size a string should be
+ * @return - An array of strings, each of which will not be larger than maxLength
+ */
+ protected static String[] splitString(String str, int maxLength) {
+ if (str == null) {
+ return null;
+ }
+
+ int len = str.length();
+ if (len <= maxLength) {
+ return new String[] {str};
+ }
+
+ int chunkCount = (int) Math.ceil((float) len / maxLength);
+ String[] chunks = new String[chunkCount];
+ for (int i = 0; i < chunkCount; i++) {
+ chunks[i] = str.substring(i * maxLength, Math.min(i * maxLength + maxLength, len));
+ }
+ return chunks;
+ }
+
+ /**
* Initializes cache. Eventually reads back cache index. Index persistent
* storage is accessed through inOpenIndex().
*/
Added: trunk/jython/tests/java/org/python/core/packagecache/CachedJarsOver64kTest.java
===================================================================
--- trunk/jython/tests/java/org/python/core/packagecache/CachedJarsOver64kTest.java (rev 0)
+++ trunk/jython/tests/java/org/python/core/packagecache/CachedJarsOver64kTest.java 2010-06-03 03:53:11 UTC (rev 7063)
@@ -0,0 +1,63 @@
+package org.python.core.packagecache;
+
+import java.io.File;
+
+import org.python.core.PyJavaPackage;
+import org.python.core.PyList;
+import org.python.core.packagecache.CachedJarsPackageManager;
+
+import junit.framework.TestCase;
+
+public class CachedJarsOver64kTest extends TestCase {
+
+ private TestCachePackageManager packageManager = null;
+
+ private File jarFile = null;
+
+ @Override
+ public void setUp() {
+ // Find the jar to use
+ packageManager = new TestCachePackageManager(new File(System
+ .getProperty("java.io.tmpdir")));
+ File cwd = new File(System.getProperty("python.test.source.dir"),
+ getClass().getPackage().getName().replace(".", "/"));
+ jarFile = new File(cwd, "vim25-small.jar");
+ }
+
+ public void testJarOver64k() {
+ assertTrue(jarFile.exists());
+ packageManager.addJarToPackages(jarFile, true);
+ assertFalse(packageManager.failed);
+ }
+
+ private class TestCachePackageManager extends CachedJarsPackageManager {
+
+ public boolean failed;
+
+ public TestCachePackageManager(File cachedir) {
+ if (useCacheDir(cachedir)){
+ initCache();
+ }
+ }
+
+ @Override
+ protected void warning(String msg){
+ failed = true;
+ }
+
+ @Override
+ public void addDirectory(File dir) {}
+ @Override
+ public void addJar(String jarfile, boolean cache) {}
+ @Override
+ public void addJarDir(String dir, boolean cache) {}
+ @Override
+ public PyList doDir(PyJavaPackage jpkg, boolean instantiate, boolean exclpkgs) {
+ return null;
+ }
+ @Override
+ public Class<?> findClass(String pkg, String name, String reason) { return null; }
+ @Override
+ public boolean packageExists(String pkg, String name) { return false; }
+ }
+}
Added: trunk/jython/tests/java/org/python/core/packagecache/vim25-small.jar
===================================================================
(Binary files differ)
Property changes on: trunk/jython/tests/java/org/python/core/packagecache/vim25-small.jar
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
