[Jython-checkins] CVS: jython/org/python/core CachedJarsPackageManager.java,NONE,1.1 PathPackageManager.java,NONE,1.1 SysPackageManager.java,NONE,1.1 PyJavaDirPackage.java,2.2,NONE

[Samuele P. <ped...@us...>]

Update of /cvsroot/jython/jython/org/python/core
In directory slayer.i.sourceforge.net:/tmp/cvs-serv1809/core

Added Files:
	CachedJarsPackageManager.java PathPackageManager.java 
	SysPackageManager.java 
Removed Files:
	PyJavaDirPackage.java 
Log Message:
Loading logic fixes:
  additions and deletions.
For issues involved see the jython-dev list archives.


--- NEW FILE ---

package org.python.core;

import java.util.Hashtable;
import java.util.Vector;
import java.util.Enumeration;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipEntry;
import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLDecoder;
import java.lang.reflect.Modifier;

/** Abstract package manager that gathers info about statically known classes
 * from a set of jars. This info can be eventually cached.
 * Off-the-shelf this class offers a local file-system based cache impl.
 */
public abstract class CachedJarsPackageManager extends PackageManager {

    /** Message log method - hook. This default impl does nothing.
     * @param msg message text
     */
    protected void message(String msg) {
    }
    /** Warning log method - hook. This default impl does nothing.
     * @param warn warning text
     */
    protected void warning(String warn) {
    }
    /** Comment log method - hook. This default impl does nothing.
     * @param msg message text
     */
    protected void comment(String msg) {
    }
    /** Debug log method - hook. This default impl does nothing.
     * @param msg message text
     */
    protected void debug(String msg) {
    }

    /** Filter class/pkg by name helper method - hook.
     * The default impl. is used by {@link #addJarToPackages} in order to filter out classes
     * whose name contains '$' (e.g. inner classes,...).
     * Should be used or overriden by derived classes too. Also to be used in {@link #doDir}.
     * @param name class/pkg name
     * @param pkg if true, name refers to a pkg
     * @return true if name must be filtered out
     */
    protected boolean filterByName(String name,boolean pkg) {
        return name.indexOf('$') != -1;
    }

    /** Filter class by access perms helper method - hook.
     * The default impl. is used by {@link #addJarToPackages} in order to filter out non-public classes.
     * Should be used or overriden by derived classes too. Also to be used in {@link #doDir}.
     * Access perms can be read with {@link #checkAccess}.
     * @param name class name
     * @param acc class access permissions as int
     * @return true if name must be filtered out
     */
    protected boolean filterByAccess(String name,int acc) {
        return (acc & Modifier.PUBLIC) != Modifier.PUBLIC;
    }

    private boolean indexModified;
    private Hashtable jarfiles;

    private static String vectorToString(Vector vec) {
        int n = vec.size();
        StringBuffer ret = new StringBuffer();
        for(int i=0; i<n; i++) {
            ret.append((String)vec.elementAt(i));
            if (i<n-1) ret.append(",");
        }
        return ret.toString();
    }

    // Add a single class from zipFile to zipPackages
    // Only add valid, public classes
    private void addZipEntry(Hashtable zipPackages, ZipEntry entry,ZipInputStream zip) throws IOException
    {
        String name = entry.getName();
        //System.err.println("entry: "+name);
        if (!name.endsWith(".class")) return;

        char sep = '/';
        int breakPoint = name.lastIndexOf(sep);
        if (breakPoint == -1) {
            breakPoint = name.lastIndexOf('\\');
            sep = '\\';
        }

        String packageName;
        if (breakPoint == -1) {
            packageName = "";
        } else {
            packageName = name.substring(0,breakPoint).replace(sep, '.');
        }

        String className = name.substring(breakPoint+1, name.length()-6);

        if (filterByName(className,false)) return;

        // An extra careful test, maybe should be ignored????
        int access = checkAccess(zip);

        if ((access == -1) || filterByAccess(name,access)) return;

        Vector vec = (Vector)zipPackages.get(packageName);
        if (vec == null) {
            vec = new Vector();
            zipPackages.put(packageName, vec);
        }
        vec.addElement(className);
    }

    // Extract all of the packages in a single jarfile
    private Hashtable getZipPackages(InputStream jarin) throws IOException {
        Hashtable zipPackages = new Hashtable();

        ZipInputStream zip=new ZipInputStream(jarin);

        ZipEntry entry;
        while ((entry = zip.getNextEntry()) != null) {
            addZipEntry(zipPackages, entry, zip);
            zip.closeEntry();
        }

        // Turn each vector into a comma-separated String
        for (Enumeration e = zipPackages.keys() ; e.hasMoreElements() ;) {
            Object key = e.nextElement();
            Vector vec = (Vector)zipPackages.get(key);
            zipPackages.put(key, vectorToString(vec));
        }

        return zipPackages;
    }

    /** Gathers classes info from jar specified by jarurl URL.
     * Eventually just using previously cached info.
     * Eventually updated info is not cached.
     * Persistent cache storage access goes through {@link #inOpenCacheFile}, {@link #outCreateCacheFile}.
     */
    public void addJarToPackages(java.net.URL jarurl) {
        addJarToPackages(jarurl,null,false);
    }

    /** Gathers classes info from jar specified by jarurl URL.
     * Eventually just using previously cached info.
     * Eventually updated info is (re-)cached if param cache is true.
     * Persistent cache storage access goes through {@link #inOpenCacheFile}, {@link #outCreateCacheFile}.
     */
    public void addJarToPackages(URL jarurl,boolean cache) {
        addJarToPackages(jarurl,null,cache);
    }

    /** Gathers classes info from jar specified by File jarfile.
     * Eventually just using previously cached info.
     * Eventually updated info is not cached.
     * Persistent cache storage access goes through {@link #inOpenCacheFile}, {@link #outCreateCacheFile}.
     */
    public void addJarToPackages(File jarfile) {
        addJarToPackages(null,jarfile,false);
    }

    /** Gathers classes info from jar specified by File jarfile.
     * Eventually just using previously cached info.
     * Eventually updated info is (re-)cached if param cache is true.
     * Persistent cache storage access goes through {@link #inOpenCacheFile}, {@link #outCreateCacheFile}.
     */
    public void addJarToPackages(File jarfile,boolean cache) {
        addJarToPackages(null,jarfile,cache);
    }
    
    private void addJarToPackages(URL jarurl,File jarfile,boolean cache) {
        try {            
            boolean caching = jarfiles!=null;

            URLConnection jarconn = null;
            boolean localfile = true;
            
            if (jarfile == null) {
                jarconn = jarurl.openConnection();
                // This is necessary because 'file:' url-connections
                // return always 0 through getLastModified (bug?).
                // And in order to handle localfiles (from urls too) uniformly.
                if(jarconn.getURL().getProtocol().equals("file")) {
                    // ??pending: need to use java2 URLDecoder.decode?
                    // but under 1.1 this is absent and should be simulated.                    
                    jarfile = new File(jarurl.getPath().replace('/',File.separatorChar)); 
                }
                else localfile = false;
            }
            
            if (localfile && !jarfile.exists()) return;

            Hashtable zipPackages = null;

            long mtime = 0;
            String jarcanon = null;
            JarXEntry entry = null;
            boolean brandNew = false;

            if(caching) {

                if(localfile) {
                    mtime = jarfile.lastModified();
                    jarcanon = jarfile.getCanonicalPath();
                } else {
                    mtime = jarconn.getLastModified();
                    jarcanon = jarurl.toString();
                }

                entry = (JarXEntry)jarfiles.get(jarcanon);

                if (entry == null) {
                    message("processing new jar, '"+
                    jarcanon+"'");

                    String jarname;
                    if(localfile) {
                        jarname = jarfile.getName();
                    } else {
                        jarname = jarurl.getPath();
                        int slash = jarname.lastIndexOf('/');
                        if (slash != -1)
                        jarname=jarname.substring(slash+1);
                    }
                    jarname=jarname.substring(0,jarname.length()-4);

                    entry = new JarXEntry(jarname);
                    jarfiles.put(jarcanon, entry);

                    brandNew = true;
                }


                if (mtime != 0 && entry.mtime == mtime) {
                    zipPackages = readCacheFile(entry, jarcanon);
                }

            }

            if (zipPackages == null) {
                caching = caching && cache;

                if(caching) {
                    indexModified = true;
                    if (entry.mtime != 0) {
                        message("processing modified jar, '"+
                        jarcanon+"'");
                    }
                    entry.mtime = mtime;
                }

                InputStream jarin;
                if (jarconn == null) jarin = new BufferedInputStream(new FileInputStream(jarfile));
                else jarin = jarconn.getInputStream();
                
                zipPackages = getZipPackages(jarin);

                if(caching) writeCacheFile(entry, jarcanon, zipPackages, brandNew); // Write the cache file
            }

            addPackages(zipPackages, jarcanon);
        } catch (IOException ioe) {
            // silently skip any bad directories
            warning("skipping bad jar, '" +
            jarurl.toString() + "'");
        }

    }

    private void addPackages(Hashtable zipPackages, String jarfile) {
        for (Enumeration e = zipPackages.keys() ; e.hasMoreElements() ;) {
            String pkg = (String)e.nextElement();
            String classes = (String)zipPackages.get(pkg);

            makeJavaPackage(pkg, classes, jarfile);
        }
    }

    // Read in cache file storing package info for a single .jar
    // Return null and delete this cachefile if it is invalid
    private Hashtable readCacheFile(JarXEntry entry,String jarcanon)
    {
        String cachefile = entry.cachefile;
        long mtime = entry.mtime;

        debug("reading cache, '"+jarcanon+"'");

        try {
            DataInputStream istream = inOpenCacheFile(cachefile);
            String old_jarcanon = istream.readUTF();
            long old_mtime = istream.readLong();
            if ((!old_jarcanon.equals(jarcanon)) ||
            (old_mtime != mtime))
            {
                comment("invalid cache file: "+
                cachefile+", "+jarcanon+":"+
                old_jarcanon+", "+mtime+":"+old_mtime);
                deleteCacheFile(cachefile);
                return null;
            }
            Hashtable packs = new Hashtable();
            try {
                while (true) {
                    String packageName = istream.readUTF();
                    String classes = istream.readUTF();
                    packs.put(packageName, classes);
                }
            } catch (EOFException eof) {
                ;
            }
            istream.close();

            return packs;
        } catch (IOException ioe) {
            // if (cachefile.exists()) cachefile.delete();
            return null;
        }
    }

    // Write a cache file storing package info for a single .jar
    private void writeCacheFile(JarXEntry entry,String jarcanon,
    Hashtable zipPackages,boolean brandNew)
    {
        try {
            DataOutputStream ostream = outCreateCacheFile(entry, brandNew);
            ostream.writeUTF(jarcanon);
            ostream.writeLong(entry.mtime);
            comment("rewriting cachefile for '"+jarcanon+"'");

            for (Enumeration e = zipPackages.keys() ; e.hasMoreElements() ;) {
                String packageName = (String)e.nextElement();
                String classes = (String)zipPackages.get(packageName);
                ostream.writeUTF(packageName);
                ostream.writeUTF(classes);
            }
            ostream.close();
        } catch (IOException ioe) {
            warning("can't write cache file for '"+jarcanon+"'");
        }
    }

    /** Initializes cache. Eventually reads back cache index.
     * Index persistent storage is accessed through {@link #inOpenIndex}.
     */
    protected void initCache() {
        indexModified = false;
        jarfiles = new Hashtable();

        try {
            DataInputStream istream = inOpenIndex();
            if (istream == null) return;

            try {
                while (true) {
                    String jarcanon = istream.readUTF();
                    String cachefile = istream.readUTF();
                    long mtime = istream.readLong();
                    jarfiles.put(jarcanon, new JarXEntry(cachefile,mtime));
                }
            } catch (EOFException eof) {
                ;
            }
            istream.close();
        } catch (IOException ioe) {
            warning("invalid index file");
        }

    }

    /** Write back cache index.
     * Index persistent storage is accessed through {@link #outOpenIndex}.
     */
    public void saveCache() {
        if(jarfiles == null || !indexModified ) return;

        indexModified = false;

        comment("writing modified index file");

        try {
            DataOutputStream ostream = outOpenIndex();
            for (Enumeration e = jarfiles.keys(); e.hasMoreElements();) {
                String jarcanon = (String)e.nextElement();
                JarXEntry entry = (JarXEntry)jarfiles.get(jarcanon);
                ostream.writeUTF(jarcanon);
                ostream.writeUTF(entry.cachefile);
                ostream.writeLong(entry.mtime);
            }
            ostream.close();
        } catch (IOException ioe) {
            warning("can't write index file");
        }
    }

    // hooks for changing cache storage
    
    /** To pass a cachefile id by ref. And for internal use.
     * @see #outCreateCacheFile
     */
    public static class JarXEntry extends Object {
        /** cachefile id */
        public String cachefile;

        public long mtime;

        public JarXEntry(String cachefile) {
            this.cachefile = cachefile;
        }

        public JarXEntry(String cachefile,long mtime) {
            this.cachefile = cachefile;
            this.mtime = mtime;
        }


    }

    /** Open cache index for reading from persistent storage - hook.
     * Must Return null if this is absent.
     * This default impl is part of the off-the-shelf local file-system cache impl.
     * Can be overriden.
     */
    protected DataInputStream inOpenIndex() throws IOException {
        File indexFile = new File(cachedir, "packages.idx");

        if (!indexFile.exists()) return null;

        DataInputStream istream = new DataInputStream(
        new BufferedInputStream(new FileInputStream(indexFile)));

        return istream;
    }

    /** Open cache index for writing back to persistent storage - hook.
     * This default impl is part of the off-the-shelf local file-system cache impl.
     * Can be overriden.
     */
    protected DataOutputStream outOpenIndex() throws IOException {
        File indexFile = new File(cachedir, "packages.idx");

        return new DataOutputStream(
        new BufferedOutputStream(new FileOutputStream(indexFile)));
    }

    /** Open cache file for reading from persistent storage - hook.
     * This default impl is part of the off-the-shelf local file-system cache impl.
     * Can be overriden.
     */
    protected DataInputStream inOpenCacheFile(String cachefile) throws IOException {
        return new DataInputStream(new BufferedInputStream(new FileInputStream(cachefile)));
    }

    /** Delete (invalidated) cache file from persistent storage - hook.
     * This default impl is part of the off-the-shelf local file-system cache impl.
     * Can be overriden.
     */
    protected void deleteCacheFile(String cachefile) {
        new File(cachefile).delete();
    }

    /** Create/open cache file for rewriting back to persistent storage - hook.
     * If create is false, cache file is supposed to exist and must be opened
     * for rewriting, entry.cachefile is a valid cachefile id.
     * If create is true, cache file must be created. entry.cachefile is a flat
     * jarname to be used to produce a valid cachefile id (to be put back in entry.cachefile
     * on exit).
     * This default impl is part of the off-the-shelf local file-system cache impl.
     * Can be overriden.
     */
    protected DataOutputStream outCreateCacheFile(JarXEntry entry, boolean create) throws IOException {
        File cachefile = null;

        if(create) {
            int index = 1;
            String suffix = "";
            String jarname = entry.cachefile;
            while (true) {
                cachefile = new File(cachedir, jarname+suffix+".pkc");
                //System.err.println("try cachefile: "+cachefile);
                if (!cachefile.exists()) break;
                suffix = "$"+index;
                index += 1;
            }
            entry.cachefile = cachefile.getCanonicalPath();
        } else cachefile = new File(entry.cachefile);

        return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(cachefile)));
    }

    // for default cache (local fs based) impl

    private File cachedir;

    /** Initialize off-the-shelf (default) local file-system cache impl.
     * Must be called before {@link #initCache}.
     * cachedir is the cache repository directory, this is eventually created.
     * Returns true if dir works.
     */
    protected boolean useCacheDir(File cachedir) {
        if(cachedir == null) return false;
        if (!cachedir.isDirectory() && cachedir.mkdirs() == false) {
            warning("can't create package cache dir, '"+cachedir+"'");
            return false;
        }

        this.cachedir = cachedir;

        return true;
    }

}

--- NEW FILE ---

package org.python.core;

import java.io.*;
import java.lang.reflect.Modifier;

/** Path package manager. Gathering classes info dynamically from a set of directories
 * in path {@link #searchPath}, and statically from a set of jars, like {@link CachedJarsPackageManager}.
 */
public abstract class PathPackageManager extends CachedJarsPackageManager {

    public PyList searchPath;

    public PathPackageManager() {
        searchPath = new PyList();
    }

    /** Helper for {@link #packageExists(java.lang.String,java.lang.String)}.
     * Scans for package pkg.name the directories in path.
     *
     */
    protected boolean packageExists(PyList path,String pkg,String name) {
        String child = pkg.replace('.',File.separatorChar) + File.separator + name;

        for (int i=0; i < path.__len__(); i++) {
            String dir = path.get(i).__str__().toString();
            if(dir.length() == 0) dir = null;

            if (new File(dir,child).isDirectory()) return true;
        }
        return false;
    }

    /** Helper for {@link #doDir(PyJavaPackage,boolean,boolean)}.
     * Scans for package jpkg content over the directories in path.
     * Add to ret the founded classes/pkgs.
     * Filter out classes using {@link #filterByName},{@link #filterByAccess}.
     */
    protected void doDir(PyList path,PyList ret, PyJavaPackage jpkg,boolean instantiate,boolean exclpkgs) {
        String child=jpkg.__name__.replace('.',File.separatorChar);

        for (int i=0; i < path.__len__(); i++) {
            String dir = path.get(i).__str__().toString();
            if(dir.length() == 0) dir = null;

            File childFile = new File(dir,child);

            String[] list=childFile.list();
            if(list == null) continue;

            doList:
            for (int j=0; j < list.length; j++) {
                String jname = list[j];

                File cand = new File(childFile, jname);

                int jlen = jname.length();

                boolean pkgCand=false;

                if (cand.isDirectory()) {
                    if (!instantiate && exclpkgs) continue;
                    pkgCand = true;
                } else {
                    if(!jname.endsWith(".class")) continue;
                    jlen-=6;
                }

                jname = jname.substring(0,jlen);
                PyString name = new PyString(jname);

                if (filterByName(jname,pkgCand)) continue;

                // for opt maybe we should some hash-set for ret
                if (jpkg.__dict__.has_key(name) || jpkg.clsSet.has_key(name) || ret.__contains__(name)) {
                    continue;
                }

                if(!Character.isJavaIdentifierStart(jname.charAt(0))) continue;
                for(int k = 1; k < jlen; k++) {
                    if(!Character.isJavaIdentifierPart(jname.charAt(k))) continue doList;
                }

                if(!pkgCand) {
                    try {
                        int acc = checkAccess(new BufferedInputStream(new FileInputStream(cand)));
                        if ((acc == -1) || filterByAccess(jname, acc) ) continue;
                    } catch(IOException e) { 
                        continue; 
                    }
                }

                if(instantiate) {
                    if(pkgCand) jpkg.addPackage(jname);
                    else jpkg.addLazyClass(jname);
                }

                ret.append(name);

            }
        }

    }

    /** Add directory dir (if exists) to {@link #searchPath}.
     */
    public void addDirectory(File dir) {
        try {
            if (dir.getPath().length() == 0) searchPath.append(Py.EmptyString);
            else searchPath.append(new PyString(dir.getCanonicalPath()));
        } catch(IOException e) {
            warning("skipping bad directory, '" +dir+ "'");
        }
    }


    // ??pending: 
    // Uses simply split and not a StringTokenizer+trim to adhere to sun jvm parsing of classpath.
    // E.g. "a;" is parsed by sun jvm as a, ""; the latter is interpreted as cwd.
    // jview trims and cwd is per default in classpath. The logic here should work
    // for both(...). Need to distinguish?
    // This code does not avoid duplicates in searchPath. Should cause no problem (?).
    /** Adds "classpath" entry. Calls {@link #addDirectory} if path refers to a dir, {@link #addJarToPackages(java.io.File, boolean)}
      with param cache true if path refers to a jar.
    */
    public void addClassPath(String path) {
        PyList paths = new PyString(path).split(java.io.File.pathSeparator);

        for (int i = 0; i < paths.__len__(); i++) {
            String entry = paths.get(i).toString();
            if (entry.endsWith(".jar") || entry.endsWith(".zip")) {
                addJarToPackages(new File(entry),true);
            } else {
                File dir = new File(entry);
                if (entry.length() == 0 || dir.isDirectory()) addDirectory(dir);
            }
        }
    }            

    public PyList doDir(PyJavaPackage jpkg,boolean instantiate,boolean exclpkgs) {
        PyList basic = basicDoDir(jpkg,instantiate,exclpkgs);
        PyList ret = new PyList();

        doDir(searchPath,ret,jpkg,instantiate,exclpkgs);

        return merge(basic,ret);
    }

    public boolean packageExists(String pkg,String name) {
        return packageExists(searchPath,pkg,name);
    }

}

--- NEW FILE ---

package org.python.core;

import java.util.Properties;
import java.util.StringTokenizer;
import java.io.*;

/** System package manager.
 * Used by org.python.core.PySystemState.
 */
public class SysPackageManager extends PathPackageManager {

protected void message(String msg) { Py.writeMessage("*sys-package-mgr*",msg); }
protected void warning(String warn) { Py.writeWarning("*sys-package-mgr*",warn); }
protected void comment(String msg) { Py.writeComment("*sys-package-mgr*",msg); }
protected void debug(String msg) {Py.writeDebug("*sys-package-mgr*",msg); }

    public SysPackageManager(File cachedir, Properties registry) {
        if(useCacheDir(cachedir)) {
            initCache();
            findAllPackages(registry);
            saveCache();
        }
    }

    private void addJarDir(String jdir) {
        File file = new File(jdir);
        if (!file.isDirectory()) return;
        String[] files = file.list();
        for(int i=0; i<files.length; i++) {
            String entry = files[i];
            if (entry.endsWith(".jar") || entry.endsWith(".zip")) {
                addJarToPackages(new File(jdir,entry),true);
            }
        }
    }

    private void addJarPath(String path) {
        StringTokenizer tok =
        new StringTokenizer(path, java.io.File.pathSeparator);
        while  (tok.hasMoreTokens())  {
            // ??pending: do jvms trim? how is interpreted entry=""?
            String entry = tok.nextToken();            
            addJarDir(entry);
        }
    }

    private void findAllPackages(Properties registry) {
        String paths = registry.getProperty(
        "python.packages.paths",
        "java.class.path,sun.boot.class.path");
        String directories = registry.getProperty(
        "python.packages.directories",
        "java.ext.dirs");
        String fakepath = registry.getProperty("python.packages.fakepath", null);
        StringTokenizer tok = new StringTokenizer(paths, ",");
        while  (tok.hasMoreTokens())  {
            String entry = tok.nextToken().trim();
            String tmp = registry.getProperty(entry);
            if (tmp == null) continue;
            addClassPath(tmp);
        }

        tok = new StringTokenizer(directories, ",");
        while  (tok.hasMoreTokens())  {
            String entry = tok.nextToken().trim();
            String tmp = registry.getProperty(entry);
            if (tmp == null) continue;
            addJarPath(tmp);
        }

        if (fakepath != null) addClassPath(fakepath);
    }

    public Class findClass(String pkg,String name) {
        Class c;
        if (pkg.length()>0) c = Py.findClassEx(pkg+'.'+name);
        else c = Py.findClassEx(name);

        return c;
    }

    public PyList doDir(PyJavaPackage jpkg,boolean instantiate,boolean exclpkgs) {
        PyList basic = basicDoDir(jpkg,instantiate,exclpkgs);
        PyList ret = new PyList();

        doDir(searchPath,ret,jpkg,instantiate,exclpkgs);

        PySystemState system = Py.getSystemState();

        if (system.getClassLoader() == null) doDir(system.path,ret,jpkg,instantiate,exclpkgs);

        return merge(basic,ret);
    }

    public boolean packageExists(String pkg,String name) {
        if (packageExists(searchPath,pkg,name)) return true;

        PySystemState system = Py.getSystemState();

        if (system.getClassLoader() == null &&  packageExists(Py.getSystemState().path,pkg,name)) {
            return true;
        }

        return false;
    }

}

--- PyJavaDirPackage.java DELETED ---