[Pydev-cvs] org.python.pydev/src_completions/org/python/pydev/editor/codecompletion/revisited/modules AbstractModule.java, 1.15, 1.16 EmptyModule.java, 1.4, 1.5 SourceModule.java, 1.20, 1.21 SourceToken.java, 1.3, 1.4 CompiledModule.java, 1.8, 1.9

[Fabio Z. <fa...@us...>]

Update of /cvsroot/pydev/org.python.pydev/src_completions/org/python/pydev/editor/codecompletion/revisited/modules
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv4313/src_completions/org/python/pydev/editor/codecompletion/revisited/modules

Modified Files:
	AbstractModule.java EmptyModule.java SourceModule.java 
	SourceToken.java CompiledModule.java 
Log Message:
Synching to latest changes:

Pydev
<ul>
<li><strong>Editor</strong>: Cursor settings no longer overridden</li>
<li><strong>Code-completion</strong>: If __all__ is defined with runtime elements (and not only in a single assign statement), it's ignored for code-completion purposes</li>
<li><strong>Debugger</strong>: Pythonpath the same in debug and regular modes (sys.path[0] is the same directory as the file run)</li>
<li><strong>Debugger</strong>: Persist choices done in the debugger when files from the debugger are not found</li>
<li><strong>Interpreter config</strong>: "email" automatically added to the "forced builtins"</li>
<li><strong>Parser</strong>: Correctly recognizing absolute import with 3 or more levels</li>
<li><strong>Syntax check</strong>: Option to do only on active editor</li>
</ul>

Also: tabs changed for spaces

Index: SourceModule.java
===================================================================
RCS file: /cvsroot/pydev/org.python.pydev/src_completions/org/python/pydev/editor/codecompletion/revisited/modules/SourceModule.java,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** SourceModule.java	17 Aug 2008 00:26:47 -0000	1.20
--- SourceModule.java	28 Sep 2008 12:45:40 -0000	1.21
***************
*** 102,105 ****
--- 102,112 ----
      public String zipFilePath;
      
+     
+     /**
+      * This is a parse error that was found when parsing the code that generated this module
+      */
+     public final Throwable parseError;
+     
+     
      public String getZipFilePath(){
          return zipFilePath;
***************
*** 115,118 ****
--- 122,140 ----
       */
      private HashMap<Integer, TreeMap<String, Object>> tokensCache = new HashMap<Integer, TreeMap<String,Object>>();
+     
+     /**
+      * Set when the visiting is done (can hold some metadata, such as __all__ token assign)
+      */
+     private GlobalModelVisitor globalModelVisitorCache = null;
+     
+     /**
+      * 
+      * @return the visitor that was used to generate the internal tokens for this module (if any).
+      * 
+      * May be null
+      */
+     public GlobalModelVisitor getGlobalModelVisitorCache() {
+         return globalModelVisitorCache;
+     }
  
      /**
***************
*** 123,127 ****
       */
      public IToken[] getWildImportedModules() {
!         return getTokens(GlobalModelVisitor.WILD_MODULES, null);
      }
  
--- 145,149 ----
       */
      public IToken[] getWildImportedModules() {
!         return getTokens(GlobalModelVisitor.WILD_MODULES, null, null);
      }
  
***************
*** 136,140 ****
       */
      public IToken[] getTokenImportedModules() {
!         return getTokens(GlobalModelVisitor.ALIAS_MODULES, null);
      }
  
--- 158,162 ----
       */
      public IToken[] getTokenImportedModules() {
!         return getTokens(GlobalModelVisitor.ALIAS_MODULES, null, null);
      }
  
***************
*** 153,157 ****
       */
      public IToken[] getGlobalTokens() {
!         return getTokens(GlobalModelVisitor.GLOBAL_TOKENS, null);
      }
  
--- 175,179 ----
       */
      public IToken[] getGlobalTokens() {
!         return getTokens(GlobalModelVisitor.GLOBAL_TOKENS, null, null);
      }
  
***************
*** 160,164 ****
       */
      public String getDocString() {
!         IToken[] l = getTokens(GlobalModelVisitor.MODULE_DOCSTRING, null);
          if (l.length > 0) {
              SimpleNode a = ((SourceToken) l[0]).getAst();
--- 182,186 ----
       */
      public String getDocString() {
!         IToken[] l = getTokens(GlobalModelVisitor.MODULE_DOCSTRING, null, null);
          if (l.length > 0) {
              SimpleNode a = ((SourceToken) l[0]).getAst();
***************
*** 178,218 ****
      public boolean isInDirectGlobalTokens(String tok, ICompletionCache completionCache){
          TreeMap<String, Object> tokens = tokensCache.get(GlobalModelVisitor.GLOBAL_TOKENS);
!     	if(tokens == null){
!     		getGlobalTokens();
!     	}
!     	
!     	boolean ret = false;
!     	if(tokens != null){
!         	synchronized (tokens) {
!         		ret = tokens.containsKey(tok);
!         	}
!     	}
!     	
!     	if(ret == false){
!     		ret = isInDirectImportTokens(tok);
!     	}
!     	return ret;
      }
      
! 	public boolean isInDirectImportTokens(String tok) {
          TreeMap<String, Object> tokens = tokensCache.get(GlobalModelVisitor.ALIAS_MODULES);
          if(tokens != null){
! 			getTokenImportedModules();
! 		}
! 		
! 		boolean ret = false;
! 		if(tokens != null){
! 			synchronized (tokens) {
! 				ret = tokens.containsKey(tok);
! 			}
! 		}
! 		return ret; 
! 	}
! 
!     
! 	private IToken[] getTokens(int which, ICompletionState state) {
!         return getTokens(which, state, null);
      }
!     
      /**
       * @param lookOnlyForNameStartingWith: if not null, well only get from the cache tokens starting with the given representation
--- 200,236 ----
      public boolean isInDirectGlobalTokens(String tok, ICompletionCache completionCache){
          TreeMap<String, Object> tokens = tokensCache.get(GlobalModelVisitor.GLOBAL_TOKENS);
!         if(tokens == null){
!             getGlobalTokens();
!         }
!         
!         boolean ret = false;
!         if(tokens != null){
!             synchronized (tokens) {
!                 ret = tokens.containsKey(tok);
!             }
!         }
!         
!         if(ret == false){
!             ret = isInDirectImportTokens(tok);
!         }
!         return ret;
      }
      
!     public boolean isInDirectImportTokens(String tok) {
          TreeMap<String, Object> tokens = tokensCache.get(GlobalModelVisitor.ALIAS_MODULES);
          if(tokens != null){
!             getTokenImportedModules();
!         }
!         
!         boolean ret = false;
!         if(tokens != null){
!             synchronized (tokens) {
!                 ret = tokens.containsKey(tok);
!             }
!         }
!         return ret; 
      }
! 
!         
      /**
       * @param lookOnlyForNameStartingWith: if not null, well only get from the cache tokens starting with the given representation
***************
*** 224,228 ****
              throw new RuntimeException("Cannot do this one with caches");
          }
!     	//cache
          TreeMap<String, Object> tokens = tokensCache.get(which);
          
--- 242,246 ----
              throw new RuntimeException("Cannot do this one with caches");
          }
!         //cache
          TreeMap<String, Object> tokens = tokensCache.get(which);
          
***************
*** 230,236 ****
              return createArrayFromCacheValues(tokens, lookOnlyForNameStartingWith);
          }
!     	//end cache
!     	
!     	
          try {
              tokensCache.put(GlobalModelVisitor.ALIAS_MODULES, new TreeMap<String, Object>());
--- 248,254 ----
              return createArrayFromCacheValues(tokens, lookOnlyForNameStartingWith);
          }
!         //end cache
!         
!         
          try {
              tokensCache.put(GlobalModelVisitor.ALIAS_MODULES, new TreeMap<String, Object>());
***************
*** 243,247 ****
              
              //we request all and put it into the cache (partitioned), because that's faster than making multiple runs through it
!             List<IToken> ret = GlobalModelVisitor.getTokens(ast, all, name, state, true);
              
              
--- 261,269 ----
              
              //we request all and put it into the cache (partitioned), because that's faster than making multiple runs through it
!             GlobalModelVisitor globalModelVisitor = GlobalModelVisitor.getGlobalModuleVisitorWithTokens(ast, all, name, state, false);
!             
!             this.globalModelVisitorCache = globalModelVisitor;
!             
!             List<IToken> ret = globalModelVisitor.getTokens();
              
              
***************
*** 250,257 ****
              }
              //cache
!     		for (IToken token : ret) {
                  int choice;
!         		if(token.isWildImport()){
!         		    choice = GlobalModelVisitor.WILD_MODULES;
                  }else if(token.isImportFrom() || token.isImport()){
                      choice = GlobalModelVisitor.ALIAS_MODULES;
--- 272,279 ----
              }
              //cache
!             for (IToken token : ret) {
                  int choice;
!                 if(token.isWildImport()){
!                     choice = GlobalModelVisitor.WILD_MODULES;
                  }else if(token.isImportFrom() || token.isImport()){
                      choice = GlobalModelVisitor.ALIAS_MODULES;
***************
*** 263,267 ****
                  String rep = token.getRepresentation();
                  if(DEBUG_INTERNAL_GLOBALS_CACHE){
!             		System.out.println("Adding choice:"+choice+" name:"+rep);
                      if(choice != which){
                          System.out.println("Looking for:"+which+"found:"+choice);
--- 285,289 ----
                  String rep = token.getRepresentation();
                  if(DEBUG_INTERNAL_GLOBALS_CACHE){
!                     System.out.println("Adding choice:"+choice+" name:"+rep);
                      if(choice != which){
                          System.out.println("Looking for:"+which+"found:"+choice);
***************
*** 290,294 ****
                      }
                  }
!     		}
              //end cache
              
--- 312,316 ----
                      }
                  }
!             }
              //end cache
              
***************
*** 332,341 ****
       * @param n
       */
!     public SourceModule(String name, File f, SimpleNode n) {
          super(name);
          this.ast = n;
          this.file = f;
!         if(f != null)
              this.lastModified = f.lastModified();
      }
  
--- 354,365 ----
       * @param n
       */
!     public SourceModule(String name, File f, SimpleNode n, Throwable parseError) {
          super(name);
          this.ast = n;
          this.file = f;
!         this.parseError = parseError;
!         if(f != null){
              this.lastModified = f.lastModified();
+         }
      }
  
***************
*** 397,408 ****
                                      ClassDef classDef = (ClassDef) d.scope.getClassDef();
                                      if(classDef != null){
!                                     	FindDefinitionModelVisitor visitor = new FindDefinitionModelVisitor(actToks[actToks.length-1], d.line, d.col, d.module);
                                          try {
! 											classDef.accept(visitor);
! 										} catch (StopVisitingException e) {
! 											//expected exception
! 										}
                                          if(visitor.definitions.size() == 0){
!                                         	return EMPTY_ITOKEN_ARRAY;
                                          }
                                          d = visitor.definitions.get(0);
--- 421,432 ----
                                      ClassDef classDef = (ClassDef) d.scope.getClassDef();
                                      if(classDef != null){
!                                         FindDefinitionModelVisitor visitor = new FindDefinitionModelVisitor(actToks[actToks.length-1], d.line, d.col, d.module);
                                          try {
!                                             classDef.accept(visitor);
!                                         } catch (StopVisitingException e) {
!                                             //expected exception
!                                         }
                                          if(visitor.definitions.size() == 0){
!                                             return EMPTY_ITOKEN_ARRAY;
                                          }
                                          d = visitor.definitions.get(0);
***************
*** 412,428 ****
                                          }
                                      }else{
!                                     	if(d.module instanceof SourceModule){
!                                     		SourceModule m = (SourceModule) d.module;
!                                     		String joined = FullRepIterable.joinFirstParts(actToks);
                                              Definition[] definitions2 = m.findDefinition(initialState.getCopyWithActTok(joined), d.line, d.col, manager.getNature());
!                                     		if(definitions2.length == 0){
!                                     			return EMPTY_ITOKEN_ARRAY;
!                                     		}
!                                     		d = definitions2[0];
!                                     		value = d.value+"."+actToks[actToks.length-1];
!                                     		if(d instanceof AssignDefinition){
!                                     			return ((SourceModule)d.module).getValueCompletions(initialState, manager, value, d.module);
!                                     		}
!                                     	}
                                      }
                                      
--- 436,452 ----
                                          }
                                      }else{
!                                         if(d.module instanceof SourceModule){
!                                             SourceModule m = (SourceModule) d.module;
!                                             String joined = FullRepIterable.joinFirstParts(actToks);
                                              Definition[] definitions2 = m.findDefinition(initialState.getCopyWithActTok(joined), d.line, d.col, manager.getNature());
!                                             if(definitions2.length == 0){
!                                                 return EMPTY_ITOKEN_ARRAY;
!                                             }
!                                             d = definitions2[0];
!                                             value = d.value+"."+actToks[actToks.length-1];
!                                             if(d instanceof AssignDefinition){
!                                                 return ((SourceModule)d.module).getValueCompletions(initialState, manager, value, d.module);
!                                             }
!                                         }
                                      }
                                      
***************
*** 584,588 ****
       */
      @SuppressWarnings("unchecked")
! 	public Definition[] findDefinition(ICompletionState state, int line, int col, final IPythonNature nature) throws Exception{
          String rep = state.getActivationToken();
          //the line passed in starts at 1 and the lines for the visitor start at 0
--- 608,612 ----
       */
      @SuppressWarnings("unchecked")
!     public Definition[] findDefinition(ICompletionState state, int line, int col, final IPythonNature nature) throws Exception{
          String rep = state.getActivationToken();
          //the line passed in starts at 1 and the lines for the visitor start at 0
***************
*** 596,616 ****
          
          if(visitor.definitions.size() > 0){
!         	//ok, it is an assign, so, let's get it
  
              for (Iterator iter = visitor.definitions.iterator(); iter.hasNext();) {
! 	            Object next = iter.next();
                  if(next instanceof AssignDefinition){
                      AssignDefinition element = (AssignDefinition) next;
!     	            if(element.target.startsWith("self") == false){
!     		            if(element.scope.isOuterOrSameScope(scopeVisitor.scope) || element.foundAsGlobal){
!     		                toRet.add(element);
!     		            }
!     	            }else{
!     		            toRet.add(element);
!     	            }
                  }else{
                      toRet.add((Definition) next);
                  }
! 	        }
              if(toRet.size() > 0){
                  return (Definition[]) toRet.toArray(new Definition[0]);
--- 620,640 ----
          
          if(visitor.definitions.size() > 0){
!             //ok, it is an assign, so, let's get it
  
              for (Iterator iter = visitor.definitions.iterator(); iter.hasNext();) {
!                 Object next = iter.next();
                  if(next instanceof AssignDefinition){
                      AssignDefinition element = (AssignDefinition) next;
!                     if(element.target.startsWith("self") == false){
!                         if(element.scope.isOuterOrSameScope(scopeVisitor.scope) || element.foundAsGlobal){
!                             toRet.add(element);
!                         }
!                     }else{
!                         toRet.add(element);
!                     }
                  }else{
                      toRet.add((Definition) next);
                  }
!             }
              if(toRet.size() > 0){
                  return (Definition[]) toRet.toArray(new Definition[0]);
***************
*** 623,629 ****
          IToken[] localTokens = scopeVisitor.scope.getAllLocalTokens();
          for (IToken tok : localTokens) {
!         	if(tok.getRepresentation().equals(rep)){
!         		return new Definition[]{new Definition(tok, scopeVisitor.scope, this, true)};
!         	}
          }
          
--- 647,653 ----
          IToken[] localTokens = scopeVisitor.scope.getAllLocalTokens();
          for (IToken tok : localTokens) {
!             if(tok.getRepresentation().equals(rep)){
!                 return new Definition[]{new Definition(tok, scopeVisitor.scope, this, true)};
!             }
          }
          
***************
*** 632,636 ****
          ICodeCompletionASTManager astManager = nature.getAstManager();
          for (IToken tok : localImportedModules) {
!         	if(tok.getRepresentation().equals(rep)){
                  Tuple3<IModule, String, IToken> o = astManager.findOnImportedMods(new IToken[]{tok}, state.getCopyWithActTok(rep), this.getName());
                  if(o != null && o.o1 instanceof SourceModule){
--- 656,660 ----
          ICodeCompletionASTManager astManager = nature.getAstManager();
          for (IToken tok : localImportedModules) {
!             if(tok.getRepresentation().equals(rep)){
                  Tuple3<IModule, String, IToken> o = astManager.findOnImportedMods(new IToken[]{tok}, state.getCopyWithActTok(rep), this.getName());
                  if(o != null && o.o1 instanceof SourceModule){
***************
*** 641,693 ****
                  }
                  if(toRet.size() > 0){
!                 	return (Definition[]) toRet.toArray(new Definition[0]);
                  }
!         	}
          }
!         	
          
          //ok, not assign nor import, let's check if it is some self (we do not check for only 'self' because that would map to a
          //local (which has already been covered).
          if (rep.startsWith("self.")){
!         	//ok, it is some self, now, that is only valid if we are in some class definition
!         	ClassDef classDef = (ClassDef) scopeVisitor.scope.getClassDef();
!         	if(classDef != null){
!         		//ok, we are in a class, so, let's get the self completions
!         		String classRep = NodeUtils.getRepresentationString(classDef);
! 				IToken[] globalTokens = getGlobalTokens(
!         				new CompletionState(line-1, col-1, classRep, nature,"", state), //use the old state as the cache 
!         				astManager);
! 				
!         		String withoutSelf = rep.substring(5);
!         		for (IToken token : globalTokens) {
! 					if(token.getRepresentation().equals(withoutSelf)){
! 						String parentPackage = token.getParentPackage();
! 						IModule module = astManager.getModule(parentPackage, nature, true);
                          
! 						if(token instanceof SourceToken && (module != null || this.name.equals(parentPackage))){
                              if(module == null){
                                  module = this;
                              }
                              
! 			                SimpleNode ast2 = ((SourceToken)token).getAst();
! 							Tuple<Integer, Integer> def = getLineColForDefinition(ast2);
! 							FastStack<SimpleNode> stack = new FastStack<SimpleNode>();
                              if(module instanceof SourceModule){
                                  stack.push(((SourceModule)module).getAst());
                              }
! 							stack.push(classDef);
! 							ILocalScope scope = new LocalScope(stack);
! 							return new Definition[]{new Definition(def.o1, def.o2, token.getRepresentation(), ast2, scope, module)};
                              
! 						}else{
! 							return new Definition[0];
! 						}
! 					}
! 				}
!         	}
          }
          
!         	
!     	//ok, it is not an assign, so, let's search the global tokens (and imports)
          String tok = rep;
          SourceModule mod = this;
--- 665,717 ----
                  }
                  if(toRet.size() > 0){
!                     return (Definition[]) toRet.toArray(new Definition[0]);
                  }
!             }
          }
!             
          
          //ok, not assign nor import, let's check if it is some self (we do not check for only 'self' because that would map to a
          //local (which has already been covered).
          if (rep.startsWith("self.")){
!             //ok, it is some self, now, that is only valid if we are in some class definition
!             ClassDef classDef = (ClassDef) scopeVisitor.scope.getClassDef();
!             if(classDef != null){
!                 //ok, we are in a class, so, let's get the self completions
!                 String classRep = NodeUtils.getRepresentationString(classDef);
!                 IToken[] globalTokens = getGlobalTokens(
!                         new CompletionState(line-1, col-1, classRep, nature,"", state), //use the old state as the cache 
!                         astManager);
!                 
!                 String withoutSelf = rep.substring(5);
!                 for (IToken token : globalTokens) {
!                     if(token.getRepresentation().equals(withoutSelf)){
!                         String parentPackage = token.getParentPackage();
!                         IModule module = astManager.getModule(parentPackage, nature, true);
                          
!                         if(token instanceof SourceToken && (module != null || this.name.equals(parentPackage))){
                              if(module == null){
                                  module = this;
                              }
                              
!                             SimpleNode ast2 = ((SourceToken)token).getAst();
!                             Tuple<Integer, Integer> def = getLineColForDefinition(ast2);
!                             FastStack<SimpleNode> stack = new FastStack<SimpleNode>();
                              if(module instanceof SourceModule){
                                  stack.push(((SourceModule)module).getAst());
                              }
!                             stack.push(classDef);
!                             ILocalScope scope = new LocalScope(stack);
!                             return new Definition[]{new Definition(def.o1, def.o2, token.getRepresentation(), ast2, scope, module)};
                              
!                         }else{
!                             return new Definition[0];
!                         }
!                     }
!                 }
!             }
          }
          
!             
!         //ok, it is not an assign, so, let's search the global tokens (and imports)
          String tok = rep;
          SourceModule mod = this;
***************
*** 706,710 ****
                      return new Definition[]{new Definition(1,1,"",null,null,o.o1)};
                  }else{
!                 	state.checkFindDefinitionMemory(o.o1, tok);
                      return (Definition[]) o.o1.findDefinition(state.getCopyWithActTok(tok), -1, -1, nature);
                  }
--- 730,734 ----
                      return new Definition[]{new Definition(1,1,"",null,null,o.o1)};
                  }else{
!                     state.checkFindDefinitionMemory(o.o1, tok);
                      return (Definition[]) o.o1.findDefinition(state.getCopyWithActTok(tok), -1, -1, nature);
                  }
***************
*** 724,731 ****
          copy.setActivationToken(tok);
          try{
!         	state.checkFindDefinitionMemory(mod, tok);
!         	findDefinitionsFromModAndTok(nature, toRet, visitor.moduleImported, mod, copy);
          }catch(CompletionRecursionException e){
!         	//ignore (will return what we've got so far)
  //            e.printStackTrace();
          }
--- 748,755 ----
          copy.setActivationToken(tok);
          try{
!             state.checkFindDefinitionMemory(mod, tok);
!             findDefinitionsFromModAndTok(nature, toRet, visitor.moduleImported, mod, copy);
          }catch(CompletionRecursionException e){
!             //ignore (will return what we've got so far)
  //            e.printStackTrace();
          }
***************
*** 738,780 ****
       * @throws Exception 
       */
! 	private void findDefinitionsFromModAndTok(IPythonNature nature, ArrayList<Definition> toRet, String moduleImported, SourceModule mod, ICompletionState state) throws Exception {
          String tok = state.getActivationToken();
! 		if(tok != null){
!         	if(tok.length() > 0){
! 	            Definition d = mod.findGlobalTokDef(state.getCopyWithActTok(tok), nature);
! 				if(d != null){
! 	                toRet.add(d);
! 	                
! 	            }else if(moduleImported != null){
! 	            	//if it was found as some import (and is already stored as a dotted name), we must check for
! 	            	//multiple representations in the absolute form:
! 	            	//as a relative import
! 	            	//as absolute import
! 	            	getModuleDefinition(nature, toRet, mod, moduleImported);
! 	            }
  
!         	}else{
!         		//we found it, but it is an empty tok (which means that what we found is the actual module).
!         		toRet.add(new Definition(1,1,"",null,null,mod)); 
!         	}
          }
! 	}
  
! 	private IDefinition getModuleDefinition(IPythonNature nature, ArrayList<Definition> toRet, SourceModule mod, String moduleImported) {
! 		String rel = AbstractToken.makeRelative(mod.getName(), moduleImported);
! 		IModule modFound = nature.getAstManager().getModule(rel, nature, false);
! 		if(modFound == null){
! 			modFound = nature.getAstManager().getModule(moduleImported, nature, false);
! 		}
! 		if(modFound != null){
! 			//ok, found it
! 			Definition definition = new Definition(1,1,"", null, null, modFound);
! 			if(toRet != null){
! 				toRet.add(definition);
! 			}
! 			return definition;
! 		}
! 		return null;
! 	}
  
  
--- 762,804 ----
       * @throws Exception 
       */
!     private void findDefinitionsFromModAndTok(IPythonNature nature, ArrayList<Definition> toRet, String moduleImported, SourceModule mod, ICompletionState state) throws Exception {
          String tok = state.getActivationToken();
!         if(tok != null){
!             if(tok.length() > 0){
!                 Definition d = mod.findGlobalTokDef(state.getCopyWithActTok(tok), nature);
!                 if(d != null){
!                     toRet.add(d);
!                     
!                 }else if(moduleImported != null){
!                     //if it was found as some import (and is already stored as a dotted name), we must check for
!                     //multiple representations in the absolute form:
!                     //as a relative import
!                     //as absolute import
!                     getModuleDefinition(nature, toRet, mod, moduleImported);
!                 }
  
!             }else{
!                 //we found it, but it is an empty tok (which means that what we found is the actual module).
!                 toRet.add(new Definition(1,1,"",null,null,mod)); 
!             }
          }
!     }
  
!     private IDefinition getModuleDefinition(IPythonNature nature, ArrayList<Definition> toRet, SourceModule mod, String moduleImported) {
!         String rel = AbstractToken.makeRelative(mod.getName(), moduleImported);
!         IModule modFound = nature.getAstManager().getModule(rel, nature, false);
!         if(modFound == null){
!             modFound = nature.getAstManager().getModule(moduleImported, nature, false);
!         }
!         if(modFound != null){
!             //ok, found it
!             Definition definition = new Definition(1,1,"", null, null, modFound);
!             if(toRet != null){
!                 toRet.add(definition);
!             }
!             return definition;
!         }
!         return null;
!     }
  
  
***************
*** 787,800 ****
      public Definition findGlobalTokDef(ICompletionState state, IPythonNature nature) throws Exception {
          String tok = state.getActivationToken();
!     	String[] headAndTail = FullRepIterable.headAndTail(tok);
!     	String firstPart = headAndTail[0];
!     	String rep = headAndTail[1];
!     	
!     	IToken[] tokens = null;
!     	if(nature != null){
!     		tokens = nature.getAstManager().getCompletionsForModule(this, state.getCopyWithActTok(firstPart), true);
!     	}else{
!     		tokens = getGlobalTokens();
!     	}
          for (IToken token : tokens) {
              boolean sameRep = token.getRepresentation().equals(rep);
--- 811,824 ----
      public Definition findGlobalTokDef(ICompletionState state, IPythonNature nature) throws Exception {
          String tok = state.getActivationToken();
!         String[] headAndTail = FullRepIterable.headAndTail(tok);
!         String firstPart = headAndTail[0];
!         String rep = headAndTail[1];
!         
!         IToken[] tokens = null;
!         if(nature != null){
!             tokens = nature.getAstManager().getCompletionsForModule(this, state.getCopyWithActTok(firstPart), true);
!         }else{
!             tokens = getGlobalTokens();
!         }
          for (IToken token : tokens) {
              boolean sameRep = token.getRepresentation().equals(rep);
***************
*** 806,810 ****
                          continue; 
                      }
!                 	//ok, we found it
                      SimpleNode a = ((SourceToken)token).getAst();
                      Tuple<Integer, Integer> def = getLineColForDefinition(a);
--- 830,834 ----
                          continue; 
                      }
!                     //ok, we found it
                      SimpleNode a = ((SourceToken)token).getAst();
                      Tuple<Integer, Integer> def = getLineColForDefinition(a);
***************
*** 813,820 ****
                      IModule module = this;
                      if(nature != null){
!                     	IModule mod = nature.getAstManager().getModule(parentPackage, nature, true);
!                     	if(mod != null){
!                     		module = mod;
!                     	}
                      }
                      
--- 837,844 ----
                      IModule module = this;
                      if(nature != null){
!                         IModule mod = nature.getAstManager().getModule(parentPackage, nature, true);
!                         if(mod != null){
!                             module = mod;
!                         }
                      }
                      
***************
*** 830,846 ****
                      }
                  }else if(token instanceof ConcreteToken){
!                 	//a contrete token represents a module
!                 	String modName = token.getParentPackage();
!                 	if(modName.length() > 0){
!                 		modName+= ".";
!                 	}
!                 	modName += token.getRepresentation();
!                 	IModule module = nature.getAstManager().getModule(modName, nature, true);
!                 	if(module == null){
!                 		return null;
!                 	}else{
!                 		return new Definition(0+1, 0+1, "", null, null, module); // it is the module itself
!                 	}
!                 	
                  }else if(token instanceof CompiledToken){
                      String parentPackage = token.getParentPackage();
--- 854,870 ----
                      }
                  }else if(token instanceof ConcreteToken){
!                     //a contrete token represents a module
!                     String modName = token.getParentPackage();
!                     if(modName.length() > 0){
!                         modName+= ".";
!                     }
!                     modName += token.getRepresentation();
!                     IModule module = nature.getAstManager().getModule(modName, nature, true);
!                     if(module == null){
!                         return null;
!                     }else{
!                         return new Definition(0+1, 0+1, "", null, null, module); // it is the module itself
!                     }
!                     
                  }else if(token instanceof CompiledToken){
                      String parentPackage = token.getParentPackage();
***************
*** 874,878 ****
                      }
                  }else{
!                 	throw new RuntimeException("Unexpected token:"+token.getClass());
                  }
              }
--- 898,902 ----
                      }
                  }else{
!                     throw new RuntimeException("Unexpected token:"+token.getClass());
                  }
              }
***************
*** 883,901 ****
      
      public Tuple<Integer, Integer> getLineColForDefinition(SimpleNode a){
!     	int line = a.beginLine;
!     	int col = a.beginColumn;
!     	
!     	if(a instanceof ClassDef){
!     		ClassDef c = (ClassDef)a;
!     		line = c.name.beginLine;
!     		col = c.name.beginColumn;
!     		
!     	} else if(a instanceof FunctionDef){
!     		FunctionDef c = (FunctionDef)a;
!     		line = c.name.beginLine;
!     		col = c.name.beginColumn;
!     	}
!     	
!     	return new Tuple<Integer, Integer>(line,col);
      }
      
--- 907,925 ----
      
      public Tuple<Integer, Integer> getLineColForDefinition(SimpleNode a){
!         int line = a.beginLine;
!         int col = a.beginColumn;
!         
!         if(a instanceof ClassDef){
!             ClassDef c = (ClassDef)a;
!             line = c.name.beginLine;
!             col = c.name.beginColumn;
!             
!         } else if(a instanceof FunctionDef){
!             FunctionDef c = (FunctionDef)a;
!             line = c.name.beginLine;
!             col = c.name.beginColumn;
!         }
!         
!         return new Tuple<Integer, Integer>(line,col);
      }
      
***************
*** 907,915 ****
          try {
              if(scope == null){
!     	        FindScopeVisitor scopeVisitor = getScopeVisitor(line, col);
                  scope = scopeVisitor.scope;
              }
! 	        
! 	        return scope.getLocalTokens(line, col, false);
          } catch (Exception e) {
              e.printStackTrace();
--- 931,939 ----
          try {
              if(scope == null){
!                 FindScopeVisitor scopeVisitor = getScopeVisitor(line, col);
                  scope = scopeVisitor.scope;
              }
!             
!             return scope.getLocalTokens(line, col, false);
          } catch (Exception e) {
              e.printStackTrace();
***************
*** 954,959 ****
          try {
              FindScopeVisitor scopeVisitor = getScopeVisitor(node.beginLine, node.beginColumn);
! 	        
! 	        return scopeVisitor.scope.getScopeEndLine();
          } catch (Exception e) {
              e.printStackTrace();
--- 978,983 ----
          try {
              FindScopeVisitor scopeVisitor = getScopeVisitor(node.beginLine, node.beginColumn);
!             
!             return scopeVisitor.scope.getScopeEndLine();
          } catch (Exception e) {
              e.printStackTrace();
***************
*** 968,973 ****
          try {
              FindScopeVisitor scopeVisitor = getScopeVisitor(-1, -1);
! 	        
! 	        return scopeVisitor.scope.getIfMainLine();
          } catch (Exception e) {
              e.printStackTrace();
--- 992,997 ----
          try {
              FindScopeVisitor scopeVisitor = getScopeVisitor(-1, -1);
!             
!             return scopeVisitor.scope.getIfMainLine();
          } catch (Exception e) {
              e.printStackTrace();
***************
*** 995,1006 ****
      }
  
! 	public void setName(String n) {
! 		this.name = n;
! 	}
  
! 	/**
! 	 * @return true if this is a bootstrap module (i.e.: a module that's only used to load a compiled module with the
! 	 * same name -- that used in eggs)
! 	 * 
       * A bootstrapped module is the way that egg handles pyd files: 
       * it'll create a file with the same name of the dll (e.g.:
--- 1019,1030 ----
      }
  
!     public void setName(String n) {
!         this.name = n;
!     }
  
!     /**
!      * @return true if this is a bootstrap module (i.e.: a module that's only used to load a compiled module with the
!      * same name -- that used in eggs)
!      * 
       * A bootstrapped module is the way that egg handles pyd files: 
       * it'll create a file with the same name of the dll (e.g.:
***************
*** 1017,1021 ****
       * __bootstrap__()
       * 
! 	 */
      public boolean isBootstrapModule() {
          if(bootstrap == null){
--- 1041,1045 ----
       * __bootstrap__()
       * 
!      */
      public boolean isBootstrapModule() {
          if(bootstrap == null){

Index: AbstractModule.java
===================================================================
RCS file: /cvsroot/pydev/org.python.pydev/src_completions/org/python/pydev/editor/codecompletion/revisited/modules/AbstractModule.java,v
retrieving revision 1.15
retrieving revision 1.16
diff -C2 -d -r1.15 -r1.16
*** AbstractModule.java	2 May 2008 13:42:01 -0000	1.15
--- AbstractModule.java	28 Sep 2008 12:45:40 -0000	1.16
***************
*** 38,49 ****
  public abstract class AbstractModule implements IModule {
  
! 	private static final IToken[] EMPTY_TOKEN_ARRAY = new IToken[0];
      
      /**
! 	 * May be changed for tests
! 	 */
      public static String MODULE_NAME_WHEN_FILE_IS_UNDEFINED = "";
  
! 	/** 
       * @see org.python.pydev.core.IModule#getWildImportedModules()
       */
--- 38,49 ----
  public abstract class AbstractModule implements IModule {
  
!     private static final IToken[] EMPTY_TOKEN_ARRAY = new IToken[0];
      
      /**
!      * May be changed for tests
!      */
      public static String MODULE_NAME_WHEN_FILE_IS_UNDEFINED = "";
  
!     /** 
       * @see org.python.pydev.core.IModule#getWildImportedModules()
       */
***************
*** 91,95 ****
       */
      public boolean isInGlobalTokens(String tok, IPythonNature nature, ICompletionCache completionCache) throws CompletionRecursionException{
!     	return isInGlobalTokens(tok, nature, true, completionCache);
      }
      
--- 91,95 ----
       */
      public boolean isInGlobalTokens(String tok, IPythonNature nature, ICompletionCache completionCache) throws CompletionRecursionException{
!         return isInGlobalTokens(tok, nature, true, completionCache);
      }
      
***************
*** 109,115 ****
          //it's worth checking it if it is not dotted... (much faster as it's in a map already)
          if(tok.indexOf(".") == -1){
!         	if(isInDirectGlobalTokens(tok, completionCache)){
!         		return IModule.FOUND_TOKEN;
!         	}
          }
          
--- 109,115 ----
          //it's worth checking it if it is not dotted... (much faster as it's in a map already)
          if(tok.indexOf(".") == -1){
!             if(isInDirectGlobalTokens(tok, completionCache)){
!                 return IModule.FOUND_TOKEN;
!             }
          }
          
***************
*** 231,240 ****
          String path = REF.getFileAbsolutePath(f);
          if(PythonPathHelper.isValidFileMod(path)){
! 	        if(PythonPathHelper.isValidSourceFile(path)){
                  return createModuleFromDoc(name, f, REF.getDocFromFile(f), nature, currLine);
! 	
! 	        }else{ //this should be a compiled extension... we have to get completions from the python shell.
! 	            return new CompiledModule(name, nature.getAstManager());
! 	        }
          }
          
--- 231,240 ----
          String path = REF.getFileAbsolutePath(f);
          if(PythonPathHelper.isValidFileMod(path)){
!             if(PythonPathHelper.isValidSourceFile(path)){
                  return createModuleFromDoc(name, f, REF.getDocFromFile(f), nature, currLine);
!     
!             }else{ //this should be a compiled extension... we have to get completions from the python shell.
!                 return new CompiledModule(name, nature.getAstManager());
!             }
          }
          
***************
*** 255,265 ****
          
          if(f != null){
! 	        if(!checkForPath || PythonPathHelper.isValidSourceFile(REF.getFileAbsolutePath(f))){
!                 Tuple<SimpleNode, Throwable> obj = PyParser.reparseDocument(new PyParser.ParserInfo(doc, true, nature, currLine));
! 		        return new SourceModule(name, f, obj.o1);
! 	        }
          } else {
!             Tuple<SimpleNode, Throwable> obj = PyParser.reparseDocument(new PyParser.ParserInfo(doc, true, nature, currLine));
! 	        return new SourceModule(name, f, obj.o1);
          }
          return null;
--- 255,265 ----
          
          if(f != null){
!             if(!checkForPath || PythonPathHelper.isValidSourceFile(REF.getFileAbsolutePath(f))){
!                 Tuple<SimpleNode, Throwable> obj = PyParser.reparseDocument(new PyParser.ParserInfo(doc, true, nature, currLine, name, f));
!                 return new SourceModule(name, f, obj.o1, obj.o2);
!             }
          } else {
!             Tuple<SimpleNode, Throwable> obj = PyParser.reparseDocument(new PyParser.ParserInfo(doc, true, nature, currLine, name, f));
!             return new SourceModule(name, f, obj.o1, obj.o2);
          }
          return null;
***************
*** 269,283 ****
       * This function creates a module and resolves the module name (use this function if only the file is available).
       */
! 	public static IModule createModuleFromDoc(File file, IDocument doc, IPythonNature pythonNature, int line, IModulesManager projModulesManager) {
! 		String moduleName = null;
! 	    if(file != null){
! 			moduleName = projModulesManager.resolveModule(REF.getFileAbsolutePath(file));
! 	    }
! 	    if(moduleName == null){
! 	        moduleName = MODULE_NAME_WHEN_FILE_IS_UNDEFINED;
! 	    }
! 		IModule module = createModuleFromDoc(moduleName, file, doc, pythonNature, line, false);
! 	    return module;
! 	}
  
  
--- 269,283 ----
       * This function creates a module and resolves the module name (use this function if only the file is available).
       */
!     public static IModule createModuleFromDoc(File file, IDocument doc, IPythonNature pythonNature, int line, IModulesManager projModulesManager) {
!         String moduleName = null;
!         if(file != null){
!             moduleName = projModulesManager.resolveModule(REF.getFileAbsolutePath(file));
!         }
!         if(moduleName == null){
!             moduleName = MODULE_NAME_WHEN_FILE_IS_UNDEFINED;
!         }
!         IModule module = createModuleFromDoc(moduleName, file, doc, pythonNature, line, false);
!         return module;
!     }
  
  
***************
*** 288,292 ****
       */
      public static IModule createModule(SimpleNode n) {
!         return new SourceModule(null, null, n);
      }
      
--- 288,292 ----
       */
      public static IModule createModule(SimpleNode n) {
!         return new SourceModule(null, null, n, null);
      }
      
***************
*** 301,305 ****
       */
      public static IModule createModule(SimpleNode n, File file, String moduleName) {
!         return new SourceModule(moduleName, file, n);
      }
      
--- 301,305 ----
       */
      public static IModule createModule(SimpleNode n, File file, String moduleName) {
!         return new SourceModule(moduleName, file, n, null);
      }
      
***************
*** 314,325 ****
       */
      public static IModule createModule(SimpleNode n, File file, IModulesManager projModulesManager) {
! 		String moduleName = null;
! 	    if(file != null){
! 			moduleName = projModulesManager.resolveModule(REF.getFileAbsolutePath(file));
! 	    }
! 	    if(moduleName == null){
! 	        moduleName = MODULE_NAME_WHEN_FILE_IS_UNDEFINED;
! 	    }
!     	return createModule(n, file, moduleName);
      }
      
--- 314,325 ----
       */
      public static IModule createModule(SimpleNode n, File file, IModulesManager projModulesManager) {
!         String moduleName = null;
!         if(file != null){
!             moduleName = projModulesManager.resolveModule(REF.getFileAbsolutePath(file));
!         }
!         if(moduleName == null){
!             moduleName = MODULE_NAME_WHEN_FILE_IS_UNDEFINED;
!         }
!         return createModule(n, file, moduleName);
      }
      
***************
*** 346,352 ****
      @Override
      public String toString() {
!     	String n2 = this.getClass().getName();
!     	String n = n2.substring(n2.lastIndexOf('.')+1);
! 		return this.getName()+" ("+n+")";
      }
      
--- 346,352 ----
      @Override
      public String toString() {
!         String n2 = this.getClass().getName();
!         String n = n2.substring(n2.lastIndexOf('.')+1);
!         return this.getName()+" ("+n+")";
      }
      

Index: SourceToken.java
===================================================================
RCS file: /cvsroot/pydev/org.python.pydev/src_completions/org/python/pydev/editor/codecompletion/revisited/modules/SourceToken.java,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** SourceToken.java	1 Oct 2007 00:26:54 -0000	1.3
--- SourceToken.java	28 Sep 2008 12:45:40 -0000	1.4
***************
*** 127,157 ****
      int[] colLineEndComplete;
      public int getLineEnd(boolean getOnlyToFirstDot){
!     	if(getOnlyToFirstDot){
!     		if(colLineEndToFirstDot == null){
!     			colLineEndToFirstDot = NodeUtils.getColLineEnd(getRepresentationNode(), getOnlyToFirstDot);
!     		}
!     		return colLineEndToFirstDot[0];
!     		
!     	}else{
!     		if(colLineEndComplete == null){
!     			colLineEndComplete = NodeUtils.getColLineEnd(getRepresentationNode(), getOnlyToFirstDot);
!     		}
!     		return colLineEndComplete[0];
!     	}
      }
      
      public int getColEnd(boolean getOnlyToFirstDot){
!     	if(getOnlyToFirstDot){
!     		if(colLineEndToFirstDot == null){
!     			colLineEndToFirstDot = NodeUtils.getColLineEnd(getRepresentationNode(), getOnlyToFirstDot);
!     		}
!     		return colLineEndToFirstDot[1];
!     		
!     	}else{
!     		if(colLineEndComplete == null){
!     			colLineEndComplete = NodeUtils.getColLineEnd(getRepresentationNode(), getOnlyToFirstDot);
!     		}
!     		return colLineEndComplete[1];
!     	}
      }
      
--- 127,157 ----
      int[] colLineEndComplete;
      public int getLineEnd(boolean getOnlyToFirstDot){
!         if(getOnlyToFirstDot){
!             if(colLineEndToFirstDot == null){
!                 colLineEndToFirstDot = NodeUtils.getColLineEnd(getRepresentationNode(), getOnlyToFirstDot);
!             }
!             return colLineEndToFirstDot[0];
!             
!         }else{
!             if(colLineEndComplete == null){
!                 colLineEndComplete = NodeUtils.getColLineEnd(getRepresentationNode(), getOnlyToFirstDot);
!             }
!             return colLineEndComplete[0];
!         }
      }
      
      public int getColEnd(boolean getOnlyToFirstDot){
!         if(getOnlyToFirstDot){
!             if(colLineEndToFirstDot == null){
!                 colLineEndToFirstDot = NodeUtils.getColLineEnd(getRepresentationNode(), getOnlyToFirstDot);
!             }
!             return colLineEndToFirstDot[1];
!             
!         }else{
!             if(colLineEndComplete == null){
!                 colLineEndComplete = NodeUtils.getColLineEnd(getRepresentationNode(), getOnlyToFirstDot);
!             }
!             return colLineEndComplete[1];
!         }
      }
      
***************
*** 187,195 ****
      
      public boolean isImportFrom() {
!     	return ast instanceof ImportFrom;
      }
  
      public boolean isWildImport() {
!     	return AbstractVisitor.isWildImport(ast);
      }
      
--- 187,195 ----
      
      public boolean isImportFrom() {
!         return ast instanceof ImportFrom;
      }
  
      public boolean isWildImport() {
!         return AbstractVisitor.isWildImport(ast);
      }
      
***************
*** 202,210 ****
       */
      public int[] getLineColEnd() {
!     	if(ast instanceof NameTok || ast instanceof Name){
!     		//those are the ones that we can be certain of...
!     		return new int[]{getLineDefinition(), getColDefinition()+getRepresentation().length()};
!     	}
!     	throw new RuntimeException("Unable to get the lenght of the token:"+ast.getClass().getName());
      }
  
--- 202,210 ----
       */
      public int[] getLineColEnd() {
!         if(ast instanceof NameTok || ast instanceof Name){
!             //those are the ones that we can be certain of...
!             return new int[]{getLineDefinition(), getColDefinition()+getRepresentation().length()};
!         }
!         throw new RuntimeException("Unable to get the lenght of the token:"+ast.getClass().getName());
      }
  

Index: CompiledModule.java
===================================================================
RCS file: /cvsroot/pydev/org.python.pydev/src_completions/org/python/pydev/editor/codecompletion/revisited/modules/CompiledModule.java,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** CompiledModule.java	4 May 2008 16:48:46 -0000	1.8
--- CompiledModule.java	28 Sep 2008 12:45:40 -0000	1.9
***************
*** 61,65 ****
      @Override
      public File getFile() {
!     	return file;
      }
  
--- 61,65 ----
      @Override
      public File getFile() {
!         return file;
      }
  
***************
*** 80,107 ****
          super(name);
          if(COMPILED_MODULES_ENABLED){
! 	        try {
! 	            setTokens(name, manager);
! 	        } catch (Exception e) {
! 	        	//ok, something went wrong... let's give it another shot...
! 	        	synchronized (this) {
! 	        		try {
! 						wait(10);
! 					} catch (InterruptedException e1) {
! 						//empty block
! 					} //just wait a little before a retry...
! 				}
! 				
! 	        	try {
! 	        		AbstractShell shell = AbstractShell.getServerShell(manager.getNature(), AbstractShell.COMPLETION_SHELL);
! 	        		synchronized(shell){
! 	        			shell.clearSocket();
! 	        		}
! 					setTokens(name, manager);
! 				} catch (Exception e2) {
! 					tokens = new HashMap<String, CompiledToken>();
! 					e2.printStackTrace();
! 					PydevPlugin.log(e2);
! 				}
! 	        }
              if(tokens != null && tokens.size() > 0){
                  List<IModulesObserver> participants = ExtensionHelper.getParticipants(ExtensionHelper.PYDEV_MODULES_OBSERVER);
--- 80,107 ----
          super(name);
          if(COMPILED_MODULES_ENABLED){
!             try {
!                 setTokens(name, manager);
!             } catch (Exception e) {
!                 //ok, something went wrong... let's give it another shot...
!                 synchronized (this) {
!                     try {
!                         wait(10);
!                     } catch (InterruptedException e1) {
!                         //empty block
!                     } //just wait a little before a retry...
!                 }
!                 
!                 try {
!                     AbstractShell shell = AbstractShell.getServerShell(manager.getNature(), AbstractShell.COMPLETION_SHELL);
!                     synchronized(shell){
!                         shell.clearSocket();
!                     }
!                     setTokens(name, manager);
!                 } catch (Exception e2) {
!                     tokens = new HashMap<String, CompiledToken>();
!                     e2.printStackTrace();
!                     PydevPlugin.log(e2);
!                 }
!             }
              if(tokens != null && tokens.size() > 0){
                  List<IModulesObserver> participants = ExtensionHelper.getParticipants(ExtensionHelper.PYDEV_MODULES_OBSERVER);
***************
*** 117,126 ****
      }
  
! 	private void setTokens(String name, ICodeCompletionASTManager manager) throws IOException, Exception, CoreException {
! 		if(TRACE_COMPILED_MODULES){
! 			PydevPlugin.log(IStatus.INFO, "Compiled modules: getting info for:"+name, null);
! 		}
! 		AbstractShell shell = AbstractShell.getServerShell(manager.getNature(), AbstractShell.COMPLETION_SHELL);
! 		synchronized(shell){
              Tuple<String, List<String[]>> completions = shell.getImportCompletions(name, manager.getModulesManager().getCompletePythonPath(null)); //default
              String fPath = completions.o1;
--- 117,126 ----
      }
  
!     private void setTokens(String name, ICodeCompletionASTManager manager) throws IOException, Exception, CoreException {
!         if(TRACE_COMPILED_MODULES){
!             PydevPlugin.log(IStatus.INFO, "Compiled modules: getting info for:"+name, null);
!         }
!         AbstractShell shell = AbstractShell.getServerShell(manager.getNature(), AbstractShell.COMPLETION_SHELL);
!         synchronized(shell){
              Tuple<String, List<String[]>> completions = shell.getImportCompletions(name, manager.getModulesManager().getCompletePythonPath(null)); //default
              String fPath = completions.o1;
***************
*** 137,188 ****
                  }
              }
! 		    ArrayList<IToken> array = new ArrayList<IToken>();
! 		    
! 		    for (Iterator iter = completions.o2.iterator(); iter.hasNext();) {
! 		        String[] element = (String[]) iter.next();
! 		        //let's make this less error-prone.
! 		        try {
! 		            String o1 = element[0]; //this one is really, really needed
! 		            String o2 = "";
! 		            String o3 = "";
! 		            String o4;
! 		            
! 		            if(element.length > 0)
! 		                o2 = element[1];
! 		            
! 		            if(element.length > 0)
! 		                o3 = element[2];
! 		            
! 		            if(element.length > 0)
! 		                o4 = element[3];
! 		            else
! 		                o4 = ""+IToken.TYPE_BUILTIN;
! 		            
! 		            IToken t = new CompiledToken(o1, o2, o3, name, Integer.parseInt(o4));
! 		            array.add(t);
! 		        } catch (Exception e) {
! 		            String received = "";
! 		            for (int i = 0; i < element.length; i++) {
! 		                received += element[i];
! 		                received += "  ";
! 		            }
! 		            
! 		            PydevPlugin.log(IStatus.ERROR, "Error getting completions for compiled module "+name+" received = '"+received+"'", e);
! 		        }
! 		    }
! 		    
! 		    //as we will use it for code completion on sources that map to modules, the __file__ should also
! 		    //be added...
! 		    if(array.size() > 0 && name.equals("__builtin__")){
! 		        array.add(new CompiledToken("__file__","","",name,IToken.TYPE_BUILTIN));
! 		        array.add(new CompiledToken("__builtins__","","",name,IToken.TYPE_BUILTIN));
! 		    }
! 		    
! 		    this.tokens = new HashMap<String, CompiledToken>();
! 		    for (IToken token : array) {
! 		    	this.tokens.put(token.getRepresentation(), (CompiledToken) token);
! 			} 
! 		}
! 	}
      
      /**
--- 137,188 ----
                  }
              }
!             ArrayList<IToken> array = new ArrayList<IToken>();
!             
!             for (Iterator iter = completions.o2.iterator(); iter.hasNext();) {
!                 String[] element = (String[]) iter.next();
!                 //let's make this less error-prone.
!                 try {
!                     String o1 = element[0]; //this one is really, really needed
!                     String o2 = "";
!                     String o3 = "";
!                     String o4;
!                     
!                     if(element.length > 0)
!                         o2 = element[1];
!                     
!                     if(element.length > 0)
!                         o3 = element[2];
!                     
!                     if(element.length > 0)
!                         o4 = element[3];
!                     else
!                         o4 = ""+IToken.TYPE_BUILTIN;
!                     
!                     IToken t = new CompiledToken(o1, o2, o3, name, Integer.parseInt(o4));
!                     array.add(t);
!                 } catch (Exception e) {
!                     String received = "";
!                     for (int i = 0; i < element.length; i++) {
!                         received += element[i];
!                         received += "  ";
!                     }
!                     
!                     PydevPlugin.log(IStatus.ERROR, "Error getting completions for compiled module "+name+" received = '"+received+"'", e);
!                 }
!             }
!             
!             //as we will use it for code completion on sources that map to modules, the __file__ should also
!             //be added...
!             if(array.size() > 0 && name.equals("__builtin__")){
!                 array.add(new CompiledToken("__file__","","",name,IToken.TYPE_BUILTIN));
!                 array.add(new CompiledToken("__builtins__","","",name,IToken.TYPE_BUILTIN));
!             }
!             
!             this.tokens = new HashMap<String, CompiledToken>();
!             for (IToken token : array) {
!                 this.tokens.put(token.getRepresentation(), (CompiledToken) token);
!             } 
!         }
!     }
      
      /**
***************
*** 206,215 ****
       */
      public IToken[] getGlobalTokens() {
!     	if(tokens == null){
!     		return new IToken[0];
!     	}
!     	
          Collection<CompiledToken> values = tokens.values();
! 		return values.toArray(new IToken[values.size()]);
      }
  
--- 206,215 ----
       */
      public IToken[] getGlobalTokens() {
!         if(tokens == null){
!             return new IToken[0];
!         }
!         
          Collection<CompiledToken> values = tokens.values();
!         return values.toArray(new IToken[values.size()]);
      }
  
***************
*** 225,231 ****
       */
      public IToken[] getGlobalTokens(ICompletionState state, ICodeCompletionASTManager manager) {
!     	Map<String, IToken> v = cache.get(state.getActivationToken());
          if(v != null){
!         	Collection<IToken> values = v.values();
              return values.toArray(new IToken[values.size()]);
          }
--- 225,231 ----
       */
      public IToken[] getGlobalTokens(ICompletionState state, ICodeCompletionASTManager manager) {
!         Map<String, IToken> v = cache.get(state.getActivationToken());
          if(v != null){
!             Collection<IToken> values = v.values();
              return values.toArray(new IToken[values.size()]);
          }
***************
*** 234,265 ****
  
          if(COMPILED_MODULES_ENABLED){
! 	        try {
! 	            AbstractShell shell = AbstractShell.getServerShell(manager.getNature(), AbstractShell.COMPLETION_SHELL);
! 	            synchronized(shell){
! 		            String act = name+"."+state.getActivationToken();
                      List<String[]> completions = shell.getImportCompletions(act, manager.getModulesManager().getCompletePythonPath(null)).o2;//default
! 		            
! 		            ArrayList<IToken> array = new ArrayList<IToken>();
! 		            
! 		            for (Iterator iter = completions.iterator(); iter.hasNext();) {
! 		                ...
 
[truncated message content]