Notepad++ Python Script / Discussion / Script Showcase: Auto Detect File Language (Shebang &...

Still a work in progress, but I've begun writing a script to auto-detect the language of a file when Notepad++ is unable to by default. (Extension-less files, etc) At the moment, it has a few simple detections, all of which are configurable in the config file generated at config\py_autolang.cfg. I'll start putting together some documentation for the different options, but for now, you can learn more about each option by checking the method LanguageAutoDetector.__default_config. So far, it does the following detections:
Filename matching - Files named configure, for instance are automatically set to the bash language by default.
Partial filename matching - By default, Makefile\..+ (so Makefile.win32 for instance) will have their language set to makefile.
Shebang - Checks the first line for the familiar #!/usr/bin/env or #!/usr/bin/. It will match windows paths, linux paths, or even situations where there is no path, such as #!python.exe
Try_Shebang - If a premapped shebang does not exist, try using the program name as the language.
Contains_String - If a file contains a regular expression, set the language accordingly.
I still need to write code for some of the configuration values, such as the file_load_event options. (Right now, you'll need to run the script to add the callback, and then it will automatically try to detect on any file load. I'll change that soon to reflect the options set) I'm also going to integrate some caching so that the load time ends up a lot quicker.
Anyways, any feedback would be appreciated.
import pickle, re
from os import path
__author__ = 'Charles Grunwald (Juntalis) <cgrunwald@gmail.com>'
def __setup_configobj__(deps, info):
    """ Setup configobj either in the lib folder or the local folder """
    from os import unlink, mkdir
    import shutil

    # Download and unzip
    tmpfile = deps.download(info['url'], info['filename'])
    tmpfolder = path.splitext(tmpfile)[0]
    deps.unzip(tmpfile, tmpfolder)
    unlink(tmpfile)

    modfolder = path.join(tmpfolder, 'configobj-4.7.2')
    modpath = [path.join(modfolder, 'configobj.py'), path.join(modfolder, 'validate.py')]

    (modpath, modconfigobj) = deps.install(modpath, 'configobj')
    shutil.rmtree(tmpfolder, True)
    return (modconfigobj, modpath)
""" Our dependencies """
__dependencies__ = {
    'configobj':
        (__setup_configobj__, {
            'filename' : 'configobj.zip',
            'url' : 'http://www.voidspace.org.uk/downloads/configobj-4.7.2.zip'
        })
}
class ScriptDeps:
    """ Simple class to install any script dependencies we don't have at startup. """
    __modules__ = {}

    def __init__(self):
        deps = __dependencies__
        for dep in deps.keys():
            (setup, info) = deps[dep]
            self.__modules__[dep] = setup(self, info)
    def get(self, name):
        if self.__modules__.has_key(name):
            return self.__modules__[name]
        return None
    def unzip(self, filename, dir):
        """ Extract zip file: filename to folder: dir. """
        import zipfile, os
        from cStringIO import StringIO
        zf = zipfile.ZipFile( filename )
        namelist = zf.namelist()
        dirlist = filter( lambda x: x.endswith( '/' ), namelist )
        filelist = filter( lambda x: not x.endswith( '/' ), namelist )
        # make base
        pushd = os.getcwd()
        if not path.isdir( dir ):
            os.mkdir( dir )
        os.chdir( dir )
        # create directory structure
        dirlist.sort()
        for dirs in dirlist:
            dirs = dirs.split( '/' )
            prefix = ''
            for dir in dirs:
                dirname = path.join( prefix, dir )
                if dir and not path.isdir( dirname ):
                    os.mkdir( dirname )
                prefix = dirname
        # extract files
        for fn in filelist:
            try:
                out = open( fn, 'wb' )
                buffer = StringIO( zf.read( fn ))
                buflen = 2 ** 20
                datum = buffer.read( buflen )
                while datum:
                    out.write( datum )
                    datum = buffer.read( buflen )
                out.close()
            except:
                import sys
                sys.stderr.write('Error while unzipping %s..\n' % filename)
        os.chdir( pushd )

    def download(self, url, filename):
        """ Download dependencies specified by dictionary __dependencies__"""
        from urllib import urlretrieve as download
        from tempfile import gettempdir as tempdir
        # Iterate through dependencies, downloading and moving.
        result = path.join(tempdir(), filename)
        download(url, result)
        return result

    def install(self, modpath, modname):
        libdir = path.join(notepad.getNppDir(), 'plugins', 'PythonScript', 'lib')
        modconfigobj = None
        try:
            # install to pythonscript lib folder.
            for modfile in modpath:
                shutil.move(modfile, libdir)
        except:
            # install to scripts/lib
            libdir = path.join(path.abspath(path.dirname(__file__)), 'lib')
            if not path.exists(libdir) or not path.isdir(libdir):
                mkdir(libdir)
            for modfile in modpath:
                shutil.move(modfile, libdir)
            initfile = path.join(libdir, '__init__.py')
            if not path.exists(initfile):
                initfile = open(initfile,'w')
                initfile.write('# Stub')
                initfile.close()
            modconfigobj = __import__('lib.%s' % modname)
        else:
            modconfigobj = __import__(modname)
        finally:
            modpath = path.join(libdir, '%s.py' % modname)

        return (modpath, modconfigobj)
class SimpleCache(dict):
    """
    Simple local cache.
    It saves local data in singleton dictionary with convenient interface

    Downloaded from http://code.activestate.com/recipes/577492-simple-local-cache-and-cache-decorator/

    Author: Andrey Nikishaev
    License: GPL
    Copyright 2010, http://creotiv.in.ua
    """

    def __new__(cls,*args):
        if not hasattr(cls,'_instance'):
            cls._instance = dict.__new__(cls)
        else:
            raise Exception('SimpleCache already initialized')
        return cls._instance

    @classmethod
    def getInstance(cls):
        if not hasattr(cls,'_instance'):
            cls._instance = dict.__new__(cls)
        return cls._instance
    def get(self,name,default=None):
        """Multilevel get function.
        Code:       
        Config().get('opt.opt_level2.key','default_value')
        """
        if not name: 
            return default
        levels = name.split('.')
        data = self         
        for level in levels:
            try:            
                data = data[level]
            except:
                return default
        return data

    def set(self,name,value):
        """Multilevel set function
        Code:       
        Config().set('opt.opt_level2.key','default_value')
        """
        levels = name.split('.')
        arr = self      
        for name in levels[:-1]:
            if not arr.has_key(name):        
                arr[name] = {}   
            arr = arr[name]
        arr[levels[-1]] = value

    def getset(self,name,value):
        """Get cache, if not exists set it and return set value
        Code:       
        Config().getset('opt.opt_level2.key','default_value')
        """
        g = self.get(name)
        if not g:
            g = value
            self.set(name,g)
        return g
def scache(func):
    def wrapper(*args, **kwargs):
        cache = SimpleCache.getInstance()
        fn = "scache." + func.__module__ + func.__class__.__name__ + \
             func.__name__ + str(args) + str(kwargs)        
        val = cache.get(fn)
        if not val:
            res = func(*args, **kwargs)
            cache.set(fn,res)
            return res
        return val
    return wrapper
# Try to import configobj module. If we cant, download it and set it up.
try:
    import configobj
    has_configobj = True
except ImportError:
    notepad.messageBox('Could not find module "configobj". Downloading and setting it up now..', 'Dependencies')
    deps = ScriptDeps()
    (configobj, configobj_path) = deps.get('configobj')
    has_configobj = configobj is not None
    if has_configobj:
        notepad.messageBox('Module "configobj" setup successfully.You can find it at:\n\n%s' % configobj_path, 'Setup Successful')
    else:
        notepad.messageBox('Error: Could not import configobj.\nDownload at: http://www.voidspace.org.uk/python/configobj.html', 'Import Error')
        exit()
class LanguageAutoDetector:
    """ Main class """
    __log = None
    __config = None
    __config_path = None
    __cache = None
    __detections = [
        'filename',
        'partial_filename',
        'xml',
        'shebang',
        'try_shebang',
        'contains_string'
        #'regex'
    ]
    __cache_ignore = {
        'contains_string' : None,
        'try_shebang' : 'shebang',
        'partial_filename' : 'filename',
        'regex' : None
    }

    def __init__(self, config_file=None):
        config = self.__load_config(config_file)
        if config['cache']['enabled']:
            self.__load_cache()
        #self.__log = logger.FileLogger()

    def __new__(cls,*args):
        if not hasattr(cls,'_instance'):
            cls._instance = dict.__init__(cls, args)
        else:
            raise Exception('LanguageAutoDetector already initialized')
        return cls._instance

    @classmethod
    def getInstance(cls):
        if not hasattr(cls,'_instance'):
            cls._instance = dict.__new__(cls)
        return cls._instance
    @scache
    def config(self, key=None, default=None):
        if self.__config is None:
            config = self.__load_config()
        else:
            config = self.__config

        if key is None:
            return config

        return self.__getdict(config, key, default)
    @scache
    def cache(self, key=None, default=None):
        if self.__cache is None:
            cache = self.__load_cache()
        else:
            cache = self.__cache

        if key is None:
            return cache

        return self.__getdict(cache, key, default)

    def set_lang(self, result, bufferID):
        ret = False
        lang = self.__test_language(result)
        if lang is None:
            if self.config('errors.invalid_lang'):
                notepad.messageBox('Error: Specified language %s invalid.' % result, 'Config Error')
        else:
            notepad.setLangType(lang, bufferID)
            ret = True
        return ret

    def detect(self, args):
        detections = self.config('detections.order')
        bufferID = args['bufferID']
        args['filename'] = path.basename(notepad.getBufferFilename(bufferID))
        notepad.activateBufferID(bufferID)
        for detection in detections:
            func = getattr(self, 'detection_%s' % detection.lower())
            result = func(args)
            console.write('Detection %s: %s\n' % (detection.lower(),result))
            if result is not None:
                if self.set_lang(result, bufferID):
                    break
    # filename
    # partial_filename
    # xml
    # shebang
    # try_shebang
    # contains_string

    def detection_filename(self, args):
        filename = args['filename']
        config = self.config('detections.filename')
        for lang in config.keys():
            if filename in config[lang]:
                return lang
        return None

    @scache
    def detection_partial_filename(self, args):
        filename = args['filename']
        config = self.config('detections.partial_filename')
        for lang in config.keys():
            for pattern in config[lang]:
                rgx = re.compile("^%s$" % pattern, re.IGNORECASE)
                if rgx.match(filename):
                    # TODO: Cache here filename -> lang
                    return lang
        return None

    def detection_xml(self, args):
        # Check for xml stuff.
        xml_config = self.config('detections.xml')
        xml_filename = args['filename']
        self.xml_lang = None
        def check(contents, lineNumber, totalLines):
            val = contents.strip().lower()
            if len(val) == 0:
                return 0
            else:
                for pattern in xml_config:
                    pattern = pattern.lower()
                    if val.startswith(pattern):
                        self.xml_lang = 'xml'
                        ext = path.splitext(xml_filename)[1]
                        if len(ext) > 0:
                            xml_cache = { ext : 'xml' } # TODO: Cache here
            return totalLines - lineNumber
        editor.forEachLine(check)
        return self.xml_lang

    def detection_shebang(self, args):
        shebang = self.__getshebang()
        if shebang is None:
            return None
        config = self.config('detections.shebang')
        for lang in config.keys():
            for pattern in config[lang]:
                rgx = re.compile(r"(?:^#!((?:/[^\s]+/env(?:\.[a-z]+) |/[^\s]+/|[A-Z]:[^\s]*\\|[A-Z]:[^\s]*\\env\.[a-z]+ ?)?%s[\d.-_]*(?:\.[0-9a-z-_.])*)\b)\Z" % pattern, re.IGNORECASE)
                if rgx.match(shebang):
                    # TODO: Cache here shebang -> lang
                    return lang
        return None

    def detection_try_shebang(self, args):
        shebang = self.__getshebang()
        if shebang is None:
            return None
        rgx = re.compile(r"^#!(?:/[^\s]+/env(?:\.[a-z]+) |/[^\s]+/|[A-Z]:[^\s]*\\|[A-Z]:[^\s]*\\env\.[a-z]+ ?)?([^\s]+)[\d.-_]*(?:\.[0-9a-z-_.])*\b", re.IGNORECASE)
        match = rgx.search(shebang)
        if match:
            result = match.group(1)
            lang = self.__test_language(result)
            if lang is None:
                return None
            return result
        return None

    def detection_contains_string(self, args):
        text = editor.getText()
        config = self.config('detections.contains_string')
        for lang in config.keys():
            for pattern in config[lang]:
                rgx = re.compile("%s" % pattern, re.IGNORECASE | re.MULTILINE)
                if rgx.search(text):
                    # TODO: Cache here shebang -> lang
                    return lang
        return None
    def detection_regex(self, args):
        pass

    def __getshebang(self):
        line = editor.getLine(0)
        if line[0:2] == '#!':
            return line
        return None

    def __getdict(self, dict, name, default=None):
        """Multilevel get function.
        Code:       
        Config().get('opt.opt_level2.key','default_value')
        """
        if not name: 
            return default
        levels = name.split('.')
        data = dict
        for level in levels:
            try:            
                data = data[level]
            except:
                return default
        return data

    def __load_config(self, cfg=None):
        """ Load configuration for script. If it doesn't exist, write the default
            configuration to file. """

        # Figure out config path.
        if cfg is None:
            cfg = path.join(notepad.getPluginConfigDir(), 'py_autolang.cfg')
        self.__config_path = cfg
        if path.exists(cfg):
            self.__config = configobj.ConfigObj(cfg)
        else:
            self.__config = self.__default_config()
            self.__save_config()
        return self.__config

    def __default_config(self, cfg=None):
        """ Default configuration for script. """

        # Figure out config path.
        if cfg is None and self.__config_path is None:
            cfg = path.join(notepad.getPluginConfigDir(), 'py_autolang.cfg')
        elif cfg is None and self.__config_path is not None:
            cfg = self.__config_path

        config = configobj.ConfigObj()
        config.filename = cfg

        # Main config
        config['script'] = {}
        config['script']['enabled'] = True
        config['script']['autoload'] = True

        # Errors
        config['errors'] = {}
        config['errors']['invalid_lang'] = True # Message box on invalid lexer specified.
        config['errors']['invalid_lexer'] = True # Message box on invalid lang specified.

        # Script cache
        config['cache'] = {}
        config['cache']['enabled'] = True
        cache_folder = path.abspath(path.join(path.dirname(cfg), 'cache'))
        valid_folder = False
        while not valid_folder:
            if path.exists(cache_folder):
                if not path.isdir(cache_folder):
                    import string, random
                    cache_folder += '-' + ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(3))
                else:
                    valid_folder = True
            else:
                from os import mkdir
                mkdir(cache_folder)
        config['cache']['folder'] = cache_folder

        # Logging
        config['logging'] = {}
        config['logging']['console'] = False
        config['logging']['console_auto_open'] = False
        config['logging']['file'] = False

        # Loading event
        config['file_load_event'] = {}
        config['file_load_event']['no_extension'] = True
        config['file_load_event']['default_lexer'] = True
        config['file_load_event']['always'] = False

        """
            Detection methods order - This tells what order to use detection methods.
            So if filename is executed before shebang, and filename matches a detection,
            the shebang line wont be run. To disable a detection method, set to 0. If
            any two methods have the same number, an error will be thrown.

            Possible detections:
                shebang
                try_shebang
                xml
                filename
                partial_filename
                contains_string
                regex - Not yet implemented
        """
        order = ['filename', 'partial_filename', 'xml', 'shebang', 'try_shebang', 'contains_string']
        # Default Detections
        ## Shebangs
        ## These all match the follow regex pattern:
        ### r"^#!(?:/[^\s]+/env(?:\.[a-z]+) |/[^\s]+/|[A-Z]:[^\s]*\\|[A-Z]:[^\s]*\\env\.[a-z]+ ?)?%s[\d.-_]*(?:\.[0-9a-z-_.])*\b" % key
        shebang = {

            # Shell
            ## Bash/Korn Shell/C Shell/Z Shell/etc
            'bash' : ['(?:[czk]|ba)?sh'],

            # Python
            ## CPython - http://python.org/
            ## Cross Twine Linker (xtpython) - http://crosstwine.com/linker/python.html
            ## Unpython Python to C compiler (unpython) - http://code.google.com/p/unpython/
            ## IPython (ipython) - http://ipython.org/
            ## PyPy - http://pypy.org/
            ## Iron Python - http://ironpython.net/
            ## Mozilla Embedded Python Console - http://www.thomas-schilz.de/MozPython/
            ## TinyPy - http://www.tinypy.org/
            ## Snipy - Personal project, you can remove if you want.
            ## Enthought SciPy distribution - http://www.enthought.com/
            ## Jython (jython) - http://www.jython.org/
            ## Cython (Optimizing Python to C Compiler) - http://cython.org/
            ## Typhon (typhon) - https://github.com/vic/typhon
            ## Mython (mython) - http://mython.org/

            # Languages Close Enough to Python """
            ## Nimrod  - http://force7.de/nimrod/download.html
            ## Serpent - http://sourceforge.net/projects/serpent/
            ## Boo - http://boo.codehaus.org/

            'python' : [
                '(?:xt|un|i)?pythonw?',
                '(?:py|i|moz|tiny|sni)pyw?(?:-c)?',
                'epdw?',
                '[jctm]ythonw?',
                '(?:nimrod|ser?pent)',
                'boo(?:c|i|ish)?'
            ],
            # Perl
            ## Perl - http://www.perl.org/
            ## Parrot - http://parrot.org/
            ### Hm, this one might be hard. Leaving parrot as perl for now
            'perl' : [ 'w?perl', 'parrot' ],

            # Ruby
            ## Ruby - http://www.ruby-lang.org/en/
            ## Iron Ruby (ir, etc) - http://ironruby.net/
            ## JRuby - http://jruby.org/
            ## Ruby on Rails - http://rubyonrails.org/
            'ruby' : [
                '[ji]?[ie]r[wbi]{0,2}?(?:_swing)?',
                '[ej]?ruby[wc]?',
                'rake'
            ],
            # Javascript
            ## Node.Js - http://nodejs.org/
            ## Narwhal - https://github.com/tlrobinson/narwhal
            ## JSDB - http://www.jsdb.org/
            ## Ringo Javascript - http://ringojs.org/
            ## GlueScript - http://gluescript.sourceforge.net/
            ## Rhino - http://www.mozilla.org/rhino/
            'javascript' : [
                '(?:node|npm)',
                '(?:narwhal|tusk)',
                '(?:jsdb|ringo|gluew?)',
                '(?:rhino|js)'
            ],

            # PHP
            'php' : [
                '(?:i?php(?:-cgi|-cli|-win)?|pharc?)'
            ]
        }

        filename = {
            'bash' : ['configure'],
            'makefile' : ['Makefile']
        }

        partial_filename = {
            'makefile' : ['Makefile\..+']
        }

        contains_string = {
            'bash' : ['^mk_add_options', '^ac_add_options']
        }

        config['detections'] = {
            'order' : order,
            'filename' : filename,
            'partial_filename' : partial_filename,
            'xml' : ['<?xml ', '<!DOCTYPE'],
            'shebang' : shebang,
            'contains_string' : contains_string
        }

        return config

    def __save_config(self):
        self.__config.write()

    def __load_cache(self):
        config = self.config()
        folder = config['cache']['folder']

        cache = {}
        for detection in config['detections']['order']:
            if detection in self.__cache_ignore:
                if self.__cache_ignore[detection] is not None:
                    detection = self.__cache_ignore[detection]
                    if cache.has_key(detection):
                        continue
                else:
                    continue
            f = path.join(folder, detection)
            if path.exists(f) and path.isfile(f):
                input = open(f, 'rb')
                cache[detection] = {
                    'changed' : False,
                    'value' : pickle.load(input)
                }
                input.close()
            else:
                cache[detection] = {
                    'changed' : False,
                    'value' : None
                }
        self.__cache = cache
        return self.__cache

    def __save_cache(self):
        config = self.config()
        folder = config['cache']['folder']
        saved = []
        for detection in config['detections']['order']:
            if detection in self.__cache_ignore:
                if self.__cache_ignore[detection] is not None:
                    detection = self.__cache_ignore[detection]
                    if detection in saved:
                        continue
                else:
                    continue
            f = path.join(folder, detection)
            cache = self.cache(detection)
            if cache['changed'] and cache['value'] is not None:
                output = open(f, 'wb')
                pickle.dump(cache['value'], output)
                output.close()
                self.__cache[detection]['changed'] = False
            saved.append(detection)

    def __test_lexer(self, lexer):
        result = True
        old_lexer = editor.getLexerLanguage()
        editor.setLexerLanguage(lexer)
        if editor.getLexerLanguage() == 'null':
            result = False
        editor.setLexerLanguage(old_lexer)
        return result
    def __test_language(self, lang):
        import Npp
        lang = lang.upper()
        result = None
        try:
            result = getattr(Npp.LANGTYPE, lang)
        except AttributeError:
            result = None
        return result
import sys
sys.stdout = console
def detect_test(args):
    console.write('Language: %s\n' % notepad.getLangType(args["bufferID"]).__str__().lower())

detector = LanguageAutoDetector()
notepad.clearCallbacks([NOTIFICATION.FILEOPENED])
notepad.callback(detector.detect, [NOTIFICATION.FILEOPENED])
Auto Detect File Language (Shebang &...

A Python Scripting plugin for Notepad++

Forums

Help

Auto Detect File Language (Shebang &...

Auto Detect File Language (Shebang &...

A Python Scripting plugin for Notepad++

Forums

Help

Auto Detect File Language (Shebang &... document.SUBSCRIPTION_OPTIONS = { "thing": "topic", "subscribed": false, "url": "subscribe", "icon": { "css": "fa fa-envelope-o" } };

Auto Detect File Language (Shebang &...
