Diff of /pyke/target_pkg.py [473574] .. [ff7730]  Maximize  Restore

  Switch to unified view

a/pyke/target_pkg.py b/pyke/target_pkg.py
...
...
36
debug = False
36
debug = False
37
37
38
Name_test = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*$')
38
Name_test = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*$')
39
39
40
class target_pkg(object):
40
class target_pkg(object):
41
    '''
41
    r'''This manages all of the target files in a compiled_krb directory.
42
        This maintains a list of:
42
43
    There is one instance per compiled_krb directory.  It keeps track of
44
    everything in that directory and manages recompiling the sources when
45
    the compiled targets are missing or out of date.
46
47
    This instance is stored permanently in the "targets" variable of the
48
    compiled_pyke_files.py module in the compiled_krb directory.
49
50
    This maintains the following information for each compiled target file:
43
            source_package, source_filepath, compile_time, target_filenames.
51
        source_package, source_filepath, compile_time, target_filename.
44
    '''
52
    '''
45
    def __init__(self, module_name, filename, pyke_version = pyke.version,
53
    def __init__(self, module_name, filename, pyke_version = pyke.version,
46
                       loader = None, sources = None):
54
                       loader = None, sources = None, compiler_version = 0):
47
        self.package_name = module_name.rsplit('.', 1)[0]
55
        r'''
56
57
        The parameters are:
58
59
            module_name:  the complete dotted name of the compiled_pyke_files
60
                          module for this object.
61
            filename:     the absolute path to the compiled_pyke_files.py/c/o
62
                          file.
63
            pyke_version: the version of pyke used to compile the target files.
64
            loader:       the __loader__ attribute of the compiled_pyke_files
65
                          module (only set if the compiled_krb directory has
66
                          been zipped).
67
            sources:      {(source_module, source_filepath):
68
                            [compile_time, target_file...]}
69
            compiler_version:
70
                          the version of the pyke compiler used to compile all
71
                          of the targets in this compiled_krb directory.
72
73
        This class is instantiated in two different circumstances:
74
75
        1.  From compiled_krb/compiled_pyke_files.py with a list of all of the
76
            compiled files in that compiled_krb directory.
77
78
            In this case, all of the parameters are passed to __init__.
79
80
        2.  From knowledge_engine.engine.__init__ (actually _init_path).
81
82
            In this case, only the first two parameters are passed to __init__.
83
84
        Either way, after importing compiled_pyke_files or creating a new
85
        instance directly, reset is called by
86
        knowledge_engine.engine._init_path.
87
        '''
88
        self.package_name = os.path.splitext(module_name)[0]
89
        if filename.endswith('.py'):
48
        self.filename = filename
90
            self.filename = filename
91
        else:
92
            self.filename = filename[:-1]
49
        self.directory = os.path.dirname(self.filename)
93
        self.directory = os.path.dirname(self.filename)
50
        if debug:
94
        if debug:
51
            print >> sys.stderr, "target_pkg:", self.package_name, self.filename
95
            print >> sys.stderr, "target_pkg:", self.package_name, self.filename
52
        self.loader = loader
96
        self.loader = loader
53
97
54
        if pyke_version == pyke.version:
98
        if compiler_version == pyke.compiler_version:
55
            # {(source_package_name, source_filepath):
99
            # {(source_package_name, source_filepath):
56
            #  [compile_time, target_filename, ...]}
100
            #  [compile_time, target_filename, ...]}
57
            self.sources = sources if sources is not None else {}
101
            self.sources = sources if sources is not None else {}
58
        elif self.loader is None:
102
        elif self.loader is None:
59
            self.sources = {}
103
            self.sources = {}
60
        else:
104
        else:
105
            # loading incorrect version from zip file
61
            raise AssertionError("%s: wrong version of pyke, "
106
            raise AssertionError("%s: wrong version of pyke, "
62
                                 "running %s, compiled for %s" % 
107
                                 "running %s, compiled for %s" % 
63
                                 (pyke.version, pyke_version))
108
                                 (module_name, pyke.version, pyke_version))
109
64
    def reset(self):
110
    def reset(self):
65
        ''' This should be called once by engine.__init__ prior to calling
111
        ''' This should be called once by engine.__init__ prior to calling
66
            add_source_package.
112
            add_source_package.
67
        '''
113
        '''
68
        if debug: print >> sys.stderr, "target_pkg.reset"
114
        if debug: print >> sys.stderr, "target_pkg.reset"
69
        self.dirty = False
115
        self.dirty = False
70
        self.source_packages = {}  # {source_package_name: source_package_dir}
116
        self.source_packages = {}  # {source_package_name: source_package_dir}
71
        self.compiled_targets = set([])  # set of target_filename
117
        self.compiled_targets = set()  # set of target_filename
72
        self.rb_names = set()
118
        self.rb_names = set()
119
73
    def add_source_package(self, source_package_name):
120
    def add_source_package(self, source_package_name):
74
        if debug:
121
        if debug:
75
            print >> sys.stderr, "target_pkg.add_source_package:", \
122
            print >> sys.stderr, "target_pkg.add_source_package:", \
76
                                 source_package_name
123
                                 source_package_name
77
        if not self.loader:
124
        if not self.loader:
...
...
84
            self.source_packages[source_package_name] = source_package_dir
131
            self.source_packages[source_package_name] = source_package_dir
85
            sources = set([])
132
            sources = set([])
86
            for dirpath, dirnames, filenames \
133
            for dirpath, dirnames, filenames \
87
             in os.walk(source_package_dir, onerror=_raise_exc):
134
             in os.walk(source_package_dir, onerror=_raise_exc):
88
                for filename in filenames:
135
                for filename in filenames:
89
                    if len(filename) > 4 \
90
                       and filename[-4:] in ('.krb', '.kfb', '.kqb'):
136
                    if filename.endswith(('.krb', '.kfb', '.kqb')):
91
                        source_abspath = os.path.join(dirpath, filename)
137
                        source_abspath = os.path.join(dirpath, filename)
92
                        assert dirpath.startswith(source_package_dir)
138
                        assert dirpath.startswith(source_package_dir)
93
                        source_relpath = \
139
                        source_relpath = \
94
                            os.path.join(dirpath[len(source_package_dir)+1:],
140
                            os.path.join(dirpath[len(source_package_dir)+1:],
95
                                         filename)
141
                                         filename)
...
...
103
                                 if src_pkg_name == source_package_name and \
149
                                 if src_pkg_name == source_package_name and \
104
                                    src_filepath not in sources]:
150
                                    src_filepath not in sources]:
105
                if debug:
151
                if debug:
106
                    print >> sys.stderr, "del:", source_package_name, filepath
152
                    print >> sys.stderr, "del:", source_package_name, filepath
107
                del self.sources[source_package_name, filepath]
153
                del self.sources[source_package_name, filepath]
154
108
    def add_source(self, source_package_name, source_filepath, source_mtime):
155
    def add_source(self, source_package_name, source_filepath, source_mtime):
109
        if debug:
156
        if debug:
110
            print >> sys.stderr, "target_pkg.add_source:", \
157
            print >> sys.stderr, "target_pkg.add_source:", \
111
                                 source_package_name, source_filepath
158
                                 source_package_name, source_filepath
112
        rb_name = os.path.basename(source_filepath)[:-4]
159
        rb_name = os.path.splitext(os.path.basename(source_filepath))[0]
113
        if debug: print >> sys.stderr, "rb_name:", rb_name
160
        if debug: print >> sys.stderr, "rb_name:", rb_name
114
        if not Name_test.match(rb_name):
161
        if not Name_test.match(rb_name):
115
            raise ValueError("%s: %s illegal as python identifier" %
162
            raise ValueError("%s: %s illegal as python identifier" %
116
                             (source_filepath, rb_name))
163
                             (source_filepath, rb_name))
117
        if rb_name in self.rb_names:
164
        if rb_name in self.rb_names:
...
...
122
        if self.sources.get(key, (0,))[0] < source_mtime:
169
        if self.sources.get(key, (0,))[0] < source_mtime:
123
            if debug:
170
            if debug:
124
                print >> sys.stderr, source_filepath, "needs to be compiled"
171
                print >> sys.stderr, source_filepath, "needs to be compiled"
125
            self.sources[key] = []
172
            self.sources[key] = []
126
            self.dirty = True
173
            self.dirty = True
174
127
    def do_by_ext(self, prefix, filename, *args):
175
    def do_by_ext(self, prefix, filename, *args):
128
        ext = os.path.splitext(filename)[1][1:]
176
        ext = os.path.splitext(filename)[1][1:]
129
        return getattr(self, "%s_%s" % (prefix, ext))(filename, *args)
177
        return getattr(self, "%s_%s" % (prefix, ext))(filename, *args)
178
130
    def compile(self, engine):
179
    def compile(self, engine):
131
        if debug: print >> sys.stderr, "%s.compile:" % self.package_name
180
        if debug: print >> sys.stderr, "%s.compile:" % self.package_name
132
        global krb_compiler
181
        global krb_compiler
133
        if not self.loader:
182
        if not self.loader:
134
            initialized = False
183
            initialized = False
...
...
148
                                source_filename))
197
                                source_filename))
149
                    if debug: print >> sys.stderr, "target_files:", target_files
198
                    if debug: print >> sys.stderr, "target_files:", target_files
150
                    value.append(time.time())
199
                    value.append(time.time())
151
                    value.extend(target_files)
200
                    value.extend(target_files)
152
                    self.compiled_targets.update(target_files)
201
                    self.compiled_targets.update(target_files)
202
153
    def compile_krb(self, source_filename):
203
    def compile_krb(self, source_filename):
154
        if debug: print >> sys.stderr, "compile_krb:", source_filename
204
        if debug: print >> sys.stderr, "compile_krb:", source_filename
155
        rb_name = os.path.basename(source_filename)[:-4]
205
        rb_name = os.path.basename(source_filename)[:-4]
156
        return krb_compiler.compile_krb(rb_name, self.package_name,
206
        return krb_compiler.compile_krb(rb_name, self.package_name,
157
                                        self.directory, source_filename)
207
                                        self.directory, source_filename)
208
158
    def compile_kfb(self, source_filename):
209
    def compile_kfb(self, source_filename):
159
        if debug: print >> sys.stderr, "compile_kfb:", source_filename
210
        if debug: print >> sys.stderr, "compile_kfb:", source_filename
160
        try:
211
        try:
161
            fbc_name = os.path.basename(source_filename)[:-4] + '.fbc'
212
            fbc_name = os.path.basename(source_filename)[:-4] + '.fbc'
162
            fbc_path = os.path.join(self.directory, fbc_name)
213
            fbc_path = os.path.join(self.directory, fbc_name)
163
            self.pickle_it(krb_compiler.compile_kfb(source_filename), fbc_path)
214
            self.pickle_it(krb_compiler.compile_kfb(source_filename), fbc_path)
164
            return (fbc_name,)
215
            return (fbc_name,)
165
        except:
216
        except:
166
            if os.path.lexists(fbc_path): os.remove(fbc_path)
217
            if os.path.lexists(fbc_path): os.remove(fbc_path)
167
            raise
218
            raise
219
168
    def compile_kqb(self, source_filename):
220
    def compile_kqb(self, source_filename):
169
        if debug: print >> sys.stderr, "compile_kqb:", source_filename
221
        if debug: print >> sys.stderr, "compile_kqb:", source_filename
170
        try:
222
        try:
171
            qbc_name = os.path.basename(source_filename)[:-4] + '.qbc'
223
            qbc_name = os.path.basename(source_filename)[:-4] + '.qbc'
172
            qbc_path = os.path.join(self.directory, qbc_name)
224
            qbc_path = os.path.join(self.directory, qbc_name)
173
            self.pickle_it(krb_compiler.compile_kqb(source_filename), qbc_path)
225
            self.pickle_it(krb_compiler.compile_kqb(source_filename), qbc_path)
174
            return (qbc_name,)
226
            return (qbc_name,)
175
        except:
227
        except:
176
            if os.path.lexists(qbc_path): os.remove(qbc_path)
228
            if os.path.lexists(qbc_path): os.remove(qbc_path)
177
            raise
229
            raise
230
178
    def write(self):
231
    def write(self):
179
        if debug: print >> sys.stderr, "target_pkg.write"
232
        if debug: print >> sys.stderr, "target_pkg.write"
180
        if self.dirty:
233
        if self.dirty:
181
            sys.stderr.write('writing [%s]/%s\n' % 
234
            sys.stderr.write('writing [%s]/%s\n' % 
182
                               (self.package_name,
235
                               (self.package_name,
183
                                os.path.basename(self.filename)))
236
                                os.path.basename(self.filename)))
184
            with open(self.filename, 'w') as f:
237
            with open(self.filename, 'w') as f:
185
                f.write("# compiled_pyke_files.py\n\n")
238
                f.write("# compiled_pyke_files.py\n\n")
186
                f.write("from pyke import target_pkg\n\n")
239
                f.write("from pyke import target_pkg\n\n")
187
                f.write("version = %r\n\n" % pyke.version)
240
                f.write("pyke_version = %r\n\n" % pyke.version)
241
                f.write("compiler_version = %r\n\n" % pyke.compiler_version)
188
                f.write("try:\n");
242
                f.write("try:\n");
189
                f.write("    loader = __loader__\n")
243
                f.write("    loader = __loader__\n")
190
                f.write("except NameError:\n");
244
                f.write("except NameError:\n");
191
                f.write("    loader = None\n\n");
245
                f.write("    loader = None\n\n");
192
                f.write("targets = "
193
                        "target_pkg.target_pkg(__name__, __file__, "
246
                f.write("targets = target_pkg.target_pkg(__name__, __file__, "
194
                        "version, loader, {\n")
247
                        "pyke_version, loader, {\n")
195
                for item in self.sources.iteritems():
248
                for item in self.sources.iteritems():
196
                    if debug: print >> sys.stderr, "write got:", item
249
                    if debug: print >> sys.stderr, "write got:", item
197
                    if item[0][0] in self.source_packages:
250
                    if item[0][0] in self.source_packages:
198
                        if debug: print >> sys.stderr, "writing:", item
251
                        if debug: print >> sys.stderr, "writing:", item
199
                        f.write("    %r: %r,\n" % item)
252
                        f.write("    %r: %r,\n" % item)
200
                f.write("})\n")
253
                f.write("  },\n  compiler_version)\n")
254
201
    def load(self, engine, load_fc = True, load_bc = True,
255
    def load(self, engine, load_fc = True, load_bc = True,
202
                           load_fb = True, load_qb = True):
256
                           load_fb = True, load_qb = True):
203
        load_flags = {'load_fc': load_fc, 'load_bc': load_bc,
257
        load_flags = {'load_fc': load_fc, 'load_bc': load_bc,
204
                      'load_fb': load_fb, 'load_qb': load_qb}
258
                      'load_fb': load_fb, 'load_qb': load_qb}
205
        if debug: print >> sys.stderr, "target_pkg.load:", load_flags
259
        if debug: print >> sys.stderr, "target_pkg.load:", load_flags
...
...
207
         in self.sources.iteritems():
261
         in self.sources.iteritems():
208
            if self.loader or source_package_name in self.source_packages:
262
            if self.loader or source_package_name in self.source_packages:
209
                for target_filename in value[1:]:
263
                for target_filename in value[1:]:
210
                    if debug: print >> sys.stderr, "load:", target_filename
264
                    if debug: print >> sys.stderr, "load:", target_filename
211
                    self.do_by_ext('load', target_filename, engine, load_flags)
265
                    self.do_by_ext('load', target_filename, engine, load_flags)
266
212
    def load_py(self, target_filename, engine, flags):
267
    def load_py(self, target_filename, engine, flags):
213
        if debug: print >> sys.stderr, "load_py:", target_filename
268
        if debug: print >> sys.stderr, "load_py:", target_filename
214
        target_module = target_filename[:-3]  # strip '.py' extension.
269
        target_module = target_filename[:-3]  # strip '.py' extension.
215
        module_path = self.package_name + '.' + target_module
270
        module_path = self.package_name + '.' + target_module
216
        if target_module.endswith('_fc'):
271
        if target_module.endswith('_fc'):
...
...
224
                self.load_module(module_path, target_filename, engine, False)
279
                self.load_module(module_path, target_filename, engine, False)
225
        else:
280
        else:
226
            raise AssertionError("target_pkg.load_py: "
281
            raise AssertionError("target_pkg.load_py: "
227
                                 "unknown target file type: %s" %
282
                                 "unknown target file type: %s" %
228
                                   target_filename)
283
                                   target_filename)
284
229
    def load_fbc(self, target_filename, engine, flags):
285
    def load_fbc(self, target_filename, engine, flags):
230
        if debug: print >> sys.stderr, "load_fbc:", target_filename
286
        if debug: print >> sys.stderr, "load_fbc:", target_filename
231
        if flags['load_fb']:
287
        if flags['load_fb']:
232
            self.load_pickle(target_filename, engine)
288
            self.load_pickle(target_filename, engine)
289
233
    def load_qbc(self, target_filename, engine, flags):
290
    def load_qbc(self, target_filename, engine, flags):
234
        if debug: print >> sys.stderr, "load_qbc:", target_filename
291
        if debug: print >> sys.stderr, "load_qbc:", target_filename
235
        if flags['load_qb']:
292
        if flags['load_qb']:
236
            self.load_pickle(target_filename, engine)
293
            self.load_pickle(target_filename, engine)
294
237
    def load_module(self, module_path, filename, engine, do_import = True):
295
    def load_module(self, module_path, filename, engine, do_import = True):
238
        if debug: print >> sys.stderr, "load_module:", module_path, filename
296
        if debug: print >> sys.stderr, "load_module:", module_path, filename
239
        module = None
297
        module = None
240
        if module_path in sys.modules:
298
        if module_path in sys.modules:
241
            if debug: print >> sys.stderr, "load_module: already imported"
299
            if debug: print >> sys.stderr, "load_module: already imported"
...
...
245
                module = reload(module)
303
                module = reload(module)
246
        elif do_import:
304
        elif do_import:
247
            if debug: print >> sys.stderr, "load_module: importing"
305
            if debug: print >> sys.stderr, "load_module: importing"
248
            module = import_(module_path)
306
            module = import_(module_path)
249
        if module is not None and \
307
        if module is not None and \
250
           (not hasattr(module, 'version') or module.version != pyke.version):
308
           getattr(module, 'compiler_version', 0) != pyke.compiler_version:
251
            raise AssertionError("%s: incorrect pyke version: "
309
            raise AssertionError("%s: incorrect pyke version: running "
252
                                 "%s, expected %s" %
310
                                 "%s, expected %s" %
253
                                   (filename, module.version, pyke.version))
311
                                   (filename, pyke.version,
312
                                    module.pyke_version))
254
        if do_import: module.populate(engine)
313
        if do_import: module.populate(engine)
314
255
    def load_pickle(self, filename, engine):
315
    def load_pickle(self, filename, engine):
256
        global pickle
316
        global pickle
257
        if debug: print >> sys.stderr, "load_pickle:", filename
317
        if debug: print >> sys.stderr, "load_pickle:", filename
258
        try:
318
        try:
259
            pickle
319
            pickle
...
...
267
                contextlib.closing(
327
                contextlib.closing(
268
                    StringIO.StringIO(self.loader.get_data(full_path)))
328
                    StringIO.StringIO(self.loader.get_data(full_path)))
269
        else:
329
        else:
270
            ctx_lib = open(full_path, 'rb')
330
            ctx_lib = open(full_path, 'rb')
271
        with ctx_lib as f:
331
        with ctx_lib as f:
272
            version = pickle.load(f)
332
            versions = pickle.load(f)
333
            if isinstance(versions, tuple):
334
                pyke_version, compiler_version = versions
335
            else:
336
                pyke_version, compiler_version = versions, 0
273
            if version != pyke.version:
337
            if compiler_version != pyke.compiler_version:
274
                raise AssertionError("%s: incorrect pyke version: "
338
                raise AssertionError("%s: incorrect pyke version: running "
275
                                     "%s, expected %s" %
339
                                     "%s, expected %s" %
276
                                       (filename, version, pyke.version))
340
                                       (filename, pyke.version, pyke_version))
277
            pickle.load(f).register(engine)
341
            pickle.load(f).register(engine)
342
278
    def pickle_it(self, obj, path):
343
    def pickle_it(self, obj, path):
279
        global pickle
344
        global pickle
280
        try:
345
        try:
281
            pickle
346
            pickle
282
        except NameError:
347
        except NameError:
...
...
284
            import copy_reg
349
            import copy_reg
285
            copy_reg.pickle(slice, lambda s: (slice, (s.start, s.stop, s.step)))
350
            copy_reg.pickle(slice, lambda s: (slice, (s.start, s.stop, s.step)))
286
        sys.stderr.write("writing [%s]/%s\n" %
351
        sys.stderr.write("writing [%s]/%s\n" %
287
                           (self.package_name, os.path.basename(path)))
352
                           (self.package_name, os.path.basename(path)))
288
        with open(path, 'wb') as f:
353
        with open(path, 'wb') as f:
289
            pickle.dump(pyke.version, f)
354
            pickle.dump((pyke.version, pyke.compiler_version), f)
290
            pickle.dump(obj, f)
355
            pickle.dump(obj, f)
291
356
292
def _raise_exc(exc): raise exc
357
def _raise_exc(exc): raise exc
293
358
294
def import_(modulename):
359
def import_(modulename):
...
...
298
    mod = __import__(modulename)
363
    mod = __import__(modulename)
299
    for comp in modulename.split('.')[1:]:
364
    for comp in modulename.split('.')[1:]:
300
        mod = getattr(mod, comp)
365
        mod = getattr(mod, comp)
301
    return mod
366
    return mod
302
367
303
def test():
304
    import doctest
305
    import sys
306
    sys.exit(doctest.testmod()[0])
307
308
if __name__ == "__main__":
309
    test()