Re: [Modeling-users] entity import takes a lot of cpu

[Sebastien B. <sbi...@us...>]

        Hi,

  I'm answering very quickly so please excuse if my typos and bad engli=
sh.

Yannick Gingras <yan...@sa...> writes:
> On Wednesday 19 February 2003 01:35 pm, Yannick Gingras wrote:
> > This delay is not showed in the python profiler so I assume that th=
e time
> > is spent in a C module.  Is the XML parser that CPU hungry ?
>=20
> Oups !
>=20
> The delay clearly shows in the profiler, the import statement was exe=
cuted=20
> before I start profiling.=20=20
>=20
> [snip]
> It seems clear to me that either we our DOM implementation is really
> unefficient or my model is too complex.  Having 40 entities in my
> model, I think it should be easy to parse.
>=20
> Anyone had the same problem ?

You're not alone! Loading the xml-model *does* consume a lot of cpu. At
the very beginning I had a simple sax parser, but I quickly switched to
DOM & XPath because it made the implementation a lot clearer. However, I
kind of regret this by now because it takes a lot of time to load an xml
model.

  I plan to release another script for ``compiling'' a model, with the
  generated __init__.py for that purpose. In the meantime I can give you
  a sample script you may want to adapt to your model (you'll find it at
  the end of the message). It does something really simple: it loads a
  model, pickles it, and drops the result into a dedicated module's
  string-field (dropping it in a model makes it possible to install it
  w/ distutils, as usual). This makes the process of loading a model a
  lot quicker; however pickle is very sensitive wrt to its environment,
  hence great care must be taken so that the python executable compiling
  the model is the very same than the one loading it, being executed on
  the same platform, etc.

  It also requires a little adaptation of the generated __init__.py,
  you'll also find sample code for that at the end of the message (don't
  forget to query/replace all occurences of MODEL_NAME with yours).

Sorry, I do not have the time to check that everything will work as
expected but it's mostly copy-pasted from a working project, hopefully
the changes you may have to make won't be too expensive.

  You can consider that these two samples of code are in public domain.

  Please tell me if you have any problem with this.

-- S=E9bastien.


------------------------------------------------------------------------
#! /usr/bin/env python
"""
compile_model.py

  ``Compile'' an xml_model into two python modules, model_<modelName>.p=
y and
  model_<modelName>_pickled.py:

    - model_<model_name>.py: contains the whole xml-file in a single mo=
dule
        attribute: model_src
=20=20
    - model_<modelName>_pickled.py: the second one contains a single at=
tribute
        as well: model_pickle, which is a raw string to be loaded by
        cPickle.loads()
=20=20=20=20=20=20
  Written by: Sebastien Bigaret
"""
import sys, getopt

def loadModel(xmlFilePath):
  """
  Creates a python module (file: model_<modelName>.py) and dumps the
  content of the file 'xmlFilePath' in a string attribute named 'model_=
src'
  """
  from Modeling import ModelSet
  ms=3DModelSet.ModelSet()
  ms.addModelFromXML({'file': xmlFilePath})
  model=3Dms.models()[0]
  model._modelSet=3DNone
  return model

def write_xml_model_in_python_module(xmlFilePath, modelName):
  """
  Creates a python module (file: model_<modelName>.py) and dumps the
  content of the file 'xmlFilePath' in a string attribute named 'model_=
src'
  """
  xml=3Dopen(xmlFilePath, 'r')
  file=3Dopen('model_%s.py'%modelName, 'w')
  file.write('model_src=3D"""')
  s=3Dxml.readline()
  while s!=3D'':
    file.write(s)
    s=3Dxml.readline()
  file.write('"""\n')
  xml.close()
  file.close()
=20=20
def write_pickled_model(model):
  """
  Creates a python module (file: model_<modelName>_pickle.py) and dumps=
 a
  pickled version of the supplied model in module's attribute 'model_pi=
ckle'.
  """
  import cPickle
  file=3Dopen('model_%s_pickle.py'%model.name(), 'w')
  file.write('model_pickle=3Dr"""')
  pickled=3DcPickle.dump(model, file)
  file.write('"""\n')
  file.close()

def usage(prgName, exitStatus=3DNone):
  print """%s <model.xml>
``Compile'' an xml_model into two python modules, model_<modelName>.py =
and
model_<modelName>_pickled.py.

  - model_<model_name>.py: contains the whole xml-file in a single modu=
le
      attribute: model_src

  - model_<modelName>_pickled.py: the second one contains a single attr=
ibute
      as well: model_pickle, which is a raw string to be loaded by
      cPickle.loads()

Options
-------
  -h           Prints this message
""" % prgName
  if exitStatus is not None:
    sys.exit(exitStatus)

def main(argv):
  me=3Dargv[0]

  try:
    options, args =3D getopt.getopt(sys.argv[1:], 'h')
  except getopt.GetoptError,msg:
    print me, msg
    usage(me, 1)
=20=20=20=20
  verbose=3D0
=20=20
  for k, v in options:
    if k=3D=3D'-h': usage(me, 0)

  if len(args)!=3D1:
    usage(me,1)
=20=20=20=20
  model=3DloadModel(args[0])
  write_xml_model_in_python_module(args[0], model.name())
  write_pickled_model(model)

=20
if __name__ =3D=3D "__main__":
  main(sys.argv)

------------------------------------------------------------------------

------------------------------------------------------------------------
# modified __init__.py for your package
##
## !!! REPLACE ALL OCCURRENCES OF MODEL_NAME WITH _YOUR_ modelName !!!
##

from Modeling import ModelSet
import os, warnings, cStringIO, traceback

def loadModel():
  """
  Loads the model. It searches for:

    1. a pickled raw string stored in attribute 'model_pickle' in module
       'model_<modelName>_pickle', if it exists

    2. then, a xml-string stored in attribute 'model_str' in module
       'model_<modelName>', if it exists

    3. last, a xml file 'model_<model_Name>.xml'


  cf. script compile_model.py and its usage
  """
  try:
      from model_MODEL_NAME_pickle import model_pickle
      import cPickle
      m=3DcPickle.loads(model_pickle)
      ModelSet.defaultModelSet().addModel(m)
      model=3DModelSet.defaultModelSet().modelNamed('MODEL_NAME')
  except:
    exc=3DcStringIO.StringIO()
    traceback.print_exc(file=3Dexc)
    warnings.warn("Couldn't load model from pickle'\n"\
                  "Reason:\n%s"%exc.getvalue())
    del exc
  else:
    return
=20=20
  try:
    from model_MODEL_NAME import model_src
    ModelSet.defaultModelSet().addModelFromXML({'string': model_src})
    model=3DModelSet.defaultModelSet().modelNamed('MODEL_NAME')
  except:
    exc=3DcStringIO.StringIO()
    traceback.print_exc(file=3Dexc)
    warnings.warn("Couldn't load model from 'model_MODEL_NAME.py'\nReas=
on:\n%s"%exc.getvalue())
    del exc
  else:
    return

  # Or, alternatively: use the xml file (ok for dev. mode, not for inst=
all w/
  # distutils)
  #
  from os import getcwd, path
  mydir =3D os.path.abspath(os.path.dirname(__file__))
  xmlmodelPath=3Dpath.join(mydir,'model_MODEL_NAME.xml')
  ModelSet.defaultModelSet().addModelFromXML({'file': xmlmodelPath})
  model=3DModelSet.defaultModelSet().modelNamed('MODEL_NAME')

if ModelSet.defaultModelSet().modelNamed('MODEL_NAME') is None:
  loadModel()
------------------------------------------------------------------------