From 322809d31b04325eaf90e2a2ec3e90147eaa8772 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Fillion-Robin Date: Tue, 31 May 2016 03:28:41 -0400 Subject: [PATCH] PERF: Improve startup time by 8% with lazy loading of wrapped libraries *** WORK IN PROGRESS: For now, you have to make sure SlicerApp-real and Slicer launcher are built to ensure the successful generation of the json files *** Startup time reduced from 3.8s to 3.5s with a "cold cache" and from 2.7s to 2.38s with a "warm cache". For each logic/mrml/dm/widgets python modules, a json files listing the associated attributes is generated. Then, when the application is initialized, the "slicer" module is created as a "lazy" module with the attributes associated with logic/mrml/dm/widgets set as "not loaded". Finally, as soon as an attribute not yet loaded is accessed, the specialized __getattribute__ loads the associated python module and update the module dictionary. The "lazy" module has been adapted from "itkLazy.py" Results have been gathered on Ubuntu 15.10 on a workstation with the following specs: 64GB / M.2 PCIe NVMe SSD / Quad Core 3.80GHz --- Base/Python/CMakeLists.txt | 1 + Base/Python/lazy.py | 107 ++++++++++++++++++ Base/Python/slicer/__init__.py | 14 ++- Base/Python/slicer/util.py | 38 ++++--- Base/QTGUI/qSlicerLoadableModule.cxx | 8 +- CMake/SlicerMacroBuildModuleQtLibrary.cmake | 16 +++ ...licerMacroPythonWrapModuleVTKLibrary.cmake | 19 ++++ 7 files changed, 179 insertions(+), 24 deletions(-) create mode 100644 Base/Python/lazy.py diff --git a/Base/Python/CMakeLists.txt b/Base/Python/CMakeLists.txt index af7df247168..cf66931aecf 100644 --- a/Base/Python/CMakeLists.txt +++ b/Base/Python/CMakeLists.txt @@ -7,6 +7,7 @@ set(Slicer_PYTHON_SCRIPTS slicer/testing slicer/util freesurfer + lazy mrml saferef teem diff --git a/Base/Python/lazy.py b/Base/Python/lazy.py new file mode 100644 index 00000000000..8aacae9f15d --- /dev/null +++ b/Base/Python/lazy.py @@ -0,0 +1,107 @@ +import imp +import json +import os +import sys +import types + +not_loaded = 'not loaded' + +def library_loader(module_name): + #print("Loading %s" % module_name) + fp, pathname, description = imp.find_module(module_name) + module = imp.load_module(module_name, fp, pathname, description) + return module + +class LazyModule(types.ModuleType): + """Subclass of ModuleType that implements a custom __getattribute__ method + to allow lazy-loading of attributes from slicer sub-modules.""" + + def __init__(self, name): + types.ModuleType.__init__(self, name) + self.__lazy_attributes = {} + #print("__lazy_attributes: %s" % len(self.__lazy_attributes)) + + def _update_lazy_attributes(self, lazy_attributes): + self.__lazy_attributes.update(lazy_attributes) + for k in lazy_attributes: + setattr(self, k, not_loaded) + + def __getattribute__(self, attr): + value = types.ModuleType.__getattribute__(self, attr) + #print("__getattribute__ %s" % (attr)) + if value is not_loaded: + module_name = self.__lazy_attributes[attr] + + module = library_loader(module_name) + namespace = module.__dict__ + + # Load into 'namespace' first, then self.__dict__ (via setattr) to + # prevent the warnings about overwriting the 'NotLoaded' values + # already in self.__dict__ we would get if we just update + # self.__dict__. + for k, v in namespace.items(): + if not k.startswith('_'): + setattr(self, k, v) + value = namespace[attr] + return value + +def writeModuleAttributeFile(module_name, config_dir='.'): + try: + exec("import %s as module" % module_name) + except ImportError as details: + print("%s [skipped: failed to import: %s]" % (module_name, details)) + return + attributes = [] + for attr in dir(module): + if not attr.startswith('__'): + attributes.append(attr) + filename = os.path.join(config_dir, "%s.json" % module_name) + with open(filename, 'w') as output: + print("%s [done: %s]" % (module_name, filename)) + output.write(json.dumps({"attributes":attributes}, indent=4)) + +def updateLazyModule(module, input_module_names=[], config_dir=None): + if isinstance(module, basestring): + if module not in sys.modules: + print("updateLazyModule failed: Couldn't find %s module" % module) + return + module = sys.modules[module] + if not isinstance(module, LazyModule): + print("updateLazyModule failed: module '%s' is not a LazyModule" % module) + return + if isinstance(input_module_names, basestring): + input_module_names = [input_module_names] + if config_dir is None: + config_dir = os.path.dirname(module.__path__[0]) + for input_module_name in input_module_names: + filename = os.path.join(config_dir, "%s.json" % input_module_name) + with open(filename) as input: + module_attributes = json.load(input)['attributes'] + #print("Updating %s with %d attributes" % (filename, len(module_attributes))) + module._update_lazy_attributes({attribute: input_module_name for attribute in module_attributes}) + + #print("Updated %s module with %d attributes from %s" % (module, len(module._LazyModule__lazy_attributes), input_module_name)) + +def createLazyModule(module_name, module_path, input_module_names=[], config_dir=None): + + thisModule = sys.modules[module_name] if module_name in sys.modules else None + + if isinstance(thisModule, LazyModule): + # Handle reload case where we've already done this once. + # If we made a new module every time, multiple reload()s would fail + # because the identity of sys.modules['itk'] would always be changing. + #print("slicer: Calling ctor of LazyModule") + thisModule.__init__(module_name) + else: + print("slicer: Creating new LazyModule") + thisModule = LazyModule(module_name) + + # Set the __path__ attribute, which is required for this module to be used as a + # package + setattr(thisModule, '__path__', module_path) + + sys.modules[module_name] = thisModule + + updateLazyModule(thisModule, input_module_names, config_dir) + + return thisModule diff --git a/Base/Python/slicer/__init__.py b/Base/Python/slicer/__init__.py index 1ad906f3bab..ba1aeb533ce 100644 --- a/Base/Python/slicer/__init__.py +++ b/Base/Python/slicer/__init__.py @@ -1,5 +1,9 @@ """ This module sets up root logging and loads the Slicer library modules into its namespace.""" +import lazy +thisModule = lazy.createLazyModule(__name__, __path__) +del lazy + #----------------------------------------------------------------------------- def _createModule(name, globals, docstring): import imp @@ -14,14 +18,14 @@ def _createModule(name, globals, docstring): #----------------------------------------------------------------------------- # Create slicer.modules and slicer.moduleNames -_createModule('slicer.modules', globals(), +_createModule('slicer.modules', vars(thisModule), """This module provides an access to all instantiated Slicer modules. The module attributes are the lower-cased Slicer module names, the associated value is an instance of ``qSlicerAbstractCoreModule``. """) -_createModule('slicer.moduleNames', globals(), +_createModule('slicer.moduleNames', vars(thisModule), """This module provides an access to all instantiated Slicer module names. The module attributes are the Slicer modules names, the associated @@ -36,15 +40,19 @@ def _createModule(name, globals, docstring): except ImportError: available_kits = [] +from .util import importModuleObjects + for kit in available_kits: try: - exec "from %s import *" % (kit) + importModuleObjects(kit, thisModule) + #exec "from %s import *" % (kit) except ImportError as detail: print detail #----------------------------------------------------------------------------- # Cleanup: Removing things the user shouldn't have to see. +del thisModule del _createModule del available_kits del kit diff --git a/Base/Python/slicer/util.py b/Base/Python/slicer/util.py index a75ef75d0bc..34b234d6dbe 100644 --- a/Base/Python/slicer/util.py +++ b/Base/Python/slicer/util.py @@ -53,11 +53,11 @@ def sourceDir(): # Custom Import # -def importVTKClassesFromDirectory(directory, dest_module_name, filematch = '*'): - importClassesFromDirectory(directory, dest_module_name, 'vtkclass', filematch) +def importVTKClassesFromDirectory(directory, dest_module_name, filematch = '*', lazy=False): + importClassesFromDirectory(directory, dest_module_name, 'vtkclass', filematch, lazy) -def importQtClassesFromDirectory(directory, dest_module_name, filematch = '*'): - importClassesFromDirectory(directory, dest_module_name, 'PythonQtClassWrapper', filematch) +def importQtClassesFromDirectory(directory, dest_module_name, filematch = '*', lazy=False): + importClassesFromDirectory(directory, dest_module_name, 'PythonQtClassWrapper', filematch, lazy) # To avoid globbing multiple times the same directory, successful # call to ``importClassesFromDirectory()`` will be indicated by @@ -66,7 +66,7 @@ def importQtClassesFromDirectory(directory, dest_module_name, filematch = '*'): # Each entry is a tuple of form (directory, dest_module_name, type_name, filematch) __import_classes_cache = set() -def importClassesFromDirectory(directory, dest_module_name, type_name, filematch = '*'): +def importClassesFromDirectory(directory, dest_module_name, type_name, filematch = '*', lazy=False): # Create entry for __import_classes_cache cache_key = ",".join([directory, dest_module_name, type_name, filematch]) @@ -74,27 +74,31 @@ def importClassesFromDirectory(directory, dest_module_name, type_name, filematch if cache_key in __import_classes_cache: return - import glob, os, re, fnmatch + import glob, lazy, os, re, fnmatch re_filematch = re.compile(fnmatch.translate(filematch)) for fname in glob.glob(os.path.join(directory, filematch)): if not re_filematch.match(os.path.basename(fname)): continue - try: - from_module_name = os.path.splitext(os.path.basename(fname))[0] - importModuleObjects(from_module_name, dest_module_name, type_name) - except ImportError as detail: - import sys - print(detail, file=sys.stderr) + from_module_name = os.path.splitext(os.path.basename(fname))[0] + if lazy: + lazy.updateLazyModule(dest_module_name, from_module_name, os.path.dirname(fname)) + else: + try: + importModuleObjects(from_module_name, dest_module_name, type_name) + except ImportError as detail: + import sys + print(detail, file=sys.stderr) __import_classes_cache.add(cache_key) -def importModuleObjects(from_module_name, dest_module_name, type_name): +def importModuleObjects(from_module_name, dest_module, type_name='*'): """Import object of type 'type_name' from module identified - by 'from_module_name' into the module identified by 'dest_module_name'.""" + by 'from_module_name' into the module identified by 'dest_module'.""" - # Obtain a reference to the module identifed by 'dest_module_name' + # Obtain a reference to the module identifed by 'dest_module' import sys - dest_module = sys.modules[dest_module_name] + if isinstance(dest_module, basestring): + dest_module = sys.modules[dest_module] # Skip if module has already been loaded if from_module_name in sys.modules: @@ -112,7 +116,7 @@ def importModuleObjects(from_module_name, dest_module_name, type_name): item = getattr(module, item_name) # Add the object to dest_module_globals_dict if any - if type(item).__name__ == type_name: + if type(item).__name__ == type_name or (type_name == '*' and not item_name.startswith('_')): setattr(dest_module, item_name, item) # diff --git a/Base/QTGUI/qSlicerLoadableModule.cxx b/Base/QTGUI/qSlicerLoadableModule.cxx index 370050a081f..334c6739cd9 100644 --- a/Base/QTGUI/qSlicerLoadableModule.cxx +++ b/Base/QTGUI/qSlicerLoadableModule.cxx @@ -74,13 +74,13 @@ bool qSlicerLoadableModule::importModulePythonExtensions( ctkScopedCurrentDir scopedCurrentDir(QFileInfo(modulePath).absolutePath()); pythonManager->executeString(QString( "from slicer.util import importVTKClassesFromDirectory;" - "importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleLogicPython.*');" - "importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleMRMLPython.*');" - "importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleMRMLDisplayableManagerPython.*');" + "importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleLogicPython.*', lazy=True);" + "importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleMRMLPython.*', lazy=True);" + "importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleMRMLDisplayableManagerPython.*', lazy=True);" ).arg(scopedCurrentDir.currentPath())); pythonManager->executeString(QString( "from slicer.util import importQtClassesFromDirectory;" - "importQtClassesFromDirectory('%1', 'slicer', filematch='qSlicer*PythonQt.*');" + "importQtClassesFromDirectory('%1', 'slicer', filematch='qSlicer*PythonQt.*', lazy=True);" ).arg(scopedCurrentDir.currentPath())); return !pythonManager->pythonErrorOccured(); #else diff --git a/CMake/SlicerMacroBuildModuleQtLibrary.cmake b/CMake/SlicerMacroBuildModuleQtLibrary.cmake index d74dfa07053..efb72aefcc9 100644 --- a/CMake/SlicerMacroBuildModuleQtLibrary.cmake +++ b/CMake/SlicerMacroBuildModuleQtLibrary.cmake @@ -240,6 +240,22 @@ macro(SlicerMacroBuildModuleQtLibrary) if(NOT "${MODULEQTLIBRARY_FOLDER}" STREQUAL "") set_target_properties(${lib_name}PythonQt PROPERTIES FOLDER ${MODULEQTLIBRARY_FOLDER}) endif() + + # XXX Check if Slicer_LAUNCHER_EXECUTABLE available at during a clean build + # XXX Install .json file. Should be taking care of by ctkMacroCompilePythonScript + + # Add target to generate module attributes file to allow lazy loading + set(module_name "${lib_name}PythonQt") + set(config_dir "${CMAKE_BINARY_DIR}/${Slicer_QTLOADABLEMODULES_LIB_DIR}/") + set(code "import sys; sys.path.append('${Slicer_SOURCE_DIR}/Base/Python/');") + set(code "${code}import lazy;") + set(code "${code}lazy.writeModuleAttributeFile('${module_name}', config_dir='${config_dir}')") + add_custom_command(TARGET ${module_name} POST_BUILD + COMMAND ${Slicer_LAUNCHER_EXECUTABLE} --no-splash -c "${code}" + COMMENT "Generating ${module_name}.json" + VERBATIM + ) + endif() endmacro() diff --git a/CMake/SlicerMacroPythonWrapModuleVTKLibrary.cmake b/CMake/SlicerMacroPythonWrapModuleVTKLibrary.cmake index f718459bf3c..4c9673ea62b 100644 --- a/CMake/SlicerMacroPythonWrapModuleVTKLibrary.cmake +++ b/CMake/SlicerMacroPythonWrapModuleVTKLibrary.cmake @@ -81,4 +81,23 @@ macro(SlicerMacroPythonWrapModuleVTKLibrary) KIT_PYTHON_LIBRARIES ${PYTHONWRAPMODULEVTKLIBRARY_Wrapped_LIBRARIES} ) + # XXX Check if Slicer_LAUNCHER_EXECUTABLE available at during a clean build + # XXX Install .json file. Should be taking care of by ctkMacroCompilePythonScript + + # Get path to real executable + get_filename_component(python_bin_dir ${PYTHON_EXECUTABLE} PATH) + set(real_python_executable ${python_bin_dir}/python${CMAKE_EXECUTABLE_SUFFIX}) + + # Add target to generate module attributes file to allow lazy loading + set(module_name "${PYTHONWRAPMODULEVTKLIBRARY_NAME}Python") + set(config_dir "${CMAKE_BINARY_DIR}/${Slicer_QTLOADABLEMODULES_LIB_DIR}/") + set(code "import sys; sys.path.append('${Slicer_SOURCE_DIR}/Base/Python/');") + set(code "${code}import lazy;") + set(code "${code}lazy.writeModuleAttributeFile('${module_name}', config_dir='${config_dir}')") + add_custom_command(TARGET ${module_name} POST_BUILD + COMMAND ${Slicer_LAUNCHER_EXECUTABLE} --launch ${real_python_executable} -c "${code}" + COMMENT "Generating ${module_name}.json" + VERBATIM + ) + endmacro()