From cdfd7aaaec2eade727a1f166d87bab098d34528a Mon Sep 17 00:00:00 2001 From: Kay Hayen Date: Fri, 27 Mar 2026 16:45:10 +0100 Subject: [PATCH] Plugins: Performance enhancements * Added option to show how much time plugin hooks consume and how often they get called, such that we can identify bottlenecks in plugins. * The anti-bloat "decideAssertions" overload was misnamed, but we don't use it yet anyway. * For some frequently used plugin hooks, track which ones overload the method in question and only iterate over those, instead of all active plugins. * Allow registering what implicit imports a plugin has to make, so it doesn't have to be called all the time. --- doc/nuitka-run.1 | 3 + doc/nuitka.1 | 3 + nuitka/MainControl.py | 4 + nuitka/States.py | 4 + nuitka/options/OptionParsing.py | 9 + nuitka/options/Options.py | 6 + nuitka/plugins/Hooks.py | 7 + nuitka/plugins/PluginBase.py | 21 +- nuitka/plugins/Plugins.py | 259 +++++++++++++++------ nuitka/plugins/PluginsUsage.py | 69 ++++++ nuitka/plugins/standard/AntiBloatPlugin.py | 9 +- 11 files changed, 325 insertions(+), 69 deletions(-) create mode 100644 nuitka/plugins/PluginsUsage.py diff --git a/doc/nuitka-run.1 b/doc/nuitka-run.1 index 18e831c4e..05c6acc2f 100644 --- a/doc/nuitka-run.1 +++ b/doc/nuitka-run.1 @@ -346,6 +346,9 @@ Provide memory information and statistics. Defaults to off. \fB\-\-show\-modules\fR Provide information for included modules and DLLs Obsolete: You should use '\-\-report' file instead. Defaults to off. .TP +\fB\-\-show\-plugin\-usage\fR +Provide information on plugin usage. Defaults to off. +.TP \fB\-\-show\-modules\-output\fR=\fI\,PATH\/\fR Where to output '\-\-show\-modules', should be a filename. Default is standard output. .TP diff --git a/doc/nuitka.1 b/doc/nuitka.1 index dddec8ff5..82beba67a 100644 --- a/doc/nuitka.1 +++ b/doc/nuitka.1 @@ -346,6 +346,9 @@ Provide memory information and statistics. Defaults to off. \fB\-\-show\-modules\fR Provide information for included modules and DLLs Obsolete: You should use '\-\-report' file instead. Defaults to off. .TP +\fB\-\-show\-plugin\-usage\fR +Provide information on plugin usage. Defaults to off. +.TP \fB\-\-show\-modules\-output\fR=\fI\,PATH\/\fR Where to output '\-\-show\-modules', should be a filename. Default is standard output. .TP diff --git a/nuitka/MainControl.py b/nuitka/MainControl.py index 4a84c5217..004fb1a5c 100644 --- a/nuitka/MainControl.py +++ b/nuitka/MainControl.py @@ -127,6 +127,7 @@ from nuitka.plugins.Hooks import ( onStandaloneDistributionFinished, writeExtraCodeFiles, ) +from nuitka.plugins.PluginsUsage import printPluginUsageStats from nuitka.PostProcessing import executePostProcessing from nuitka.Progress import ( closeProgressBar, @@ -1213,6 +1214,8 @@ def _main(): if isShowMemory(): showMemoryTrace() + printPluginUsageStats() + sys.exit(0) executePostProcessing(scons_options["result_exe"]) @@ -1330,6 +1333,7 @@ exist, out e.g. '--output-dir=output' to sure is importable.""" % base_path, createDmgFile(general) writeCompilationReports(aborted=False) + printPluginUsageStats() run_filename = OutputDirectories.getResultRunFilename(onefile=isOnefileMode()) diff --git a/nuitka/States.py b/nuitka/States.py index ca01dc99a..72d0490f8 100644 --- a/nuitka/States.py +++ b/nuitka/States.py @@ -14,6 +14,8 @@ as a global instance. class GlobalState(object): """The global state of Nuitka compilation.""" + # pylint: disable=too-many-instance-attributes + __slots__ = ( "is_debug", "is_non_debug", @@ -22,6 +24,7 @@ class GlobalState(object): "report_missing_trust", "is_verbose", "data_composer_verbose", + "show_plugin_usage", ) def __init__(self): @@ -32,6 +35,7 @@ class GlobalState(object): self.report_missing_trust = None self.is_verbose = None self.data_composer_verbose = None + self.show_plugin_usage = None states = GlobalState() diff --git a/nuitka/options/OptionParsing.py b/nuitka/options/OptionParsing.py index 1fc66de37..0b2dd0d13 100644 --- a/nuitka/options/OptionParsing.py +++ b/nuitka/options/OptionParsing.py @@ -1656,6 +1656,15 @@ Provide information for included modules and DLLs Obsolete: You should use '--report' file instead. Defaults to off.""", ) +tracing_group.add_option( + "--show-plugin-usage", + action="store_true", + dest="show_plugin_usage", + github_action=False, + default=False, + help="""Provide information on plugin usage. Defaults to off.""", +) + tracing_group.add_option( "--show-modules-output", action="store", diff --git a/nuitka/options/Options.py b/nuitka/options/Options.py index d56f14769..f18e08a29 100644 --- a/nuitka/options/Options.py +++ b/nuitka/options/Options.py @@ -448,6 +448,7 @@ longer part of Winlibs and therefore no more available this way. Use only \ states.is_verbose = options.verbose states.data_composer_verbose = options.data_composer_verbose + states.show_plugin_usage = options.show_plugin_usage optimization_logger.is_quiet = not options.verbose @@ -2093,6 +2094,11 @@ def isShowInclusion(): return options.show_inclusion +def isShowPluginUsage(): + """:returns: bool derived from ``--show-plugin-usage``""" + return options is not None and options.show_plugin_usage + + def isRemoveBuildDir(): """:returns: bool derived from ``--remove-output``""" return options.remove_build and not options.generate_c_only diff --git a/nuitka/plugins/Hooks.py b/nuitka/plugins/Hooks.py index c328ecf57..bc7d122a1 100644 --- a/nuitka/plugins/Hooks.py +++ b/nuitka/plugins/Hooks.py @@ -153,6 +153,13 @@ def decideCompilation(module_name): return Plugins.decideCompilation(module_name=module_name) +def registerDecisionCompilation(plugin_name, module_name, decision): + """Let plugins register a decision whether to C compile a module or include as bytecode.""" + return Plugins.registerDecisionCompilation( + plugin_name=plugin_name, module_name=module_name, decision=decision + ) + + def decideRecompileExtensionModules(module_name): """Let plugins decide whether to re-compile an extension module from source code. diff --git a/nuitka/plugins/PluginBase.py b/nuitka/plugins/PluginBase.py index 49024c2be..ac4212e21 100644 --- a/nuitka/plugins/PluginBase.py +++ b/nuitka/plugins/PluginBase.py @@ -110,7 +110,12 @@ from nuitka.utils.Utils import ( withNoWarning, ) -from .Hooks import decideAnnotations, decideAssertions, decideDocStrings +from .Hooks import ( + decideAnnotations, + decideAssertions, + decideDocStrings, + registerDecisionCompilation, +) _warned_unused_plugins = set() @@ -1219,6 +1224,20 @@ Unwanted import of '%(unwanted)s' that %(problem)s '%(binding_name)s' encountere # Virtual method, pylint: disable=no-self-use,unused-argument return None + def registerDecisionCompilation(self, module_name, decision): + """Register a decision whether to C compile a module or include as bytecode. + + Notes: + Registers the decision statically to be considered during compilation. + + Args: + module_name: (str) name of module + decision: "compiled" or "bytecode" + """ + registerDecisionCompilation( + plugin_name=self.plugin_name, module_name=module_name, decision=decision + ) + def decideRecompileExtensionModules(self, module_name): # Virtual method, pylint: disable=no-self-use,unused-argument return None diff --git a/nuitka/plugins/Plugins.py b/nuitka/plugins/Plugins.py index f52454346..3d3330051 100644 --- a/nuitka/plugins/Plugins.py +++ b/nuitka/plugins/Plugins.py @@ -28,7 +28,7 @@ from nuitka.containers.OrderedSets import OrderedSet from nuitka.Errors import NuitkaForbiddenImportEncounter, NuitkaSyntaxError from nuitka.freezer.IncludedDataFiles import IncludedDataFile from nuitka.freezer.IncludedEntryPoints import IncludedEntryPoint -from nuitka.importing.Importing import getModuleNameAndKindFromFilename +from nuitka.importing.Importing import locateModule from nuitka.importing.Recursion import decideRecursion, recurseTo from nuitka.ModuleRegistry import ( addUsedModule, @@ -74,9 +74,15 @@ from nuitka.utils.ModuleNames import ( from nuitka.Version import getCommercialVersion from .PluginBase import NuitkaPluginBase, control_tags +from .PluginsUsage import counted_plugin_method # Maps plugin name to plugin instances. active_plugins = OrderedDict() +active_plugins_with_implicit_imports = [] +active_plugins_with_decide_compilation = [] +active_plugins_with_decide_annotations = [] +active_plugins_with_decide_doc_strings = [] +active_plugins_with_decide_assertions = [] plugin_name2plugin_classes = {} plugin_options = OrderedDict() plugin_values = {} @@ -139,6 +145,30 @@ def _addActivePlugin(plugin_class, args, force=False): active_plugins[plugin_name] = plugin_instance + if ( + type(plugin_instance).getImplicitImports + is not NuitkaPluginBase.getImplicitImports + ): + active_plugins_with_implicit_imports.append(plugin_instance) + + if ( + type(plugin_instance).decideCompilation + is not NuitkaPluginBase.decideCompilation + ): + active_plugins_with_decide_compilation.append(plugin_instance) + + if ( + type(plugin_instance).decideAnnotations + is not NuitkaPluginBase.decideAnnotations + ): + active_plugins_with_decide_annotations.append(plugin_instance) + + if type(plugin_instance).decideDocStrings is not NuitkaPluginBase.decideDocStrings: + active_plugins_with_decide_doc_strings.append(plugin_instance) + + if type(plugin_instance).decideAssertions is not NuitkaPluginBase.decideAssertions: + active_plugins_with_decide_assertions.append(plugin_instance) + is_gui_toolkit_plugin = getattr(plugin_class, "plugin_gui_toolkit", False) # Singleton, pylint: disable=global-statement @@ -430,6 +460,7 @@ class Plugins(object): extra_scan_paths_cache = {} @staticmethod + @counted_plugin_method def _considerImplicitImports(plugin, module): result = [] @@ -479,7 +510,9 @@ class Plugins(object): continue try: - module_filename = plugin.locateModule(full_name) + _module_name, module_filename, module_kind, _finding = locateModule( + module_name=full_name, parent_package=None, level=0 + ) except Exception: plugin.warning( "Problem locating '%s' for implicit imports of '%s'." @@ -496,7 +529,7 @@ class Plugins(object): continue - result.append((full_name, module_filename)) + result.append((full_name, module_filename, module_kind)) if result and isShowInclusion(): plugin.info( @@ -507,12 +540,9 @@ class Plugins(object): return result @staticmethod + @counted_plugin_method def _reportImplicitImports(plugin, module, implicit_imports): - for full_name, module_filename in implicit_imports: - # TODO: The module_kind should be forwarded from previous in the class using locateModule code. - _module_name2, module_kind = getModuleNameAndKindFromFilename( - module_filename - ) + for full_name, module_filename, module_kind in implicit_imports: # This will get back to all other plugins allowing them to inhibit it though. decision, decision_reason = decideRecursion( @@ -533,7 +563,7 @@ class Plugins(object): using_module_name=module.module_name, ) except NuitkaForbiddenImportEncounter as e: - plugin.sysexit( + return plugin.sysexit( """\ Error, forbidden import of '%s' (intending to avoid '%s') in module '%s' \ through implicit import by '%s' plugin encountered.""" @@ -561,6 +591,7 @@ through implicit import by '%s' plugin encountered.""" yield path @classmethod + @counted_plugin_method def getPackageExtraScanPaths(cls, package_name, package_dir): key = package_name, package_dir @@ -579,6 +610,7 @@ through implicit import by '%s' plugin encountered.""" return cls.extra_scan_paths_cache[key] @classmethod + @counted_plugin_method def considerImplicitImports(cls, module): """Let plugins add implicit imports for a module. @@ -588,7 +620,7 @@ through implicit import by '%s' plugin encountered.""" iterable of module names """ - for plugin in getActivePlugins(): + for plugin in active_plugins_with_implicit_imports: key = (module.getFullName(), plugin) if key not in cls.implicit_imports_cache: @@ -597,11 +629,12 @@ through implicit import by '%s' plugin encountered.""" cls._considerImplicitImports(plugin=plugin, module=module) ) - cls._reportImplicitImports( - plugin=plugin, - module=module, - implicit_imports=cls.implicit_imports_cache[key], - ) + if cls.implicit_imports_cache[key]: + cls._reportImplicitImports( + plugin=plugin, + module=module, + implicit_imports=cls.implicit_imports_cache[key], + ) # Pre and post load code may have been created, if so indicate it's used. full_name = module.getFullName() @@ -635,6 +668,7 @@ through implicit import by '%s' plugin encountered.""" ) @staticmethod + @counted_plugin_method def onCopiedDLLs(dist_dir, standalone_entry_points): """Lets the plugins modify entry points on disk.""" for entry_point in standalone_entry_points: @@ -648,12 +682,14 @@ through implicit import by '%s' plugin encountered.""" plugin.onCopiedDLL(dll_path) @staticmethod + @counted_plugin_method def onBeforeCodeParsing(): """Let plugins prepare for code parsing""" for plugin in getActivePlugins(): plugin.onBeforeCodeParsing() @staticmethod + @counted_plugin_method def onStandaloneDistributionFinished(dist_dir, standalone_binary): """Let plugins post-process the distribution folder in standalone mode""" for plugin in getActivePlugins(): @@ -663,24 +699,28 @@ through implicit import by '%s' plugin encountered.""" plugin.onStandaloneBinary(standalone_binary) @staticmethod + @counted_plugin_method def onGeneratedSourceCode(source_dir, onefile): """Let plugins modify the generated source code""" for plugin in getActivePlugins(): plugin.onGeneratedSourceCode(source_dir, onefile) @staticmethod + @counted_plugin_method def onOnefileFinished(filename): """Let plugins post-process the onefile executable in onefile mode""" for plugin in getActivePlugins(): plugin.onOnefileFinished(filename) @staticmethod + @counted_plugin_method def onBootstrapBinary(filename): """Let plugins add to bootstrap binary in some way""" for plugin in getActivePlugins(): plugin.onBootstrapBinary(filename) @staticmethod + @counted_plugin_method def onFinalResult(filename): """Let plugins add to final binary in some way""" for plugin in getActivePlugins(): @@ -724,6 +764,7 @@ through implicit import by '%s' plugin encountered.""" return result @staticmethod + @counted_plugin_method def getModuleSpecificDllPaths(module_name): """Provide a list of directories, where DLLs should be searched for this package (or module). @@ -741,6 +782,7 @@ through implicit import by '%s' plugin encountered.""" _uncompiled_decorator_names = None @classmethod + @counted_plugin_method def getUncompiledDecoratorNames(cls): """Provide a list of decorators that should cause a function to be uncompiled. @@ -761,6 +803,7 @@ through implicit import by '%s' plugin encountered.""" sys_path_additions_cache = {} @classmethod + @counted_plugin_method def getModuleSysPathAdditions(cls, module_name): """Provide a list of directories, that should be considered in 'PYTHONPATH' when this module is used. @@ -780,6 +823,7 @@ through implicit import by '%s' plugin encountered.""" return cls.sys_path_additions_cache[module_name] @staticmethod + @counted_plugin_method def removeDllDependencies(dll_filename, dll_filenames): """Create list of removable shared libraries by scanning through the plugins. @@ -838,11 +882,13 @@ through implicit import by '%s' plugin encountered.""" yield included_datafile @staticmethod + @counted_plugin_method def onDataFileTags(included_datafile): for plugin in getActivePlugins(): plugin.onDataFileTags(included_datafile) @staticmethod + @counted_plugin_method def onDllTags(included_entry_point): for plugin in getActivePlugins(): plugin.onDllTags(included_entry_point) @@ -1061,6 +1107,7 @@ through implicit import by '%s' plugin encountered.""" cls.onModuleDiscovered(fake_module) @staticmethod + @counted_plugin_method def onModuleSourceCode(module_name, source_filename, source_code): assert type(module_name) is ModuleName assert type(source_code) is str @@ -1083,6 +1130,7 @@ through implicit import by '%s' plugin encountered.""" return source_code, contributing_plugins @staticmethod + @counted_plugin_method def onFrozenModuleBytecode(module_name, is_package, bytecode): assert type(module_name) is ModuleName assert bytecode.__class__.__name__ == "code" @@ -1094,6 +1142,7 @@ through implicit import by '%s' plugin encountered.""" return bytecode @staticmethod + @counted_plugin_method def onModuleEncounter(using_module_name, module_name, module_filename, module_kind): result = None deciding_plugins = [] @@ -1151,8 +1200,6 @@ Error, follow decision '%s' for module '%s' of plugin '%s' does not match other # Do parent package look ahead first. parent_package_name = module_name.getPackageName() if parent_package_name is not None: - from nuitka.importing.Importing import locateModule - ( _parent_package_name, parent_module_filename, @@ -1208,6 +1255,7 @@ Error, follow decision '%s' for module '%s' of plugin '%s' does not match other pass @staticmethod + @counted_plugin_method def onModuleRecursion( module_name, module_filename, module_kind, using_module_name, source_ref, reason ): @@ -1222,6 +1270,7 @@ Error, follow decision '%s' for module '%s' of plugin '%s' does not match other ) @staticmethod + @counted_plugin_method def onCompilationStartChecks(): """The compilation is setup, locating modules if expected to work.""" @@ -1230,6 +1279,7 @@ Error, follow decision '%s' for module '%s' of plugin '%s' does not match other plugin.onCompilationStartChecks() @staticmethod + @counted_plugin_method def onModuleInitialSet(): """The initial set of root modules is complete, plugins may add more.""" @@ -1240,6 +1290,7 @@ Error, follow decision '%s' for module '%s' of plugin '%s' does not match other addRootModule(module) @staticmethod + @counted_plugin_method def considerIncompleteModuleSet(): """The module set is incomplete, giving plugins a chance to add more.""" @@ -1265,6 +1316,11 @@ Error, follow decision '%s' for module '%s' of plugin '%s' does not match other ): del modules_to_add[module_namespace_to_add] + if modules_to_add: + Plugins._addIncompleteModules(modules_to_add) + + @staticmethod + def _addIncompleteModules(modules_to_add): for module in getDoneModules(): for module_usage_attempt in module.getUsedModules(): if module_usage_attempt.filename is not None: @@ -1313,6 +1369,7 @@ through incomplete set import by '%s' plugin encountered.""" break @staticmethod + @counted_plugin_method def onModuleCompleteSet(): """The final set of modules is determined, this is only for inspection, cannot change.""" @@ -1323,6 +1380,7 @@ through incomplete set import by '%s' plugin encountered.""" plugin.onModuleCompleteSet(module_set) @staticmethod + @counted_plugin_method def suppressUnknownImportWarning(importing, source_ref, module_name): """Let plugins decide whether to suppress import warnings for an unknown module. @@ -1343,26 +1401,42 @@ through incomplete set import by '%s' plugin encountered.""" return False - @staticmethod - def decideCompilation(module_name): + registered_compilation_decisions = {} + + @classmethod + def registerDecisionCompilation(cls, plugin_name, module_name, decision): + if type(module_name) is str: + module_name = ModuleName(module_name) + + if module_name not in cls.registered_compilation_decisions: + cls.registered_compilation_decisions[module_name] = OrderedDict() + + cls.registered_compilation_decisions[module_name][plugin_name] = decision + + @classmethod + @counted_plugin_method + def decideCompilation(cls, module_name): """Let plugins decide whether to C compile a module or include as bytecode. Notes: - The decision is made by the first plugin not returning None. + The decision is made by plugins answering, with collision checks + if multiple plugins provide conflicting decisions. Returns: "compiled" (default) or "bytecode". """ - for plugin in getActivePlugins(): - value = plugin.decideCompilation(module_name) - - if value is not None: - assert value in ("compiled", "bytecode") - return value - - return None + return cls._decideWithoutDisagreement( + method_name="decideCompilation", + call_per_plugin=lambda plugin: plugin.decideCompilation(module_name), + legal_values=("compiled", "bytecode", None), + abstain_values=(None,), + default_value=None, + plugins_list=active_plugins_with_decide_compilation, + registered_values=cls.registered_compilation_decisions.get(module_name), + ) @staticmethod + @counted_plugin_method def decideRecompileExtensionModules(module_name): """Let plugins decide whether to re-compile an extension module from source code. @@ -1399,6 +1473,7 @@ through incomplete set import by '%s' plugin encountered.""" preprocessor_symbols = None @classmethod + @counted_plugin_method def getPreprocessorSymbols(cls): """Let plugins provide C defines to be used in compilation. @@ -1433,6 +1508,7 @@ through incomplete set import by '%s' plugin encountered.""" build_definitions = None @classmethod + @counted_plugin_method def getBuildDefinitions(cls): """Let plugins provide C source defines to be used in compilation. @@ -1465,6 +1541,7 @@ through incomplete set import by '%s' plugin encountered.""" extra_include_directories = None @classmethod + @counted_plugin_method def getExtraIncludeDirectories(cls): """Let plugins extra directories to use for C includes in compilation. @@ -1487,6 +1564,7 @@ through incomplete set import by '%s' plugin encountered.""" return cls.extra_include_directories @staticmethod + @counted_plugin_method def _getExtraCodeFiles(for_onefile): result = OrderedDict() @@ -1539,6 +1617,7 @@ through incomplete set import by '%s' plugin encountered.""" extra_link_libraries = None @classmethod + @counted_plugin_method def getExtraLinkLibraries(cls): if cls.extra_link_libraries is None: cls.extra_link_libraries = OrderedSet() @@ -1558,6 +1637,7 @@ through incomplete set import by '%s' plugin encountered.""" extra_link_directories = None @classmethod + @counted_plugin_method def getExtraLinkDirectories(cls): if cls.extra_link_directories is None: cls.extra_link_directories = OrderedSet() @@ -1575,11 +1655,13 @@ through incomplete set import by '%s' plugin encountered.""" return cls.extra_link_directories @classmethod + @counted_plugin_method def onDataComposerRun(cls): for plugin in getActivePlugins(): plugin.onDataComposerRun() @classmethod + @counted_plugin_method def onDataComposerResult(cls, blob_filename): for plugin in getActivePlugins(): plugin.onDataComposerResult(blob_filename) @@ -1591,6 +1673,7 @@ through incomplete set import by '%s' plugin encountered.""" return cls.encodeDataComposerName(result) @classmethod + @counted_plugin_method def encodeDataComposerName(cls, name): # Encoding needs to match generated source code output. if str is not bytes: @@ -1606,6 +1689,7 @@ through incomplete set import by '%s' plugin encountered.""" return name @classmethod + @counted_plugin_method def onFunctionBodyParsing(cls, provider, function_name, body): module_name = provider.getParentModule().getFullName() @@ -1622,6 +1706,7 @@ through incomplete set import by '%s' plugin encountered.""" ) @classmethod + @counted_plugin_method def onClassBodyParsing(cls, provider, class_name, node): module_name = provider.getParentModule().getFullName() @@ -1634,15 +1719,26 @@ through incomplete set import by '%s' plugin encountered.""" node=node, ) + cache_contribution_values_cache = {} + @classmethod + @counted_plugin_method def getPluginsCacheContributionValues(cls, module_name): """Let plugins provide values that need to be taken into account for caching.""" - for plugin in getActivePlugins(): - for value in plugin.getCacheContributionValues(module_name): - yield value + if module_name not in cls.cache_contribution_values_cache: + result = [] + + for plugin in getActivePlugins(): + for value in plugin.getCacheContributionValues(module_name): + result.append(value) + + cls.cache_contribution_values_cache[module_name] = tuple(result) + + return cls.cache_contribution_values_cache[module_name] @classmethod + @counted_plugin_method def getExtraConstantDefaultPopulation(cls): for plugin in getActivePlugins(): for value in plugin.getExtraConstantDefaultPopulation(): @@ -1655,34 +1751,53 @@ through incomplete set import by '%s' plugin encountered.""" call_per_plugin, legal_values, abstain_values, - get_default_value, + default_value, + plugins_list, + registered_values, ): - result = abstain_values[0] - plugin_name = None + per_plugin_decisions = [] - for plugin in getActivePlugins(): + if registered_values is not None: + for deciding_plugin_name, value in registered_values.items(): + if value not in legal_values: + return plugins_logger.sysexit( + "Error, can only register '%s' for '%s' not %r" + % (legal_values, method_name, value) + ) + + if value not in abstain_values: + per_plugin_decisions.append( + (deciding_plugin_name, value, plugins_logger) + ) + + for plugin in plugins_list: value = call_per_plugin(plugin) if value not in legal_values: - plugin.sysexit( + return plugin.sysexit( "Error, can only return '%s' from '%s' not %r" % (legal_values, method_name, value) ) - if value in abstain_values: - continue + if value not in abstain_values: + per_plugin_decisions.append((plugin.plugin_name, value, plugin)) + result = abstain_values[0] + plugin_name = None + + for deciding_plugin_name, value, deciding_plugin_logger in per_plugin_decisions: if value != result: if result in abstain_values: result = value - plugin_name = plugin.plugin_name + plugin_name = deciding_plugin_name else: - plugin.sysexit( + return deciding_plugin_logger.sysexit( "Error, conflicting value '%s' with plug-in '%s' value '%s'." % (value, plugin_name, result) ) + if result in abstain_values: - result = get_default_value() + result = default_value return result @@ -1691,17 +1806,23 @@ through incomplete set import by '%s' plugin encountered.""" @classmethod def decideAnnotations(cls, module_name): # For Python2 it's not a thing. - if str is bytes: - return False - if module_name not in cls.decide_annotations_cache: - cls.decide_annotations_cache[module_name] = cls._decideWithoutDisagreement( - call_per_plugin=lambda plugin: plugin.decideAnnotations(module_name), - legal_values=(None, True, False), - abstain_values=(None,), - method_name="decideAnnotations", - get_default_value=lambda: not hasPythonFlagNoAnnotations(), - ) + if str is bytes: + cls.decide_annotations_cache[module_name] = False + else: + cls.decide_annotations_cache[module_name] = ( + cls._decideWithoutDisagreement( + call_per_plugin=lambda plugin: plugin.decideAnnotations( + module_name + ), + legal_values=(None, True, False), + abstain_values=(None,), + method_name="decideAnnotations", + default_value=not hasPythonFlagNoAnnotations(), + plugins_list=active_plugins_with_decide_annotations, + registered_values=None, + ) + ) return cls.decide_annotations_cache[module_name] @@ -1715,7 +1836,9 @@ through incomplete set import by '%s' plugin encountered.""" legal_values=(None, True, False), abstain_values=(None,), method_name="decideDocStrings", - get_default_value=lambda: not hasPythonFlagNoDocStrings(), + default_value=not hasPythonFlagNoDocStrings(), + plugins_list=active_plugins_with_decide_doc_strings, + registered_values=None, ) return cls.decide_doc_strings_cache[module_name] @@ -1730,12 +1853,15 @@ through incomplete set import by '%s' plugin encountered.""" legal_values=(None, True, False), abstain_values=(None,), method_name="decideAssertions", - get_default_value=lambda: not hasPythonFlagNoAsserts(), + default_value=not hasPythonFlagNoAsserts(), + plugins_list=active_plugins_with_decide_assertions, + registered_values=None, ) return cls.decide_assertions_cache[module_name] @classmethod + @counted_plugin_method def decideAllowOutsideDependencies(cls, module_name): result = None plugin_name = None @@ -1747,7 +1873,7 @@ through incomplete set import by '%s' plugin encountered.""" if value is True: if result is False: - plugin.sysexit( + return plugin.sysexit( "Error, conflicting allow/disallow outside dependencies of plug-in '%s'." % plugin_name ) @@ -1757,7 +1883,7 @@ through incomplete set import by '%s' plugin encountered.""" elif value is False: if result is False: - plugin.sysexit( + return plugin.sysexit( "Error, conflicting allow/disallow outside dependencies of plug-in '%s'." % plugin_name ) @@ -1765,7 +1891,7 @@ through incomplete set import by '%s' plugin encountered.""" result = False plugin_name = plugin.plugin_name elif value is not None: - plugin.sysexit( + return plugin.sysexit( "Error, can only return True, False, None from 'decideAllowOutsideDependencies' not %r" % value ) @@ -1773,6 +1899,7 @@ through incomplete set import by '%s' plugin encountered.""" return result @classmethod + @counted_plugin_method def isAcceptableMissingDLL(cls, package_name, filename): dll_basename = getDllBasename(os.path.basename(filename)) @@ -1791,7 +1918,7 @@ through incomplete set import by '%s' plugin encountered.""" if value is True: if result is False: - plugin.sysexit( + return plugin.sysexit( "Error, conflicting accept/reject missing DLLs of plug-in '%s'." % plugin_name ) @@ -1801,7 +1928,7 @@ through incomplete set import by '%s' plugin encountered.""" elif value is False: if result is False: - plugin.sysexit( + return plugin.sysexit( "Error, conflicting accept/reject missing DLLs of plug-in '%s'." % plugin_name ) @@ -1809,7 +1936,7 @@ through incomplete set import by '%s' plugin encountered.""" result = False plugin_name = plugin.plugin_name elif value is not None: - plugin.sysexit( + return plugin.sysexit( "Error, can only return True, False, None from 'isAcceptableMissingDLL' not %r" % value ) @@ -1889,7 +2016,7 @@ def loadUserPlugin(plugin_filename): None """ if not os.path.exists(plugin_filename): - plugins_logger.sysexit("Error, cannot find '%s'." % plugin_filename) + return plugins_logger.sysexit("Error, cannot find '%s'." % plugin_filename) user_plugin_module = importFileAsModule(plugin_filename) @@ -1908,7 +2035,9 @@ def loadUserPlugin(plugin_filename): break # do not look for more in that module if not valid_file: # this is not a plugin file ... - plugins_logger.sysexit("Error, '%s' is not a plugin file." % plugin_filename) + return plugins_logger.sysexit( + "Error, '%s' is not a plugin file." % plugin_filename + ) return plugin_class @@ -1997,12 +2126,12 @@ def activatePlugins(): # ensure plugin is known and not both, enabled and disabled for plugin_name in getPluginsEnabled() + getPluginsDisabled(): if plugin_name not in plugin_name2plugin_classes: - plugins_logger.sysexit( + return plugins_logger.sysexit( "Error, unknown plug-in '%s' referenced." % plugin_name ) if plugin_name in getPluginsEnabled() and plugin_name in getPluginsDisabled(): - plugins_logger.sysexit( + return plugins_logger.sysexit( "Error, conflicting enable/disable of plug-in '%s'." % plugin_name ) @@ -2081,7 +2210,7 @@ def _addPluginCommandLineOptions(parser, plugin_class, plugin_help_mode): e.option_id in other_plugin_option._long_opts or other_plugin_option._short_opts ): - plugins_logger.sysexit( + return plugins_logger.sysexit( "Plugin '%s' failed to add options due to conflict with '%s' from plugin '%s." % (plugin_name, e.option_id, other_plugin_name) ) @@ -2159,7 +2288,7 @@ def getPluginOptions(plugin_name): if "[REQUIRED]" in option.help: if not arg_value: - plugins_logger.sysexit( + return plugins_logger.sysexit( "Error, required plugin argument '%s' of Nuitka plugin '%s' not given." % (option_name, plugin_name) ) diff --git a/nuitka/plugins/PluginsUsage.py b/nuitka/plugins/PluginsUsage.py new file mode 100644 index 000000000..a8c94227f --- /dev/null +++ b/nuitka/plugins/PluginsUsage.py @@ -0,0 +1,69 @@ +# Copyright 2026, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file + + +"""Plugins usage statistics""" + +from nuitka.States import states +from nuitka.Tracing import printIndented, printLine +from nuitka.utils.Timing import TimerReport + +counted_plugin_methods = {} + + +def counted_plugin_method(plugin_method): + name = "Plugins." + plugin_method.__name__ + + def wrapped_plugin_method(*args, **kw): + if states.show_plugin_usage: + if name not in counted_plugin_methods: + counted_plugin_methods[name] = [0, 0.0] + + counted_plugin_methods[name][0] += 1 + + timer_report = TimerReport( + message="", decider=False, include_sleep_time=False + ) + with timer_report: + result = plugin_method(*args, **kw) + + counted_plugin_methods[name][1] += timer_report.getTimer().getDelta() + + return result + + return plugin_method(*args, **kw) + + return wrapped_plugin_method + + +def printPluginUsageStats(): + if not states.show_plugin_usage: + return + + if counted_plugin_methods: + printLine("Plugin method calls:") + + for name, (count, total_time) in sorted( + counted_plugin_methods.items(), key=lambda x: x[1][1], reverse=True + ): + average_time = total_time / count if count > 0 else 0.0 + printIndented( + 1, + "%s calls: %d, total time: %.3fs, avg time: %.5fs" + % (name, count, total_time, average_time), + ) + + +# Part of "Nuitka", an optimizing Python compiler that is compatible and +# integrates with CPython, but also works on its own. +# +# Licensed under the GNU Affero General Public License, Version 3 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.gnu.org/licenses/agpl.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nuitka/plugins/standard/AntiBloatPlugin.py b/nuitka/plugins/standard/AntiBloatPlugin.py index 48da1f720..6329d0f2b 100644 --- a/nuitka/plugins/standard/AntiBloatPlugin.py +++ b/nuitka/plugins/standard/AntiBloatPlugin.py @@ -272,6 +272,9 @@ instead of '--noinclude-custom-mode=%s'""" % (module_name, custom_choice)) # Cache execution context for anti-bloat configs. self.context_codes = {} + # Precompute this for getCacheContributionValues to avoid sorting each time + self.handled_modules_hash = str(tuple(sorted(self.handled_modules.items()))) + def getEvaluationConditionControlTags(self): return self.control_tags @@ -284,7 +287,7 @@ instead of '--noinclude-custom-mode=%s'""" % (module_name, custom_choice)) # TODO: Until we can change the evaluation to tell us exactly what # control tag values were used, we have to make this one. We sort # the values, to try and have order changes in code not matter. - yield str(tuple(sorted(self.handled_modules.items()))) + yield self.handled_modules_hash @classmethod def addPluginCommandLineOptions(cls, group): @@ -809,7 +812,7 @@ class %(class_name)s: return None - def decideAsserts(self, module_name): + def decideAssertions(self, module_name): # Finding a matching configuration aborts the search, not finding one # means default behavior should apply. for _config_module_name, asserts_config_value in self.getYamlConfigItem( @@ -1052,7 +1055,7 @@ slow down compilation.""" return "compiled" def onIncompleteModuleSet(self, module_names): - for module_name in list(module_names): + for module_name in module_names: for ( config_of_module_name, module_to_check,