UI: Added options for compile time profiling

* Also renamed old runtime profiling option, that is however still broken.

UserPlugin-Creation.rst

@@ -174,8 +174,6 @@ the methods have no argument.
 +--------------------------------------+-----------------------------------------------------------------------------------+
 | isMingw64                            | *bool* = ``--mingw64``                                                            |
 +--------------------------------------+-----------------------------------------------------------------------------------+
-| isProfile                            | *bool* = ``--profile``                                                            |
-+--------------------------------------+-----------------------------------------------------------------------------------+
 | shallUsePythonDebug                  | *bool* = ``--python-debug`` or ``sys.flags.debug``                                |
 +--------------------------------------+-----------------------------------------------------------------------------------+
 | isRemoveBuildDir                     | *bool* = ``--remove-output``                                                      |

doc/nuitka-run.1

@@ -210,7 +210,7 @@ Disable check normally done with "\-\-debug". The C compilation may produce warn
 \fB\-\-unstripped\fR
 Keep debug info in the resulting object file for better debugger interaction. Defaults to off.
 .TP
-\fB\-\-profile\fR
+\fB\-\-debug\-profile\-runtime\fR
 Enable vmprof based profiling of time spent. Not working currently. Defaults to off.
 .TP
 \fB\-\-trace\-execution\fR
@@ -244,6 +244,9 @@ This is not incremental compilation, but for Nuitka development only. Takes exis
 .TP
 \fB\-\-devel\-internal\-graph\fR
 Create graph of optimization process internals, do not use for whole programs, but only for small test cases. Defaults to off.
+.TP
+\fB\-\-devel\-profile\-compilation\fR
+Enable cProfile based profiling of time spent during compilation. Defaults to off.
 .IP
 Backend C compiler choice:
 .TP

doc/nuitka.1

@@ -210,7 +210,7 @@ Disable check normally done with "\-\-debug". The C compilation may produce warn
 \fB\-\-unstripped\fR
 Keep debug info in the resulting object file for better debugger interaction. Defaults to off.
 .TP
-\fB\-\-profile\fR
+\fB\-\-debug\-profile\-runtime\fR
 Enable vmprof based profiling of time spent. Not working currently. Defaults to off.
 .TP
 \fB\-\-trace\-execution\fR
@@ -244,6 +244,9 @@ This is not incremental compilation, but for Nuitka development only. Takes exis
 .TP
 \fB\-\-devel\-internal\-graph\fR
 Create graph of optimization process internals, do not use for whole programs, but only for small test cases. Defaults to off.
+.TP
+\fB\-\-devel\-profile\-compilation\fR
+Enable cProfile based profiling of time spent during compilation. Defaults to off.
 .IP
 Backend C compiler choice:
 .TP

nuitka/MainControl.py

@@ -81,6 +81,7 @@ from nuitka.PythonVersions import (
 )
 from nuitka.Serialization import ConstantAccessor
 from nuitka.Tracing import (
+    code_generation_logger,
     doNotBreakSpaces,
     general,
     inclusion_logger,
@@ -109,6 +110,7 @@ from nuitka.utils.MemoryUsage import reportMemoryUsage, showMemoryTrace
 from nuitka.utils.ModuleNames import ModuleName
 from nuitka.utils.ReExecute import callExecProcess, reExecuteNuitka
 from nuitka.utils.StaticLibraries import getSystemStaticLibPythonPath
+from nuitka.utils.Timing import withProfiling
 from nuitka.utils.Utils import getArchitecture, isMacOS, isWin32Windows
 from nuitka.Version import getCommercialVersion, getNuitkaVersion
 
@@ -665,7 +667,7 @@ def runSconsBackend():
     if Options.getForcedStderrPath():
         scons_options["forced_stderr_path"] = Options.getForcedStderrPath()
 
-    if Options.isProfile():
+    if Options.isRuntimeProfile():
         scons_options["profile_mode"] = asBoolStr(True)
 
     if Options.shallTreatUninstalledPython():
@@ -895,7 +897,12 @@ def compileTree():
         )
 
         # Now build the target language code for the whole tree.
-        makeSourceDirectory()
+        with withProfiling(
+            name="code-generation",
+            logger=code_generation_logger,
+            enabled=Options.isCompileTimeProfile(),
+        ):
+            makeSourceDirectory()
 
         bytecode_accessor = ConstantAccessor(
             data_filename="__bytecode.const", top_level_name="bytecode_data"

nuitka/OptionParsing.py

@@ -945,9 +945,9 @@ Defaults to off.""",
 )
 
 debug_group.add_option(
-    "--profile",
+    "--debug-profile-runtime",
     action="store_true",
-    dest="profile",
+    dest="debug_profile_runtime",
     default=False,
     github_action=False,
     help="""\
@@ -1092,6 +1092,16 @@ development_group.add_option(
     help=SUPPRESS_HELP,
 )
 
+development_group.add_option(
+    "--devel-profile-compilation",
+    action="store_true",
+    dest="devel_profile_compilation",
+    default=False,
+    github_action=False,
+    help="""\
+Enable cProfile based profiling of time spent during compilation. Defaults to off.""",
+)
+
 del development_group
 
 # This is for testing framework, "coverage.py" hates to loose the process. And

nuitka/Options.py

@@ -1789,12 +1789,17 @@ def isUnstripped():
     Passed to Scons as ``unstripped_mode`` to it can ask the linker to
     include symbol information.
     """
-    return options.unstripped or options.profile or is_debug
+    return options.unstripped or isRuntimeProfile() or is_debug
 
 
-def isProfile():
-    """:returns: bool derived from ``--profile``"""
-    return options.profile
+def isRuntimeProfile():
+    """:returns: bool derived from ``--debug-profile-runtime``"""
+    return options.debug_profile_runtime
+
+
+def isCompileTimeProfile():
+    """:returns: bool derived from ``--devel-profile-compilation``"""
+    return options.devel_profile_compilation
 
 
 def shallCreateGraph():

nuitka/optimizations/Optimization.py

@@ -21,7 +21,7 @@ from nuitka.Progress import (
 )
 from nuitka.Tracing import general, optimization_logger, progress_logger
 from nuitka.utils.MemoryUsage import MemoryWatch, reportMemoryUsage
-from nuitka.utils.Timing import TimerReport
+from nuitka.utils.Timing import TimerReport, withProfiling
 
 from . import Graphs
 from .BytecodeDemotion import demoteCompiledModuleToBytecode
@@ -326,7 +326,7 @@ def makeOptimizationPass():
     return finished
 
 
-def optimizeModules(output_filename):
+def _optimizeModules(output_filename):
     Graphs.startGraph()
 
     finished = makeOptimizationPass()
@@ -347,6 +347,15 @@ def optimizeModules(output_filename):
     Graphs.endGraph(output_filename)
 
 
+def optimizeModules(output_filename):
+    with withProfiling(
+        name="module-optimization",
+        logger=general,
+        enabled=Options.isCompileTimeProfile(),
+    ):
+        _optimizeModules(output_filename)
+
+
 #     Part of "Nuitka", an optimizing Python compiler that is compatible and
 #     integrates with CPython, but also works on its own.
 #

nuitka/utils/Timing.py

@@ -7,8 +7,10 @@ Mostly for measurements of Nuitka of itself, e.g. how long did it take to
 call an external tool.
 """
 
+from contextlib import contextmanager
 from timeit import default_timer as timer
 
+from nuitka.__past__ import StringIO
 from nuitka.Tracing import general
 
 
@@ -86,6 +88,36 @@ class TimerReport(object):
             self.logger.info(self.message % self.timer.getDelta(), keep_format=True)
 
 
+@contextmanager
+def withProfiling(name, logger, enabled):
+    if enabled:
+        import cProfile
+        import pstats
+
+        from nuitka.Options import getOutputPath
+
+        pr = cProfile.Profile()
+        pr.enable()
+
+        yield
+
+        pr.disable()
+
+        s = StringIO()
+        ps = pstats.Stats(pr, stream=s).sort_stats(pstats.SortKey.CUMULATIVE)
+
+        ps.print_stats()
+        for line in s.getvalue().splitlines():
+            logger.info(line)
+
+        profile_filename = getOutputPath(name + ".prof")
+
+        pr.dump_stats(profile_filename)
+        logger.info("Profiling data for '%s' saved to '%s'." % (name, profile_filename))
+    else:
+        yield
+
+
 #     Part of "Nuitka", an optimizing Python compiler that is compatible and
 #     integrates with CPython, but also works on its own.
 #