https://github.com/python/cpython/commit/50369e6c34d05222e5a0ec9443a9f7b230e83112
commit: 50369e6c34d05222e5a0ec9443a9f7b230e83112
branch: main
author: Michael Droettboom <mdboom@gmail.com>
committer: Fidget-Spinner <kenjin4096@gmail.com>
date: 2024-03-22T01:27:46+08:00
summary:
gh-116996: Add pystats about _Py_uop_analyse_and_optimize (GH-116997)
files:
M Include/cpython/pystats.h
M Include/internal/pycore_code.h
M Python/optimizer_analysis.c
M Python/specialize.c
M Tools/scripts/summarize_stats.py
diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h
index 887fbbedf88502..5bf7bacd514699 100644
--- a/Include/cpython/pystats.h
+++ b/Include/cpython/pystats.h
@@ -19,6 +19,8 @@
// Define _PY_INTERPRETER macro to increment interpreter_increfs and
// interpreter_decrefs. Otherwise, increment increfs and decrefs.
+#include "pycore_uop_ids.h"
+
#ifndef Py_CPYTHON_PYSTATS_H
# error "this header file must not be included directly"
#endif
@@ -116,7 +118,7 @@ typedef struct _optimization_stats {
uint64_t recursive_call;
uint64_t low_confidence;
uint64_t executors_invalidated;
- UOpStats opcode[512];
+ UOpStats opcode[MAX_UOP_ID];
uint64_t unsupported_opcode[256];
uint64_t trace_length_hist[_Py_UOP_HIST_SIZE];
uint64_t trace_run_length_hist[_Py_UOP_HIST_SIZE];
@@ -124,6 +126,9 @@ typedef struct _optimization_stats {
uint64_t optimizer_attempts;
uint64_t optimizer_successes;
uint64_t optimizer_failure_reason_no_memory;
+ uint64_t remove_globals_builtins_changed;
+ uint64_t remove_globals_incorrect_keys;
+ uint64_t error_in_opcode[MAX_UOP_ID];
} OptimizationStats;
typedef struct _rare_event_stats {
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index 8eabd49a18afa9..e004783ee48198 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -308,6 +308,7 @@ extern int _PyStaticCode_Init(PyCodeObject *co);
#define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0)
#define UOP_STAT_INC(opname, name) do { if (_Py_stats) { assert(opname < 512); _Py_stats->optimization_stats.opcode[opname].name++; } } while (0)
#define OPT_UNSUPPORTED_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.unsupported_opcode[opname]++; } while (0)
+#define OPT_ERROR_IN_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.error_in_opcode[opname]++; } while (0)
#define OPT_HIST(length, name) \
do { \
if (_Py_stats) { \
@@ -334,6 +335,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
#define OPT_STAT_INC(name) ((void)0)
#define UOP_STAT_INC(opname, name) ((void)0)
#define OPT_UNSUPPORTED_OPCODE(opname) ((void)0)
+#define OPT_ERROR_IN_OPCODE(opname) ((void)0)
#define OPT_HIST(length, name) ((void)0)
#define RARE_EVENT_STAT_INC(name) ((void)0)
#endif // !Py_STATS
diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c
index 603ac6815665ca..6c460c5359d71e 100644
--- a/Python/optimizer_analysis.c
+++ b/Python/optimizer_analysis.c
@@ -139,6 +139,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer,
PyInterpreterState *interp = _PyInterpreterState_GET();
PyObject *builtins = frame->f_builtins;
if (builtins != interp->builtins) {
+ OPT_STAT_INC(remove_globals_builtins_changed);
return 1;
}
PyObject *globals = frame->f_globals;
@@ -170,6 +171,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer,
switch(opcode) {
case _GUARD_BUILTINS_VERSION:
if (incorrect_keys(inst, builtins)) {
+ OPT_STAT_INC(remove_globals_incorrect_keys);
return 0;
}
if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) {
@@ -190,6 +192,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer,
break;
case _GUARD_GLOBALS_VERSION:
if (incorrect_keys(inst, globals)) {
+ OPT_STAT_INC(remove_globals_incorrect_keys);
return 0;
}
uint64_t watched_mutations = get_mutations(globals);
@@ -238,6 +241,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer,
globals = func->func_globals;
builtins = func->func_builtins;
if (builtins != interp->builtins) {
+ OPT_STAT_INC(remove_globals_builtins_changed);
return 1;
}
break;
@@ -358,6 +362,7 @@ optimize_uops(
_Py_UOpsContext context;
_Py_UOpsContext *ctx = &context;
+ uint32_t opcode = UINT16_MAX;
if (_Py_uop_abstractcontext_init(ctx) < 0) {
goto out_of_space;
@@ -374,8 +379,7 @@ optimize_uops(
this_instr++) {
int oparg = this_instr->oparg;
- uint32_t opcode = this_instr->opcode;
-
+ opcode = this_instr->opcode;
_Py_UopsSymbol **stack_pointer = ctx->frame->stack_pointer;
#ifdef Py_DEBUG
@@ -410,6 +414,9 @@ optimize_uops(
error:
DPRINTF(3, "\n");
DPRINTF(1, "Encountered error in abstract interpreter\n");
+ if (opcode <= MAX_UOP_ID) {
+ OPT_ERROR_IN_OPCODE(opcode);
+ }
_Py_uop_abstractcontext_fini(ctx);
return -1;
diff --git a/Python/specialize.c b/Python/specialize.c
index b1f9eb756c3665..801ab1f2e64e5d 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -247,6 +247,8 @@ print_optimization_stats(FILE *out, OptimizationStats *stats)
fprintf(out, "Optimization optimizer successes: %" PRIu64 "\n", stats->optimizer_successes);
fprintf(out, "Optimization optimizer failure no memory: %" PRIu64 "\n",
stats->optimizer_failure_reason_no_memory);
+ fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed);
+ fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys);
const char* const* names;
for (int i = 0; i <= MAX_UOP_ID; i++) {
@@ -268,6 +270,17 @@ print_optimization_stats(FILE *out, OptimizationStats *stats)
);
}
}
+
+ for (int i = 0; i < MAX_UOP_ID; i++) {
+ if (stats->error_in_opcode[i]) {
+ fprintf(
+ out,
+ "error_in_opcode[%s].count : %" PRIu64 "\n",
+ _PyUOpName(i),
+ stats->error_in_opcode[i]
+ );
+ }
+ }
}
static void
diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py
index 6af14e1b769b80..d40106b8682388 100644
--- a/Tools/scripts/summarize_stats.py
+++ b/Tools/scripts/summarize_stats.py
@@ -513,6 +513,8 @@ def get_optimizer_stats(self) -> dict[str, tuple[int, int | None]]:
attempts = self._data["Optimization optimizer attempts"]
successes = self._data["Optimization optimizer successes"]
no_memory = self._data["Optimization optimizer failure no memory"]
+ builtins_changed = self._data["Optimizer remove globals builtins changed"]
+ incorrect_keys = self._data["Optimizer remove globals incorrect keys"]
return {
Doc(
@@ -527,6 +529,14 @@ def get_optimizer_stats(self) -> dict[str, tuple[int, int | None]]:
"Optimizer no memory",
"The number of optimizations that failed due to no memory.",
): (no_memory, attempts),
+ Doc(
+ "Remove globals builtins changed",
+ "The builtins changed during optimization",
+ ): (builtins_changed, attempts),
+ Doc(
+ "Remove globals incorrect keys",
+ "The keys in the globals dictionary aren't what was expected",
+ ): (incorrect_keys, attempts),
}
def get_histogram(self, prefix: str) -> list[tuple[int, int]]:
@@ -1177,6 +1187,17 @@ def calc_unsupported_opcodes_table(stats: Stats) -> Rows:
reverse=True,
)
+ def calc_error_in_opcodes_table(stats: Stats) -> Rows:
+ error_in_opcodes = stats.get_opcode_stats("error_in_opcode")
+ return sorted(
+ [.
+ (opcode, Count(count))
+ for opcode, count in error_in_opcodes.get_opcode_counts().items()
+ ],
+ key=itemgetter(1),
+ reverse=True,
+ )
+
def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None):
if not base_stats.get_optimization_stats() or (
head_stats is not None and not head_stats.get_optimization_stats()
@@ -1223,6 +1244,11 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None)
)
],
)
+ yield Section(
+ "Optimizer errored out with opcode",
+ "Optimization stopped after encountering this opcode",
+ [Table(("Opcode", "Count:"), calc_error_in_opcodes_table, JoinMode.CHANGE)],
+ )
return Section(
"Optimization (Tier 2) stats",
_______________________________________________
Python-checkins mailing list -- python-checkins@python.org
To unsubscribe send an email to python-checkins-leave@python.org
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: list-python-checkins@lists.gossamer-threads.com
commit: 50369e6c34d05222e5a0ec9443a9f7b230e83112
branch: main
author: Michael Droettboom <mdboom@gmail.com>
committer: Fidget-Spinner <kenjin4096@gmail.com>
date: 2024-03-22T01:27:46+08:00
summary:
gh-116996: Add pystats about _Py_uop_analyse_and_optimize (GH-116997)
files:
M Include/cpython/pystats.h
M Include/internal/pycore_code.h
M Python/optimizer_analysis.c
M Python/specialize.c
M Tools/scripts/summarize_stats.py
diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h
index 887fbbedf88502..5bf7bacd514699 100644
--- a/Include/cpython/pystats.h
+++ b/Include/cpython/pystats.h
@@ -19,6 +19,8 @@
// Define _PY_INTERPRETER macro to increment interpreter_increfs and
// interpreter_decrefs. Otherwise, increment increfs and decrefs.
+#include "pycore_uop_ids.h"
+
#ifndef Py_CPYTHON_PYSTATS_H
# error "this header file must not be included directly"
#endif
@@ -116,7 +118,7 @@ typedef struct _optimization_stats {
uint64_t recursive_call;
uint64_t low_confidence;
uint64_t executors_invalidated;
- UOpStats opcode[512];
+ UOpStats opcode[MAX_UOP_ID];
uint64_t unsupported_opcode[256];
uint64_t trace_length_hist[_Py_UOP_HIST_SIZE];
uint64_t trace_run_length_hist[_Py_UOP_HIST_SIZE];
@@ -124,6 +126,9 @@ typedef struct _optimization_stats {
uint64_t optimizer_attempts;
uint64_t optimizer_successes;
uint64_t optimizer_failure_reason_no_memory;
+ uint64_t remove_globals_builtins_changed;
+ uint64_t remove_globals_incorrect_keys;
+ uint64_t error_in_opcode[MAX_UOP_ID];
} OptimizationStats;
typedef struct _rare_event_stats {
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index 8eabd49a18afa9..e004783ee48198 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -308,6 +308,7 @@ extern int _PyStaticCode_Init(PyCodeObject *co);
#define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0)
#define UOP_STAT_INC(opname, name) do { if (_Py_stats) { assert(opname < 512); _Py_stats->optimization_stats.opcode[opname].name++; } } while (0)
#define OPT_UNSUPPORTED_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.unsupported_opcode[opname]++; } while (0)
+#define OPT_ERROR_IN_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.error_in_opcode[opname]++; } while (0)
#define OPT_HIST(length, name) \
do { \
if (_Py_stats) { \
@@ -334,6 +335,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
#define OPT_STAT_INC(name) ((void)0)
#define UOP_STAT_INC(opname, name) ((void)0)
#define OPT_UNSUPPORTED_OPCODE(opname) ((void)0)
+#define OPT_ERROR_IN_OPCODE(opname) ((void)0)
#define OPT_HIST(length, name) ((void)0)
#define RARE_EVENT_STAT_INC(name) ((void)0)
#endif // !Py_STATS
diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c
index 603ac6815665ca..6c460c5359d71e 100644
--- a/Python/optimizer_analysis.c
+++ b/Python/optimizer_analysis.c
@@ -139,6 +139,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer,
PyInterpreterState *interp = _PyInterpreterState_GET();
PyObject *builtins = frame->f_builtins;
if (builtins != interp->builtins) {
+ OPT_STAT_INC(remove_globals_builtins_changed);
return 1;
}
PyObject *globals = frame->f_globals;
@@ -170,6 +171,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer,
switch(opcode) {
case _GUARD_BUILTINS_VERSION:
if (incorrect_keys(inst, builtins)) {
+ OPT_STAT_INC(remove_globals_incorrect_keys);
return 0;
}
if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) {
@@ -190,6 +192,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer,
break;
case _GUARD_GLOBALS_VERSION:
if (incorrect_keys(inst, globals)) {
+ OPT_STAT_INC(remove_globals_incorrect_keys);
return 0;
}
uint64_t watched_mutations = get_mutations(globals);
@@ -238,6 +241,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer,
globals = func->func_globals;
builtins = func->func_builtins;
if (builtins != interp->builtins) {
+ OPT_STAT_INC(remove_globals_builtins_changed);
return 1;
}
break;
@@ -358,6 +362,7 @@ optimize_uops(
_Py_UOpsContext context;
_Py_UOpsContext *ctx = &context;
+ uint32_t opcode = UINT16_MAX;
if (_Py_uop_abstractcontext_init(ctx) < 0) {
goto out_of_space;
@@ -374,8 +379,7 @@ optimize_uops(
this_instr++) {
int oparg = this_instr->oparg;
- uint32_t opcode = this_instr->opcode;
-
+ opcode = this_instr->opcode;
_Py_UopsSymbol **stack_pointer = ctx->frame->stack_pointer;
#ifdef Py_DEBUG
@@ -410,6 +414,9 @@ optimize_uops(
error:
DPRINTF(3, "\n");
DPRINTF(1, "Encountered error in abstract interpreter\n");
+ if (opcode <= MAX_UOP_ID) {
+ OPT_ERROR_IN_OPCODE(opcode);
+ }
_Py_uop_abstractcontext_fini(ctx);
return -1;
diff --git a/Python/specialize.c b/Python/specialize.c
index b1f9eb756c3665..801ab1f2e64e5d 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -247,6 +247,8 @@ print_optimization_stats(FILE *out, OptimizationStats *stats)
fprintf(out, "Optimization optimizer successes: %" PRIu64 "\n", stats->optimizer_successes);
fprintf(out, "Optimization optimizer failure no memory: %" PRIu64 "\n",
stats->optimizer_failure_reason_no_memory);
+ fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed);
+ fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys);
const char* const* names;
for (int i = 0; i <= MAX_UOP_ID; i++) {
@@ -268,6 +270,17 @@ print_optimization_stats(FILE *out, OptimizationStats *stats)
);
}
}
+
+ for (int i = 0; i < MAX_UOP_ID; i++) {
+ if (stats->error_in_opcode[i]) {
+ fprintf(
+ out,
+ "error_in_opcode[%s].count : %" PRIu64 "\n",
+ _PyUOpName(i),
+ stats->error_in_opcode[i]
+ );
+ }
+ }
}
static void
diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py
index 6af14e1b769b80..d40106b8682388 100644
--- a/Tools/scripts/summarize_stats.py
+++ b/Tools/scripts/summarize_stats.py
@@ -513,6 +513,8 @@ def get_optimizer_stats(self) -> dict[str, tuple[int, int | None]]:
attempts = self._data["Optimization optimizer attempts"]
successes = self._data["Optimization optimizer successes"]
no_memory = self._data["Optimization optimizer failure no memory"]
+ builtins_changed = self._data["Optimizer remove globals builtins changed"]
+ incorrect_keys = self._data["Optimizer remove globals incorrect keys"]
return {
Doc(
@@ -527,6 +529,14 @@ def get_optimizer_stats(self) -> dict[str, tuple[int, int | None]]:
"Optimizer no memory",
"The number of optimizations that failed due to no memory.",
): (no_memory, attempts),
+ Doc(
+ "Remove globals builtins changed",
+ "The builtins changed during optimization",
+ ): (builtins_changed, attempts),
+ Doc(
+ "Remove globals incorrect keys",
+ "The keys in the globals dictionary aren't what was expected",
+ ): (incorrect_keys, attempts),
}
def get_histogram(self, prefix: str) -> list[tuple[int, int]]:
@@ -1177,6 +1187,17 @@ def calc_unsupported_opcodes_table(stats: Stats) -> Rows:
reverse=True,
)
+ def calc_error_in_opcodes_table(stats: Stats) -> Rows:
+ error_in_opcodes = stats.get_opcode_stats("error_in_opcode")
+ return sorted(
+ [.
+ (opcode, Count(count))
+ for opcode, count in error_in_opcodes.get_opcode_counts().items()
+ ],
+ key=itemgetter(1),
+ reverse=True,
+ )
+
def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None):
if not base_stats.get_optimization_stats() or (
head_stats is not None and not head_stats.get_optimization_stats()
@@ -1223,6 +1244,11 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None)
)
],
)
+ yield Section(
+ "Optimizer errored out with opcode",
+ "Optimization stopped after encountering this opcode",
+ [Table(("Opcode", "Count:"), calc_error_in_opcodes_table, JoinMode.CHANGE)],
+ )
return Section(
"Optimization (Tier 2) stats",
_______________________________________________
Python-checkins mailing list -- python-checkins@python.org
To unsubscribe send an email to python-checkins-leave@python.org
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: list-python-checkins@lists.gossamer-threads.com