From dbf4c60fa95be34e417d584f4b6b5f6293ca4b68 Mon Sep 17 00:00:00 2001 From: Sahil Jain Date: Mon, 1 Sep 2025 22:00:40 +0530 Subject: [PATCH 1/3] Support new opcodes --- ASTree.cpp | 28 ++++++++++++++++++++++++++ pyc_code.cpp | 32 +++++++++++++++++++++++++++++- pyc_code.h | 10 +++++++++- pyc_sequence.h | 5 +++++ tests/compiled/non_local.3.12.pyc | Bin 0 -> 404 bytes tests/input/non_local.py | 7 +++++++ tests/run_tests.py | 2 +- tests/tokenized/non_local.txt | 10 ++++++++++ 8 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 tests/compiled/non_local.3.12.pyc create mode 100644 tests/input/non_local.py create mode 100644 tests/tokenized/non_local.txt diff --git a/ASTree.cpp b/ASTree.cpp index 6635808e1..749d4a248 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -2584,6 +2584,19 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(value); } break; + case Pyc::MAKE_CELL_A: + // no op + break; + case Pyc::COPY_FREE_VARS_A: + // nonlocal should be inserted + // some nonlocals may be redundant but we're not optimizing currently + if (operand != code->freeVars()->size()) { + throw std::runtime_error("Different number of free variables copied"); + } + for (int i = 0; i < code->freeVars()->size(); i++) { + code->markNonLocal(code->freeVars()->get(i).cast()); + } + break; default: fprintf(stderr, "Unsupported opcode: %s (%d)\n", Pyc::OpcodeName(opcode), opcode); cleanBuild = false; @@ -3567,6 +3580,21 @@ void decompyle(PycRef code, PycModule* mod, std::ostream& pyc_output) } pyc_output << "\n"; } + + PycCode::nonlocals_t non_locals = code->getNonLocals(); + if (non_locals.size()) { + start_line(cur_indent + 1, pyc_output); + pyc_output << "nonlocal "; + bool first = true; + for (const auto& non_local : non_locals) { + if (!first) + pyc_output << ", "; + pyc_output << non_local->value(); + first = false; + } + pyc_output << "\n"; + } + printDocstringAndGlobals = false; } diff --git a/pyc_code.cpp b/pyc_code.cpp index b88f666e1..7428dbe36 100644 --- a/pyc_code.cpp +++ b/pyc_code.cpp @@ -26,6 +26,10 @@ lntable Obj Obj Obj Obj Obj Obj exceptiontable Obj */ +#define CO_FAST_LOCAL 0x20 +#define CO_FAST_CELL 0x40 +#define CO_FAST_FREE 0x80 + void PycCode::load(PycData* stream, PycModule* mod) { if (mod->verCompare(1, 3) >= 0 && mod->verCompare(2, 3) < 0) @@ -75,23 +79,49 @@ void PycCode::load(PycData* stream, PycModule* mod) m_consts = LoadObject(stream, mod).cast(); m_names = LoadObject(stream, mod).cast(); + // Represents localsplusnames for py 3.11+ if (mod->verCompare(1, 3) >= 0) m_localNames = LoadObject(stream, mod).cast(); else m_localNames = new PycTuple; - if (mod->verCompare(3, 11) >= 0) + PycSimpleSequence::value_t free_vars_extra, cell_vars_extra; + + if (mod->verCompare(3, 11) >= 0) { m_localKinds = LoadObject(stream, mod).cast(); + + if (m_localKinds->length() != m_localNames->size()) { + throw std::runtime_error("All variables kinds are not available"); + } + + // Starting py3.11, all variables are now part of localsplusnames + + for (int i = 0; i < m_localKinds->length(); i++) { + const char kind = m_localKinds->value()[i]; + auto name = m_localNames->get(i); + + if (kind & CO_FAST_CELL) { + cell_vars_extra.push_back(name); + } + else if (kind & CO_FAST_FREE) { + free_vars_extra.push_back(name); + } + } + } else m_localKinds = new PycString; if (mod->verCompare(2, 1) >= 0 && mod->verCompare(3, 11) < 0) m_freeVars = LoadObject(stream, mod).cast(); + else if (mod->verCompare(3, 11) >= 0) + m_freeVars = new PycTuple(free_vars_extra); else m_freeVars = new PycTuple; if (mod->verCompare(2, 1) >= 0 && mod->verCompare(3, 11) < 0) m_cellVars = LoadObject(stream, mod).cast(); + else if (mod->verCompare(3, 11) >= 0) + m_cellVars = new PycTuple(cell_vars_extra); else m_cellVars = new PycTuple; diff --git a/pyc_code.h b/pyc_code.h index 6485729a7..3b3b0590a 100644 --- a/pyc_code.h +++ b/pyc_code.h @@ -22,7 +22,8 @@ class PycExceptionTableEntry { class PycCode : public PycObject { public: - typedef std::vector> globals_t; + typedef std::vector> globals_t, nonlocals_t; + enum CodeFlags { CO_OPTIMIZED = 0x1, // 1.3 -> CO_NEWLOCALS = 0x2, // 1.3 -> @@ -99,6 +100,12 @@ class PycCode : public PycObject { m_globalsUsed.emplace_back(std::move(varname)); } + const nonlocals_t& getNonLocals() const { return m_nonlocalsUsed; } + + void markNonLocal(PycRef varname) { + m_nonlocalsUsed.emplace_back(std::move(varname)); + } + std::vector exceptionTableEntries() const; private: @@ -118,6 +125,7 @@ class PycCode : public PycObject { PycRef m_lnTable; PycRef m_exceptTable; globals_t m_globalsUsed; /* Global vars used in this code */ + nonlocals_t m_nonlocalsUsed; // Nonlocal vars used in this code }; #endif diff --git a/pyc_sequence.h b/pyc_sequence.h index 64ff66ca5..eed250350 100644 --- a/pyc_sequence.h +++ b/pyc_sequence.h @@ -38,6 +38,11 @@ class PycTuple : public PycSimpleSequence { typedef PycSimpleSequence::value_t value_t; PycTuple(int type = TYPE_TUPLE) : PycSimpleSequence(type) { } + PycTuple(value_t values, int type = TYPE_TUPLE): PycSimpleSequence(type) { + m_values = values; + m_size = values.size(); + } + void load(class PycData* stream, class PycModule* mod) override; }; diff --git a/tests/compiled/non_local.3.12.pyc b/tests/compiled/non_local.3.12.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d87ee2e5f751fdd8fde7eb918c7d64c3f2ff6c27 GIT binary patch literal 404 zcmY*VF-yZx5Wagc^+lT@Se%`Nw*=fv(Om-K=2n`Bg_x9=6iiXbP%texikrJQ`z!nd z5*!4lP8Q-IPTr->;DfvS?t99$zj(s-N662iKrt$piQ>g)_)di9w^_ zDkuz!YM0V2k`Cus;5qtYDsSK#PC*&of-d3T9M9}fRQr_fX>LZ76B}R-Y2!@X^BN;$ zpoF*JR$*z5^e{>Z;8ns@{=0GDjQ%8$N + +x = 0 +def inner ( ) : + +nonlocal x +x += 1 +print ( x ) + +return inner From 43417a8b0c1a5e477dff590627605cd915ef149b Mon Sep 17 00:00:00 2001 From: Sahil Jain Date: Tue, 2 Sep 2025 15:22:08 +0530 Subject: [PATCH 2/3] Create enum for kinds --- pyc_code.cpp | 8 ++------ pyc_code.h | 11 +++++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pyc_code.cpp b/pyc_code.cpp index 7428dbe36..45aa2165f 100644 --- a/pyc_code.cpp +++ b/pyc_code.cpp @@ -26,10 +26,6 @@ lntable Obj Obj Obj Obj Obj Obj exceptiontable Obj */ -#define CO_FAST_LOCAL 0x20 -#define CO_FAST_CELL 0x40 -#define CO_FAST_FREE 0x80 - void PycCode::load(PycData* stream, PycModule* mod) { if (mod->verCompare(1, 3) >= 0 && mod->verCompare(2, 3) < 0) @@ -100,10 +96,10 @@ void PycCode::load(PycData* stream, PycModule* mod) const char kind = m_localKinds->value()[i]; auto name = m_localNames->get(i); - if (kind & CO_FAST_CELL) { + if (kind & Kinds::CO_FAST_CELL) { cell_vars_extra.push_back(name); } - else if (kind & CO_FAST_FREE) { + else if (kind & Kinds::CO_FAST_FREE) { free_vars_extra.push_back(name); } } diff --git a/pyc_code.h b/pyc_code.h index 3b3b0590a..0fd7d6312 100644 --- a/pyc_code.h +++ b/pyc_code.h @@ -50,6 +50,17 @@ class PycCode : public PycObject { CO_NO_MONITORING_EVENTS = 0x2000000, // 3.13 -> }; + enum Kinds { // 3.11 -> + CO_FAST_ARG_POS = 0x2, + CO_FAST_ARG_KW = 0x4, + CO_FAST_ARG_VAR = 0x8, + CO_FAST_ARG = CO_FAST_ARG_POS | CO_FAST_ARG_KW | CO_FAST_ARG_VAR , + CO_FAST_HIDDEN = 0x10, + CO_FAST_LOCAL = 0x20, + CO_FAST_CELL = 0x40, + CO_FAST_FREE = 0x80, + }; + PycCode(int type = TYPE_CODE) : PycObject(type), m_argCount(), m_posOnlyArgCount(), m_kwOnlyArgCount(), m_numLocals(), m_stackSize(), m_flags(), m_firstLine() { } From 8401a00344c191bdd6e76851d006c70374499b8d Mon Sep 17 00:00:00 2001 From: Sahil Jain Date: Tue, 2 Sep 2025 15:39:12 +0530 Subject: [PATCH 3/3] todo --- pyc_code.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/pyc_code.cpp b/pyc_code.cpp index 45aa2165f..9efb831c0 100644 --- a/pyc_code.cpp +++ b/pyc_code.cpp @@ -26,6 +26,7 @@ lntable Obj Obj Obj Obj Obj Obj exceptiontable Obj */ +// TODO : Simplify this function void PycCode::load(PycData* stream, PycModule* mod) { if (mod->verCompare(1, 3) >= 0 && mod->verCompare(2, 3) < 0)