Support Python 3.13 and make tests for older versions green (#17)

Author: maxim-krikun

README.rst

@@ -9,7 +9,7 @@ This debugger extension provides visualizations for Python objects and stacktrac
 
 The goal of this project is to provide a similar debugging experience in WinDbg/CDB/NTSD as `already exists in GDB <https://wiki.python.org/moin/DebuggingWithGdb>`_.
 
-Currently, the extension is tested against 32bit and 64bit builds of Python versions 2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10 and 3.11.
+Currently, the extension is tested against 32bit and 64bit builds of Python versions 2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 and 3.13.
 
 Installation
 ============

include/PyFunctionObject.h

@@ -29,7 +29,7 @@ namespace PyExt::Remote {
 		auto closure() const -> std::unique_ptr<PyTupleObject>;
 		auto doc() const -> std::unique_ptr<PyObject>;
 		auto name() const -> std::unique_ptr<PyStringValue>;
-		auto dict() const -> std::unique_ptr<PyDictObject>;
+		auto dict() const -> std::unique_ptr<PyDict> override;
 		auto weakreflist() const -> std::unique_ptr<PyListObject>;
 		auto module() const -> std::unique_ptr<PyObject>;
 		auto annotations() const -> std::unique_ptr<PyDictObject>;

include/PyObject.h

@@ -39,7 +39,8 @@ namespace PyExt::Remote {
 		auto refCount() const -> SSize;
 		auto type() const -> PyTypeObject;
 		auto slots() const -> std::vector<std::pair<std::string, std::unique_ptr<PyObject>>>;
-		auto dict() const -> std::unique_ptr<PyDict>;
+		auto managedDict() const -> std::unique_ptr<PyDict>;
+		virtual auto dict() const -> std::unique_ptr<PyDict>;
 		virtual auto repr(bool pretty = true) const -> std::string;
 		virtual auto details() const -> std::string;
 

include/PyTypeObject.h

@@ -3,6 +3,7 @@
 #include "PyVarObject.h"
 #include "PyMemberDef.h"
 #include "PyTupleObject.h"
+#include "PyDictObject.h"
 #include <array>
 #include <string>
 
@@ -24,9 +25,12 @@ namespace PyExt::Remote {
 		auto documentation() const -> std::string;
 		auto members() const -> std::vector<std::unique_ptr<PyMemberDef>>;
 		auto isManagedDict() const -> bool;
+		auto getStaticBuiltinIndex() const -> SSize;
+		auto hasInlineValues() const -> bool;
 		auto dictOffset() const -> SSize;
 		auto mro() const -> std::unique_ptr<PyTupleObject>;
 		auto isPython2() const -> bool;
+		auto dict() const -> std::unique_ptr<PyDict> override;
 		auto repr(bool pretty = true) const -> std::string override;
 
 	};

src/ExtHelpers.cpp

@@ -43,4 +43,12 @@ namespace utils {
 		return oss.str();
 	}
 
+
+	auto getFullSymbolName(const string& symbolName) -> string
+	{
+		ExtBuffer<char> buffer;
+		g_Ext->FindFirstModule("python???", &buffer, 0);
+		return buffer.GetBuffer() + "!"s + symbolName;
+	}
+
 }

src/ExtHelpers.h

@@ -70,5 +70,6 @@ namespace utils {
 	auto getPointerSize() -> int;
 	auto escapeDml(const std::string& str) -> std::string;
 	auto link(const std::string& text, const std::string& cmd, const std::string& alt = ""s) -> std::string;
+	auto getFullSymbolName(const std::string& symbolName) -> std::string;
 
 }

src/objects/PyDictObject.cpp

@@ -139,10 +139,12 @@ namespace PyExt::Remote {
 		optional<ExtRemoteTyped> valuesArray;
 		if (!isCombined) {
 			valuesArray = remoteType().Field("ma_values");
-
-			// Since Python 3.11 values are stored in `values` of `ma_values`.
-			if (valuesArray->HasField("values"))
+			utils::ignoreExtensionError([&] {
+				// Python >= 3.11
+				// Find full symbol name because there may be a name collision leading to truncated type info.
+				valuesArray = ExtRemoteTyped(utils::getFullSymbolName("_dictvalues").c_str(), valuesArray->GetPtr(), true);
 				valuesArray = valuesArray->Field("values");
+			});				
 		}
 
 		for (SSize i = 0; i < tableSize; ++i) {

src/objects/PyFunctionObject.cpp

@@ -65,7 +65,7 @@ namespace PyExt::Remote {
 	}
 
 
-	auto PyFunctionObject::dict() const -> unique_ptr<PyDictObject>
+	auto PyFunctionObject::dict() const -> unique_ptr<PyDict>
 	{
 		return utils::fieldAsPyObject<PyDictObject>(remoteType(), "func_dict");
 	}

src/objects/PyObject.cpp

@@ -62,14 +62,30 @@ namespace PyExt::Remote {
 	}
 
 
-	auto PyObject::dict() const -> unique_ptr<PyDict>
+	auto PyObject::managedDict() const -> unique_ptr<PyDict>
 	{
-		if (type().isManagedDict()) {
-			// Python >= 3.11, see PyObject_GenericGetDict
-			auto pointerSize = utils::getPointerSize();
-			Offset valuesPtr;
+		// Python >= 3.11, see PyObject_GenericGetDict
+		if (!type().isManagedDict())
+			return { };
+
+		auto pointerSize = utils::getPointerSize();
+
+		Offset dictPtr = 0;
+		utils::ignoreExtensionError([&] {
+			// Python >= 3.13
+			auto managedDictPtr = offset() - 3 * pointerSize;
+			dictPtr = ExtRemoteTyped("PyManagedDictPointer", managedDictPtr, true).Field("dict").GetPtr();
+		});
+		if (dictPtr)
+			return make_unique<PyDictObject>(dictPtr);
+
+		Offset valuesPtr;
+		if (type().hasInlineValues()) {
+			// Python >= 3.13
+			auto dictValues = ExtRemoteTyped("(_dictvalues*)((PyObject*)(@$extin)+1)", offset());
+			valuesPtr = dictValues.Field("values").GetPtr();
+		} else {
 			optional<ExtRemoteTyped> dictOrValues;
-
 			utils::ignoreExtensionError([&] {
 				// Python 3.12
 				auto dictOrValuesPtr = offset() - 3 * pointerSize;
@@ -92,12 +108,20 @@ namespace PyExt::Remote {
 					return dictPtr ? make_unique<PyDictObject>(dictPtr) : nullptr;
 				}
 			}
-
-			auto ht = ExtRemoteTyped("PyHeapTypeObject", type().offset(), true);
-			auto cachedKeys = ht.Field("ht_cached_keys");
-			return make_unique<PyManagedDict>(cachedKeys.GetPtr(), valuesPtr);
 		}
 
+		auto ht = ExtRemoteTyped("PyHeapTypeObject", type().offset(), true);
+		auto cachedKeys = ht.Field("ht_cached_keys");
+		return make_unique<PyManagedDict>(cachedKeys.GetPtr(), valuesPtr);
+	}
+
+
+	auto PyObject::dict() const -> unique_ptr<PyDict>
+	{
+		auto managedDict_ = managedDict();
+		if (managedDict_ != nullptr)
+			return managedDict_;
+
 		// see https://docs.python.org/3.10/c-api/typeobj.html#c.PyTypeObject.tp_dictoffset
 		auto dictOffset_ = type().dictOffset();
 		if (dictOffset_ == 0)

src/objects/PyTypeObject.cpp

@@ -117,6 +117,29 @@ namespace PyExt::Remote {
 	}
 
 
+	auto PyTypeObject::getStaticBuiltinIndex() const -> SSize
+	{
+		auto type = remoteType();
+		auto flagsRaw = type.Field("tp_flags");
+		auto flags = utils::readIntegral<unsigned long>(flagsRaw);
+		if (flags & (1 << 1)) {  // _Py_TPFLAGS_STATIC_BUILTIN
+			// Python >= 3.12
+			auto subclassesRaw = type.Field("tp_subclasses");
+			auto subclasses = utils::readIntegral<SSize>(subclassesRaw);
+			return subclasses - 1;
+		}
+		return -1;
+	}
+
+
+	auto PyTypeObject::hasInlineValues() const -> bool
+	{
+		auto flagsRaw = remoteType().Field("tp_flags");
+		auto flags = utils::readIntegral<unsigned long>(flagsRaw);
+		return flags & (1 << 2);  // Py_TPFLAGS_INLINE_VALUES
+	}
+
+
 	auto PyTypeObject::dictOffset() const -> SSize
 	{
 		auto dictOffset = remoteType().Field("tp_dictoffset");
@@ -138,6 +161,23 @@ namespace PyExt::Remote {
 	}
 
 
+	auto PyTypeObject::dict() const -> unique_ptr<PyDict>
+	{
+		Offset dictAddr;
+		auto builtinIndex = getStaticBuiltinIndex();
+		if (builtinIndex != -1) {
+			auto builtins = ExtRemoteTyped("_PyRuntime.interpreters.main->types.builtins");  // TODO: Support multiple interpreters
+			if (builtins.HasField("initialized"))  // Python >= 3.13
+				builtins = builtins.Field("initialized");
+			auto builtinState = builtins.ArrayElement(builtinIndex);
+			dictAddr = builtinState.Field("tp_dict").GetPtr();
+		} else {
+			dictAddr = remoteType().Field("tp_dict").GetPtr();
+		}
+		return make_unique<PyDictObject>(dictAddr);
+	}
+
+
 	auto PyTypeObject::repr(bool pretty) const -> string
 	{
 		string repr = "<class '" + name() + "'>";