From 614b78a98013ab8496b6e22bd24138520efc4d3c Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:00:59 +0000 Subject: [PATCH 001/131] Add missing `.clang-format` file --- .clang-format | 152 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 .clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..039810c --- /dev/null +++ b/.clang-format @@ -0,0 +1,152 @@ +--- +Language: Cpp +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Right +AlignOperands: false +AlignTrailingComments: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: None +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterClass: true + AfterControlStatement: true + AfterEnum: true + AfterFunction: true + AfterNamespace: true + AfterObjCDeclaration: true + AfterStruct: true + AfterUnion: true + AfterExternBlock: true + BeforeCatch: true + BeforeElse: true + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +BreakBeforeTernaryOperators: false +BreakConstructorInitializers: AfterColon +BreakStringLiterals: false +ColumnLimit: 0 +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: false +DerivePointerAlignment: false +FixNamespaceComments: false +IncludeBlocks: Preserve +IndentCaseLabels: false +IndentPPDirectives: None +IndentWidth: 4 +IndentWrappedFunctionNames: true +KeepEmptyLinesAtTheStartOfBlocks: true +MacroBlockBegin: 'EXPOSURE_BEGIN' +MacroBlockEnd: 'EXPOSURE_END|EXPOSURE_CHAINTO' +MaxEmptyLinesToKeep: 4 +NamespaceIndentation: None +PointerAlignment: Left +ReflowComments: false +SortIncludes: false +SortUsingDeclarations: false +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: Never +SpaceInEmptyParentheses: false +SpacesInAngles: false +SpacesInCStyleCastParentheses: false +SpacesInContainerLiterals: false +SpacesInParentheses: true +SpacesInSquareBrackets: false +Standard: Cpp11 +TabWidth: 4 +UseTab: ForContinuationAndIndentation +--- +Language: ObjC +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Right +AlignOperands: false +AlignTrailingComments: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: None +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterClass: true + AfterControlStatement: true + AfterEnum: true + AfterFunction: true + AfterNamespace: true + AfterObjCDeclaration: true + AfterStruct: true + AfterUnion: true + AfterExternBlock: true + BeforeCatch: true + BeforeElse: true + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +BreakBeforeTernaryOperators: false +BreakConstructorInitializers: AfterColon +BreakStringLiterals: false +ColumnLimit: 120 +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: false +DerivePointerAlignment: false +FixNamespaceComments: false +IncludeBlocks: Preserve +IndentCaseLabels: false +IndentPPDirectives: None +IndentWidth: 4 +IndentWrappedFunctionNames: true +KeepEmptyLinesAtTheStartOfBlocks: true +MacroBlockBegin: 'EXPOSURE_BEGIN' +MacroBlockEnd: 'EXPOSURE_END|EXPOSURE_CHAINTO' +MaxEmptyLinesToKeep: 4 +NamespaceIndentation: None +ObjCBlockIndentWidth: 4 +PointerAlignment: Left +ReflowComments: false +SortIncludes: false +SortUsingDeclarations: false +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: Never +SpaceInEmptyParentheses: false +SpacesInAngles: false +SpacesInCStyleCastParentheses: false +SpacesInContainerLiterals: false +SpacesInParentheses: true +SpacesInSquareBrackets: false +Standard: Cpp11 +TabWidth: 4 +UseTab: ForContinuationAndIndentation +... From 63253bbac97870d4e511c5c861c26f6e6f46357a Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:06:59 +0000 Subject: [PATCH 002/131] Stop tracking `BlueAsyncRes` as memory allocations Because it was kind of abusing the alloc / free annotations for an unknown benefit. --- include/BlueAsyncRes.h | 4 ---- src/BlueAsyncRes.cpp | 3 --- 2 files changed, 7 deletions(-) diff --git a/include/BlueAsyncRes.h b/include/BlueAsyncRes.h index 9bdb943..9e95c43 100644 --- a/include/BlueAsyncRes.h +++ b/include/BlueAsyncRes.h @@ -35,10 +35,6 @@ BLUE_BLUEIMPORT_CLASS( BlueAsyncRes ) : virtual ~BlueAsyncRes() { CCP_ASSERT( m_reloadNotifyTargetsCount == 0 ); - - #if CCP_TELEMETRY_ENABLED - tmFree( TMCM_GENERAL, this ); - #endif } // This gets called right before destruction - safe to call virtual functions. diff --git a/src/BlueAsyncRes.cpp b/src/BlueAsyncRes.cpp index 122e2df..6563e17 100644 --- a/src/BlueAsyncRes.cpp +++ b/src/BlueAsyncRes.cpp @@ -199,9 +199,6 @@ void BlueAsyncRes::InitializeImpl( const wchar_t* name, const wchar_t* ext ) CW2A asciiName( name ); CCP_LOG( "BlueAsyncRes::Initialize: %s", (const char*)asciiName ); -#if CCP_TELEMETRY_ENABLED - tmAlloc( TMCM_GENERAL, this, 1111111, "%s", tmDynamicString( TMCM_GENERAL, (const char*)asciiName ) ); -#endif m_ext = ext; From a4d019bce2eb2c9ee4763d8fdce432edf01e78e7 Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:11:16 +0000 Subject: [PATCH 003/131] Replace RadGameTools' Telemetry API calls with Tracy This tries to keep as much functionality from Telemetry available as possible. However, there are a few subtle changes, notable the concept of a "Time Span" doesn't exist in Tracy in the same way as it does in Telemetry. --- CMakeLists.txt | 7 +- include/BlueStatistics.h | 26 +++++- src/BlueOS.cpp | 6 +- src/BlueStatistics.cpp | 154 +++++++++++++++++++++++++++--------- src/BlueStatistics_Blue.cpp | 14 ++-- src/StdAfx.h | 2 + src/TaskletTimer.cpp | 6 +- 7 files changed, 156 insertions(+), 59 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b74533..10326c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -338,7 +338,12 @@ if(WIN32) winhttp psapi iphlpapi -) + ) + add_custom_command( + TARGET Blue POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy $ $ + COMMAND_EXPAND_LISTS + ) endif() target_include_directories(Blue diff --git a/include/BlueStatistics.h b/include/BlueStatistics.h index bbf8138..c837c58 100644 --- a/include/BlueStatistics.h +++ b/include/BlueStatistics.h @@ -152,13 +152,35 @@ void BLUEIMPORT tmTaskletEnter( uint32_t ctx, const char* name ); void BLUEIMPORT tmTaskletLeave( uint32_t ctx ); void tmTaskletAppendText( uint32_t ctx, const char* appendText ); +class BLUEIMPORT TracyZone +{ +public: + TracyZone() = delete; + TracyZone( uint32_t ctx, const char* name, const char* filename, uint32_t lineno, uint32_t color = 3766446u ); + ~TracyZone(); + TracyZone( TracyZone&& other ) noexcept; + TracyZone( const TracyZone& ) = delete; + TracyZone& operator=( TracyZone&& ) = delete; + TracyZone& operator=( const TracyZone& ) = delete; + + void text( const char* text ) const; + +private: + std::optional m_telemetryContext; + PyTaskletObject* m_fiber{nullptr}; +}; + +void BLUEIMPORT TracyEnterZone( void* key, const char* name, const char* filename, uint32_t lineno ); +void BLUEIMPORT TracyLeaveZone( void* key ); +void TracyZoneAddText( void* key, const char* text ); + #define CCP_STATS_SCOPED_TIME( identifier ) \ - tmTaskletZone zone_##_COUNTER_( TMCM_CPP, g_ccpStatistics_##identifier.GetName().c_str() );\ + TracyZone tracy_zone_##__COUNTER__( TMCM_CPP, g_ccpStatistics_##identifier.GetName().c_str(), __FILE__, __LINE__ );\ CcpStatisticsStopwatch ccpStatsStopwatch_##identifier( g_ccpStatistics_##identifier ) #undef CCP_STATS_ZONE #define CCP_STATS_ZONE( name ) \ - tmTaskletZone zone_##_COUNTER_( TMCM_CPP, name ) + TracyZone tracy_zone_##__COUNTER__( TMCM_CPP, name, __FILE__, __LINE__ ); #else #define CCP_STATS_SCOPED_TIME( identifier ) CcpStatisticsStopwatch ccpStatsStopwatch_##identifier( g_ccpStatistics_##identifier ) diff --git a/src/BlueOS.cpp b/src/BlueOS.cpp index ce9b746..b675240 100644 --- a/src/BlueOS.cpp +++ b/src/BlueOS.cpp @@ -913,11 +913,7 @@ void BlueOS::PumpOSInternal() // Sleep until we need to wake up { - //CCP_STATS_ZONE( "BlueOS/PumpOS/DoSleep" ); - // Defining this directly to be able to mark this as an "idle" telemetry zone -#if CCP_TELEMETRY_ENABLED - tmZone( TMCM_GENERAL, TMZF_IDLE, "BlueOS/PumpOS/DoSleep" ); -#endif + CCP_STATS_ZONE( "BlueOS/PumpOS/DoSleep" ); DoSleep(); mNextScheduledEvent = int( mSleepTime ); } diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 8d22084..531e572 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -68,13 +68,19 @@ const TaskletInfo s_fallbackInfo = { "Tasklet?", "", 0 }; TaskletInfo s_lastTasklet = s_fallbackInfo; // need to remember last activated tasklet for tmEnd std::unordered_map s_taskletFree; // original tp_free functions for tasklet types -tm_uint32 s_taskletTrackID = 0; // telemetry track ID for tasklet time spans +uint32_t s_taskletTrackID = 0; // telemetry track ID for tasklet time spans + +thread_local PyTaskletObject* g_activeFiber{nullptr}; +typedef std::unordered_map FiberNameStore; +thread_local FiberNameStore g_fiberNameStore; + +typedef std::unordered_map TasketZoneStore; +thread_local TasketZoneStore g_taskletZoneStore; // Overriden tp_free function for tasklets: notify telemetry and call original tp_free void OnTaskletFree( void* tasklet ) { - tmEndFiber( 0, reinterpret_cast( tasklet ) ); - + g_fiberNameStore.erase( (PyTaskletObject*) tasklet ); auto found = s_taskletFree.find( Py_TYPE( tasklet ) ); if( found != end( s_taskletFree ) ) { @@ -148,13 +154,16 @@ int PythonProfiler( PyObject* obj, PyFrameObject* frame, int what, PyObject* arg { switch( what ) { - case PyTrace_CALL: - tmEnterEx( 0, nullptr, 0, 0, PyString_AsString( frame->f_code->co_filename ), PyFrame_GetLineNumber( frame ), TMZF_NONE, "%s", PyString_AsString( frame->f_code->co_name ) ); - tmZoneColor( 0, 94.f / 255.f, 32.f / 255.f ); - break; + case PyTrace_CALL: { + auto zoneName = Immortalize( frame->f_code->co_name ); + auto fileName = Immortalize( frame->f_code->co_filename ); + if( zoneName && fileName ) + TracyEnterZone( frame, zoneName, fileName, PyFrame_GetLineNumber( frame ) ); + } + break; case PyTrace_EXCEPTION: case PyTrace_RETURN: - tmLeave( 0 ); + TracyLeaveZone( frame ); break; default: break; @@ -200,7 +209,6 @@ void BlueStatistics::StartTimedTelemetry( const std::string& server, float sampl } s_telemetryServerOrFileSystemDumpPath = server; - s_telemetryConnectionType = TMCT_TCP; s_telemetrySamplePeriod = (float)samplePeriod; s_isTelemetryConnectionRequested = true; #else @@ -215,7 +223,6 @@ void BlueStatistics::StartTelemetryDump( const std::string& dumpFolder, float sa CCP_LOGERR( "Telemetry is already running!" ); return; } - s_telemetryConnectionType = TMCT_FILE; s_telemetryServerOrFileSystemDumpPath = dumpFolder; s_telemetrySamplePeriod = (float)samplePeriod; s_isTelemetryConnectionRequested = true; @@ -226,7 +233,6 @@ void BlueStatistics::StartTelemetryDump( const std::string& dumpFolder, float sa void BlueStatistics::PauseTelemetry() { #if CCP_TELEMETRY_ENABLED - tmPause( TMCM_GENERAL, 1 ); s_isTelemetryPaused = true; #endif } @@ -234,7 +240,6 @@ void BlueStatistics::PauseTelemetry() void BlueStatistics::ResumeTelemetry() { #if CCP_TELEMETRY_ENABLED - tmPause( TMCM_GENERAL, 0 ); s_isTelemetryPaused = false; #endif } @@ -250,8 +255,6 @@ void BlueStatistics::StopTelemetry() { s_isTelemetryConnected = false; s_isTelemetryShuttingDown = true; - - tmPause( TMCM_GENERAL, 1 ); } #endif } @@ -291,13 +294,6 @@ void BlueStatistics::UpdateTelemetry() // There doesn't seem to be an easy way to back up the previous profiler function, so it gets lost PyEval_SetProfile( &PythonProfiler, nullptr ); } - - if( s_isTelemetryTaskletCaptureEnabled ) - { - s_taskletTrackID = tmNewTimeSpanTrackID(); - tmTrackName( 0, s_taskletTrackID, "Tasklets" ); - tmTrackOrder( 0, s_taskletTrackID, 0 ); - } #endif } s_isTelemetryConnectionRequested = false; @@ -343,6 +339,24 @@ void BlueStatistics::UpdateTelemetry() #endif } +void SwitchToFiber( PyTaskletObject* to ) +{ + if ( to ) + { + auto existing = g_fiberNameStore.find( to ); + if( existing == g_fiberNameStore.cend() ) + { + auto toInfo = GetTaskletInfo( to ); + // Construct the tracy fiber name as `function_name (address_of_tasklet)` + // This should deal with the case of `function_name` running in two different tasklets + g_fiberNameStore[to] = std::string( toInfo.name ) + " (" + std::to_string( reinterpret_cast( to ) ) + ")"; + existing = g_fiberNameStore.find( to ); + } + TracyFiberEnter( existing->second.c_str() ); + } + g_activeFiber = to; +} + #if CCP_STACKLESS void BlueStatistics::OnTaskletSwitch( PyObject* _from, PyObject* _to ) { @@ -350,29 +364,24 @@ void BlueStatistics::OnTaskletSwitch( PyObject* _from, PyObject* _to ) PyTaskletObject* from = (PyTaskletObject*)_from; PyTaskletObject* to = (PyTaskletObject*)_to; - if( tmRunning() ) + if( TracyIsStarted && TracyIsConnected ) { StoreFree( from ); StoreFree( to ); - if( s_isTelemetryTaskletCaptureEnabled && from && !PyTasklet_IsMain( from ) ) - { - tmEndTimeSpanEx( 0, reinterpret_cast( from ), s_lastTasklet.filename, s_lastTasklet.line ); - } - if( !to || PyTasklet_IsMain( to ) ) { - tmSwitchToFiber( 0, 0 ); + if ( g_activeFiber ) + { + TracyFiberLeave; + } + g_activeFiber = nullptr; } else { - tmSwitchToFiber( 0, reinterpret_cast( to ) ); - if( s_isTelemetryTaskletCaptureEnabled ) { - auto toInfo = GetTaskletInfo( to ); - tmBeginColoredTimeSpanEx( 0, reinterpret_cast( to ), s_taskletTrackID, 0, TMZF_NONE, toInfo.filename, toInfo.line, "%s", toInfo.name ); - s_lastTasklet = toInfo; + SwitchToFiber( to ); } } } @@ -477,9 +486,6 @@ ICcpStatisticsAccumulator* BlueStatistics::GetAccumulator( const std::string& na void BlueStatistics::SetTimelineSectionName( const char* name ) { -#if CCP_TELEMETRY_ENABLED - tmSetTimelineSectionName( TMCM_GENERAL, name ); -#endif } void BlueStatistics::SetCppCaptureEnabled( bool b ) @@ -720,7 +726,7 @@ void tmTaskletEnter( uint32_t ctx, const char* name ) // so we have to filter out zones manually if( s_isTelemetryCppCaptureEnabled || (ctx & TMCM_CPP) == 0 ) { - tmEnter( ctx, TMZF_NONE, "%s", name ); + tmEnter( TMCM_CPP, TMZF_NONE, "%s", name ); } } @@ -738,7 +744,41 @@ void tmTaskletAppendText( uint32_t ctx, const char* appendText ) tmMessage( ctx, TMMF_ZONE_SUBLABEL, "%s", tmDynamicString( TMCM_GENERAL, appendText ) ); } -tmTaskletZone::tmTaskletZone( uint32_t ctx, const char* name ) : m_telemetryContext( ctx ) +void TracyEnterZone( void* key, const char* name, const char* filename, uint32_t lineno ) +{ + if( TracyIsStarted && TracyIsConnected ) + { + if( g_taskletZoneStore.find( key ) == g_taskletZoneStore.end() ) + { + g_taskletZoneStore.insert( { key, TracyZone( TMCM_CPP, name, filename, lineno, 16765248u ) } ); + } + } +} + +void TracyLeaveZone( void* key ) +{ + if( TracyIsStarted && TracyIsConnected ) + { + if( g_taskletZoneStore.find( key ) != g_taskletZoneStore.end() ) + { + g_taskletZoneStore.erase( key ); + } + } +} + +void TracyZoneAddText( void* key, const char* text ) +{ + if( TracyIsStarted && TracyIsConnected ) + { + auto zone = g_taskletZoneStore.find( key ); + if( zone != g_taskletZoneStore.end() ) + { + zone->second.text( text ); + } + } +} + +tmTaskletZone::tmTaskletZone( uint32_t ctx, const char* name ) { tmTaskletEnter( ctx, name ); } @@ -748,6 +788,42 @@ tmTaskletZone::~tmTaskletZone() tmTaskletLeave( m_telemetryContext ); } -#endif +TracyZone::TracyZone( uint32_t ctx, const char* name, const char* filename, uint32_t lineno, uint32_t color ) : m_fiber( g_activeFiber ) +{ + if( TracyIsStarted && TracyIsConnected ) + { + CCP_ASSERT( filename != nullptr ); + CCP_ASSERT( name != nullptr ); + auto data = ___tracy_alloc_srcloc( lineno, filename, strlen( filename ), name, strlen( name ), color ); + m_telemetryContext.emplace( ___tracy_emit_zone_begin_alloc( data, ctx & TMCM_CPP ) ); + } +} + +TracyZone::TracyZone( TracyZone&& other ) noexcept +{ + m_telemetryContext = other.m_telemetryContext; + // mark this instance's zone as inactive in case the destructor runs + other.m_telemetryContext.reset(); +} + +TracyZone::~TracyZone() +{ + if( TracyIsStarted && TracyIsConnected && m_telemetryContext ) + { + auto previous = g_activeFiber; + SwitchToFiber( m_fiber ); + TracyCZoneEnd( m_telemetryContext.value() ); + SwitchToFiber( previous ); + } +} +void TracyZone::text( const char* text ) const +{ + if( TracyIsStarted && TracyIsConnected && m_telemetryContext ) + { + CCP_ASSERT( text != nullptr ); + TracyCZoneText( m_telemetryContext.value(), text, strlen( text ) ); + } +} +#endif diff --git a/src/BlueStatistics_Blue.cpp b/src/BlueStatistics_Blue.cpp index 855c57c..7e61154 100644 --- a/src/BlueStatistics_Blue.cpp +++ b/src/BlueStatistics_Blue.cpp @@ -293,7 +293,7 @@ PyObject* PyEnterZone( PyObject* self, PyObject* args ) return nullptr; } - tmTaskletEnter( TMCM_GENERAL, zone ); + TracyEnterZone( self, zone, __FILE__, __LINE__ ); #endif Py_RETURN_NONE; } @@ -301,7 +301,7 @@ PyObject* PyEnterZone( PyObject* self, PyObject* args ) PyObject* PyLeaveZone( PyObject* self, PyObject* args ) { #if CCP_TELEMETRY_ENABLED - tmTaskletLeave( TMCM_GENERAL ); + TracyLeaveZone( self ); #endif Py_RETURN_NONE; } @@ -328,9 +328,9 @@ PyObject* PyAppendToZone( PyObject* self, PyObject* args ) } #if CCP_TELEMETRY_ENABLED -static TmU64 s_timespanId = 0xf00000000; +static uint64_t s_timespanId = 0xf00000000; #endif - + PyObject* PyBeginTimeSpan( PyObject* self, PyObject* args ) { #if CCP_TELEMETRY_ENABLED @@ -348,7 +348,6 @@ PyObject* PyBeginTimeSpan( PyObject* self, PyObject* args ) } ++s_timespanId; - tmBeginTimeSpan( TMCM_GENERAL, s_timespanId, TMTSF_NONE, label ); return PyLong_FromLongLong( s_timespanId ); #else @@ -359,7 +358,7 @@ PyObject* PyBeginTimeSpan( PyObject* self, PyObject* args ) PyObject* PyEndTimeSpan( PyObject* self, PyObject* args ) { #if CCP_TELEMETRY_ENABLED - TmU64 id = 0; + uint64_t id = 0; PyObject* labelO; if( !PyArg_ParseTuple( args, "LO", &id, &labelO ) ) @@ -372,8 +371,6 @@ PyObject* PyEndTimeSpan( PyObject* self, PyObject* args ) { return nullptr; } - - tmEndTimeSpan( TMCM_GENERAL, id, TMTSF_NONE, label ); #endif Py_RETURN_NONE; } @@ -825,4 +822,3 @@ const Be::ClassInfo* BlueStatistics::ExposeToBlue() EXPOSURE_END() } - diff --git a/src/StdAfx.h b/src/StdAfx.h index 97f3027..ea60561 100644 --- a/src/StdAfx.h +++ b/src/StdAfx.h @@ -20,6 +20,8 @@ #endif #endif +#include + // include all python stuff #if BLUE_WITH_PYTHON #include diff --git a/src/TaskletTimer.cpp b/src/TaskletTimer.cpp index 0f8e284..2827760 100644 --- a/src/TaskletTimer.cpp +++ b/src/TaskletTimer.cpp @@ -3,7 +3,7 @@ TaskletTimer.cpp - Author: Kristján Valur Jónsson + Author: Kristján Valur Jónsson Created: Sept 2004 OS: Win32 Project: Yep @@ -177,7 +177,7 @@ PyObject *TaskletTimer::EnterTaskletEx(PyObject *newContext, TASKLETFLAGS flags) Py_DECREF(strifiedName); #if CCP_TELEMETRY_ENABLED if (canonicalName) { - tmTaskletEnter(TMCM_GENERAL, PyString_AsString(canonicalName)); + TracyEnterZone( mSimpleCtxt, PyString_AsString(canonicalName), __FILE__, __LINE__ ); } #else CCP_UNUSED( canonicalName ); @@ -243,7 +243,7 @@ bool TaskletTimer::ReturnFromTasklet(PyObject *backContext) #if CCP_TELEMETRY_ENABLED if (mDoTelemetry) { - tmTaskletLeave( TMCM_GENERAL ); + TracyLeaveZone( mSimpleCtxt ); } #endif // Temporary hack to support IDLE From fbfbb9a167027771f3311f9cda700d2ec8656426 Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Tue, 7 Jan 2025 13:06:55 +0000 Subject: [PATCH 004/131] Replace magic number with tracy color constant It's not exactly the same color, but should be close enough. --- include/BlueStatistics.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/BlueStatistics.h b/include/BlueStatistics.h index c837c58..d805031 100644 --- a/include/BlueStatistics.h +++ b/include/BlueStatistics.h @@ -156,7 +156,7 @@ class BLUEIMPORT TracyZone { public: TracyZone() = delete; - TracyZone( uint32_t ctx, const char* name, const char* filename, uint32_t lineno, uint32_t color = 3766446u ); + TracyZone( uint32_t ctx, const char* name, const char* filename, uint32_t lineno, uint32_t color = tracy::Color::SteelBlue4 ); ~TracyZone(); TracyZone( TracyZone&& other ) noexcept; TracyZone( const TracyZone& ) = delete; From b0bd6a051ad4c1c0f2e6d7cb8b19f74a015fe96e Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Thu, 9 Jan 2025 11:06:30 +0000 Subject: [PATCH 005/131] Replace magic number with tracy color constant It's not exactly the same color, but should be close enough. --- src/BlueStatistics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 531e572..1adf009 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -750,7 +750,7 @@ void TracyEnterZone( void* key, const char* name, const char* filename, uint32_t { if( g_taskletZoneStore.find( key ) == g_taskletZoneStore.end() ) { - g_taskletZoneStore.insert( { key, TracyZone( TMCM_CPP, name, filename, lineno, 16765248u ) } ); + g_taskletZoneStore.insert( { key, TracyZone( TMCM_CPP, name, filename, lineno, tracy::Color::Yellow ) } ); } } } From ea28cc2ac98ff048e9f02b5f880653b1fbf75d19 Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Thu, 9 Jan 2025 11:06:50 +0000 Subject: [PATCH 006/131] Add explanation for switching fibers --- src/BlueStatistics.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 1adf009..25ba16b 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -810,6 +810,7 @@ TracyZone::~TracyZone() { if( TracyIsStarted && TracyIsConnected && m_telemetryContext ) { + // Zones need to end on the same fiber they were started from, so do a little song and dance to ensure that auto previous = g_activeFiber; SwitchToFiber( m_fiber ); TracyCZoneEnd( m_telemetryContext.value() ); From 875dfb2c64abefc687526db37dc13dbe093cfdd6 Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Thu, 9 Jan 2025 11:09:14 +0000 Subject: [PATCH 007/131] Remove telemetry instrumentation from TaskletTimer First of all, it's an _extra_ zone created; tasklets are already tracked through `OnTaskletSwitch` in `BlueStatistics`. Secondly, for some unknown reason the instrumentation here is causing weird errors in Tracy around ending a zone. No investigation happened because of point 1, e.g. removing this does not lose any information. --- src/TaskletTimer.cpp | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/src/TaskletTimer.cpp b/src/TaskletTimer.cpp index 2827760..c0410b4 100644 --- a/src/TaskletTimer.cpp +++ b/src/TaskletTimer.cpp @@ -96,7 +96,6 @@ TaskletTimer::TaskletTimer() : mMaxWarn(1), mStackMap( "TaskletTimer/mStackMap" ), mSimpleCtxt(Py_None, true), - mCanonicalizationDict(PyDict_New()), m_BlueOSPumpCountAtStart( 0 ) { mCurrentStack = 0; @@ -153,37 +152,6 @@ PyObject *TaskletTimer::EnterTaskletStr(const char *context, TASKLETFLAGS flags) PyObject *TaskletTimer::EnterTaskletEx(PyObject *newContext, TASKLETFLAGS flags) { - if (mDoTelemetry) { - PyObject *strifiedName = PyObject_Str(newContext); - - if (strifiedName == NULL) { - return NULL; - } - - PyObject *canonicalName = PyDict_GetItem(mCanonicalizationDict, strifiedName); - - if (!canonicalName) { - // First time we've seen this particular name, set it in the dict and use it directly - int err = PyDict_SetItem(mCanonicalizationDict, strifiedName, strifiedName); - if (err != 0) { - // Not being able to set an entry in our dict is pretty fatal, boil the exception on up. - return NULL; - } - - // Success, we are the canonical string for this forever and ever and ever now - canonicalName = strifiedName; - } - - Py_DECREF(strifiedName); -#if CCP_TELEMETRY_ENABLED - if (canonicalName) { - TracyEnterZone( mSimpleCtxt, PyString_AsString(canonicalName), __FILE__, __LINE__ ); - } -#else - CCP_UNUSED( canonicalName ); -#endif - } - mFlags = flags; //temporary hack to support the IDLE flag if (!mActive) { //timer is not enabled. @@ -240,12 +208,6 @@ PyObject *TaskletTimer::EnterTaskletEx(PyObject *newContext, TASKLETFLAGS flags) bool TaskletTimer::ReturnFromTasklet(PyObject *backContext) { -#if CCP_TELEMETRY_ENABLED - if (mDoTelemetry) - { - TracyLeaveZone( mSimpleCtxt ); - } -#endif // Temporary hack to support IDLE if (mFlags) { TimesliceReset(); From 19b6d6d912050fe733db63ddcc7c9a47fa93d8e3 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Thu, 9 Jan 2025 15:48:20 +0000 Subject: [PATCH 008/131] Fix compile errors --- include/BlueStatistics.h | 3 ++- src/BlueStatistics.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/include/BlueStatistics.h b/include/BlueStatistics.h index d805031..9832003 100644 --- a/include/BlueStatistics.h +++ b/include/BlueStatistics.h @@ -11,6 +11,7 @@ #define BlueStatistics_h #include "ICcpStatisticsAccumulator.h" +#include BLUE_DECLARE( CcpStatisticsEntry ); @@ -167,7 +168,7 @@ class BLUEIMPORT TracyZone private: std::optional m_telemetryContext; - PyTaskletObject* m_fiber{nullptr}; + void* m_fiber{nullptr}; }; void BLUEIMPORT TracyEnterZone( void* key, const char* name, const char* filename, uint32_t lineno ); diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 25ba16b..a16d303 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -812,7 +812,7 @@ TracyZone::~TracyZone() { // Zones need to end on the same fiber they were started from, so do a little song and dance to ensure that auto previous = g_activeFiber; - SwitchToFiber( m_fiber ); + SwitchToFiber( (PyTaskletObject*) m_fiber ); TracyCZoneEnd( m_telemetryContext.value() ); SwitchToFiber( previous ); } From 076c2a2e6418dbe39b0c2e888a9f2c29db5c0395 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Mon, 13 Jan 2025 13:51:34 +0000 Subject: [PATCH 009/131] Fix `TracyZone` `C4251` compiler warning This warning originates from a public header file. Any external component which includes `BlueStatistics.h` will also emit this warning when building. This causes issues for components such as `Trinity` which treat build warnings as errors. The reason for the warning is that the implementation of `std:optional` is dependent on the compiler. Exporting a symbol of that type through a DLL is therefore generally unsafe as there can be no __general__ guarantee that any two DLLs were built using the same toolchain. With that said, this is warning is not indicative of a problem for Carbon components where the toolchain is pre-determined. The warning only becomes an issue when elevated in severity. --- include/BlueStatistics.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/BlueStatistics.h b/include/BlueStatistics.h index 9832003..b271a90 100644 --- a/include/BlueStatistics.h +++ b/include/BlueStatistics.h @@ -153,12 +153,12 @@ void BLUEIMPORT tmTaskletEnter( uint32_t ctx, const char* name ); void BLUEIMPORT tmTaskletLeave( uint32_t ctx ); void tmTaskletAppendText( uint32_t ctx, const char* appendText ); -class BLUEIMPORT TracyZone +class TracyZone { public: TracyZone() = delete; - TracyZone( uint32_t ctx, const char* name, const char* filename, uint32_t lineno, uint32_t color = tracy::Color::SteelBlue4 ); - ~TracyZone(); + BLUEIMPORT TracyZone( uint32_t ctx, const char* name, const char* filename, uint32_t lineno, uint32_t color = tracy::Color::SteelBlue4 ); + BLUEIMPORT ~TracyZone(); TracyZone( TracyZone&& other ) noexcept; TracyZone( const TracyZone& ) = delete; TracyZone& operator=( TracyZone&& ) = delete; From ef79e186b6a0a274690d0699f57d63d82555bd25 Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Wed, 15 Jan 2025 14:03:18 +0000 Subject: [PATCH 010/131] Set Tracy program name This way discovering the correct process to profile can be somewhat easier, depending on the passed in name. Re-using the existing but now obsolete "server or dump file name" parameter for this to keep the amount of changes from the outside to be low. Should get a "proper" clean-up in the Python 3 version since we can break some backwards compatibility there. --- src/BlueStatistics.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index a16d303..c0a0ee5 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -288,6 +288,7 @@ void BlueStatistics::UpdateTelemetry() if( s_isTelemetryConnected ) { + TracySetProgramName( s_telemetryServerOrFileSystemDumpPath.c_str() ); #if CCP_STACKLESS if( s_isTelemetryPythonCaptureEnabled ) { From c86c37652c0806a66e5d5b8a62a0448994b59e7f Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Wed, 15 Jan 2025 14:06:36 +0000 Subject: [PATCH 011/131] Fix client locking up when stopping profiler The problem here was that simply destroying the profiler using `tracy::ShutdownProfiler` causes the tracy network thread to wait indefinately for more communication. Calling `tracy::RequestShutdown` first stops the network communication thread, and thus avoids the lock up. However, there still is an outstanding issue that requires terminating the session in the tracy visualizer tool in order to restart profiling in the same exefile process session. --- src/BlueStatistics.cpp | 72 +++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index c0a0ee5..66b9a89 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -251,11 +251,40 @@ void BlueStatistics::StopTelemetry() { return; } - if( s_isTelemetryConnected ) + + if( s_isTelemetryPythonCaptureEnabled ) + { + PyEval_SetProfile( nullptr, nullptr ); + } + +#if CCP_STACKLESS + for( auto& free : s_taskletFree ) + { + free.first->tp_free = free.second; + } + s_taskletFree.clear(); + + s_lastTasklet = s_fallbackInfo; +#endif + // Wrap up instrumentation data + g_taskletZoneStore.clear(); + if ( g_activeFiber ) { - s_isTelemetryConnected = false; - s_isTelemetryShuttingDown = true; + TracyFiberLeave; + g_activeFiber = nullptr; } + + // Ensure it gets flushed + CcpTelemetryTick(); + + // Ensure we stop profiling + s_isTelemetryConnected = false; + s_isTelemetryShuttingDown = true; + + // Finally, ask for it to be shutdown + tracy::GetProfiler().RequestShutdown(); + + CcpTelemetryTick(); #endif } @@ -302,27 +331,18 @@ void BlueStatistics::UpdateTelemetry() return; } - CcpTelemetryTick(); + if ( s_isTelemetryConnected && !s_isTelemetryShuttingDown ) + { + CcpTelemetryTick(); + } if( s_isTelemetryShuttingDown ) { - CcpStopTelemetry(); - - if( s_isTelemetryPythonCaptureEnabled ) - { - PyEval_SetProfile( nullptr, nullptr ); - } - -#if CCP_STACKLESS - for( auto& free : s_taskletFree ) + if ( tracy::GetProfiler().HasShutdownFinished() ) { - free.first->tp_free = free.second; + CcpStopTelemetry(); + s_isTelemetryShuttingDown = false; } - s_taskletFree.clear(); - - s_lastTasklet = s_fallbackInfo; -#endif - s_isTelemetryShuttingDown = false; } else if(s_telemetrySamplePeriod > 0.0f ) // Check if we have passed our timed sample time { @@ -365,7 +385,7 @@ void BlueStatistics::OnTaskletSwitch( PyObject* _from, PyObject* _to ) PyTaskletObject* from = (PyTaskletObject*)_from; PyTaskletObject* to = (PyTaskletObject*)_to; - if( TracyIsStarted && TracyIsConnected ) + if( s_isTelemetryConnected ) { StoreFree( from ); StoreFree( to ); @@ -747,7 +767,7 @@ void tmTaskletAppendText( uint32_t ctx, const char* appendText ) void TracyEnterZone( void* key, const char* name, const char* filename, uint32_t lineno ) { - if( TracyIsStarted && TracyIsConnected ) + if( s_isTelemetryConnected ) { if( g_taskletZoneStore.find( key ) == g_taskletZoneStore.end() ) { @@ -758,7 +778,7 @@ void TracyEnterZone( void* key, const char* name, const char* filename, uint32_t void TracyLeaveZone( void* key ) { - if( TracyIsStarted && TracyIsConnected ) + if( s_isTelemetryConnected ) { if( g_taskletZoneStore.find( key ) != g_taskletZoneStore.end() ) { @@ -769,7 +789,7 @@ void TracyLeaveZone( void* key ) void TracyZoneAddText( void* key, const char* text ) { - if( TracyIsStarted && TracyIsConnected ) + if( s_isTelemetryConnected ) { auto zone = g_taskletZoneStore.find( key ); if( zone != g_taskletZoneStore.end() ) @@ -791,7 +811,7 @@ tmTaskletZone::~tmTaskletZone() TracyZone::TracyZone( uint32_t ctx, const char* name, const char* filename, uint32_t lineno, uint32_t color ) : m_fiber( g_activeFiber ) { - if( TracyIsStarted && TracyIsConnected ) + if( s_isTelemetryConnected ) { CCP_ASSERT( filename != nullptr ); CCP_ASSERT( name != nullptr ); @@ -809,7 +829,7 @@ TracyZone::TracyZone( TracyZone&& other ) noexcept TracyZone::~TracyZone() { - if( TracyIsStarted && TracyIsConnected && m_telemetryContext ) + if( s_isTelemetryConnected && m_telemetryContext ) { // Zones need to end on the same fiber they were started from, so do a little song and dance to ensure that auto previous = g_activeFiber; @@ -821,7 +841,7 @@ TracyZone::~TracyZone() void TracyZone::text( const char* text ) const { - if( TracyIsStarted && TracyIsConnected && m_telemetryContext ) + if( s_isTelemetryConnected && m_telemetryContext ) { CCP_ASSERT( text != nullptr ); TracyCZoneText( m_telemetryContext.value(), text, strlen( text ) ); From bb48cc20b83d41368e2df0fcd5773f3d5248700c Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 18 Feb 2025 13:43:59 +0000 Subject: [PATCH 012/131] Fix `TracyZone` copy constructor The copied TracyZone object must remember the fiber it should switch to before notifying tracy of zone end. --- src/BlueStatistics.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 66b9a89..a74408b 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -822,6 +822,7 @@ TracyZone::TracyZone( uint32_t ctx, const char* name, const char* filename, uint TracyZone::TracyZone( TracyZone&& other ) noexcept { + m_fiber = other.m_fiber; m_telemetryContext = other.m_telemetryContext; // mark this instance's zone as inactive in case the destructor runs other.m_telemetryContext.reset(); From d0b11e67d2a836fc6c0f8c43a17fe3cf0f58c551 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 18 Feb 2025 13:45:30 +0000 Subject: [PATCH 013/131] Cleanup `SwitchToFiber` logic `SwitchToFiber` also needs to consider cases where `TracyFiberLeave` is to be called. Therefore this logic has been consolidated away from `OnTaskletSwitch` into `SwitchToFiber`. --- src/BlueStatistics.cpp | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index a74408b..c32af89 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -362,7 +362,15 @@ void BlueStatistics::UpdateTelemetry() void SwitchToFiber( PyTaskletObject* to ) { - if ( to ) + if( !to || PyTasklet_IsMain( to ) ) + { + if ( g_activeFiber ) + { + TracyFiberLeave; + } + g_activeFiber = nullptr; + } + else { auto existing = g_fiberNameStore.find( to ); if( existing == g_fiberNameStore.cend() ) @@ -374,8 +382,8 @@ void SwitchToFiber( PyTaskletObject* to ) existing = g_fiberNameStore.find( to ); } TracyFiberEnter( existing->second.c_str() ); + g_activeFiber = to; } - g_activeFiber = to; } #if CCP_STACKLESS @@ -390,20 +398,9 @@ void BlueStatistics::OnTaskletSwitch( PyObject* _from, PyObject* _to ) StoreFree( from ); StoreFree( to ); - if( !to || PyTasklet_IsMain( to ) ) + if( s_isTelemetryTaskletCaptureEnabled ) { - if ( g_activeFiber ) - { - TracyFiberLeave; - } - g_activeFiber = nullptr; - } - else - { - if( s_isTelemetryTaskletCaptureEnabled ) - { - SwitchToFiber( to ); - } + SwitchToFiber( to ); } } #endif From 045e41992aa14b6f06b46cf7214f46a03f1d32f2 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 18 Feb 2025 13:48:18 +0000 Subject: [PATCH 014/131] Make fiber name and tasklet zone store global This state needs to be shared between threads. --- src/BlueStatistics.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index c32af89..a656203 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -72,10 +72,10 @@ uint32_t s_taskletTrackID = 0; // telemetry track ID for tasklet time spans thread_local PyTaskletObject* g_activeFiber{nullptr}; typedef std::unordered_map FiberNameStore; -thread_local FiberNameStore g_fiberNameStore; +FiberNameStore g_fiberNameStore; typedef std::unordered_map TasketZoneStore; -thread_local TasketZoneStore g_taskletZoneStore; +TasketZoneStore g_taskletZoneStore; // Overriden tp_free function for tasklets: notify telemetry and call original tp_free void OnTaskletFree( void* tasklet ) From 8615ecd1a93a8715151913f9aa09ba06cd71d70d Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Thu, 20 Feb 2025 14:26:01 +0000 Subject: [PATCH 015/131] Fix Tracy invalid zone order instrumentation failure `TracyCZone` start and end order need to mimic stack behaviour. Keys passed to the `g_taskletZoneStore` map through `TracyEnterZone` and `TracyLeaveZone` need to be unique, a side effect of being a map, and also accessible from the instrumentation code. Passing `self` as a key to `g_taskletZoneStore` is problematic because `self` is a pointer to the module in-memory where the relevant function is exposed from. `self` is therefore not a reliable, unique key. This result in some skipped zone creations. Regardless a given zone is added, the instrumentation code always calls `TracyLeaveZone` on the same key, resulting in an out-of-order instrumentation failure as Tracy is expecting a different zone to end. Using a frame object fixes the issue as the object is unique to the current frame. This, however, imposes a requirement towards instrumented python code where `TracyLeaveZone` needs to be called from the same stack frame as where the call to `TracyEnterZone` is made. Reference: https://ccpgames.atlassian.net/browse/PLAT-8399 --- src/BlueStatistics_Blue.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BlueStatistics_Blue.cpp b/src/BlueStatistics_Blue.cpp index 7e61154..ef6987a 100644 --- a/src/BlueStatistics_Blue.cpp +++ b/src/BlueStatistics_Blue.cpp @@ -293,7 +293,7 @@ PyObject* PyEnterZone( PyObject* self, PyObject* args ) return nullptr; } - TracyEnterZone( self, zone, __FILE__, __LINE__ ); + TracyEnterZone( PyEval_GetFrame(), zone, __FILE__, __LINE__ ); #endif Py_RETURN_NONE; } @@ -301,7 +301,7 @@ PyObject* PyEnterZone( PyObject* self, PyObject* args ) PyObject* PyLeaveZone( PyObject* self, PyObject* args ) { #if CCP_TELEMETRY_ENABLED - TracyLeaveZone( self ); + TracyLeaveZone( PyEval_GetFrame() ); #endif Py_RETURN_NONE; } From 55fbcff9de836606fb5559f9c4428edc512a0ce8 Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Thu, 20 Feb 2025 18:32:09 +0000 Subject: [PATCH 016/131] Fix crash when calling `StopTelemetry` without `StartTelemetry` being called first Managing the state of the connection to a telemetry solution using multiple booleans is not very maintainable. Case in point, with the recent change to request the tracy profiler to shut down, a crash condition was introduced when calling `StopTelemetry` without ever calling `StartTelemetry`. This is because Tracy expects that profiler has been initialized before it's requested to shut down, and if it wasn't initialized then bad memory is accessed. Alas, this simplifies a lot of the nested if-else logic around the booleans by introducing a state enumeration. While this is still not perfect, then at least checking for the correct state is a lot simpler than before, and therefore it's more straight forward to correctly transition between states, as well as use the expected update behaviour for the current state. https://ccpgames.atlassian.net/browse/PLAT-8456 --- src/BlueStatistics.cpp | 142 +++++++++++++++++++++++------------------ 1 file changed, 80 insertions(+), 62 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index a656203..beac1d9 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -13,16 +13,22 @@ BlueStatistics* g_statistics = &s_statisticsInstance; BLUE_REGISTER_GLOBAL_AS_MODULE_OBJECT( "statistics", g_statistics ); -static bool s_isTelemetryConnectionRequested = false; -static bool s_isTelemetryConnected = false; +enum ProfilerState { + Stopped, + StartRequested, + Started, + Paused, + StopRequested, +}; + +ProfilerState s_profilerState{ProfilerState::Stopped}; + static bool s_isTelemetryCppCaptureEnabled = true; static bool s_isTelemetryTaskletCaptureEnabled = true; static bool s_isTelemetryPythonCaptureEnabled = false; -static bool s_isTelemetryPaused = false; static float s_telemetrySamplePeriod = 0.0f; // In seconds #if CCP_TELEMETRY_ENABLED -static bool s_isTelemetryShuttingDown = false; static int s_telemetryConnectionType = 0; static std::string s_telemetryServerOrFileSystemDumpPath; static Be::Time s_telemetryStartTime; @@ -202,15 +208,15 @@ void BlueStatistics::StartTelemetry( const std::string& server ) void BlueStatistics::StartTimedTelemetry( const std::string& server, float samplePeriod ) { #if CCP_TELEMETRY_ENABLED - if( s_isTelemetryConnected ) + if (s_profilerState != ProfilerState::Stopped ) { - CCP_LOGERR( "Telemetry is already running!" ); return; } s_telemetryServerOrFileSystemDumpPath = server; s_telemetrySamplePeriod = (float)samplePeriod; - s_isTelemetryConnectionRequested = true; + + s_profilerState = ProfilerState::StartRequested; #else #endif } @@ -218,14 +224,14 @@ void BlueStatistics::StartTimedTelemetry( const std::string& server, float sampl void BlueStatistics::StartTelemetryDump( const std::string& dumpFolder, float samplePeriod ) { #if CCP_TELEMETRY_ENABLED - if( s_isTelemetryConnected ) + if (s_profilerState != ProfilerState::Stopped ) { - CCP_LOGERR( "Telemetry is already running!" ); return; } s_telemetryServerOrFileSystemDumpPath = dumpFolder; s_telemetrySamplePeriod = (float)samplePeriod; - s_isTelemetryConnectionRequested = true; + + s_profilerState = ProfilerState::StartRequested; #else #endif } @@ -233,21 +239,28 @@ void BlueStatistics::StartTelemetryDump( const std::string& dumpFolder, float sa void BlueStatistics::PauseTelemetry() { #if CCP_TELEMETRY_ENABLED - s_isTelemetryPaused = true; + if ( s_profilerState != ProfilerState::Started ) { + return; + } + + s_profilerState = ProfilerState::Paused; #endif } void BlueStatistics::ResumeTelemetry() { #if CCP_TELEMETRY_ENABLED - s_isTelemetryPaused = false; + if ( s_profilerState != ProfilerState::Paused ) { + return; + } + s_profilerState = ProfilerState::Started; #endif } void BlueStatistics::StopTelemetry() { #if CCP_TELEMETRY_ENABLED - if( s_isTelemetryShuttingDown ) + if ( s_profilerState != ProfilerState::Started && s_profilerState != ProfilerState::Paused ) { return; } @@ -278,8 +291,7 @@ void BlueStatistics::StopTelemetry() CcpTelemetryTick(); // Ensure we stop profiling - s_isTelemetryConnected = false; - s_isTelemetryShuttingDown = true; + s_profilerState = ProfilerState::StopRequested; // Finally, ask for it to be shutdown tracy::GetProfiler().RequestShutdown(); @@ -290,12 +302,12 @@ void BlueStatistics::StopTelemetry() bool BlueStatistics::IsTelemetryConnected() { - return s_isTelemetryConnected; + return s_profilerState == ProfilerState::Started || s_profilerState == ProfilerState::Paused; } bool BlueStatistics::IsTelemetryConnectionRequested() { - return s_isTelemetryConnectionRequested; + return s_profilerState == ProfilerState::StartRequested; } float BlueStatistics::TelemetrySamplingTimeLeft() @@ -305,57 +317,63 @@ float BlueStatistics::TelemetrySamplingTimeLeft() bool BlueStatistics::IsTelemetryPaused() { - return s_isTelemetryPaused; + return s_profilerState == ProfilerState::Paused; } void BlueStatistics::UpdateTelemetry() { #if CCP_TELEMETRY_ENABLED - if( s_isTelemetryConnectionRequested && !s_isTelemetryShuttingDown ) + switch ( s_profilerState ) { - s_isTelemetryConnected = CcpStartTelemetry( s_telemetryServerOrFileSystemDumpPath.c_str(), s_telemetryConnectionType, m_telemetryMaxThreadCount ); - - if( s_isTelemetryConnected ) + case ProfilerState::StartRequested: { - TracySetProgramName( s_telemetryServerOrFileSystemDumpPath.c_str() ); -#if CCP_STACKLESS - if( s_isTelemetryPythonCaptureEnabled ) - { - // There doesn't seem to be an easy way to back up the previous profiler function, so it gets lost - PyEval_SetProfile( &PythonProfiler, nullptr ); + if ( CcpStartTelemetry( s_telemetryServerOrFileSystemDumpPath.c_str(), s_telemetryConnectionType, m_telemetryMaxThreadCount ) ) { + TracySetProgramName( s_telemetryServerOrFileSystemDumpPath.c_str() ); + if ( s_isTelemetryPythonCaptureEnabled ) { + PyEval_SetProfile( &PythonProfiler, nullptr ); + } + s_profilerState = ProfilerState::Started; + s_telemetryLastCheckTime = s_telemetryStartTime = BeOS->GetActualTime(); + } else { + CCP_LOGERR( "Failed to start Telemetry" ); + s_profilerState = ProfilerState::Stopped; } -#endif + break; } - s_isTelemetryConnectionRequested = false; - s_telemetryLastCheckTime = s_telemetryStartTime = BeOS->GetActualTime(); - return; - } - - if ( s_isTelemetryConnected && !s_isTelemetryShuttingDown ) - { - CcpTelemetryTick(); - } - - if( s_isTelemetryShuttingDown ) - { - if ( tracy::GetProfiler().HasShutdownFinished() ) + case ProfilerState::Started: { - CcpStopTelemetry(); - s_isTelemetryShuttingDown = false; + CcpTelemetryTick(); + if(s_telemetrySamplePeriod > 0.0f ) // Check if we have passed our timed sample time + { + Be::Time newTime = BeOS->GetActualTime(); + Be::Time delta = newTime - s_telemetryLastCheckTime; + s_telemetryLastCheckTime = newTime; + s_telemetrySamplePeriod -= ((float)delta / Be::Time(1e7)); + + if(s_telemetrySamplePeriod < 0.0f) + { + CCP_LOG( "Finalising timed Telemetry run." ); + StopTelemetry(); + } + } + break; } - } - else if(s_telemetrySamplePeriod > 0.0f ) // Check if we have passed our timed sample time - { - Be::Time newTime = BeOS->GetActualTime(); - Be::Time delta = newTime - s_telemetryLastCheckTime; - s_telemetryLastCheckTime = newTime; - s_telemetrySamplePeriod -= ((float)delta / Be::Time(1e7)); - - if(s_telemetrySamplePeriod < 0.0f) + case ProfilerState::StopRequested: { - CCP_LOG( "Finalising timed Telemetry run." ); - StopTelemetry(); + if ( tracy::GetProfiler().HasShutdownFinished() ) + { + CcpStopTelemetry(); + s_profilerState = ProfilerState::Stopped; + } + break; } + case ProfilerState::Paused: + case ProfilerState::Stopped: + // Nothing to do + break; + default: + CCP_LOGERR( "BlueStatistics::UpdateTelemetry - unhandled profiler state %d", s_profilerState ); + break; } #endif } @@ -393,7 +411,7 @@ void BlueStatistics::OnTaskletSwitch( PyObject* _from, PyObject* _to ) PyTaskletObject* from = (PyTaskletObject*)_from; PyTaskletObject* to = (PyTaskletObject*)_to; - if( s_isTelemetryConnected ) + if( s_profilerState == ProfilerState::Started ) { StoreFree( from ); StoreFree( to ); @@ -764,7 +782,7 @@ void tmTaskletAppendText( uint32_t ctx, const char* appendText ) void TracyEnterZone( void* key, const char* name, const char* filename, uint32_t lineno ) { - if( s_isTelemetryConnected ) + if( s_profilerState == ProfilerState::Started ) { if( g_taskletZoneStore.find( key ) == g_taskletZoneStore.end() ) { @@ -775,7 +793,7 @@ void TracyEnterZone( void* key, const char* name, const char* filename, uint32_t void TracyLeaveZone( void* key ) { - if( s_isTelemetryConnected ) + if( s_profilerState == ProfilerState::Started ) { if( g_taskletZoneStore.find( key ) != g_taskletZoneStore.end() ) { @@ -786,7 +804,7 @@ void TracyLeaveZone( void* key ) void TracyZoneAddText( void* key, const char* text ) { - if( s_isTelemetryConnected ) + if( s_profilerState == ProfilerState::Started ) { auto zone = g_taskletZoneStore.find( key ); if( zone != g_taskletZoneStore.end() ) @@ -808,7 +826,7 @@ tmTaskletZone::~tmTaskletZone() TracyZone::TracyZone( uint32_t ctx, const char* name, const char* filename, uint32_t lineno, uint32_t color ) : m_fiber( g_activeFiber ) { - if( s_isTelemetryConnected ) + if( s_profilerState == ProfilerState::Started ) { CCP_ASSERT( filename != nullptr ); CCP_ASSERT( name != nullptr ); @@ -827,7 +845,7 @@ TracyZone::TracyZone( TracyZone&& other ) noexcept TracyZone::~TracyZone() { - if( s_isTelemetryConnected && m_telemetryContext ) + if( s_profilerState == ProfilerState::Started && m_telemetryContext ) { // Zones need to end on the same fiber they were started from, so do a little song and dance to ensure that auto previous = g_activeFiber; @@ -839,7 +857,7 @@ TracyZone::~TracyZone() void TracyZone::text( const char* text ) const { - if( s_isTelemetryConnected && m_telemetryContext ) + if( s_profilerState == ProfilerState::Started && m_telemetryContext ) { CCP_ASSERT( text != nullptr ); TracyCZoneText( m_telemetryContext.value(), text, strlen( text ) ); From 14dd4769929577f99f91920329a281273e06f9ff Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 26 Feb 2025 17:10:59 +0000 Subject: [PATCH 017/131] Delay Tracy instrumentation until profiler is connected This change fixes an issue where the Tracy profiler would emit a zone double end failure if connected to early at the start of the client lifetime. Ending a zone prior to connecting to the profiler should be a no-op as Tracy is guarded against such eventualities via the Telemetry context active state. This does not appear to sufficiently guard against such eventualities as ending a zone on an empty zone stack. --- src/BlueStatistics.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index beac1d9..7ff26d2 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -327,14 +327,20 @@ void BlueStatistics::UpdateTelemetry() { case ProfilerState::StartRequested: { - if ( CcpStartTelemetry( s_telemetryServerOrFileSystemDumpPath.c_str(), s_telemetryConnectionType, m_telemetryMaxThreadCount ) ) { - TracySetProgramName( s_telemetryServerOrFileSystemDumpPath.c_str() ); - if ( s_isTelemetryPythonCaptureEnabled ) { - PyEval_SetProfile( &PythonProfiler, nullptr ); + if (tracy::IsProfilerStarted()) + { + if (tracy::GetProfiler().IsConnected()) + { + TracySetProgramName( s_telemetryServerOrFileSystemDumpPath.c_str() ); + if ( s_isTelemetryPythonCaptureEnabled ) { + PyEval_SetProfile( &PythonProfiler, nullptr ); + } + s_profilerState = ProfilerState::Started; + s_telemetryLastCheckTime = s_telemetryStartTime = BeOS->GetActualTime(); } - s_profilerState = ProfilerState::Started; - s_telemetryLastCheckTime = s_telemetryStartTime = BeOS->GetActualTime(); - } else { + } + else if (!CcpStartTelemetry( s_telemetryServerOrFileSystemDumpPath.c_str(), s_telemetryConnectionType, m_telemetryMaxThreadCount )) + { CCP_LOGERR( "Failed to start Telemetry" ); s_profilerState = ProfilerState::Stopped; } From f39db777e3a95f93c5b9a9ea939ba328db7f5939 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 26 Feb 2025 17:20:08 +0000 Subject: [PATCH 018/131] Propagate `ProfilerState::StartRequested` to python This allows elements such as the telemetry panel to react to `ProfilerState::StartRequested`. --- src/BlueStatistics.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 7ff26d2..ca0e689 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -260,7 +260,7 @@ void BlueStatistics::ResumeTelemetry() void BlueStatistics::StopTelemetry() { #if CCP_TELEMETRY_ENABLED - if ( s_profilerState != ProfilerState::Started && s_profilerState != ProfilerState::Paused ) + if ( !IsTelemetryConnected() ) { return; } @@ -302,7 +302,7 @@ void BlueStatistics::StopTelemetry() bool BlueStatistics::IsTelemetryConnected() { - return s_profilerState == ProfilerState::Started || s_profilerState == ProfilerState::Paused; + return s_profilerState == ProfilerState::Started || s_profilerState == ProfilerState::StartRequested || s_profilerState == ProfilerState::Paused; } bool BlueStatistics::IsTelemetryConnectionRequested() From bf4c20b740a4808fcf358bac6cd39bdc466199b0 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Thu, 27 Feb 2025 10:42:27 +0000 Subject: [PATCH 019/131] Revert "Propagate `ProfilerState::StartRequested` to python" This reverts commit f39db777e3a95f93c5b9a9ea939ba328db7f5939. --- src/BlueStatistics.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index ca0e689..7ff26d2 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -260,7 +260,7 @@ void BlueStatistics::ResumeTelemetry() void BlueStatistics::StopTelemetry() { #if CCP_TELEMETRY_ENABLED - if ( !IsTelemetryConnected() ) + if ( s_profilerState != ProfilerState::Started && s_profilerState != ProfilerState::Paused ) { return; } @@ -302,7 +302,7 @@ void BlueStatistics::StopTelemetry() bool BlueStatistics::IsTelemetryConnected() { - return s_profilerState == ProfilerState::Started || s_profilerState == ProfilerState::StartRequested || s_profilerState == ProfilerState::Paused; + return s_profilerState == ProfilerState::Started || s_profilerState == ProfilerState::Paused; } bool BlueStatistics::IsTelemetryConnectionRequested() From ba0f7b106b40a305ebd65170bdc745f45b47af0d Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 4 Mar 2025 13:42:20 +0000 Subject: [PATCH 020/131] Stop tracy profiler from fully shutting down An edge case exists in Tracy where long-living threads exceeding the lifetime of a single profiler session delimited by `tracy::StartupProfiler` and `Tracy::ShutdownProfiler` will cause a crash. This means we cannot fully shutdown the profiler in a safe manner without putting the application at risk. The workaround provided in this commit removes the ability for the tracy client to be fully disabled via `tracy::ShutdownProfiler` when calling `blue.statistics.StopTelemetry`. The consequence of this is that it's not longer possible to fully eliminate profiling overhead after a profiling session is started. See https://github.com/wolfpld/tracy/issues/999 for further details. --- src/BlueStatistics.cpp | 118 +++++++++++++++-------------------------- 1 file changed, 43 insertions(+), 75 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 7ff26d2..114e463 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -18,7 +18,6 @@ enum ProfilerState { StartRequested, Started, Paused, - StopRequested, }; ProfilerState s_profilerState{ProfilerState::Stopped}; @@ -239,7 +238,8 @@ void BlueStatistics::StartTelemetryDump( const std::string& dumpFolder, float sa void BlueStatistics::PauseTelemetry() { #if CCP_TELEMETRY_ENABLED - if ( s_profilerState != ProfilerState::Started ) { + if( s_profilerState != ProfilerState::Started ) + { return; } @@ -250,9 +250,11 @@ void BlueStatistics::PauseTelemetry() void BlueStatistics::ResumeTelemetry() { #if CCP_TELEMETRY_ENABLED - if ( s_profilerState != ProfilerState::Paused ) { + if ( s_profilerState != ProfilerState::Paused ) + { return; } + s_profilerState = ProfilerState::Started; #endif } @@ -260,43 +262,7 @@ void BlueStatistics::ResumeTelemetry() void BlueStatistics::StopTelemetry() { #if CCP_TELEMETRY_ENABLED - if ( s_profilerState != ProfilerState::Started && s_profilerState != ProfilerState::Paused ) - { - return; - } - - if( s_isTelemetryPythonCaptureEnabled ) - { - PyEval_SetProfile( nullptr, nullptr ); - } - -#if CCP_STACKLESS - for( auto& free : s_taskletFree ) - { - free.first->tp_free = free.second; - } - s_taskletFree.clear(); - - s_lastTasklet = s_fallbackInfo; -#endif - // Wrap up instrumentation data - g_taskletZoneStore.clear(); - if ( g_activeFiber ) - { - TracyFiberLeave; - g_activeFiber = nullptr; - } - - // Ensure it gets flushed - CcpTelemetryTick(); - - // Ensure we stop profiling - s_profilerState = ProfilerState::StopRequested; - - // Finally, ask for it to be shutdown - tracy::GetProfiler().RequestShutdown(); - - CcpTelemetryTick(); + PauseTelemetry(); #endif } @@ -364,15 +330,6 @@ void BlueStatistics::UpdateTelemetry() } break; } - case ProfilerState::StopRequested: - { - if ( tracy::GetProfiler().HasShutdownFinished() ) - { - CcpStopTelemetry(); - s_profilerState = ProfilerState::Stopped; - } - break; - } case ProfilerState::Paused: case ProfilerState::Stopped: // Nothing to do @@ -414,18 +371,20 @@ void SwitchToFiber( PyTaskletObject* to ) void BlueStatistics::OnTaskletSwitch( PyObject* _from, PyObject* _to ) { #if CCP_TELEMETRY_ENABLED - PyTaskletObject* from = (PyTaskletObject*)_from; - PyTaskletObject* to = (PyTaskletObject*)_to; - - if( s_profilerState == ProfilerState::Started ) + if (s_profilerState == ProfilerState::Stopped || s_profilerState == ProfilerState::StartRequested) { - StoreFree( from ); - StoreFree( to ); + return; + } - if( s_isTelemetryTaskletCaptureEnabled ) - { - SwitchToFiber( to ); - } + auto from = (PyTaskletObject*)_from; + auto to = (PyTaskletObject*)_to; + + StoreFree( from ); + StoreFree( to ); + + if( s_isTelemetryTaskletCaptureEnabled ) + { + SwitchToFiber( to ); } #endif } @@ -799,12 +758,16 @@ void TracyEnterZone( void* key, const char* name, const char* filename, uint32_t void TracyLeaveZone( void* key ) { - if( s_profilerState == ProfilerState::Started ) + // An edge case exists where TracyEnterZone is called whilst the profiler is started with a matching call to TracyLeaveZone when paused. + // Therefore, we have to notify Tracy of any valid calls to TracyLeaveZone, even when active instrumentation is paused. + if (!(s_profilerState == ProfilerState::Started || s_profilerState == ProfilerState::Paused)) { - if( g_taskletZoneStore.find( key ) != g_taskletZoneStore.end() ) - { - g_taskletZoneStore.erase( key ); - } + return; + } + + if( g_taskletZoneStore.find( key ) != g_taskletZoneStore.end() ) + { + g_taskletZoneStore.erase( key ); } } @@ -832,13 +795,15 @@ tmTaskletZone::~tmTaskletZone() TracyZone::TracyZone( uint32_t ctx, const char* name, const char* filename, uint32_t lineno, uint32_t color ) : m_fiber( g_activeFiber ) { - if( s_profilerState == ProfilerState::Started ) + if( s_profilerState != ProfilerState::Started ) { - CCP_ASSERT( filename != nullptr ); - CCP_ASSERT( name != nullptr ); - auto data = ___tracy_alloc_srcloc( lineno, filename, strlen( filename ), name, strlen( name ), color ); - m_telemetryContext.emplace( ___tracy_emit_zone_begin_alloc( data, ctx & TMCM_CPP ) ); + return; } + + CCP_ASSERT( filename != nullptr ); + CCP_ASSERT( name != nullptr ); + auto data = ___tracy_alloc_srcloc( lineno, filename, strlen( filename ), name, strlen( name ), color ); + m_telemetryContext.emplace( ___tracy_emit_zone_begin_alloc( data, ctx & TMCM_CPP ) ); } TracyZone::TracyZone( TracyZone&& other ) noexcept @@ -851,14 +816,17 @@ TracyZone::TracyZone( TracyZone&& other ) noexcept TracyZone::~TracyZone() { - if( s_profilerState == ProfilerState::Started && m_telemetryContext ) + // Notify Tracy of all zones ended with a valid context, regardless of profiler state + if( !m_telemetryContext ) { - // Zones need to end on the same fiber they were started from, so do a little song and dance to ensure that - auto previous = g_activeFiber; - SwitchToFiber( (PyTaskletObject*) m_fiber ); - TracyCZoneEnd( m_telemetryContext.value() ); - SwitchToFiber( previous ); + return; } + + // Zones need to end on the same fiber they were started from, so do a little song and dance to ensure that + auto previous = g_activeFiber; + SwitchToFiber( (PyTaskletObject*) m_fiber ); + TracyCZoneEnd( m_telemetryContext.value() ); + SwitchToFiber( previous ); } void TracyZone::text( const char* text ) const From 4acb1cde335078858f46de9c5328b35ae107a951 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Thu, 6 Mar 2025 14:03:22 +0000 Subject: [PATCH 021/131] Make `s_profilerState` atomic Profiler state is accessed outside the main thread and should be made atomic in order to avoid race conditions. --- src/BlueStatistics.cpp | 44 +++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 114e463..270b811 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -20,7 +20,7 @@ enum ProfilerState { Paused, }; -ProfilerState s_profilerState{ProfilerState::Stopped}; +std::atomic s_profilerState{ProfilerState::Stopped}; static bool s_isTelemetryCppCaptureEnabled = true; static bool s_isTelemetryTaskletCaptureEnabled = true; @@ -207,7 +207,7 @@ void BlueStatistics::StartTelemetry( const std::string& server ) void BlueStatistics::StartTimedTelemetry( const std::string& server, float samplePeriod ) { #if CCP_TELEMETRY_ENABLED - if (s_profilerState != ProfilerState::Stopped ) + if (s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Stopped ) { return; } @@ -215,7 +215,7 @@ void BlueStatistics::StartTimedTelemetry( const std::string& server, float sampl s_telemetryServerOrFileSystemDumpPath = server; s_telemetrySamplePeriod = (float)samplePeriod; - s_profilerState = ProfilerState::StartRequested; + s_profilerState.store( ProfilerState::StartRequested, std::memory_order_release ); #else #endif } @@ -223,14 +223,14 @@ void BlueStatistics::StartTimedTelemetry( const std::string& server, float sampl void BlueStatistics::StartTelemetryDump( const std::string& dumpFolder, float samplePeriod ) { #if CCP_TELEMETRY_ENABLED - if (s_profilerState != ProfilerState::Stopped ) + if (s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Stopped ) { return; } s_telemetryServerOrFileSystemDumpPath = dumpFolder; s_telemetrySamplePeriod = (float)samplePeriod; - s_profilerState = ProfilerState::StartRequested; + s_profilerState.store( ProfilerState::StartRequested, std::memory_order_release ); #else #endif } @@ -238,24 +238,24 @@ void BlueStatistics::StartTelemetryDump( const std::string& dumpFolder, float sa void BlueStatistics::PauseTelemetry() { #if CCP_TELEMETRY_ENABLED - if( s_profilerState != ProfilerState::Started ) + if( s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Started ) { return; } - s_profilerState = ProfilerState::Paused; + s_profilerState.store( ProfilerState::Paused, std::memory_order_release ); #endif } void BlueStatistics::ResumeTelemetry() { #if CCP_TELEMETRY_ENABLED - if ( s_profilerState != ProfilerState::Paused ) + if ( s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Paused ) { return; } - s_profilerState = ProfilerState::Started; + s_profilerState.store( ProfilerState::Started, std::memory_order_release ); #endif } @@ -268,12 +268,12 @@ void BlueStatistics::StopTelemetry() bool BlueStatistics::IsTelemetryConnected() { - return s_profilerState == ProfilerState::Started || s_profilerState == ProfilerState::Paused; + return s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started || s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Paused; } bool BlueStatistics::IsTelemetryConnectionRequested() { - return s_profilerState == ProfilerState::StartRequested; + return s_profilerState.load( std::memory_order_acquire ) == ProfilerState::StartRequested; } float BlueStatistics::TelemetrySamplingTimeLeft() @@ -283,13 +283,13 @@ float BlueStatistics::TelemetrySamplingTimeLeft() bool BlueStatistics::IsTelemetryPaused() { - return s_profilerState == ProfilerState::Paused; + return s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Paused; } void BlueStatistics::UpdateTelemetry() { #if CCP_TELEMETRY_ENABLED - switch ( s_profilerState ) + switch ( s_profilerState.load(std::memory_order_acquire) ) { case ProfilerState::StartRequested: { @@ -301,14 +301,14 @@ void BlueStatistics::UpdateTelemetry() if ( s_isTelemetryPythonCaptureEnabled ) { PyEval_SetProfile( &PythonProfiler, nullptr ); } - s_profilerState = ProfilerState::Started; + s_profilerState.store( ProfilerState::Started, std::memory_order_release ); s_telemetryLastCheckTime = s_telemetryStartTime = BeOS->GetActualTime(); } } else if (!CcpStartTelemetry( s_telemetryServerOrFileSystemDumpPath.c_str(), s_telemetryConnectionType, m_telemetryMaxThreadCount )) { CCP_LOGERR( "Failed to start Telemetry" ); - s_profilerState = ProfilerState::Stopped; + s_profilerState.store( ProfilerState::Stopped, std::memory_order_release ); } break; } @@ -335,7 +335,7 @@ void BlueStatistics::UpdateTelemetry() // Nothing to do break; default: - CCP_LOGERR( "BlueStatistics::UpdateTelemetry - unhandled profiler state %d", s_profilerState ); + CCP_LOGERR( "BlueStatistics::UpdateTelemetry - unhandled profiler state %d", s_profilerState.load(std::memory_order_acquire)); break; } #endif @@ -371,7 +371,7 @@ void SwitchToFiber( PyTaskletObject* to ) void BlueStatistics::OnTaskletSwitch( PyObject* _from, PyObject* _to ) { #if CCP_TELEMETRY_ENABLED - if (s_profilerState == ProfilerState::Stopped || s_profilerState == ProfilerState::StartRequested) + if (s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Started) { return; } @@ -747,7 +747,7 @@ void tmTaskletAppendText( uint32_t ctx, const char* appendText ) void TracyEnterZone( void* key, const char* name, const char* filename, uint32_t lineno ) { - if( s_profilerState == ProfilerState::Started ) + if( s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started ) { if( g_taskletZoneStore.find( key ) == g_taskletZoneStore.end() ) { @@ -760,7 +760,7 @@ void TracyLeaveZone( void* key ) { // An edge case exists where TracyEnterZone is called whilst the profiler is started with a matching call to TracyLeaveZone when paused. // Therefore, we have to notify Tracy of any valid calls to TracyLeaveZone, even when active instrumentation is paused. - if (!(s_profilerState == ProfilerState::Started || s_profilerState == ProfilerState::Paused)) + if (!(s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started || s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Paused)) { return; } @@ -773,7 +773,7 @@ void TracyLeaveZone( void* key ) void TracyZoneAddText( void* key, const char* text ) { - if( s_profilerState == ProfilerState::Started ) + if( s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started ) { auto zone = g_taskletZoneStore.find( key ); if( zone != g_taskletZoneStore.end() ) @@ -795,7 +795,7 @@ tmTaskletZone::~tmTaskletZone() TracyZone::TracyZone( uint32_t ctx, const char* name, const char* filename, uint32_t lineno, uint32_t color ) : m_fiber( g_activeFiber ) { - if( s_profilerState != ProfilerState::Started ) + if( s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Started ) { return; } @@ -831,7 +831,7 @@ TracyZone::~TracyZone() void TracyZone::text( const char* text ) const { - if( s_profilerState == ProfilerState::Started && m_telemetryContext ) + if( s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started && m_telemetryContext ) { CCP_ASSERT( text != nullptr ); TracyCZoneText( m_telemetryContext.value(), text, strlen( text ) ); From 6241f1658683c496da711c35976737b1c9595ef4 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Thu, 6 Mar 2025 15:08:19 +0000 Subject: [PATCH 022/131] Remove unused `TracyZoneAddText` function This function was not exported or exposed to python. We used to allow appending text to zones via `PyAppendToZone`, which is meant to append to the latest zone's text. This functionality is currently unsupported but will get revisited at a later point. --- include/BlueStatistics.h | 1 - src/BlueStatistics.cpp | 12 ------------ 2 files changed, 13 deletions(-) diff --git a/include/BlueStatistics.h b/include/BlueStatistics.h index b271a90..56f065a 100644 --- a/include/BlueStatistics.h +++ b/include/BlueStatistics.h @@ -173,7 +173,6 @@ class TracyZone void BLUEIMPORT TracyEnterZone( void* key, const char* name, const char* filename, uint32_t lineno ); void BLUEIMPORT TracyLeaveZone( void* key ); -void TracyZoneAddText( void* key, const char* text ); #define CCP_STATS_SCOPED_TIME( identifier ) \ TracyZone tracy_zone_##__COUNTER__( TMCM_CPP, g_ccpStatistics_##identifier.GetName().c_str(), __FILE__, __LINE__ );\ diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 270b811..1a267ce 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -771,18 +771,6 @@ void TracyLeaveZone( void* key ) } } -void TracyZoneAddText( void* key, const char* text ) -{ - if( s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started ) - { - auto zone = g_taskletZoneStore.find( key ); - if( zone != g_taskletZoneStore.end() ) - { - zone->second.text( text ); - } - } -} - tmTaskletZone::tmTaskletZone( uint32_t ctx, const char* name ) { tmTaskletEnter( ctx, name ); From b449e6280c03149fd765f19a23cf4049e7906134 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Thu, 6 Mar 2025 16:10:49 +0000 Subject: [PATCH 023/131] Use stack container as tasklet zone store The key map used to record zones had two drawbacks. Firstly, it was not possible to determine the last zone entered for a given fiber. This is required by some of the API, `PyAppendToZone` as an example. Secondly, and edge case exists where instrumentation may attempt to add multiple zones sharing a frame object to the tasklet zone store. Since the frame object is used as a key to the underlying map, we could only have one zone per frame object active at any moment in time, resulting in some missing instrumentation. Another implementation detail is moving to the tasklet store to thread local storage. This is the recommended approach acc. to the documentation, and there are good reasons for it. Most importantly, non-fiber zones should never intermingle across threads as it's not safe to leave a zone from a thread which did not enter it. --- src/BlueStatistics.cpp | 32 ++++++++++++++++++-------------- src/BlueStatistics_Blue.cpp | 4 ++-- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 1a267ce..f3c4487 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -77,14 +77,15 @@ uint32_t s_taskletTrackID = 0; // telemetry track ID for tasklet time spans thread_local PyTaskletObject* g_activeFiber{nullptr}; typedef std::unordered_map FiberNameStore; -FiberNameStore g_fiberNameStore; +thread_local FiberNameStore g_fiberNameStore; // Per-thread record of valid fiber names -typedef std::unordered_map TasketZoneStore; -TasketZoneStore g_taskletZoneStore; +typedef std::unordered_map>> TasketZoneStore; +thread_local TasketZoneStore g_taskletZoneStore; // Per-thread record of zones instrumented from python // Overriden tp_free function for tasklets: notify telemetry and call original tp_free void OnTaskletFree( void* tasklet ) { + g_taskletZoneStore.erase( (PyTaskletObject*) tasklet ); g_fiberNameStore.erase( (PyTaskletObject*) tasklet ); auto found = s_taskletFree.find( Py_TYPE( tasklet ) ); if( found != end( s_taskletFree ) ) @@ -365,6 +366,12 @@ void SwitchToFiber( PyTaskletObject* to ) TracyFiberEnter( existing->second.c_str() ); g_activeFiber = to; } + + if( auto existing = g_taskletZoneStore.find( g_activeFiber ); existing == g_taskletZoneStore.cend() ) + { + // Initialize an empty zone stack, main tasklet is mapped to nullptr + g_taskletZoneStore[g_activeFiber] = {}; + } } #if CCP_STACKLESS @@ -749,25 +756,22 @@ void TracyEnterZone( void* key, const char* name, const char* filename, uint32_t { if( s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started ) { - if( g_taskletZoneStore.find( key ) == g_taskletZoneStore.end() ) + if (auto existing = g_taskletZoneStore.find( g_activeFiber ); existing != g_taskletZoneStore.end()) { - g_taskletZoneStore.insert( { key, TracyZone( TMCM_CPP, name, filename, lineno, tracy::Color::Yellow ) } ); + existing->second.emplace( PyEval_GetFrame(), TracyZone(TMCM_CPP, name, filename, lineno, tracy::Color::Yellow )); } } } void TracyLeaveZone( void* key ) { - // An edge case exists where TracyEnterZone is called whilst the profiler is started with a matching call to TracyLeaveZone when paused. - // Therefore, we have to notify Tracy of any valid calls to TracyLeaveZone, even when active instrumentation is paused. - if (!(s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started || s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Paused)) + if (auto existing = g_taskletZoneStore.find( g_activeFiber ); existing != g_taskletZoneStore.end() && !existing->second.empty()) { - return; - } - - if( g_taskletZoneStore.find( key ) != g_taskletZoneStore.end() ) - { - g_taskletZoneStore.erase( key ); + // Frame object guards against whether this function had a prior call to `TracyEnterZone` result in a new Zone pushed to the stack + if ( existing->second.top().first == PyEval_GetFrame()) + { + existing->second.pop(); + } } } diff --git a/src/BlueStatistics_Blue.cpp b/src/BlueStatistics_Blue.cpp index ef6987a..08d34b4 100644 --- a/src/BlueStatistics_Blue.cpp +++ b/src/BlueStatistics_Blue.cpp @@ -293,7 +293,7 @@ PyObject* PyEnterZone( PyObject* self, PyObject* args ) return nullptr; } - TracyEnterZone( PyEval_GetFrame(), zone, __FILE__, __LINE__ ); + TracyEnterZone( nullptr, zone, __FILE__, __LINE__ ); #endif Py_RETURN_NONE; } @@ -301,7 +301,7 @@ PyObject* PyEnterZone( PyObject* self, PyObject* args ) PyObject* PyLeaveZone( PyObject* self, PyObject* args ) { #if CCP_TELEMETRY_ENABLED - TracyLeaveZone( PyEval_GetFrame() ); + TracyLeaveZone( nullptr ); #endif Py_RETURN_NONE; } From 3cd504500dc2ce9e20de6a90929d494f4f628240 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Thu, 6 Mar 2025 17:04:10 +0000 Subject: [PATCH 024/131] Suspend active instrumentation when no zones are active --- src/BlueStatistics.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index f3c4487..6f191af 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -82,6 +82,8 @@ thread_local FiberNameStore g_fiberNameStore; // Per-thread record of valid fibe typedef std::unordered_map>> TasketZoneStore; thread_local TasketZoneStore g_taskletZoneStore; // Per-thread record of zones instrumented from python +thread_local int g_zoneCount = 0; // Per-thread zone count used to determine when it is safe to suspend instrumentation + // Overriden tp_free function for tasklets: notify telemetry and call original tp_free void OnTaskletFree( void* tasklet ) { @@ -332,6 +334,15 @@ void BlueStatistics::UpdateTelemetry() break; } case ProfilerState::Paused: + if (g_zoneCount > 0) + { + CcpTelemetryTick(); + } + else + { + PyEval_SetProfile( nullptr, nullptr ); + s_profilerState.store( ProfilerState::Stopped, std::memory_order_release ); + } case ProfilerState::Stopped: // Nothing to do break; @@ -378,7 +389,7 @@ void SwitchToFiber( PyTaskletObject* to ) void BlueStatistics::OnTaskletSwitch( PyObject* _from, PyObject* _to ) { #if CCP_TELEMETRY_ENABLED - if (s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Started) + if (!TracyIsStarted || s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Stopped) { return; } @@ -796,6 +807,7 @@ TracyZone::TracyZone( uint32_t ctx, const char* name, const char* filename, uint CCP_ASSERT( name != nullptr ); auto data = ___tracy_alloc_srcloc( lineno, filename, strlen( filename ), name, strlen( name ), color ); m_telemetryContext.emplace( ___tracy_emit_zone_begin_alloc( data, ctx & TMCM_CPP ) ); + g_zoneCount++; } TracyZone::TracyZone( TracyZone&& other ) noexcept @@ -819,6 +831,7 @@ TracyZone::~TracyZone() SwitchToFiber( (PyTaskletObject*) m_fiber ); TracyCZoneEnd( m_telemetryContext.value() ); SwitchToFiber( previous ); + g_zoneCount--; } void TracyZone::text( const char* text ) const From 03049660fba3246442dd1e0d30991149f4465eef Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Thu, 6 Mar 2025 17:24:04 +0000 Subject: [PATCH 025/131] Stop telemetry when profiler is disconnected --- src/BlueStatistics.cpp | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 6f191af..1516bb5 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -317,20 +317,27 @@ void BlueStatistics::UpdateTelemetry() } case ProfilerState::Started: { - CcpTelemetryTick(); - if(s_telemetrySamplePeriod > 0.0f ) // Check if we have passed our timed sample time + if (TracyIsConnected) { - Be::Time newTime = BeOS->GetActualTime(); - Be::Time delta = newTime - s_telemetryLastCheckTime; - s_telemetryLastCheckTime = newTime; - s_telemetrySamplePeriod -= ((float)delta / Be::Time(1e7)); - - if(s_telemetrySamplePeriod < 0.0f) + CcpTelemetryTick(); + if(s_telemetrySamplePeriod > 0.0f ) // Check if we have passed our timed sample time { - CCP_LOG( "Finalising timed Telemetry run." ); - StopTelemetry(); + Be::Time newTime = BeOS->GetActualTime(); + Be::Time delta = newTime - s_telemetryLastCheckTime; + s_telemetryLastCheckTime = newTime; + s_telemetrySamplePeriod -= ((float)delta / Be::Time(1e7)); + + if(s_telemetrySamplePeriod < 0.0f) + { + CCP_LOG( "Finalising timed Telemetry run." ); + StopTelemetry(); + } } } + else + { + StopTelemetry(); + } break; } case ProfilerState::Paused: From f7de9d7fb3c6f27df4c0cf105fdab5b9a4275000 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Thu, 6 Mar 2025 17:26:14 +0000 Subject: [PATCH 026/131] Cleanup profiler state --- src/BlueStatistics.cpp | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 1516bb5..36dbdb8 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -17,7 +17,7 @@ enum ProfilerState { Stopped, StartRequested, Started, - Paused, + StopRequested, }; std::atomic s_profilerState{ProfilerState::Stopped}; @@ -241,19 +241,14 @@ void BlueStatistics::StartTelemetryDump( const std::string& dumpFolder, float sa void BlueStatistics::PauseTelemetry() { #if CCP_TELEMETRY_ENABLED - if( s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Started ) - { - return; - } - - s_profilerState.store( ProfilerState::Paused, std::memory_order_release ); + StopTelemetry(); #endif } void BlueStatistics::ResumeTelemetry() { #if CCP_TELEMETRY_ENABLED - if ( s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Paused ) + if ( s_profilerState.load( std::memory_order_acquire ) != ProfilerState::StopRequested ) { return; } @@ -265,13 +260,18 @@ void BlueStatistics::ResumeTelemetry() void BlueStatistics::StopTelemetry() { #if CCP_TELEMETRY_ENABLED - PauseTelemetry(); + if( s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Started ) + { + return; + } + + s_profilerState.store( ProfilerState::StopRequested, std::memory_order_release ); #endif } bool BlueStatistics::IsTelemetryConnected() { - return s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started || s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Paused; + return TracyIsConnected; } bool BlueStatistics::IsTelemetryConnectionRequested() @@ -286,7 +286,7 @@ float BlueStatistics::TelemetrySamplingTimeLeft() bool BlueStatistics::IsTelemetryPaused() { - return s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Paused; + return s_profilerState.load( std::memory_order_acquire ) == ProfilerState::StopRequested; } void BlueStatistics::UpdateTelemetry() @@ -296,9 +296,9 @@ void BlueStatistics::UpdateTelemetry() { case ProfilerState::StartRequested: { - if (tracy::IsProfilerStarted()) + if (TracyIsStarted) { - if (tracy::GetProfiler().IsConnected()) + if (TracyIsConnected) { TracySetProgramName( s_telemetryServerOrFileSystemDumpPath.c_str() ); if ( s_isTelemetryPythonCaptureEnabled ) { @@ -340,7 +340,8 @@ void BlueStatistics::UpdateTelemetry() } break; } - case ProfilerState::Paused: + case ProfilerState::StopRequested: + // Wait until zone starvation to suspend active instrumentation if (g_zoneCount > 0) { CcpTelemetryTick(); From 65101824c91356cd5985f08ce7abc7997828fbe8 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Fri, 7 Mar 2025 10:37:18 +0000 Subject: [PATCH 027/131] Revert "Cleanup profiler state" This reverts commit f7de9d7fb3c6f27df4c0cf105fdab5b9a4275000. --- src/BlueStatistics.cpp | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 36dbdb8..1516bb5 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -17,7 +17,7 @@ enum ProfilerState { Stopped, StartRequested, Started, - StopRequested, + Paused, }; std::atomic s_profilerState{ProfilerState::Stopped}; @@ -241,14 +241,19 @@ void BlueStatistics::StartTelemetryDump( const std::string& dumpFolder, float sa void BlueStatistics::PauseTelemetry() { #if CCP_TELEMETRY_ENABLED - StopTelemetry(); + if( s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Started ) + { + return; + } + + s_profilerState.store( ProfilerState::Paused, std::memory_order_release ); #endif } void BlueStatistics::ResumeTelemetry() { #if CCP_TELEMETRY_ENABLED - if ( s_profilerState.load( std::memory_order_acquire ) != ProfilerState::StopRequested ) + if ( s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Paused ) { return; } @@ -260,18 +265,13 @@ void BlueStatistics::ResumeTelemetry() void BlueStatistics::StopTelemetry() { #if CCP_TELEMETRY_ENABLED - if( s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Started ) - { - return; - } - - s_profilerState.store( ProfilerState::StopRequested, std::memory_order_release ); + PauseTelemetry(); #endif } bool BlueStatistics::IsTelemetryConnected() { - return TracyIsConnected; + return s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started || s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Paused; } bool BlueStatistics::IsTelemetryConnectionRequested() @@ -286,7 +286,7 @@ float BlueStatistics::TelemetrySamplingTimeLeft() bool BlueStatistics::IsTelemetryPaused() { - return s_profilerState.load( std::memory_order_acquire ) == ProfilerState::StopRequested; + return s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Paused; } void BlueStatistics::UpdateTelemetry() @@ -296,9 +296,9 @@ void BlueStatistics::UpdateTelemetry() { case ProfilerState::StartRequested: { - if (TracyIsStarted) + if (tracy::IsProfilerStarted()) { - if (TracyIsConnected) + if (tracy::GetProfiler().IsConnected()) { TracySetProgramName( s_telemetryServerOrFileSystemDumpPath.c_str() ); if ( s_isTelemetryPythonCaptureEnabled ) { @@ -340,8 +340,7 @@ void BlueStatistics::UpdateTelemetry() } break; } - case ProfilerState::StopRequested: - // Wait until zone starvation to suspend active instrumentation + case ProfilerState::Paused: if (g_zoneCount > 0) { CcpTelemetryTick(); From ed25396871fb592ba1350c8623732e8d9cbed59f Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Fri, 7 Mar 2025 13:21:56 +0000 Subject: [PATCH 028/131] Cleanup profiler state machine Pause and resume functionality has been deprecated and consolidated into `StartTelemetry` and `StopTelemetry`. --- include/BlueStatistics.h | 1 + src/BlueStatistics.cpp | 48 +++++++++++++++++-------------------- src/BlueStatistics_Blue.cpp | 21 +++++++++------- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/include/BlueStatistics.h b/include/BlueStatistics.h index 56f065a..0eed1a4 100644 --- a/include/BlueStatistics.h +++ b/include/BlueStatistics.h @@ -86,6 +86,7 @@ BLUE_CLASS( BlueStatistics ) : public IRoot float TelemetrySamplingTimeLeft(); bool IsTelemetryConnected(); bool IsTelemetryPaused(); + bool IsTelemetryStarted(); void SetCppCaptureEnabled( bool b ); bool IsCppCaptureEnabled(); diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 1516bb5..b15637d 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -17,7 +17,7 @@ enum ProfilerState { Stopped, StartRequested, Started, - Paused, + StopRequested, }; std::atomic s_profilerState{ProfilerState::Stopped}; @@ -210,7 +210,7 @@ void BlueStatistics::StartTelemetry( const std::string& server ) void BlueStatistics::StartTimedTelemetry( const std::string& server, float samplePeriod ) { #if CCP_TELEMETRY_ENABLED - if (s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Stopped ) + if( s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started || s_profilerState.load( std::memory_order_acquire ) == ProfilerState::StartRequested ) { return; } @@ -226,7 +226,7 @@ void BlueStatistics::StartTimedTelemetry( const std::string& server, float sampl void BlueStatistics::StartTelemetryDump( const std::string& dumpFolder, float samplePeriod ) { #if CCP_TELEMETRY_ENABLED - if (s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Stopped ) + if( s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started || s_profilerState.load( std::memory_order_acquire ) == ProfilerState::StartRequested ) { return; } @@ -240,43 +240,34 @@ void BlueStatistics::StartTelemetryDump( const std::string& dumpFolder, float sa void BlueStatistics::PauseTelemetry() { -#if CCP_TELEMETRY_ENABLED - if( s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Started ) - { - return; - } - - s_profilerState.store( ProfilerState::Paused, std::memory_order_release ); -#endif + // Deprecated } void BlueStatistics::ResumeTelemetry() { -#if CCP_TELEMETRY_ENABLED - if ( s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Paused ) - { - return; - } - - s_profilerState.store( ProfilerState::Started, std::memory_order_release ); -#endif + // Deprecated } void BlueStatistics::StopTelemetry() { #if CCP_TELEMETRY_ENABLED - PauseTelemetry(); + if( s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Stopped || s_profilerState.load( std::memory_order_acquire ) == ProfilerState::StopRequested ) + { + return; + } + + s_profilerState.store( ProfilerState::StopRequested, std::memory_order_release ); #endif } bool BlueStatistics::IsTelemetryConnected() { - return s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started || s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Paused; + return TracyIsStarted && TracyIsConnected; } bool BlueStatistics::IsTelemetryConnectionRequested() { - return s_profilerState.load( std::memory_order_acquire ) == ProfilerState::StartRequested; + return TracyIsStarted && !TracyIsConnected && s_profilerState.load( std::memory_order_acquire ) == ProfilerState::StartRequested; } float BlueStatistics::TelemetrySamplingTimeLeft() @@ -286,7 +277,12 @@ float BlueStatistics::TelemetrySamplingTimeLeft() bool BlueStatistics::IsTelemetryPaused() { - return s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Paused; + return false; // Deprecated +} + +bool BlueStatistics::IsTelemetryStarted() +{ + return s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started; } void BlueStatistics::UpdateTelemetry() @@ -296,9 +292,9 @@ void BlueStatistics::UpdateTelemetry() { case ProfilerState::StartRequested: { - if (tracy::IsProfilerStarted()) + if (TracyIsStarted) { - if (tracy::GetProfiler().IsConnected()) + if (TracyIsConnected) { TracySetProgramName( s_telemetryServerOrFileSystemDumpPath.c_str() ); if ( s_isTelemetryPythonCaptureEnabled ) { @@ -340,7 +336,7 @@ void BlueStatistics::UpdateTelemetry() } break; } - case ProfilerState::Paused: + case ProfilerState::StopRequested: if (g_zoneCount > 0) { CcpTelemetryTick(); diff --git a/src/BlueStatistics_Blue.cpp b/src/BlueStatistics_Blue.cpp index 08d34b4..0ce0d3a 100644 --- a/src/BlueStatistics_Blue.cpp +++ b/src/BlueStatistics_Blue.cpp @@ -664,18 +664,14 @@ const Be::ClassInfo* BlueStatistics::ExposeToBlue() ( "PauseTelemetry", PauseTelemetry, - "Pauses Telemetry capture. Ticking and frame boundary information are" - "\nstill sent over, but high frequency data such as memory events, mutex" - "\nstates, and zones are discarded. An application can use this function" - "\nto keep Telemetry live but with very low overhead until a specific" - "\nproblem area is encountered." + "Pauses Telemetry capture. (deprecated)" ) MAP_METHOD_AND_WRAP ( "ResumeTelemetry", ResumeTelemetry, - "Resumes Telemetry captures." + "Resumes Telemetry capture. (deprecated)" ) MAP_METHOD_AND_WRAP @@ -689,7 +685,7 @@ const Be::ClassInfo* BlueStatistics::ExposeToBlue() ( "isTelemetryConnectionRequested", IsTelemetryConnectionRequested, - "Is Telemetry connection pending?" + "Is Telemetry profiler connection pending?" ) MAP_PROPERTY_READONLY @@ -703,14 +699,21 @@ const Be::ClassInfo* BlueStatistics::ExposeToBlue() ( "isTelemetryConnected", IsTelemetryConnected, - "Is Telemetry connected?" + "Is Telemetry connected to a profiler?" ) MAP_PROPERTY_READONLY ( "isTelemetryPaused", IsTelemetryPaused, - "Is Telemetry paused?" + "Is Telemetry paused? (deprecated)" + ) + + MAP_PROPERTY_READONLY + ( + "isTelemetryStarted", + IsTelemetryStarted, + "Is Telemetry instrumentation active?" ) MAP_PROPERTY From 12578aa22d7ee8dec74daaf431735f837c211c96 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Fri, 7 Mar 2025 14:13:49 +0000 Subject: [PATCH 029/131] Extend logging for profiler state machine --- src/BlueStatistics.cpp | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index b15637d..ef3fd75 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -8,6 +8,8 @@ #include "StdAfx.h" #include "IBlueOS.h" +static CcpLogChannel_t s_ch = CCP_LOG_DEFINE_CHANNEL( "Telemetry" ); + static CBlueStatistics s_statisticsInstance; BlueStatistics* g_statistics = &s_statisticsInstance; @@ -218,6 +220,7 @@ void BlueStatistics::StartTimedTelemetry( const std::string& server, float sampl s_telemetryServerOrFileSystemDumpPath = server; s_telemetrySamplePeriod = (float)samplePeriod; + CCP_LOG_CH( s_ch, "StartTelemetry - initiating lazy start" ); s_profilerState.store( ProfilerState::StartRequested, std::memory_order_release ); #else #endif @@ -256,6 +259,7 @@ void BlueStatistics::StopTelemetry() return; } + CCP_LOG_CH( s_ch, "StopTelemetry - initiating lazy stop" ); s_profilerState.store( ProfilerState::StopRequested, std::memory_order_release ); #endif } @@ -296,6 +300,7 @@ void BlueStatistics::UpdateTelemetry() { if (TracyIsConnected) { + CCP_LOG_CH( s_ch, "UpdateTelemetry - Telemetry server connected to Profiler" ); TracySetProgramName( s_telemetryServerOrFileSystemDumpPath.c_str() ); if ( s_isTelemetryPythonCaptureEnabled ) { PyEval_SetProfile( &PythonProfiler, nullptr ); @@ -304,10 +309,14 @@ void BlueStatistics::UpdateTelemetry() s_telemetryLastCheckTime = s_telemetryStartTime = BeOS->GetActualTime(); } } - else if (!CcpStartTelemetry( s_telemetryServerOrFileSystemDumpPath.c_str(), s_telemetryConnectionType, m_telemetryMaxThreadCount )) + else { - CCP_LOGERR( "Failed to start Telemetry" ); - s_profilerState.store( ProfilerState::Stopped, std::memory_order_release ); + CCP_LOG_CH( s_ch, "UpdateTelemetry - Starting Telemetry Server" ); + if (!CcpStartTelemetry( s_telemetryServerOrFileSystemDumpPath.c_str(), s_telemetryConnectionType, m_telemetryMaxThreadCount )) + { + CCP_LOGERR_CH( s_ch, "UpdateTelemetry - Failed to start Telemetry server" ); + s_profilerState.store( ProfilerState::Stopped, std::memory_order_release ); + } } break; } @@ -325,7 +334,7 @@ void BlueStatistics::UpdateTelemetry() if(s_telemetrySamplePeriod < 0.0f) { - CCP_LOG( "Finalising timed Telemetry run." ); + CCP_LOG_CH( s_ch, "UpdateTelemetry - Finalizing timed Telemetry run" ); StopTelemetry(); } } @@ -343,6 +352,7 @@ void BlueStatistics::UpdateTelemetry() } else { + CCP_LOG_CH( s_ch, "UpdateTelemetry - Lazy stop finished." ); PyEval_SetProfile( nullptr, nullptr ); s_profilerState.store( ProfilerState::Stopped, std::memory_order_release ); } @@ -350,7 +360,7 @@ void BlueStatistics::UpdateTelemetry() // Nothing to do break; default: - CCP_LOGERR( "BlueStatistics::UpdateTelemetry - unhandled profiler state %d", s_profilerState.load(std::memory_order_acquire)); + CCP_LOGERR_CH( s_ch, "UpdateTelemetry - Unhandled profiler state %d", s_profilerState.load(std::memory_order_acquire)); break; } #endif From a548a72012f09c873feae4c56f316826217e96d7 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Fri, 7 Mar 2025 14:42:57 +0000 Subject: [PATCH 030/131] Always tick telemetry during lazy stop --- src/BlueStatistics.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index ef3fd75..b0891eb 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -346,16 +346,16 @@ void BlueStatistics::UpdateTelemetry() break; } case ProfilerState::StopRequested: - if (g_zoneCount > 0) - { - CcpTelemetryTick(); - } - else + { + CcpTelemetryTick(); + + if( g_zoneCount <= 0 ) { CCP_LOG_CH( s_ch, "UpdateTelemetry - Lazy stop finished." ); PyEval_SetProfile( nullptr, nullptr ); s_profilerState.store( ProfilerState::Stopped, std::memory_order_release ); } + } case ProfilerState::Stopped: // Nothing to do break; From 6fb7f2988b0edf27764bbb92cc2154d1857fd368 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Fri, 7 Mar 2025 15:27:50 +0000 Subject: [PATCH 031/131] Tick telemetry during active profiling regardless of profiler connection This could result in a missing frame-tick when disconnecting the profiler. --- src/BlueStatistics.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index b0891eb..17628b1 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -322,9 +322,10 @@ void BlueStatistics::UpdateTelemetry() } case ProfilerState::Started: { + CcpTelemetryTick(); + if (TracyIsConnected) { - CcpTelemetryTick(); if(s_telemetrySamplePeriod > 0.0f ) // Check if we have passed our timed sample time { Be::Time newTime = BeOS->GetActualTime(); From 382383177581265a4143e6540b40742ab252ee0b Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Fri, 7 Mar 2025 15:28:35 +0000 Subject: [PATCH 032/131] Add logging when profiler is disconnected --- src/BlueStatistics.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 17628b1..c713ef9 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -342,6 +342,7 @@ void BlueStatistics::UpdateTelemetry() } else { + CCP_LOG_CH( s_ch, "UpdateTelemetry - Disconnected from profiler" ); StopTelemetry(); } break; From a9504a5596773d45df5a41e1b5b71f8c9484b4df Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Mon, 17 Mar 2025 10:57:19 +0000 Subject: [PATCH 033/131] Disconnect profiler trace function when stopping telemetry --- src/BlueStatistics.cpp | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index c713ef9..f23169e 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -84,8 +84,6 @@ thread_local FiberNameStore g_fiberNameStore; // Per-thread record of valid fibe typedef std::unordered_map>> TasketZoneStore; thread_local TasketZoneStore g_taskletZoneStore; // Per-thread record of zones instrumented from python -thread_local int g_zoneCount = 0; // Per-thread zone count used to determine when it is safe to suspend instrumentation - // Overriden tp_free function for tasklets: notify telemetry and call original tp_free void OnTaskletFree( void* tasklet ) { @@ -351,12 +349,8 @@ void BlueStatistics::UpdateTelemetry() { CcpTelemetryTick(); - if( g_zoneCount <= 0 ) - { - CCP_LOG_CH( s_ch, "UpdateTelemetry - Lazy stop finished." ); - PyEval_SetProfile( nullptr, nullptr ); - s_profilerState.store( ProfilerState::Stopped, std::memory_order_release ); - } + PyEval_SetProfile( nullptr, nullptr ); + s_profilerState.store( ProfilerState::Stopped, std::memory_order_release ); } case ProfilerState::Stopped: // Nothing to do @@ -822,7 +816,6 @@ TracyZone::TracyZone( uint32_t ctx, const char* name, const char* filename, uint CCP_ASSERT( name != nullptr ); auto data = ___tracy_alloc_srcloc( lineno, filename, strlen( filename ), name, strlen( name ), color ); m_telemetryContext.emplace( ___tracy_emit_zone_begin_alloc( data, ctx & TMCM_CPP ) ); - g_zoneCount++; } TracyZone::TracyZone( TracyZone&& other ) noexcept @@ -846,7 +839,6 @@ TracyZone::~TracyZone() SwitchToFiber( (PyTaskletObject*) m_fiber ); TracyCZoneEnd( m_telemetryContext.value() ); SwitchToFiber( previous ); - g_zoneCount--; } void TracyZone::text( const char* text ) const From 42c285d7f4fefb71768fee076e4b7cc2b2956fbe Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Mon, 17 Mar 2025 11:15:34 +0000 Subject: [PATCH 034/131] Only key zones to python frame objects when instrumenting from python Some C++ instrumentation may attempt to call `TracyEnterZone` or `TracyLeaveZone` with no access to a python frame object, f.ex. a thread spun up from c++. --- src/BlueStatistics.cpp | 4 ++-- src/BlueStatistics_Blue.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index f23169e..81e6cc5 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -778,7 +778,7 @@ void TracyEnterZone( void* key, const char* name, const char* filename, uint32_t { if (auto existing = g_taskletZoneStore.find( g_activeFiber ); existing != g_taskletZoneStore.end()) { - existing->second.emplace( PyEval_GetFrame(), TracyZone(TMCM_CPP, name, filename, lineno, tracy::Color::Yellow )); + existing->second.emplace( key, TracyZone(TMCM_CPP, name, filename, lineno, tracy::Color::Yellow )); } } } @@ -788,7 +788,7 @@ void TracyLeaveZone( void* key ) if (auto existing = g_taskletZoneStore.find( g_activeFiber ); existing != g_taskletZoneStore.end() && !existing->second.empty()) { // Frame object guards against whether this function had a prior call to `TracyEnterZone` result in a new Zone pushed to the stack - if ( existing->second.top().first == PyEval_GetFrame()) + if ( existing->second.top().first == key ) { existing->second.pop(); } diff --git a/src/BlueStatistics_Blue.cpp b/src/BlueStatistics_Blue.cpp index 0ce0d3a..c74e88f 100644 --- a/src/BlueStatistics_Blue.cpp +++ b/src/BlueStatistics_Blue.cpp @@ -293,7 +293,7 @@ PyObject* PyEnterZone( PyObject* self, PyObject* args ) return nullptr; } - TracyEnterZone( nullptr, zone, __FILE__, __LINE__ ); + TracyEnterZone( PyEval_GetFrame(), zone, __FILE__, __LINE__ ); #endif Py_RETURN_NONE; } @@ -301,7 +301,7 @@ PyObject* PyEnterZone( PyObject* self, PyObject* args ) PyObject* PyLeaveZone( PyObject* self, PyObject* args ) { #if CCP_TELEMETRY_ENABLED - TracyLeaveZone( nullptr ); + TracyLeaveZone( PyEval_GetFrame() ); #endif Py_RETURN_NONE; } From 90cb65f335e14e49976625c24c5c05b0da05ef9d Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 19 Mar 2025 11:35:40 +0000 Subject: [PATCH 035/131] Cleanup logging when starting and stopping telemetry --- src/BlueStatistics.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 81e6cc5..4f0c58c 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -218,7 +218,7 @@ void BlueStatistics::StartTimedTelemetry( const std::string& server, float sampl s_telemetryServerOrFileSystemDumpPath = server; s_telemetrySamplePeriod = (float)samplePeriod; - CCP_LOG_CH( s_ch, "StartTelemetry - initiating lazy start" ); + CCP_LOG_CH( s_ch, "Profiler start requested" ); s_profilerState.store( ProfilerState::StartRequested, std::memory_order_release ); #else #endif @@ -257,7 +257,7 @@ void BlueStatistics::StopTelemetry() return; } - CCP_LOG_CH( s_ch, "StopTelemetry - initiating lazy stop" ); + CCP_LOG_CH( s_ch, "Profiler stop requested" ); s_profilerState.store( ProfilerState::StopRequested, std::memory_order_release ); #endif } From 697e2c4aef0dcadf7f1e69182c915e664aaca395 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Mon, 24 Mar 2025 10:53:53 +0000 Subject: [PATCH 036/131] Fix Tracy profiler crash during early telemetry sessions The crash in question happens when calling `TracyLeaveFiber` when the profiler was not connected during the pre-requisite call to `TracyFiberEnter`. --- src/BlueStatistics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 4f0c58c..1326603 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -398,7 +398,7 @@ void SwitchToFiber( PyTaskletObject* to ) void BlueStatistics::OnTaskletSwitch( PyObject* _from, PyObject* _to ) { #if CCP_TELEMETRY_ENABLED - if (!TracyIsStarted || s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Stopped) + if (s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Started) { return; } From 879951afae8591f996859e64f8e16a1ba03a2538 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Mon, 24 Mar 2025 11:42:40 +0000 Subject: [PATCH 037/131] Fix a crash when switching back to a fiber whose tasklet has been freed Tracy zone objects switch back to the previous fiber when destroyed. This fiber's tasklet representation may be freed from memory if the profiler is stopped since tasklet switches are not tracked at that point. --- src/BlueStatistics.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 1326603..dc4579c 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -89,6 +89,12 @@ void OnTaskletFree( void* tasklet ) { g_taskletZoneStore.erase( (PyTaskletObject*) tasklet ); g_fiberNameStore.erase( (PyTaskletObject*) tasklet ); + if (g_activeFiber && g_activeFiber == tasklet) + { + // Catch an edge case where the profiler is stopped and a tasklet switch + // away from this tasklet is not reflected in the active fiber + g_activeFiber = nullptr; + } auto found = s_taskletFree.find( Py_TYPE( tasklet ) ); if( found != end( s_taskletFree ) ) { From 4c294f4e3c6744a4d9eae8b2443fb45d82d2c60c Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 25 Mar 2025 11:55:30 +0000 Subject: [PATCH 038/131] Delay fiber name erasure following tasklet closure The profiler may not receive valid fiber name string information for short-lived tasklets if the underlying fiber name is deallocated immediately upon tasklet closure. This change delays fiber name deallocation by five seconds. This is a temporary measure as it's not a great synchronization mechanism. Ideally we'd be able to clear the fiber name store when the profiler is disconnected, but that'll require us to mark all reported zones as inactive, which we don't support at the time of writing. Reference: https://ccpgames.atlassian.net/browse/PLAT-8855 --- src/BlueStatistics.cpp | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index dc4579c..664df3f 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -79,7 +79,9 @@ uint32_t s_taskletTrackID = 0; // telemetry track ID for tasklet time spans thread_local PyTaskletObject* g_activeFiber{nullptr}; typedef std::unordered_map FiberNameStore; -thread_local FiberNameStore g_fiberNameStore; // Per-thread record of valid fiber names +FiberNameStore g_fiberNameStore; // Persisted fiber name string store + +std::unordered_map g_fiberEraseMap; // Map of fibers scheduled for erasure typedef std::unordered_map>> TasketZoneStore; thread_local TasketZoneStore g_taskletZoneStore; // Per-thread record of zones instrumented from python @@ -88,7 +90,7 @@ thread_local TasketZoneStore g_taskletZoneStore; // Per-thread record of zones i void OnTaskletFree( void* tasklet ) { g_taskletZoneStore.erase( (PyTaskletObject*) tasklet ); - g_fiberNameStore.erase( (PyTaskletObject*) tasklet ); + g_fiberEraseMap.emplace( (PyTaskletObject*) tasklet, BeOS->GetActualTime()); if (g_activeFiber && g_activeFiber == tasklet) { // Catch an edge case where the profiler is stopped and a tasklet switch @@ -330,11 +332,23 @@ void BlueStatistics::UpdateTelemetry() if (TracyIsConnected) { + Be::Time now = BeOS->GetActualTime(); + + // Erase fibers scheduled for erasure + for ( const auto& [tasklet, time] : g_fiberEraseMap) + { + // Give the profiler a few seconds to receive information from the fiber name store before deallocating + // the underlying string + if ((time - now) / Be::Time(1e7) >= 5) + { + g_fiberNameStore.erase( tasklet ); + g_fiberEraseMap.erase( tasklet ); + } + } if(s_telemetrySamplePeriod > 0.0f ) // Check if we have passed our timed sample time { - Be::Time newTime = BeOS->GetActualTime(); - Be::Time delta = newTime - s_telemetryLastCheckTime; - s_telemetryLastCheckTime = newTime; + Be::Time delta = now - s_telemetryLastCheckTime; + s_telemetryLastCheckTime = now; s_telemetrySamplePeriod -= ((float)delta / Be::Time(1e7)); if(s_telemetrySamplePeriod < 0.0f) From 62b2404d4da68e01e1cbb9724a24b174bb29b334 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 25 Mar 2025 15:29:08 +0000 Subject: [PATCH 039/131] Pass python source data into zones created using python markup In order to perform the change, the frame object's `co_filename` string is immortalized which increases memory consumption. --- src/BlueStatistics_Blue.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/BlueStatistics_Blue.cpp b/src/BlueStatistics_Blue.cpp index c74e88f..a387dea 100644 --- a/src/BlueStatistics_Blue.cpp +++ b/src/BlueStatistics_Blue.cpp @@ -292,8 +292,14 @@ PyObject* PyEnterZone( PyObject* self, PyObject* args ) { return nullptr; } + auto frame = PyEval_GetFrame(); - TracyEnterZone( PyEval_GetFrame(), zone, __FILE__, __LINE__ ); + auto fileName = Immortalize( frame->f_code->co_filename ); + if (!fileName) + { + return nullptr; + } + TracyEnterZone( frame, zone, fileName, frame->f_lineno ); #endif Py_RETURN_NONE; } From 07da5fae4bd24d207ed79022925e1a88cccbd97f Mon Sep 17 00:00:00 2001 From: Filipp Pavlov Date: Tue, 6 May 2025 14:28:23 +0000 Subject: [PATCH 040/131] Fix for an occasional crash in Graphite. When querying AllReference objects when they are in the process of cleaning up outdated references from the previous generation we may end up accessing dangling "parent" pointers. Added checks for parent "generation" to guard against that. --- src/AllReferences.cpp | 23 ++++++++++++++++++----- src/AllReferences.h | 2 ++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/AllReferences.cpp b/src/AllReferences.cpp index 7e948a9..ad170c0 100644 --- a/src/AllReferences.cpp +++ b/src/AllReferences.cpp @@ -201,12 +201,15 @@ BluePy AllReferences::GetReferences( IRoot* obj ) Py_XDECREF( element ); }; - if( found != end( m_references ) ) + if( found != end( m_references ) && !IsOutdated( found->second ) ) { AddRef( found->second.first ); for( auto& rec : found->second.rest ) { - AddRef( rec ); + if( !IsOutdated( rec ) ) + { + AddRef( rec ); + } } } return BluePy( result ); @@ -280,7 +283,7 @@ bool AllReferences::HasRoute( IRoot* from, IRoot* to, std::unordered_mapsecond ) ) { hasRoute[to] = false; return false; @@ -292,7 +295,7 @@ bool AllReferences::HasRoute( IRoot* from, IRoot* to, std::unordered_mapsecond.rest ) { - if( HasRoute( from, ref.parent, hasRoute ) ) + if( !IsOutdated( ref ) && HasRoute( from, ref.parent, hasRoute ) ) { hasRoute[to] = true; return true; @@ -300,4 +303,14 @@ bool AllReferences::HasRoute( IRoot* from, IRoot* to, std::unordered_map& hasRoute ) const; + bool IsOutdated( const Reference& ref ) const; + bool IsOutdated( const References& refs ) const; IRootPtr m_root; std::vector m_stack; From 0fe868d0b8da0e3e486d796b7bc921912a1eb2ac Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Wed, 16 Jul 2025 14:55:41 +0000 Subject: [PATCH 041/131] Fix crash when attempting to call `del` on a `blue.DBRow` column As per [Python documentation](https://docs.python.org/2/c-api/typeobj.html#c.PyTypeObject.tp_setattro): > The signature is the same as for PyObject_SetAttr(), but setting v to NULL to delete an attribute must be supported. --- src/PyRowSet.cpp | 8 ++- tests/python/bluetests/test/test_dbrow.py | 59 +++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 tests/python/bluetests/test/test_dbrow.py diff --git a/src/PyRowSet.cpp b/src/PyRowSet.cpp index d850a10..afedb1c 100644 --- a/src/PyRowSet.cpp +++ b/src/PyRowSet.cpp @@ -1623,6 +1623,12 @@ int DBRow::SetAttr(PyObject *self, PyObject *key, PyObject *val) const ColumnDescriptor *cd = tis->GetCD(idx, key, NULL); if (cd != 0) { + // Guard against `del DBRow.column` / `delattr(DBRow, "column")` + if (!val) + { + PyErr_SetString(PyExc_RuntimeError, "Cannot delete a column from a DBRow"); + return -1; + } if (!tis->Set(*cd, idx, val)) return -1; return 0; @@ -1830,4 +1836,4 @@ bool DBRowsetInit(PyObject *module) return true; } -#endif \ No newline at end of file +#endif diff --git a/tests/python/bluetests/test/test_dbrow.py b/tests/python/bluetests/test/test_dbrow.py new file mode 100644 index 0000000..ceae0de --- /dev/null +++ b/tests/python/bluetests/test/test_dbrow.py @@ -0,0 +1,59 @@ +from . import blueunittest +import blue + + +class TestDBRow(blueunittest.TestCase): + """ + A set of test cases for the DBRow class. + """ + + def setUp(self): + self.columns = (("nodeID", 4), ("ipAddress", 129), ("port", 3)) + self.row = blue.DBRow(blue.DBRowDescriptor(self.columns)) + + def testAttemptingToDeleteColumnDoesntCrash(self): + foo = self.row + with self.assertRaises(RuntimeError): + delattr(foo, "nodeID") + with self.assertRaises(RuntimeError): + del foo.ipAddress + + def testSliceSubscript(self): + sliceTest = self.row[:] + + self.assertIsInstance(sliceTest, list) + self.assertEqual(self.row, blue.DBRow(blue.DBRowDescriptor(self.columns), sliceTest)) + self.assertListEqual(sliceTest, [0.0, None, 0]) + self.assertListEqual(self.row[:1], [0.0]) + self.assertListEqual(self.row[1:], [None, 0]) + self.assertListEqual(self.row[1:2], [None]) + self.assertListEqual(self.row[:-1], [0.0, None]) + self.assertListEqual(self.row[0:], [0.0, None, 0]) + + with self.assertRaises(NotImplementedError): + self.row[::2] + + def testIndexSubscript(self): + self.assertEqual(self.row[0], 0.0) + self.assertEqual(self.row[-1], 0) + + with self.assertRaises(IndexError): + self.row[len(self.row)] + + def testUnicodeSubscript(self): + self.assertEqual(self.row["nodeID"], 0) + + with self.assertRaises(KeyError): + self.row["test"] + + def testEquivalencyComparison(self): + self.assertFalse(self.row == None) + self.assertNotEqual(self.row, None) + + self.assertFalse(self.row == 1) + self.assertNotEqual(self.row, 1) + + self.assertFalse(self.row == blue.DBRowDescriptor(self.columns)) + self.assertNotEqual(self.row, blue.DBRowDescriptor(self.columns)) + + self.assertEqual(self.row, blue.DBRow(blue.DBRowDescriptor(self.columns))) From b794f8087d6dd90917b0bdb58decb9d9ae047ea1 Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:46:37 +0000 Subject: [PATCH 042/131] Fix `testSliceSubscript` In Python 2 the slice subscript with a step raises a `TypeError`, as opposed to a `NotImplementedError`. --- tests/python/bluetests/test/test_dbrow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_dbrow.py b/tests/python/bluetests/test/test_dbrow.py index ceae0de..d101d2e 100644 --- a/tests/python/bluetests/test/test_dbrow.py +++ b/tests/python/bluetests/test/test_dbrow.py @@ -30,7 +30,7 @@ def testSliceSubscript(self): self.assertListEqual(self.row[:-1], [0.0, None]) self.assertListEqual(self.row[0:], [0.0, None, 0]) - with self.assertRaises(NotImplementedError): + with self.assertRaises(TypeError): self.row[::2] def testIndexSubscript(self): From a6af20397ec0d76928e8cc3ec47a3cdea8399280 Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Wed, 23 Jul 2025 13:57:17 +0000 Subject: [PATCH 043/131] Add typechecks for Python objects stored in `blue.DBRow` Previously it was possible to store arbitrary Python objects inside a `blue.DBRow`. This can be abused to smuggle callable Python objects through as part of malicious payloads. Therefore, it is necessary to ensure only the expected types are stored inside `blue.DBRow`. --- src/PyRowSet.cpp | 59 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/src/PyRowSet.cpp b/src/PyRowSet.cpp index afedb1c..c0f0c7c 100644 --- a/src/PyRowSet.cpp +++ b/src/PyRowSet.cpp @@ -1047,13 +1047,39 @@ PyObject *DBRow::Get(const ColumnDescriptor &c, Py_ssize_t i) const return PyFloat_FromDouble(*(double*)data); case DBTYPE_CY: return PyFloat_FromDouble(double(*(int64_t*)data) / 10000.0); - case DBTYPE_STR: - case DBTYPE_WSTR: + case DBTYPE_STR: { + //a python object + PyObject* result = *(PyObject**)data; + if( result && result != Py_None && !PyString_CheckExact( result ) ) + { + PyErr_Format( PyExc_TypeError, "Unexpected value for db column %d", c.mType ); + return nullptr; + } + Py_XINCREF( result ); + return result; + } + case DBTYPE_WSTR: { + //a python object + PyObject* result = *(PyObject**)data; + if( result && result != Py_None && !PyUnicode_CheckExact( result ) ) + { + PyErr_Format( PyExc_TypeError, "Unexpected value for db column %d", c.mType ); + return nullptr; + } + Py_XINCREF( result ); + return result; + } case DBTYPE_BYTES: { //a python object - PyObject *result = *(PyObject**)data; - Py_XINCREF(result); - return result;} + PyObject* result = *(PyObject**)data; + if( result && result != Py_None && !PyBytes_CheckExact( result ) ) + { + PyErr_Format( PyExc_TypeError, "Unexpected value for db column %d", c.mType ); + return nullptr; + } + Py_XINCREF( result ); + return result; + } case DBTYPE_EMPTY: // A virtual column! return mRD->VirtualGet(c.mOffset, this); @@ -1174,8 +1200,31 @@ bool DBRow::SetNotNull(const ColumnDescriptor &c, PyObject *o) return true; } case DBTYPE_STR: + if ( o != Py_None && !PyString_CheckExact( o ) ) + { + PyErr_Format( PyExc_TypeError, "Unexpected value for db column %d", c.mType ); + return false; + } + Py_INCREF(o); + Py_XDECREF(*(PyObject**)data); + *(PyObject**)data = o; + return true; case DBTYPE_WSTR: + if ( o != Py_None && !PyUnicode_CheckExact( o ) ) + { + PyErr_Format( PyExc_TypeError, "Unexpected value for db column %d", c.mType ); + return false; + } + Py_INCREF(o); + Py_XDECREF(*(PyObject**)data); + *(PyObject**)data = o; + return true; case DBTYPE_BYTES: + if ( o != Py_None && !PyBytes_CheckExact( o ) ) + { + PyErr_Format( PyExc_TypeError, "Unexpected value for db column %d", c.mType ); + return false; + } //a python object Py_INCREF(o); Py_XDECREF(*(PyObject**)data); From 82cbde6193edd0d8b807b5802e44f122c304965b Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Wed, 23 Jul 2025 15:21:55 +0000 Subject: [PATCH 044/131] Add unittest for Python object validation in `blue.DBRow` Had forgotten to add this in the previous CL. --- tests/python/bluetests/test/test_dbrow.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/python/bluetests/test/test_dbrow.py b/tests/python/bluetests/test/test_dbrow.py index d101d2e..709b259 100644 --- a/tests/python/bluetests/test/test_dbrow.py +++ b/tests/python/bluetests/test/test_dbrow.py @@ -57,3 +57,20 @@ def testEquivalencyComparison(self): self.assertNotEqual(self.row, blue.DBRowDescriptor(self.columns)) self.assertEqual(self.row, blue.DBRow(blue.DBRowDescriptor(self.columns))) + + def testPythonObjectValidation(self): + objectStoringRow = blue.DBRow(blue.DBRowDescriptor((("text", 128),("unicodeText", 129),("data", 130)))) + + for illegalValue in ( + object(), + 4711, + 0.815, + (1,2,3,), + ['a','b', 'c'] + ): + with self.assertRaises(TypeError): + setattr(objectStoringRow, "text", illegalValue) + with self.assertRaises(TypeError): + setattr(objectStoringRow, "unicodeText", illegalValue) + with self.assertRaises(TypeError): + setattr(objectStoringRow, "data", illegalValue) From fb3c323f97d0890908a9f9c044129e97a1a72ab1 Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Tue, 29 Jul 2025 16:00:00 +0000 Subject: [PATCH 045/131] Make `blue.DBRow`'s virtual columns read-only The virtual column mechanism in `blue.DBRow` is a convenient way to avoid creating custom types that wrap a result returned from a DB call. Unfortunately, it is also a convenient way for a malicious actor to smuggle malicious payloads over the network. Specifically, using a virtual column's setter method can be used to protect attributes on spoofed machoNet packets from being overridden by certain sanity checks. By making the virtual columns read-only it is possible to detect if this is about to happen. Ideally, the whole virtual column mechanism would be removed, because the getter of a virtual column can still be any callable that is whitelisted in `blue.marshal`. Unfortunately, virtual columns and `blue.DBRow` are used all over the place in the Python code, making it virtually impossible to remove this mechanism without rewriting large parts of the game code. However, these virtual columns are only used to read values - they are never set. Therefore, this limits the ways in which executable code can be injected. --- src/PyRowSet.cpp | 21 ++++++++++++++++----- tests/python/bluetests/test/test_dbrow.py | 12 ++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/PyRowSet.cpp b/src/PyRowSet.cpp index c0f0c7c..a42f347 100644 --- a/src/PyRowSet.cpp +++ b/src/PyRowSet.cpp @@ -130,8 +130,9 @@ struct ColumnDescriptor case DBTYPE_BYTES: size = 5; break; //signal an object case DBTYPE_EMPTY: - size = -1; break; //virtual - + // PyErr_SetString(PyExc_RuntimeError, "Virtual columns are no longer supported"); + // return false; + size = -1; break; default: PyErr_Format(PyExc_TypeError, "DBRowDescriptor doesn't support data type %d", type); return false; @@ -595,6 +596,9 @@ Py_ssize_t DBRowDescriptor::SequenceLength(DBRowDescriptor *row) //Return a shallow copy of the dude PyObject *DBRowDescriptor::Get_virtual() { + // The virtual column mechanism was an open door for remote code execution, there it was removed. + // PyErr_SetString(PyExc_RuntimeError, "Virtual columns are no longer supported"); + // return nullptr; BluePyList newList(0); if (!newList) return 0; @@ -609,6 +613,9 @@ PyObject *DBRowDescriptor::Get_virtual() bool DBRowDescriptor::Set_virtual(PyObject *l) { + // The virtual column mechanism was an open door for remote code execution, there it was removed. + // PyErr_SetString(PyExc_RuntimeError, "Virtual columns are ready only"); + // return false; //make sure we have a list of tuples at least three if (!PyList_Check(l)) return PyErr_SetString(PyExc_TypeError, "list required"), false; @@ -625,7 +632,7 @@ bool DBRowDescriptor::Set_virtual(PyObject *l) goto ERR1; for (Py_ssize_t i = 0; i= PyList_GET_SIZE(mVirtualGetSet.o)) @@ -1081,7 +1090,8 @@ PyObject *DBRow::Get(const ColumnDescriptor &c, Py_ssize_t i) const return result; } case DBTYPE_EMPTY: - // A virtual column! + // PyErr_SetString(PyExc_RuntimeError, "Virtual columns are no longer supported"); + // return nullptr; return mRD->VirtualGet(c.mOffset, this); default: PyErr_Format(PyExc_RuntimeError, "Unexpected db column type encountered: %d", c.mType); @@ -1232,7 +1242,8 @@ bool DBRow::SetNotNull(const ColumnDescriptor &c, PyObject *o) return true; case DBTYPE_EMPTY: - return mRD->VirtualSet(c.mOffset, this, o); + PyErr_SetString(PyExc_RuntimeError, "Virtual columns are no longer supported"); + return false; default: PyErr_Format(PyExc_RuntimeError, "Unexpected db column type encountered: %d", c.mType); return false; diff --git a/tests/python/bluetests/test/test_dbrow.py b/tests/python/bluetests/test/test_dbrow.py index 709b259..ba5736f 100644 --- a/tests/python/bluetests/test/test_dbrow.py +++ b/tests/python/bluetests/test/test_dbrow.py @@ -18,6 +18,18 @@ def testAttemptingToDeleteColumnDoesntCrash(self): with self.assertRaises(RuntimeError): del foo.ipAddress + def testVirtualColumnsAreReadOnly(self): + def getFoo(*args, **kwargs): + return 4711 + def setFoo(*args, **kwargs): + pass + desc = blue.DBRowDescriptor(self.columns) + with self.assertRaises(TypeError): + desc.virtual = [("foo", getFoo, setFoo)] + desc.virtual = [("foo", getFoo)] + row = blue.DBRow(desc) + self.assertEqual(row.foo, 4711) + def testSliceSubscript(self): sliceTest = self.row[:] From 65610cc52e4810f378a6c0cdfdecfd69d547b229 Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Tue, 29 Jul 2025 16:01:55 +0000 Subject: [PATCH 046/131] Revert "Add typechecks for Python objects stored in `blue.DBRow`" --- src/PyRowSet.cpp | 59 ++--------------------- tests/python/bluetests/test/test_dbrow.py | 17 ------- 2 files changed, 5 insertions(+), 71 deletions(-) diff --git a/src/PyRowSet.cpp b/src/PyRowSet.cpp index c0f0c7c..afedb1c 100644 --- a/src/PyRowSet.cpp +++ b/src/PyRowSet.cpp @@ -1047,39 +1047,13 @@ PyObject *DBRow::Get(const ColumnDescriptor &c, Py_ssize_t i) const return PyFloat_FromDouble(*(double*)data); case DBTYPE_CY: return PyFloat_FromDouble(double(*(int64_t*)data) / 10000.0); - case DBTYPE_STR: { - //a python object - PyObject* result = *(PyObject**)data; - if( result && result != Py_None && !PyString_CheckExact( result ) ) - { - PyErr_Format( PyExc_TypeError, "Unexpected value for db column %d", c.mType ); - return nullptr; - } - Py_XINCREF( result ); - return result; - } - case DBTYPE_WSTR: { - //a python object - PyObject* result = *(PyObject**)data; - if( result && result != Py_None && !PyUnicode_CheckExact( result ) ) - { - PyErr_Format( PyExc_TypeError, "Unexpected value for db column %d", c.mType ); - return nullptr; - } - Py_XINCREF( result ); - return result; - } + case DBTYPE_STR: + case DBTYPE_WSTR: case DBTYPE_BYTES: { //a python object - PyObject* result = *(PyObject**)data; - if( result && result != Py_None && !PyBytes_CheckExact( result ) ) - { - PyErr_Format( PyExc_TypeError, "Unexpected value for db column %d", c.mType ); - return nullptr; - } - Py_XINCREF( result ); - return result; - } + PyObject *result = *(PyObject**)data; + Py_XINCREF(result); + return result;} case DBTYPE_EMPTY: // A virtual column! return mRD->VirtualGet(c.mOffset, this); @@ -1200,31 +1174,8 @@ bool DBRow::SetNotNull(const ColumnDescriptor &c, PyObject *o) return true; } case DBTYPE_STR: - if ( o != Py_None && !PyString_CheckExact( o ) ) - { - PyErr_Format( PyExc_TypeError, "Unexpected value for db column %d", c.mType ); - return false; - } - Py_INCREF(o); - Py_XDECREF(*(PyObject**)data); - *(PyObject**)data = o; - return true; case DBTYPE_WSTR: - if ( o != Py_None && !PyUnicode_CheckExact( o ) ) - { - PyErr_Format( PyExc_TypeError, "Unexpected value for db column %d", c.mType ); - return false; - } - Py_INCREF(o); - Py_XDECREF(*(PyObject**)data); - *(PyObject**)data = o; - return true; case DBTYPE_BYTES: - if ( o != Py_None && !PyBytes_CheckExact( o ) ) - { - PyErr_Format( PyExc_TypeError, "Unexpected value for db column %d", c.mType ); - return false; - } //a python object Py_INCREF(o); Py_XDECREF(*(PyObject**)data); diff --git a/tests/python/bluetests/test/test_dbrow.py b/tests/python/bluetests/test/test_dbrow.py index 709b259..d101d2e 100644 --- a/tests/python/bluetests/test/test_dbrow.py +++ b/tests/python/bluetests/test/test_dbrow.py @@ -57,20 +57,3 @@ def testEquivalencyComparison(self): self.assertNotEqual(self.row, blue.DBRowDescriptor(self.columns)) self.assertEqual(self.row, blue.DBRow(blue.DBRowDescriptor(self.columns))) - - def testPythonObjectValidation(self): - objectStoringRow = blue.DBRow(blue.DBRowDescriptor((("text", 128),("unicodeText", 129),("data", 130)))) - - for illegalValue in ( - object(), - 4711, - 0.815, - (1,2,3,), - ['a','b', 'c'] - ): - with self.assertRaises(TypeError): - setattr(objectStoringRow, "text", illegalValue) - with self.assertRaises(TypeError): - setattr(objectStoringRow, "unicodeText", illegalValue) - with self.assertRaises(TypeError): - setattr(objectStoringRow, "data", illegalValue) From e44372b321a2997b45693554fb8558f1635633ac Mon Sep 17 00:00:00 2001 From: CCP Cookies <69311849+ccpcookies@users.noreply.github.com> Date: Wed, 30 Jul 2025 14:22:36 +0000 Subject: [PATCH 047/131] Clean debug comments left from previous commit --- src/PyRowSet.cpp | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/src/PyRowSet.cpp b/src/PyRowSet.cpp index a42f347..d543efc 100644 --- a/src/PyRowSet.cpp +++ b/src/PyRowSet.cpp @@ -130,8 +130,6 @@ struct ColumnDescriptor case DBTYPE_BYTES: size = 5; break; //signal an object case DBTYPE_EMPTY: - // PyErr_SetString(PyExc_RuntimeError, "Virtual columns are no longer supported"); - // return false; size = -1; break; default: PyErr_Format(PyExc_TypeError, "DBRowDescriptor doesn't support data type %d", type); @@ -596,9 +594,6 @@ Py_ssize_t DBRowDescriptor::SequenceLength(DBRowDescriptor *row) //Return a shallow copy of the dude PyObject *DBRowDescriptor::Get_virtual() { - // The virtual column mechanism was an open door for remote code execution, there it was removed. - // PyErr_SetString(PyExc_RuntimeError, "Virtual columns are no longer supported"); - // return nullptr; BluePyList newList(0); if (!newList) return 0; @@ -613,9 +608,6 @@ PyObject *DBRowDescriptor::Get_virtual() bool DBRowDescriptor::Set_virtual(PyObject *l) { - // The virtual column mechanism was an open door for remote code execution, there it was removed. - // PyErr_SetString(PyExc_RuntimeError, "Virtual columns are ready only"); - // return false; //make sure we have a list of tuples at least three if (!PyList_Check(l)) return PyErr_SetString(PyExc_TypeError, "list required"), false; @@ -679,19 +671,6 @@ bool DBRowDescriptor::VirtualSet(int n, PyObject *row, PyObject *val) { PyErr_SetString(PyExc_RuntimeError, "Virtual columns are read-only"); return false; - if (!mVirtualGetSet || !PyList_Check(mVirtualGetSet.o)) - return PyErr_SetString(PyExc_RuntimeError, "Internal error in VirtualSet"), false; - if (n < 0 || n >= PyList_GET_SIZE(mVirtualGetSet.o)) - return PyErr_SetString(PyExc_RuntimeError, "Internal error in VirtualSet"), false; - PyObject *t = PyList_GET_ITEM(mVirtualGetSet.o, n); - if (!PyTuple_Check(t)) - return PyErr_SetString(PyExc_RuntimeError, "Internal error in VirtualSet"), false; - if (PyTuple_GET_SIZE(t)<3) - return (PyErr_SetString(PyExc_AttributeError, "read only attribute")), false; - PyObject *res = PyObject_CallFunctionObjArgs(PyTuple_GET_ITEM(t, 2), row, val, 0); - if (!res) return false; - Py_DECREF(res); - return true; } @@ -1090,8 +1069,7 @@ PyObject *DBRow::Get(const ColumnDescriptor &c, Py_ssize_t i) const return result; } case DBTYPE_EMPTY: - // PyErr_SetString(PyExc_RuntimeError, "Virtual columns are no longer supported"); - // return nullptr; + // A virtual column! return mRD->VirtualGet(c.mOffset, this); default: PyErr_Format(PyExc_RuntimeError, "Unexpected db column type encountered: %d", c.mType); From fae73ff450e979d1108540c8c973b112a002b04b Mon Sep 17 00:00:00 2001 From: CCP Cookies <69311849+ccpcookies@users.noreply.github.com> Date: Wed, 30 Jul 2025 14:34:34 +0000 Subject: [PATCH 048/131] Update error message to make it clearer that virtual columns are readonly --- src/PyRowSet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PyRowSet.cpp b/src/PyRowSet.cpp index d543efc..86dea15 100644 --- a/src/PyRowSet.cpp +++ b/src/PyRowSet.cpp @@ -1220,7 +1220,7 @@ bool DBRow::SetNotNull(const ColumnDescriptor &c, PyObject *o) return true; case DBTYPE_EMPTY: - PyErr_SetString(PyExc_RuntimeError, "Virtual columns are no longer supported"); + PyErr_SetString(PyExc_RuntimeError, "Virtual columns are read-only"); return false; default: PyErr_Format(PyExc_RuntimeError, "Unexpected db column type encountered: %d", c.mType); From 0cdc8625af253991423b5106a623281c74fc2a48 Mon Sep 17 00:00:00 2001 From: CCP Cookies <69311849+ccpcookies@users.noreply.github.com> Date: Fri, 1 Aug 2025 13:12:54 +0000 Subject: [PATCH 049/131] Add check to DBRow to ensure that unmarshalling creates expected DBRowDescriptorCheck When unmarshalling a DBRow the DBRowDescriptor type is read from the marshal string. It is possible to change this value to create a different object. However, the object must be in the whitelist for it to be created. It is very likely that changing the object type created would result in a crash, but for safety a check is added. It is also a good indicator that the input marshal string has been manually altered. --- src/PyRowSet.cpp | 8 ++++++++ tests/python/bluetests/test/test_marshal.py | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/PyRowSet.cpp b/src/PyRowSet.cpp index d0d8df2..43fc33d 100644 --- a/src/PyRowSet.cpp +++ b/src/PyRowSet.cpp @@ -1745,6 +1745,14 @@ PyObject *DBRow::Read(Marshal &m, ReadStream &s) { BluePy rd(m.ReadObject(&s)); //read row descriptor if (!rd) return 0; + + //Check object is of correct DBRowDescriptorType + if( DBRowDescriptor::GetType() != rd.o->ob_type ) + { + PyErr_Format( PyExc_TypeError, "Hacker Warning! Invalid type in marshal data, expected blue.DBRowDescriptor." ); + return 0; + } + //Create object BluePy rowO(DBRow::CreateFromRowDescriptor( rd ) ); if (!rowO) return 0; diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 4d843d9..74def7e 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -181,3 +181,17 @@ def test_wstream(self): obj = [OldSchoolObject(), SimpleObject(), "this is a test"] ws = blue.marshal.Save(obj) self.verify_round_trip(ws) + + def test_dbrow(self): + rowDesc = blue.DBRowDescriptor((("Test", 20),)) + sourceRow = blue.DBRow(rowDesc, (123, )) + self.verify_round_trip(sourceRow) + + def test_dbrow_with_invalid_descriptor_in_stream_raises_error(self): + # Unmarshalled bytes will attempt to craete a DBRow and create an eveexception.SQLError rather than + # expected blue.DBRowDescriptor + bytes = b'~\x00\x00\x00\x00*",\x02\tblue.Dict$--' + with self.assertRaises(RuntimeError) as raisedValue: + blue.marshal.Load(bytes) + + self.assertEqual(raisedValue.exception.args[0], TypeError) From b83932e85fb20f69829d55dc1964a472dc6e637f Mon Sep 17 00:00:00 2001 From: CCP Cookies <69311849+ccpcookies@users.noreply.github.com> Date: Fri, 1 Aug 2025 13:17:56 +0000 Subject: [PATCH 050/131] Fix typo in comment --- tests/python/bluetests/test/test_marshal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 74def7e..609414f 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -188,7 +188,7 @@ def test_dbrow(self): self.verify_round_trip(sourceRow) def test_dbrow_with_invalid_descriptor_in_stream_raises_error(self): - # Unmarshalled bytes will attempt to craete a DBRow and create an eveexception.SQLError rather than + # Unmarshalled bytes will attempt to create a DBRow and create a blue.Dict rather than # expected blue.DBRowDescriptor bytes = b'~\x00\x00\x00\x00*",\x02\tblue.Dict$--' with self.assertRaises(RuntimeError) as raisedValue: From a79a1eb9a9ea3c807029cf955750a78e73127915 Mon Sep 17 00:00:00 2001 From: CCP Cookies <69311849+ccpcookies@users.noreply.github.com> Date: Fri, 1 Aug 2025 15:21:26 +0000 Subject: [PATCH 051/131] Switch to use Py_TYPE to get Python object type. --- src/PyRowSet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PyRowSet.cpp b/src/PyRowSet.cpp index 43fc33d..ef87a4b 100644 --- a/src/PyRowSet.cpp +++ b/src/PyRowSet.cpp @@ -1747,7 +1747,7 @@ PyObject *DBRow::Read(Marshal &m, ReadStream &s) if (!rd) return 0; //Check object is of correct DBRowDescriptorType - if( DBRowDescriptor::GetType() != rd.o->ob_type ) + if( DBRowDescriptor::GetType() != Py_TYPE( rd.o ) ) { PyErr_Format( PyExc_TypeError, "Hacker Warning! Invalid type in marshal data, expected blue.DBRowDescriptor." ); return 0; From e7de39338cf1471a1e35c12076886bb81fa4b4a8 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Mon, 1 Sep 2025 15:30:42 +0000 Subject: [PATCH 052/131] Marshal single-character strings as `TY_BUFFER` types This allows Python3 environments to unmarshal the object in a manner which is consistent to other Python2 strings. --- src/Marshal.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Marshal.cpp b/src/Marshal.cpp index c7994bc..2eb7637 100644 --- a/src/Marshal.cpp +++ b/src/Marshal.cpp @@ -1864,22 +1864,21 @@ bool Marshal::WriteObject(WriteStream* stream, PyObject* o) { RETFAIL(WriteType(stream, TY_STR_EMPTY)); } - else if (size == 1) + else { - RETFAIL(WriteType(stream, TY_STR_CHAR)); - RETFAIL(stream->Write(string[0])); - } else { PyObject* index = PyDict_GetItem(mStrTable, o); if (index) { RETFAIL(WriteType(stream, TY_STR_TABLE)); RETFAIL(stream->Write((char)PyInt_AS_LONG(index))); - } else { + } + else + { //Write the string as buffer. This gives us object sharing and all //The old TY_STR_SHORT and TY_STR are kept for backwards compatibility CHECKREF(); RETFAIL(WriteType(stream, TY_BUFFER)); - RETFAIL(stream->WriteBuff(string, size)); + RETFAIL(stream->WriteBuff(string, size)); } } From 32e0c928b8857ca2f3a29973e4f43e67551c6127 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 2 Sep 2025 13:26:43 +0000 Subject: [PATCH 053/131] Fix typo in test name --- tests/python/bluetests/test/test_marshal.py | 394 ++++++++++---------- 1 file changed, 197 insertions(+), 197 deletions(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 609414f..435259a 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -1,197 +1,197 @@ -__author__ = 'snorri.sturluson' - -from . import blueunittest -import blue -import sys - -class EmptyObject(object): - pass - - -class SimpleObject(object): - def __init__(self): - self.a = "this is a string" - self.b = 42 - self.c = 3.14159267 - - -class OldSchoolObject: - def __init__(self): - self.a = "this is a string" - self.b = 42 - self.c = 3.14159267 - - -def SaveCallback(obj): - if isinstance(obj, OldSchoolObject): - return "magic" - return None - - -def LoadCallback(obj): - if obj == "magic": - return OldSchoolObject() - return None - - -class testMarshal(blueunittest.TestCase): - loaded = [] - saved = [] - - @classmethod - def setUpClass(cls): - blue.marshal.ResetTypeStats() - cls.loaded = [0]*48 - cls.saved = [0]*48 - - @classmethod - def tearDownClass(cls): - IGNORE_TYPES = [0, 3, 13, 16, 12, 24, 26, 29, 30, 33] - for i in range(48): - if i in IGNORE_TYPES: - continue - if cls.loaded[i] == 0: - sys.stderr.write("Missing coverage for type %d when loading" % i) - if cls.saved[i] == 0: - sys.stderr.write("Missing coverage for type %d when saving" % i) - - def _update_coverage(self): - typeStats = blue.marshal.GetTypeStats() - for i in range(48): - self.loaded[i] += typeStats[0][i] - self.saved[i] += typeStats[1][i] - - def verify_round_trip(self, obj): - blue.marshal.ResetTypeStats() - s = blue.marshal.Save(obj) - obj2 = blue.marshal.Load(s) - self.assertBlueObjectsEqual(obj, obj2) - typeStats = blue.marshal.GetTypeStats() - self.assertEqual(typeStats[0], typeStats[1]) - self._update_coverage() - - def test_none(self): - self.verify_round_trip(None) - - def test_empty_string(self): - self.verify_round_trip("") - - def test_string(self): - self.verify_round_trip("this is a test") - - def test_string_from_stringtable(self): - self.verify_round_trip("ballID") - - def test_empty_unicode(self): - self.verify_round_trip(u"") - - def test_single_char_unicode(self): - self.verify_round_trip(u"A") - - def test_unicode(self): - self.verify_round_trip(u"\u20A8\u20B1\u20A9") - - def test_unicode_as_utf8(self): - self.verify_round_trip(u"this is a unicode test") - - def test_integer(self): - self.verify_round_trip(0) - self.verify_round_trip(1) - self.verify_round_trip(-1) - self.verify_round_trip(42) - self.verify_round_trip(32767) - self.verify_round_trip(2147483647) - self.verify_round_trip(-2147483648) - - def test_long(self): - self.verify_round_trip(42) - self.verify_round_trip(9223372036854775807) - - def test_float(self): - self.verify_round_trip(0.0) - self.verify_round_trip(3.14159267) - self.verify_round_trip(-2.781431508934509809834) - - def test_bool(self): - self.verify_round_trip(True) - self.verify_round_trip(False) - - def test_empty_dict(self): - self.verify_round_trip({}) - - def test_empty_object(self): - self.verify_round_trip(EmptyObject()) - - def test_simple_object(self): - self.verify_round_trip(SimpleObject()) - - def test_empty_list(self): - self.verify_round_trip([]) - - def test_list_of_one_string(self): - self.verify_round_trip(["this is a test"]) - - def test_list_of_strings(self): - self.verify_round_trip(["this", "is", "a", "test"]) - - def test_empty_tuple(self): - self.verify_round_trip(()) - - def test_tuple_of_one_string(self): - self.verify_round_trip(("this is a test",)) - - def test_tuple_of_two_strings(self): - self.verify_round_trip(("this is", "a test")) - - def test_tuple_of_strings(self): - self.verify_round_trip(("this", "is", "a", "test")) - - def test_instanced_object(self): - obj = SimpleObject() - self.verify_round_trip([obj, obj, obj]) - - def test_instanced_old_shool_object(self): - obj = OldSchoolObject() - self.verify_round_trip([obj, obj, obj]) - - def test_callback(self): - obj = [OldSchoolObject(), SimpleObject(), "this is a test"] - s = blue.marshal.Save(obj, callback=SaveCallback) - obj2 = blue.marshal.Load(s, callback=LoadCallback) - self.assertBlueObjectsEqual(obj, obj2) - typeStats = blue.marshal.GetTypeStats() - self.assertEqual(typeStats[0], typeStats[1]) - self._update_coverage() - - def test_checksum(self): - obj = [OldSchoolObject(), SimpleObject(), "this is a test"] - s = blue.marshal.Save(obj, useChecksum=1) - obj2 = blue.marshal.Load(s) - self.assertBlueObjectsEqual(obj, obj2) - typeStats = blue.marshal.GetTypeStats() - self.assertEqual(typeStats[0], typeStats[1]) - self._update_coverage() - - def test_empty_dbrow(self): - rd = blue.DBRowDescriptor(()) - d = blue.DBRow(rd) - self.verify_round_trip(d) - - def test_wstream(self): - obj = [OldSchoolObject(), SimpleObject(), "this is a test"] - ws = blue.marshal.Save(obj) - self.verify_round_trip(ws) - - def test_dbrow(self): - rowDesc = blue.DBRowDescriptor((("Test", 20),)) - sourceRow = blue.DBRow(rowDesc, (123, )) - self.verify_round_trip(sourceRow) - - def test_dbrow_with_invalid_descriptor_in_stream_raises_error(self): - # Unmarshalled bytes will attempt to create a DBRow and create a blue.Dict rather than - # expected blue.DBRowDescriptor - bytes = b'~\x00\x00\x00\x00*",\x02\tblue.Dict$--' - with self.assertRaises(RuntimeError) as raisedValue: - blue.marshal.Load(bytes) - - self.assertEqual(raisedValue.exception.args[0], TypeError) +__author__ = 'snorri.sturluson' + +from . import blueunittest +import blue +import sys + +class EmptyObject(object): + pass + + +class SimpleObject(object): + def __init__(self): + self.a = "this is a string" + self.b = 42 + self.c = 3.14159267 + + +class OldSchoolObject: + def __init__(self): + self.a = "this is a string" + self.b = 42 + self.c = 3.14159267 + + +def SaveCallback(obj): + if isinstance(obj, OldSchoolObject): + return "magic" + return None + + +def LoadCallback(obj): + if obj == "magic": + return OldSchoolObject() + return None + + +class testMarshal(blueunittest.TestCase): + loaded = [] + saved = [] + + @classmethod + def setUpClass(cls): + blue.marshal.ResetTypeStats() + cls.loaded = [0]*48 + cls.saved = [0]*48 + + @classmethod + def tearDownClass(cls): + IGNORE_TYPES = [0, 3, 13, 16, 12, 24, 26, 29, 30, 33] + for i in range(48): + if i in IGNORE_TYPES: + continue + if cls.loaded[i] == 0: + sys.stderr.write("Missing coverage for type %d when loading" % i) + if cls.saved[i] == 0: + sys.stderr.write("Missing coverage for type %d when saving" % i) + + def _update_coverage(self): + typeStats = blue.marshal.GetTypeStats() + for i in range(48): + self.loaded[i] += typeStats[0][i] + self.saved[i] += typeStats[1][i] + + def verify_round_trip(self, obj): + blue.marshal.ResetTypeStats() + s = blue.marshal.Save(obj) + obj2 = blue.marshal.Load(s) + self.assertBlueObjectsEqual(obj, obj2) + typeStats = blue.marshal.GetTypeStats() + self.assertEqual(typeStats[0], typeStats[1]) + self._update_coverage() + + def test_none(self): + self.verify_round_trip(None) + + def test_empty_string(self): + self.verify_round_trip("") + + def test_string(self): + self.verify_round_trip("this is a test") + + def test_string_from_stringtable(self): + self.verify_round_trip("ballID") + + def test_empty_unicode(self): + self.verify_round_trip(u"") + + def test_single_char_unicode(self): + self.verify_round_trip(u"A") + + def test_unicode(self): + self.verify_round_trip(u"\u20A8\u20B1\u20A9") + + def test_unicode_as_utf8(self): + self.verify_round_trip(u"this is a unicode test") + + def test_integer(self): + self.verify_round_trip(0) + self.verify_round_trip(1) + self.verify_round_trip(-1) + self.verify_round_trip(42) + self.verify_round_trip(32767) + self.verify_round_trip(2147483647) + self.verify_round_trip(-2147483648) + + def test_long(self): + self.verify_round_trip(42) + self.verify_round_trip(9223372036854775807) + + def test_float(self): + self.verify_round_trip(0.0) + self.verify_round_trip(3.14159267) + self.verify_round_trip(-2.781431508934509809834) + + def test_bool(self): + self.verify_round_trip(True) + self.verify_round_trip(False) + + def test_empty_dict(self): + self.verify_round_trip({}) + + def test_empty_object(self): + self.verify_round_trip(EmptyObject()) + + def test_simple_object(self): + self.verify_round_trip(SimpleObject()) + + def test_empty_list(self): + self.verify_round_trip([]) + + def test_list_of_one_string(self): + self.verify_round_trip(["this is a test"]) + + def test_list_of_strings(self): + self.verify_round_trip(["this", "is", "a", "test"]) + + def test_empty_tuple(self): + self.verify_round_trip(()) + + def test_tuple_of_one_string(self): + self.verify_round_trip(("this is a test",)) + + def test_tuple_of_two_strings(self): + self.verify_round_trip(("this is", "a test")) + + def test_tuple_of_strings(self): + self.verify_round_trip(("this", "is", "a", "test")) + + def test_instanced_object(self): + obj = SimpleObject() + self.verify_round_trip([obj, obj, obj]) + + def test_instanced_old_school_object(self): + obj = OldSchoolObject() + self.verify_round_trip([obj, obj, obj]) + + def test_callback(self): + obj = [OldSchoolObject(), SimpleObject(), "this is a test"] + s = blue.marshal.Save(obj, callback=SaveCallback) + obj2 = blue.marshal.Load(s, callback=LoadCallback) + self.assertBlueObjectsEqual(obj, obj2) + typeStats = blue.marshal.GetTypeStats() + self.assertEqual(typeStats[0], typeStats[1]) + self._update_coverage() + + def test_checksum(self): + obj = [OldSchoolObject(), SimpleObject(), "this is a test"] + s = blue.marshal.Save(obj, useChecksum=1) + obj2 = blue.marshal.Load(s) + self.assertBlueObjectsEqual(obj, obj2) + typeStats = blue.marshal.GetTypeStats() + self.assertEqual(typeStats[0], typeStats[1]) + self._update_coverage() + + def test_empty_dbrow(self): + rd = blue.DBRowDescriptor(()) + d = blue.DBRow(rd) + self.verify_round_trip(d) + + def test_wstream(self): + obj = [OldSchoolObject(), SimpleObject(), "this is a test"] + ws = blue.marshal.Save(obj) + self.verify_round_trip(ws) + + def test_dbrow(self): + rowDesc = blue.DBRowDescriptor((("Test", 20),)) + sourceRow = blue.DBRow(rowDesc, (123, )) + self.verify_round_trip(sourceRow) + + def test_dbrow_with_invalid_descriptor_in_stream_raises_error(self): + # Unmarshalled bytes will attempt to create a DBRow and create a blue.Dict rather than + # expected blue.DBRowDescriptor + bytes = b'~\x00\x00\x00\x00*",\x02\tblue.Dict$--' + with self.assertRaises(RuntimeError) as raisedValue: + blue.marshal.Load(bytes) + + self.assertEqual(raisedValue.exception.args[0], TypeError) From 9e10a1b2a88e4d76be0ac62fbdf913b0111b10f3 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 2 Sep 2025 13:28:57 +0000 Subject: [PATCH 054/131] Add `test_marshal.TestBackwardsCompatibility` test class This class will eventually add test coverage for marshalled Python 3 objects. The class is mostly in parity with `testMarshal`. The exception being some tests such as `test_string`, which should be consistently marshalled as Unicode objects in Python 3 and are therefore covered by `test_unicode`. The real difference between the two classes is that `testMarshal` adds coverage for round-trip transactions, saving and loading objects, whereas `TestBackwardsCompatibility` is solely concerned with loading pre-marshalled Python 3 objects. --- tests/python/bluetests/test/test_marshal.py | 96 +++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 435259a..26e1864 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -195,3 +195,99 @@ def test_dbrow_with_invalid_descriptor_in_stream_raises_error(self): blue.marshal.Load(bytes) self.assertEqual(raisedValue.exception.args[0], TypeError) + +class TestBackwardsCompatibility(blueunittest.TestCase): + """ + This class adds coverage for objects marshalled in Python 3. + """ + def test_load_old_style_object(self): + pass + + def test_load_new_style_object(self): + pass + + def test_load_object_with_data(self): + pass + + def test_none(self): + pass + + def test_string_from_stringtable(self): + pass + + def test_empty_unicode(self): + pass + + def test_single_char_unicode(self): + pass + + def test_unicode(self): + pass + + def test_unicode_as_utf8(self): + pass + + def test_integer(self): + pass + + def test_long(self): + pass + + def test_float(self): + pass + + def test_bool(self): + pass + def test_empty_dict(self): + pass + + def test_empty_object(self): + pass + + def test_simple_object(self): + pass + + def test_empty_list(self): + pass + + def test_list_of_one_string(self): + pass + + def test_list_of_strings(self): + pass + + def test_empty_tuple(self): + pass + + def test_tuple_of_one_string(self): + pass + + def test_tuple_of_two_strings(self): + pass + + def test_tuple_of_strings(self): + pass + + def test_instanced_object(self): + pass + + def test_instanced_old_school_object(self): + pass + + def test_callback(self): + pass + + def test_checksum(self): + pass + + def test_empty_dbrow(self): + pass + + def test_dbrow(self): + pass + + def test_dbrow_with_invalid_descriptor_in_stream_raises_error(self): + pass + + def test_wstream(self): + pass From eea1eea0d465c8e809578204888d38f65c2835a3 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 2 Sep 2025 15:55:54 +0000 Subject: [PATCH 055/131] Implement `TestBackwardsCompatibility.test_none` unmarshal test --- tests/python/bluetests/test/test_marshal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 26e1864..10d9076 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -210,7 +210,8 @@ def test_load_object_with_data(self): pass def test_none(self): - pass + bytes = b'~\x00\x00\x00\x00\x01' + self.assertEqual(blue.marshal.Load(bytes), None) def test_string_from_stringtable(self): pass From 1370cc3c16ba0ca0785233f7f815e529864b31f0 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 2 Sep 2025 15:56:18 +0000 Subject: [PATCH 056/131] Implement `TestBackwardsCompatibility.test_string_from_stringtable` --- tests/python/bluetests/test/test_marshal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 10d9076..cce9a85 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -214,7 +214,8 @@ def test_none(self): self.assertEqual(blue.marshal.Load(bytes), None) def test_string_from_stringtable(self): - pass + bytes = b'~\x00\x00\x00\x00\x11\x06' + self.assertEqual(blue.marshal.Load(bytes), "ballID") def test_empty_unicode(self): pass From 01c90914445e7b61a4256a23c4066cb036005f3b Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 2 Sep 2025 15:56:28 +0000 Subject: [PATCH 057/131] Implement `TestBackwardsCompatibility.test_empty_unicode` --- tests/python/bluetests/test/test_marshal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index cce9a85..16e9f0c 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -218,7 +218,8 @@ def test_string_from_stringtable(self): self.assertEqual(blue.marshal.Load(bytes), "ballID") def test_empty_unicode(self): - pass + bytes = b'~\x00\x00\x00\x00(' + self.assertEqual(blue.marshal.Load(bytes), u"") def test_single_char_unicode(self): pass From 0eed95b74b1447154fa49bc7e90a5a0f5c5bb3af Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 2 Sep 2025 15:56:35 +0000 Subject: [PATCH 058/131] Implement `TestBackwardsCompatibility.test_single_char_unicode` --- tests/python/bluetests/test/test_marshal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 16e9f0c..0c2a5c7 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -222,7 +222,8 @@ def test_empty_unicode(self): self.assertEqual(blue.marshal.Load(bytes), u"") def test_single_char_unicode(self): - pass + bytes = b'~\x00\x00\x00\x00.\x01A' + self.assertEqual(blue.marshal.Load(bytes), u"A") def test_unicode(self): pass From 08e46f48af3495abf8ade64faca3ba9a6f80e11d Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 2 Sep 2025 15:56:43 +0000 Subject: [PATCH 059/131] Implement `TestBackwardsCompatibility.test_unicode` --- tests/python/bluetests/test/test_marshal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 0c2a5c7..56a14b6 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -226,7 +226,8 @@ def test_single_char_unicode(self): self.assertEqual(blue.marshal.Load(bytes), u"A") def test_unicode(self): - pass + bytes = b'~\x00\x00\x00\x00.\t\xe2\x82\xa8\xe2\x82\xb1\xe2\x82\xa9' + self.assertEqual(blue.marshal.Load(bytes), u"\u20A8\u20B1\u20A9") def test_unicode_as_utf8(self): pass From 719eac09caef89d0358be4b05c01fd459eb690de Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 2 Sep 2025 15:57:00 +0000 Subject: [PATCH 060/131] Implement `TestBackwardsCompatibility.test_load_object_with_data` --- tests/python/bluetests/test/test_marshal.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 56a14b6..b3a76f4 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -22,6 +22,11 @@ def __init__(self): self.c = 3.14159267 +class ObjectWithData(object): + def __init__(self, data): + self.data = data + + def SaveCallback(obj): if isinstance(obj, OldSchoolObject): return "magic" @@ -207,7 +212,9 @@ def test_load_new_style_object(self): pass def test_load_object_with_data(self): - pass + bytes = b'~\x00\x00\x00\x00#,%\x02*bluetests.test.test_marshal.ObjectWithData\x16\x01.\x04test.\x04data--' + obj = blue.marshal.Load(bytes) + self.assertEqual(obj.data, "test") def test_none(self): bytes = b'~\x00\x00\x00\x00\x01' From 64f3e433d685805507522c92c7217944ccf1d520 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 2 Sep 2025 16:03:42 +0000 Subject: [PATCH 061/131] Implement `TestBackwardsCompatibility.test_load_new_style_object` --- tests/python/bluetests/test/test_marshal.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index b3a76f4..b75414e 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -22,6 +22,10 @@ def __init__(self): self.c = 3.14159267 +class NewStyleObject(object): + pass + + class ObjectWithData(object): def __init__(self, data): self.data = data @@ -209,7 +213,8 @@ def test_load_old_style_object(self): pass def test_load_new_style_object(self): - pass + bytes = b'~\x00\x00\x00\x00#%%\x02*bluetests.test.test_marshal.NewStyleObject--' + self.assertIsInstance(blue.marshal.Load(bytes), NewStyleObject) def test_load_object_with_data(self): bytes = b'~\x00\x00\x00\x00#,%\x02*bluetests.test.test_marshal.ObjectWithData\x16\x01.\x04test.\x04data--' From d0d27ca200f60bfec8c006b528b227cbb8359cd7 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 2 Sep 2025 16:03:57 +0000 Subject: [PATCH 062/131] Implement `TestBackwardsCompatibility.test_load_old_style_object` --- tests/python/bluetests/test/test_marshal.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index b75414e..9a3f931 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -26,6 +26,10 @@ class NewStyleObject(object): pass +class OldStyleObject: + pass + + class ObjectWithData(object): def __init__(self, data): self.data = data @@ -210,7 +214,8 @@ class TestBackwardsCompatibility(blueunittest.TestCase): This class adds coverage for objects marshalled in Python 3. """ def test_load_old_style_object(self): - pass + bytes = b'~\x00\x00\x00\x00#%%\x02*bluetests.test.test_marshal.OldStyleObject--' + self.assertIsInstance(blue.marshal.Load(bytes), OldStyleObject) def test_load_new_style_object(self): bytes = b'~\x00\x00\x00\x00#%%\x02*bluetests.test.test_marshal.NewStyleObject--' From 848f199d0b6be190deed1a5fdca9387fd9dbc8ed Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 2 Sep 2025 17:01:58 +0000 Subject: [PATCH 063/131] Allow marshalled new-style objects to construct as old-style objects Python3 marshals all class objects as new-style objects. This is a problem because an old-style object marshalled as a new-style object may not be constructable as a new-style object in Python 2. In those cases, we cannot assume the class definition of the object in question contains a `__new__` operator, and must therefore create the class instance using a similar approach to old-style objects in `Marshall::ReadObjectsInstance`. --- src/Marshal.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Marshal.cpp b/src/Marshal.cpp index 2eb7637..8bf307e 100644 --- a/src/Marshal.cpp +++ b/src/Marshal.cpp @@ -1549,8 +1549,13 @@ PyObject *Marshal::ReadObjectNewobj(ReadStream* stream, bool shared) if (!cls) return 0; BluePy __new__(PyObject_GetAttr(cls, mStock_New)); +#ifdef PY3_COMPATIBILITY_MODE + // Marshalled new-style objects from Python3 may have to be constructed as old-style objects in Python2.7 + BluePy r(PyObject_HasAttr( cls, mStock_New ) ? BluePy(PyObject_CallObject(__new__, args)) : BluePy(PyInstance_NewRaw(cls, 0))); +#else if (!__new__) return 0; BluePy r(PyObject_CallObject(__new__, args)); +#endif if (!r) return 0; //object is constructed, now update r From 3f0ef62250f1a0eb88babd7c5ffe8590b9e453c5 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 2 Sep 2025 17:02:31 +0000 Subject: [PATCH 064/131] Add `PY3_COMPATIBILITY_MODE` compile definition --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 10326c7..968a732 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -308,6 +308,7 @@ target_precompile_headers(Blue PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src/StdAfx.h) target_compile_definitions(Blue PUBLIC CCP_STACKLESS + PY3_COMPATIBILITY_MODE ) if(APPLE) From 726f11acad2eb1b430b766c146810294ac7d4568 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Tue, 2 Sep 2025 17:04:55 +0000 Subject: [PATCH 065/131] Extend backwards compatibility old-style coverage This covers a case where an old-style object is constructed from a new-style marshalled object with state. --- tests/python/bluetests/test/test_marshal.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 9a3f931..86878aa 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -30,6 +30,11 @@ class OldStyleObject: pass +class OldStyleObjectWithData: + def __init__(self, data): + self.data = data + + class ObjectWithData(object): def __init__(self, data): self.data = data @@ -217,6 +222,10 @@ def test_load_old_style_object(self): bytes = b'~\x00\x00\x00\x00#%%\x02*bluetests.test.test_marshal.OldStyleObject--' self.assertIsInstance(blue.marshal.Load(bytes), OldStyleObject) + def test_load_old_style_object_with_data(self): + bytes = b'~\x00\x00\x00\x00#,%\x022bluetests.test.test_marshal.OldStyleObjectWithData\x16\x01.\x04test.\x04data--' + self.assertBlueObjectsEqual(blue.marshal.Load(bytes), OldStyleObjectWithData("test")) + def test_load_new_style_object(self): bytes = b'~\x00\x00\x00\x00#%%\x02*bluetests.test.test_marshal.NewStyleObject--' self.assertIsInstance(blue.marshal.Load(bytes), NewStyleObject) From 4107a08014426e75e195adda4448dd08861eeb77 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 11:21:40 +0000 Subject: [PATCH 066/131] Implement `TestBackwardsCompatibility.test_unicode_as_utf8` --- tests/python/bluetests/test/test_marshal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 86878aa..83ebc8f 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -256,7 +256,8 @@ def test_unicode(self): self.assertEqual(blue.marshal.Load(bytes), u"\u20A8\u20B1\u20A9") def test_unicode_as_utf8(self): - pass + bytes = b'~\x00\x00\x00\x00.\x16this is a unicode test' + self.assertEqual(blue.marshal.Load(bytes), u"this is a unicode test") def test_integer(self): pass From a12aabf6c1410b6bc4ce5cd29d84a13675eceea1 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 11:24:51 +0000 Subject: [PATCH 067/131] Implement `TestBackwardsCompatibility.test_integer` --- tests/python/bluetests/test/test_marshal.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 83ebc8f..2d679cf 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -260,7 +260,13 @@ def test_unicode_as_utf8(self): self.assertEqual(blue.marshal.Load(bytes), u"this is a unicode test") def test_integer(self): - pass + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x08'), 0) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\t'), 1) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x07'), -1) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x06*'), 42) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x05\xff\x7f'), 32767) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x04\xff\xff\xff\x7f'), 2147483647) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x04\x00\x00\x00\x80'), -2147483648) def test_long(self): pass From 095cb357b7791024e1fbd37bcea05b82d2c7a360 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 11:26:07 +0000 Subject: [PATCH 068/131] Implement `TestBackwardsCompatibility.test_long` --- tests/python/bluetests/test/test_marshal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 2d679cf..ce5eca1 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -269,7 +269,8 @@ def test_integer(self): self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x04\x00\x00\x00\x80'), -2147483648) def test_long(self): - pass + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00/\x08\xff\xff\xff\xff\xff\xff\xff\x7f'), 9223372036854775807) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00/\t\x00\x00\x00\x00\x00\x00\x00\x80\x00'), 9223372036854775808) def test_float(self): pass From 6999a4b1ec407dfcba640220c0ea9c440ffcce8b Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 11:30:29 +0000 Subject: [PATCH 069/131] Implement `TestBackwardsCompatibility.test_float` --- tests/python/bluetests/test/test_marshal.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index ce5eca1..071ab43 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -273,7 +273,9 @@ def test_long(self): self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00/\t\x00\x00\x00\x00\x00\x00\x00\x80\x00'), 9223372036854775808) def test_float(self): - pass + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x0b'), 0.0) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\n\xcd\x06xV\xfb!\t@'), 3.14159267) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\nO\x80\xb7)_@\x06\xc0'), -2.781431508934509809834) def test_bool(self): pass From 4c1a9c0f245d7b00c5eca10e6dc29d59c209995c Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 11:31:31 +0000 Subject: [PATCH 070/131] Implement `TestBackwardsCompatibility.test_bool` --- tests/python/bluetests/test/test_marshal.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 071ab43..41411a5 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -278,7 +278,9 @@ def test_float(self): self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\nO\x80\xb7)_@\x06\xc0'), -2.781431508934509809834) def test_bool(self): - pass + self.assertTrue(blue.marshal.Load(b'~\x00\x00\x00\x00\x1f'), True) + self.assertFalse(blue.marshal.Load(b'~\x00\x00\x00\x00 '), False) + def test_empty_dict(self): pass From b688db0bef96a6b2f647c7887525ab2279e21cc3 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 11:32:30 +0000 Subject: [PATCH 071/131] Implement `TestBackwardsCompatibility.test_empty_dict` --- tests/python/bluetests/test/test_marshal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 41411a5..62b28f3 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -282,7 +282,7 @@ def test_bool(self): self.assertFalse(blue.marshal.Load(b'~\x00\x00\x00\x00 '), False) def test_empty_dict(self): - pass + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x16\x00'), {}) def test_empty_object(self): pass From 8535490261e9116fa7417961f825e82567243f37 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 11:59:52 +0000 Subject: [PATCH 072/131] Add marshalling test coverage for non-empty dictionaries --- tests/python/bluetests/test/test_marshal.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 62b28f3..51e7507 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -137,6 +137,9 @@ def test_bool(self): def test_empty_dict(self): self.verify_round_trip({}) + def test_dict(self): + self.verify_round_trip({"key": "test"}) + def test_empty_object(self): self.verify_round_trip(EmptyObject()) @@ -284,6 +287,16 @@ def test_bool(self): def test_empty_dict(self): self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x16\x00'), {}) + def test_dict(self): + bytes = b'~\x00\x00\x00\x00\x16\x01.\x04test.\x03key' + loaded = blue.marshal.Load(bytes) + self.assertEqual(loaded, {"key": "test"}) + + # Explicit type checking due to Unicode and str types being implicitly comparable + for key, value in loaded.items(): + self.assertTrue(type(key) == unicode) + self.assertTrue(type(value) == unicode) + def test_empty_object(self): pass From 344a43976995b1faba6ec8de82f2f2cfd2780698 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 12:09:33 +0000 Subject: [PATCH 073/131] Add type checking to marshal compatibility tests handling unicode Unicode and string types are implicitly comparable, so compatibility tests need to verify whether the unmarshalled object is of the expected type. --- tests/python/bluetests/test/test_marshal.py | 36 ++++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 51e7507..cd6614c 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -227,7 +227,10 @@ def test_load_old_style_object(self): def test_load_old_style_object_with_data(self): bytes = b'~\x00\x00\x00\x00#,%\x022bluetests.test.test_marshal.OldStyleObjectWithData\x16\x01.\x04test.\x04data--' - self.assertBlueObjectsEqual(blue.marshal.Load(bytes), OldStyleObjectWithData("test")) + loaded = blue.marshal.Load(bytes) + + self.assertTrue(type(loaded.data) == unicode) + self.assertBlueObjectsEqual(loaded, OldStyleObjectWithData("test")) def test_load_new_style_object(self): bytes = b'~\x00\x00\x00\x00#%%\x02*bluetests.test.test_marshal.NewStyleObject--' @@ -235,8 +238,10 @@ def test_load_new_style_object(self): def test_load_object_with_data(self): bytes = b'~\x00\x00\x00\x00#,%\x02*bluetests.test.test_marshal.ObjectWithData\x16\x01.\x04test.\x04data--' - obj = blue.marshal.Load(bytes) - self.assertEqual(obj.data, "test") + loaded = blue.marshal.Load(bytes) + + self.assertTrue(type(loaded.data) == unicode) + self.assertEqual(loaded.data, "test") def test_none(self): bytes = b'~\x00\x00\x00\x00\x01' @@ -244,23 +249,38 @@ def test_none(self): def test_string_from_stringtable(self): bytes = b'~\x00\x00\x00\x00\x11\x06' - self.assertEqual(blue.marshal.Load(bytes), "ballID") + loaded = blue.marshal.Load(bytes) + + self.assertTrue(type(loaded) == str) + self.assertEqual(loaded, "ballID") def test_empty_unicode(self): bytes = b'~\x00\x00\x00\x00(' - self.assertEqual(blue.marshal.Load(bytes), u"") + loaded = blue.marshal.Load(bytes) + + self.assertTrue(type(loaded) == unicode) + self.assertEqual(loaded, "") def test_single_char_unicode(self): bytes = b'~\x00\x00\x00\x00.\x01A' - self.assertEqual(blue.marshal.Load(bytes), u"A") + loaded = blue.marshal.Load(bytes) + + self.assertTrue(type(loaded) == unicode) + self.assertEqual(loaded, "A") def test_unicode(self): bytes = b'~\x00\x00\x00\x00.\t\xe2\x82\xa8\xe2\x82\xb1\xe2\x82\xa9' - self.assertEqual(blue.marshal.Load(bytes), u"\u20A8\u20B1\u20A9") + loaded = blue.marshal.Load(bytes) + + self.assertTrue(type(loaded) == unicode) + self.assertEqual(loaded, u"\u20A8\u20B1\u20A9") def test_unicode_as_utf8(self): bytes = b'~\x00\x00\x00\x00.\x16this is a unicode test' - self.assertEqual(blue.marshal.Load(bytes), u"this is a unicode test") + loaded = blue.marshal.Load(bytes) + + self.assertTrue(type(loaded) == unicode) + self.assertEqual(loaded, "this is a unicode test") def test_integer(self): self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x08'), 0) From e740de4fb248976e4288a726ecd87e232f74d76b Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 12:11:43 +0000 Subject: [PATCH 074/131] Format existing marshal compatibility tests for consistency --- tests/python/bluetests/test/test_marshal.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index cd6614c..0effdfb 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -223,7 +223,9 @@ class TestBackwardsCompatibility(blueunittest.TestCase): """ def test_load_old_style_object(self): bytes = b'~\x00\x00\x00\x00#%%\x02*bluetests.test.test_marshal.OldStyleObject--' - self.assertIsInstance(blue.marshal.Load(bytes), OldStyleObject) + loaded = blue.marshal.Load(bytes) + + self.assertIsInstance(loaded, OldStyleObject) def test_load_old_style_object_with_data(self): bytes = b'~\x00\x00\x00\x00#,%\x022bluetests.test.test_marshal.OldStyleObjectWithData\x16\x01.\x04test.\x04data--' @@ -234,7 +236,9 @@ def test_load_old_style_object_with_data(self): def test_load_new_style_object(self): bytes = b'~\x00\x00\x00\x00#%%\x02*bluetests.test.test_marshal.NewStyleObject--' - self.assertIsInstance(blue.marshal.Load(bytes), NewStyleObject) + loaded = blue.marshal.Load(bytes) + + self.assertIsInstance(loaded, NewStyleObject) def test_load_object_with_data(self): bytes = b'~\x00\x00\x00\x00#,%\x02*bluetests.test.test_marshal.ObjectWithData\x16\x01.\x04test.\x04data--' @@ -245,7 +249,9 @@ def test_load_object_with_data(self): def test_none(self): bytes = b'~\x00\x00\x00\x00\x01' - self.assertEqual(blue.marshal.Load(bytes), None) + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, None) def test_string_from_stringtable(self): bytes = b'~\x00\x00\x00\x00\x11\x06' @@ -310,8 +316,8 @@ def test_empty_dict(self): def test_dict(self): bytes = b'~\x00\x00\x00\x00\x16\x01.\x04test.\x03key' loaded = blue.marshal.Load(bytes) - self.assertEqual(loaded, {"key": "test"}) + self.assertEqual(loaded, {"key": "test"}) # Explicit type checking due to Unicode and str types being implicitly comparable for key, value in loaded.items(): self.assertTrue(type(key) == unicode) From 39cc75ef9a4091ca9a300cf2d0fee92fc4e717c8 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 12:40:46 +0000 Subject: [PATCH 075/131] Implement `TestBackwardsCompatibility.test_empty_object` --- tests/python/bluetests/test/test_marshal.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 0effdfb..c78beac 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -5,8 +5,8 @@ import sys class EmptyObject(object): - pass - + def __eq__(self, other): + return isinstance(other, type(self)) class SimpleObject(object): def __init__(self): @@ -324,7 +324,10 @@ def test_dict(self): self.assertTrue(type(value) == unicode) def test_empty_object(self): - pass + bytes = b"~\x00\x00\x00\x00#%%\x02'bluetests.test.test_marshal.EmptyObject--" + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, EmptyObject()) def test_simple_object(self): pass From fc142c43f9af0d4b91e82e7d901c8acd9089de89 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 13:06:47 +0000 Subject: [PATCH 076/131] Implement `TestBackwardsCompatibility.test_simple_object` --- tests/python/bluetests/test/test_marshal.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index c78beac..c7d242c 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -330,7 +330,10 @@ def test_empty_object(self): self.assertEqual(loaded, EmptyObject()) def test_simple_object(self): - pass + bytes = b'~\x00\x00\x00\x00#,%\x02(bluetests.test.test_marshal.SimpleObject\x16\x05.\x10this is a string.\x01a\x06*.\x01b\n\xcd\x06xV\xfb!\t@.\x01c\x13\x07x\x01\x8d\x98{t\xd3.\x01d.\x10this is a string.\x01e--' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, SimpleObject()) def test_empty_list(self): pass From 3fb1d05a3044306fc850dfc63f6c0a4f2c6f5b3c Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 13:10:34 +0000 Subject: [PATCH 077/131] Extend `test_marshal.SimpleObject` comparison Adding byte string and Unicode fields extends our test coverage. Since string, unicode and byte expressions compare in a type-agnostic manner, it's sufficient to compare `__dict__` values across `SimpleObject` instances in order to ascertain object equivalency. --- tests/python/bluetests/test/test_marshal.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index c7d242c..8494d4e 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -8,11 +8,20 @@ class EmptyObject(object): def __eq__(self, other): return isinstance(other, type(self)) + class SimpleObject(object): def __init__(self): self.a = "this is a string" self.b = 42 self.c = 3.14159267 + self.d = b'x\x01\x8d\x98{t\xd3' + self.e = u"this is a string" + + def __eq__(self, other): + # String, byte and unicode comparisons are type-agnostic + # Therefore, an unmarshalled Python3 SimpleObject instance should compare truthfully + # even though d and e fields differ in type + return isinstance(self, type(other)) and self.__dict__ == other.__dict__ class OldSchoolObject: From 347c08b7d83d6f720308ec9f497872400421804d Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 13:11:58 +0000 Subject: [PATCH 078/131] Implement `TestBackwardsCompatibility.test_empty_list` --- tests/python/bluetests/test/test_marshal.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 8494d4e..453c06c 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -345,7 +345,10 @@ def test_simple_object(self): self.assertEqual(loaded, SimpleObject()) def test_empty_list(self): - pass + bytes = b'~\x00\x00\x00\x00&' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, []) def test_list_of_one_string(self): pass From d64bfb372db2931637f6529f61371084f48201d5 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 13:13:05 +0000 Subject: [PATCH 079/131] Implement `TestBackwardsCompatibility.test_list_of_one_string` --- tests/python/bluetests/test/test_marshal.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 453c06c..5178308 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -351,7 +351,10 @@ def test_empty_list(self): self.assertEqual(loaded, []) def test_list_of_one_string(self): - pass + bytes = b"~\x00\x00\x00\x00'.\x0ethis is a test" + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, ["this is a test"]) def test_list_of_strings(self): pass From cb584c38cd6f4772c541afa20d2c8dd4d4017d0b Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 13:14:18 +0000 Subject: [PATCH 080/131] Implement `TestBackwardsCompatibility.test_list_of_strings` --- tests/python/bluetests/test/test_marshal.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 5178308..a73435b 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -357,7 +357,10 @@ def test_list_of_one_string(self): self.assertEqual(loaded, ["this is a test"]) def test_list_of_strings(self): - pass + bytes = b'~\x00\x00\x00\x00\x15\x04.\x04this.\x02is.\x01a.\x04test' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, ["this", "is", "a", "test"]) def test_empty_tuple(self): pass From ad51c041ce43952bf95ec2a5379e17a21c46e602 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 13:15:06 +0000 Subject: [PATCH 081/131] Implement `TestBackwardsCompatibility.test_empty_tuple` --- tests/python/bluetests/test/test_marshal.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index a73435b..edee36a 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -363,7 +363,10 @@ def test_list_of_strings(self): self.assertEqual(loaded, ["this", "is", "a", "test"]) def test_empty_tuple(self): - pass + bytes = b'~\x00\x00\x00\x00$' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, ()) def test_tuple_of_one_string(self): pass From bff4796be4558f4d5496d24efe12c761eee3f17d Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 13:16:33 +0000 Subject: [PATCH 082/131] Implement `TestBackwardsCompatibility.test_tuple_of_one_string` --- tests/python/bluetests/test/test_marshal.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index edee36a..46346cb 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -369,7 +369,10 @@ def test_empty_tuple(self): self.assertEqual(loaded, ()) def test_tuple_of_one_string(self): - pass + bytes = b'~\x00\x00\x00\x00%.\x0ethis is a test' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, ("this is a test",)) def test_tuple_of_two_strings(self): pass From eef74ad89cf41abcc841e84d0374db5d707ed753 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 13:18:24 +0000 Subject: [PATCH 083/131] Implement `TestBackwardsCompatibility.test_tuple_of_two_strings` --- tests/python/bluetests/test/test_marshal.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 46346cb..aa7eeaf 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -375,7 +375,10 @@ def test_tuple_of_one_string(self): self.assertEqual(loaded, ("this is a test",)) def test_tuple_of_two_strings(self): - pass + bytes = b'~\x00\x00\x00\x00,.\x07this is.\x06a test' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, ("this is", "a test")) def test_tuple_of_strings(self): pass From e763f28f1f7b4e5be297397250767aa7ad63791c Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 13:22:33 +0000 Subject: [PATCH 084/131] Add type checks to prior compatibility tests --- tests/python/bluetests/test/test_marshal.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index aa7eeaf..948e9c9 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -355,12 +355,15 @@ def test_list_of_one_string(self): loaded = blue.marshal.Load(bytes) self.assertEqual(loaded, ["this is a test"]) + self.assertIsInstance(loaded[0], unicode) def test_list_of_strings(self): bytes = b'~\x00\x00\x00\x00\x15\x04.\x04this.\x02is.\x01a.\x04test' loaded = blue.marshal.Load(bytes) self.assertEqual(loaded, ["this", "is", "a", "test"]) + for item in loaded: + self.assertIsInstance(item, unicode) def test_empty_tuple(self): bytes = b'~\x00\x00\x00\x00$' @@ -373,12 +376,15 @@ def test_tuple_of_one_string(self): loaded = blue.marshal.Load(bytes) self.assertEqual(loaded, ("this is a test",)) + self.assertIsInstance(loaded[0], unicode) def test_tuple_of_two_strings(self): bytes = b'~\x00\x00\x00\x00,.\x07this is.\x06a test' loaded = blue.marshal.Load(bytes) self.assertEqual(loaded, ("this is", "a test")) + for item in loaded: + self.assertIsInstance(item, unicode) def test_tuple_of_strings(self): pass From 3b992d92bb96bb87d62a170a848fc37144eefc3c Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 13:22:55 +0000 Subject: [PATCH 085/131] Implement `TestBackwardsCompatibility.test_tuple_of_strings` --- tests/python/bluetests/test/test_marshal.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 948e9c9..e388d6c 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -387,7 +387,12 @@ def test_tuple_of_two_strings(self): self.assertIsInstance(item, unicode) def test_tuple_of_strings(self): - pass + bytes = b'~\x00\x00\x00\x00\x14\x04.\x04this.\x02is.\x01a.\x04test' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, ("this", "is", "a", "test")) + for item in loaded: + self.assertIsInstance(item, unicode) def test_instanced_object(self): pass From 7e10a2c578e0c28bd04ab4c71fb5c82b9d49622a Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 13:25:20 +0000 Subject: [PATCH 086/131] Implement `TestBackwardsCompatibility.test_instanced_object` --- tests/python/bluetests/test/test_marshal.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index e388d6c..6bd5c30 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -395,7 +395,11 @@ def test_tuple_of_strings(self): self.assertIsInstance(item, unicode) def test_instanced_object(self): - pass + bytes = b'~\x01\x00\x00\x00\x15\x03c,%\x02(bluetests.test.test_marshal.SimpleObject\x16\x05.\x10this is a string.\x01a\x06*.\x01b\n\xcd\x06xV\xfb!\t@.\x01c\x13\x07x\x01\x8d\x98{t\xd3.\x01d.\x10this is a string.\x01e--\x1b\x01\x1b\x01\x01\x00\x00\x00' + loaded = blue.marshal.Load(bytes) + instance = SimpleObject() + + self.assertEqual(loaded, [instance, instance, instance]) def test_instanced_old_school_object(self): pass From c47b64c4d7adc9616ca7e5fa37f995e57fae8d1c Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 13:57:11 +0000 Subject: [PATCH 087/131] Consolidate and reformat `test_marshal` data `OldSchoolObject` has been renamed to `OldStyleObject`. `NewStyleObject` and `SimpleObject` have been consolidated into one object as they're both new style objects and have an overlap in utility afforded to the test suite. `OldStyleObject` and `NewStyleObject` have had their state extended to afford better coverage and make the tests more consistent. --- tests/python/bluetests/test/test_marshal.py | 96 +++++++-------------- 1 file changed, 31 insertions(+), 65 deletions(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 6bd5c30..3bb7ac0 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -9,55 +9,42 @@ def __eq__(self, other): return isinstance(other, type(self)) -class SimpleObject(object): +class NewStyleObject(object): def __init__(self): self.a = "this is a string" - self.b = 42 - self.c = 3.14159267 - self.d = b'x\x01\x8d\x98{t\xd3' - self.e = u"this is a string" + self.b = b"this is a string" + self.c = u"this is a string" + self.d = 42 + self.e = 3.14159267 def __eq__(self, other): # String, byte and unicode comparisons are type-agnostic - # Therefore, an unmarshalled Python3 SimpleObject instance should compare truthfully + # Therefore, an unmarshalled Python3 NewStyleObject instance should compare truthfully # even though d and e fields differ in type return isinstance(self, type(other)) and self.__dict__ == other.__dict__ -class OldSchoolObject: +class OldStyleObject: def __init__(self): self.a = "this is a string" - self.b = 42 - self.c = 3.14159267 - - -class NewStyleObject(object): - pass - + self.b = b"this is a string" + self.c = u"this is a string" + self.d = 42 + self.e = 3.14159267 -class OldStyleObject: - pass - - -class OldStyleObjectWithData: - def __init__(self, data): - self.data = data - - -class ObjectWithData(object): - def __init__(self, data): - self.data = data + def __eq__(self, other): + return isinstance(self, type(other)) and self.__dict__ == other.__dict__ def SaveCallback(obj): - if isinstance(obj, OldSchoolObject): + if isinstance(obj, OldStyleObject): return "magic" return None def LoadCallback(obj): if obj == "magic": - return OldSchoolObject() + return OldStyleObject() return None @@ -152,8 +139,8 @@ def test_dict(self): def test_empty_object(self): self.verify_round_trip(EmptyObject()) - def test_simple_object(self): - self.verify_round_trip(SimpleObject()) + def test_new_style_object(self): + self.verify_round_trip(NewStyleObject()) def test_empty_list(self): self.verify_round_trip([]) @@ -177,15 +164,15 @@ def test_tuple_of_strings(self): self.verify_round_trip(("this", "is", "a", "test")) def test_instanced_object(self): - obj = SimpleObject() + obj = NewStyleObject() self.verify_round_trip([obj, obj, obj]) - def test_instanced_old_school_object(self): - obj = OldSchoolObject() + def test_instanced_old_style_object(self): + obj = OldStyleObject() self.verify_round_trip([obj, obj, obj]) def test_callback(self): - obj = [OldSchoolObject(), SimpleObject(), "this is a test"] + obj = [OldStyleObject(), NewStyleObject(), "this is a test"] s = blue.marshal.Save(obj, callback=SaveCallback) obj2 = blue.marshal.Load(s, callback=LoadCallback) self.assertBlueObjectsEqual(obj, obj2) @@ -194,7 +181,7 @@ def test_callback(self): self._update_coverage() def test_checksum(self): - obj = [OldSchoolObject(), SimpleObject(), "this is a test"] + obj = [OldStyleObject(), NewStyleObject(), "this is a test"] s = blue.marshal.Save(obj, useChecksum=1) obj2 = blue.marshal.Load(s) self.assertBlueObjectsEqual(obj, obj2) @@ -208,7 +195,7 @@ def test_empty_dbrow(self): self.verify_round_trip(d) def test_wstream(self): - obj = [OldSchoolObject(), SimpleObject(), "this is a test"] + obj = [OldStyleObject(), NewStyleObject(), "this is a test"] ws = blue.marshal.Save(obj) self.verify_round_trip(ws) @@ -231,30 +218,10 @@ class TestBackwardsCompatibility(blueunittest.TestCase): This class adds coverage for objects marshalled in Python 3. """ def test_load_old_style_object(self): - bytes = b'~\x00\x00\x00\x00#%%\x02*bluetests.test.test_marshal.OldStyleObject--' - loaded = blue.marshal.Load(bytes) - - self.assertIsInstance(loaded, OldStyleObject) - - def test_load_old_style_object_with_data(self): - bytes = b'~\x00\x00\x00\x00#,%\x022bluetests.test.test_marshal.OldStyleObjectWithData\x16\x01.\x04test.\x04data--' + bytes = b'~\x00\x00\x00\x00#,%\x02*bluetests.test.test_marshal.OldStyleObject\x16\x05.\x10this is a string.\x01a\x13\x10this is a string.\x01b.\x10this is a string.\x01c\x06*.\x01d\n\xcd\x06xV\xfb!\t@.\x01e--' loaded = blue.marshal.Load(bytes) - self.assertTrue(type(loaded.data) == unicode) - self.assertBlueObjectsEqual(loaded, OldStyleObjectWithData("test")) - - def test_load_new_style_object(self): - bytes = b'~\x00\x00\x00\x00#%%\x02*bluetests.test.test_marshal.NewStyleObject--' - loaded = blue.marshal.Load(bytes) - - self.assertIsInstance(loaded, NewStyleObject) - - def test_load_object_with_data(self): - bytes = b'~\x00\x00\x00\x00#,%\x02*bluetests.test.test_marshal.ObjectWithData\x16\x01.\x04test.\x04data--' - loaded = blue.marshal.Load(bytes) - - self.assertTrue(type(loaded.data) == unicode) - self.assertEqual(loaded.data, "test") + self.assertEqual(loaded, OldStyleObject()) def test_none(self): bytes = b'~\x00\x00\x00\x00\x01' @@ -338,11 +305,11 @@ def test_empty_object(self): self.assertEqual(loaded, EmptyObject()) - def test_simple_object(self): - bytes = b'~\x00\x00\x00\x00#,%\x02(bluetests.test.test_marshal.SimpleObject\x16\x05.\x10this is a string.\x01a\x06*.\x01b\n\xcd\x06xV\xfb!\t@.\x01c\x13\x07x\x01\x8d\x98{t\xd3.\x01d.\x10this is a string.\x01e--' + def test_new_style_object(self): + bytes = b'~\x00\x00\x00\x00#,%\x02*bluetests.test.test_marshal.NewStyleObject\x16\x05.\x10this is a string.\x01a\x13\x10this is a string.\x01b.\x10this is a string.\x01c\x06*.\x01d\n\xcd\x06xV\xfb!\t@.\x01e--' loaded = blue.marshal.Load(bytes) - self.assertEqual(loaded, SimpleObject()) + self.assertEqual(loaded, NewStyleObject()) def test_empty_list(self): bytes = b'~\x00\x00\x00\x00&' @@ -395,14 +362,13 @@ def test_tuple_of_strings(self): self.assertIsInstance(item, unicode) def test_instanced_object(self): - bytes = b'~\x01\x00\x00\x00\x15\x03c,%\x02(bluetests.test.test_marshal.SimpleObject\x16\x05.\x10this is a string.\x01a\x06*.\x01b\n\xcd\x06xV\xfb!\t@.\x01c\x13\x07x\x01\x8d\x98{t\xd3.\x01d.\x10this is a string.\x01e--\x1b\x01\x1b\x01\x01\x00\x00\x00' + bytes = b'~\x01\x00\x00\x00\x15\x03c,%\x02*bluetests.test.test_marshal.NewStyleObject\x16\x05.\x10this is a string.\x01a\x13\x10this is a string.\x01b.\x10this is a string.\x01c\x06*.\x01d\n\xcd\x06xV\xfb!\t@.\x01e--\x1b\x01\x1b\x01\x01\x00\x00\x00' loaded = blue.marshal.Load(bytes) - instance = SimpleObject() + instance = NewStyleObject() self.assertEqual(loaded, [instance, instance, instance]) - def test_instanced_old_school_object(self): - pass + def test_instanced_old_style_object(self): def test_callback(self): pass From 7dea5fe3ba691aa0184578ddd291a566f6fcea8c Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 13:58:08 +0000 Subject: [PATCH 088/131] Implement `TestBackwardsCompatibility.test_instanced_old_style_object` --- tests/python/bluetests/test/test_marshal.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 3bb7ac0..4012b98 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -369,6 +369,11 @@ def test_instanced_object(self): self.assertEqual(loaded, [instance, instance, instance]) def test_instanced_old_style_object(self): + bytes = b'~\x01\x00\x00\x00\x15\x03c,%\x02*bluetests.test.test_marshal.OldStyleObject\x16\x05.\x10this is a string.\x01a\x13\x10this is a string.\x01b.\x10this is a string.\x01c\x06*.\x01d\n\xcd\x06xV\xfb!\t@.\x01e--\x1b\x01\x1b\x01\x01\x00\x00\x00' + loaded = blue.marshal.Load(bytes) + instance = OldStyleObject() + + self.assertEqual(loaded, [instance, instance, instance]) def test_callback(self): pass From 6a0c9ee503122444049aac7c49164b9629641c62 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 14:39:11 +0000 Subject: [PATCH 089/131] Implement `TestBackwardsCompatibility.test_dbrow` --- tests/python/bluetests/test/test_marshal.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 4012b98..499287d 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -385,7 +385,10 @@ def test_empty_dbrow(self): pass def test_dbrow(self): - pass + bytes = b'~\x00\x00\x00\x00*",\x02\x14blue.DBRowDescriptor%%,.\x04Test\x06\x14--\x02\xf7{' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, blue.DBRow(blue.DBRowDescriptor((("Test", 20),)), (123, ))) def test_dbrow_with_invalid_descriptor_in_stream_raises_error(self): pass From 369e9ae9828b9186b4f29e26c3d1b4a6ebec3f5d Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 14:41:04 +0000 Subject: [PATCH 090/131] Implement `TestBackwardsCompatibility.test_empty_dbrow` --- tests/python/bluetests/test/test_marshal.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 499287d..4ad85cb 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -382,7 +382,10 @@ def test_checksum(self): pass def test_empty_dbrow(self): - pass + bytes = b'~\x00\x00\x00\x00*",\x02\x14blue.DBRowDescriptor%$--\x00' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, blue.DBRow(blue.DBRowDescriptor(()))) def test_dbrow(self): bytes = b'~\x00\x00\x00\x00*",\x02\x14blue.DBRowDescriptor%%,.\x04Test\x06\x14--\x02\xf7{' From dff688a324696233efedec5e3c60a5c809194e06 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 14:44:19 +0000 Subject: [PATCH 091/131] Implement `TestBackwardsCompatibility.test_checksum` --- tests/python/bluetests/test/test_marshal.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 4ad85cb..ea84965 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -379,7 +379,13 @@ def test_callback(self): pass def test_checksum(self): - pass + # Marshalled Python3 object using checksum + bytes = b'~\x00\x00\x00\x00\x1c\xb9/\x0fL\x15\x02#,%\x02*bluetests.test.test_marshal.NewStyleObject\x16\x05.\x10this is a string.\x01a\x13\x10this is a string.\x01b.\x10this is a string.\x01c\x06*.\x01d\n\xcd\x06xV\xfb!\t@.\x01e--.\x0ethis is a test' + loaded = blue.marshal.Load(bytes) + comparison = blue.marshal.Save([NewStyleObject(), "this is a test"], useChecksum=1) + + # Marshalled data will differ due to string fields, so we must load both objects for comparison + self.assertBlueObjectsEqual(loaded, blue.marshal.Load(comparison)) def test_empty_dbrow(self): bytes = b'~\x00\x00\x00\x00*",\x02\x14blue.DBRowDescriptor%$--\x00' From 99579fd4b4bb93d784f4170d5c116fb994304c1e Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 15:51:46 +0000 Subject: [PATCH 092/131] Implement `TestBackwardsCompatibility.test_read_callback_called` --- tests/python/bluetests/test/test_marshal.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index ea84965..0fdeecc 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -375,8 +375,14 @@ def test_instanced_old_style_object(self): self.assertEqual(loaded, [instance, instance, instance]) - def test_callback(self): - pass + def test_read_callback_called(self): + def read_callback(obj): + read_callback.called = True + read_callback.called = False + + bytes = b'~\x00\x00\x00\x00\x19.\x04test' + blue.marshal.Load(bytes, callback=read_callback) + self.assertTrue(read_callback.called) def test_checksum(self): # Marshalled Python3 object using checksum From 50562190feeae4f736880048a95088ee81051846 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Wed, 3 Sep 2025 15:59:07 +0000 Subject: [PATCH 093/131] Remove `test_wstream` marshalling compatibility test This test is intended to test write stream operations, which is not part of `TestBackwardsCompatibility` coverage. --- tests/python/bluetests/test/test_marshal.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 0fdeecc..84d2318 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -407,6 +407,3 @@ def test_dbrow(self): def test_dbrow_with_invalid_descriptor_in_stream_raises_error(self): pass - - def test_wstream(self): - pass From 94dd38d69180e20e51c880af40adf88530496cf1 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Thu, 4 Sep 2025 13:03:58 +0000 Subject: [PATCH 094/131] Control Python3 forward compatibility features through CMake --- CMakeLists.txt | 8 +++++--- tests/python/bluetests/test/test_marshal.py | 4 ++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 968a732..141ec80 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -304,11 +304,13 @@ target_sources(Blue PRIVATE ${PUBLIC_HEADER_FILES_INCLUDE} ) +option(ENABLE_PY3_COMPATIBILITY "Build with Python3 forward compatibility enabled" ON) + target_precompile_headers(Blue PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src/StdAfx.h) target_compile_definitions(Blue PUBLIC CCP_STACKLESS - PY3_COMPATIBILITY_MODE + $<$:PY3_COMPATIBILITY_MODE> ) if(APPLE) @@ -388,9 +390,9 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests/python ) if(WIN32) - set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$\;${BRANCH_ROOT_DIR}/carbon/common/stdlib\;${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}\;${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>") + set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$\;${BRANCH_ROOT_DIR}/carbon/common/stdlib\;${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}\;${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$<$:PY3_COMPATIBILITY_MODE=True>") elseif(APPLE) - set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$:${BRANCH_ROOT_DIR}/carbon/common/stdlib:${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}:${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>") + set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$:${BRANCH_ROOT_DIR}/carbon/common/stdlib:${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}:${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$<$:PY3_COMPATIBILITY_MODsE=True") else() message(FATAL_ERROR "Unsupported platform") endif() diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 84d2318..1f0a9b6 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -2,7 +2,10 @@ from . import blueunittest import blue + +import unittest import sys +import os class EmptyObject(object): def __eq__(self, other): @@ -213,6 +216,7 @@ def test_dbrow_with_invalid_descriptor_in_stream_raises_error(self): self.assertEqual(raisedValue.exception.args[0], TypeError) +@unittest.skipUnless(os.environ.has_key("PY3_COMPATIBILITY_MODE"), "Skipping because these tests will fail if `PY3_COMPATIBILITY_MODE` is not enabled") class TestBackwardsCompatibility(blueunittest.TestCase): """ This class adds coverage for objects marshalled in Python 3. From 215a967b46c1189beaed19c8fb5ad6d3a78e9111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Fri, 5 Sep 2025 12:04:07 +0000 Subject: [PATCH 095/131] Add forwards compatibility support for things in the builtins module (#22) --- src/Marshal.cpp | 9 +++++++++ tests/python/bluetests/test/test_marshal.py | 16 ++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/Marshal.cpp b/src/Marshal.cpp index 8bf307e..c4e470f 100644 --- a/src/Marshal.cpp +++ b/src/Marshal.cpp @@ -1659,7 +1659,16 @@ PyObject *Marshal::GetGlobalObject(PyObject *nameO) const char *dot = strrchr(name, '.'); BluePyStr modulename; if (dot){ +#if PY3_COMPATIBILITY_MODE + if( strncmp( name, "builtins.", 9 ) == 0 ) { + modulename = BluePyStr( "__builtin__" ); + } + else { + modulename = BluePyStr( dot-name, name ); + } +#else modulename = BluePyStr(dot-name, name); +#endif name = dot+1; } else { modulename = BluePyStr("__builtin__"); diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 1f0a9b6..f732296 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -411,3 +411,19 @@ def test_dbrow(self): def test_dbrow_with_invalid_descriptor_in_stream_raises_error(self): pass + + def test_set(self): + blue.marshal.globalsWhitelist = {set: None} + blue.marshal.collectWhitelist = False + bytes = b'~\x00\x00\x00\x00",\x02\x0cbuiltins.set%\x15\x03\t\x06\x02\x06\x03--' + loaded = blue.marshal.Load(bytes) + self.assertSetEqual(loaded, {1, 2, 3}) + + def test_runtime_error(self): + blue.marshal.globalsWhitelist = {RuntimeError: None} + blue.marshal.collectWhitelist = False + bytes = b'~\x00\x00\x00\x00",\x02\x15builtins.RuntimeError%.\x05Boom!--' + loaded = blue.marshal.Load(bytes) + self.assertIsInstance(loaded, RuntimeError) + self.assertIsInstance(loaded.message, unicode) + self.assertEqual(loaded.message, u"Boom!") From 580daf91661a0c927971d84b950898807e49470b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Fri, 5 Sep 2025 13:59:00 +0000 Subject: [PATCH 096/131] Toggle single character string optimization (#23) Reintroduce single character string optimization when not running in compatibility mode --- src/Marshal.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Marshal.cpp b/src/Marshal.cpp index c4e470f..7b13a31 100644 --- a/src/Marshal.cpp +++ b/src/Marshal.cpp @@ -1878,6 +1878,13 @@ bool Marshal::WriteObject(WriteStream* stream, PyObject* o) { RETFAIL(WriteType(stream, TY_STR_EMPTY)); } +#ifndef PY3_COMPATIBILITY_MODE + else if (size == 1) + { + RETFAIL(WriteType(stream, TY_STR_CHAR)); + RETFAIL(stream->Write(string[0])); + } +#endif else { PyObject* index = PyDict_GetItem(mStrTable, o); From 7b6fdc2c1f0a767d0af392f1f2907a2cc4012bac Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Mon, 8 Sep 2025 10:53:42 +0000 Subject: [PATCH 097/131] Use `assertIsInstance` in place of `assertTrue` for type checking --- tests/python/bluetests/test/test_marshal.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index f732296..2a22501 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -213,7 +213,7 @@ def test_dbrow_with_invalid_descriptor_in_stream_raises_error(self): bytes = b'~\x00\x00\x00\x00*",\x02\tblue.Dict$--' with self.assertRaises(RuntimeError) as raisedValue: blue.marshal.Load(bytes) - + self.assertEqual(raisedValue.exception.args[0], TypeError) @unittest.skipUnless(os.environ.has_key("PY3_COMPATIBILITY_MODE"), "Skipping because these tests will fail if `PY3_COMPATIBILITY_MODE` is not enabled") @@ -244,28 +244,28 @@ def test_empty_unicode(self): bytes = b'~\x00\x00\x00\x00(' loaded = blue.marshal.Load(bytes) - self.assertTrue(type(loaded) == unicode) + self.assertIsInstance(loaded, unicode) self.assertEqual(loaded, "") def test_single_char_unicode(self): bytes = b'~\x00\x00\x00\x00.\x01A' loaded = blue.marshal.Load(bytes) - self.assertTrue(type(loaded) == unicode) + self.assertIsInstance(loaded, unicode) self.assertEqual(loaded, "A") def test_unicode(self): bytes = b'~\x00\x00\x00\x00.\t\xe2\x82\xa8\xe2\x82\xb1\xe2\x82\xa9' loaded = blue.marshal.Load(bytes) - self.assertTrue(type(loaded) == unicode) + self.assertIsInstance(loaded, unicode) self.assertEqual(loaded, u"\u20A8\u20B1\u20A9") def test_unicode_as_utf8(self): bytes = b'~\x00\x00\x00\x00.\x16this is a unicode test' loaded = blue.marshal.Load(bytes) - self.assertTrue(type(loaded) == unicode) + self.assertIsInstance(loaded, unicode) self.assertEqual(loaded, "this is a unicode test") def test_integer(self): @@ -300,8 +300,8 @@ def test_dict(self): self.assertEqual(loaded, {"key": "test"}) # Explicit type checking due to Unicode and str types being implicitly comparable for key, value in loaded.items(): - self.assertTrue(type(key) == unicode) - self.assertTrue(type(value) == unicode) + self.assertIsInstance(key, unicode) + self.assertIsInstance(value, unicode) def test_empty_object(self): bytes = b"~\x00\x00\x00\x00#%%\x02'bluetests.test.test_marshal.EmptyObject--" From 281b588e93acc113fda22e38ee3248f8eee45bf8 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Mon, 8 Sep 2025 10:56:21 +0000 Subject: [PATCH 098/131] Fix marshalling forward compatibility string table unit test Since a string table entry is marshalled as an index and not a string we don't need to make assumptions about string encoding sent over the wire. --- tests/python/bluetests/test/test_marshal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 2a22501..b11aa58 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -237,7 +237,8 @@ def test_string_from_stringtable(self): bytes = b'~\x00\x00\x00\x00\x11\x06' loaded = blue.marshal.Load(bytes) - self.assertTrue(type(loaded) == str) + # We expect a str type constructed from marshalled string table index + self.assertIsInstance(loaded, str) self.assertEqual(loaded, "ballID") def test_empty_unicode(self): From fcf2dea47dd751e3d3874cefea156751556fed28 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Mon, 8 Sep 2025 12:07:13 +0000 Subject: [PATCH 099/131] Set `ENABLE_PY3_COMPATIBILITY` to OFF by default --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 141ec80..c6f0ede 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -304,7 +304,7 @@ target_sources(Blue PRIVATE ${PUBLIC_HEADER_FILES_INCLUDE} ) -option(ENABLE_PY3_COMPATIBILITY "Build with Python3 forward compatibility enabled" ON) +option(ENABLE_PY3_COMPATIBILITY "Build with Python3 forward compatibility enabled" OFF) target_precompile_headers(Blue PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src/StdAfx.h) target_compile_definitions(Blue From eb140b9741fb8fb22aad5035af0bc4c411da76f0 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Mon, 8 Sep 2025 13:44:32 +0000 Subject: [PATCH 100/131] Remove redundant unit test for marshalling forwards compatibility This test does not require added test coverage when building in compatibility mode. --- tests/python/bluetests/test/test_marshal.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index b11aa58..1ca4f76 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -410,9 +410,6 @@ def test_dbrow(self): self.assertEqual(loaded, blue.DBRow(blue.DBRowDescriptor((("Test", 20),)), (123, ))) - def test_dbrow_with_invalid_descriptor_in_stream_raises_error(self): - pass - def test_set(self): blue.marshal.globalsWhitelist = {set: None} blue.marshal.collectWhitelist = False From 3ed39e66f5f5a1436675d75d006a913725161b27 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Thu, 4 Sep 2025 13:08:25 +0000 Subject: [PATCH 101/131] Apply forward compatibility changes for testing --- tests/python/bluetests/test/test_marshal.py | 32 ++++++--------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 1ca4f76..0f6cdf1 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -237,36 +237,35 @@ def test_string_from_stringtable(self): bytes = b'~\x00\x00\x00\x00\x11\x06' loaded = blue.marshal.Load(bytes) - # We expect a str type constructed from marshalled string table index - self.assertIsInstance(loaded, str) + self.assertTrue(type(loaded) == str) self.assertEqual(loaded, "ballID") def test_empty_unicode(self): bytes = b'~\x00\x00\x00\x00(' loaded = blue.marshal.Load(bytes) - self.assertIsInstance(loaded, unicode) + self.assertTrue(type(loaded) == unicode) self.assertEqual(loaded, "") def test_single_char_unicode(self): bytes = b'~\x00\x00\x00\x00.\x01A' loaded = blue.marshal.Load(bytes) - self.assertIsInstance(loaded, unicode) + self.assertTrue(type(loaded) == unicode) self.assertEqual(loaded, "A") def test_unicode(self): bytes = b'~\x00\x00\x00\x00.\t\xe2\x82\xa8\xe2\x82\xb1\xe2\x82\xa9' loaded = blue.marshal.Load(bytes) - self.assertIsInstance(loaded, unicode) + self.assertTrue(type(loaded) == unicode) self.assertEqual(loaded, u"\u20A8\u20B1\u20A9") def test_unicode_as_utf8(self): bytes = b'~\x00\x00\x00\x00.\x16this is a unicode test' loaded = blue.marshal.Load(bytes) - self.assertIsInstance(loaded, unicode) + self.assertTrue(type(loaded) == unicode) self.assertEqual(loaded, "this is a unicode test") def test_integer(self): @@ -301,8 +300,8 @@ def test_dict(self): self.assertEqual(loaded, {"key": "test"}) # Explicit type checking due to Unicode and str types being implicitly comparable for key, value in loaded.items(): - self.assertIsInstance(key, unicode) - self.assertIsInstance(value, unicode) + self.assertTrue(type(key) == unicode) + self.assertTrue(type(value) == unicode) def test_empty_object(self): bytes = b"~\x00\x00\x00\x00#%%\x02'bluetests.test.test_marshal.EmptyObject--" @@ -410,18 +409,5 @@ def test_dbrow(self): self.assertEqual(loaded, blue.DBRow(blue.DBRowDescriptor((("Test", 20),)), (123, ))) - def test_set(self): - blue.marshal.globalsWhitelist = {set: None} - blue.marshal.collectWhitelist = False - bytes = b'~\x00\x00\x00\x00",\x02\x0cbuiltins.set%\x15\x03\t\x06\x02\x06\x03--' - loaded = blue.marshal.Load(bytes) - self.assertSetEqual(loaded, {1, 2, 3}) - - def test_runtime_error(self): - blue.marshal.globalsWhitelist = {RuntimeError: None} - blue.marshal.collectWhitelist = False - bytes = b'~\x00\x00\x00\x00",\x02\x15builtins.RuntimeError%.\x05Boom!--' - loaded = blue.marshal.Load(bytes) - self.assertIsInstance(loaded, RuntimeError) - self.assertIsInstance(loaded.message, unicode) - self.assertEqual(loaded.message, u"Boom!") + def test_dbrow_with_invalid_descriptor_in_stream_raises_error(self): + pass From ef2d7fb2061991135382bc319f1dcc4f855bac28 Mon Sep 17 00:00:00 2001 From: ccp serpent <162123512+ccp-serpent@users.noreply.github.com> Date: Mon, 8 Sep 2025 10:53:42 +0000 Subject: [PATCH 102/131] Use `assertIsInstance` in place of `assertTrue` for type checking --- tests/python/bluetests/test/test_marshal.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 0f6cdf1..d1f087e 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -244,28 +244,28 @@ def test_empty_unicode(self): bytes = b'~\x00\x00\x00\x00(' loaded = blue.marshal.Load(bytes) - self.assertTrue(type(loaded) == unicode) + self.assertIsInstance(loaded, unicode) self.assertEqual(loaded, "") def test_single_char_unicode(self): bytes = b'~\x00\x00\x00\x00.\x01A' loaded = blue.marshal.Load(bytes) - self.assertTrue(type(loaded) == unicode) + self.assertIsInstance(loaded, unicode) self.assertEqual(loaded, "A") def test_unicode(self): bytes = b'~\x00\x00\x00\x00.\t\xe2\x82\xa8\xe2\x82\xb1\xe2\x82\xa9' loaded = blue.marshal.Load(bytes) - self.assertTrue(type(loaded) == unicode) + self.assertIsInstance(loaded, unicode) self.assertEqual(loaded, u"\u20A8\u20B1\u20A9") def test_unicode_as_utf8(self): bytes = b'~\x00\x00\x00\x00.\x16this is a unicode test' loaded = blue.marshal.Load(bytes) - self.assertTrue(type(loaded) == unicode) + self.assertIsInstance(loaded, unicode) self.assertEqual(loaded, "this is a unicode test") def test_integer(self): @@ -300,8 +300,8 @@ def test_dict(self): self.assertEqual(loaded, {"key": "test"}) # Explicit type checking due to Unicode and str types being implicitly comparable for key, value in loaded.items(): - self.assertTrue(type(key) == unicode) - self.assertTrue(type(value) == unicode) + self.assertIsInstance(key, unicode) + self.assertIsInstance(value, unicode) def test_empty_object(self): bytes = b"~\x00\x00\x00\x00#%%\x02'bluetests.test.test_marshal.EmptyObject--" From 5611f1b603ea62815f8ce10d9d230fefd25f35df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Tue, 9 Sep 2025 15:56:14 +0000 Subject: [PATCH 103/131] Change byte order in CarbonIO packet headers This is to be forward compatible with carbon-io, which uses network byte order for the packet headers. --- src/CarbonIO/CarbonIO.cpp | 21 ++-- src/CarbonIO/CarbonIO.h | 20 ++-- tests/python/bluetests/test/test_carbonio.py | 102 +++++++++++++++++++ 3 files changed, 123 insertions(+), 20 deletions(-) create mode 100644 tests/python/bluetests/test/test_carbonio.py diff --git a/src/CarbonIO/CarbonIO.cpp b/src/CarbonIO/CarbonIO.cpp index 30e6851..a324fbf 100644 --- a/src/CarbonIO/CarbonIO.cpp +++ b/src/CarbonIO/CarbonIO.cpp @@ -2778,7 +2778,7 @@ void CarbonIO::dataReceived( SCompletionUnit *completion, const char* data, cons } // a partial packet has been recieved, call for completion - packetSize = (ceHeaderSizeMask & *(int *)completion->packetListTail->data) + sizeof(int); + packetSize = ( ceHeaderSizeMask & ntohl( *reinterpret_cast( completion->packetListTail->data ) ) ) + sizeof(int); need = packetSize - completion->packetListTail->packetLen; D_HANDLEREAD(ciolog("working packet[%d:%d] need[%d] avail[%d] for [%d]", packetSize, completion->packetListTail->packetLen, need, available, (int)completion->workHandle )); @@ -2787,7 +2787,7 @@ void CarbonIO::dataReceived( SCompletionUnit *completion, const char* data, cons else { // packet is empty, call for the full amount - packetSize = (ceHeaderSizeMask & *(int *)indat) + sizeof(int); + packetSize = ( ceHeaderSizeMask & ntohl( *reinterpret_cast( indat ) ) ) + sizeof( uint32_t ); need = packetSize; D_HANDLEREAD(ciolog("not working packet, needing full amount[%d] for[%d] [0x%08X]:[0x%08X]", need, (int)completion->workHandle, *(unsigned int *)indat, ceHeaderSizeMask & *(int *)indat)); @@ -2811,11 +2811,11 @@ void CarbonIO::dataReceived( SCompletionUnit *completion, const char* data, cons available -= need; indat += need; - unsigned int header = *(int *)completion->packetListTail->data; + unsigned int header = ntohl( *reinterpret_cast( completion->packetListTail->data ) ); if ( header & ceHeaderExpectPayloadOffset ) { completion->packetListTail->oobData = sizeof(int)*2 + completion->packetListTail->data; - completion->packetListTail->oobLen = *(int *)(completion->packetListTail->data + sizeof(int)); + completion->packetListTail->oobLen = ntohl( *reinterpret_cast( completion->packetListTail->data + sizeof( uint32_t ) ) ); completion->packetListTail->payloadOffset = sizeof(int)*2 + completion->packetListTail->oobLen; if ( completion->packetListTail->payloadOffset > completion->packetListTail->packetLen ) @@ -3289,10 +3289,10 @@ bool CarbonIO::processRawQueuedData( SCompletionUnit *completion ) // find the offset to the body (the part we compress) unsigned int offset = sizeof(unsigned int); // the header itself - if ( *(unsigned int *)packet->data & ceHeaderExpectPayloadOffset ) // was there out-of-band data? + if ( ntohl( *reinterpret_cast( packet->data ) ) & ceHeaderExpectPayloadOffset ) // was there out-of-band data? { offset += sizeof(unsigned int); // the size of the len param - offset += *(unsigned int *)(packet->data + sizeof(unsigned int)); // the size of the data + offset += *reinterpret_cast( packet->data + sizeof( uint32_t ) ); // the size of the data } char *outbuf = 0; @@ -3305,9 +3305,9 @@ bool CarbonIO::processRawQueuedData( SCompletionUnit *completion ) { completion->stats.bytesSentCompressed += outlen + offset; - *(unsigned int *)packet->data &= ceHeaderBitsMask; // knock off old size - *(unsigned int *)packet->data |= (outlen + offset) - sizeof(unsigned int); // plug in NEW size - *(unsigned int *)packet->data |= m_compressionType; + *reinterpret_cast( packet->data ) &= htonl( ceHeaderBitsMask ); // knock off old size + *reinterpret_cast( packet->data ) |= htonl( ( outlen + offset ) - sizeof( uint32_t ) ); // plug in NEW size + *reinterpret_cast( packet->data ) |= htonl( m_compressionType ); // if this is not the root packet, append the data TO the // root packet @@ -3936,7 +3936,8 @@ bool CarbonIO::compress( const char *in, unsigned int inLen, char **out, unsigne //------------------------------------------------------------------------------ bool CarbonIO::decompress( SPacket* packet ) { - if ( *(unsigned int *)packet->data & ceHeaderBitZlibCompressed ) + unsigned long packetHeader = ntohl( *reinterpret_cast( packet->data ) ); + if ( packetHeader & ceHeaderBitZlibCompressed ) { CPerformanceTime timeCompress( "decompress" ); diff --git a/src/CarbonIO/CarbonIO.h b/src/CarbonIO/CarbonIO.h index 4a667dc..0538f07 100644 --- a/src/CarbonIO/CarbonIO.h +++ b/src/CarbonIO/CarbonIO.h @@ -558,15 +558,15 @@ bool CarbonIO::formatPacket( char* buf, int pos; if ( OOBData && OOBLen ) { - *(unsigned int *)buf = (dataLen + OOBLen + sizeof(unsigned int)) | ceHeaderExpectPayloadOffset; - *(unsigned int *)(buf + sizeof(unsigned int)) = OOBLen; - memcpy( buf + sizeof(unsigned int)*2, OOBData, OOBLen ); - pos = OOBLen + sizeof(unsigned int)*2; + *reinterpret_cast( buf ) = htonl( (dataLen + OOBLen + sizeof( uint32_t ) ) | ceHeaderExpectPayloadOffset ); + *reinterpret_cast( buf + sizeof( uint32_t ) ) = htonl( OOBLen ); + memcpy( buf + sizeof( uint32_t )*2, OOBData, OOBLen ); + pos = OOBLen + sizeof( uint32_t )*2; } else { - *(unsigned int *)buf = dataLen; - pos = sizeof(unsigned int); + *reinterpret_cast( buf ) = htonl( dataLen ); + pos = sizeof( uint32_t ); } char *outbuf = 0; @@ -576,9 +576,9 @@ bool CarbonIO::formatPacket( char* buf, && checkAndCompress(data, dataLen, &outbuf, &outlen) ) { *len = pos + outlen; - *(unsigned int *)buf &= ceHeaderBitsMask; // knock off old size - *(unsigned int *)buf |= *len - sizeof(unsigned int); // plug in NEW size - *(unsigned int *)buf |= m_compressionType; + *reinterpret_cast( buf ) &= htonl( ceHeaderBitsMask ); // knock off old size + *reinterpret_cast( buf ) |= htonl( *len - sizeof( uint32_t ) ); // plug in NEW size + *reinterpret_cast( buf ) |= htonl( m_compressionType ); memcpy( buf + pos, outbuf, outlen ); delete[] outbuf; } @@ -816,7 +816,7 @@ bool CarbonIO::isPacketValid( SPacket *packet ) return packet && packet->packetLen >= 4 && (packet->auxData - || (packet->packetLen == (*(int *)packet->data & ceHeaderSizeMask) + sizeof(int))); + || ( packet->packetLen == ( ntohl( *reinterpret_cast( packet->data ) ) & ceHeaderSizeMask ) + sizeof( int ) ) ); } //------------------------------------------------------------------------------ diff --git a/tests/python/bluetests/test/test_carbonio.py b/tests/python/bluetests/test/test_carbonio.py new file mode 100644 index 0000000..2c1640f --- /dev/null +++ b/tests/python/bluetests/test/test_carbonio.py @@ -0,0 +1,102 @@ +import functools +import socket +import sys +import unittest + +import blue +import stackless + +LOCALHOST = "127.0.0.1" + +def find_port(): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.bind((LOCALHOST, 0)) + port = s.getsockname()[1] + return port + finally: + s.close() + +def run_on_server(host_address, function): + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + server.setmaxpacketsize(10 * 1024 * 1024) + server.bind(host_address) + server.listen(0) + connection, address = server.accept() + function(connection) + finally: + server.close() + +def run_on_client(host_address, function): + client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + client.connect(host_address) + client.setmaxpacketsize(10 * 1024 * 1024) + client.setblockingsend(True) + function(client) + finally: + client.close() + +def send_packet(packet, oob_data, connection): + connection.sendpacket(packet, oob_data) + +def receive_packet(connection): + receive_packet.received = connection.recvpacketoob() +receive_packet.received = None + +def send(data, connection): + connection.send(data) + +def receive(connection): + receive.received = connection.recv(65535) +receive.received = None + +def run_connected(client_func, server_func): + port = find_port() + address = (LOCALHOST, port) + c = stackless.tasklet(run_on_client)(address, client_func) + s = stackless.tasklet(run_on_server)(address, server_func) + while c.alive or s.alive: + blue.os.Pump() + +class TestCarbonIO(unittest.TestCase): + + def setUp(self): + import carbonio + import _slsocket + _slsocket.use_carbonio(True) + carbonio._socket = _slsocket + sys.modules["_socket"] = _slsocket + sys.modules["select"] = None + reload(socket) # Make sure the socket module is using _slsocket instead of _socket + + def test_sendpacket(self): + PACKET_DATA = "data" + OOB_DATA = None + run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), receive_packet) + self.assertEqual(receive_packet.received, (PACKET_DATA, "", 1)) + + def test_sent_packet_format(self): + PACKET_DATA = "data" + OOB_DATA = None + run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), receive) + self.assertEqual(receive.received, "\x00\x00\x00\x04data") + + + def test_receive_formatted_packet(self): + PACKET_DATA = "\x00\x00\x00\x04data" + run_connected(functools.partial(send, PACKET_DATA), receive_packet) + self.assertEqual(receive_packet.received, ("data", "", 1)) + + def test_sent_packet_format_with_oob_data(self): + PACKET_DATA = "Hello" + OOB_DATA = "World" + run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), receive) + self.assertEqual(receive.received, "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello") + + def test_receive_formatted_packet_with_oob_data(self): + PACKET_DATA = "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello" + run_connected(functools.partial(send, PACKET_DATA), receive_packet) + self.assertEqual(receive_packet.received, ("Hello", "World", 1)) From eac7df6612113b82d954bcbbc9edcac620c1fe21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Wed, 10 Sep 2025 14:13:51 +0000 Subject: [PATCH 104/131] Change byte order in StacklessIO packet headers on Windows This is to be forward compatible with carbon-io, which uses network byte order for the packet headers. Further changes are required to effect this change on macOS. --- src/slsocket.cpp | 16 +++++---- .../{test_carbonio.py => test_slsocket.py} | 34 ++++++++++++++++++- 2 files changed, 42 insertions(+), 8 deletions(-) rename tests/python/bluetests/test/{test_carbonio.py => test_slsocket.py} (72%) diff --git a/src/slsocket.cpp b/src/slsocket.cpp index 228641e..2adeb73 100644 --- a/src/slsocket.cpp +++ b/src/slsocket.cpp @@ -1150,14 +1150,15 @@ class SendPacketResult : public SendResult buffers.Push(header); SSIZE_T result = header.len; result += SendResult::Prepare(buffers, obj, pbuf); - mHeader = (DWORD)(result-sizeof(mHeader)); //number of cargo bytes - if (mHeader > (DWORD)mXtra->GetMaxPacketSize()) { + auto cargoBytes = static_cast( result - sizeof(mHeader) ); //number of cargo bytes + if ( cargoBytes > static_cast( mXtra->GetMaxPacketSize() ) ) { char tmp[128]; sprintf_s(tmp, "packet too long at %d bytes, max size is %d", - mHeader, mXtra->GetMaxPacketSize()); + cargoBytes, mXtra->GetMaxPacketSize()); OutputDebugString(tmp); throw std::length_error(tmp); } + mHeader = htonl( cargoBytes ); return result; } @@ -1276,30 +1277,31 @@ class RecvPacketResult : public IOOverlappedRecv { mBytesRead += bytesTransfered; bool more; + uint32_t cargoBytes{ntohl( mHeader )}; if (mBytesRead < sizeof(mHeader)) { //continue reading header more = true; } else if (mBytesRead == sizeof(mHeader)) { //just finished reading header, allocate buffer. - if (mHeader > (DWORD)mXtra->GetMaxPacketSize()) { + if ( cargoBytes > static_cast( mXtra->GetMaxPacketSize() ) ) { char tmp[128]; sprintf_s(tmp, "too large a packet detected at %d bytes, max is %d", mHeader, mXtra->GetMaxPacketSize()); throw std::length_error(tmp); } more = mHeader!=0; - } else if (mBytesRead < (int)(sizeof(mHeader)+mHeader)) { + } else if (mBytesRead < static_cast( sizeof( mHeader ) + cargoBytes ) ) { //header here but packet not finished more = true; } else { //packet finished. //did we read any of the next guy's header? (we requested next header's amount of data :) - DWORD rest = mBytesRead - (sizeof(mHeader) + mHeader); + uint32_t rest = mBytesRead - ( sizeof( mHeader ) + cargoBytes ); if (rest) { RecvPacketResult *next = mXtra->NextRecv(this); _ASSERT(next); next->OnHeaderBytesRead(rest); - mBytesRead = (int)(sizeof(mHeader)+mHeader); + mBytesRead = static_cast( sizeof( mHeader ) + cargoBytes ); } mXtra->mStats.PacketReceived(); more = false; diff --git a/tests/python/bluetests/test/test_carbonio.py b/tests/python/bluetests/test/test_slsocket.py similarity index 72% rename from tests/python/bluetests/test/test_carbonio.py rename to tests/python/bluetests/test/test_slsocket.py index 2c1640f..6d96bd0 100644 --- a/tests/python/bluetests/test/test_carbonio.py +++ b/tests/python/bluetests/test/test_slsocket.py @@ -40,7 +40,10 @@ def run_on_client(host_address, function): client.close() def send_packet(packet, oob_data, connection): - connection.sendpacket(packet, oob_data) + if oob_data is not None: + connection.sendpacket(packet, oob_data) + else: + connection.sendpacket(packet) def receive_packet(connection): receive_packet.received = connection.recvpacketoob() @@ -100,3 +103,32 @@ def test_receive_formatted_packet_with_oob_data(self): PACKET_DATA = "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello" run_connected(functools.partial(send, PACKET_DATA), receive_packet) self.assertEqual(receive_packet.received, ("Hello", "World", 1)) + +class TestStacklessIO(unittest.TestCase): + + def setUp(self): + import stacklessio + import _slsocket + import slselect + _slsocket.use_carbonio(False) + stacklessio._socket = _slsocket + sys.modules["_socket"] = _slsocket + sys.modules["select"] = slselect + reload(socket) # Make sure the socket module is using _slsocket instead of _socket + + def test_sendpacket(self): + PACKET_DATA = "data" + OOB_DATA = None + run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), receive_packet) + self.assertEqual(receive_packet.received, (PACKET_DATA, None, 0)) + + def test_sent_packet_format(self): + PACKET_DATA = "data" + OOB_DATA = None + run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), receive) + self.assertEqual(receive.received, "\x00\x00\x00\x04data") + + def test_receive_formatted_packet(self): + PACKET_DATA = "\x00\x00\x00\x04data" + run_connected(functools.partial(send, PACKET_DATA), receive_packet) + self.assertEqual(receive_packet.received, ("data", None, 0)) From 4b8af5e86df8b1ef27e8a71a4c93c47e68acb221 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Thu, 11 Sep 2025 16:17:08 +0000 Subject: [PATCH 105/131] Get unit tests running through CTest --- CMakeLists.txt | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c6f0ede..4027ef5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ find_package(BlueExposure REQUIRED NO_CMAKE_PATH) find_package(CURL REQUIRED NO_CMAKE_PATH) find_package(OpenSSL REQUIRED NO_CMAKE_PATH) find_package(PDM REQUIRED NO_CMAKE_PATH) +find_package(Python REQUIRED NO_CMAKE_PATH) find_package(YAML REQUIRED NO_CMAKE_PATH) find_package(ZLIB REQUIRED NO_CMAKE_PATH) @@ -371,7 +372,27 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) else() message(FATAL_ERROR "Unsupported platform: " ${CMAKE_SYSTEM_NAME}) endif() - execute_process(COMMAND ${PYTHON_EXECUTABLE} discover.py + if(WIN32) + set(EXE_EXTENSION ".exe") + set(SCRIPT_EXTENSION ".bat") + else() + set(SCRIPT_EXTENSION ".sh") + endif() + set(PYTHON_INTERPRETER_PATH "${BRANCH_ROOT_DIR}/eve/client/pythonInterpreter${SCRIPT_EXTENSION}") + get_target_property(LIBPYTHON Python IMPORTED_LOCATION) + if(DEFINED CMAKE_BUILD_TYPE AND NOT CMAKE_BUILD_TYPE MATCHES "Release") + set(EXEFILE_NAME exefile_${CMAKE_BUILD_TYPE}${EXE_EXTENSION}) + else() + set(EXEFILE_NAME exefile${EXE_EXTENSION}) + endif() + set(EXEFILE_PATH $/${EXEFILE_NAME}) + add_custom_command( + TARGET Blue POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy -t $ + ${Exefile_ROOT}/${CCP_VENDOR_BIN_PATH}/${EXEFILE_NAME} + ${LIBPYTHON} + ) + execute_process(COMMAND ${PYTHON_INTERPRETER_PATH} discover.py OUTPUT_VARIABLE PYTHON_TESTS_STR OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE @@ -386,13 +407,13 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) foreach(PYTHON_TEST ${PYTHON_TESTS}) message(STATUS "Found test ${PYTHON_TEST}") add_test(NAME ${PYTHON_TEST} - COMMAND ${PYTHON_EXECUTABLE} -m unittest -v ${PYTHON_TEST} + COMMAND ${EXEFILE_PATH} /inherit /buildflavor=$> /py -m unittest -v ${PYTHON_TEST} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests/python ) if(WIN32) set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$\;${BRANCH_ROOT_DIR}/carbon/common/stdlib\;${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}\;${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$<$:PY3_COMPATIBILITY_MODE=True>") elseif(APPLE) - set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$:${BRANCH_ROOT_DIR}/carbon/common/stdlib:${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}:${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$<$:PY3_COMPATIBILITY_MODsE=True") + set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$:${BRANCH_ROOT_DIR}/carbon/common/stdlib:${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_LIB_PATH}:${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$<$:PY3_COMPATIBILITY_MODE=True") else() message(FATAL_ERROR "Unsupported platform") endif() From 76fb41b01383268a238578245ad70b4095e3bf9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Thu, 11 Sep 2025 16:19:40 +0000 Subject: [PATCH 106/131] Implement rebind for PyAllocator The absence of rebind was causing a static assertion to fail when building on macOS with the latest SDK. Rebinding an allocator to the same type did not result in the original allocator. --- include/BluePyCpp.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/include/BluePyCpp.h b/include/BluePyCpp.h index a5c1f11..eafde50 100644 --- a/include/BluePyCpp.h +++ b/include/BluePyCpp.h @@ -157,7 +157,12 @@ class PyAllocator : public std::allocator typename _p::pointer allocate(typename _p::size_type _Count, const void *) { return allocate(_Count); } -}; + + template + struct rebind { + using other = PyAllocator; + }; +}; // A simple smartpointer to own references. From b16e1d4d136de8a4542a66f09099615cf6ff3f1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Thu, 11 Sep 2025 16:22:30 +0000 Subject: [PATCH 107/131] Fix failing unit test on macOS This was failing because the integer got converted to a long during the round trip. Mostly, this should be fine, and this will eventually get resolved as part of the Python 3 migration, so I'm updating the test to start with a long to begin with. --- tests/python/bluetests/test/test_marshal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index d1f087e..47d5b25 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -122,7 +122,7 @@ def test_integer(self): def test_long(self): self.verify_round_trip(42) - self.verify_round_trip(9223372036854775807) + self.verify_round_trip(9223372036854775807L) def test_float(self): self.verify_round_trip(0.0) From 1fc777f51ca44df920bdf7222bc597bea13d24d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Thu, 11 Sep 2025 16:24:49 +0000 Subject: [PATCH 108/131] Fix up slsocket unit tests to run on macOS carbonio simply does not exist on macOS, so skip any related tests. Fix up StacklessIO test that expected all the data for a packet to get sent at once, but the header got received separately, ahead of the body. --- tests/python/bluetests/test/test_slsocket.py | 44 ++++++++++++++------ 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/tests/python/bluetests/test/test_slsocket.py b/tests/python/bluetests/test/test_slsocket.py index 6d96bd0..3c6c16d 100644 --- a/tests/python/bluetests/test/test_slsocket.py +++ b/tests/python/bluetests/test/test_slsocket.py @@ -10,6 +10,7 @@ def find_port(): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: s.bind((LOCALHOST, 0)) port = s.getsockname()[1] @@ -17,21 +18,23 @@ def find_port(): finally: s.close() -def run_on_server(host_address, function): +def run_on_server(channel, function): server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: server.setmaxpacketsize(10 * 1024 * 1024) - server.bind(host_address) + server.bind(("127.0.0.1", 0)) + channel.send(server.getsockname()) server.listen(0) connection, address = server.accept() function(connection) finally: server.close() -def run_on_client(host_address, function): +def run_on_client(channel, function): client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: + host_address = channel.receive() client.connect(host_address) client.setmaxpacketsize(10 * 1024 * 1024) client.setblockingsend(True) @@ -56,18 +59,32 @@ def receive(connection): receive.received = connection.recv(65535) receive.received = None +def receive_bytes(byte_count, connection): + receive_bytes.received = "" + while len(receive_bytes.received) < byte_count: + data = connection.recv(byte_count - len(receive_bytes.received)) + receive_bytes.received += data +receive_bytes.received = None + def run_connected(client_func, server_func): - port = find_port() - address = (LOCALHOST, port) - c = stackless.tasklet(run_on_client)(address, client_func) - s = stackless.tasklet(run_on_server)(address, server_func) - while c.alive or s.alive: - blue.os.Pump() + channel = stackless.channel() + c = stackless.tasklet(run_on_client)(channel, client_func) + s = stackless.tasklet(run_on_server)(channel, server_func) + sleeptime = blue.os.sleeptime + try: + blue.os.sleeptime = 100 + while c.alive or s.alive: + blue.os.Pump() + finally: + blue.os.sleeptime = sleeptime class TestCarbonIO(unittest.TestCase): def setUp(self): - import carbonio + try: + import carbonio + except ImportError: + raise unittest.SkipTest("CarbonIO not supported on this system") import _slsocket _slsocket.use_carbonio(True) carbonio._socket = _slsocket @@ -110,7 +127,8 @@ def setUp(self): import stacklessio import _slsocket import slselect - _slsocket.use_carbonio(False) + if hasattr(_slsocket, "use_carbonio"): + _slsocket.use_carbonio(False) stacklessio._socket = _slsocket sys.modules["_socket"] = _slsocket sys.modules["select"] = slselect @@ -125,8 +143,8 @@ def test_sendpacket(self): def test_sent_packet_format(self): PACKET_DATA = "data" OOB_DATA = None - run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), receive) - self.assertEqual(receive.received, "\x00\x00\x00\x04data") + run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), functools.partial(receive_bytes, 8)) + self.assertEqual(receive_bytes.received, "\x00\x00\x00\x04data") def test_receive_formatted_packet(self): PACKET_DATA = "\x00\x00\x00\x04data" From 89aa182365782ef849ec111f150cf1a6ef840197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Thu, 11 Sep 2025 16:26:52 +0000 Subject: [PATCH 109/131] Use network byte order for macho packet headers on macOS This should make StacklessIO forward compatible with carbon-io. --- src/slsocket_posix.cpp | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/slsocket_posix.cpp b/src/slsocket_posix.cpp index 8cb4719..7872073 100644 --- a/src/slsocket_posix.cpp +++ b/src/slsocket_posix.cpp @@ -1601,19 +1601,6 @@ class SendAllResult : public SendResult } }; -static void FlipHeader(uint32_t &header) -{ - if (htons(1) == 1) { - // Big endian machine. Our protocol uses little endian notation, so we must cheat - uint32_t h = header; - uint32_t tmp = (h & 0xff ) << 24; - tmp |= ((h & 0xff00) << 8); - tmp |= ((h & 0xff0000) >> 8); - tmp |= ((h & 0xff000000) >> 24); - header = tmp; - } -} - class SendPacketResult : public SendBase { public: @@ -1626,7 +1613,7 @@ class SendPacketResult : public SendBase // steal the buffer StealBuffer(buf); mHeader = mKeeper.len; - FlipHeader(mHeader); + mHeader = htonl(mHeader); Request(s->sock_timeout); return mResult; } @@ -1908,7 +1895,7 @@ class RecvPacketResult : public SocketResult rcvd = recv(handle, (char*)&mHeader + mBytesRead, sizeof(mHeader) - mBytesRead, 0); if (rcvd > 0 && mBytesRead+rcvd == 4) { // We completed reading the header, do stuff! - FlipHeader(mHeader); + mHeader = htonl(mHeader); mPacketSize = (mHeader & ceHeaderSizeMask); if (mPacketSize > (uint32_t)GetXtra()->GetMaxPacketSize()) { char tmp[128] = {'\0'}; From 800796c16c4168547507f378a2bc06e7fad301ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Thu, 11 Sep 2025 17:29:34 +0000 Subject: [PATCH 110/131] Fix OOB Data handling for StacklessIO on macOS OOB Data only gets handled in the macOS build, not in the Windows one. As far as I can tell, the expected behavior is for the OOB data header and the OOB data to leak into the beginning of the packet data if the packet does not get eaten by BlueNet. I guess the expectation is that all packets containing OOB data should get eaten. --- src/slsocket_posix.cpp | 2 +- tests/python/bluetests/test/test_slsocket.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/slsocket_posix.cpp b/src/slsocket_posix.cpp index 7872073..5c5731b 100644 --- a/src/slsocket_posix.cpp +++ b/src/slsocket_posix.cpp @@ -1821,7 +1821,7 @@ class RecvPacketResult : public SocketResult if (mBytesRead < (sizeof(mHeader) + sizeof(uint32_t))) return; - uint32_t oobDataLen = *(uint32_t *)(mData); + uint32_t oobDataLen = ntohl(*(uint32_t *)(mData)); // sanity check the out-of-band data length; mPacketSize was sanity checked already if (oobDataLen > mPacketSize) { char tmp[128] = {'\0'}; diff --git a/tests/python/bluetests/test/test_slsocket.py b/tests/python/bluetests/test/test_slsocket.py index 3c6c16d..e359f36 100644 --- a/tests/python/bluetests/test/test_slsocket.py +++ b/tests/python/bluetests/test/test_slsocket.py @@ -150,3 +150,10 @@ def test_receive_formatted_packet(self): PACKET_DATA = "\x00\x00\x00\x04data" run_connected(functools.partial(send, PACKET_DATA), receive_packet) self.assertEqual(receive_packet.received, ("data", None, 0)) + + def test_receive_formatted_packet_with_oob_data(self): + if sys.platform == "win32": + raise unittest.SkipTest("StacklessIO does not support OOB data on Windows") + PACKET_DATA = "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello" + run_connected(functools.partial(send, PACKET_DATA), receive_packet) + self.assertEqual(receive_packet.received, ("\x00\x00\x00\x05WorldHello", None, 0)) From a9b97e26b10ac550ce24cc1cd9fb35a51bc8ad49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Fri, 12 Sep 2025 16:18:37 +0000 Subject: [PATCH 111/131] Prevent noise when starting up Python interpreter from causing invalid tests Since the string "test" is used for pattern matching by the unittest module when finding test cases, it is guaranteed to be a part of the path for any valid Python test. Filtering out strings that do not contain this seems to be sufficient to weed out the noise when starting up the interpreter batch file. --- CMakeLists.txt | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4027ef5..dbc4031 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -405,17 +405,20 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) endif() separate_arguments(PYTHON_TESTS UNIX_COMMAND ${PYTHON_TESTS_STR}) foreach(PYTHON_TEST ${PYTHON_TESTS}) - message(STATUS "Found test ${PYTHON_TEST}") - add_test(NAME ${PYTHON_TEST} - COMMAND ${EXEFILE_PATH} /inherit /buildflavor=$> /py -m unittest -v ${PYTHON_TEST} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests/python - ) - if(WIN32) - set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$\;${BRANCH_ROOT_DIR}/carbon/common/stdlib\;${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}\;${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$<$:PY3_COMPATIBILITY_MODE=True>") - elseif(APPLE) - set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$:${BRANCH_ROOT_DIR}/carbon/common/stdlib:${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_LIB_PATH}:${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$<$:PY3_COMPATIBILITY_MODE=True") - else() - message(FATAL_ERROR "Unsupported platform") + string(FIND ${PYTHON_TEST} "test" TEST_INDEX) + if( TEST_INDEX GREATER -1 ) # The line contains the string "test" + message(STATUS "Found test ${PYTHON_TEST}") + add_test(NAME ${PYTHON_TEST} + COMMAND ${EXEFILE_PATH} /inherit /buildflavor=$> /py -m unittest -v ${PYTHON_TEST} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests/python + ) + if(WIN32) + set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$\;${BRANCH_ROOT_DIR}/carbon/common/stdlib\;${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}\;${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$<$:PY3_COMPATIBILITY_MODE=True>") + elseif(APPLE) + set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$:${BRANCH_ROOT_DIR}/carbon/common/stdlib:${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_LIB_PATH}:${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$<$:PY3_COMPATIBILITY_MODE=True") + else() + message(FATAL_ERROR "Unsupported platform") + endif() endif() endforeach() From 128801f139879552268dbf6bdcd5f13750e3f4be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Fri, 12 Sep 2025 17:23:33 +0000 Subject: [PATCH 112/131] Make Python 3 compatibility mode definition control packet byte order We don't want to break network compatibility between branches until we have to. --- src/CarbonIO/CarbonIO.cpp | 8 ++++ src/CarbonIO/CarbonIO.h | 21 +++++++++ src/slsocket.cpp | 15 ++++++ src/slsocket_posix.cpp | 23 +++++++++ tests/python/bluetests/test/test_slsocket.py | 49 +++++++++++++++----- 5 files changed, 104 insertions(+), 12 deletions(-) diff --git a/src/CarbonIO/CarbonIO.cpp b/src/CarbonIO/CarbonIO.cpp index a324fbf..d0af2d7 100644 --- a/src/CarbonIO/CarbonIO.cpp +++ b/src/CarbonIO/CarbonIO.cpp @@ -2778,7 +2778,11 @@ void CarbonIO::dataReceived( SCompletionUnit *completion, const char* data, cons } // a partial packet has been recieved, call for completion +#ifdef PY3_COMPATIBILITY_MODE packetSize = ( ceHeaderSizeMask & ntohl( *reinterpret_cast( completion->packetListTail->data ) ) ) + sizeof(int); +#else + packetSize = (ceHeaderSizeMask & *(int *)completion->packetListTail->data) + sizeof(int); +#endif need = packetSize - completion->packetListTail->packetLen; D_HANDLEREAD(ciolog("working packet[%d:%d] need[%d] avail[%d] for [%d]", packetSize, completion->packetListTail->packetLen, need, available, (int)completion->workHandle )); @@ -2787,7 +2791,11 @@ void CarbonIO::dataReceived( SCompletionUnit *completion, const char* data, cons else { // packet is empty, call for the full amount +#ifdef PY3_COMPATIBILITY_MODE packetSize = ( ceHeaderSizeMask & ntohl( *reinterpret_cast( indat ) ) ) + sizeof( uint32_t ); +#else + packetSize = (ceHeaderSizeMask & *(int *)indat) + sizeof(int); +#endif need = packetSize; D_HANDLEREAD(ciolog("not working packet, needing full amount[%d] for[%d] [0x%08X]:[0x%08X]", need, (int)completion->workHandle, *(unsigned int *)indat, ceHeaderSizeMask & *(int *)indat)); diff --git a/src/CarbonIO/CarbonIO.h b/src/CarbonIO/CarbonIO.h index 0538f07..68683b6 100644 --- a/src/CarbonIO/CarbonIO.h +++ b/src/CarbonIO/CarbonIO.h @@ -556,6 +556,7 @@ bool CarbonIO::formatPacket( char* buf, } int pos; +#ifdef PY3_COMPATIBILITY_MODE if ( OOBData && OOBLen ) { *reinterpret_cast( buf ) = htonl( (dataLen + OOBLen + sizeof( uint32_t ) ) | ceHeaderExpectPayloadOffset ); @@ -568,6 +569,20 @@ bool CarbonIO::formatPacket( char* buf, *reinterpret_cast( buf ) = htonl( dataLen ); pos = sizeof( uint32_t ); } +#else + if ( OOBData && OOBLen ) + { + *(unsigned int *)buf = (dataLen + OOBLen + sizeof(unsigned int)) | ceHeaderExpectPayloadOffset; + *(unsigned int *)(buf + sizeof(unsigned int)) = OOBLen; + memcpy( buf + sizeof(unsigned int)*2, OOBData, OOBLen ); + pos = OOBLen + sizeof(unsigned int)*2; + } + else + { + *(unsigned int *)buf = dataLen; + pos = sizeof(unsigned int); + } +#endif char *outbuf = 0; unsigned int outlen; @@ -576,9 +591,15 @@ bool CarbonIO::formatPacket( char* buf, && checkAndCompress(data, dataLen, &outbuf, &outlen) ) { *len = pos + outlen; +#ifdef PY3_COMPATIBILITY_MODE *reinterpret_cast( buf ) &= htonl( ceHeaderBitsMask ); // knock off old size *reinterpret_cast( buf ) |= htonl( *len - sizeof( uint32_t ) ); // plug in NEW size *reinterpret_cast( buf ) |= htonl( m_compressionType ); +#else + *(unsigned int *)buf &= ceHeaderBitsMask; // knock off old size + *(unsigned int *)buf |= *len - sizeof(unsigned int); // plug in NEW size + *(unsigned int *)buf |= m_compressionType; +#endif memcpy( buf + pos, outbuf, outlen ); delete[] outbuf; } diff --git a/src/slsocket.cpp b/src/slsocket.cpp index 2adeb73..1fa5027 100644 --- a/src/slsocket.cpp +++ b/src/slsocket.cpp @@ -1150,6 +1150,7 @@ class SendPacketResult : public SendResult buffers.Push(header); SSIZE_T result = header.len; result += SendResult::Prepare(buffers, obj, pbuf); +#ifdef PY3_COMPATIBILITY_MODE auto cargoBytes = static_cast( result - sizeof(mHeader) ); //number of cargo bytes if ( cargoBytes > static_cast( mXtra->GetMaxPacketSize() ) ) { char tmp[128]; @@ -1159,6 +1160,16 @@ class SendPacketResult : public SendResult throw std::length_error(tmp); } mHeader = htonl( cargoBytes ); +#else + mHeader = (DWORD)(result-sizeof(mHeader)); //number of cargo bytes + if (mHeader > (DWORD)mXtra->GetMaxPacketSize()) { + char tmp[128]; + sprintf_s(tmp, "packet too long at %d bytes, max size is %d", + mHeader, mXtra->GetMaxPacketSize()); + OutputDebugString(tmp); + throw std::length_error(tmp); + } +#endif return result; } @@ -1277,7 +1288,11 @@ class RecvPacketResult : public IOOverlappedRecv { mBytesRead += bytesTransfered; bool more; +#ifdef PY3_COMPATIBILITY_MODE uint32_t cargoBytes{ntohl( mHeader )}; +#else + uint32_t cargoBytes{mHeader}; +#endif if (mBytesRead < sizeof(mHeader)) { //continue reading header more = true; diff --git a/src/slsocket_posix.cpp b/src/slsocket_posix.cpp index 5c5731b..896c54f 100644 --- a/src/slsocket_posix.cpp +++ b/src/slsocket_posix.cpp @@ -1601,6 +1601,19 @@ class SendAllResult : public SendResult } }; +static void FlipHeader(uint32_t &header) +{ + if (htons(1) == 1) { + // Big endian machine. Our protocol uses little endian notation, so we must cheat + uint32_t h = header; + uint32_t tmp = (h & 0xff ) << 24; + tmp |= ((h & 0xff00) << 8); + tmp |= ((h & 0xff0000) >> 8); + tmp |= ((h & 0xff000000) >> 24); + header = tmp; + } +} + class SendPacketResult : public SendBase { public: @@ -1613,7 +1626,10 @@ class SendPacketResult : public SendBase // steal the buffer StealBuffer(buf); mHeader = mKeeper.len; +#ifdef PY3_COMPATIBILITY_MODE mHeader = htonl(mHeader); +#endif + FlipHeader(mHeader); Request(s->sock_timeout); return mResult; } @@ -1821,7 +1837,11 @@ class RecvPacketResult : public SocketResult if (mBytesRead < (sizeof(mHeader) + sizeof(uint32_t))) return; +#ifdef PY3_COMPATIBILITY_MODE uint32_t oobDataLen = ntohl(*(uint32_t *)(mData)); +#else + uint32_t oobDataLen = *(uint32_t *)(mData); +#endif // sanity check the out-of-band data length; mPacketSize was sanity checked already if (oobDataLen > mPacketSize) { char tmp[128] = {'\0'}; @@ -1895,7 +1915,10 @@ class RecvPacketResult : public SocketResult rcvd = recv(handle, (char*)&mHeader + mBytesRead, sizeof(mHeader) - mBytesRead, 0); if (rcvd > 0 && mBytesRead+rcvd == 4) { // We completed reading the header, do stuff! +#ifdef PY3_COMPATIBILITY_MODE mHeader = htonl(mHeader); +#endif + FlipHeader(mHeader); mPacketSize = (mHeader & ceHeaderSizeMask); if (mPacketSize > (uint32_t)GetXtra()->GetMaxPacketSize()) { char tmp[128] = {'\0'}; diff --git a/tests/python/bluetests/test/test_slsocket.py b/tests/python/bluetests/test/test_slsocket.py index e359f36..d7d9f2c 100644 --- a/tests/python/bluetests/test/test_slsocket.py +++ b/tests/python/bluetests/test/test_slsocket.py @@ -1,4 +1,5 @@ import functools +import os import socket import sys import unittest @@ -91,8 +92,8 @@ def setUp(self): sys.modules["_socket"] = _slsocket sys.modules["select"] = None reload(socket) # Make sure the socket module is using _slsocket instead of _socket - - def test_sendpacket(self): + self.expectNetworkByteOrder = os.environ.has_key("PY3_COMPATIBILITY_MODE") + def test_send_receive_packet(self): PACKET_DATA = "data" OOB_DATA = None run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), receive_packet) @@ -102,11 +103,16 @@ def test_sent_packet_format(self): PACKET_DATA = "data" OOB_DATA = None run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), receive) - self.assertEqual(receive.received, "\x00\x00\x00\x04data") - + if self.expectNetworkByteOrder: + self.assertEqual(receive_bytes.received, "\x00\x00\x00\x04data") + else: + self.assertEqual(receive_bytes.received, "\x04\x00\x00\x00data") def test_receive_formatted_packet(self): - PACKET_DATA = "\x00\x00\x00\x04data" + if self.expectNetworkByteOrder: + PACKET_DATA = "\x00\x00\x00\x04data" + else: + PACKET_DATA = "\x04\x00\x00\x00data" run_connected(functools.partial(send, PACKET_DATA), receive_packet) self.assertEqual(receive_packet.received, ("data", "", 1)) @@ -114,10 +120,16 @@ def test_sent_packet_format_with_oob_data(self): PACKET_DATA = "Hello" OOB_DATA = "World" run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), receive) - self.assertEqual(receive.received, "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello") + if self.expectNetworkByteOrder: + self.assertEqual(receive.received, "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello") + else: + self.assertEqual(receive.received, "\x0E\x00\x00\x10\x05\x00\x00\x00HelloWorld") def test_receive_formatted_packet_with_oob_data(self): - PACKET_DATA = "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello" + if self.expectNetworkByteOrder: + PACKET_DATA = "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello" + else: + PACKET_DATA = "\x0E\x00\x00\x10\x05\x00\x00\x00HelloWorld" run_connected(functools.partial(send, PACKET_DATA), receive_packet) self.assertEqual(receive_packet.received, ("Hello", "World", 1)) @@ -133,8 +145,9 @@ def setUp(self): sys.modules["_socket"] = _slsocket sys.modules["select"] = slselect reload(socket) # Make sure the socket module is using _slsocket instead of _socket + self.expectNetworkByteOrder = os.environ.has_key("PY3_COMPATIBILITY_MODE") - def test_sendpacket(self): + def test_send_receive_packet(self): PACKET_DATA = "data" OOB_DATA = None run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), receive_packet) @@ -144,16 +157,28 @@ def test_sent_packet_format(self): PACKET_DATA = "data" OOB_DATA = None run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), functools.partial(receive_bytes, 8)) - self.assertEqual(receive_bytes.received, "\x00\x00\x00\x04data") + if self.expectNetworkByteOrder: + self.assertEqual(receive_bytes.received, "\x00\x00\x00\x04data") + else: + self.assertEqual(receive_bytes.received, "\x04\x00\x00\x00data") def test_receive_formatted_packet(self): - PACKET_DATA = "\x00\x00\x00\x04data" + if self.expectNetworkByteOrder: + PACKET_DATA = "\x00\x00\x00\x04data" + else: + PACKET_DATA = "\x04\x00\x00\x00data" run_connected(functools.partial(send, PACKET_DATA), receive_packet) self.assertEqual(receive_packet.received, ("data", None, 0)) def test_receive_formatted_packet_with_oob_data(self): if sys.platform == "win32": raise unittest.SkipTest("StacklessIO does not support OOB data on Windows") - PACKET_DATA = "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello" + if self.expectNetworkByteOrder: + PACKET_DATA = "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello" + else: + PACKET_DATA = "\x0E\x00\x00\x10\x05\x00\x00\x00HelloWorld" run_connected(functools.partial(send, PACKET_DATA), receive_packet) - self.assertEqual(receive_packet.received, ("\x00\x00\x00\x05WorldHello", None, 0)) + if self.expectNetworkByteOrder: + self.assertEqual(receive_packet.received, ("\x00\x00\x00\x05WorldHello", None, 0)) + else: + self.assertEqual(receive_packet.received, ("\x05\x00\x00\x00HelloWorld", None, 0)) From 4b1dde3653748b7869a1caf8a11b8dab818ad2bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Fri, 12 Sep 2025 17:25:01 +0000 Subject: [PATCH 113/131] Fix Environment variable flag sent to tests indicating compatibility mode being enabled This was not setting the variable as intended. Instead one got created that had an unexpected name containing the result of the boolean evaluation. --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dbc4031..3b28b4b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -413,9 +413,9 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests/python ) if(WIN32) - set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$\;${BRANCH_ROOT_DIR}/carbon/common/stdlib\;${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}\;${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$<$:PY3_COMPATIBILITY_MODE=True>") + set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$\;${BRANCH_ROOT_DIR}/carbon/common/stdlib\;${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}\;${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$,PY3_COMPATIBILITY_MODE=True,>") elseif(APPLE) - set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$:${BRANCH_ROOT_DIR}/carbon/common/stdlib:${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_LIB_PATH}:${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$<$:PY3_COMPATIBILITY_MODE=True") + set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$:${BRANCH_ROOT_DIR}/carbon/common/stdlib:${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_LIB_PATH}:${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$,PY3_COMPATIBILITY_MODE=True,>") else() message(FATAL_ERROR "Unsupported platform") endif() From de5337a7de61be20aea4c9959c5ba1655291abb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Fri, 12 Sep 2025 17:34:06 +0000 Subject: [PATCH 114/131] Fix received data checking in unit test This was checking the "received" attribute of the wrong function. --- tests/python/bluetests/test/test_slsocket.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python/bluetests/test/test_slsocket.py b/tests/python/bluetests/test/test_slsocket.py index d7d9f2c..ccdf912 100644 --- a/tests/python/bluetests/test/test_slsocket.py +++ b/tests/python/bluetests/test/test_slsocket.py @@ -104,9 +104,9 @@ def test_sent_packet_format(self): OOB_DATA = None run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), receive) if self.expectNetworkByteOrder: - self.assertEqual(receive_bytes.received, "\x00\x00\x00\x04data") + self.assertEqual(receive.received, "\x00\x00\x00\x04data") else: - self.assertEqual(receive_bytes.received, "\x04\x00\x00\x00data") + self.assertEqual(receive.received, "\x04\x00\x00\x00data") def test_receive_formatted_packet(self): if self.expectNetworkByteOrder: From 2ca5f91767f1bc4e08bc980813ea6725b762cc83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Fri, 12 Sep 2025 17:51:44 +0000 Subject: [PATCH 115/131] Undo accidental changes to test_marshal.py This got messed up during rebase. --- tests/python/bluetests/test/test_marshal.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 47d5b25..0dbf8d9 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -237,7 +237,8 @@ def test_string_from_stringtable(self): bytes = b'~\x00\x00\x00\x00\x11\x06' loaded = blue.marshal.Load(bytes) - self.assertTrue(type(loaded) == str) + # We expect a str type constructed from marshalled string table index + self.assertIsInstance(loaded, str) self.assertEqual(loaded, "ballID") def test_empty_unicode(self): @@ -409,5 +410,18 @@ def test_dbrow(self): self.assertEqual(loaded, blue.DBRow(blue.DBRowDescriptor((("Test", 20),)), (123, ))) - def test_dbrow_with_invalid_descriptor_in_stream_raises_error(self): - pass + def test_set(self): + blue.marshal.globalsWhitelist = {set: None} + blue.marshal.collectWhitelist = False + bytes = b'~\x00\x00\x00\x00",\x02\x0cbuiltins.set%\x15\x03\t\x06\x02\x06\x03--' + loaded = blue.marshal.Load(bytes) + self.assertSetEqual(loaded, {1, 2, 3}) + + def test_runtime_error(self): + blue.marshal.globalsWhitelist = {RuntimeError: None} + blue.marshal.collectWhitelist = False + bytes = b'~\x00\x00\x00\x00",\x02\x15builtins.RuntimeError%.\x05Boom!--' + loaded = blue.marshal.Load(bytes) + self.assertIsInstance(loaded, RuntimeError) + self.assertIsInstance(loaded.message, unicode) + self.assertEqual(loaded.message, u"Boom!") From 99f56b7d0e817e2473ee1b79c11122db76c1e8b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Fri, 12 Sep 2025 18:03:17 +0000 Subject: [PATCH 116/131] Add missing #ifefs around Python 3 compatibility behavior --- src/CarbonIO/CarbonIO.cpp | 28 ++++++++++++++++++++++++++-- src/CarbonIO/CarbonIO.h | 4 ++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/CarbonIO/CarbonIO.cpp b/src/CarbonIO/CarbonIO.cpp index d0af2d7..a88dd13 100644 --- a/src/CarbonIO/CarbonIO.cpp +++ b/src/CarbonIO/CarbonIO.cpp @@ -2818,12 +2818,19 @@ void CarbonIO::dataReceived( SCompletionUnit *completion, const char* data, cons appendDataToPacket( completion->packetListTail, indat, need ); available -= need; indat += need; - +#ifdef PY3_COMPATIBILITY_MODE unsigned int header = ntohl( *reinterpret_cast( completion->packetListTail->data ) ); +#else + unsigned int header = *(int *)completion->packetListTail->data; +#endif if ( header & ceHeaderExpectPayloadOffset ) { completion->packetListTail->oobData = sizeof(int)*2 + completion->packetListTail->data; +#ifdef PY3_COMPATIBILITY_MODE completion->packetListTail->oobLen = ntohl( *reinterpret_cast( completion->packetListTail->data + sizeof( uint32_t ) ) ); +#else + completion->packetListTail->oobLen = *(int *)(completion->packetListTail->data + sizeof(int)); +#endif completion->packetListTail->payloadOffset = sizeof(int)*2 + completion->packetListTail->oobLen; if ( completion->packetListTail->payloadOffset > completion->packetListTail->packetLen ) @@ -3297,11 +3304,19 @@ bool CarbonIO::processRawQueuedData( SCompletionUnit *completion ) // find the offset to the body (the part we compress) unsigned int offset = sizeof(unsigned int); // the header itself +#ifdef PY3_COMPATIBILITY_MODE if ( ntohl( *reinterpret_cast( packet->data ) ) & ceHeaderExpectPayloadOffset ) // was there out-of-band data? { offset += sizeof(unsigned int); // the size of the len param offset += *reinterpret_cast( packet->data + sizeof( uint32_t ) ); // the size of the data } +#else + if ( *(unsigned int *)packet->data & ceHeaderExpectPayloadOffset ) // was there out-of-band data? + { + offset += sizeof(unsigned int); // the size of the len param + offset += *(unsigned int *)(packet->data + sizeof(unsigned int)); // the size of the data + } +#endif char *outbuf = 0; unsigned int outlen; @@ -3313,10 +3328,15 @@ bool CarbonIO::processRawQueuedData( SCompletionUnit *completion ) { completion->stats.bytesSentCompressed += outlen + offset; +#ifdef PY3_COMPATIBILITY_MODE *reinterpret_cast( packet->data ) &= htonl( ceHeaderBitsMask ); // knock off old size *reinterpret_cast( packet->data ) |= htonl( ( outlen + offset ) - sizeof( uint32_t ) ); // plug in NEW size *reinterpret_cast( packet->data ) |= htonl( m_compressionType ); - +#else + *(unsigned int *)packet->data &= ceHeaderBitsMask; // knock off old size + *(unsigned int *)packet->data |= (outlen + offset) - sizeof(unsigned int); // plug in NEW size + *(unsigned int *)packet->data |= m_compressionType; +#endif // if this is not the root packet, append the data TO the // root packet if ( packet != completion->queuedData ) @@ -3944,7 +3964,11 @@ bool CarbonIO::compress( const char *in, unsigned int inLen, char **out, unsigne //------------------------------------------------------------------------------ bool CarbonIO::decompress( SPacket* packet ) { +#ifdef PY3_COMPATIBILITY_MODE unsigned long packetHeader = ntohl( *reinterpret_cast( packet->data ) ); +#else + unsigned long packetHeader = *reinterpret_cast( packet->data ); +#endif if ( packetHeader & ceHeaderBitZlibCompressed ) { CPerformanceTime timeCompress( "decompress" ); diff --git a/src/CarbonIO/CarbonIO.h b/src/CarbonIO/CarbonIO.h index 68683b6..351cc23 100644 --- a/src/CarbonIO/CarbonIO.h +++ b/src/CarbonIO/CarbonIO.h @@ -837,7 +837,11 @@ bool CarbonIO::isPacketValid( SPacket *packet ) return packet && packet->packetLen >= 4 && (packet->auxData +#ifdef PY3_COMPATIBILITY_MODE || ( packet->packetLen == ( ntohl( *reinterpret_cast( packet->data ) ) & ceHeaderSizeMask ) + sizeof( int ) ) ); +#else + || (packet->packetLen == (*(int *)packet->data & ceHeaderSizeMask) + sizeof(int))); +#endif } //------------------------------------------------------------------------------ From b84f3f75309c27694509f75c0dd3062b429aa854 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Mon, 15 Sep 2025 12:33:34 +0000 Subject: [PATCH 117/131] Fix order of packet data in unit tests Changing the header does not affect the order of the packet data. The packet data was out of order in some of the tests. --- tests/python/bluetests/test/test_slsocket.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/python/bluetests/test/test_slsocket.py b/tests/python/bluetests/test/test_slsocket.py index ccdf912..b088f4f 100644 --- a/tests/python/bluetests/test/test_slsocket.py +++ b/tests/python/bluetests/test/test_slsocket.py @@ -123,13 +123,13 @@ def test_sent_packet_format_with_oob_data(self): if self.expectNetworkByteOrder: self.assertEqual(receive.received, "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello") else: - self.assertEqual(receive.received, "\x0E\x00\x00\x10\x05\x00\x00\x00HelloWorld") + self.assertEqual(receive.received, "\x0E\x00\x00\x10\x05\x00\x00\x00WorldHello") def test_receive_formatted_packet_with_oob_data(self): if self.expectNetworkByteOrder: PACKET_DATA = "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello" else: - PACKET_DATA = "\x0E\x00\x00\x10\x05\x00\x00\x00HelloWorld" + PACKET_DATA = "\x0E\x00\x00\x10\x05\x00\x00\x00WorldHello" run_connected(functools.partial(send, PACKET_DATA), receive_packet) self.assertEqual(receive_packet.received, ("Hello", "World", 1)) @@ -176,9 +176,9 @@ def test_receive_formatted_packet_with_oob_data(self): if self.expectNetworkByteOrder: PACKET_DATA = "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello" else: - PACKET_DATA = "\x0E\x00\x00\x10\x05\x00\x00\x00HelloWorld" + PACKET_DATA = "\x0E\x00\x00\x10\x05\x00\x00\x00WorldHello" run_connected(functools.partial(send, PACKET_DATA), receive_packet) if self.expectNetworkByteOrder: self.assertEqual(receive_packet.received, ("\x00\x00\x00\x05WorldHello", None, 0)) else: - self.assertEqual(receive_packet.received, ("\x05\x00\x00\x00HelloWorld", None, 0)) + self.assertEqual(receive_packet.received, ("\x05\x00\x00\x00WorldHello", None, 0)) From 92899dac27c6bfef577a1522b3bf272b2fec472a Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Thu, 25 Sep 2025 12:11:24 +0000 Subject: [PATCH 118/131] Add filename to "Couldn't open file" error messages This provides more context on which file failed to be opened. Ideally the error also includes information about the underlying system failure, but that information isn't easy to pipe through the current systems. --- src/BackgroundReader.cpp | 4 +++- src/BluePaths.cpp | 12 +++++++++--- src/BlueResFile2_Blue.cpp | 4 +++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/BackgroundReader.cpp b/src/BackgroundReader.cpp index 0037353..eafb23d 100644 --- a/src/BackgroundReader.cpp +++ b/src/BackgroundReader.cpp @@ -32,7 +32,9 @@ void BackgroundReader::Perform() if (!resFile->OpenW( m_filename.c_str(), true ) ) { - m_result = Be::Result( "Couldn't open file" ); + std::string msg = "Couldn't open file: "; + msg += CW2A(m_filename.c_str()); + m_result = Be::Result( msg ); return; } diff --git a/src/BluePaths.cpp b/src/BluePaths.cpp index cf399f0..482c005 100644 --- a/src/BluePaths.cpp +++ b/src/BluePaths.cpp @@ -300,7 +300,9 @@ Be::Result BluePaths::Open( const std::wstring& filename, Be::Optio *stream = fileStream.Detach(); return Be::Result(); } - return Be::Result("Couldn't create file"); + std::string msg = "Couldn't open file "; + msg += CW2A( filename.c_str() ); + return Be::Result( msg ); } else if( fileMode[0] == 'a' ) { @@ -312,7 +314,9 @@ Be::Result BluePaths::Open( const std::wstring& filename, Be::Optio *stream = fileStream.Detach(); return Be::Result(); } - return Be::Result("Couldn't open file"); + std::string msg = "Couldn't open file "; + msg += CW2A( filename.c_str() ); + return Be::Result( msg ); } } } @@ -322,7 +326,9 @@ Be::Result BluePaths::Open( const std::wstring& filename, Be::Optio return Be::Result(); } - return Be::Result("Couldn't open file"); + std::string msg = "Couldn't open file "; + msg += CW2A( filename.c_str() ); + return Be::Result( msg ); } Be::Result BluePaths::GetFileContentsWithYield( const std::wstring& path, IBlueStream** contents ) diff --git a/src/BlueResFile2_Blue.cpp b/src/BlueResFile2_Blue.cpp index 72505a9..7ff2bd6 100644 --- a/src/BlueResFile2_Blue.cpp +++ b/src/BlueResFile2_Blue.cpp @@ -56,7 +56,9 @@ static PyObject* PyOpenAlways( PyObject* self, PyObject* args ) if( !ok ) { - PyErr_SetString( PyExc_BlueError, "Couldn't open file" ); + std::string msg = "Couldn't open file "; + msg += CW2A( (const wchar_t*)PyUnicode_AsUnicode( filename ) ); + PyErr_SetString( PyExc_BlueError, msg.c_str() ); return nullptr; } From 4c3272478dde8232a429a4373d7fb95509f19dff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Mon, 29 Sep 2025 16:37:16 +0000 Subject: [PATCH 119/131] Guard access to expanded search paths behind mutex The tests were crashing because of one thread attempting to iterate through the search paths, while another was modifying them. --- src/BlueResFileSystemLocal.cpp | 15 +++++++++++++++ src/BlueResFileSystemLocal.h | 6 +++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/BlueResFileSystemLocal.cpp b/src/BlueResFileSystemLocal.cpp index 7c12a17..b06ea4f 100644 --- a/src/BlueResFileSystemLocal.cpp +++ b/src/BlueResFileSystemLocal.cpp @@ -80,6 +80,7 @@ bool BlueResFileSystemLocal::IsDirectory( const std::wstring& dir ) // We have a prefix, look it up in our search paths map. // If found, the value associated with it is a list of paths. + std::shared_lock guard( m_expandedSearchPathsMutex ); ExpandedSearchPathMap_t::iterator found = m_expandedSearchPaths.find( key ); if( found != m_expandedSearchPaths.end() ) { @@ -123,6 +124,7 @@ void BlueResFileSystemLocal::GetDirectoryContents( const wchar_t* dir, std::set< // We have a prefix, look it up in our search paths map. // If found, the value associated with it is a list of paths. + std::shared_lock guard( m_expandedSearchPathsMutex ); ExpandedSearchPathMap_t::iterator found = m_expandedSearchPaths.find( key ); if( found != m_expandedSearchPaths.end() ) { @@ -211,6 +213,7 @@ const wchar_t* BlueResFileSystemLocal::GetSearchPathW( const char* key ) void BlueResFileSystemLocal::ClearSearchPaths() { + std::unique_lock guard( m_expandedSearchPathsMutex ); m_searchPaths.clear(); m_expandedSearchPaths.clear(); } @@ -229,6 +232,7 @@ bool BlueResFileSystemLocal::ResolvePathW( const std::wstring& path, std::wstrin // We have a prefix, look it up in our search paths map. // If found, the value associated with it is a list of paths. + std::shared_lock guard( m_expandedSearchPathsMutex ); ExpandedSearchPathMap_t::iterator found = m_expandedSearchPaths.find( key ); if( found != m_expandedSearchPaths.end() ) { @@ -294,6 +298,7 @@ std::wstring BlueResFileSystemLocal::ResolvePathForWritingW( const std::wstring& std::string key = (const char*)CW2A( keyW.c_str() ); ToLower( key ); + std::shared_lock guard( m_expandedSearchPathsMutex ); ExpandedSearchPathMap_t::iterator found = m_expandedSearchPaths.find( key ); if( found != m_expandedSearchPaths.end() ) { @@ -320,6 +325,7 @@ std::wstring BlueResFileSystemLocal::ResolvePathToRootW( const std::string& root std::string rootLower = root; ToLower( rootLower ); + std::shared_lock guard( m_expandedSearchPathsMutex ); ExpandedSearchPathMap_t::iterator found = m_expandedSearchPaths.find( rootLower ); if( found != m_expandedSearchPaths.end() ) { @@ -358,6 +364,7 @@ void BlueResFileSystemLocal::GetExpandedSearchPaths( const char* key, std::vecto std::string keyLower = key; ToLower( keyLower ); + std::shared_lock guard( m_expandedSearchPathsMutex ); ExpandedSearchPathMap_t::iterator found = m_expandedSearchPaths.find( keyLower ); if( found != m_expandedSearchPaths.end() ) { @@ -467,6 +474,7 @@ std::optional> ExpandSearchPath( const std::wstring& p bool BlueResFileSystemLocal::ExpandSearchPaths() { + std::unique_lock guard( m_expandedSearchPathsMutex ); m_expandedSearchPaths.clear(); for( const auto& searchPath : m_searchPaths ) { @@ -494,6 +502,7 @@ bool BlueResFileSystemLocal::ExpandSearchPaths() void BlueResFileSystemLocal::LogPaths() { + std::shared_lock guard( m_expandedSearchPathsMutex ); CCP_LOG( "BlueOS search paths:" ); for( ExpandedSearchPathMap_t::const_iterator it = m_expandedSearchPaths.begin(); it != m_expandedSearchPaths.end(); ++it ) { @@ -638,3 +647,9 @@ std::vector BlueResFileSystemLocal::ListDirFromScript( const std:: std::vector list( begin( results ), end( results ) ); return list; } + +BlueResFileSystemLocal::ExpandedSearchPathMap_t BlueResFileSystemLocal::GetExpandedSearchPaths() { + std::shared_lock guard( m_expandedSearchPathsMutex ); + return m_expandedSearchPaths; +} + diff --git a/src/BlueResFileSystemLocal.h b/src/BlueResFileSystemLocal.h index f380e87..5e4b407 100644 --- a/src/BlueResFileSystemLocal.h +++ b/src/BlueResFileSystemLocal.h @@ -10,6 +10,7 @@ #define BlueResFileSystemLocal_h #include "IBlueResFileSystem.h" +#include BLUE_CLASS( BlueResFileSystemLocal ) : public IBlueResFileSystem { @@ -45,7 +46,7 @@ BLUE_CLASS( BlueResFileSystemLocal ) : public IBlueResFileSystem typedef std::map> ExpandedSearchPathMap_t; SearchPathMap_t GetAllSearchPaths() { return m_searchPaths; } - ExpandedSearchPathMap_t GetExpandedSearchPaths() { return m_expandedSearchPaths; } + ExpandedSearchPathMap_t GetExpandedSearchPaths(); std::wstring ResolvePath( const wchar_t* path ); @@ -61,6 +62,9 @@ BLUE_CLASS( BlueResFileSystemLocal ) : public IBlueResFileSystem // Search paths expanded, for quicker resolving of paths ExpandedSearchPathMap_t m_expandedSearchPaths; + // Expanded search paths can be accessed from multiple threads. + std::shared_timed_mutex m_expandedSearchPathsMutex; + // Helper function to expand search paths - called after any entry is changed bool ExpandSearchPaths(); }; From cbb22b91773832730ae680da0e35b340f06f37ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Wed, 1 Oct 2025 18:46:29 +0000 Subject: [PATCH 120/131] Fix crash when tasklets die before loading TextResources This class inherits from BlueAsyncRes, which has a Shutdown method. This method must get called before ending the load, or we won't unregister any outstanding callbacks, resulting in use of freed memory. Therefore, in order to ensure that the shutdown method gets called, use the TYPEDEF_BLUECLASS_SHUTDOWN macro. --- src/BlueTextResource.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BlueTextResource.h b/src/BlueTextResource.h index 9b3c2c7..b28cd05 100644 --- a/src/BlueTextResource.h +++ b/src/BlueTextResource.h @@ -32,6 +32,6 @@ public BlueAsyncRes, std::string m_text; }; -TYPEDEF_BLUECLASS( BlueTextResource ); +TYPEDEF_BLUECLASS_SHUTDOWN( BlueTextResource ); #endif // BlueTextResource_h From 577dc4952e77e09be63d70e8e1546e0b745cfa7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hrafn=20J=C3=B6kull=20Geirsson?= <5185245+hrafn@users.noreply.github.com> Date: Wed, 1 Oct 2025 18:59:22 +0000 Subject: [PATCH 121/131] Remove preprocessor macro for toggling Pyhon 3 compatibility This will make the code forward compatible by default. These changes will break network compatibility with older versions of Blue, due to the changed macho packet byte order. --- CMakeLists.txt | 5 +-- src/CarbonIO/CarbonIO.cpp | 34 ---------------- src/CarbonIO/CarbonIO.h | 25 ------------ src/Marshal.cpp | 16 -------- src/slsocket.cpp | 15 ------- src/slsocket_posix.cpp | 11 +---- tests/python/bluetests/test/test_marshal.py | 1 - tests/python/bluetests/test/test_slsocket.py | 42 ++++---------------- 8 files changed, 11 insertions(+), 138 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b28b4b..8c4582f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -311,7 +311,6 @@ target_precompile_headers(Blue PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src/StdAfx.h) target_compile_definitions(Blue PUBLIC CCP_STACKLESS - $<$:PY3_COMPATIBILITY_MODE> ) if(APPLE) @@ -413,9 +412,9 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests/python ) if(WIN32) - set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$\;${BRANCH_ROOT_DIR}/carbon/common/stdlib\;${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}\;${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$,PY3_COMPATIBILITY_MODE=True,>") + set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$\;${BRANCH_ROOT_DIR}/carbon/common/stdlib\;${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}\;${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>") elseif(APPLE) - set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$:${BRANCH_ROOT_DIR}/carbon/common/stdlib:${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_LIB_PATH}:${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$,PY3_COMPATIBILITY_MODE=True,>") + set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$:${BRANCH_ROOT_DIR}/carbon/common/stdlib:${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_LIB_PATH}:${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>") else() message(FATAL_ERROR "Unsupported platform") endif() diff --git a/src/CarbonIO/CarbonIO.cpp b/src/CarbonIO/CarbonIO.cpp index a88dd13..3b2523a 100644 --- a/src/CarbonIO/CarbonIO.cpp +++ b/src/CarbonIO/CarbonIO.cpp @@ -2778,11 +2778,7 @@ void CarbonIO::dataReceived( SCompletionUnit *completion, const char* data, cons } // a partial packet has been recieved, call for completion -#ifdef PY3_COMPATIBILITY_MODE packetSize = ( ceHeaderSizeMask & ntohl( *reinterpret_cast( completion->packetListTail->data ) ) ) + sizeof(int); -#else - packetSize = (ceHeaderSizeMask & *(int *)completion->packetListTail->data) + sizeof(int); -#endif need = packetSize - completion->packetListTail->packetLen; D_HANDLEREAD(ciolog("working packet[%d:%d] need[%d] avail[%d] for [%d]", packetSize, completion->packetListTail->packetLen, need, available, (int)completion->workHandle )); @@ -2791,11 +2787,7 @@ void CarbonIO::dataReceived( SCompletionUnit *completion, const char* data, cons else { // packet is empty, call for the full amount -#ifdef PY3_COMPATIBILITY_MODE packetSize = ( ceHeaderSizeMask & ntohl( *reinterpret_cast( indat ) ) ) + sizeof( uint32_t ); -#else - packetSize = (ceHeaderSizeMask & *(int *)indat) + sizeof(int); -#endif need = packetSize; D_HANDLEREAD(ciolog("not working packet, needing full amount[%d] for[%d] [0x%08X]:[0x%08X]", need, (int)completion->workHandle, *(unsigned int *)indat, ceHeaderSizeMask & *(int *)indat)); @@ -2818,19 +2810,11 @@ void CarbonIO::dataReceived( SCompletionUnit *completion, const char* data, cons appendDataToPacket( completion->packetListTail, indat, need ); available -= need; indat += need; -#ifdef PY3_COMPATIBILITY_MODE unsigned int header = ntohl( *reinterpret_cast( completion->packetListTail->data ) ); -#else - unsigned int header = *(int *)completion->packetListTail->data; -#endif if ( header & ceHeaderExpectPayloadOffset ) { completion->packetListTail->oobData = sizeof(int)*2 + completion->packetListTail->data; -#ifdef PY3_COMPATIBILITY_MODE completion->packetListTail->oobLen = ntohl( *reinterpret_cast( completion->packetListTail->data + sizeof( uint32_t ) ) ); -#else - completion->packetListTail->oobLen = *(int *)(completion->packetListTail->data + sizeof(int)); -#endif completion->packetListTail->payloadOffset = sizeof(int)*2 + completion->packetListTail->oobLen; if ( completion->packetListTail->payloadOffset > completion->packetListTail->packetLen ) @@ -3304,19 +3288,11 @@ bool CarbonIO::processRawQueuedData( SCompletionUnit *completion ) // find the offset to the body (the part we compress) unsigned int offset = sizeof(unsigned int); // the header itself -#ifdef PY3_COMPATIBILITY_MODE if ( ntohl( *reinterpret_cast( packet->data ) ) & ceHeaderExpectPayloadOffset ) // was there out-of-band data? { offset += sizeof(unsigned int); // the size of the len param offset += *reinterpret_cast( packet->data + sizeof( uint32_t ) ); // the size of the data } -#else - if ( *(unsigned int *)packet->data & ceHeaderExpectPayloadOffset ) // was there out-of-band data? - { - offset += sizeof(unsigned int); // the size of the len param - offset += *(unsigned int *)(packet->data + sizeof(unsigned int)); // the size of the data - } -#endif char *outbuf = 0; unsigned int outlen; @@ -3328,15 +3304,9 @@ bool CarbonIO::processRawQueuedData( SCompletionUnit *completion ) { completion->stats.bytesSentCompressed += outlen + offset; -#ifdef PY3_COMPATIBILITY_MODE *reinterpret_cast( packet->data ) &= htonl( ceHeaderBitsMask ); // knock off old size *reinterpret_cast( packet->data ) |= htonl( ( outlen + offset ) - sizeof( uint32_t ) ); // plug in NEW size *reinterpret_cast( packet->data ) |= htonl( m_compressionType ); -#else - *(unsigned int *)packet->data &= ceHeaderBitsMask; // knock off old size - *(unsigned int *)packet->data |= (outlen + offset) - sizeof(unsigned int); // plug in NEW size - *(unsigned int *)packet->data |= m_compressionType; -#endif // if this is not the root packet, append the data TO the // root packet if ( packet != completion->queuedData ) @@ -3964,11 +3934,7 @@ bool CarbonIO::compress( const char *in, unsigned int inLen, char **out, unsigne //------------------------------------------------------------------------------ bool CarbonIO::decompress( SPacket* packet ) { -#ifdef PY3_COMPATIBILITY_MODE unsigned long packetHeader = ntohl( *reinterpret_cast( packet->data ) ); -#else - unsigned long packetHeader = *reinterpret_cast( packet->data ); -#endif if ( packetHeader & ceHeaderBitZlibCompressed ) { CPerformanceTime timeCompress( "decompress" ); diff --git a/src/CarbonIO/CarbonIO.h b/src/CarbonIO/CarbonIO.h index 351cc23..0538f07 100644 --- a/src/CarbonIO/CarbonIO.h +++ b/src/CarbonIO/CarbonIO.h @@ -556,7 +556,6 @@ bool CarbonIO::formatPacket( char* buf, } int pos; -#ifdef PY3_COMPATIBILITY_MODE if ( OOBData && OOBLen ) { *reinterpret_cast( buf ) = htonl( (dataLen + OOBLen + sizeof( uint32_t ) ) | ceHeaderExpectPayloadOffset ); @@ -569,20 +568,6 @@ bool CarbonIO::formatPacket( char* buf, *reinterpret_cast( buf ) = htonl( dataLen ); pos = sizeof( uint32_t ); } -#else - if ( OOBData && OOBLen ) - { - *(unsigned int *)buf = (dataLen + OOBLen + sizeof(unsigned int)) | ceHeaderExpectPayloadOffset; - *(unsigned int *)(buf + sizeof(unsigned int)) = OOBLen; - memcpy( buf + sizeof(unsigned int)*2, OOBData, OOBLen ); - pos = OOBLen + sizeof(unsigned int)*2; - } - else - { - *(unsigned int *)buf = dataLen; - pos = sizeof(unsigned int); - } -#endif char *outbuf = 0; unsigned int outlen; @@ -591,15 +576,9 @@ bool CarbonIO::formatPacket( char* buf, && checkAndCompress(data, dataLen, &outbuf, &outlen) ) { *len = pos + outlen; -#ifdef PY3_COMPATIBILITY_MODE *reinterpret_cast( buf ) &= htonl( ceHeaderBitsMask ); // knock off old size *reinterpret_cast( buf ) |= htonl( *len - sizeof( uint32_t ) ); // plug in NEW size *reinterpret_cast( buf ) |= htonl( m_compressionType ); -#else - *(unsigned int *)buf &= ceHeaderBitsMask; // knock off old size - *(unsigned int *)buf |= *len - sizeof(unsigned int); // plug in NEW size - *(unsigned int *)buf |= m_compressionType; -#endif memcpy( buf + pos, outbuf, outlen ); delete[] outbuf; } @@ -837,11 +816,7 @@ bool CarbonIO::isPacketValid( SPacket *packet ) return packet && packet->packetLen >= 4 && (packet->auxData -#ifdef PY3_COMPATIBILITY_MODE || ( packet->packetLen == ( ntohl( *reinterpret_cast( packet->data ) ) & ceHeaderSizeMask ) + sizeof( int ) ) ); -#else - || (packet->packetLen == (*(int *)packet->data & ceHeaderSizeMask) + sizeof(int))); -#endif } //------------------------------------------------------------------------------ diff --git a/src/Marshal.cpp b/src/Marshal.cpp index 7b13a31..618e07b 100644 --- a/src/Marshal.cpp +++ b/src/Marshal.cpp @@ -1549,13 +1549,8 @@ PyObject *Marshal::ReadObjectNewobj(ReadStream* stream, bool shared) if (!cls) return 0; BluePy __new__(PyObject_GetAttr(cls, mStock_New)); -#ifdef PY3_COMPATIBILITY_MODE // Marshalled new-style objects from Python3 may have to be constructed as old-style objects in Python2.7 BluePy r(PyObject_HasAttr( cls, mStock_New ) ? BluePy(PyObject_CallObject(__new__, args)) : BluePy(PyInstance_NewRaw(cls, 0))); -#else - if (!__new__) return 0; - BluePy r(PyObject_CallObject(__new__, args)); -#endif if (!r) return 0; //object is constructed, now update r @@ -1659,16 +1654,12 @@ PyObject *Marshal::GetGlobalObject(PyObject *nameO) const char *dot = strrchr(name, '.'); BluePyStr modulename; if (dot){ -#if PY3_COMPATIBILITY_MODE if( strncmp( name, "builtins.", 9 ) == 0 ) { modulename = BluePyStr( "__builtin__" ); } else { modulename = BluePyStr( dot-name, name ); } -#else - modulename = BluePyStr(dot-name, name); -#endif name = dot+1; } else { modulename = BluePyStr("__builtin__"); @@ -1878,13 +1869,6 @@ bool Marshal::WriteObject(WriteStream* stream, PyObject* o) { RETFAIL(WriteType(stream, TY_STR_EMPTY)); } -#ifndef PY3_COMPATIBILITY_MODE - else if (size == 1) - { - RETFAIL(WriteType(stream, TY_STR_CHAR)); - RETFAIL(stream->Write(string[0])); - } -#endif else { PyObject* index = PyDict_GetItem(mStrTable, o); diff --git a/src/slsocket.cpp b/src/slsocket.cpp index 1fa5027..2adeb73 100644 --- a/src/slsocket.cpp +++ b/src/slsocket.cpp @@ -1150,7 +1150,6 @@ class SendPacketResult : public SendResult buffers.Push(header); SSIZE_T result = header.len; result += SendResult::Prepare(buffers, obj, pbuf); -#ifdef PY3_COMPATIBILITY_MODE auto cargoBytes = static_cast( result - sizeof(mHeader) ); //number of cargo bytes if ( cargoBytes > static_cast( mXtra->GetMaxPacketSize() ) ) { char tmp[128]; @@ -1160,16 +1159,6 @@ class SendPacketResult : public SendResult throw std::length_error(tmp); } mHeader = htonl( cargoBytes ); -#else - mHeader = (DWORD)(result-sizeof(mHeader)); //number of cargo bytes - if (mHeader > (DWORD)mXtra->GetMaxPacketSize()) { - char tmp[128]; - sprintf_s(tmp, "packet too long at %d bytes, max size is %d", - mHeader, mXtra->GetMaxPacketSize()); - OutputDebugString(tmp); - throw std::length_error(tmp); - } -#endif return result; } @@ -1288,11 +1277,7 @@ class RecvPacketResult : public IOOverlappedRecv { mBytesRead += bytesTransfered; bool more; -#ifdef PY3_COMPATIBILITY_MODE uint32_t cargoBytes{ntohl( mHeader )}; -#else - uint32_t cargoBytes{mHeader}; -#endif if (mBytesRead < sizeof(mHeader)) { //continue reading header more = true; diff --git a/src/slsocket_posix.cpp b/src/slsocket_posix.cpp index 896c54f..7462c86 100644 --- a/src/slsocket_posix.cpp +++ b/src/slsocket_posix.cpp @@ -1625,10 +1625,7 @@ class SendPacketResult : public SendBase // Copy address // steal the buffer StealBuffer(buf); - mHeader = mKeeper.len; -#ifdef PY3_COMPATIBILITY_MODE - mHeader = htonl(mHeader); -#endif + mHeader = htonl(mKeeper.len); FlipHeader(mHeader); Request(s->sock_timeout); return mResult; @@ -1837,11 +1834,7 @@ class RecvPacketResult : public SocketResult if (mBytesRead < (sizeof(mHeader) + sizeof(uint32_t))) return; -#ifdef PY3_COMPATIBILITY_MODE uint32_t oobDataLen = ntohl(*(uint32_t *)(mData)); -#else - uint32_t oobDataLen = *(uint32_t *)(mData); -#endif // sanity check the out-of-band data length; mPacketSize was sanity checked already if (oobDataLen > mPacketSize) { char tmp[128] = {'\0'}; @@ -1915,9 +1908,7 @@ class RecvPacketResult : public SocketResult rcvd = recv(handle, (char*)&mHeader + mBytesRead, sizeof(mHeader) - mBytesRead, 0); if (rcvd > 0 && mBytesRead+rcvd == 4) { // We completed reading the header, do stuff! -#ifdef PY3_COMPATIBILITY_MODE mHeader = htonl(mHeader); -#endif FlipHeader(mHeader); mPacketSize = (mHeader & ceHeaderSizeMask); if (mPacketSize > (uint32_t)GetXtra()->GetMaxPacketSize()) { diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 0dbf8d9..ab9d71d 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -216,7 +216,6 @@ def test_dbrow_with_invalid_descriptor_in_stream_raises_error(self): self.assertEqual(raisedValue.exception.args[0], TypeError) -@unittest.skipUnless(os.environ.has_key("PY3_COMPATIBILITY_MODE"), "Skipping because these tests will fail if `PY3_COMPATIBILITY_MODE` is not enabled") class TestBackwardsCompatibility(blueunittest.TestCase): """ This class adds coverage for objects marshalled in Python 3. diff --git a/tests/python/bluetests/test/test_slsocket.py b/tests/python/bluetests/test/test_slsocket.py index b088f4f..632f1ce 100644 --- a/tests/python/bluetests/test/test_slsocket.py +++ b/tests/python/bluetests/test/test_slsocket.py @@ -92,7 +92,6 @@ def setUp(self): sys.modules["_socket"] = _slsocket sys.modules["select"] = None reload(socket) # Make sure the socket module is using _slsocket instead of _socket - self.expectNetworkByteOrder = os.environ.has_key("PY3_COMPATIBILITY_MODE") def test_send_receive_packet(self): PACKET_DATA = "data" OOB_DATA = None @@ -103,16 +102,10 @@ def test_sent_packet_format(self): PACKET_DATA = "data" OOB_DATA = None run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), receive) - if self.expectNetworkByteOrder: - self.assertEqual(receive.received, "\x00\x00\x00\x04data") - else: - self.assertEqual(receive.received, "\x04\x00\x00\x00data") + self.assertEqual(receive.received, "\x00\x00\x00\x04data") def test_receive_formatted_packet(self): - if self.expectNetworkByteOrder: - PACKET_DATA = "\x00\x00\x00\x04data" - else: - PACKET_DATA = "\x04\x00\x00\x00data" + PACKET_DATA = "\x00\x00\x00\x04data" run_connected(functools.partial(send, PACKET_DATA), receive_packet) self.assertEqual(receive_packet.received, ("data", "", 1)) @@ -120,16 +113,10 @@ def test_sent_packet_format_with_oob_data(self): PACKET_DATA = "Hello" OOB_DATA = "World" run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), receive) - if self.expectNetworkByteOrder: - self.assertEqual(receive.received, "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello") - else: - self.assertEqual(receive.received, "\x0E\x00\x00\x10\x05\x00\x00\x00WorldHello") + self.assertEqual(receive.received, "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello") def test_receive_formatted_packet_with_oob_data(self): - if self.expectNetworkByteOrder: - PACKET_DATA = "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello" - else: - PACKET_DATA = "\x0E\x00\x00\x10\x05\x00\x00\x00WorldHello" + PACKET_DATA = "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello" run_connected(functools.partial(send, PACKET_DATA), receive_packet) self.assertEqual(receive_packet.received, ("Hello", "World", 1)) @@ -145,7 +132,6 @@ def setUp(self): sys.modules["_socket"] = _slsocket sys.modules["select"] = slselect reload(socket) # Make sure the socket module is using _slsocket instead of _socket - self.expectNetworkByteOrder = os.environ.has_key("PY3_COMPATIBILITY_MODE") def test_send_receive_packet(self): PACKET_DATA = "data" @@ -157,28 +143,16 @@ def test_sent_packet_format(self): PACKET_DATA = "data" OOB_DATA = None run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), functools.partial(receive_bytes, 8)) - if self.expectNetworkByteOrder: - self.assertEqual(receive_bytes.received, "\x00\x00\x00\x04data") - else: - self.assertEqual(receive_bytes.received, "\x04\x00\x00\x00data") + self.assertEqual(receive_bytes.received, "\x00\x00\x00\x04data") def test_receive_formatted_packet(self): - if self.expectNetworkByteOrder: - PACKET_DATA = "\x00\x00\x00\x04data" - else: - PACKET_DATA = "\x04\x00\x00\x00data" + PACKET_DATA = "\x00\x00\x00\x04data" run_connected(functools.partial(send, PACKET_DATA), receive_packet) self.assertEqual(receive_packet.received, ("data", None, 0)) def test_receive_formatted_packet_with_oob_data(self): if sys.platform == "win32": raise unittest.SkipTest("StacklessIO does not support OOB data on Windows") - if self.expectNetworkByteOrder: - PACKET_DATA = "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello" - else: - PACKET_DATA = "\x0E\x00\x00\x10\x05\x00\x00\x00WorldHello" + PACKET_DATA = "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello" run_connected(functools.partial(send, PACKET_DATA), receive_packet) - if self.expectNetworkByteOrder: - self.assertEqual(receive_packet.received, ("\x00\x00\x00\x05WorldHello", None, 0)) - else: - self.assertEqual(receive_packet.received, ("\x05\x00\x00\x00WorldHello", None, 0)) + self.assertEqual(receive_packet.received, ("\x00\x00\x00\x05WorldHello", None, 0)) From a4458e9165cd5156a4b349b9e0ea174ebbe56c2e Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Thu, 22 Jan 2026 10:05:57 +0100 Subject: [PATCH 122/131] Fix crash for bad payloads in `ReadObjectReference` Back-porting the fix from Python 3 --- src/Marshal.cpp | 15 ++++++++++----- tests/python/bluetests/test/test_marshal.py | 10 ++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/Marshal.cpp b/src/Marshal.cpp index 618e07b..23473c8 100644 --- a/src/Marshal.cpp +++ b/src/Marshal.cpp @@ -1223,17 +1223,22 @@ PyObject * Marshal::ReadObjectReference( ReadStream * stream, bool isShared ) int len; PyObject* ret; - if( !stream->ReadInteger( len ) ) return 0; + if( !stream->ReadInteger( len ) ) + { + return nullptr; + } if( stream->GetVersion() == 0 ) { if( len < 1 || len > stream->mMapCount || !(ret = stream->mShared[len - 1]) ) { PyErr_SetString( PyExc_ValueError, "Invalid TY_REFERENCE in stream" ); - return 0; + return nullptr; } } else { - if( len < 0 || len >= (int)stream->mShared.size() ) - return PyErr_SetString( PyExc_ValueError, "Invalid TY_REFERENCE in stream" ), nullptr; - ret = stream->mShared[len]; + if( len < 0 || len >= (int)stream->mShared.size() || !(ret = stream->mShared[len]) ) + { + PyErr_SetString( PyExc_ValueError, "Invalid TY_REFERENCE in stream" ); + return nullptr; + } } Py_INCREF( ret ); return ret; diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index ab9d71d..0d773d4 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -216,6 +216,16 @@ def test_dbrow_with_invalid_descriptor_in_stream_raises_error(self): self.assertEqual(raisedValue.exception.args[0], TypeError) + def test_nullptr_deref_in_readobjectreference(self): + known_bad_payloads = ( + b"\x7D\x01\x57\x1B\x00", # entering via `TY_INSTANCE` + b"\x7D\x01\x62\x1B\x00", # entering via `TY_REDUCE` + b"\x7D\x01\x63\x1B\x00", # entering via `TY_NEWOBJ` + ) + for bad_payload in known_bad_payloads: + with self.assertRaises(RuntimeError): + blue.marshal.Load(bad_payload) + class TestBackwardsCompatibility(blueunittest.TestCase): """ This class adds coverage for objects marshalled in Python 3. From d963b2512a2c0ee563996aeb430f78b7e6b235b1 Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Thu, 22 Jan 2026 15:18:33 +0100 Subject: [PATCH 123/131] Fix `-Wchar-subscript` warning from `ColumnDescriptor` A `signed` type may be negative on some platforms. --- src/PyRowSet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PyRowSet.cpp b/src/PyRowSet.cpp index ef87a4b..b7effd9 100644 --- a/src/PyRowSet.cpp +++ b/src/PyRowSet.cpp @@ -144,7 +144,7 @@ struct ColumnDescriptor std::string mName; int mOffset; DBTYPE mType; - char mSize; + unsigned char mSize; }; bool operator == (const ColumnDescriptor &a, const ColumnDescriptor &b) From ab4ffd0efdfc87b74fd22944c8f59cf61c85a186 Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Thu, 22 Jan 2026 15:34:19 +0100 Subject: [PATCH 124/131] Add constraints to column names used in `blue.DBRowDescriptor` First of all, an empty column name does not make much sense - unless one wants a `blue.DBRow` that can _only_ access it's data through offsets. That is not something a `blue.DBRow` is meant for, though (e.g. it is to wrap the result of a database query, which always contains a column name). Secondly, column names must not conflict with Python magic attributes. Otherwise, interesting things can happen when combining, say, `__dict__` or `__class__` as a name with any of the column types that accept arbitrary Python objects. Most notably, this can be used to craft potentially malicious network packets. --- src/PyRowSet.cpp | 40 ++++++++++++++++++++--- tests/python/bluetests/test/test_dbrow.py | 15 +++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/PyRowSet.cpp b/src/PyRowSet.cpp index b7effd9..c4923ef 100644 --- a/src/PyRowSet.cpp +++ b/src/PyRowSet.cpp @@ -303,6 +303,29 @@ bool RowDescriptor::InitFromTypedList() size_t i; for(i = 0; i 1 && name[0] == '_' && name[1] == '_') + { + PyErr_SetString(PyExc_ValueError, "Column names cannot begin with an underscore"); + return false; + } + int size; //logarithm of size plus 1 if (!ColumnDescriptor::TypeSize(cd.mType, size)) return false; @@ -635,18 +658,27 @@ bool DBRowDescriptor::Set_virtual(PyObject *l) if (!newList.Append(t)) goto ERR2; } - InitFromTypedList(); + if ( ! InitFromTypedList() ) + { + return false; + } if (newLen) mVirtualGetSet = newList; return true; ERR1: mColumnList = oldlist; - InitFromTypedList(); + if ( ! InitFromTypedList() ) + { + return false; + } return PyErr_SetString(PyExc_TypeError, "expected list of tuples of size two"), false; ERR2: mColumnList = oldlist; - InitFromTypedList(); - return 0; + if ( ! InitFromTypedList() ) + { + return false; + } + return false; } diff --git a/tests/python/bluetests/test/test_dbrow.py b/tests/python/bluetests/test/test_dbrow.py index 368fa32..a8282ce 100644 --- a/tests/python/bluetests/test/test_dbrow.py +++ b/tests/python/bluetests/test/test_dbrow.py @@ -69,3 +69,18 @@ def testEquivalencyComparison(self): self.assertNotEqual(self.row, blue.DBRowDescriptor(self.columns)) self.assertEqual(self.row, blue.DBRow(blue.DBRowDescriptor(self.columns))) + + def testColumnNameConstraints(self): + invalid_column_names = ( + # An empty column name makes no sense + "", + # Python's magic attributes can lead to type confusion and other interesting bits. + # Therefore, disallow column names starting with a double underscore. + "__this_might_be_a_python_magic_attribute" + ) + for invalid_column_name in invalid_column_names: + with self.assertRaises(ValueError): + blue.DBRowDescriptor(((invalid_column_name, 0x80),)) + + # However, a single underscore as starting character is fine + _ = blue.DBRowDescriptor((("_dummy", 0x80),)) From 9e8ea4f25051fb4ff2f21efba28e4bedf1fcb975 Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Thu, 22 Jan 2026 16:03:42 +0100 Subject: [PATCH 125/131] Disallow multiple `blue.DBRow` columns with the same name Having multiple columns with the same name is an odd construct for a `blue.DBRow` as only the first column can be accessed by name, while the others require index based access. This goes against the nature of how `blue.DBRow` is supposed to be used, e.g. database queries don't return columns that can only be accessed by index. --- src/PyRowSet.cpp | 10 +++++++++- tests/python/bluetests/test/test_dbrow.py | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/PyRowSet.cpp b/src/PyRowSet.cpp index c4923ef..1788767 100644 --- a/src/PyRowSet.cpp +++ b/src/PyRowSet.cpp @@ -400,7 +400,15 @@ bool RowDescriptor::InitFromTypedList() //Also initialize the map, while we're at it. mColumnMap.insert(columnMap_t::value_type(cd.mName.c_str(), std::pair(&cd, (int)i))); } - return true; + + // check if there are multiple columns with the same name + if ( mColumnMap.size() != mColumnList.size() ) + { + PyErr_SetString( PyExc_ValueError, "Multiple columns with the same name are not allowed" ); + return false; + } + + return true; } diff --git a/tests/python/bluetests/test/test_dbrow.py b/tests/python/bluetests/test/test_dbrow.py index a8282ce..26de4e9 100644 --- a/tests/python/bluetests/test/test_dbrow.py +++ b/tests/python/bluetests/test/test_dbrow.py @@ -84,3 +84,7 @@ def testColumnNameConstraints(self): # However, a single underscore as starting character is fine _ = blue.DBRowDescriptor((("_dummy", 0x80),)) + + # cannot have multiple columns with the same name + with self.assertRaises(ValueError): + blue.DBRowDescriptor((('foo', 0x80), ('foo', 0x80),)) From 35601ad280d10c1318853721dae8bd394e1c266e Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+CCP-Aporia@users.noreply.github.com> Date: Fri, 23 Jan 2026 12:50:55 +0100 Subject: [PATCH 126/131] Fix error message --- src/PyRowSet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PyRowSet.cpp b/src/PyRowSet.cpp index 1788767..fcce639 100644 --- a/src/PyRowSet.cpp +++ b/src/PyRowSet.cpp @@ -322,7 +322,7 @@ bool RowDescriptor::InitFromTypedList() // All names starting with a double underscore are thus disallowed. if ( name.length() > 1 && name[0] == '_' && name[1] == '_') { - PyErr_SetString(PyExc_ValueError, "Column names cannot begin with an underscore"); + PyErr_SetString(PyExc_ValueError, "Column names cannot begin with a double underscore"); return false; } From 936a1c9ea8c359418400c634147784c6bd38aa60 Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+ccp-aporia@users.noreply.github.com> Date: Wed, 28 Jan 2026 13:18:28 +0000 Subject: [PATCH 127/131] Revert some sanity checks for `blue.DBRow` column names Turns out that there is plenty of SQL code that uses both anonymous column names, as well as multiple columns with the same name. --- src/PyRowSet.cpp | 13 ------------- tests/python/bluetests/test/test_dbrow.py | 8 +------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/src/PyRowSet.cpp b/src/PyRowSet.cpp index fcce639..1184083 100644 --- a/src/PyRowSet.cpp +++ b/src/PyRowSet.cpp @@ -305,12 +305,6 @@ bool RowDescriptor::InitFromTypedList() ColumnDescriptor &cd = mColumnList[i]; const auto& name = cd.mName; - // Empty column names make no sense - if ( name.empty() ) - { - PyErr_SetString(PyExc_ValueError, "Column names cannot be empty"); - return false; - } // It is possible to do all sorts of strange things by allowing columns that have the same // name as Python's magic double-underscore attributes. For example, it is possible to use @@ -401,13 +395,6 @@ bool RowDescriptor::InitFromTypedList() mColumnMap.insert(columnMap_t::value_type(cd.mName.c_str(), std::pair(&cd, (int)i))); } - // check if there are multiple columns with the same name - if ( mColumnMap.size() != mColumnList.size() ) - { - PyErr_SetString( PyExc_ValueError, "Multiple columns with the same name are not allowed" ); - return false; - } - return true; } diff --git a/tests/python/bluetests/test/test_dbrow.py b/tests/python/bluetests/test/test_dbrow.py index 26de4e9..82a485f 100644 --- a/tests/python/bluetests/test/test_dbrow.py +++ b/tests/python/bluetests/test/test_dbrow.py @@ -72,11 +72,9 @@ def testEquivalencyComparison(self): def testColumnNameConstraints(self): invalid_column_names = ( - # An empty column name makes no sense - "", # Python's magic attributes can lead to type confusion and other interesting bits. # Therefore, disallow column names starting with a double underscore. - "__this_might_be_a_python_magic_attribute" + "__this_might_be_a_python_magic_attribute", ) for invalid_column_name in invalid_column_names: with self.assertRaises(ValueError): @@ -84,7 +82,3 @@ def testColumnNameConstraints(self): # However, a single underscore as starting character is fine _ = blue.DBRowDescriptor((("_dummy", 0x80),)) - - # cannot have multiple columns with the same name - with self.assertRaises(ValueError): - blue.DBRowDescriptor((('foo', 0x80), ('foo', 0x80),)) From 01f46264058a06b6ce3dddaad98dc785fe43829e Mon Sep 17 00:00:00 2001 From: CCP Toebeans <105929497+ccptoebeans@users.noreply.github.com> Date: Wed, 29 Apr 2026 14:04:18 +0000 Subject: [PATCH 128/131] Ensure marshal.load raises a runtimeError when the recursionLimit is reached --- src/Marshal.cpp | 4 ++++ tests/python/bluetests/test/test_marshal.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/src/Marshal.cpp b/src/Marshal.cpp index 23473c8..f3c2623 100644 --- a/src/Marshal.cpp +++ b/src/Marshal.cpp @@ -1381,6 +1381,10 @@ PyObject* Marshal::ReadObject( ReadStream *stream ) { CCP_STATS_ZONE( __FUNCTION__ ); + Incrementor _inc(mRecursionLevel); + if( !_inc.Test( sRecursionLimit ) ) + return nullptr; + PYTYPES type; if (!stream->ReadType(type)) return 0; diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 0d773d4..acbab94 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -226,6 +226,12 @@ def test_nullptr_deref_in_readobjectreference(self): with self.assertRaises(RuntimeError): blue.marshal.Load(bad_payload) + def test_deeply_nested_read(self): + depth = 10000 + payload = b"\x7D\x01" + b"\x25" * depth + b"\x01" + with self.assertRaises(RuntimeError): + blue.marshal.Load(payload) + class TestBackwardsCompatibility(blueunittest.TestCase): """ This class adds coverage for objects marshalled in Python 3. From c872b59ba4e952915cd98cfda1812296de4b7f3e Mon Sep 17 00:00:00 2001 From: CCP Toebeans <105929497+ccptoebeans@users.noreply.github.com> Date: Fri, 8 May 2026 16:46:37 +0000 Subject: [PATCH 129/131] Revert #88-macho-packet-network-byte-order --- CMakeLists.txt | 47 ++---- include/BluePyCpp.h | 8 +- src/BlueResFileSystemLocal.cpp | 15 -- src/BlueResFileSystemLocal.h | 6 +- src/BlueTextResource.h | 2 +- src/CarbonIO/CarbonIO.cpp | 22 +-- src/CarbonIO/CarbonIO.h | 20 +-- src/Marshal.cpp | 16 ++ src/slsocket.cpp | 16 +- src/slsocket_posix.cpp | 5 +- tests/python/bluetests/test/test_marshal.py | 3 +- tests/python/bluetests/test/test_slsocket.py | 158 ------------------- 12 files changed, 66 insertions(+), 252 deletions(-) delete mode 100644 tests/python/bluetests/test/test_slsocket.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c4582f..e81800f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,7 +22,6 @@ find_package(BlueExposure REQUIRED NO_CMAKE_PATH) find_package(CURL REQUIRED NO_CMAKE_PATH) find_package(OpenSSL REQUIRED NO_CMAKE_PATH) find_package(PDM REQUIRED NO_CMAKE_PATH) -find_package(Python REQUIRED NO_CMAKE_PATH) find_package(YAML REQUIRED NO_CMAKE_PATH) find_package(ZLIB REQUIRED NO_CMAKE_PATH) @@ -311,6 +310,7 @@ target_precompile_headers(Blue PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src/StdAfx.h) target_compile_definitions(Blue PUBLIC CCP_STACKLESS + $<$:PY3_COMPATIBILITY_MODE> ) if(APPLE) @@ -371,27 +371,7 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) else() message(FATAL_ERROR "Unsupported platform: " ${CMAKE_SYSTEM_NAME}) endif() - if(WIN32) - set(EXE_EXTENSION ".exe") - set(SCRIPT_EXTENSION ".bat") - else() - set(SCRIPT_EXTENSION ".sh") - endif() - set(PYTHON_INTERPRETER_PATH "${BRANCH_ROOT_DIR}/eve/client/pythonInterpreter${SCRIPT_EXTENSION}") - get_target_property(LIBPYTHON Python IMPORTED_LOCATION) - if(DEFINED CMAKE_BUILD_TYPE AND NOT CMAKE_BUILD_TYPE MATCHES "Release") - set(EXEFILE_NAME exefile_${CMAKE_BUILD_TYPE}${EXE_EXTENSION}) - else() - set(EXEFILE_NAME exefile${EXE_EXTENSION}) - endif() - set(EXEFILE_PATH $/${EXEFILE_NAME}) - add_custom_command( - TARGET Blue POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy -t $ - ${Exefile_ROOT}/${CCP_VENDOR_BIN_PATH}/${EXEFILE_NAME} - ${LIBPYTHON} - ) - execute_process(COMMAND ${PYTHON_INTERPRETER_PATH} discover.py + execute_process(COMMAND ${PYTHON_EXECUTABLE} discover.py OUTPUT_VARIABLE PYTHON_TESTS_STR OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE @@ -404,20 +384,17 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) endif() separate_arguments(PYTHON_TESTS UNIX_COMMAND ${PYTHON_TESTS_STR}) foreach(PYTHON_TEST ${PYTHON_TESTS}) - string(FIND ${PYTHON_TEST} "test" TEST_INDEX) - if( TEST_INDEX GREATER -1 ) # The line contains the string "test" - message(STATUS "Found test ${PYTHON_TEST}") - add_test(NAME ${PYTHON_TEST} - COMMAND ${EXEFILE_PATH} /inherit /buildflavor=$> /py -m unittest -v ${PYTHON_TEST} + message(STATUS "Found test ${PYTHON_TEST}") + add_test(NAME ${PYTHON_TEST} + COMMAND ${PYTHON_EXECUTABLE} -m unittest -v ${PYTHON_TEST} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests/python - ) - if(WIN32) - set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$\;${BRANCH_ROOT_DIR}/carbon/common/stdlib\;${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}\;${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>") - elseif(APPLE) - set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$:${BRANCH_ROOT_DIR}/carbon/common/stdlib:${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_LIB_PATH}:${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>") - else() - message(FATAL_ERROR "Unsupported platform") - endif() + ) + if(WIN32) + set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$\;${BRANCH_ROOT_DIR}/carbon/common/stdlib\;${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}\;${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$<$:PY3_COMPATIBILITY_MODE=True>") + elseif(APPLE) + set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$:${BRANCH_ROOT_DIR}/carbon/common/stdlib:${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}:${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$<$:PY3_COMPATIBILITY_MODsE=True") + else() + message(FATAL_ERROR "Unsupported platform") endif() endforeach() diff --git a/include/BluePyCpp.h b/include/BluePyCpp.h index eafde50..6242985 100644 --- a/include/BluePyCpp.h +++ b/include/BluePyCpp.h @@ -158,10 +158,10 @@ class PyAllocator : public std::allocator return allocate(_Count); } - template - struct rebind { - using other = PyAllocator; - }; +// template +// struct rebind { +// using other = PyAllocator; +// }; }; diff --git a/src/BlueResFileSystemLocal.cpp b/src/BlueResFileSystemLocal.cpp index b06ea4f..7c12a17 100644 --- a/src/BlueResFileSystemLocal.cpp +++ b/src/BlueResFileSystemLocal.cpp @@ -80,7 +80,6 @@ bool BlueResFileSystemLocal::IsDirectory( const std::wstring& dir ) // We have a prefix, look it up in our search paths map. // If found, the value associated with it is a list of paths. - std::shared_lock guard( m_expandedSearchPathsMutex ); ExpandedSearchPathMap_t::iterator found = m_expandedSearchPaths.find( key ); if( found != m_expandedSearchPaths.end() ) { @@ -124,7 +123,6 @@ void BlueResFileSystemLocal::GetDirectoryContents( const wchar_t* dir, std::set< // We have a prefix, look it up in our search paths map. // If found, the value associated with it is a list of paths. - std::shared_lock guard( m_expandedSearchPathsMutex ); ExpandedSearchPathMap_t::iterator found = m_expandedSearchPaths.find( key ); if( found != m_expandedSearchPaths.end() ) { @@ -213,7 +211,6 @@ const wchar_t* BlueResFileSystemLocal::GetSearchPathW( const char* key ) void BlueResFileSystemLocal::ClearSearchPaths() { - std::unique_lock guard( m_expandedSearchPathsMutex ); m_searchPaths.clear(); m_expandedSearchPaths.clear(); } @@ -232,7 +229,6 @@ bool BlueResFileSystemLocal::ResolvePathW( const std::wstring& path, std::wstrin // We have a prefix, look it up in our search paths map. // If found, the value associated with it is a list of paths. - std::shared_lock guard( m_expandedSearchPathsMutex ); ExpandedSearchPathMap_t::iterator found = m_expandedSearchPaths.find( key ); if( found != m_expandedSearchPaths.end() ) { @@ -298,7 +294,6 @@ std::wstring BlueResFileSystemLocal::ResolvePathForWritingW( const std::wstring& std::string key = (const char*)CW2A( keyW.c_str() ); ToLower( key ); - std::shared_lock guard( m_expandedSearchPathsMutex ); ExpandedSearchPathMap_t::iterator found = m_expandedSearchPaths.find( key ); if( found != m_expandedSearchPaths.end() ) { @@ -325,7 +320,6 @@ std::wstring BlueResFileSystemLocal::ResolvePathToRootW( const std::string& root std::string rootLower = root; ToLower( rootLower ); - std::shared_lock guard( m_expandedSearchPathsMutex ); ExpandedSearchPathMap_t::iterator found = m_expandedSearchPaths.find( rootLower ); if( found != m_expandedSearchPaths.end() ) { @@ -364,7 +358,6 @@ void BlueResFileSystemLocal::GetExpandedSearchPaths( const char* key, std::vecto std::string keyLower = key; ToLower( keyLower ); - std::shared_lock guard( m_expandedSearchPathsMutex ); ExpandedSearchPathMap_t::iterator found = m_expandedSearchPaths.find( keyLower ); if( found != m_expandedSearchPaths.end() ) { @@ -474,7 +467,6 @@ std::optional> ExpandSearchPath( const std::wstring& p bool BlueResFileSystemLocal::ExpandSearchPaths() { - std::unique_lock guard( m_expandedSearchPathsMutex ); m_expandedSearchPaths.clear(); for( const auto& searchPath : m_searchPaths ) { @@ -502,7 +494,6 @@ bool BlueResFileSystemLocal::ExpandSearchPaths() void BlueResFileSystemLocal::LogPaths() { - std::shared_lock guard( m_expandedSearchPathsMutex ); CCP_LOG( "BlueOS search paths:" ); for( ExpandedSearchPathMap_t::const_iterator it = m_expandedSearchPaths.begin(); it != m_expandedSearchPaths.end(); ++it ) { @@ -647,9 +638,3 @@ std::vector BlueResFileSystemLocal::ListDirFromScript( const std:: std::vector list( begin( results ), end( results ) ); return list; } - -BlueResFileSystemLocal::ExpandedSearchPathMap_t BlueResFileSystemLocal::GetExpandedSearchPaths() { - std::shared_lock guard( m_expandedSearchPathsMutex ); - return m_expandedSearchPaths; -} - diff --git a/src/BlueResFileSystemLocal.h b/src/BlueResFileSystemLocal.h index 5e4b407..f380e87 100644 --- a/src/BlueResFileSystemLocal.h +++ b/src/BlueResFileSystemLocal.h @@ -10,7 +10,6 @@ #define BlueResFileSystemLocal_h #include "IBlueResFileSystem.h" -#include BLUE_CLASS( BlueResFileSystemLocal ) : public IBlueResFileSystem { @@ -46,7 +45,7 @@ BLUE_CLASS( BlueResFileSystemLocal ) : public IBlueResFileSystem typedef std::map> ExpandedSearchPathMap_t; SearchPathMap_t GetAllSearchPaths() { return m_searchPaths; } - ExpandedSearchPathMap_t GetExpandedSearchPaths(); + ExpandedSearchPathMap_t GetExpandedSearchPaths() { return m_expandedSearchPaths; } std::wstring ResolvePath( const wchar_t* path ); @@ -62,9 +61,6 @@ BLUE_CLASS( BlueResFileSystemLocal ) : public IBlueResFileSystem // Search paths expanded, for quicker resolving of paths ExpandedSearchPathMap_t m_expandedSearchPaths; - // Expanded search paths can be accessed from multiple threads. - std::shared_timed_mutex m_expandedSearchPathsMutex; - // Helper function to expand search paths - called after any entry is changed bool ExpandSearchPaths(); }; diff --git a/src/BlueTextResource.h b/src/BlueTextResource.h index b28cd05..9b3c2c7 100644 --- a/src/BlueTextResource.h +++ b/src/BlueTextResource.h @@ -32,6 +32,6 @@ public BlueAsyncRes, std::string m_text; }; -TYPEDEF_BLUECLASS_SHUTDOWN( BlueTextResource ); +TYPEDEF_BLUECLASS( BlueTextResource ); #endif // BlueTextResource_h diff --git a/src/CarbonIO/CarbonIO.cpp b/src/CarbonIO/CarbonIO.cpp index 3b2523a..f22578a 100644 --- a/src/CarbonIO/CarbonIO.cpp +++ b/src/CarbonIO/CarbonIO.cpp @@ -2778,7 +2778,7 @@ void CarbonIO::dataReceived( SCompletionUnit *completion, const char* data, cons } // a partial packet has been recieved, call for completion - packetSize = ( ceHeaderSizeMask & ntohl( *reinterpret_cast( completion->packetListTail->data ) ) ) + sizeof(int); + packetSize = (ceHeaderSizeMask & *(int *)completion->packetListTail->data) + sizeof(int); need = packetSize - completion->packetListTail->packetLen; D_HANDLEREAD(ciolog("working packet[%d:%d] need[%d] avail[%d] for [%d]", packetSize, completion->packetListTail->packetLen, need, available, (int)completion->workHandle )); @@ -2787,7 +2787,7 @@ void CarbonIO::dataReceived( SCompletionUnit *completion, const char* data, cons else { // packet is empty, call for the full amount - packetSize = ( ceHeaderSizeMask & ntohl( *reinterpret_cast( indat ) ) ) + sizeof( uint32_t ); + packetSize = (ceHeaderSizeMask & *(int *)indat) + sizeof(int); need = packetSize; D_HANDLEREAD(ciolog("not working packet, needing full amount[%d] for[%d] [0x%08X]:[0x%08X]", need, (int)completion->workHandle, *(unsigned int *)indat, ceHeaderSizeMask & *(int *)indat)); @@ -2810,11 +2810,12 @@ void CarbonIO::dataReceived( SCompletionUnit *completion, const char* data, cons appendDataToPacket( completion->packetListTail, indat, need ); available -= need; indat += need; - unsigned int header = ntohl( *reinterpret_cast( completion->packetListTail->data ) ); + + unsigned int header = *(int *)completion->packetListTail->data; if ( header & ceHeaderExpectPayloadOffset ) { completion->packetListTail->oobData = sizeof(int)*2 + completion->packetListTail->data; - completion->packetListTail->oobLen = ntohl( *reinterpret_cast( completion->packetListTail->data + sizeof( uint32_t ) ) ); + completion->packetListTail->oobLen = *(int *)(completion->packetListTail->data + sizeof(int)); completion->packetListTail->payloadOffset = sizeof(int)*2 + completion->packetListTail->oobLen; if ( completion->packetListTail->payloadOffset > completion->packetListTail->packetLen ) @@ -3288,10 +3289,10 @@ bool CarbonIO::processRawQueuedData( SCompletionUnit *completion ) // find the offset to the body (the part we compress) unsigned int offset = sizeof(unsigned int); // the header itself - if ( ntohl( *reinterpret_cast( packet->data ) ) & ceHeaderExpectPayloadOffset ) // was there out-of-band data? + if ( *(unsigned int *)packet->data & ceHeaderExpectPayloadOffset ) // was there out-of-band data? { offset += sizeof(unsigned int); // the size of the len param - offset += *reinterpret_cast( packet->data + sizeof( uint32_t ) ); // the size of the data + offset += *(unsigned int *)(packet->data + sizeof(unsigned int)); // the size of the data } char *outbuf = 0; @@ -3304,9 +3305,9 @@ bool CarbonIO::processRawQueuedData( SCompletionUnit *completion ) { completion->stats.bytesSentCompressed += outlen + offset; - *reinterpret_cast( packet->data ) &= htonl( ceHeaderBitsMask ); // knock off old size - *reinterpret_cast( packet->data ) |= htonl( ( outlen + offset ) - sizeof( uint32_t ) ); // plug in NEW size - *reinterpret_cast( packet->data ) |= htonl( m_compressionType ); + *(unsigned int *)packet->data &= ceHeaderBitsMask; // knock off old size + *(unsigned int *)packet->data |= (outlen + offset) - sizeof(unsigned int); // plug in NEW size + *(unsigned int *)packet->data |= m_compressionType; // if this is not the root packet, append the data TO the // root packet if ( packet != completion->queuedData ) @@ -3934,8 +3935,7 @@ bool CarbonIO::compress( const char *in, unsigned int inLen, char **out, unsigne //------------------------------------------------------------------------------ bool CarbonIO::decompress( SPacket* packet ) { - unsigned long packetHeader = ntohl( *reinterpret_cast( packet->data ) ); - if ( packetHeader & ceHeaderBitZlibCompressed ) + if ( *(unsigned int *)packet->data & ceHeaderBitZlibCompressed ) { CPerformanceTime timeCompress( "decompress" ); diff --git a/src/CarbonIO/CarbonIO.h b/src/CarbonIO/CarbonIO.h index 0538f07..c616c5e 100644 --- a/src/CarbonIO/CarbonIO.h +++ b/src/CarbonIO/CarbonIO.h @@ -558,15 +558,15 @@ bool CarbonIO::formatPacket( char* buf, int pos; if ( OOBData && OOBLen ) { - *reinterpret_cast( buf ) = htonl( (dataLen + OOBLen + sizeof( uint32_t ) ) | ceHeaderExpectPayloadOffset ); - *reinterpret_cast( buf + sizeof( uint32_t ) ) = htonl( OOBLen ); - memcpy( buf + sizeof( uint32_t )*2, OOBData, OOBLen ); - pos = OOBLen + sizeof( uint32_t )*2; + *(unsigned int *)buf = (dataLen + OOBLen + sizeof(unsigned int)) | ceHeaderExpectPayloadOffset; + *(unsigned int *)(buf + sizeof(unsigned int)) = OOBLen; + memcpy( buf + sizeof(unsigned int)*2, OOBData, OOBLen ); + pos = OOBLen + sizeof(unsigned int)*2; } else { - *reinterpret_cast( buf ) = htonl( dataLen ); - pos = sizeof( uint32_t ); + *(unsigned int *)buf = dataLen; + pos = sizeof(unsigned int); } char *outbuf = 0; @@ -576,9 +576,9 @@ bool CarbonIO::formatPacket( char* buf, && checkAndCompress(data, dataLen, &outbuf, &outlen) ) { *len = pos + outlen; - *reinterpret_cast( buf ) &= htonl( ceHeaderBitsMask ); // knock off old size - *reinterpret_cast( buf ) |= htonl( *len - sizeof( uint32_t ) ); // plug in NEW size - *reinterpret_cast( buf ) |= htonl( m_compressionType ); + *(unsigned int *)buf &= ceHeaderBitsMask; // knock off old size + *(unsigned int *)buf |= *len - sizeof(unsigned int); // plug in NEW size + *(unsigned int *)buf |= m_compressionType; memcpy( buf + pos, outbuf, outlen ); delete[] outbuf; } @@ -816,7 +816,7 @@ bool CarbonIO::isPacketValid( SPacket *packet ) return packet && packet->packetLen >= 4 && (packet->auxData - || ( packet->packetLen == ( ntohl( *reinterpret_cast( packet->data ) ) & ceHeaderSizeMask ) + sizeof( int ) ) ); + || (packet->packetLen == (*(int *)packet->data & ceHeaderSizeMask) + sizeof(int))); } //------------------------------------------------------------------------------ diff --git a/src/Marshal.cpp b/src/Marshal.cpp index f3c2623..b78cb42 100644 --- a/src/Marshal.cpp +++ b/src/Marshal.cpp @@ -1558,8 +1558,13 @@ PyObject *Marshal::ReadObjectNewobj(ReadStream* stream, bool shared) if (!cls) return 0; BluePy __new__(PyObject_GetAttr(cls, mStock_New)); +#ifdef PY3_COMPATIBILITY_MODE // Marshalled new-style objects from Python3 may have to be constructed as old-style objects in Python2.7 BluePy r(PyObject_HasAttr( cls, mStock_New ) ? BluePy(PyObject_CallObject(__new__, args)) : BluePy(PyInstance_NewRaw(cls, 0))); +#else +if (!__new__) return 0; +BluePy r(PyObject_CallObject(__new__, args)); +#endif if (!r) return 0; //object is constructed, now update r @@ -1663,6 +1668,7 @@ PyObject *Marshal::GetGlobalObject(PyObject *nameO) const char *dot = strrchr(name, '.'); BluePyStr modulename; if (dot){ +#if PY3_COMPATIBILITY_MODE if( strncmp( name, "builtins.", 9 ) == 0 ) { modulename = BluePyStr( "__builtin__" ); } @@ -1673,6 +1679,9 @@ PyObject *Marshal::GetGlobalObject(PyObject *nameO) } else { modulename = BluePyStr("__builtin__"); } +#else + modulename = BluePyStr(dot-name, name); +#endif BluePy module(PyImport_ImportModule((char*)modulename.Str())); if (!module) return 0; @@ -1878,6 +1887,13 @@ bool Marshal::WriteObject(WriteStream* stream, PyObject* o) { RETFAIL(WriteType(stream, TY_STR_EMPTY)); } +#ifndef PY3_COMPATIBILITY_MODE + else if (size == 1) + { + RETFAIL(WriteType(stream, TY_STR_CHAR)); + RETFAIL(stream->Write(string[0])); + } +#endif else { PyObject* index = PyDict_GetItem(mStrTable, o); diff --git a/src/slsocket.cpp b/src/slsocket.cpp index 2adeb73..89c88eb 100644 --- a/src/slsocket.cpp +++ b/src/slsocket.cpp @@ -1150,15 +1150,14 @@ class SendPacketResult : public SendResult buffers.Push(header); SSIZE_T result = header.len; result += SendResult::Prepare(buffers, obj, pbuf); - auto cargoBytes = static_cast( result - sizeof(mHeader) ); //number of cargo bytes - if ( cargoBytes > static_cast( mXtra->GetMaxPacketSize() ) ) { + mHeader = (DWORD)(result-sizeof(mHeader)); //number of cargo bytes + if (mHeader > (DWORD)mXtra->GetMaxPacketSize()) { char tmp[128]; sprintf_s(tmp, "packet too long at %d bytes, max size is %d", - cargoBytes, mXtra->GetMaxPacketSize()); + mHeader, mXtra->GetMaxPacketSize()); OutputDebugString(tmp); throw std::length_error(tmp); } - mHeader = htonl( cargoBytes ); return result; } @@ -1277,31 +1276,30 @@ class RecvPacketResult : public IOOverlappedRecv { mBytesRead += bytesTransfered; bool more; - uint32_t cargoBytes{ntohl( mHeader )}; if (mBytesRead < sizeof(mHeader)) { //continue reading header more = true; } else if (mBytesRead == sizeof(mHeader)) { //just finished reading header, allocate buffer. - if ( cargoBytes > static_cast( mXtra->GetMaxPacketSize() ) ) { + if (mHeader > (DWORD)mXtra->GetMaxPacketSize()) { char tmp[128]; sprintf_s(tmp, "too large a packet detected at %d bytes, max is %d", mHeader, mXtra->GetMaxPacketSize()); throw std::length_error(tmp); } more = mHeader!=0; - } else if (mBytesRead < static_cast( sizeof( mHeader ) + cargoBytes ) ) { + } else if (mBytesRead < (int)(sizeof(mHeader)+mHeader)) { //header here but packet not finished more = true; } else { //packet finished. //did we read any of the next guy's header? (we requested next header's amount of data :) - uint32_t rest = mBytesRead - ( sizeof( mHeader ) + cargoBytes ); + DWORD rest = mBytesRead - (sizeof(mHeader) + mHeader); if (rest) { RecvPacketResult *next = mXtra->NextRecv(this); _ASSERT(next); next->OnHeaderBytesRead(rest); - mBytesRead = static_cast( sizeof( mHeader ) + cargoBytes ); + mBytesRead = (int)(sizeof(mHeader)+mHeader); } mXtra->mStats.PacketReceived(); more = false; diff --git a/src/slsocket_posix.cpp b/src/slsocket_posix.cpp index 7462c86..b1ed15a 100644 --- a/src/slsocket_posix.cpp +++ b/src/slsocket_posix.cpp @@ -1625,7 +1625,7 @@ class SendPacketResult : public SendBase // Copy address // steal the buffer StealBuffer(buf); - mHeader = htonl(mKeeper.len); + mHeader = mKeeper.len; FlipHeader(mHeader); Request(s->sock_timeout); return mResult; @@ -1834,7 +1834,7 @@ class RecvPacketResult : public SocketResult if (mBytesRead < (sizeof(mHeader) + sizeof(uint32_t))) return; - uint32_t oobDataLen = ntohl(*(uint32_t *)(mData)); + uint32_t oobDataLen = *(uint32_t *)(mData); // sanity check the out-of-band data length; mPacketSize was sanity checked already if (oobDataLen > mPacketSize) { char tmp[128] = {'\0'}; @@ -1908,7 +1908,6 @@ class RecvPacketResult : public SocketResult rcvd = recv(handle, (char*)&mHeader + mBytesRead, sizeof(mHeader) - mBytesRead, 0); if (rcvd > 0 && mBytesRead+rcvd == 4) { // We completed reading the header, do stuff! - mHeader = htonl(mHeader); FlipHeader(mHeader); mPacketSize = (mHeader & ceHeaderSizeMask); if (mPacketSize > (uint32_t)GetXtra()->GetMaxPacketSize()) { diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index acbab94..6d30700 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -122,7 +122,7 @@ def test_integer(self): def test_long(self): self.verify_round_trip(42) - self.verify_round_trip(9223372036854775807L) + self.verify_round_trip(9223372036854775807) def test_float(self): self.verify_round_trip(0.0) @@ -232,6 +232,7 @@ def test_deeply_nested_read(self): with self.assertRaises(RuntimeError): blue.marshal.Load(payload) +@unittest.skipUnless(os.environ.has_key("PY3_COMPATIBILITY_MODE"), "Skipping because these tests will fail if `PY3_COMPATIBILITY_MODE` is not enabled") class TestBackwardsCompatibility(blueunittest.TestCase): """ This class adds coverage for objects marshalled in Python 3. diff --git a/tests/python/bluetests/test/test_slsocket.py b/tests/python/bluetests/test/test_slsocket.py deleted file mode 100644 index 632f1ce..0000000 --- a/tests/python/bluetests/test/test_slsocket.py +++ /dev/null @@ -1,158 +0,0 @@ -import functools -import os -import socket -import sys -import unittest - -import blue -import stackless - -LOCALHOST = "127.0.0.1" - -def find_port(): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - try: - s.bind((LOCALHOST, 0)) - port = s.getsockname()[1] - return port - finally: - s.close() - -def run_on_server(channel, function): - server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - try: - server.setmaxpacketsize(10 * 1024 * 1024) - server.bind(("127.0.0.1", 0)) - channel.send(server.getsockname()) - server.listen(0) - connection, address = server.accept() - function(connection) - finally: - server.close() - -def run_on_client(channel, function): - client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - host_address = channel.receive() - client.connect(host_address) - client.setmaxpacketsize(10 * 1024 * 1024) - client.setblockingsend(True) - function(client) - finally: - client.close() - -def send_packet(packet, oob_data, connection): - if oob_data is not None: - connection.sendpacket(packet, oob_data) - else: - connection.sendpacket(packet) - -def receive_packet(connection): - receive_packet.received = connection.recvpacketoob() -receive_packet.received = None - -def send(data, connection): - connection.send(data) - -def receive(connection): - receive.received = connection.recv(65535) -receive.received = None - -def receive_bytes(byte_count, connection): - receive_bytes.received = "" - while len(receive_bytes.received) < byte_count: - data = connection.recv(byte_count - len(receive_bytes.received)) - receive_bytes.received += data -receive_bytes.received = None - -def run_connected(client_func, server_func): - channel = stackless.channel() - c = stackless.tasklet(run_on_client)(channel, client_func) - s = stackless.tasklet(run_on_server)(channel, server_func) - sleeptime = blue.os.sleeptime - try: - blue.os.sleeptime = 100 - while c.alive or s.alive: - blue.os.Pump() - finally: - blue.os.sleeptime = sleeptime - -class TestCarbonIO(unittest.TestCase): - - def setUp(self): - try: - import carbonio - except ImportError: - raise unittest.SkipTest("CarbonIO not supported on this system") - import _slsocket - _slsocket.use_carbonio(True) - carbonio._socket = _slsocket - sys.modules["_socket"] = _slsocket - sys.modules["select"] = None - reload(socket) # Make sure the socket module is using _slsocket instead of _socket - def test_send_receive_packet(self): - PACKET_DATA = "data" - OOB_DATA = None - run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), receive_packet) - self.assertEqual(receive_packet.received, (PACKET_DATA, "", 1)) - - def test_sent_packet_format(self): - PACKET_DATA = "data" - OOB_DATA = None - run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), receive) - self.assertEqual(receive.received, "\x00\x00\x00\x04data") - - def test_receive_formatted_packet(self): - PACKET_DATA = "\x00\x00\x00\x04data" - run_connected(functools.partial(send, PACKET_DATA), receive_packet) - self.assertEqual(receive_packet.received, ("data", "", 1)) - - def test_sent_packet_format_with_oob_data(self): - PACKET_DATA = "Hello" - OOB_DATA = "World" - run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), receive) - self.assertEqual(receive.received, "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello") - - def test_receive_formatted_packet_with_oob_data(self): - PACKET_DATA = "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello" - run_connected(functools.partial(send, PACKET_DATA), receive_packet) - self.assertEqual(receive_packet.received, ("Hello", "World", 1)) - -class TestStacklessIO(unittest.TestCase): - - def setUp(self): - import stacklessio - import _slsocket - import slselect - if hasattr(_slsocket, "use_carbonio"): - _slsocket.use_carbonio(False) - stacklessio._socket = _slsocket - sys.modules["_socket"] = _slsocket - sys.modules["select"] = slselect - reload(socket) # Make sure the socket module is using _slsocket instead of _socket - - def test_send_receive_packet(self): - PACKET_DATA = "data" - OOB_DATA = None - run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), receive_packet) - self.assertEqual(receive_packet.received, (PACKET_DATA, None, 0)) - - def test_sent_packet_format(self): - PACKET_DATA = "data" - OOB_DATA = None - run_connected(functools.partial(send_packet, PACKET_DATA, OOB_DATA), functools.partial(receive_bytes, 8)) - self.assertEqual(receive_bytes.received, "\x00\x00\x00\x04data") - - def test_receive_formatted_packet(self): - PACKET_DATA = "\x00\x00\x00\x04data" - run_connected(functools.partial(send, PACKET_DATA), receive_packet) - self.assertEqual(receive_packet.received, ("data", None, 0)) - - def test_receive_formatted_packet_with_oob_data(self): - if sys.platform == "win32": - raise unittest.SkipTest("StacklessIO does not support OOB data on Windows") - PACKET_DATA = "\x10\x00\x00\x0E\x00\x00\x00\x05WorldHello" - run_connected(functools.partial(send, PACKET_DATA), receive_packet) - self.assertEqual(receive_packet.received, ("\x00\x00\x00\x05WorldHello", None, 0)) From f1c3ed4ddf0d5aab13c2fcf3123515e5768111ca Mon Sep 17 00:00:00 2001 From: CCP Toebeans <105929497+ccptoebeans@users.noreply.github.com> Date: Fri, 8 May 2026 17:05:39 +0000 Subject: [PATCH 130/131] Fix bad revert in Marhsal.cpp --- src/Marshal.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Marshal.cpp b/src/Marshal.cpp index b78cb42..cbaf785 100644 --- a/src/Marshal.cpp +++ b/src/Marshal.cpp @@ -1675,13 +1675,13 @@ PyObject *Marshal::GetGlobalObject(PyObject *nameO) else { modulename = BluePyStr( dot-name, name ); } +#else + modulename = BluePyStr(dot-name, name); +#endif name = dot+1; } else { modulename = BluePyStr("__builtin__"); } -#else - modulename = BluePyStr(dot-name, name); -#endif BluePy module(PyImport_ImportModule((char*)modulename.Str())); if (!module) return 0; From d525c8aa25c6cdaba4380a5fe33e92fb121df01f Mon Sep 17 00:00:00 2001 From: CCP Aporia <28982391+ccp-aporia@users.noreply.github.com> Date: Tue, 8 Sep 2026 10:24:28 +0000 Subject: [PATCH 131/131] Fix type size handling The bit masking could silently return the wrong size for unsupported types. Fixes PLAT-11751 --- src/PyRowSet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PyRowSet.cpp b/src/PyRowSet.cpp index 1184083..c726eb8 100644 --- a/src/PyRowSet.cpp +++ b/src/PyRowSet.cpp @@ -104,7 +104,7 @@ struct ColumnDescriptor ColumnDescriptor(const char *name) : mName(name), mType(0), mOffset(0), mSize(0){} static bool TypeSize(const DBTYPE &type, int &size) { - switch(type & 0xff) { + switch(type) { case DBTYPE_BOOL: size = 0; break; case DBTYPE_I1: