diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..039810c --- /dev/null +++ b/.clang-format @@ -0,0 +1,152 @@ +--- +Language: Cpp +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Right +AlignOperands: false +AlignTrailingComments: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: None +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterClass: true + AfterControlStatement: true + AfterEnum: true + AfterFunction: true + AfterNamespace: true + AfterObjCDeclaration: true + AfterStruct: true + AfterUnion: true + AfterExternBlock: true + BeforeCatch: true + BeforeElse: true + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +BreakBeforeTernaryOperators: false +BreakConstructorInitializers: AfterColon +BreakStringLiterals: false +ColumnLimit: 0 +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: false +DerivePointerAlignment: false +FixNamespaceComments: false +IncludeBlocks: Preserve +IndentCaseLabels: false +IndentPPDirectives: None +IndentWidth: 4 +IndentWrappedFunctionNames: true +KeepEmptyLinesAtTheStartOfBlocks: true +MacroBlockBegin: 'EXPOSURE_BEGIN' +MacroBlockEnd: 'EXPOSURE_END|EXPOSURE_CHAINTO' +MaxEmptyLinesToKeep: 4 +NamespaceIndentation: None +PointerAlignment: Left +ReflowComments: false +SortIncludes: false +SortUsingDeclarations: false +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: Never +SpaceInEmptyParentheses: false +SpacesInAngles: false +SpacesInCStyleCastParentheses: false +SpacesInContainerLiterals: false +SpacesInParentheses: true +SpacesInSquareBrackets: false +Standard: Cpp11 +TabWidth: 4 +UseTab: ForContinuationAndIndentation +--- +Language: ObjC +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Right +AlignOperands: false +AlignTrailingComments: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: None +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterClass: true + AfterControlStatement: true + AfterEnum: true + AfterFunction: true + AfterNamespace: true + AfterObjCDeclaration: true + AfterStruct: true + AfterUnion: true + AfterExternBlock: true + BeforeCatch: true + BeforeElse: true + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +BreakBeforeTernaryOperators: false +BreakConstructorInitializers: AfterColon +BreakStringLiterals: false +ColumnLimit: 120 +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: false +DerivePointerAlignment: false +FixNamespaceComments: false +IncludeBlocks: Preserve +IndentCaseLabels: false +IndentPPDirectives: None +IndentWidth: 4 +IndentWrappedFunctionNames: true +KeepEmptyLinesAtTheStartOfBlocks: true +MacroBlockBegin: 'EXPOSURE_BEGIN' +MacroBlockEnd: 'EXPOSURE_END|EXPOSURE_CHAINTO' +MaxEmptyLinesToKeep: 4 +NamespaceIndentation: None +ObjCBlockIndentWidth: 4 +PointerAlignment: Left +ReflowComments: false +SortIncludes: false +SortUsingDeclarations: false +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: Never +SpaceInEmptyParentheses: false +SpacesInAngles: false +SpacesInCStyleCastParentheses: false +SpacesInContainerLiterals: false +SpacesInParentheses: true +SpacesInSquareBrackets: false +Standard: Cpp11 +TabWidth: 4 +UseTab: ForContinuationAndIndentation +... diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b74533..e81800f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -304,10 +304,13 @@ target_sources(Blue PRIVATE ${PUBLIC_HEADER_FILES_INCLUDE} ) +option(ENABLE_PY3_COMPATIBILITY "Build with Python3 forward compatibility enabled" OFF) + target_precompile_headers(Blue PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src/StdAfx.h) target_compile_definitions(Blue PUBLIC CCP_STACKLESS + $<$:PY3_COMPATIBILITY_MODE> ) if(APPLE) @@ -338,7 +341,12 @@ if(WIN32) winhttp psapi iphlpapi -) + ) + add_custom_command( + TARGET Blue POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy $ $ + COMMAND_EXPAND_LISTS + ) endif() target_include_directories(Blue @@ -378,13 +386,13 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) foreach(PYTHON_TEST ${PYTHON_TESTS}) message(STATUS "Found test ${PYTHON_TEST}") add_test(NAME ${PYTHON_TEST} - COMMAND ${PYTHON_EXECUTABLE} -m unittest -v ${PYTHON_TEST} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests/python + COMMAND ${PYTHON_EXECUTABLE} -m unittest -v ${PYTHON_TEST} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests/python ) if(WIN32) - set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$\;${BRANCH_ROOT_DIR}/carbon/common/stdlib\;${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}\;${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>") + set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$\;${BRANCH_ROOT_DIR}/carbon/common/stdlib\;${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}\;${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$<$:PY3_COMPATIBILITY_MODE=True>") elseif(APPLE) - set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$:${BRANCH_ROOT_DIR}/carbon/common/stdlib:${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}:${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>") + set_tests_properties(${PYTHON_TEST} PROPERTIES ENVIRONMENT "PYTHONPATH=$:${BRANCH_ROOT_DIR}/carbon/common/stdlib:${BRANCH_ROOT_DIR}/vendor/python/2.7.1+ccp-stackless/${CCP_VENDOR_BIN_PATH}:${BRANCH_ROOT_DIR}/packages;BUILDFLAVOR=$>;$<$:PY3_COMPATIBILITY_MODsE=True") else() message(FATAL_ERROR "Unsupported platform") endif() diff --git a/include/BlueAsyncRes.h b/include/BlueAsyncRes.h index 9bdb943..9e95c43 100644 --- a/include/BlueAsyncRes.h +++ b/include/BlueAsyncRes.h @@ -35,10 +35,6 @@ BLUE_BLUEIMPORT_CLASS( BlueAsyncRes ) : virtual ~BlueAsyncRes() { CCP_ASSERT( m_reloadNotifyTargetsCount == 0 ); - - #if CCP_TELEMETRY_ENABLED - tmFree( TMCM_GENERAL, this ); - #endif } // This gets called right before destruction - safe to call virtual functions. diff --git a/include/BluePyCpp.h b/include/BluePyCpp.h index a5c1f11..6242985 100644 --- a/include/BluePyCpp.h +++ b/include/BluePyCpp.h @@ -157,7 +157,12 @@ class PyAllocator : public std::allocator typename _p::pointer allocate(typename _p::size_type _Count, const void *) { return allocate(_Count); } -}; + +// template +// struct rebind { +// using other = PyAllocator; +// }; +}; // A simple smartpointer to own references. diff --git a/include/BlueStatistics.h b/include/BlueStatistics.h index bbf8138..0eed1a4 100644 --- a/include/BlueStatistics.h +++ b/include/BlueStatistics.h @@ -11,6 +11,7 @@ #define BlueStatistics_h #include "ICcpStatisticsAccumulator.h" +#include BLUE_DECLARE( CcpStatisticsEntry ); @@ -85,6 +86,7 @@ BLUE_CLASS( BlueStatistics ) : public IRoot float TelemetrySamplingTimeLeft(); bool IsTelemetryConnected(); bool IsTelemetryPaused(); + bool IsTelemetryStarted(); void SetCppCaptureEnabled( bool b ); bool IsCppCaptureEnabled(); @@ -152,13 +154,34 @@ void BLUEIMPORT tmTaskletEnter( uint32_t ctx, const char* name ); void BLUEIMPORT tmTaskletLeave( uint32_t ctx ); void tmTaskletAppendText( uint32_t ctx, const char* appendText ); +class TracyZone +{ +public: + TracyZone() = delete; + BLUEIMPORT TracyZone( uint32_t ctx, const char* name, const char* filename, uint32_t lineno, uint32_t color = tracy::Color::SteelBlue4 ); + BLUEIMPORT ~TracyZone(); + TracyZone( TracyZone&& other ) noexcept; + TracyZone( const TracyZone& ) = delete; + TracyZone& operator=( TracyZone&& ) = delete; + TracyZone& operator=( const TracyZone& ) = delete; + + void text( const char* text ) const; + +private: + std::optional m_telemetryContext; + void* m_fiber{nullptr}; +}; + +void BLUEIMPORT TracyEnterZone( void* key, const char* name, const char* filename, uint32_t lineno ); +void BLUEIMPORT TracyLeaveZone( void* key ); + #define CCP_STATS_SCOPED_TIME( identifier ) \ - tmTaskletZone zone_##_COUNTER_( TMCM_CPP, g_ccpStatistics_##identifier.GetName().c_str() );\ + TracyZone tracy_zone_##__COUNTER__( TMCM_CPP, g_ccpStatistics_##identifier.GetName().c_str(), __FILE__, __LINE__ );\ CcpStatisticsStopwatch ccpStatsStopwatch_##identifier( g_ccpStatistics_##identifier ) #undef CCP_STATS_ZONE #define CCP_STATS_ZONE( name ) \ - tmTaskletZone zone_##_COUNTER_( TMCM_CPP, name ) + TracyZone tracy_zone_##__COUNTER__( TMCM_CPP, name, __FILE__, __LINE__ ); #else #define CCP_STATS_SCOPED_TIME( identifier ) CcpStatisticsStopwatch ccpStatsStopwatch_##identifier( g_ccpStatistics_##identifier ) diff --git a/src/AllReferences.cpp b/src/AllReferences.cpp index 7e948a9..ad170c0 100644 --- a/src/AllReferences.cpp +++ b/src/AllReferences.cpp @@ -201,12 +201,15 @@ BluePy AllReferences::GetReferences( IRoot* obj ) Py_XDECREF( element ); }; - if( found != end( m_references ) ) + if( found != end( m_references ) && !IsOutdated( found->second ) ) { AddRef( found->second.first ); for( auto& rec : found->second.rest ) { - AddRef( rec ); + if( !IsOutdated( rec ) ) + { + AddRef( rec ); + } } } return BluePy( result ); @@ -280,7 +283,7 @@ bool AllReferences::HasRoute( IRoot* from, IRoot* to, std::unordered_mapsecond ) ) { hasRoute[to] = false; return false; @@ -292,7 +295,7 @@ bool AllReferences::HasRoute( IRoot* from, IRoot* to, std::unordered_mapsecond.rest ) { - if( HasRoute( from, ref.parent, hasRoute ) ) + if( !IsOutdated( ref ) && HasRoute( from, ref.parent, hasRoute ) ) { hasRoute[to] = true; return true; @@ -300,4 +303,14 @@ bool AllReferences::HasRoute( IRoot* from, IRoot* to, std::unordered_map& hasRoute ) const; + bool IsOutdated( const Reference& ref ) const; + bool IsOutdated( const References& refs ) const; IRootPtr m_root; std::vector m_stack; diff --git a/src/BackgroundReader.cpp b/src/BackgroundReader.cpp index 0037353..eafb23d 100644 --- a/src/BackgroundReader.cpp +++ b/src/BackgroundReader.cpp @@ -32,7 +32,9 @@ void BackgroundReader::Perform() if (!resFile->OpenW( m_filename.c_str(), true ) ) { - m_result = Be::Result( "Couldn't open file" ); + std::string msg = "Couldn't open file: "; + msg += CW2A(m_filename.c_str()); + m_result = Be::Result( msg ); return; } diff --git a/src/BlueAsyncRes.cpp b/src/BlueAsyncRes.cpp index 122e2df..6563e17 100644 --- a/src/BlueAsyncRes.cpp +++ b/src/BlueAsyncRes.cpp @@ -199,9 +199,6 @@ void BlueAsyncRes::InitializeImpl( const wchar_t* name, const wchar_t* ext ) CW2A asciiName( name ); CCP_LOG( "BlueAsyncRes::Initialize: %s", (const char*)asciiName ); -#if CCP_TELEMETRY_ENABLED - tmAlloc( TMCM_GENERAL, this, 1111111, "%s", tmDynamicString( TMCM_GENERAL, (const char*)asciiName ) ); -#endif m_ext = ext; diff --git a/src/BlueOS.cpp b/src/BlueOS.cpp index ce9b746..b675240 100644 --- a/src/BlueOS.cpp +++ b/src/BlueOS.cpp @@ -913,11 +913,7 @@ void BlueOS::PumpOSInternal() // Sleep until we need to wake up { - //CCP_STATS_ZONE( "BlueOS/PumpOS/DoSleep" ); - // Defining this directly to be able to mark this as an "idle" telemetry zone -#if CCP_TELEMETRY_ENABLED - tmZone( TMCM_GENERAL, TMZF_IDLE, "BlueOS/PumpOS/DoSleep" ); -#endif + CCP_STATS_ZONE( "BlueOS/PumpOS/DoSleep" ); DoSleep(); mNextScheduledEvent = int( mSleepTime ); } diff --git a/src/BluePaths.cpp b/src/BluePaths.cpp index cf399f0..482c005 100644 --- a/src/BluePaths.cpp +++ b/src/BluePaths.cpp @@ -300,7 +300,9 @@ Be::Result BluePaths::Open( const std::wstring& filename, Be::Optio *stream = fileStream.Detach(); return Be::Result(); } - return Be::Result("Couldn't create file"); + std::string msg = "Couldn't open file "; + msg += CW2A( filename.c_str() ); + return Be::Result( msg ); } else if( fileMode[0] == 'a' ) { @@ -312,7 +314,9 @@ Be::Result BluePaths::Open( const std::wstring& filename, Be::Optio *stream = fileStream.Detach(); return Be::Result(); } - return Be::Result("Couldn't open file"); + std::string msg = "Couldn't open file "; + msg += CW2A( filename.c_str() ); + return Be::Result( msg ); } } } @@ -322,7 +326,9 @@ Be::Result BluePaths::Open( const std::wstring& filename, Be::Optio return Be::Result(); } - return Be::Result("Couldn't open file"); + std::string msg = "Couldn't open file "; + msg += CW2A( filename.c_str() ); + return Be::Result( msg ); } Be::Result BluePaths::GetFileContentsWithYield( const std::wstring& path, IBlueStream** contents ) diff --git a/src/BlueResFile2_Blue.cpp b/src/BlueResFile2_Blue.cpp index 72505a9..7ff2bd6 100644 --- a/src/BlueResFile2_Blue.cpp +++ b/src/BlueResFile2_Blue.cpp @@ -56,7 +56,9 @@ static PyObject* PyOpenAlways( PyObject* self, PyObject* args ) if( !ok ) { - PyErr_SetString( PyExc_BlueError, "Couldn't open file" ); + std::string msg = "Couldn't open file "; + msg += CW2A( (const wchar_t*)PyUnicode_AsUnicode( filename ) ); + PyErr_SetString( PyExc_BlueError, msg.c_str() ); return nullptr; } diff --git a/src/BlueStatistics.cpp b/src/BlueStatistics.cpp index 8d22084..664df3f 100644 --- a/src/BlueStatistics.cpp +++ b/src/BlueStatistics.cpp @@ -8,21 +8,28 @@ #include "StdAfx.h" #include "IBlueOS.h" +static CcpLogChannel_t s_ch = CCP_LOG_DEFINE_CHANNEL( "Telemetry" ); + static CBlueStatistics s_statisticsInstance; BlueStatistics* g_statistics = &s_statisticsInstance; BLUE_REGISTER_GLOBAL_AS_MODULE_OBJECT( "statistics", g_statistics ); -static bool s_isTelemetryConnectionRequested = false; -static bool s_isTelemetryConnected = false; +enum ProfilerState { + Stopped, + StartRequested, + Started, + StopRequested, +}; + +std::atomic s_profilerState{ProfilerState::Stopped}; + static bool s_isTelemetryCppCaptureEnabled = true; static bool s_isTelemetryTaskletCaptureEnabled = true; static bool s_isTelemetryPythonCaptureEnabled = false; -static bool s_isTelemetryPaused = false; static float s_telemetrySamplePeriod = 0.0f; // In seconds #if CCP_TELEMETRY_ENABLED -static bool s_isTelemetryShuttingDown = false; static int s_telemetryConnectionType = 0; static std::string s_telemetryServerOrFileSystemDumpPath; static Be::Time s_telemetryStartTime; @@ -68,13 +75,28 @@ const TaskletInfo s_fallbackInfo = { "Tasklet?", "", 0 }; TaskletInfo s_lastTasklet = s_fallbackInfo; // need to remember last activated tasklet for tmEnd std::unordered_map s_taskletFree; // original tp_free functions for tasklet types -tm_uint32 s_taskletTrackID = 0; // telemetry track ID for tasklet time spans +uint32_t s_taskletTrackID = 0; // telemetry track ID for tasklet time spans + +thread_local PyTaskletObject* g_activeFiber{nullptr}; +typedef std::unordered_map FiberNameStore; +FiberNameStore g_fiberNameStore; // Persisted fiber name string store + +std::unordered_map g_fiberEraseMap; // Map of fibers scheduled for erasure + +typedef std::unordered_map>> TasketZoneStore; +thread_local TasketZoneStore g_taskletZoneStore; // Per-thread record of zones instrumented from python // Overriden tp_free function for tasklets: notify telemetry and call original tp_free void OnTaskletFree( void* tasklet ) { - tmEndFiber( 0, reinterpret_cast( tasklet ) ); - + g_taskletZoneStore.erase( (PyTaskletObject*) tasklet ); + g_fiberEraseMap.emplace( (PyTaskletObject*) tasklet, BeOS->GetActualTime()); + if (g_activeFiber && g_activeFiber == tasklet) + { + // Catch an edge case where the profiler is stopped and a tasklet switch + // away from this tasklet is not reflected in the active fiber + g_activeFiber = nullptr; + } auto found = s_taskletFree.find( Py_TYPE( tasklet ) ); if( found != end( s_taskletFree ) ) { @@ -148,13 +170,16 @@ int PythonProfiler( PyObject* obj, PyFrameObject* frame, int what, PyObject* arg { switch( what ) { - case PyTrace_CALL: - tmEnterEx( 0, nullptr, 0, 0, PyString_AsString( frame->f_code->co_filename ), PyFrame_GetLineNumber( frame ), TMZF_NONE, "%s", PyString_AsString( frame->f_code->co_name ) ); - tmZoneColor( 0, 94.f / 255.f, 32.f / 255.f ); - break; + case PyTrace_CALL: { + auto zoneName = Immortalize( frame->f_code->co_name ); + auto fileName = Immortalize( frame->f_code->co_filename ); + if( zoneName && fileName ) + TracyEnterZone( frame, zoneName, fileName, PyFrame_GetLineNumber( frame ) ); + } + break; case PyTrace_EXCEPTION: case PyTrace_RETURN: - tmLeave( 0 ); + TracyLeaveZone( frame ); break; default: break; @@ -193,16 +218,16 @@ void BlueStatistics::StartTelemetry( const std::string& server ) void BlueStatistics::StartTimedTelemetry( const std::string& server, float samplePeriod ) { #if CCP_TELEMETRY_ENABLED - if( s_isTelemetryConnected ) + if( s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started || s_profilerState.load( std::memory_order_acquire ) == ProfilerState::StartRequested ) { - CCP_LOGERR( "Telemetry is already running!" ); return; } s_telemetryServerOrFileSystemDumpPath = server; - s_telemetryConnectionType = TMCT_TCP; s_telemetrySamplePeriod = (float)samplePeriod; - s_isTelemetryConnectionRequested = true; + + CCP_LOG_CH( s_ch, "Profiler start requested" ); + s_profilerState.store( ProfilerState::StartRequested, std::memory_order_release ); #else #endif } @@ -210,60 +235,49 @@ void BlueStatistics::StartTimedTelemetry( const std::string& server, float sampl void BlueStatistics::StartTelemetryDump( const std::string& dumpFolder, float samplePeriod ) { #if CCP_TELEMETRY_ENABLED - if( s_isTelemetryConnected ) + if( s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started || s_profilerState.load( std::memory_order_acquire ) == ProfilerState::StartRequested ) { - CCP_LOGERR( "Telemetry is already running!" ); return; } - s_telemetryConnectionType = TMCT_FILE; s_telemetryServerOrFileSystemDumpPath = dumpFolder; s_telemetrySamplePeriod = (float)samplePeriod; - s_isTelemetryConnectionRequested = true; + + s_profilerState.store( ProfilerState::StartRequested, std::memory_order_release ); #else #endif } void BlueStatistics::PauseTelemetry() { -#if CCP_TELEMETRY_ENABLED - tmPause( TMCM_GENERAL, 1 ); - s_isTelemetryPaused = true; -#endif + // Deprecated } void BlueStatistics::ResumeTelemetry() { -#if CCP_TELEMETRY_ENABLED - tmPause( TMCM_GENERAL, 0 ); - s_isTelemetryPaused = false; -#endif + // Deprecated } void BlueStatistics::StopTelemetry() { #if CCP_TELEMETRY_ENABLED - if( s_isTelemetryShuttingDown ) + if( s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Stopped || s_profilerState.load( std::memory_order_acquire ) == ProfilerState::StopRequested ) { return; } - if( s_isTelemetryConnected ) - { - s_isTelemetryConnected = false; - s_isTelemetryShuttingDown = true; - tmPause( TMCM_GENERAL, 1 ); - } + CCP_LOG_CH( s_ch, "Profiler stop requested" ); + s_profilerState.store( ProfilerState::StopRequested, std::memory_order_release ); #endif } bool BlueStatistics::IsTelemetryConnected() { - return s_isTelemetryConnected; + return TracyIsStarted && TracyIsConnected; } bool BlueStatistics::IsTelemetryConnectionRequested() { - return s_isTelemetryConnectionRequested; + return TracyIsStarted && !TracyIsConnected && s_profilerState.load( std::memory_order_acquire ) == ProfilerState::StartRequested; } float BlueStatistics::TelemetrySamplingTimeLeft() @@ -273,108 +287,151 @@ float BlueStatistics::TelemetrySamplingTimeLeft() bool BlueStatistics::IsTelemetryPaused() { - return s_isTelemetryPaused; + return false; // Deprecated +} + +bool BlueStatistics::IsTelemetryStarted() +{ + return s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started; } void BlueStatistics::UpdateTelemetry() { #if CCP_TELEMETRY_ENABLED - if( s_isTelemetryConnectionRequested && !s_isTelemetryShuttingDown ) + switch ( s_profilerState.load(std::memory_order_acquire) ) { - s_isTelemetryConnected = CcpStartTelemetry( s_telemetryServerOrFileSystemDumpPath.c_str(), s_telemetryConnectionType, m_telemetryMaxThreadCount ); - - if( s_isTelemetryConnected ) + case ProfilerState::StartRequested: { -#if CCP_STACKLESS - if( s_isTelemetryPythonCaptureEnabled ) + if (TracyIsStarted) { - // There doesn't seem to be an easy way to back up the previous profiler function, so it gets lost - PyEval_SetProfile( &PythonProfiler, nullptr ); + if (TracyIsConnected) + { + CCP_LOG_CH( s_ch, "UpdateTelemetry - Telemetry server connected to Profiler" ); + TracySetProgramName( s_telemetryServerOrFileSystemDumpPath.c_str() ); + if ( s_isTelemetryPythonCaptureEnabled ) { + PyEval_SetProfile( &PythonProfiler, nullptr ); + } + s_profilerState.store( ProfilerState::Started, std::memory_order_release ); + s_telemetryLastCheckTime = s_telemetryStartTime = BeOS->GetActualTime(); + } } - - if( s_isTelemetryTaskletCaptureEnabled ) + else { - s_taskletTrackID = tmNewTimeSpanTrackID(); - tmTrackName( 0, s_taskletTrackID, "Tasklets" ); - tmTrackOrder( 0, s_taskletTrackID, 0 ); + CCP_LOG_CH( s_ch, "UpdateTelemetry - Starting Telemetry Server" ); + if (!CcpStartTelemetry( s_telemetryServerOrFileSystemDumpPath.c_str(), s_telemetryConnectionType, m_telemetryMaxThreadCount )) + { + CCP_LOGERR_CH( s_ch, "UpdateTelemetry - Failed to start Telemetry server" ); + s_profilerState.store( ProfilerState::Stopped, std::memory_order_release ); + } } -#endif + break; } - s_isTelemetryConnectionRequested = false; - s_telemetryLastCheckTime = s_telemetryStartTime = BeOS->GetActualTime(); - return; - } - - CcpTelemetryTick(); - - if( s_isTelemetryShuttingDown ) - { - CcpStopTelemetry(); + case ProfilerState::Started: + { + CcpTelemetryTick(); - if( s_isTelemetryPythonCaptureEnabled ) + if (TracyIsConnected) + { + Be::Time now = BeOS->GetActualTime(); + + // Erase fibers scheduled for erasure + for ( const auto& [tasklet, time] : g_fiberEraseMap) + { + // Give the profiler a few seconds to receive information from the fiber name store before deallocating + // the underlying string + if ((time - now) / Be::Time(1e7) >= 5) + { + g_fiberNameStore.erase( tasklet ); + g_fiberEraseMap.erase( tasklet ); + } + } + if(s_telemetrySamplePeriod > 0.0f ) // Check if we have passed our timed sample time + { + Be::Time delta = now - s_telemetryLastCheckTime; + s_telemetryLastCheckTime = now; + s_telemetrySamplePeriod -= ((float)delta / Be::Time(1e7)); + + if(s_telemetrySamplePeriod < 0.0f) + { + CCP_LOG_CH( s_ch, "UpdateTelemetry - Finalizing timed Telemetry run" ); + StopTelemetry(); + } + } + } + else + { + CCP_LOG_CH( s_ch, "UpdateTelemetry - Disconnected from profiler" ); + StopTelemetry(); + } + break; + } + case ProfilerState::StopRequested: { + CcpTelemetryTick(); + PyEval_SetProfile( nullptr, nullptr ); + s_profilerState.store( ProfilerState::Stopped, std::memory_order_release ); } + case ProfilerState::Stopped: + // Nothing to do + break; + default: + CCP_LOGERR_CH( s_ch, "UpdateTelemetry - Unhandled profiler state %d", s_profilerState.load(std::memory_order_acquire)); + break; + } +#endif +} -#if CCP_STACKLESS - for( auto& free : s_taskletFree ) +void SwitchToFiber( PyTaskletObject* to ) +{ + if( !to || PyTasklet_IsMain( to ) ) + { + if ( g_activeFiber ) { - free.first->tp_free = free.second; + TracyFiberLeave; } - s_taskletFree.clear(); - - s_lastTasklet = s_fallbackInfo; -#endif - s_isTelemetryShuttingDown = false; + g_activeFiber = nullptr; } - else if(s_telemetrySamplePeriod > 0.0f ) // Check if we have passed our timed sample time + else { - Be::Time newTime = BeOS->GetActualTime(); - Be::Time delta = newTime - s_telemetryLastCheckTime; - s_telemetryLastCheckTime = newTime; - s_telemetrySamplePeriod -= ((float)delta / Be::Time(1e7)); - - if(s_telemetrySamplePeriod < 0.0f) + auto existing = g_fiberNameStore.find( to ); + if( existing == g_fiberNameStore.cend() ) { - CCP_LOG( "Finalising timed Telemetry run." ); - StopTelemetry(); + auto toInfo = GetTaskletInfo( to ); + // Construct the tracy fiber name as `function_name (address_of_tasklet)` + // This should deal with the case of `function_name` running in two different tasklets + g_fiberNameStore[to] = std::string( toInfo.name ) + " (" + std::to_string( reinterpret_cast( to ) ) + ")"; + existing = g_fiberNameStore.find( to ); } + TracyFiberEnter( existing->second.c_str() ); + g_activeFiber = to; + } + + if( auto existing = g_taskletZoneStore.find( g_activeFiber ); existing == g_taskletZoneStore.cend() ) + { + // Initialize an empty zone stack, main tasklet is mapped to nullptr + g_taskletZoneStore[g_activeFiber] = {}; } -#endif } #if CCP_STACKLESS void BlueStatistics::OnTaskletSwitch( PyObject* _from, PyObject* _to ) { #if CCP_TELEMETRY_ENABLED - PyTaskletObject* from = (PyTaskletObject*)_from; - PyTaskletObject* to = (PyTaskletObject*)_to; - - if( tmRunning() ) + if (s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Started) { - StoreFree( from ); - StoreFree( to ); + return; + } - if( s_isTelemetryTaskletCaptureEnabled && from && !PyTasklet_IsMain( from ) ) - { - tmEndTimeSpanEx( 0, reinterpret_cast( from ), s_lastTasklet.filename, s_lastTasklet.line ); - } + auto from = (PyTaskletObject*)_from; + auto to = (PyTaskletObject*)_to; - if( !to || PyTasklet_IsMain( to ) ) - { - tmSwitchToFiber( 0, 0 ); - } - else - { - tmSwitchToFiber( 0, reinterpret_cast( to ) ); + StoreFree( from ); + StoreFree( to ); - if( s_isTelemetryTaskletCaptureEnabled ) - { - auto toInfo = GetTaskletInfo( to ); - tmBeginColoredTimeSpanEx( 0, reinterpret_cast( to ), s_taskletTrackID, 0, TMZF_NONE, toInfo.filename, toInfo.line, "%s", toInfo.name ); - s_lastTasklet = toInfo; - } - } + if( s_isTelemetryTaskletCaptureEnabled ) + { + SwitchToFiber( to ); } #endif } @@ -477,9 +534,6 @@ ICcpStatisticsAccumulator* BlueStatistics::GetAccumulator( const std::string& na void BlueStatistics::SetTimelineSectionName( const char* name ) { -#if CCP_TELEMETRY_ENABLED - tmSetTimelineSectionName( TMCM_GENERAL, name ); -#endif } void BlueStatistics::SetCppCaptureEnabled( bool b ) @@ -720,7 +774,7 @@ void tmTaskletEnter( uint32_t ctx, const char* name ) // so we have to filter out zones manually if( s_isTelemetryCppCaptureEnabled || (ctx & TMCM_CPP) == 0 ) { - tmEnter( ctx, TMZF_NONE, "%s", name ); + tmEnter( TMCM_CPP, TMZF_NONE, "%s", name ); } } @@ -738,7 +792,30 @@ void tmTaskletAppendText( uint32_t ctx, const char* appendText ) tmMessage( ctx, TMMF_ZONE_SUBLABEL, "%s", tmDynamicString( TMCM_GENERAL, appendText ) ); } -tmTaskletZone::tmTaskletZone( uint32_t ctx, const char* name ) : m_telemetryContext( ctx ) +void TracyEnterZone( void* key, const char* name, const char* filename, uint32_t lineno ) +{ + if( s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started ) + { + if (auto existing = g_taskletZoneStore.find( g_activeFiber ); existing != g_taskletZoneStore.end()) + { + existing->second.emplace( key, TracyZone(TMCM_CPP, name, filename, lineno, tracy::Color::Yellow )); + } + } +} + +void TracyLeaveZone( void* key ) +{ + if (auto existing = g_taskletZoneStore.find( g_activeFiber ); existing != g_taskletZoneStore.end() && !existing->second.empty()) + { + // Frame object guards against whether this function had a prior call to `TracyEnterZone` result in a new Zone pushed to the stack + if ( existing->second.top().first == key ) + { + existing->second.pop(); + } + } +} + +tmTaskletZone::tmTaskletZone( uint32_t ctx, const char* name ) { tmTaskletEnter( ctx, name ); } @@ -748,6 +825,49 @@ tmTaskletZone::~tmTaskletZone() tmTaskletLeave( m_telemetryContext ); } -#endif +TracyZone::TracyZone( uint32_t ctx, const char* name, const char* filename, uint32_t lineno, uint32_t color ) : m_fiber( g_activeFiber ) +{ + if( s_profilerState.load( std::memory_order_acquire ) != ProfilerState::Started ) + { + return; + } + + CCP_ASSERT( filename != nullptr ); + CCP_ASSERT( name != nullptr ); + auto data = ___tracy_alloc_srcloc( lineno, filename, strlen( filename ), name, strlen( name ), color ); + m_telemetryContext.emplace( ___tracy_emit_zone_begin_alloc( data, ctx & TMCM_CPP ) ); +} + +TracyZone::TracyZone( TracyZone&& other ) noexcept +{ + m_fiber = other.m_fiber; + m_telemetryContext = other.m_telemetryContext; + // mark this instance's zone as inactive in case the destructor runs + other.m_telemetryContext.reset(); +} +TracyZone::~TracyZone() +{ + // Notify Tracy of all zones ended with a valid context, regardless of profiler state + if( !m_telemetryContext ) + { + return; + } + + // Zones need to end on the same fiber they were started from, so do a little song and dance to ensure that + auto previous = g_activeFiber; + SwitchToFiber( (PyTaskletObject*) m_fiber ); + TracyCZoneEnd( m_telemetryContext.value() ); + SwitchToFiber( previous ); +} +void TracyZone::text( const char* text ) const +{ + if( s_profilerState.load( std::memory_order_acquire ) == ProfilerState::Started && m_telemetryContext ) + { + CCP_ASSERT( text != nullptr ); + TracyCZoneText( m_telemetryContext.value(), text, strlen( text ) ); + } +} + +#endif diff --git a/src/BlueStatistics_Blue.cpp b/src/BlueStatistics_Blue.cpp index 855c57c..a387dea 100644 --- a/src/BlueStatistics_Blue.cpp +++ b/src/BlueStatistics_Blue.cpp @@ -292,8 +292,14 @@ PyObject* PyEnterZone( PyObject* self, PyObject* args ) { return nullptr; } + auto frame = PyEval_GetFrame(); - tmTaskletEnter( TMCM_GENERAL, zone ); + auto fileName = Immortalize( frame->f_code->co_filename ); + if (!fileName) + { + return nullptr; + } + TracyEnterZone( frame, zone, fileName, frame->f_lineno ); #endif Py_RETURN_NONE; } @@ -301,7 +307,7 @@ PyObject* PyEnterZone( PyObject* self, PyObject* args ) PyObject* PyLeaveZone( PyObject* self, PyObject* args ) { #if CCP_TELEMETRY_ENABLED - tmTaskletLeave( TMCM_GENERAL ); + TracyLeaveZone( PyEval_GetFrame() ); #endif Py_RETURN_NONE; } @@ -328,9 +334,9 @@ PyObject* PyAppendToZone( PyObject* self, PyObject* args ) } #if CCP_TELEMETRY_ENABLED -static TmU64 s_timespanId = 0xf00000000; +static uint64_t s_timespanId = 0xf00000000; #endif - + PyObject* PyBeginTimeSpan( PyObject* self, PyObject* args ) { #if CCP_TELEMETRY_ENABLED @@ -348,7 +354,6 @@ PyObject* PyBeginTimeSpan( PyObject* self, PyObject* args ) } ++s_timespanId; - tmBeginTimeSpan( TMCM_GENERAL, s_timespanId, TMTSF_NONE, label ); return PyLong_FromLongLong( s_timespanId ); #else @@ -359,7 +364,7 @@ PyObject* PyBeginTimeSpan( PyObject* self, PyObject* args ) PyObject* PyEndTimeSpan( PyObject* self, PyObject* args ) { #if CCP_TELEMETRY_ENABLED - TmU64 id = 0; + uint64_t id = 0; PyObject* labelO; if( !PyArg_ParseTuple( args, "LO", &id, &labelO ) ) @@ -372,8 +377,6 @@ PyObject* PyEndTimeSpan( PyObject* self, PyObject* args ) { return nullptr; } - - tmEndTimeSpan( TMCM_GENERAL, id, TMTSF_NONE, label ); #endif Py_RETURN_NONE; } @@ -667,18 +670,14 @@ const Be::ClassInfo* BlueStatistics::ExposeToBlue() ( "PauseTelemetry", PauseTelemetry, - "Pauses Telemetry capture. Ticking and frame boundary information are" - "\nstill sent over, but high frequency data such as memory events, mutex" - "\nstates, and zones are discarded. An application can use this function" - "\nto keep Telemetry live but with very low overhead until a specific" - "\nproblem area is encountered." + "Pauses Telemetry capture. (deprecated)" ) MAP_METHOD_AND_WRAP ( "ResumeTelemetry", ResumeTelemetry, - "Resumes Telemetry captures." + "Resumes Telemetry capture. (deprecated)" ) MAP_METHOD_AND_WRAP @@ -692,7 +691,7 @@ const Be::ClassInfo* BlueStatistics::ExposeToBlue() ( "isTelemetryConnectionRequested", IsTelemetryConnectionRequested, - "Is Telemetry connection pending?" + "Is Telemetry profiler connection pending?" ) MAP_PROPERTY_READONLY @@ -706,14 +705,21 @@ const Be::ClassInfo* BlueStatistics::ExposeToBlue() ( "isTelemetryConnected", IsTelemetryConnected, - "Is Telemetry connected?" + "Is Telemetry connected to a profiler?" ) MAP_PROPERTY_READONLY ( "isTelemetryPaused", IsTelemetryPaused, - "Is Telemetry paused?" + "Is Telemetry paused? (deprecated)" + ) + + MAP_PROPERTY_READONLY + ( + "isTelemetryStarted", + IsTelemetryStarted, + "Is Telemetry instrumentation active?" ) MAP_PROPERTY @@ -825,4 +831,3 @@ const Be::ClassInfo* BlueStatistics::ExposeToBlue() EXPOSURE_END() } - diff --git a/src/CarbonIO/CarbonIO.cpp b/src/CarbonIO/CarbonIO.cpp index 30e6851..f22578a 100644 --- a/src/CarbonIO/CarbonIO.cpp +++ b/src/CarbonIO/CarbonIO.cpp @@ -3308,7 +3308,6 @@ bool CarbonIO::processRawQueuedData( SCompletionUnit *completion ) *(unsigned int *)packet->data &= ceHeaderBitsMask; // knock off old size *(unsigned int *)packet->data |= (outlen + offset) - sizeof(unsigned int); // plug in NEW size *(unsigned int *)packet->data |= m_compressionType; - // if this is not the root packet, append the data TO the // root packet if ( packet != completion->queuedData ) diff --git a/src/CarbonIO/CarbonIO.h b/src/CarbonIO/CarbonIO.h index 4a667dc..c616c5e 100644 --- a/src/CarbonIO/CarbonIO.h +++ b/src/CarbonIO/CarbonIO.h @@ -816,7 +816,7 @@ bool CarbonIO::isPacketValid( SPacket *packet ) return packet && packet->packetLen >= 4 && (packet->auxData - || (packet->packetLen == (*(int *)packet->data & ceHeaderSizeMask) + sizeof(int))); + || (packet->packetLen == (*(int *)packet->data & ceHeaderSizeMask) + sizeof(int))); } //------------------------------------------------------------------------------ diff --git a/src/Marshal.cpp b/src/Marshal.cpp index c7994bc..cbaf785 100644 --- a/src/Marshal.cpp +++ b/src/Marshal.cpp @@ -1223,17 +1223,22 @@ PyObject * Marshal::ReadObjectReference( ReadStream * stream, bool isShared ) int len; PyObject* ret; - if( !stream->ReadInteger( len ) ) return 0; + if( !stream->ReadInteger( len ) ) + { + return nullptr; + } if( stream->GetVersion() == 0 ) { if( len < 1 || len > stream->mMapCount || !(ret = stream->mShared[len - 1]) ) { PyErr_SetString( PyExc_ValueError, "Invalid TY_REFERENCE in stream" ); - return 0; + return nullptr; } } else { - if( len < 0 || len >= (int)stream->mShared.size() ) - return PyErr_SetString( PyExc_ValueError, "Invalid TY_REFERENCE in stream" ), nullptr; - ret = stream->mShared[len]; + if( len < 0 || len >= (int)stream->mShared.size() || !(ret = stream->mShared[len]) ) + { + PyErr_SetString( PyExc_ValueError, "Invalid TY_REFERENCE in stream" ); + return nullptr; + } } Py_INCREF( ret ); return ret; @@ -1376,6 +1381,10 @@ PyObject* Marshal::ReadObject( ReadStream *stream ) { CCP_STATS_ZONE( __FUNCTION__ ); + Incrementor _inc(mRecursionLevel); + if( !_inc.Test( sRecursionLimit ) ) + return nullptr; + PYTYPES type; if (!stream->ReadType(type)) return 0; @@ -1549,8 +1558,13 @@ PyObject *Marshal::ReadObjectNewobj(ReadStream* stream, bool shared) if (!cls) return 0; BluePy __new__(PyObject_GetAttr(cls, mStock_New)); - if (!__new__) return 0; - BluePy r(PyObject_CallObject(__new__, args)); +#ifdef PY3_COMPATIBILITY_MODE + // Marshalled new-style objects from Python3 may have to be constructed as old-style objects in Python2.7 + BluePy r(PyObject_HasAttr( cls, mStock_New ) ? BluePy(PyObject_CallObject(__new__, args)) : BluePy(PyInstance_NewRaw(cls, 0))); +#else +if (!__new__) return 0; +BluePy r(PyObject_CallObject(__new__, args)); +#endif if (!r) return 0; //object is constructed, now update r @@ -1654,7 +1668,16 @@ PyObject *Marshal::GetGlobalObject(PyObject *nameO) const char *dot = strrchr(name, '.'); BluePyStr modulename; if (dot){ +#if PY3_COMPATIBILITY_MODE + if( strncmp( name, "builtins.", 9 ) == 0 ) { + modulename = BluePyStr( "__builtin__" ); + } + else { + modulename = BluePyStr( dot-name, name ); + } +#else modulename = BluePyStr(dot-name, name); +#endif name = dot+1; } else { modulename = BluePyStr("__builtin__"); @@ -1864,22 +1887,28 @@ bool Marshal::WriteObject(WriteStream* stream, PyObject* o) { RETFAIL(WriteType(stream, TY_STR_EMPTY)); } +#ifndef PY3_COMPATIBILITY_MODE else if (size == 1) { RETFAIL(WriteType(stream, TY_STR_CHAR)); RETFAIL(stream->Write(string[0])); - } else { + } +#endif + else + { PyObject* index = PyDict_GetItem(mStrTable, o); if (index) { RETFAIL(WriteType(stream, TY_STR_TABLE)); RETFAIL(stream->Write((char)PyInt_AS_LONG(index))); - } else { + } + else + { //Write the string as buffer. This gives us object sharing and all //The old TY_STR_SHORT and TY_STR are kept for backwards compatibility CHECKREF(); RETFAIL(WriteType(stream, TY_BUFFER)); - RETFAIL(stream->WriteBuff(string, size)); + RETFAIL(stream->WriteBuff(string, size)); } } diff --git a/src/PyRowSet.cpp b/src/PyRowSet.cpp index d850a10..c726eb8 100644 --- a/src/PyRowSet.cpp +++ b/src/PyRowSet.cpp @@ -104,7 +104,7 @@ struct ColumnDescriptor ColumnDescriptor(const char *name) : mName(name), mType(0), mOffset(0), mSize(0){} static bool TypeSize(const DBTYPE &type, int &size) { - switch(type & 0xff) { + switch(type) { case DBTYPE_BOOL: size = 0; break; case DBTYPE_I1: @@ -130,8 +130,7 @@ struct ColumnDescriptor case DBTYPE_BYTES: size = 5; break; //signal an object case DBTYPE_EMPTY: - size = -1; break; //virtual - + size = -1; break; default: PyErr_Format(PyExc_TypeError, "DBRowDescriptor doesn't support data type %d", type); return false; @@ -145,7 +144,7 @@ struct ColumnDescriptor std::string mName; int mOffset; DBTYPE mType; - char mSize; + unsigned char mSize; }; bool operator == (const ColumnDescriptor &a, const ColumnDescriptor &b) @@ -304,6 +303,23 @@ bool RowDescriptor::InitFromTypedList() size_t i; for(i = 0; i 1 && name[0] == '_' && name[1] == '_') + { + PyErr_SetString(PyExc_ValueError, "Column names cannot begin with a double underscore"); + return false; + } + int size; //logarithm of size plus 1 if (!ColumnDescriptor::TypeSize(cd.mType, size)) return false; @@ -378,7 +394,8 @@ bool RowDescriptor::InitFromTypedList() //Also initialize the map, while we're at it. mColumnMap.insert(columnMap_t::value_type(cd.mName.c_str(), std::pair(&cd, (int)i))); } - return true; + + return true; } @@ -625,7 +642,7 @@ bool DBRowDescriptor::Set_virtual(PyObject *l) goto ERR1; for (Py_ssize_t i = 0; i= PyList_GET_SIZE(mVirtualGetSet.o)) - return PyErr_SetString(PyExc_RuntimeError, "Internal error in VirtualSet"), false; - PyObject *t = PyList_GET_ITEM(mVirtualGetSet.o, n); - if (!PyTuple_Check(t)) - return PyErr_SetString(PyExc_RuntimeError, "Internal error in VirtualSet"), false; - if (PyTuple_GET_SIZE(t)<3) - return (PyErr_SetString(PyExc_AttributeError, "read only attribute")), false; - PyObject *res = PyObject_CallFunctionObjArgs(PyTuple_GET_ITEM(t, 2), row, val, 0); - if (!res) return false; - Py_DECREF(res); - return true; + PyErr_SetString(PyExc_RuntimeError, "Virtual columns are read-only"); + return false; } @@ -1183,7 +1198,8 @@ bool DBRow::SetNotNull(const ColumnDescriptor &c, PyObject *o) return true; case DBTYPE_EMPTY: - return mRD->VirtualSet(c.mOffset, this, o); + PyErr_SetString(PyExc_RuntimeError, "Virtual columns are read-only"); + return false; default: PyErr_Format(PyExc_RuntimeError, "Unexpected db column type encountered: %d", c.mType); return false; @@ -1623,6 +1639,12 @@ int DBRow::SetAttr(PyObject *self, PyObject *key, PyObject *val) const ColumnDescriptor *cd = tis->GetCD(idx, key, NULL); if (cd != 0) { + // Guard against `del DBRow.column` / `delattr(DBRow, "column")` + if (!val) + { + PyErr_SetString(PyExc_RuntimeError, "Cannot delete a column from a DBRow"); + return -1; + } if (!tis->Set(*cd, idx, val)) return -1; return 0; @@ -1750,6 +1772,14 @@ PyObject *DBRow::Read(Marshal &m, ReadStream &s) { BluePy rd(m.ReadObject(&s)); //read row descriptor if (!rd) return 0; + + //Check object is of correct DBRowDescriptorType + if( DBRowDescriptor::GetType() != Py_TYPE( rd.o ) ) + { + PyErr_Format( PyExc_TypeError, "Hacker Warning! Invalid type in marshal data, expected blue.DBRowDescriptor." ); + return 0; + } + //Create object BluePy rowO(DBRow::CreateFromRowDescriptor( rd ) ); if (!rowO) return 0; @@ -1830,4 +1860,4 @@ bool DBRowsetInit(PyObject *module) return true; } -#endif \ No newline at end of file +#endif diff --git a/src/StdAfx.h b/src/StdAfx.h index 97f3027..ea60561 100644 --- a/src/StdAfx.h +++ b/src/StdAfx.h @@ -20,6 +20,8 @@ #endif #endif +#include + // include all python stuff #if BLUE_WITH_PYTHON #include diff --git a/src/TaskletTimer.cpp b/src/TaskletTimer.cpp index 0f8e284..c0410b4 100644 --- a/src/TaskletTimer.cpp +++ b/src/TaskletTimer.cpp @@ -3,7 +3,7 @@ TaskletTimer.cpp - Author: Kristján Valur Jónsson + Author: Kristján Valur Jónsson Created: Sept 2004 OS: Win32 Project: Yep @@ -96,7 +96,6 @@ TaskletTimer::TaskletTimer() : mMaxWarn(1), mStackMap( "TaskletTimer/mStackMap" ), mSimpleCtxt(Py_None, true), - mCanonicalizationDict(PyDict_New()), m_BlueOSPumpCountAtStart( 0 ) { mCurrentStack = 0; @@ -153,37 +152,6 @@ PyObject *TaskletTimer::EnterTaskletStr(const char *context, TASKLETFLAGS flags) PyObject *TaskletTimer::EnterTaskletEx(PyObject *newContext, TASKLETFLAGS flags) { - if (mDoTelemetry) { - PyObject *strifiedName = PyObject_Str(newContext); - - if (strifiedName == NULL) { - return NULL; - } - - PyObject *canonicalName = PyDict_GetItem(mCanonicalizationDict, strifiedName); - - if (!canonicalName) { - // First time we've seen this particular name, set it in the dict and use it directly - int err = PyDict_SetItem(mCanonicalizationDict, strifiedName, strifiedName); - if (err != 0) { - // Not being able to set an entry in our dict is pretty fatal, boil the exception on up. - return NULL; - } - - // Success, we are the canonical string for this forever and ever and ever now - canonicalName = strifiedName; - } - - Py_DECREF(strifiedName); -#if CCP_TELEMETRY_ENABLED - if (canonicalName) { - tmTaskletEnter(TMCM_GENERAL, PyString_AsString(canonicalName)); - } -#else - CCP_UNUSED( canonicalName ); -#endif - } - mFlags = flags; //temporary hack to support the IDLE flag if (!mActive) { //timer is not enabled. @@ -240,12 +208,6 @@ PyObject *TaskletTimer::EnterTaskletEx(PyObject *newContext, TASKLETFLAGS flags) bool TaskletTimer::ReturnFromTasklet(PyObject *backContext) { -#if CCP_TELEMETRY_ENABLED - if (mDoTelemetry) - { - tmTaskletLeave( TMCM_GENERAL ); - } -#endif // Temporary hack to support IDLE if (mFlags) { TimesliceReset(); diff --git a/src/slsocket.cpp b/src/slsocket.cpp index 228641e..89c88eb 100644 --- a/src/slsocket.cpp +++ b/src/slsocket.cpp @@ -1154,7 +1154,7 @@ class SendPacketResult : public SendResult if (mHeader > (DWORD)mXtra->GetMaxPacketSize()) { char tmp[128]; sprintf_s(tmp, "packet too long at %d bytes, max size is %d", - mHeader, mXtra->GetMaxPacketSize()); + mHeader, mXtra->GetMaxPacketSize()); OutputDebugString(tmp); throw std::length_error(tmp); } @@ -1281,25 +1281,25 @@ class RecvPacketResult : public IOOverlappedRecv more = true; } else if (mBytesRead == sizeof(mHeader)) { //just finished reading header, allocate buffer. - if (mHeader > (DWORD)mXtra->GetMaxPacketSize()) { + if (mHeader > (DWORD)mXtra->GetMaxPacketSize()) { char tmp[128]; sprintf_s(tmp, "too large a packet detected at %d bytes, max is %d", mHeader, mXtra->GetMaxPacketSize()); throw std::length_error(tmp); } more = mHeader!=0; - } else if (mBytesRead < (int)(sizeof(mHeader)+mHeader)) { + } else if (mBytesRead < (int)(sizeof(mHeader)+mHeader)) { //header here but packet not finished more = true; } else { //packet finished. //did we read any of the next guy's header? (we requested next header's amount of data :) - DWORD rest = mBytesRead - (sizeof(mHeader) + mHeader); + DWORD rest = mBytesRead - (sizeof(mHeader) + mHeader); if (rest) { RecvPacketResult *next = mXtra->NextRecv(this); _ASSERT(next); next->OnHeaderBytesRead(rest); - mBytesRead = (int)(sizeof(mHeader)+mHeader); + mBytesRead = (int)(sizeof(mHeader)+mHeader); } mXtra->mStats.PacketReceived(); more = false; diff --git a/src/slsocket_posix.cpp b/src/slsocket_posix.cpp index 8cb4719..b1ed15a 100644 --- a/src/slsocket_posix.cpp +++ b/src/slsocket_posix.cpp @@ -1625,7 +1625,7 @@ class SendPacketResult : public SendBase // Copy address // steal the buffer StealBuffer(buf); - mHeader = mKeeper.len; + mHeader = mKeeper.len; FlipHeader(mHeader); Request(s->sock_timeout); return mResult; @@ -1834,7 +1834,7 @@ class RecvPacketResult : public SocketResult if (mBytesRead < (sizeof(mHeader) + sizeof(uint32_t))) return; - uint32_t oobDataLen = *(uint32_t *)(mData); + uint32_t oobDataLen = *(uint32_t *)(mData); // sanity check the out-of-band data length; mPacketSize was sanity checked already if (oobDataLen > mPacketSize) { char tmp[128] = {'\0'}; diff --git a/tests/python/bluetests/test/test_dbrow.py b/tests/python/bluetests/test/test_dbrow.py new file mode 100644 index 0000000..82a485f --- /dev/null +++ b/tests/python/bluetests/test/test_dbrow.py @@ -0,0 +1,84 @@ +from . import blueunittest +import blue + + +class TestDBRow(blueunittest.TestCase): + """ + A set of test cases for the DBRow class. + """ + + def setUp(self): + self.columns = (("nodeID", 4), ("ipAddress", 129), ("port", 3)) + self.row = blue.DBRow(blue.DBRowDescriptor(self.columns)) + + def testAttemptingToDeleteColumnDoesntCrash(self): + foo = self.row + with self.assertRaises(RuntimeError): + delattr(foo, "nodeID") + with self.assertRaises(RuntimeError): + del foo.ipAddress + + def testVirtualColumnsAreReadOnly(self): + def getFoo(*args, **kwargs): + return 4711 + def setFoo(*args, **kwargs): + pass + desc = blue.DBRowDescriptor(self.columns) + with self.assertRaises(TypeError): + desc.virtual = [("foo", getFoo, setFoo)] + desc.virtual = [("foo", getFoo)] + row = blue.DBRow(desc) + self.assertEqual(row.foo, 4711) + + def testSliceSubscript(self): + sliceTest = self.row[:] + + self.assertIsInstance(sliceTest, list) + self.assertEqual(self.row, blue.DBRow(blue.DBRowDescriptor(self.columns), sliceTest)) + self.assertListEqual(sliceTest, [0.0, None, 0]) + self.assertListEqual(self.row[:1], [0.0]) + self.assertListEqual(self.row[1:], [None, 0]) + self.assertListEqual(self.row[1:2], [None]) + self.assertListEqual(self.row[:-1], [0.0, None]) + self.assertListEqual(self.row[0:], [0.0, None, 0]) + + with self.assertRaises(TypeError): + self.row[::2] + + def testIndexSubscript(self): + self.assertEqual(self.row[0], 0.0) + self.assertEqual(self.row[-1], 0) + + with self.assertRaises(IndexError): + self.row[len(self.row)] + + def testUnicodeSubscript(self): + self.assertEqual(self.row["nodeID"], 0) + + with self.assertRaises(KeyError): + self.row["test"] + + def testEquivalencyComparison(self): + self.assertFalse(self.row == None) + self.assertNotEqual(self.row, None) + + self.assertFalse(self.row == 1) + self.assertNotEqual(self.row, 1) + + self.assertFalse(self.row == blue.DBRowDescriptor(self.columns)) + self.assertNotEqual(self.row, blue.DBRowDescriptor(self.columns)) + + self.assertEqual(self.row, blue.DBRow(blue.DBRowDescriptor(self.columns))) + + def testColumnNameConstraints(self): + invalid_column_names = ( + # Python's magic attributes can lead to type confusion and other interesting bits. + # Therefore, disallow column names starting with a double underscore. + "__this_might_be_a_python_magic_attribute", + ) + for invalid_column_name in invalid_column_names: + with self.assertRaises(ValueError): + blue.DBRowDescriptor(((invalid_column_name, 0x80),)) + + # However, a single underscore as starting character is fine + _ = blue.DBRowDescriptor((("_dummy", 0x80),)) diff --git a/tests/python/bluetests/test/test_marshal.py b/tests/python/bluetests/test/test_marshal.py index 4d843d9..6d30700 100644 --- a/tests/python/bluetests/test/test_marshal.py +++ b/tests/python/bluetests/test/test_marshal.py @@ -1,183 +1,443 @@ -__author__ = 'snorri.sturluson' - -from . import blueunittest -import blue -import sys - -class EmptyObject(object): - pass - - -class SimpleObject(object): - def __init__(self): - self.a = "this is a string" - self.b = 42 - self.c = 3.14159267 - - -class OldSchoolObject: - def __init__(self): - self.a = "this is a string" - self.b = 42 - self.c = 3.14159267 - - -def SaveCallback(obj): - if isinstance(obj, OldSchoolObject): - return "magic" - return None - - -def LoadCallback(obj): - if obj == "magic": - return OldSchoolObject() - return None - - -class testMarshal(blueunittest.TestCase): - loaded = [] - saved = [] - - @classmethod - def setUpClass(cls): - blue.marshal.ResetTypeStats() - cls.loaded = [0]*48 - cls.saved = [0]*48 - - @classmethod - def tearDownClass(cls): - IGNORE_TYPES = [0, 3, 13, 16, 12, 24, 26, 29, 30, 33] - for i in range(48): - if i in IGNORE_TYPES: - continue - if cls.loaded[i] == 0: - sys.stderr.write("Missing coverage for type %d when loading" % i) - if cls.saved[i] == 0: - sys.stderr.write("Missing coverage for type %d when saving" % i) - - def _update_coverage(self): - typeStats = blue.marshal.GetTypeStats() - for i in range(48): - self.loaded[i] += typeStats[0][i] - self.saved[i] += typeStats[1][i] - - def verify_round_trip(self, obj): - blue.marshal.ResetTypeStats() - s = blue.marshal.Save(obj) - obj2 = blue.marshal.Load(s) - self.assertBlueObjectsEqual(obj, obj2) - typeStats = blue.marshal.GetTypeStats() - self.assertEqual(typeStats[0], typeStats[1]) - self._update_coverage() - - def test_none(self): - self.verify_round_trip(None) - - def test_empty_string(self): - self.verify_round_trip("") - - def test_string(self): - self.verify_round_trip("this is a test") - - def test_string_from_stringtable(self): - self.verify_round_trip("ballID") - - def test_empty_unicode(self): - self.verify_round_trip(u"") - - def test_single_char_unicode(self): - self.verify_round_trip(u"A") - - def test_unicode(self): - self.verify_round_trip(u"\u20A8\u20B1\u20A9") - - def test_unicode_as_utf8(self): - self.verify_round_trip(u"this is a unicode test") - - def test_integer(self): - self.verify_round_trip(0) - self.verify_round_trip(1) - self.verify_round_trip(-1) - self.verify_round_trip(42) - self.verify_round_trip(32767) - self.verify_round_trip(2147483647) - self.verify_round_trip(-2147483648) - - def test_long(self): - self.verify_round_trip(42) - self.verify_round_trip(9223372036854775807) - - def test_float(self): - self.verify_round_trip(0.0) - self.verify_round_trip(3.14159267) - self.verify_round_trip(-2.781431508934509809834) - - def test_bool(self): - self.verify_round_trip(True) - self.verify_round_trip(False) - - def test_empty_dict(self): - self.verify_round_trip({}) - - def test_empty_object(self): - self.verify_round_trip(EmptyObject()) - - def test_simple_object(self): - self.verify_round_trip(SimpleObject()) - - def test_empty_list(self): - self.verify_round_trip([]) - - def test_list_of_one_string(self): - self.verify_round_trip(["this is a test"]) - - def test_list_of_strings(self): - self.verify_round_trip(["this", "is", "a", "test"]) - - def test_empty_tuple(self): - self.verify_round_trip(()) - - def test_tuple_of_one_string(self): - self.verify_round_trip(("this is a test",)) - - def test_tuple_of_two_strings(self): - self.verify_round_trip(("this is", "a test")) - - def test_tuple_of_strings(self): - self.verify_round_trip(("this", "is", "a", "test")) - - def test_instanced_object(self): - obj = SimpleObject() - self.verify_round_trip([obj, obj, obj]) - - def test_instanced_old_shool_object(self): - obj = OldSchoolObject() - self.verify_round_trip([obj, obj, obj]) - - def test_callback(self): - obj = [OldSchoolObject(), SimpleObject(), "this is a test"] - s = blue.marshal.Save(obj, callback=SaveCallback) - obj2 = blue.marshal.Load(s, callback=LoadCallback) - self.assertBlueObjectsEqual(obj, obj2) - typeStats = blue.marshal.GetTypeStats() - self.assertEqual(typeStats[0], typeStats[1]) - self._update_coverage() - - def test_checksum(self): - obj = [OldSchoolObject(), SimpleObject(), "this is a test"] - s = blue.marshal.Save(obj, useChecksum=1) - obj2 = blue.marshal.Load(s) - self.assertBlueObjectsEqual(obj, obj2) - typeStats = blue.marshal.GetTypeStats() - self.assertEqual(typeStats[0], typeStats[1]) - self._update_coverage() - - def test_empty_dbrow(self): - rd = blue.DBRowDescriptor(()) - d = blue.DBRow(rd) - self.verify_round_trip(d) - - def test_wstream(self): - obj = [OldSchoolObject(), SimpleObject(), "this is a test"] - ws = blue.marshal.Save(obj) - self.verify_round_trip(ws) +__author__ = 'snorri.sturluson' + +from . import blueunittest +import blue + +import unittest +import sys +import os + +class EmptyObject(object): + def __eq__(self, other): + return isinstance(other, type(self)) + + +class NewStyleObject(object): + def __init__(self): + self.a = "this is a string" + self.b = b"this is a string" + self.c = u"this is a string" + self.d = 42 + self.e = 3.14159267 + + def __eq__(self, other): + # String, byte and unicode comparisons are type-agnostic + # Therefore, an unmarshalled Python3 NewStyleObject instance should compare truthfully + # even though d and e fields differ in type + return isinstance(self, type(other)) and self.__dict__ == other.__dict__ + + +class OldStyleObject: + def __init__(self): + self.a = "this is a string" + self.b = b"this is a string" + self.c = u"this is a string" + self.d = 42 + self.e = 3.14159267 + + def __eq__(self, other): + return isinstance(self, type(other)) and self.__dict__ == other.__dict__ + + +def SaveCallback(obj): + if isinstance(obj, OldStyleObject): + return "magic" + return None + + +def LoadCallback(obj): + if obj == "magic": + return OldStyleObject() + return None + + +class testMarshal(blueunittest.TestCase): + loaded = [] + saved = [] + + @classmethod + def setUpClass(cls): + blue.marshal.ResetTypeStats() + cls.loaded = [0]*48 + cls.saved = [0]*48 + + @classmethod + def tearDownClass(cls): + IGNORE_TYPES = [0, 3, 13, 16, 12, 24, 26, 29, 30, 33] + for i in range(48): + if i in IGNORE_TYPES: + continue + if cls.loaded[i] == 0: + sys.stderr.write("Missing coverage for type %d when loading" % i) + if cls.saved[i] == 0: + sys.stderr.write("Missing coverage for type %d when saving" % i) + + def _update_coverage(self): + typeStats = blue.marshal.GetTypeStats() + for i in range(48): + self.loaded[i] += typeStats[0][i] + self.saved[i] += typeStats[1][i] + + def verify_round_trip(self, obj): + blue.marshal.ResetTypeStats() + s = blue.marshal.Save(obj) + obj2 = blue.marshal.Load(s) + self.assertBlueObjectsEqual(obj, obj2) + typeStats = blue.marshal.GetTypeStats() + self.assertEqual(typeStats[0], typeStats[1]) + self._update_coverage() + + def test_none(self): + self.verify_round_trip(None) + + def test_empty_string(self): + self.verify_round_trip("") + + def test_string(self): + self.verify_round_trip("this is a test") + + def test_string_from_stringtable(self): + self.verify_round_trip("ballID") + + def test_empty_unicode(self): + self.verify_round_trip(u"") + + def test_single_char_unicode(self): + self.verify_round_trip(u"A") + + def test_unicode(self): + self.verify_round_trip(u"\u20A8\u20B1\u20A9") + + def test_unicode_as_utf8(self): + self.verify_round_trip(u"this is a unicode test") + + def test_integer(self): + self.verify_round_trip(0) + self.verify_round_trip(1) + self.verify_round_trip(-1) + self.verify_round_trip(42) + self.verify_round_trip(32767) + self.verify_round_trip(2147483647) + self.verify_round_trip(-2147483648) + + def test_long(self): + self.verify_round_trip(42) + self.verify_round_trip(9223372036854775807) + + def test_float(self): + self.verify_round_trip(0.0) + self.verify_round_trip(3.14159267) + self.verify_round_trip(-2.781431508934509809834) + + def test_bool(self): + self.verify_round_trip(True) + self.verify_round_trip(False) + + def test_empty_dict(self): + self.verify_round_trip({}) + + def test_dict(self): + self.verify_round_trip({"key": "test"}) + + def test_empty_object(self): + self.verify_round_trip(EmptyObject()) + + def test_new_style_object(self): + self.verify_round_trip(NewStyleObject()) + + def test_empty_list(self): + self.verify_round_trip([]) + + def test_list_of_one_string(self): + self.verify_round_trip(["this is a test"]) + + def test_list_of_strings(self): + self.verify_round_trip(["this", "is", "a", "test"]) + + def test_empty_tuple(self): + self.verify_round_trip(()) + + def test_tuple_of_one_string(self): + self.verify_round_trip(("this is a test",)) + + def test_tuple_of_two_strings(self): + self.verify_round_trip(("this is", "a test")) + + def test_tuple_of_strings(self): + self.verify_round_trip(("this", "is", "a", "test")) + + def test_instanced_object(self): + obj = NewStyleObject() + self.verify_round_trip([obj, obj, obj]) + + def test_instanced_old_style_object(self): + obj = OldStyleObject() + self.verify_round_trip([obj, obj, obj]) + + def test_callback(self): + obj = [OldStyleObject(), NewStyleObject(), "this is a test"] + s = blue.marshal.Save(obj, callback=SaveCallback) + obj2 = blue.marshal.Load(s, callback=LoadCallback) + self.assertBlueObjectsEqual(obj, obj2) + typeStats = blue.marshal.GetTypeStats() + self.assertEqual(typeStats[0], typeStats[1]) + self._update_coverage() + + def test_checksum(self): + obj = [OldStyleObject(), NewStyleObject(), "this is a test"] + s = blue.marshal.Save(obj, useChecksum=1) + obj2 = blue.marshal.Load(s) + self.assertBlueObjectsEqual(obj, obj2) + typeStats = blue.marshal.GetTypeStats() + self.assertEqual(typeStats[0], typeStats[1]) + self._update_coverage() + + def test_empty_dbrow(self): + rd = blue.DBRowDescriptor(()) + d = blue.DBRow(rd) + self.verify_round_trip(d) + + def test_wstream(self): + obj = [OldStyleObject(), NewStyleObject(), "this is a test"] + ws = blue.marshal.Save(obj) + self.verify_round_trip(ws) + + def test_dbrow(self): + rowDesc = blue.DBRowDescriptor((("Test", 20),)) + sourceRow = blue.DBRow(rowDesc, (123, )) + self.verify_round_trip(sourceRow) + + def test_dbrow_with_invalid_descriptor_in_stream_raises_error(self): + # Unmarshalled bytes will attempt to create a DBRow and create a blue.Dict rather than + # expected blue.DBRowDescriptor + bytes = b'~\x00\x00\x00\x00*",\x02\tblue.Dict$--' + with self.assertRaises(RuntimeError) as raisedValue: + blue.marshal.Load(bytes) + + self.assertEqual(raisedValue.exception.args[0], TypeError) + + def test_nullptr_deref_in_readobjectreference(self): + known_bad_payloads = ( + b"\x7D\x01\x57\x1B\x00", # entering via `TY_INSTANCE` + b"\x7D\x01\x62\x1B\x00", # entering via `TY_REDUCE` + b"\x7D\x01\x63\x1B\x00", # entering via `TY_NEWOBJ` + ) + for bad_payload in known_bad_payloads: + with self.assertRaises(RuntimeError): + blue.marshal.Load(bad_payload) + + def test_deeply_nested_read(self): + depth = 10000 + payload = b"\x7D\x01" + b"\x25" * depth + b"\x01" + with self.assertRaises(RuntimeError): + blue.marshal.Load(payload) + +@unittest.skipUnless(os.environ.has_key("PY3_COMPATIBILITY_MODE"), "Skipping because these tests will fail if `PY3_COMPATIBILITY_MODE` is not enabled") +class TestBackwardsCompatibility(blueunittest.TestCase): + """ + This class adds coverage for objects marshalled in Python 3. + """ + def test_load_old_style_object(self): + bytes = b'~\x00\x00\x00\x00#,%\x02*bluetests.test.test_marshal.OldStyleObject\x16\x05.\x10this is a string.\x01a\x13\x10this is a string.\x01b.\x10this is a string.\x01c\x06*.\x01d\n\xcd\x06xV\xfb!\t@.\x01e--' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, OldStyleObject()) + + def test_none(self): + bytes = b'~\x00\x00\x00\x00\x01' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, None) + + def test_string_from_stringtable(self): + bytes = b'~\x00\x00\x00\x00\x11\x06' + loaded = blue.marshal.Load(bytes) + + # We expect a str type constructed from marshalled string table index + self.assertIsInstance(loaded, str) + self.assertEqual(loaded, "ballID") + + def test_empty_unicode(self): + bytes = b'~\x00\x00\x00\x00(' + loaded = blue.marshal.Load(bytes) + + self.assertIsInstance(loaded, unicode) + self.assertEqual(loaded, "") + + def test_single_char_unicode(self): + bytes = b'~\x00\x00\x00\x00.\x01A' + loaded = blue.marshal.Load(bytes) + + self.assertIsInstance(loaded, unicode) + self.assertEqual(loaded, "A") + + def test_unicode(self): + bytes = b'~\x00\x00\x00\x00.\t\xe2\x82\xa8\xe2\x82\xb1\xe2\x82\xa9' + loaded = blue.marshal.Load(bytes) + + self.assertIsInstance(loaded, unicode) + self.assertEqual(loaded, u"\u20A8\u20B1\u20A9") + + def test_unicode_as_utf8(self): + bytes = b'~\x00\x00\x00\x00.\x16this is a unicode test' + loaded = blue.marshal.Load(bytes) + + self.assertIsInstance(loaded, unicode) + self.assertEqual(loaded, "this is a unicode test") + + def test_integer(self): + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x08'), 0) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\t'), 1) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x07'), -1) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x06*'), 42) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x05\xff\x7f'), 32767) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x04\xff\xff\xff\x7f'), 2147483647) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x04\x00\x00\x00\x80'), -2147483648) + + def test_long(self): + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00/\x08\xff\xff\xff\xff\xff\xff\xff\x7f'), 9223372036854775807) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00/\t\x00\x00\x00\x00\x00\x00\x00\x80\x00'), 9223372036854775808) + + def test_float(self): + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x0b'), 0.0) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\n\xcd\x06xV\xfb!\t@'), 3.14159267) + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\nO\x80\xb7)_@\x06\xc0'), -2.781431508934509809834) + + def test_bool(self): + self.assertTrue(blue.marshal.Load(b'~\x00\x00\x00\x00\x1f'), True) + self.assertFalse(blue.marshal.Load(b'~\x00\x00\x00\x00 '), False) + + def test_empty_dict(self): + self.assertEqual(blue.marshal.Load(b'~\x00\x00\x00\x00\x16\x00'), {}) + + def test_dict(self): + bytes = b'~\x00\x00\x00\x00\x16\x01.\x04test.\x03key' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, {"key": "test"}) + # Explicit type checking due to Unicode and str types being implicitly comparable + for key, value in loaded.items(): + self.assertIsInstance(key, unicode) + self.assertIsInstance(value, unicode) + + def test_empty_object(self): + bytes = b"~\x00\x00\x00\x00#%%\x02'bluetests.test.test_marshal.EmptyObject--" + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, EmptyObject()) + + def test_new_style_object(self): + bytes = b'~\x00\x00\x00\x00#,%\x02*bluetests.test.test_marshal.NewStyleObject\x16\x05.\x10this is a string.\x01a\x13\x10this is a string.\x01b.\x10this is a string.\x01c\x06*.\x01d\n\xcd\x06xV\xfb!\t@.\x01e--' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, NewStyleObject()) + + def test_empty_list(self): + bytes = b'~\x00\x00\x00\x00&' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, []) + + def test_list_of_one_string(self): + bytes = b"~\x00\x00\x00\x00'.\x0ethis is a test" + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, ["this is a test"]) + self.assertIsInstance(loaded[0], unicode) + + def test_list_of_strings(self): + bytes = b'~\x00\x00\x00\x00\x15\x04.\x04this.\x02is.\x01a.\x04test' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, ["this", "is", "a", "test"]) + for item in loaded: + self.assertIsInstance(item, unicode) + + def test_empty_tuple(self): + bytes = b'~\x00\x00\x00\x00$' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, ()) + + def test_tuple_of_one_string(self): + bytes = b'~\x00\x00\x00\x00%.\x0ethis is a test' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, ("this is a test",)) + self.assertIsInstance(loaded[0], unicode) + + def test_tuple_of_two_strings(self): + bytes = b'~\x00\x00\x00\x00,.\x07this is.\x06a test' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, ("this is", "a test")) + for item in loaded: + self.assertIsInstance(item, unicode) + + def test_tuple_of_strings(self): + bytes = b'~\x00\x00\x00\x00\x14\x04.\x04this.\x02is.\x01a.\x04test' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, ("this", "is", "a", "test")) + for item in loaded: + self.assertIsInstance(item, unicode) + + def test_instanced_object(self): + bytes = b'~\x01\x00\x00\x00\x15\x03c,%\x02*bluetests.test.test_marshal.NewStyleObject\x16\x05.\x10this is a string.\x01a\x13\x10this is a string.\x01b.\x10this is a string.\x01c\x06*.\x01d\n\xcd\x06xV\xfb!\t@.\x01e--\x1b\x01\x1b\x01\x01\x00\x00\x00' + loaded = blue.marshal.Load(bytes) + instance = NewStyleObject() + + self.assertEqual(loaded, [instance, instance, instance]) + + def test_instanced_old_style_object(self): + bytes = b'~\x01\x00\x00\x00\x15\x03c,%\x02*bluetests.test.test_marshal.OldStyleObject\x16\x05.\x10this is a string.\x01a\x13\x10this is a string.\x01b.\x10this is a string.\x01c\x06*.\x01d\n\xcd\x06xV\xfb!\t@.\x01e--\x1b\x01\x1b\x01\x01\x00\x00\x00' + loaded = blue.marshal.Load(bytes) + instance = OldStyleObject() + + self.assertEqual(loaded, [instance, instance, instance]) + + def test_read_callback_called(self): + def read_callback(obj): + read_callback.called = True + read_callback.called = False + + bytes = b'~\x00\x00\x00\x00\x19.\x04test' + blue.marshal.Load(bytes, callback=read_callback) + self.assertTrue(read_callback.called) + + def test_checksum(self): + # Marshalled Python3 object using checksum + bytes = b'~\x00\x00\x00\x00\x1c\xb9/\x0fL\x15\x02#,%\x02*bluetests.test.test_marshal.NewStyleObject\x16\x05.\x10this is a string.\x01a\x13\x10this is a string.\x01b.\x10this is a string.\x01c\x06*.\x01d\n\xcd\x06xV\xfb!\t@.\x01e--.\x0ethis is a test' + loaded = blue.marshal.Load(bytes) + comparison = blue.marshal.Save([NewStyleObject(), "this is a test"], useChecksum=1) + + # Marshalled data will differ due to string fields, so we must load both objects for comparison + self.assertBlueObjectsEqual(loaded, blue.marshal.Load(comparison)) + + def test_empty_dbrow(self): + bytes = b'~\x00\x00\x00\x00*",\x02\x14blue.DBRowDescriptor%$--\x00' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, blue.DBRow(blue.DBRowDescriptor(()))) + + def test_dbrow(self): + bytes = b'~\x00\x00\x00\x00*",\x02\x14blue.DBRowDescriptor%%,.\x04Test\x06\x14--\x02\xf7{' + loaded = blue.marshal.Load(bytes) + + self.assertEqual(loaded, blue.DBRow(blue.DBRowDescriptor((("Test", 20),)), (123, ))) + + def test_set(self): + blue.marshal.globalsWhitelist = {set: None} + blue.marshal.collectWhitelist = False + bytes = b'~\x00\x00\x00\x00",\x02\x0cbuiltins.set%\x15\x03\t\x06\x02\x06\x03--' + loaded = blue.marshal.Load(bytes) + self.assertSetEqual(loaded, {1, 2, 3}) + + def test_runtime_error(self): + blue.marshal.globalsWhitelist = {RuntimeError: None} + blue.marshal.collectWhitelist = False + bytes = b'~\x00\x00\x00\x00",\x02\x15builtins.RuntimeError%.\x05Boom!--' + loaded = blue.marshal.Load(bytes) + self.assertIsInstance(loaded, RuntimeError) + self.assertIsInstance(loaded.message, unicode) + self.assertEqual(loaded.message, u"Boom!")