Rewrite history to squash #142

.github/actions/setup-caches/action.yml

@@ -10,6 +10,7 @@ inputs:
       - 'cp39'
       - 'cp310'
       - 'cp311'
+      - 'cp312'
     default: 'cp39'
 
 runs:

.github/actions/setup-dependencies/action.yml

@@ -10,6 +10,7 @@ inputs:
       - 'cp39'
       - 'cp310'
       - 'cp311'
+      - 'cp312'
     default: 'cp39'
 
 runs:

.github/actions/setup-python/action.yml

@@ -10,6 +10,7 @@ inputs:
       - '3.9'
       - '3.10'
       - '3.11'
+      - '3.12'
     default: '3.9'
 
 runs:
@@ -33,4 +34,4 @@ runs:
 
     - name: Install cibuildwheel and twine
       shell: bash
-      run: pip install cibuildwheel==2.11.2 twine
+      run: pip install cibuildwheel==2.16.5 twine

.github/workflows/build.yml

@@ -179,11 +179,13 @@ jobs:
             - "3.9"
             - "3.10"
             - "3.11"
+            - "3.12"
           cibuildwheel:
             - "cp38"
             - "cp39"
             - "cp310"
             - "cp311"
+            - "cp312"
           is-full-run:
             - ${{ needs.initialize.outputs.FULL_RUN == 'true' }}
           exclude:
@@ -198,24 +200,41 @@ jobs:
               cibuildwheel: "cp310"
             - python-version: "3.8"
               cibuildwheel: "cp311"
+            - python-version: "3.8"
+              cibuildwheel: "cp312"
             - python-version: "3.9"
               cibuildwheel: "cp38"
             - python-version: "3.9"
               cibuildwheel: "cp310"
             - python-version: "3.9"
               cibuildwheel: "cp311"
+            - python-version: "3.9"
+              cibuildwheel: "cp312"
             - python-version: "3.10"
               cibuildwheel: "cp38"
             - python-version: "3.10"
               cibuildwheel: "cp39"
             - python-version: "3.10"
               cibuildwheel: "cp311"
+            - python-version: "3.10"
+              cibuildwheel: "cp312"
             - python-version: "3.11"
               cibuildwheel: "cp38"
             - python-version: "3.11"
               cibuildwheel: "cp39"
             - python-version: "3.11"
               cibuildwheel: "cp310"
+            - python-version: "3.11"
+              cibuildwheel: "cp312"
+            - python-version: "3.12"
+              cibuildwheel: "cp38"
+            - python-version: "3.12"
+              cibuildwheel: "cp39"
+            - python-version: "3.12"
+              cibuildwheel: "cp310"
+            - python-version: "3.12"
+              cibuildwheel: "cp311"
+
 
             ##############################################
             # Things to exclude if not a full matrix run #
@@ -402,6 +421,7 @@ jobs:
             - 3.9
             - "3.10"
             - 3.11
+            - 3.12
           is-full-run:
             - ${{ needs.initialize.outputs.FULL_RUN == 'true' }}
           exclude:
@@ -625,6 +645,20 @@ jobs:
           run: make test
           if: ${{ contains( 'numpy', matrix.package )}}
 
+    ###########################
+    #~~~~~~~~~~~~~~~~~~~~~~~~~#
+    #~~~~~~|#############|~~~~#
+    #~~~~~~|#|~~~~~~~/##/~~~~~#
+    #~~~~~~|#|~~~~~/##/~~~~~~~#
+    #~~~~~~~~~~~~/##/~~~~~~~~~#
+    #~~~~~~~~~~/##/~~~~~~~~~~~#
+    #~~~~~~~~/##/~~~~~~~~~~~~~#
+    #~~~~~~/##/~~~~~~~~~~~~~~~#
+    #~~~~~~~~~~~~~~~~~~~~~~~~~#
+    #  Test Service Adapters  #
+    #~~~~~~~~~~~~~~~~~~~~~~~~~#
+    # Coming soon!
+
     #############################
     #~~~~~~~~~~~~~~~~~~~~~~~~~~~#
     #~~~~~~|#############|~~~~~~#

.github/workflows/conda.yml

@@ -52,7 +52,7 @@ jobs:
         - name: Set up Caches
           uses: ./.github/actions/setup-caches
           with:
-            cibuildwheel: 'cp311'
+            cibuildwheel: 'cp312'
 
         - name: Python Lint Steps
           run: make lint

CMakeLists.txt

@@ -3,7 +3,7 @@
 #########################
 cmake_minimum_required(VERSION 3.20.0)
 project(csp VERSION "0.0.3")
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 20)
 
 ###################################################################################################################################################
 # CMake Dependencies #
@@ -98,7 +98,7 @@ if(CSP_USE_CCACHE)
 endif()
 
 if(NOT DEFINED CSP_PYTHON_VERSION)
-    set(CSP_PYTHON_VERSION 3.11)
+    set(CSP_PYTHON_VERSION 3.12)
 endif()
 
 # Path to python folder for autogen
@@ -153,9 +153,6 @@ endif()
 ###################################################################################################################################################
 # Flags #
 #########
-# Compiler version flags
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
-
 # Optimization Flags
 if(WIN32)
     if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug)

Makefile

@@ -11,7 +11,7 @@ endif
 #########
 # BUILD #
 #########
-.PHONY: requirements develop build build-conda install
+.PHONY: requirements develop build build-debug build-conda install
 
 requirements:  ## install python dev and runtime dependencies
 	python -m pip install toml
@@ -24,6 +24,9 @@ develop: requirements  ## install dependencies and build library
 build:  ## build the library
 	python setup.py build build_ext --inplace -- -- -j$(NPROC)
 
+build-debug:  ## build the library ( DEBUG ) - May need a make clean when switching from regular build to build-debug and vice versa
+	SKBUILD_CONFIGURE_OPTIONS="" DEBUG=1 python setup.py build build_ext --inplace -- -- -j$(NPROC)
+
 build-conda:  ## build the library in Conda
 	CSP_USE_VCPKG=0 python setup.py build build_ext --inplace -- -- -j$(NPROC)
 
@@ -38,14 +41,15 @@ install:  ## install library
 lint-py:
 	python -m isort --check csp/ examples/ setup.py
 	python -m ruff check csp/ examples/ setup.py
+	python -m ruff format --check csp/ examples/ setup.py
 
 lint-cpp:
 	# clang-format --dry-run -Werror -i -style=file `find ./cpp/ -name "*.*pp"`
 	echo "C++ linting disabled for now"
 
 lint-docs:
-	python -m mdformat --check docs/wiki/ README.md examples/README.md
-	python -m codespell_lib docs/wiki/ README.md examples/README.md
+	python -m mdformat --check docs/wiki/ README.md examples/
+	python -m codespell_lib docs/wiki/ README.md examples/ --skip "*.cpp,*.h"
 
 # lint: lint-py lint-cpp  ## run lints
 lint: lint-py lint-docs ## run lints
@@ -62,8 +66,8 @@ fix-cpp:
 	echo "C++ autoformatting disabled for now"
 
 fix-docs:
-	python -m mdformat docs/wiki/ README.md examples/README.md
-	python -m codespell_lib --write docs/wiki/ README.md examples/README.md
+	python -m mdformat docs/wiki/ README.md examples/
+	python -m codespell_lib --write docs/wiki/ README.md examples/ --skip "*.cpp,*.h"
 
 fix: fix-py fix-cpp fix-docs ## run autofixers
 
@@ -191,9 +195,6 @@ dependencies-debian:  ## install dependencies for linux
 dependencies-fedora:  ## install dependencies for linux
 	yum install -y automake bison ccache cmake curl flex perl-IPC-Cmd tar unzip zip
 
-dependencies-alma:  ## install dependencies for linux
-	dnf install -y automake bison ccache cmake curl flex perl-IPC-Cmd tar unzip zip
-
 dependencies-vcpkg:  ## install dependnecies via vcpkg
 	cd vcpkg && ./bootstrap-vcpkg.sh && ./vcpkg install
 

README.md

@@ -61,7 +61,7 @@ See [our wiki!](https://github.com/Point72/csp/wiki)
 
 ## Development
 
-Check out the [Developer Documentation](https://github.com/Point72/csp/wiki/99.-Developer)
+Check out the [contribution guide](https://github.com/Point72/csp/wiki/Contribute) and [local development instructions](https://github.com/Point72/csp/wiki/Local-Development-Setup).
 
 ## Authors
 

conda/dev-environment-unix.yml

@@ -5,45 +5,50 @@ channels:
 dependencies:
  - bison
  - brotli
+ - build
  - bump2version>=1
  - cmake
- - codespell
+ - codespell>=2.2.6,<2.3
  - compilers
  - cyrus-sasl
  - exprtk
  - flex
+ - graphviz
  - python-graphviz
  - gtest
- - httpx
- - isort
- - libarrow=15
+ - httpx>=0.20,<1
+ - isort>=5,<6
+ - libarrow=16
  - librdkafka
  - libboost-headers
  - lz4-c
  - mamba
- - mdformat
+ - mdformat>=0.7.17,<0.8
  - ninja
  - numpy
+ - pillow
  - psutil
- - pyarrow=15
+ - pyarrow=16
  - pandas
  - pillow
  - polars
+ - psutil
  - pytz
  - pytest
+ - pytest-asyncio
  - pytest-cov
  - pytest-sugar
- - pytest-asyncio
- - python<3.12
+ - python<3.13
  - python-rapidjson
  - rapidjson
  - requests
  - ruamel.yaml
- - ruff
+ - ruff>=0.3,<0.4
  - scikit-build
  - slack-sdk
  - sqlalchemy
  - tar
+ - threadpoolctl
  - tornado
  - twine
  - unzip

cpp/csp/adapters/websocket/CMakeLists.txt

@@ -1,5 +1,8 @@
 csp_autogen( csp.adapters.websocket_types websocket_types WEBSOCKET_HEADER WEBSOCKET_SOURCE )
 
+# Need to build websocket adapter under cpp17 standard due to websocketpp incompatibility issues
+set(CMAKE_CXX_STANDARD 17)
+
 set(WS_CLIENT_HEADER_FILES
         ClientAdapterManager.h
         ClientInputAdapter.h

cpp/csp/adapters/websocket/ClientInputAdapter.cpp

@@ -30,7 +30,7 @@ void ClientInputAdapter::processMessage( std::string payload, PushBatch* batch )
 
     if( type() -> type() == CspType::Type::STRUCT )
     {
-        auto tick = m_converter -> asStruct( &payload, payload.length() );
+        auto tick = m_converter -> asStruct( (void*)payload.data(), payload.length() );
         pushTick( std::move(tick), batch );
     } else if ( type() -> type() == CspType::Type::STRING )
     {

cpp/csp/adapters/websocket/ClientInputAdapter.h

@@ -1,8 +1,6 @@
 #ifndef _IN_CSP_ADAPTERS_WEBSOCKETS_CLIENT_INPUTADAPTER_H
 #define _IN_CSP_ADAPTERS_WEBSOCKETS_CLIENT_INPUTADAPTER_H
 
-#include <websocketpp/config/core_client.hpp>
-#include <websocketpp/client.hpp>
 #include <csp/engine/Dictionary.h>
 #include <csp/adapters/utils/MessageStructConverter.h>
 #include <csp/engine/PushInputAdapter.h>

cpp/csp/adapters/websocket/ClientOutputAdapter.h

@@ -5,7 +5,6 @@
 #include <csp/engine/Dictionary.h>
 #include <csp/engine/OutputAdapter.h>
 #include <csp/adapters/utils/MessageWriter.h>
-#include <websocketpp/client.hpp>
 
 namespace csp::adapters::websocket
 {

cpp/csp/cppnodes/baselibimpl.cpp

@@ -379,11 +379,11 @@ DECLARE_CPPNODE( unroll )
                 {
                     size_t idx = 0;
                     if( !s_pending )
-                        CSP_OUTPUT( v[idx++] );
+                        CSP_OUTPUT( static_cast<ElemT>( v[idx++] ) );
 
                     s_pending += sz - idx;
                     for( ; idx < sz; ++idx )
-                        csp.schedule_alarm( alarm, TimeDelta::ZERO(), v[idx] );
+                        csp.schedule_alarm( alarm, TimeDelta::ZERO(), static_cast<ElemT>( v[idx] ) );
                 }
             }
 

cpp/csp/engine/PushPullInputAdapter.cpp

@@ -81,7 +81,7 @@ PushPullInputAdapter::PullDataEvent * PushPullInputAdapter::nextPullEvent()
     auto * event = m_poppedPullEvents.front();
     m_poppedPullEvents.pop();
 
-    if( m_adjustOutOfOrderTime )
+    if( m_adjustOutOfOrderTime && event )
         event -> time = std::max( event -> time, rootEngine() -> now() );
 
     return event;    

cpp/csp/engine/RootEngine.cpp

@@ -11,12 +11,24 @@
 namespace csp
 {
 
-static volatile bool    g_SIGNALED = false;
+static volatile int     g_SIGNAL_COUNT = 0;
+/*
+The signal count variable is maintained to ensure that multiple engine threads shutdown properly.
+
+An interrupt should cause all running engines to stop, but should not affect future runs in the same process.
+Thus, each root engine keeps track of the signal count when its created. When an interrupt occurs, one engine thread
+handles the interrupt by incrementing the count. Then, all other root engines detect the signal by comparing their
+initial count to the current count. 
+
+Future runs after the interrupt remain unaffected since they are initialized with the updated signal count, and will
+only consider themselves "interupted" if another signal is received during their execution.
+*/
+
 static struct sigaction g_prevSIGTERMaction;
 
 static void handle_SIGTERM( int signum )
 {
-    g_SIGNALED = true;
+    g_SIGNAL_COUNT++;
     if( g_prevSIGTERMaction.sa_handler )
         (*g_prevSIGTERMaction.sa_handler)( signum );
 }
@@ -58,6 +70,7 @@ RootEngine::RootEngine( const Dictionary & settings ) : Engine( m_cycleStepTable
                                                         m_cycleCount( 0 ),
                                                         m_settings( settings ),
                                                         m_inRealtime( false ),
+                                                        m_initSignalCount( g_SIGNAL_COUNT ),
                                                         m_pushEventQueue( m_settings.queueWaitTime > TimeDelta::ZERO() )
 {
     if( settings.get<bool>( "profile",  false ) )
@@ -78,7 +91,7 @@ RootEngine::~RootEngine()
 
 bool RootEngine::interrupted() const
 {
-    return g_SIGNALED;
+    return g_SIGNAL_COUNT != m_initSignalCount;
 }
 
 void RootEngine::preRun( DateTime start, DateTime end )
@@ -131,7 +144,7 @@ void RootEngine::processEndCycle()
 void RootEngine::runSim( DateTime end )
 {
     m_inRealtime = false;
-    while( m_scheduler.hasEvents() && m_state == State::RUNNING && !g_SIGNALED )
+    while( m_scheduler.hasEvents() && m_state == State::RUNNING && !interrupted() )
     {
         m_now = m_scheduler.nextTime();
         if( m_now > end )
@@ -161,7 +174,7 @@ void RootEngine::runRealtime( DateTime end )
 
     m_inRealtime = true;
     bool haveEvents = false;
-    while( m_state == State::RUNNING && !g_SIGNALED )
+    while( m_state == State::RUNNING && !interrupted() )
     {
         TimeDelta waitTime;
         if( !m_pendingPushEvents.hasEvents() )