Merge pull request #3333 from neuronsimulator/jelic/clean-nmodl-merge

Bring NMODL transpiler under NEURON

Author: JCGoran

.bbp-project.yaml

@@ -9,6 +9,13 @@ tools:
   ClangFormat:
     enable: True
     version: == 12.0.1
+    exclude:
+      match:
+      - src/nmodl/language/templates/*
   CMakeFormat:
     enable: True
     version: == 0.6.13
+    exclude:
+      match:
+      - src/nmodl/language/templates/*
+      - test/nmodl/usecases/*/*

.github/workflows/coverage.yml

@@ -79,11 +79,6 @@ jobs:
         with:
           fetch-depth: 2
 
-      - name: Clone nmodl
-        working-directory: ${{runner.workspace}}/nrn
-        run: |
-            git submodule update --init --recursive --force --depth 1 -- external/nmodl
-
       - name: Set up Python@${{ env.PY_MIN_VERSION }}
         uses: actions/setup-python@v5
         with:
@@ -92,7 +87,6 @@ jobs:
       - name: Install Python@${{ env.PY_MIN_VERSION }} dependencies
         working-directory: ${{runner.workspace}}/nrn
         run: |
-          python -m pip install --upgrade -r external/nmodl/requirements.txt
           python -m pip install --upgrade -r ci_requirements.txt
           python -m pip install --upgrade pip -r nrn_requirements.txt
 
@@ -109,7 +103,6 @@ jobs:
       - name: Install Python@${{ env.PY_MAX_VERSION }} dependencies
         working-directory: ${{runner.workspace}}/nrn
         run: |
-          python -m pip install --upgrade -r external/nmodl/requirements.txt
           python -m pip install --upgrade -r ci_requirements.txt
           python -m pip install --upgrade pip -r nrn_requirements.txt
 

.github/workflows/neuron-ci.yml

@@ -159,11 +159,6 @@ jobs:
         with:
           fetch-depth: 2
 
-      - name: Clone nmodl
-        working-directory: ${{runner.workspace}}/nrn
-        run: |
-            git submodule update --init --recursive --force --depth 1 -- external/nmodl
-
       - name: Set up Python@${{ env.PY_MIN_VERSION }}
         if: ${{matrix.config.python_dynamic == 'ON'}}
         uses: actions/setup-python@v5
@@ -174,7 +169,6 @@ jobs:
         if: ${{ matrix.config.python_dynamic == 'ON' }}
         working-directory: ${{runner.workspace}}/nrn
         run: |
-          python -m pip install --upgrade -r external/nmodl/requirements.txt
           python -m pip install --upgrade -r ci_requirements.txt
           python -m pip install --upgrade pip -r nrn_requirements.txt
 
@@ -186,7 +180,6 @@ jobs:
       - name: Install Python@${{ env.PY_MAX_VERSION }} dependencies
         working-directory: ${{runner.workspace}}/nrn
         run: |
-          python -m pip install --upgrade -r external/nmodl/requirements.txt
           python -m pip install --upgrade -r ci_requirements.txt
           python -m pip install --upgrade pip -r nrn_requirements.txt
 

.gitmodules

@@ -16,9 +16,6 @@
 [submodule "external/Random123"]
 	path = external/Random123
 	url = https://github.com/BlueBrain/Random123.git
-[submodule "external/nmodl"]
-	path = external/nmodl
-	url = https://github.com/BlueBrain/nmodl
 [submodule "external/CLI11"]
 	path = external/CLI11
 	url = https://github.com/CLIUtils/CLI11.git
@@ -31,3 +28,12 @@
 [submodule "external/fmt"]
 	path = external/fmt
 	url = https://github.com/fmtlib/fmt
+[submodule "external/pybind11"]
+	path = external/pybind11
+	url = https://github.com/pybind/pybind11.git
+[submodule "external/json"]
+	path = external/json
+	url = https://github.com/nlohmann/json.git
+[submodule "external/spdlog"]
+	path = external/spdlog
+	url = https://github.com/gabime/spdlog.git

CMakeLists.txt

@@ -209,6 +209,15 @@ set(NRN_PROFILER
     "${NRN_PROFILER_DEFAULT}"
     CACHE STRING "Set which profiler to build against ('caliper', 'likwid')")
 
+# =============================================================================
+# Build options for NMODL
+# =============================================================================
+option(NMODL_ENABLE_PYTHON_BINDINGS "Enable pybind11 based python bindings" ON)
+set(NMODL_EXTRA_CXX_FLAGS
+    ""
+    CACHE STRING "Add extra compile flags for NMODL sources")
+separate_arguments(NMODL_EXTRA_CXX_FLAGS)
+
 # =============================================================================
 # Include cmake modules
 # =============================================================================
@@ -507,22 +516,50 @@ cpp_cc_git_submodule(CLI11 BUILD PACKAGE CLI11 REQUIRED)
 # Enable NMODL code-generator support
 # =============================================================================
 if(NRN_ENABLE_NMODL OR NRN_ENABLE_CORENEURON)
+  # NMODL Python build dependencies
+  nrn_find_python_module(MODULE jinja2 REQUIRED)
+  # N.B. `nrn_find_python_module` uses the _importable_ name of the package to find it, _not_ the
+  # actual package name. In this case, `pip install pyyaml` actually installs an importable package
+  # called `yaml`
+  nrn_find_python_module(MODULE yaml REQUIRED)
+
+  # NMODL non-Python build dependencies
+  cpp_cc_git_submodule(json BUILD PACKAGE nlohmann_json REQUIRED)
+  cpp_cc_git_submodule(pybind11 BUILD PACKAGE pybind11 REQUIRED)
+
+  option(SPDLOG_FMT_EXTERNAL "Force to use an external {{fmt}}" ON)
+  option(SPDLOG_SYSTEM_INCLUDE "Include spdlog as a system lib" ON)
+  cpp_cc_git_submodule(spdlog BUILD PACKAGE spdlog REQUIRED)
+  if(${CODING_CONV_PREFIX}_3RDPARTY_USE_SPDLOG)
+    # See above, same logic as fmt
+    set_target_properties(spdlog PROPERTIES POSITION_INDEPENDENT_CODE ON)
+  endif()
 
   set(NMODL_ENABLE_PYTHON_BINDINGS
       OFF
       CACHE BOOL "Enable NMODL python bindings")
-  nrn_add_external_project(nmodl)
+
+  include(${PROJECT_SOURCE_DIR}/cmake/nmodl/PythonLinkHelper.cmake)
+
+  # where the platform-specific sources are located
+  set(NMODL_PROJECT_PLATLIB_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/nmodl)
+  set(NMODL_PROJECT_PLATLIB_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/src/nmodl)
+  # where the platform-independent sources are located
+  set(NMODL_PROJECT_PURELIB_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/share/lib/python/neuron/nmodl)
+  set(NMODL_PROJECT_PURELIB_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/lib/python/neuron/nmodl)
+
+  add_subdirectory(src/nmodl)
   set(CORENRN_NMODL_BINARY ${CMAKE_BINARY_DIR}/bin/nmodl${CMAKE_EXECUTABLE_SUFFIX})
   set(NMODL_TARGET_TO_DEPEND nmodl)
-  set(NMODL_PROJECT_BINARY_DIR ${CMAKE_BINARY_DIR}/external/nmodl)
+  set(NMODL_PROJECT_PLATLIB_BINARY_DIR ${CMAKE_BINARY_DIR}/src/nmodl)
   # install nrnunits.lib and libpywrapper.so from external/nmodl
   install(
-    FILES ${NMODL_PROJECT_BINARY_DIR}/lib/libpywrapper${CMAKE_SHARED_LIBRARY_SUFFIX}
+    FILES ${NMODL_PROJECT_PLATLIB_BINARY_DIR}/lib/libpywrapper${CMAKE_SHARED_LIBRARY_SUFFIX}
     DESTINATION lib
     COMPONENT pywrapper
     OPTIONAL)
   install(
-    FILES ${NMODL_PROJECT_BINARY_DIR}/share/nmodl/nrnunits.lib
+    FILES ${NMODL_PROJECT_PLATLIB_BINARY_DIR}/share/nmodl/nrnunits.lib
     DESTINATION share/nmodl
     COMPONENT nrnunits)
 
@@ -648,6 +685,13 @@ function(prepend_to_var var new_path)
 endfunction()
 set(NRN_RUN_FROM_BUILD_DIR_ENV "NEURONHOME=${PROJECT_BINARY_DIR}/share/nrn"
                                "NRNHOME=${PROJECT_BINARY_DIR}")
+
+# if we do not link against Python, NMODL will complain that some of the env variables are missing
+# when building coreNEURON mod files
+if(NOT NRN_LINK_AGAINST_PYTHON)
+  set(NRN_RUN_FROM_BUILD_DIR_ENV "${NRN_RUN_FROM_BUILD_DIR_ENV}" "NMODLHOME=${PROJECT_BINARY_DIR}"
+                                 "NMODL_PYLIB=${PYTHON_LIBRARY}")
+endif()
 prepend_to_var(PATH "${PROJECT_BINARY_DIR}/bin")
 if(APPLE)
   prepend_to_var(DYLD_LIBRARY_PATH "${PROJECT_BINARY_DIR}/lib")

cmake/CompilerHelper.cmake

@@ -2,11 +2,11 @@
 # Compiler specific settings
 # =============================================================================
 if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR NRN_MACOS_BUILD)
-  set(UNDEFINED_SYMBOLS_IGNORE_FLAG "-undefined dynamic_lookup")
+  set(UNDEFINED_SYMBOLS_IGNORE_FLAG "-Wl,-undefined,dynamic_lookup")
   string(APPEND CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS " ${UNDEFINED_SYMBOLS_IGNORE_FLAG}")
   string(APPEND CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS " ${UNDEFINED_SYMBOLS_IGNORE_FLAG}")
 else()
-  set(UNDEFINED_SYMBOLS_IGNORE_FLAG "--unresolved-symbols=ignore-all")
+  set(UNDEFINED_SYMBOLS_IGNORE_FLAG "-Wl,--unresolved-symbols=ignore-all")
 endif()
 
 if(CMAKE_C_COMPILER_ID MATCHES "PGI" OR CMAKE_C_COMPILER_ID MATCHES "NVHPC")

cmake/MacroHelper.cmake

@@ -169,6 +169,7 @@ macro(nocmodl_mod_to_cpp modfile_basename modfile_compat)
     OUTPUT ${CPPFILE_OUTPUT_PATH}
     COMMAND
       ${CMAKE_COMMAND} -E env "MODLUNIT=${PROJECT_BINARY_DIR}/share/nrn/lib/nrnunits.lib"
+      "NMODL_PYLIB=${PYTHON_LIBRARY}" "NMODLHOME=${PROJECT_BINARY_DIR}"
       ${NRN_NOCMODL_SANITIZER_ENVIRONMENT} $<TARGET_FILE:${NRN_CODEGENERATOR_TARGET}>
       ${MODFILE_OUTPUT_PATH} ${NRN_NMODL_--neuron} -o ${modfile_output_dir}
     COMMAND sed ${NOCMODL_SED_EXPR} ${CPPFILE_OUTPUT_PATH} > ${CPPFILE_OUTPUT_PATH}.tmp

cmake/nmodl/ClangTidyHelper.cmake

@@ -0,0 +1,20 @@
+set(ENABLE_CLANG_TIDY
+    OFF
+    CACHE BOOL "Add clang-tidy automatically to builds")
+if(ENABLE_CLANG_TIDY)
+  find_program(CLANG_TIDY_EXE NAMES "clang-tidy")
+  if(CLANG_TIDY_EXE)
+    message(STATUS "clang-tidy found: ${CLANG_TIDY_EXE}")
+    set(CLANG_TIDY_CHECKS
+        "-*,modernize-*,readability-*,performance-*,cppcoreguidelines-*,clang-analyzer-core*,google-*"
+    )
+    set(CMAKE_CXX_CLANG_TIDY
+        "${CLANG_TIDY_EXE};-checks=${CLANG_TIDY_CHECKS};-fix;-header-filter='${CMAKE_SOURCE_DIR}/*'"
+        CACHE STRING "" FORCE)
+  else()
+    message(AUTHOR_WARNING "clang-tidy not found!")
+    set(CMAKE_CXX_CLANG_TIDY
+        ""
+        CACHE STRING "" FORCE) # delete it
+  endif()
+endif()

cmake/nmodl/CompilerHelper.cmake

@@ -0,0 +1,48 @@
+if(CMAKE_CXX_COMPILER_ID MATCHES "PGI" OR CMAKE_CXX_COMPILER_ID MATCHES "NVHPC")
+  set(NMODL_PGI_COMPILER TRUE)
+
+  # CMake adds standard complaint PGI flag "-A" which breaks compilation of of spdlog and fmt
+  set(CMAKE_CXX14_STANDARD_COMPILE_OPTION --c++14)
+
+  # Avoid errors related to "excessive recursion at instantiation of function ...", Eigen-related
+  # (in accelerated regions), e.g., transposeInPlace()
+  list(APPEND NMODL_EXTRA_CXX_FLAGS "-Wc,--pending_instantiations=0")
+
+  # ~~~
+  # PGI enables number of diagnostic messages by default classes which results into thousands of
+  # messages specifically for AST. Disable these verbose warnings for now.
+  # TODO : fix these warnings from template modification (#272)
+  # ~~~
+  if(${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 20.7)
+    set(NMODL_COMPILER_WARNING_SUPPRESSIONS --diag_suppress=1,82,111,115,177,186,611,997,1097,1625)
+  else()
+    # https://forums.developer.nvidia.com/t/many-all-diagnostic-numbers-increased-by-1-from-previous-values/146268/3
+    # changed the numbering scheme in newer versions. The following list is from a clean start 13
+    # August 2021. It would clearly be nicer to apply these suppressions only to relevant files.
+    # Examples of the suppressed warnings are given below.
+    # ~~~
+    # "ext/spdlog/include/spdlog/fmt/fmt.h", warning #1-D: last line of file ends without a newline
+    # "ext/fmt/include/fmt/format.h", warning #111-D: statement is unreachable
+    # "ext/fmt/include/fmt/format.h", warning #128-D: loop is not reachable
+    # "ext/spdlog/include/spdlog/fmt/bundled/format.h", warning #185-D: dynamic initialization in unreachable code
+    # "ext/json/json.hpp", warning #186-D: pointless comparison of unsigned integer with zero
+    # "src/ast/all.hpp", warning #998-D: function "..." is hidden by "..." -- virtual function override intended?
+    # "ext/spdlog/include/spdlog/fmt/bundled/format.h", warning #1098-D: unknown attribute "fallthrough"
+    # "ext/pybind11/include/pybind11/detail/common.h", warning #1626-D: routine is both "inline" and "noinline"
+    # "ext/spdlog/include/spdlog/fmt/bundled/core.h", warning #1676-D: unrecognized GCC pragma
+    # ~~~
+    # The following warnings do not seem to be suppressible with --diag_suppress:
+    # ~~~
+    # "src/codegen/codegen_cuda_visitor.cpp", NVC++-W-0277-Cannot inline function - data type mismatch
+    # "nvc++IkWUbMugiSgNH.s: Warning: stand-alone `data16' prefix
+    # ~~~
+    # The situation may be better once https://github.com/fmtlib/fmt/pull/2582 is included in a
+    # release.
+    set(NMODL_COMPILER_WARNING_SUPPRESSIONS --diag_suppress=1,111,128,185,186,998,1098,1626,1676)
+    # There are a few more warnings produced by the unit test infrastructure.
+    # ~~~
+    # "test/unit/visitor/constant_folder.cpp", warning #177-D: variable "..." was declared but never referenced
+    # ~~~
+    set(NMODL_TESTS_COMPILER_WARNING_SUPPRESSIONS --diag_suppress=177)
+  endif()
+endif()

cmake/nmodl/Config.cmake.in

@@ -0,0 +1,7 @@
+@PACKAGE_INIT@
+
+include("${CMAKE_CURRENT_LIST_DIR}/nmodlTargets.cmake")
+
+set(nmodl_BINARY ${PACKAGE_PREFIX_DIR}/@nmodl_BINARY@)
+
+check_required_components(nmodl)

cmake/nmodl/ExternalProjectHelper.cmake

@@ -0,0 +1,12 @@
+find_package(Git QUIET)
+
+# initialize submodule with given path
+function(initialize_submodule path)
+  if(NOT ${GIT_FOUND})
+    message(
+      FATAL_ERROR "git not found and ${path} sub-module not cloned (use git clone --recursive)")
+  endif()
+  message(STATUS "Sub-module : missing ${path}: running git submodule update --init")
+  execute_process(COMMAND git submodule update --init -- ${path}
+                  WORKING_DIRECTORY ${NMODL_PROJECT_PLATLIB_SOURCE_DIR})
+endfunction()

cmake/nmodl/FlexHelper.cmake

@@ -0,0 +1,17 @@
+# Often older version of flex is available in /usr. Even we set PATH for newer flex, CMake will set
+# FLEX_INCLUDE_DIRS to /usr/include. This will result in compilation errors. Hence we check for flex
+# include directory for the corresponding FLEX_EXECUTABLE. If found, we add that first and then we
+# include include path from CMake.
+
+get_filename_component(FLEX_BIN_DIR ${FLEX_EXECUTABLE} DIRECTORY)
+
+if(NOT FLEX_BIN_DIR MATCHES "/usr/bin")
+  if(NOT FLEX_INCLUDE_PATH)
+    get_filename_component(FLEX_INCLUDE_PATH ${FLEX_BIN_DIR} PATH)
+    set(FLEX_INCLUDE_PATH ${FLEX_INCLUDE_PATH}/include/)
+  endif()
+  if(EXISTS "${FLEX_INCLUDE_PATH}/FlexLexer.h")
+    message(STATUS " Adding Flex include path as : ${FLEX_INCLUDE_PATH}")
+    include_directories(${FLEX_INCLUDE_PATH})
+  endif()
+endif()

cmake/nmodl/GitRevision.cmake

@@ -0,0 +1,49 @@
+# For now use simple approach to get version information as git is always avaialble on the machine
+# where we are building
+
+find_package(Git)
+
+if(GIT_FOUND)
+
+  # get last commit sha1
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} log -1 --format=%h
+    WORKING_DIRECTORY ${NMODL_PROJECT_PLATLIB_SOURCE_DIR}
+    OUTPUT_VARIABLE NMODL_GIT_REVISION_SHA1
+    RESULT_VARIABLE NMODL_GIT_STATUS
+    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+  if(NOT NMODL_GIT_STATUS EQUAL 0)
+    set(NMODL_GIT_REVISION_SHA1 "git-error")
+  endif()
+
+  # get last commit date
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} show -s --format=%ci
+    WORKING_DIRECTORY ${NMODL_PROJECT_PLATLIB_SOURCE_DIR}
+    OUTPUT_VARIABLE NMODL_GIT_REVISION_DATE
+    RESULT_VARIABLE NMODL_GIT_STATUS
+    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+  if(NOT NMODL_GIT_STATUS EQUAL 0)
+    set(NMODL_GIT_REVISION_SHA1 "git-error")
+  endif()
+
+  # remove extra double quotes
+  string(REGEX REPLACE "\"" "" NMODL_GIT_REVISION_DATE "${NMODL_GIT_REVISION_DATE}")
+  set(NMODL_GIT_REVISION "${NMODL_GIT_REVISION_SHA1} ${NMODL_GIT_REVISION_DATE}")
+
+  # get the last version tag from git
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} describe --abbrev=0 --tags
+    WORKING_DIRECTORY ${NMODL_PROJECT_PLATLIB_SOURCE_DIR}
+    OUTPUT_VARIABLE NMODL_GIT_LAST_TAG
+    RESULT_VARIABLE NMODL_GIT_STATUS
+    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+  if(NOT NMODL_GIT_STATUS EQUAL 0)
+    # Must be a valid version from CMake's perspective.
+    set(NMODL_GIT_LAST_TAG "0.0")
+  endif()
+else()
+  set(NMODL_GIT_REVISION "unknown")
+  # Must be a valid version from CMake's perspective.
+  set(NMODL_GIT_LAST_TAG "0.0")
+endif()

cmake/nmodl/PythonLinkHelper.cmake

@@ -0,0 +1,23 @@
+# =============================================================================
+# Support for building NMODL without python library
+# =============================================================================
+# ~~~
+# When NMODL is built under environment  (e.g. manylinux) without python-dev
+# package i.e. libpython.so, we have to build extension and binaries by
+# ignoring undefned symbols. The python library will be loaded later on target
+# system.
+# ~~~
+
+# use same variable name as NEURON as it won't be user option
+set(NRN_LINK_AGAINST_PYTHON
+    TRUE
+    CACHE BOOL "Disable linking to python library")
+
+mark_as_advanced(NRN_LINK_AGAINST_PYTHON)
+
+if(NOT NRN_LINK_AGAINST_PYTHON)
+  string(APPEND CMAKE_EXE_LINKER_FLAGS " ${UNDEFINED_SYMBOLS_IGNORE_FLAG}")
+  set(NMODL_WRAPPER_LIBS pyembed dl)
+else()
+  set(NMODL_WRAPPER_LIBS pyembed pywrapper dl)
+endif()