diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3ba3d5da..8fadfb34 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,4 +10,9 @@ updates: # Check for updates once a week schedule: interval: "weekly" + # Group actions version bumps into a single PR + groups: + actions-deps: + patterns: + - "*" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 771659bf..d1621e0b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -10,7 +10,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 0 # Fetch full history for git describe to work diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index e74324d3..301237df 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -199,7 +199,7 @@ jobs: FMT_REF: "11.1.4" steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 # unshallow for setuptools-scm.get_version to discover tags @@ -281,7 +281,7 @@ jobs: path: C:\vcpkg\installed key: ${{ matrix.os }}-${{ env.cache-name }}-a - - uses: pypa/cibuildwheel@v3.4.1 + - uses: pypa/cibuildwheel@v4.1.0 if: startsWith(matrix.os, 'ubuntu-') env: CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} @@ -334,7 +334,7 @@ jobs: output-dir: wheelhouse config-file: "{package}/pyproject.toml" - - uses: pypa/cibuildwheel@v3.4.1 + - uses: pypa/cibuildwheel@v4.1.0 if: startsWith(matrix.os, 'macos-') env: CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 9f14738a..e69e98eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,13 @@ endif() # ############################################################################## option(AL_PYTHON_BINDINGS "Build Python bindings" OFF) +# When ON, the al C++ library is not built here; instead it is located with +# find_package(al-core CONFIG). Enables a two-stage build where libimas-core +# (the C/C++ library) and imas-core (the Python wrapper) are packaged +# separately, e.g. as two conda packages. +option(AL_USE_INSTALLED_CORE + "Link Python bindings against a pre-installed al-core (find_package)" OFF) + # Configuration options for shared libraries # ############################################################################## @@ -127,8 +134,31 @@ if(WIN32) find_package(dlfcn-win32 CONFIG REQUIRED) endif() -# build AL_CORE library only if backend is enabled -if(AL_BACKEND_HDF5 OR AL_BACKEND_MDSPLUS OR AL_BACKEND_UDA OR AL_BACKEND_UDAFAT OR AL_PYTHON_BINDINGS) +# Two-stage build entry: locate a pre-installed al-core instead of building it. +# Exposes target `al-core::al` and an unqualified alias `al` so the python/ +# subdirectory and any consumers can link to `al` transparently. +if(AL_USE_INSTALLED_CORE) + find_package(al-core CONFIG) + if(NOT al-core_FOUND) + message(FATAL_ERROR + "AL_USE_INSTALLED_CORE=ON requires an installed al-core that ships " + "al-coreConfig.cmake (introduced in IMAS-Core 5.7.1 or later). " + "find_package(al-core CONFIG) could not locate it.\n" + "Point CMAKE_PREFIX_PATH (or al-core_DIR) at an install prefix that " + "contains lib/cmake/al-core/al-coreConfig.cmake — typically the " + "stage-1 install tree of this branch. Releases built before this " + "change (e.g. IMAS-Core/5.6.0) only ship al-core.pc and will NOT " + "satisfy AL_USE_INSTALLED_CORE; either install stage 1 from source " + "or wait for a release that includes the CMake package config.") + endif() + if(NOT TARGET al) + add_library(al ALIAS al-core::al) + endif() +endif() + +# build AL_CORE library only if backend is enabled and we are not reusing an +# already-installed al-core +if((AL_BACKEND_HDF5 OR AL_BACKEND_MDSPLUS OR AL_BACKEND_UDA OR AL_BACKEND_UDAFAT OR AL_PYTHON_BINDINGS) AND NOT AL_USE_INSTALLED_CORE) # Core dependencies set(Boost_USE_MULTITHREADED FALSE) @@ -199,14 +229,43 @@ add_dependencies( imas_print_version al ) # ############################################################################## # Install al library +include(GNUInstallDirs) install( TARGETS al + EXPORT al-coreTargets RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core ) +# Export CMake package config so downstream projects (notably the Python +# wrapper built with AL_USE_INSTALLED_CORE=ON) can `find_package(al-core CONFIG)`. +include(CMakePackageConfigHelpers) +set(_al_core_cmake_dir ${CMAKE_INSTALL_LIBDIR}/cmake/al-core) +install( + EXPORT al-coreTargets + FILE al-coreTargets.cmake + NAMESPACE al-core:: + DESTINATION ${_al_core_cmake_dir} +) +configure_package_config_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/al-coreConfig.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/al-coreConfig.cmake + INSTALL_DESTINATION ${_al_core_cmake_dir} +) +write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/al-coreConfigVersion.cmake + VERSION ${PROJECT_VERSION} + COMPATIBILITY SameMajorVersion +) +install( + FILES + ${CMAKE_CURRENT_BINARY_DIR}/al-coreConfig.cmake + ${CMAKE_CURRENT_BINARY_DIR}/al-coreConfigVersion.cmake + DESTINATION ${_al_core_cmake_dir} +) + # TODO: put public heades in a separate directory? install( FILES ${PUBLIC_HEADER_FILES} @@ -234,20 +293,20 @@ install( ) # Install common files -# Note: The environment variable $AL_COMMON_PATH must be set to this installation directory at runtime install(DIRECTORY common TYPE DATA) # Install Dummy install(TARGETS imas_print_version DESTINATION bin) -# Scikit-build-core entry point for python bindings +endif() # AL core library: (AL_BACKEND_* OR AL_PYTHON_BINDINGS) AND NOT AL_USE_INSTALLED_CORE + +# Scikit-build-core entry point for python bindings — works whether `al` was +# just built here or imported via find_package(al-core). # ############################################################################## if(AL_PYTHON_BINDINGS) include(skbuild.cmake) endif() -endif() # AL core library (AL_BACKEND_HDF5 OR AL_BACKEND_MDSPLUS OR AL_BACKEND_UDA OR AL_BACKEND_UDAFAT OR AL_PYTHON_BINDINGS) - # MDSplus models # ############################################################################## diff --git a/cmake/al-coreConfig.cmake.in b/cmake/al-coreConfig.cmake.in new file mode 100644 index 00000000..4d8b30ed --- /dev/null +++ b/cmake/al-coreConfig.cmake.in @@ -0,0 +1,8 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) +find_dependency(Boost COMPONENTS filesystem) + +include("${CMAKE_CURRENT_LIST_DIR}/al-coreTargets.cmake") + +check_required_components(al-core) diff --git a/common/README.md b/common/README.md index e1b4899b..a6e6ae75 100644 --- a/common/README.md +++ b/common/README.md @@ -11,7 +11,7 @@ During the CMake configure step of any of the other Access Layer repositories (`al-core`, `al-fortran`, etc.) this folder is imported by CMake: ```cmake -# Configuration options for common assets +# Configuration options for common repos ################################################################################ option( AL_DOWNLOAD_DEPENDENCIES "Automatically download assets from the AL git repository" ON ) set( AL_CORE_GIT_REPOSITORY "git@github.com:iterorganization/IMAS-Core.git" CACHE STRING "Git repository of AL-core" ) @@ -19,29 +19,22 @@ set( AL_CORE_VERSION "main" CACHE STRING "Git commit/tag/branch of AL-core" ) include(FetchContent) -# Load common assets ################################################################################ -if( DEFINED ENV{AL_COMMON_PATH} ) - # Take common assets from the path in this environment variable instead of al-core - set( AL_COMMON_PATH $ENV{AL_COMMON_PATH} ) +if( ${AL_DOWNLOAD_DEPENDENCIES} ) + # Download common assets from the ITER git: + FetchContent_Declare( + al-core + GIT_REPOSITORY "${AL_CORE_GIT_REPOSITORY}" + GIT_TAG "${AL_CORE_VERSION}" + ) else() - if( ${AL_DOWNLOAD_DEPENDENCIES} ) - # Download common assets from the ITER git: - FetchContent_Declare( - al-core - GIT_REPOSITORY "${AL_CORE_GIT_REPOSITORY}" - GIT_TAG "${AL_CORE_VERSION}" - ) - else() - FetchContent_Declare( - al-core - SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../al-core" - ) - endif() - FetchContent_MakeAvailable( al-core ) - set( AL_COMMON_PATH "${al-core_SOURCE_DIR}/common" ) + FetchContent_Declare( + al-core + SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../al-core" + ) endif() -add_subdirectory( ${AL_COMMON_PATH} _common ) +FetchContent_MakeAvailable( al-core ) +add_subdirectory( "${al-core_SOURCE_DIR}/common" _common ) ``` Contrary to what the name implies, the `GIT_TAG` can be a branch name, tag diff --git a/common/cmake/ALCommonConfig.cmake b/common/cmake/ALCommonConfig.cmake index 2431a5d9..b25cebf1 100644 --- a/common/cmake/ALCommonConfig.cmake +++ b/common/cmake/ALCommonConfig.cmake @@ -17,12 +17,7 @@ option( AL_DOCS_ONLY "Don't build anything, except the Sphinx-based High Level I # No longer need to find SaxonHE - saxonche is installed automatically via pip in virtual environments if( NOT AL_DOWNLOAD_DEPENDENCIES ) - if( DEFINED ENV{AL_COMMON_PATH} ) - set( _DEV OFF ) - else() - set( _DEV ON ) - endif() - option( AL_DEVELOPMENT_LAYOUT "Look into parent directories for dependencies" ${_DEV} ) + option( AL_DEVELOPMENT_LAYOUT "Look into parent directories for dependencies" ON ) endif() # Enable CTest? diff --git a/common/doc_common/building_installing.rst b/common/doc_common/building_installing.rst deleted file mode 100644 index 6394abeb..00000000 --- a/common/doc_common/building_installing.rst +++ /dev/null @@ -1,413 +0,0 @@ -Building and installing the Access Layer -======================================== - -This page describes how to build and install the Access Layer. - -Documentation for developers wishing to contribute to the Access Layer can be found in -the :ref:`Access Layer development guide`. Please refer to that guide if you wish to set -up a development environment. - -.. note:: - - For Windows-specific installation instructions, please refer to the - :doc:`Windows Installation Guide `. - - -.. _`build prerequisites`: - -Prerequisites -------------- - -To build the Access Layer you need: - -- Git -- A C++11 compiler (tested with GCC and Intel compilers) -- CMake (3.16 or newer) -- Saxon-HE XSLT processor -- Boost C++ libraries (1.66 or newer) -- PkgConfig - -The following dependencies are only required for some of the components: - -- Backends - - - **HDF5 backend**: HDF5 C/C++ libraries (1.8.12 or newer) - - **MDSplus backend**: MDSplus libraries (7.84.8 or newer) - - **UDA backend**: `UDA `__ libraries - (2.7.5 or newer) [#uda_install]_ - -.. [#uda_install] When installing UDA, make sure you have - `Cap'n'Proto `__ installed in your system - and add its support by adding the CMake switch `-DENABLE_CAPNP=ON` when configuring UDA. - - -- High Level Interfaces - - - All HLIs require the ``xsltproc`` program - - **C++ High Level Interface**: Blitz++ libraries - - **Fortran High Level Interface**: A fortran compiler, ideally with F2008 support - (tested with ``gfortran``, ``ifort`` and ``nagfor``) - - **Java High Level Interface**: Java Development Kit and Java Native Interface - (tested with Java versions 11, 17 and 21) - - **MATLAB High Level Interface**: A working MATLAB installation (tested with - version 2020b) - - **Python High Level Interface**: Python (version 3.8 or newer) with the pip - packages ``build``, ``cython`` and ``numpy`` installed. - - - - -Standard environments: - -.. md-tab-set:: - - .. md-tab-item:: SDCC ``intel-2020b`` - - The following modules provide all the requirements when using the - ``intel-2020b`` toolchain: - - .. code-block:: bash - - module load intel/2020b CMake/3.24.3-GCCcore-10.2.0 Saxon-HE/11.4-Java-11 \ - Boost/1.74.0-GCC-10.2.0 HDF5/1.10.7-iimpi-2020b \ - MDSplus/7.96.17-GCCcore-10.2.0 MDSplus-Java/7.96.17-GCCcore-10.2.0-Java-11 \ - UDA/2.7.4-GCCcore-10.2.0 Blitz++/1.0.2-GCCcore-10.2.0 \ - MATLAB/2020b-GCCcore-10.2.0-Java-11 SciPy-bundle/2020.11-intel-2020b - - .. md-tab-item:: SDCC ``foss-2023b`` - - The following modules provide all the requirements when using the - ``foss-2023b`` toolchain: - - .. code-block:: bash - - module load CMake/3.27.6-GCCcore-13.2.0 Saxon-HE/12.4-Java-21 \ - Boost/1.83.0-GCC-13.2.0 HDF5/1.14.3-gompi-2023b \ - MDSplus/7.132.0-GCCcore-13.2.0 \ - UDA/2.8.0-GCC-13.2.0 Blitz++/1.0.2-GCCcore-13.2.0 \ - MATLAB/2023b-r5 SciPy-bundle/2023.11-gfbf-2023b \ - build/1.0.3-foss-2023b - - .. admonition:: The MATLAB/2023b-r5 installation is lightly tweaked - - The installation at ITER uses `EB PR#20508 - `__ - and its tweak resolves `IMAS-5162 `__ - by removing ``libstdc++.so.6`` from the MATLAB installation. It also adds - ``extern/bin/glnxa64`` to ``LD_LIBRARY_PATH`` to address the - ``MatlabEngine not found`` issue. - - .. caution:: - - When using the HDF5 backend within MATLAB, depending on the HDF5 library being used - you may need to add ``LD_PRELOAD=/lib/libhdf5_hl.so`` when starting - MATLAB. - - .. md-tab-item:: Ubuntu 22.04 - - The following packages provide most requirements when using Ubuntu 22.04: - - .. code-block:: bash - - apt install git build-essential cmake libsaxonhe-java libboost-all-dev \ - pkg-config libhdf5-dev xsltproc libblitz0-dev gfortran \ - default-jdk-headless python3-dev python3-venv python3-pip - - The following dependencies are not available from the package repository, - you will need to install them yourself: - - - MDSplus: see their `GitHub repository - `__ or `home page - `__ for installation instructions. - - UDA: see their `GitHub repository `__ for more - details. - - MATLAB, which is not freely available. - - -Building and installing a single High Level Interface ------------------------------------------------------ - -This section explains how to install a single High Level Interface. Please make sure you -have the :ref:`build prerequisites` installed. - - -Clone the repository -```````````````````` - -First you need to clone the repository of the High Level Interface you want to build: - -.. code-block:: bash - - # For the C++ HLI use: - git clone ssh://git@git.iter.org/imas/al-cpp.git - # For the Fortran HLI use: - git clone ssh://git@git.iter.org/imas/al-fortran.git - # For the Java HLI use: - git clone ssh://git@git.iter.org/imas/al-java.git - # For the MATLAB HLI use: - git clone ssh://git@git.iter.org/imas/al-matlab.git - # For the Python HLI use: - git clone ssh://git@git.iter.org/imas/al-python.git - - -Configuration -````````````` - -Once you have cloned the repository, navigate your shell to the folder and run cmake. -You can pass configuration options with ``-D OPTION=VALUE``. See below list for an -overview of configuration options. - -.. code-block:: bash - - cd al-cpp # al-fortran, al-java, al-matlab or al-python - cmake -B build -D CMAKE_INSTALL_PREFIX=$HOME/al-install -D OPTION1=VALUE1 -D OPTION2=VALUE2 [...] - -.. note:: - - CMake will automatically fetch dependencies from other Access Layer GIT repositories - for you. You may need to provide credentials to clone the following repositories: - - - `imas-core (git@github.com:iterorganization/IMAS-Core.git) - `__ - - `al-plugins (ssh://git@git.iter.org/imas/al-plugins.git) - `__ - - `imas-data-dictionary (git@github.com:iterorganization/IMAS-Data-Dictionary.git) - `__ - - If you need to change the git repositories, for example to point to a mirror of the - repository or to use a HTTPS URL instead of the default SSH URLs, you can update the - :ref:`configuration options`. For example, add the following options to your - ``cmake`` command to download the repositories over HTTPS instead of SSH: - - .. code-block:: text - :caption: Use explicit options to download dependent repositories over HTTPS - - cmake -B build \ - -D AL_CORE_GIT_REPOSITORY=git@github.com:iterorganization/IMAS-Core.git \ - -D AL_PLUGINS_GIT_REPOSITORY=https://git.iter.org/scm/imas/al-plugins.git \ - -D DD_GIT_REPOSITORY=git@github.com:iterorganization/IMAS-Data-Dictionary.git - - If you use CMake 3.21 or newer, you can also use the ``https`` preset: - - .. code-block:: text - :caption: Use CMake preset to set to download dependent repositories over HTTPS - - cmake -B build --preset=https - - -Choosing the compilers -'''''''''''''''''''''' - -You can instruct CMake to use compilers with the following environment variables: - -- ``CC``: C compiler, for example ``gcc`` or ``icc``. -- ``CXX``: C++ compiler, for example ``g++`` or ``icpc``. -- ``FC``: Fortran compiler, for example ``gfortran``, ``ifort`` or ``nagfor``. - -If you don't specify a compiler, CMake will take a default (usually from the Gnu -Compiler Collection). - -.. important:: - - These environment variables must be set before the first time you configure - ``cmake``! - - If you have an existing ``build`` folder and want to use a different compiler, you - should delete the ``build`` folder first, or use a differently named folder for the - build tree. - - -Configuration options -''''''''''''''''''''' - -- **Backend configuration options** - - - ``AL_BACKEND_HDF5``, allowed values ``ON`` *(default)* or ``OFF``. - Enable/disable the HDF5 backend. - - ``AL_BACKEND_MDSPLUS``, allowed values ``ON`` or ``OFF`` *(default)*. - Enable/disable the MDSplus backend. - - - ``AL_BUILD_MDSPLUS_MODELS``, allowed values ``ON`` or ``OFF``. - Enable building MDSplus models for the selected Data Dictionary version. - Defaults to ``ON`` when ``AL_BACKEND_MDSPLUS`` is enabled, ``OFF`` otherwise. - Can be overridden by user to explicitly enable or disable building MDSplus models. - - - ``AL_BACKEND_UDA``, allowed values ``ON`` or ``OFF`` *(default)*. Enable/disable - the UDA backend. - - ``AL_BACKEND_UDAFAT``, allowed values ``ON`` or ``OFF`` *(default)*. - Enable/disable the UDA backend and use FAT UDA instead of the client/server - model. See the `UDA documentation `__ for more - information. - -- **Control what to build** - - - ``AL_EXAMPLES``, allowed values ``ON`` *(default)* or ``OFF``. Enable/disable - building the example programs in the ``examples`` directory. - - ``AL_TESTS``, allowed values ``ON`` *(default)* or ``OFF``. Enable/disable - building the test programs in the ``tests`` folder. - - ``AL_PLUGINS``, allowed values ``ON`` or ``OFF`` *(default)* . Enable/disable - building the plugins from the ``al-plugins`` repository. - - ``AL_HLI_DOCS``, allowed values ``ON`` or ``OFF`` *(default)*. Enable/disable - building the documentation. - - ``AL_DOCS_ONLY``, allowed values ``ON`` or ``OFF`` *(default)*. When enabled, - ONLY the documentation will be built (needs ``AL_HLI_DOCS=ON``). Regardless of - other configuration options, nothing else will be built. - - ``AL_PYTHON_BINDINGS``, allowed values ``ON`` *(default when building the Python - HLI)* or ``OFF`` *(default when not building the Python HLI)*. When enabled, this - builds the Access Layer Python lowlevel bindings. - -- **Dependency configuration options** - - - ``AL_DOWNLOAD_DEPENDENCIES``, allowed values ``ON`` *(default)* or ``OFF``. - Enable or disable the automatic downloading of dependencies. Should be disabled - when using a :ref:`development environment `. - - .. important:: - - The following environment variables must be set before the first time you - configure ``cmake``! - - If you have an existing ``build`` folder and want to use a different compiler, - you should delete the ``build`` folder first, or use a differently named folder - for the build tree. - - When ``AL_DOWNLOAD_DEPENDENCIES`` is enabled, the following settings can be used to - configure the location and/or version of the dependencies that should be used. - - - ``AL_CORE_GIT_REPOSITORY``, - ``AL_PLUGINS_GIT_REPOSITORY``, ``DD_GIT_REPOSITORY``. Configure the git URLs - where the ``imas-core``, ``al-plugins`` c.q. - ``imas-data-dictionary`` repositories should be fetched from. - - ``AL_CORE_VERSION``, ``AL_PLUGINS_VERSION``, - ``DD_VERSION``. Configure the version of the repository that should be used. - This can point to any valid branch name, tag or commit hash. - - This setting can be used to control which version of the Data Dictionary you - want to use. For example: ``-D DD_VERSION=3.38.1`` will use DD version 3.38.1 - instead of the default. - - .. code-block:: text - :caption: Default values for ``*_GIT_REPOSITORY`` and ``*_VERSION`` options - - AL_CORE_GIT_REPOSITORY: git@github.com:iterorganization/IMAS-Core.git - AL_CORE_VERSION: main - - AL_PLUGINS_GIT_REPOSITORY: ssh://git@git.iter.org/imas/al-plugins.git - AL_PLUGINS_VERSION: main - - DD_GIT_REPOSITORY: git@github.com:iterorganization/IMAS-Data-Dictionary.git - DD_VERSION: main - -- **Useful CMake options** - - - ``CMAKE_INSTALL_PREFIX``. Configure the path where the Access Layer will be - installed, for example ``-D CMAKE_INSTALL_PREFIX=$HOME/al-install`` will install - the Access Layer inside the ``al-install`` folder in your home directory. - - ``CMAKE_BUILD_TYPE``. Configure the build type for compiled languages. - Supported values (case sensitive): - - - ``Debug``: build with debug symbols and minimal optimizations. - - ``Release``: build with optimizations enabled, without debug symbols. - - ``RelWithDebInfo`` *(default)*: build with optimizations and debug symbols - enabled. - - ``MinSizeRel``: build optimized for minimizing the size of the resulting - binaries. - -More advanced options are available as well, these can be used to configure where CMake -searches for the prerequisite dependencies (such as the Boost libraries). To show all -available configuration options, use the command-line tool ``ccmake`` or the gui tool -``cmake-gui``: - -.. code-block:: bash - - # for the CLI tool - ccmake -B build -S . - # for the GUI tool - cmake-gui -B build -S . - - -Build the High Level Interface -`````````````````````````````` - -Use ``make`` to build everything. You can speed things up by using parallel compiling -as shown with the ``-j`` option. Be careful with the amount of parallel processes -though: it's easy to exhaust your machine's available hardware (CPU or memory) which may -cause the build to fail. This is especially the case with the C++ High Level Interface. - -.. code-block:: bash - - # Instruct make to build "all" in the "build" folder, using at most "8" parallel - # processes: - make -C build -j8 all - -.. note:: - - By default CMake on Linux will create ``Unix Makefiles`` for actually building - everything, as assumed in this section. - - You can select different generators (such as Ninja) if you prefer, but these are not - tested. See the `CMake documentation - `__ for more - details. - - -Optional: Test the High Level Interface -``````````````````````````````````````` - -If you set either of the options ``AL_EXAMPLES`` or ``AL_TESTS`` to ``ON``, you can run -the corresponding test programs as follows: - -.. code-block:: bash - - # Use make: - make -C build test - # Directly invoke ctest - ctest --test-dir build - -This executes ``ctest`` to run all test and example programs. Note that this may take a -long time to complete. - - -Install the High Level Interface -```````````````````````````````` - -Run ``make install`` to install the high level interface in the folder that you chose in -the configuration step above. - - -Use the High Level Interface -```````````````````````````` - -After installing the HLI, you need to ensure that your code can find the installed -Access Layer. To help you with this, a file ``al_env.sh`` is installed. You can -``source`` this file to set all required environment variables: - -.. code-block:: bash - :caption: Set environment variables (replace ```` with your install folder) - - source /bin/al_env.sh - -You may want to add this to your ``$HOME/.bashrc`` file to automatically make the Access -Layer installation available for you. - -.. note:: - - To use a ``public`` dataset, you also need to set the ``IMAS_HOME`` environment - variable. For example, on SDCC, this would be ``export IMAS_HOME=/work/imas``. - - Some programs may rely on an environment variable ``IMAS_VERSION`` to detect which - version of the data dictionary is used in the current IMAS environment. You may set - it manually with the DD version you've build the HLI with, for example: ``export - IMAS_VERSION=3.41.0``. - -Once you have set the required environment variables, you may continue :ref:`Using the -Access Layer`. - - -Troubleshooting -``````````````` - -**Problem:** ``Target Boost::log already has an imported location`` - This problem is known to occur with the ``2020b`` toolchain on SDCC. Add the CMake - configuration option ``-D Boost_NO_BOOST_CMAKE=ON`` to work around the problem. - diff --git a/common/doc_common/ci_build_docs.sh b/common/doc_common/ci_build_docs.sh deleted file mode 100755 index 398d94a1..00000000 --- a/common/doc_common/ci_build_docs.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -# ITER Bamboo CI (ci.iter.org) script to build sphinx documentation for all HLIs - -# Script assumes that the pwd is the access layer git root directory (i.e. the -# parent directory of where this script is located). - -# Quit with error when any command is unsuccessful -set -e - -# Set up environment such that module files can be loaded -if test -f /etc/profile.d/modules.sh ;then -. /etc/profile.d/modules.sh -else -. /usr/share/Modules/init/sh -fi - -# Make Python available -module purge -module load Python/3.8.6-GCCcore-10.2.0 - -# Create and activate a venv -rm -rf docs_venv -python -m venv docs_venv -. docs_venv/bin/activate - -# Install dependencies -pip install --upgrade pip -pip install -r doc_common/requirements.txt - -# Output all installed packages -pip list -v - -# Update the ##VERSION## string in the deploy script -if test `git describe` = `git describe --abbrev=0`; then - # Strip patch version from the release tag, e.g. 5.0.1 -> 5.0 - VERSION=`git describe | cut -d. -f1-2` -else - VERSION=dev -fi -sed -i "s/##VERSION##/$VERSION/g" doc_common/deploy.ps1 - -# Instruct sphinx to treat all warnings as errors -export SPHINXOPTS='-W --keep-going' - -# Make all the docs -# We're not using `make docs`, because some HLI Makefiles -# won't do anything unless the right environment parameters exist -make -C cppinterface/doc html -make -C fortraninterface/doc html -make -C javainterface/doc html -make -C mexinterface/doc html -make -C pythoninterface/doc html diff --git a/common/doc_common/conf.rst b/common/doc_common/conf.rst deleted file mode 100644 index 1959f351..00000000 --- a/common/doc_common/conf.rst +++ /dev/null @@ -1,141 +0,0 @@ -Configuring the Access Layer -============================ - -Some functionality of the Access Layer can be configured through enironment -variables. This page provides an overview of the available options. - - -Environment variables controlling HLI behaviour ------------------------------------------------ - -``IMAS_AL_DEFAULT_BACKEND`` [#backend_env]_ - Specify which backend to use by default with open/create methods that do not - pass this information as an argument. Values for this environment - variable correspond to the targeted backend ID, see below table. If not - specified, the MDS+ backend is the default. - - .. csv-table:: Backend IDs - :header-rows: 1 - - Backend, Backend ID - :ref:`ASCII `, 11 - :ref:`MDSplus `, 12 - :ref:`HDF5 `, 13 - :ref:`Memory `, 14 - :ref:`UDA `, 15 - - -``IMAS_AL_FALLBACK_BACKEND`` [#backend_env]_ - Specify a fallback backend to be tried if opening the given data-entry was - not successful with the primary/default backend. Values for this environment - variable correspond to the targeted backend ID, see above table. If not - specified, no secondary attempt will be made. This does not have any effect - on calls to create new dataentries. - - -``IMAS_AL_DISABLE_OBSOLESCENT_WARNING`` - Since version 4.10.0, all interfaces print warnings when putting an IDS that - contains data in fields marked as `obsolescent` in the DD. Setting this - variable to ``1`` disables these printouts. - - -``IMAS_AL_DISABLE_VALIDATE`` - Set this environment variable to ``1`` to disable the automatic run of :ref:`IDS - validation` when doing a ``put`` or ``put_slice``. - - -.. [#backend_env] These environment variables are not applicable when using - :ref:`Data Entry URIs`, which explicitly specify the backend. Also not applicable - in the Python HLI. - - -Environment variables controlling access layer plugins ------------------------------------------------------- - -``IMAS_AL_ENABLE_PLUGINS`` - Execution of C++ plugins in AL5 is a new feature which can be tested by - users who are interested in. It's currently an experimental feature which is - disabled by default. - - When the plugins framework is disabled: - - - Low level plugins registering/search functions are disabled. - - The behavior of writing data for nodes with default values is the same - that AL4. HLI write requests for these `empty` nodes are not sent to the - LL. - - When the plugins framework is enabled: - - - Low level plugins registering/search functions are enabled. - - The behavior of writing data for nodes with default values differs from - AL4. HLI write requests for these `empty` nodes are sent to the LL - allowing eventually to execute low level C++ plugins bound to these nodes - whose content can be handled by these plugins. - - To enable the plugins framework, set the global environment variable - ``IMAS_AL_ENABLE_PLUGINS`` before executing the access layer: - - .. code-block:: bash - - export IMAS_AL_ENABLE_PLUGINS=TRUE - - -Backend specific environment variables --------------------------------------- - - -``HDF5_BACKEND_READ_CACHE`` [#uri_precedence]_ - Specify the size of the read cache in MB (default is 5). It may improve - reading performance at the cost of increased memory consumption. Obtained - performance and best size of cache is heavily depending on the data. - - -``HDF5_BACKEND_WRITE_CACHE`` [#uri_precedence]_ - Specify the size of the write cache in MB (default is 5). It may improve - writing performance at the cost of increased memory consumption. Obtained - performance and best size of cache is heavily depending on the data. - - -.. [#uri_precedence] These settings can also be configured in the IMAS URI, see - :ref:`Query keys specific for the HDF5 backend`. The URI provided settings - will be used if both are present. - - -``IMAS_AL_SERIALIZER_TMP_DIR`` - Specify the path to storing temporary data. If it is not set, the default - location `/dev/shm/` or the current working directory will be chosen. - - - -UDA client configuration to reach the server at ITER ----------------------------------------------------- - -``UDA_HOST=uda.iter.org`` [#uda_uri]_ - If set, all queries with the UDA backend will be directed at the ITER UDA - server `uda.iter.org`, unless directly specified in the Access Layer's URI. - -``UDA_PORT=56565`` [#uda_uri]_ - If set, all queries will be directed to the port `56565` of the selected UDA - server, unless directly specified in the Access Layer's URI. - - - The ITER UDA server uses SSL authentication through a Personal Key Infrastructure - (PKI). You can download your PKI certificate at `pkiuda.iter.org`. Extract the - obtained `bundle.zip` in a folder in which only you have read permission (e.g. - `$HOME/.uda`). Then set the following environment variables: - - .. code-block:: bash - - export UDA_CLIENT_SSL_KEY=$HOME/.uda/private.key - export UDA_CLIENT_CA_SSL_CERT=$HOME/.uda/ca-server-certificate.pem - export UDA_CLIENT_SSL_CERT=$HOME/.uda/certificate.pem - export UDA_CLIENT_SSL_AUTHENTICATE=1 - - Do the same on all the systems from which you want to access ITER's UDA server. - - -.. [#uda_uri] An Access Layer URI of the form - `imas:uda?path=;backend=` will only work if - `UDA_HOST` and `UDA_PORT` environment variables are set. If not, the information - needs to be directly passed in the URI, as - `imas://:/uda?path=;backend=`. diff --git a/common/doc_common/deploy.ps1 b/common/doc_common/deploy.ps1 deleted file mode 100644 index 19fe1592..00000000 --- a/common/doc_common/deploy.ps1 +++ /dev/null @@ -1,95 +0,0 @@ -# Stop when any command fails -$ErrorActionPreference = "Stop" -# Enable debug output -$DebugPreference = "Continue" - -$sharepoint_root = "\\sharepoint.iter.org@SSL\departments\POP\CM\IMDesign\Code Documentation\ACCESS-LAYER-doc" -$root_url = "https://sharepoint.iter.org/departments/POP/CM/IMDesign/Code%20Documentation/ACCESS-LAYER-doc" -$version = "##VERSION##" - -Net use X: $sharepoint_root - -foreach ($hli in $("cpp", "fortran", "java", "matlab", "python")) { - $deploy_folder = "${sharepoint_root}\${hli}\${version}" - Write-Debug "Deploying to folder: $deploy_folder" - - New-Item -ItemType Directory -Path $deploy_folder -Force - Copy-Item -Path ".\${hli}_html\*" -Destination $deploy_folder -Recurse -Force - - # Generate versions.js - $latest = "IDM" - $latest_version = [version]"0.0" - $latest_url = "https://user.iter.org/default.aspx?uid=YSQENW" - - $versions = New-Object System.Collections.Generic.List[string] - Get-ChildItem -Path "${sharepoint_root}\${hli}" -Directory -Name | Foreach-Object { - try { - $as_version = [version]$_ - if ($as_version -gt $latest_version) { - $latest_version = $as_version - $latest = $_ - $latest_url = "${root_url}/${hli}/${_}/index.html" - } - } catch [System.InvalidCastException] { - sleep 0.01 - Write-Debug "Could not convert $_ to a [version]. Ignoring for version comparisons" - } - } - # Generate the versions.js file for use by the sphinx-immaterial version - # switcher. See - # https://jbms.github.io/sphinx-immaterial/customization.html#version-dropdown - # - # Notes: - # - The "version" item is used for generating the URL. Since sharepoint - # doesn't automatically forward to `index.html`, we add it manually. We - # also need to include a pound (#) because sphinx-immaterial will add an - # additional `/` to the url which must be ignored by the sharepoint - # server. - # - The "title" is just the folder name (dev, or MAJOR.MINOR release) - # - Aliases are used to check if the current documentation page is the - # latest release yes or no. - # - The latest release will get the alias "latest" - # - We always include the folder name. This is required due to the way - # that sphinx-immaterial checks which version the current page is for. - # This is checked (in javascript) by iterating over all versions.js - # entries and selecting the one which matches: - # - $redirect_url == $base_url - # - or $base_url / .. / $alias matches for any alias in aliases - # The redirect_url is constructed from the version, but that will - # include a "/index.html#" so doesn't match the $base_url. By - # including an alias with the folder name we can still let this - # algorithm succeed. - Get-ChildItem -Path "${sharepoint_root}\${hli}" -Directory -Name | Foreach-Object { - $aliases = "[`"$_`"]" - if ($_ -eq $latest) { - $aliases = "[`"$_`", `"latest`"]" - } - $versions.Add(" {`"version`": `"${_}/index.html#`", `"title`": `"$_`", `"aliases`": $aliases}") - } - Write-Debug "Latest documentation version: $latest" - $versions_content = "[`n" + ($versions -Join ",`n") + "`n]" - - # Deploy versions.js - Set-Content -Path "${sharepoint_root}\${hli}\versions.json" -Value $versions_content - Set-Content -Path "${sharepoint_root}\${hli}\versions.js" -Value $versions_content - - # Deploy redirect html page - $latest_html_content = " - - - - - - - -

If this page does not refresh automatically, then please direct your browser to - our latest docs. -

- -" - Set-Content -Path "${sharepoint_root}\${hli}\latest.html" -Value $latest_html_content - - Get-ChildItem -Path "${sharepoint_root}\${hli}" -} - -Net use X: /delete \ No newline at end of file diff --git a/common/doc_common/dev_guide.rst b/common/doc_common/dev_guide.rst deleted file mode 100644 index f794b7e4..00000000 --- a/common/doc_common/dev_guide.rst +++ /dev/null @@ -1,247 +0,0 @@ -Access Layer development guide -============================== - - -Access Layer repositories -------------------------- - -The IMAS Access Layer consists of a number of components which are developed in separate -repositories: - -- `imas-core `__: the - Access Layer core repository, MDSplus model generator and Python lowlevel - bindings. -- `data-dictionary - `__: the IMAS Data - Dictionary definitions, used for generating MDSplus models and the traditional High - Level Interfaces. -- `al-plugins `__: Access - Layer plugins. -- Traditional (code-generated) High Level Interfaces - - - `al-cpp `__: C++ HLI - - `al-fortran `__: - Fortran HLI - - `al-java `__: Java HLI - - `al-matlab `__: - MATLAB HLI - - `al-python `__: - Python HLI - -- Non-generated code HLIs. The following High Level Interfaces load the Data - Dictionary definitions at runtime - - - `IMASPy `__: alternative - Python HLI - - `al-hdc `__: alternative - HLI based on the HDC (Hierarchical Data Containers) library - -The documentation on this page covers everything except the Non-generated HLIs, those -are documented in their own projects. - - -Development environment ------------------------ - -.. note:: - - This is the first iteration of the development process after the Access Layer split. - The process is not set in stone or sacred. Please suggest improvements to the - development flow based on your experience, if you feel the process the can be - streamlined. - -See the :ref:`build prerequisites` section for an overview of modules you need to load -when on SDCC or packages to install when using Ubuntu 22.04. - -The recommended development folder layout is to clone all :ref:`Access Layer -repositories` in a single root folder (``al-dev`` in below example, but the name of that -folder is not important). - -.. code-block:: text - - al-dev/ # Feel free to name this folder however you want - ├── al-core/ - ├── al-plugins/ # Optional - ├── al-cpp/ # Optional - ├── al-fortran/ # Optional - ├── al-java/ # Optional - ├── al-matlab/ # Optional - ├── al-python/ # Optional - └── data-dictionary/ - -Then, when you configure a project for building (see :ref:`Configuration`), set the -option ``-D AL_DOWNLOAD_DEPENDENCIES=OFF``. Instead of fetching requirements from the -ITER git, CMake will now use the repositories as they are checked out in your -development folders. - -The additional option ``-D AL_DEVELOPMENT_LAYOUT=ON`` may be required when you have set -the environment variable ``$AL_COMMON_PATH``. This environment variable is set by the -easybuild module ``IMAS-AL-Core`` and is used in the CMake configuration to build -easybuild modules. - -.. important:: - - With this setup, it is your responsibility to update the repositories to their - latest versions (if needed). The ``_VERSION`` configuration options are - ignored when ``AL_DOWNLOAD_DEPENDENCIES=OFF``. - -This setup allows you to develop in multiple repositories in parallel. - - -Dependency management ---------------------- - -With all Access Layer components spread over different repositories, managing -dependencies is more complex than before. Below diagram expresses the dependencies -between the different repositories: - -.. md-mermaid:: - :name: repository-dependencies - - flowchart - core[al-core] -->|"MDSplus
models"| dd[data-dictionary] - plugins[al-plugins] --> core - hli["al-{hli}"] --> core - hli --> dd - hli --> plugins - -To manage the "correct" version of each of the dependencies, the CMake configuration -specifies which branch to use from each repository: - -- Each HLI indicates which commit to use from the ``al-core`` repository. This is - defined by the ``AL_CORE_VERSION`` cache string in the main ``CMakeLists.txt`` of - the repository. - - The default version used is ``main``, which is the last stable release of - ``al-core``. -- Inside the ``al-core`` repository, the commits to use for the - ``al-plugins`` and ``data-dictionary`` are set in `ALCommonConfig.cmake - `__. - - The default versions used are ``main`` for ``al-plugins``, and ``main`` for - ``data-dictionary``. - - -.. info:: - - CMake supports setting branch names, tags and commit hashes for the dependencies. - - -CMake ------ - -We're using CMake for the build configuration. See `the CMake documentation -`__ for more details about CMake. - -The ``FetchContent`` CMake module for making :ref:`dependencies from other repositories -` available. For more information on this module we refer to the -`FetchContent CMake documentation -`__ - - -Documentation overview ----------------------- - -The documentation is generated with Sphinx. Because the documentation of each HLI -depends on the contents of the ``al-plugins`` and ``al-core`` repositories, it is -configured with CMake. For more information on Sphinx, see the `Sphinx docs -`__ and the `documentation of the theme -(sphinx-immaterial) that we're using -`__. - -Documentation of the HLI is inside the ``doc`` folder of the repository. This folder -contains the configuration (``conf.py``), and documentation pages (``*.rst``). -Documentation that is common to all High Level Interfaces (such as this developer guide) -is in the `common/doc_common folder in the al-core repository -`__. - - -Building the documentation -'''''''''''''''''''''''''' - -Use the option ``-D AL_HLI_DOCS`` to enable building documentation. This will create a -target ``al--docs``, e.g. ``al-python-docs`` that will only build the -documentation. You could also use ``-D AL_DOCS_ONLY`` to only build the documentation, -and nothing else. - -.. code-block:: console - :caption: Example: building the documentation for the Python HLI - - al-dev$ cd al-python - al-python$ # Configure cmake to only create the documentation: - al-python$ cmake -B build -D AL_HLI_DOCS -D AL_DOCS_ONLY - [...] - al-python$ make -C build al-python-docs - [...] - - -CI and deployment overview --------------------------- - -Main CI plans. These plans execute the script ``ci/build_and_test.sh`` in each of the -repositories. This allows for easy reproduction of the CI plans on SDCC: just execute -``bash ci/build_and_test.sh`` in a clone of the repository. - -- `AL-Core `__ tests building the Access - Layer core. Note that there are no unit tests in this repository, the CI plan - only checks that the AL can be built successfully. -- `AL-Cpp `__ builds and tests the C++ HLI. -- `AL-Fortran `__ builds and tests the Fortran - HLI. - - .. note:: - There are three Jobs inside this plan. One uses the GCC compilers, one uses the - Intel compilers and the third uses the NAGfor Fortran compiler. - - All three execute ``ci/build_and_test.sh``, but some set the ``CC``, ``CXX`` and - ``FC`` environment variables to select the compiler. - -- `AL-Java `__ builds and tests the Java HLI. -- `AL-Matlab `__ builds and tests the Matlab HLI. -- `AL-Python `__ builds and tests the Python HLI. - -Documentation CI plan: - -- `Access Layer Doc `__ builds the AL - documentation for all HLIs (only on the ``main`` and ``develop`` branches). The - output of this CI plan is used to deploy the documentation to sharepoint with the - `Access-Layer doc deployment project - `__. - -Other CI plans: - -- `Data-Dictionary Dev `__ builds and tests all - HLIs (develop branch) with the Data Dictionary branch that triggered the build. - - This project is used to test the compatibility of new Data Dictionary developments - with the generated Access Layer HLIs. This CI plan is triggered for the develop - branches of the Data Dictionary, and with every PR in the Data-Dictionary - repository. - - .. note:: - The Main CI plans use the last released Data Dictionary version for testing. - -- `IMAS AL DEV `__ builds development modules - for all components. These modules are published to SDCC by the `IMAS AL DEV Deploy - `__ - deployment project. - - These development modules can be used on SDCC as follows: - - .. code-block:: bash - :caption: Development modules based on the Data Dictionary ``develop/3`` branch - - module use /work/imas/opt/bamboo_deploy/imas3-dev-modules/modules/all/ - # For intel: - module load IMAS/develop3-develop-intel-2020b - # For foss: - module load IMAS/develop3-develop-foss-2020b - - .. code-block:: bash - :caption: Development modules based on the Data Dictionary ``develop/4`` branch - - module use /work/imas/opt/bamboo_deploy/imas4-dev-modules/modules/all/ - # For intel: - module load IMAS/develop4-develop-intel-2020b - # For foss: - module load IMAS/develop4-develop-foss-2020b diff --git a/common/doc_common/identifiers.rst b/common/doc_common/identifiers.rst deleted file mode 100644 index 69a86f1c..00000000 --- a/common/doc_common/identifiers.rst +++ /dev/null @@ -1,54 +0,0 @@ -Identifiers -=========== - -The "identifier" structure is used to provide an enumerated list of options for -defining, for example: - -- A particular coordinate system, such as Cartesian, cylindrical, or spherical. -- A particle, which may be either an electron, an ion, a neutral atom, a - molecule, a neutron, or a photon. -- Plasma heating may come from neutral beam injection, electron cyclotron - heating, ion cyclotron heating, lower hybrid heating, alpha particles. - -Identifiers are a list of possible valid labels. Each label has three -representations: - -1. An index (integer) -2. A name (short string) -3. A description (long string). - -.. csv-table:: Identifier examples (from part of the ``core_sources/source`` identifier) - :header-rows: 1 - - Index, Name, Description - 2, NBI, Source from Neutral Beam Injection - 3, EC, Sources from heating at the electron cyclotron heating and current drive - 4, LH, Sources from lower hybrid heating and current drive - 5, IC, Sources from heating at the ion cyclotron range of frequencies - 6, fusion, "Sources from fusion reactions, e.g. alpha particle heating" - -The list of possible labels for a given identifier structure in the Data -Dictionary can be found in the |DD| documentation. - -The use of private indices or names in identifiers structure is discouraged, -since this would defeat the purpose of having a standard enumerated list. Please -create a `JIRA `_ tracker when you want to add a new -identifier value. - - -Using the identifiers library ------------------------------ - -|identifiers_link_instructions| - -Below examples illustrates how to use the identifiers in your |lang| programs. - -.. literalinclude:: code_samples/identifier_example1 - :caption: |lang| example 1: obtain identifier information of coordinate identifier ``phi`` - -.. literalinclude:: code_samples/identifier_example2 - :caption: |lang| example 2: Use the identifier library to fill the ``NBI`` label in the ``core_sources`` IDS - -.. literalinclude:: code_samples/identifier_example3 - :caption: |lang| example 3: Use the identifier library to fill the type of coordinate system used in the ``equilibrium`` IDS - diff --git a/common/doc_common/imas.rst b/common/doc_common/imas.rst deleted file mode 100644 index aa739fca..00000000 --- a/common/doc_common/imas.rst +++ /dev/null @@ -1,32 +0,0 @@ -IMAS overview -============= - -IMAS is the Integrated Modeling and Analysis Suite of ITER. It consists of -numerous infrastructure components, physics components and tools. An up-to-date -overview of these can be found at ``_ (ITER -Organization account required). - -The IMAS core consists of: - -1. Standardized data structures for storing experimental and simulation data. -2. Infrastructure for storing and loading these data structures. - -The standardized data structures are defined in the |DD|. -The documentation for the Data Dictionary (DD) can be found there as well, for -example: - -- Which data structures (IDSs) exist -- What data is contained in these structures -- What units a data field has -- What are the coordinates belonging to a data field - -.. todo:: - - Add links to the HLI documentation pages. - -The Access Layer, of which you are currently reading the documentation, -provides the libraries for working with these data structures, for example: - -- Loading an IDS from disk -- Storing an IDS to disk -- Using and manipulating IDSs in your program diff --git a/common/doc_common/imas_uri.rst b/common/doc_common/imas_uri.rst deleted file mode 100644 index 30acbf6d..00000000 --- a/common/doc_common/imas_uri.rst +++ /dev/null @@ -1,253 +0,0 @@ -Data entry URIs -=============== - -Data entry URIs specify where and how IMAS data is stored (or should be stored -to). When you :ref:`load or store IMAS data `, you -need to provide a data entry URI. - -This page documents the URI structure and the options that are supported. - - -Data entry URI structure ------------------------- - -The general structure of an IMAS URI is the following, with optional elements -indicated with square brackets: - -.. code-block:: text - - imas:[//host/]backend?query - -Let's break down each of the components: - -1. ``imas:`` this part indicates that this is an IMAS URI -2. ``host`` when the data is located at another machine, you use this - section to indicate the address of that machine. See :ref:`UDA backend` for - further details. -3. ``backend`` select the Access Layer backend. See :ref:`Backends` for the - options. -4. ``query`` the query consists of ``key=value`` pairs, separated by a - semicolon ``;``. See :ref:`Query keys` for further details. - -.. - Commenting this out, as no backend currently supports URI fragments - - 5. ``fragment`` In order to identify a subset from a given data-entry a - ``fragment`` can be added to the URI. Such ``fragment``, which starts with a - hash ``#``, is optional and allows to identify a specific IDS, or a part of - an IDS. See :ref:`URI fragment` for further details. - - -Backends --------- - -Several backends exist for storing and loading IMAS data. Each backend uses a -different format for storing the data. - -.. note:: - - Depending on local install choices, some backends may be unavailable in - your Access Layer installation. - - -Backend comparison -'''''''''''''''''' - -.. csv-table:: Comparison of backend functionality - :header-rows: 1 - :stub-columns: 1 - - , :ref:`HDF5 `, :ref:`MDSplus `, :ref:`UDA `, :ref:`Memory `, :ref:`ASCII `, :ref:`Flexbuffers ` - :ref:`get `, Yes, Yes, Yes, Yes, Yes, Yes [#fb_get]_ - :ref:`get_slice `, Yes, Yes, Yes, Yes, \-, \- - :ref:`put `, Yes, Yes, \-, Yes, Yes, Yes [#fb_put]_ - :ref:`put_slice `, Yes, Yes, \-, Yes, \-, \- - Persistent storage, Yes, Yes, Yes [#uda]_, \-, Yes [#ascii]_, \- - -.. [#uda] The UDA backend is read-only. -.. [#ascii] Suitable for tests and small data, but not recommended for large - datasets or long-term storage. -.. [#fb_get] Only when using ``OPEN_PULSE`` mode -.. [#fb_put] Only when not using ``OPEN_PULSE`` mode - - -HDF5 backend -'''''''''''' - -The HDF5 backend is identified by ``hdf5`` in the IMAS URI, and stores data in -the `hdf5 data format `_ - - -MDSplus backend -''''''''''''''' - -The MDSplus backend is identified by ``mdsplus`` in the IMAS URI. The data is -stored in the `MDSplus format `_. - -This backend imposes some limitations on the data that can be stored, see -`maxoccur` in the |DD| documentation. - -This backend has been around and stable for a longer time, so most older IMAS -data is stored in this format. - - -UDA backend -''''''''''' - -The UDA backend is used when a host is provided. `UDA (Universal Data Access) -`_ is the mechanism for contacting the server that -stores the data. - -A number of UDA plugins already exist for these, but their availability depends -on how UDA has been installed on the local cluster. Therefore it's recommended -that you contact the IMAS support team when you want to use this functionality. - -.. todo:: - - Provide a sample URI string - - -Memory backend -'''''''''''''' - -The memory backend is identified by ``memory`` in the IMAS URI. When storing or -loading IMAS data with this backend, the data is stored in-memory. This is -therefore not persistent. - -The memory backend can still be useful to transfer data between languages in the -same program (for example, storing an IDS in C++ and then loading it with the -Fortran HLI) or to :ref:`store a number of time slices ` and then :ref:`loading all time slices `. - - -ASCII backend -''''''''''''' - -The ASCII backend is identified by ``ascii`` in the IMAS URI. The ASCII backend -can be used to store IDS data in a plain-text human readable format. The -performance and size of the stored data is worse than the other backends, so -this is typically only used for debugging. - - -Flexbuffers backend -''''''''''''''''''' - -The Flexbuffers backend is identified by ``flexbuffers`` in the IMAS URI. The -Flexbuffers backend is used when (de)serializing IDSs with the -``FLEXBUFFERS_SERIALIZER_PROTOCOL``. It is optimized for (de)serialization speed and -therefore has very limited functionality. It is not intended to be used outside of IDS -serialization. - - -Query keys ----------- - -You can use query keys to indicate to the backend where the data is stored and -(optionally) set backend-specific configuration options. The following query -keys are currently recognized. - -.. note:: - - Query keys are case-sensitive and unknown query keys are silently ignored. - -``path`` [#mandatory]_ - Provide the path to the folder where the IMAS data is (or will be) stored. - Paths can be absolute (starting with a ``/`` on UNIX, or with a drive letter - on Windows) or relative to the current working directory. - - The backend manages how your IMAS data is stored within the folder. - - .. code-block:: text - :caption: URI examples using path - - imas:hdf5?path=/absolute/path/to/data - imas:hdf5?path=relative_path - -``user``, ``database``, ``version``, ``pulse``, ``run`` [#mandatory]_ - Use `legacy` (Access Layer version 4 and earlier) way to indicate where the - IMAS data is (or will be) stored. - - .. code-block:: text - :caption: URI example using legacy data identifiers - - imas:mdsplus?user=public;pulse=131024;run=41;database=ITER;version=3 - - In Access Layer version 5.0.0 and earlier use key ``shot`` instead of ``pulse``. - - .. code-block:: text - :caption: URI example using legacy data identifiers in Access Layer 5.0.0 and earlier. - - imas:mdsplus?user=public;shot=131024;run=41;database=ITER;version=3 - - - -.. [#mandatory] Either ``path`` or `all` of the legacy query keys must be - provided. - - -Query keys specific for the HDF5 backend -'''''''''''''''''''''''''''''''''''''''' - -The :ref:`HDF5 backend` also recognizes these backend-specific query keys. - -``hdf5_compression`` - Data compression is enabled by default. Set ``hdf5_compression=no`` or - ``hdf5_compression=n`` to disable data compression. - -``hdf5_write_buffering`` - During a `put` operation, 0D and 1D buffers are first - stored in memory. Buffers are flushed at the end of the put. - - This feature is enabled by default. Set ``hdf5_write_buffering=no`` or - ``hdf5_write_buffering=n`` to disable write buffering. - -``write_cache_option`` - Set the size of the HDF5 chunk cache used during chunked datasets write - operations. Default to 100x1024x1024 bytes (100 MiB). - -``read_cache_option`` - Set the size of the HDF5 chunk cache used during chunked datasets read - operations. Default to 5x1024x1024 bytes (5 MiB). - -``open_read_only`` - Open master file and IDSs files in read only if ``open_read_only=yes`` or - ``open_read_only=y``, overwriting the files access modes default behavior (see IMAS-5274 for an example use-case). - -``hdf5_debug`` - HDF5 debug output is disabled by default. Set ``hdf5_debug=yes`` or - ``hdf5_debug=y`` to enable HDF5 debug output. - - -Query keys specific for the ASCII backend -''''''''''''''''''''''''''''''''''''''''' - -The :ref:`ASCII backend` also recognizes these backend-specific query keys. - -``filename`` - Specify the exact filename in which the IDS data will be stored, instead - of the default `.ids`. - - -Query keys specific for the UDA backend -''''''''''''''''''''''''''''''''''''''' - -The :ref:`UDA backend` also recognizes these backend-specific query keys. - -``verbose`` - UDA verbosity is disabled by default. Set ``verbose=1`` to obtain - more information and ease debugging. - -``cache_mode`` - UDA cache_mode is ``ids`` by default. Set ``cache_mode=none``or ``cache_mode=ids`` to specify the mode of caching. - - ``none``: No caching is performed. - - ``ids``: Caches the entire IDS (Interface Data Structure). - -``fetch`` - UDA ``fetch`` is disabled by default. Set ``fetch=1`` to enable fetching - and downloading IDS files to the local ``local_cache`` directory. - -``local_cache`` - UDA ``local_cache`` is set to ``tmp/uda-cache-of-$USER/path_in_uri`` by default. This is used along with ``fetch=1`` in the query. - Set ``local_cache=/path/to/local/cache/directory`` and the download directory will be ``local_cache/path_in_uri``. - ``local_cache`` specifies the path to the local cache directory where IDSs will be downloaded. diff --git a/common/doc_common/load_store_ids.rst b/common/doc_common/load_store_ids.rst deleted file mode 100644 index 7f4f4467..00000000 --- a/common/doc_common/load_store_ids.rst +++ /dev/null @@ -1,261 +0,0 @@ -Loading and storing IMAS data -============================= - -IMAS data is grouped together in Data Entries. A Data Entry is a collection of -:ref:`IDSs ` and their (potentially) multiple -occurrences, which groups and stores data over multiple IDSs as a single -dataset. The Data Entry concept is used whether the collection of IDSs is stored -in a database or only exists temporarily (for example for communication in an -integrated workflow). - -Loading and storing IMAS data happens through an IMAS Database Entry. A Database -Entry tracks the information required for locating where the Data Entry is (or -will be) stored on disk. In |lang| this object is modeled as |dbentry|. - -You may :ref:`open an existing IMAS Database Entry`, which you can use for -loading data that was stored previously. Alternatively you can :ref:`create a -new IMAS Database Entry` to store IDS data. - - -Open an existing IMAS Database Entry ------------------------------------- - -To open an IMAS Database Entry, you need to know the URI indicating where the -Access Layer can find the data. IMAS URIs start with ``imas:`` and indicate -the format and the location of the stored data. You can find a detailed -description of the IMAS URI syntax on the :ref:`Data entry URIs` page. - -.. literalinclude:: code_samples/dbentry_open - :caption: |lang| example: open an existing IMAS Database Entry - -.. seealso:: - - API documentation for |dbentry_open|. - - -Loading IMAS data ------------------ - -After you open a database entry, you can request to load data from disk. - -.. contents:: - :local: - - -Load an entire IDS -'''''''''''''''''' - -With |dbentry_get| you can load ("get") an entire IDS from the database entry. - -Multiple `occurrences` of an IDS may be stored in a data entry. By default, if -you don't specify an occurrence number, occurrence 0 is loaded. By providing an -occurrence number you can load a specific occurrence. How different occurrences -are used depends on the experiment. They could, for example, correspond to: - -- different methods for computing the physical quantities of the IDS, or -- different functionalities in a workflow (e.g. initial values, prescribed - values, values at next time step, …), or -- multiple subsystems (e.g. diagnostics) of the same type in an experiment, etc. - -.. todo:: extend docs after Task 2c. is implemented (get multiple occurrences) - -.. literalinclude:: code_samples/dbentry_get - :caption: |lang| example: get an IDS from an IMAS Database Entry - -.. seealso:: - - - API documentation for |dbentry_get|. - - -Load a single `time slice` of an IDS -'''''''''''''''''''''''''''''''''''' - -Instead of loading a full IDS from disk, the Access Layer allows you to load a -specific `time slice`. This is often useful when you're not interested in the -full time evolution, but instead want data of a specific time. You can use -|dbentry_getslice| for this. - -Most of the time there are no entries at that specific time, so you also need to -indicate an `interpolation method`. This determines what values the access layer -returns when your requested time is in between available time points in the -data. Three interpolation methods currently exist: - -|PREVIOUS_INTERP| - Returns the `previous` time slice if the requested time does not exactly - exist in the original IDS. - - For example, when data exists at :math:`t=\{1, 3, 4\}`, requesting - :math:`t_r=2.1` will give you the data at :math:`t=1`. - - .. csv-table:: Edge case behaviour. :math:`\{t_i\}, i=1..N` represents the time series stored in the IDS. - :header-rows: 1 - - Case, Behaviour - :math:`t_r \lt t_1`, Return data at :math:`t_1`. - :math:`t_r = t_i` [#equal_note]_, Return data at :math:`t_i`. - - -|CLOSEST_INTERP| - Returns the `closest` time slice in the original IDS. This can also be - `after` the requested time. - - For example, when data exists at :math:`t=\{1, 3, 4\}`, requesting - :math:`t=2.1` will give you the data at :math:`t=3`. - - .. csv-table:: Edge case behaviour. :math:`\{t_i\}, i=1..N` represents the time series stored in the IDS. - :header-rows: 1 - - Case, Behaviour - :math:`t_r \lt t_1`, Return data at :math:`t_1`. - :math:`t_r = t_i` [#equal_note]_, Return data at :math:`t_i`. - :math:`t_r - t_i = t_{i+1} - t_r` [#equal_note]_, Return data at :math:`t_{i+1}`. - -.. [#equal_note] Equality for floating point numbers is tricky. For example, - :code:`3.0/7.0 + 2.0/7.0 + 2.0/7.0` is not exactly equal to :code:`1.0`. It - is therefore advised not to depend on this behaviour. - -|LINEAR_INTERP| - Returns a linear interpolation between the existing slices before and after - the requested time. - - For example, when data exists at :math:`t=\{1, 3, 4\}`, requesting - :math:`t=2.1` will give you a linear interpolation of the data at - :math:`t=1` and the data at :math:`t=3`. - - Note that the linear interpolation will be successful only if, between the - two time slices of an interpolated dynamic array of structure, the same - leaves are populated and they have the same size. Otherwise - |dbentry_getslice| will interpolate all fields with a compatible size and - leave others empty. - - .. csv-table:: Edge case behaviour. :math:`\{t_i\}, i=1..N` represents the time series stored in the IDS. - :header-rows: 1 - - Case, Behaviour - :math:`t_r \lt t_1`, Return data at :math:`t_1`. - :math:`t_r \gt t_N`, Return data at :math:`t_N`. - -.. literalinclude:: code_samples/dbentry_getslice - :caption: |lang| example: get a time slice from an IMAS Database Entry - -.. note:: - - The access layer assumes that all time arrays are stored in increasing - order. |dbentry_getslice| may return unexpected results if your data does - not adhere to this assumption. - -.. seealso:: - - API documentation for |dbentry_getslice|. - - -.. include:: partial_get - - -Create a new IMAS Database Entry --------------------------------- - -To create a new IMAS Database Entry, you need to provide the URI to indicate the -format and the location where you want to store the data. You can find a -detailed description of the IMAS URI syntax and the options available on the -:ref:`Data entry URIs` page. - -.. caution:: - - This function erases any existing database entry on the specified URI! - - -.. literalinclude:: code_samples/dbentry_create - :caption: |lang| example: create a new IMAS Database Entry - -.. seealso:: - - API documentation for |dbentry_create|. - - -Store IMAS data ---------------- - -After you have created an IMAS Database Entry, you can use it for storing IDS -data. There are two ways to do this: - -.. contents:: - :local: - - -Store an entire IDS -''''''''''''''''''' - -With |dbentry_put| you can store ("put") an entire IDS in a database entry. -First you need to have an IDS with data: you can create a new one or :ref:`load -an IDS ` which you modify. See :ref:`Use Interface Data -Structures` for more information on using and manipulating IDSs. - -.. caution:: - - This function erases the existing IDS in the data entry if any was already - stored previously. - -Multiple `occurrences` of an IDS may be stored in a data entry. By default, if -you don't specify an occurrence number, the IDS is stored as occurrence 0. By -providing an occurrence number you can store the IDS as a specific occurrence. - -.. note:: - - The MDS+ backend has a limitation on the number of occurrences of a given - IDS. This number is indicated in the |DD| documentation in the "Max. - occurrence number" column of the list of IDSs. This limitation doesn't apply - to other backends. - -.. literalinclude:: code_samples/dbentry_put - :caption: |lang| example: put an IDS to an IMAS Database Entry - -.. seealso:: - - API documentation for |dbentry_put|. - - -Append a time slice to an already-stored IDS -'''''''''''''''''''''''''''''''''''''''''''' - -With |dbentry_put_slice| you can append a time slice to an existing database -entry. This is useful when you generate data inside a time loop (for example in -simulations, or when taking measurements of an experiment). - -It means you can put a time slice with every iteration of your loop such that -you don't have to keep track of the complete time evolution in memory. Instead, -the Access Layer will keep appending the data to the Database Entry in the -storage backend. - -.. note:: - - Although being put progressively time slice by time slice, the final IDS - must be compliant with the data dictionary. A typical error when - constructing IDS variables time slice by time slice is to change the size of - the IDS fields during the time loop, which is not allowed but for the - children of an array of structure which has time as its coordinate. - -.. literalinclude:: code_samples/dbentry_put_slice - :caption: |lang| example: iteratively put time slices to an IMAS Database Entry - -.. seealso:: - - API documentation for |dbentry_put_slice|. - - -Listing all occurrences of an IDS from a backend -'''''''''''''''''''''''''''''''''''''''''''''''' - -With |list_all_occurrences| you can List all non-empty occurrences of an IDS -using its name in the dataset, and optionnally return the content of a -descriptive node path. - -.. note:: - - The MDS+ backend is storing IDS occurrences infos (pulse file metadata) - for AL version > 5.0.0. Pulse files created with AL version <= 5.0.0. - do not provide these informations (an exception will occur for such - pulse files when calling |list_all_occurrences|). - -.. literalinclude:: code_samples/dbentry_list_all_occurrences - :caption: |lang| example: listing all occurrences of a magnetics IDS from an IMAS Database Entry diff --git a/common/doc_common/media/image1.png b/common/doc_common/media/image1.png deleted file mode 100644 index d76f03e0..00000000 Binary files a/common/doc_common/media/image1.png and /dev/null differ diff --git a/common/doc_common/media/image2.png b/common/doc_common/media/image2.png deleted file mode 100644 index 157bb270..00000000 Binary files a/common/doc_common/media/image2.png and /dev/null differ diff --git a/common/doc_common/media/image3.png b/common/doc_common/media/image3.png deleted file mode 100644 index 51035edd..00000000 Binary files a/common/doc_common/media/image3.png and /dev/null differ diff --git a/common/doc_common/media/image4.png b/common/doc_common/media/image4.png deleted file mode 100644 index 8fa3a7fa..00000000 Binary files a/common/doc_common/media/image4.png and /dev/null differ diff --git a/common/doc_common/media/image5.png b/common/doc_common/media/image5.png deleted file mode 100644 index 278f00ab..00000000 Binary files a/common/doc_common/media/image5.png and /dev/null differ diff --git a/common/doc_common/media/image6.png b/common/doc_common/media/image6.png deleted file mode 100644 index 7f24dd08..00000000 Binary files a/common/doc_common/media/image6.png and /dev/null differ diff --git a/common/doc_common/media/image7.png b/common/doc_common/media/image7.png deleted file mode 100644 index 534b89ab..00000000 Binary files a/common/doc_common/media/image7.png and /dev/null differ diff --git a/common/doc_common/plugins.rst b/common/doc_common/plugins.rst deleted file mode 100644 index f7b1acb2..00000000 --- a/common/doc_common/plugins.rst +++ /dev/null @@ -1,75 +0,0 @@ -=========================================== -Plugins framework for the IMAS access layer -=========================================== - -Plugins are C++ software components compiled in separate -libraries from the Access Layer (AL) library, which make use of them. Using a -modified Low Level architecture, we demonstrate that plugins satisfy many -use-cases requirements, providing new features available from all existing HLIs. -We describe the Access Layer plugins architecture and some plugins examples. - - - -Plugins offer the following advantages: - -- Developers write plugins in C++ and compile plugins separately from - the AL library, which discovers them at runtime, according to some - Data Dictionary-defined or user-defined plugins activation - directives. More precisely, the AL Low Level layer loads plugins and - calls them according to the previous directives. **Plugins provide - new features with common behavior between High Level - Interfaces**. There is no need for developing specific High Level - Interface (HLI) implementation of plugins features, **decreasing - therefore drastically development time**, **simplifying maintenance** - and **reducing the risk of potential bugs**. - -- Contributing also to **reduced time development**, code **plugin - compilation is fast** since plugins have no dependency on HLI - classes. The compilation time must be compared to the one required - for specific HLI code whose change may (depending on which source - file has been modified) trigger the full recompilation of the HLI - layer. - -- Features provided by a plugin behave the same in all HLIs with same - exceptions, same outputs, same bugs... **Users experience remains - unchanged** from one HLI to another. - -- Plugins source files are stored in separate repositories with - separate development lifecycles. Developers group plugins into - separate projects, improving **code management, lowering code - coupling** (plugins are not coupled, they only implement the plugin - interface described later) between AL components and **making clear - separation of features** provided by each plugin. - -- Plugins depend on the AL library, not the other way around. - Therefore, at development time, plugins developers do not need to - recompile the AL, which lowers time development. Moreover, in - production, upon new plugins release, no change are required to the - AL sources or to the AL installed binaries, **preventing AL updates - procedures and reducing AL deployment maintenance.** - -AL plugins provide many features available for all HLIs as for example: - -- **Creating fast C++ post processing** using low-level R/W data access - functions. Plugins have flexibility to run on-the-fly data - transformations such as decompression/compression, calibration, - filtering, ... - -- **Patching values from/to DD leaves** by overriding ``get()``/``put()`` - operations. An example is the unified NBC plugin (which overrides - the ``get()`` operation) which provides data format backward - compatibility. - -- **Displaying specific values** of DD leaves to users. Building - reports for some specific Interface Data Structure (IDS) nodes. - **Debugging**: developers check expected DD leaves values according - to some logic implemented in the plugin (for some specific - data/context). Plugins can display important warnings/infos to users - if necessary. - -- **Selecting partial data to speed up IDS writing/reading.** Let us - consider a partial read operation. Since a plugin has the control to - read data from any DD node, plugin logic allows reading few specific - nodes of an IDS and skip/ignore (large) nodes not required by the use - case. We will show later an example, which shows how to read only - some part of the data from an IDS, reducing IDS loading time. diff --git a/common/doc_common/plugins_architecture.rst b/common/doc_common/plugins_architecture.rst deleted file mode 100644 index 083f8e74..00000000 --- a/common/doc_common/plugins_architecture.rst +++ /dev/null @@ -1,339 +0,0 @@ -========================================= -Plugin implementation in the Access Layer -========================================= - - -Overview of the AL architecture before AL-plugins -================================================= - -`Figure 1`_ depicts the layered AL architecture model. It comprises -the upper High Level Interface (HLI) and the so-called Low Level (LL), -which receive HLIs requests. The LL includes a C layer with C functions -(wrappers) for calling the functions of the (C++) LL API located in the -C++ layer of the LL. - -.. figure:: ./doc_common/media/image1.png - :name: Figure 1 - - **Figure 1:** Layered AL architecture model - -When calling an AL API function (``get()``/``get_slice()``, ``put()``/``put_slice()``), -the HLI iterates over all nodes of the IDS where each visited node is -either a scalar, or an array (with dimensions from 1 to 6) or an array -of structures. - -Distinguishing Read and Write operations, we have two use-cases: - -- When calling ``get()``/``get_slice()``, the HLI calls the LL for each visited - node, passing the data contained in the node (scalar or array), the - parameters which define the dimension(s), the shape(s) of the data - and the identifier (path) of the node. Then the LL calls the backend, - which returns the data to the LL, which in turn returns the data - (allocated pointers) to the HLI. - -- When calling ``put()``/``put_slice()``, the HLI calls the LL for each node, - passing a pointer to the data to be written, the parameters which - define the dimension(s), the shape(s) of the data and the identifier - (path) of the node. Then the LL calls the backend, which writes the - data to some storage. - - -Modified AL architecture for plugins management and execution -============================================================= - -`Figure 2`_ depicts the modified AL architecture for enabling the use of -plugins. It introduces new components: - -- The C wrappers for plugins management - -- The plugins API - -- The plugin interface - -- The C++ plugins - -From now on, the Low Level C wrappers call the Plugins API functions. -The LL API is remained unchanged. - -.. figure:: ./doc_common/media/image2.png - :name: Figure 2 - - **Figure 2:** Modified layered AL architecture model for plugins - orchestration - - -The C wrappers for plugins management -------------------------------------- - -`Figure 3`_ displays the list of new C wrappers for plugins management and -the new plugins API. The latter includes: - -- functions for plugins management (registering and binding to a node) - -- functions which allow plugins to override the behavior of the LL data - access functions - -In the new AL plugin architecture, HLIs call the current C wrappers, which call -the new plugins API, which in turn call the plugins if they are registered and -bound to at least one node of the DD. For example, the -``al_begin_global_action(...)`` function calls the new plugins API -``beginGlobalActionPlugin(...)`` function of the plugins API. The functions -``al_read_data(...) and :code:``al_write_data(...)` delegate to the -``readDataSPlugin(...)`` and ``writeDataPlugin(...)`` plugins API -functions. - -.. figure:: ./doc_common/media/image3.png - :name: Figure 3 - - **Figure 3:** The new C wrappers for plugins management and the new plugins API - -Since the plugins API functions located in the LL C++ layer are not -accessible from the HLIs, the new C wrappers depicted in `Figure 3`_ allow -for accessing some plugins API functions from HLIs. These functions are -mainly devoted to plugin registering and activation, features supported -by the plugins API described in the next section: - -- The ``al_register_plugin(...)`` wrapper (resp. - ``al_unregister_plugin(...)``) delegates to the dedicated C++ - ``registerPlugin(...)`` (resp. ``unregisterPlugin(...)``) function - from the plugins API. - -- The ``al_bind_plugin(...)`` wrapper (``al_unbind_plugin(...)``) - delegates to the dedicated C++ :code:`bindPlugin(...) (resp. - ``unbindPlugin(...)``) function from the plugins API. - - -The Plugins API and the low level holder plugin class ------------------------------------------------------ - -`Figure 4`_ depicts the ``LLplugin`` plugin holder class. - -.. figure:: ./doc_common/media/image4.png - :name: Figure 4 - - **Figure 4:** The plugin holder class (``LLplugin``) - - -Plugin registration -~~~~~~~~~~~~~~~~~~~ - -Among the plugins API functions (`Figure 3`_), we find -``registerPlugin(plugin_name)`` (``unregisterPlugin(plugin_name)``) -which creates (resp. destroys) a plugin instance from a C++ class located in a -shared library (.so). The ``registerPlugin(...)`` function creates a -``LLplugin`` object for holding the plugin instance using the -``al_plugin`` pointer attribute defined in the ``LLplugin`` class (see -`Figure 4`_). The ``LLplugin`` object is then stored in a static map (named -````LLplugin``sStore``) of the ``LLplugin`` class. This map allows the -plugin framework to retrieve the plugin later using the name of the plugin as a -key (``plugin_name`` function argument). - - -Plugin activation -~~~~~~~~~~~~~~~~~ - -Once a plugin is registered, it is available from the plugins framework as long -as the AL process is running and as long as users have not called the -``unregister_plugin(...)`` C wrapper. However, during put/get operations, -the AL plugin framework will ignore a registered plugin not bound to any DD -node. To activate a plugin, an HLI code bounds the plugin to at least one -particular IDS node using the ``al_bind_plugin(...)`` wrapper. This function -updates the ``boundPlugins`` static map of the ``LLplugin`` class, adding the name -of the plugin to a list which is mapped to the identifier of the DD node (the -identifier is the path to the node). To disable a plugin, HLIs use the -``al_unbind_plugin(...)`` function which removes the plugin name from the -list hold by the ``boundPlugins`` map. - -Unregistering a plugin using ``al_unregister_plugin(plugin_name)`` will -remove the ``LLplugin`` object from the LL store ``LLpluginsStore`` and -destroy the underlying plugin instance. - - -Calling plugins from the Low Level -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -All plugins API functions in `Figure 3`_ (except functions for plugins -registering/activation) call plugins operations from the plugin -interface described in the next section during LL operations. For -example, the ``beginGlobalActionPlugin(...)`` function calls the -``begin_global_action(...)`` function of the plugin interface. More details -are provided in sequence diagrams presented in section :ref:`get() sequence -diagram` and :ref:`put() sequence diagram`. - - -The Access Layer plugin interface ---------------------------------- - -When creating a plugin, the plugin class has to inherit from the -``access_layer_plugin`` interface and implement each (pure virtual) -operation declared in this interface (see ``access_layer_plugin.h``). - -The ``begin_global_action(...)`` (resp. ``begin_slice_action(...)``) functions are -plugins operations called when the HLI calls ``al_begin_global_action(...)`` -(resp. ``al_begin_slice_action(...)``). These functions allow developers to -initialize the plugin before iteration of the IDS nodes performed by the -HLI. For example, during a slice operation using ``get_slice(...)`` or -``put_slice(...)``, the ``begin_slice_action(...)`` allows to store in the plugin -object the time of the slice and the interpolation method to perform the -slice. - -The node_operation(``const std::string &path``) function returns the type of -operation applied to the data node (located at the value given by the -``path`` argument) by the plugin. This function must return one of the -following value: ``plugin::OPERATION::GET_ONLY``, -``plugin::OPERATION::PUT_ONLY`` or ``plugin::OPERATION::PUT_AND_GET``. The -``node_operation(...)`` function, called by the plugins framework, has several -purposes: - -- It allows the plugins framework to sort the plugins that are contributing to - the ``get``/``get_slice`` operation, ``put``/``put_slice`` operation, or - both. Only plugins returning ``plugin::OPERATION::GET_ONLY`` or - ``plugin::OPERATION::PUT_AND_GET`` are added to the - ``ids_properties/plugins/get_operation`` data structure during a - ``get``/``get_slice`` operation. Moreover, only plugins returning - ``plugin::OPERATION::PUT_ONLY`` or ``plugin::OPERATION::PUT_AND_GET`` are - added to the ``ids_properties/plugins/put_operation`` data structure during a - ``put``/``put_slice`` operation. - -- The plugins framework will call the plugin operation ``read_data(...)`` for - a given IDS node path only if the ``node_operation(...)`` function returns - ``plugin::OPERATION::GET_ONLY`` or ``plugin::OPERATION::PUT_AND_GET`` for - this path. - -- The plugins framework will call the plugin operation ``write_data(...)`` - for a given IDS node path only if the ``node_operation(...)`` function - returns ``plugin::OPERATION::PUT_ONLY`` or ``plugin::OPERATION::PUT_AND_GET`` - for this path. - -The ``read_data(...)`` and (resp. ``write_data(...)``) allows a plugin to -intercept a data pointer coming from/going to the backend to read or modify it -on the fly. This documentation provides some plugins examples to illustrate the -use of these functions. - - -Calling low level data access API functions from plugin code ------------------------------------------------------------- - -New C wrappers allow plugins to call LL data access operations (the -latter should not be called by plugins code to avoid infinite -recursion). These functions have the same name that the existing -counterpart AL wrappers with prefix ``al`` replaced by ``al_plugin`` -(`Figure 6`_). - -.. code-block:: C++ - :caption: **Figure 6:** new C wrappers for calling LL data access operations from plugins - :name: Figure 6 - - void al_plugin_begin_global_action(const std::string &plugin_name, int pulseCtx, const char* dataobjectname, int mode, int opCtx); - void al_plugin_slice_action(const std::string &plugin_name, int pulseCtx, const char* dataobjectname, int mode, double time, int interp, int opCtx); - void al_plugin_arraystruct_action(const std::string &plugin_name, int ctx, int *actxID, const char* fieldPath, const char* timeBasePath, int *arraySize); - void al_plugin_read_data(const std::string &plugin_name, int ctx, const char* fieldPath, const char* timeBasePath, void **data, int datatype, int dim, int *size); - void al_plugin_write_data(const std::string &plugin_name, int ctxID, const char *field, const char *timebase, void *data, int datatype, int dim, int *size); - - -Plugins orchestration -===================== - -A registered and activated plugin (see :ref:`Plugin activation`) will be called -by the LL whenever a ``get()``/``get_slice()`` or ``put()``/``put_slice()`` -operation is performed by an HLI. The next sections describe the dynamic of -plugins calls during a ``get()``/``get_slice()`` or a ``put()``/``put_slice()`` -operation through sequence diagrams. - -.. _`get() sequence diagram`: - -``get()`` sequence diagram --------------------------- - -.. figure:: ./doc_common/media/image5.png - :name: Figure 7 - - **Figure 7:** ``get()`` operation sequence diagram - -`Figure 7`_ depicts an example of the ``get()`` operations sequence of a -``camera_ir`` IDS using the ``camera_ir`` plugin developed at WEST. The -client code uses HLI operations to read data of a ``camera_ir`` IDS. In -the first two calls, the client registers the plugin and binds it to the -node identified by the path ``camera_ir/frame/surface_temperature``, then -the AL API ``get()`` operation is called on the ``camera_ir`` IDS object (the -preliminary operation for creating the IDS object is not shown in the -figure). The ``al_begin_global_action(...)`` function calls first the -``beginGlobalActionPlugin(...)`` function of the plugins API (whatever the -node to which the plugin is bound) which in turn calls the -``begin_arraystruct_action(...)`` of the plugin interface. Similarly, each -call to ``al_begin_arraystruct_action(...)`` generates first a call to -``beginArraystructActionPlugin(...)``, followed by a call to the -``begin_arraystruct_action(...)`` function of the plugin interface. - -If several plugins are bound to the same IDS node, the plugin -framework will iterate over each plugin and call the appropriate plugin -interface function for each plugin sequentially. The iteration order -follows the order in which plugins bindings have been performed by the -HLI client using ``al_bind_plugin(...)``. - -.. _`put() sequence diagram`: - -``put()`` sequence diagram --------------------------- - -The ``put()`` sequence diagram (`Figure 8`_) is quite similar to the ``get()`` -sequence diagram. The example uses the ``camera_ir_write`` plugin -described later in this document. - -.. figure:: ./doc_common/media/image6.png - :name: Figure 8 - - **Figure 8:** ``put()`` operation sequence diagram - -Note that the wrapper ``al_begin_arraystruct_action(...)`` is called in this -modified AL plugin architecture even if the corresponding array of -structure (AOS) has a 0-shape (this is not true for the previous AL -architecture where HLIs were not calling ``al_begin_arraystruct_action(...)`` -if the AOS was found to be empty). The reason is to make plugins able to -write/update the size of AOSs. - - -Access Layer plugin provenance -============================== - -Recent versions of the Data Dictionary store information concerning plugin -provenance. See https://jira.iter.org/browse/IMAS-4491 and the Pull Request -https://git.iter.org/projects/IMAS/repos/data-dictionary/pull-requests/534/overview. - -To take into account these change, we have defined new plugins requirements with -the ``provenance_plugin_feature`` and the ``readback_plugin_feature`` interfaces -(`Figure 8`_). Moreover, in order to extend the ability of the LL to accept any -type of plugins, we have introduced a new access_layer_base_plugin interface. -The latter is more general and does not hold the read/write data access and -*readback* requirements. - -It may be necessary to define *readback* plugins that can read data -written by other plugins. For instance, the ``camera_ir_write`` plugin -(used currently on WEST) writes compressed data of the ``camera_ir`` IDS -to the backend. In order to read these data during a ``get()`` or -``get_slice()`` operation, the ``camera_ir_write`` plugin implements the -``readback_plugin_feature`` interface, which provides information to the -Access Layer about a *readback* plugin capable of reading and -uncompressing the data. For this example, the ``camera_ir`` readback -plugin is able to read and decompress data stored by the -``camera_ir_write`` plugin, providing uncompressed data to the HLI. The -name of the *readback* plugin is obtained using the -``getReadbackName(const std::string &path, int *index)`` function of the -``readback_plugin_feature`` interface for a given IDS node path. It is worth -noting that several plugins can be applied to the same IDS node path, -and the ``index`` parameter indicates which *readback* plugin should be -used. For instance, if two *readback* plugins are defined for the same -node path, a value of 1 for the ``index`` parameter indicates that the -plugin should be applied after the plugin defined at ``index=0``. - -During a ``put()``/``put_slice()`` operation, the **readback** -informations specified by the **readback_plugin_feature** interface are -stored in the backend. These data are read during a ``get()``/``get_slice()`` -operation and used to bind and execute the *readback* plugins. - -.. figure:: ./doc_common/media/image7.png - :name: Figure 9 - - **Figure 9:** The ``provenance_plugin_feature``, ``readback_plugin_feature`` - and ``access_layer_base_plugin`` interfaces - diff --git a/common/doc_common/plugins_examples.rst b/common/doc_common/plugins_examples.rst deleted file mode 100644 index 03bbf340..00000000 --- a/common/doc_common/plugins_examples.rst +++ /dev/null @@ -1,958 +0,0 @@ -Plugins examples -================ - -.. note:: - - The plugin examples referenced in this documentation are maintained in the al-plugins repository. - Please refer to: https://git.iter.org/projects/IMAS/repos/al-plugins/browse for the complete source code. - - -The ``debug`` plugin --------------------- - -In this first example, we want to display the value of the field -``ids_properties/version_put/access_layer`` for a given IDS during the -execution of a ``get()`` operation. - -The debug plugin is a C++ class named ``Debug_plugin``. The header code shows: - -- The Debug_plugin class inherits from the access_layer_plugin plugin - interface - -- All operations of the plugin interface are declared - -- The private attributes shot, dataobjectname and occurrence will be - initialized during the initialization of the plugin - -.. note:: - - The complete source code for the debug_plugin.h header is available in the al-plugins repository. - -The plugin implementation code includes: - -- Plugin initialization occurs in the ``begin_global_action(...)`` function. - However, no initialization is required in this example. - -- ``read_data(...)`` calls the backend using the LL - ``al_plugin_read_data(...)`` function. If the data type is a string, the - plugin print its value to the screen. - -- ``begin_arraystruct_action(...)`` creates a new ``ArraystructContext`` and - calls the backend, then the plugin prints the size of the array of - structure. - -- The *readback* information are empty in this example (the function - ``getReadbackName(path, index)`` returns an empty string, meaning that - the ``debug`` plugin does not define any *readback* plugin. - -.. note:: - - The complete source code for the debug_plugin.cpp implementation is available in the al-plugins repository. - - -Plugin compilation: creating a shared library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The Makefile for compiling the plugin will also compile the C++ HLI code ``test_debug_plugin.cpp``, which uses -this plugin. - -Executing the Makefile generates the shared library ``debug_plugin.so`` -and creates an executable ``test_debug_plugin`` for the HLI test -code described later. - -The ``AL_BUILD`` variable has to be set to the root directory of the IMAS -Access Layer sources. The dependency ``$AL_BUILD/lowlevel`` is required -since plugins depend on the LL API. The dependency -``$AL_BUILD/cppinterface/src`` is used only for the compilation of the HLI -test code. - - -.. code-block:: Makefile - :caption: **Figure 11:** plugin compilation Makefile - :name: Figure 11 - - include ../Makefile.common - - ifeq (,$(AL_BUILD)) - AL_BUILD=../ - $(warning AL_BUILD variable unset, probably not running 'make al_test' from the Installer. Assuming AL_BUILD=$(AL_BUILD).) - endif - - all sources sources_install install sources_uninstall uninstall clean clean-src test: - - DBGFLAGS= -g - DBGFLAGS+= -DDEBUG -DBOOST_ALL_DYN_LINK - NOT_SUPPORTED_COMPILER= - CC = gcc - CXX = g++ - ifeq "$(strip $(CC))" "icc" - CXXFLAGS= -O0 -fPIC -shared-intel ${DBGFLAGS} - LDFLAGS= - else ifeq "$(strip $(CC))" "gcc" - CXXFLAGS= -std=c++11 -pthread -O0 -fPIC ${DBGFLAGS} - LDFLAGS= -shared -pthread -Wl,--no-undefined - LDFLAGS_HLI=-pthread -Wl,--no-undefined - else - NOT_SUPPORTED_COMPILER=unsupported_compiler - endif - INCDIR_PKGCONFIG=`pkg-config blitz --cflags` - INCDIR= -I$(AL_BUILD)/lowlevel - LIBS= -lal `pkg-config blitz --libs` -L$(BOOST_ROOT)/lib -L$(AL_BUILD)/lowlevel -lboost_log -lboost_thread - INCDIR_HLI=$(INCDIR) -I$(AL_BUILD)/cppinterface/src $(INCDIR_PKGCONFIG) - LIBS_HLI=-L$(AL_BUILD)/cppinterface/lib $(LIBS) -lal-cpp - - OBJ_DEBUG_PLUGIN = debug_plugin.o simple_logger.o - PLUGINS = debug_plugin - - install: all - - EXE = $(addprefix test_, $(PLUGINS)) - - all: $(NOT_SUPPORTED_COMPILER) $(EXE) - - test_debug_plugin: debug_plugin - @echo Compiling test: $@ - $(CXX) $(INCDIR_HLI) $(CXXFLAGS) -c $@.cpp -o $@.o - $(CXX) $(LDFLAGS_HLI) -o $@ $@.o $(LIBDIR) $(LIBS_HLI) - - debug_plugin: $(OBJ_DEBUG_PLUGIN) - @echo Linking $^ - $(CXX) $(LDFLAGS) -o $@.so $^ $(LIBDIR) $(LIBS) - - .cpp.o: - @echo compiling $< - $(CXX) $(INCDIR) $(INCDIR_PKGCONFIG) $(CXXFLAGS) -c $< -o $@ - - clean: - $(RM) *.log *.o *.so $(OBJ) $(EXE) $(PLUGINS) - - unsupported_compiler: - @echo Makefile does not support $(CC) compiler \(try CC=gcc or icc\). - exit 1 - - -Execution of the Makefile gives the following output: - -.. code-block:: console - - $ make -f Makefile_debug_plugin - g++ `pkg-config blitz --cflags` -I/ZONE_TRAVAIL/LF218007/installer/src/3.35.0/ual/feature/al_plugins/lowlevel -std=c++11 -pthread -O0 -fPIC -g -DDEBUG -DBOOST_ALL_DYN_LINK -c debug_plugin.cpp -o debug_plugin.o - g++ `pkg-config blitz --cflags` -I/ZONE_TRAVAIL/LF218007/installer/src/3.35.0/ual/feature/al_plugins/lowlevel -std=c++11 -pthread -O0 -fPIC -g -DDEBUG -DBOOST_ALL_DYN_LINK -c simple_logger.cpp -o simple_logger.o - g++ -shared -pthread -Wl,--no-undefined -o debug_plugin.so debug_plugin.o simple_logger.o -L/ZONE_TRAVAIL/LF218007/installer/src/3.35.0/ual/feature/al_plugins/lowlevel -L/Applications/libraries/boost/1.76.0/gcc/6.4.0/lib -limas `pkg-config blitz --libs` -lboost_log -lboost_thread - - -Client code for plugin execution -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -.. md-tab-set:: - - .. md-tab-item:: Using Python - - `Figure 12`_ shows a python client code to execute the ``debug`` plugin: - - - The shot is opened using ``open(uri, mode)`` - - - The plugin ``debug`` is registered using ``register_plugin(...)`` - - - The plugin is bound to the node - ``ids_properties/version_put/access_layer`` of the ``magnetics`` IDS, - occurrence 0 - - - The ``get()`` operation is called for the IDS name found in the path - which is the last argument passed to the python script - - - Finally, the plugin is removed from memory using ``unregister_plugin(...)`` - - - .. code-block:: python - :caption: **Figure 12:** Client implementation code using a python HLI - :name: Figure 12 - - import imas - from imas import imasdef - from imas.hli_plugins import HLIPlugins - import sys - - uri = sys.argv[1] - - data_entry = imas.DBEntry(uri, "r") - data_entry.open() - - HLIPlugins.register_plugin("debug") - HLIPlugins.bind_plugin("magnetics:0/ids_properties/version_put/access_layer", "debug") - HLIPlugins.bind_plugin("magnetics:0/flux_loop", "debug") - data_entry.get('magnetics', occurrence=0) - HLIPlugins.unregister_plugin("debug") - data_entry.close() - - - Before executing the plugin, we need to specify the location of the - plugin shared library to the AL plugin framework using the environment - variable **IMAS_AL_PLUGINS**. For example: - - .. code-block:: console - - $ export IMAS_AL_PLUGINS=/Home/LF218007/access_layer_plugins - - In the example below, we are printing the value of the field - ``magnetics:0/ids_properties/version_put/access_layer`` for the shot - specified by the **uri** passed to the python script. The execution of - the python code shown above gives the following output: - - .. code-block:: console - - $ python test_debug_plugin.py imas:hdf5?path=/Imas_public/public/imasdb/west/3/54914/0/ - 0000001 | [debug] : read_data - reading data for field path= ids_properties/version_put/access_layer - 0000002 | [debug] : read_data - ids_properties/access_layer= 4.10.0-2-g367115bb - visiting AOS=flux_loop with size=17 - - - .. md-tab-item:: Using C++ - - .. code-block:: C++ - :caption: **Figure 13:** C++ test code, file ``test_debug_plugin.cpp`` - :name: Figure 13 - - #include - #include "ALClasses.h" - - using namespace IdsNs; - - void execute(char** argv); - void exitIfError(al_status_t &status); - - void execute(char** argv) { - - int pulse=54; - char* userName = NULL; - - userName = getenv("USER"); - if(userName == NULL) - { - printf( "PANIC: $USER not found! Exiting..."); - exit(1); - } - - /* Get Full */ - IdsNs::IDS data_entry(pulse,1,-1,-1); - data_entry.openEnv(userName, "test", "3"); - - al_status_t status = al_register_plugin("debug"); - exitIfError(status); - printf("Using the magnetics IDS for demo purpose\n"); - al_bind_plugin("magnetics:0/ids_properties/version_put/access_layer", "debug"); - al_bind_plugin("magnetics:0/flux_loop", "debug"); - - IDS::magnetics ids = data_entry._magnetics; - ids.get(0); - - data_entry.close(); - status = al_unregister_plugin("debug"); - exitIfError(status); - } - - void exitIfError(al_status_t &status) { - if (status.code != 0) { - printf("%s\n", status.message); - exit(-1); - } - } - - int main(int argc, char** argv){ - execute(argv); - } - - `Figure 13`_ shows a C++ client implementation test for executing the - ``debug`` plugin. The output is the same than previously as - expected: - - .. code-block:: console - - $ ./test_debug_plugin imas:hdf5?path=/Imas_public/public/imasdb/west/3/54914/0/ - Using the magnetics IDS for demo purpose - 0000001 | [debug] : read_data - reading data for field path= ids_properties/version_put/access_layer - 0000002 | [debug] : read_data - ids_properties/creation_date= 4.10.0-2-g367115bb - visiting AOS=flux_loop with size=17 - - - .. md-tab-item:: Using gfortran - - .. code-block:: Fortran - :caption: **Figure 14:** client implementation code using a Fortran HLI - :name: Figure 14 - - program test_debug_plugin - - use ids_routines - implicit none - - integer :: idx, mode, status - - type (ids_magnetics) :: mag ! Declaration of the ids - character(STRMAXLEN) :: uri - character(len=132):: usr - integer :: pulse = 54 - integer :: run = 1 - integer :: pulsectx - - call get_environment_variable("USER",usr) - - ! Registering the 'debug' plugin - call al_register_plugin ('debug', status) - - ! Binding the 'debug' plugin to the access_layer node of the magnetics IDS (as a demo purpose) - call al_bind_plugin ('magnetics:0/ids_properties/version_put/access_layer', 'debug', status) - - ! Opening the pulse file - call al_build_uri_from_legacy_parameters(MDSPLUS_BACKEND, pulse, run, usr, "test", "3", "", uri, status) - call al_begin_dataentry_action(uri, OPEN_PULSE, pulsectx, status); - write(*,*) 'Opened pulse file, pulsectx = ', pulsectx - - ! Calling 'get' will call the 'debug' plugin - call ids_get(pulsectx,"magnetics", mag) - - call imas_close(pulsectx) - - end program test_debug_plugin - - - `Figure 14`_ shows a Fortran client implementation for executing the - ``debug`` plugin. The output is the same than previously as expected: - - .. code-block:: console - - $ ./gfortran_test_debug_plugin imas:hdf5?path=/Imas_public/public/imasdb/west/3/54914/0/ - 0000001 | [debug] : read_data - reading data for field path= ids_properties/version_put/access_layer - 0000002 | [debug] : read_data - ids_properties/creation_date= 4.10.0-2-g367115bb - visiting AOS=flux_loop with size=17 - - -.. _`simplifying plugin code`: - -Simplifying plugin code: introducing the ``AL_reader_helper_plugin`` class --------------------------------------------------------------------------- - -In this section, we show how to simplify the ``debug`` plugin code -presented previously. For this purpose, we introduce the new helper -class ``AL_reader_helper_plugin`` whose part of the header (provenance -feature operations have been removed for clarity) is shown in Figure 15. -Figure 16 depicts its implementation code. - -.. note:: - - The complete source code for the al_reader_helper_plugin.h header is available in the al-plugins repository. - -By inheriting the helper class, we obtain the header of the ``Debug_plugin`` -class depicted in Figure 17. The header code declares only the -``read_data(..)`` function (from the plugin interface) whose implementation -is overridden in the simplified implementation code of the ``Debug_plugin`` -class (Figure 18). - -.. note:: - - The complete source code for the al_reader_helper_plugin.cpp implementation is available in the al-plugins repository. - -.. code-block:: C++ - :caption: **Figure 17:** the simplified ``Debug_plugin`` class header - :name: Figure 17 - - #ifndef DEBUG_PLUGIN_H - #define DEBUG_PLUGIN_H 1 - #include "al_reader_helper_plugin.h" - #include "access_layer_plugin.h" - - class Debug_plugin: public AL_reader_helper_plugin - { - public: - Debug_plugin(); - ~Debug_plugin(); - - int read_data(int ctx, const char* fieldPath, const char* timeBasePath, - void **data, int datatype, int dim, int *size); - }; - - extern "C" access_layer_plugin* create() { - return new Debug_plugin; - } - extern "C" void destroy (access_layer_plugin* al_plugin) { - delete al_plugin; - } - #endif - -.. code-block:: C++ - :caption: **Figure 18:** the simplified ``Debug_plugin`` class implementation - :name: Figure 18 - - #include "debug_plugin.h" - #include "simple_logger.h" - - Debug_plugin::Debug_plugin() - {} - - Debug_plugin::~Debug_plugin() - {} - - void Debug_plugin::begin_arraystruct_action(int ctx, int *aosctx, const char* fieldPath, const char* timeBasePath, int *arraySize) { - if (*arraySize != 0) - printf("visiting AOS=%s with size=%d\n", fieldPath, *arraySize); - } - - int Debug_plugin::read_data(int ctx, const char* fieldPath, const char* timeBasePath, - void **data, int datatype, int dim, int *size) { - al_plugin_read_data(ctx, fieldPath, timeBasePath, data, datatype, dim, size); - if (datatype == CHAR_DATA) { - LOG_DEBUG << "reading data for field path= " << fieldPath; - char buff[100]; - snprintf(buff, sizeof(buff), "%s", (char*) *data); - std::string buffAsStdStr = buff; - LOG_DEBUG << "ids_properties/access_layer= " << buffAsStdStr; - } - else { - LOG_DEBUG << "debug plugin prints only STRING data" << fieldPath; - } - return 1; - } - - -A plugin to automatically fill ``ids_properties/creation_date`` ---------------------------------------------------------------- - -The requirement of IMAS-3121 ITER JIRA ticket is to fill in the -``ids_properties/creation_date`` node during a ``put()`` operation with the -current date in the form YYYY-MM-DD. - -.. note:: - - The complete source code for the creation_date_plugin.cpp implementation is available in the al-plugins repository. - -The ``Creation_date_plugin`` class implements this feature. The header file -content is also available in the al-plugins repository. Provenance feature operations have been removed for clarity in -these files. - -.. code-block:: python - :caption: **Figure 27:** python client of the ``creation_date`` plugin - :name: Figure 27 - - import imas - import matplotlib.pyplot as plt - from imas.hli_plugins import HLIPlugins - from imas import imasdef - import sys - - uri = sys.argv[1] - - data_entry = imas.DBEntry(uri, "w") - data_entry.create() - - m= imas.magnetics() - m.ids_properties.homogeneous_time=1 - m.time.resize(1) - m.time[0]=0. - - HLIPlugins.register_plugin("creation_date") - - path= "magnetics:0/ids_properties/creation_date" - HLIPlugins.bind_plugin(path, "creation_date") - - data_entry.put(m) - - HLIPlugins.unregister_plugin("creation_date") - - data_entry.close() - - print("checking value of creation_date...") - data_entry.open(uri=uri, mode=imasdef.FORCE_OPEN_PULSE) - ids = data_entry.get('magnetics') - print(ids.ids_properties.creation_date) - data_entry.close() - -.. code-block:: console - :caption: **Figure 28:** executing the ``creation_date`` plugin and checking the result - :name: Figure 28 - - $ python test_creation_date_plugin.py imas:hdf5?path=/Home/LF218007/public/imasdb/test/3/55000/0 - Patching creation_date... - checking value of creation_date... - 2022-03-11 - -After execution of the ``creation_date`` plugin using the python client code -depicted in `Figure 27`_, we check that the ``ids_properties/creation_date`` has -been successfully updated as expected (`Figure 28`_). - -The C++ code below (`Figure 29`_) uses the same plugin. The test is making -a little bit more than the previous python test, it prints at the end -if some *readback* plugins have been called during ``get()``. Since no -*readback* plugin has been defined in the ``creation_date`` -plugin code, the test prints “No readback plugins have been called -during ``get()``.”. We will see later an example, which shows how to -define a *readback* plugin. - -.. code-block:: C++ - :caption: **Figure 29:** executing the ``Creation_date`` plugin - :name: Figure 29 - - #include "creation_date_plugin.h" - - #include - #include - #include - #include - #include - #include - #include - - Creation_date_plugin::Creation_date_plugin() - { - } - - Creation_date_plugin::~Creation_date_plugin() - { - } - - /** - The following functions are used to provide metadata about the plugin to identify and manage the plugin. - - getName() returns the name of the plugin, which is "creation_date" in this example. - getCommit() returns a unique identifier for the version of the plugin, which is "8f2e7cd64daf9e35a6e6c5850dd80fc198f11d86" in this example. - getVersion() returns the version number of the plugin, which is "1.0.0" in this example. - getRepository() returns the URI for the Git repository where the plugin is located, which is "ssh://git@git.iter.org/imas/access-layer-plugins.git" in this example. - getParameters() returns a string representing the parameters for the plugin. - */ - - std::string Creation_date_plugin::getName() { - return "creation_date"; - } - - std::string Creation_date_plugin::getDescription() { - return ""; - } - - std::string Creation_date_plugin::getCommit() { - return "8f2e7cd64daf9e35a6e6c5850dd80fc198f11d86"; - } - - std::string Creation_date_plugin::getVersion() { - return "1.0.0"; - } - std::string Creation_date_plugin::getRepository() { - return "ssh://git@git.iter.org/imas/access-layer-plugins.git"; - } - std::string Creation_date_plugin::getParameters() { - return "creation_date plugin parameters"; - } - - /** - The following functions are used by the Access Layer to determine which readback plugin to use for a given field path. - - The getReadbackName() function returns the name of the readback plugin to use for a given field path. If the field path contains the string "creation_date", it returns the string "debug". Otherwise, it returns an empty string. - - The getReadbackCommit() function returns the commit hash of the readback plugin to use for a given field path. If the field path contains the string "creation_date", it returns a specific commit hash. Otherwise, it returns an empty string. - - The getReadbackVersion() function returns the version of the readback plugin to use for a given field path. If the field path contains the string "creation_date", it returns the string "1.1.0". Otherwise, it returns an empty string. - - The getReadbackRepository() function returns the repository location of the readback plugin to use for a given field path. If the field path contains the string "creation_date", it returns a specific repository location. Otherwise, it returns an empty string. - - The getReadbackParameters() function returns the parameters to pass to the readback plugin for a given field path. If the field path contains the string "creation_date", it returns a specific string. Otherwise, it returns an empty string. - */ - - std::string Creation_date_plugin::getReadbackName(const std::string &path, int* index) { - return ""; - } - - std::string Creation_date_plugin::getReadbackDescription(const std::string &path) { - return ""; - } - - std::string Creation_date_plugin::getReadbackCommit(const std::string &path) { - if (path.rfind("creation_date")) - return "d92bb2f30384bc618574508b56e9542d00f0e97a"; - return ""; - } - - std::string Creation_date_plugin::getReadbackVersion(const std::string &path) { - if (path.rfind("creation_date")) - return "1.1.0"; - return ""; - } - - std::string Creation_date_plugin::getReadbackRepository(const std::string &path) { - if (path.rfind("creation_date")) - return "ssh://git@git.iter.org/imas/access-layer-plugins.git"; - return ""; - } - - std::string Creation_date_plugin::getReadbackParameters(const std::string &path) { - if (path.rfind("creation_date")) - return "readback plugin parameters"; - return ""; - } - - plugin::OPERATION Creation_date_plugin::node_operation(const std::string &path) { - return plugin::OPERATION::PUT_ONLY; - } - - void Creation_date_plugin::begin_global_action(int pulseCtx, const char* dataobjectname, const char* datapath, int mode, int opCtx) { - } - - void Creation_date_plugin::begin_slice_action(int pulseCtx, const char* dataobjectname, int mode, double time, int interp, int opCtx) { - } - - void Creation_date_plugin::begin_arraystruct_action(int ctx, int *aosctx, const char* fieldPath, const char* timeBasePath, int *arraySize) { - LLenv lle = Lowlevel::getLLenv(ctx); - ArraystructContext* actx = lle.create(fieldPath, timeBasePath); - *aosctx = Lowlevel::addLLenv(lle.backend, actx); - lle.backend->beginArraystructAction(actx, arraySize); - } - - int Creation_date_plugin::read_data(int ctx, const char* fieldPath, const char* timeBasePath, - void **data, int datatype, int dim, int *size) { - return 0; - } - - - /** - This function is used to write data to the specific IDS field 'ids_properties/creation_date'. - - The function takes several parameters, including an integer "ctx", which is a handle to the current low level context, a character array "fieldPath" representing the path to the field being modified, a character array "timeBasePath" representing the path to the timebase for the field, a void pointer "data" containing the data to be written, an integer "datatype" specifying the type of data being written, an integer "dim" specifying the number of dimensions of the data, and an array of integers "size" specifying the size of each dimension. - - The function prints a message indicating that the "Creation_date" plugin is patching the creation date. It then uses the standard library function "std::time" to obtain the current system time and local time, and formats it using the standard library function "std::put_time" to create a string representing the current date in the format of "YYYY-MM-DD". This string is stored in a std::ostringstream object called "oss" and then converted to a C-style string using the "str()" function, and then to a void pointer using the "c_str()" function. - - Finally, the function creates an array of sizes for the data being written, with a single element representing the length of the string. It then calls the "al_plugin_write_data" function to write the data to the specified field, passing in the current context, the field path, the timebase path, the void pointer to the data, the data type, the number of dimensions, and the array of sizes. After writing the data, the function prints a message indicating that the patching is complete. - */ - void Creation_date_plugin::write_data(int ctx, const char* fieldPath, const char* timeBasePath, void *data, int datatype, int dim, int *size) { - printf("Patching creation_date from the creation_date plugin... \n"); - auto t = std::time(nullptr); - auto tm = *std::localtime(&t); - std::ostringstream oss; - oss << std::put_time(&tm, "%Y-%m-%d"); - auto text = oss.str(); - void* ptrData = (void *) (text.c_str()); - int arrayOfSizes[1] = {(int)text.size()}; - al_plugin_write_data(ctx, fieldPath, timeBasePath, ptrData, CHAR_DATA, 1, arrayOfSizes); - printf("End of patching.\n"); - } - - void Creation_date_plugin::setParameter(const char* parameter_name, int datatype, int dim, int *size, void *data) { - } - - void Creation_date_plugin::end_action(int ctx) {} - - - -.. code-block:: Fortran - :caption: **Figure 30:** Fortran test code for executing the ``creation_date`` plugin - :name: Figure 30 - - program test - - use ids_routines - implicit none - - integer :: idx, mode, status - - type (ids_magnetics) :: mag ! Declaration of the empty ids to be filled - character(STRMAXLEN) :: uri - character(len=132):: usr - integer :: pulse = 54 - integer :: run = 1 - - call get_environment_variable("USER",usr) - - call al_build_uri_from_legacy_parameters(MDSPLUS_BACKEND, pulse, run, usr, "test", "3", "", uri, status) - mag%ids_properties%homogeneous_time = 1 ! Mandatory to define this property - allocate(mag%time(1)) - mag%time = 0.0 - - - ! Registering the 'creation_date' plugin - call al_register_plugin ('creation_date', status) - - write(*,*) 'Using the magnetics IDS for demo purpose' - - ! Binding the 'creation_date' node of the magnetics IDS to the 'creation_date' plugin (only for demo purpose) - call al_bind_plugin ('magnetics:0/ids_properties/creation_date', 'creation_date', status) - - ! Creating the pulse file - - call al_begin_dataentry_action(uri, FORCE_CREATE_PULSE, idx, status) - write(*,*) 'Creating pulse file, idx = ', idx - call ids_put(idx,"magnetics",mag) - - write(*,*) 'Closing pulse file, idx = ', idx - call imas_close(idx) - - ! Unregistering the plugin since we do not need it anymore - call al_unregister_plugin ('creation_date', status) - - ! Opening the pulse file to check the value of 'creation_date' - - call al_begin_dataentry_action(uri, OPEN_PULSE, idx, status) - write(*,*) 'Opening pulse file, idx = ', idx - - write(*,*) 'Reading pulse file, idx = ', idx - call ids_get(idx,"magnetics",mag) - - print *, 'creation_date = ', mag%ids_properties%creation_date - call imas_close(idx) - - end program test - -Executing the Fortran code above gives the following output: - -.. code-block:: console - - $ ./gfortran_test_creation_date_plugin 55000 0 LF218007 test 13 - - Using the magnetics IDS for demo purpose - - Creating pulse file, idx = 1 - - Patching creation_date... - - End of patching. - - Closing pulse file, idx = 1 - - Opening pulse file, idx = 1 - - Reading pulse file, idx = 1 - - creation_date = 2022-03-16 - - -Binding a *readback* plugin -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Let us consider again the ``creation_date`` plugin. Suppose that -we want to display the value of the field ``ids_properties/creation_date`` -(using the ``debug`` plugin) just after it has been patched. In order to -execute the ``debug`` plugin, we replace the *readback* feature -function ``getReadbackName(...)`` of the ``creation_date`` plugin: - -.. code-block:: C++ - - std::string Creation_date_plugin::getReadbackName(const std::string &path, int* index) { - return ""; - } - -by the following code: - -.. code-block:: C++ - - std::string Creation_date_plugin::getReadbackName(const std::string &path, int* index) { - if (path.rfind("ids_properties/creation_date")) { - *index = 0; - return "debug"; - } - } - -In the code above, we are binding the ``debug`` plugin to the path -``ids_properties/creation_date`` - -After recompilation of the plugin, the execution of the C++ test of -`Figure 29`_ gives the following output: - -.. code-block:: console - - $ ./test_creation_date_plugin imas:hdf5?path=/Home/LF218007/public/imasdb/test/3/55000/0 - Patching the field 'ids_properties/creation_date' of a magnetics IDS for demo purpose. - Patching creation_date from the creation_date plugin... - End of patching. - Reading IDS... - 0000001 | [debug] : read_data - reading data for field path= ids_properties/creation_date - 0000002 | [debug] : read_data - ids_properties/creation_date= 2023-03-06 - visiting AOS=ids_properties/plugins/node with size=1 - node at path=ids_properties/creation_date - readback plugin --> name=debug - --> version=1.1.0 - --> commit=d92bb2f30384bc618574508b56e9542d00f0e97a - -The *readback* plugin ``debug`` is executed during the -``get()`` operation (when visiting the node ``ids_properties/creation_date``) -as expected. - - -Building a partial ``get()`` operation --------------------------------------- - -Skipping the read of an array of structure -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. note:: - - The complete source code for the PartialGetPlugin class implementation (partial_get_plugin.cpp) - and header (partial_get_plugin.h) is available in the al-plugins repository. - -In a first use-case, the user wants to access only few attributes of the -``equilibrium`` IDS for many shots. In order to speed up reading, he -decides to skip the loading of the ``grids_ggd`` array of structures (AOS). -The ``PartialGetPlugin`` class provides an efficient solution. During the ``get()`` -operation, the plugin intercepts the HLI call to the function -``al_begin_arraystruct_action(...)`` for the ``grids_ggd`` AOS and sets its size -(using the arraySize pointer) to 0 after displaying a warning to the -user: - -.. code-block:: C++ - - void PartialGetPlugin::begin_arraystruct_action(int ctx, int *aosctx, const char* fieldPath, const char* timeBasePath, int *arraySize) { - if (std::string(fieldPath) == "grids_ggd") { - LOG_WARNING << "ignoring gids_ggd"; - *arraySize = 0; - } - } - -The value of arraySize (set to 0) is returned to the HLI which will -therefore skip the iteration of the ``grids_ggd`` AOS. - - -Executing the ``partial_get`` plugin -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. md-tab-set:: - - .. md-tab-item:: Using Python - - `Figure 33`_ depicts a HLI python client code to test the plugin. - - .. code-block:: python - :caption: **Figure 33:** python client code to test the ``PartialGetPlugin`` plugin - :name: Figure 33 - - import imas - from imas import imasdef - import matplotlib.pyplot as plt - from imas.hli_plugins import HLIPlugins - import sys - - uri = sys.argv[1] - - data_entry = imas.DBEntry(uri) - data_entry.open() - - HLIPlugins.register_plugin("partial_get"); - HLIPlugins.bind_plugin("equilibrium:0/grids_ggd", "partial_get") - - equilibrium_ids = data_entry.get('equilibrium', occurrence=0) - - print("Grids_ggd AOS size=",len(equilibrium_ids.grids_ggd)) - - HLIPlugins.unregister_plugin("partial_get") - - The execution of the plugin gives the following output: - - .. code-block:: console - - $ python test_partial_get_plugin.py imas:hdf5?path=/Imas_public/public/imasdb/west/3/54914/0/ - 0000001 | [warning] : begin_arraystruct_action - ignoring grids_ggd - Grids_ggd AOS size= 0 - - - .. md-tab-item:: Using C++ - - .. code-block:: C++ - :caption: **Figure 34:** C++ client code to test the ``partial_get`` plugin - :name: Figure 34 - - #include - #include "ALClasses.h" - - using namespace IdsNs; - - void execute(char** argv); - void exitIfError(al_status_t &status); - - void execute(char** argv) { - - int pulse=54; - char* userName = NULL; - - userName = getenv("USER"); - if(userName == NULL) - { - printf( "PANIC: $USER not found! Exiting..."); - exit(1); - } - - /* Get Full */ - IdsNs::IDS data_entry(pulse,1,-1,-1); - data_entry.openEnv(userName, "test", "3"); - - al_status_t status = al_register_plugin("partial_get"); - exitIfError(status); - status = al_bind_plugin("magnetics:0/flux_loop", "partial_get"); - IDS::magnetics ids = data_entry._magnetics; - ids.get(0); - printf("magnetics AOS size=%d\n",ids.flux_loop.size()); - data_entry.close(); - - status = al_unregister_plugin("partial_get"); - exitIfError(status); - } - - void exitIfError(al_status_t &status) { - if (status.code != 0) { - printf("%s\n", status.message); - exit(-1); - } - } - - - int main(int argc, char** argv){ - execute(argv); - } - - Execution of C++ client code gives: - - .. code-block:: console - - $ ./test_partial_get_plugin imas:hdf5?path=/Imas_public/public/imasdb/west/3/54914/0/ - 0000001 | [warning] : begin_arraystruct_action - ignoring grids_ggd - Grids_ggd AOS size=0 - - - .. md-tab-item:: Using Fortran - - .. code-block:: Fortran - :caption: **Figure 35:** fortran client code to test the ``partial_get`` plugin - :name: Figure 35 - - program test - - use ids_routines - implicit none - - integer :: idx, mode, status - - type (ids_magnetics) :: mag ! Declaration of the ids - character(STRMAXLEN) :: uri - character(len=132):: usr - integer :: pulse = 54 - integer :: run = 1 - - call get_environment_variable("USER",usr) - - call al_build_uri_from_legacy_parameters(MDSPLUS_BACKEND, pulse, run, usr, "test", "3", "", uri, status) - - ! Registering the 'partial_get' plugin - call al_register_plugin ('partial_get', status) - - ! Binding the 'partial_get' plugin to the 'grids_ggd' node of the equilibrium IDS (as a demo purpose) - call al_bind_plugin ('magnetics:0/flux_loop', 'partial_get', status) - - ! Opening the pulse file - call al_begin_dataentry_action(uri, OPEN_PULSE, idx, status); - write(*,*) 'Opening pulse file, idx = ', idx - - ! Calling 'get' will call the 'partial_get' plugin - call ids_get(idx,"magnetics", mag) - print *, 'flux_loop pointer associated = ', associated(mag%flux_loop) - call imas_close(idx) - - end program test - - .. code-block:: console - - $ ./gfortran_test_partial_get_plugin imas:hdf5?path=/Imas_public/public/imasdb/west/3/54914/0/ - 0000001 | [warning] : begin_arraystruct_action - ignoring grids_ggd - Grids_ggd pointer associated = F diff --git a/common/doc_common/requirements.txt b/common/doc_common/requirements.txt deleted file mode 100644 index 299cf3a5..00000000 --- a/common/doc_common/requirements.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Sphinx and theme -sphinx >= 6.0, < 7.0 -sphinx_immaterial >= 0.11.4, < 0.12 - -# Fortran domain -sphinx-fortran -six # un-listed dependency of sphinx-fortran - -# Java domain -# Note: package is deprecated, but no alternatives exist... -javasphinx - -# Matlab domain -sphinxcontrib-matlabdomain - -# Requirements for the HLI files we import for the Python docs -numpy diff --git a/common/doc_common/static/.keep b/common/doc_common/static/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/common/doc_common/templates/base.html b/common/doc_common/templates/base.html deleted file mode 100644 index 0f8c7af1..00000000 --- a/common/doc_common/templates/base.html +++ /dev/null @@ -1,24 +0,0 @@ -{% extends "!base.html" %} - -{% block styles %} -{{ super() }} - -{% endblock %} - -{% block outdated %} - {% if is_develop %} - You're viewing a development version ({{ version }}). - {% else %} - You're viewing an older version ({{ version }}). - {% endif %} - - Click here to go to the documentation of the latest release. - -{% endblock %} \ No newline at end of file diff --git a/common/doc_common/use_ids.rst b/common/doc_common/use_ids.rst deleted file mode 100644 index 6f10b4bc..00000000 --- a/common/doc_common/use_ids.rst +++ /dev/null @@ -1,339 +0,0 @@ -Use Interface Data Structures -============================= - -The Interface Data Structures (IDSs) are the main way to interact with IMAS -data. An IDS is a tree-like structure with one root element (the IDS) and -several branches with data at the leave nodes. - -Many types of IDSs exist: check out the documentation of the |DD| for a complete -overview. - - -Creating IDSs -------------- - -IDSs can be created in multiple ways: - -1. :ref:`Load an IDS from disk ` -2. :ref:`Create an empty IDS` -3. :ref:`Create a copy of an IDS` - - -Create an empty IDS -''''''''''''''''''' - -You can create an empty instance of an IDS by |create_ids_text| creates an empty -``core_profiles`` IDS. This initializes all items in the IDS to their -:ref:`default values`. - -.. literalinclude:: code_samples/ids_create - :caption: |lang| example: create an empty IDS - - -Create a copy of an IDS -''''''''''''''''''''''' - -You can create a copy of another IDS |copy_ids|. - -.. literalinclude:: code_samples/ids_copy - :caption: |lang| example: create a copy of an IDS - - -Deallocate an IDS -''''''''''''''''' - -If you no longer need an IDS, you can deallocate it so it releases the (memory) -resources in use by the data. |deallocate_ids_text| - -.. literalinclude:: code_samples/ids_deallocate - :caption: |lang| example: deallocate an IDS - - -Mandatory and recommended IDS attributes ----------------------------------------- - -Some attributes in an IDS are mandatory or recommended to always fill. Below -list provides a short overview: - -.. todo:: - - Link to DD documentation - -1. ``ids_properties/homogeneous_time`` `[mandatory]`: see :ref:`Time coordinates - and time handling`. -2. ``ids_properties/comment`` `[recommended]`: a comment describing the content - of this IDS. -3. ``ids_properties/provider`` `[recommended]`: name of the person in charge of - producing this data. -4. ``ids_properties/creation_date`` `[recommended]`: date at which this data has - been produced, recommended to use the `ISO 8601 - `_ ``YYYY-MM-DD`` format. - -.. note:: - - ``ids_properties/version_put`` is filled by the access layer when you - :ref:`put an IDS `. - - -Understanding the IDS structure -------------------------------- - -An IDS is a `tree structure -`_. You can think of it -similar as a directory structure with files: the IDS is the root "directory", -and inside it you can find "subdirectories" and "files" with data. - -We will use the general Computer Science terminology for tree structures and -call these "files" and "directories" of our IDSs `nodes`. IDSs can have a -limited number of different types of nodes: - -1. :ref:`Structure`: think of these as the directories of your file - system. Structures contain one or more child nodes (files and - subdirectories). Child nodes can be of any node type again. - -2. :ref:`Array of structures`: this is an array of structures (see - previous point). - -3. :ref:`Data`: this is a data element. Like files on your file system - these nodes contain the actual data stored in the IDS. - - -Structure -''''''''' - -Structure nodes in an IDS are a container for other nodes. In |lang| they are -implemented as a |structures_type|. You can address child nodes as -|structures_child_attribute|, see the code sample below. - -.. literalinclude:: code_samples/ids_structures_node - :caption: |lang| example: address the child node of an IDS structure node - - -Array of structures -''''''''''''''''''' - -Array of structure nodes in an IDS are one-dimensional arrays, containing structure -nodes. In |lang| they are implemented as a |aos_type|. The default value (for -example, when creating a new IDS) for these nodes is |aos_default|. - -.. literalinclude:: code_samples/ids_array_of_structures_node - :caption: |lang| example: address the child node of an IDS arrays of structure node - - -Resizing an array of structures -``````````````````````````````` - -You can resize an array of structures with |aos_resize_meth|. After calling -this, the array of structures will have ``n`` elements. - -.. caution:: - - Resizing an array of structures with |aos_resize_meth| will clear all data - inside the array of structure! Use |aos_resize_keep_meth| to keep existing - data. - -.. literalinclude:: code_samples/ids_array_of_structures_resize - :caption: |lang| example: resizing an array of structures - - -Data -'''' - -Data nodes in an IDS contain numerical or textual data. The data type and -dimensions of a node are defined in the |DD|. - -.. literalinclude:: code_samples/ids_data_node - :caption: |lang| example: get the data contained in a data node of an IDS - - -Data types -`````````` - -The following data types exist: - -- Textual data (|str_type|) -- Whole numbers (|int_type|) -- Floating point numbers (|double_type|) -- Complex floating point numbers (|complex_type|) - -Data nodes can be 0-dimensional, which means that the node accepts a single -value of the specified type. Multi-dimensional data nodes also exist: - -- Textual data: at most 1 dimension (|str_1d_type|) -- Whole numbers: 1-3 dimensions (|int_nd_type|) -- Floating point numbers: 1-6 dimensions (|double_nd_type|) -- Complex floating point numbers: 1-6 dimensions (|complex_nd_type|) - - -.. _Empty fields: - -Default values -`````````````` - -The default values for data fields (for example when creating an empty IDS) are -indicated in the following table. |isFieldValid| - -.. csv-table:: - :header-rows: 1 - :stub-columns: 1 - - , 0D, 1+ dimension - "Textual - - data", |str_default|, |str_1D_default| - "Whole - - numbers", |int_default|, |ND_default| - "Floating - - point - - numbers", |double_default|, |ND_default| - "Complex - - numbers", |complex_default|, |ND_default| - - -Time coordinates and time handling -'''''''''''''''''''''''''''''''''' - -Some quantities (and array of structures) are time dependent. In the |DD| -documentation this is indicated by a coordinate that refers to a time quantity. - -This time-dependent coordinate is treated specially in the access layer, and it -depends on the value of ``ids_properties/homogeneous_time``. There are three -valid values for this property: - -.. todo:: - - Add links to DD. - -1. |tm_heterogeneous| (=0): time-dependent quantities in the IDS may have - different time coordinates. The time coordinates are stored as indicated by - the path in the documentation. This is known as `heterogeneous time`. -2. |tm_homogeneous| (=1): All time-dependent quantities in this IDS use the same - time coordinate. This is known as `homogeneous time`. This time coordinate is - located in the root of the IDS, for example ``core_profiles/time``. The paths - time paths indicated in the documentation are unused in this case. -3. |tm_independent| (=2): The IDS stores no time-dependent data. - - - -IDS validation --------------- - -The IDSs you fill should be consistent. To help you in validating that, the -Access Layer provides a validation method (|ids_validate|) that executes the -following checks. - -.. contents:: Validation checks - :local: - :depth: 1 - -If you call this method and your IDS fails validation, the Access Layer -|validate_error| explaining the problem. See the following example: - -.. literalinclude:: code_samples/ids_validate - :caption: |lang| example: call IDS validation - -The Access Layer automatically validates an IDS every time you do a -`put` or `put_slice`. To disable this feature, you must set the environment -variable ``IMAS_AL_DISABLE_VALIDATE`` to ``1``. - -.. seealso:: - - API documentation: |ids_validate| - - -Validate the time mode -'''''''''''''''''''''' - -The time mode of an IDS is stored in ``ids_properties.homogeneous_time``. This -property must be filled with a valid time mode (|tm_homogeneous|, -|tm_heterogeneous| or |tm_independent|). When the time -mode is |tm_independent|, all time-dependent quantities must be empty. - - -Validate coordinates -'''''''''''''''''''' - -If a quantity in your IDS has coordinates, then these coordinates must be filled. The -size of your data must match the size of the coordinates: - -.. todo:: link to DD docs - -1. Some dimensions must have a fixed size. This is indicated by the Data Dictionary - as, for example, ``1...3``. - - For example, in the ``magnetics`` IDS, ``b_field_pol_probe(i1)/bandwidth_3db`` has - ``1...2`` as coordinate 1. This means that, if you fill this data field, the first - (and only) dimension of this field must be of size 2. - -2. If the coordinate is another quantity in the IDS, then that coordinate must be - filled and have the same size as your data. - - For example, in the ``pf_active`` IDS, ``coil(i1)/current_limit_max`` is a - two-dimensional quantity with coordinates ``coil(i1)/b_field_max`` and - ``coil(i1)/temperature``. This means that, if you fill this data field, their - coordinate fields must be filled as well. The first dimension of - ``current_limit_max`` must have the same size as ``b_field_max`` and the second - dimension the same size as ``temperature``. - - Time coordinates are handled depending on the value of - ``ids_properties/homogeneous_time``: - - - When using |tm_homogeneous|, all time coordinates look at the root - ``time`` node of the IDS. - - When using |tm_heterogeneous|, all time coordinates look at the time - path specified as coordinate by the Data Dictionary. - - For dynamic array of structures, the time coordinates is a ``FLT_0D`` inside the - AoS (see, for example, ``profiles_1d`` in the ``core_profiles`` IDS). In such - cases the time node must be set to something different than ``EMPTY_FLOAT``. - This is the only case in which values of the coordinates are verified, in all - other cases only the sizes of coordinates are validated. - - .. rubric:: Alternative coordinates - - Version 4 of the Data Dictionary introduces alternative coordinates. An - example of this can be found in the ``core_profiles`` IDS in - ``profiles_1d(itime)/grid/rho_tor_norm``. Alternatives for this coordinate - are: - - - ``profiles_1d(itime)/grid/rho_tor`` - - ``profiles_1d(itime)/grid/psi`` - - ``profiles_1d(itime)/grid/volume`` - - ``profiles_1d(itime)/grid/area`` - - ``profiles_1d(itime)/grid/surface`` - - ``profiles_1d(itime)/grid/rho_pol_norm`` - - Multiple alternative coordinates may be filled (for example, an IDS might - fill both the normalized and non-normalized toroidal flux coordinate). In - that case, the size must be the same. - - When a quantity refers to this set of alternatives (for example - ``profiles_1d(itime)/electrons/temperature``), at least one of the - alternative coordinates must be set and its size match the size of the - quantity. - -3. The Data Dictionary can indicate exclusive alternative coordinates. See for - example the ``distribution(i1)/profiles_2d(itime)/density(:,:)`` quantity in the - ``distributions`` IDS, which has as first coordinate - ``distribution(i1)/profiles_2d(itime)/grid/r OR - distribution(i1)/profiles_2d(itime)/grid/rho_tor_norm``. This means that - either ``r`` or ``rho_tor_norm`` can be used as coordinate. - - Validation works the same as explained in the previous point, except that - exactly one of the alternative coordinate must be filled. Its size must, of - course, still match the size of the data in the specified dimension. - -4. Some quantites indicate a coordinate must be the same size as another quantity - through the property ``coordinateX_same_as``. In this case, the other quantity is - not a coordinate, but their data is related and must be of the same size. - - An example can be found in the ``edge_profiles`` IDS, quantity - ``ggd(itime)/neutral(i1)/velocity(i2)/diamagnetic``. This is a two-dimensional field - for which the first coordinate must be the same as - ``ggd(itime)/neutral(i1)/velocity(i2)/radial``. When the diamagnetic velocity - component is filled, the radial component must be filled as well, and have a - matching size. diff --git a/common/doc_common/using_al.rst b/common/doc_common/using_al.rst deleted file mode 100644 index 91b9f6ab..00000000 --- a/common/doc_common/using_al.rst +++ /dev/null @@ -1,38 +0,0 @@ -Using the Access Layer -====================== - -Making the Access Layer available for use ------------------------------------------ - -To use the Access Layer in your programs, you need to make it available. On the -ITER cluster (SDCC) this is achieved by loading the ``IMAS`` module from the -command line: - -.. code-block:: bash - - module load IMAS - -This will select the default IMAS module. You can also select a specific -version. To see which versions of the IMAS module are available, enter on the -command line: - -.. code-block:: bash - - module avail IMAS - -Please check with your HPC administrators which module you need to load when not working -on SDCC. When you're working with a local installation (see :ref:`Building and -installing the Access Layer`), you can source the installed environment file: - -.. code-block:: bash - :caption: Set environment variables (replace ```` with the folder of your local install) - - source /bin/al_env.sh - - -.. - HLI-specific documentation should include the following items: - - - Code samples for loading the library (import, include, use, etc.) - - Example program printing the Access Layer version used - - Instructions for (compiling if relevant) and running the example program diff --git a/docs/source/user_guide/backends_guide.rst b/docs/source/user_guide/backends_guide.rst index f40dec61..d322ffe1 100644 --- a/docs/source/user_guide/backends_guide.rst +++ b/docs/source/user_guide/backends_guide.rst @@ -166,27 +166,39 @@ Query keys specific for the HDF5 backend The :ref:`HDF5 backend` also recognizes these backend-specific query keys. ``hdf5_compression`` - Data compression is enabled by default. Set ``hdf5_compression=no`` or - ``hdf5_compression=n`` to disable data compression. + Data compression is enabled by default. Set ``hdf5_compression=no`` + to disable data compression. ``hdf5_write_buffering`` - During a `put` operation, 0D and 1D buffers are first - stored in memory. Buffers are flushed at the end of the put. + During a ``put`` operation, 0D and 1D field buffers are first stored in + memory and flushed to HDF5 in a single write at the end of the put. + + This feature is enabled by default. Set ``hdf5_write_buffering=no`` to + disable write buffering (data is written to HDF5 field-by-field as it + arrives, which reduces peak memory usage at the cost of slower writes). + +``hdf5_read_buffering`` + During a get operation, supported 0D and 1D field data read from HDF5 is + buffered in memory before being returned to the caller. Slice and time range + reads, as well as fields with higher dimensionality, are read directly. - This feature is enabled by default. Set ``hdf5_write_buffering=no`` or - ``hdf5_write_buffering=n`` to disable write buffering. + This feature is enabled by default. Set ``hdf5_read_buffering=no`` to + disable read buffering. + +``hdf5_write_cache`` + Set the size (in MiB) of the HDF5 per-dataset chunk cache used during write + operations. The cache is allocated per open dataset, so the aggregate memory + grows with the number of datasets written simultaneously. -``write_cache_option`` - Set the size of the HDF5 chunk cache used during chunked datasets write - operations. Default to 100x1024x1024 bytes (100 MiB). + Default: ``100`` (MiB). Can also be set via the environment variable + ``HDF5_BACKEND_WRITE_CACHE=``. -``read_cache_option`` - Set the size of the HDF5 chunk cache used during chunked datasets read - operations. Default to 5x1024x1024 bytes (5 MiB). +``hdf5_read_cache`` + Set the size (in MiB) of the HDF5 per-dataset chunk cache used during read + operations. -``open_read_only`` - Open master file and IDSs files in read only if ``open_read_only=yes`` or - ``open_read_only=y``, overwriting the files access modes default behavior (see IMAS-5274 for an example use-case). + Default: ``5`` (MiB). Can also be set via the environment variable + ``HDF5_BACKEND_READ_CACHE=``. ``hdf5_debug`` HDF5 debug output is disabled by default. Set ``hdf5_debug=yes`` or diff --git a/docs/source/user_guide/configuration.rst b/docs/source/user_guide/configuration.rst index 97186c06..1d89e89d 100644 --- a/docs/source/user_guide/configuration.rst +++ b/docs/source/user_guide/configuration.rst @@ -10,42 +10,12 @@ variables. This page provides an overview of the available options. Environment variables controlling IMAS-Core behaviour ----------------------------------------------------- -``IMAS_AL_DEFAULT_BACKEND`` [#backend_env]_ - Specify which backend to use by default with open/create methods that do not - pass this information as an argument. Values for this environment - variable correspond to the targeted backend ID, see below table. If not - specified, the MDS+ backend is the default. - - .. csv-table:: Backend IDs - :header-rows: 1 - - Backend, Backend ID - :ref:`ASCII `, 11 - :ref:`MDSplus `, 12 - :ref:`HDF5 `, 13 - :ref:`Memory `, 14 - :ref:`UDA `, 15 - - -``IMAS_AL_FALLBACK_BACKEND`` [#backend_env]_ - Specify a fallback backend to be tried if opening the given data-entry was - not successful with the primary/default backend. Values for this environment - variable correspond to the targeted backend ID, see above table. If not - specified, no secondary attempt will be made. This does not have any effect - on calls to create new dataentries. - - ``IMAS_AL_DISABLE_OBSOLESCENT_WARNING`` Since version 4.10.0, all interfaces print warnings when putting an IDS that contains data in fields marked as `obsolescent` in the DD. Setting this variable to ``1`` disables these printouts. -.. [#backend_env] These environment variables are not applicable when using - Data entry URIs (see :doc:`../user_guide/uris_guide`), which explicitly specify the backend. Also not applicable - in the Pythonens-user API. - - ``IMAS_LOCAL_HOSTS`` If you have a UDA server on a site where users have direct access to the IMAS data files, the IMAS-Core backend can decide to use a direct file access (via the relevant backend, e.g. HDF5) @@ -97,7 +67,7 @@ Backend specific environment variables ``HDF5_BACKEND_WRITE_CACHE`` [#uri_precedence]_ - Specify the size of the write cache in MB (default is 5). It may improve + Specify the size of the write cache in MB (default is 100). It may improve writing performance at the cost of increased memory consumption. Obtained performance and best size of cache is heavily depending on the data. diff --git a/pyproject.toml b/pyproject.toml index e0ce230c..68165362 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,7 @@ AL_BACKEND_UDA = { env = "AL_BACKEND_UDA", default = "ON" } AL_BACKEND_MDSPLUS = { env = "AL_BACKEND_MDSPLUS", default = "OFF" } AL_BUILD_MDSPLUS_MODELS = { env = "AL_BUILD_MDSPLUS_MODELS", default = "OFF" } AL_PYTHON_BINDINGS = "ON" +AL_USE_INSTALLED_CORE = { env = "AL_USE_INSTALLED_CORE", default = "OFF" } DOWNLOAD_DEPENDENCIES = { env = "DOWNLOAD_DEPENDENCIES", default = "ON" } DD_GIT_REPOSITORY = { env = "DD_GIT_REPOSITORY", default = "EMPTY" } DD_VERSION = { env = "DD_VERSION", default = "EMPTY" } diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 528e33f6..42341ece 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -8,6 +8,14 @@ python_add_library(al_defs MODULE ${al_defs_source} WITH_SOABI) target_link_libraries(_al_lowlevel PRIVATE Python::NumPy al) target_link_libraries(al_defs PRIVATE Python::NumPy al) +if(AL_USE_INSTALLED_CORE) + # libal is provided externally (e.g. the libimas-core conda package). + # Install only the Python extensions — the dynamic linker resolves libal + # at load time via the host's normal library search path / rpath. + install(TARGETS _al_lowlevel al_defs DESTINATION imas_core) + return() +endif() + # Handling RPATH in macOS: if(APPLE) set_target_properties(al PROPERTIES INSTALL_RPATH "@loader_path") diff --git a/python/tests/test_hdf5_permissions.py b/python/tests/test_hdf5_permissions.py new file mode 100644 index 00000000..424937d3 --- /dev/null +++ b/python/tests/test_hdf5_permissions.py @@ -0,0 +1,230 @@ +"""Regression test for issue #40. + +When master.h5 is world-writable but the IDS file (e.g. equilibrium.h5) is +read-only for the current user, reading used to fail with: + + ALBackendException = Unable to open HDF5 group: equilibrium + +This was because openMasterFile opened master.h5 in H5F_ACC_RDWR (because +the user had write access), and then H5Gopen2 tried to follow the external +link in RDWR mode — conflicting with the RDONLY handle for the IDS file. +""" + +import os + +import numpy as np +import pytest + +import imas_core + +ll = imas_core._al_lowlevel +defs = imas_core.al_defs + + +def _hdf5_available() -> bool: + """Return True if a working HDF5 backend is available.""" + uri = f"imas:hdf5?path=dummy" + try: + imas_core.exception.raise_error_flag = True + status, ctx = ll.al_begin_dataentry_action(uri, defs.OPEN_PULSE) + imas_core.exception.raise_error_flag = False + return True + except imas_core.exception.ImasCoreBackendException as iex: + if "not available" in str(iex.message): + return False + pass + + +pytestmark = pytest.mark.skipif( + not _hdf5_available(), reason="HDF5 backend not available" +) + + +def _write_ids(tmpdir: str) -> None: + """Create an HDF5 pulse and write a minimal equilibrium IDS.""" + uri = f"imas:hdf5?path={tmpdir}" + status, ctx = ll.al_begin_dataentry_action(uri, defs.CREATE_PULSE) + assert status == 0, f"Failed to create pulse: status={status}" + + status, opctx = ll.al_begin_global_action(ctx, "equilibrium", defs.WRITE_OP) + assert status == 0, f"Failed to begin write action: status={status}" + + ll.al_write_data(opctx, "ids_properties/homogeneous_time", "", 2) + ll.al_write_data(opctx, "ids_properties/comment", "", "TEST") + ll.al_end_action(opctx) + ll.al_close_pulse(ctx, defs.CLOSE_PULSE) + + +def _read_ids(tmpdir: str) -> None: + """Open the HDF5 pulse and read back the equilibrium IDS.""" + uri = f"imas:hdf5?path={tmpdir}" + status, ctx = ll.al_begin_dataentry_action(uri, defs.OPEN_PULSE) + assert status == 0, f"Failed to open pulse: status={status}" + + status, opctx = ll.al_begin_global_action(ctx, "equilibrium", defs.READ_OP) + assert status == 0, f"Failed to begin read action: status={status}" + + result = ll.al_read_data(opctx, "ids_properties/comment", "", defs.CHAR_DATA, 1) + assert result == (0,"TEST"), f"Failed to read back comment: {result}" + + ll.al_end_action(opctx) + ll.al_close_pulse(ctx, defs.CLOSE_PULSE) + return result + + +def test_read_with_writable_master_and_readonly_ids(tmp_path): + """Reading must succeed when master.h5 is world-writable but IDS is read-only. + + This reproduces issue #40: a user reads an IDS they don't own where the + owner left master.h5 world-writable (e.g. 0o666) but the IDS file is + only readable (e.g. 0o444). + """ + _write_ids(tmp_path) + + master_path = os.path.join(tmp_path, "master.h5") + ids_path = os.path.join(tmp_path, "equilibrium.h5") + + assert os.path.exists(master_path), "master.h5 not created" + assert os.path.exists(ids_path), "equilibrium.h5 not created" + + # Simulate the problematic permission setup from issue #40: + # master is world-writable, IDS is read-only for everyone. + os.chmod(master_path, 0o666) + os.chmod(ids_path, 0o444) + + # This must not raise an exception. + result = _read_ids(tmp_path) + assert result is not None + + +def test_write_to_readonly_ids_raises(tmp_path): + """Writing to a read-only IDS file must fail, not silently delete and + recreate the file with modified permissions. + + Regression test for the side-effect of the issue #40 fix: when master.h5 + is writable but equilibrium.h5 is read-only, a PUT operation must return + an error status, and the file permissions must not change. + """ + _write_ids(tmp_path) + + master_path = os.path.join(tmp_path, "master.h5") + ids_path = os.path.join(tmp_path, "equilibrium.h5") + + os.chmod(master_path, 0o666) # world-writable master + os.chmod(ids_path, 0o444) # read-only IDS + + uri = f"imas:hdf5?path={tmp_path}" + status, ctx = ll.al_begin_dataentry_action(uri, defs.OPEN_PULSE) + assert status == 0 + + # First do a successful read (reproduces the scenario from issue #40) + status, opctx = ll.al_begin_global_action(ctx, "equilibrium", defs.READ_OP) + assert status == 0, f"GET should succeed: status={status}" + result = ll.al_read_data(opctx, "ids_properties/comment", "", defs.CHAR_DATA, 1) + assert result == (0,"TEST") + ll.al_end_action(opctx) + + # Now try to write — must fail (permission denied), not change permissions + status, opctx = ll.al_begin_global_action(ctx, "equilibrium", defs.WRITE_OP) + assert status != 0, ( + "PUT to read-only IDS should fail, but returned status={status}" + ) + + ll.al_close_pulse(ctx, defs.CLOSE_PULSE) + + # Permissions must be unchanged + assert oct(os.stat(ids_path).st_mode & 0o777) == oct(0o444), ( + "equilibrium.h5 permissions were modified during failed write" + ) + + +def test_read_with_readonly_master_and_readonly_ids(tmp_path): + """Reading must also succeed when both master and IDS are read-only.""" + _write_ids(tmp_path) + + master_path = os.path.join(tmp_path, "master.h5") + ids_path = os.path.join(tmp_path, "equilibrium.h5") + + os.chmod(master_path, 0o444) + os.chmod(ids_path, 0o444) + + try: + result = _read_ids(tmp_path) + assert result is not None + finally: + # Restore write permissions so tempdir cleanup works. + os.chmod(master_path, 0o644) + os.chmod(ids_path, 0o644) + + +def test_put_with_readonly_master_and_existing_rw_ids(tmp_path): + """PUT must succeed when master.h5 is read-only but the IDS file already + exists with a link in master and has read-write permissions. + + This represents the case where a user wants to prevent creation of new IDS + types in a data-entry (by making master.h5 read-only) while still allowing + writes to already-linked IDS files. + """ + _write_ids(tmp_path) + + master_path = os.path.join(tmp_path, "master.h5") + ids_path = os.path.join(tmp_path, "equilibrium.h5") + + # master is read-only (no new links can be added), IDS is read-write + os.chmod(master_path, 0o444) + # equilibrium.h5 stays at default writable permissions + + uri = f"imas:hdf5?path={tmp_path}" + status, ctx = ll.al_begin_dataentry_action(uri, defs.OPEN_PULSE) + assert status == 0 + + # PUT must succeed: the link already exists in master and IDS is writable + status, opctx = ll.al_begin_global_action(ctx, "equilibrium", defs.WRITE_OP) + assert status == 0, ( + f"PUT to existing RW IDS should succeed with RDONLY master, " + f"got status={status}" + ) + ll.al_write_data(opctx, "time", "", np.array([1.0, 2.0, 3.0])) + ll.al_end_action(opctx) + ll.al_close_pulse(ctx, defs.CLOSE_PULSE) + + # master permissions must not have changed + assert oct(os.stat(master_path).st_mode & 0o777) == oct(0o444), ( + "master.h5 permissions were modified during write to existing IDS" + ) + + # Restore so tempdir cleanup works + os.chmod(master_path, 0o644) + + +def test_put_with_readonly_master_and_no_ids_link_raises(tmp_path): + """PUT must fail when master.h5 is read-only and the target IDS has no + existing link in master. + + A read-only master prevents adding new external links, so writing to an + IDS type that was never stored in this data-entry must raise an error. + """ + # Only write 'equilibrium'; leave 'core_profiles' absent + _write_ids(tmp_path) + + master_path = os.path.join(tmp_path, "master.h5") + os.chmod(master_path, 0o444) + + uri = f"imas:hdf5?path={tmp_path}" + status, ctx = ll.al_begin_dataentry_action(uri, defs.OPEN_PULSE) + assert status == 0 + + # PUT to a new IDS type must fail: master is RDONLY so no link can be added + status, _ = ll.al_begin_global_action(ctx, "core_profiles", defs.WRITE_OP) + assert status != 0, ( + "PUT to a new IDS type with RDONLY master should fail" + ) + + ll.al_close_pulse(ctx, defs.CLOSE_PULSE) + + # Restore so tempdir cleanup works + os.chmod(master_path, 0o644) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/skbuild.cmake b/skbuild.cmake index facd0058..b19e0964 100644 --- a/skbuild.cmake +++ b/skbuild.cmake @@ -34,19 +34,36 @@ elseif(NOT ${AL_PYTHON_BINDINGS} MATCHES "[Oo][Nn]") message(FATAL_ERROR "AL_PYTHON_BINDINGS=${AL_PYTHON_BINDINGS} not e|editable|no-build-isolation|[Oo][Nn]") endif() -add_custom_command( - TARGET al POST_BUILD - COMMAND - ${CMAKE_COMMAND} -E env +if(AL_USE_INSTALLED_CORE) + # `al` is an ALIAS to an imported target — it is not built in this tree, + # so it cannot carry a POST_BUILD hook. Drive pip wheel from the custom + # target itself, and put it in ALL so `cmake --build` triggers it. + add_custom_target(al-python-bindings ALL + COMMAND + ${CMAKE_COMMAND} -E env CMAKE_ARGS=${CMAKE_ARGS} - SKBUILD_BUILD_DIR=${CMAKE_CURRENT_BINARY_DIR}/{wheel_tag} + SKBUILD_BUILD_DIR=${CMAKE_CURRENT_BINARY_DIR}/{wheel_tag} ${Python_EXECUTABLE} - -m pip wheel + -m pip wheel ${CMAKE_CURRENT_SOURCE_DIR} - ${PIP_OPTIONS} - --wheel-dir ${CMAKE_CURRENT_BINARY_DIR}/dist/ -) -add_custom_target(al-python-bindings DEPENDS al) + ${PIP_OPTIONS} + --wheel-dir ${CMAKE_CURRENT_BINARY_DIR}/dist/ + ) +else() + add_custom_command( + TARGET al POST_BUILD + COMMAND + ${CMAKE_COMMAND} -E env + CMAKE_ARGS=${CMAKE_ARGS} + SKBUILD_BUILD_DIR=${CMAKE_CURRENT_BINARY_DIR}/{wheel_tag} + ${Python_EXECUTABLE} + -m pip wheel + ${CMAKE_CURRENT_SOURCE_DIR} + ${PIP_OPTIONS} + --wheel-dir ${CMAKE_CURRENT_BINARY_DIR}/dist/ + ) + add_custom_target(al-python-bindings DEPENDS al) +endif() set_target_properties( al-python-bindings PROPERTIES DIST_FOLDER ${CMAKE_CURRENT_BINARY_DIR}/dist/) install(CODE "execute_process(COMMAND ${Python_EXECUTABLE} diff --git a/src/hdf5/hdf5_reader.cpp b/src/hdf5/hdf5_reader.cpp index a4a553b9..7c8e9230 100644 --- a/src/hdf5/hdf5_reader.cpp +++ b/src/hdf5/hdf5_reader.cpp @@ -1288,6 +1288,19 @@ int HDF5Reader::readPersistentShapes_Get(Context *ctx, hid_t gid, const std::str new_data_set->open(tensorized_path.c_str(), gid, &dataset_id, 1, nullptr, alconst::integer_data, true, true, dec->getURI()); if (!(new_data_set->dataset_id >= 0)) throw ALBackendException("HDF5Backend: unexpected dataset_id value in HDF5Reader::readPersistentShapes_Get()", LOG); + + { + int shape_rank = new_data_set->getRank(); + int expected_rank = static_cast(current_arrctx_indices.size()) + 1; + if (shape_rank != expected_rank) { + throw ALBackendException( + "HDF5Backend: SHAPE dataset '" + tensorized_path + + "' has rank " + std::to_string(shape_rank) + + " but rank " + std::to_string(expected_rank) + " was expected. ", + + "This may indicate issues in the data production process. " + LOG); + } + } *dataset_id_shapes = dataset_id; int dim = -1; auto hsSelectionReader = std::unique_ptr(new HDF5HsSelectionReader(new_data_set->getRank(), new_data_set->dataset_id, @@ -1325,6 +1338,19 @@ int HDF5Reader::readPersistentShapes_GetSlice(Context *ctx, DataEntryContext *dec = getDataEntryContext(ctx); std::unique_ptr new_data_set(new HDF5DataSetHandler(false, dec->getURI())); new_data_set->open(tensorized_path.c_str(), gid, &dataset_id, 1, nullptr, alconst::integer_data, true, true, dec->getURI()); + + { + int shape_rank = new_data_set->getRank(); + int expected_rank = static_cast(aos_indices.size()) + 1; + if (shape_rank != expected_rank) { + throw ALBackendException( + "HDF5Backend: SHAPE dataset '" + tensorized_path + + "' has rank " + std::to_string(shape_rank) + + " but rank " + std::to_string(expected_rank) + " was expected. ", + + "This may indicate issues in the data production process. " + LOG); + } + } int dim = -1; auto hsSelectionReader = std::unique_ptr(new HDF5HsSelectionReader(new_data_set->getRank(), dataset_id, new_data_set->getDataSpace(), new_data_set->getLargestDims(), alconst::integer_data, aos_indices.size(), &dim)); diff --git a/src/hdf5/hdf5_utils.cpp b/src/hdf5/hdf5_utils.cpp index f945ce62..c51c227b 100644 --- a/src/hdf5/hdf5_utils.cpp +++ b/src/hdf5/hdf5_utils.cpp @@ -6,6 +6,13 @@ #include #include #include +#ifdef _WIN32 +# include // _access() +# define access _access +# define W_OK 2 +#else +# include +#endif #include #include @@ -268,7 +275,15 @@ void HDF5Utils::openIDSFile(OperationContext * ctx, std::string &IDSpulseFile, h } } else { - if (errno == EAGAIN || errno == EACCES || errno == EBUSY) { + // errno after H5Fopen is unreliable: HDF5 may overwrite it + // internally before returning. Use access(W_OK) to check write + // permission explicitly. + if (access(IDSpulseFile.c_str(), W_OK) != 0) { + char error_message[200]; + sprintf(error_message, "Unable to open external file in Read-Write mode for IDS: %s. Permission denied.\n", ctx->getDataobjectName().c_str()); + throw ALBackendException(error_message, LOG); + } + else if (errno == EAGAIN || errno == EBUSY) { char error_message[200]; sprintf(error_message, "Unable to open external file in Read-Write mode for IDS: %s. It might indicate that the file is being currently handled by a writing concurrent process.\n", ctx->getDataobjectName().c_str()); throw ALBackendException(error_message, LOG); @@ -281,8 +296,6 @@ void HDF5Utils::openIDSFile(OperationContext * ctx, std::string &IDSpulseFile, h createIDSFile(ctx, IDSpulseFile, backend_version, IDS_file_id); } - - } } } @@ -649,7 +662,25 @@ hid_t > &opened_IDS_files, std::string & files_directory, std::string & relative opened_IDS_files[IDS_link_name] = IDS_file_id; } - *IDS_group_id = openHDF5Group(ctx->getDataobjectName().c_str(), file_id); + // Detect access mode mismatch: master was opened in RDWR (e.g. because + // master.h5 has world-write bits set), but the IDS file could only be + // opened in RDONLY (the user has no write permission on that file). + // In that case H5Gopen2 on the master would try to follow the external + // link in RDWR mode, conflicting with the already-open RDONLY IDS handle. + // Fix: open the group directly from the IDS file handle, bypassing the + // external link, so the access mode stays consistent. + bool mode_mismatch = false; + if (IDS_file_id > 0) { + unsigned master_intent = 0, ids_intent = 0; + H5Fget_intent(file_id, &master_intent); + H5Fget_intent(IDS_file_id, &ids_intent); + mode_mismatch = (master_intent & H5F_ACC_RDWR) && !(ids_intent & H5F_ACC_RDWR); + } + + if (mode_mismatch) + *IDS_group_id = openHDF5Group(ctx->getDataobjectName().c_str(), IDS_file_id); + else + *IDS_group_id = openHDF5Group(ctx->getDataobjectName().c_str(), file_id); } void HDF5Utils::showStatus(hid_t file_id) {