pax_global_header00006660000000000000000000000064151317502530014514gustar00rootroot0000000000000052 comment=c4034fde84b51f2f8bbb3683feba9157d946581b atlas-0.45.0/000077500000000000000000000000001513175025300127065ustar00rootroot00000000000000atlas-0.45.0/.clang-format000066400000000000000000000115221513175025300152620ustar00rootroot00000000000000--- Language: Cpp AccessModifierOffset: -4 AlignAfterOpenBracket: Align AlignArrayOfStructures: None AlignConsecutiveMacros: None AlignConsecutiveAssignments: Consecutive AlignConsecutiveBitFields: None AlignConsecutiveDeclarations: None AlignEscapedNewlines: Left AlignOperands: Align AlignTrailingComments: true AllowAllArgumentsOnNextLine: true AllowAllConstructorInitializersOnNextLine: true AllowAllParametersOfDeclarationOnNextLine: true AllowShortEnumsOnASingleLine: true AllowShortBlocksOnASingleLine: Never AllowShortCaseLabelsOnASingleLine: false AllowShortFunctionsOnASingleLine: Inline AllowShortLambdasOnASingleLine: All AllowShortIfStatementsOnASingleLine: Never AllowShortLoopsOnASingleLine: false AlwaysBreakAfterDefinitionReturnType: None AlwaysBreakAfterReturnType: None AlwaysBreakBeforeMultilineStrings: true AlwaysBreakTemplateDeclarations: Yes AttributeMacros: - __capability BinPackArguments: true BinPackParameters: true BraceWrapping: AfterCaseLabel: false AfterClass: false AfterControlStatement: Never AfterEnum: true AfterFunction: false AfterNamespace: false AfterObjCDeclaration: false AfterStruct: false AfterUnion: false AfterExternBlock: false BeforeCatch: true BeforeElse: true BeforeLambdaBody: false BeforeWhile: false IndentBraces: false SplitEmptyFunction: true SplitEmptyRecord: true SplitEmptyNamespace: true BreakBeforeBinaryOperators: None BreakBeforeConceptDeclarations: true BreakBeforeBraces: Custom BreakBeforeInheritanceComma: false BreakInheritanceList: BeforeColon BreakBeforeTernaryOperators: true BreakConstructorInitializersBeforeComma: false BreakConstructorInitializers: AfterColon BreakAfterJavaFieldAnnotations: false BreakStringLiterals: true ColumnLimit: 120 CommentPragmas: '^ IWYU pragma:' CompactNamespaces: false ConstructorInitializerAllOnOneLineOrOnePerLine: true ConstructorInitializerIndentWidth: 4 ContinuationIndentWidth: 4 Cpp11BracedListStyle: true DeriveLineEnding: true DerivePointerAlignment: false DisableFormat: false EmptyLineAfterAccessModifier: Never EmptyLineBeforeAccessModifier: LogicalBlock ExperimentalAutoDetectBinPacking: false FixNamespaceComments: true ForEachMacros: - foreach - Q_FOREACH - BOOST_FOREACH IfMacros: - KJ_IF_MAYBE IncludeBlocks: Preserve IncludeCategories: - Regex: '^<.*\.h>' Priority: 1 SortPriority: 0 CaseSensitive: false - Regex: '^<.*' Priority: 2 SortPriority: 0 CaseSensitive: false - Regex: '.*' Priority: 3 SortPriority: 0 CaseSensitive: false IncludeIsMainRegex: '([-_](test|unittest))?$' IncludeIsMainSourceRegex: '' IndentAccessModifiers: false IndentCaseLabels: true IndentCaseBlocks: false IndentGotoLabels: true IndentPPDirectives: None IndentExternBlock: AfterExternBlock IndentRequires: false IndentWidth: 4 IndentWrappedFunctionNames: false InsertTrailingCommas: None JavaScriptQuotes: Leave JavaScriptWrapImports: true KeepEmptyLinesAtTheStartOfBlocks: false LambdaBodyIndentation: Signature MacroBlockBegin: '' MacroBlockEnd: '' MaxEmptyLinesToKeep: 2 NamespaceIndentation: None ObjCBinPackProtocolList: Auto ObjCBlockIndentWidth: 2 ObjCBreakBeforeNestedBlockParam: true ObjCSpaceAfterProperty: false ObjCSpaceBeforeProtocolList: false PenaltyBreakAssignment: 2 PenaltyBreakBeforeFirstCallParameter: 1 PenaltyBreakComment: 300 PenaltyBreakFirstLessLess: 120 PenaltyBreakString: 1000 PenaltyBreakTemplateDeclaration: 10 PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 200 PenaltyIndentedWhitespace: 0 PointerAlignment: Left PPIndentWidth: -1 ReferenceAlignment: Pointer ReflowComments: false ShortNamespaceLines: 1 SortIncludes: CaseSensitive SortJavaStaticImport: Before SortUsingDeclarations: true SpaceAfterCStyleCast: false SpaceAfterLogicalNot: false SpaceAfterTemplateKeyword: true SpaceBeforeAssignmentOperators: true SpaceBeforeCaseColon: false SpaceBeforeCpp11BracedList: false SpaceBeforeCtorInitializerColon: false SpaceBeforeInheritanceColon: true SpaceBeforeParens: ControlStatements SpaceAroundPointerQualifiers: Default SpaceBeforeRangeBasedForLoopColon: true SpaceInEmptyBlock: false SpaceInEmptyParentheses: false SpacesBeforeTrailingComments: 2 SpacesInAngles: Never SpacesInConditionalStatement: false SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInLineCommentPrefix: Minimum: 1 Maximum: -1 SpacesInParentheses: false SpacesInSquareBrackets: false SpaceBeforeSquareBrackets: false BitFieldColonSpacing: Both Standard: Auto StatementAttributeLikeMacros: - Q_EMIT StatementMacros: - Q_UNUSED - QT_REQUIRE_VERSION TabWidth: 4 UseCRLF: false UseTab: Never WhitespaceSensitiveMacros: - STRINGIZE - PP_STRINGIZE - BOOST_PP_STRINGIZE - NS_SWIFT_NAME - CF_SWIFT_NAME ... atlas-0.45.0/.github/000077500000000000000000000000001513175025300142465ustar00rootroot00000000000000atlas-0.45.0/.github/ci-config.yml000066400000000000000000000001551513175025300166300ustar00rootroot00000000000000dependencies: | ecmwf/ecbuild ecmwf/eckit ecmwf/fckit dependency_branch: develop parallelism_factor: 8 atlas-0.45.0/.github/ci-hpc-config.yml000066400000000000000000000011661513175025300174030ustar00rootroot00000000000000matrix: - mpi_on - mpi_off mpi_on: build: modules: - ninja modules_package: - atlas:fftw,eigen,qhull,python3/3.12,openmpi - fckit:python3/3.12,openmpi - eckit:openmpi dependencies: - ecmwf/ecbuild@develop - ecmwf/eckit@develop - ecmwf/fckit@develop parallel: 64 ntasks: 16 env: - ATLAS_FPE=0 mpi_off: build: modules: - ninja modules_package: - atlas:fftw,eigen,qhull,python3/3.12 - fckit:python3/3.12 dependencies: - ecmwf/ecbuild@develop - ecmwf/eckit@develop - ecmwf/fckit@develop parallel: 64 atlas-0.45.0/.github/docs-config.yml000066400000000000000000000012711513175025300171650ustar00rootroot00000000000000# note: each step is executed in own process build-steps: - git clone --depth 1 https://github.com/ecmwf/atlas-docs.git $RUNNER_TEMP/atlas-docs - | wget https://github.com/doxygen/doxygen/archive/refs/tags/Release_1_13_2.tar.gz tar xzf Release_1_13_2.tar.gz cmake -S doxygen-Release_1_13_2 -B doxygen cmake --build doxygen -j16 sudo cmake --install doxygen - sudo apt install -y -q ghostscript texlive-full - | cd $RUNNER_TEMP/atlas-docs make PUBLIC=1 WITH_ECKIT=1 WITH_DOXYGEN=1 ATLAS_SOURCE_DIR=$GITHUB_WORKSPACE clean html echo "DOC_BUILD_PATH=$RUNNER_TEMP/atlas-docs/build/html" >> "$GITHUB_ENV" hosts: ecmwf-sites: space: docs name: atlas atlas-0.45.0/.github/static-analyzer-skip.txt000066400000000000000000000003401513175025300210620ustar00rootroot00000000000000# This file can be used to suppress warnings # For syntax, see https://codechecker.readthedocs.io/en/latest/analyzer/user_guide/#skip-file # Avoid warnings from external pocket_hdronly.h -*/atlas/linalg/fft/pocketfft.cc atlas-0.45.0/.github/workflows/000077500000000000000000000000001513175025300163035ustar00rootroot00000000000000atlas-0.45.0/.github/workflows/build-hpc.yml000066400000000000000000000177171513175025300207120ustar00rootroot00000000000000name: build-hpc # Controls when the action will run on: # Trigger the workflow on all pushes to main and develop, except on tag creation push: branches: - main - develop - ci tags-ignore: - '**' # Trigger the workflow on all pull requests pull_request: ~ # Allow workflow to be dispatched on demand workflow_dispatch: ~ # Trigger after public PR approved for CI pull_request_target: types: [labeled] env: ATLAS_TOOLS: ${{ github.workspace }}/tools CTEST_PARALLEL_LEVEL: 1 CACHE_SUFFIX: v1 # Increase to force new cache to be created jobs: ci-hpc: name: ci-hpc if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} strategy: fail-fast: false # false: try to complete all jobs matrix: name: - ac-gpu nvhpc include: - name: ac-gpu nvhpc site: ac-batch troika_user_secret: HPC_CI_GPU_SSH_USER sbatch_options: | #SBATCH --time=00:20:00 #SBATCH --nodes=1 #SBATCH --ntasks=4 #SBATCH --cpus-per-task=32 #SBATCH --gpus-per-task=1 #SBATCH --mem=200G #SBATCH --qos=dg modules: - cmake - ninja - prgenv/nvidia - hpcx-openmpi/2.14.0-cuda - fftw - qhull - eigen cmake_options: -DMPI_SLOTS=4 -DENABLE_WARNING_AS_ERROR=ON -DENABLE_PMR=OFF # ENABLE_PMR=OFF because the default host-compiler (gcc 8.5) is not std::pmr capable runs-on: [self-hosted, linux, hpc] env: GH_TOKEN: ${{ github.token }} steps: - uses: ecmwf/reusable-workflows/ci-hpc-generic@v2 with: site: ${{ matrix.site }} troika_user: ${{ secrets[matrix.troika_user_secret] }} sbatch_options: ${{ matrix.sbatch_options }} output_dir: ${{ matrix.output_dir || '' }} workdir: ${{ matrix.workdir || '' }} template_data: | cmake_options: - -DENABLE_MPI=ON - -DENABLE_ACC=ON - -DENABLE_CUDA=ON - -DSKIP_TEST_atlas_test_field_foreach=TRUE - ${{ matrix.cmake_options || '' }} ctest_options: ${{ matrix.ctest_options || '' }} dependencies: ecmwf/ecbuild: version: develop ecmwf/eckit: version: develop cmake_options: - -DENABLE_MPI=ON - -DENABLE_CUDA=OFF - -DENABLE_TESTS=OFF - -DENABLE_ECKIT_SQL=OFF - -DENABLE_ECKIT_CMD=OFF - -DENABLE_AIO=OFF - -DENABLE_ECKIT_GEO=OFF - -DENABLE_WARNINGS=OFF - ${{ matrix.cmake_options || '' }} ecmwf/fckit: version: develop cmake_options: - -DENABLE_TESTS=OFF - ${{ matrix.cmake_options || '' }} ecmwf-ifs/fiat: version: develop cmake_options: - -DENABLE_MPI=ON - -DENABLE_TESTS=OFF - ${{ matrix.cmake_options || '' }} ecmwf-ifs/ectrans: version: develop cmake_options: - -DENABLE_MPI=ON - -DENABLE_ACC=ON - -DENABLE_GPU=ON - -DENABLE_TESTS=OFF - ${{ matrix.cmake_options || '' }} template: | set +x module_load() { echo "+ module load $1" module load $1 } {% for module in "${{ join(matrix.modules, ',') }}".split(',') %} module_load {{module}} {% endfor %} echo "+ module list" module list BASEDIR=$PWD export CMAKE_TEST_LAUNCHER="srun;-n;1" export CMAKE_PREFIX_PATH=$BASEDIR/install:$CMAKE_PREFIX_PATH {% for repo_name, options in dependencies.items() %} name=$(basename {{repo_name}}) echo "::group::Get dependency $name" echo "+ mkdir -p $name" mkdir -p $name echo "+ pushd $name" pushd $name echo "+ git init" git init echo "+ git remote add origin ${{ github.server_url }}/{{repo_name}}" git remote add origin ${{ github.server_url }}/{{repo_name}} echo "+ git fetch origin {{options['version']}}" git fetch origin {{options['version']}} echo "+ git reset --hard FETCH_HEAD" git reset --hard FETCH_HEAD echo "+ cmake -G Ninja -S . -B build {{ options['cmake_options']|join(' ') }}" cmake -G Ninja -S . -B build {{ options['cmake_options']|join(' ') }} start=`date +%s` echo "+ cmake --build build" cmake --build build end=`date +%s` runtime=$((end-start)) echo "Build $name took $runtime seconds" echo "+ cmake --install build --prefix \$BASEDIR/install/$name" cmake --install build --prefix $BASEDIR/install/$name echo "+ export PATH=\$BASEDIR/install/$name/bin:\$PATH" export PATH=$BASEDIR/install/$name/bin:$PATH echo "+ popd" popd echo "::endgroup::" {% endfor %} REPO=${{ github.event.pull_request.head.repo.full_name || github.repository }} SHA=${{ github.event.pull_request.head.sha || github.sha }} name=$(basename $REPO) echo "::group::Checkout $name" echo "+ mkdir -p $name" mkdir -p $name echo "+ pushd $name" pushd $name echo "+ git init" git init echo "+ git remote add origin ${{ github.server_url }}/$REPO" git remote add origin ${{ github.server_url }}/$REPO echo "+ git fetch origin $SHA" git fetch origin $SHA echo "+git reset --hard FETCH_HEAD" git reset --hard FETCH_HEAD echo "+ popd" popd echo "::endgroup::" echo "::group::Build $name" echo "+ cmake -G Ninja -S $name -B build {{ cmake_options|join(' ') }}" cmake -G Ninja -S $name -B build {{ cmake_options|join(' ') }} start=`date +%s` echo "+ cmake --build build" cmake --build build end=`date +%s` runtime=$((end-start)) echo "Build $name took $runtime seconds" echo "::endgroup::" echo "::group::Test $name" export ATLAS_FINALISES_MPI=1 export ECKIT_MPI_WORKAROUND_SEGV_AT_EXIT=1 echo "+ ctest --test-dir build --output-on-failure {{ ctest_options }}" ctest --test-dir build --output-on-failure {{ ctest_options }} echo "::endgroup::" echo "::group::Install $name" echo "+ cmake --install build --prefix $BASEDIR/install/$name" cmake --install build --prefix $BASEDIR/install/$name echo "+ export PATH=\$BASEDIR/install/$name/bin:\$PATH" export PATH=$BASEDIR/install/$name/bin:$PATH echo "::endgroup::" echo "::group::Verify $name installation" echo "+ atlas --info" atlas --info echo "::endgroup::" echo "::group::Cleanup" set -x rm -r $name rm -r build rm -r $BASEDIR/install {% for repo_name in dependencies.keys() %} name=$(basename {{repo_name}}) rm -r $name {% endfor %} echo "::endgroup::" set +x atlas-0.45.0/.github/workflows/build-wheel-wrapper.yml000066400000000000000000000021761513175025300227130ustar00rootroot00000000000000# (C) Copyright 2024- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. name: Build Python Wrapper Wheel on: # Trigger the workflow manually workflow_dispatch: inputs: use_test_pypi: description: Use test pypi instead of the regular one required: false type: boolean default: false # Allow to be called from another workflow -- eg `cd.yml` workflow_call: inputs: use_test_pypi: description: Use test pypi instead of the regular one required: false type: boolean default: false jobs: python-wrapper-wheel: name: Python Wrapper Wheel uses: ecmwf/reusable-workflows/.github/workflows/python-wrapper-wheel.yml@main with: use_test_pypi: ${{ inputs.use_test_pypi }} wheel_directory: python/atlaslib-ecmwf secrets: inherit atlas-0.45.0/.github/workflows/build.yml000066400000000000000000000224651513175025300201360ustar00rootroot00000000000000name: build # Controls when the action will run on: # Trigger the workflow on all pushes, except on tag creation push: branches: - '**' tags-ignore: - '**' # Trigger the workflow on all pull requests pull_request: ~ # Allow workflow to be dispatched on demand workflow_dispatch: ~ env: ATLAS_TOOLS: ${{ github.workspace }}/tools CTEST_PARALLEL_LEVEL: 1 CACHE_SUFFIX: v1 jobs: ci: name: ci strategy: fail-fast: false # false: try to complete all jobs matrix: build_type: [Release,Debug] name: - linux gnu-10 - linux gnu-12 - linux gnu-14 - linux clang-13 - linux clang-18 - linux intel - linux intel-classic - linux nvhpc-25.1 - macos include: - name: linux gnu-10 os: ubuntu-22.04 compiler: gnu-10 compiler_cc: gcc-10 compiler_cxx: g++-10 compiler_fc: gfortran-10 caching: true coverage: true - name: linux gnu-12 os: ubuntu-24.04 compiler: gnu-12 compiler_cc: gcc-12 compiler_cxx: g++-12 compiler_fc: gfortran-12 caching: true coverage: false - name: linux gnu-14 os: ubuntu-24.04 compiler: gnu-14 compiler_cc: gcc-14 compiler_cxx: g++-14 compiler_fc: gfortran-14 caching: true coverage: false - name: linux clang-13 os: ubuntu-22.04 compiler: clang-13 compiler_cc: clang-13 compiler_cxx: clang++-13 compiler_fc: gfortran-10 caching: true coverage: false - name: linux clang-18 os: ubuntu-24.04 compiler: clang-18 compiler_cc: clang-18 compiler_cxx: clang++-18 compiler_fc: gfortran-14 caching: true coverage: false - name : linux intel os: ubuntu-24.04 compiler: intel compiler_cc: icx compiler_cxx: icpx compiler_fc: ifx caching: true coverage: false - name : linux intel-classic os: ubuntu-22.04 compiler: intel-classic compiler_cc: icc compiler_cxx: icpc compiler_fc: ifort caching: true coverage: false - name: linux nvhpc-25.1 os: ubuntu-24.04 compiler: nvhpc-25.1 compiler_cc: nvc compiler_cxx: nvc++ compiler_fc: nvfortran caching: true coverage: false cmake_options: -DSKIP_TEST_atlas_test_omp_sort=TRUE - name: macos # Xcode compiler requires empty environment variables, so we pass null (~) here os: macos-14 compiler: clang-15 compiler_cc: ~ compiler_cxx: ~ compiler_fc: gfortran-13 caching: true coverage: false cmake_options: -DMPI_SLOTS=4 runs-on: ${{ matrix.os }} steps: - name: Checkout Repository uses: actions/checkout@v2 - name: Set up Homebrew id: set-up-homebrew uses: Homebrew/actions/setup-homebrew@master - name: Environment run: | echo "DEPS_DIR=${{ runner.temp }}/deps" >> $GITHUB_ENV echo "CC=${{ matrix.compiler_cc }}" >> $GITHUB_ENV echo "CXX=${{ matrix.compiler_cxx }}" >> $GITHUB_ENV echo "FC=${{ matrix.compiler_fc }}" >> $GITHUB_ENV if [[ "${{ matrix.os }}" =~ macos ]]; then export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 export HOMEBREW_NO_AUTO_UPDATE=1 export HOMEBREW_NO_INSTALL_CLEANUP=1 echo "HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1" >> $GITHUB_ENV echo "HOMEBREW_NO_AUTO_UPDATE=1" >> $GITHUB_ENV echo "HOMEBREW_NO_INSTALL_CLEANUP=1" >> $GITHUB_ENV brew install ninja brew install libomp else sudo apt-get update if [[ "${{ matrix.compiler }}" =~ gnu ]]; then sudo apt-get install -y libomp5 libomp-dev fi sudo apt-get install ninja-build fi printenv - name: Free Disk Space (Ubuntu) # Free up disk space for nvhpc uses: jlumbroso/free-disk-space@main if: contains( matrix.compiler, 'nvhpc' ) continue-on-error: true with: # this might remove tools that are actually needed, # if set to "true" but frees about 6 GB tool-cache: false # all of these default to true, but feel free to set to # "false" if necessary for your workflow android: true dotnet: true haskell: true large-packages: false # takes too long docker-images: false # takes too long swap-storage: true - name: Retrieve cached dependencies if: matrix.caching id: deps-restore-cache uses: actions/cache/restore@v4 with: path: ${{ env.DEPS_DIR }} key: deps-${{ matrix.os }}-${{ matrix.compiler }}-${{ env.CACHE_SUFFIX }} - name: Install NVHPC compiler if: contains( matrix.compiler, 'nvhpc' ) shell: bash -eux {0} run: | ${ATLAS_TOOLS}/install-nvhpc.sh --prefix ${DEPS_DIR}/nvhpc --version 25.1 source ${DEPS_DIR}/nvhpc/env.sh echo "${NVHPC_DIR}/compilers/bin" >> $GITHUB_PATH [ -z ${MPI_HOME+x} ] || echo "MPI_HOME=${MPI_HOME}" >> $GITHUB_ENV - name: Install Intel oneAPI compiler if: contains( matrix.compiler, 'intel' ) run: | ${ATLAS_TOOLS}/install-intel-oneapi.sh source /opt/intel/oneapi/setvars.sh printenv >> $GITHUB_ENV echo "CACHE_SUFFIX=$CC-$($CC -dumpversion)" >> $GITHUB_ENV - name: Install MPI shell: bash -eux {0} run: | FCFLAGS=-fPIC CFLAGS=-fPIC FFLAGS=-fPIC ${ATLAS_TOOLS}/install-mpi.sh --mpi openmpi --prefix ${DEPS_DIR}/openmpi if [[ "${{ matrix.os }}" =~ macos ]]; then echo "CACHE_SUFFIX=${CACHE_SUFFIX}-mpi_$(mpirun --version | head -1 | awk '{print $4}')" >> $GITHUB_ENV fi [ -f ${DEPS_DIR}/openmpi/env.sh ] && source ${DEPS_DIR}/openmpi/env.sh [ -z ${MPI_HOME+x} ] || echo "MPI_HOME=${MPI_HOME}" >> $GITHUB_ENV - name: Install FFTW shell: bash -eux {0} run: | ${ATLAS_TOOLS}/install-fftw.sh --version 3.3.10 --prefix ${DEPS_DIR}/fftw echo "FFTW_ROOT=${DEPS_DIR}/fftw" >> $GITHUB_ENV - name: Install Qhull if: "!contains( matrix.compiler, 'intel-classic' )" shell: bash -eux {0} run: | ${ATLAS_TOOLS}/install-qhull.sh --version 8.1-alpha3 --prefix ${DEPS_DIR}/qhull echo "Qhull_ROOT=${DEPS_DIR}/qhull" >> $GITHUB_ENV - name: Install LZ4 if: "!contains( matrix.compiler, 'nvhpc' )" run: | brew install lz4 echo "LZ4_ROOT=$(brew --prefix lz4)" >> $GITHUB_ENV - name: Install Eigen if: "!contains( matrix.compiler, 'nvhpc' )" run: | brew install eigen@3 echo "Eigen3_ROOT=$(brew --prefix eigen@3)" >> $GITHUB_ENV - name: Save cached dependencies # There seems to be a problem with cached NVHPC dependencies, leading to SIGILL perhaps due to slightly different architectures if: matrix.caching && matrix.build_type == 'Debug' && github.ref_name == github.event.repository.default_branch id: deps-save-cache uses: actions/cache/save@v4 with: path: ${{ env.DEPS_DIR }} key: ${{ steps.deps-restore-cache.outputs.cache-primary-key }} - name: Set Build & Test Environment run: | # Only add --oversubscribe arg for openmpi [ -z ${I_MPI_ROOT+x} ] && echo "ATLAS_RUN_MPI_ARGS=--oversubscribe" >> $GITHUB_ENV # Add mpirun to path for testing [ -z ${MPI_HOME+x} ] || echo "${MPI_HOME}/bin" >> $GITHUB_PATH - name: Build & Test id: build-test uses: ecmwf/build-package@v2 with: self_coverage: ${{ matrix.coverage }} force_build: true cache_suffix: "${{ matrix.build_type }}-${{ env.CACHE_SUFFIX }}" recreate_cache: ${{ matrix.caching == false }} dependencies: | ecmwf/ecbuild ecmwf/eckit ecmwf/fckit dependency_branch: develop dependency_cmake_options: | ecmwf/eckit: "-G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DENABLE_TESTS=OFF -DENABLE_ECKIT_CMD=OFF -DENABLE_ECKIT_SQL=OFF -DENABLE_WARNINGS=OFF" ecmwf/fckit: "-G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DENABLE_TESTS=OFF" cmake_options: "-G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DATLAS_BITS_LOCAL=64 -DENABLE_WARNING_AS_ERROR=ON -DENABLE_PMR=ON ${{ matrix.cmake_options }}" ctest_options: "${{ matrix.ctest_options }}" - name: Verify tools run: | export PATH=${{ steps.build-test.outputs.bin_path }}:$PATH echo "+ atlas --info" atlas --info echo "+ atlas-grids --list" atlas-grids --list - name: Codecov Upload if: steps.build-test.outputs.coverage_file uses: codecov/codecov-action@v2 with: files: ${{ steps.build-test.outputs.coverage_file }} atlas-0.45.0/.github/workflows/cd.yml000066400000000000000000000003621513175025300174150ustar00rootroot00000000000000name: cd on: push: tags: - '**' jobs: deploy: uses: ecmwf/reusable-workflows/.github/workflows/create-package.yml@v2 secrets: inherit wheel: uses: ./.github/workflows/build-wheel-wrapper.yml secrets: inherit atlas-0.45.0/.github/workflows/check-release-version.yml000066400000000000000000000003071513175025300232040ustar00rootroot00000000000000name: Check VERSION file on: push: branches: - "release/**" - "hotfix/**" jobs: check_version: uses: ecmwf/reusable-workflows/.github/workflows/check-release-version.yml@v2 atlas-0.45.0/.github/workflows/ci.yml000066400000000000000000000064001513175025300174210ustar00rootroot00000000000000name: ci on: # Trigger the workflow on push to master or develop, except tag creation push: branches: - 'master' - 'develop' - 'ci' tags-ignore: - '**' # Trigger the workflow on pull request pull_request: ~ # Trigger the workflow manually workflow_dispatch: ~ # Trigger after public PR approved for CI pull_request_target: types: [labeled] jobs: # Run CI including downstream packages on self-hosted runners downstream-ci: name: downstream-ci if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} uses: ecmwf/downstream-ci/.github/workflows/downstream-ci.yml@main with: atlas: ecmwf/atlas@${{ github.event.pull_request.head.sha || github.sha }} secrets: inherit # Run CI of private downstream packages on self-hosted runners private-downstream-ci: name: private-downstream-ci needs: [downstream-ci] if: ${{ (success() || failure()) && (!github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }} runs-on: ubuntu-latest permissions: pull-requests: write steps: - name: Dispatch private downstream CI uses: ecmwf/dispatch-private-downstream-ci@v1 with: token: ${{ secrets.GH_REPO_READ_TOKEN }} owner: ecmwf repository: private-downstream-ci event_type: downstream-ci payload: '{"atlas": "ecmwf/atlas@${{ github.event.pull_request.head.sha || github.sha }}"}' # Build downstream packages on HPC downstream-ci-hpc: name: downstream-ci-hpc if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} uses: ecmwf/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main with: atlas: ecmwf/atlas@${{ github.event.pull_request.head.sha || github.sha }} secrets: inherit # Run CI of private downstream packages on HPC private-downstream-ci-hpc: name: private-downstream-ci-hpc needs: [downstream-ci-hpc] if: ${{ (success() || failure()) && (!github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }} runs-on: ubuntu-latest permissions: pull-requests: write steps: - name: Dispatch private downstream CI uses: ecmwf/dispatch-private-downstream-ci@v1 with: token: ${{ secrets.GH_REPO_READ_TOKEN }} owner: ecmwf repository: private-downstream-ci event_type: downstream-ci-hpc payload: '{"atlas": "ecmwf/atlas@${{ github.event.pull_request.head.sha || github.sha }}"}' notify: runs-on: ubuntu-latest needs: - downstream-ci - private-downstream-ci - downstream-ci-hpc - private-downstream-ci-hpc if: ${{ always() && !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} steps: - name: Trigger Teams notification uses: ecmwf/notify-teams@v1 with: incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} needs_context: ${{ toJSON(needs) }} atlas-0.45.0/.github/workflows/docs.yml000066400000000000000000000003421513175025300177550ustar00rootroot00000000000000name: docs on: push: tags: - '**' workflow_dispatch: ~ jobs: publish: uses: ecmwf/reusable-workflows/.github/workflows/cd-docs.yml@v2 secrets: inherit with: config: .github/docs-config.yml atlas-0.45.0/.github/workflows/label-public-pr.yml000066400000000000000000000003431513175025300220000ustar00rootroot00000000000000# Manage labels of pull requests that originate from forks name: label-public-pr on: pull_request_target: types: [opened, synchronize] jobs: label: uses: ecmwf/reusable-workflows/.github/workflows/label-pr.yml@v2 atlas-0.45.0/.github/workflows/notify-new-issue.yml000066400000000000000000000004251513175025300222540ustar00rootroot00000000000000name: Notify new issue on: issues: types: - "opened" jobs: notify: runs-on: ubuntu-latest steps: - name: Notify new issue uses: ecmwf/notify-teams-issue@v1 with: incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} atlas-0.45.0/.github/workflows/notify-new-pr.yml000066400000000000000000000005361513175025300215500ustar00rootroot00000000000000name: Notify new PR # Needs the worklow to be located in the branche the PR is merged to on: pull_request_target: types: - "opened" jobs: notify: runs-on: ubuntu-latest steps: - name: Notify new PR uses: ecmwf/notify-teams-pr@v1 with: incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} atlas-0.45.0/.github/workflows/static-analyzer.yml000066400000000000000000000126471513175025300221520ustar00rootroot00000000000000name: static-analyzer on: pull_request: types: [opened, synchronize, reopened, closed] jobs: build: if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed' || github.event_name == 'push'}} runs-on: ubuntu-latest outputs: artifact-id: ${{ steps.upload-sareport.outputs.artifact-id }} steps: - name: Checkout atlas uses: actions/checkout@v4 with: path: atlas - name: Checkout ecbuild uses: actions/checkout@v4 with: repository: ecmwf/ecbuild path: ecbuild ref: 'develop' - name: Checkout eckit uses: actions/checkout@v4 with: repository: ecmwf/eckit path: eckit ref: 'develop' - name: Checkout fckit uses: actions/checkout@v4 with: repository: ecmwf/fckit path: fckit ref: 'develop' - name: Install dependencies run: | python -m pip install CodeChecker - name: Generate CMakeLists.txt shell: bash # default on ubuntu-latest run: | # Write the file atomically; fail if anything goes wrong cat > CMakeLists.txt <<'CML' cmake_minimum_required(VERSION 3.18 FATAL_ERROR) #################################################################### macro(ecbundle_add_project package_name) # # add_subdirectory depending on BUILD_${package_name} # set(BUILD_${package_name} ON CACHE BOOL "") if(BUILD_${package_name}) set(dir ${ARGV1}) if(NOT dir) set(dir ${package_name}) endif() add_subdirectory(${dir}) endif() endmacro() macro(ecbundle_set key value) set(${key} ${value} CACHE STRING "") if("${${key}}" STREQUAL "${value}") message(" - ${key} = ${value}") else() message(" - ${key} = ${${key}} [default=${value}]") endif() endmacro() #################################################################### message(" - source : ${CMAKE_CURRENT_SOURCE_DIR}") message(" - build : ${CMAKE_CURRENT_BINARY_DIR}") message(" - install : ${CMAKE_INSTALL_PREFIX}") message(" - build type : ${CMAKE_BUILD_TYPE}") message("Bundle variables set for this build:") ecbundle_set(CMAKE_EXPORT_COMPILE_COMMANDS ON) ecbundle_set(ENABLE_ECKIT_CMD OFF) ecbundle_set(ENABLE_ECKIT_SQL OFF) ecbundle_set(ENABLE_ECKIT_GEO OFF) ecbundle_set(ENABLE_OMP OFF) ecbundle_set(ECKIT_ENABLE_TESTS OFF) ecbundle_set(FCKIT_ENABLE_TESTS OFF) message("") #################################################################### find_package(ecbuild 3.8 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR}/ecbuild) project(bundle VERSION 0.0.0) ## Initialize include(${CMAKE_CURRENT_BINARY_DIR}/init.cmake OPTIONAL) ## Projects ecbundle_add_project(ecbuild) ecbundle_add_project(eckit) ecbundle_add_project(fckit) ecbundle_add_project(atlas) ## Finalize include(${CMAKE_CURRENT_BINARY_DIR}/final.cmake OPTIONAL) ecbuild_install_project(NAME ${PROJECT_NAME}) ecbuild_print_summary() CML - name: Build sources run: | mkdir build cd build cmake -GNinja .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ninja - name: Analyze sources run: | jq '[ .[] | select(.file | test(".*/atlas/src/.*") ) ]' build/compile_commands.json > atlas_compile_commands.json jq '[ .[] | select(.file | test(".*/pluto/src/.*") ) ]' build/compile_commands.json > pluto_compile_commands.json jq -n --slurpfile atlas atlas_compile_commands.json --slurpfile pluto pluto_compile_commands.json '$atlas[] + $pluto[]' \ > build/filtered_compile_commands.json CodeChecker analyze build/filtered_compile_commands.json -o cc_out --analyzers clangsa -j 4 \ -i atlas/.github/static-analyzer-skip.txt CodeChecker parse -e html cc_out -o ./reports_html --trim-path-prefix ${PWD} || true - name: Archive static-analyzer report id: upload-sareport uses: actions/upload-artifact@v4 with: name: sareport path: reports_html preview-publish: if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed' }} needs: build uses: ecmwf/reusable-workflows/.github/workflows/pr-preview-publish.yml@main with: artifact-id: ${{ needs.build.outputs.artifact-id }} space: docs name: atlas path: static-analyzer link-text: 💣💥☠️ Static Analyzer Report ☠️💥💣 link-tag: STATIC-ANALYSIS secrets: sites-token: ${{ secrets.ECMWF_SITES_TOKEN }} preview-unpublish: if: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' }} uses: ecmwf/reusable-workflows/.github/workflows/pr-preview-unpublish.yml@main with: space: docs name: atlas path: static-analyzer secrets: sites-token: ${{ secrets.ECMWF_SITES_TOKEN }} atlas-0.45.0/.gitignore000077500000000000000000000002071513175025300147000ustar00rootroot00000000000000.tags* CMakeLists.txt.user* *.autosave doc/html doc/latex *.sublime-workspace *.swp .nfs* build/* install/* env.sh *.DS_Store .vscode atlas-0.45.0/.travis.yml000066400000000000000000000212241513175025300150200ustar00rootroot00000000000000 language: cpp cache: directories: - ${HOME}/deps/cmake - ${HOME}/deps/openmpi - ${HOME}/deps/mpich - ${HOME}/deps/eckit - ${HOME}/deps/fckit jobs: include: - os: linux compiler: clang env: - CACHE_NAME=linux-clang38-mpich - CXX_COMPILER='clang++-3.8' C_COMPILER='clang-3.8' Fortran_COMPILER='gfortran' - MPI='mpich' - ATLAS_CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=DEBUG" addons: apt: sources: ['llvm-toolchain-precise', 'ubuntu-toolchain-r-test'] packages: ['clang-3.8', 'gfortran'] - os: linux compiler: gcc env: - CACHE_NAME=linux-gcc5-openmpi - CXX_COMPILER='g++-5' C_COMPILER='gcc-5' Fortran_COMPILER='gfortran-5' - MPI='openmpi' - ATLAS_CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=DEBUG" addons: apt: sources: ['ubuntu-toolchain-r-test'] packages: ['g++-5', 'gcc-5', 'gfortran-5', 'libfftw3-dev'] - os: linux compiler: gcc env: - CACHE_NAME=linux-gcc7-mpich - CXX_COMPILER='g++-7' C_COMPILER='gcc-7' Fortran_COMPILER='gfortran-7' - MPI='mpich' - ATLAS_CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=DEBUG -DENABLE_GPROF=ON" - COVERAGE=ON addons: apt: sources: ['ubuntu-toolchain-r-test'] packages: ['g++-7', 'gcc-7', 'gfortran-7', 'libfftw3-dev', 'lcov'] - os: osx env: - CACHE_NAME=osx-clang-openmpi - CXX_COMPILER='clang++' C_COMPILER='clang' Fortran_COMPILER='gfortran' - MPI=openmpi - ATLAS_CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=DEBUG" - ATLAS_CTEST_OPTIONS="-E atlas_test_stencil_parallel_mpi16" osx_image: xcode12 - os: osx env: - CACHE_NAME=osx-clang-mpich - CXX_COMPILER='clang++' C_COMPILER='clang' Fortran_COMPILER='gfortran' - MPI=mpich osx_image: xcode12 - os: linux compiler: gcc env: - CACHE_NAME=linux-pgi-openmpi - CXX_COMPILER='pgc++' C_COMPILER='pgcc' Fortran_COMPILER='pgfortran' - MPI='openmpi' - PGI_VERSION="CommunityEdition" - ECKIT_CMAKE_OPTIONS="-DENABLE_ECKIT-395=OFF -DRT_LIB=/usr/lib/x86_64-linux-gnu/librt.so -DCURSES_LIBRARY=/usr/lib/x86_64-linux-gnu/libcurses.so -DMPI_ARGS=--oversubscribe" - ATLAS_CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=DEBUG -DMPI_ARGS=--oversubscribe" before_install: ################################################################# # Set compilers ################################################################# - | ### Set compilers export CC=${C_COMPILER} export CXX=${CXX_COMPILER} export FC=${Fortran_COMPILER} - | ### Load scripts source ${TRAVIS_BUILD_DIR}/tools/source-me.sh install: ################################################################# # All dependencies are installed in ${TRAVIS_BUILD_DIR}/deps/ ################################################################# - DEPS_DIR=${HOME}/deps - mkdir -p ${DEPS_DIR} && cd ${DEPS_DIR} - | DEPS_BRANCH="master" if [[ "${TRAVIS_BRANCH}" != "master" ]]; then DEPS_BRANCH="develop" fi ################################################################# # Install Compilers ################################################################# - | ### Install gcc (homebrew) if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then export HOMEBREW_NO_AUTO_UPDATE=1 brew install gcc brew link gcc fi - | ### Install PGI community edition if [[ "${PGI_VERSION:-notset}" == "CommunityEdition" ]]; then install-pgi.sh --prefix ${DEPS_DIR}/pgi source ${DEPS_DIR}/pgi/env.sh fi ################################################################# # Install MPI ################################################################# - | ### Install MPI install-mpi.sh ${MPI} source ${DEPS_DIR}/${MPI}/env.sh echo "${MPI_HOME}" echo "${PATH}" ################################################################# # Install CMake ################################################################# - | ### Install CMake if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then CMAKE_VERSION=3.18.4 if [[ -z "$(ls -A ${DEPS_DIR}/cmake)" ]]; then if [[ "${TRAVIS_ARCH}" == "amd64" ]]; then CMAKE_URL="https://cmake.org/files/v${CMAKE_VERSION%.*}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" mkdir -p ${DEPS_DIR}/cmake && travis_retry wget --no-check-certificate --quiet -O - ${CMAKE_URL} | tar --strip-components=1 -xz -C ${DEPS_DIR}/cmake fi if [[ "${TRAVIS_ARCH}" = "ppc64le" ]]; then # Build from source CMAKE_URL="https://cmake.org/files/v${CMAKE_VERSION%.*}/cmake-${CMAKE_VERSION}.tar.gz" mkdir -p ${DEPS_DIR}/cmake-source && travis_retry wget --quiet -O - ${CMAKE_URL} | tar --strip-components=1 -xz -C ${DEPS_DIR}/cmake-source ( cd ${DEPS_DIR}/cmake-source mkdir -p build && cd build ../bootstrap --prefix=${DEPS_DIR}/cmake && make -j4 install ) fi else echo "CMake already found in cache" fi export PATH=${DEPS_DIR}/cmake/bin:${PATH} else brew upgrade cmake || brew install cmake brew install openssl export OPENSSL_ROOT_DIR=/usr/local/opt/openssl fi cmake --version ################################################################# # Install FFTW ################################################################# # - | # ### Install FFTW # if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then # brew ls --versions fftw || brew install fftw # fi ################################################################# # Install CGAL ################################################################# # - | # ### Install CGAL # if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then # brew ls --versions cgal || brew install cgal # fi ################################################################# # Install ecbuild ################################################################# - | ### Install ecbuild git clone --depth 1 -b ${DEPS_BRANCH} https://github.com/ecmwf/ecbuild ${DEPS_DIR}/ecbuild export PATH=${DEPS_DIR}/ecbuild/bin:${PATH} export ECBUILD_MODULE_PATH=${DEPS_DIR}/ecbuild/cmake ecbuild --version ################################################################# # Install eckit ################################################################# - | ### Install eckit install-dep.sh --repo eckit --branch ${DEPS_BRANCH} --prefix ${DEPS_DIR}/eckit --cmake "-DENABLE_TESTS=OFF -DCMAKE_BUILD_TYPE=DEBUG ${ECKIT_CMAKE_OPTIONS}" - export ECKIT_PATH=${DEPS_DIR}/eckit - ${DEPS_DIR}/eckit/bin/eckit-version ################################################################# # Install fckit ################################################################# - | ### Install fckit install-dep.sh --repo fckit --branch ${DEPS_BRANCH} --prefix ${DEPS_DIR}/fckit --cmake "-DENABLE_TESTS=ON -DCMAKE_BUILD_TYPE=DEBUG" --depends "eckit" - export FCKIT_PATH=${DEPS_DIR}/fckit script: ################################################################# # Environment variables ################################################################# - echo ${CXX} - echo ${CC} - echo ${FC} - echo ${MPI_HOME} - echo ${PATH} - | ATLAS_SOURCE_DIR=${TRAVIS_BUILD_DIR} ATLAS_BUILD_DIR=${TRAVIS_BUILD_DIR}/builds/atlas ################################################################# # Build Atlas ################################################################# - mkdir -p ${ATLAS_BUILD_DIR} && cd ${ATLAS_BUILD_DIR} - cmake -DCMAKE_MODULE_PATH=${ECBUILD_MODULE_PATH} ${ATLAS_CMAKE_OPTIONS} ${ATLAS_SOURCE_DIR} - make -j$(nproc) - bin/atlas --info ################################################################# # Test Atlas ################################################################# - ATLAS_TRACE=1 ATLAS_DEBUG=1 ctest --output-on-failure --timeout 300 ${ATLAS_CTEST_OPTIONS} after_success: - | if [[ "${COVERAGE}" == "ON" ]]; then # Creating report cd ${ATLAS_BUILD_DIR} lcov --directory . --capture --output-file coverage.info # capture coverage info lcov --remove coverage.info '/usr/*' --output-file coverage.info # filter out system lcov --list coverage.info #debug info # Uploading report to CodeCov bash <(curl -s https://codecov.io/bash) -t 9c489980-d292-499c-8615-af02df3b20d1 || echo "Codecov did not collect coverage reports" fi after_failure: - cd ${ATLAS_BUILD_DIR} - ATLAS_TRACE=1 ctest -VV --rerun-failed --timeout 60 ${ATLAS_CTEST_OPTIONS} - cat ecbuild.log atlas-0.45.0/AUTHORS000066400000000000000000000012551513175025300137610ustar00rootroot00000000000000Authors ======= - Willem Deconinck - Pedro Maciel - Tiago Quintino Thanks for contributions from ============================= - Carlos Osuna - Avi Bahra - Andreas Mueller - Baudouin Raoult - Florian Rathgeber - Oliver Lomax - Marek Wlasak - Daniel Holdaway - Daan Degrauwe - Philippe Marguinaud - Slavko Brdar - Gianmarco Mengaldo - Dušan Figala - Benjamin Menetrier - James Hawkes - Mats Hamrud - Rahul Mahajan - Toby Searle - Olivier Iffrig - Christian Kuehnlein - Iain Russell - Marco Milan - Daniel Tipping - Domokos Sármány - Lorenzo Milazzo - Francois Hebert - Yannick Trémolet - Mark J. Olah - Sam Hatfield - Peter Bispham - Paul Cresswell - Steven Vahl - Michael Lange atlas-0.45.0/CHANGELOG.md000066400000000000000000001007611513175025300145240ustar00rootroot00000000000000Changelog ========= All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] ## [0.45.0] - 2026-01-14 ### Fixed - Fixes and workaround failing atlas_fctest_functionspace when fckit is compiled with -Mnotarget_temps with NVHPC by @wdeconinck in https://github.com/ecmwf/atlas/pull/332 - Pluto fixes and improvements by @wdeconinck in https://github.com/ecmwf/atlas/pull/335 - [pluto] Fix installation of missing header - FunctionSpace::scatter sets local field to dirty halo - Remove unused header eckit/memory/Noncopyable.h - Fix GatherScatter when global index with index 1 in in halo as is the case with ORCA grids ### Changed - Increase support for parallel interpolation from LAM grids by @wdeconinck in https://github.com/ecmwf/atlas/pull/331 - Detect and treat multiple (2+) discontiguity when wrapping arrays w/ atlas::Field by @sbrdar in https://github.com/ecmwf/atlas/pull/327 - Improve UnstructuredGrid::spec() performance by @wdeconinck in https://github.com/ecmwf/atlas/pull/336 - DelaunayMeshGenerator/ExtendNodesGlobal custom extension grid (defaul… by @pmaciel in https://github.com/ecmwf/atlas/pull/337 - Add cell_maximum_diagonal_on_unit_sphere metadata for global StructuredGrid without projections ### Added - Installation scripts for lz4 and metis - Add cell_maximum_diagonal_on_unit_sphere metadata for global StructuredGrid without projections to improve search in FiniteElement interpolation ## [0.44.1] - 2025-11-07 ### Fixed - Fix loop bounds for gcc+OpenMP compiler by @fmahebert in https://github.com/ecmwf/atlas/pull/324 - Fix compilation of atlas_fctest_fvm_nabla with NAG compiler (#326) by @wdeconinck in https://github.com/ecmwf/atlas/pull/330 - Set Eigen version to eigen@3 in CI to prevent 5.0.0 update ## [0.44.0] - 2025-10-06 ### Added - Add Collect communication pattern by @wdeconinck in https://github.com/ecmwf/atlas/pull/301 - Add array::View interoperability with mdspan by @wdeconinck in https://github.com/ecmwf/atlas/pull/304 - Add support for pocketfft by @wdeconinck in https://github.com/ecmwf/atlas/pull/308, https://github.com/ecmwf/atlas/pull/320 - Improvement in spherical-polygon intersection by @sbrdar in https://github.com/ecmwf/atlas/pull/310 - Improvement in the conservative interpolation for general meshes by @sbrdar in https://github.com/ecmwf/atlas/pull/318 - Add Field::syncHost and Field::syncDevice by @wdeconinck in https://github.com/ecmwf/atlas/pull/311 - Add more GPU tracing capabilities for device allocations and host-device data transfers by @wdeconinck in https://github.com/ecmwf/atlas/pull/312 - Improve HaloExchange on_device, checking state flags by @wdeconinck in https://github.com/ecmwf/atlas/pull/313 - Update interpolation to support hicsparse backend by @l90lpa in https://github.com/ecmwf/atlas/pull/275 - Add Locator to find at which partition and at which index a global index is located by @wdeconinck in https://github.com/ecmwf/atlas/pull/317 - Add matrix halo exchange to "binning" interpolation method. by @odlomax in https://github.com/ecmwf/atlas/pull/315 - Introduce atlas-interpolations app by @wdeconinck in https://github.com/ecmwf/atlas/pull/321 - Support building of StructuredColumns with halos for grids with uneven points at poles e.g. N80 grid ### Changed - Move mdspan from atlas to pluto and update mdspan to latest by @wdeconinck in https://github.com/ecmwf/atlas/pull/303 - Update MeshBuilder API by @wdeconinck in https://github.com/ecmwf/atlas/pull/305 ### Fixed - Fix out-of-range in RegionalLinear2D with FORTRAN feature disabled by @wdeconinck in https://github.com/ecmwf/atlas/pull/322 - Fix empty adjoint matrix for SphericalVector for scalar fields by @wdeconinck in https://github.com/ecmwf/atlas/pull/314 - Support CUDA 13.0 by @wdeconinck in https://github.com/ecmwf/atlas/pull/316 - Fix out-of-range in RegionalLinear2D with FORTRAN feature disabled by @wdeconinck in https://github.com/ecmwf/atlas/pull/322 - Allow BlockStructuredColumns to be used for MatchingPartitioner ## [0.43.1] - 2025-07-09 ### Fixed - Pluto installation of pluto_module.mod file - Avoid temporary array creation in Fortran pluto_allocator%allocate() function ## [0.43.0] - 2025-06-25 ### Added - Python wheel by @tmi in https://github.com/ecmwf/atlas/pull/274, https://github.com/ecmwf/atlas/pull/280 - Add Grid getter to FunctionSpace by @tom-j-h in https://github.com/ecmwf/atlas/pull/264 - Fortran: Export atlas_functionspace_BlockStructuredColumns by @wdeconinck in https://github.com/ecmwf/atlas/pull/276 - Feature/spectral updates by @wdeconinck in https://github.com/ecmwf/atlas/pull/285 - Add Field::halo() to manage and describe field halo's by @wdeconinck in https://github.com/ecmwf/atlas/pull/291 - Add a vector_component option utility by @odlomax in https://github.com/ecmwf/atlas/pull/294 - Implement floating-point-exception trapping for macos by @wdeconinck in https://github.com/ecmwf/atlas/pull/295 ### Fixed - Bugfix/device strides on cpu by @sbrdar in https://github.com/ecmwf/atlas/pull/297 - Support nvidia 24.5 by @wdeconinck in https://github.com/ecmwf/atlas/pull/278 - Set device_updated=false when calling Array::deallocateDevice() (fixes #243) by @wdeconinck in https://github.com/ecmwf/atlas/pull/277 - Fix StructuredGrid periodicity for certain domains (fixes #282) by @wdeconinck in https://github.com/ecmwf/atlas/pull/283 ### Changed - Make FiniteElement Interpolation weight computation multithreaded using OpenMP by @wdeconinck in https://github.com/ecmwf/atlas/pull/292 - Added OpenMP to the cubed-sphere interpolator and the matching cubed sphere partitioner. by @odlomax in https://github.com/ecmwf/atlas/pull/293 - Deprecate aliases for structured interpolation methods by @wdeconinck in https://github.com/ecmwf/atlas/pull/296 ## [0.42.0] - 2025-04-09 ### Added - Cubed sphere 2 grid builder by @mo-jonasganderton in https://github.com/ecmwf/atlas/pull/253 - Grid/Distribution constructor for PointCloud by @tom-j-h in https://github.com/ecmwf/atlas/pull/262 - Add distribution of serial interpolation matrix by @sbrdar in https://github.com/ecmwf/atlas/pull/258 - Add offloading of discontiguous arrays in https://github.com/ecmwf/atlas/pull/267 - Add device offload operators on FieldSet in https://github.com/ecmwf/atlas/pull/268 - Introducing Pluto by @wdeconinck in https://github.com/ecmwf/atlas/pull/269 - Python wheel: initial setup by @tmi in https://github.com/ecmwf/atlas/pull/257 ### Fixed - Fix inconsistent index-base used in StructuredColumns::remote_index by @sbrdar in https://github.com/ecmwf/atlas/pull/265 - Fix OpenMP related bug in DistributionArray ### Changed - Use pluto where possible instead of hic calls by @wdeconinck in https://github.com/ecmwf/atlas/pull/270 - Use pluto default memory resources in atlas::array by @wdeconinck in https://github.com/ecmwf/atlas/pull/273 ## [0.41.1] - 2025-02-18 ### Fixed - Add CI with gpu - Add -Werror in CI to avoid new warnings in the future - Fix compilation with ATLAS_BITS_LOCAL=64 and add it to CI to avoid errors in the future ## [0.41.0] - 2025-02-10 ### Added - Add hicsparse as wrapper to cusparse and hipsparse (#237) - Add SparseMatrix multiply_add functionality (#240) - Replace eckit::SparseMatrix with SparseMatrixStorage and SparseMatrixView to support host/device memory spaces (#247) - Add hicSparse backend to sparse matrix multiply (#246) - New simplified cubed sphere grid (#245) - Add functionality to gather global (serial) sparse matrix from an interpolation (#255) ### Fixed - Fix warnings (#256) - bugfix: DIV_BY_ZERO in MatchingFunctionSpacePartitionerLonLatPolygon for small grids with OMP (#244) - bugfix: Apply normalisation also to SphereT as was already done for unit-sphere (#242) ## [0.40.0] - 2024-11-18 ### Added - Add MultiField to pack field allocations by @sbrdar in https://github.com/ecmwf/atlas/pull/229 - Add function to create `std::variant` for multiple array views. by @odlomax in https://github.com/ecmwf/atlas/pull/220 - Add Fortran access to device data through Field and FieldSet by @sbrdar in https://github.com/ecmwf/atlas/pull/232 - Add Fortran API for structured stencil computations by @wdeconinck in https://github.com/ecmwf/atlas/pull/228 ### Changed - Support OpenACC with Cray compiler + use OpenACC_Fortran_FLAGS by @wdeconinck in https://github.com/ecmwf/atlas/pull/225 - Refactor `SphericalVector` interpolation method to use `array::ArrayViewVariant`. by @odlomax in https://github.com/ecmwf/atlas/pull/227 - Integrate `pack_vector_fields` into `SphericalVector` Interpolation method. by @odlomax in https://github.com/ecmwf/atlas/pull/224 - Refactor `util::pack_vector_fields` to use `array::ArrayViewVariant` by @odlomax in https://github.com/ecmwf/atlas/pull/226 ### Fixed - Fix application of limiter for StructuredInterpolation2D by @wdeconinck in https://github.com/ecmwf/atlas/pull/236 - Fix use of ATLAS_ENABLE_CUDA ON/OFF - Bugfix for Qhull by @benjaminmenetrier in https://github.com/ecmwf/atlas/pull/230 - Fixed ordering of `fixup_halos` and `halo_exhange.execute_adjoint` in `StructuredColumns.cc` by @odlomax in https://github.com/ecmwf/atlas/pull/223 ## [0.39.0] - 2024-09-18 ### Added - Add HIC abstraction layer for HIP and CUDA by @wdeconinck in https://github.com/ecmwf/atlas/pull/219 - Support for HIP via HIC - Add regional interpolation by @benjaminmenetrier in https://github.com/ecmwf/atlas/pull/215 - Pack vector components into higher-rank vector fields. by @odlomax in https://github.com/ecmwf/atlas/pull/218 ### Fixed - Fix missing header in FieldImpl.h by @tehrengruber in https://github.com/ecmwf/atlas/pull/214 - Bug fixes to vector interpolation with StructuredColumns and spherical vector interpolation by @MarekWlasak in https://github.com/ecmwf/atlas/pull/222 ## [0.38.1] - 2024-07-15 ### Fixed - Compilation and running with NAG compiler - Wrap out-of-bounds source cells into the domain (#211) ## [0.38.0] - 2024-06-20 ### Added - Make non_linear interpolation independent of a chosen value type by @wdeconinck in https://github.com/ecmwf/atlas/pull/176 - Procedure to carry out a regridding from high to low resolution (binning) by @mo-lormi in https://github.com/ecmwf/atlas/pull/191 - Add Fortran interface for node-to-edge connectivity building by @benjaminmenetrier in https://github.com/ecmwf/atlas/pull/209 - CUDA/OpenACC capable fields with Native storage backend @sbrdar and @wdeconinck in https://github.com/ecmwf/atlas/pull/182 ### Changed - Make non_linear interpolation independent of a chosen value type by @wdeconinck in https://github.com/ecmwf/atlas/pull/176 ### Fixed - Remove float in Triag2D intersection algorithm by @fmahebert in https://github.com/ecmwf/atlas/pull/203 - Allow zero-sized interpolation target functionspace by @odlomax in https://github.com/ecmwf/atlas/pull/206 - Made sure cubed-sphere interpolation method always sets metadata. by @odlomax in https://github.com/ecmwf/atlas/pull/208 - Avoid silent errors accessing Fieldset fields by ambiguous names by @wdeconinck in https://github.com/ecmwf/atlas/pull/210 - Fixes opposite pole coordinates by @benjaminmenetrier in https://github.com/ecmwf/atlas/pull/202 ## [0.37.0] - 2024-04-09 ### Added - Add SphericalVector interpolation method using parallel transport (#163) - Added arrayForEachDim method ### Fixed - Bugfix for regional grids with ny > nx - Fix build for configuration setting ATLAS_BITS_LOCAL=64 ### Changed - Use new LocalConfiguration baseclass functions in util::Config and util::Metadata instead of eckit::Value backdoor - atlas_io is an adaptor library when eckit_codec is available (#181) ## [0.36.0] - 2023-12-11 ### Added - Add TriangularMeshBuilder with Fortran API, so far for serial meshes only - Add Fortran API for CellCollumns functionspace - Add EqualAreaPartitioner which is geometry based rather than loadbalanced like EqualRegionsPartitioner ### Fixed - Compilation with Intel LLVM compiler - Fix 180 degrees phase shift error in MDPI_gulfstream function ### Changed - Update install scripts - Preparation for using eckit::codec as backend for atlas_io ## [0.35.1] - 2023-10-24 ### Added - Don't output with output::Gmsh the triangle elements with wrong orientation when coordinates are "lonlat" - Add control to skip Gmsh-output of triangles with too large edge length ratio - Configurable geometry in KDTree - Use configurable KDTree geometry in PointCloud - atlas-grid-points executable to list grid points - Allow constructor of atlas::io::ArrayShape with different integer types - Support atlas_io with vector ### Fixed - Control FPE in StructuredColumns::checksum to avoid overflow and invalid in unimportant halo regions - Fixes to MeshBuilder validate_mesh_vs_grid ### Changed - Use search radius in FiniteElement interpolation when mesh defines metadata to do so ## [0.35.0] - 2023-02-10 ### Added - Add accessors for the GridBox class (MIR-632) - Add FunctionSpace::partition() field - Add configurable MPI communicator to data structures (#131) - Add single precision support for fvm::Nabla - Implement PointCloud parallel construction with halo_radius ### Fixed - Fix StructuredMeshGenerator for Gaussian grids that don't start at 0 degrees (ATLAS-375) - Remove functionspace::Spectral via Trans initialisation (fixes #153) - Enable MeshBuilder to set up the Grid (#152) - Fix use of cmake option ATLAS_ENABLE_TRANS as in a bundle ### Changed - Cleanup long-standing temporary element types. API change but with deprecated old API - Deprecated rename Mesh::nb_partitions -> Mesh::nb_parts - Implement Delaunay triangulation using Qhull instead of CGAL but CGAL can still be enabled instead for now ## [0.34.0] - 2023-07-10 ### Added - Fieldset::metadata (#126) - Fieldset::adjointHaloExchange - Field/Fieldset::clone method - Functions to enable/disable FPE - Add function to build mesh from imported connectivity data (#135) - Implement field::for_each capabilities (#139) - Introduce colon-separated environment variable ATLAS_PLUGIN_PATH to simplify plugin detection - Introduce atlas::mdspan, contributed from github.com/kokkos/mdspan - Add function Field::horizontal_dimension() -> std::vector - Setup horizontal_dimensions() for BlockStructuredColumns fields - Upgrade the halo exchange procedure for the function space 'PointCloud' (#120) ### Fixed - Enable latitude normalisation in KDTree coordinate transform (#140) - Fix LocalView indexing bug for non-contiguous slices - C++17 flag public propagation to downstream C++ CMake packages - Fix cells().global_index() metadata for RegularLonLat grids in StructuredMeshGenerator - BuildHalo: mark interior added cells as ghost ## [0.33.0] - 2023-04-03 ### Added - Add support for StructuredPartitionPolygon with halo > 0 - Add information on atlas having PROJ support ### Changed - C++17 standard is now a requirement ### Fixed Fix StructuredInterpolation2D with retry for failed stencils ## [0.32.1] - 2023-02-09 ### Added - Added (lon, lat) to (alpha, beta) transforms to cubed sphere projection ## [0.32.0] - 2023-01-23 ### Added - Added BlockStructuredColumns FunctionSpace - Added function SolidBodyRotation - Added convenience Fortran constructors for ShiftedLonLat, SHiftedLon, ShiftedLat - Support for more FunctionSpaces in various interpolation methods ### Changed - PolygonLocator now wraps around longitudes for non-matching domains - StructuredMeshGenerator can generate meshes with partitions without elements - SerialDistribution makes every MPI task have the entire mesh ### Fixed - Remove assertion for checking of empty mesh in NodeColumns FunctionSpace - Gaussian latitudes can now be computed for very high resolution, without running out of memory. - Interpolation to functionspaces with empty partitions ## [0.31.1] - 2022-11-11 ### Fixed - Fix bug introduced in StructuredMeshGenerator global numbering with release 0.31.0 in commit a63fc62a2 - Fix healpix global numbering for pentagon pole elements in parallel - Fix validity check of atlas::HealpixGrid ## [0.31.0] - 2022-11-10 ### Added - Extend PointCloud functionspace to do halo exchanges, including Fortran API - Add FunctionSpace::gather and FunctionSpace::scatter abstraction ### Changed - Improve performance of MatchingMeshPartitionerLonLatPolygon using OpenMP - Improve performance of BuildHalo using OpenMP and unordered_map - Improve performance of StructuredMeshGenerator using OpenMP - Improve performance of ATLAS_TRACE - Reduce memory peak in GatherScatter setup - Global element numbering of RegularLonLat grids is now following rows independent of partitioning ### Fixed - Running CI with Github Actions - Fix output of atlas-grids y-range for shifted grids - Fix building with ATLAS_BITS_LOCAL=64 ## [0.30.0] - 2022-08-22 ### Added - Fortran API for Interpolation::execute_adjoint() - Fortran API for FieldSet::name() - Fortran API for MeshGenerator::generate(grid,partitioner) - Pentagon element type - SphericalHarmonic function - New interpolation method: ConservativeSphericalPolygonInterpolation - Support 'variables' option in functionspace::PointCloud::createField ### Changed - Atlas-IO is now standalone project, still embedded but only depending on eckit - Deprecate Trans naming of 'ifs' or 'trans' in favour of 'ectrans' - Default StructuredMeshGenerator partitioner is equal_regions instead of trans/ectrans ### Fixed - Fix global numbering HEALPix grid to standard - Fix NodeColumns remote_index for parallel orca grids - Fix use of ATLAS_LINALG_DENSE_BACKEND environment variable ## [0.29.0] - 2022-04-21 ### Added - MatchingMeshPartitioner "cubedsphere" - Interpolator "cubedsphere-bilinear" - Improvements to Interpolation::Cache - Add support for rank 2 fields when a nonlinear action is added to the interpolator - Create Array using ArraySpec only ### Changed - FieldSet::has(...) replaces FieldSet::has_field(...) - Metadata return value to Interpolation::execute() - Rename BilinearRemapping to UnstructuredBilinearLonLat ### Fixed - Compatibility with proj version >= 8 - Compatibility with eckit version <= 1.18.5 - Compatibility with GridTools backend and using 64bit idx_t - Wrongly computed Jacobian::transpose() introduced in 0.28.0 - Fix bug where using ectrans was not enabling adjoint of invtrans - Avoid segfault when OpenMP tasking is broken, as it is with AppleClang and LLVM libomp ## [0.28.1] - 2022-03-14 ### Fixed - Fix compilation for GNU 7.3 ## [0.28.0] - 2022-03-02 ### Added - Assignment of ArrayView from ArrayView - Grid "regional_variable_resolution" via a new VariableResolutionProjection - CubedSphereDualMeshGenerator - VortexRollup function as analytical field for initialising data - ConvexSphericalPolygon utility class - Improve Projection::Jacobian - Initial implementation for bilinear interpolation for unstructured meshes ### Changed - Use new eckit (1.19.0) Sparse and Dense linear algebra API - General robustness improvements to CubedSphere to using functionspaces with various halos ### Fixed - Workarounds to fix compilation with Fujitsu compiler - Workarounds to avoid Cray compiler problems with certain flag combinations - CellColumns::haloExchange for meshes with multiple element types - Computation of HEALPix mesh remote indices. ## [0.27.0] - 2021-12-03 ### Added - Adjoint interpolation with some restrictions - Cubed sphere grid partitioner - Cubed sphere parallel mesh generation - Cubed sphere function spaces - Fortran interfaces to Projection methods - Support discovery of open-source ectrans - Dense linear Algebra matrix_multiply abstraction ### Changed - Remove etc/atlas/config.yaml because defaults should be in code - Naming of sparse_matrix_multiply backend 'omp' -> 'openmp' - Applied clang-format 13.0.0 (all files touched) ## [0.26.0] - 2021-08-23 ### Added - Support for cubed sphere grids and preliminary support for cubes sphere mesh generation. ### Fixed - Compilation with altas_bits_local=64 - Too aggressive optimisation with gnu 11 - Compatibility with cmake 3.20 and nvhpc compilers ## [0.25.0] - 2021-05-18 ### Added - New concept Redistribution to redistribute field on same grid but with different partitioner - Support for Trans::invtrans_adj() and related methods - Introduce ~atlas/etc/atlas/config.yaml - atlas-io with support for encoding/decoding std::array - atlas-io print support for tiny arrays ### Changed - atlas-io version 0.2 - Move util::SpecRegistry to non-templated grid::SpecRegistry ### Fixed - minor bug fixes ## [0.24.1] - 2021-04-06 ### Fixed - Periodic halo for HEALPix mesh - General warnings and suggestions by DeepCode bot - Compilation problems with clang 3.8 ## [0.24.0] - 2021-03-19 ### Fixed - Fixed hang in band distribution for L160000x8000 grid - Fixes to Spectral functionspace and TransIFS regarding vertical levels ### Changed - Requires eckit 1.16 ### Added - atlas-io first version, not for operational use - Spec registry for grids ## [0.23.0] - 2021-01-19 ### Fixed - Structured interpolation method interpolating to area straddling Greenwich. - Fixes when compiling with ATLAS_BITS_LOCAL=64 ### Changed - Possibility to link to alternative open-source version of IFS trans library. ### Added - Caching mechanism for interpolation ## [0.22.1] - 2020-10-22 ### Fixed - Installation of PGI compilers via tools/install-pgi.sh - Allow dependency on older Eigen 3.2 which does not use CMake targets ## [0.22.0] - 2020-10-14 ### Fixed - Feature INIT_SNAN was not working - Support KNearestNeighbour interpolation for functionspace with smaller halo than the mesh contains - Support array size up to size_t limit ### Changed - Migration to use ecbuild 3.4 - ATLAS_BITS_LOCAL can be configured to 32 or 64 ### Added - Fields can be created with given alignment, which adds padding in innermost dimension - Added conservative interpolation with "grid-box average" and "grid-box maximum" - Missing value definition for fields - Add support for missing values in matrix-based interpolation methods - Floating point trapping and signal handling mechanism - Fortran: GridDistribution constructors - Fortran: Domain access - Fortran: Get lonlat_bounding_box via domain - Possibility to access Jacobian of projections (with only some projections implemented) ## [0.21.0] - 2020-06-23 ### Fixed - Fixed Rotation order of applying the rotation angle - Fix nabla computation of surface vector fields - Fix registration and destruction issues of halo-exchange caches - Workaround Clang compiler problem with OpenMP tasking, using introspection - Fix bug in conversion of negative degrees to microdegrees - Fix problem in distribution of work amongst OpenMP threads in StructuredColumns::setup - Fix problem with StructuredColumns creation for grids defined with domains with negative West - Fix computation of Grid::lonlatBoundingBox for arbitrary projections crossing the dateline. ### Changed - Snap LinearSpacing values to start and endpoint to allow exact comparisons - Improved performance and memory requirement of cropping of large StructuredGrids - Regional grids by default now have a positive y-numbering (previously negative). ### Added - PolygonLocator based on functionspace partition polygons. - KDTree class which abstracts eckit concrete implementations, including Fortran interface - Geometry class with factory mechanism to configure which spheroid to use, including Fortran interface - StrcutredGrid function for i,j to index and inverse - Fortran interface to create StructuredColumns with custom distribution - Fortran interface to grid spec - Fortran interface to Projection - Adjoint of HaloExchange - Plugin mechanism to load plugins at runtime. - Fortran constructors for atlas_RegionalGrid - Fortran constructors for projected reduced Gaussian grids - Add copy constructor and assignment operator for atlas::vector - Mercator projection support for scaling, and operation on ellipsoid. - Grid Distribution can now also be created as a function, e.g. for Serial or Bands ## [0.20.2] - 2020-04-27 ### Fixed - Avoid 100ds of compilation warnings introduced in version 0.20.0 ## [0.20.1] - 2020-04-08 ### Fixed - Make feature BOUNDSCHECKING work again. It was not turned on for DEBUG builds - Workaround clang OpenMP bug - Fix Segfault due to unexpected order of destruction of singleton objects ### Added - atlas-grids tool can now be used to compute approximate North-South grid resolution ## [0.20.0] - 2020-03-06 ### Fixed - Pole edges hould not be created for global regular grids with points at poles - Update compatibility with more recent GridTools - HaloExchange with CUDA - Self registration from external library ### Added - Proj-based projections - OpenMP functions for sorting, filling, copying - Parallel structured grid interpolation ### Changed - Grid iterators can have random access - Speed improvements for StructuredColumns constructor - Speed improvements for LonLatPolygon::contains() - Port to ecbuild 3 (also minimum required version) - Tidying up of atlas tools ## [0.19.2] - 2020-01-28 ### Changed - Compatibility with eckit 1.7 due to API change in eckit::LocalConfiguration ## [0.19.1] - 2019-12-19 ### Fixed - Keep Gaussian identity of a Gaussian grid if a given domain does not crop any latitudes - Fix naming for LegendreCache, to be more specific, and platform independent ## [0.19.0] - 2019-10-01 ### Fixed - Lambert ( conformal conic ) projection xy coordinates are now corrected ### Changed - LambertProjection renamed to LambertConformalConic ### Added - Reordering of nodes strategies (Hilbert curve, ReverseCuthillMckee) - Preliminary CellColumns functionspace with Gmsh IO; halos are not yet fully supported ## [0.18.1] - 2019-08-10 ### Fixed - Match vertical structured interpolation to IFS - Fix in creating vertical dimension in StructuredColumns using interval - Fix in caching StructuredColumnsHaloExchange ## [0.18.0] - 2019-07-15 ### Changed - Make grid hashes crossplatform ### Added - Fortran: Can now use TransLocal - Fortran: Can now create unstructured grids - LonLat Bounding box computations for grids using its projection - Serialisation of Mesh Connectivity tables to/from eckit::Stream ### Fixed - Structured interpolation bugs - StructuredColumns bug with iSend - Memory corruption in Spectral functionspace with GT CUDA backend ## [0.17.2] - 2019-06-04 ### Fixed - Compilation with PGI 19.4 - Structured Grid creation for periodic domains that do not start at 0 degrees longitude (Greenwich) ## [0.17.1] - 2019-04-22 ### Added - Option to declaration of field type (vector/scalar) when creating field in FunctionSpace - New projection: Lambert Azimuthal Equal Area - StructuredInterpolation2D to target FunctionSpace StructuredColumns ### Fixed - Compilation with IBM XL 19 - Compilation with Intel 19 ## [0.17.0] - 2019-04-02 ### Changed - OpenMP is now private dependency - Dependencies are now added in a modern CMake3 way - Fortran modules are installed in /module/atlas ### Added - Spectral functionspace better used in distributed context - Nabla now holds shared_ptr to Method - Fortran access to vertical coordinates from StructuredColumns ### Fixed - Compilation with PGI/19.1 (regression) - Add missing halo_exchange for Fortran rank-4 arrays - Memory leaks with StructuredColumns ## [0.16.0] - 2019-02-14 ### Changed - Interpolation makes use of OpenMP - Cleanup of header includes - fypp Fortran preprocessor is ported to fckit 0.6 ### Added - Parallel structured interpolation methods (2D,3D): linear, cubic, quasicubic - Interpolation for multi-level and multi-variable fields - atlas_Trace: Fortran API and use within OpenMP parallel regions - StructuredColumns halo-exchange for vector fields - Field::halo_exchange() function ### Fixed - Fortran compilation with PGI 18.10 - Access to Field view within OpenMP parallel region - FunctionSpaces use only required halo, even if larger halo is available in mesh - Fixed faulty name of a Field when created through Fortran API, wrapping existing memory - Fix NodeColumns functionspace when mesh is created from projected grid. - Parallel interpolation from regular lonlat grid. - Spectral spherical harmonics transforms for large cases ## [0.15.2] - 2018-08-31 ### Changed - Initialisation of Fields to signalling NaN in debug builds, uninitialised in non-debug builds (used to be initialised to zero as part of std::vector construction) ### Added - Implementation of cropped unstructured grids so that spectral transforms to unstructured grids are allowed ### Fixed - Spectral transforms to grids including pole and equator - Build with gridtools CUDA backend ## [0.15.1] - 2018-07-17 ### Fixed - Compilation for Intel 18 debug - Memory bug for spherical harmonics - Compatibility with fckit 0.5.1 ## [0.15.0] - 2018-06-19 ### Changed - Native Array data storage uses now a raw C pointer instead of std::vector - Significant performance improvements to Spherical harmonics transforms ### Fixed - Various bugs related to parallel halos - Bit reproducibility for parallel interpolation ## [0.14.0] - 2018-03-22 ### Added - Spherical Harmonics transforms can receive a cache memory handle ### Changed - Earth interface (C++) - Requires eckit 0.20.0, fckit 0.5.0 ## [0.13.2] - 2018-03-20 ### Fixed - C++ compilation using PGI 17, 18 and GCC 7 - Support Python 3 to generate Fortran bindings - Travis CI linked with github repository - Problem with CUDA allocated memory ## [0.13.1] - 2018-03-01 ### Fixed - Fortran compilation using Intel 18 - GridTools compatibility ## 0.13.0 - 2018-02-16 [Unreleased]: https://github.com/ecmwf/atlas/compare/master...develop [0.45.0]: https://github.com/ecmwf/atlas/compare/0.44.1...0.45.0 [0.44.1]: https://github.com/ecmwf/atlas/compare/0.44.0...0.44.1 [0.44.0]: https://github.com/ecmwf/atlas/compare/0.43.1...0.44.0 [0.43.1]: https://github.com/ecmwf/atlas/compare/0.43.0...0.43.1 [0.43.0]: https://github.com/ecmwf/atlas/compare/0.42.0...0.43.0 [0.42.0]: https://github.com/ecmwf/atlas/compare/0.41.0...0.42.0 [0.41.1]: https://github.com/ecmwf/atlas/compare/0.41.0...0.41.1 [0.41.0]: https://github.com/ecmwf/atlas/compare/0.40.0...0.41.0 [0.40.0]: https://github.com/ecmwf/atlas/compare/0.39.0...0.40.0 [0.39.0]: https://github.com/ecmwf/atlas/compare/0.38.1...0.39.0 [0.38.1]: https://github.com/ecmwf/atlas/compare/0.38.0...0.38.1 [0.38.0]: https://github.com/ecmwf/atlas/compare/0.37.0...0.38.0 [0.37.0]: https://github.com/ecmwf/atlas/compare/0.36.0...0.37.0 [0.36.0]: https://github.com/ecmwf/atlas/compare/0.35.1...0.36.0 [0.35.1]: https://github.com/ecmwf/atlas/compare/0.35.0...0.35.1 [0.35.0]: https://github.com/ecmwf/atlas/compare/0.34.0...0.35.0 [0.34.0]: https://github.com/ecmwf/atlas/compare/0.33.0...0.34.0 [0.33.0]: https://github.com/ecmwf/atlas/compare/0.32.1...0.33.0 [0.32.1]: https://github.com/ecmwf/atlas/compare/0.32.0...0.32.1 [0.32.0]: https://github.com/ecmwf/atlas/compare/0.31.1...0.32.0 [0.31.1]: https://github.com/ecmwf/atlas/compare/0.31.0...0.31.1 [0.31.0]: https://github.com/ecmwf/atlas/compare/0.30.0...0.31.0 [0.30.0]: https://github.com/ecmwf/atlas/compare/0.29.0...0.30.0 [0.29.0]: https://github.com/ecmwf/atlas/compare/0.28.1...0.29.0 [0.28.1]: https://github.com/ecmwf/atlas/compare/0.28.0...0.28.1 [0.28.0]: https://github.com/ecmwf/atlas/compare/0.27.0...0.28.0 [0.27.0]: https://github.com/ecmwf/atlas/compare/0.26.0...0.27.0 [0.26.0]: https://github.com/ecmwf/atlas/compare/0.25.0...0.26.0 [0.25.0]: https://github.com/ecmwf/atlas/compare/0.24.1...0.25.0 [0.24.1]: https://github.com/ecmwf/atlas/compare/0.24.0...0.24.1 [0.24.0]: https://github.com/ecmwf/atlas/compare/0.23.0...0.24.0 [0.23.0]: https://github.com/ecmwf/atlas/compare/0.22.1...0.23.0 [0.22.1]: https://github.com/ecmwf/atlas/compare/0.22.0...0.22.1 [0.22.0]: https://github.com/ecmwf/atlas/compare/0.21.0...0.22.0 [0.21.0]: https://github.com/ecmwf/atlas/compare/0.20.2...0.21.0 [0.20.2]: https://github.com/ecmwf/atlas/compare/0.20.1...0.20.2 [0.20.1]: https://github.com/ecmwf/atlas/compare/0.20.0...0.20.1 [0.20.0]: https://github.com/ecmwf/atlas/compare/0.20.0...0.19.2 [0.19.2]: https://github.com/ecmwf/atlas/compare/0.19.1...0.19.2 [0.19.1]: https://github.com/ecmwf/atlas/compare/0.19.0...0.19.1 [0.19.0]: https://github.com/ecmwf/atlas/compare/0.18.1...0.19.0 [0.18.1]: https://github.com/ecmwf/atlas/compare/0.18.0...0.18.1 [0.18.0]: https://github.com/ecmwf/atlas/compare/0.17.2...0.18.0 [0.17.2]: https://github.com/ecmwf/atlas/compare/0.17.1...0.17.2 [0.17.1]: https://github.com/ecmwf/atlas/compare/0.17.0...0.17.1 [0.17.0]: https://github.com/ecmwf/atlas/compare/0.16.0...0.17.0 [0.16.0]: https://github.com/ecmwf/atlas/compare/0.15.2...0.16.0 [0.15.2]: https://github.com/ecmwf/atlas/compare/0.15.1...0.15.2 [0.15.1]: https://github.com/ecmwf/atlas/compare/0.15.0...0.15.1 [0.15.0]: https://github.com/ecmwf/atlas/compare/0.14.0...0.15.0 [0.14.0]: https://github.com/ecmwf/atlas/compare/0.13.2...0.14.0 [0.13.2]: https://github.com/ecmwf/atlas/compare/0.13.1...0.13.2 [0.13.1]: https://github.com/ecmwf/atlas/compare/0.13.0...0.13.1 atlas-0.45.0/CMakeLists.txt000066400000000000000000000100431513175025300154440ustar00rootroot00000000000000# (C) Copyright 2013 ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. ############################################################################################ # Atlas cmake_minimum_required( VERSION 3.12 FATAL_ERROR ) find_package( ecbuild 3.4 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild ) ################################################################################ # Initialise project Atlas project( atlas LANGUAGES CXX ) set( ATLAS_BITS_GLOBAL 64 CACHE STRING "bits used to define a global index (atlas::gidx_t)" ) set( ATLAS_BITS_LOCAL 32 CACHE STRING "bits used to define a local index (atlas::idx_t)" ) ################################################################################ # Required packages if( ATLAS_BITS_LOCAL MATCHES 64 ) ecbuild_find_package( NAME eckit VERSION 1.17.1 REQUIRED ) else() ecbuild_find_package( NAME eckit VERSION 1.16 REQUIRED ) endif() ecbuild_debug( " eckit_FEATURES : [${eckit_FEATURES}]" ) add_subdirectory(atlas_io) find_package(atlas_io) ################################################################################ # Features that can be enabled / disabled with -DENABLE_ ecbuild_add_option( FEATURE ATLAS_GRID DESCRIPTION "Build grid related features" ) ecbuild_add_option( FEATURE ATLAS_FIELD DESCRIPTION "Build field and memory management related features" ) ecbuild_add_option( FEATURE ATLAS_FUNCTIONSPACE DESCRIPTION "Build functionspace related features (mesh, functionspace, parallel)" CONDITION atlas_HAVE_ATLAS_GRID AND atlas_HAVE_ATLAS_FIELD ) ecbuild_add_option( FEATURE ATLAS_INTERPOLATION DESCRIPTION "Build interpolation related features" CONDITION atlas_HAVE_ATLAS_FUNCTIONSPACE ) ecbuild_add_option( FEATURE ATLAS_TRANS DESCRIPTION "Build transform related features" CONDITION atlas_HAVE_ATLAS_FUNCTIONSPACE ) ecbuild_add_option( FEATURE ATLAS_NUMERICS DESCRIPTION "Build numerics related features" CONDITION atlas_HAVE_ATLAS_FUNCTIONSPACE ) ecbuild_add_option( FEATURE ECKIT_DEVELOP DESCRIPTION "Used to enable new features or API depending on eckit develop branch, not yet in a tagged release" DEFAULT OFF ) ecbuild_add_option( FEATURE FCKIT_DEVELOP DESCRIPTION "Used to enable new features or API depending on eckit develop branch, not yet in a tagged release" DEFAULT OFF ) if (DEFINED ATLAS_ENABLE_CUDA AND NOT DEFINED HIC_ENABLE_CUDA ) set( HIC_ENABLE_CUDA ${ATLAS_ENABLE_CUDA} ) endif() if (DEFINED ATLAS_ENABLE_HIP AND NOT DEFINED HIC_ENABLE_HIP ) set( HIC_ENABLE_HIP ${ATLAS_ENABLE_HIP} ) endif() add_subdirectory( hic ) find_package(hic REQUIRED) add_subdirectory( pluto ) find_package(pluto REQUIRED) include( features/BOUNDSCHECKING ) include( features/FORTRAN ) include( features/CUDA ) include( features/MPI ) include( features/OMP ) include( features/POCKETFFT ) include( features/FFTW ) include( features/ECTRANS ) include( features/TESSELATION ) include( features/GRIDTOOLS_STORAGE ) include( features/ACC ) include( features/EIGEN ) include( features/PROJ ) include( features/SANDBOX ) include( features/CLANG_TIDY ) include( features/INCLUDE_WHAT_YOU_USE ) include( features/INIT_SNAN ) include( features/DOCS ) include( features/ATLAS_RUN ) ################################################################################ # sources include( atlas_compile_flags ) add_subdirectory( src ) add_subdirectory( doc ) ################################################################################ # Export and summarize include( atlas_export ) ecbuild_print_summary() atlas-0.45.0/LICENSE000066400000000000000000000250031513175025300137130ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 1996-2018 ECMWF Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. atlas-0.45.0/README.md000066400000000000000000000130121513175025300141620ustar00rootroot00000000000000Atlas ===== [![atlas release version](https://img.shields.io/github/release/ecmwf/atlas.svg)](https://github.com/ecmwf/atlas/releases/latest) [![build](https://github.com/ecmwf/atlas/actions/workflows/build.yml/badge.svg)](https://github.com/ecmwf/atlas/actions/workflows/build.yml) [![codecov](https://codecov.io/gh/ecmwf/atlas/branch/develop/graph/badge.svg)](https://codecov.io/gh/ecmwf/atlas) Project home: https://confluence.ecmwf.int/display/ATLAS Contact: Willem Deconinck (willem.deconinck@ecmwf.int) Publication: > [Deconinck et al, 2017](https://doi.org/10.1016/j.cpc.2017.07.006) --- > Atlas: A library for numerical weather prediction and climate modelling Atlas is a ECMWF library for parallel data-structures supporting unstructured grids and function spaces, with the aim to investigate alternative more scalable dynamical core options for Earth System models, and to support modern interpolation and product generation software Atlas is predominantly C++ code, with main features available to Fortran codes through a F2003 interface. It requires some flavour of Unix (such as Linux). It is known to run on a number of systems, some of which are directly supported by ECMWF. Requirements ------------ Tested compilers include: - GCC 4.9.1, 5.3.0, 6.3.0, 7.2.0 - Intel 15.0.2, 16.0.3, 17, 18 - CCE 8.4.5, 8.5.8, 8.6.2 - PGI-Fortran 17.7 combined with GNU-C/C++ 6.3 - PGI 17.7 Known compilers to fail include: - PGI-Fortran 17.10, 18.1 Required dependencies: - CMake --- For use and installation see http://www.cmake.org/ - ecbuild --- ECMWF library of CMake macros - eckit (with MPI support) --- C++ support library Recommended dependencies: - fckit --- For enabling Fortran interfaces - python (only when Fortran bindings are required) Optional dependencies: - gridtools-storage --- For GPU interoperability - transi --- For enabling IFS spherical harmonics transforms ( not open-source ) - CGAL --- For enabling Delaunay triangulation of unstructured grids - Eigen3 -- For certain linear algebra operations - FFTW -- For enabling inverse spherical harmonics transforms (TransLocal) Installation ------------ Atlas employs an out-of-source build/install based on CMake. Make sure ecbuild, eckit and fckit are installed and the ecbuild executable script is found ( `which ecbuild` ). Following environment variables help the build system to detect the right dependencies: ```bash # For finding eckit ECKIT_PATH # Path to eckit prefix # For finding fckit FCKIT_PATH # Path to fckit prefix ``` Other environment variables that could be required for optional features: ```bash # For finding gridtools-storage GRIDTOOLS_STORAGE_PATH # Path to gridtools-storage prefix # For finding transi TRANSI_PATH # Path to transi prefix # For finding CGAL BOOST_ROOT # Path to Boost prefix CGAL_DIR # Path to directory containing CGALConfig.cmake Eigen3_DIR # Path to directory containing Eigen3Config.cmake FFTW_PATH # Path to FFTW prefix ``` Now proceed with installation as follows ```bash # Environment --- Edit as needed ATLAS_SRC=$(pwd) ATLAS_BUILD=build ATLAS_INSTALL=$HOME/local # 1. Create the build directory: mkdir $ATLAS_BUILD cd $ATLAS_BUILD # 2. Run CMake ecbuild --prefix=$ATLAS_INSTALL -- $ATLAS_SRC # 3. Compile / Install make -j10 make install # 4. Check installation $ATLAS_INSTALL/bin/atlas --info ``` Extra flags maybe added to step 2 to fine-tune configuration. - `--build=DEBUG|RELEASE|BIT` --- Optimisation level * DEBUG: No optimisation (`-O0 -g`) * BIT: Maximum optimisation while remaning bit-reproducible (`-O2 -g`) * RELEASE: Maximum optimisation (`-O3`) - `-DENABLE_OMP=OFF` --- Disable OpenMP - `-DENABLE_FORTRAN=OFF` --- Disable Compilation of Fortran bindings > **Note:** > By default compilation is done using shared libraries. Some systems have linking > problems with static libraries that have not been compiled with `-fPIC`. > In this case, also compile atlas using static linking, by adding to step 2: `--static` Runtime Configuration --------------------- Atlas behaviour can be configured through some environment variables with defaults marked in square brackets - `ATLAS_INFO=<0|[1]>` --- Control printing of Atlas standard information - `ATLAS_DEBUG=<[0]|1>` --- Control printing of Atlas debug information - `ATLAS_TRACE=<[0]|1>` --- Control printing of Atlas traces (includes timings) Contributing ------------ Contributions to Atlas are welcome. In order to do so, please open an issue where a feature request or bug can be discussed. Then issue a pull request with your contribution. Pull requests must be issued against the develop branch. Citing Atlas ------------ If you publish work which mentions Atlas, or Atlas has been useful in your research, please cite the following paper: ```bibtex @article{DECONINCK2017188, title = "Atlas : A library for numerical weather prediction and climate modelling", journal = "Computer Physics Communications", volume = "220", pages = "188 - 204", year = "2017", issn = "0010-4655", doi = "https://doi.org/10.1016/j.cpc.2017.07.006", url = "http://www.sciencedirect.com/science/article/pii/S0010465517302138", author = "Willem Deconinck and Peter Bauer and Michail Diamantakis and Mats Hamrud and Christian Kühnlein and Pedro Maciel and Gianmarco Mengaldo and Tiago Quintino and Baudouin Raoult and Piotr K. Smolarkiewicz and Nils P. Wedi", keywords = "Numerical weather prediction, Climate, Earth system, High performance computing, Meteorology, Flexible mesh data structure" } ``` atlas-0.45.0/VERSION000066400000000000000000000000071513175025300137530ustar00rootroot000000000000000.45.0 atlas-0.45.0/atlas_io/000077500000000000000000000000001513175025300145015ustar00rootroot00000000000000atlas-0.45.0/atlas_io/AUTHORS000066400000000000000000000001421513175025300155460ustar00rootroot00000000000000Authors ======= - Willem Deconinck Thanks for contributions from ============================= atlas-0.45.0/atlas_io/CHANGELOG.md000066400000000000000000000000001513175025300163000ustar00rootroot00000000000000atlas-0.45.0/atlas_io/CMakeLists.txt000066400000000000000000000065441513175025300172520ustar00rootroot00000000000000# (C) Copyright 2021 ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. ############################################################################################ # Atlas-IO cmake_minimum_required( VERSION 3.12 FATAL_ERROR ) find_package( ecbuild 3.4 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild ) ################################################################################ # Initialise project Atlas project( atlas_io VERSION ${atlas_VERSION} LANGUAGES CXX ) ################################################################################ # Required packages ecbuild_find_package( NAME eckit REQUIRED ) ecbuild_debug( " eckit_FEATURES : [${eckit_FEATURES}]" ) ################################################################################ # Features that can be enabled / disabled with -DENABLE_ ecbuild_add_option( FEATURE ECKIT_DEVELOP DESCRIPTION "Used to enable new features or API depending on eckit develop branch, not yet in a tagged release" DEFAULT OFF ) set( eckit_HAVE_ECKIT_CODEC 0 ) if( TARGET eckit_codec ) set( eckit_HAVE_ECKIT_CODEC 1 ) endif() ecbuild_add_option( FEATURE ECKIT_CODEC DEFAULT ON DESCRIPTION "Use eckit::codec with adaptor" CONDITION eckit_HAVE_ECKIT_CODEC ) ################################################################################ # sources set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) ecbuild_add_option( FEATURE WARNING_AS_ERROR DEFAULT OFF DESCRIPTION "Treat compile warning as error" ) if(HAVE_WARNING_AS_ERROR) ecbuild_add_cxx_flags("-Werror" NO_FAIL NAME atlas_io_cxx_warning_as_error) endif() ecbuild_add_option( FEATURE WARNINGS DEFAULT ON DESCRIPTION "Add warnings to compiler" ) if(HAVE_WARNINGS) ecbuild_add_cxx_flags("-Wall" NO_FAIL) ecbuild_add_cxx_flags("-Wextra" NO_FAIL) ecbuild_add_cxx_flags("-Wno-unused-parameter" NO_FAIL) endif() if( CMAKE_CXX_COMPILER_ID STREQUAL Intel ) ecbuild_add_cxx_flags("-diag-disable=10441" NO_FAIL) # Deprecated classic compiler endif() check_cxx_source_compiles( "#include \n int main() { char * type; int status; char * r = abi::__cxa_demangle(type, 0, 0, &status); }" ATLAS_IO_HAVE_CXXABI_H ) test_big_endian( _BIG_ENDIAN ) if( _BIG_ENDIAN ) set( ATLAS_IO_BIG_ENDIAN 1 ) set( ATLAS_IO_LITTLE_ENDIAN 0 ) else() set( ATLAS_IO_BIG_ENDIAN 0 ) set( ATLAS_IO_LITTLE_ENDIAN 1 ) endif() if( HAVE_ECKIT_CODEC ) ecbuild_info("atlas_io is configured to be an adaptor library which delegates calls to eckit_codec") add_subdirectory(eckit_codec_adaptor) else() add_subdirectory( src ) endif() add_subdirectory( tests ) ################################################################################ # Export and summarize ecbuild_add_resources( TARGET atlas_io-others SOURCES_PACK README.md CHANGELOG.md LICENSE ) ecbuild_install_project( NAME Atlas-IO ) ecbuild_print_summary() atlas-0.45.0/atlas_io/LICENSE000066400000000000000000000250031513175025300155060ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 1996-2018 ECMWF Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. atlas-0.45.0/atlas_io/README.md000066400000000000000000000003051513175025300157560ustar00rootroot00000000000000atlas-io ======== An IO API for serializing arbitrary data, supporting compression. The API contains a message format (like GRIB). Multiple messages can be written to file or sent over network. atlas-0.45.0/atlas_io/cmake/000077500000000000000000000000001513175025300155615ustar00rootroot00000000000000atlas-0.45.0/atlas_io/cmake/atlas-io-import.cmake.in000066400000000000000000000002201513175025300222030ustar00rootroot00000000000000 include( CMakeFindDependencyMacro ) ## eckit find_dependency( eckit HINTS ${CMAKE_CURRENT_LIST_DIR}/../eckit @eckit_DIR@ @eckit_BINARY_DIR@ ) atlas-0.45.0/atlas_io/eckit_codec_adaptor/000077500000000000000000000000001513175025300204475ustar00rootroot00000000000000atlas-0.45.0/atlas_io/eckit_codec_adaptor/CMakeLists.txt000066400000000000000000000000261513175025300232050ustar00rootroot00000000000000add_subdirectory(src) atlas-0.45.0/atlas_io/eckit_codec_adaptor/src/000077500000000000000000000000001513175025300212365ustar00rootroot00000000000000atlas-0.45.0/atlas_io/eckit_codec_adaptor/src/CMakeLists.txt000066400000000000000000000000341513175025300237730ustar00rootroot00000000000000add_subdirectory(atlas_io) atlas-0.45.0/atlas_io/eckit_codec_adaptor/src/atlas_io/000077500000000000000000000000001513175025300230315ustar00rootroot00000000000000atlas-0.45.0/atlas_io/eckit_codec_adaptor/src/atlas_io/CMakeLists.txt000066400000000000000000000006271513175025300255760ustar00rootroot00000000000000 ecbuild_add_library( TARGET atlas_io INSTALL_HEADERS ALL HEADER_DESTINATION include/atlas_io PUBLIC_LIBS eckit_codec PUBLIC_INCLUDES $ SOURCES atlas-io.h Trace.cc Trace.h detail/BlackMagic.h LINKER_LANGUAGE CXX ) target_compile_features( atlas_io PUBLIC cxx_std_17 ) atlas-0.45.0/atlas_io/eckit_codec_adaptor/src/atlas_io/Trace.cc000066400000000000000000000025241513175025300244010ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "Trace.h" #include "eckit/log/CodeLocation.h" namespace atlas { namespace io { atlas::io::Trace::Trace(const eckit::CodeLocation& loc) { for (size_t id = 0; id < TraceHookRegistry::size(); ++id) { if (TraceHookRegistry::enabled(id)) { hooks_.emplace_back(TraceHookRegistry::hook(id)(loc, loc.func())); } } } Trace::Trace(const eckit::CodeLocation& loc, const std::string& title) { for (size_t id = 0; id < TraceHookRegistry::size(); ++id) { if (TraceHookRegistry::enabled(id)) { hooks_.emplace_back(TraceHookRegistry::hook(id)(loc, title)); } } } Trace::Trace(const eckit::CodeLocation& loc, const std::string& title, const Labels& labels) { for (size_t id = 0; id < TraceHookRegistry::size(); ++id) { if (TraceHookRegistry::enabled(id)) { hooks_.emplace_back(TraceHookRegistry::hook(id)(loc, title)); } } } } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/eckit_codec_adaptor/src/atlas_io/Trace.h000066400000000000000000000047211513175025300242440ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include #include #include namespace eckit { class CodeLocation; } namespace atlas { namespace io { struct TraceHook { TraceHook() = default; virtual ~TraceHook() = default; }; struct TraceHookRegistry { using TraceHookBuilder = std::function(const eckit::CodeLocation&, const std::string&)>; std::vector hooks; std::vector enabled_; static TraceHookRegistry& instance() { static TraceHookRegistry instance; return instance; } static size_t add(TraceHookBuilder&& hook) { instance().hooks.emplace_back(hook); instance().enabled_.emplace_back(true); return instance().hooks.size() - 1; } static size_t add(const TraceHookBuilder& hook) { instance().hooks.emplace_back(hook); instance().enabled_.emplace_back(true); return instance().hooks.size() - 1; } static void enable(size_t id) { instance().enabled_[id] = true; } static void disable(size_t id) { instance().enabled_[id] = false; } static bool enabled(size_t id) { return instance().enabled_[id]; } static size_t size() { return instance().hooks.size(); } static TraceHookBuilder& hook(size_t id) { return instance().hooks[id]; } static size_t invalidId() { return std::numeric_limits::max(); } private: TraceHookRegistry() = default; }; struct Trace { using Labels = std::vector; Trace(const eckit::CodeLocation& loc); Trace(const eckit::CodeLocation& loc, const std::string& title); Trace(const eckit::CodeLocation& loc, const std::string& title, const Labels& labels); private: std::vector> hooks_; }; } // namespace io } // namespace atlas #include "atlas_io/detail/BlackMagic.h" #define ATLAS_IO_TRACE(...) __ATLAS_IO_TYPE(::atlas::io::Trace, Here() __ATLAS_IO_COMMA_ARGS(__VA_ARGS__)) #define ATLAS_IO_TRACE_SCOPE(...) __ATLAS_IO_TYPE_SCOPE(::atlas::io::Trace, Here() __ATLAS_IO_COMMA_ARGS(__VA_ARGS__)) atlas-0.45.0/atlas_io/eckit_codec_adaptor/src/atlas_io/atlas-io.h000066400000000000000000000066761513175025300247320ustar00rootroot00000000000000/* * (C) Copyright 2023- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/codec/codec.h" namespace atlas::io { // Encoding/Decoding using Metadata = ::eckit::codec::Metadata; using DataType = ::eckit::codec::DataType; using Data = ::eckit::codec::Data; using Encoder = ::eckit::codec::Encoder; using Decoder = ::eckit::codec::Decoder; // Record using Record = ::eckit::codec::Record; using RecordWriter = ::eckit::codec::RecordWriter; using RecordPrinter = ::eckit::codec::RecordPrinter; using RecordItemReader = ::eckit::codec::RecordItemReader; using RecordReader = ::eckit::codec::RecordReader; // I/O using Session = ::eckit::codec::Session; using Mode = ::eckit::codec::Mode; using Stream = ::eckit::codec::Stream; using FileStream = ::eckit::codec::FileStream; using InputFileStream = ::eckit::codec::InputFileStream; using OutputFileStream = ::eckit::codec::OutputFileStream; // Array using ArrayReference = ::eckit::codec::ArrayReference; using ArrayMetadata = ::eckit::codec::ArrayMetadata; using ArrayShape = ::eckit::codec::ArrayShape; // Exceptions using Exception = ::eckit::codec::Exception; using NotEncodable = ::eckit::codec::NotEncodable; using NotDecodable = ::eckit::codec::NotDecodable; using InvalidRecord = ::eckit::codec::InvalidRecord; using DataCorruption = ::eckit::codec::DataCorruption; using WriteError = ::eckit::codec::WriteError; // Type traits template static constexpr bool is_interpretable() { return ::eckit::codec::is_interpretable(); } template static constexpr bool is_encodable() { return ::eckit::codec::is_encodable(); } template static constexpr bool is_decodable() { return ::eckit::codec::is_decodable(); } template static constexpr bool can_encode_metadata() { return ::eckit::codec::can_encode_metadata(); } template static constexpr bool can_encode_data() { return ::eckit::codec::can_encode_metadata(); } namespace tag { using disable_static_assert = ::eckit::codec::tag::disable_static_assert; using enable_static_assert = ::eckit::codec::tag::enable_static_assert; } // Functions using ::eckit::codec::ref; using ::eckit::codec::copy; using ::eckit::codec::encode; using ::eckit::codec::decode; using ::eckit::codec::interprete; using ::eckit::codec::link; using ::eckit::codec::make_datatype; // template // using make_datatype = eckit::codec::make_datatype; namespace defaults { using ::eckit::codec::defaults::compression_algorithm; using ::eckit::codec::defaults::checksum_algorithm; using ::eckit::codec::defaults::checksum_read; using ::eckit::codec::defaults::checksum_write; } } #define ATLAS_IO_ASSERT(X) ASSERT(X) #define ATLAS_IO_ASSERT_MSG(X, M) ASSERT_MSG(X, M) atlas-0.45.0/atlas_io/eckit_codec_adaptor/src/atlas_io/detail/000077500000000000000000000000001513175025300242735ustar00rootroot00000000000000atlas-0.45.0/atlas_io/eckit_codec_adaptor/src/atlas_io/detail/BlackMagic.h000066400000000000000000000125651513175025300264320ustar00rootroot00000000000000/* * (C) Copyright 2013 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once // This file contains preprocessor black magic. It contains macros that // can return the number of arguments passed //----------------------------------------------------------------------------------------------------------- // Public /// Returns the number of passed arguments #define __ATLAS_IO_NARG(...) /// Splice a and b together #define __ATLAS_IO_SPLICE(a, b) #define __ATLAS_IO_STRINGIFY(a) a #define __ATLAS_IO_TYPE(Type, ...) #define __ATLAS_IO_TYPE_SCOPE(Type, ...) //----------------------------------------------------------------------------------------------------------- // Details // Undefine these, to be redefined further down. #undef __ATLAS_IO_NARG #undef __ATLAS_IO_SPLICE #undef __ATLAS_IO_TYPE #undef __ATLAS_IO_TYPE_SCOPE #define __ATLAS_IO_REVERSE 5, 4, 3, 2, 1, 0 #define __ATLAS_IO_ARGN(_1, _2, _3, _4, _5, N, ...) N #define __ATLAS_IO_NARG__(dummy, ...) __ATLAS_IO_ARGN(__VA_ARGS__) #define __ATLAS_IO_NARG_(...) __ATLAS_IO_NARG__(dummy, ##__VA_ARGS__, __ATLAS_IO_REVERSE) #define __ATLAS_IO_SPLICE(a, b) __ATLAS_IO_SPLICE_1(a, b) #define __ATLAS_IO_SPLICE_1(a, b) __ATLAS_IO_SPLICE_2(a, b) #define __ATLAS_IO_SPLICE_2(a, b) a##b #define __ATLAS_IO_ARG16(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, ...) _15 #define __ATLAS_IO_HAS_COMMA(...) __ATLAS_IO_ARG16(__VA_ARGS__, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0) #define __ATLAS_IO_TRIGGER_PARENTHESIS(...) , #define __ATLAS_IO_ISEMPTY(...) \ __ATLAS_IO_ISEMPTY_(/* test if there is just one argument, eventually an empty \ one */ \ __ATLAS_IO_HAS_COMMA(__VA_ARGS__), /* test if \ _TRIGGER_PARENTHESIS_ \ together with the \ argument adds a comma */ \ __ATLAS_IO_HAS_COMMA( \ __ATLAS_IO_TRIGGER_PARENTHESIS __VA_ARGS__), /* test if the argument together with \ a parenthesis adds a comma */ \ __ATLAS_IO_HAS_COMMA(__VA_ARGS__(/*empty*/)), /* test if placing it between \ __ATLAS_IO_TRIGGER_PARENTHESIS and the \ parenthesis adds a comma */ \ __ATLAS_IO_HAS_COMMA(__ATLAS_IO_TRIGGER_PARENTHESIS __VA_ARGS__(/*empty*/))) #define __ATLAS_IO_PASTE5(_0, _1, _2, _3, _4) _0##_1##_2##_3##_4 #define __ATLAS_IO_ISEMPTY_(_0, _1, _2, _3) \ __ATLAS_IO_HAS_COMMA(__ATLAS_IO_PASTE5(__ATLAS_IO_IS_EMPTY_CASE_, _0, _1, _2, _3)) #define __ATLAS_IO_IS_EMPTY_CASE_0001 , #define __ATLAS_IO_NARG(...) __ATLAS_IO_SPLICE(__ATLAS_IO_CALL_NARG_, __ATLAS_IO_ISEMPTY(__VA_ARGS__))(__VA_ARGS__) #define __ATLAS_IO_CALL_NARG_1(...) 0 #define __ATLAS_IO_CALL_NARG_0 __ATLAS_IO_NARG_ #define __ATLAS_IO_COMMA_ARGS(...) \ __ATLAS_IO_SPLICE(__ATLAS_IO_COMMA_ARGS_, __ATLAS_IO_ISEMPTY(__VA_ARGS__))(__VA_ARGS__) #define __ATLAS_IO_COMMA_ARGS_1(...) #define __ATLAS_IO_COMMA_ARGS_0(...) , __VA_ARGS__ #define __ATLAS_IO_ARGS_OR_DUMMY(...) \ __ATLAS_IO_SPLICE(__ATLAS_IO_ARGS_OR_DUMMY_, __ATLAS_IO_ISEMPTY(__VA_ARGS__)) \ (__VA_ARGS__) #define __ATLAS_IO_ARGS_OR_DUMMY_0(...) __VA_ARGS__ #define __ATLAS_IO_ARGS_OR_DUMMY_1(...) 0 #define __ATLAS_IO_TYPE(Type, ...) \ __ATLAS_IO_SPLICE(__ATLAS_IO_TYPE_, __ATLAS_IO_ISEMPTY(__VA_ARGS__)) \ (Type, __ATLAS_IO_ARGS_OR_DUMMY(__VA_ARGS__)) #define __ATLAS_IO_TYPE_1(Type, dummy) Type __ATLAS_IO_SPLICE(__variable_, __LINE__) #define __ATLAS_IO_TYPE_0(Type, ...) Type __ATLAS_IO_SPLICE(__variable_, __LINE__)(__VA_ARGS__) #define __ATLAS_IO_TYPE_SCOPE(Type, ...) \ __ATLAS_IO_SPLICE(__ATLAS_IO_TYPE_SCOPE_, __ATLAS_IO_ISEMPTY(__VA_ARGS__)) \ (Type, __ATLAS_IO_ARGS_OR_DUMMY(__VA_ARGS__)) #define __ATLAS_IO_TYPE_SCOPE_1(Type, ...) \ for (bool __ATLAS_IO_SPLICE(__done_, __LINE__) = false; __ATLAS_IO_SPLICE(__done_, __LINE__) != true;) \ for (Type __ATLAS_IO_SPLICE(__variable_, __LINE__); __ATLAS_IO_SPLICE(__done_, __LINE__) != true; \ __ATLAS_IO_SPLICE(__done_, __LINE__) = true) #define __ATLAS_IO_TYPE_SCOPE_0(Type, ...) \ for (bool __ATLAS_IO_SPLICE(__done_, __LINE__) = false; __ATLAS_IO_SPLICE(__done_, __LINE__) != true;) \ for (Type __ATLAS_IO_SPLICE(__variable_, __LINE__)(__VA_ARGS__); __ATLAS_IO_SPLICE(__done_, __LINE__) != true; \ __ATLAS_IO_SPLICE(__done_, __LINE__) = true) atlas-0.45.0/atlas_io/src/000077500000000000000000000000001513175025300152705ustar00rootroot00000000000000atlas-0.45.0/atlas_io/src/CMakeLists.txt000066400000000000000000000000631513175025300200270ustar00rootroot00000000000000add_subdirectory(atlas_io) add_subdirectory(tools) atlas-0.45.0/atlas_io/src/atlas_io/000077500000000000000000000000001513175025300170635ustar00rootroot00000000000000atlas-0.45.0/atlas_io/src/atlas_io/CMakeLists.txt000066400000000000000000000053131513175025300216250ustar00rootroot00000000000000 if( HAVE_ECKIT_DEVELOP ) set( ATLAS_IO_ECKIT_DEVELOP 1 ) else() set( ATLAS_IO_ECKIT_DEVELOP 0 ) endif() ecbuild_parse_version( ${eckit_VERSION} PREFIX ATLAS_IO_ECKIT ) math( EXPR ATLAS_IO_ECKIT_VERSION_INT "( 10000 * ${ATLAS_IO_ECKIT_VERSION_MAJOR} ) + ( 100 * ${ATLAS_IO_ECKIT_VERSION_MINOR} ) + ${ATLAS_IO_ECKIT_VERSION_PATCH}" ) configure_file( detail/defines.h.in detail/defines.h ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/detail/defines.h DESTINATION ${INSTALL_INCLUDE_DIR}/atlas_io/detail ) ecbuild_add_library( TARGET atlas_io INSTALL_HEADERS ALL HEADER_DESTINATION include/atlas_io PUBLIC_LIBS eckit PUBLIC_INCLUDES $ $ SOURCES atlas-io.h Data.cc Data.h detail/Assert.h detail/Base64.cc detail/Base64.h detail/Checksum.h detail/Checksum.cc detail/DataInfo.h detail/DataType.cc detail/DataType.h detail/Decoder.cc detail/Decoder.h detail/Defaults.h detail/Encoder.cc detail/Encoder.h detail/Endian.h detail/Link.cc detail/Link.h detail/ParsedRecord.h detail/RecordInfo.h detail/RecordSections.h detail/Reference.h detail/sfinae.h detail/StaticAssert.h detail/tag.h detail/Time.cc detail/Time.h detail/Type.h detail/TypeTraits.h detail/Version.h Exceptions.cc Exceptions.h FileStream.cc FileStream.h Metadata.cc Metadata.h print/TableFormat.cc print/TableFormat.h print/JSONFormat.cc print/JSONFormat.h print/Bytes.cc print/Bytes.h ReadRequest.cc ReadRequest.h Record.cc Record.h RecordItem.cc RecordItem.h RecordItemReader.cc RecordItemReader.h RecordPrinter.cc RecordPrinter.h RecordReader.cc RecordReader.h RecordWriter.cc RecordWriter.h Session.cc Session.h Stream.cc Stream.h Trace.cc Trace.h types/array.h types/array/ArrayMetadata.cc types/array/ArrayMetadata.h types/array/ArrayReference.cc types/array/ArrayReference.h types/array/adaptors/StdArrayAdaptor.h types/array/adaptors/StdVectorAdaptor.h types/array/adaptors/StdVectorOfStdArrayAdaptor.h types/string.h types/scalar.h types/scalar.cc ${CMAKE_CURRENT_BINARY_DIR}/detail/defines.h ) target_compile_features( atlas_io PUBLIC cxx_std_17 ) atlas-0.45.0/atlas_io/src/atlas_io/Data.cc000066400000000000000000000065551513175025300202560ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "Data.h" #include #include "eckit/utils/Compressor.h" #include "atlas_io/Stream.h" #include "atlas_io/Trace.h" #include "atlas_io/detail/Assert.h" #include "atlas_io/detail/Checksum.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- Data::Data(void* p, size_t size): buffer_(p, size), size_(size) {} std::uint64_t Data::write(Stream& out) const { ATLAS_IO_TRACE(); if (size()) { ATLAS_IO_ASSERT(buffer_.size() >= size()); return out.write(buffer_.data(), size()); } return 0; } std::uint64_t Data::read(Stream& in, size_t size) { if (size > size_) { buffer_.resize(size); size_ = size; } return in.read(buffer_, size); } void Data::compress(const std::string& compression) { ATLAS_IO_TRACE("compress(" + compression + ")"); if (size_) { auto compressor = std::unique_ptr(eckit::CompressorFactory::instance().build(compression)); if (dynamic_cast(compressor.get())) { return; } eckit::Buffer compressed(size_t(1.2 * size_)); size_ = compressor->compress(buffer_, size_, compressed); buffer_ = std::move(compressed); } } void Data::decompress(const std::string& compression, size_t uncompressed_size) { ATLAS_IO_TRACE("decompress(" + compression + ")"); auto compressor = std::unique_ptr(eckit::CompressorFactory::instance().build(compression)); if (dynamic_cast(compressor.get())) { return; } eckit::Buffer uncompressed(size_t(1.2 * uncompressed_size)); compressor->uncompress(buffer_, size_, uncompressed, uncompressed_size); size_ = uncompressed_size; buffer_ = std::move(uncompressed); } void Data::clear() { buffer_ = eckit::Buffer{}; size_ = 0; } std::string Data::checksum(const std::string& algorithm) const { return atlas::io::checksum(buffer_, size_, algorithm); } void Data::assign(const Data& other) { if (other.size() > buffer_.size()) { buffer_.resize(other.size()); } size_ = other.size(); buffer_.copy(other.buffer_, size_); } void Data::assign(const void* p, size_t s) { if (s > size()) { buffer_.resize(s); } size_ = s; buffer_.copy(p, size_); } Data& compress(Data& data, const std::string& compression) { data.compress(compression); return data; } Data& decompress(Data& data, const std::string& compression, size_t uncompressed_size) { data.decompress(compression, uncompressed_size); return data; } //--------------------------------------------------------------------------------------------------------------------- void encode(const Data& in, Data& out) { out.assign(in); } //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/Data.h000066400000000000000000000032371513175025300201120ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include "eckit/io/Buffer.h" namespace atlas { namespace io { class Stream; //--------------------------------------------------------------------------------------------------------------------- class Data { public: Data() = default; Data(void*, size_t); Data(Data&&) = default; Data& operator=(Data&&) = default; operator const void*() const { return data(); } const void* data() const { return buffer_.data(); } size_t size() const { return size_; } void assign(const Data& other); void assign(const void*, size_t); void clear(); std::uint64_t write(Stream& out) const; std::uint64_t read(Stream& in, size_t size); void compress(const std::string& compression); void decompress(const std::string& compression, size_t uncompressed_size); std::string checksum(const std::string& algorithm = "") const; private: eckit::Buffer buffer_; size_t size_{0}; }; //--------------------------------------------------------------------------------------------------------------------- void encode_data(const Data&, Data& out); //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/Exceptions.cc000066400000000000000000000067321513175025300215230ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "Exceptions.h" #include "atlas_io/detail/defines.h" #if ATLAS_HAVE_CXXABI_H #include #endif #include #include namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- std::string demangle(const char* name) { #if ATLAS_HAVE_CXXABI_H int status = -4; std::unique_ptr res{abi::__cxa_demangle(name, nullptr, nullptr, &status), std::free}; return (status == 0) ? res.get() : name; #else return name; #endif } //--------------------------------------------------------------------------------------------------------------------- NotEncodable::NotEncodable(const std::string& type_name): Exception{[&type_name] { std::stringstream message; message << "atlas::io::NotEncodable: Cannot encode values of type " << type_name << "."; message << "\n Implement the functions" "\n" "\n void encode_data(const " << type_name << "&, atlas::io::Data& );" "\n size_t encode_metadata(const " << type_name << "&, atlas::io::Metadata& );" "\n" "\n or alternatively a conversion function to atlas::io::types::ArrayView" "\n" "\n void interprete(const " << type_name << "&, atlas::io::types::ArrayView& )" "\n" "\n Rules of argument-dependent-lookup apply." "\n --> Functions need to be declared in namespace of any of the arguments."; return message.str(); }()} {} //--------------------------------------------------------------------------------------------------------------------- NotDecodable::NotDecodable(const std::string& type_name): Exception{[&type_name] { std::stringstream message; message << "atlas::io::NotDecodable: Cannot decode values of type " << type_name << "."; message << "\n Implement the functions" "\n" "\n void decode( const atlas::io::Metadata&, const atlas::io::Data&, " << type_name << "& );" "\n" "\n Rules of argument-dependent-lookup apply." "\n --> Functions need to be declared in namespace of any of the arguments."; return message.str(); }()} {} //--------------------------------------------------------------------------------------------------------------------- Exception::~Exception() = default; NotEncodable::~NotEncodable() = default; NotDecodable::~NotDecodable() = default; InvalidRecord::~InvalidRecord() = default; DataCorruption::~DataCorruption() = default; WriteError::~WriteError() = default; //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/Exceptions.h000066400000000000000000000054131513175025300213600ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/exception/Exceptions.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- std::string demangle(const char*); template std::string demangle() { return demangle(typeid(T).name()); } //--------------------------------------------------------------------------------------------------------------------- class Exception : public eckit::Exception { public: using eckit::Exception::Exception; ~Exception() override; }; //--------------------------------------------------------------------------------------------------------------------- class NotEncodable : Exception { public: NotEncodable(const std::string& type_name); template NotEncodable(const T&): NotEncodable{demangle::type>()} {} ~NotEncodable() override; }; //--------------------------------------------------------------------------------------------------------------------- class NotDecodable : public Exception { public: NotDecodable(const std::string& type_name); template NotDecodable(const T&): NotDecodable{demangle::type>()} {} ~NotDecodable() override; }; //--------------------------------------------------------------------------------------------------------------------- class InvalidRecord : public Exception { public: InvalidRecord(const std::string& message): Exception("atlas::io::InvalidRecord: " + message) {} ~InvalidRecord() override; }; //--------------------------------------------------------------------------------------------------------------------- class DataCorruption : public Exception { public: DataCorruption(const std::string& message): Exception("atlas::io::DataCorruption: " + message) {} ~DataCorruption() override; }; //--------------------------------------------------------------------------------------------------------------------- class WriteError : public Exception { public: WriteError(const std::string& message): Exception("atlas::io::WriteError: " + message) {} ~WriteError() override; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/FileStream.cc000066400000000000000000000120471513175025300214310ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "atlas_io/FileStream.h" #include "eckit/io/FileHandle.h" #include "eckit/io/PooledHandle.h" #include "atlas_io/Session.h" #include "atlas_io/Trace.h" namespace atlas { namespace io { namespace { //--------------------------------------------------------------------------------------------------------------------- /// DataHandle that implements file IO for reading and writing /// Main differences with eckit::FileHandle: /// - Automatic opening and closing of file /// - Openmode argument: /// * read: for reading /// * write: for writing, will overwrite existing file /// * append: for appending implemented via write and seek to eof. /// - ATLAS_IO_TRACE recording class FileHandle : public eckit::FileHandle { public: FileHandle(const eckit::PathName& path, char openmode): eckit::FileHandle(path, openmode == 'a' /*overwrite*/) { ATLAS_IO_TRACE("FileHandle::open(" + eckit::FileHandle::path() + "," + openmode + ")"); if (openmode == 'r') { openForRead(); } else if (openmode == 'w' || (openmode == 'a' && not path.exists())) { openForWrite(0); } else if (openmode == 'a') { openForWrite(path.size()); seek(eckit::Offset(path.size())); } } void close() override { if (not closed_) { ATLAS_IO_TRACE("FileHandle::close(" + path() + ")"); eckit::FileHandle::close(); closed_ = true; } } FileHandle(const eckit::PathName& path, Mode openmode): FileHandle(path, openmode == Mode::read ? 'r' : openmode == Mode::write ? 'w' : 'a') {} FileHandle(const eckit::PathName& path, const std::string& openmode): FileHandle(path, openmode[0]) {} ~FileHandle() override { close(); } private: bool closed_{false}; }; //--------------------------------------------------------------------------------------------------------------------- /// DataHandle that implements file reading only. /// Internally there is a registry of opened files which avoids /// opening the same file multiple times. /// Note that close() will not actually close the file when there /// is another PooledHandle referencing the same file. /// /// Main difference with eckit::PooledHandle /// - Automatic opening and closing of file /// - ATLAS_IO_TRACE recording class PooledHandle : public eckit::PooledHandle { public: PooledHandle(const eckit::PathName& path): eckit::PooledHandle(path), path_(path) { ATLAS_IO_TRACE("PooledHandle::open(" + path_.baseName() + ")"); openForRead(); } ~PooledHandle() override { ATLAS_IO_TRACE("PooledHandle::close(" + path_.baseName() + ")"); close(); } eckit::PathName path_; }; } // namespace //--------------------------------------------------------------------------------------------------------------------- FileStream::FileStream(const eckit::PathName& path, char openmode): Stream([&path, &openmode]() -> eckit::DataHandle* { eckit::DataHandle* datahandle; if (openmode == 'r') { datahandle = new PooledHandle(path); } else { datahandle = new FileHandle(path, openmode); } return datahandle; }()) { if (openmode == 'r') { // Keep the PooledHandle alive until the end of active session Session::store(*this); } } FileStream::FileStream(const eckit::PathName& path, Mode openmode): FileStream(path, openmode == Mode::read ? 'r' : openmode == Mode::write ? 'w' : 'a') {} FileStream::FileStream(const eckit::PathName& path, const std::string& openmode): FileStream(path, openmode[0]) {} //--------------------------------------------------------------------------------------------------------------------- InputFileStream::InputFileStream(const eckit::PathName& path): FileStream(path, Mode::read) {} //--------------------------------------------------------------------------------------------------------------------- OutputFileStream::OutputFileStream(const eckit::PathName& path, Mode openmode): FileStream(path, openmode) {} OutputFileStream::OutputFileStream(const eckit::PathName& path, const std::string& openmode): FileStream(path, openmode) {} OutputFileStream::OutputFileStream(const eckit::PathName& path, char openmode): FileStream(path, openmode) {} void OutputFileStream::close() { datahandle().close(); } //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/FileStream.h000066400000000000000000000035751513175025300213010ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include "eckit/filesystem/PathName.h" #include "atlas_io/Stream.h" namespace eckit { class DataHandle; } namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- enum class Mode { read, append, write, }; //--------------------------------------------------------------------------------------------------------------------- class FileStream : public Stream { public: FileStream(const eckit::PathName& path, Mode openmode); FileStream(const eckit::PathName& path, char openmode); FileStream(const eckit::PathName& path, const std::string& openmode); }; //--------------------------------------------------------------------------------------------------------------------- class InputFileStream : public FileStream { public: InputFileStream(const eckit::PathName& path); }; //--------------------------------------------------------------------------------------------------------------------- class OutputFileStream : public FileStream { public: OutputFileStream(const eckit::PathName& path, Mode openmode = Mode::write); OutputFileStream(const eckit::PathName& path, const std::string& openmode); OutputFileStream(const eckit::PathName& path, char openmode); void close(); }; //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/Metadata.cc000066400000000000000000000047151513175025300211210ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "Metadata.h" #include #include #include #include "eckit/log/JSON.h" #include "atlas_io/atlas_compat.h" #include "atlas_io/detail/Assert.h" #include "atlas_io/Exceptions.h" #include "atlas_io/types/array/ArrayReference.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- size_t uncompressed_size(const atlas::io::Metadata& m) { if (m.has("data.size")) { return m.getUnsigned("data.size"); } else if (m.has("type")) { if (m.getString("type") == "array") { atlas::io::ArrayMetadata array(m); return array.bytes(); } } std::stringstream err; err << "Could not compute uncompressed data size from metadata \n"; write(m, err); throw Exception(err.str()); } //--------------------------------------------------------------------------------------------------------------------- void write(const atlas::io::Metadata& metadata, std::ostream& out) { eckit::JSON js(out, eckit::JSON::Formatting::indent(4)); js << metadata; } void write(const atlas::io::Metadata& metadata, atlas::io::Stream& out) { std::stringstream ss; write(metadata, ss); std::string s = ss.str(); out.write(s.data(), s.size()); } //--------------------------------------------------------------------------------------------------------------------- void Metadata::link(Metadata&& linked) { std::string initial_link = link(); ATLAS_IO_ASSERT(initial_link.size()); data = std::move(linked.data); record = std::move(linked.record); set(linked); // Set link to initial_link, in case the link is itself another link set("link", initial_link); } std::string Metadata::json() const { std::stringstream s; eckit::JSON js(s, eckit::JSON::Formatting::compact()); js << *this; return s.str(); } //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/Metadata.h000066400000000000000000000065701513175025300207640ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include "atlas_io/Stream.h" #include "atlas_io/detail/Checksum.h" #include "atlas_io/detail/DataInfo.h" #include "atlas_io/detail/Endian.h" #include "atlas_io/detail/Link.h" #include "atlas_io/detail/RecordInfo.h" #include "atlas_io/detail/Type.h" #include "eckit/config/LocalConfiguration.h" #include "eckit/value/Value.h" namespace atlas { namespace io { class Metadata; class Stream; //--------------------------------------------------------------------------------------------------------------------- size_t uncompressed_size(const atlas::io::Metadata& m); //--------------------------------------------------------------------------------------------------------------------- class Metadata : public eckit::LocalConfiguration { public: using eckit::LocalConfiguration::LocalConfiguration; Metadata(): eckit::LocalConfiguration() {} Link link() const { return Link{getString("link", "")}; } Type type() const { return Type{getString("type", "")}; } void link(atlas::io::Metadata&&); std::string json() const; DataInfo data; RecordInfo record; // extended LocalConfiguration: using eckit::LocalConfiguration::set; Metadata& set(const eckit::Configuration& other) { #if ATLAS_IO_ECKIT_VERSION_AT_LEAST(1, 26, 0) || ATLAS_IO_ECKIT_DEVELOP LocalConfiguration::set(other); #else eckit::Value& root = const_cast(get()); auto& other_root = other.get(); std::vector other_keys; eckit::fromValue(other_keys, other_root.keys()); for (auto& key : other_keys) { root[key] = other_root[key]; } #endif return *this; } /// @brief Constructor immediately setting a value. template Metadata(const std::string& name, const ValueT& value) { set(name, value); } /// @brief Constructor starting from a Configuration Metadata(const eckit::Configuration& other): eckit::LocalConfiguration(other) {} Metadata& remove(const std::string& name) { #if ATLAS_IO_ECKIT_VERSION_AT_LEAST(1, 26, 0) || ATLAS_IO_ECKIT_DEVELOP LocalConfiguration::remove(name); #else eckit::Value& root = const_cast(get()); root.remove(name); #endif return *this; } std::vector keys() const { #if ATLAS_IO_ECKIT_VERSION_AT_LEAST(1, 26, 0) || ATLAS_IO_ECKIT_DEVELOP return LocalConfiguration::keys(); #else std::vector result; eckit::fromValue(result, get().keys()); return result; #endif } }; //--------------------------------------------------------------------------------------------------------------------- void write(const atlas::io::Metadata&, std::ostream& out); void write(const atlas::io::Metadata&, atlas::io::Stream& out); //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/ReadRequest.cc000066400000000000000000000111311513175025300216130ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "ReadRequest.h" #include "eckit/log/Log.h" #include "atlas_io/Exceptions.h" #include "atlas_io/RecordItemReader.h" #include "atlas_io/Trace.h" #include "atlas_io/detail/Assert.h" #include "atlas_io/detail/Checksum.h" #include "atlas_io/detail/Defaults.h" namespace atlas { namespace io { static std::string stream_path(Stream stream) { std::stringstream s; s << &stream.datahandle(); return s.str(); } //--------------------------------------------------------------------------------------------------------------------- ReadRequest::ReadRequest(const std::string& URI, atlas::io::Decoder* decoder): uri_(URI), decoder_(decoder), item_(new RecordItem()) { do_checksum_ = defaults::checksum_read(); ATLAS_IO_ASSERT(uri_.size()); } ReadRequest::ReadRequest(Stream stream, size_t offset, const std::string& key, Decoder* decoder): stream_{stream}, offset_{offset}, key_{key}, uri_{"stream:" + stream_path(stream) + "?offset=key=" + key_}, decoder_(decoder), item_(new RecordItem()) { do_checksum_ = defaults::checksum_read(); ATLAS_IO_ASSERT(stream_); } //--------------------------------------------------------------------------------------------------------------------- ReadRequest::ReadRequest(ReadRequest&& other): stream_{other.stream_}, offset_{other.offset_}, key_{other.key_}, uri_(std::move(other.uri_)), decoder_(std::move(other.decoder_)), item_(std::move(other.item_)), do_checksum_{other.do_checksum_}, finished_{other.finished_} { other.do_checksum_ = true; other.finished_ = true; } //--------------------------------------------------------------------------------------------------------------------- ReadRequest::~ReadRequest() { if (item_) { if (not finished_) { eckit::Log::error() << "Request for " << uri_ << " was not completed." << std::endl; } } } //--------------------------------------------------------------------------------------------------------------------- void ReadRequest::read() { if (item_->empty()) { if (stream_) { RecordItemReader{stream_, offset_, key_}.read(*item_); } else { RecordItemReader(uri_).read(*item_); } } } //--------------------------------------------------------------------------------------------------------------------- void ReadRequest::checksum(bool b) { do_checksum_ = b; } void ReadRequest::checksum() { if (not do_checksum_) { return; } Checksum encoded_checksum{item_->metadata().data.checksum()}; if (not encoded_checksum.available()) { return; } Checksum computed_checksum{item_->data().checksum(encoded_checksum.algorithm())}; if (computed_checksum.available() && (computed_checksum.str() != encoded_checksum.str())) { std::stringstream err; err << "Mismatch in checksums for " << uri_ << ".\n"; err << " Encoded: [" << encoded_checksum.str() << "].\n"; err << " Computed: [" << computed_checksum.str() << "]."; throw DataCorruption(err.str()); } do_checksum_ = false; } //--------------------------------------------------------------------------------------------------------------------- void ReadRequest::decompress() { read(); item_->decompress(); } //--------------------------------------------------------------------------------------------------------------------- void ReadRequest::decode() { decompress(); io::decode(item_->metadata(), item_->data(), *decoder_); if (item_->data().size()) { ATLAS_IO_TRACE_SCOPE("deallocate"); item_->clear(); } else { item_->clear(); } } //--------------------------------------------------------------------------------------------------------------------- void ReadRequest::wait() { ATLAS_IO_TRACE("ReadRequest::wait(" + uri_ + ")"); if (item_) { if (not finished_) { read(); checksum(); decompress(); decode(); } finished_ = true; } } //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/ReadRequest.h000066400000000000000000000036111513175025300214610ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include #include "atlas_io/RecordItem.h" #include "atlas_io/detail/Decoder.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- class ReadRequest { public: ReadRequest(ReadRequest&& other); template ReadRequest(Stream stream, size_t offset, const std::string& key, T& value): ReadRequest{stream, offset, key, new Decoder(value)} {} template ReadRequest(const std::string& URI, T& value): ReadRequest{URI, new Decoder(value)} {} template ReadRequest(const RecordItem::URI& URI, T& value): ReadRequest{URI.str(), value} {} ~ReadRequest(); void read(); void checksum(); void decompress(); void decode(); void wait(); void checksum(bool); private: ReadRequest(const std::string& URI, Decoder* decoder); ReadRequest(Stream, size_t offset, const std::string& key, Decoder*); ReadRequest() = delete; ReadRequest(const ReadRequest&) = delete; Stream stream_; size_t offset_; std::string key_; std::string uri_; std::unique_ptr decoder_; std::unique_ptr item_; bool do_checksum_{true}; bool finished_{false}; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/Record.cc000066400000000000000000000262461513175025300206220ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "atlas_io/Record.h" #include "eckit/config/YAMLConfiguration.h" #include "eckit/filesystem/URI.h" #include "atlas_io/Exceptions.h" #include "atlas_io/Trace.h" #include "atlas_io/detail/Assert.h" #include "atlas_io/detail/ParsedRecord.h" #include "atlas_io/detail/Version.h" namespace atlas { namespace io { namespace { //--------------------------------------------------------------------------------------------------------------------- template inline size_t read_struct(IStream& in, Struct& s) { static_assert(Struct::bytes == sizeof(Struct), ""); return in.read(reinterpret_cast(&s), sizeof(Struct)); } //--------------------------------------------------------------------------------------------------------------------- template inline Struct read_struct(IStream& in) { Struct s; if (read_struct(in, s) != sizeof(Struct)) { throw InvalidRecord("Unexpected EOF reached"); } return s; } //--------------------------------------------------------------------------------------------------------------------- } // namespace //--------------------------------------------------------------------------------------------------------------------- Endian RecordHead::endian() const { if (magic_number == 1234) { return Endian::native; } else if (magic_number == 3523477504) { return Endian::swapped; } throw Exception("Mixed endianness is not supported", Here()); } //--------------------------------------------------------------------------------------------------------------------- std::string Record::URI::str() const { eckit::URI uri("file", eckit::PathName(path)); uri.query("offset", std::to_string(offset)); return uri.asRawString(); } //--------------------------------------------------------------------------------------------------------------------- Record::Record(): record_(new ParsedRecord()) {} //--------------------------------------------------------------------------------------------------------------------- bool Record::empty() const { return record_->head.record_length == 0; } //--------------------------------------------------------------------------------------------------------------------- const Metadata& Record::metadata(const std::string& key) const { if (record_->items.find(key) == record_->items.end()) { throw Exception("Record does not contain key \"" + key + "\"", Here()); } return record_->items.at(key); } //--------------------------------------------------------------------------------------------------------------------- Endian Record::endian() const { return record_->head.endian(); } //--------------------------------------------------------------------------------------------------------------------- Version Record::version() const { return record_->head.version; } //--------------------------------------------------------------------------------------------------------------------- Time Record::time() const { return record_->head.time; } //--------------------------------------------------------------------------------------------------------------------- uint64_t Record::size() const { return record_->head.record_length; } //--------------------------------------------------------------------------------------------------------------------- const std::vector& Record::keys() const { return record_->keys; } //--------------------------------------------------------------------------------------------------------------------- bool Record::has(const std::string& key) { return record_->items.find(key) != record_->items.end(); } //--------------------------------------------------------------------------------------------------------------------- Record::operator const ParsedRecord&() const { return *record_; } //--------------------------------------------------------------------------------------------------------------------- static void parse_record(ParsedRecord& record, const std::string& key, const Metadata& metadata) { if (metadata.type() || metadata.link()) { record.items.emplace(key, metadata); record.keys.emplace_back(key); } else { for (auto& next_key : metadata.keys()) { parse_record(record, key + "." + next_key, metadata.getSubConfiguration(next_key)); } } } //--------------------------------------------------------------------------------------------------------------------- Record& Record::read(Stream& in, bool read_to_end) { if (not empty()) { return *this; } ATLAS_IO_TRACE("read_metadata"); auto& r = record_->head; auto rbegin = in.position(); // Begin Record // ------------ if (atlas::io::read_struct(in, r) != sizeof(r)) { if (in.position() > sizeof(r.begin.string)) { if (not r.begin.valid()) { std::stringstream err; err << "Format is not recognized. Received: " << r.begin.string; throw InvalidRecord(err.str()); } } throw InvalidRecord("Unexpected EOF reached"); } if (not r.valid()) { std::stringstream err; err << "Format is not recognized. Received: " << r.begin.string; throw InvalidRecord(err.str()); } if (r.version < RecordHead{}.version) { throw InvalidRecord("Version of record (" + r.version.str() + ") is too old."); } if (r.metadata_length < sizeof(RecordMetadataSection::Begin) + sizeof(RecordMetadataSection::End)) { throw InvalidRecord("Unexpected metadata section length: " + std::to_string(r.metadata_length) + " < " + std::to_string(sizeof(RecordMetadataSection::Begin) + sizeof(RecordMetadataSection::End))); } if (r.index_length < sizeof(RecordDataIndexSection::Begin) + sizeof(RecordDataIndexSection::End)) { throw InvalidRecord("Unexpected data index section length."); } r.metadata_offset += rbegin; r.index_offset += rbegin; // Metadata section // ---------------- in.seek(r.metadata_offset); auto metadata_begin = atlas::io::read_struct(in); if (not metadata_begin.valid()) { throw InvalidRecord("Metadata section is not valid. Invalid section begin marker: [" + metadata_begin.str() + "]"); } std::string metadata_str; metadata_str.resize(size_t(r.metadata_length) - sizeof(RecordMetadataSection::Begin) - sizeof(RecordMetadataSection::End)); if (in.read(const_cast(metadata_str.data()), metadata_str.size()) != metadata_str.size()) { throw InvalidRecord("Unexpected EOF reached"); } auto metadata_end = atlas::io::read_struct(in); if (not metadata_end.valid()) { throw InvalidRecord("Metadata section is not valid. Invalid section end marker: [" + metadata_end.str() + "]"); } Checksum encoded_metadata_checksum(r.metadata_checksum); Checksum computed_metadata_checksum( atlas::io::checksum(metadata_str.data(), metadata_str.size(), encoded_metadata_checksum.algorithm())); if (computed_metadata_checksum.available() && encoded_metadata_checksum.str() != computed_metadata_checksum.str()) { std::stringstream err; err << "Mismatch in metadata checksum.\n"; err << " Encoded: [" << encoded_metadata_checksum.str() << "].\n"; err << " Computed: [" << computed_metadata_checksum.str() << "]."; throw DataCorruption(err.str()); } ATLAS_IO_ASSERT(r.metadata_format == "yaml"); Metadata metadata = eckit::YAMLConfiguration(metadata_str); for (auto& key : metadata.keys()) { parse_record(*record_, key, metadata.getSubConfiguration(key)); } // DataIndex section // ----------------- in.seek(r.index_offset); auto index_begin = atlas::io::read_struct(in); if (not index_begin.valid()) { throw InvalidRecord("Data index section is not valid. Invalid section begin marker: [" + index_begin.str() + "]"); } const auto index_length = (size_t(r.index_length) - sizeof(RecordDataIndexSection::Begin) - sizeof(RecordDataIndexSection::End)); const auto index_size = index_length / sizeof(RecordDataIndexSection::Entry); auto& data_sections = record_->data_sections; data_sections.resize(index_size); if (in.read(data_sections.data(), index_length) != index_length) { throw InvalidRecord("Unexpected EOF reached"); } auto index_end = atlas::io::read_struct(in); if (not index_end.valid()) { throw InvalidRecord("Data index section is not valid. Invalid section end marker: [" + index_end.str() + "]"); } for (auto& data_section : data_sections) { data_section.offset += rbegin; if (data_section.length < sizeof(RecordDataSection::Begin) + sizeof(RecordDataSection::End)) { throw InvalidRecord("Unexpected data section length: [" + std::to_string(data_section.length) + "]"); } } record_->parse(); if (read_to_end) { in.seek(rbegin + r.record_length - sizeof(RecordEnd)); RecordEnd record_end; read_struct(in, record_end); if (not record_end.valid()) { throw InvalidRecord("RecordEnd has unexpected format: [" + record_end.str() + "]"); } if (in.position() != rbegin + r.record_length) { throw InvalidRecord("RecordEnd has unexpected format"); } } return *this; } //--------------------------------------------------------------------------------------------------------------------- void ParsedRecord::parse() { for (auto& key : keys) { auto& item = items.at(key); item.record.version_ = head.version; item.record.created_ = head.time; item.data.section(item.getInt("data.section", 0)); item.data.endian(head.endian()); item.data.compression(item.getString("data.compression.type", "none")); if (item.data.section()) { auto& data_section = data_sections.at(size_t(item.data.section() - 1)); item.data.checksum(data_section.checksum); item.data.compressed_size(data_section.length - sizeof(RecordDataSection::Begin) - sizeof(RecordDataSection::End)); if (item.data.compressed()) { item.data.size(atlas::io::uncompressed_size(item)); } else { item.data.size(item.data.compressed_size()); } } } } //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/Record.h000066400000000000000000000033361513175025300204570ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include #include #include "atlas_io/detail/Endian.h" #include "atlas_io/detail/Time.h" #include "atlas_io/detail/Version.h" namespace atlas { namespace io { class Stream; class ParsedRecord; class Metadata; //--------------------------------------------------------------------------------------------------------------------- class Record { public: struct URI { std::string str() const; std::string path; std::uint64_t offset; URI() = default; URI(const std::string& _path, std::uint64_t _offset = 0): path(_path), offset(_offset) {} URI(const URI& other): path(other.path), offset(other.offset) {} }; private: std::shared_ptr record_; public: Record(); bool empty() const; Record& read(Stream& in, bool verify_end = false); const Metadata& metadata(const std::string& key) const; Endian endian() const; Version version() const; Time time() const; std::uint64_t size() const; const std::vector& keys() const; bool has(const std::string& key); operator const ParsedRecord&() const; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/RecordItem.cc000066400000000000000000000107611513175025300214340ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "RecordItem.h" #include "eckit/filesystem/URI.h" #include "atlas_io/atlas_compat.h" #include "atlas_io/detail/Assert.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- RecordItem::URI::URI(const std::string& _uri) { eckit::URI uri{_uri}; ATLAS_IO_ASSERT(uri.scheme() == "file"); ATLAS_IO_ASSERT(not uri.query("key").empty()); path = uri.path(); offset = 0; if (not uri.query("offset").empty()) { offset = std::stoul(uri.query("offset")); } key = uri.query("key"); } //--------------------------------------------------------------------------------------------------------------------- RecordItem::URI::URI(const std::string& _path, uint64_t _offset, const std::string& _key): path(_path), offset(_offset), key(_key) {} //--------------------------------------------------------------------------------------------------------------------- std::string RecordItem::URI::str() const { eckit::URI uri("file", eckit::PathName(path)); uri.query("offset", std::to_string(offset)); uri.query("key", key); return uri.asRawString(); } //--------------------------------------------------------------------------------------------------------------------- RecordItem::RecordItem(RecordItem&& other): metadata_(std::move(other.metadata_)), data_(std::move(other.data_)) {} //--------------------------------------------------------------------------------------------------------------------- RecordItem::RecordItem(Metadata&& metadata, Data&& data): metadata_(new Metadata(metadata)), data_(std::move(data)) {} //--------------------------------------------------------------------------------------------------------------------- const Data& RecordItem::data() const { return data_; } //--------------------------------------------------------------------------------------------------------------------- const Metadata& RecordItem::metadata() const { return *metadata_; } //--------------------------------------------------------------------------------------------------------------------- void RecordItem::metadata(const Metadata& m) { *metadata_ = m; } //--------------------------------------------------------------------------------------------------------------------- void RecordItem::data(Data&& d) { data_ = std::move(d); } //--------------------------------------------------------------------------------------------------------------------- bool RecordItem::empty() const { return metadata().empty(); } //--------------------------------------------------------------------------------------------------------------------- void RecordItem::clear() { data_.clear(); metadata_.reset(new atlas::io::Metadata()); } //--------------------------------------------------------------------------------------------------------------------- void RecordItem::decompress() { ATLAS_IO_ASSERT(not empty()); if (metadata().data.compressed()) { data_.decompress(metadata().data.compression(), metadata().data.size()); } metadata_->data.compressed(false); } //--------------------------------------------------------------------------------------------------------------------- void RecordItem::compress() { ATLAS_IO_ASSERT(not empty()); if (not metadata().data.compressed() && metadata().data.compression() != "none") { data_.compress(metadata().data.compression()); metadata_->data.compressed(true); } } //--------------------------------------------------------------------------------------------------------------------- size_t encode_metadata(const RecordItem& in, Metadata& metadata) { metadata.set(in.metadata()); return in.data().size(); } //--------------------------------------------------------------------------------------------------------------------- void encode_data(const RecordItem& in, Data& out) { out.assign(in.data()); } //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/RecordItem.h000066400000000000000000000052641513175025300213000ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include #include "atlas_io/Data.h" #include "atlas_io/Metadata.h" #include "atlas_io/detail/tag.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- struct RecordItem { public: struct URI { URI(const std::string& uri); URI(const std::string& _path, std::uint64_t _offset, const std::string& _key); std::string str() const; operator std::string() { return str(); } std::string path; std::uint64_t offset; std::string key; }; public: RecordItem() = default; template explicit RecordItem(T&& x, tag::enable_static_assert = tag::enable_static_assert()); template explicit RecordItem(T&& x, tag::disable_static_assert); RecordItem(Metadata&&, Data&&); RecordItem(RecordItem&& other); const Data& data() const; const Metadata& metadata() const; void metadata(const Metadata& m); void data(atlas::io::Data&& d); bool empty() const; void clear(); void decompress(); void compress(); private: std::unique_ptr metadata_{new Metadata()}; Data data_; }; //--------------------------------------------------------------------------------------------------------------------- template RecordItem::RecordItem(T&& x, tag::enable_static_assert) { encode(x, *metadata_, data_); } //--------------------------------------------------------------------------------------------------------------------- template RecordItem::RecordItem(T&& x, tag::disable_static_assert) { encode(x, *metadata_, data_, tag::disable_static_assert()); } //--------------------------------------------------------------------------------------------------------------------- size_t encode_metadata(const RecordItem& in, atlas::io::Metadata& metadata); //--------------------------------------------------------------------------------------------------------------------- void encode_data(const RecordItem& in, atlas::io::Data& out); //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/RecordItemReader.cc000066400000000000000000000175231513175025300225620ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "RecordItemReader.h" #include "atlas_io/Exceptions.h" #include "atlas_io/FileStream.h" #include "atlas_io/Record.h" #include "atlas_io/Session.h" #include "atlas_io/Trace.h" #include "atlas_io/detail/Assert.h" #include "atlas_io/detail/ParsedRecord.h" #include "atlas_io/detail/RecordSections.h" namespace atlas { namespace io { namespace { //--------------------------------------------------------------------------------------------------------------------- template inline size_t read_struct(IStream& in, Struct& s) { static_assert(Struct::bytes == sizeof(Struct), ""); return in.read(reinterpret_cast(&s), sizeof(Struct)); } //--------------------------------------------------------------------------------------------------------------------- template inline Struct read_struct(IStream& in) { Struct s; if (read_struct(in, s) != sizeof(Struct)) { throw InvalidRecord("Unexpected EOF reached"); } return s; } //--------------------------------------------------------------------------------------------------------------------- static Data read_data(const Record& record, int data_section_index, Stream in) { ATLAS_IO_TRACE("read_data(data_section=" + std::to_string(data_section_index) + ")"); if (data_section_index == 0) { return atlas::io::Data(); } const auto& parsed = static_cast(record); const auto& data_section = parsed.data_sections.at(size_t(data_section_index) - 1); atlas::io::Data data; auto offset = data_section.offset; in.seek(offset); auto data_begin = atlas::io::read_struct(in); if (not data_begin.valid()) { throw InvalidRecord("Data section is not valid"); } auto data_size = size_t(data_section.length) - sizeof(RecordDataSection::Begin) - sizeof(RecordDataSection::End); if (data_size) { if (data.read(in, data_size) != data_size) { throw InvalidRecord("Data section is not valid"); } ATLAS_IO_ASSERT(data.size() == data_size); } auto data_end = atlas::io::read_struct(in); if (not data_end.valid()) { throw InvalidRecord("Data section is not valid"); } return data; } //--------------------------------------------------------------------------------------------------------------------- static eckit::PathName make_absolute_path(const std::string& reference_path, RecordItem::URI& uri) { eckit::PathName absolute_path = uri.path; if (reference_path.size() && uri.path[0] != '/' && uri.path[0] != '~') { absolute_path = eckit::PathName{reference_path} / absolute_path; } return absolute_path.fullName(); } //--------------------------------------------------------------------------------------------------------------------- static Record read_record(const std::string& path, size_t offset) { auto record = Session::record(path, offset); if (record.empty()) { auto in = InputFileStream(path); in.seek(offset); record.read(in); } return record; } //--------------------------------------------------------------------------------------------------------------------- static Record read_record(Stream in, size_t offset) { auto record = Session::record(in, offset); if (record.empty()) { in.seek(offset); record.read(in); } return record; } //--------------------------------------------------------------------------------------------------------------------- } // anonymous namespace //--------------------------------------------------------------------------------------------------------------------- RecordItemReader::RecordItemReader(Stream in, size_t offset, const std::string& key): in_(in), uri_{"", offset, key} { ATLAS_IO_TRACE("RecordItemReader(Stream,offset,key"); record_ = read_record(in, uri_.offset); if (not record_.has(uri_.key)) { throw InvalidRecord(uri_.key + " not found in record " + uri_.path); } } RecordItemReader::RecordItemReader(Stream in, const std::string& key): in_(in), uri_{"", 0, key} { record_ = read_record(in, uri_.offset); if (not record_.has(uri_.key)) { throw InvalidRecord(uri_.key + " not found in record " + uri_.path); } } //--------------------------------------------------------------------------------------------------------------------- RecordItemReader::RecordItemReader(const std::string& uri): RecordItemReader("", uri) {} //--------------------------------------------------------------------------------------------------------------------- RecordItemReader::RecordItemReader(const std::string& ref, const std::string& uri): ref_{ref}, uri_{uri} { auto absolute_path = make_absolute_path(ref_, uri_); if (not absolute_path.exists()) { throw InvalidRecord("Item " + uri_.str() + " refers to non existing file: " + absolute_path); } record_ = read_record(absolute_path, uri_.offset); if (not record_.has(uri_.key)) { throw InvalidRecord(uri_.key + " not found in record " + uri_.path); } } //--------------------------------------------------------------------------------------------------------------------- void RecordItemReader::read(RecordItem& item) { io::Metadata metadata; io::Data data; read(metadata, data); item.metadata(metadata); item.data(std::move(data)); }; //--------------------------------------------------------------------------------------------------------------------- void RecordItemReader::read(Metadata& metadata, bool follow_links) { ATLAS_IO_TRACE("RecordItemReader::read_metadata(" + uri_.path + ":" + uri_.key + ")"); metadata = record_.metadata(uri_.key); if (follow_links && metadata.link()) { auto absolute_path = make_absolute_path(ref_, uri_); Metadata linked; RecordItemReader{absolute_path.dirName(), metadata.link()}.read(linked); metadata.link(std::move(linked)); } }; //--------------------------------------------------------------------------------------------------------------------- static void read_from_stream(Record record, Stream in, const std::string& key, io::Metadata& metadata, io::Data& data) { ATLAS_IO_TRACE("RecordItemReader::read( Stream, " + key + ")"); metadata = record.metadata(key); if (metadata.link()) { throw atlas::io::Exception("Cannot follow links in records that are not file based"); } else { if (metadata.data.section()) { data = atlas::io::read_data(record, metadata.data.section(), in); } } } void RecordItemReader::read(io::Metadata& metadata, io::Data& data) { if (in_) { read_from_stream(record_, in_, uri_.key, metadata, data); return; } ATLAS_IO_TRACE("RecordItemReader::read(" + uri_.path + ":" + uri_.key + ")"); metadata = record_.metadata(uri_.key); auto absolute_path = make_absolute_path(ref_, uri_); if (metadata.link()) { Metadata linked; RecordItemReader{absolute_path.dirName(), metadata.link()}.read(linked, data); metadata.link(std::move(linked)); } else { if (metadata.data.section()) { data = atlas::io::read_data(record_, metadata.data.section(), InputFileStream(absolute_path)); } } }; //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/RecordItemReader.h000066400000000000000000000025471513175025300224240ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include "atlas_io/Record.h" #include "atlas_io/RecordItem.h" #include "atlas_io/Stream.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- class RecordItemReader { public: RecordItemReader(Stream, size_t offset, const std::string& key); RecordItemReader(Stream, const std::string& key); RecordItemReader(const std::string& uri); void read(RecordItem& item); void read(Metadata&, bool follow_links = true); void read(Metadata&, Data&); private: RecordItemReader(const std::string& ref, const std::string& uri); Stream in_; Record record_; std::string ref_{}; // directory to which relative URI's are evaluated RecordItem::URI uri_; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/RecordPrinter.cc000066400000000000000000000070041513175025300221550ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "RecordPrinter.h" #include #include "atlas_io/Exceptions.h" #include "atlas_io/FileStream.h" #include "atlas_io/detail/Assert.h" #include "atlas_io/print/JSONFormat.h" #include "atlas_io/print/TableFormat.h" #include "atlas_io/atlas_compat.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- RecordPrinter::RecordPrinter(const eckit::PathName& path, const eckit::Configuration& config): RecordPrinter(path, 0, config) {} //--------------------------------------------------------------------------------------------------------------------- RecordPrinter::RecordPrinter(const eckit::PathName& path, const std::uint64_t offset, const eckit::Configuration& config): RecordPrinter(Record::URI{path, offset}, config) {} //--------------------------------------------------------------------------------------------------------------------- RecordPrinter::RecordPrinter(const Record::URI& ref, const eckit::Configuration& config): uri_(ref), record_(Session::record(ref.path, ref.offset)) { if (record_.empty()) { auto in = InputFileStream(uri_.path); in.seek(uri_.offset); record_.read(in, true); ATLAS_IO_ASSERT(not record_.empty()); } config.get("format", options_.format); config.get("details", options_.details); // Check if format is supported { std::vector supported_formats{"json", "yaml", "table"}; bool format_supported{false}; for (auto& supported_format : supported_formats) { if (options_.format == supported_format) { format_supported = true; break; } } if (not format_supported) { std::stringstream s; s << "Format '" + options_.format + "' not supported. Supported formats:"; for (auto& supported_format : supported_formats) { s << "\n - " << supported_format; } throw Exception(s.str(), Here()); } } } //--------------------------------------------------------------------------------------------------------------------- void RecordPrinter::print(std::ostream& out) const { eckit::LocalConfiguration config; config.set("details", options_.details); if (options_.format == "json") { JSONFormat{uri_, config}.print(out); } else if (options_.format == "yaml") { JSONFormat{uri_, config}.print(out); } else if (options_.format == "table") { TableFormat{uri_, config}.print(out); } else { throw Exception("Cannot print record: Unrecognized format " + options_.format + ".", Here()); } } //--------------------------------------------------------------------------------------------------------------------- std::ostream& operator<<(std::ostream& out, const RecordPrinter& info) { info.print(out); return out; } //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/RecordPrinter.h000066400000000000000000000033431513175025300220210ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/config/Configuration.h" #include "eckit/filesystem/PathName.h" #include "atlas_io/Record.h" #include "atlas_io/Session.h" #include "atlas_io/detail/NoConfig.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- class RecordPrinter { public: RecordPrinter(const Record::URI&, const eckit::Configuration& = NoConfig()); RecordPrinter(const eckit::PathName&, const eckit::Configuration& = NoConfig()); RecordPrinter(const eckit::PathName&, std::uint64_t offset, const eckit::Configuration& = NoConfig()); Record record() const { return record_; } size_t size() const { return record_.size(); } Version version() const { return record_.version(); } Time time() const { return record_.time(); } void print(std::ostream& out) const; friend std::ostream& operator<<(std::ostream&, const RecordPrinter&); private: Session session_; Record::URI uri_; struct { std::string format{"table"}; bool details{false}; } options_; Record record_; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/RecordReader.cc000066400000000000000000000057641513175025300217470ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "RecordReader.h" #include "atlas_io/Metadata.h" #include "atlas_io/RecordItemReader.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- RecordReader::RecordReader(const Record::URI& ref): RecordReader(ref.path, ref.offset) {} //--------------------------------------------------------------------------------------------------------------------- RecordReader::RecordReader(const std::string& path, uint64_t offset): session_{}, path_{path}, offset_{offset} {} RecordReader::RecordReader(Stream stream, uint64_t offset): session_{}, stream_{stream}, path_{}, offset_{offset} {} //--------------------------------------------------------------------------------------------------------------------- Record::URI RecordReader::uri() const { Record::URI _uri; _uri.path = path_; _uri.offset = offset_; return _uri; } //--------------------------------------------------------------------------------------------------------------------- RecordItem::URI RecordReader::uri(const std::string& key) const { return RecordItem::URI(path_, offset_, key); } //--------------------------------------------------------------------------------------------------------------------- void RecordReader::trace(const std::string&, const char* file, int line, const char* func) {} //--------------------------------------------------------------------------------------------------------------------- void RecordReader::wait(const std::string& key) { request(key).wait(); } //--------------------------------------------------------------------------------------------------------------------- void RecordReader::wait() { // This can be optimized perhaps to overlap IO with decoding in multithreaded environment for (auto& pair : requests_) { auto& request = pair.second; request.wait(); } } //--------------------------------------------------------------------------------------------------------------------- ReadRequest& RecordReader::request(const std::string& key) { return requests_.at(key); } //--------------------------------------------------------------------------------------------------------------------- Metadata RecordReader::metadata(const std::string& key) { Metadata metadata; RecordItemReader{uri(key)}.read(metadata); return metadata; } void RecordReader::checksum(bool b) { do_checksum_ = b; } //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/RecordReader.h000066400000000000000000000041731513175025300216020ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include #include "atlas_io/Metadata.h" #include "atlas_io/ReadRequest.h" #include "atlas_io/Session.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- class RecordReader { public: RecordReader(const Record::URI& ref); RecordReader(const std::string& path, std::uint64_t offset = 0); RecordReader(Stream stream, std::uint64_t offset = 0); template ReadRequest& read(const std::string& key, T& value) { trace("read(" + key + ")", __FILE__, __LINE__, __func__); if (stream_) { trace("stream", __FILE__, __LINE__, __func__); requests_.emplace(key, ReadRequest{stream_, offset_, key, value}); } else { requests_.emplace(key, ReadRequest{uri(key), value}); } if (do_checksum_ >= 0) { requests_.at(key).checksum(do_checksum_); } return requests_.at(key); } void wait(const std::string& key); void wait(); ReadRequest& request(const std::string& key); Metadata metadata(const std::string& key); void checksum(bool); private: Record::URI uri() const; RecordItem::URI uri(const std::string& key) const; void trace(const std::string&, const char* file, int line, const char* func); private: Session session_; Stream stream_; std::map requests_; std::string path_; std::uint64_t offset_; int do_checksum_{-1}; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/RecordWriter.cc000066400000000000000000000214151513175025300220100ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "atlas_io/RecordWriter.h" #include "atlas_io/Exceptions.h" #include "atlas_io/RecordWriter.h" #include "atlas_io/Trace.h" #include "atlas_io/detail/Checksum.h" #include "atlas_io/detail/Defaults.h" #include "atlas_io/detail/Encoder.h" #include "atlas_io/detail/RecordSections.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- template inline void write_struct(OStream& out, const Struct& s) { static_assert(Struct::bytes == sizeof(Struct), ""); if (out.write(reinterpret_cast(&s), sizeof(s)) != sizeof(s)) { throw WriteError("Could not write struct to stream"); } } //--------------------------------------------------------------------------------------------------------------------- template inline void write_string(OStream& out, const std::string& s) { if (out.write(s.data(), s.size()) != s.size()) { throw WriteError("Could not write string to stream"); } } //--------------------------------------------------------------------------------------------------------------------- size_t RecordWriter::write(Stream out) const { ATLAS_IO_TRACE("RecordWriter::write"); RecordHead r; auto begin_of_record = out.position(); auto position = [&begin_of_record, &out]() { return out.position() - begin_of_record; }; std::vector index; // Begin Record // ------------ atlas::io::write_struct(out, r); // Metadata section // ---------------- ATLAS_IO_TRACE_SCOPE("metadata section") { r.metadata_offset = position(); atlas::io::write_struct(out, RecordMetadataSection::Begin()); auto metadata_str = metadata(); atlas::io::write_string(out, metadata_str); atlas::io::write_struct(out, RecordMetadataSection::End()); r.metadata_length = position() - r.metadata_offset; r.metadata_checksum = do_checksum_ ? atlas::io::checksum(metadata_str.data(), metadata_str.size()) : std::string("none:"); // Index section // ------------- size_t nb_data_sections = static_cast(nb_data_sections_); r.index_offset = position(); atlas::io::write_struct(out, RecordDataIndexSection::Begin()); index.resize(nb_data_sections); for (size_t i = 0; i < nb_data_sections; ++i) { atlas::io::write_struct(out, index[i]); } atlas::io::write_struct(out, RecordDataIndexSection::End()); r.index_length = position() - r.index_offset; } // Data sections // ------------- ATLAS_IO_TRACE_SCOPE("data sections") { size_t i{0}; for (auto& key : keys_) { auto& encoder = encoders_.at(key); auto& info = info_.at(key); if (info.section() == 0) { continue; } atlas::io::Data data; encode_data(encoder, data); data.compress(info.compression()); auto& data_section = index[i]; data_section.offset = position(); atlas::io::write_struct(out, RecordDataSection::Begin()); if (data.write(out) != data.size()) { throw WriteError("Could not write data for item " + key + " to stream"); } atlas::io::write_struct(out, RecordDataSection::End()); data_section.length = position() - data_section.offset; data_section.checksum = do_checksum_ ? data.checksum() : std::string("none:"); ++i; } } // End Record // ---------- atlas::io::write_struct(out, RecordEnd()); auto end_of_record = out.position(); r.record_length = end_of_record - begin_of_record; r.time = atlas::io::Time::now(); out.seek(begin_of_record); atlas::io::write_struct(out, r); out.seek(begin_of_record + r.index_offset + sizeof(RecordDataIndexSection::Begin)); for (auto& entry : index) { atlas::io::write_struct(out, entry); } out.seek(end_of_record); // So that following writes will not overwrite return r.record_length; } //--------------------------------------------------------------------------------------------------------------------- void RecordWriter::compression(const std::string& c) { compression_ = c; } //--------------------------------------------------------------------------------------------------------------------- void RecordWriter::compression(bool on) { if (on) { compression_ = defaults::compression_algorithm(); // still possible to be "none" } else { compression_ = "none"; } } //--------------------------------------------------------------------------------------------------------------------- void RecordWriter::checksum(bool on) { if (on) { do_checksum_ = defaults::checksum_write(); // still possible to be off via environment } else { do_checksum_ = false; } } //--------------------------------------------------------------------------------------------------------------------- void RecordWriter::set(const RecordWriter::Key& key, Link&& link, const eckit::Configuration&) { keys_.emplace_back(key); encoders_[key] = Encoder{link}; info_.emplace(key, DataInfo{}); } //--------------------------------------------------------------------------------------------------------------------- void RecordWriter::set(const RecordWriter::Key& key, Encoder&& encoder, const eckit::Configuration& config) { DataInfo info; if (encoder.encodes_data()) { ++nb_data_sections_; info.compression(config.getString("compression", compression_)); info.section(nb_data_sections_); } keys_.emplace_back(key); encoders_[key] = std::move(encoder); info_.emplace(key, std::move(info)); } //--------------------------------------------------------------------------------------------------------------------- size_t RecordWriter::write(const eckit::PathName& path, Mode mode) const { return write(OutputFileStream(path, mode)); } //--------------------------------------------------------------------------------------------------------------------- size_t RecordWriter::write(eckit::DataHandle& out) const { return write(Stream(out)); } //--------------------------------------------------------------------------------------------------------------------- size_t RecordWriter::estimateMaximumSize() const { size_t size{0}; size += sizeof(RecordHead); size += sizeof(RecordMetadataSection::Begin); size += metadata().size(); size += sizeof(RecordMetadataSection::End); size += sizeof(RecordDataIndexSection::Begin); size += size_t(nb_data_sections_) * sizeof(RecordDataIndexSection::Entry); size += sizeof(RecordDataIndexSection::End); for (auto& key : keys_) { auto& encoder = encoders_.at(key); auto& info = info_.at(key); if (info.section() == 0) { continue; } size += sizeof(RecordDataSection::Begin); { atlas::io::Metadata m; size_t max_data_size = encode_metadata(encoder, m); if (info.compression() != "none") { max_data_size = size_t(1.2 * max_data_size); max_data_size = std::max(max_data_size, 10 * 1024); // minimum 10KB } size += max_data_size; } size += sizeof(RecordDataSection::End); } size += sizeof(RecordEnd); return size; } //--------------------------------------------------------------------------------------------------------------------- std::string RecordWriter::metadata() const { atlas::io::Metadata metadata; for (auto& key : keys_) { auto& encoder = encoders_.at(key); auto& info = info_.at(key); atlas::io::Metadata m; encode_metadata(encoder, m); if (info.section()) { m.set("data.section", info.section()); if (info.compression() != "none") { m.set("data.compression.type", info.compression()); } } metadata.set(key, m); } std::stringstream ss; atlas::io::write(metadata, ss); return ss.str(); } //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/RecordWriter.h000066400000000000000000000106361513175025300216550ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include #include "atlas_io/FileStream.h" #include "atlas_io/RecordItem.h" #include "atlas_io/Stream.h" #include "atlas_io/detail/Encoder.h" #include "atlas_io/detail/NoConfig.h" #include "atlas_io/detail/Reference.h" #include "atlas_io/detail/TypeTraits.h" #include "atlas_io/types/array/ArrayReference.h" #include "atlas_io/types/scalar.h" #include "atlas_io/types/string.h" #include "atlas_io/detail/Defaults.h" namespace atlas { namespace io { template Interpreted interprete(T& in) { Interpreted interpreted; interprete(in, interpreted); return interpreted; } //--------------------------------------------------------------------------------------------------------------------- /// @class RecordWriter /// @brief Write record class RecordWriter { public: using Key = std::string; public: /// @brief Set compression void compression(const std::string&); /// @brief Set compression off or to default void compression(bool); /// @brief Set checksum off or to default void checksum(bool); // -- set( Key, Value ) where Value can be a variety of things /// @brief Add link to other record item (RecordItem::URI) void set(const Key&, Link&&, const eckit::Configuration& = NoConfig()); /// @brief Add item to record void set(const Key&, Encoder&&, const eckit::Configuration& = NoConfig()); /// @brief Add item to record template = 0> void set(const Key& key, Value&& value, const eckit::Configuration& config = NoConfig()) { set(key, Encoder{std::move(value)}, config); } /// @brief Add item to record template void set(const Key& key, const Reference& value, const eckit::Configuration& config = NoConfig()) { set(key, std::move(value), config); } /// @brief Add item to record template = 0> void set(const Key& key, const Value& value, const eckit::Configuration& config = NoConfig()) { set(key, RecordItem(interprete(value)), config); } /// @brief Add item to record template = 0> void set(const Key& key, const Value& value, const eckit::Configuration& config = NoConfig()) { set(key, Encoder{value}, config); } void set(const Key& key, const char* value, const eckit::Configuration& config = NoConfig()) { set(key, Encoder{std::string(value)}, config); } void set(const Key& key, const std::string& value, const eckit::Configuration& config = NoConfig()) { set(key, Encoder{std::string(value)}, config); } /// @brief Write new record to path size_t write(const eckit::PathName&, Mode = Mode::write) const; /// @brief Write new record to a DataHandle /// @pre The DataHandle must be opened for Write access. size_t write(eckit::DataHandle&) const; /// @brief Write new record to a Stream /// @pre The Stream must be opened for Write access. size_t write(Stream) const; /// @brief estimate maximum size of record /// /// This could be useful to write a record to a fixed size MemoryHandle /// /// @note Without compression this matches exactly the required record size. /// With compression active, the data sizes are assumed to be 120% of uncompressed sizes for robustness, /// which may seem contradictory. size_t estimateMaximumSize() const; private: std::vector keys_; std::map encoders_; std::map info_; std::string compression_{defaults::compression_algorithm()}; int do_checksum_{defaults::checksum_write()}; int nb_data_sections_{0}; std::string metadata() const; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/Session.cc000066400000000000000000000124061513175025300210200ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "Session.h" #include #include #include #include #include #include #include "eckit/filesystem/PathName.h" #include "atlas_io/Exceptions.h" #include "atlas_io/atlas_compat.h" #include "atlas_io/detail/Assert.h" namespace atlas { namespace io { using lock_guard = std::lock_guard; class SessionImpl { public: void store(Stream stream); Record record(const std::string& path, size_t offset); private: std::recursive_mutex mutex_; std::vector handles_; std::map records_; }; //--------------------------------------------------------------------------------------------------------------------- class ActiveSession { public: static ActiveSession& instance(); SessionImpl& current(); void push(); void pop(); Record record(const std::string& path, size_t offset); void store(Stream stream); private: friend class Session; std::recursive_mutex mutex_; std::unique_ptr session_; std::atomic count_{0}; }; //--------------------------------------------------------------------------------------------------------------------- ActiveSession& ActiveSession::instance() { static ActiveSession instance; return instance; } //--------------------------------------------------------------------------------------------------------------------- Record ActiveSession::record(const std::string& path, size_t offset) { if (count_) { return current().record(path, offset); } else { return Record(); } } //--------------------------------------------------------------------------------------------------------------------- void ActiveSession::store(Stream stream) { if (count_) { return current().store(stream); } } //--------------------------------------------------------------------------------------------------------------------- SessionImpl& ActiveSession::current() { lock_guard lock(mutex_); if (count_ == 0) { throw Exception("No atlas::io session is currently active", Here()); } return *session_; } //--------------------------------------------------------------------------------------------------------------------- void ActiveSession::push() { lock_guard lock(mutex_); if (count_ == 0) { ATLAS_IO_ASSERT(session_ == nullptr); session_.reset(new SessionImpl()); } ++count_; } //--------------------------------------------------------------------------------------------------------------------- void ActiveSession::pop() { lock_guard lock(mutex_); if (count_ == 0) { throw Exception("No atlas::io session is currently active", Here()); } --count_; if (count_ == 0) { session_.reset(); } } //--------------------------------------------------------------------------------------------------------------------- void SessionImpl::store(Stream stream) { lock_guard lock(mutex_); handles_.emplace_back(stream); } //--------------------------------------------------------------------------------------------------------------------- Record SessionImpl::record(const std::string& path, size_t offset) { lock_guard lock(mutex_); auto key = Record::URI{eckit::PathName(path).fullName(), offset}.str(); if (records_.find(key) == records_.end()) { records_.emplace(key, Record{}); } return records_.at(key); } //--------------------------------------------------------------------------------------------------------------------- Session::Session() { ActiveSession::instance().push(); } //--------------------------------------------------------------------------------------------------------------------- Session::~Session() { ActiveSession::instance().pop(); } //--------------------------------------------------------------------------------------------------------------------- bool Session::active() { return ActiveSession::instance().count_ > 0; } //--------------------------------------------------------------------------------------------------------------------- Record Session::record(const std::string& path, size_t offset) { return ActiveSession::instance().record(path, offset); } //--------------------------------------------------------------------------------------------------------------------- Record Session::record(Stream stream, size_t offset) { std::stringstream id; id << &stream.datahandle(); return ActiveSession::instance().record(id.str(), offset); } //--------------------------------------------------------------------------------------------------------------------- void Session::store(Stream stream) { ActiveSession::instance().store(stream); } //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/Session.h000066400000000000000000000020471513175025300206620ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include "atlas_io/Record.h" #include "atlas_io/Stream.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- class Session { public: Session(); ~Session(); static bool active(); static Record record(const std::string& path, size_t offset); static Record record(Stream, size_t offset); static void store(Stream stream); }; //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/Stream.cc000066400000000000000000000034301513175025300206250ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "atlas_io/Stream.h" #include "eckit/io/DataHandle.h" #include "atlas_io/atlas_compat.h" #include "atlas_io/detail/Assert.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- Stream::Stream(eckit::DataHandle& datahandle): ptr_(&datahandle) {} Stream::Stream(eckit::DataHandle* datahandle): shared_(datahandle), ptr_(shared_.get()) {} Stream::Stream(std::shared_ptr datahandle): shared_(datahandle), ptr_(shared_.get()) {} eckit::DataHandle& Stream::datahandle() { ATLAS_IO_ASSERT(ptr_ != nullptr); return *ptr_; } uint64_t Stream::seek(uint64_t offset) { ATLAS_IO_ASSERT(ptr_ != nullptr); return std::uint64_t(ptr_->seek(static_cast(offset))); } uint64_t Stream::position() { ATLAS_IO_ASSERT(ptr_ != nullptr); return std::uint64_t(ptr_->position()); } uint64_t Stream::write(const void* data, size_t length) { ATLAS_IO_ASSERT(ptr_ != nullptr); return std::uint64_t(ptr_->write(data, static_cast(length))); } uint64_t Stream::read(void* data, size_t length) { ATLAS_IO_ASSERT(ptr_ != nullptr); return std::uint64_t(ptr_->read(data, static_cast(length))); } //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/Stream.h000066400000000000000000000046211513175025300204720ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include namespace eckit { class DataHandle; } namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- /// @class Stream /// @brief Handle to a shared eckit::DataHandle /// /// Note, a Stream is not intended to be opened and closed within atlas::io context /// The derived classes InputFileStream and OutputFileStream automatically open and close /// on construction an destruction. class Stream { public: /// Default constructor /// @post Stream is not usable but can be assigned to become valid Stream() = default; /// Constructor taking ownership of datahandle Stream(eckit::DataHandle*); /// Constructor to share datahandle with a shared_ptr Stream(std::shared_ptr); /// Constructor referencing datahandle, no ownership is taken /// @note The usability depends on the usable lifetime of /// the referenced datahandle Stream(eckit::DataHandle&); /// Access internal eckit::DataHandle eckit::DataHandle& datahandle(); /// Move position to given offset std::uint64_t seek(std::uint64_t offset); /// Return offset of current position std::uint64_t position(); /// Write data of given length (bytes) /// @return number of bytes written /// @post The position is increased with number of bytes written std::uint64_t write(const void* data, size_t length); /// Read data of given length (bytes) /// @return number of bytes read /// @post The position is increased with number of bytes read std::uint64_t read(void* data, size_t length); /// Return true if pointer is valid; operator bool() const { return ptr_; } private: std::shared_ptr shared_; eckit::DataHandle* ptr_{nullptr}; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/Trace.cc000066400000000000000000000025151513175025300204330ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "Trace.h" #include "eckit/log/CodeLocation.h" namespace atlas { namespace io { atlas::io::Trace::Trace(const eckit::CodeLocation& loc) { for (size_t id = 0; id < TraceHookRegistry::size(); ++id) { if (TraceHookRegistry::enabled(id)) { hooks_.emplace_back(TraceHookRegistry::hook(id)(loc, loc.func())); } } } Trace::Trace(const eckit::CodeLocation& loc, const std::string& title) { for (size_t id = 0; id < TraceHookRegistry::size(); ++id) { if (TraceHookRegistry::enabled(id)) { hooks_.emplace_back(TraceHookRegistry::hook(id)(loc, title)); } } } Trace::Trace(const eckit::CodeLocation& loc, const std::string& title, const Labels&) { for (size_t id = 0; id < TraceHookRegistry::size(); ++id) { if (TraceHookRegistry::enabled(id)) { hooks_.emplace_back(TraceHookRegistry::hook(id)(loc, title)); } } } } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/Trace.h000066400000000000000000000047211513175025300202760ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include #include #include namespace eckit { class CodeLocation; } namespace atlas { namespace io { struct TraceHook { TraceHook() = default; virtual ~TraceHook() = default; }; struct TraceHookRegistry { using TraceHookBuilder = std::function(const eckit::CodeLocation&, const std::string&)>; std::vector hooks; std::vector enabled_; static TraceHookRegistry& instance() { static TraceHookRegistry instance; return instance; } static size_t add(TraceHookBuilder&& hook) { instance().hooks.emplace_back(hook); instance().enabled_.emplace_back(true); return instance().hooks.size() - 1; } static size_t add(const TraceHookBuilder& hook) { instance().hooks.emplace_back(hook); instance().enabled_.emplace_back(true); return instance().hooks.size() - 1; } static void enable(size_t id) { instance().enabled_[id] = true; } static void disable(size_t id) { instance().enabled_[id] = false; } static bool enabled(size_t id) { return instance().enabled_[id]; } static size_t size() { return instance().hooks.size(); } static TraceHookBuilder& hook(size_t id) { return instance().hooks[id]; } static size_t invalidId() { return std::numeric_limits::max(); } private: TraceHookRegistry() = default; }; struct Trace { using Labels = std::vector; Trace(const eckit::CodeLocation& loc); Trace(const eckit::CodeLocation& loc, const std::string& title); Trace(const eckit::CodeLocation& loc, const std::string& title, const Labels& labels); private: std::vector> hooks_; }; } // namespace io } // namespace atlas #include "atlas_io/detail/BlackMagic.h" #define ATLAS_IO_TRACE(...) __ATLAS_IO_TYPE(::atlas::io::Trace, Here() __ATLAS_IO_COMMA_ARGS(__VA_ARGS__)) #define ATLAS_IO_TRACE_SCOPE(...) __ATLAS_IO_TYPE_SCOPE(::atlas::io::Trace, Here() __ATLAS_IO_COMMA_ARGS(__VA_ARGS__)) atlas-0.45.0/atlas_io/src/atlas_io/atlas-io.h000066400000000000000000000110051513175025300207420ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include #include "atlas_io/detail/Link.h" #include "atlas_io/detail/Reference.h" #include "atlas_io/detail/StaticAssert.h" #include "atlas_io/detail/sfinae.h" #include "atlas_io/Exceptions.h" #include "atlas_io/FileStream.h" #include "atlas_io/Record.h" #include "atlas_io/RecordItemReader.h" #include "atlas_io/RecordPrinter.h" #include "atlas_io/RecordReader.h" #include "atlas_io/RecordWriter.h" #include "atlas_io/Session.h" #include "atlas_io/Stream.h" #include "atlas_io/Trace.h" #include "atlas_io/types/array.h" #include "atlas_io/types/scalar.h" #include "atlas_io/types/string.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- inline Link link(const std::string& uri) { return Link{uri}; } //--------------------------------------------------------------------------------------------------------------------- template = 0> Reference ref(const T& x, tag::enable_static_assert = tag::enable_static_assert()) { static_assert(is_encodable(), "in atlas::io::ref(const Value&)" "\n" "\n Static assertion failed" "\n -----------------------" "\n" "\n Cannot encode values of referenced type." "\n" "\n Implement the functions" "\n" "\n void encode_data(const Value& in, atlas::io::Data& out);" "\n size_t encode_metadata(const Value& value, atlas::io::Metadata& metadata);" "\n" "\n or alternatively a conversion function to atlas::io::types::ArrayView" "\n" "\n void interprete(const Value& in, atlas::io::types::ArrayView& out)" "\n" "\n Rules of argument-dependent-lookup apply." "\n --> Functions need to be declared in namespace of any of the arguments." "\n" "\n Note, turn this into a runtime exception by calling this function instead:" "\n" "\n atlas::io::ref(const T&, atlas::io::no_static_assert() )" "\n"); return Reference(x); } template = 0> Reference ref(const T& x, tag::disable_static_assert) { if (not is_encodable()) { throw NotEncodable(x); } return Reference(x); } template = 0> ArrayReference ref(const T& x, tag::enable_static_assert = tag::enable_static_assert()) { ArrayReference w; interprete(x, w); return w; } //--------------------------------------------------------------------------------------------------------------------- template RecordItem copy(T&& value, tag::disable_static_assert) { return RecordItem(std::forward(value), tag::disable_static_assert()); } template RecordItem copy(T&& value) { return RecordItem(std::forward(value)); } //--------------------------------------------------------------------------------------------------------------------- template void encode(const T& in, atlas::io::Metadata& metadata, atlas::io::Data& data, tag::enable_static_assert = tag::enable_static_assert()) { auto referenced = ref(in, tag::enable_static_assert()); sfinae::encode_metadata(referenced, metadata); sfinae::encode_data(referenced, data); } template void encode(const T& in, atlas::io::Metadata& metadata, atlas::io::Data& data, tag::disable_static_assert) { auto referenced = ref(in, tag::disable_static_assert()); sfinae::encode_metadata(referenced, metadata); sfinae::encode_data(referenced, data); } //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/atlas_compat.h000066400000000000000000000000151513175025300216770ustar00rootroot00000000000000#pragma once atlas-0.45.0/atlas_io/src/atlas_io/detail/000077500000000000000000000000001513175025300203255ustar00rootroot00000000000000atlas-0.45.0/atlas_io/src/atlas_io/detail/Assert.h000066400000000000000000000010261513175025300217360ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/exception/Exceptions.h" #define ATLAS_IO_ASSERT(X) ASSERT(X) #define ATLAS_IO_ASSERT_MSG(X, M) ASSERT_MSG(X, M) atlas-0.45.0/atlas_io/src/atlas_io/detail/Base64.cc000066400000000000000000000110231513175025300216550ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "Base64.h" #include namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- namespace { static std::array b64_decode_table{ /* ASCII table */ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 62, 64, 64, 64, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 64, 64, 64, 64, 64, 64, 64, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 64, 64, 64, 64, 64, 64, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64}; static std::array b64_encode_table{ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; } // namespace //--------------------------------------------------------------------------------------------------------------------- std::string Base64::encode(const void* data, size_t len) { const auto& table = b64_encode_table; const unsigned char* src = reinterpret_cast(data); unsigned char *out, *pos; const unsigned char *end, *in; size_t out_len = 4 * ((len + 2) / 3); /* 3-byte blocks to 4-byte */ if (out_len < len) { return std::string(); /* integer overflow */ } std::string str; str.resize(out_len); out = reinterpret_cast(const_cast(str.data())); end = src + len; in = src; pos = out; while (end - in >= 3) { *pos++ = table[in[0] >> 2]; *pos++ = table[((in[0] & 0x03) << 4) | (in[1] >> 4)]; *pos++ = table[((in[1] & 0x0f) << 2) | (in[2] >> 6)]; *pos++ = table[in[2] & 0x3f]; in += 3; } if (end - in) { *pos++ = table[in[0] >> 2]; if (end - in == 1) { *pos++ = table[(in[0] & 0x03) << 4]; *pos++ = '='; } else { *pos++ = table[((in[0] & 0x03) << 4) | (in[1] >> 4)]; *pos++ = table[(in[1] & 0x0f) << 2]; } *pos++ = '='; } return str; } //--------------------------------------------------------------------------------------------------------------------- std::string Base64::decode(const void* data, size_t len) { const auto& table = b64_decode_table; const unsigned char* p = reinterpret_cast(data); int pad = len > 0 && (len % 4 || p[len - 1] == '='); const size_t L = ((len + 3) / 4 - pad) * 4; std::string str(L / 4 * 3 + pad, '\0'); for (size_t i = 0, j = 0; i < L; i += 4) { int n = table[p[i]] << 18 | table[p[i + 1]] << 12 | table[p[i + 2]] << 6 | table[p[i + 3]]; str[j++] = n >> 16; str[j++] = n >> 8 & 0xFF; str[j++] = n & 0xFF; } if (pad) { int n = table[p[L]] << 18 | table[p[L + 1]] << 12; str[str.size() - 1] = n >> 16; if (len > L + 2 && p[L + 2] != '=') { n |= table[p[L + 2]] << 6; str.push_back(n >> 8 & 0xFF); } } return str; } //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/Base64.h000066400000000000000000000023231513175025300215220ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- class Base64 { public: static std::string encode(const void* data, size_t len); static std::string decode(const void* data, size_t len); template static std::string encode(const T& value) { return encode(&value, sizeof(value)); } template static T decode(const std::string& in) { std::string decoded = decode(in.data(), in.size()); return *reinterpret_cast(decoded.data()); } }; //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/BlackMagic.h000066400000000000000000000125651513175025300224640ustar00rootroot00000000000000/* * (C) Copyright 2013 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once // This file contains preprocessor black magic. It contains macros that // can return the number of arguments passed //----------------------------------------------------------------------------------------------------------- // Public /// Returns the number of passed arguments #define __ATLAS_IO_NARG(...) /// Splice a and b together #define __ATLAS_IO_SPLICE(a, b) #define __ATLAS_IO_STRINGIFY(a) a #define __ATLAS_IO_TYPE(Type, ...) #define __ATLAS_IO_TYPE_SCOPE(Type, ...) //----------------------------------------------------------------------------------------------------------- // Details // Undefine these, to be redefined further down. #undef __ATLAS_IO_NARG #undef __ATLAS_IO_SPLICE #undef __ATLAS_IO_TYPE #undef __ATLAS_IO_TYPE_SCOPE #define __ATLAS_IO_REVERSE 5, 4, 3, 2, 1, 0 #define __ATLAS_IO_ARGN(_1, _2, _3, _4, _5, N, ...) N #define __ATLAS_IO_NARG__(dummy, ...) __ATLAS_IO_ARGN(__VA_ARGS__) #define __ATLAS_IO_NARG_(...) __ATLAS_IO_NARG__(dummy, ##__VA_ARGS__, __ATLAS_IO_REVERSE) #define __ATLAS_IO_SPLICE(a, b) __ATLAS_IO_SPLICE_1(a, b) #define __ATLAS_IO_SPLICE_1(a, b) __ATLAS_IO_SPLICE_2(a, b) #define __ATLAS_IO_SPLICE_2(a, b) a##b #define __ATLAS_IO_ARG16(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, ...) _15 #define __ATLAS_IO_HAS_COMMA(...) __ATLAS_IO_ARG16(__VA_ARGS__, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0) #define __ATLAS_IO_TRIGGER_PARENTHESIS(...) , #define __ATLAS_IO_ISEMPTY(...) \ __ATLAS_IO_ISEMPTY_(/* test if there is just one argument, eventually an empty \ one */ \ __ATLAS_IO_HAS_COMMA(__VA_ARGS__), /* test if \ _TRIGGER_PARENTHESIS_ \ together with the \ argument adds a comma */ \ __ATLAS_IO_HAS_COMMA( \ __ATLAS_IO_TRIGGER_PARENTHESIS __VA_ARGS__), /* test if the argument together with \ a parenthesis adds a comma */ \ __ATLAS_IO_HAS_COMMA(__VA_ARGS__(/*empty*/)), /* test if placing it between \ __ATLAS_IO_TRIGGER_PARENTHESIS and the \ parenthesis adds a comma */ \ __ATLAS_IO_HAS_COMMA(__ATLAS_IO_TRIGGER_PARENTHESIS __VA_ARGS__(/*empty*/))) #define __ATLAS_IO_PASTE5(_0, _1, _2, _3, _4) _0##_1##_2##_3##_4 #define __ATLAS_IO_ISEMPTY_(_0, _1, _2, _3) \ __ATLAS_IO_HAS_COMMA(__ATLAS_IO_PASTE5(__ATLAS_IO_IS_EMPTY_CASE_, _0, _1, _2, _3)) #define __ATLAS_IO_IS_EMPTY_CASE_0001 , #define __ATLAS_IO_NARG(...) __ATLAS_IO_SPLICE(__ATLAS_IO_CALL_NARG_, __ATLAS_IO_ISEMPTY(__VA_ARGS__))(__VA_ARGS__) #define __ATLAS_IO_CALL_NARG_1(...) 0 #define __ATLAS_IO_CALL_NARG_0 __ATLAS_IO_NARG_ #define __ATLAS_IO_COMMA_ARGS(...) \ __ATLAS_IO_SPLICE(__ATLAS_IO_COMMA_ARGS_, __ATLAS_IO_ISEMPTY(__VA_ARGS__))(__VA_ARGS__) #define __ATLAS_IO_COMMA_ARGS_1(...) #define __ATLAS_IO_COMMA_ARGS_0(...) , __VA_ARGS__ #define __ATLAS_IO_ARGS_OR_DUMMY(...) \ __ATLAS_IO_SPLICE(__ATLAS_IO_ARGS_OR_DUMMY_, __ATLAS_IO_ISEMPTY(__VA_ARGS__)) \ (__VA_ARGS__) #define __ATLAS_IO_ARGS_OR_DUMMY_0(...) __VA_ARGS__ #define __ATLAS_IO_ARGS_OR_DUMMY_1(...) 0 #define __ATLAS_IO_TYPE(Type, ...) \ __ATLAS_IO_SPLICE(__ATLAS_IO_TYPE_, __ATLAS_IO_ISEMPTY(__VA_ARGS__)) \ (Type, __ATLAS_IO_ARGS_OR_DUMMY(__VA_ARGS__)) #define __ATLAS_IO_TYPE_1(Type, dummy) Type __ATLAS_IO_SPLICE(__variable_, __LINE__) #define __ATLAS_IO_TYPE_0(Type, ...) Type __ATLAS_IO_SPLICE(__variable_, __LINE__)(__VA_ARGS__) #define __ATLAS_IO_TYPE_SCOPE(Type, ...) \ __ATLAS_IO_SPLICE(__ATLAS_IO_TYPE_SCOPE_, __ATLAS_IO_ISEMPTY(__VA_ARGS__)) \ (Type, __ATLAS_IO_ARGS_OR_DUMMY(__VA_ARGS__)) #define __ATLAS_IO_TYPE_SCOPE_1(Type, ...) \ for (bool __ATLAS_IO_SPLICE(__done_, __LINE__) = false; __ATLAS_IO_SPLICE(__done_, __LINE__) != true;) \ for (Type __ATLAS_IO_SPLICE(__variable_, __LINE__); __ATLAS_IO_SPLICE(__done_, __LINE__) != true; \ __ATLAS_IO_SPLICE(__done_, __LINE__) = true) #define __ATLAS_IO_TYPE_SCOPE_0(Type, ...) \ for (bool __ATLAS_IO_SPLICE(__done_, __LINE__) = false; __ATLAS_IO_SPLICE(__done_, __LINE__) != true;) \ for (Type __ATLAS_IO_SPLICE(__variable_, __LINE__)(__VA_ARGS__); __ATLAS_IO_SPLICE(__done_, __LINE__) != true; \ __ATLAS_IO_SPLICE(__done_, __LINE__) = true) atlas-0.45.0/atlas_io/src/atlas_io/detail/Checksum.cc000066400000000000000000000040411513175025300223750ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "Checksum.h" #include #include "eckit/utils/Hash.h" #include "eckit/utils/Tokenizer.h" #include "atlas_io/Trace.h" #include "atlas_io/detail/Defaults.h" namespace atlas { namespace io { Checksum::Checksum(const std::string& checksum) { std::vector tokens; eckit::Tokenizer tokenizer(':'); eckit::Tokenizer{':'}(checksum, tokens); if (tokens.size() == 1) { algorithm_ = "none"; checksum_ = ""; } else { algorithm_ = tokens[0]; checksum_ = tokens[1]; } } bool Checksum::available() const { return checksum_.size() && algorithm_ != "none"; } std::string Checksum::str() const { if (algorithm_.empty()) { return ""; } return algorithm_ + ":" + checksum_; } std::string Checksum::str(size_t size) const { if (algorithm_.empty()) { return ""; } return algorithm_ + ":" + checksum_.substr(0, std::min(size, checksum_.size())); } std::string checksum(const void* buffer, size_t size, const std::string& algorithm) { auto is_available = [](const std::string& alg) -> bool { return eckit::HashFactory::instance().has(alg); }; auto hash = [&](const std::string& alg) -> std::string { std::unique_ptr hasher(eckit::HashFactory::instance().build(alg)); ATLAS_IO_TRACE("checksum(" + alg + ")"); return std::string(alg) + ":" + hasher->compute(buffer, long(size)); }; std::string alg = algorithm.empty() ? defaults::checksum_algorithm() : algorithm; if (is_available(alg)) { return hash(alg); } else { return hash("none"); } } } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/Checksum.h000066400000000000000000000016401513175025300222410ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include namespace atlas { namespace io { class Checksum { public: Checksum() = default; Checksum(const std::string& checksum); bool available() const; std::string str() const; std::string str(size_t size) const; std::string algorithm() const { return algorithm_; } private: std::string algorithm_; std::string checksum_; }; std::string checksum(const void* buffer, size_t size, const std::string& algorithm = ""); } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/DataInfo.h000066400000000000000000000032611513175025300221650ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include "atlas_io/detail/Checksum.h" #include "atlas_io/detail/Endian.h" namespace atlas { namespace io { class DataInfo { public: int section() const { return section_; } const std::string& compression() const { return compression_; } Endian endian() const { return endian_; } void endian(Endian e) { endian_ = e; } void compression(const std::string& c) { compression_ = c; } void size(size_t s) { uncompressed_size_ = s; } size_t size() const { return uncompressed_size_; } void compressed_size(size_t s) { compressed_size_ = s; } size_t compressed_size() const { return compressed_size_; } void compressed(bool f) { if (f == false) { compression("none"); } } bool compressed() const { return compression_ != "none"; } operator bool() const { return section_ > 0; } const Checksum& checksum() const { return checksum_; } void checksum(const std::string& c) { checksum_ = Checksum(c); } void section(int s) { section_ = s; } private: int section_{0}; std::string compression_{"none"}; Checksum checksum_; Endian endian_{Endian::native}; size_t uncompressed_size_{0}; size_t compressed_size_{0}; }; } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/DataType.cc000066400000000000000000000021401513175025300223440ustar00rootroot00000000000000/* * (C) Copyright 2013 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "DataType.h" #include #include "atlas_io/Exceptions.h" //------------------------------------------------------------------------------------------------------ namespace atlas { namespace io { void DataType::throw_not_recognised(kind_t kind) { std::stringstream msg; msg << "kind [" << kind << "] not recognised."; throw Exception(msg.str(), Here()); } void DataType::throw_not_recognised(std::string datatype) { std::stringstream msg; msg << "datatype [" << datatype << "] not recognised."; throw Exception(msg.str(), Here()); } //------------------------------------------------------------------------------------------------------ } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/DataType.h000066400000000000000000000261661513175025300222240ustar00rootroot00000000000000/* * (C) Copyright 2013 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include //------------------------------------------------------------------------------------------------------ // For type safety we want to use std::byte for the DataType "BYTE", but it is a C++17 feature. // Backport std::byte here without any operations #if __cplusplus >= 201703L #include #else #ifndef STD_BYTE_DEFINED #define STD_BYTE_DEFINED namespace std { #ifdef _CRAYC struct byte { unsigned char byte_; }; #else enum class byte : unsigned char { }; #endif } // namespace std #endif #endif //------------------------------------------------------------------------------------------------------ namespace atlas { namespace io { class DataType { public: typedef long kind_t; static const kind_t KIND_BYTE = 1; static const kind_t KIND_INT32 = -4; static const kind_t KIND_INT64 = -8; static const kind_t KIND_REAL32 = 4; static const kind_t KIND_REAL64 = 8; static const kind_t KIND_UINT64 = -16; template static DataType create(); static DataType byte() { return DataType(KIND_BYTE); } static DataType int32() { return DataType(KIND_INT32); } static DataType int64() { return DataType(KIND_INT64); } static DataType real32() { return DataType(KIND_REAL32); } static DataType real64() { return DataType(KIND_REAL64); } static DataType uint64() { return DataType(KIND_UINT64); } template static kind_t kind(); template static kind_t kind(const DATATYPE&); template static std::string str(); template static std::string str(const DATATYPE); static kind_t str_to_kind(const std::string&); static std::string kind_to_str(kind_t); static bool kind_valid(kind_t); private: static std::string byte_str() { return "byte"; } static std::string int32_str() { return "int32"; } static std::string int64_str() { return "int64"; } static std::string real32_str() { return "real32"; } static std::string real64_str() { return "real64"; } static std::string uint64_str() { return "uint64"; } [[noreturn]] static void throw_not_recognised(kind_t); [[noreturn]] static void throw_not_recognised(std::string datatype); public: DataType(const std::string&); DataType(long); DataType(const DataType&); DataType& operator=(const DataType&); std::string str() const { return kind_to_str(kind_); } kind_t kind() const { return kind_; } size_t size() const { return (kind_ == KIND_UINT64) ? 8 : std::abs(kind_); } friend bool operator==(DataType dt1, DataType dt2); friend bool operator!=(DataType dt1, DataType dt2); friend bool operator==(DataType dt, kind_t kind); friend bool operator!=(DataType dt, kind_t kind); friend bool operator==(kind_t kind, DataType dt); friend bool operator!=(kind_t kind, DataType dt2); private: kind_t kind_; }; template <> inline std::string DataType::str() { return byte_str(); } template <> inline std::string DataType::str() { return byte_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(int) == 4, ""); return int32_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(int) == 4, ""); return int32_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(long) == 8, ""); return int64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(long) == 8, ""); return int64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(long long) == 8, ""); return int64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(long long) == 8, ""); return int64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(float) == 4, ""); return real32_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(float) == 4, ""); return real32_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(double) == 8, ""); return real64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(double) == 8, ""); return real64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(unsigned long) == 8, ""); return uint64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(unsigned long) == 8, ""); return uint64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(unsigned long long) == 8, ""); return uint64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(unsigned long long) == 8, ""); return uint64_str(); } template <> inline std::string DataType::str(const int&) { return str(); } template <> inline std::string DataType::str(const long&) { return str(); } template <> inline std::string DataType::str(const long long&) { return str(); } template <> inline std::string DataType::str(const unsigned long&) { return str(); } template <> inline std::string DataType::str(const unsigned long long&) { return str(); } template <> inline std::string DataType::str(const float&) { return str(); } template <> inline std::string DataType::str(const double&) { return str(); } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(std::byte) == 1, ""); return KIND_BYTE; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(std::byte) == 1, ""); return KIND_BYTE; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(int) == 4, ""); return KIND_INT32; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(int) == 4, ""); return KIND_INT32; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(long) == 8, ""); return KIND_INT64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(long) == 8, ""); return KIND_INT64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(long long) == 8, ""); return KIND_INT64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(long long) == 8, ""); return KIND_INT64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(unsigned long) == 8, ""); return KIND_UINT64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(unsigned long) == 8, ""); return KIND_UINT64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(unsigned long long) == 8, ""); return KIND_UINT64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(unsigned long long) == 8, ""); return KIND_UINT64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(float) == 4, ""); return KIND_REAL32; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(float) == 4, ""); return KIND_REAL32; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(double) == 8, ""); return KIND_REAL64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(double) == 8, ""); return KIND_REAL64; } template <> inline DataType::kind_t DataType::kind(const int&) { return kind(); } template <> inline DataType::kind_t DataType::kind(const long&) { return kind(); } template <> inline DataType::kind_t DataType::kind(const unsigned long&) { return kind(); } template <> inline DataType::kind_t DataType::kind(const float&) { return kind(); } template <> inline DataType::kind_t DataType::kind(const double&) { return kind(); } inline DataType::kind_t DataType::str_to_kind(const std::string& datatype) { if (datatype == "int32") return KIND_INT32; else if (datatype == "int64") return KIND_INT64; else if (datatype == "uint64") return KIND_UINT64; else if (datatype == "real32") return KIND_REAL32; else if (datatype == "real64") return KIND_REAL64; else if (datatype == "byte") { return KIND_BYTE; } else { throw_not_recognised(datatype); } } inline std::string DataType::kind_to_str(kind_t kind) { switch (kind) { case KIND_INT32: return int32_str(); case KIND_INT64: return int64_str(); case KIND_UINT64: return uint64_str(); case KIND_REAL32: return real32_str(); case KIND_REAL64: return real64_str(); case KIND_BYTE: return byte_str(); default: throw_not_recognised(kind); } } inline bool DataType::kind_valid(kind_t kind) { switch (kind) { case KIND_BYTE: case KIND_INT32: case KIND_INT64: case KIND_UINT64: case KIND_REAL32: case KIND_REAL64: return true; default: return false; } } inline DataType::DataType(const DataType& other): kind_(other.kind_) {} inline DataType& DataType::operator=(const DataType& other) { kind_ = other.kind_; return *this; } inline DataType::DataType(const std::string& datatype): kind_(str_to_kind(datatype)) {} inline DataType::DataType(long kind): kind_(kind) {} inline bool operator==(DataType dt1, DataType dt2) { return dt1.kind_ == dt2.kind_; } inline bool operator!=(DataType dt1, DataType dt2) { return dt1.kind_ != dt2.kind_; } inline bool operator==(DataType dt, DataType::kind_t kind) { return dt.kind_ == kind; } inline bool operator!=(DataType dt, DataType::kind_t kind) { return dt.kind_ != kind; } inline bool operator==(DataType::kind_t kind, DataType dt) { return dt.kind_ == kind; } inline bool operator!=(DataType::kind_t kind, DataType dt) { return dt.kind_ != kind; } template inline DataType DataType::create() { return DataType(DataType::kind()); } template inline DataType make_datatype() { return DataType(DataType::kind()); } //------------------------------------------------------------------------------------------------------ } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/Decoder.cc000066400000000000000000000015541513175025300222060ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "Decoder.h" #include "atlas_io/Trace.h" namespace atlas { namespace io { void decode(const atlas::io::Metadata& metadata, const atlas::io::Data& data, Decoder& decoder) { ATLAS_IO_TRACE("decode"); decoder.self_->decode_(metadata, data); } void decode(const atlas::io::Metadata& metadata, const atlas::io::Data& data, Decoder&& decoder) { ATLAS_IO_TRACE_SCOPE("decode"); decoder.self_->decode_(metadata, data); } } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/Decoder.h000066400000000000000000000031341513175025300220440ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include "atlas_io/Data.h" #include "atlas_io/Metadata.h" #include "atlas_io/detail/TypeTraits.h" namespace atlas { namespace io { class Decoder { public: template = 0> explicit Decoder(T& value): self_(new DecodableItem(value)) {} friend void decode(const atlas::io::Metadata& metadata, const atlas::io::Data& data, Decoder&); friend void decode(const atlas::io::Metadata& metadata, const atlas::io::Data& data, Decoder&&); private: struct Decodable { virtual ~Decodable() = default; virtual void decode_(const atlas::io::Metadata&, const atlas::io::Data&) = 0; }; template struct DecodableItem : Decodable { explicit DecodableItem(T& value): data_(value) {} void decode_(const atlas::io::Metadata& metadata, const atlas::io::Data& encoded) override { decode(metadata, encoded, data_); } T& data_; }; std::shared_ptr self_; }; void decode(const Metadata&, const Data&, Decoder&); void decode(const Metadata&, const Data&, Decoder&&); } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/Defaults.h000066400000000000000000000025331513175025300222500ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include "eckit/config/Resource.h" namespace atlas { namespace io { namespace defaults { [[maybe_unused]] static std::string checksum_algorithm() { static std::string checksum = eckit::Resource("atlas.io.checksum.algorithm;$ATLAS_IO_CHECKSUM", "xxh64"); return checksum; } [[maybe_unused]] static bool checksum_read() { static bool checksum = eckit::Resource("atlas.io.checksum.read;$ATLAS_IO_CHECKSUM_READ", true); return checksum; } [[maybe_unused]] static bool checksum_write() { static bool checksum = eckit::Resource("atlas.io.checksum.write;$ATLAS_IO_CHECKSUM_WRITE", true); return checksum; } [[maybe_unused]] static const std::string& compression_algorithm() { static std::string compression = eckit::Resource("atlas.io.compression;$ATLAS_IO_COMPRESSION", "none"); return compression; } } // namespace defaults } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/Encoder.cc000066400000000000000000000015171513175025300222170ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "Encoder.h" #include "atlas_io/Trace.h" namespace atlas { namespace io { size_t encode_metadata(const Encoder& encoder, atlas::io::Metadata& metadata) { ATLAS_IO_TRACE(); ASSERT(encoder); return encoder.self_->encode_metadata_(metadata); } void encode_data(const Encoder& encoder, atlas::io::Data& out) { ATLAS_IO_TRACE(); ASSERT(encoder); encoder.self_->encode_data_(out); } } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/Encoder.h000066400000000000000000000064661513175025300220710ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include #include #include "atlas_io/Data.h" #include "atlas_io/RecordItem.h" #include "atlas_io/detail/DataInfo.h" #include "atlas_io/detail/Link.h" #include "atlas_io/detail/Reference.h" #include "atlas_io/detail/TypeTraits.h" namespace atlas { namespace io { class Encoder { public: Encoder() = default; operator bool() const { return bool(self_); } template = 0> explicit Encoder(T&& x): self_(new EncodableValue(std::move(x))) {} Encoder(const Link& link): self_(new EncodableLink(link)) {} Encoder(Encoder&& other): self_(std::move(other.self_)) {} template = 0> explicit Encoder(const T& x): self_(new EncodableValue(x)) {} Encoder& operator=(Encoder&& rhs) { self_ = std::move(rhs.self_); return *this; } friend size_t encode_metadata(const Encoder&, atlas::io::Metadata&); friend void encode_data(const Encoder&, atlas::io::Data&); bool encodes_data() const { return self_->encodes_data_(); } private: struct Encodable { virtual ~Encodable() = default; virtual size_t encode_metadata_(atlas::io::Metadata&) const = 0; virtual void encode_data_(atlas::io::Data&) const = 0; virtual bool encodes_data_() const = 0; }; template struct EncodableValue : Encodable { EncodableValue(Value&& v): value_{std::move(v)} { sfinae::encode_metadata(value_, metadata_, data_size_); } template = 0> EncodableValue(const Value& v): value_{v} { sfinae::encode_metadata(value_, metadata_, data_size_); } size_t encode_metadata_(atlas::io::Metadata& metadata) const override { metadata.set(metadata_); return data_size_; } void encode_data_(atlas::io::Data& out) const override { sfinae::encode_data(value_, out); } bool encodes_data_() const override { return data_size_ > 0; } const Value value_; Metadata metadata_; size_t data_size_{0}; }; struct EncodableLink : Encodable { EncodableLink(const Link& link): link_(link) {} size_t encode_metadata_(atlas::io::Metadata& metadata) const override { metadata.set(atlas::io::Metadata("link", link_.uri)); return 0; } void encode_data_(atlas::io::Data& /*out*/) const override {} bool encodes_data_() const override { return false; } Link link_; }; std::unique_ptr self_; }; size_t encode_metadata(const Encoder& encoder, atlas::io::Metadata& metadata); void encode_data(const Encoder& encoder, atlas::io::Data& out); } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/Endian.h000066400000000000000000000016701513175025300217000ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "atlas_io/detail/defines.h" #ifndef ATLAS_IO_BIG_ENDIAN #error ATLAS_IO_BIG_ENDIAN not defined #endif #ifndef ATLAS_IO_LITTLE_ENDIAN #error ATLAS_IO_LITTLE_ENDIAN not defined #endif namespace atlas { namespace io { enum class Endian { little = 0, big = 1, #if ATLAS_IO_BIG_ENDIAN native = big, swapped = little #elif ATLAS_IO_LITTLE_ENDIAN native = little, swapped = big #else #error Neither ATLAS_IO_BIG_ENDIAN nor ATLAS_IO_LITTLE_ENDIAN equals true #endif }; } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/Link.cc000066400000000000000000000021571513175025300215360ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "Link.h" #include "eckit/filesystem/PathName.h" #include "atlas_io/RecordItem.h" namespace atlas { namespace io { //bool Link::relative() const { // std::string path = RecordItem::URI{uri}.path; // if( path.size() == 0 ) { // return true; // } // if( path[0] == '/' ) { // return false; // } // if( path[0] == '~' ) { // return false; // } // return true; //} //Link operator/( const eckit::PathName& dir, const Link& link ) { // auto relative_path = eckit::PathName{ RecordItem::URI{link}.path }; // auto absolute_uri = RecordItem::URI(link); // absolute_uri.path = dir / relative_path; // return Link{absolute_uri.str()}; //}; } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/Link.h000066400000000000000000000015241513175025300213750ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include namespace eckit { class PathName; } namespace atlas { namespace io { struct Link { std::string uri; const std::string& str() const { return uri; } operator bool() const { return uri.size(); } operator const std::string&() const { return str(); } // bool relative() const; // friend Link operator/( const eckit::PathName& dir, const Link& link ); }; } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/NoConfig.h000066400000000000000000000016071513175025300222040ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/config/LocalConfiguration.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- class NoConfig : public eckit::LocalConfiguration { public: NoConfig() = default; virtual ~NoConfig() = default; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/ParsedRecord.h000066400000000000000000000027121513175025300230550ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include #include "atlas_io/Metadata.h" #include "atlas_io/detail/RecordSections.h" namespace atlas { namespace io { /// Low-level Record information container. /// /// No big data is kept here, only metadata, and information /// on how to retrieve data at a later stage class ParsedRecord { public: RecordHead head; ///< head section of parsed record std::vector keys; ///< Keys of items encoded in parsed record std::map items; ///< Items encoded in parsed record std::vector data_sections; ///< Description of data sections in parsed record /// The parse() function needs to be called during the reading of the record and /// completes the "items" through introspection of the "data_sections". /// It also computes uncompressed data size using available metadata in the "items" void parse(); }; } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/RecordInfo.h000066400000000000000000000013301513175025300225250ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "atlas_io/detail/Time.h" #include "atlas_io/detail/Version.h" namespace atlas { namespace io { struct RecordInfo { Version version_; Time created_; const Version& version() const { return version_; } const Time& created() const { return created_; } }; } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/RecordSections.h000066400000000000000000000137571513175025300234410ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/types/FixedString.h" #include "atlas_io/detail/Endian.h" #include "atlas_io/detail/Time.h" #include "atlas_io/detail/Version.h" namespace atlas { namespace io { // ------------------------------------------------------------------------------------------------------------------------------------ struct RecordBegin { eckit::FixedString<8> string{"ATLAS-IO"}; ///< 16 ATLAS-IO string eckit::FixedString<8> spare{"\n"}; ///< bool valid() const { return string == "ATLAS-IO"; } std::string str() const { return string; } }; struct RecordEnd { // 32 bytes static constexpr size_t bytes = 32; eckit::FixedString<1> eol{"\n"}; eckit::FixedString<12> string{"ATLAS-IO-END"}; eckit::FixedString<19> padding{" \n\n\n\n"}; bool valid() const { return string == "ATLAS-IO-END"; } std::string str() const { return string; } }; // ------------------------------------------------------------------------------------------------------------------------------------ struct RecordHead { static constexpr size_t bytes = 256; static constexpr size_t padding_ = bytes - 16 - 8 - 16 - 8 * 4 - 64 - 8 * 2 - 4 - 1; RecordBegin begin; ///< 16 beginning of record Version version; ///< 8 version of this record Time time; ///< 16 time since system_clock epoch (1970-1-1 00:00) uint64_t record_length{0}; ///< 8 length of entire record eckit::FixedString<8> metadata_format{"yaml"}; ///< 8 format of metadata section in this record std::uint64_t metadata_offset{bytes}; ///< 8 offset where metadata section starts std::uint64_t metadata_length{0}; ///< 8 length of metadata section eckit::FixedString<64> metadata_checksum; ///< 64 checksum of metadata std::uint64_t index_offset{0}; ///< 8 offset where data section starts std::uint64_t index_length; ///< 8 length of data section std::uint32_t magic_number{1234}; ///< 4 number 1234 encoded in binary, used to detect encoded endianness eckit::FixedString padding__; ///< Extra padding to get to eckit::FixedString<1> eol{"\n"}; static constexpr size_t size() { return bytes; } ///< Size in bytes of this section Endian endian() const; ///< Endianness determined from magic_number bool valid() const { return begin.valid(); } ///< Check if this is a valid RecordRoot std::string str() const { return begin.str(); } }; // ------------------------------------------------------------------------------------------------------------------------------------ struct RecordMetadataSection { struct Begin { // 32 bytes static constexpr size_t bytes = 32; eckit::FixedString<1> eol{"\n"}; eckit::FixedString<14> string{"METADATA-BEGIN"}; eckit::FixedString<17> padding{" \n"}; bool valid() const { return string == "METADATA-BEGIN"; } std::string str() const { return string; } }; struct End { // 32 bytes static constexpr size_t bytes = 32; eckit::FixedString<1> eol{"\n"}; eckit::FixedString<12> string{"METADATA-END"}; eckit::FixedString<19> padding{" \n"}; bool valid() const { return string == "METADATA-END"; } std::string str() const { return string; } }; }; // ------------------------------------------------------------------------------------------------------------------------------------ struct RecordDataIndexSection { struct Begin { // 32 bytes static constexpr size_t bytes = 32; eckit::FixedString<1> eol{"\n"}; eckit::FixedString<11> string{"INDEX-BEGIN"}; eckit::FixedString<20> padding{" \n"}; bool valid() const { return string == "INDEX-BEGIN"; } std::string str() const { return string; } }; struct End { // 32 bytes static constexpr size_t bytes = 32; eckit::FixedString<1> eol{"\n"}; eckit::FixedString<9> string{"INDEX-END"}; eckit::FixedString<22> padding{" \n"}; bool valid() const { return string == "INDEX-END"; } std::string str() const { return string; } }; struct Entry { static constexpr size_t bytes = 80; std::uint64_t offset; std::uint64_t length; eckit::FixedString<64> checksum; }; }; // ------------------------------------------------------------------------------------------------------------------------------------ struct RecordDataSection { struct Begin { // 32 bytes static constexpr size_t bytes = 32; eckit::FixedString<1> eol{"\n"}; eckit::FixedString<10> string{"DATA-BEGIN"}; eckit::FixedString<21> padding{" \n"}; bool valid() const { return string == "DATA-BEGIN"; } std::string str() const { return string; } }; struct End { // 32 bytes static constexpr size_t bytes = 32; eckit::FixedString<1> eol{"\n"}; eckit::FixedString<8> string{"DATA-END"}; eckit::FixedString<23> padding{" \n"}; bool valid() const { return string == "DATA-END"; } std::string str() const { return string; } }; }; // ------------------------------------------------------------------------------------------------------------------------------------ } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/Reference.h000066400000000000000000000022111513175025300223700ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "atlas_io/Data.h" #include "atlas_io/Metadata.h" #include "atlas_io/detail/sfinae.h" #include "atlas_io/Exceptions.h" namespace atlas { namespace io { template struct Reference { const T* ref; Reference(const T& r): ref(&r) {} friend size_t encode_metadata(const Reference& in, atlas::io::Metadata& metadata) { size_t size{0}; if (not sfinae::encode_metadata(*in.ref, metadata, size)) { throw NotEncodable(*in.ref); } return size; } friend void encode_data(const Reference& in, atlas::io::Data& out) { if (not sfinae::encode_data(*in.ref, out)) { throw NotEncodable(*in.ref); } } }; } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/StaticAssert.h000066400000000000000000000055571513175025300231230ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #ifndef ATLAS_IO_STATIC_ASSERT #if defined(__clang_analyzer__) #define ATLAS_IO_STATIC_ASSERT 0 #elif defined(__clang__) && (__clang_major__ < 4) #define ATLAS_IO_STATIC_ASSERT 0 #else #define ATLAS_IO_STATIC_ASSERT 1 #endif #endif #if ATLAS_IO_STATIC_ASSERT #include #include "atlas_io/detail/TypeTraits.h" namespace atlas { namespace io { template = 0> void encode_data(const T& in, atlas::io::Data& out) { static_assert( can_encode_data(), "\n\n" "\n Static assertion failed" "\n -----------------------" "\n" "\n Values of template type T cannot be encoded into Data because following function is not defined:" "\n" "\n void encode_data(const T& value, atlas::io::Data& out);" "\n" "\n Note that argument-dependent-lookup rules apply." "\n --> The function must be declared in the namespace of type T." "\n\n"); } template = 0> size_t encode_metadata(const T&, atlas::io::Metadata&) { static_assert( can_encode_metadata(), "\n\n" "\n Static assertion failed" "\n -----------------------" "\n" "\n Values of template type T cannot be incoded into Metadata because following function is not defined:" "\n" "\n size_t encode_metadata(const T& value, atlas::io::Metadata& metadata);" "\n" "\n Note that argument-dependent-lookup rules apply." "\n --> The function must be declared in the namespace of type T" "\n\n"); return 0; } template = 0> void decode(const atlas::io::Metadata&, const atlas::io::Data&, T&) { static_assert(is_decodable(), "\n\n" "\n Static assertion failed" "\n -----------------------" "\n" "\n Values of template type T cannot be decoded because following function is not defined:" "\n" "\n void decode(const atlas::io::Metadata&, const atlas::io::Data&, T& out);" "\n" "\n Note that argument-dependent-lookup rules apply." "\n The function must be declared in the namespace of type T" "\n\n"); } } // namespace io } // namespace atlas #endif atlas-0.45.0/atlas_io/src/atlas_io/detail/Time.cc000066400000000000000000000072221513175025300215350ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "Time.h" // ------------------------------------------------------------------------------------------------------- // Some older compilers, e.g. intel 17 rely on GCC 4, which does not have std::put_time implemented yet, even though it is in C++11. // So implement it here. #include // std::ios_base #include // std::ostreambuf_iterator #include // std::use_facet, std::time_put #include // std::basic_ostream namespace atlas_std { namespace { template struct _put_time { const std::tm* time; const char* fmt; }; template inline _put_time put_time(const std::tm* time, const CharT* fmt) { return {time, fmt}; } template std::basic_ostream& operator<<(std::basic_ostream& os, _put_time f) { using Iter = typename std::ostreambuf_iterator; using TimePut = std::time_put; const CharT* const fmt_end = f.fmt + Traits::length(f.fmt); const TimePut& mp = std::use_facet(os.getloc()); std::ios_base::iostate err = std::ios_base::goodbit; try { if (mp.put(Iter(os.rdbuf()), os, os.fill(), f.time, f.fmt, fmt_end).failed()) { err |= std::ios_base::badbit; } } catch (...) { err |= std::ios_base::badbit; } if (err) { os.setstate(err); } return os; } } // namespace } // namespace atlas_std // ------------------------------------------------------------------------------------------------------- #include #include #include #include #include #include "eckit/log/JSON.h" namespace atlas { namespace io { namespace { static std::time_t to_time_t(Time time) { return std::time_t(time.tv_sec); } static Time from_time_point(std::chrono::time_point t) { using namespace std::chrono; auto since_epoch = t.time_since_epoch(); auto sec_since_epoch = duration_cast(since_epoch); auto nsec_since_epoch = duration_cast(since_epoch); auto extra_nsec = duration_cast(nsec_since_epoch - sec_since_epoch); Time time; time.tv_sec = static_cast(sec_since_epoch.count()); time.tv_nsec = static_cast(extra_nsec.count()); return time; } } // namespace Time Time::now() { return from_time_point(std::chrono::system_clock::now()); } void Time::print(std::ostream& out) const { // Will print time-date in ISO 8601 format: 1970-01-01T00:00:00.123456789Z auto time = to_time_t(*this); out << atlas_std::put_time(::gmtime(&time), "%FT%T") << "." << tv_nsec << "Z"; // Note, normally we should be using std::put_time instead of above implemented // atlas_std::put_time but some installations that we support don't implement it. } std::ostream& operator<<(std::ostream& out, const Time& time) { time.print(out); return out; } eckit::JSON& operator<<(eckit::JSON& out, const Time& time) { std::stringstream s; s << time; out << s.str(); return out; } std::string Time::str() const { std::stringstream s; print(s); return s.str(); } } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/Time.h000066400000000000000000000022721513175025300213770ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include namespace eckit { class JSON; } namespace atlas { namespace io { /// Store UTC time up to nanosecond precision struct Time { std::uint64_t tv_sec{0}; ///< seconds since Epoch (1970-01-01T00:00:00Z) std::uint64_t tv_nsec{0}; ///< additional nanoseconds /// Create current time using system clock static Time now(); /// print UTC time in ISO 8601 format: "1970-01-01T00:00:00.123456789Z" void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream&, const Time&); friend eckit::JSON& operator<<(eckit::JSON&, const Time&); /// @return string of UTC time in ISO 8601 format: "1970-01-01T00:00:00.123456789Z" std::string str() const; }; } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/Type.h000066400000000000000000000014661513175025300214260ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include namespace atlas { namespace io { struct Type { const std::string name_; operator const std::string&() { return name_; } operator bool() const { return name_.size(); } Type(const char* name): name_(name) {} Type(const std::string& name): name_(name) {} bool operator==(const Type& other) const { return name_ == other.name_; } }; } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/TypeTraits.h000066400000000000000000000120061513175025300226050ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include "atlas_io/detail/DataType.h" namespace atlas { namespace io { class Metadata; class Data; class Stream; namespace adl_tests { using std::declval; using std::is_pod; template std::false_type can_interprete(...) noexcept(false); template (), declval()))> std::true_type can_interprete(int) noexcept(noexcept(interprete(declval(), declval()))); template std::false_type can_encode_data(...) noexcept(false); template (), declval()))> std::true_type can_encode_data(int) noexcept(noexcept(encode_data(declval(), declval()))); template std::false_type can_encode_metadata(...) noexcept(false); template (), declval()))> std::true_type can_encode_metadata(int) noexcept(noexcept(encode_metadata(declval(), declval()))); template std::false_type can_decode(...) noexcept(false); template (), declval(), declval()))> std::true_type can_decode(int) noexcept(noexcept(decode(declval(), declval(), declval()))); } // namespace adl_tests template static constexpr bool is_interpretable() { return decltype(adl_tests::can_interprete::type, A>(0))::value; } template static constexpr bool can_encode_data() { return decltype(adl_tests::can_encode_data::type>(0))::value; } template static constexpr bool can_encode_metadata() { return decltype(adl_tests::can_encode_metadata::type>(0))::value; } template static constexpr bool is_encodable() { return can_encode_data() && can_encode_metadata(); } template static constexpr bool is_decodable() { return decltype(adl_tests::can_decode::type>(0))::value; } template static constexpr bool is_encodable_rvalue() { return is_encodable() && std::is_rvalue_reference::value; }; template using enable_if_t = typename std::enable_if::type; template using disable_if_t = typename std::enable_if::type; template using enable_if_encodable_t = enable_if_t()>; template using disable_if_encodable_t = disable_if_t()>; template using enable_if_decodable_t = enable_if_t()>; template using disable_if_decodable_t = disable_if_t()>; template using enable_if_can_encode_metadata_t = enable_if_t()>; template using disable_if_can_encode_metadata_t = disable_if_t()>; template using enable_if_can_encode_data_t = enable_if_t()>; template using disable_if_can_encode_data_t = disable_if_t()>; template using enable_if_interpretable_t = enable_if_t()>; template using disable_if_interpretable_t = disable_if_t()>; template using enable_if_rvalue_t = enable_if_t::value>; template using enable_if_move_constructible_encodable_rvalue_t = enable_if_t() && std::is_rvalue_reference() && std::is_move_constructible()>; template using enable_if_move_constructible_decodable_rvalue_t = enable_if_t() && std::is_rvalue_reference() && std::is_move_constructible()>; template using enable_if_scalar_t = enable_if_t::value && EnableBool>; template constexpr bool is_array_datatype() { return std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v; } template using enable_if_array_datatype = typename std::enable_if(), int>::type; } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/Version.h000066400000000000000000000032051513175025300221230ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include "eckit/types/SemanticVersion.h" namespace atlas { namespace io { struct Version { // 8 bytes std::uint32_t major{0}; ///< Major version std::uint32_t minor{2}; ///< Minor version std::string str() const { return std::to_string(major) + "." + std::to_string(minor); } operator std::string() const { return str(); } operator eckit::SemanticVersion() const { return eckit::SemanticVersion{major, minor, 0}; } bool operator<(const Version& v) const { return eckit::SemanticVersion{major, minor, 0} < eckit::SemanticVersion{v.major, v.minor, 0}; } bool operator==(const Version& v) const { return eckit::SemanticVersion{major, minor, 0} == eckit::SemanticVersion{v.major, v.minor, 0}; } bool operator!=(const Version& v) const { return !(*this == v); } bool operator<=(const Version& v) const { return (*this < v) or (*this == v); } bool operator>(const Version& v) const { return !(*this <= v); } bool operator>=(const Version& v) const { return (*this > v) or (*this == v); } friend std::ostream& operator<<(std::ostream& out, const Version& v) { out << v.str(); return out; } }; } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/defines.h.in000066400000000000000000000015301513175025300225170ustar00rootroot00000000000000#if 0 /* * (C) Copyright 2022 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ // clang-format off #endif #ifndef atlas_io_defines_h #define atlas_io_defines_h #cmakedefine01 ATLAS_IO_HAVE_CXXABI_H #cmakedefine01 ATLAS_IO_LITTLE_ENDIAN #cmakedefine01 ATLAS_IO_BIG_ENDIAN #define ATLAS_IO_ECKIT_VERSION_INT @ATLAS_IO_ECKIT_VERSION_INT@ #define ATLAS_IO_ECKIT_DEVELOP @ATLAS_IO_ECKIT_DEVELOP@ #define ATLAS_IO_ECKIT_VERSION_AT_LEAST(x, y, z) (ATLAS_IO_ECKIT_VERSION_INT >= x * 10000 + y * 100 + z) #endif atlas-0.45.0/atlas_io/src/atlas_io/detail/sfinae.h000066400000000000000000000060101513175025300217400ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "atlas_io/detail/TypeTraits.h" namespace atlas { namespace io { // ------------------------------------------------------------------------------------------------------- namespace { // These anonymous namespace functions are to avoid recursive behaviour in // following sfinae namespace template = 0> inline void do_interprete(const T& in, A& interpreted) { interprete(in, interpreted); } template = 0> inline size_t do_encode_metadata(const T& in, Metadata& out) { size_t size = encode_metadata(in, out); return size; } template = 0> inline void do_encode_data(const T& in, Data& out) { encode_data(in, out); } } // namespace // ------------------------------------------------------------------------------------------------------- namespace sfinae { // ------------------------------------------------------------------------------------------------------- template = 0> bool interprete(const T& in, A& interpreted) { do_interprete(in, interpreted); return true; } template = 0> bool interprete(const T& /*in*/, A& /*interpreted*/) { return false; } // ------------------------------------------------------------------------------------------------------- template = 0> bool encode_data(const T& in, Data& out) { do_encode_data(in, out); return true; } template = 0> bool encode_data(const T&, Data&) { return false; } // ------------------------------------------------------------------------------------------------------- template = 0> bool encode_metadata(const T& in, Metadata& out) { do_encode_metadata(in, out); return true; } template = 0> bool encode_metadata(const T&, Metadata&) { return false; } template = 0> bool encode_metadata(const T& in, Metadata& out, size_t& data_size) { data_size = do_encode_metadata(in, out); return true; } template = 0> bool encode_metadata(const T&, Metadata&, size_t& data_size) { data_size = 0; return false; } // ------------------------------------------------------------------------------------------------------- } // namespace sfinae } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/detail/tag.h000066400000000000000000000011071513175025300212500ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once namespace atlas { namespace io { namespace tag { struct disable_static_assert {}; struct enable_static_assert {}; } // namespace tag } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/print/000077500000000000000000000000001513175025300202175ustar00rootroot00000000000000atlas-0.45.0/atlas_io/src/atlas_io/print/Bytes.cc000066400000000000000000000056451513175025300216260ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "Bytes.h" #include #include #include #include #include namespace atlas { namespace io { namespace { template class FixedFormat { public: using value_type = Value; FixedFormat(value_type x, long precision): x_(x), precision_(precision > 0 ? precision : 20) {} void print(std::ostream& out) const { for (long precision = 0; precision <= precision_; ++precision) { if (is_precision(precision) || precision == precision_) { out << std::setprecision(precision); out << std::fixed << x_; break; } } } bool is_precision(long precision) const { std::stringstream ss; ss << std::setprecision(precision); ss << std::fixed << x_; value_type _x; ss >> _x; return std::abs(x_ - _x) < 1.e-20; } friend std::ostream& operator<<(std::ostream& out, const FixedFormat& This) { This.print(out); return out; } private: value_type x_; long precision_; }; inline FixedFormat fixed_format(double x, long precision) { return FixedFormat(x, precision); } /* static std::pair reduce_to_10( size_t bytes ) { static const std::vector magnitudes{"B", "K", "M", "G", "T", "P", "E", "Z", "Y"}; double x = bytes; size_t n = 0; while ( x >= 10 && n < magnitudes.size() ) { x /= 1024.; n++; } return std::make_pair( x, magnitudes[n] ); } */ static std::pair reduce_to_1000(size_t bytes) { static const std::vector magnitudes{"B", "K", "M", "G", "T", "P", "E", "Z", "Y"}; double x = bytes; size_t n = 0; while (x >= 1000 && n < magnitudes.size()) { x /= 1024.; n++; } return std::make_pair(x, magnitudes[n]); } } // namespace void Bytes::print(std::ostream& out, int decimals, int width) const { if (bytes_ < 1000 && width >= 4) { out << std::setw(width - 1) << std::right << bytes_ << 'B'; } else { auto pair = reduce_to_1000(bytes_); out << std::setw(width - 1) << std::right << fixed_format(pair.first, decimals); out << pair.second; } } std::ostream& operator<<(std::ostream& out, const Bytes& bytes) { bytes.print(out); return out; } std::string Bytes::str(int decimals, int width) const { std::stringstream s; print(s, decimals, width); return s.str(); } } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/print/Bytes.h000066400000000000000000000015661513175025300214660ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include namespace atlas { namespace io { struct Bytes { public: Bytes(size_t bytes): bytes_(bytes) {} operator size_t() const { return bytes_; } std::string str(int decimals = 2, int width = 7) const; void print(std::ostream& out, int decimals = 2, int width = 7) const; friend std::ostream& operator<<(std::ostream&, const Bytes&); private: size_t bytes_; }; } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/print/JSONFormat.cc000066400000000000000000000043001513175025300224450ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "JSONFormat.h" #include "eckit/log/JSON.h" #include "atlas_io/Record.h" #include "atlas_io/RecordItemReader.h" #include "atlas_io/Session.h" namespace atlas { namespace io { JSONFormat::JSONFormat(const Record::URI& record, const eckit::Configuration& config): record_(Session::record(record.path, record.offset)) { for (const auto& key : record_.keys()) { items_.emplace(key, Metadata()); RecordItemReader{RecordItem::URI{record.path, record.offset, key}}.read(items_.at(key)); } config.get("details", print_details_); } void JSONFormat::print(std::ostream& out) const { eckit::JSON js(out, eckit::JSON::Formatting::indent(4)); Metadata metadata; for (const auto& key : record_.keys()) { const auto& item = items_.at(key); Metadata m = record_.metadata(key); if (record_.metadata(key).link()) { m.set(item); if (m.has("data")) { // removes data.section m.remove("data"); } } if (not print_details_) { if (m.has("data")) { m.remove("data"); } } if (print_details_) { if (item.data.size()) { m.set("data.compression.type", item.data.compression()); m.set("data.compression.size", item.data.compressed_size()); m.set("data.size", item.data.size()); m.set("data.byte_order", (item.data.endian() == Endian::little) ? "little endian" : "big endian"); m.set("data.checksum", item.data.checksum().str()); } m.set("version", item.record.version().str()); m.set("created", item.record.created().str()); } metadata.set(key, m); } js << metadata; } } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/print/JSONFormat.h000066400000000000000000000015731513175025300223200ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/config/Configuration.h" #include "atlas_io/Metadata.h" #include "atlas_io/Record.h" namespace atlas { namespace io { class JSONFormat { public: JSONFormat(const Record::URI& record, const eckit::Configuration&); void print(std::ostream&) const; private: const Record record_; std::map items_; bool print_details_{false}; }; } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/print/TableFormat.cc000066400000000000000000000230641513175025300227330ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "TableFormat.h" #include #include "atlas_io/Exceptions.h" #include "atlas_io/Metadata.h" #include "atlas_io/Record.h" #include "atlas_io/RecordItemReader.h" #include "atlas_io/Session.h" #include "atlas_io/detail/Assert.h" #include "atlas_io/print/Bytes.h" #include "atlas_io/types/array/ArrayReference.h" #include "atlas_io/types/scalar.h" #include "atlas_io/atlas_compat.h" namespace atlas { namespace io { std::ostream& operator<<(std::ostream& out, const MetadataPrettyPrintBase& p) { p.print(out); return out; } std::string MetadataPrettyPrintBase::str() const { std::stringstream s; print(s); return s.str(); } class DefaultMetadataPrettyPrint : public MetadataPrettyPrintBase { public: DefaultMetadataPrettyPrint(const Metadata&) {} void print(std::ostream&) const override {} }; class ArrayMetadataPrettyPrint : public MetadataPrettyPrintBase { template void print_value(std::ostream& out) const { std::vector value; metadata_.get("value", value); out << "{"; for (size_t i = 0; i < value.size(); ++i) { out << value[i]; if (i < value.size() - 1) { out << ","; } } out << "}"; } public: ArrayMetadataPrettyPrint(const Metadata& m): metadata_(m) {} void print(std::ostream& out) const override { std::string type = metadata_.getString("type"); ATLAS_IO_ASSERT(type == "array"); ArrayMetadata array(metadata_); out << std::setw(7) << std::left << array.datatype().str(); if (metadata_.has("value")) { out << ": "; std::string datatype = metadata_.getString("datatype"); if (datatype == DataType::str()) { print_value(out); } else if (datatype == DataType::str()) { print_value(out); } else if (datatype == DataType::str()) { print_value(out); } else if (datatype == DataType::str()) { print_value(out); } else if (datatype == DataType::str()) { print_value(out); } } else { out << "["; for (int i = 0; i < array.rank(); ++i) { out << array.shape(i); if (i < array.rank() - 1) { out << ","; } } out << "]"; } } private: Metadata metadata_; }; class StringMetadataPrettyPrint : public MetadataPrettyPrintBase { public: StringMetadataPrettyPrint(const Metadata& m): metadata_(m) {} void print(std::ostream& out) const override { std::string type = metadata_.getString("type"); ATLAS_IO_ASSERT(type == "string"); std::string value = metadata_.getString("value"); if (value.size() <= 32) { out << value; } else { out << value.substr(0, 32) << "..."; } } private: Metadata metadata_; }; class ScalarMetadataPrettyPrint : public MetadataPrettyPrintBase { public: ScalarMetadataPrettyPrint(const Metadata& m): metadata_(m) {} template T decode() const { T value; Data dummy; atlas::io::decode(metadata_, dummy, value); return value; } void print(std::ostream& out) const override { std::string type = metadata_.getString("type"); ATLAS_IO_ASSERT(type == "scalar"); std::string datatype = metadata_.getString("datatype"); out << std::setw(7) << std::left << datatype << ": "; if (datatype == DataType::str()) { out << decode(); } else if (datatype == DataType::str()) { out << decode(); } else if (datatype == DataType::str()) { out << decode(); } else if (datatype == DataType::str()) { out << decode(); } else if (datatype == DataType::str()) { out << decode(); } } private: Metadata metadata_; }; MetadataPrettyPrint::MetadataPrettyPrint(const atlas::io::Metadata& m) { std::string type = m.getString("type"); // Poor man factory builder if (type == "array") { impl_.reset(new ArrayMetadataPrettyPrint(m)); } else if (type == "scalar") { impl_.reset(new ScalarMetadataPrettyPrint(m)); } else if (type == "string") { impl_.reset(new StringMetadataPrettyPrint(m)); } else { impl_.reset(new DefaultMetadataPrettyPrint(m)); } } std::ostream& operator<<(std::ostream& out, const MetadataPrettyPrint& p) { out << *p.impl_; return out; } std::string MetadataPrettyPrint::str() const { return impl_->str(); } struct TablePrinter { std::vector > columns; std::vector widths; std::string indent{" "}; std::string sep{" "}; int col{0}; int row{1}; std::vector optional; std::vector underline; TablePrinter() = default; TablePrinter& column(const std::string& title, size_t width = 0) { columns.emplace_back(std::vector{title}); widths.emplace_back(std::max(title.size(), width)); optional.push_back(false); underline.push_back(true); return *this; } TablePrinter& column() { columns.emplace_back(std::vector{""}); widths.emplace_back(0); optional.push_back(true); underline.push_back(false); return *this; } TablePrinter& operator<<(const MetadataPrettyPrint p) { *this << p.str(); return *this; } TablePrinter& operator<<(const std::string& s) { columns[col].emplace_back(s); widths[col] = std::max(widths[col], s.size()); if (optional[col] && widths[col] > 0) { optional[col] = false; widths[col] = std::max(widths[col], columns[col][0].size()); } ++col; if (col == static_cast(columns.size())) { ++row; col = 0; } return *this; } void print(std::ostream& out) const { out << " "; for (size_t c = 0; c < columns.size(); ++c) { out << " " << sep << " " << std::setw(widths[c]) << std::left << columns[c][0]; } out << " " << sep << std::endl; out << " "; for (size_t c = 0; c < columns.size(); ++c) { const char u = underline[c] ? '-' : ' '; out << " " << sep << " " << std::string(widths[c], u); // underline } out << " " << sep << std::endl; for (int r = 1; r < row; ++r) { out << " "; for (size_t c = 0; c < columns.size(); ++c) { out << " " << sep << " " << std::setw(widths[c]) << std::left << columns[c][r]; } out << " " << sep << std::endl; } } }; TableFormat::TableFormat(const Record::URI& record, const eckit::Parametrisation& config): record_(Session::record(record.path, record.offset)) { for (const auto& key : record_.keys()) { items_.emplace(key, Metadata()); RecordItemReader{RecordItem::URI{record.path, record.offset, key}}.read(items_.at(key)); } config.get("details", print_details_); } void TableFormat::print(std::ostream& out) const { ATLAS_IO_ASSERT(not record_.empty()); TablePrinter table; table.column("name"); table.column("type"); table.column("description"); if (print_details_) { table.column("ver."); table.column("comp."); table.column("size"); table.column("checksum[:8]"); table.column("endian"); table.column("created"); } table.column(); for (auto& key : record_.keys()) { auto& item = items_.at(key); size_t cbytes = item.data.compressed_size(); size_t ubytes = item.data.size(); std::string endian = (item.data.endian() == Endian::little) ? "little" : "big"; std::string created = item.record.created().str().substr(0, 16); created[10] = ' '; table << key; table << item.getString("type"); table << MetadataPrettyPrint{item}; if (print_details_) { table << item.record.version(); if (item.data.section()) { table << item.data.compression(); table << Bytes{cbytes}.str() + (item.data.compressed() ? " < " + Bytes{ubytes}.str() : ""); table << Checksum{item.data.checksum()}.str(8); table << endian; } else { table << "" << "" << "" << ""; } table << created; } if (record_.metadata(key).link()) { table << "-> " + record_.metadata(key).link().str(); } else { table << ""; } } table.print(out); } } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/print/TableFormat.h000066400000000000000000000027301513175025300225720ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include #include #include #include "atlas_io/Metadata.h" #include "atlas_io/Record.h" #include "atlas_io/RecordItemReader.h" #include "atlas_io/Session.h" namespace atlas { namespace io { class MetadataPrettyPrintBase { public: virtual ~MetadataPrettyPrintBase() = default; virtual void print(std::ostream&) const = 0; friend std::ostream& operator<<(std::ostream&, const MetadataPrettyPrintBase&); std::string str() const; }; class MetadataPrettyPrint { public: MetadataPrettyPrint(const atlas::io::Metadata&); friend std::ostream& operator<<(std::ostream& out, const MetadataPrettyPrint& p); std::string str() const; private: std::unique_ptr impl_; }; class TableFormat { public: TableFormat(const Record::URI& record, const eckit::Parametrisation& config); void print(std::ostream&) const; private: const Record record_; std::map items_; bool print_details_{false}; }; } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/types/000077500000000000000000000000001513175025300202275ustar00rootroot00000000000000atlas-0.45.0/atlas_io/src/atlas_io/types/array.h000066400000000000000000000012041513175025300215130ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "atlas_io/types/array/ArrayReference.h" #include "atlas_io/types/array/adaptors/StdArrayAdaptor.h" #include "atlas_io/types/array/adaptors/StdVectorAdaptor.h" #include "atlas_io/types/array/adaptors/StdVectorOfStdArrayAdaptor.h" atlas-0.45.0/atlas_io/src/atlas_io/types/array/000077500000000000000000000000001513175025300213455ustar00rootroot00000000000000atlas-0.45.0/atlas_io/src/atlas_io/types/array/ArrayMetadata.cc000066400000000000000000000072511513175025300244000ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "ArrayMetadata.h" #include #include #include #include "atlas_io/Exceptions.h" #include "atlas_io/atlas_compat.h" #include "atlas_io/detail/Assert.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- size_t encode_metadata(const ArrayMetadata& value, atlas::io::Metadata& out) { out.set("type", value.type()); out.set("shape", value.shape_); out.set("datatype", value.datatype_.str()); return value.bytes(); } //--------------------------------------------------------------------------------------------------------------------- int ArrayMetadata::shape(int i) const { if (i >= rank()) { throw Exception( "ArrayMetadata::shape(i=" + std::to_string(i) + ") goes out of bounds. rank=" + std::to_string(rank()), Here()); } return shape_[size_t(i)]; } //--------------------------------------------------------------------------------------------------------------------- ArrayMetadata::ArrayMetadata(const Metadata& metadata): datatype_(DataType::KIND_REAL64) /* circumvent absense of default constructor */ { std::string encoded_type; ATLAS_IO_ASSERT_MSG(metadata.get("type", encoded_type), "metadata is missing 'type'"); ATLAS_IO_ASSERT_MSG(encoded_type == type(), "metadata has unexpected type '" + encoded_type + "'"); ATLAS_IO_ASSERT_MSG(metadata.get("shape", shape_), "metadata is missing 'shape'"); std::string datatype_str; ATLAS_IO_ASSERT_MSG(metadata.get("datatype", datatype_str), "metadata is missing 'datatype'"); datatype_ = DataType(datatype_str); } //--------------------------------------------------------------------------------------------------------------------- size_t ArrayMetadata::size() const { return size_t(std::accumulate(shape_.begin(), shape_.end(), 1, std::multiplies())); } //--------------------------------------------------------------------------------------------------------------------- ArrayMetadata::ArrayMetadata(): shape_(), datatype_(DataType::KIND_REAL64) /* circumvent absense of default constructor */ {} //--------------------------------------------------------------------------------------------------------------------- ArrayMetadata::ArrayMetadata(const DataType& datatype, const ArrayShape& shape): shape_(shape), datatype_(datatype) {} //--------------------------------------------------------------------------------------------------------------------- ArrayMetadata::ArrayMetadata(const ArrayMetadata& other): ArrayMetadata{other.datatype_, other.shape_} {} //--------------------------------------------------------------------------------------------------------------------- ArrayMetadata::ArrayMetadata(ArrayMetadata&& other): shape_(std::move(other.shape_)), datatype_{other.datatype_} {} //--------------------------------------------------------------------------------------------------------------------- ArrayMetadata& ArrayMetadata::operator=(ArrayMetadata&& rhs) { shape_ = std::move(rhs.shape_); datatype_ = rhs.datatype_; return *this; } //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/types/array/ArrayMetadata.h000066400000000000000000000073301513175025300242400ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include #include "atlas_io/Metadata.h" #include "atlas_io/detail/DataType.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- class ArrayShape : public std::vector { private: using Base = std::vector; public: ArrayShape() {} ArrayShape(Base&& base): Base(std::forward(base)) {} template ArrayShape(std::initializer_list list): Base(list.begin(), list.end()) {} template ArrayShape(idx_t data[], size_t size): Base(data, data + size) {} template ArrayShape(const std::array& list): Base(list.begin(), list.end()) {} template ArrayShape(const std::vector& list): Base(list.begin(), list.end()) {} template >> ArrayShape(Int1 i) { resize(1); operator[](0) = i; } template && std::is_integral_v>> ArrayShape(Int1 i, Int2 j) { resize(2); operator[](0) = i; operator[](1) = j; } template && std::is_integral_v && std::is_integral_v>> ArrayShape(Int1 i, Int2 j, Int3 k) { resize(3); operator[](0) = i; operator[](1) = j; operator[](2) = k; } template && std::is_integral_v && std::is_integral_v && std::is_integral_v>> ArrayShape(Int1 i, Int2 j, Int3 k, Int4 l) { resize(4); operator[](0) = i; operator[](1) = j; operator[](2) = k; operator[](3) = l; } }; //--------------------------------------------------------------------------------------------------------------------- class ArrayMetadata { public: using ArrayShape = io::ArrayShape; using DataType = io::DataType; static std::string type() { return "array"; } public: ArrayMetadata(); explicit ArrayMetadata(const Metadata&); explicit ArrayMetadata(const DataType&, const ArrayShape&); explicit ArrayMetadata(const ArrayMetadata&); ArrayMetadata(ArrayMetadata&&); ArrayMetadata& operator=(ArrayMetadata&&); int rank() const { return int(shape_.size()); } int shape(int i) const; const ArrayShape& shape() const { return shape_; } DataType datatype() const { return datatype_; } size_t size() const; size_t bytes() const { return size() * datatype_.size(); } friend size_t encode_metadata(const ArrayMetadata& value, atlas::io::Metadata& out); private: ArrayShape shape_; DataType datatype_; }; //--------------------------------------------------------------------------------------------------------------------- size_t encode_metadata(const ArrayMetadata& value, atlas::io::Metadata& out); //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/types/array/ArrayReference.cc000066400000000000000000000063171513175025300245600ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "ArrayReference.h" #include "atlas_io/atlas_compat.h" #include "atlas_io/detail/Assert.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- void encode_data(const ArrayReference& value, atlas::io::Data& out) { out = atlas::io::Data(value.data(), value.bytes()); } //--------------------------------------------------------------------------------------------------------------------- namespace { template void encode_metadata_value(const ArrayReference& value, atlas::io::Metadata& out) { ATLAS_IO_ASSERT(value.datatype() == make_datatype()); const T* array = reinterpret_cast(value.data()); std::vector vector(value.size()); std::copy(array, array + vector.size(), vector.begin()); out.set("value", vector); } } // namespace size_t encode_metadata(const ArrayReference& value, atlas::io::Metadata& out) { auto bytes = encode_metadata(static_cast(value), out); if (value.rank() == 1 && value.size() <= 4) { auto kind = value.datatype().kind(); using DataType = ArrayReference::DataType; if (kind == DataType::kind()) { encode_metadata_value(value, out); } else if (kind == DataType::kind()) { encode_metadata_value(value, out); } else if (kind == DataType::kind()) { encode_metadata_value(value, out); } else if (kind == DataType::kind()) { encode_metadata_value(value, out); } else if (kind == DataType::kind()) { encode_metadata_value(value, out); } } return bytes; } //--------------------------------------------------------------------------------------------------------------------- ArrayReference::ArrayReference(const void* data, ArrayMetadata::DataType datatype, const ArrayMetadata::ArrayShape& shape): ArrayMetadata(datatype, shape), data_(const_cast(data)) {} //--------------------------------------------------------------------------------------------------------------------- ArrayReference::ArrayReference(ArrayReference&& other): ArrayMetadata(std::move(other)), data_(other.data_) { other.data_ = nullptr; } //--------------------------------------------------------------------------------------------------------------------- ArrayReference& ArrayReference::operator=(ArrayReference&& rhs) { ArrayMetadata::operator=(std::move(rhs)); data_ = rhs.data_; rhs.data_ = nullptr; return *this; } //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/types/array/ArrayReference.h000066400000000000000000000032331513175025300244140ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "atlas_io/Data.h" #include "atlas_io/Metadata.h" #include "atlas_io/types/array/ArrayMetadata.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- class ArrayReference : public ArrayMetadata { public: ArrayReference() = default; ArrayReference(const void* data, DataType, const ArrayShape&); template ArrayReference(const T* data, const ArrayShape& shape): ArrayMetadata(DataType::create(), shape), data_(const_cast(data)) {} ArrayReference(ArrayReference&&); ArrayReference& operator=(ArrayReference&&); void* data() const { return data_; } friend void decode(const atlas::io::Metadata&, const atlas::io::Data&, ArrayReference&); private: void* data_{nullptr}; }; //--------------------------------------------------------------------------------------------------------------------- size_t encode_metadata(const ArrayReference& value, atlas::io::Metadata& out); void encode_data(const ArrayReference& value, atlas::io::Data& out); //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/types/array/adaptors/000077500000000000000000000000001513175025300231625ustar00rootroot00000000000000atlas-0.45.0/atlas_io/src/atlas_io/types/array/adaptors/StdArrayAdaptor.h000066400000000000000000000040571513175025300264050ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include "atlas_io/Data.h" #include "atlas_io/Exceptions.h" #include "atlas_io/Metadata.h" #include "atlas_io/types/array/ArrayMetadata.h" #include "atlas_io/types/array/ArrayReference.h" namespace std { //--------------------------------------------------------------------------------------------------------------------- template void interprete(const std::array& vector, atlas::io::ArrayReference& out) { using atlas::io::ArrayReference; out = ArrayReference{vector.data(), {N}}; } //--------------------------------------------------------------------------------------------------------------------- template void decode(const atlas::io::Metadata& m, const atlas::io::Data& encoded, std::array& out) { atlas::io::ArrayMetadata array(m); if (array.datatype().kind() != atlas::io::ArrayMetadata::DataType::kind()) { std::stringstream err; err << "Could not decode " << m.json() << " into std::vector<" << atlas::io::demangle() << ">. " << "Incompatible datatype!"; throw atlas::io::Exception(err.str(), Here()); } if (array.size() != N) { std::stringstream err; err << "Could not decode " << m.json() << " into std::array<" << atlas::io::demangle() << "," << N << ">. " << "Incompatible size!"; throw atlas::io::Exception(err.str(), Here()); } const T* data = static_cast(encoded.data()); std::copy(data, data + N, out.begin()); } //--------------------------------------------------------------------------------------------------------------------- } // end namespace std atlas-0.45.0/atlas_io/src/atlas_io/types/array/adaptors/StdVectorAdaptor.h000066400000000000000000000037611513175025300265720ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include "atlas_io/Data.h" #include "atlas_io/Exceptions.h" #include "atlas_io/Metadata.h" #include "atlas_io/detail/TypeTraits.h" #include "atlas_io/types/array/ArrayMetadata.h" #include "atlas_io/types/array/ArrayReference.h" namespace std { //--------------------------------------------------------------------------------------------------------------------- template = 0> void interprete(const std::vector& vector, atlas::io::ArrayReference& out) { using atlas::io::ArrayReference; out = ArrayReference{vector.data(), {int(vector.size())}}; } //--------------------------------------------------------------------------------------------------------------------- template = 0> void decode(const atlas::io::Metadata& m, const atlas::io::Data& encoded, std::vector& out) { atlas::io::ArrayMetadata array(m); if (array.datatype().kind() != atlas::io::ArrayMetadata::DataType::kind()) { std::stringstream err; err << "Could not decode " << m.json() << " into std::vector<" << atlas::io::demangle() << ">. " << "Incompatible datatypes: " << array.datatype().str() << " and " << atlas::io::ArrayMetadata::DataType::str() << "."; throw atlas::io::Exception(err.str(), Here()); } const T* data = static_cast(encoded.data()); out.assign(data, data + array.size()); } //--------------------------------------------------------------------------------------------------------------------- } // end namespace std atlas-0.45.0/atlas_io/src/atlas_io/types/array/adaptors/StdVectorOfStdArrayAdaptor.h000066400000000000000000000050361513175025300305260ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include "atlas_io/Data.h" #include "atlas_io/Exceptions.h" #include "atlas_io/Metadata.h" #include "atlas_io/types/array/ArrayMetadata.h" #include "atlas_io/types/array/ArrayReference.h" namespace std { //--------------------------------------------------------------------------------------------------------------------- template void interprete(const std::vector>& vector_of_array, atlas::io::ArrayReference& out) { using atlas::io::ArrayReference; out = ArrayReference{vector_of_array.front().data(), {vector_of_array.size(),N}}; } //--------------------------------------------------------------------------------------------------------------------- template void decode(const atlas::io::Metadata& m, const atlas::io::Data& encoded, std::vector>& out) { atlas::io::ArrayMetadata array(m); if (array.datatype().kind() != atlas::io::ArrayMetadata::DataType::kind()) { std::stringstream err; err << "Could not decode " << m.json() << " into std::vector<" << atlas::io::demangle() << ">. " << "Incompatible datatype!"; throw atlas::io::Exception(err.str(), Here()); } if (array.rank() != 2) { std::stringstream err; err << "Could not decode " << m.json() << " into std::vector() << "," << N << ">>. " << "Incompatible rank!"; throw atlas::io::Exception(err.str(), Here()); } if (array.shape(1) != N) { std::stringstream err; err << "Could not decode " << m.json() << " into std::vector() << "," << N << ">>. " << "Incompatible size!"; throw atlas::io::Exception(err.str(), Here()); } const std::array* data = static_cast*>(encoded.data()); // std::copy(data, data + array.shape(0), out.begin()); out.assign(data, data + array.shape(0)); } //--------------------------------------------------------------------------------------------------------------------- } // end namespace std atlas-0.45.0/atlas_io/src/atlas_io/types/scalar.cc000066400000000000000000000130601513175025300220030ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ // Cray C++ compiler should not try to optimize this code #ifdef _CRAYC #pragma _CRI noopt #endif // GNU C++ compiler (version 11) should not try to optimize this code #if defined(__GNUC__) && !defined(__NVCOMPILER) && !defined(__clang__) #pragma GCC optimize("O0") #endif #include "scalar.h" #include #include #include "eckit/utils/ByteSwap.h" #include "atlas_io/atlas_compat.h" #include "atlas_io/detail/Assert.h" #include "atlas_io/detail/DataType.h" #include "atlas_io/Data.h" #include "atlas_io/Metadata.h" #include "atlas_io/detail/Base64.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- template void decode_scalar(const atlas::io::Metadata& metadata, T& value) { ATLAS_IO_ASSERT(metadata.getString("type") == "scalar"); ATLAS_IO_ASSERT(metadata.getString("datatype") == DataType::str()); metadata.get("value", value); } template void decode_scalar_b64(const atlas::io::Metadata& metadata, T& value) { ATLAS_IO_ASSERT(metadata.getString("type") == "scalar"); ATLAS_IO_ASSERT(metadata.getString("datatype") == DataType::str()); std::string base64 = metadata.getString("base64"); T value_ns = Base64::decode(base64); if (Endian::native == Endian::little) { T tmp = value_ns; eckit::byteswap(tmp); value_ns = tmp; } value = value_ns; } //--------------------------------------------------------------------------------------------------------------------- template void encode_scalar_metadata(const T& value, atlas::io::Metadata& out) { out.set("type", "scalar"); out.set("datatype", DataType::str()); out.set("value", value); } inline void encode_scalar_metadata(const unsigned long& value, atlas::io::Metadata& out) { out.set("type", "scalar"); out.set("datatype", DataType::str()); out.set("value", size_t(value)); } inline void encode_scalar_metadata(const unsigned long long& value, atlas::io::Metadata& out) { out.set("type", "scalar"); out.set("datatype", DataType::str()); out.set("value", size_t(value)); } template size_t encode_scalar_metadata_b64(const T& value, atlas::io::Metadata& out) { encode_scalar_metadata(value, out); T value_ns = value; if (Endian::native == Endian::little) { eckit::byteswap(value_ns); } out.set("base64", Base64::encode(value_ns)); return 0; } //--------------------------------------------------------------------------------------------------------------------- size_t encode_metadata(const int& value, atlas::io::Metadata& out) { return encode_scalar_metadata_b64(value, out); } size_t encode_metadata(const long& value, atlas::io::Metadata& out) { return encode_scalar_metadata_b64(value, out); } size_t encode_metadata(const long long& value, atlas::io::Metadata& out) { return encode_scalar_metadata_b64(value, out); } size_t encode_metadata(const unsigned long& value, atlas::io::Metadata& out) { return encode_scalar_metadata_b64(value, out); } size_t encode_metadata(const unsigned long long& value, atlas::io::Metadata& out) { return encode_scalar_metadata_b64(value, out); } size_t encode_metadata(const float& value, atlas::io::Metadata& out) { return encode_scalar_metadata_b64(value, out); } size_t encode_metadata(const double& value, atlas::io::Metadata& out) { return encode_scalar_metadata_b64(value, out); } //--------------------------------------------------------------------------------------------------------------------- void encode_data(const int&, atlas::io::Data&) {} void encode_data(const long&, atlas::io::Data&) {} void encode_data(const long long&, atlas::io::Data&) {} void encode_data(const unsigned long&, atlas::io::Data&) {} void encode_data(const unsigned long long&, atlas::io::Data&) {} void encode_data(const float&, atlas::io::Data&) {} void encode_data(const double&, atlas::io::Data&) {} //--------------------------------------------------------------------------------------------------------------------- void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, int& value) { decode_scalar(metadata, value); } void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, long& value) { decode_scalar(metadata, value); } void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, long long& value) { decode_scalar(metadata, value); } void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, unsigned long& value) { decode_scalar(metadata, value); } void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, unsigned long long& value) { decode_scalar_b64(metadata, value); } void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, float& value) { decode_scalar_b64(metadata, value); } void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, double& value) { decode_scalar_b64(metadata, value); } //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/types/scalar.h000066400000000000000000000045111513175025300216460ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "atlas_io/Data.h" #include "atlas_io/Metadata.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- size_t encode_metadata(const int&, atlas::io::Metadata&); size_t encode_metadata(const long&, atlas::io::Metadata&); size_t encode_metadata(const long long&, atlas::io::Metadata&); size_t encode_metadata(const unsigned long&, atlas::io::Metadata&); size_t encode_metadata(const unsigned long long&, atlas::io::Metadata&); size_t encode_metadata(const float&, atlas::io::Metadata&); size_t encode_metadata(const double&, atlas::io::Metadata&); //--------------------------------------------------------------------------------------------------------------------- void encode_data(const int&, atlas::io::Data&); void encode_data(const long&, atlas::io::Data&); void encode_data(const long long&, atlas::io::Data&); void encode_data(const unsigned long&, atlas::io::Data&); void encode_data(const unsigned long long&, atlas::io::Data&); void encode_data(const float&, atlas::io::Data&); void encode_data(const double&, atlas::io::Data&); //--------------------------------------------------------------------------------------------------------------------- void decode(const atlas::io::Metadata&, const atlas::io::Data&, int&); void decode(const atlas::io::Metadata&, const atlas::io::Data&, long&); void decode(const atlas::io::Metadata&, const atlas::io::Data&, long long&); void decode(const atlas::io::Metadata&, const atlas::io::Data&, unsigned long&); void decode(const atlas::io::Metadata&, const atlas::io::Data&, unsigned long long&); void decode(const atlas::io::Metadata&, const atlas::io::Data&, float&); void decode(const atlas::io::Metadata&, const atlas::io::Data&, double&); //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/atlas_io/types/string.h000066400000000000000000000024231513175025300217070ustar00rootroot00000000000000/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include "atlas_io/Data.h" #include "atlas_io/Metadata.h" #include "atlas_io/atlas_compat.h" #include "atlas_io/detail/Assert.h" namespace atlas { namespace io { //--------------------------------------------------------------------------------------------------------------------- inline size_t encode_metadata(const std::string& value, atlas::io::Metadata& out) { out.set("type", "string"); out.set("value", value); return 0; } inline void encode_data(const std::string&, atlas::io::Data&) {} inline void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, std::string& value) { ATLAS_IO_ASSERT(metadata.getString("type") == "string"); metadata.get("value", value); } //--------------------------------------------------------------------------------------------------------------------- } // namespace io } // namespace atlas atlas-0.45.0/atlas_io/src/tools/000077500000000000000000000000001513175025300164305ustar00rootroot00000000000000atlas-0.45.0/atlas_io/src/tools/CMakeLists.txt000066400000000000000000000007631513175025300211760ustar00rootroot00000000000000# (C) Copyright 2013 ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. ecbuild_add_executable( TARGET atlas-io-list SOURCES atlas-io-list.cc LIBS atlas_io eckit_option ) atlas-0.45.0/atlas_io/src/tools/atlas-io-list.cc000066400000000000000000000162721513175025300214310ustar00rootroot00000000000000/* * (C) Copyright 2013 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include #include #include #include "eckit/option/CmdArgs.h" #include "eckit/option/Separator.h" #include "eckit/option/SimpleOption.h" #include "eckit/option/VectorOption.h" #include "eckit/runtime/Tool.h" #include "eckit/filesystem/PathName.h" #include "atlas_io/Exceptions.h" #include "atlas_io/RecordPrinter.h" #include "atlas_io/print/Bytes.h" //-------------------------------------------------------------------------------- using eckit::Log; class AtlasIOTool : public eckit::Tool { public: using Options = std::vector; using Args = eckit::option::CmdArgs; protected: virtual std::string indent() { return " "; } virtual std::string briefDescription() { return ""; } virtual std::string longDescription() { return ""; } virtual std::string usage() { return name() + " [OPTION]... [--help,-h]"; } void add_option(eckit::option::Option* option) { options_.push_back(option); } virtual void help(std::ostream& out = Log::info()) { auto indented = [&](const std::string& s) -> std::string { std::string str = indent() + s; size_t pos = 0; while ((pos = str.find('\n', pos)) != std::string::npos) { str.replace(pos, 1, '\n' + indent()); ++pos; } return str; }; out << "NAME\n" << indented(name()); std::string brief = briefDescription(); if (brief.size()) { out << " - " << brief << '\n'; } std::string usg = usage(); if (usg.size()) { out << '\n'; out << "SYNOPSIS\n" << indented(usg) << '\n'; } std::string desc = longDescription(); if (desc.size()) { out << '\n'; out << "DESCRIPTION\n" << indented(desc) << '\n'; } out << '\n'; out << "OPTIONS\n"; for (Options::const_iterator it = options_.begin(); it != options_.end(); ++it) { std::stringstream s; s << **it; out << indented(s.str()) << "\n\n"; } out << std::flush; } virtual int numberOfPositionalArguments() { return -1; } virtual int minimumPositionalArguments() { return 0; } bool handle_help() { for (int i = 1; i < argc(); ++i) { if (argv(i) == "--help" || argv(i) == "-h") { help(std::cout); return true; } } return false; } public: AtlasIOTool(int argc, char** argv): eckit::Tool(argc, argv) { add_option(new eckit::option::SimpleOption("help", "Print this help")); } int start() { try { if (handle_help()) { return success(); } if (argc() - 1 < minimumPositionalArguments()) { Log::error() << "Usage: " << usage() << std::endl; return failed(); } Options opts = options_; std::function dummy = [](const std::string&) {}; Args args(dummy, opts, numberOfPositionalArguments(), minimumPositionalArguments() > 0); int err_code = execute(args); return err_code; } catch (eckit::Exception& e) { Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl; Log::error() << "** Exception terminates " << name() << std::endl; } catch (std::exception& e) { Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl; Log::error() << "** Exception terminates " << name() << std::endl; } return failed(); } void run() final {} // unused virtual int execute(const Args&) = 0; static constexpr int success() { return 0; } static constexpr int failed() { return 1; } private: Options options_; }; //---------------------------------------------------------------------------------------------------------------------- struct AtlasIOList : public AtlasIOTool { std::string briefDescription() override { return "Inspection of atlas-io files"; } std::string usage() override { return name() + " [OPTION]... [--help,-h]"; } std::string longDescription() override { return "Inspection of atlas-io files\n" "\n" " : path to atlas-io file"; } AtlasIOList(int argc, char** argv): AtlasIOTool(argc, argv) { add_option(new eckit::option::SimpleOption("format", "Output format")); add_option(new eckit::option::SimpleOption("version", "Print version of records")); add_option(new eckit::option::SimpleOption("details", "Print detailed information")); } int execute(const Args& args) override { auto return_code = success(); using namespace atlas; // User sanity checks if (args.count() == 0) { Log::error() << "No file specified." << std::endl; help(Log::error()); return failed(); } // Configuration eckit::LocalConfiguration config; config.set("format", args.getString("format", "table")); config.set("details", args.getBool("details", false)); // Loop over files for (size_t f = 0; f < args.count(); ++f) { eckit::PathName file(args(f)); if (!file.exists()) { Log::error() << "File does not exist: " << file << std::endl; return failed(); } auto filesize = size_t(file.size()); io::Session session; std::uint64_t pos = 0; try { while (pos < filesize) { auto uri = io::Record::URI{file, pos}; auto record = io::RecordPrinter{uri, config}; std::stringstream out; out << "\n# " << uri.path << " [" << uri.offset << "] " << "{ size: " << atlas::io::Bytes{record.size()}.str(0) << ", version: " << record.version() << ", created: " << record.time() << " }"; out << '\n' << (config.getString("format") == "table" ? "" : "---") << '\n'; out << record << std::endl; std::cout << out.str(); pos += record.size(); } } catch (const io::Exception& e) { Log::error() << " ATLAS-IO-ERROR: " << e.what() << std::endl; return_code = failed(); } } return return_code; } }; //------------------------------------------------------------------------------------------------------ int main(int argc, char** argv) { return AtlasIOList{argc, argv}.start(); } atlas-0.45.0/atlas_io/tests/000077500000000000000000000000001513175025300156435ustar00rootroot00000000000000atlas-0.45.0/atlas_io/tests/CMakeLists.txt000066400000000000000000000022721513175025300204060ustar00rootroot00000000000000# (C) Copyright 2022 ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. ecbuild_add_test( TARGET atlas_io_test_encoding SOURCES test_io_encoding.cc LIBS atlas_io ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) ecbuild_add_test( TARGET atlas_io_test_stream SOURCES test_io_stream.cc LIBS atlas_io ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ) ecbuild_add_executable( TARGET atlas_io_test_record SOURCES test_io_record.cc LIBS atlas_io NOINSTALL ) foreach( algorithm none bzip2 aec lz4 snappy ) string( TOUPPER ${algorithm} feature ) if( eckit_HAVE_${feature} OR algorithm MATCHES "none" ) ecbuild_add_test( TARGET atlas_io_test_record_COMPRESSION_${algorithm} COMMAND atlas_io_test_record ARGS --suffix ".${algorithm}" ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ATLAS_IO_COMPRESSION=${algorithm} ) endif() endforeach() atlas-0.45.0/atlas_io/tests/TestEnvironment.h000066400000000000000000000261021513175025300211610ustar00rootroot00000000000000/* * (C) Copyright 2013 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include #include #include #include #include #include "eckit/config/LibEcKit.h" #include "eckit/config/Resource.h" #include "eckit/eckit.h" #include "eckit/log/PrefixTarget.h" #include "eckit/runtime/Main.h" #include "eckit/testing/Test.h" #include "eckit/types/Types.h" #include "atlas_io/atlas-io.h" #include "atlas_io/Trace.h" namespace atlas { namespace test { using eckit::types::is_approximately_equal; class Test; static Test* current_test_{nullptr}; static size_t ATLAS_MAX_FAILED_EXPECTS() { static size_t v = size_t(eckit::Resource("$ATLAS_MAX_FAILED_EXPECTS", 100)); return v; } class Test { struct Failure { std::string message; eckit::CodeLocation location; }; public: Test(const std::string& description, const eckit::CodeLocation& location): description_(description), location_(location) { current_test_ = this; } ~Test() { current_test_ = nullptr; } void expect_failed(const std::string& message, const eckit::CodeLocation& location) { failures_.emplace_back(Failure{message, location}); eckit::Log::error() << message << std::endl; if (failures_.size() == ATLAS_MAX_FAILED_EXPECTS()) { std::stringstream msg; msg << "Maximum number of allowed EXPECTS have failed (${ATLAS_MAX_FAILED_EXPECTS}=" << ATLAS_MAX_FAILED_EXPECTS() << ")."; throw eckit::testing::TestException(msg.str(), location_); } } bool failed() const { return failures_.size() > 0; } void throw_on_failed_expects() { if (failed()) { std::stringstream msg; msg << failures_.size() << " EXPECTS have failed"; throw eckit::testing::TestException(msg.str(), location_); } } const std::string& description() const { return description_; } private: std::vector failures_; std::string description_; eckit::CodeLocation location_; }; Test& current_test() { ATLAS_IO_ASSERT(current_test_); return *current_test_; } //---------------------------------------------------------------------------------------------------------------------- #ifdef MAYBE_UNUSED #define MAYBE_UNUSED [[maybe_unused]] #endif #ifdef EXPECT_EQ #undef EXPECT_EQ #endif #ifdef EXPECT_APPROX_EQ #undef EXPECT_APPROX_EQ #endif #ifdef EXPECT #undef EXPECT #endif #define REQUIRE(expr) \ do { \ if (!(expr)) { \ throw eckit::testing::TestException("EXPECT condition failed: " #expr, Here()); \ } \ } while (false) #define EXPECT(expr) \ do { \ if (!(expr)) { \ current_test().expect_failed("EXPECT condition failed: " #expr, Here()); \ } \ } while (false) template struct Printer { static void print(std::ostream& out, const Value& v) { out << v; } }; template <> struct Printer { static void print(std::ostream& out, const double& v) { out << std::fixed << std::setprecision(12) << v; } }; template <> struct Printer { static void print(std::ostream& out, const eckit::CodeLocation& location) { out << eckit::PathName{location.file()}.baseName() << " +" << location.line(); } }; template struct PrintValue { const Value& value; PrintValue(const Value& v): value(v) {} void print(std::ostream& out) const { Printer::print(out, value); } friend std::ostream& operator<<(std::ostream& out, const PrintValue& v) { v.print(out); return out; } }; template PrintValue print(const Value& v) { return PrintValue(v); } bool approx_eq(const float& v1, const float& v2) { return is_approximately_equal(v1, v2); } bool approx_eq(const float& v1, const float& v2, const float& t) { return is_approximately_equal(v1, v2, t); } bool approx_eq(const double& v1, const double& v2) { return is_approximately_equal(v1, v2); } bool approx_eq(const double& v1, const double& v2, const double& t) { return is_approximately_equal(v1, v2, t); } //bool approx_eq(const Point2& v1, const Point2& v2) { // return approx_eq(v1[0], v2[0]) && approx_eq(v1[1], v2[1]); //} //bool approx_eq(const Point2& v1, const Point2& v2, const double& t) { // return approx_eq(v1[0], v2[0], t) && approx_eq(v1[1], v2[1], t); //} template std::string expect_message(const std::string& condition, const T1& lhs, const T2& rhs, const eckit::CodeLocation& loc) { std::stringstream msg; msg << eckit::Colour::red << condition << " FAILED @ " << print(loc) << eckit::Colour::reset << "\n" << eckit::Colour::red << " --> lhs = " << print(lhs) << eckit::Colour::reset << "\n" << eckit::Colour::red << " --> rhs = " << print(rhs) << eckit::Colour::reset; return msg.str(); } #define EXPECT_EQ(lhs, rhs) \ do { \ if (!(lhs == rhs)) { \ current_test().expect_failed(expect_message("EXPECT_EQ( " #lhs ", " #rhs " )", lhs, rhs, Here()), Here()); \ } \ } while (false) #define __EXPECT_APPROX_EQ(lhs, rhs) \ do { \ if (!(approx_eq(lhs, rhs))) { \ current_test().expect_failed(expect_message("EXPECT_APPROX_EQ( " #lhs ", " #rhs " )", lhs, rhs, Here()), \ Here()); \ } \ } while (false) #define __EXPECT_APPROX_EQ_TOL(lhs, rhs, tol) \ do { \ if (!(approx_eq(lhs, rhs, tol))) { \ current_test().expect_failed( \ expect_message("EXPECT_APPROX_EQ( " #lhs ", " #rhs ", " #tol " )", lhs, rhs, Here()), Here()); \ } \ } while (false) #define EXPECT_APPROX_EQ(...) __ATLAS_IO_SPLICE(__EXPECT_APPROX_EQ__, __ATLAS_IO_NARG(__VA_ARGS__))(__VA_ARGS__) #define __EXPECT_APPROX_EQ__2 __EXPECT_APPROX_EQ #define __EXPECT_APPROX_EQ__3 __EXPECT_APPROX_EQ_TOL //---------------------------------------------------------------------------------------------------------------------- namespace { [[maybe_unused]] int digits(int number) { int d = 0; while (number) { number /= 10; d++; } return d; } [[maybe_unused]] std::string debug_prefix(const std::string& libname) { std::string s = libname; std::transform(s.begin(), s.end(), s.begin(), ::toupper); s += "_DEBUG"; return s; } [[maybe_unused]] void debug_addTarget(eckit::LogTarget* target) { for (std::string libname : eckit::system::Library::list()) { const eckit::system::Library& lib = eckit::system::Library::lookup(libname); if (lib.debug()) { lib.debugChannel().addTarget(new eckit::PrefixTarget(debug_prefix(libname), target)); } } if (eckit::Log::debug()) eckit::Log::debug().addTarget(target); } [[maybe_unused]] void debug_setTarget(eckit::LogTarget* target) { for (std::string libname : eckit::system::Library::list()) { const eckit::system::Library& lib = eckit::system::Library::lookup(libname); if (lib.debug()) { lib.debugChannel().setTarget(new eckit::PrefixTarget(debug_prefix(libname), target)); } } if (eckit::Log::debug()) eckit::Log::debug().setTarget(target); } [[maybe_unused]] void debug_reset() { for (std::string libname : eckit::system::Library::list()) { const eckit::system::Library& lib = eckit::system::Library::lookup(libname); if (lib.debug()) { lib.debugChannel().reset(); } } if (eckit::Log::debug()) eckit::Log::debug().reset(); } [[maybe_unused]] bool getEnv(const std::string& env, bool default_value) { const char* cenv = ::getenv(env.c_str()); if (cenv != nullptr) { return eckit::Translator()(cenv); } return default_value; } [[maybe_unused]] int getEnv(const std::string& env, int default_value) { const char* cenv = ::getenv(env.c_str()); if (cenv != nullptr) { return eckit::Translator()(cenv); } return default_value; } [[maybe_unused]] void setEnv(const std::string& env, bool value) { constexpr int DO_NOT_REPLACE_IF_EXISTS = 0; ::setenv(env.c_str(), eckit::Translator()(value).c_str(), DO_NOT_REPLACE_IF_EXISTS); } } // namespace struct TestEnvironment { TestEnvironment(int argc, char* argv[]) { eckit::Main::initialise(argc, argv); } ~TestEnvironment() {} }; //---------------------------------------------------------------------------------------------------------------------- template int run(int argc, char* argv[]) { Environment env(argc, argv); int errors = eckit::testing::run_tests(argc, argv, false); return errors; } int run(int argc, char* argv[]) { return run(argc, argv); } //---------------------------------------------------------------------------------------------------------------------- } // namespace test } // namespace atlas atlas-0.45.0/atlas_io/tests/test_io_encoding.cc000066400000000000000000000504171513175025300214750ustar00rootroot00000000000000/* * (C) Copyright 2013 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include #include #include "atlas_io/atlas-io.h" #include "TestEnvironment.h" namespace atlas { namespace test { using io::ArrayReference; // ------------------------------------------------------------------------------------------------------- struct UnencodableType { std::string _; }; // ------------------------------------------------------------------------------------------------------- // Example type that can be encoded / decoded with atlas::io // The "operations" performed on this type are stored for unit-test purposes class EncodableType { public: using Operations = std::vector; EncodableType(std::string s, std::shared_ptr operations = std::make_shared()): str(s), ops(operations) { ATLAS_IO_TRACE("EncodableType[" + str + "] construct"); ops->push_back("constructor"); } EncodableType(): ops(std::make_shared()) {} EncodableType(const EncodableType& other) { // This constructor should not be called. str = other.str; ATLAS_IO_TRACE("EncodableType[" + str + "] copy constructor"); ops = other.ops; ops->push_back("copy constructor"); } EncodableType(EncodableType&& other) { str = std::move(other.str); ATLAS_IO_TRACE("EncodableType[" + str + "] move constructor"); ops = other.ops; ops->push_back("move constructor"); } EncodableType& operator=(const EncodableType& rhs) { // This assignment should not be called. str = rhs.str; ATLAS_IO_TRACE("EncodableType[" + str + "] assignment"); ops = rhs.ops; ops->push_back("assignment"); return *this; } EncodableType& operator=(const EncodableType&& rhs) { // This assignment should not be called. str = std::move(rhs.str); ATLAS_IO_TRACE("EncodableType[" + str + "] move"); ops = rhs.ops; ops->push_back("move"); return *this; } friend void encode_data(const EncodableType& in, atlas::io::Data& out) { in.ops->push_back("encode_data"); out.assign(in.str.data(), in.str.size()); } friend size_t encode_metadata(const EncodableType& in, atlas::io::Metadata& metadata) { in.ops->push_back("encode_metadata"); metadata.set("type", "EncodableType"); metadata.set("bytes", in.str.size()); return in.str.size(); } friend void decode(const atlas::io::Metadata&, const atlas::io::Data& b, EncodableType& out) { out.ops->push_back("decode"); const char* data = static_cast(b.data()); out.str = std::string(data, data + b.size()); } const std::vector& operations() const { return *ops; } private: std::string str; mutable std::shared_ptr> ops; }; // ------------------------------------------------------------------------------------------------------- CASE("test exceptions") { EXPECT(not(io::is_interpretable())); EXPECT(not io::is_encodable()); EXPECT(not io::can_encode_metadata()); EXPECT(not io::can_encode_data()); UnencodableType in; atlas::io::Data data; atlas::io::Metadata metadata; EXPECT_THROWS_AS(io::ref(in, io::tag::disable_static_assert()), io::NotEncodable); EXPECT_THROWS_AS(io::copy(in, io::tag::disable_static_assert()), io::NotEncodable); EXPECT_THROWS_AS(io::encode(in, metadata, data, io::tag::disable_static_assert()), io::NotEncodable); } // ------------------------------------------------------------------------------------------------------- CASE("encoding test::EncodableType") { static_assert(not io::is_interpretable(), ""); static_assert(io::is_encodable(), ""); static_assert(io::is_decodable(), ""); const std::string encoded_string{"encoded string"}; EncodableType in(encoded_string); atlas::io::Data data; atlas::io::Metadata metadata; EXPECT_NO_THROW(encode(in, metadata, data)); EXPECT(metadata.type() == "EncodableType"); EXPECT(data.size() == encoded_string.size()); EXPECT(::memcmp(data, encoded_string.data(), encoded_string.size()) == 0); } // ------------------------------------------------------------------------------------------------------- CASE("encoding atlas::io::types::ArrayView") { static_assert(not io::is_interpretable(), ""); static_assert(io::can_encode_data(), ""); static_assert(io::can_encode_metadata(), ""); static_assert(io::is_encodable(), ""); } // ------------------------------------------------------------------------------------------------------- template void assert_StdVector() { static_assert(io::is_interpretable, ArrayReference>(), ""); static_assert(not io::can_encode_data>(), ""); static_assert(not io::can_encode_metadata>(), ""); static_assert(not io::is_encodable>(), ""); } template void encode_StdVector() { std::vector in{1, 2, 3, 4, 5}; ArrayReference interpreted; interprete(in, interpreted); atlas::io::Data data; atlas::io::Metadata metadata; encode(interpreted, metadata, data); EXPECT(data.size() == in.size() * sizeof(T)); EXPECT(::memcmp(in.data(), data.data(), data.size()) == 0); EXPECT(metadata.type() == "array"); EXPECT(metadata.getString("datatype") == atlas::io::DataType::str()); } CASE("encoding std::vector") { assert_StdVector(); assert_StdVector(); assert_StdVector(); assert_StdVector(); assert_StdVector(); encode_StdVector(); encode_StdVector(); encode_StdVector(); encode_StdVector(); { using T = std::byte; std::bitset<8> bits; std::vector in; in.resize(5); size_t n{0}; for (auto& byte : in) { bits.set(n++, true); byte = *reinterpret_cast(&bits); } ArrayReference interpreted; interprete(in, interpreted); atlas::io::Data data; atlas::io::Metadata metadata; encode(interpreted, metadata, data); EXPECT(data.size() == in.size() * sizeof(T)); EXPECT(::memcmp(in.data(), data.data(), data.size()) == 0); EXPECT(metadata.type() == "array"); EXPECT(metadata.getString("datatype") == atlas::io::DataType::str()); } } // ------------------------------------------------------------------------------------------------------- template void assert_StdArray() { static_assert(io::is_interpretable, ArrayReference>(), ""); static_assert(not io::can_encode_data>(), ""); static_assert(not io::can_encode_metadata>(), ""); static_assert(not io::is_encodable>(), ""); } template void encode_StdArray() { std::array in{1, 2, 3, 4, 5}; ArrayReference interpreted; interprete(in, interpreted); atlas::io::Data data; atlas::io::Metadata metadata; encode(interpreted, metadata, data); EXPECT(data.size() == in.size() * sizeof(T)); EXPECT(::memcmp(in.data(), data.data(), data.size()) == 0); EXPECT(metadata.type() == "array"); EXPECT(metadata.getString("datatype") == atlas::io::DataType::str()); } CASE("encoding std::array") { assert_StdArray(); assert_StdArray(); assert_StdArray(); assert_StdArray(); assert_StdArray(); encode_StdVector(); encode_StdVector(); encode_StdVector(); encode_StdVector(); { using T = std::byte; std::bitset<8> bits; std::vector in; in.resize(5); size_t n{0}; for (auto& byte : in) { bits.set(n++, true); byte = *reinterpret_cast(&bits); } ArrayReference interpreted; interprete(in, interpreted); atlas::io::Data data; atlas::io::Metadata metadata; encode(interpreted, metadata, data); EXPECT(data.size() == in.size() * sizeof(T)); EXPECT(::memcmp(in.data(), data.data(), data.size()) == 0); EXPECT(metadata.type() == "array"); EXPECT(metadata.getString("datatype") == atlas::io::DataType::str()); } } // ------------------------------------------------------------------------------------------------------- CASE("test Encoder") { SECTION("default constructor") { io::Encoder encoder; EXPECT(encoder == false); io::Metadata metadata; io::Data data; EXPECT_THROWS_AS(encode(encoder, metadata, data), eckit::AssertionFailed); } SECTION("Encoder via reference") { io::Encoder encoder; auto ops = std::make_shared(); EncodableType encodable("string", ops); EXPECT_EQ(ops->size(), 1); EXPECT_EQ(ops->back(), "constructor"); io::ref(encodable); EXPECT_EQ(ops->size(), 1); EXPECT_EQ(ops->back(), "constructor"); encoder = io::Encoder{io::ref(encodable)}; EXPECT_EQ(ops->size(), 2); EXPECT_EQ(ops->back(), "encode_metadata"); io::Metadata metadata; io::Data data; encode(encoder, metadata, data); EXPECT_EQ(ops->size(), 3); EXPECT_EQ(ops->back(), "encode_data"); } SECTION("Encoder via copy") { io::Encoder encoder; auto ops = std::make_shared(); EncodableType encodable("string", ops); EXPECT_EQ(ops->size(), 1); EXPECT_EQ(ops->back(), "constructor"); encoder = io::Encoder{io::copy(encodable)}; EXPECT_EQ(ops->size(), 3); EXPECT_EQ(ops->at(1), "encode_metadata"); EXPECT_EQ(ops->at(2), "encode_data"); io::Metadata metadata; io::Data data; encode(encoder, metadata, data); EXPECT_EQ(ops->size(), 3); EXPECT_EQ(ops->at(2), "encode_data"); } SECTION("Encoder via move") { io::Encoder encoder; auto ops = std::make_shared(); EncodableType encodable("string", ops); EXPECT_EQ(ops->size(), 1); EXPECT_EQ(ops->back(), "constructor"); encoder = io::Encoder{std::move(encodable)}; EXPECT_EQ(ops->size(), 3); EXPECT_EQ(ops->at(1), "move constructor"); EXPECT_EQ(ops->at(2), "encode_metadata"); io::Metadata metadata; io::Data data; encode(encoder, metadata, data); EXPECT_EQ(ops->size(), 4); EXPECT_EQ(ops->at(3), "encode_data"); } } // ------------------------------------------------------------------------------------------------------- CASE("Encoder for std::vector") { SECTION("ref") { using T = double; std::vector v{1, 2, 3, 4, 5, 6, 7, 8}; io::Encoder encoder(io::ref(v)); // We can only encode with reference to original vector (no copies were made) io::Metadata metadata; io::Data data; encode(encoder, metadata, data); EXPECT(data.size() == v.size() * sizeof(T)); EXPECT(::memcmp(data, v.data(), data.size()) == 0); } SECTION("copy") { using T = double; std::vector v{1, 2, 3, 4, 5, 6, 7, 8}; io::Encoder encoder; { std::vector scoped = v; encoder = io::Encoder(io::copy(scoped)); scoped.assign(scoped.size(), 0); // zero out before destruction } // We can now encode with scoped vector destroyed io::Metadata metadata; io::Data data; encode(encoder, metadata, data); EXPECT_EQ(data.size(), v.size() * sizeof(T)); EXPECT(::memcmp(data, v.data(), data.size()) == 0); } } // ------------------------------------------------------------------------------------------------------- CASE("Encoder for std::array") { SECTION("ref") { using T = double; std::array v{1, 2, 3, 4, 5, 6, 7, 8}; io::Encoder encoder(io::ref(v)); // We can only encode with reference to original vector (no copies were made) io::Metadata metadata; io::Data data; encode(encoder, metadata, data); EXPECT(data.size() == v.size() * sizeof(T)); EXPECT(::memcmp(data, v.data(), data.size()) == 0); } SECTION("copy") { using T = double; std::array v{1, 2, 3, 4, 5, 6, 7, 8}; io::Encoder encoder; { std::array scoped = v; encoder = io::Encoder(io::copy(scoped)); std::fill(std::begin(scoped), std::end(scoped), 0); // zero out before destruction } // We can now encode with scoped vector destroyed io::Metadata metadata; io::Data data; encode(encoder, metadata, data); EXPECT_EQ(data.size(), v.size() * sizeof(T)); EXPECT(::memcmp(data, v.data(), data.size()) == 0); } } // ------------------------------------------------------------------------------------------------------- CASE("Encoder of encoder") { using T = double; std::vector v{1, 2, 3, 4, 5, 6, 7, 8}; io::Encoder encoder(io::ref(v)); io::Encoder encoder_of_encoder(io::ref(encoder)); io::Metadata metadata; io::Data data; encode(encoder_of_encoder, metadata, data); EXPECT_EQ(data.size(), v.size() * sizeof(T)); EXPECT(::memcmp(data, v.data(), data.size()) == 0); } // ------------------------------------------------------------------------------------------------------- /// Helper class to be used in testing decoding of arrays. template struct EncodedArray { atlas::io::Data data; atlas::io::Metadata metadata; EncodedArray(): in{1, 2, 3, 4, 5, 6, 7, 8} { encode(in, metadata, data); } friend bool operator==(const std::vector& lhs, const EncodedArray& rhs) { if (lhs.size() != rhs.in.size()) { return false; } return ::memcmp(lhs.data(), rhs.in.data(), rhs.in.size() * sizeof(T)) == 0; } friend bool operator==(const std::array& lhs, const EncodedArray& rhs) { if (lhs.size() != rhs.in.size()) { return false; } return ::memcmp(lhs.data(), rhs.in.data(), rhs.in.size() * sizeof(T)) == 0; } private: std::vector in; }; template <> struct EncodedArray { using T = std::byte; atlas::io::Data data; atlas::io::Metadata metadata; EncodedArray() { std::bitset<8> bits; in.resize(5); size_t n{0}; for (auto& byte : in) { bits.set(n++, true); byte = *reinterpret_cast(&bits); } encode(in, metadata, data); } friend bool operator==(const std::vector& lhs, const EncodedArray& rhs) { if (lhs.size() != rhs.in.size()) { return false; } return ::memcmp(lhs.data(), rhs.in.data(), rhs.in.size() * sizeof(T)) == 0; } private: std::vector in; }; // ------------------------------------------------------------------------------------------------------- CASE("Decoding to std::vector") { using T = double; EncodedArray encoded; std::vector out; SECTION("decode std::vector directly") { EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, out)); EXPECT(out == encoded); } SECTION("decode using rvalue io::Decoder (type erasure)") { EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(out))); EXPECT(out == encoded); } SECTION("decode using lvalue io::Decoder (type erasure)") { io::Decoder decoder(out); EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(out))); EXPECT(out == encoded); } SECTION("decode using decoder of decoder") { io::Decoder decoder(out); EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(decoder))); EXPECT(out == encoded); } } // ------------------------------------------------------------------------------------------------------- CASE("Decoding to std::array") { using T = double; EncodedArray encoded; std::array out; SECTION("decode std::vector directly") { EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, out)); EXPECT(out == encoded); } SECTION("decode using rvalue io::Decoder (type erasure)") { EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(out))); EXPECT(out == encoded); } SECTION("decode using lvalue io::Decoder (type erasure)") { io::Decoder decoder(out); EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(out))); EXPECT(out == encoded); } SECTION("decode using decoder of decoder") { io::Decoder decoder(out); EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(decoder))); EXPECT(out == encoded); } } // ------------------------------------------------------------------------------------------------------- CASE("Encode/Decode byte array") { using T = std::byte; EncodedArray encoded; std::vector out; auto validate = [&]() { EXPECT(out == encoded); auto to_byte = []( const char* str) { return std::byte(std::bitset<8>(str).to_ulong()); }; EXPECT(out[0] == to_byte("00000001")); EXPECT(out[1] == to_byte("00000011")); EXPECT(out[2] == to_byte("00000111")); EXPECT(out[3] == to_byte("00001111")); EXPECT(out[4] == to_byte("00011111")); }; SECTION("decode directly") { EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, out)); validate(); } SECTION("decode using rvalue io::Decoder (type erasure)") { EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(out))); validate(); } SECTION("decode using lvalue io::Decoder (type erasure)") { io::Decoder decoder(out); EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(out))); validate(); } SECTION("decode using decoder of decoder") { io::Decoder decoder(out); EXPECT_NO_THROW(decode(encoded.metadata, encoded.data, io::Decoder(decoder))); validate(); } } // ------------------------------------------------------------------------------------------------------- CASE("Encode/Decode string") { std::string in{"short string"}; io::Metadata metadata; io::Data data; encode(in, metadata, data); EXPECT_EQ(data.size(), 0); std::string out; decode(metadata, data, out); EXPECT_EQ(out, in); } // ------------------------------------------------------------------------------------------------------- template void test_encode_decode_scalar() { T in{std::numeric_limits::max()}, out; io::Metadata metadata; io::Data data; encode(in, metadata, data); EXPECT_EQ(data.size(), 0); decode(metadata, data, out); EXPECT_EQ(out, in); } CASE("Encode/Decode scalar") { // bit identical encoding via Base64 string within the metadata! SECTION("int32") { test_encode_decode_scalar(); } SECTION("int64") { test_encode_decode_scalar(); } SECTION("real32") { test_encode_decode_scalar(); } SECTION("real64") { test_encode_decode_scalar(); } SECTION("uint64") { test_encode_decode_scalar(); } } // ------------------------------------------------------------------------------------------------------- } // namespace test } // namespace atlas int main(int argc, char** argv) { return atlas::test::run(argc, argv); } atlas-0.45.0/atlas_io/tests/test_io_record.cc000066400000000000000000000532021513175025300211600ustar00rootroot00000000000000/* * (C) Copyright 2013 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include #include #include #include "eckit/io/MemoryHandle.h" #include "TestEnvironment.h" namespace atlas { namespace test { template struct Matrix { std::vector data_; size_t rows_; size_t cols_; size_t size() const { return rows_ * cols_; } T* data() { return data_.data(); } const T* data() const { return data_.data(); } atlas::io::DataType datatype() const { return atlas::io::make_datatype(); } Matrix(size_t rows, size_t cols) { resize(rows, cols); } void resize(size_t rows, size_t cols) { rows_ = rows; cols_ = cols; data_.resize(size()); } void assign(std::initializer_list list) { data_.assign(list); } void assign(T value) { data_.assign(size(), value); } }; template void interprete(const Matrix& in, atlas::io::ArrayReference& out) { out = io::ArrayReference(in.data(), in.datatype(), atlas::io::ArrayShape{in.rows_, in.cols_}); } template void decode(const atlas::io::Metadata& metadata, const atlas::io::Data& data, Matrix& out) { atlas::io::ArrayMetadata array(metadata); out.resize(array.shape(0), array.shape(1)); ::memcpy(out.data(), data, data.size()); } struct Arrays { std::vector v1; std::vector v2; Matrix v3{0, 0}; bool operator==(const Arrays& other) const { return v1 == other.v1 && ::memcmp(v2.data(), other.v2.data(), v2.size() * sizeof(float)) == 0 && ::memcmp(v3.data(), other.v3.data(), v3.size() * v3.datatype().size()) == 0; } bool operator!=(const Arrays& other) const { return not operator==(other); } }; //----------------------------------------------------------------------------- static eckit::LocalConfiguration no_compression = [] { eckit::LocalConfiguration c; c.set("compression", "none"); return c; }(); //----------------------------------------------------------------------------- std::string suffix() { static std::string suffix = eckit::Resource("--suffix", ""); return suffix; } //----------------------------------------------------------------------------- namespace globals { struct TestRecord { Arrays data; TestRecord() = default; TestRecord(const std::function& initializer) { initializer(data); } }; static TestRecord record1{[](Arrays& data) { data.v1 = {0, 1, 2, 3, 4}; data.v2 = {3, 2, 1}; data.v3.resize(3, 2); data.v3.assign({11, 12, 21, 22, 31, 32}); }}; static TestRecord record2{[](Arrays& data) { data.v1 = {0, 10, 20, 30, 40, 50}; data.v2 = {30, 20, 10, 40}; data.v3.resize(2, 3); data.v3.assign({11, 12, 13, 21, 22, 23}); }}; static TestRecord record3{[](Arrays& data) { data.v1.assign(1024 / 8 - 1, 2.); data.v2.assign(1023 * 1024 / 4 + 512 / 4, 1.); data.v3.resize(1024, 1024); data.v3.assign(3); }}; std::vector records; } // namespace globals //----------------------------------------------------------------------------- template void write_length(Length length, const std::string& path) { std::ofstream file(path); file << length; file.close(); } //-----------------------------------------------------------------------------// // // // Writing records // // // //-----------------------------------------------------------------------------// CASE("Write records, each in separate file (offset=0)") { auto write_record = [&](const Arrays& data, const eckit::PathName& path) { io::RecordWriter record; record.set("v1", io::ref(data.v1), no_compression); record.set("v2", io::ref(data.v2), no_compression); record.set("v3", io::ref(data.v3)); auto length = record.write(path); write_length(length, path + ".length"); }; SECTION("record1.atlas" + suffix()) { write_record(globals::record1.data, "record1.atlas" + suffix()); } SECTION("record2.atlas" + suffix()) { write_record(globals::record2.data, "record2.atlas" + suffix()); } SECTION("record3.atlas" + suffix()) { write_record(globals::record3.data, "record3.atlas" + suffix()); } } //----------------------------------------------------------------------------- CASE("Write records to same file using record.write(path,io::Mode)") { // This will reopen files upon every append, bad for performance static std::vector lengths; static std::vector offsets{0}; auto write_record = [&](Arrays& data, const eckit::PathName& path, io::Mode mode) { io::RecordWriter record; record.set("v1", io::ref(data.v1), no_compression); record.set("v2", io::ref(data.v2), no_compression); record.set("v3", io::ref(data.v3)); globals::records.emplace_back(io::Record::URI{path, offsets.back()}); lengths.emplace_back(record.write(path, mode)); offsets.emplace_back(offsets.back() + lengths.back()); }; SECTION("record1 -> records.atlas" + suffix()) { write_record(globals::record1.data, "records.atlas" + suffix(), io::Mode::write); } SECTION("record2 -> records.atlas" + suffix()) { write_record(globals::record2.data, "records.atlas" + suffix(), io::Mode::append); } SECTION("record3 -> records.atlas" + suffix()) { write_record(globals::record3.data, "records.atlas" + suffix(), io::Mode::append); } } //----------------------------------------------------------------------------- CASE("Write records to same file using record.write(Stream) keeping Stream open") { // This should give exactly the same output file as previous, except no auto write_record = [&](const Arrays& data, io::Stream stream) { io::RecordWriter record; record.set("v1", io::ref(data.v1), no_compression); record.set("v2", io::ref(data.v2), no_compression); record.set("v3", io::ref(data.v3)); record.write(stream); }; static io::OutputFileStream stream("records.atlas" + suffix() + ".duplicate"); SECTION("record1 -> records.atlas" + suffix() + ".duplicate") { EXPECT_EQ(stream.position(), globals::records[0].offset); write_record(globals::record1.data, stream); } SECTION("record2 -> records.atlas" + suffix() + ".duplicate") { EXPECT_EQ(stream.position(), globals::records[1].offset); write_record(globals::record2.data, stream); } SECTION("record3 -> records.atlas" + suffix() + ".duplicate") { EXPECT_EQ(stream.position(), globals::records[2].offset); write_record(globals::record3.data, stream); } SECTION("close stream") { stream.close(); // required because stream is a static variable } } //----------------------------------------------------------------------------- CASE("Write master record referencing record1 and record2 and record3") { io::RecordWriter record; record.set("v1", io::link("file:record1.atlas" + suffix() + "?key=v1")); record.set("v2", io::link("file:record1.atlas" + suffix() + "?key=v2")); record.set("v3", io::link("file:record1.atlas" + suffix() + "?key=v3")); record.set("v4", io::link("file:record2.atlas" + suffix() + "?key=v1")); record.set("v5", io::link("file:record2.atlas" + suffix() + "?key=v2")); record.set("v6", io::link("file:record2.atlas" + suffix() + "?key=v3")); record.set("v7", io::link("file:record3.atlas" + suffix() + "?key=v1")); record.set("v8", io::link("file:record3.atlas" + suffix() + "?key=v2")); record.set("v9", io::link("file:record3.atlas" + suffix() + "?key=v3")); record.write("record.atlas" + suffix()); } //----------------------------------------------------------------------------- CASE("Write records in nested subdirectories") { auto reference_path = eckit::PathName{"atlas_test_io_refpath"}; { eckit::PathName{reference_path / "links" / "1"}.mkdir(); io::RecordWriter record; record.set("v1", io::ref(globals::record1.data.v1)); record.set("v2", io::ref(globals::record1.data.v2)); record.set("v3", io::ref(globals::record1.data.v3)); record.set("s1", std::string("short string")); record.set("s2", double(1. / 3.)); record.write(reference_path / "links" / "1" / "record.atlas" + suffix()); } { eckit::PathName{reference_path / "links" / "2"}.mkdir(); io::RecordWriter record; record.set("v1", io::ref(globals::record2.data.v1)); record.set("v2", io::ref(globals::record2.data.v2)); record.set("v3", io::ref(globals::record2.data.v3)); record.set("s1", size_t(10000000000)); record.write(reference_path / "links" / "2" / "record.atlas" + suffix()); } { io::RecordWriter record; record.set("l1", io::link("file:1/record.atlas" + suffix() + "?key=v1")); record.set("l2", io::link("file:1/record.atlas" + suffix() + "?key=v2")); record.set("l3", io::link("file:1/record.atlas" + suffix() + "?key=v3")); record.set("l4", io::link("file:2/record.atlas" + suffix() + "?key=v1")); record.set("l5", io::link("file:2/record.atlas" + suffix() + "?key=v2")); record.set("l6", io::link("file:2/record.atlas" + suffix() + "?key=v3")); record.set("l7", io::link("file:1/record.atlas" + suffix() + "?key=s1")); record.set("l8", io::link("file:1/record.atlas" + suffix() + "?key=s2")); record.set("l9", io::link("file:2/record.atlas" + suffix() + "?key=s1")); record.write(reference_path / "links" / "record.atlas" + suffix()); } { io::RecordWriter record; record.set("l1", io::link("file:links/record.atlas" + suffix() + "?key=l1")); record.set("l2", io::link("file:links/record.atlas" + suffix() + "?key=l2")); record.set("l3", io::link("file:links/record.atlas" + suffix() + "?key=l3")); record.set("l4", io::link("file:links/record.atlas" + suffix() + "?key=l4")); record.set("l5", io::link("file:links/record.atlas" + suffix() + "?key=l5")); record.set("l6", io::link("file:links/record.atlas" + suffix() + "?key=l6")); record.set("l7", io::link("file:links/record.atlas" + suffix() + "?key=l7")); record.set("l8", io::link("file:links/record.atlas" + suffix() + "?key=l8")); record.set("l9", io::link("file:links/record.atlas" + suffix() + "?key=l9")); record.write(reference_path / "record.atlas" + suffix()); } } //-----------------------------------------------------------------------------// // // // Reading tests // // // //-----------------------------------------------------------------------------// CASE("Test RecordItemReader") { SECTION("file:record1.atlas" + suffix() + "?key=v2") { io::RecordItemReader reader{"file:record1.atlas" + suffix() + "?key=v2"}; { // When we only want to read metadata io::Metadata metadata; reader.read(metadata); EXPECT(metadata.link() == false); EXPECT_EQ(metadata.type(), "array"); EXPECT(metadata.data.compressed() == false); EXPECT_EQ(metadata.data.compression(), "none"); EXPECT_EQ(metadata.data.size(), globals::record1.data.v2.size() * sizeof(float)); } { // When we want to read both metadata and data io::Metadata metadata; io::Data data; reader.read(metadata, data); EXPECT(metadata.data.compressed() == false); EXPECT_EQ(metadata.data.compression(), "none"); EXPECT_EQ(metadata.data.size(), globals::record1.data.v2.size() * sizeof(float)); EXPECT_EQ(data.size(), metadata.data.size()); EXPECT_EQ(data.size(), metadata.data.compressed_size()); EXPECT(::memcmp(data, globals::record1.data.v2.data(), data.size()) == 0); } } SECTION("file:record.atlas" + suffix() + "?key=v9") { io::RecordItemReader reader{"file:record.atlas" + suffix() + "?key=v9"}; { // When we only want to read metadata io::Metadata metadata; reader.read(metadata); EXPECT(metadata.link() == true); EXPECT_EQ(metadata.link().str(), "file:record3.atlas" + suffix() + "?key=v3"); EXPECT_EQ(metadata.type(), "array"); } { // When we want to read both metadata and data io::Metadata metadata; io::Data data; reader.read(metadata, data); EXPECT(metadata.data.compressed() == (io::defaults::compression_algorithm() != "none")); EXPECT_EQ(metadata.data.compression(), io::defaults::compression_algorithm()); EXPECT_EQ(metadata.data.size(), globals::record3.data.v3.size() * sizeof(int)); EXPECT_EQ(data.size(), metadata.data.compressed_size()); } } } //----------------------------------------------------------------------------- CASE("Read records from different files") { Arrays data1, data2, data3; auto read_record = [&](const eckit::PathName& path, Arrays& data) { io::RecordReader record(path); record.read("v1", data.v1).wait(); record.read("v2", data.v2).wait(); record.read("v3", data.v3).wait(); }; read_record("record1.atlas" + suffix(), data1); read_record("record2.atlas" + suffix(), data2); read_record("record3.atlas" + suffix(), data3); EXPECT(data1 == globals::record1.data); EXPECT(data2 == globals::record2.data); EXPECT(data3 == globals::record3.data); } //----------------------------------------------------------------------------- CASE("Read multiple records from same file") { Arrays data1, data2; io::RecordReader record1(globals::records[0]); io::RecordReader record2(globals::records[1]); record1.read("v1", data1.v1).wait(); record1.read("v2", data1.v2).wait(); record1.read("v3", data1.v3).wait(); record2.read("v1", data2.v1).wait(); record2.read("v2", data2.v2).wait(); record2.read("v3", data2.v3).wait(); EXPECT(data1 == globals::record1.data); EXPECT(data2 == globals::record2.data); } //----------------------------------------------------------------------------- CASE("Write master record referencing record1 and record2") { io::RecordWriter record; record.set("v1", io::link("file:record1.atlas" + suffix() + "?key=v1")); record.set("v2", io::link("file:record1.atlas" + suffix() + "?key=v2")); record.set("v3", io::link("file:record1.atlas" + suffix() + "?key=v3")); record.set("v4", io::link("file:record2.atlas" + suffix() + "?key=v1")); record.set("v5", io::link("file:record2.atlas" + suffix() + "?key=v2")); record.set("v6", io::link("file:record2.atlas" + suffix() + "?key=v3")); record.write("record.atlas" + suffix()); } //----------------------------------------------------------------------------- CASE("Read master record") { Arrays data1, data2; io::RecordReader record("record.atlas" + suffix()); eckit::Log::info() << "record.metadata(\"v1\"): " << record.metadata("v1") << std::endl; record.read("v1", data1.v1).wait(); record.read("v2", data1.v2).wait(); record.read("v3", data1.v3).wait(); record.read("v4", data2.v1).wait(); record.read("v5", data2.v2).wait(); record.read("v6", data2.v3).wait(); EXPECT(data1 == globals::record1.data); EXPECT(data2 == globals::record2.data); } //----------------------------------------------------------------------------- CASE("Async read") { Arrays data1, data2; io::RecordReader record("record.atlas" + suffix()); // Request reads record.read("v1", data1.v1); record.read("v2", data1.v2); record.read("v3", data1.v3); record.read("v4", data2.v1); record.read("v5", data2.v2); record.read("v6", data2.v3); // Wait for specific requests record.wait("v4"); record.wait("v5"); record.wait("v6"); // Should have completed EXPECT(data2 == globals::record2.data); // Should not be complete yet EXPECT(data1 != globals::record1.data); // Wait for all requests; record.wait(); // Should have completed EXPECT(data1 == globals::record1.data); } //----------------------------------------------------------------------------- CASE("Recursive Write/read records in nested subdirectories") { auto reference_path = eckit::PathName{"atlas_test_io_refpath"}; // Read Arrays data1, data2; io::RecordReader record(reference_path / "record.atlas" + suffix()); record.read("l1", data1.v1).wait(); record.read("l2", data1.v2).wait(); record.read("l3", data1.v3).wait(); record.read("l4", data2.v1).wait(); record.read("l5", data2.v2).wait(); record.read("l6", data2.v3).wait(); std::string l7; double l8; size_t l9; record.read("l7", l7).wait(); record.read("l8", l8).wait(); record.read("l9", l9).wait(); EXPECT(data1 == globals::record1.data); EXPECT(data2 == globals::record2.data); EXPECT_EQ(l7, "short string"); EXPECT_EQ(l8, 1. / 3.); EXPECT_EQ(l9, 10000000000ul); } //----------------------------------------------------------------------------- CASE("Write record to memory") { const auto& data_write = globals::record3.data; const auto& v1 = globals::record3.data.v1; const auto& v2 = globals::record3.data.v2; const auto& v3 = globals::record3.data.v3; eckit::Buffer memory; // write { ATLAS_IO_TRACE("write"); io::RecordWriter record; record.compression(false); record.checksum(false); record.set("v1", io::ref(v1)); record.set("v2", io::ref(v2)); record.set("v3", io::ref(v3)); memory.resize(record.estimateMaximumSize()); eckit::Log::info() << "memory.size() : " << memory.size() << std::endl; ; eckit::MemoryHandle datahandle_out{memory}; datahandle_out.openForWrite(0); auto record_length = record.write(datahandle_out); datahandle_out.close(); // Without compression, this should be exact EXPECT_EQ(memory.size(), record_length); } // read with individual RecordItemReader { ATLAS_IO_TRACE("read with RecordItemReader"); io::Session session; eckit::MemoryHandle datahandle_in{memory}; datahandle_in.openForRead(); { io::RecordItemReader reader(datahandle_in, "v1"); io::Metadata metadata; io::Data data; reader.read(metadata, data); EXPECT(::memcmp(data, data_write.v1.data(), data.size()) == 0); } { io::RecordItemReader reader(datahandle_in, "v2"); io::Metadata metadata; io::Data data; reader.read(metadata, data); EXPECT(::memcmp(data, data_write.v2.data(), data.size()) == 0); } { io::RecordItemReader reader(datahandle_in, "v3"); io::Metadata metadata; io::Data data; reader.read(metadata, data); EXPECT(::memcmp(data, data_write.v3.data(), data.size()) == 0); } datahandle_in.close(); } // read with RecordReader { ATLAS_IO_TRACE("read with RecordReader"); Arrays data_read; eckit::MemoryHandle datahandle_in{memory}; datahandle_in.openForRead(); io::RecordReader reader(datahandle_in); reader.read("v1", data_read.v1); reader.read("v2", data_read.v2); reader.read("v3", data_read.v3); reader.wait(); datahandle_in.close(); EXPECT(data_read == data_write); } } //-----------------------------------------------------------------------------// // // // Reading tests // // // //-----------------------------------------------------------------------------// CASE("RecordPrinter") { SECTION("table") { eckit::LocalConfiguration table_with_details; table_with_details.set("format", "table"); table_with_details.set("details", true); io::RecordPrinter record{eckit::PathName("record1.atlas" + suffix()), table_with_details}; std::stringstream out; EXPECT_NO_THROW(out << record); eckit::Log::debug() << out.str(); } SECTION("yaml") { eckit::LocalConfiguration yaml_with_details; yaml_with_details.set("format", "yaml"); yaml_with_details.set("details", true); io::RecordPrinter record{eckit::PathName("record1.atlas" + suffix()), yaml_with_details}; std::stringstream out; EXPECT_NO_THROW(out << record); eckit::Log::debug() << out.str(); } } //----------------------------------------------------------------------------- } // namespace test } // namespace atlas int main(int argc, char** argv) { return atlas::test::run(argc, argv); } atlas-0.45.0/atlas_io/tests/test_io_stream.cc000066400000000000000000000150561513175025300212020ustar00rootroot00000000000000/* * (C) Copyright 2013 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "atlas_io/atlas-io.h" #include "eckit/io/FileHandle.h" #include "eckit/io/PooledHandle.h" #include "TestEnvironment.h" namespace atlas { namespace test { CASE("Stream interoperability with eckit::DataHandle") { SECTION("own pointer") { io::Stream s; { eckit::DataHandle* datahandle = new eckit::FileHandle("test_io_session.data"); datahandle->openForWrite(0); s = io::Stream{datahandle}; } s.datahandle().close(); } SECTION("shared pointer") { io::Stream s; { std::shared_ptr datahandle = std::make_shared("test_io_session.data"); datahandle->openForWrite(0); s = io::Stream{datahandle}; } s.datahandle().close(); } SECTION("reference") { io::Stream s; eckit::FileHandle datahandle("test_io_session.data"); datahandle.openForWrite(0); s = io::Stream{datahandle}; s.datahandle().close(); } } CASE("Test seek-for-write works when opening OutputFileStream for append") { std::string s1("write \n"); std::string s2("append \n"); std::string s3("overwrite\n"); { ATLAS_IO_TRACE("write"); io::Stream f = io::OutputFileStream("append-test"); f.write(s1.c_str(), s1.size()); } { ATLAS_IO_TRACE("append"); io::Stream f = io::OutputFileStream("append-test", io::Mode::append); auto offset = f.position(); f.write(s2.c_str(), s2.size()); // Rewind to beginning of append f.seek(offset); f.write(s3.c_str(), s3.size()); } { ATLAS_IO_TRACE("read"); io::Stream f = io::InputFileStream("append-test"); std::string expected = s1 + s3; std::string read(expected.size(), ' '); f.read(const_cast(read.data()), read.size()); EXPECT_EQ(read, expected); } } CASE("Opening same file in same scope") { // Opening same file within same scope will avoid opening it multiple times, good for perfmance // write a file { io::OutputFileStream out("test_io_session.data"); out.write("line1\n", 6); out.write("line2\n", 6); out.write("line3\n", 6); } std::string l1(5, ' '), l2(5, ' '), l3(5, ' '); io::Stream f1 = io::InputFileStream{"test_io_session.data"}; f1.seek(0 * 6); f1.read(const_cast(l1.data()), 5); io::Stream f2 = io::InputFileStream{"test_io_session.data"}; f2.seek(1 * 6); f2.read(const_cast(l2.data()), 5); io::Stream f3 = io::InputFileStream{"test_io_session.data"}; f3.seek(2 * 6); f3.read(const_cast(l3.data()), 5); EXPECT_EQ(l1, "line1"); EXPECT_EQ(l2, "line2"); EXPECT_EQ(l3, "line3"); auto& pooled_handle = dynamic_cast(f1.datahandle()); EXPECT_EQ(pooled_handle.nbOpens(), 1); EXPECT_EQ(pooled_handle.nbSeeks(), 3); EXPECT_EQ(pooled_handle.nbReads(), 3); } CASE("Opening same file in parallel scopes") { // Files are opened and closed within each scope, bad for performance // write a file { io::OutputFileStream out("test_io_session.data"); out.write("line1\n", 6); out.write("line2\n", 6); out.write("line3\n", 6); } std::string l1(5, ' '), l2(5, ' '), l3(5, ' '); { io::Stream f1 = io::InputFileStream{"test_io_session.data"}; f1.seek(0 * 6); f1.read(const_cast(l1.data()), 5); auto& pooled_handle = dynamic_cast(f1.datahandle()); EXPECT_EQ(pooled_handle.nbOpens(), 1); EXPECT_EQ(pooled_handle.nbSeeks(), 1); EXPECT_EQ(pooled_handle.nbReads(), 1); } { io::Stream f2 = io::InputFileStream{"test_io_session.data"}; f2.seek(1 * 6); f2.read(const_cast(l2.data()), 5); auto& pooled_handle = dynamic_cast(f2.datahandle()); EXPECT_EQ(pooled_handle.nbOpens(), 1); EXPECT_EQ(pooled_handle.nbSeeks(), 1); EXPECT_EQ(pooled_handle.nbReads(), 1); } { io::Stream f3 = io::InputFileStream{"test_io_session.data"}; f3.seek(2 * 6); f3.read(const_cast(l3.data()), 5); auto& pooled_handle = dynamic_cast(f3.datahandle()); EXPECT_EQ(pooled_handle.nbOpens(), 1); EXPECT_EQ(pooled_handle.nbSeeks(), 1); EXPECT_EQ(pooled_handle.nbReads(), 1); } } CASE("Opening same file in parallel scopes with Session") { // Declaring this in an outer scope will keep storage of InputFileStream // within nested scopes, so that files will not be opened/closed repeatedly io::Session session; // write a file { io::OutputFileStream out("test_io_session.data"); out.write("line1\n", 6); out.write("line2\n", 6); out.write("line3\n", 6); } std::string l1(5, ' '), l2(5, ' '), l3(5, ' '); { io::Stream f1 = io::InputFileStream{"test_io_session.data"}; f1.seek(0 * 6); f1.read(const_cast(l1.data()), 5); auto& pooled_handle = dynamic_cast(f1.datahandle()); EXPECT_EQ(pooled_handle.nbOpens(), 1); EXPECT_EQ(pooled_handle.nbSeeks(), 1); EXPECT_EQ(pooled_handle.nbReads(), 1); } { io::Stream f2 = io::InputFileStream{"test_io_session.data"}; f2.seek(1 * 6); f2.read(const_cast(l2.data()), 5); auto& pooled_handle = dynamic_cast(f2.datahandle()); EXPECT_EQ(pooled_handle.nbOpens(), 1); EXPECT_EQ(pooled_handle.nbSeeks(), 2); EXPECT_EQ(pooled_handle.nbReads(), 2); } { io::Stream f3 = io::InputFileStream{"test_io_session.data"}; f3.seek(2 * 6); f3.read(const_cast(l3.data()), 5); auto& pooled_handle = dynamic_cast(f3.datahandle()); EXPECT_EQ(pooled_handle.nbOpens(), 1); EXPECT_EQ(pooled_handle.nbSeeks(), 3); EXPECT_EQ(pooled_handle.nbReads(), 3); } } } // namespace test } // namespace atlas int main(int argc, char** argv) { return atlas::test::run(argc, argv); } atlas-0.45.0/bamboo/000077500000000000000000000000001513175025300141455ustar00rootroot00000000000000atlas-0.45.0/bamboo/CLANG-env.sh000066400000000000000000000010441513175025300161120ustar00rootroot00000000000000#!/bin/bash if [[ $(uname) == "Darwin" ]]; then # Up to date CMake version required export PATH=${HOME}/Applications/CMake.app/Contents/bin:${PATH} # No module environment on the Mac return fi # initialise module environment if it is not if [[ ! $(command -v module > /dev/null 2>&1) ]]; then . /usr/local/apps/module/init/bash fi module unload grib_api module unload eccodes module unload emos module unload fftw module unload libemos module unload metview module unload netcdf4 module load cmake/3.16.5 module switch gnu clang atlas-0.45.0/bamboo/CLANG-flags.cmake000066400000000000000000000005231513175025300170650ustar00rootroot00000000000000set( ENABLE_TRANS ON CACHE BOOL "Enable TRANS" ) set( ENABLE_BOUNDSCHECKING ON CACHE BOOL "Enable bounds checking") set( ENABLE_TESSELATION OFF CACHE BOOL "Disable CGAL" ) # cgal is old in leap42 set( ENABLE_OMP_CXX OFF CACHE BOOL "Disable OpenMP for C++" ) # because of problems with clang OpenMP atlas-0.45.0/bamboo/CMakeLists.txt000066400000000000000000000002541513175025300167060ustar00rootroot00000000000000file( GLOB_RECURSE bamboo_files RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*" ) ecbuild_add_resources( TARGET ${PROJECT_NAME}_bamboo SOURCES_DONT_PACK ${bamboo_files} ) atlas-0.45.0/bamboo/GCC-env.sh000066400000000000000000000006511513175025300156650ustar00rootroot00000000000000#!/bin/bash [[ $(uname) == "Darwin" ]] && return # no module environment on the Mac # initialise module environment if it is not if [[ ! $(command -v module > /dev/null 2>&1) ]]; then . /usr/local/apps/module/init/bash fi module unload grib_api module unload eccodes module unload emos module unload fftw module unload libemos module unload metview module unload netcdf4 module load cmake/3.16.5 module load proj/8.2.1 atlas-0.45.0/bamboo/GCC-flags.cmake000066400000000000000000000007341513175025300166410ustar00rootroot00000000000000set( ENABLE_MPI OFF CACHE BOOL "Disable MPI under Intel compilation" ) set( ENABLE_TRANS ON CACHE BOOL "Enable TRANS" ) set( ENABLE_BOUNDSCHECKING ON CACHE BOOL "Enable bounds checking") set( ENABLE_PROJ ON CACHE BOOL "Enable proj" ) set( ENABLE_TESSELATION OFF CACHE BOOL "Disable CGAL" ) # cgal is old in leap42 #set( ECBUILD_2_COMPAT OFF CACHE BOOL "Disable ecbuild 2 compat mode for bamboo testing" ) atlas-0.45.0/bamboo/INTEL-env.sh000066400000000000000000000006621513175025300161460ustar00rootroot00000000000000#!/bin/bash [[ $(uname) == "Darwin" ]] && return # no module environment on the Mac # initialise module environment if it is not if [[ ! $(command -v module > /dev/null 2>&1) ]]; then . /usr/local/apps/module/init/bash fi module unload grib_api module unload eccodes module unload emos module unload fftw module unload libemos module unload metview module unload netcdf4 module load cmake/3.16.5 module switch gnu intel/17.0.3 atlas-0.45.0/bamboo/INTEL-flags.cmake000066400000000000000000000004701513175025300171150ustar00rootroot00000000000000set( ENABLE_MPI OFF CACHE BOOL "Disable MPI under Intel compilation" ) set( ENABLE_TRANS ON CACHE BOOL "Enable TRANS" ) set( ENABLE_BOUNDSCHECKING ON CACHE BOOL "Enable bounds checking") set( ENABLE_TESSELATION OFF CACHE BOOL "Disable CGAL" ) # cgal is old in leap42 atlas-0.45.0/bamboo/MACOSX-env.sh000066400000000000000000000000751513175025300162630ustar00rootroot00000000000000export PATH=$HOME/Applications/CMake.app/Contents/bin:$PATH atlas-0.45.0/bamboo/env.sh000066400000000000000000000000631513175025300152700ustar00rootroot00000000000000#!/usr/bin/env bash # export ctest_parallel="no" atlas-0.45.0/bamboo/flags.cmake000066400000000000000000000014701513175025300162450ustar00rootroot00000000000000 # the following line does not work (CMAKE_SOURCE_DIR seems to point to the build dir) #include(${CMAKE_SOURCE_DIR}/cmake/atlas_compiler_flags.cmake) #include(../../git/atlas/cmake/atlas_compiler_flags.cmake) # on openSUSE 11.3 machines, we need to point to a newer compiler #SET(CMAKE_Fortran_COMPILER /usr/local/apps/gcc/4.8.1/LP64/bin/gfortran CACHE STRING "Fortran compiler") #SET(CMAKE_C_COMPILER /usr/local/apps/gcc/4.8.1/LP64/bin/gcc CACHE STRING "C compiler") #SET(CMAKE_CXX_COMPILER /usr/local/apps/gcc/4.8.1/LP64/bin/g++ CACHE STRING "C++ compiler") #set(CMAKE_Fortran_LINK_FLAGS "-L/usr/local/apps/gcc/4.8.1/LP64/lib/gcc/x86_64-suse-linux/4.8.1/") #link_directories("/usr/local/apps/gcc/4.8.1/LP64/lib/gcc/x86_64-suse-linux/4.8.1/") SET(ENABLE_SANDBOX OFF CACHE BOOL "Disable sandbox") atlas-0.45.0/bamboo/leap42-env.sh000077500000000000000000000001041513175025300163540ustar00rootroot00000000000000#!/bin/bash # CC=mpicc # CXX=mpicxx # FC=mpif90 ulimit -s unlimitedatlas-0.45.0/cmake/000077500000000000000000000000001513175025300137665ustar00rootroot00000000000000atlas-0.45.0/cmake/FindPROJ.cmake000066400000000000000000000057331513175025300163530ustar00rootroot00000000000000# (C) Copyright 2011- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. # - Try to find the proj library # Once done this will define # # PROJ_FOUND - system has proj # PROJ_INCLUDE_DIRS - the proj include directory # PROJ_LIBRARIES - Link these to use proj # PROJ_VERSION - semantic version of proj # ### Set search paths from environment if ( NOT PROJ_PATH AND PROJ_ROOT ) set( PROJ_PATH ${PROJ_ROOT} ) endif() if ( NOT PROJ_PATH AND NOT "$ENV{PROJ_ROOT}" STREQUAL "" ) set( PROJ_PATH "$ENV{PROJ_ROOT}" ) endif() if ( NOT PROJ_PATH AND NOT "$ENV{PROJ_PATH}" STREQUAL "" ) set( PROJ_PATH "$ENV{PROJ_PATH}" ) endif() if ( NOT PROJ_PATH AND NOT "$ENV{PROJ_DIR}" STREQUAL "" ) set( PROJ_PATH "$ENV{PROJ_DIR}" ) endif() ### If search paths given, use it, otherwise, use pkg-config if( PROJ_PATH ) find_path(PROJ_INCLUDE_DIR NAMES proj.h PATHS ${PROJ_PATH} ${PROJ_PATH}/include PATH_SUFFIXES PROJ NO_DEFAULT_PATH ) find_library(PROJ_LIBRARY NAMES proj PATHS ${PROJ_PATH} ${PROJ_PATH}/lib PATH_SUFFIXES PROJ NO_DEFAULT_PATH ) else() find_package(PkgConfig) if(PKG_CONFIG_FOUND) if(PROJ_FIND_VERSION) pkg_check_modules(PKPROJ ${_pkgconfig_REQUIRED} QUIET PROJ>=${PROJ_FIND_VERSION}) else() pkg_check_modules(PKPROJ ${_pkgconfig_REQUIRED} QUIET PROJ) endif() if( PKPROJ_FOUND ) find_path(PROJ_INCLUDE_DIR proj.h HINTS ${PKPROJ_INCLUDEDIR} ${PKPROJ_INCLUDE_DIRS} PATH_SUFFIXES PROJ NO_DEFAULT_PATH ) find_library(PROJ_LIBRARY proj HINTS ${PKPROJ_LIBDIR} ${PKPROJ_LIBRARY_DIRS} PATH_SUFFIXES PROJ NO_DEFAULT_PATH ) endif() endif() endif() find_path(PROJ_INCLUDE_DIR NAMES proj.h PATHS PATH_SUFFIXES PROJ ) find_library( PROJ_LIBRARY NAMES proj PATHS PATH_SUFFIXES PROJ ) ### Detect version set( PROJ_VERSION 0 ) file(READ ${PROJ_INCLUDE_DIR}/proj.h proj_version) string(REGEX REPLACE "^.*PROJ_VERSION_MAJOR +([0-9]+).*$" "\\1" PROJ_VERSION_MAJOR "${proj_version}") string(REGEX REPLACE "^.*PROJ_VERSION_MINOR +([0-9]+).*$" "\\1" PROJ_VERSION_MINOR "${proj_version}") string(REGEX REPLACE "^.*PROJ_VERSION_PATCH +([0-9]+).*$" "\\1" PROJ_VERSION_PATCH "${proj_version}") string(CONCAT PROJ_VERSION ${PROJ_VERSION_MAJOR} "." ${PROJ_VERSION_MINOR} "." ${PROJ_VERSION_PATCH}) ### Handle the QUIETLY and REQUIRED arguments and set PROJ_FOUND include(FindPackageHandleStandardArgs) find_package_handle_standard_args(PROJ REQUIRED_VARS PROJ_LIBRARY PROJ_INCLUDE_DIR VERSION_VAR PROJ_VERSION) set( PROJ_LIBRARIES ${PROJ_LIBRARY} ) set( PROJ_INCLUDE_DIRS ${PROJ_INCLUDE_DIR} ) mark_as_advanced( PROJ_INCLUDE_DIR PROJ_LIBRARY ) atlas-0.45.0/cmake/Findpocketfft.cmake000066400000000000000000000032371513175025300175630ustar00rootroot00000000000000# (C) Copyright 2025- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. # - Try to find the pocketfft library # Once done this will define # # pocketfft_FOUND - pocketfft found # pocketfft_INCLUDE_DIRS - the pocketfft include directories # pocketfft_VERSION - semantic version of pocketfft # ### Set search paths from environment if ( NOT pocketfft_PATH AND pocketfft_ROOT ) set( pocketfft_PATH ${pocketfft_ROOT} ) endif() if ( NOT pocketfft_PATH AND NOT "$ENV{pocketfft_ROOT}" STREQUAL "" ) set( pocketfft_PATH "$ENV{pocketfft_ROOT}" ) endif() if ( NOT pocketfft_PATH AND NOT "$ENV{pocketfft_PATH}" STREQUAL "" ) set( pocketfft_PATH "$ENV{pocketfft_PATH}" ) endif() ### If search paths given, use it if( pocketfft_PATH ) find_path(pocketfft_INCLUDE_DIR NAMES pocketfft_hdronly.h PATHS ${pocketfft_PATH} ${pocketfft_PATH}/include pocketfft NO_DEFAULT_PATH ) else() if( NOT pocketfft_INCLUDE_DIR ) find_path(pocketfft_INCLUDE_DIR NAMES pocketfft_hdronly.h ) endif() endif() ### As of 2025 pocketfft does not yet have any version. ### Handle the QUIETLY and REQUIRED arguments and set pocketfft_FOUND include(FindPackageHandleStandardArgs) find_package_handle_standard_args(pocketfft REQUIRED_VARS pocketfft_INCLUDE_DIR) set( POCKETFFT_INCLUDE_DIRS ${pocketfft_INCLUDE_DIR} ) mark_as_advanced( pocketfft_INCLUDE_DIR ) atlas-0.45.0/cmake/atlas-import.cmake.in000066400000000000000000000115151513175025300200140ustar00rootroot00000000000000 include( CMakeFindDependencyMacro ) set( atlas_HAVE_MPI @atlas_HAVE_MPI@ ) set( atlas_HAVE_OMP @atlas_HAVE_OMP@ ) set( atlas_HAVE_OMP_CXX @atlas_HAVE_OMP_CXX@ ) set( atlas_HAVE_OMP_Fortran @atlas_HAVE_OMP_Fortran@ ) set( atlas_HAVE_ECTRANS @atlas_HAVE_ECTRANS@ ) set( atlas_HAVE_FORTRAN @atlas_HAVE_FORTRAN@ ) set( atlas_HAVE_EIGEN @atlas_HAVE_EIGEN@ ) set( atlas_HAVE_GRIDTOOLS_STORAGE @atlas_HAVE_GRIDTOOLS_STORAGE@ ) set( atlas_HAVE_TESSELATION @atlas_HAVE_TESSELATION@ ) set( ATLAS_LIBRARIES @ATLAS_LIBRARIES@ ) set( atlas_VERSION_STR @atlas_VERSION_STR@ ) set( atlas_REQUIRES_PRIVATE_DEPENDENCIES @atlas_REQUIRES_PRIVATE_DEPENDENCIES@ ) ## eckit find_dependency( eckit HINTS ${CMAKE_CURRENT_LIST_DIR}/../eckit @eckit_DIR@ @eckit_BINARY_DIR@ ) ## fckit if( atlas_HAVE_FORTRAN ) find_dependency( fckit HINTS ${CMAKE_CURRENT_LIST_DIR}/../fckit @fckit_DIR@ @fckit_BINARY_DIR@ ) endif() find_dependency( atlas_io HINTS ${CMAKE_CURRENT_LIST_DIR}/../atlas_io @atlas_io_DIR@ @atlas_io_BINARY_DIR@ ) find_dependency( hic HINTS ${CMAKE_CURRENT_LIST_DIR}/../hic @hic_DIR@ @hic_BINARY_DIR@ ) find_dependency( pluto HINTS ${CMAKE_CURRENT_LIST_DIR}/../pluto @pluto_DIR@ @pluto_BINARY_DIR@ ) ## Eigen3 set( Eigen3_HINT @Eigen3_DIR@ ) if( atlas_HAVE_EIGEN AND Eigen3_HINT ) find_dependency( Eigen3 HINTS ${Eigen3_HINT} ) endif() ## gridtools_storage if( atlas_HAVE_GRIDTOOLS_STORAGE ) # Required for GridTools' find_package( MPI COMPONENTS CXX ) if( NOT CMAKE_CXX_COMPILER_LOADED ) enable_language( CXX ) endif() find_dependency( GridTools HINTS ${GridTools_ROOT}/lib/cmake # non-standard install (reported upstream) $ENV{GridTools_ROOT}/lib/cmake # non-standard install (reported upstream) ${CMAKE_CURRENT_LIST_DIR}/.. # non-standard install (reported upstream) ${CMAKE_PREFIX_PATH}/lib/cmake # non-standard install (reported upstream) ${CMAKE_INSTALL_PREFIX}/lib/cmake # non-standard install (reported upstream) ${CMAKE_CURRENT_LIST_DIR}/../gridtools @GridTools_DIR@ @GridTools_BINARY_DIR@ ) endif() ## OpenMP unset( atlas_OMP_COMPONENTS ) if( atlas_HAVE_OMP_Fortran AND CMAKE_Fortran_COMPILER_LOADED AND atlas_REQUIRES_PRIVATE_DEPENDENCIES ) list( APPEND atlas_OMP_COMPONENTS Fortran ) endif() if( atlas_HAVE_OMP_CXX ) if( NOT CMAKE_CXX_COMPILER_LOADED ) enable_language( CXX ) endif() list( APPEND atlas_OMP_COMPONENTS CXX ) endif() if( atlas_OMP_COMPONENTS ) find_dependency( OpenMP COMPONENTS ${atlas_OMP_COMPONENTS} ) endif() ## transi if( atlas_HAVE_ECTRANS AND atlas_REQUIRES_PRIVATE_DEPENDENCIES ) set( transi_DIR @transi_DIR@ ) if( transi_DIR ) find_dependency( transi HINTS ${CMAKE_CURRENT_LIST_DIR}/../transi @transi_DIR@ ) else() find_dependency( ectrans COMPONENTS transi double HINTS ${CMAKE_CURRENT_LIST_DIR}/../ectrans @ectrans_DIR@ ) endif() endif() ## Qhull if( atlas_HAVE_TESSELATION AND atlas_REQUIRES_PRIVATE_DEPENDENCIES ) find_dependency( Qhull HINTS @Qhull_DIR@ ) endif() ## Fortran set( atlas_FORTRAN_FOUND 0 ) if( atlas_HAVE_FORTRAN ) set( atlas_FORTRAN_FOUND 1 ) elseif( atlas_FIND_REQUIRED_FORTRAN ) message( FATAL_ERROR "atlas was not compiled with FORTRAN enabled" ) endif() set( @PROJECT_NAME@_DIR ${CMAKE_CURRENT_LIST_DIR} CACHE STRING "" ) function( atlas_create_plugin name ) set( options ) set( single_value_args VERSION LIBRARY URL NAMESPACE) set( multi_value_args ) cmake_parse_arguments( _PAR "${options}" "${single_value_args}" "${multi_value_args}" ${_FIRST_ARG} ${ARGN} ) set( _plugin_file share/plugins/${name}.yml ) if( NOT DEFINED _PAR_VERSION ) set( _version ${${PROJECT_NAME}_VERSION} ) else() set( _version ${_PAR_VERSION} ) endif() if( NOT DEFINED _PAR_LIBRARY ) set( _library "${name}" ) else() set( _library "${_PAR_LIBRARY}" ) endif() if( NOT DEFINED _PAR_URL ) set( _url "http://www.ecmwf.int" ) else() set( _url ${_PAR_URL} ) endif() if( NOT DEFINED _PAR_NAMESPACE ) set( _namespace "int.ecmwf" ) else() set( _namespace ${_PAR_NAMESPACE} ) endif() file( WRITE ${CMAKE_BINARY_DIR}/${_plugin_file} "plugin:\n" ) file( APPEND ${CMAKE_BINARY_DIR}/${_plugin_file} " name: ${name}\n" ) file( APPEND ${CMAKE_BINARY_DIR}/${_plugin_file} " namespace: ${_namespace}\n" ) file( APPEND ${CMAKE_BINARY_DIR}/${_plugin_file} " url: ${_url}\n" ) file( APPEND ${CMAKE_BINARY_DIR}/${_plugin_file} " version: ${_version}\n" ) file( APPEND ${CMAKE_BINARY_DIR}/${_plugin_file} " library: ${_library}\n" ) install( FILES ${CMAKE_BINARY_DIR}/${_plugin_file} DESTINATION share/plugins ) endfunction() atlas-0.45.0/cmake/atlas_compile_flags.cmake000066400000000000000000000055361513175025300207710ustar00rootroot00000000000000# (C) Copyright 2013 ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) ecbuild_add_option( FEATURE WARNING_AS_ERROR DEFAULT OFF DESCRIPTION "Treat compile warning as error" ) if(HAVE_WARNING_AS_ERROR) ecbuild_add_cxx_flags("-Werror" NO_FAIL NAME atlas_cxx_warning_as_error) endif() ecbuild_add_option( FEATURE WARNINGS DEFAULT ON DESCRIPTION "Add warnings to compiler" ) # activate warnings, ecbuild macros check the compiler recognises the options if(HAVE_WARNINGS) ecbuild_add_cxx_flags("-Wall" NO_FAIL) ecbuild_add_cxx_flags("-Wextra" NO_FAIL) ecbuild_add_cxx_flags("-Wno-unused-parameter" NO_FAIL) ecbuild_add_cxx_flags("-Wno-sign-compare" NO_FAIL) endif() # nvfortran does not provide robust flags to prevent forcefully repacking of discontinuguous argument arrays set( HAVE_NOREPACK 1 ) if( CMAKE_Fortran_COMPILER_ID MATCHES NVHPC ) set( HAVE_NOREPACK 0 ) endif() if( CMAKE_CXX_COMPILER_ID STREQUAL Intel ) ecbuild_add_cxx_flags("-diag-disable=11074" NO_FAIL) # Inline limits ecbuild_add_cxx_flags("-diag-disable=11076" NO_FAIL) # Inline limits ecbuild_add_cxx_flags("-diag-disable=10441" NO_FAIL) # Deprecated classic compiler endif() if( CMAKE_Fortran_COMPILER_ID MATCHES Intel) # Both for Intel and Intel-LLVM ecbuild_add_fortran_flags("-diag-disable=5462" NO_FAIL) # Global name too long, shortened endif() if( CMAKE_CXX_COMPILER_ID MATCHES Cray ) if( NOT CMAKE_CXX_COMPILER_ID MATCHES CrayClang ) ecbuild_add_cxx_flags("-hnomessage=3140" NAME atlas_cxx_disable_warnings ) # colon separated numbers endif() ecbuild_add_fortran_flags("-hnomessage=3140" NAME atlas_fortran_disable_warnings ) # colon separated numbers # CC-3140 crayc++: WARNING File = atlas/functionspace/NodeColumns.cc, Line = 1, Column = 1 # The IPA optimization level was changed to "1" due to the presence of OMP # directives, ACC directives, or ASM intrinsics. endif() if( CMAKE_CXX_COMPILER_ID MATCHES NVHPC ) ecbuild_add_cxx_flags("--diag_suppress declared_but_not_referenced --display_error_number" NAME atlas_cxx_disable_warnings ) # For all the variables with side effects (constructor/destructor functionality) endif() if( CMAKE_CXX_COMPILER_ID MATCHES IntelLLVM ) # Turn off -ffinite-math-only which gets included by some optimisation levels which assumes values can never be NaN. # Then results in std::isnan(value) always return false. ecbuild_add_cxx_flags("-fno-finite-math-only") endif() atlas-0.45.0/cmake/atlas_ecbuild2_compatibility.cmake000066400000000000000000000026621513175025300226040ustar00rootroot00000000000000macro( filter_tpl _tpl ) string( TOUPPER ${_tpl} _TPL ) unset( ${_tpl}_INCLUDE_DIRS ) unset( ${_TPL}_INCLUDE_DIRS ) unset( ${_tpl}_INCLUDE_DIR ) unset( ${_TPL}_INCLUDE_DIR ) unset( ${_tpl}_LIBRARIES ) unset( ${_TPL}_LIBRARIES ) unset( ${_tpl}_LIBRARY ) unset( ${_TPL}_LIBRARY ) if( ${_tpl}_INCLUDE_DIRS ) unset( ${_tpl}_INCLUDE_DIRS CACHE ) endif() if( ${_TPL}_INCLUDE_DIRS ) unset( ${_TPL}_INCLUDE_DIRS CACHE ) endif() if( ${_tpl}_INCLUDE_DIR ) unset( ${_tpl}_INCLUDE_DIR CACHE ) endif() if( ${_TPL}_INCLUDE_DIR ) unset( ${_TPL}_INCLUDE_DIR CACHE ) endif() if( ${_tpl}_LIBRARIES ) unset( ${_tpl}_LIBRARIES CACHE ) endif() if( ${_TPL}_LIBRARIES ) unset( ${_TPL}_LIBRARIES CACHE ) endif() if( ${_tpl}_LIBRARY ) unset( ${_tpl}_LIBRARY CACHE ) endif() if( ${_TPL}_LIBRARY ) unset( ${_TPL}_LIBRARY CACHE ) endif() endmacro() filter_tpl( fckit ) filter_tpl( CGAL ) filter_tpl( FFTW ) filter_tpl( Eigen3 ) filter_tpl( transi ) get_target_property( eckit_intf_incl_dirs eckit INTERFACE_INCLUDE_DIRECTORIES ) set( eckit_can_be_filtered FALSE ) foreach( eckit_intf_incl_dir ${eckit_intf_incl_dirs} ) string(FIND ${eckit_intf_incl_dir} "eckit" _found ) if( NOT( _found EQUAL "-1" ) ) # if eckit INTERFACE_INCLUDE_DIRECTORIES contains "eckit" set( eckit_can_be_filtered TRUE ) endif() endforeach() if( eckit_can_be_filtered ) filter_tpl(eckit) endif() unset( ATLAS_TPLS ) atlas-0.45.0/cmake/atlas_export.cmake000066400000000000000000000032121513175025300174730ustar00rootroot00000000000000################################################################################ # export package info if( TARGET atlas_f ) list( APPEND ATLAS_LIBRARIES atlas_f ) endif() list( APPEND ATLAS_LIBRARIES atlas ) ################################################################################ # pkg-config ecbuild_add_option( FEATURE PKGCONFIG DESCRIPTION "Atlas pkgconfig" ) set( ATLAS_URL "https://software.ecmwf.int/wiki/display/ATLAS" ) set( ATLAS_DESCRIPTION "Atlas framework for parallel mesh datastructures" ) if( atlas_HAVE_PKGCONFIG ) ecbuild_pkgconfig( NAME atlas LIBRARIES ${ATLAS_LIBRARIES} ) ecbuild_pkgconfig( NAME atlas-c++ LANGUAGES CXX LIBRARIES atlas ) if( atlas_HAVE_FORTRAN ) ecbuild_pkgconfig( NAME atlas-fortran LANGUAGES Fortran LIBRARIES atlas_f NO_PRIVATE_INCLUDE_DIRS ) endif() endif() ################################################################################ # finalize ecbuild_add_resources( TARGET atlas-others SOURCES_PACK README.md CHANGELOG.md LICENSE ) if( atlas_HAVE_FORTRAN AND ECBUILD_INSTALL_FORTRAN_MODULES ) install( DIRECTORY ${CMAKE_Fortran_MODULE_DIRECTORY}/${CMAKE_CFG_INTDIR} DESTINATION module/atlas COMPONENT modules ) endif() set( atlas_REQUIRES_PRIVATE_DEPENDENCIES FALSE ) get_target_property( target_build_type atlas TYPE ) if( target_build_type STREQUAL STATIC_LIBRARY ) set( atlas_REQUIRES_PRIVATE_DEPENDENCIES TRUE ) endif() include( atlas_ecbuild2_compatibility ) ecbuild_install_project( NAME Atlas ) atlas-0.45.0/cmake/atlas_host_device.cmake000066400000000000000000000036371513175025300204610ustar00rootroot00000000000000# (C) Copyright 2013 ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. function( create_hic_wrapper variable ) set( options "" ) set( single_value_args SOURCE ) set( multi_value_args "" ) cmake_parse_arguments( _PAR "${options}" "${single_value_args}" "${multi_value_args}" ${_FIRST_ARG} ${ARGN} ) get_filename_component(directory ${_PAR_SOURCE} DIRECTORY) get_filename_component(base ${_PAR_SOURCE} NAME_WE) get_filename_component(name ${_PAR_SOURCE} NAME) get_filename_component(abspath ${_PAR_SOURCE} ABSOLUTE) if( HAVE_CUDA ) set( extension "cu" ) elseif( HAVE_HIP ) set( extension "hip" ) endif() if( directory ) set(hic_wrapper ${CMAKE_CURRENT_BINARY_DIR}/${directory}/${base}.${extension}) else() set(hic_wrapper ${CMAKE_CURRENT_BINARY_DIR}/${base}.${extension}) endif() set(${variable} ${hic_wrapper} PARENT_SCOPE) set(content " #include \"atlas/${directory}/${name}\" ") if( ${abspath} IS_NEWER_THAN ${hic_wrapper} ) file(WRITE ${hic_wrapper} "${content}") endif() endfunction() function( atlas_host_device srclist ) set( options "" ) set( single_value_args "" ) set( multi_value_args SOURCES ) cmake_parse_arguments( _PAR "${options}" "${single_value_args}" "${multi_value_args}" ${ARGN} ) if( HAVE_GPU ) set( use_hic_srclist ${_PAR_SOURCES} ) foreach( src ${use_hic_srclist} ) create_hic_wrapper( hic_wrapper SOURCE ${src} ) list( APPEND ${srclist} ${hic_wrapper} ) ecbuild_list_exclude_pattern( LIST ${srclist} REGEX ${src} ) endforeach() set( ${srclist} "${${srclist}}" PARENT_SCOPE ) endif() endfunction() atlas-0.45.0/cmake/features/000077500000000000000000000000001513175025300156045ustar00rootroot00000000000000atlas-0.45.0/cmake/features/ACC.cmake000066400000000000000000000011231513175025300171710ustar00rootroot00000000000000### OpenACC if( atlas_HAVE_ATLAS_FIELD AND HAVE_GPU ) if( DEFINED ATLAS_ENABLE_ACC ) set( ENABLE_ACC ${ATLAS_ENABLE_ACC} ) endif() if( ENABLE_ACC ) if( NOT HAVE_FORTRAN ) enable_language(Fortran) endif() find_package( OpenACC COMPONENTS Fortran CXX ) endif() ecbuild_add_option( FEATURE ACC DESCRIPTION "OpenACC capable data structures" CONDITION OpenACC_Fortran_FOUND ) if( HAVE_ACC ) set( ACC_LINK_OPTIONS ${OpenACC_Fortran_FLAGS} ) endif() else() set( HAVE_ACC 0 ) set( atlas_HAVE_ACC 0 ) endif() atlas-0.45.0/cmake/features/ATLAS_RUN.cmake000066400000000000000000000011501513175025300201730ustar00rootroot00000000000000### use of atlas-run for tests ecbuild_add_option( FEATURE ATLAS_RUN DEFAULT ON DESCRIPTION "Use atlas/tools/atlas-run to run atlas tests" ) if( HAVE_ATLAS_RUN ) set( MPIEXEC_EXECUTABLE ${CMAKE_CURRENT_SOURCE_DIR}/tools/atlas-run ) set( MPIEXEC_NUMPROC_FLAG='-n' ) set( MPIEXEC_NUMTHREAD_FLAG='-c' ) if( CMAKE_VERSION VERSION_LESS 3.29 ) set( CMAKE_CROSSCOMPILING_EMULATOR ${CMAKE_CURRENT_SOURCE_DIR}/tools/atlas-run ) else() unset(CMAKE_CROSSCOMPILING_EMULATOR) set( CMAKE_TEST_LAUNCHER ${CMAKE_CURRENT_SOURCE_DIR}/tools/atlas-run ) endif() endif() atlas-0.45.0/cmake/features/BOUNDSCHECKING.cmake000066400000000000000000000010401513175025300206670ustar00rootroot00000000000000### Bounds checking if( ${CMAKE_BUILD_TYPE} MATCHES "Debug" ) set( DEFAULT_BOUNDSCHECKING ON ) else() set( DEFAULT_BOUNDSCHECKING OFF ) endif() ecbuild_add_option( FEATURE BOUNDSCHECKING DEFAULT ${DEFAULT_BOUNDSCHECKING} DESCRIPTION "Bounds checking for atlas::ArrayView and atlas::IndexView" ) if( ${CMAKE_BUILD_TYPE} MATCHES "Debug" ) if( NOT atlas_HAVE_BOUNDSCHECKING ) ecbuild_info( "Turning BOUNDSCHECKING ON for Debug build" ) set( atlas_HAVE_BOUNDSCHECKING 1 ) endif() endif() atlas-0.45.0/cmake/features/CLANG_TIDY.cmake000066400000000000000000000027521513175025300202710ustar00rootroot00000000000000set( CLANG_TIDY_SUPPORTED_COMPILERS GNU Clang ) set( CLANG_TIDY_DEFAULT OFF ) foreach( _clang_tidy_supported_compiler ${CLANG_TIDY_SUPPORTED_COMPILERS} ) if( CMAKE_CXX_COMPILER_ID MATCHES ${_clang_tidy_supported_compiler} ) set( CLANG_TIDY_DEFAULT ON ) endif() endforeach() find_program( CLANG_TIDY_EXE NAMES "clang-tidy" ) if( CLANG_TIDY_EXE ) ecbuild_info( "Found clang-tidy: ${CLANG_TIDY_EXE}" ) if( NOT CLANG_TIDY_DEFAULT AND NOT DEFINED ENABLE_CLANG_TIDY ) ecbuild_info( "Feature CLANG_TIDY default OFF for compiler ${CMAKE_CXX_COMPILER_ID}" ) endif() endif() ecbuild_add_option( FEATURE CLANG_TIDY DEFAULT ${CLANG_TIDY_DEFAULT} DESCRIPTION "Use clang-tidy" CONDITION CLANG_TIDY_EXE ) if (HAVE_CLANG_TIDY) # Uncomment to apply fixes. Make sure to use a clean build, and apply clang-format afterwards! # set( CLANG_TIDY_FIXIT ";-fix" ) set( CLANG_TIDY_CHECKS "-*" ) foreach( _clang_tidy_check readability-braces-around-statements redundant-string-init modernize-use-nullptr modernize-use-using modernize-use-override modernize-use-emplace modernize-use-equals-default modernize-use-equals-delete ) set( CLANG_TIDY_CHECKS "${CLANG_TIDY_CHECKS},${_clang_tidy_check}" ) endforeach() set( CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_EXE};-checks=${CLANG_TIDY_CHECKS};-extra-arg-before=-D__clang_analyzer__;-header-filter='${CMAKE_SOURCE_DIR}/*'${CLANG_TIDY_FIXIT}" ) endif() atlas-0.45.0/cmake/features/CUDA.cmake000066400000000000000000000025151513175025300173250ustar00rootroot00000000000000 # ecbuild_add_option( FEATURE CUDA # DESCRIPTION "Enable CUDA support" # DEFAULT OFF # ) # ecbuild_add_option( FEATURE HIP # DESCRIPTION "Enable CUDA support" # DEFAULT OFF # ) set( atlas_HAVE_CUDA 0 ) set( atlas_HAVE_HIP 0 ) set( atlas_HAVE_GPU 0 ) if( hic_HAVE_CUDA ) enable_language( CUDA ) ecbuild_info( "CUDA language enabled" ) find_package(CUDAToolkit REQUIRED) set( atlas_HAVE_CUDA 1 ) set( atlas_HAVE_GPU 1 ) elseif( hic_HAVE_HIP ) enable_language( HIP ) ecbuild_info( "HIP language enabled" ) find_package(hip CONFIG REQUIRED) set( atlas_HAVE_HIP 1 ) set( atlas_HAVE_GPU 1 ) endif() set( HAVE_CUDA ${atlas_HAVE_CUDA} ) set( HAVE_HIP ${atlas_HAVE_HIP} ) set( HAVE_GPU ${atlas_HAVE_GPU} ) if( HAVE_GPU ) ecbuild_info("GPU support enabled") else() ecbuild_info("GPU support not enabled") endif() if( HAVE_GPU ) set( GPU_AWARE_MPI_default ON ) else() set( GPU_AWARE_MPI_default OFF ) endif() ecbuild_add_option( FEATURE GPU_AWARE_MPI DESCRIPTION "MPI supports GPU to GPU transfers" DEFAULT ${GPU_AWARE_MPI_default} ) if( HAVE_GPU_AWARE_MPI ) ecbuild_info("GPU aware MPI support enabled") else() ecbuild_info("GPU aware MPI support not enabled") endif() atlas-0.45.0/cmake/features/DOCS.cmake000066400000000000000000000005121513175025300173340ustar00rootroot00000000000000################################################################################ # documentation ecbuild_add_option( FEATURE DOCS DESCRIPTION "Atlas documentation" DEFAULT OFF REQUIRED_PACKAGES "LATEX COMPONENTS PDFLATEX BIBTEX OPTIONAL_COMPONENTS MAKEINDEX HTLATEX" ) atlas-0.45.0/cmake/features/ECTRANS.cmake000066400000000000000000000032331513175025300177060ustar00rootroot00000000000000if( atlas_HAVE_ATLAS_TRANS ) ### trans ... if( NOT DEFINED ATLAS_ENABLE_ECTRANS AND DEFINED ATLAS_ENABLE_TRANS ) ecbuild_warn("Atlas option ATLAS_ENABLE_TRANS is deprecated in favour of ATLAS_ENABLE_ECTRANS") set( ATLAS_ENABLE_ECTRANS ${ATLAS_ENABLE_TRANS} ) endif() if( NOT DEFINED ENABLE_ECTRANS AND DEFINED ENABLE_TRANS ) ecbuild_warn("Atlas option ENABLE_TRANS is deprecated in favour of ENABLE_ECTRANS") set( ENABLE_ECTRANS ${ENABLE_TRANS} ) endif() if( DEFINED ATLAS_ENABLE_ECTRANS ) set( ENABLE_ECTRANS ${ATLAS_ENABLE_ECTRANS} ) endif() set( atlas_HAVE_PACKAGE_ECTRANS 0 ) if( atlas_HAVE_ATLAS_FUNCTIONSPACE AND (ENABLE_ECTRANS OR NOT DEFINED ENABLE_ECTRANS) ) find_package( ectrans 1.1 COMPONENTS transi double QUIET ) if( TARGET transi_dp ) set( transi_FOUND TRUE ) if( NOT TARGET transi ) if( CMAKE_VERSION VERSION_LESS 3.18 ) # Before CMake 3.18 it is not possible to alias a non-global imported target # Make the import global. Warning, this may break further find_package get_target_property( transi_dp_IMPORTED transi_dp IMPORTED ) if( transi_dp_IMPORTED ) set_target_properties( transi_dp PROPERTIES IMPORTED_GLOBAL TRUE) endif() endif() add_library( transi ALIAS transi_dp ) endif() set( atlas_HAVE_PACKAGE_ECTRANS 1 ) else() find_package( transi 0.8 QUIET ) endif() endif() ecbuild_add_option( FEATURE ECTRANS DESCRIPTION "Support for IFS spectral transforms" CONDITION atlas_HAVE_ATLAS_FUNCTIONSPACE AND transi_FOUND ) endif() atlas-0.45.0/cmake/features/EIGEN.cmake000066400000000000000000000011001513175025300174250ustar00rootroot00000000000000### Eigen if( atlas_HAVE_ATLAS_FUNCTIONSPACE ) ecbuild_add_option( FEATURE EIGEN DESCRIPTION "Use Eigen linear algebra library" REQUIRED_PACKAGES Eigen3 ) if( HAVE_EIGEN AND NOT TARGET Eigen3::Eigen ) # This is the case for older Eigen versions (e.g. 3.2.0) ecbuild_add_library( TARGET atlas_eigen3 TYPE INTERFACE ) target_include_directories( atlas_eigen3 INTERFACE ${EIGEN3_INCLUDE_DIRS} ) add_library( Eigen3::Eigen ALIAS atlas_eigen3 ) endif() else() set( HAVE_EIGEN 0 ) set( atlas_HAVE_EIGEN 0 ) endif()atlas-0.45.0/cmake/features/FFTW.cmake000066400000000000000000000004431513175025300173550ustar00rootroot00000000000000### FFTW ... if( atlas_HAVE_ATLAS_TRANS ) ecbuild_add_option( FEATURE FFTW DESCRIPTION "Support for fftw" REQUIRED_PACKAGES "FFTW COMPONENTS double QUIET" ) if( NOT HAVE_FFTW ) unset( FFTW_LIBRARIES ) unset( FFTW_INCLUDES ) endif() endif()atlas-0.45.0/cmake/features/FORTRAN.cmake000066400000000000000000000014721513175025300177250ustar00rootroot00000000000000### Fortran ... ecbuild_add_option( FEATURE FORTRAN DESCRIPTION "Provide Fortran bindings" REQUIRED_PACKAGES "fckit VERSION 0.6.2 COMPONENTS ECKIT" CONDITION atlas_HAVE_ATLAS_FUNCTIONSPACE ) if( atlas_HAVE_FORTRAN ) if( fckit_HAVE_ECKIT ) ecbuild_enable_fortran( REQUIRED MODULE_DIRECTORY ${PROJECT_BINARY_DIR}/module ) set( Fortran Fortran ) if( HAVE_TESTS ) set( HAVE_FCTEST ON ) set( atlas_HAVE_FCTEST ON ) else() set( HAVE_FCTEST OFF ) set( atlas_HAVE_FCTEST OFF ) endif() else() ecbuild_warn( "In order to compile atlas_f, fckit is required to be compiled with eckit. Turning off fortran." ) set( HAVE_FORTRAN 0 ) set( atlas_HAVE_FORTRAN 0 ) endif() endif() ecbuild_find_python( NO_LIBS ) atlas-0.45.0/cmake/features/GRIDTOOLS_STORAGE.cmake000066400000000000000000000034011513175025300213360ustar00rootroot00000000000000if( atlas_HAVE_ATLAS_FIELD AND (ENABLE_GRIDTOOLS_STORAGE OR atlas_ENABLE_GRIDTOOLS_STORAGE) ) ### GridTools storage module ### GridTools may search for CUDA, which searches for "Threads" ### Set THREADS_HAVE_PTHREAD_ARG variable to false so that it can be recomputed based on ### THREADS_PREFER_PTHREAD_FLAG, in case other project had it on a different setting. ### This is certainly a CMake bug ( see ECKIT-426 ) set( THREADS_HAVE_PTHREAD_ARG FALSE ) if( NOT DEFINED THREADS_PREFER_PTHREAD_FLAG ) set( THREADS_PREFER_PTHREAD_FLAG 1 ) endif() find_package( GridTools QUIET HINTS ${GridTools_ROOT}/lib/cmake $ENV{GridTools_ROOT}/lib/cmake ${CMAKE_PREFIX_PATH}/lib/cmake ${CMAKE_INSTALL_PREFIX}/lib/cmake ${GridTools_BINARY_DIR} ) ecbuild_add_option( FEATURE GRIDTOOLS_STORAGE DESCRIPTION "Arrays internally use GridTools storage layer" CONDITION GridTools_FOUND ) set( ATLAS_GRIDTOOLS_STORAGE_BACKEND_HOST 0 ) set( ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA 0 ) if( atlas_HAVE_GRIDTOOLS_STORAGE ) if( GRIDTOOLS_HAS_BACKEND_CUDA ) if( atlas_HAVE_CUDA ) ecbuild_info( "GridTools found with CUDA support -> backend CUDA" ) set( ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA 1 ) else() ecbuild_info( "GridTools found with CUDA support, but atlas does not have CUDA enabled -> backend HOST" ) set( ATLAS_GRIDTOOLS_STORAGE_BACKEND_HOST 1 ) endif() else() ecbuild_info( "GridTools found without CUDA support -> backend HOST" ) set( ATLAS_GRIDTOOLS_STORAGE_BACKEND_HOST 1 ) endif() endif() else() set( HAVE_GRIDTOOLS_STORAGE 0 ) set( atlas_HAVE_GRIDTOOLS_STORAGE 0 ) set( ATLAS_GRIDTOOLS_STORAGE_BACKEND_HOST 0 ) set( ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA 0 ) endif() atlas-0.45.0/cmake/features/INCLUDE_WHAT_YOU_USE.cmake000066400000000000000000000010221513175025300217570ustar00rootroot00000000000000find_program( INCLUDE_WHAT_YOU_USE_EXE NAMES "include-what-you-use" ) if( INCLUDE_WHAT_YOU_USE_EXE ) ecbuild_info( "Found include-what-you-use: ${INCLUDE_WHAT_YOU_USE_EXE}" ) endif() ecbuild_add_option( FEATURE INCLUDE_WHAT_YOU_USE DEFAULT OFF # Need clang compiler? DESCRIPTION "Use include-what-you-use clang-tool" CONDITION INCLUDE_WHAT_YOU_USE_EXE ) if( HAVE_INCLUDE_WHAT_YOU_USE ) set( CMAKE_CXX_INCLUDE_WHAT_YOU_USE ${INCLUDE_WHAT_YOU_USE_EXE} ) endif() atlas-0.45.0/cmake/features/INIT_SNAN.cmake000066400000000000000000000022041513175025300201660ustar00rootroot00000000000000### Init signaling NaN if( ${CMAKE_BUILD_TYPE} MATCHES "Debug" ) set( DEFAULT_INIT_SNAN ON ) else() set( DEFAULT_INIT_SNAN OFF ) endif() ecbuild_add_option( FEATURE INIT_SNAN DEFAULT ${DEFAULT_INIT_SNAN} DESCRIPTION "Initialise atlas arrays with signaling_NaN (real types) or other invalid values (other types)" ) if( ${CMAKE_BUILD_TYPE} MATCHES "Debug" ) if( NOT atlas_HAVE_INIT_SNAN ) ecbuild_info( "Turning INIT_SNAN ON for Debug build" ) set( atlas_HAVE_INIT_SNAN 1 ) endif() endif() cmake_push_check_state(RESET) include(CheckSymbolExists) set(CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE) if(UNIX) set(CMAKE_REQUIRED_LIBRARIES m) endif() check_symbol_exists(feenableexcept "fenv.h" atlas_HAVE_FEENABLEEXCEPT) check_symbol_exists(fedisableexcept "fenv.h" atlas_HAVE_FEDISABLEEXCEPT) if( atlas_HAVE_FEENABLEEXCEPT ) set( atlas_HAVE_FEENABLEEXCEPT 1 ) else() set( atlas_HAVE_FEENABLEEXCEPT 0 ) endif() if( atlas_HAVE_FEDISABLEEXCEPT ) set( atlas_HAVE_FEDISABLEEXCEPT 1 ) else() set( atlas_HAVE_FEDISABLEEXCEPT 0 ) endif() cmake_pop_check_state() atlas-0.45.0/cmake/features/MPI.cmake000066400000000000000000000003271513175025300172350ustar00rootroot00000000000000 ### MPI ... if( NOT eckit_HAVE_MPI ) ecbuild_warn("ecKit has been compiled without MPI. This causes Atlas to not be able to run parallel jobs.") set( atlas_HAVE_MPI 0 ) else() set( atlas_HAVE_MPI 1 ) endif() atlas-0.45.0/cmake/features/OMP.cmake000066400000000000000000000065361513175025300172530ustar00rootroot00000000000000### OMP ... if( ENABLE_OMP OR NOT DEFINED ENABLE_OMP ) find_package( OpenMP COMPONENTS CXX ${Fortran} ) endif() ecbuild_add_option( FEATURE OMP DESCRIPTION "support for OpenMP shared memory parallelism" CONDITION OpenMP_Fortran_FOUND OR OpenMP_CXX_FOUND ) ecbuild_add_option( FEATURE OMP_Fortran DESCRIPTION "support for Fortran OpenMP shared memory parallelism" CONDITION HAVE_OMP AND OpenMP_Fortran_FOUND ) ecbuild_add_option( FEATURE OMP_CXX DESCRIPTION "support for CXX OpenMP shared memory parallelism" CONDITION HAVE_OMP AND OpenMP_CXX_FOUND ) if( TARGET OpenMP::OpenMP_CXX ) set( OMP_CXX OpenMP::OpenMP_CXX ) endif() if( TARGET OpenMP::OpenMP_Fortran ) set( OMP_Fortran OpenMP::OpenMP_Fortran ) endif() if( HAVE_OMP_CXX ) if( NOT CMAKE_CXX_COMPILER_ID MATCHES Clang ) set( ATLAS_OMP_TASK_SUPPORTED 1 ) endif() if( NOT DEFINED ATLAS_OMP_TASK_SUPPORTED ) try_run( execute_result compile_result ${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/cmake/features/OMP/test_omp_task.cc LINK_LIBRARIES ${OMP_CXX} COMPILE_OUTPUT_VARIABLE compile_output RUN_OUTPUT_VARIABLE execute_output ) ecbuild_debug("Compiling and running ${PROJECT_SOURCE_DIR}/cmake/features/OMP/test_omp_task.cc") ecbuild_debug_var( compile_result ) ecbuild_debug_var( compile_output ) ecbuild_debug_var( execute_result ) ecbuild_debug_var( execute_output ) if( compile_result ) if( execute_result MATCHES 0 ) set( ATLAS_OMP_TASK_SUPPORTED 1 ) else() ecbuild_info(" Compiler failed to correctly run program with 'omp task' pragma." "Sorting with OMP is disabled.") set( ATLAS_OMP_TASK_SUPPORTED 0 ) endif() else() set( ATLAS_OMP_TASK_SUPPORTED 0 ) endif() endif() if( ATLAS_OMP_TASK_SUPPORTED ) if( NOT CMAKE_CXX_COMPILER_ID MATCHES Clang ) set( ATLAS_OMP_TASK_UNTIED_SUPPORTED 1 ) endif() if( NOT DEFINED ATLAS_OMP_TASK_UNTIED_SUPPORTED ) try_run( execute_result compile_result ${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/cmake/features/OMP/test_omp_untied.cc LINK_LIBRARIES ${OMP_CXX} COMPILE_OUTPUT_VARIABLE compile_output RUN_OUTPUT_VARIABLE execute_output ) ecbuild_debug("Compiling and running ${PROJECT_SOURCE_DIR}/cmake/features/OMP/test_omp_untied.cc") ecbuild_debug_var( compile_result ) ecbuild_debug_var( compile_output ) ecbuild_debug_var( execute_result ) ecbuild_debug_var( execute_output ) if( compile_result ) if( execute_result MATCHES 0 ) set( ATLAS_OMP_TASK_UNTIED_SUPPORTED 1 ) else() ecbuild_info(" Compiler failed to run program with omp pragma with 'untied if' construct." "Workaround will be enabled.") set( ATLAS_OMP_TASK_UNTIED_SUPPORTED 0 ) endif() else() set( ATLAS_OMP_TASK_UNTIED_SUPPORTED 0 ) endif() endif() else() set( ATLAS_OMP_TASK_UNTIED_SUPPORTED 0 ) endif() else() set( ATLAS_OMP_TASK_SUPPORTED 0 ) set( ATLAS_OMP_TASK_UNTIED_SUPPORTED 0 ) endif() atlas-0.45.0/cmake/features/OMP/000077500000000000000000000000001513175025300162375ustar00rootroot00000000000000atlas-0.45.0/cmake/features/OMP/test_omp_task.cc000066400000000000000000000027261513175025300214310ustar00rootroot00000000000000/* * (C) Copyright 2013 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ // This test mirrors functionality from src/atlas/parallel/omp/sort.h // It appears that on some installations / compilers (such as apple-clang with llvm omp) // the tasking does not seem to work properly. // It all compiles but leads to runtime errors. This executable is // added to allow compiler introspection for this feature. #include #include #include #include void recursive_task( int begin, int end ) { int size = end - begin; if ( size >= 2 ) { // should be much larger in real case (e.g. 256) int mid = begin + size / 2; std::cout << begin << " - " << end << " [" < #include #include template void merge_sort_recursive( const RandomAccessIterator& iterator, size_t begin, size_t end ) { auto size = end - begin; if ( size >= 2 ) { // should be much larger in real case (e.g. 256) auto mid = begin + size / 2; { #pragma omp task shared( iterator ) untied if ( size >= ( 1 << 15 ) ) merge_sort_recursive( iterator, begin, mid ); #pragma omp task shared( iterator ) untied if ( size >= ( 1 << 15 ) ) merge_sort_recursive( iterator, mid, end ); #pragma omp taskwait } std::inplace_merge( iterator + begin, iterator + mid, iterator + end ); } else { std::sort( iterator + begin, iterator + end ); } } template void omp_sort( RandomAccessIterator first, RandomAccessIterator last ) { #pragma omp parallel #pragma omp single merge_sort_recursive( first, 0, std::distance( first, last ) ); } int main() { auto integers = std::vector( 8 ); omp_sort( integers.begin(), integers.end() ); return 0; } atlas-0.45.0/cmake/features/POCKETFFT.cmake000066400000000000000000000054531513175025300201420ustar00rootroot00000000000000### POCKETFFT ... if( atlas_HAVE_ATLAS_TRANS ) set (_user_disabled_pocketfft OFF) if (DEFINED ATLAS_ENABLE_POCKETFFT AND NOT ATLAS_ENABLE_POCKETFFT) set(ENABLE_POCKETFFT ${ATLAS_ENABLE_POCKETFFT}) endif() if (DEFINED ENABLE_POCKETFFT AND NOT ENABLE_POCKETFFT) set (_user_disabled_pocketfft ON) endif() if (NOT _user_disabled_pocketfft) # Find pocketfft header-only library! # If pocketfft is not found, the source codes will be downloaded, unless ENABLE_DOWNLOAD=OFF if (NOT DEFINED ENABLE_DOWNLOAD) set(ENABLE_DOWNLOAD ON) endif() if (NOT ENABLE_DOWNLOAD) ecbuild_find_package(pocketfft) else() ecbuild_find_package(pocketfft QUIET) if (pocketfft_FOUND AND EXISTS ${POCKETFFT_INCLUDE_DIRS}) ecbuild_info("atlas FOUND pocketfft") ecbuild_info(" POCKETFFT_INCLUDE_DIRS : [${POCKETFFT_INCLUDE_DIRS}]") else() set(pocketfft_ROOT ${PROJECT_BINARY_DIR}/pocketfft) if (NOT EXISTS ${pocketfft_ROOT} ) set(pocketfft_COMMIT 0fa0ef591e38c2758e3184c6c23e497b9f732ffa) # from branch cpp, dated 30 Nov 2024 set(pocketfft_URL https://github.com/mreineck/pocketfft/archive/${pocketfft_COMMIT}.zip) ecbuild_info("Downloading pocketfft (${pocketfft_URL}) ...") file(DOWNLOAD ${pocketfft_URL} ${PROJECT_BINARY_DIR}/download/pocketfft.zip STATUS DOWNLOAD_STATUS) # Separate the returned status code, and error message. list(GET DOWNLOAD_STATUS 0 STATUS_CODE) list(GET DOWNLOAD_STATUS 1 ERROR_MESSAGE) # Check if download was successful. if(${STATUS_CODE} EQUAL 0) file(ARCHIVE_EXTRACT INPUT ${PROJECT_BINARY_DIR}/download/pocketfft.zip DESTINATION ${PROJECT_BINARY_DIR}/download) file(REMOVE ${PROJECT_BINARY_DIR}/download/pocketfft.zip) file(RENAME ${PROJECT_BINARY_DIR}/download/pocketfft-${pocketfft_COMMIT} ${pocketfft_ROOT}) ecbuild_info("Downloading pocketfft (${pocketfft_URL}) ... done") else() ecbuild_info("Downloading pocketfft (${pocketfft_URL}) ... FAILED: ${ERROR_MESSAGE}") endif() endif() ecbuild_find_package(pocketfft) endif() endif() endif() ecbuild_add_option( FEATURE POCKETFFT DEFAULT ON DESCRIPTION "Enable use of pocketfft for FFT's used in Trans" CONDITION pocketfft_FOUND ) endif() if( NOT atlas_HAVE_POCKETFFT ) unset( POCKETFFT_INCLUDE_DIRS ) endif() atlas-0.45.0/cmake/features/PROJ.cmake000066400000000000000000000035621513175025300173660ustar00rootroot00000000000000if( atlas_HAVE_ATLAS_GRID ) ### Proj # From proj 9 onwards, it is guaranteed that it was built/installed using CMake and the CMake export is available. # Then we can safely remove the file FindPROJ.cmake from atlas # and the following atrocity could be simply replaced with # # ecbuild_add_option( FEATURE PROJ # DESCRIPTION "PROJ-based projections" # DEFAULT OFF # REQUIRED_PACKAGES PROJ ) # if( ENABLE_PROJ ) ecbuild_find_package_search_hints( NAME PROJ ) # 1) Try to find PROJ the CMake way (proj >= 7) find_package( PROJ CONFIG ) if( PROJ_FOUND ) ecbuild_debug("Found PROJ via CONFIG") else() # 2) Try to find PROJ4 the CMake way (proj 6) find_package( PROJ4 CONFIG ) if( PROJ4_FOUND ) ecbuild_debug("Found PROJ4 via CONFIG") set( PROJ_FOUND ${PROJ4_FOUND} ) set( PROJ_LIBRARIES ${PROJ4_LIBRARIES} ) set( PROJ_INCLUDE_DIRS ${PROJ4_INCLUDE_DIRS} ) set( PROJ_VERSION ${PROJ4_VERSION} ) endif() endif() if( NOT PROJ_FOUND ) # 3) Try to find PROJ via FindPROJ.cmake provided within atlas (proj < 9) find_package( PROJ MODULE ) if( PROJ_FOUND ) ecbuild_debug("Found PROJ via FindPROJ.cmake") endif() endif() else() ecbuild_debug("Skipping search for PROJ as ENABLE_PROJ=OFF (default)") endif() if( PROJ_FOUND ) ecbuild_info( "Found PROJ (${PROJ_VERSION}):") ecbuild_info( " PROJ_LIBRARIES : ${PROJ_LIBRARIES}") ecbuild_info( " PROJ_INCLUDE_DIRS : ${PROJ_INCLUDE_DIRS}") endif() ecbuild_add_option( FEATURE PROJ DESCRIPTION "PROJ-based projections" DEFAULT OFF CONDITION PROJ_FOUND ) if( NOT HAVE_PROJ ) unset( PROJ_LIBRARIES ) unset( PROJ_INCLUDE_DIRS ) endif() endif()atlas-0.45.0/cmake/features/SANDBOX.cmake000066400000000000000000000002151513175025300177020ustar00rootroot00000000000000### sandbox ecbuild_add_option( FEATURE SANDBOX DEFAULT OFF DESCRIPTION "Build the sandbox stuff" ) atlas-0.45.0/cmake/features/TESSELATION.cmake000066400000000000000000000035441513175025300204060ustar00rootroot00000000000000if( atlas_HAVE_ATLAS_FUNCTIONSPACE ) ### tesselation ... ecbuild_add_option( FEATURE TESSELATION DESCRIPTION "Support for unstructured mesh generation" CONDITION atlas_HAVE_ATLAS_FUNCTIONSPACE REQUIRED_PACKAGES "Qhull" ) if(HAVE_TESSELATION) set(QHULL_LIBRARIES Qhull::qhullcpp Qhull::qhull_r) set(atlas_HAVE_QHULL 1) else() set(atlas_HAVE_QHULL 0) endif() ### NOTE # # CGAL is deprecated as TESSELATION backend. Qhull is to be used instead. # To use CGAL regardless, turn ON CGAL feature (-DENABLE_CGAL=ON) set(Boost_USE_MULTITHREADED ON ) ecbuild_add_option( FEATURE CGAL DEFAULT OFF DESCRIPTION "Support for unstructured mesh generation" CONDITION atlas_HAVE_ATLAS_FUNCTIONSPACE REQUIRED_PACKAGES "CGAL" "Boost VERSION 1.45.0 QUIET" ) if( HAVE_CGAL ) list( APPEND CGAL_INCLUDE_DIRS ${Boost_INCLUDE_DIRS} ) if ( TARGET CGAL::CGAL ) list( APPEND CGAL_LIBRARIES CGAL::CGAL ${CGAL_3RD_PARTY_LIBRARIES} ${GMP_LIBRARIES} ${MPFR_LIBRARIES} ${Boost_THREAD_LIBRARY} ${Boost_SYSTEM_LIBRARY} ) set( _cgal_target CGAL::CGAL ) get_target_property(_aliased CGAL::CGAL ALIASED_TARGET) if(_aliased) set( _cgal_target ${_aliased} ) endif() # Reset INTERFACE_COMPILE_OPTIONS ( see ATLAS-193 ) get_target_property( CGAL_COMPILE_FLAGS ${_cgal_target} INTERFACE_COMPILE_OPTIONS ) set_target_properties( ${_cgal_target} PROPERTIES INTERFACE_COMPILE_OPTIONS "" ) else() list( APPEND CGAL_LIBRARIES ${CGAL_LIBRARY} ${CGAL_3RD_PARTY_LIBRARIES} ${GMP_LIBRARIES} ${MPFR_LIBRARIES} ${Boost_THREAD_LIBRARY} ${Boost_SYSTEM_LIBRARY} ) endif() endif() if( NOT HAVE_CGAL ) unset( CGAL_LIBRARIES ) unset( CGAL_INCLUDE_DIRS ) endif() endif() atlas-0.45.0/cmake/project_summary.cmake000066400000000000000000000040161513175025300202140ustar00rootroot00000000000000# (C) Copyright 2013 ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. if( MPI_CXX_FOUND AND MPI_Fortan_FOUND ) ecbuild_info( "MPI" ) ecbuild_info( " MPI_CXX_COMPILER : [${MPI_CXX_COMPILER}]" ) ecbuild_info( " MPI_CXX_INCLUDE_PATH : [${MPI_CXX_INCLUDE_PATH}]" ) ecbuild_info( " MPI_CXX_LIBRARIES : [${MPI_CXX_LIBRARIES}]" ) ecbuild_info( " MPI_Fortan_COMPILER : [${MPI_Fortan_COMPILER}]" ) ecbuild_info( " MPI_Fortan_INCLUDE_PATH : [${MPI_Fortan_INCLUDE_PATH}]" ) ecbuild_info( " MPI_Fortan_LIBRARIES : [${MPI_Fortan_LIBRARIES}]" ) ecbuild_info( " MPIEXEC : [${MPIEXEC}]" ) endif() if( CGAL_FOUND ) ecbuild_info( "CGAL (${CGAL_VERSION})" ) ecbuild_info( " includes : [${CGAL_INCLUDE_DIRS}]" ) ecbuild_info( " libs : [${CGAL_LIBRARY}]" ) endif() if( atlas_HAVE_CUDA ) ecbuild_info( "CUDA (${CUDA_VERSION})" ) ecbuild_info( " CUDA_NVCC_COMPILER : [${CUDA_NVCC_EXECUTABLE}]" ) ecbuild_info( " CUDA_CUDART_LIBRARY : [${CUDA_CUDART_LIBRARY}]" ) ecbuild_info( " CUDA_NVCC_FLAGS : [${CUDA_NVCC_FLAGS}]" ) endif() if( atlas_HAVE_ACC ) ecbuild_info( "ACC" ) ecbuild_info( " OpenACC_Fortran_FLAGS : [${OpenACC_Fortran_FLAGS}]" ) endif() if( atlas_HAVE_GRIDTOOLS_STORAGE ) if( ATLAS_GRIDTOOLS_STORAGE_BACKEND_HOST ) ecbuild_info( "Array storage backend: GridTools [HOST]" ) endif() if( ATLAS_GRIDTOOLS_STORAGE_BACKEND_CUDA ) ecbuild_info( "Array storage backend: GridTools [CUDA]" ) endif() else() if( NOT atlas_HAVE_GPU ) ecbuild_info( "Array storage backend: Native [HOST]" ) else() ecbuild_info( "Array storage backend: Native [GPU]" ) endif() endif() atlas-0.45.0/doc/000077500000000000000000000000001513175025300134535ustar00rootroot00000000000000atlas-0.45.0/doc/CMakeLists.txt000066400000000000000000000053201513175025300162130ustar00rootroot00000000000000# (C) Copyright 2013 ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. if(atlas_HAVE_DOCS) if(atlas_HAVE_FORTRAN) add_subdirectory(example-fortran) endif() add_subdirectory(user-guide) list( APPEND DOC_CODE_TARGETS atlas_c-hello-world atlas_c-global-grids-Structured atlas_c-global-grids-Unstructured atlas_c-meshes-Structured atlas_c-fields atlas_c-fields-on-grid atlas_c-NodeColumns atlas_c-StructuredColumns ) if( atlas_HAVE_FORTRAN ) list( APPEND DOC_CODE_TARGETS atlas_f-hello-world atlas_f-global-grids-Structured atlas_f-meshes-Structured atlas_f-fields atlas_f-fields-on-grid atlas_f-NodeColumns ) endif() add_custom_target(atlas_doc_code) add_dependencies (atlas_doc_code ${DOC_CODE_TARGETS} ) add_custom_target(atlas_doc_pdf) add_dependencies (atlas_doc_pdf atlas-user-guide-pdf ) if( LATEX_HTLATEX_FOUND ) add_custom_target(atlas_doc_html) add_dependencies (atlas_doc_html atlas-user-guide-html ) endif() add_custom_target(atlas_doc) add_dependencies (atlas_doc atlas_doc_code atlas_doc_pdf ) if( PROJECT_NAME STREQUAL CMAKE_PROJECT_NAME ) add_custom_target(doc) add_dependencies(doc atlas_doc) endif() if( NOT DEFINED ATLAS_DOC_VERSION ) set( ATLAS_DOC_VERSION ${ATLAS_VERSION_STR} ) endif() if( NOT DEFINED ATLAS_DOXYGEN_GENERATOR ) set( ATLAS_DOXYGEN_GENERATOR "stock") endif() if( ATLAS_DOXYGEN_GENERATOR STREQUAL "m.css" ) set( ATLAS_DOXYFILE Doxyfile-mcss ) set( ATLAS_DOXYGEN_EXECUTABLE doxygen.py ) else() set( ATLAS_DOXYFILE Doxyfile-stock ) endif() if( NOT DEFINED ATLAS_DOXYGEN_EXECUTABLE ) find_package(Doxygen COMPONENTS dot) if( DOXYGEN_FOUND ) set( ATLAS_DOXYGEN_EXECUTABLE ${DOXYGEN_EXECUTABLE} ) else() set( ATLAS_DOXYGEN_EXECUTABLE doxygen ) endif() endif() foreach( doxyfile Doxyfile-default Doxyfile-custom Doxyfile-mcss Doxyfile-stock ) execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_CURRENT_SOURCE_DIR}/${doxyfile} ${CMAKE_CURRENT_BINARY_DIR}/${doxyfile} ) endforeach() configure_file(Doxyfile.in ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY) add_custom_target( atlas_doxygen COMMAND ${ATLAS_DOXYGEN_EXECUTABLE} Doxyfile WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Generating Doxygen documentation" VERBATIM ) set_property(TARGET atlas_doxygen PROPERTY EXCLUDE_FROM_ALL TRUE) endif() atlas-0.45.0/doc/Doxyfile-custom000066400000000000000000000025141513175025300164730ustar00rootroot00000000000000@INCLUDE = Doxyfile-default # Nested folders will be ignored without this. You may not need it. RECURSIVE = YES # Enable preprocessing and related preprocessor necessities ENABLE_PREPROCESSING = YES MACRO_EXPANSION = YES EXPAND_ONLY_PREDEF = YES SKIP_FUNCTION_MACROS = NO # extra defs for to help with building the _right_ version of the docs PREDEFINED = DOXYGEN_DOCUMENTATION_BUILD PREDEFINED += DOXYGEN_SHOULD_SKIP_THIS # Using `=` instead of `+=` overrides HIDE_FRIEND_COMPOUNDS = YES PREDEFINED += protected=private PREDEFINED += DOXYGEN_HIDE(X)= PREDEFINED += ATLAS_HOST_DEVICE= PREDEFINED += INDEX_REF=idx_t& EXTRACT_PRIVATE = NO EXCLUDE_PATTERNS = */detail/* EXCLUDE_PATTERNS += detail/* EXCLUDE_PATTERNS += */test/* EXCLUDE_PATTERNS += */gridtools/* EXCLUDE_PATTERNS += */native/* EXCLUDE_PATTERNS += */helpers/* EXCLUDE_PATTERNS += */Object.h EXCLUDE_PATTERNS += */ObjectHandle.h EXCLUDE_PATTERNS += */State.h EXCLUDE_PATTERNS += *.cc EXCLUDE_PATTERNS += CMakeLists.txt EXCLUDE_PATTERNS += */TableView.h EXCLUDE_PATTERNS += */Table.h EXCLUDE_SYMBOLS = detail EXCLUDE_SYMBOLS += eckit INLINE_INHERITED_MEMB = YES HIDE_UNDOC_RELATIONS = NO GENERATE_TODOLIST = NO GENERATE_BUGLIST = NO #BUILTIN_STL_SUPPORT = NO TOC_INCLUDE_HEADINGS = 5 MARKDOWN_SUPPORT = YES atlas-0.45.0/doc/Doxyfile-default000066400000000000000000003303041513175025300166060ustar00rootroot00000000000000# Doxyfile 1.8.17 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project. # # All text after a double hash (##) is considered a comment and is placed in # front of the TAG it is preceding. # # All text after a single hash (#) is considered a comment and will be ignored. # The format is: # TAG = value [value, ...] # For lists, items can also be appended using: # TAG += value [value, ...] # Values that contain spaces should be placed between quotes (\" \"). #--------------------------------------------------------------------------- # Project related configuration options #--------------------------------------------------------------------------- # This tag specifies the encoding used for all characters in the configuration # file that follow. The default is UTF-8 which is also the encoding used for all # text before the first occurrence of this tag. Doxygen uses libiconv (or the # iconv built into libc) for the transcoding. See # https://www.gnu.org/software/libiconv/ for the list of possible encodings. # The default value is: UTF-8. DOXYFILE_ENCODING = UTF-8 # The PROJECT_NAME tag is a single word (or a sequence of words surrounded by # double-quotes, unless you are using Doxywizard) that should identify the # project for which the documentation is generated. This name is used in the # title of most generated pages and in a few other places. # The default value is: My Project. PROJECT_NAME = "My Project" # The PROJECT_NUMBER tag can be used to enter a project or revision number. This # could be handy for archiving the generated documentation or if some version # control system is used. PROJECT_NUMBER = # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a # quick idea about the purpose of the project. Keep the description short. PROJECT_BRIEF = # With the PROJECT_LOGO tag one can specify a logo or an icon that is included # in the documentation. The maximum height of the logo should not exceed 55 # pixels and the maximum width should not exceed 200 pixels. Doxygen will copy # the logo to the output directory. PROJECT_LOGO = # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path # into which the generated documentation will be written. If a relative path is # entered, it will be relative to the location where doxygen was started. If # left blank the current directory will be used. OUTPUT_DIRECTORY = # If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- # directories (in 2 levels) under the output directory of each output format and # will distribute the generated files over these directories. Enabling this # option can be useful when feeding doxygen a huge amount of source files, where # putting all generated files in the same directory would otherwise causes # performance problems for the file system. # The default value is: NO. CREATE_SUBDIRS = NO # If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII # characters to appear in the names of generated files. If set to NO, non-ASCII # characters will be escaped, for example _xE3_x81_x84 will be used for Unicode # U+3044. # The default value is: NO. ALLOW_UNICODE_NAMES = NO # The OUTPUT_LANGUAGE tag is used to specify the language in which all # documentation generated by doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. # Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, # Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), # Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, # Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), # Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, # Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, # Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, # Ukrainian and Vietnamese. # The default value is: English. OUTPUT_LANGUAGE = English # The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all # documentation generated by doxygen is written. Doxygen will use this # information to generate all generated output in the proper direction. # Possible values are: None, LTR, RTL and Context. # The default value is: None. OUTPUT_TEXT_DIRECTION = None # If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member # descriptions after the members that are listed in the file and class # documentation (similar to Javadoc). Set to NO to disable this. # The default value is: YES. BRIEF_MEMBER_DESC = YES # If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief # description of a member or function before the detailed description # # Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the # brief descriptions will be completely suppressed. # The default value is: YES. REPEAT_BRIEF = YES # This tag implements a quasi-intelligent brief description abbreviator that is # used to form the text in various listings. Each string in this list, if found # as the leading text of the brief description, will be stripped from the text # and the result, after processing the whole list, is used as the annotated # text. Otherwise, the brief description is used as-is. If left blank, the # following values are used ($name is automatically replaced with the name of # the entity):The $name class, The $name widget, The $name file, is, provides, # specifies, contains, represents, a, an and the. ABBREVIATE_BRIEF = "The $name class" \ "The $name widget" \ "The $name file" \ is \ provides \ specifies \ contains \ represents \ a \ an \ the # If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then # doxygen will generate a detailed section even if there is only a brief # description. # The default value is: NO. ALWAYS_DETAILED_SEC = NO # If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all # inherited members of a class in the documentation of that class as if those # members were ordinary class members. Constructors, destructors and assignment # operators of the base classes will not be shown. # The default value is: NO. INLINE_INHERITED_MEMB = NO # If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path # before files name in the file list and in the header files. If set to NO the # shortest path that makes the file name unique will be used # The default value is: YES. FULL_PATH_NAMES = YES # The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. # Stripping is only done if one of the specified strings matches the left-hand # part of the path. The tag can be used to show relative paths in the file list. # If left blank the directory from which doxygen is run is used as the path to # strip. # # Note that you can specify absolute paths here, but also relative paths, which # will be relative from the directory where doxygen is started. # This tag requires that the tag FULL_PATH_NAMES is set to YES. STRIP_FROM_PATH = # The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the # path mentioned in the documentation of a class, which tells the reader which # header file to include in order to use a class. If left blank only the name of # the header file containing the class definition is used. Otherwise one should # specify the list of include paths that are normally passed to the compiler # using the -I flag. STRIP_FROM_INC_PATH = # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but # less readable) file names. This can be useful is your file systems doesn't # support long names like on DOS, Mac, or CD-ROM. # The default value is: NO. SHORT_NAMES = NO # If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the # first line (until the first dot) of a Javadoc-style comment as the brief # description. If set to NO, the Javadoc-style will behave just like regular Qt- # style comments (thus requiring an explicit @brief command for a brief # description.) # The default value is: NO. JAVADOC_AUTOBRIEF = NO # If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line # such as # /*************** # as being the beginning of a Javadoc-style comment "banner". If set to NO, the # Javadoc-style will behave just like regular comments and it will not be # interpreted by doxygen. # The default value is: NO. JAVADOC_BANNER = NO # If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first # line (until the first dot) of a Qt-style comment as the brief description. If # set to NO, the Qt-style will behave just like regular Qt-style comments (thus # requiring an explicit \brief command for a brief description.) # The default value is: NO. QT_AUTOBRIEF = NO # The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a # multi-line C++ special comment block (i.e. a block of //! or /// comments) as # a brief description. This used to be the default behavior. The new default is # to treat a multi-line C++ comment block as a detailed description. Set this # tag to YES if you prefer the old behavior instead. # # Note that setting this tag to YES also means that rational rose comments are # not recognized any more. # The default value is: NO. MULTILINE_CPP_IS_BRIEF = NO # If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the # documentation from any documented member that it re-implements. # The default value is: YES. INHERIT_DOCS = YES # If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new # page for each member. If set to NO, the documentation of a member will be part # of the file/class/namespace that contains it. # The default value is: NO. SEPARATE_MEMBER_PAGES = NO # The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen # uses this value to replace tabs by spaces in code fragments. # Minimum value: 1, maximum value: 16, default value: 4. TAB_SIZE = 4 # This tag can be used to specify a number of aliases that act as commands in # the documentation. An alias has the form: # name=value # For example adding # "sideeffect=@par Side Effects:\n" # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading # "Side Effects:". You can put \n's in the value part of an alias to insert # newlines (in the resulting output). You can put ^^ in the value part of an # alias to insert a newline as if a physical newline was in the original file. # When you need a literal { or } or , in the value part of an alias you have to # escape them by means of a backslash (\), this can lead to conflicts with the # commands \{ and \} for these it is advised to use the version @{ and @} or use # a double escape (\\{ and \\}) ALIASES = # This tag can be used to specify a number of word-keyword mappings (TCL only). # A mapping has the form "name=value". For example adding "class=itcl::class" # will allow you to use the command class in the itcl::class meaning. TCL_SUBST = # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources # only. Doxygen will then generate output that is more tailored for C. For # instance, some of the names that are used will be different. The list of all # members will be omitted, etc. # The default value is: NO. OPTIMIZE_OUTPUT_FOR_C = NO # Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or # Python sources only. Doxygen will then generate output that is more tailored # for that language. For instance, namespaces will be presented as packages, # qualified scopes will look different, etc. # The default value is: NO. OPTIMIZE_OUTPUT_JAVA = NO # Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran # sources. Doxygen will then generate output that is tailored for Fortran. # The default value is: NO. OPTIMIZE_FOR_FORTRAN = NO # Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL # sources. Doxygen will then generate output that is tailored for VHDL. # The default value is: NO. OPTIMIZE_OUTPUT_VHDL = NO # Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice # sources only. Doxygen will then generate output that is more tailored for that # language. For instance, namespaces will be presented as modules, types will be # separated into more groups, etc. # The default value is: NO. OPTIMIZE_OUTPUT_SLICE = NO # Doxygen selects the parser to use depending on the extension of the files it # parses. With this tag you can assign which parser to use for a given # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and # language is one of the parsers supported by doxygen: IDL, Java, Javascript, # Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, # Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: # FortranFree, unknown formatted Fortran: Fortran. In the later case the parser # tries to guess whether the code is fixed or free formatted code, this is the # default for Fortran type files), VHDL, tcl. For instance to make doxygen treat # .inc files as Fortran files (default is PHP), and .f files as C (default is # Fortran), use: inc=Fortran f=C. # # Note: For files without extension you can use no_extension as a placeholder. # # Note that for custom extensions you also need to set FILE_PATTERNS otherwise # the files are not read by doxygen. EXTENSION_MAPPING = # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments # according to the Markdown format, which allows for more readable # documentation. See https://daringfireball.net/projects/markdown/ for details. # The output of markdown processing is further processed by doxygen, so you can # mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in # case of backward compatibilities issues. # The default value is: YES. MARKDOWN_SUPPORT = YES # When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up # to that level are automatically included in the table of contents, even if # they do not have an id attribute. # Note: This feature currently applies only to Markdown headings. # Minimum value: 0, maximum value: 99, default value: 5. # This tag requires that the tag MARKDOWN_SUPPORT is set to YES. TOC_INCLUDE_HEADINGS = 5 # When enabled doxygen tries to link words that correspond to documented # classes, or namespaces to their corresponding documentation. Such a link can # be prevented in individual cases by putting a % sign in front of the word or # globally by setting AUTOLINK_SUPPORT to NO. # The default value is: YES. AUTOLINK_SUPPORT = YES # If you use STL classes (i.e. std::string, std::vector, etc.) but do not want # to include (a tag file for) the STL sources as input, then you should set this # tag to YES in order to let doxygen match functions declarations and # definitions whose arguments contain STL classes (e.g. func(std::string); # versus func(std::string) {}). This also make the inheritance and collaboration # diagrams that involve STL classes more complete and accurate. # The default value is: NO. BUILTIN_STL_SUPPORT = NO # If you use Microsoft's C++/CLI language, you should set this option to YES to # enable parsing support. # The default value is: NO. CPP_CLI_SUPPORT = NO # Set the SIP_SUPPORT tag to YES if your project consists of sip (see: # https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen # will parse them like normal C++ but will assume all classes use public instead # of private inheritance when no explicit protection keyword is present. # The default value is: NO. SIP_SUPPORT = NO # For Microsoft's IDL there are propget and propput attributes to indicate # getter and setter methods for a property. Setting this option to YES will make # doxygen to replace the get and set methods by a property in the documentation. # This will only work if the methods are indeed getting or setting a simple # type. If this is not the case, or you want to show the methods anyway, you # should set this option to NO. # The default value is: YES. IDL_PROPERTY_SUPPORT = YES # If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC # tag is set to YES then doxygen will reuse the documentation of the first # member in the group (if any) for the other members of the group. By default # all members of a group must be documented explicitly. # The default value is: NO. DISTRIBUTE_GROUP_DOC = NO # If one adds a struct or class to a group and this option is enabled, then also # any nested class or struct is added to the same group. By default this option # is disabled and one has to add nested compounds explicitly via \ingroup. # The default value is: NO. GROUP_NESTED_COMPOUNDS = NO # Set the SUBGROUPING tag to YES to allow class member groups of the same type # (for instance a group of public functions) to be put as a subgroup of that # type (e.g. under the Public Functions section). Set it to NO to prevent # subgrouping. Alternatively, this can be done per class using the # \nosubgrouping command. # The default value is: YES. SUBGROUPING = YES # When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions # are shown inside the group in which they are included (e.g. using \ingroup) # instead of on a separate page (for HTML and Man pages) or section (for LaTeX # and RTF). # # Note that this feature does not work in combination with # SEPARATE_MEMBER_PAGES. # The default value is: NO. INLINE_GROUPED_CLASSES = NO # When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions # with only public data fields or simple typedef fields will be shown inline in # the documentation of the scope in which they are defined (i.e. file, # namespace, or group documentation), provided this scope is documented. If set # to NO, structs, classes, and unions are shown on a separate page (for HTML and # Man pages) or section (for LaTeX and RTF). # The default value is: NO. INLINE_SIMPLE_STRUCTS = NO # When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or # enum is documented as struct, union, or enum with the name of the typedef. So # typedef struct TypeS {} TypeT, will appear in the documentation as a struct # with name TypeT. When disabled the typedef will appear as a member of a file, # namespace, or class. And the struct will be named TypeS. This can typically be # useful for C code in case the coding convention dictates that all compound # types are typedef'ed and only the typedef is referenced, never the tag name. # The default value is: NO. TYPEDEF_HIDES_STRUCT = NO # The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This # cache is used to resolve symbols given their name and scope. Since this can be # an expensive process and often the same symbol appears multiple times in the # code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small # doxygen will become slower. If the cache is too large, memory is wasted. The # cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range # is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 # symbols. At the end of a run doxygen will report the cache usage and suggest # the optimal cache size from a speed point of view. # Minimum value: 0, maximum value: 9, default value: 0. LOOKUP_CACHE_SIZE = 0 #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- # If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in # documentation are documented, even if no documentation was available. Private # class members and static file members will be hidden unless the # EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. # Note: This will also disable the warnings about undocumented members that are # normally produced when WARNINGS is set to YES. # The default value is: NO. EXTRACT_ALL = NO # If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will # be included in the documentation. # The default value is: NO. EXTRACT_PRIVATE = NO # If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual # methods of a class will be included in the documentation. # The default value is: NO. EXTRACT_PRIV_VIRTUAL = NO # If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal # scope will be included in the documentation. # The default value is: NO. EXTRACT_PACKAGE = NO # If the EXTRACT_STATIC tag is set to YES, all static members of a file will be # included in the documentation. # The default value is: NO. EXTRACT_STATIC = NO # If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined # locally in source files will be included in the documentation. If set to NO, # only classes defined in header files are included. Does not have any effect # for Java sources. # The default value is: YES. EXTRACT_LOCAL_CLASSES = YES # This flag is only useful for Objective-C code. If set to YES, local methods, # which are defined in the implementation section but not in the interface are # included in the documentation. If set to NO, only methods in the interface are # included. # The default value is: NO. EXTRACT_LOCAL_METHODS = NO # If this flag is set to YES, the members of anonymous namespaces will be # extracted and appear in the documentation as a namespace called # 'anonymous_namespace{file}', where file will be replaced with the base name of # the file that contains the anonymous namespace. By default anonymous namespace # are hidden. # The default value is: NO. EXTRACT_ANON_NSPACES = NO # If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all # undocumented members inside documented classes or files. If set to NO these # members will be included in the various overviews, but no documentation # section is generated. This option has no effect if EXTRACT_ALL is enabled. # The default value is: NO. HIDE_UNDOC_MEMBERS = NO # If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all # undocumented classes that are normally visible in the class hierarchy. If set # to NO, these classes will be included in the various overviews. This option # has no effect if EXTRACT_ALL is enabled. # The default value is: NO. HIDE_UNDOC_CLASSES = NO # If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend # (class|struct|union) declarations. If set to NO, these declarations will be # included in the documentation. # The default value is: NO. HIDE_FRIEND_COMPOUNDS = NO # If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any # documentation blocks found inside the body of a function. If set to NO, these # blocks will be appended to the function's detailed documentation block. # The default value is: NO. HIDE_IN_BODY_DOCS = NO # The INTERNAL_DOCS tag determines if documentation that is typed after a # \internal command is included. If the tag is set to NO then the documentation # will be excluded. Set it to YES to include the internal documentation. # The default value is: NO. INTERNAL_DOCS = NO # If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file # names in lower-case letters. If set to YES, upper-case letters are also # allowed. This is useful if you have classes or files whose names only differ # in case and if your file system supports case sensitive file names. Windows # (including Cygwin) ands Mac users are advised to set this option to NO. # The default value is: system dependent. CASE_SENSE_NAMES = YES # If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with # their full class and namespace scopes in the documentation. If set to YES, the # scope will be hidden. # The default value is: NO. HIDE_SCOPE_NAMES = NO # If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will # append additional text to a page's title, such as Class Reference. If set to # YES the compound reference will be hidden. # The default value is: NO. HIDE_COMPOUND_REFERENCE= NO # If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of # the files that are included by a file in the documentation of that file. # The default value is: YES. SHOW_INCLUDE_FILES = YES # If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each # grouped member an include statement to the documentation, telling the reader # which file to include in order to use the member. # The default value is: NO. SHOW_GROUPED_MEMB_INC = NO # If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include # files with double quotes in the documentation rather than with sharp brackets. # The default value is: NO. FORCE_LOCAL_INCLUDES = NO # If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the # documentation for inline members. # The default value is: YES. INLINE_INFO = YES # If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the # (detailed) documentation of file and class members alphabetically by member # name. If set to NO, the members will appear in declaration order. # The default value is: YES. SORT_MEMBER_DOCS = YES # If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief # descriptions of file, namespace and class members alphabetically by member # name. If set to NO, the members will appear in declaration order. Note that # this will also influence the order of the classes in the class list. # The default value is: NO. SORT_BRIEF_DOCS = NO # If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the # (brief and detailed) documentation of class members so that constructors and # destructors are listed first. If set to NO the constructors will appear in the # respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. # Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief # member documentation. # Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting # detailed member documentation. # The default value is: NO. SORT_MEMBERS_CTORS_1ST = NO # If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy # of group names into alphabetical order. If set to NO the group names will # appear in their defined order. # The default value is: NO. SORT_GROUP_NAMES = NO # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by # fully-qualified names, including namespaces. If set to NO, the class list will # be sorted only by class name, not including the namespace part. # Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. # Note: This option applies only to the class list, not to the alphabetical # list. # The default value is: NO. SORT_BY_SCOPE_NAME = NO # If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper # type resolution of all parameters of a function it will reject a match between # the prototype and the implementation of a member function even if there is # only one candidate or it is obvious which candidate to choose by doing a # simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still # accept a match between prototype and implementation in such cases. # The default value is: NO. STRICT_PROTO_MATCHING = NO # The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo # list. This list is created by putting \todo commands in the documentation. # The default value is: YES. GENERATE_TODOLIST = YES # The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test # list. This list is created by putting \test commands in the documentation. # The default value is: YES. GENERATE_TESTLIST = YES # The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug # list. This list is created by putting \bug commands in the documentation. # The default value is: YES. GENERATE_BUGLIST = YES # The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) # the deprecated list. This list is created by putting \deprecated commands in # the documentation. # The default value is: YES. GENERATE_DEPRECATEDLIST= YES # The ENABLED_SECTIONS tag can be used to enable conditional documentation # sections, marked by \if ... \endif and \cond # ... \endcond blocks. ENABLED_SECTIONS = # The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the # initial value of a variable or macro / define can have for it to appear in the # documentation. If the initializer consists of more lines than specified here # it will be hidden. Use a value of 0 to hide initializers completely. The # appearance of the value of individual variables and macros / defines can be # controlled using \showinitializer or \hideinitializer command in the # documentation regardless of this setting. # Minimum value: 0, maximum value: 10000, default value: 30. MAX_INITIALIZER_LINES = 30 # Set the SHOW_USED_FILES tag to NO to disable the list of files generated at # the bottom of the documentation of classes and structs. If set to YES, the # list will mention the files that were used to generate the documentation. # The default value is: YES. SHOW_USED_FILES = YES # Set the SHOW_FILES tag to NO to disable the generation of the Files page. This # will remove the Files entry from the Quick Index and from the Folder Tree View # (if specified). # The default value is: YES. SHOW_FILES = YES # Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces # page. This will remove the Namespaces entry from the Quick Index and from the # Folder Tree View (if specified). # The default value is: YES. SHOW_NAMESPACES = YES # The FILE_VERSION_FILTER tag can be used to specify a program or script that # doxygen should invoke to get the current version for each file (typically from # the version control system). Doxygen will invoke the program by executing (via # popen()) the command command input-file, where command is the value of the # FILE_VERSION_FILTER tag, and input-file is the name of an input file provided # by doxygen. Whatever the program writes to standard output is used as the file # version. For an example see the documentation. FILE_VERSION_FILTER = # The LAYOUT_FILE tag can be used to specify a layout file which will be parsed # by doxygen. The layout file controls the global structure of the generated # output files in an output format independent way. To create the layout file # that represents doxygen's defaults, run doxygen with the -l option. You can # optionally specify a file name after the option, if omitted DoxygenLayout.xml # will be used as the name of the layout file. # # Note that if you run doxygen from a directory containing a file called # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE # tag is left empty. LAYOUT_FILE = # The CITE_BIB_FILES tag can be used to specify one or more bib files containing # the reference definitions. This must be a list of .bib files. The .bib # extension is automatically appended if omitted. This requires the bibtex tool # to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info. # For LaTeX the style of the bibliography can be controlled using # LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the # search path. See also \cite for info how to create references. CITE_BIB_FILES = #--------------------------------------------------------------------------- # Configuration options related to warning and progress messages #--------------------------------------------------------------------------- # The QUIET tag can be used to turn on/off the messages that are generated to # standard output by doxygen. If QUIET is set to YES this implies that the # messages are off. # The default value is: NO. QUIET = NO # The WARNINGS tag can be used to turn on/off the warning messages that are # generated to standard error (stderr) by doxygen. If WARNINGS is set to YES # this implies that the warnings are on. # # Tip: Turn warnings on while writing the documentation. # The default value is: YES. WARNINGS = YES # If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate # warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag # will automatically be disabled. # The default value is: YES. WARN_IF_UNDOCUMENTED = YES # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for # potential errors in the documentation, such as not documenting some parameters # in a documented function, or documenting parameters that don't exist or using # markup commands wrongly. # The default value is: YES. WARN_IF_DOC_ERROR = YES # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return # value. If set to NO, doxygen will only warn about wrong or incomplete # parameter documentation, but not about the absence of documentation. If # EXTRACT_ALL is set to YES then this flag will automatically be disabled. # The default value is: NO. WARN_NO_PARAMDOC = NO # If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when # a warning is encountered. # The default value is: NO. WARN_AS_ERROR = NO # The WARN_FORMAT tag determines the format of the warning messages that doxygen # can produce. The string should contain the $file, $line, and $text tags, which # will be replaced by the file and line number from which the warning originated # and the warning text. Optionally the format may contain $version, which will # be replaced by the version of the file (if it could be obtained via # FILE_VERSION_FILTER) # The default value is: $file:$line: $text. WARN_FORMAT = "$file:$line: $text" # The WARN_LOGFILE tag can be used to specify a file to which warning and error # messages should be written. If left blank the output is written to standard # error (stderr). WARN_LOGFILE = #--------------------------------------------------------------------------- # Configuration options related to the input files #--------------------------------------------------------------------------- # The INPUT tag is used to specify the files and/or directories that contain # documented source files. You may enter file names like myfile.cpp or # directories like /usr/src/myproject. Separate the files or directories with # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. INPUT = # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses # libiconv (or the iconv built into libc) for the transcoding. See the libiconv # documentation (see: https://www.gnu.org/software/libiconv/) for the list of # possible encodings. # The default value is: UTF-8. INPUT_ENCODING = UTF-8 # If the value of the INPUT tag contains directories, you can use the # FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and # *.h) to filter out the source-files in the directories. # # Note that for custom extensions or not directly supported extensions you also # need to set EXTENSION_MAPPING for the extension otherwise the files are not # read by doxygen. # # If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, # *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, # *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, # *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment), # *.doc (to be provided as doxygen C comment), *.txt (to be provided as doxygen # C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f, *.for, *.tcl, *.vhd, # *.vhdl, *.ucf, *.qsf and *.ice. FILE_PATTERNS = *.c \ *.cc \ *.cxx \ *.cpp \ *.c++ \ *.java \ *.ii \ *.ixx \ *.ipp \ *.i++ \ *.inl \ *.idl \ *.ddl \ *.odl \ *.h \ *.hh \ *.hxx \ *.hpp \ *.h++ \ *.cs \ *.d \ *.php \ *.php4 \ *.php5 \ *.phtml \ *.inc \ *.m \ *.markdown \ *.md \ *.mm \ *.dox \ *.doc \ *.txt \ *.py \ *.pyw \ *.f90 \ *.f95 \ *.f03 \ *.f08 \ *.f \ *.for \ *.tcl \ *.vhd \ *.vhdl \ *.ucf \ *.qsf \ *.ice # The RECURSIVE tag can be used to specify whether or not subdirectories should # be searched for input files as well. # The default value is: NO. RECURSIVE = NO # The EXCLUDE tag can be used to specify files and/or directories that should be # excluded from the INPUT source files. This way you can easily exclude a # subdirectory from a directory tree whose root is specified with the INPUT tag. # # Note that relative paths are relative to the directory from which doxygen is # run. EXCLUDE = # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded # from the input. # The default value is: NO. EXCLUDE_SYMLINKS = NO # If the value of the INPUT tag contains directories, you can use the # EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude # certain files from those directories. # # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories for example use the pattern */test/* EXCLUDE_PATTERNS = # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names # (namespaces, classes, functions, etc.) that should be excluded from the # output. The symbol name can be a fully qualified name, a word, or if the # wildcard * is used, a substring. Examples: ANamespace, AClass, # AClass::ANamespace, ANamespace::*Test # # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories use the pattern */test/* EXCLUDE_SYMBOLS = # The EXAMPLE_PATH tag can be used to specify one or more files or directories # that contain example code fragments that are included (see the \include # command). EXAMPLE_PATH = # If the value of the EXAMPLE_PATH tag contains directories, you can use the # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and # *.h) to filter out the source-files in the directories. If left blank all # files are included. EXAMPLE_PATTERNS = * # If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be # searched for input files to be used with the \include or \dontinclude commands # irrespective of the value of the RECURSIVE tag. # The default value is: NO. EXAMPLE_RECURSIVE = NO # The IMAGE_PATH tag can be used to specify one or more files or directories # that contain images that are to be included in the documentation (see the # \image command). IMAGE_PATH = # The INPUT_FILTER tag can be used to specify a program that doxygen should # invoke to filter for each input file. Doxygen will invoke the filter program # by executing (via popen()) the command: # # # # where is the value of the INPUT_FILTER tag, and is the # name of an input file. Doxygen will then use the output that the filter # program writes to standard output. If FILTER_PATTERNS is specified, this tag # will be ignored. # # Note that the filter must not add or remove lines; it is applied before the # code is scanned, but not when the output code is generated. If lines are added # or removed, the anchors will not be placed correctly. # # Note that for custom extensions or not directly supported extensions you also # need to set EXTENSION_MAPPING for the extension otherwise the files are not # properly processed by doxygen. INPUT_FILTER = # The FILTER_PATTERNS tag can be used to specify filters on a per file pattern # basis. Doxygen will compare the file name with each pattern and apply the # filter if there is a match. The filters are a list of the form: pattern=filter # (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how # filters are used. If the FILTER_PATTERNS tag is empty or if none of the # patterns match the file name, INPUT_FILTER is applied. # # Note that for custom extensions or not directly supported extensions you also # need to set EXTENSION_MAPPING for the extension otherwise the files are not # properly processed by doxygen. FILTER_PATTERNS = # If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using # INPUT_FILTER) will also be used to filter the input files that are used for # producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). # The default value is: NO. FILTER_SOURCE_FILES = NO # The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file # pattern. A pattern will override the setting for FILTER_PATTERN (if any) and # it is also possible to disable source filtering for a specific pattern using # *.ext= (so without naming a filter). # This tag requires that the tag FILTER_SOURCE_FILES is set to YES. FILTER_SOURCE_PATTERNS = # If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that # is part of the input, its contents will be placed on the main page # (index.html). This can be useful if you have a project on for instance GitHub # and want to reuse the introduction page also for the doxygen output. USE_MDFILE_AS_MAINPAGE = #--------------------------------------------------------------------------- # Configuration options related to source browsing #--------------------------------------------------------------------------- # If the SOURCE_BROWSER tag is set to YES then a list of source files will be # generated. Documented entities will be cross-referenced with these sources. # # Note: To get rid of all source code in the generated output, make sure that # also VERBATIM_HEADERS is set to NO. # The default value is: NO. SOURCE_BROWSER = NO # Setting the INLINE_SOURCES tag to YES will include the body of functions, # classes and enums directly into the documentation. # The default value is: NO. INLINE_SOURCES = NO # Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any # special comment blocks from generated source code fragments. Normal C, C++ and # Fortran comments will always remain visible. # The default value is: YES. STRIP_CODE_COMMENTS = YES # If the REFERENCED_BY_RELATION tag is set to YES then for each documented # entity all documented functions referencing it will be listed. # The default value is: NO. REFERENCED_BY_RELATION = NO # If the REFERENCES_RELATION tag is set to YES then for each documented function # all documented entities called/used by that function will be listed. # The default value is: NO. REFERENCES_RELATION = NO # If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set # to YES then the hyperlinks from functions in REFERENCES_RELATION and # REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will # link to the documentation. # The default value is: YES. REFERENCES_LINK_SOURCE = YES # If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the # source code will show a tooltip with additional information such as prototype, # brief description and links to the definition and documentation. Since this # will make the HTML file larger and loading of large files a bit slower, you # can opt to disable this feature. # The default value is: YES. # This tag requires that the tag SOURCE_BROWSER is set to YES. SOURCE_TOOLTIPS = YES # If the USE_HTAGS tag is set to YES then the references to source code will # point to the HTML generated by the htags(1) tool instead of doxygen built-in # source browser. The htags tool is part of GNU's global source tagging system # (see https://www.gnu.org/software/global/global.html). You will need version # 4.8.6 or higher. # # To use it do the following: # - Install the latest version of global # - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file # - Make sure the INPUT points to the root of the source tree # - Run doxygen as normal # # Doxygen will invoke htags (and that will in turn invoke gtags), so these # tools must be available from the command line (i.e. in the search path). # # The result: instead of the source browser generated by doxygen, the links to # source code will now point to the output of htags. # The default value is: NO. # This tag requires that the tag SOURCE_BROWSER is set to YES. USE_HTAGS = NO # If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a # verbatim copy of the header file for each class for which an include is # specified. Set to NO to disable this. # See also: Section \class. # The default value is: YES. VERBATIM_HEADERS = YES #--------------------------------------------------------------------------- # Configuration options related to the alphabetical class index #--------------------------------------------------------------------------- # If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all # compounds will be generated. Enable this if the project contains a lot of # classes, structs, unions or interfaces. # The default value is: YES. ALPHABETICAL_INDEX = YES # The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in # which the alphabetical index list will be split. # Minimum value: 1, maximum value: 20, default value: 5. # This tag requires that the tag ALPHABETICAL_INDEX is set to YES. COLS_IN_ALPHA_INDEX = 5 # In case all classes in a project start with a common prefix, all classes will # be put under the same header in the alphabetical index. The IGNORE_PREFIX tag # can be used to specify a prefix (or a list of prefixes) that should be ignored # while generating the index headers. # This tag requires that the tag ALPHABETICAL_INDEX is set to YES. IGNORE_PREFIX = #--------------------------------------------------------------------------- # Configuration options related to the HTML output #--------------------------------------------------------------------------- # If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output # The default value is: YES. GENERATE_HTML = YES # The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of # it. # The default directory is: html. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_OUTPUT = html # The HTML_FILE_EXTENSION tag can be used to specify the file extension for each # generated HTML page (for example: .htm, .php, .asp). # The default value is: .html. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_FILE_EXTENSION = .html # The HTML_HEADER tag can be used to specify a user-defined HTML header file for # each generated HTML page. If the tag is left blank doxygen will generate a # standard header. # # To get valid HTML the header file that includes any scripts and style sheets # that doxygen needs, which is dependent on the configuration options used (e.g. # the setting GENERATE_TREEVIEW). It is highly recommended to start with a # default header using # doxygen -w html new_header.html new_footer.html new_stylesheet.css # YourConfigFile # and then modify the file new_header.html. See also section "Doxygen usage" # for information on how to generate the default header that doxygen normally # uses. # Note: The header is subject to change so you typically have to regenerate the # default header when upgrading to a newer version of doxygen. For a description # of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_HEADER = # The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each # generated HTML page. If the tag is left blank doxygen will generate a standard # footer. See HTML_HEADER for more information on how to generate a default # footer and what special commands can be used inside the footer. See also # section "Doxygen usage" for information on how to generate the default footer # that doxygen normally uses. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_FOOTER = # The HTML_STYLESHEET tag can be used to specify a user-defined cascading style # sheet that is used by each HTML page. It can be used to fine-tune the look of # the HTML output. If left blank doxygen will generate a default style sheet. # See also section "Doxygen usage" for information on how to generate the style # sheet that doxygen normally uses. # Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as # it is more robust and this tag (HTML_STYLESHEET) will in the future become # obsolete. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_STYLESHEET = # The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined # cascading style sheets that are included after the standard style sheets # created by doxygen. Using this option one can overrule certain style aspects. # This is preferred over using HTML_STYLESHEET since it does not replace the # standard style sheet and is therefore more robust against future updates. # Doxygen will copy the style sheet files to the output directory. # Note: The order of the extra style sheet files is of importance (e.g. the last # style sheet in the list overrules the setting of the previous ones in the # list). For an example see the documentation. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_EXTRA_STYLESHEET = # The HTML_EXTRA_FILES tag can be used to specify one or more extra images or # other source files which should be copied to the HTML output directory. Note # that these files will be copied to the base HTML output directory. Use the # $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these # files. In the HTML_STYLESHEET file, use the file name only. Also note that the # files will be copied as-is; there are no commands or markers available. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_EXTRA_FILES = # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the style sheet and background images according to # this color. Hue is specified as an angle on a colorwheel, see # https://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. # Minimum value: 0, maximum value: 359, default value: 220. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_COLORSTYLE_HUE = 220 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors # in the HTML output. For a value of 0 the output will use grayscales only. A # value of 255 will produce the most vivid colors. # Minimum value: 0, maximum value: 255, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_COLORSTYLE_SAT = 100 # The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the # luminance component of the colors in the HTML output. Values below 100 # gradually make the output lighter, whereas values above 100 make the output # darker. The value divided by 100 is the actual gamma applied, so 80 represents # a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not # change the gamma. # Minimum value: 40, maximum value: 240, default value: 80. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_COLORSTYLE_GAMMA = 80 # If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML # page will contain the date and time when the page was generated. Setting this # to YES can help to show when doxygen was last run and thus if the # documentation is up to date. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_TIMESTAMP = NO # If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML # documentation will contain a main index with vertical navigation menus that # are dynamically created via Javascript. If disabled, the navigation index will # consists of multiple levels of tabs that are statically embedded in every HTML # page. Disable this option to support browsers that do not have Javascript, # like the Qt help browser. # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_DYNAMIC_MENUS = YES # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML # documentation will contain sections that can be hidden and shown after the # page has loaded. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_DYNAMIC_SECTIONS = NO # With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries # shown in the various tree structured indices initially; the user can expand # and collapse entries dynamically later on. Doxygen will expand the tree to # such a level that at most the specified number of entries are visible (unless # a fully collapsed tree already exceeds this amount). So setting the number of # entries 1 will produce a full collapsed tree by default. 0 is a special value # representing an infinite number of entries and will result in a full expanded # tree by default. # Minimum value: 0, maximum value: 9999, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_INDEX_NUM_ENTRIES = 100 # If the GENERATE_DOCSET tag is set to YES, additional index files will be # generated that can be used as input for Apple's Xcode 3 integrated development # environment (see: https://developer.apple.com/xcode/), introduced with OSX # 10.5 (Leopard). To create a documentation set, doxygen will generate a # Makefile in the HTML output directory. Running make will produce the docset in # that directory and running make install will install the docset in # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at # startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy # genXcode/_index.html for more information. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_DOCSET = NO # This tag determines the name of the docset feed. A documentation feed provides # an umbrella under which multiple documentation sets from a single provider # (such as a company or product suite) can be grouped. # The default value is: Doxygen generated docs. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_FEEDNAME = "Doxygen generated docs" # This tag specifies a string that should uniquely identify the documentation # set bundle. This should be a reverse domain-name style string, e.g. # com.mycompany.MyDocSet. Doxygen will append .docset to the name. # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_BUNDLE_ID = org.doxygen.Project # The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify # the documentation publisher. This should be a reverse domain-name style # string, e.g. com.mycompany.MyDocSet.documentation. # The default value is: org.doxygen.Publisher. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_PUBLISHER_ID = org.doxygen.Publisher # The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. # The default value is: Publisher. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop # (see: https://www.microsoft.com/en-us/download/details.aspx?id=21138) on # Windows. # # The HTML Help Workshop contains a compiler that can convert all HTML output # generated by doxygen into a single compiled HTML file (.chm). Compiled HTML # files are now used as the Windows 98 help format, and will replace the old # Windows help format (.hlp) on all Windows platforms in the future. Compressed # HTML files also contain an index, a table of contents, and you can search for # words in the documentation. The HTML workshop also contains a viewer for # compressed HTML files. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_HTMLHELP = NO # The CHM_FILE tag can be used to specify the file name of the resulting .chm # file. You can add a path in front of the file if the result should not be # written to the html output directory. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. CHM_FILE = # The HHC_LOCATION tag can be used to specify the location (absolute path # including file name) of the HTML help compiler (hhc.exe). If non-empty, # doxygen will try to run the HTML help compiler on the generated index.hhp. # The file has to be specified with full path. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. HHC_LOCATION = # The GENERATE_CHI flag controls if a separate .chi index file is generated # (YES) or that it should be included in the master .chm file (NO). # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. GENERATE_CHI = NO # The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) # and project file content. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. CHM_INDEX_ENCODING = # The BINARY_TOC flag controls whether a binary table of contents is generated # (YES) or a normal table of contents (NO) in the .chm file. Furthermore it # enables the Previous and Next buttons. # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. BINARY_TOC = NO # The TOC_EXPAND flag can be set to YES to add extra items for group members to # the table of contents of the HTML help documentation and to the tree view. # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. TOC_EXPAND = NO # If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and # QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that # can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help # (.qch) of the generated HTML documentation. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_QHP = NO # If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify # the file name of the resulting .qch file. The path specified is relative to # the HTML output folder. # This tag requires that the tag GENERATE_QHP is set to YES. QCH_FILE = # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help # Project output. For more information please see Qt Help Project / Namespace # (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. QHP_NAMESPACE = org.doxygen.Project # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual # Folders (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual- # folders). # The default value is: doc. # This tag requires that the tag GENERATE_QHP is set to YES. QHP_VIRTUAL_FOLDER = doc # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom # filter to add. For more information please see Qt Help Project / Custom # Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- # filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_NAME = # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the # custom filter to add. For more information please see Qt Help Project / Custom # Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- # filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_ATTRS = # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this # project's filter section matches. Qt Help Project / Filter Attributes (see: # https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_SECT_FILTER_ATTRS = # The QHG_LOCATION tag can be used to specify the location of Qt's # qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the # generated .qhp file. # This tag requires that the tag GENERATE_QHP is set to YES. QHG_LOCATION = # If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be # generated, together with the HTML files, they form an Eclipse help plugin. To # install this plugin and make it available under the help contents menu in # Eclipse, the contents of the directory containing the HTML and XML files needs # to be copied into the plugins directory of eclipse. The name of the directory # within the plugins directory should be the same as the ECLIPSE_DOC_ID value. # After copying Eclipse needs to be restarted before the help appears. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_ECLIPSEHELP = NO # A unique identifier for the Eclipse help plugin. When installing the plugin # the directory name containing the HTML and XML files should also have this # name. Each documentation set should have its own identifier. # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. ECLIPSE_DOC_ID = org.doxygen.Project # If you want full control over the layout of the generated HTML pages it might # be necessary to disable the index and replace it with your own. The # DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top # of each HTML page. A value of NO enables the index and the value YES disables # it. Since the tabs in the index contain the same information as the navigation # tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. DISABLE_INDEX = NO # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index # structure should be generated to display hierarchical information. If the tag # value is set to YES, a side panel will be generated containing a tree-like # index structure (just like the one that is generated for HTML Help). For this # to work a browser that supports JavaScript, DHTML, CSS and frames is required # (i.e. any modern browser). Windows users are probably better off using the # HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can # further fine-tune the look of the index. As an example, the default style # sheet generated by doxygen has an example that shows how to put an image at # the root of the tree instead of the PROJECT_NAME. Since the tree basically has # the same information as the tab index, you could consider setting # DISABLE_INDEX to YES when enabling this option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = NO # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # doxygen will group on one line in the generated HTML documentation. # # Note that a value of 0 will completely suppress the enum values from appearing # in the overview section. # Minimum value: 0, maximum value: 20, default value: 4. # This tag requires that the tag GENERATE_HTML is set to YES. ENUM_VALUES_PER_LINE = 4 # If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used # to set the initial width (in pixels) of the frame in which the tree is shown. # Minimum value: 0, maximum value: 1500, default value: 250. # This tag requires that the tag GENERATE_HTML is set to YES. TREEVIEW_WIDTH = 250 # If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to # external symbols imported via tag files in a separate window. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. EXT_LINKS_IN_WINDOW = NO # Use this tag to change the font size of LaTeX formulas included as images in # the HTML documentation. When you change the font size after a successful # doxygen run you need to manually remove any form_*.png images from the HTML # output directory to force them to be regenerated. # Minimum value: 8, maximum value: 50, default value: 10. # This tag requires that the tag GENERATE_HTML is set to YES. FORMULA_FONTSIZE = 10 # Use the FORMULA_TRANSPARENT tag to determine whether or not the images # generated for formulas are transparent PNGs. Transparent PNGs are not # supported properly for IE 6.0, but are supported on all modern browsers. # # Note that when changing this option you need to delete any form_*.png files in # the HTML output directory before the changes have effect. # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. FORMULA_TRANSPARENT = YES # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see # https://www.mathjax.org) which uses client side Javascript for the rendering # instead of using pre-rendered bitmaps. Use this if you do not have LaTeX # installed or if you want to formulas look prettier in the HTML output. When # enabled you may also need to install MathJax separately and configure the path # to it using the MATHJAX_RELPATH option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. USE_MATHJAX = NO # When MathJax is enabled you can set the default output format to be used for # the MathJax output. See the MathJax site (see: # http://docs.mathjax.org/en/latest/output.html) for more details. # Possible values are: HTML-CSS (which is slower, but has the best # compatibility), NativeMML (i.e. MathML) and SVG. # The default value is: HTML-CSS. # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_FORMAT = HTML-CSS # When MathJax is enabled you need to specify the location relative to the HTML # output directory using the MATHJAX_RELPATH option. The destination directory # should contain the MathJax.js script. For instance, if the mathjax directory # is located at the same level as the HTML output directory, then # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of # MathJax from https://www.mathjax.org before deployment. # The default value is: https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/. # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_RELPATH = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/ # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_EXTENSIONS = # The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces # of code that will be used on startup of the MathJax code. See the MathJax site # (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an # example see the documentation. # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_CODEFILE = # When the SEARCHENGINE tag is enabled doxygen will generate a search box for # the HTML output. The underlying search engine uses javascript and DHTML and # should work on any modern browser. Note that when using HTML help # (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) # there is already a search function so this one should typically be disabled. # For large projects the javascript based search engine can be slow, then # enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to # search using the keyboard; to jump to the search box use + S # (what the is depends on the OS and browser, but it is typically # , /