.gitlab-ci.yml

stages:
  - pre_check
  - esp_branch_sync
  - upstream_sync
  - test_build
  - build
  - pack
  - sign
  - private_deploy
  - test
  - public_deploy
  - update_idf_tools

image: ${CI_DOCKER_REGISTRY}/llvm-test:1

default:
  interruptible: true

variables:
  ESP_LLVM_EMBEDDED_TOOLCHAIN_REF: "llvm22_support"
  ESP_GNU_TOOLCHAIN_VER: "15.2.0_20251204"
  AMD64_IMAGE: $CI_DOCKER_REGISTRY/llvm-build:8
  CROSS_ARM_IMAGE: $CI_DOCKER_REGISTRY/llvm-build-cross-arm:2
  CROSS_WIN_IMAGE: $CI_DOCKER_REGISTRY/llvm-build-cross-win:3
  CROSS_MACOS_IMAGE: $CI_DOCKER_REGISTRY/llvm-build-cross-macos:1
  DIST_DIR: "dist"
  BUILD_DIR: "build"
  GITLAB_SERVER_URL: https://gitlab-ci-token:${CI_JOB_TOKEN}@${GITLAB_HTTPS_SERVER}
  REBASE_PUSH_REMOTE: "https://gitlab-ci-token:${ESP_MAINT_TOKEN}@${GITLAB_HTTPS_SERVER}/${CI_PROJECT_PATH}.git"
  ESP_MAIN_SYNCED_TAG: "esp_main_synced"
  ESP_RELEASE_BRANCH: origin/$CI_DEFAULT_BRANCH
  CCACHE_DIR: ${CI_PROJECT_DIR}/.ccache
  CCACHE_MAXSIZE: 10G
  CCACHE_SLOPPINESS: "time_macros,file_macro,include_file_mtime,include_file_ctime"
  ESP_LLVM_PARALLEL_COMPILE_JOBS_FORCED: 32
  ESP_LLVM_PARALLEL_LINK_JOBS_FORCED: 4
  ESP_LLVM_PARALLEL_TABLEGEN_JOBS_FORCED: 16
  MAX_NINJA_JOBS: 32

workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_COMMIT_TAG =~ /^esp_main_synced.*/'
      when: never
    - if: $CI_COMMIT_TAG
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "rebase_on_main"'
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "esp_branch_sync"'
    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $ESP_MAIN_BRANCH'

.use_ci_tools: &use_ci_tools |
  curl -sSL ${CIT_LOADER_URL} -o cit_loader.sh && sh cit_loader.sh
  source citools/import_functions

.add_gitlab_key: &add_gitlab_key |
  cit_add_ssh_key "${GITLAB_KEY}"

.get_toolchain_build_scripts: &get_toolchain_build_scripts |
  git clone -b ${ESP_LLVM_EMBEDDED_TOOLCHAIN_REF} --single-branch ${GITLAB_SERVER_URL}/${ESP_LLVM_EMBEDDED_TOOLCHAIN_REPO_PATH}.git

.rules:scheduled-rebase-on-main:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "rebase_on_main"'
      variables:
        PUSH_RESULT: "ON"

.rules:scheduled-esp-branch-sync:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "esp_branch_sync"'

.if-scheduled-sync: &if-scheduled-sync
  if: '$CI_PIPELINE_SOURCE == "schedule" && ($SCHEDULE_TYPE == "rebase_on_main" || $SCHEDULE_TYPE == "esp_branch_sync")'

.if-esp-main-workflow: &if-esp-main-workflow
  if: '$CI_COMMIT_REF_NAME == $ESP_MAIN_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $ESP_MAIN_BRANCH'

.configure-build-jobs: &configure-build-jobs |
  ccache --zero-stats || true
  ccache --show-stats || true
  if [ -n "${CI_MERGE_REQUEST_LABELS:-}" ] && echo "${CI_MERGE_REQUEST_LABELS:-}" | grep -q "ccache::recache"; then
    export CCACHE_RECACHE="1"
    echo "INFO: ccache recache enabled"
  fi
  if [ -n "${CI_MERGE_REQUEST_LABELS:-}" ] && echo "${CI_MERGE_REQUEST_LABELS:-}" | grep -q "limit_ninja_jobs"; then
    export MAX_NINJA_JOBS="12"
    echo "INFO: limit ninja jobs to ${MAX_NINJA_JOBS}"
  fi

before_script:
  - *use_ci_tools
  - *add_gitlab_key
  # Set the displayed user with the commits that are about to be made
  - git config --global user.email "${GITLAB_USER_EMAIL}"
  - git config --global user.name "${GITLAB_USER_NAME}"

esp_release_branch_sync:
  stage: esp_branch_sync
  extends:
    - .rules:scheduled-esp-branch-sync
  tags: [ "amd64", "build" ]
  dependencies: []
  variables:
    MR_CREATE_PATH: "/api/v4/projects/${CI_PROJECT_ID}/merge_requests"
    MR_LABELS: "sync_fail"
    CI_TOKEN_HEADER: "PRIVATE-TOKEN: ${ESP_MAINT_TOKEN}"
    CONT_TYPE_HEADER: "Content-Type: application/json"
  script:
    - set -euxo pipefail
    - git remote add push-remote ${REBASE_PUSH_REMOTE}
    - git remote -v
    - BRANCH_NAME=${CI_COMMIT_REF_NAME}
    - git branch -D $BRANCH_NAME || true
    - git checkout -b $BRANCH_NAME
    - ORIG_TAG_HASH=$(git rev-parse $ESP_MAIN_SYNCED_TAG^{commit})
    - echo "Sync tag '$ESP_MAIN_SYNCED_TAG' original hash -> '$ORIG_TAG_HASH'"
    - rc=0
    - llvm/utils/git/cherry-pick-range.py --no-abort -b $ESP_RELEASE_BRANCH $BRANCH_NAME $ESP_MAIN_SYNCED_TAG || rc=$?
    - echo "rc=$rc"
    - |
      # if neither commits were picked, create WIP branch and MR to resolve conflict and merge
      if [ "$rc" -ne 0 ] && [ "$rc" -ne 2 ]; then
        WIP_BRANCH=$(cat .esp_rebase_WIP_BRANCH)
        if [ -z "$WIP_BRANCH" ]; then
          echo "WIP branch not found"
          exit 1
        fi
        echo "Push conflict resolution WIP branch '$WIP_BRANCH' to '$REBASE_PUSH_REMOTE'"
        git push push-remote $WIP_BRANCH
        git status
        CONFLICT_HASH=$(cat .esp_rebase_HEAD)
        MR_DESC="MR created by $CI_JOB_URL to fix conflict happened when cherry-picking $CONFLICT_HASH from \`$ESP_RELEASE_BRANCH\` to \`$BRANCH_NAME\`."
        MR_DESC="${MR_DESC}\n\nTODO:"
        MR_DESC="${MR_DESC}\n- [ ] Look for \`<<<<<<<\` and \`>>>>>>>\` markers in the files and solve conflicts."
        MR_DESC="${MR_DESC}\n- [ ] Ensure pipeline passes."
        MR_DESC="${MR_DESC}\n- [ ] DISABLE ALL SCHEDULED SYNC PIPELINES for \`$BRANCH_NAME\`!"
        MR_DESC="${MR_DESC}\n- [ ] Rebase if necessary and merge this MR."
        MR_DESC="${MR_DESC}\n- [ ] Move \`$ESP_MAIN_SYNCED_TAG\` tag to $CONFLICT_HASH."
        MR_DESC="${MR_DESC}\n- [ ] RE-ENABLE ALL SCHEDULED SYNC PIPELINES for \`$BRANCH_NAME\`!"
        MR_BODY='{
          "id": '"$CI_PROJECT_ID"',
          "source_branch": "'"${WIP_BRANCH}"'",
          "target_branch": "'"${BRANCH_NAME}"'",
          "title": "Draft: '"${WIP_BRANCH}"'",
          "description": "'"${MR_DESC}"'",
          "assignee_id": '"${GITLAB_USER_ID}"',
          "labels": "'"${MR_LABELS}"'",
          "remove_source_branch": true
        }'
        curl -X POST "${CI_SERVER_URL}${MR_CREATE_PATH}" --header "$CI_TOKEN_HEADER" --header "$CONT_TYPE_HEADER" --data "${MR_BODY}"
        exit 1
      fi
    # Push synced branch with updated tag atomically
    # if one cherry-pick failed, but some commits were picked (rc is 2), WIP branch will be created on the next run of this job.
    - echo "Push updated branch '$BRANCH_NAME' and sync tag '$ESP_MAIN_SYNCED_TAG' to '$REBASE_PUSH_REMOTE'"
    - git push --force --atomic push-remote $BRANCH_NAME $ESP_MAIN_SYNCED_TAG
    - git status

rebase_on_main:
  stage: upstream_sync
  extends:
    - .rules:scheduled-rebase-on-main
  tags: [ "amd64", "build" ]
  variables:
    PUSH_RESULT: "OFF"
  script:
    - set -euxo pipefail
    - GIT_DIR=$( git rev-parse --git-dir )
    - git fetch origin $SCHED_REBASE_ONTO_BR
    - git remote add push-remote ${REBASE_PUSH_REMOTE}
    - git remote -v
    - BRANCH_NAME=${CI_COMMIT_REF_NAME}
    - git branch -D $BRANCH_NAME || true
    - git checkout -b $BRANCH_NAME
    - |
      rc=0
      echo "Rebase branch '$BRANCH_NAME' on 'origin/$SCHED_REBASE_ONTO_BR'"
      if git rebase origin/$SCHED_REBASE_ONTO_BR ; then
        echo "Successfully rebased branch '$BRANCH_NAME'"
        if [[ "${PUSH_RESULT}" == "ON" ]]; then
          # --force-with-lease doesn't work on shallow clones
          echo "Push rebased branch '$BRANCH_NAME' to '$REBASE_PUSH_REMOTE'"
          git push push-remote $BRANCH_NAME --force
        else
          echo "Skip pushing rebased branch '$BRANCH_NAME'"
        fi
      else
        rc=$?
        echo "Rebasing branch '$BRANCH_NAME' on 'origin/$SCHED_REBASE_ONTO_BR' failed!"
        echo "Please rebase '$BRANCH_NAME' onto 'origin/$SCHED_REBASE_ONTO_BR' branch manually."
        if [ -d "${GIT_DIR}/rebase-merge" ] || [ -d "${GIT_DIR}/rebase-apply" ] ; then
            echo "Abort rebase"
            git rebase --abort
        fi
      fi
      git status
      exit $rc

.check_pre_commit_template:
  stage: pre_check
  tags: [ "amd64", "build" ]
  rules:
    # Relax only when the MR is explicitly marked (CI_MERGE_REQUEST_LABELS is set for MR pipelines).
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /(^|,)allow_format_checks_failure(,|$)/'
      when: on_success
      allow_failure: true
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      when: on_success
      allow_failure: false
    - when: never

check_code_format:
  extends: .check_pre_commit_template
  script:
    - python3 $PWD/llvm/utils/git/code-format-helper.py
      --start-rev $CI_MERGE_REQUEST_DIFF_BASE_SHA
      --end-rev HEAD

check_commit_message:
  extends: .check_pre_commit_template
  script:
    - python3 $PWD/llvm/utils/git/esp-commit-msg-check.py
      $CI_MERGE_REQUEST_DIFF_BASE_SHA..HEAD

.build_toolchain_template:
  tags: [ "amd64", "build", "nocaps" ]
  artifacts:
    paths:
      - ${DIST_DIR}/
      - ${BUILD_DIR}/*.log
      - ${BUILD_DIR}/CMakeCache.txt
    when: always
    expire_in: 3 days
  dependencies: []
  retry: 1
  variables:
    TARGET: "Xtensa;RISCV"
    USE_LINKER: "ld"
    CROSS_BUILD_MINGW: "OFF"
    BUILD_TARGET_LIBS: "OFF"
    BUILD_GNU_BINUTILS: "OFF"
    RUN_CORE_TESTS: "OFF"
    RUN_PKG_TESTS: "OFF"
    RUN_TOOLCHAIN_TESTS: "OFF"
    PACK_TOOLCHAIN: "ON"
    PACK_STANDALONE_LIBS: "OFF"
    PACK_TARGET_LIBS: "OFF"
    PACK_EXTRA_TOOLS: "OFF"
  rules:
    - <<: *if-esp-main-workflow
      when: never
    - <<: *if-scheduled-sync
      when: never
    - when: on_success
  cache:
    key: ccache-$CI_JOB_NAME
    paths:
      - $CCACHE_DIR
  before_script:
    - *configure-build-jobs
    - git config --global --add safe.directory $CI_PROJECT_DIR
  after_script:
    # help to identify that build failed due to OOM
    - >
      if [ $CI_JOB_STATUS == 'failed' ]; then
        [ ! -f "${BUILD_DIR}/build.log" ] || grep -i "internal compiler error\|Killed" ${BUILD_DIR}/build.log || true
        [ ! -f "${BUILD_DIR}/tests.log" ] || grep -i "internal compiler error\|Killed" ${BUILD_DIR}/tests.log || true
        [ ! -f "${BUILD_DIR}/compiler-rt-tests.log" ] || grep -i "internal compiler error\|Killed" ${BUILD_DIR}/compiler-rt-tests.log || true
        [ ! -f "${BUILD_DIR}/lld-tests.log" ] || grep -i "internal compiler error\|Killed" ${BUILD_DIR}/lld-tests.log || true
        [ ! -f "${BUILD_DIR}/pkg-tests.log" ] || grep -i "internal compiler error\|Killed" ${BUILD_DIR}/pkg-tests.log || true
      fi
    - ccache --show-stats
  script:
    - echo "MAX_NINJA_JOBS=${MAX_NINJA_JOBS}"
    - *get_toolchain_build_scripts
    - LLVM_PROJECT_PATH=$PWD
    - BUILD_PATH=$PWD/${BUILD_DIR}
    - INSTALL_PATH=$PWD/${BUILD_DIR}/_install
    - mkdir -p ${BUILD_PATH}
    - mkdir -p ${INSTALL_PATH}
    - BUILD_HOST=$(gcc -dumpmachine)
    # Config to build target libraries
    # TODO: do not build core tools (clang, lld, binutils  etc)
    # when PACK_TOOLCHAIN is OFF and PACK_TARGET_LIBS is ON.
    # Re-use core tools built in another job.
    # LLVM-xxx
    - >
      if [ "${PACK_TARGET_LIBS}" == "ON" ] || [ "${BUILD_TARGET_LIBS}" == "ON" ]; then
        echo "Enable target libraries build"
        export USE_LIBC="newlib";
        export USE_LIBCXX="libstdcxx;libcxx";
        export USE_RTLIB="compiler-rt;libgcc";
        # TODO: remove this once we do not need GNU binutils for target libraries build
        export BUILD_GNU_BINUTILS="ON";
      else
        echo "Disable target libraries"
        export USE_LIBC=none;
        export USE_LIBCXX="";
        if [ "${RUN_CORE_TESTS}" == "ON" ]; then
          # Need to have multilib dir structure to avoid test failures due to '-Wmissing-multilib'.
          # So enable copying of libgcc from GNU toolchain. It is fast and not heavy.
          export USE_RTLIB="libgcc";
        else
          export USE_RTLIB="";
        fi
      fi
    # build toolchain core w/o any libs and GNU components
    - time cmake $PWD/esp-llvm-embedded-toolchain -GNinja
      -DFETCHCONTENT_SOURCE_DIR_LLVMPROJECT=${LLVM_PROJECT_PATH}
      -DNEWLIB_REPO_URL="${GITLAB_SERVER_URL}/${NEWLIB_REPO_PATH}.git"
      -DBINUTILS_REPO_URL="${GITLAB_SERVER_URL}/${BINUTILS_REPO_PATH}.git"
      -DXTENSA_OVERLAYS_REPO_URL="${GITLAB_SERVER_URL}/${XTENSA_OVERLAYS_REPO_PATH}.git"
      -DFETCHCONTENT_QUIET=OFF
      -DLLVM_TOOLCHAIN_ESP_GNU_TOOLCHAIN_VER=${ESP_GNU_TOOLCHAIN_VER}
      -DLLVM_TOOLCHAIN_CROSS_BUILD_MINGW=${CROSS_BUILD_MINGW}
      -DLLVM_TOOLCHAIN_C_LIBRARY=${USE_LIBC}
      -DLLVM_TOOLCHAIN_CXX_LIBRARIES=${USE_LIBCXX}
      -DLLVM_TOOLCHAIN_RT_LIBRARIES=${USE_RTLIB}
      -DLLVM_TOOLCHAIN_INCLUDE_GNU_BINUTILS=${BUILD_GNU_BINUTILS}
      -DLLVM_TOOLCHAIN_ESPRESSIF=ON
      -DLLVM_TOOLCHAIN_HOST_TRIPLE=${CONF_HOST}
      -DLLVM_TOOLCHAIN_ENABLED_TARGETS="${TARGET}"
      -DLLVM_TOOLCHAIN_PACKAGE_NAME="esp-clang"
      -DLLVM_USE_LINKER=${USE_LINKER}
      -DLLVM_PARALLEL_COMPILE_JOBS=${ESP_LLVM_PARALLEL_COMPILE_JOBS_FORCED}
      -DLLVM_PARALLEL_LINK_JOBS=${ESP_LLVM_PARALLEL_LINK_JOBS_FORCED}
      -DLLVM_PARALLEL_TABLEGEN_JOBS=${ESP_LLVM_PARALLEL_TABLEGEN_JOBS_FORCED}
      -DCLANG_REPOSITORY_STRING="${GH_REPO_HTTPS}"
      -DCPACK_ARCHIVE_THREADS=0
      --install-prefix=${INSTALL_PATH}
      -B ${BUILD_PATH} > ${BUILD_PATH}/build.log 2>&1
    # Do not run unit tests for cross-builds.
    # Run as non-root user because permission tests fail when run by root.
    - >
      if [[ "${CONF_HOST}" == "${BUILD_HOST}" ]]; then
        export CUR_USER=$(whoami);
        useradd -m test_runner;
        chown -R test_runner ${BUILD_PATH};
        chown -R test_runner ${CCACHE_DIR};
        runuser -u test_runner -- git config --global --add safe.directory $CI_PROJECT_DIR;
        chown -R test_runner $PWD/esp-llvm-embedded-toolchain;
        runuser -u test_runner -- git config --global --add safe.directory $PWD/esp-llvm-embedded-toolchain;
        if [[ "${RUN_CORE_TESTS}" == "ON" ]]; then
          echo "Run LLVM/Clang unit tests";
          touch ${BUILD_PATH}/tests.log;
          chmod o+w ${BUILD_PATH}/tests.log;
          runuser -u test_runner -- ninja -C ${BUILD_PATH} -j${MAX_NINJA_JOBS} check-all > ${BUILD_PATH}/tests.log 2>&1;
        fi
        if [[ "${RUN_TOOLCHAIN_TESTS}" == "ON" ]]; then
          touch ${BUILD_PATH}/toolchain-tests.log;
          chmod o+w ${BUILD_PATH}/toolchain-tests.log;
          echo "Run toolchain unit tests";
          time runuser -u test_runner -- ninja -C ${BUILD_PATH} -j${MAX_NINJA_JOBS} check-llvm-toolchain > ${BUILD_PATH}/toolchain-tests.log 2>&1;
          echo "Toolchain unit tests finished";
          echo "Run toolchain runtimes unit tests";
          time runuser -u test_runner -- ninja -C ${BUILD_PATH} -j${MAX_NINJA_JOBS} check-llvm-toolchain-runtimes >> ${BUILD_PATH}/toolchain-tests.log 2>&1;
          echo "Toolchain runtimes unit tests finished";
        fi
        if [[ "${RUN_PKG_TESTS}" == "ON" ]]; then
          echo "Run toolchain package tests";
          # to avoid test failure "fatal: detected dubious ownership in repository at '/builds/llvm-project'"
          touch ${BUILD_PATH}/pkg-tests.log;
          chmod o+w ${BUILD_PATH}/pkg-tests.log;
          time runuser -u test_runner -- ninja -C ${BUILD_PATH} -j${MAX_NINJA_JOBS} check-package-llvm-toolchain > ${BUILD_PATH}/pkg-tests.log 2>&1;
          echo "Toolchain package tests finished";
        fi
        chown -R ${CUR_USER} $PWD/esp-llvm-embedded-toolchain;
        chown -R ${CUR_USER} ${BUILD_PATH};
        chown -R ${CUR_USER} ${CCACHE_DIR};
      fi
    # pack distro
    - mkdir -p ${PWD}/${DIST_DIR}
    - >
      if [[ "${PACK_TOOLCHAIN}" == "ON" ]]; then
        echo "Pack toolchain";
        time ninja -C ${BUILD_PATH} -j${MAX_NINJA_JOBS} package-llvm-toolchain >> ${BUILD_PATH}/build.log 2>&1
        echo "Pack toolchain finished";
        DISTRO_PACK_PATH=$(ninja -C ${BUILD_PATH} -j${MAX_NINJA_JOBS} print-llvm-toolchain-package-path | tail -n 1)
        echo "DISTRO_PACK_PATH=${DISTRO_PACK_PATH}"
        mv ${DISTRO_PACK_PATH} ${PWD}/${DIST_DIR}/
        ARCHIVE_NAME=$(basename ${DISTRO_PACK_PATH})
        echo "${ARCHIVE_NAME}" > ${PWD}/${DIST_DIR}/dist_name_${CONF_HOST}
      fi
    - >
      if [[ "${PACK_EXTRA_TOOLS}" == "ON" ]]; then
        echo "Unpack toolchain";
        time ninja -C ${BUILD_PATH} -j${MAX_NINJA_JOBS} unpack-llvm-toolchain >> ${BUILD_PATH}/build.log 2>&1
        echo "Unpack toolchain finished";
        # pack clangd
        echo "Pack clangd";
        time ninja -C ${BUILD_PATH} -j${MAX_NINJA_JOBS} package-clangd >> ${BUILD_PATH}/build.log 2>&1
        echo "Pack clangd finished";
        DISTRO_PACK_PATH=$(ninja -C ${BUILD_PATH} -j${MAX_NINJA_JOBS} print-clangd-package-path | tail -n 1)
        echo "DISTRO_PACK_PATH=${DISTRO_PACK_PATH}"
        mv ${DISTRO_PACK_PATH} ${PWD}/${DIST_DIR}/
        ARCHIVE_NAME=$(basename ${DISTRO_PACK_PATH})
        echo "${ARCHIVE_NAME}" > ${PWD}/${DIST_DIR}/clangd_dist_name_${CONF_HOST}
        # pack lld
        echo "Pack lld";
        time ninja -C ${BUILD_PATH} -j${MAX_NINJA_JOBS} package-lld >> ${BUILD_PATH}/build.log 2>&1
        echo "Pack lld finished";
        DISTRO_PACK_PATH=$(ninja -C ${BUILD_PATH} -j${MAX_NINJA_JOBS} print-lld-package-path | tail -n 1)
        echo "DISTRO_PACK_PATH=${DISTRO_PACK_PATH}"
        mv ${DISTRO_PACK_PATH} ${PWD}/${DIST_DIR}/
        ARCHIVE_NAME=$(basename ${DISTRO_PACK_PATH})
        echo "${ARCHIVE_NAME}" > ${PWD}/${DIST_DIR}/lld_dist_name_${CONF_HOST}
        # re-pack toolchain w/o clangd
        echo "Re-pack toolchain";
        time ninja -C ${BUILD_PATH} -j${MAX_NINJA_JOBS} repack-llvm-toolchain >> ${BUILD_PATH}/build.log 2>&1
        echo "Re-pack toolchain finished";
      fi
    # pack distro with standalone libs
    - >
      if [[ "${PACK_STANDALONE_LIBS}" == "ON" ]]; then
        echo "Clean install dir ${INSTALL_PATH}"
        rm -rf ${INSTALL_PATH}
        echo "Pack standalone libs";
        time ninja -C ${BUILD_PATH} -j${MAX_NINJA_JOBS} package-llvm-standalone-libs >> ${BUILD_PATH}/build.log 2>&1
        echo "Pack standalone libs finished";
        DISTRO_PACK_PATH=$(ninja -C ${BUILD_PATH} -j${MAX_NINJA_JOBS} print-llvm-standalone-libs-package-path | tail -n 1)
        echo "DISTRO_PACK_PATH=${DISTRO_PACK_PATH}"
        mv ${DISTRO_PACK_PATH} ${PWD}/${DIST_DIR}/
        ARCHIVE_NAME=$(basename ${DISTRO_PACK_PATH})
        echo "${ARCHIVE_NAME}" > ${PWD}/${DIST_DIR}/libs_dist_name_${CONF_HOST}
      fi
    # pack target libraries to be re-used in distros for other platforms
    - >
      if [[ "${PACK_TARGET_LIBS}" == "ON" ]]; then
        echo "Clean install dir ${INSTALL_PATH}"
        rm -rf ${INSTALL_PATH}
        echo "Pack target libs";
        time ninja -C ${BUILD_PATH} -j${MAX_NINJA_JOBS} package-llvm-toolchain-target-libs >> ${BUILD_PATH}/build.log 2>&1
        echo "Pack target libs finished";
        DISTRO_PACK_PATH=$(ninja -C ${BUILD_PATH} -j${MAX_NINJA_JOBS} print-llvm-toolchain-target-libs-package-path | tail -n 1)
        echo "DISTRO_PACK_PATH=${DISTRO_PACK_PATH}"
        mv ${DISTRO_PACK_PATH} ${PWD}/${DIST_DIR}/
        ARCHIVE_NAME=$(basename ${DISTRO_PACK_PATH})
        echo "${ARCHIVE_NAME}" > ${PWD}/${DIST_DIR}/target_libs_arch_name
      fi
    - ls -l ${PWD}/${DIST_DIR}

# some Clang/LLVM unit tests fail if we build Clang for RISCV+Xtensa only
# this job is intended to run unit tests only, so it builds Clang with X86, RISCV and Xtensa backends
# TODO: LLVM-326 and LLVM-401
build_and_test:
  tags: [ "amd64", "build", "nocaps" ]
  stage: test_build
  needs: []
  artifacts:
    paths:
      - ${DIST_DIR}/
      - ${BUILD_DIR}/*.log
      - ${BUILD_DIR}/CMakeCache.txt
    when: always
    expire_in: 3 days
  retry: 1
  variables:
    PACK_TOOL: "tar cJhvf"
    CACHE_KEY: ccache-$CI_JOB_NAME
  rules:
    - <<: *if-scheduled-sync
      when: never
    - if: '$CI_COMMIT_REF_NAME == $ESP_MAIN_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $ESP_MAIN_BRANCH'
      variables:
        CACHE_KEY: ccache-esp_main-$CI_JOB_NAME
    - when: on_success
  cache:
    key: $CACHE_KEY
    paths:
      - $CCACHE_DIR
  before_script:
    - *configure-build-jobs
    - git config --global --add safe.directory $CI_PROJECT_DIR
  after_script:
    # help to identify that build failed due to OOM
    - >
      if [ $CI_JOB_STATUS == 'failed' ]; then
        [ ! -f "${BUILD_DIR}/build.log" ] || grep -i "internal compiler error\|Killed" ${BUILD_DIR}/build.log || true
        [ ! -f "${BUILD_DIR}/tests.log" ] || grep -i "internal compiler error\|Killed" ${BUILD_DIR}/tests.log || true
        [ ! -f "${BUILD_DIR}/lld-tests.log" ] || grep -i "internal compiler error\|Killed" ${BUILD_DIR}/lld-tests.log || true
      fi
    - ccache --show-stats
  script:
    - echo "MAX_NINJA_JOBS=${MAX_NINJA_JOBS}"
    - BUILD_PATH=$PWD/${BUILD_DIR}
    - LLVM_PROJECT_PATH=$PWD
    - mkdir -p ${BUILD_PATH}
    - time cmake -G Ninja
      -S llvm
      -DLLVM_ENABLE_PROJECTS="clang;lld;clang-tools-extra"
      -DCMAKE_BUILD_TYPE=Release
      -DLLVM_ENABLE_ASSERTIONS=ON
      -DLLVM_APPEND_VC_REV=OFF
      -DLLDB_INCLUDE_TESTS=OFF
      -DLLVM_TARGETS_TO_BUILD="X86;RISCV"
      -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=Xtensa
      -DCMAKE_C_COMPILER_LAUNCHER=ccache
      -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
      -DLLVM_USE_LINKER=lld
      -DLLVM_PARALLEL_COMPILE_JOBS=${ESP_LLVM_PARALLEL_COMPILE_JOBS_FORCED}
      -DLLVM_PARALLEL_LINK_JOBS=${ESP_LLVM_PARALLEL_LINK_JOBS_FORCED}
      -DLLVM_PARALLEL_TABLEGEN_JOBS=${ESP_LLVM_PARALLEL_TABLEGEN_JOBS_FORCED}
      -DLLVM_FORCE_VC_REPOSITORY="${GH_REPO_HTTPS}"
      -B ${BUILD_PATH} > ${BUILD_PATH}/build.log 2>&1
    - export CUR_USER=$(whoami);
    - useradd -m test_runner;
    - chown -R test_runner ${BUILD_PATH};
    - chown -R test_runner ${CCACHE_DIR};
    - runuser -u test_runner -- git config --global --add safe.directory $CI_PROJECT_DIR
    - |
      # Pick which lit suites to run from the files this MR touches:
      #  - pure backend / backend-test changes -> only the RISCV/Xtensa suites;
      #  - cross-cutting backend files (ISA-info, clang target/driver) also run
      #    clang driver+preprocessor and the llvm unit tests;
      #  - anything else, an unavailable diff base, a non-MR pipeline, or the
      #    'test::all' MR label -> full check-all.
      TEST_TARGETS="check-all"
      if [ "${CI_PIPELINE_SOURCE:-}" = "merge_request_event" ] \
         && ! echo "${CI_MERGE_REQUEST_LABELS:-}" | grep -q "test::all" \
         && [ -n "${CI_MERGE_REQUEST_DIFF_BASE_SHA:-}" ] \
         && git cat-file -e "${CI_MERGE_REQUEST_DIFF_BASE_SHA}^{commit}" 2>/dev/null; then
        CHANGED=$(git diff --name-only "${CI_MERGE_REQUEST_DIFF_BASE_SHA}" HEAD)
        echo "Changed files in this MR:"; echo "${CHANGED}"
        # .gitlab-ci.yml is not test-bearing: ignore it so it neither forces
        # check-all on its own nor vetoes narrowing when bundled with backend
        # changes. A CI-only change leaves CHANGED empty -> check-all default.
        CHANGED=$(printf '%s\n' "${CHANGED}" | grep -vxF '.gitlab-ci.yml' || true)
        RISCV_LOCAL='^(llvm/lib/Target/RISCV/|llvm/test/(CodeGen|MC|MC/Disassembler)/RISCV/|clang/test/CodeGen/RISCV/)'
        XTENSA_LOCAL='^(llvm/lib/Target/Xtensa/|llvm/test/(CodeGen|MC|MC/Disassembler)/Xtensa/)'
        RISCV_CROSS='^(llvm/(lib|include/llvm)/TargetParser/RISCV|clang/lib/Basic/Targets/RISCV|clang/lib/Driver/ToolChains/Arch/RISCV|clang/test/(Driver|Preprocessor)/.*riscv)'
        XTENSA_CROSS='^(clang/lib/Basic/Targets/Xtensa|clang/lib/Driver/ToolChains/.*[Xx]tensa)'
        LLD='^lld/'
        KNOWN="${RISCV_LOCAL}|${XTENSA_LOCAL}|${RISCV_CROSS}|${XTENSA_CROSS}|${LLD}"
        OUTSIDE=$(echo "${CHANGED}" | grep -vE "${KNOWN}" || true)
        if [ -n "${CHANGED}" ] && [ -z "${OUTSIDE}" ]; then
          T=""
          echo "${CHANGED}" | grep -qE "${RISCV_LOCAL}|${RISCV_CROSS}"   && T="${T} check-llvm-codegen-riscv check-llvm-mc-riscv check-llvm-mc-disassembler-riscv check-clang-codegen-riscv"
          echo "${CHANGED}" | grep -qE "${XTENSA_LOCAL}|${XTENSA_CROSS}" && T="${T} check-llvm-codegen-xtensa check-llvm-mc-xtensa check-llvm-mc-disassembler-xtensa"
          echo "${CHANGED}" | grep -qE "${RISCV_CROSS}|${XTENSA_CROSS}"  && T="${T} check-clang-driver check-clang-preprocessor check-llvm-unit"
          echo "${CHANGED}" | grep -qE "${LLD}"                          && T="${T} check-lld"
          T=$(echo ${T} | tr ' ' '\n' | grep -v '^$' | sort -u | tr '\n' ' ')
          [ -n "${T}" ] && TEST_TARGETS="${T}"
        else
          echo "Changes fall outside the RISCV/Xtensa allowlist; running check-all"
        fi
      fi
      echo "Selected test targets: ${TEST_TARGETS}"
    - echo "Run LLVM/Clang unit tests";
    - touch ${BUILD_PATH}/tests.log;
    - chmod o+w ${BUILD_PATH}/tests.log;
    - time runuser -u test_runner -- ninja -C ${BUILD_PATH} -j${MAX_NINJA_JOBS} ${TEST_TARGETS} > ${BUILD_PATH}/tests.log 2>&1;
    - echo "LLVM/Clang unit tests finished";
    - chown -R ${CUR_USER} ${BUILD_PATH};
    - chown -R ${CUR_USER} ${CCACHE_DIR};
    - mkdir -p ${PWD}/${DIST_DIR}
    - PIE_TEST_TOOLS_ARCH=${PWD}/${DIST_DIR}/pie_test_tools.tar.xz
    - pushd ${BUILD_PATH}
    - echo "Packing tools for PIE optimization tests"
    - time ${PACK_TOOL} ${PIE_TEST_TOOLS_ARCH} bin/clang bin/llc bin/llvm-objdump bin/opt lib/clang
    - echo "Packing tools for PIE optimization tests finished";
    - popd
    - ls -l ${PWD}/${DIST_DIR}

.build_linux-gnu_template:
  extends: .build_toolchain_template
  variables:
    # we'd use LLD, but GCC in cross-build ARM docker images has problems with -fuse-ld=lld option
    # So use gold linker instead
    USE_LINKER: "gold"

# Actually this job builds and packs distro for x86_64-linux-gnu,
# but also it runs package tests. So keep it in 'test_build' stage
build_x86_64-linux-gnu:
  extends: .build_linux-gnu_template
  stage: test_build
  image: ${AMD64_IMAGE}
  needs: []
  variables:
    CONF_HOST: "x86_64-linux-gnu"
    USE_LINKER: "lld"
  rules:
    - <<: *if-esp-main-workflow
      when: never
    - <<: *if-scheduled-sync
      when: never
    # Build complete distro + archive with target libs and run all tests
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_LABELS =~ /(^|,)toolchain_tests(,|$)/'
      variables:
        BUILD_TARGET_LIBS: "ON"
        RUN_TOOLCHAIN_TESTS: "ON"
        RUN_PKG_TESTS: "ON"
    # Build all distros and run all tests
    - if: '$CI_COMMIT_TAG'
      variables:
        BUILD_TARGET_LIBS: "ON"
        RUN_TOOLCHAIN_TESTS: "ON"
        RUN_PKG_TESTS: "ON"
        PACK_STANDALONE_LIBS: "ON"
        PACK_EXTRA_TOOLS: "ON"
    - when: on_success

# Target libs are built in build_x86_64-linux-gnu, but due to artifacts
# size limit we have to produce target libs archive in this job too.
# Archive with target libs from this job will be used in pack jobs
# for all platform except for x86_64-linux-gnu.
build_target_libs:
  extends: .build_linux-gnu_template
  stage: build
  image: ${AMD64_IMAGE}
  variables:
    CONF_HOST: "x86_64-linux-gnu"
    PACK_TARGET_LIBS: "ON"
    PACK_TOOLCHAIN: "OFF"
  rules:
    - <<: *if-esp-main-workflow
      when: never
    - <<: *if-scheduled-sync
      when: never
    # Build target libs only on default branch or tag
    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
      when: on_success

build_x86_64-w64-mingw32:
  extends: .build_toolchain_template
  stage: build
  image: ${CROSS_WIN_IMAGE}
  variables:
    USE_LINKER: ""
    CONF_HOST: "x86_64-w64-mingw32"
    CROSS_BUILD_MINGW: "ON"
  rules:
    - <<: *if-esp-main-workflow
      when: never
    - <<: *if-scheduled-sync
      when: never
    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
      variables:
        BUILD_GNU_BINUTILS: "ON"
        PACK_STANDALONE_LIBS: "ON"
        PACK_EXTRA_TOOLS: "ON"
    - when: on_success

build_arm-linux-gnueabihf:
  extends: .build_linux-gnu_template
  stage: build
  image: ${CROSS_ARM_IMAGE}
  variables:
    CONF_HOST: "arm-linux-gnueabihf"
  rules:
    - <<: *if-esp-main-workflow
      when: never
    - <<: *if-scheduled-sync
      when: never
    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
      variables:
        BUILD_GNU_BINUTILS: "ON"
        PACK_STANDALONE_LIBS: "ON"
        PACK_EXTRA_TOOLS: "ON"
    - when: on_success

build_aarch64-linux-gnu:
  extends: .build_linux-gnu_template
  stage: build
  image: ${CROSS_ARM_IMAGE}
  variables:
    CONF_HOST: "aarch64-linux-gnu"
  rules:
    - <<: *if-esp-main-workflow
      when: never
    - <<: *if-scheduled-sync
      when: never
    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
      variables:
        BUILD_GNU_BINUTILS: "ON"
        PACK_STANDALONE_LIBS: "ON"
        PACK_EXTRA_TOOLS: "ON"
    - when: on_success

build_x86_64-apple-darwin:
  extends: .build_toolchain_template
  stage: build
  image: ${CROSS_MACOS_IMAGE}
  variables:
    CONF_HOST: "x86_64-apple-darwin21.1"
  rules:
    - <<: *if-esp-main-workflow
      when: never
    - <<: *if-scheduled-sync
      when: never
    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
      variables:
        BUILD_GNU_BINUTILS: "ON"
        PACK_STANDALONE_LIBS: "ON"
        PACK_EXTRA_TOOLS: "ON"
    - when: on_success

build_aarch64-apple-darwin:
  extends: .build_toolchain_template
  stage: build
  image: ${CROSS_MACOS_IMAGE}
  variables:
    CONF_HOST: "aarch64-apple-darwin21.1"
  rules:
    - <<: *if-esp-main-workflow
      when: never
    - <<: *if-scheduled-sync
      when: never
    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
      variables:
        BUILD_GNU_BINUTILS: "ON"
        PACK_STANDALONE_LIBS: "ON"
        PACK_EXTRA_TOOLS: "ON"
    - when: on_success

.unpack_distro: &unpack_distro |
    pushd ${DIST_DIR}
    ls -l
    DISTRO_PACK_FILE=$(cat dist_name_${CONF_HOST})
    echo "DISTRO_PACK_FILE=${DISTRO_PACK_FILE}"
    ${UNPACK_TOOL} ${DISTRO_PACK_FILE}
    DISTRO_PACK_DIR=$(tar tJf ${DISTRO_PACK_FILE} | sed -e 's@/.*@@' | uniq)
    ls -l $PWD/${DISTRO_PACK_DIR}/lib/clang-runtimes/
    echo "DISTRO_PACK_DIR=${DISTRO_PACK_DIR}"
    rm -f ${DISTRO_PACK_FILE}

.pack_template:
  stage: pack
  tags: [ "amd64", "build" ]
  artifacts:
    paths:
      - ${DIST_DIR}/
    when: always
    expire_in: 3 day
  variables:
    PACK_TOOL: "tar cJf"
    UNPACK_TOOL: "tar xJf"
  rules:
    - <<: *if-esp-main-workflow
      when: never
    - <<: *if-scheduled-sync
      when: never
    # Pack distros only on default branch or tag
    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
      when: on_success
  script:
    # update distro
    - *unpack_distro
    - TARGET_LIBS_PACK_FILE=$(cat target_libs_arch_name)
    - rm -f target_libs_arch_name
    - echo "TARGET_LIBS_PACK_FILE=${TARGET_LIBS_PACK_FILE}"
    - tar xJfv ${TARGET_LIBS_PACK_FILE}
    - rm -f ${TARGET_LIBS_PACK_FILE}
    - ls -l $PWD
    - ls -l $PWD/${DISTRO_PACK_DIR}
    - ls -l $PWD/${DISTRO_PACK_DIR}/lib/clang-runtimes/
    # both distro and target libs archives have the same root dir name,
    # so that dir contains everything we need to re-pack after unpacking steps above
    - ${PACK_TOOL} ${DISTRO_PACK_FILE} ${DISTRO_PACK_DIR}
    - rm -rf ${DISTRO_PACK_DIR}
    - ls -l

pack_x86_64-w64-mingw32:
  extends: .pack_template
  needs:
    # needs target libs archive from native build job
    - job: "build_target_libs"
    - job: "build_x86_64-w64-mingw32"
  variables:
    CONF_HOST: "x86_64-w64-mingw32"

pack_arm-linux-gnueabihf:
  extends: .pack_template
  needs:
    # needs target libs archive from native build job
    - job: "build_target_libs"
    - job: "build_arm-linux-gnueabihf"
  variables:
    CONF_HOST: "arm-linux-gnueabihf"

pack_aarch64-linux-gnu:
  extends: .pack_template
  needs:
    # needs target libs archive from native build job
    - job: "build_target_libs"
    - job: "build_aarch64-linux-gnu"
  variables:
    CONF_HOST: "aarch64-linux-gnu"

pack_x86_64-apple-darwin:
  extends: .pack_template
  needs:
    # needs target libs archive from native build job
    - job: "build_target_libs"
    - job: "build_x86_64-apple-darwin"
  variables:
    CONF_HOST: "x86_64-apple-darwin21.1"

pack_aarch64-apple-darwin:
  extends: .pack_template
  needs:
    # needs target libs archive from native build job
    - job: "build_target_libs"
    - job: "build_aarch64-apple-darwin"
  variables:
    CONF_HOST: "aarch64-apple-darwin21.1"

.macos_codesign_template:
  stage: sign
  rules:
    - <<: *if-esp-main-workflow
      when: never
    - <<: *if-scheduled-sync
      when: never
    - if: $CI_COMMIT_TAG
      when: on_success
      allow_failure: false
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
      when: manual
      allow_failure: true
  resource_group: macos_codesign
  tags: [ "darwin", "codesign" ]
  artifacts:
    paths:
      - ${DIST_ART_DIR}
  variables:
    # directory with distro archives
    DIST_ART_DIR: ${DIST_DIR}
    # command to archive distro
    ARCHIVE_TOOL: "tar cJf"
    # command to unarchive distro
    UNARCHIVE_TOOL: "tar xJf"
    # URL to macos codesign repo
    NOTARIZATION_SCRIPTS_GIT: "${CI_SERVER_PROTOCOL}://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/espressif/macos_codesign_notarization.git"
  script:
    - git clone -q --depth=1 ${NOTARIZATION_SCRIPTS_GIT} -b ${CI_COMMIT_REF_NAME} ||
      git clone -q --depth=1 ${NOTARIZATION_SCRIPTS_GIT}
    - ./macos_codesign_notarization/run.sh

sign_x86_64-apple-darwin:
  extends: 
    - .macos_codesign_template
  needs:
    - pack_x86_64-apple-darwin

sign_aarch64-apple-darwin:
  extends: 
    - .macos_codesign_template
  needs:
    - pack_aarch64-apple-darwin

.prepare_test_app_build: &prepare_test_app_build |
  pushd ${IDF_PATH}
  # Use custom idf in case custom branch is present
  if [ -n "${TEST_APP_IDF_CUSTOM_BRANCH:-}" ]; then
    echo "TEST_APP_IDF_CUSTOM_BRANCH=$TEST_APP_IDF_CUSTOM_BRANCH"
    git remote add gitlab https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/espressif/esp-idf.git
    git fetch gitlab ${TEST_APP_IDF_CUSTOM_BRANCH}:${TEST_APP_IDF_CUSTOM_BRANCH}
    git checkout ${TEST_APP_IDF_CUSTOM_BRANCH}
  fi
  if [ -n "${TEST_APP_SUBMODULE_URL:-}" ]; then
    git submodule add ${TEST_APP_SUBMODULE_URL} ${TEST_APP_SUBMODULE_PATH}
  fi
  git submodule update --init --recursive
  idf.py --version || true
  if [ -n "${TEST_APP_PATH:-}" ]; then
    pushd ${IDF_PATH}/$TEST_APP_PATH
  fi

.test_build_idf_app_template:
  stage: test
  image: espressif/idf:latest
  tags: [ "amd64", "build" ]
  artifacts:
    paths:
      - ${BUILD_DIR}/*.log
    when: always
    expire_in: 1 day
  parallel:
    matrix:
      - CHIP: [esp32p4, esp32p4eco4]
  needs:
    - job: "build_x86_64-linux-gnu"
  variables:
    # use IDF 'master' from docker image
    TEST_APP_IDF_CUSTOM_BRANCH: ""
    TEST_APP_PATH: ""
    TEST_APP_SUBMODULE_URL: ""
    TEST_APP_SUBMODULE_PATH: ""
    CONF_HOST: "x86_64-linux-gnu"
    UNPACK_TOOL: "tar xJf"
  rules:
    - <<: *if-esp-main-workflow
      when: never
    - <<: *if-scheduled-sync
      when: never
    # job "build_x86_64-linux-gnu" produces full distro only on default branch or tag or when toolchain_tests label is present
    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_LABELS =~ /(^|,)toolchain_tests(,|$)/'
      when: on_success
  script:
    - mkdir -p $PWD/${BUILD_DIR}
    - export BUILD_LOG=$PWD/${BUILD_DIR}/build.log
    - *unpack_distro
    - export PATH=$PWD/${DISTRO_PACK_DIR}/bin:${PATH}
    - which clang
    - *prepare_test_app_build
    - export IDF_TOOLCHAIN=clang
    - >
      if [ "${CHIP}" == "esp32p4eco4" ]; then
        P4ECO4=YES
        CHIP=esp32p4
      else
        P4ECO4=NO
      fi
    # Enable ESPV target lowering for test IDF builds (opt-in per code review !296)
    - echo "set(extra_flags -Xclang=-target-feature -Xclang=+espv-lowering)" >> CMakeLists.txt
    - echo 'idf_build_set_property(COMPILE_OPTIONS "${extra_flags}" APPEND)' >> CMakeLists.txt
    - idf.py set-target ${CHIP} 2>&1 | tee ${BUILD_LOG}
    - >
      if [ "${P4ECO4}" == "YES" ]; then
        echo "CONFIG_ESP32P4_SELECTS_REV_LESS_V3=y" >> sdkconfig
      fi
    - idf.py build 2>&1 | tee -a ${BUILD_LOG}

test_build_idf_hello_world:
  extends:
    - .test_build_idf_app_template
  variables:
    TEST_APP_PATH: examples/get-started/hello_world

test_build_esp_dsp:
  extends:
    - .test_build_idf_app_template
  allow_failure: true
  variables:
    TEST_APP_PATH: components/esp-dsp/test_app
    TEST_APP_SUBMODULE_URL: https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/idf/esp-dsp.git
    TEST_APP_SUBMODULE_PATH: components/esp-dsp

test_pie_optimizations:
  tags: [ "amd64", "build" ]
  stage: test
  needs:
    - job: "build_and_test"
  artifacts:
    paths:
      - esp-dsp/scripts/
    when: always
    expire_in: 1 day
  variables:
    TEST_APP_ESP_DSP_CUSTOM_BRANCH: "llvm-23-optimization-test-v3"
    UNPACK_TOOL: "tar xJvf"
  rules:
    - <<: *if-scheduled-sync
      when: never
    - when: on_success
  allow_failure: true
  script:
    - export LLVM_PROJECT_PATH=$PWD
    - export ESP_LLVM_BIN_PREPEND_DISABLED=1
    - export DSP_ASM_COMPARE_MODE=relaxed
    - export DSP_ASM_COMPARE_SHOW_SIMILARITY=1
    - ls -l ${PWD}/${DIST_DIR}
    - PIE_TEST_TOOLS_ARCH=${PWD}/${DIST_DIR}/pie_test_tools.tar.xz
    - echo "Unpacking tools for PIE optimization tests"
    - mkdir pie_tests
    - pushd pie_tests
    - ${UNPACK_TOOL} ${PIE_TEST_TOOLS_ARCH}
    - ls -l $PWD
    - export PATH=${PWD}/bin:${PATH}
    - popd
    - git clone --shallow-submodules --recursive --single-branch --branch $TEST_APP_ESP_DSP_CUSTOM_BRANCH -- ${GITLAB_SERVER_URL}/idf/esp-dsp.git esp-dsp
    - pushd $PWD/esp-dsp/
    - echo ${LLVM_PROJECT_PATH}/llvm/utils/update_test_checks.py
    - ./test_all.sh ${LLVM_PROJECT_PATH}/llvm/utils/update_test_checks.py

.test_xesppie_template:
  stage: test
  needs:
    - job: "build_x86_64-linux-gnu"
  allow_failure: false
  variables:
    # Extra arguments for the esp-compiler-tests clone (e.g. branch selection).
    ESP_COMPILER_TESTS_CLONE_ARGS: ""
    CONF_TARGET: "riscv32-esp-elf"
  script:
    - cd ${DIST_DIR}/
    - ls -l
    - DISTRO_PACK_FILE=$(cat dist_name_x86_64-linux-gnu)
    - tar -xf ${DISTRO_PACK_FILE}
    - ls -l
    - cd esp-clang
    - ls -l
    - pwd
    - export CC="$(pwd)/bin/clang"
    - export OBJDUMP="$(pwd)/bin/llvm-objdump"
    - git clone -q --depth=1 ${ESP_COMPILER_TESTS_CLONE_ARGS} "${GITLAB_SERVER_URL}/idf/esp-compiler-tests.git"
    - cd esp-compiler-tests/build-only/xesppie
    # Install deps for the interpreter test_xesppie.py actually runs under,
    # in case plain `pip` in run.sh maps to a different one.
    - python3 -m pip install -r requirements.txt
    - ./run.sh

test_xesppie:
  extends: .test_xesppie_template
  only:
    - tags

# Run the xesppie build-only tests whenever the files defining the
# xespv2p1/xespv2p2 or esploop instruction encodings change: the .td
# definitions plus the assembler parser and MC code emitter (tag pipelines
# are already covered by test_xesppie above).
test_xesppie_td_changes:
  extends: .test_xesppie_template
  variables:
    ESP_COMPILER_TESTS_CLONE_ARGS: "--single-branch --branch master"
  rules:
    - if: '$CI_COMMIT_TAG'
      when: never
    - changes:
        - llvm/lib/Target/RISCV/RISCVInstrInfoESPV.td
        - llvm/lib/Target/RISCV/RISCVInstrInfoESPLOOP.td
        - llvm/lib/Target/RISCV/RISCVInstrFormatsESP.td
        - llvm/lib/Target/RISCV/RISCVESPOperands.td
        - llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
        - llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp

upload_to_http:
  stage: private_deploy
  allow_failure: true
  tags: [ "deploy", "shiny" ]
  variables:
    # force the fetch strategy to clean old archives up in dist/ dir
    GIT_STRATEGY: fetch
  rules:
    - <<: *if-esp-main-workflow
      when: never
    - <<: *if-scheduled-sync
      when: never
    - when: manual
  needs:
    - job: build_x86_64-linux-gnu
  script:
    - cit_add_ssh_key "${HTTP_UPLOAD_KEY}"
    # List of archives
    - FILES=$(find ${DIST_DIR} -name dist_name_\* -exec cat {} \+)
    - cd ${DIST_DIR}
    - ls -l $FILES
    - scp ${FILES} ${HTTP_UPLOAD_DIR}/ct-ng/llvm-builds
    # Show info
    - echo -e "\nArchives were published there:\n\n$(for n in ${FILES}; do echo "${HTTP_PUBLIC_DIR}/ct-ng/llvm-builds/${n}"; done)\n"

.gen_checksum_file: &gen_checksum_file |
  ls -l ${FILES}
  for n in ${FILES}; do
    sz=$(stat -c%s "${n}") >> ${SHA256_FILE};
    printf "# %s: %s bytes\n" "${n}" "${sz}" >> ${SHA256_FILE};
    sha256sum -b "${n}" >> ${SHA256_FILE};
  done

upload_to_github:
  stage: public_deploy
  when: manual
  allow_failure: true
  only:
    - tags
  tags: [ "amd64", "internet" ]
  image: espressif/github-hub:2
  variables:
    GIT_STRATEGY: fetch
    GITHUB_TOKEN: "${GH_TOKEN}"
    GITHUB_REPO: "${GH_REPO_HTTPS}"
    TAG: "${CI_COMMIT_TAG}"
  needs:
    - job: build_x86_64-linux-gnu
    - job: pack_arm-linux-gnueabihf
    - job: pack_aarch64-linux-gnu
    - job: pack_x86_64-w64-mingw32
    - job: sign_x86_64-apple-darwin
    - job: sign_aarch64-apple-darwin
  before_script: []
  script:
    - ls -l ${DIST_DIR}
    - cd ${DIST_DIR}
    - git remote add github ${GH_REPO_HTTPS}
    - hub release show ${TAG} || { echo "Please create a release on GitHub with ${TAG} tag at first"; exit 1; }
    # Generate checksum file for toolchain
    # List of clang archives
    - FILES=$(find ${PWD} -name dist_name_\* -exec cat {} \+)
    - SHA256_FILE=clang-${CI_COMMIT_TAG}-checksum.sha256
    - *gen_checksum_file
    - DIST_FILES=${FILES}
    # Generate checksum file and add standalone libraries to the list
    - FILES=$(find ${PWD} -name libs_dist_name_\* -exec cat {} \+)
    - *gen_checksum_file
    - DIST_FILES=$(echo -e "${DIST_FILES}\n${FILES}")
    # Generate checksum file and add clangd archives to the list
    - FILES=$(find ${PWD} -name clangd_dist_name_\* -exec cat {} \+)
    - *gen_checksum_file
    - DIST_FILES=$(echo -e "${DIST_FILES}\n${FILES}")
    # Generate checksum file and add lld archives to the list
    - FILES=$(find ${PWD} -name lld_dist_name_\* -exec cat {} \+)
    - *gen_checksum_file
    - DIST_FILES=$(echo -e "${DIST_FILES}\n${FILES}")
    # Add checksum file to the list
    - DIST_FILES=$(echo -e "${DIST_FILES}\n${SHA256_FILE}")
    - ls -l ${DIST_FILES}
    # Upload archives
    - for n in ${DIST_FILES}; do hub release edit -m "" -a "${n}" "${TAG}"; done

update_idf_tools:
  stage: update_idf_tools
  when: manual
  allow_failure: true
  only:
    - tags
  variables:
    TOOL_NAME: esp-clang
    TOOL_MEMBERS: "esp-clang esp-clang-libs esp-clangd esp-lld"
    TOOL_VERSION: ${CI_COMMIT_TAG}
    TOOL_SHA256_URL: https://github.com/espressif/llvm-project/releases/download/${CI_COMMIT_TAG}/clang-${CI_COMMIT_TAG}-checksum.sha256
    esp_clang_EXTRA_OPTS: "--dist-filename-prefix clang-esp"
    esp_clang_libs_EXTRA_OPTS: "--dist-filename-prefix libs-clang-esp"
    esp_clangd_EXTRA_OPTS: "--dist-filename-prefix clangd-esp"
    esp_lld_EXTRA_OPTS: "--dist-filename-prefix lld-esp"
    RN_SECTION: Toolchain
  trigger:
    project: idf/idf-tools-updater
    strategy: depend