Merge branch 'espressif:release/v5.4' into release/v5.4

Author: Jason2866

.gitlab/ci/common.yml

@@ -56,7 +56,7 @@ variables:
 
   # Docker images
   ESP_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-env-v5.4:2"
-  ESP_IDF_DOC_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-doc-env-v5.4:1-1"
+  ESP_IDF_DOC_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-doc-env-v5.4:2-1"
   TARGET_TEST_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/target-test-env-v5.4:2"
   SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"
   PRE_COMMIT_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-pre-commit:1"
@@ -191,13 +191,21 @@ variables:
   fi
 
   # Custom OpenOCD
-  if [[ ! -z "$OOCD_DISTRO_URL" && "$CI_JOB_STAGE" == "target_test" ]]; then
-    echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
-    wget $OOCD_DISTRO_URL
-    ARCH_NAME=$(basename $OOCD_DISTRO_URL)
-    tar -x -f $ARCH_NAME
-    export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
-    export PATH=$PWD/openocd-esp32/bin:$PATH
+  if [[ "$CI_JOB_STAGE" == "target_test" ]]; then
+    machine="$(uname -m)"
+    if [[ "$machine" == "armv7l" ]] ; then
+      OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARMHF"
+    elif [[ "$machine" == "aarch64" ]] ; then
+      OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARM64"
+    fi
+    if [[ ! -z "$OOCD_DISTRO_URL" ]]; then
+      echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
+      wget $OOCD_DISTRO_URL
+      ARCH_NAME=$(basename $OOCD_DISTRO_URL)
+      tar -x -f $ARCH_NAME
+      export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
+      export PATH=$PWD/openocd-esp32/bin:$PATH
+    fi
   fi
 
   if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then

.gitlab/ci/pre_check.yml

@@ -165,6 +165,8 @@ pipeline_variables:
       if [ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]; then
         echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env
       fi
+    - echo "OOCD_DISTRO_URL_ARMHF=$OOCD_DISTRO_URL_ARMHF" >> pipeline.env
+    - echo "OOCD_DISTRO_URL_ARM64=$OOCD_DISTRO_URL_ARM64" >> pipeline.env
     - python tools/ci/ci_process_description.py
     - cat pipeline.env
     - python tools/ci/artifacts_handler.py upload --type modified_files_and_components_report

.gitmodules

@@ -54,8 +54,10 @@
 	sbom-supplier = Person: Dave Gamble
 	sbom-url = https://github.com/DaveGamble/cJSON
 	sbom-description = Ultralightweight JSON parser in ANSI C
-	sbom-hash = acc76239bee01d8e9c858ae2cab296704e52d916
+	sbom-hash = 8f2beb57ddad1f94bed899790b00f46df893ccac
 	sbom-cve-exclude-list = CVE-2024-31755 Resolved in v1.7.18
+	sbom-cve-exclude-list = CVE-2023-26819 Resolved in commit a328d65ad490b64da8c87523cbbfe16050ba5bf6
+	sbom-cve-exclude-list = CVE-2023-53154 Resolved in v1.7.18
 
 [submodule "components/mbedtls/mbedtls"]
 	path = components/mbedtls/mbedtls

.pre-commit-config.yaml

@@ -23,7 +23,8 @@ repos:
             .*.pb-c.h|
             .*.pb-c.c|
             .*.yuv|
-            .*.rgb
+            .*.rgb|
+            docs/sphinx-known-warnings\.txt
           )$
       - id: end-of-file-fixer
         exclude: *whitespace_excludes
@@ -56,6 +57,7 @@ repos:
     rev: v2.3.0
     hooks:
       - id: codespell
+        exclude: ^docs/sphinx-known-warnings\.txt$
   - repo: local
     hooks:
       - id: check-executables

components/app_trace/app_trace_membufs_proto.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0 OR MIT
  */
@@ -148,6 +148,18 @@ static esp_err_t esp_apptrace_membufs_swap_waitus(esp_apptrace_membufs_proto_dat
         if (res != ESP_OK) {
             break;
         }
+#if CONFIG_IDF_TARGET_ESP32S3
+        /*
+        * ESP32S3 has a serious data corruption issue with the transferred data to host.
+        * This delay helps reduce the failure rate by temporarily reducing heavy memory writes
+        * from RTOS-level tracing and giving OpenOCD more time to read trace memory before
+        * the current thread continues execution. While this doesn't completely prevent
+        * memory access from other threads/cores/ISRs, it has shown to significantly improve
+        * reliability when combined with CRC checks in OpenOCD. In practice, this reduces the
+        * number of retries needed to read an entire block without corruption.
+        */
+        esp_rom_delay_us(100);
+#endif
     }
     return res;
 }
@@ -339,7 +351,7 @@ uint8_t *esp_apptrace_membufs_up_buffer_get(esp_apptrace_membufs_proto_data_t *p
 esp_err_t esp_apptrace_membufs_up_buffer_put(esp_apptrace_membufs_proto_data_t *proto, uint8_t *ptr, esp_apptrace_tmo_t *tmo)
 {
     esp_apptrace_membufs_pkt_end(ptr);
-    // TODO: mark block as busy in order not to re-use it for other tracing calls until it is completely written
+    // TODO: mark block as busy in order not to reuse it for other tracing calls until it is completely written
     // TODO: avoid potential situation when all memory is consumed by low prio tasks which can not complete writing due to
     // higher prio tasks and the latter can not allocate buffers at all
     // this is abnormal situation can be detected on host which will receive only uncompleted buffers

components/app_trace/port/xtensa/port.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0 OR MIT
  */
@@ -12,7 +12,7 @@
 // ======================
 
 // Xtensa has useful feature: TRAX debug module. It allows recording program execution flow at run-time without disturbing CPU.
-// Exectution flow data are written to configurable Trace RAM block. Besides accessing Trace RAM itself TRAX module also allows to read/write
+// Execution flow data are written to configurable Trace RAM block. Besides accessing Trace RAM itself TRAX module also allows to read/write
 // trace memory via its registers by means of JTAG, APB or ERI transactions.
 // ESP32 has two Xtensa cores with separate TRAX modules on them and provides two special memory regions to be used as trace memory.
 // Chip allows muxing access to those trace memory blocks in such a way that while one block is accessed by CPUs another one can be accessed by host
@@ -47,17 +47,23 @@
 // 2. TRAX Registers layout
 // ========================
 
-// This module uses two TRAX HW registers to communicate with host SW (OpenOCD).
+// This module uses two TRAX HW registers and one Performance Monitor register to communicate with host SW (OpenOCD).
 //  - Control register uses TRAX_DELAYCNT as storage. Only lower 24 bits of TRAX_DELAYCNT are writable. Control register has the following bitfields:
 //   | 31..XXXXXX..24 | 23 .(host_connect). 23| 22..(block_id)..15 | 14..(block_len)..0 |
 //    14..0  bits - actual length of user data in trace memory block. Target updates it every time it fills memory block and exposes it to host.
 //                  Host writes zero to this field when it finishes reading exposed block;
 //    21..15 bits - trace memory block transfer ID. Block counter. It can overflow. Updated by target, host should not modify it. Actually can be 2 bits;
 //    22     bit  - 'host data present' flag. If set to one there is data from host, otherwise - no host data;
 //    23     bit  - 'host connected' flag. If zero then host is not connected and tracing module works in post-mortem mode, otherwise in streaming mode;
-// - Status register uses TRAX_TRIGGERPC as storage. If this register is not zero then current CPU is changing TRAX registers and
-//   this register holds address of the instruction which application will execute when it finishes with those registers modifications.
-//   See 'Targets Connection' setion for details.
+//  - Status register uses TRAX_TRIGGERPC as storage. If this register is not zero then current CPU is changing TRAX registers and
+//    this register holds address of the instruction which application will execute when it finishes with those registers modifications.
+//    See 'Targets Connection' section for details.
+//  - CRC16 register uses ERI_PERFMON_PM1 as storage. This register is used to store CRC16 checksum of the exposed trace memory block.
+//    The register has the following format:
+//    | 31..16 (CRC indicator) | 15..0 (CRC16 value) |
+//    CRC indicator (0xA55A) is used to distinguish valid CRC values from other data that might be in the register.
+//    CRC16 is calculated over the entire exposed block and is updated every time a block is exposed to the host.
+//    This allows the host to verify data integrity of the received trace data.
 
 // 3. Modes of operation
 // =====================
@@ -127,7 +133,7 @@
 
 // Access to internal module's data is synchronized with custom mutex. Mutex is a wrapper for portMUX_TYPE and uses almost the same sync mechanism as in
 // vPortCPUAcquireMutex/vPortCPUReleaseMutex. The mechanism uses S32C1I Xtensa instruction to implement exclusive access to module's data from tasks and
-// ISRs running on both cores. Also custom mutex allows specifying timeout for locking operation. Locking routine checks underlaying mutex in cycle until
+// ISRs running on both cores. Also custom mutex allows specifying timeout for locking operation. Locking routine checks underlying mutex in cycle until
 // it gets its ownership or timeout expires. The differences of application tracing module's mutex implementation from vPortCPUAcquireMutex/vPortCPUReleaseMutex are:
 // - Support for timeouts.
 // - Local IRQs for CPU which owns the mutex are disabled till the call to unlocking routine. This is made to avoid possible task's prio inversion.
@@ -142,9 +148,9 @@
 
 // Timeout mechanism is based on xthal_get_ccount() routine and supports timeout values in microseconds.
 // There are two situations when task/ISR can be delayed by tracing API call. Timeout mechanism takes into account both conditions:
-// - Trace data are locked by another task/ISR. When wating on trace data lock.
+// - Trace data are locked by another task/ISR. When waiting on trace data lock.
 // - Current TRAX memory input block is full when working in streaming mode (host is connected). When waiting for host to complete previous block reading.
-// When wating for any of above conditions xthal_get_ccount() is called periodically to calculate time elapsed from trace API routine entry. When elapsed
+// When waiting for any of above conditions xthal_get_ccount() is called periodically to calculate time elapsed from trace API routine entry. When elapsed
 // time exceeds specified timeout value operation is canceled and ESP_ERR_TIMEOUT code is returned.
 #include "sdkconfig.h"
 #include "soc/soc.h"
@@ -159,11 +165,15 @@
 #include "esp_log.h"
 #include "esp_app_trace_membufs_proto.h"
 #include "esp_app_trace_port.h"
+#include "esp_rom_crc.h"
 
 // TRAX is disabled, so we use its registers for our own purposes
 // | 31..XXXXXX..24 | 23 .(host_connect). 23 | 22 .(host_data). 22| 21..(block_id)..15 | 14..(block_len)..0 |
 #define ESP_APPTRACE_TRAX_CTRL_REG              ERI_TRAX_DELAYCNT
 #define ESP_APPTRACE_TRAX_STAT_REG              ERI_TRAX_TRIGGERPC
+#define ESP_APPTRACE_TRAX_CRC16_REG             ERI_PERFMON_PM1
+
+#define ESP_APPTRACE_CRC_INDICATOR              (0xA55AU << 16)
 
 #define ESP_APPTRACE_TRAX_BLOCK_LEN_MSK         0x7FFFUL
 #define ESP_APPTRACE_TRAX_BLOCK_LEN(_l_)        ((_l_) & ESP_APPTRACE_TRAX_BLOCK_LEN_MSK)
@@ -498,7 +508,8 @@ static esp_err_t esp_apptrace_trax_buffer_swap_start(uint32_t curr_block_id)
         uint32_t acked_block = ESP_APPTRACE_TRAX_BLOCK_ID_GET(ctrl_reg);
         uint32_t host_to_read = ESP_APPTRACE_TRAX_BLOCK_LEN_GET(ctrl_reg);
         if (host_to_read != 0 || acked_block != (curr_block_id & ESP_APPTRACE_TRAX_BLOCK_ID_MSK)) {
-            ESP_APPTRACE_LOGD("HC[%d]: Can not switch %" PRIx32 " %" PRIu32 " %" PRIx32 " %" PRIx32 "/%" PRIx32, esp_cpu_get_core_id(), ctrl_reg, host_to_read, acked_block,
+            ESP_APPTRACE_LOGD("HC[%d]: Can not switch %" PRIx32 " %" PRIu32 " %" PRIx32 " %" PRIx32 "/%" PRIx32,
+                esp_cpu_get_core_id(), ctrl_reg, host_to_read, acked_block,
                 curr_block_id & ESP_APPTRACE_TRAX_BLOCK_ID_MSK, curr_block_id);
             res = ESP_ERR_NO_MEM;
             goto _on_err;
@@ -514,6 +525,14 @@ static esp_err_t esp_apptrace_trax_buffer_swap_end(uint32_t new_block_id, uint32
 {
     uint32_t ctrl_reg = eri_read(ESP_APPTRACE_TRAX_CTRL_REG);
     uint32_t host_connected = ESP_APPTRACE_TRAX_HOST_CONNECT & ctrl_reg;
+
+    /* calculate CRC16 of the already switched block */
+    if (prev_block_len > 0) {
+        const uint8_t *prev_block_start = s_trax_blocks[!((new_block_id % 2))];
+        uint16_t crc16 = esp_rom_crc16_le(0, prev_block_start, prev_block_len);
+        eri_write(ESP_APPTRACE_TRAX_CRC16_REG, crc16 | ESP_APPTRACE_CRC_INDICATOR);
+        ESP_APPTRACE_LOGD("CRC16:%x %d @%x", crc16, prev_block_len, prev_block_start);
+    }
     eri_write(ESP_APPTRACE_TRAX_CTRL_REG, ESP_APPTRACE_TRAX_BLOCK_ID(new_block_id) |
               host_connected | ESP_APPTRACE_TRAX_BLOCK_LEN(prev_block_len));
     esp_apptrace_trax_buffer_swap_unlock();

components/app_trace/sys_view/Sample/FreeRTOSV10.4/SEGGER_SYSVIEW_FreeRTOS.c

@@ -108,7 +108,7 @@ static void _cbSendTaskList(void) {
 *    Called from SystemView when asked by the host, returns the
 *    current system time in micro seconds.
 */
-static U64 _cbGetTime(void) {
+__attribute__((unused)) static U64 _cbGetTime(void) {
   U64 Time;
 
   Time = xTaskGetTickCountFromISR();
@@ -260,7 +260,10 @@ void SYSVIEW_SendTaskInfo(U32 TaskID, const char* sName, unsigned Prio, U32 Stac
 */
 // Callbacks provided to SYSTEMVIEW by FreeRTOS
 const SEGGER_SYSVIEW_OS_API SYSVIEW_X_OS_TraceAPI = {
-  _cbGetTime,
+  /* Callback _cbGetTime locks xKernelLock inside xTaskGetTickCountFromISR, this can cause deadlock on multi-core.
+     To prevent deadlock, always lock xKernelLock before s_sys_view_lock. Omitting the callback here results in sending
+     SYSVIEW_EVTID_SYSTIME_CYCLES events instead of SYSVIEW_EVTID_SYSTIME_US */
+  NULL,
   _cbSendTaskList,
 };
 

components/bootloader_support/CMakeLists.txt

@@ -36,6 +36,10 @@ if(CONFIG_APP_BUILD_TYPE_APP_2NDBOOT)
         )
 endif()
 
+if(CONFIG_ESP_ROM_REV0_HAS_NO_ECDSA_INTERFACE)
+    list(APPEND srcs "src/${IDF_TARGET}/bootloader_ecdsa.c")
+endif()
+
 if(BOOTLOADER_BUILD OR CONFIG_APP_BUILD_TYPE_RAM)
     set(include_dirs "include" "bootloader_flash/include"
         "private_include")
@@ -50,10 +54,6 @@ if(BOOTLOADER_BUILD OR CONFIG_APP_BUILD_TYPE_RAM)
     "src/${IDF_TARGET}/bootloader_${IDF_TARGET}.c"
     )
     list(APPEND priv_requires hal)
-    if(CONFIG_ESP_ROM_REV0_HAS_NO_ECDSA_INTERFACE)
-        list(APPEND srcs
-            "src/${IDF_TARGET}/bootloader_ecdsa.c")
-    endif()
 else()
     list(APPEND srcs
         "src/idf/bootloader_sha.c")

components/bootloader_support/src/bootloader_common_loader.c

@@ -28,6 +28,7 @@
 
 #define ESP_PARTITION_HASH_LEN 32 /* SHA-256 digest length */
 #define IS_FIELD_SET(rev_full) (((rev_full) != 65535) && ((rev_full) != 0))
+#define ALIGN_UP(num, align) (((num) + ((align) - 1)) & ~((align) - 1))
 
 static const char* TAG = "boot_comm";
 
@@ -242,7 +243,10 @@ rtc_retain_mem_t* bootloader_common_get_rtc_retain_mem(void)
 #if ESP_ROM_HAS_LP_ROM
     #define RTC_RETAIN_MEM_ADDR (SOC_RTC_DRAM_LOW)
 #else
-    #define RTC_RETAIN_MEM_ADDR (SOC_RTC_DRAM_HIGH - sizeof(rtc_retain_mem_t))
+    /* Since the structure containing the retain_mem_t is aligned on 8 by the linker, make sure we align this
+     * structure size here too */
+    #define RETAIN_MEM_SIZE     ALIGN_UP(sizeof(rtc_retain_mem_t), 8)
+    #define RTC_RETAIN_MEM_ADDR (SOC_RTC_DRAM_HIGH - RETAIN_MEM_SIZE)
 #endif //ESP_ROM_HAS_LP_ROM
     static rtc_retain_mem_t *const s_bootloader_retain_mem = (rtc_retain_mem_t *)RTC_RETAIN_MEM_ADDR;
     return s_bootloader_retain_mem;

components/bootloader_support/src/esp_image_format.c

@@ -768,20 +768,27 @@ static esp_err_t verify_segment_header(int index, const esp_image_segment_header
     bool map_segment = should_map(load_addr);
 
 #if SOC_MMU_PAGE_SIZE_CONFIGURABLE
+    esp_err_t err = ESP_FAIL;
+
     /* ESP APP descriptor is present in the DROM segment #0 */
     if (index == 0 && metadata->start_addr != ESP_BOOTLOADER_OFFSET) {
-        const esp_app_desc_t *app_desc = (const esp_app_desc_t *)bootloader_mmap(segment_data_offs, sizeof(esp_app_desc_t));
-        if (!app_desc || app_desc->magic_word != ESP_APP_DESC_MAGIC_WORD) {
+        uint32_t mmu_page_size = 0, magic_word = 0;
+        const uint32_t mmu_page_size_offset = segment_data_offs + offsetof(esp_app_desc_t, mmu_page_size);
+        CHECK_ERR(bootloader_flash_read(segment_data_offs, &magic_word, sizeof(uint32_t), true));
+        CHECK_ERR(bootloader_flash_read(mmu_page_size_offset, &mmu_page_size, sizeof(uint32_t), true));
+        // Extract only the lowest byte from mmu_page_size (as per image format)
+        mmu_page_size &= 0xFF;
+
+        if (magic_word != ESP_APP_DESC_MAGIC_WORD) {
             ESP_LOGE(TAG, "Failed to fetch app description header!");
             return ESP_FAIL;
         }
 
         // Convert from log base 2 number to actual size while handling legacy image case (value 0)
-        metadata->mmu_page_size = (app_desc->mmu_page_size > 0) ? (1UL << app_desc->mmu_page_size) : SPI_FLASH_MMU_PAGE_SIZE;
+        metadata->mmu_page_size = (mmu_page_size > 0) ? (1UL << mmu_page_size) : SPI_FLASH_MMU_PAGE_SIZE;
         if (metadata->mmu_page_size != SPI_FLASH_MMU_PAGE_SIZE) {
             ESP_LOGI(TAG, "MMU page size mismatch, configured: 0x%x, found: 0x%"PRIx32, SPI_FLASH_MMU_PAGE_SIZE, metadata->mmu_page_size);
         }
-        bootloader_munmap(app_desc);
     } else if (index == 0 && metadata->start_addr == ESP_BOOTLOADER_OFFSET) {
         // Bootloader always uses the default MMU page size
         metadata->mmu_page_size = SPI_FLASH_MMU_PAGE_SIZE;
@@ -808,6 +815,10 @@ static esp_err_t verify_segment_header(int index, const esp_image_segment_header
     }
 
     return ESP_OK;
+#if SOC_MMU_PAGE_SIZE_CONFIGURABLE
+err:
+    return err;
+#endif
 }
 
 static bool should_map(uint32_t load_addr)