96 curl async file io

.github/workflows/debug-shutdown.yml

@@ -0,0 +1,200 @@
+name: Debug Shutdown Tests
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    services:
+      mysql:
+        image: mysql:8.3
+        env:
+          MYSQL_ROOT_PASSWORD: ''
+          MYSQL_ALLOW_EMPTY_PASSWORD: yes
+          MYSQL_DATABASE: test
+        ports: ['3306:3306']
+        options: >-
+          --health-cmd="mysqladmin ping --silent"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=5
+
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: test
+        ports: ['5432:5432']
+        options: >-
+          --health-cmd="pg_isready"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=5
+
+    steps:
+      - name: Checkout php-async repo
+        uses: actions/checkout@v4
+        with:
+          path: async
+
+      - name: Clone php-src (true-async)
+        run: |
+          git clone --depth=1 --branch=true-async https://github.com/true-async/php-src php-src
+
+      - name: Copy php-async extension into php-src
+        run: |
+          mkdir -p php-src/ext/async
+          cp -r async/* php-src/ext/async/
+
+      # Cache APT packages to speed up dependency installation
+      - name: Cache APT packages
+        uses: actions/cache@v4
+        with:
+          path: /var/cache/apt/archives
+          key: ${{ runner.os }}-apt-packages-${{ hashFiles('.github/workflows/debug-shutdown.yml') }}
+          restore-keys: |
+            ${{ runner.os }}-apt-packages-
+
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            gcc g++ autoconf bison re2c \
+            libgmp-dev libicu-dev libtidy-dev libsasl2-dev \
+            libzip-dev libbz2-dev libsqlite3-dev libonig-dev libcurl4-openssl-dev \
+            libxml2-dev libxslt1-dev libpq-dev libreadline-dev libldap2-dev libsodium-dev \
+            libargon2-dev \
+            firebird-dev \
+            valgrind cmake
+
+      # Cache LibUV 1.45.0 installation to avoid rebuilding
+      - name: Cache LibUV 1.45.0
+        id: cache-libuv
+        uses: actions/cache@v4
+        with:
+          path: |
+            /usr/local/lib/libuv*
+            /usr/local/include/uv*
+            /usr/local/lib/pkgconfig/libuv.pc
+          key: ${{ runner.os }}-libuv-1.45.0-release
+
+      - name: Install LibUV >= 1.45.0
+        run: |
+          # Check if we have cached LibUV 1.45.0
+          if [ "${{ steps.cache-libuv.outputs.cache-hit }}" == "true" ]; then
+            echo "Using cached LibUV 1.45.0 installation"
+            sudo ldconfig
+            echo "LibUV version: $(pkg-config --modversion libuv)"
+          # Check if system libuv meets requirements
+          elif pkg-config --exists libuv && pkg-config --atleast-version=1.45.0 libuv; then
+            echo "System libuv version: $(pkg-config --modversion libuv)"
+            sudo apt-get install -y libuv1-dev
+          else
+            echo "Installing LibUV 1.45.0 from source"
+            wget https://github.com/libuv/libuv/archive/v1.45.0.tar.gz
+            tar -xzf v1.45.0.tar.gz
+            cd libuv-1.45.0
+            mkdir build && cd build
+            cmake .. -DCMAKE_BUILD_TYPE=Release
+            make -j$(nproc)
+            sudo make install
+            sudo ldconfig
+            cd ../..
+            echo "LibUV 1.45.0 compiled and installed"
+          fi
+
+      - name: Configure PHP
+        working-directory: php-src
+        run: |
+          ./buildconf -f
+          ./configure \
+            --enable-zts \
+            --enable-fpm \
+            --with-pdo-mysql=mysqlnd \
+            --with-mysqli=mysqlnd \
+            --with-pgsql \
+            --with-pdo-pgsql \
+            --with-pdo-sqlite \
+            --enable-intl \
+            --without-pear \
+            --with-zip \
+            --with-zlib \
+            --enable-soap \
+            --enable-xmlreader \
+            --with-xsl \
+            --with-tidy \
+            --enable-sysvsem \
+            --enable-sysvshm \
+            --enable-shmop \
+            --enable-pcntl \
+            --with-readline \
+            --enable-mbstring \
+            --with-curl \
+            --with-gettext \
+            --enable-sockets \
+            --with-bz2 \
+            --with-openssl \
+            --with-gmp \
+            --enable-bcmath \
+            --enable-calendar \
+            --enable-ftp \
+            --enable-sysvmsg \
+            --with-ffi \
+            --enable-zend-test \
+            --enable-dl-test=shared \
+            --with-ldap \
+            --with-ldap-sasl \
+            --with-password-argon2 \
+            --with-mhash \
+            --with-sodium \
+            --enable-dba \
+            --with-cdb \
+            --enable-flatfile \
+            --enable-inifile \
+            --with-config-file-path=/etc \
+            --with-config-file-scan-dir=/etc/php.d \
+            --with-pdo-firebird \
+            --enable-address-sanitizer \
+            --enable-async
+
+      - name: Build PHP
+        working-directory: php-src
+        run: |
+          make -j"$(nproc)"
+          sudo make install
+          sudo mkdir -p /etc/php.d
+          sudo chmod 777 /etc/php.d
+          {
+            echo "opcache.enable_cli=1"
+            echo "opcache.protect_memory=1"
+          } > /etc/php.d/opcache.ini
+
+      - name: Run failing tests
+        working-directory: php-src/ext/async
+        run: |
+          /usr/local/bin/php -v
+          /usr/local/bin/php ../../run-tests.php \
+            -d opcache.enable_cli=1 \
+            -d opcache.jit_buffer_size=64M \
+            -d opcache.jit=tracing \
+            -d zend_test.observer.enabled=1 \
+            -d zend_test.observer.show_output=0 \
+            -P -q -x -j4 \
+            -g FAIL,BORK,LEAK,XLEAK \
+            --no-progress \
+            --offline \
+            --show-diff \
+            --show-slow 4000 \
+            --set-timeout 120 \
+            --repeat 2 \
+            tests/bailout/009-memory-exhaustion-during-await.phpt \
+            tests/bailout/010-memory-exhaustion-during-awaitAllOrFail.phpt \
+            tests/bailout/011-stack-overflow-during-await.phpt \
+            tests/bailout/012-memory-exhaustion-awaiting-process.phpt \
+            tests/exec/017-exec_unicode_binary.phpt \
+            tests/exec/018-exec_long_lines.phpt \
+            tests/exec/022-exec_chdir_cwd.phpt \
+            tests/include/009-require_double_include_redeclare.phpt \
+            tests/include/010-require_in_coroutine_then_main_redeclare.phpt

CHANGELOG.md

@@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [0.6.0]
 
 ### Fixed
+- **Async file IO position tracking**: Replaced bare `lseek`/`_lseeki64` with `zend_lseek` across reactor. Rewrote `libuv_io_seek` to accept `whence` and return position, eliminating double lseek in `php_stdiop_seek`. Fixed append-mode offset init and fseek behavior. On Windows, append writes now query real EOF via `lseek(SEEK_END)` before dispatch to avoid stale cached offsets.
+- **Windows concurrent append (XFAIL)**: On Windows, `WriteFile` via libuv ignores CRT `_O_APPEND` because `FILE_WRITE_DATA` coexists with `FILE_APPEND_DATA` on the HANDLE. Removing `FILE_WRITE_DATA` would fix atomic append but breaks `ftruncate`/`SetEndOfFile`. Concurrent append from multiple coroutines remains a known limitation (test 069 marked XFAIL).
+- **Reactor deadlock on pending file I/O requests**: `uv_fs_read`, `uv_fs_write`, `uv_fs_fsync`, and `uv_fs_fstat` are libuv requests (not handles) that keep `uv_loop_alive()` true but were invisible to `ZEND_ASYNC_ACTIVE_EVENT_COUNT`. The reactor loop exited prematurely (`has_handles && active_event_count > 0` → false) while file I/O callbacks were still pending, causing deadlocks in async file writes (e.g. `CURLOPT_FILE` with async I/O). Fixed by adding `ZEND_ASYNC_INCREASE_EVENT_COUNT` after successful `uv_fs_*` submission and `ZEND_ASYNC_DECREASE_EVENT_COUNT` in their completion callbacks (`io_file_read_cb`, `io_file_write_cb`, `io_file_flush_cb`, `io_file_stat_cb`).
 - **Generator segfault in fiber-coroutine mode**: Generators running inside fiber coroutines were not marked with `ZEND_GENERATOR_IN_FIBER` because `EG(active_fiber)` is not set in coroutine mode. This caused shutdown destructors to close generators while the coroutine was still suspended, leading to a NULL `execute_data` dereference in `zend_generator_resume`. Fixed by also checking `ZEND_ASYNC_CURRENT_COROUTINE` with `ZEND_COROUTINE_IS_FIBER` when setting the `IN_FIBER` flag on generators.
 
 ### Added
@@ -43,6 +46,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **`Async\iterate()` function**: Iterates over an iterable, calling the callback for each element with optional concurrency limit. Supports `cancelPending` parameter (default: `true`) that controls whether coroutines spawned inside the callback are cancelled or awaited after iteration completes.
 - **`Async\FileSystemWatcher` class**: Persistent filesystem watcher with `foreach` iteration support, suspend/resume on new events, two storage modes (coalesce with HashTable deduplication, raw with circular buffer), `close()`/`isClosed()` lifecycle, and `Awaitable` interface via `ZEND_ASYNC_EVENT_REF_FIELDS` pattern. Replaces the one-shot `Async\watch_filesystem()` function.
 - **`Async\signal()` function**: One-shot signal handler that returns a `Future` resolved when the specified signal is received. Supports optional `Cancellation` for early cancellation.
+- **Acting coroutine for error context** (`zend_async_globals_t.acting_coroutine`): New field in async globals that allows scheduler-context code to attribute errors to a suspended coroutine. When set, `zend_get_executed_filename_ex()`, `zend_get_executed_lineno()`, and `get_active_function_name()` in `Zend/zend_execute_API.c` fall back to the coroutine's suspended `execute_data` for file, line, and function name. Zero-cost: the execute_data is only read when an error actually occurs. Macros: `ZEND_ASYNC_ACTING_COROUTINE`, `ZEND_ASYNC_ACT_AS_START(coroutine)`, `ZEND_ASYNC_ACT_AS_END()`.
 
 ### Changed
 - **Bailout handling**: Added `ZEND_ASYNC_EVENT_F_BAILOUT` flag (bit 11) on `zend_async_event_t`. During bailout (e.g. OOM), PHP-level handlers are no longer called — finally handlers on coroutines and scopes are destroyed without execution, scope exception handlers (`try_to_handle_exception`) are skipped. C-level callbacks (`ZEND_ASYNC_CALLBACKS_NOTIFY`) continue to work normally. Convenience macros: `ZEND_COROUTINE_SET_BAILOUT`/`ZEND_COROUTINE_IS_BAILOUT`, `ZEND_ASYNC_SCOPE_SET_BAILOUT`/`ZEND_ASYNC_SCOPE_IS_BAILOUT`.

CURL-INTEGRATION.md

@@ -0,0 +1,153 @@
+# cURL Async Integration — Known Issues & Architecture
+
+This document covers important technical details about the async cURL integration,
+including known libcurl bugs and the workarounds applied.
+
+---
+
+## Overview
+
+The async cURL integration uses libcurl's `multi_socket` API combined with
+the PAUSE/unpause pattern for non-blocking I/O:
+
+- **File uploads** (`CURLFile`): `curl_mime_data_cb()` with a read callback that
+  returns `CURL_READFUNC_PAUSE` while async file I/O is in progress.
+- **File downloads** (`CURLOPT_FILE`): write callback returns `CURL_WRITEFUNC_PAUSE`
+  while async file write is in progress.
+- **User callbacks** (`CURLOPT_WRITEFUNCTION`): write callback pauses, spawns a
+  high-priority coroutine to run the PHP callback, then unpauses.
+
+After async I/O completes, the transfer is unpaused via `curl_easy_pause(CURLPAUSE_CONT)`
+and driven forward with `curl_multi_socket_action(CURL_SOCKET_TIMEOUT)`.
+
+---
+
+## Minimum libcurl Version
+
+**Recommended: libcurl >= 8.11.1** for fully async file upload support.
+
+On older versions, file uploads (`CURLFile`) fall back to synchronous `read()`
+inside the read callback. This is safe for local files but blocks the event loop
+briefly during each read. Downloads and user write callbacks work correctly on
+all versions.
+
+---
+
+## The PAUSE/unpause Bug (libcurl < 8.11.1)
+
+### Symptom
+
+Intermittent timeout on file uploads (~20% failure rate):
+```
+Operation timed out after 5000 milliseconds with 0 bytes received
+```
+
+### Root Cause
+
+Multiple bugs in libcurl's PAUSE/unpause mechanism prevent the transfer from
+being driven after `curl_easy_pause(CURLPAUSE_CONT)`:
+
+1. **`timer_lastcall` / `last_expire_ts` optimization** — `Curl_update_timer()`
+   skips the timer callback when the new expire timestamp matches the cached one.
+   Fixed in [curl#15627](https://github.com/curl/curl/pull/15627) (8.11.1),
+   but the fix was removed during intermediate refactors (present in 8.5.0,
+   absent in 8.6–8.10, re-added in 8.11.1).
+
+2. **`tempcount` guard on `cselect_bits`** — In `curl_easy_pause()`,
+   `data->conn->cselect_bits` is only set when `data->state.tempcount == 0`.
+   If any response data arrived while the transfer was paused, `tempcount > 0`
+   and `cselect_bits` is never set. Without `cselect_bits`, the transfer is
+   not processed even when timeouts fire correctly. Fixed in 8.11.1+ where
+   `cselect_bits` was replaced with `data->state.select_bits` (always set,
+   no `tempcount` guard).
+
+3. **`CURLINFO_ACTIVESOCKET` unreliable during transfer** — Returns
+   `CURL_SOCKET_BAD` (-1) in the `multi_socket` API because `lastconnect_id`
+   is not set until the transfer completes. Cannot be used to drive the socket
+   directly.
+
+### Workarounds Tested (all insufficient for < 8.11.1)
+
+| Approach                                                      | Result                                                   |
+|---------------------------------------------------------------|----------------------------------------------------------|
+| `curl_multi_socket_action(CURL_SOCKET_TIMEOUT)` after unpause | ~80% pass                                                |
+| Manual `curl_timer_cb(multi, 0, NULL)` to force 0ms timer     | ~94% pass                                                |
+| `CURLINFO_ACTIVESOCKET` + `CURL_CSELECT_IN\|OUT`              | ~92% pass (socket sometimes BAD)                         |
+| Track socket via `curl_socket_cb` + direct socket action      | ~82–92% pass (socket removed from sockhash during pause) |
+| `curl_multi_perform()`                                        | ~74% pass (must not mix with multi_socket API)           |
+
+### Solution Applied
+
+For libcurl < 8.11.1, the file upload read callback (`curl_async_read_cb`)
+uses **synchronous `read()`** instead of the async PAUSE/unpause pattern.
+This completely avoids the bug — no PAUSE means no broken unpause.
+
+```c
+#if LIBCURL_VERSION_NUM < 0x080B01
+    // Synchronous read — safe for local files
+    const ssize_t n = read(fd, buffer, requested);
+#else
+    // Async PAUSE/unpause pattern
+    return CURL_READFUNC_PAUSE;
+#endif
+```
+
+Compile-time check via `LIBCURL_VERSION_NUM`. Zero runtime overhead.
+With libcurl >= 8.11.1, the async path is used and works 100% reliably.
+
+---
+
+## Timer Callback Regression (libcurl 8.10.x)
+
+### Symptom
+
+`curl_exec()` hangs indefinitely when connecting to unreachable hosts (e.g. `192.0.2.1`),
+even with `CURLOPT_TIMEOUT` and `CURLOPT_CONNECTTIMEOUT` set. The timeout never fires
+in the `multi_socket` API.
+
+### Root Cause
+
+A regression introduced in libcurl 8.10.0 causes `CURLMOPT_TIMERFUNCTION` to not be
+called again after `curl_multi_socket_action(CURL_SOCKET_TIMEOUT)`. The sequence:
+
+1. curl registers a connect socket and requests a timeout timer (e.g. 2000ms)
+2. Timer fires → `curl_multi_socket_action(CURL_SOCKET_TIMEOUT)` is called
+3. curl returns `running_handles=1` — does **not** complete the transfer
+4. curl does **not** call `CURLMOPT_TIMERFUNCTION` to request a new timer
+5. The socket never becomes writable (host unreachable) → deadlock
+
+In working versions (e.g. 8.5.0), step 4 correctly re-arms the timer with a short
+interval (~35ms), which fires again and completes the transfer with `CURLE_OPERATION_TIMEDOUT`.
+
+### Affected Versions
+
+- **Broken**: libcurl 8.10.0 – 8.11.0
+- **Working**: libcurl ≤ 8.9.1, libcurl ≥ 8.11.1
+- Related: [curl#15154](https://github.com/curl/curl/issues/15154) — "curl_multi still breaks in 8.10.1 with libev"
+- Related: [curl#15639](https://github.com/curl/curl/issues/15639) — Regression in multi-curl async calls (8.9.1 → 8.10.0/8.11.0)
+- Fix: [curl#15627](https://github.com/curl/curl/pull/15627) — merged into curl 8.11.1
+
+### Recommendation
+
+Avoid libcurl 8.10.x for any `multi_socket`-based integration. Use libcurl 8.5.0
+(as in CI) or >= 8.11.1.
+
+---
+
+## References
+
+- [curl#15627](https://github.com/curl/curl/pull/15627) — Fix for `CURLMOPT_TIMERFUNCTION` not being called (merged Nov 2024, curl 8.11.1)
+- [curl#15154](https://github.com/curl/curl/issues/15154) — curl_multi breaks in 8.10.1 with libev
+- [curl#15639](https://github.com/curl/curl/issues/15639) — Regression in multi-curl async calls (8.9.1 → 8.10.0/8.11.0)
+- [curl#5299](https://github.com/curl/curl/issues/5299) — `CURLINFO_ACTIVESOCKET` reliability issues
+- libcurl `multi_socket` API: https://curl.se/libcurl/c/libcurl-multi.html
+- libcurl pause/unpause: https://curl.se/libcurl/c/curl_easy_pause.html
+
+---
+
+## Files
+
+- `ext/curl/curl_async.c` — Async cURL implementation (read/write callbacks, unpause logic)
+- `ext/curl/curl_async.h` — Struct definitions and public API
+- `ext/curl/interface.c` — `build_mime_structure_from_hash()` registers the async read callback
+- `ext/curl/curl_private.h` — `mime_data_cb_arg_t` struct with async state

README.md

@@ -110,6 +110,7 @@ PhpStorm stubs for autocompletion and inline docs are available in [`ide-stubs/`
 - **[Documentation](https://true-async.github.io/)** — full reference and guides
 - **[Supported Functions](https://true-async.github.io/docs/reference/supported-functions.html)** — complete list of async-aware PHP functions
 - **[Download & Installation](https://true-async.github.io/download.html)** — packages and build instructions
+- **[cURL Integration Notes](CURL-INTEGRATION.md)** — known libcurl bugs, minimum version requirements, and architecture details
 
 ---