diff --git a/Makefile b/Makefile index 99efa3b8..c1b25fea 100644 --- a/Makefile +++ b/Makefile @@ -3,3 +3,6 @@ start-template-server: kill-template-server: docker kill $(shell docker ps --filter expose=49999 --format {{.ID}}) + +debug-template: + cd template && python build_debug.py && python debug_logs.py diff --git a/template/README.md b/template/README.md index 19039fc4..31f89a90 100644 --- a/template/README.md +++ b/template/README.md @@ -55,3 +55,35 @@ sbx = Sandbox.create(template="code-interpreter-custom") execution = sbx.run_code("print('Hello, World!')") print(execution.logs.stdout) ``` + +## Debugging a server that won't start + +The template runs Jupyter and the code-interpreter server as **systemd** +services (`systemd/jupyter.service`, `systemd/code-interpreter.service`). This is +the path CI and production use — note it is *different* from `make +start-template-server`, which runs the Docker `start-up.sh` path. The two can +diverge, so a server that boots fine under Docker may still fail under systemd. + +When a build fails its readiness check (`Waiting for template to be ready ... +timed out`), the real cause is in the service journals. To see them: + +``` +make debug-template +``` + +This builds a debug template (gated on a fixed timeout instead of `/health`, so +it finalizes even while the server is crash-looping), spawns a sandbox, and +prints `systemctl status` + the full `journalctl` for both services. It needs +`template/.env` with your `E2B_API_KEY` and the deps from `requirements-dev.txt`. + +The debug build also applies a systemd drop-in that routes Jupyter's stdout to +the journal (`make_template(debug=True)`). Production builds keep +`StandardOutput=null`, so Jupyter's request/error logs are only captured in the +debug template. + +Inside a running sandbox you can also inspect things directly: + +``` +journalctl -u jupyter -u code-interpreter +systemctl status code-interpreter +``` diff --git a/template/build_debug.py b/template/build_debug.py new file mode 100644 index 00000000..f4dc63b4 --- /dev/null +++ b/template/build_debug.py @@ -0,0 +1,23 @@ +import os + +from dotenv import load_dotenv +from e2b import Template, default_build_logger, wait_for_timeout +from template import make_template + +load_dotenv() + +alias = os.getenv("E2B_DEBUG_TEMPLATE", "code-interpreter-debug") + +Template.build( + make_template( + kernels=["python", "javascript"], + ready=wait_for_timeout(60_000), + debug=True, + ), + alias=alias, + cpu_count=2, + memory_mb=2048, + on_build_logs=default_build_logger(min_level="debug"), +) + +print(f"Built debug template: {alias}") diff --git a/template/debug_logs.py b/template/debug_logs.py new file mode 100644 index 00000000..cdd7b49a --- /dev/null +++ b/template/debug_logs.py @@ -0,0 +1,35 @@ +import os + +from dotenv import load_dotenv +from e2b import Sandbox + +load_dotenv() + +alias = os.getenv("E2B_DEBUG_TEMPLATE", "code-interpreter-debug") + +sbx = Sandbox.create(template=alias, timeout=600) +print(f"sandbox: {sbx.sandbox_id}") + +CMDS = [ + "systemctl --no-pager status jupyter || true", + "systemctl --no-pager status code-interpreter || true", + "journalctl --no-pager -u jupyter || true", + "journalctl --no-pager -u code-interpreter || true", + "curl -s --max-time 3 -o /dev/null -w 'jupyter :8888 -> %{http_code}\\n' http://localhost:8888/api/status || true", + "curl -s --max-time 3 -o /dev/null -w 'server :49999 -> %{http_code}\\n' http://localhost:49999/health || true", +] + +try: + for cmd in CMDS: + print(f"\n===== $ {cmd} =====") + try: + result = sbx.commands.run(f"sudo bash -lc {cmd!r}", timeout=60) + if result.stdout: + print(result.stdout) + if result.stderr: + print("[stderr]", result.stderr) + except Exception as e: + # Keep going so one slow/failed command doesn't skip the rest. + print(f"[command failed] {e}") +finally: + sbx.kill() diff --git a/template/systemd/jupyter-debug.conf b/template/systemd/jupyter-debug.conf new file mode 100644 index 00000000..89377a6f --- /dev/null +++ b/template/systemd/jupyter-debug.conf @@ -0,0 +1,5 @@ +# Debug-only drop-in: route Jupyter's stdout to the journal (the base unit +# sends it to /dev/null) so ServerApp request/error logs are visible via +# `journalctl -u jupyter`. Applied only by `make_template(debug=True)`. +[Service] +StandardOutput=journal diff --git a/template/template.py b/template/template.py index 62fec9e2..fd8dd937 100644 --- a/template/template.py +++ b/template/template.py @@ -1,9 +1,11 @@ -from e2b import Template, wait_for_url +from e2b import ReadyCmd, Template, wait_for_url def make_template( kernels: list[str] = ["python", "r", "javascript", "bash", "java"], is_docker: bool = False, + ready: ReadyCmd | None = None, + debug: bool = False, ): enabled_kernels = set(["python", "javascript"] + kernels) # Start with base template @@ -104,16 +106,18 @@ def make_template( ) if not is_docker: - template = ( - template.copy( - "systemd/jupyter.service", "/etc/systemd/system/jupyter.service" - ) - .copy( - "systemd/code-interpreter.service", - "/etc/systemd/system/code-interpreter.service", - ) - .run_cmd("systemctl daemon-reload") + template = template.copy( + "systemd/jupyter.service", "/etc/systemd/system/jupyter.service" + ).copy( + "systemd/code-interpreter.service", + "/etc/systemd/system/code-interpreter.service", ) + if debug: + # Drop-in that routes Jupyter's stdout to the journal for debugging. + template = template.copy( + "systemd/jupyter-debug.conf", + "/etc/systemd/system/jupyter.service.d/debug.conf", + ) else: template = template.copy("start-up.sh", ".jupyter/start-up.sh").run_cmd( "chmod +x .jupyter/start-up.sh" @@ -136,6 +140,7 @@ def make_template( else: start_cmd = "sudo systemctl start jupyter" - return template.set_start_cmd( - start_cmd, wait_for_url("http://localhost:49999/health") - ) + if ready is None: + ready = wait_for_url("http://localhost:49999/health") + + return template.set_start_cmd(start_cmd, ready)