diff --git a/docker-compose-library.yaml b/docker-compose-library.yaml index 0a991e1cf..3c198c0a8 100644 --- a/docker-compose-library.yaml +++ b/docker-compose-library.yaml @@ -28,6 +28,8 @@ services: depends_on: mcp-mock-server: condition: service_healthy + mock-mcp: + condition: service_healthy networks: - lightspeednet volumes: @@ -93,6 +95,23 @@ services: retries: 3 start_period: 2s + mock-mcp: + build: + context: ./tests/e2e/mock_mcp_server + dockerfile: Dockerfile + container_name: mock-mcp + ports: + - "3001:3001" + networks: + - lightspeednet + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3001/health')"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 2s + + networks: lightspeednet: driver: bridge \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 339e702ea..85a38bc62 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -92,6 +92,8 @@ services: condition: service_healthy mcp-mock-server: condition: service_healthy + mock-mcp: + condition: service_healthy networks: - lightspeednet healthcheck: @@ -118,6 +120,23 @@ services: retries: 3 start_period: 2s + mock-mcp: + build: + context: ./tests/e2e/mock_mcp_server + dockerfile: Dockerfile + container_name: mock-mcp + ports: + - "3001:3001" + networks: + - lightspeednet + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3001/health')"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 2s + + volumes: llama-storage: diff --git a/src/app/endpoints/tools.py b/src/app/endpoints/tools.py index 2021ab5ed..d272a7d66 100644 --- a/src/app/endpoints/tools.py +++ b/src/app/endpoints/tools.py @@ -4,6 +4,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request from llama_stack_client import APIConnectionError, BadRequestError, AuthenticationError +from llama_stack.core.datatypes import AuthenticationRequiredError from authentication import get_auth_dependency from authentication.interface import AuthTuple @@ -90,8 +91,7 @@ async def tools_endpoint_handler( # pylint: disable=too-many-locals,too-many-st except BadRequestError: logger.error("Toolgroup %s is not found", toolgroup.identifier) continue - except AuthenticationError as e: - logger.error("Authentication error: %s", e) + except (AuthenticationError, AuthenticationRequiredError) as e: if toolgroup.mcp_endpoint: await probe_mcp_oauth_and_raise_401( toolgroup.mcp_endpoint.uri, chain_from=e diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml new file mode 100644 index 000000000..0656aa87c --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml @@ -0,0 +1,25 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds llama-stack as library + use_as_library_client: true + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-oauth" + provider_id: "model-context-protocol" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "oauth" \ No newline at end of file diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml new file mode 100644 index 000000000..a598ce441 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml @@ -0,0 +1,26 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to separate llama-stack service + use_as_library_client: false + url: http://llama-stack:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-oauth" + provider_id: "model-context-protocol" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "oauth" \ No newline at end of file diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 543803f4c..33d82fbc1 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -297,6 +297,15 @@ def before_feature(context: Context, feature: Feature) -> None: context.port = os.getenv("E2E_LSC_PORT", "8080") context.feedback_conversations = [] + if "MCP" in feature.tags: + mode_dir = "library-mode" if context.is_library_mode else "server-mode" + context.feature_config = ( + f"tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp.yaml" + ) + context.default_config_backup = create_config_backup("lightspeed-stack.yaml") + switch_config(context.feature_config) + restart_container("lightspeed-stack") + def after_feature(context: Context, feature: Feature) -> None: """Run after each feature file is exercised. @@ -324,3 +333,8 @@ def after_feature(context: Context, feature: Feature) -> None: url = f"http://{context.hostname}:{context.port}/v1/conversations/{conversation_id}" response = requests.delete(url, timeout=10) assert response.status_code == 200, f"{url} returned {response.status_code}" + + if "MCP" in feature.tags: + switch_config(context.default_config_backup) + restart_container("lightspeed-stack") + remove_config_backup(context.default_config_backup) diff --git a/tests/e2e/features/mcp.feature b/tests/e2e/features/mcp.feature new file mode 100644 index 000000000..ae3da6ab0 --- /dev/null +++ b/tests/e2e/features/mcp.feature @@ -0,0 +1,204 @@ +@MCP +Feature: MCP tests + + Background: + Given The service is started locally + And REST API service prefix is /v1 + + Scenario: Check if tools endpoint reports error when MCP requires authentication + Given The system is in default state + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + And The headers of the response contains the following header "www-authenticate" + + Scenario: Check if query endpoint reports error when MCP requires authentication + Given The system is in default state + When I use "query" to ask question + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + And The headers of the response contains the following header "www-authenticate" + + Scenario: Check if streaming_query endpoint reports error when MCP requires authentication + Given The system is in default state + When I use "streaming_query" to ask question + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + And The headers of the response contains the following header "www-authenticate" + + @skip # will be fixed in LCORE-1368 + Scenario: Check if tools endpoint succeeds when MCP auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-oauth": {"Authorization": "Bearer test-token"}} + """ + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 200 + And The body of the response is the following + """ + { + "tools": [ + { + "identifier": "", + "description": "Insert documents into memory", + "parameters": [], + "provider_id": "", + "toolgroup_id": "builtin::rag", + "server_source": "builtin", + "type": "" + }, + { + "identifier": "", + "description": "Search for information in a database.", + "parameters": [], + "provider_id": "", + "toolgroup_id": "builtin::rag", + "server_source": "builtin", + "type": "" + }, + { + "identifier": "", + "description": "Mock tool for E2E", + "parameters": [], + "provider_id": "", + "toolgroup_id": "mcp-oauth", + "server_source": "http://localhost:3001", + "type": "" + } + ] + } + """ + + @skip # will be fixed in LCORE-1366 + Scenario: Check if query endpoint succeeds when MCP auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-oauth": {"Authorization": "Bearer test-token"}} + """ + And I capture the current token metrics + When I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response should contain following fragments + | Fragments in LLM response | + | hello | + And The token metrics should have increased + + @skip # will be fixed in LCORE-1366 + Scenario: Check if streaming_query endpoint succeeds when MCP auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-oauth": {"Authorization": "Bearer test-token"}} + """ + And I capture the current token metrics + When I use "streaming_query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + When I wait for the response to be completed + Then The status code of the response is 200 + And The streamed response should contain following fragments + | Fragments in LLM response | + | hello | + And The token metrics should have increased + + @skip # will be fixed in LCORE-1368 + Scenario: Check if tools endpoint reports error when MCP invalid auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-oauth": {"Authorization": "Bearer invalid-token"}} + """ + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + And The headers of the response contains the following header "www-authenticate" + + @skip # will be fixed in LCORE-1366 + Scenario: Check if query endpoint reports error when MCP invalid auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-oauth": {"Authorization": "Bearer invalid-token"}} + """ + When I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + And The headers of the response contains the following header "www-authenticate" + + @skip # will be fixed in LCORE-1366 + Scenario: Check if streaming_query endpoint reports error when MCP invalid auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-oauth": {"Authorization": "Bearer invalid-token"}} + """ + When I use "streaming_query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + And The headers of the response contains the following header "www-authenticate" diff --git a/tests/e2e/features/steps/common_http.py b/tests/e2e/features/steps/common_http.py index d35dc6e79..8e64fe5fc 100644 --- a/tests/e2e/features/steps/common_http.py +++ b/tests/e2e/features/steps/common_http.py @@ -3,7 +3,12 @@ import json import requests -from behave import then, when, step # pyright: ignore[reportAttributeAccessIssue] +from behave import ( + then, + when, + step, + given, +) # pyright: ignore[reportAttributeAccessIssue] from behave.runner import Context from tests.e2e.utils.utils import ( normalize_endpoint, @@ -188,6 +193,15 @@ def check_prediction_result(context: Context) -> None: assert result == expected_body, f"got:\n{result}\nwant:\n{expected_body}" +@then('The headers of the response contains the following header "{header_name}"') +def check_response_headers_contains(context: Context, header_name: str) -> None: + """Check that response contains a header whose name matches.""" + assert context.response is not None, "Request needs to be performed first" + assert ( + header_name in context.response.headers.keys() + ), f"The response headers '{context.response.headers}' doesn't contain header '{header_name}'" + + @then('The body of the response, ignoring the "{field}" field, is the following') def check_prediction_result_ignoring_field(context: Context, field: str) -> None: """Check the content of the response to be exactly the same. @@ -386,3 +400,13 @@ def check_response_partially(context: Context) -> None: json_str = replace_placeholders(context, context.text or "{}") expected = json.loads(json_str) validate_json_partially(body, expected) + + +@given('I set the "{header_name}" header to') +def set_header(context: Context, header_name: str) -> None: + """Set a header in the request.""" + assert context.text is not None, "Header value needs to be specified" + + if not hasattr(context, "auth_headers"): + context.auth_headers = {} + context.auth_headers[header_name] = context.text diff --git a/tests/e2e/mock_mcp_server/Dockerfile b/tests/e2e/mock_mcp_server/Dockerfile new file mode 100644 index 000000000..b641ca4c4 --- /dev/null +++ b/tests/e2e/mock_mcp_server/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-slim +WORKDIR /app +COPY server.py . +EXPOSE 3001 +CMD ["python", "server.py"] diff --git a/tests/e2e/mock_mcp_server/server.py b/tests/e2e/mock_mcp_server/server.py new file mode 100644 index 000000000..2d2768ed6 --- /dev/null +++ b/tests/e2e/mock_mcp_server/server.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Minimal mock MCP server for E2E tests with OAuth support. + +Responds to GET (OAuth probe) with 401 and WWW-Authenticate. Accepts POST +(MCP JSON-RPC) when Authorization: Bearer is present; otherwise 401. +Uses only Python stdlib. +""" + +import json +from http.server import HTTPServer, BaseHTTPRequestHandler +from typing import Any + +# Standard OAuth-style challenge so the client can drive an OAuth flow +WWW_AUTHENTICATE = 'Bearer realm="mock-mcp", error="invalid_token"' + + +class Handler(BaseHTTPRequestHandler): + """HTTP handler: GET/POST without valid Bearer → 401; POST with Bearer → MCP.""" + + def _require_oauth(self) -> None: + """Send 401 with WWW-Authenticate.""" + self.send_response(401) + self.send_header("WWW-Authenticate", WWW_AUTHENTICATE) + self.send_header("Content-Type", "application/json") + body = b'{"error":"unauthorized"}' + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _parse_auth(self) -> str | None: + """Return Bearer token if present, else None.""" + auth = self.headers.get("Authorization") + if auth and auth.startswith("Bearer ") and "invalid" not in auth: + return auth[7:].strip() + return None + + def _json_response(self, data: dict) -> None: + """Send JSON response.""" + body = json.dumps(data).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: # pylint: disable=invalid-name + """Handle GET requests.""" + if self.path == "/health": + self._json_response({"status": "ok"}) + else: + self._require_oauth() + + def do_POST(self) -> None: # pylint: disable=invalid-name + """Handle POST requests.""" + if self._parse_auth() is None: + self._require_oauth() + return + + length = int(self.headers.get("Content-Length", 0)) + raw = self.rfile.read(length) if length else b"{}" + try: + req = json.loads(raw.decode("utf-8")) + req_id = req.get("id", 1) + method = req.get("method", "") + except (json.JSONDecodeError, UnicodeDecodeError): + req_id = 1 + method = "" + + if method == "initialize": + self._json_response( + { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "mock-mcp-e2e", "version": "1.0.0"}, + }, + } + ) + elif method == "tools/list": + self._json_response( + { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "tools": [ + { + "name": "mock_tool_e2e", + "description": "Mock tool for E2E", + "inputSchema": { + "type": "object", + "properties": { + "message": { + "type": "string", + "description": "Test message", + } + }, + }, + } + ], + }, + } + ) + else: + self._json_response({"jsonrpc": "2.0", "id": req_id, "result": {}}) + + def log_message(self, format: str, *args: Any) -> None: + """Suppress request logging for minimal output.""" + + +if __name__ == "__main__": + server = HTTPServer(("0.0.0.0", 3001), Handler) + print("Mock MCP server on :3001") + server.serve_forever() diff --git a/tests/e2e/test_list.txt b/tests/e2e/test_list.txt index 398d824c2..84bf75af7 100644 --- a/tests/e2e/test_list.txt +++ b/tests/e2e/test_list.txt @@ -12,4 +12,5 @@ features/info.feature features/query.feature features/streaming_query.feature features/rest_api.feature +features/mcp.feature features/models.feature