LCORE-1247: E2E Tests for MCP OAuth

docker-compose-library.yaml

@@ -28,6 +28,8 @@ services:
     depends_on:
       mcp-mock-server:
         condition: service_healthy
+      mock-mcp:
+        condition: service_healthy
     networks:
       - lightspeednet
     volumes:
@@ -93,6 +95,23 @@ services:
       retries: 3
       start_period: 2s
 
+  mock-mcp:
+    build:
+      context: ./tests/e2e/mock_mcp_server
+      dockerfile: Dockerfile
+    container_name: mock-mcp
+    ports:
+      - "3001:3001"
+    networks:
+      - lightspeednet
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3001/health')"]
+      interval: 5s
+      timeout: 3s
+      retries: 3
+      start_period: 2s
+
+
 networks:
   lightspeednet:
     driver: bridge

docker-compose.yaml

@@ -92,6 +92,8 @@ services:
           condition: service_healthy
         mcp-mock-server:
           condition: service_healthy
+        mock-mcp:
+          condition: service_healthy
     networks:
       - lightspeednet
     healthcheck:
@@ -118,6 +120,23 @@ services:
       retries: 3
       start_period: 2s
 
+  mock-mcp:
+    build:
+      context: ./tests/e2e/mock_mcp_server
+      dockerfile: Dockerfile
+    container_name: mock-mcp
+    ports:
+      - "3001:3001"
+    networks:
+      - lightspeednet
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3001/health')"]
+      interval: 5s
+      timeout: 3s
+      retries: 3
+      start_period: 2s
+
+
 volumes:
   llama-storage:
 

src/app/endpoints/tools.py

@@ -4,6 +4,7 @@
 
 from fastapi import APIRouter, Depends, HTTPException, Request
 from llama_stack_client import APIConnectionError, BadRequestError, AuthenticationError
+from llama_stack.core.datatypes import AuthenticationRequiredError
 
 from authentication import get_auth_dependency
 from authentication.interface import AuthTuple
@@ -90,8 +91,7 @@ async def tools_endpoint_handler(  # pylint: disable=too-many-locals,too-many-st
         except BadRequestError:
             logger.error("Toolgroup %s is not found", toolgroup.identifier)
             continue
-        except AuthenticationError as e:
-            logger.error("Authentication error: %s", e)
+        except (AuthenticationError, AuthenticationRequiredError) as e:
             if toolgroup.mcp_endpoint:
                 await probe_mcp_oauth_and_raise_401(
                     toolgroup.mcp_endpoint.uri, chain_from=e

tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml

@@ -0,0 +1,25 @@
+name: Lightspeed Core Service (LCS)
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Library mode - embeds llama-stack as library
+  use_as_library_client: true
+  library_client_config_path: run.yaml
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+authentication:
+  module: "noop"
+mcp_servers:
+  - name: "mcp-oauth"
+    provider_id: "model-context-protocol"
+    url: "http://mock-mcp:3001"
+    authorization_headers:
+      Authorization: "oauth"

tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml

@@ -0,0 +1,26 @@
+name: Lightspeed Core Service (LCS)
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Server mode - connects to separate llama-stack service
+  use_as_library_client: false
+  url: http://llama-stack:8321
+  api_key: xyzzy
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+authentication:
+  module: "noop"
+mcp_servers:
+  - name: "mcp-oauth"
+    provider_id: "model-context-protocol"
+    url: "http://mock-mcp:3001"
+    authorization_headers:
+      Authorization: "oauth"

tests/e2e/features/environment.py

@@ -297,6 +297,15 @@ def before_feature(context: Context, feature: Feature) -> None:
         context.port = os.getenv("E2E_LSC_PORT", "8080")
         context.feedback_conversations = []
 
+    if "MCP" in feature.tags:
+        mode_dir = "library-mode" if context.is_library_mode else "server-mode"
+        context.feature_config = (
+            f"tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp.yaml"
+        )
+        context.default_config_backup = create_config_backup("lightspeed-stack.yaml")
+        switch_config(context.feature_config)
+        restart_container("lightspeed-stack")
+
 
 def after_feature(context: Context, feature: Feature) -> None:
     """Run after each feature file is exercised.
@@ -324,3 +333,8 @@ def after_feature(context: Context, feature: Feature) -> None:
             url = f"http://{context.hostname}:{context.port}/v1/conversations/{conversation_id}"
             response = requests.delete(url, timeout=10)
             assert response.status_code == 200, f"{url} returned {response.status_code}"
+
+    if "MCP" in feature.tags:
+        switch_config(context.default_config_backup)
+        restart_container("lightspeed-stack")
+        remove_config_backup(context.default_config_backup)

tests/e2e/features/mcp.feature

@@ -0,0 +1,204 @@
+@MCP
+Feature: MCP tests
+
+  Background:
+    Given The service is started locally
+      And REST API service prefix is /v1
+
+  Scenario: Check if tools endpoint reports error when MCP requires authentication
+    Given The system is in default state
+    When I access REST API endpoint "tools" using HTTP GET method
+    Then The status code of the response is 401
+    And The body of the response is the following
+    """
+        {
+            "detail": {
+                "response": "Missing or invalid credentials provided by client",
+                "cause": "MCP server at http://mock-mcp:3001 requires OAuth"
+            }
+        }
+    """
+    And The headers of the response contains the following header "www-authenticate"
+
+  Scenario: Check if query endpoint reports error when MCP requires authentication
+    Given The system is in default state
+    When I use "query" to ask question
+    """
+    {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"}
+    """
+    Then The status code of the response is 401
+    And The body of the response is the following
+    """
+        {
+            "detail": {
+                "response": "Missing or invalid credentials provided by client",
+                "cause": "MCP server at http://mock-mcp:3001 requires OAuth"
+            }
+        }
+    """
+    And The headers of the response contains the following header "www-authenticate"
+
+  Scenario: Check if streaming_query endpoint reports error when MCP requires authentication
+    Given The system is in default state
+    When I use "streaming_query" to ask question
+    """
+    {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"}
+    """
+    Then The status code of the response is 401
+    And The body of the response is the following
+    """
+        {
+            "detail": {
+                "response": "Missing or invalid credentials provided by client",
+                "cause": "MCP server at http://mock-mcp:3001 requires OAuth"
+            }
+        }
+    """
+    And The headers of the response contains the following header "www-authenticate"
+
+  @skip     # will be fixed in LCORE-1368
+  Scenario: Check if tools endpoint succeeds when MCP auth token is passed
+    Given The system is in default state
+    And I set the "MCP-HEADERS" header to
+    """
+    {"mcp-oauth": {"Authorization": "Bearer test-token"}}
+    """
+    When I access REST API endpoint "tools" using HTTP GET method
+    Then The status code of the response is 200
+    And The body of the response is the following
+    """
+        {
+            "tools": [
+                {
+                    "identifier": "",
+                    "description": "Insert documents into memory",
+                    "parameters": [],
+                    "provider_id": "",
+                    "toolgroup_id": "builtin::rag",
+                    "server_source": "builtin",
+                    "type": ""
+                },
+                {
+                    "identifier": "",
+                    "description": "Search for information in a database.",
+                    "parameters": [],
+                    "provider_id": "",
+                    "toolgroup_id": "builtin::rag",
+                    "server_source": "builtin",
+                    "type": ""
+                },
+                {
+                    "identifier": "",
+                    "description": "Mock tool for E2E",
+                    "parameters": [],
+                    "provider_id": "",
+                    "toolgroup_id": "mcp-oauth",
+                    "server_source": "http://localhost:3001",
+                    "type": ""
+                }
+            ]
+        }
+    """
+
+  @skip     # will be fixed in LCORE-1366
+  Scenario: Check if query endpoint succeeds when MCP auth token is passed
+    Given The system is in default state
+    And I set the "MCP-HEADERS" header to
+    """
+    {"mcp-oauth": {"Authorization": "Bearer test-token"}}
+    """
+    And I capture the current token metrics
+    When I use "query" to ask question with authorization header
+    """
+    {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"}
+    """
+    Then The status code of the response is 200
+    And The response should contain following fragments
+        | Fragments in LLM response |
+        | hello                     |
+    And The token metrics should have increased
+
+  @skip     # will be fixed in LCORE-1366
+  Scenario: Check if streaming_query endpoint succeeds when MCP auth token is passed
+    Given The system is in default state
+    And I set the "MCP-HEADERS" header to
+    """
+    {"mcp-oauth": {"Authorization": "Bearer test-token"}}
+    """
+    And I capture the current token metrics
+    When I use "streaming_query" to ask question with authorization header
+    """
+    {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"}
+    """
+    When I wait for the response to be completed
+    Then The status code of the response is 200
+    And The streamed response should contain following fragments
+        | Fragments in LLM response |
+        | hello                     |
+    And The token metrics should have increased
+
+  @skip     # will be fixed in LCORE-1368
+  Scenario: Check if tools endpoint reports error when MCP invalid auth token is passed
+    Given The system is in default state
+    And I set the "MCP-HEADERS" header to
+    """
+    {"mcp-oauth": {"Authorization": "Bearer invalid-token"}}
+    """
+    When I access REST API endpoint "tools" using HTTP GET method
+    Then The status code of the response is 401
+    And The body of the response is the following
+    """
+        {
+            "detail": {
+                "response": "Missing or invalid credentials provided by client",
+                "cause": "MCP server at http://mock-mcp:3001 requires OAuth"
+            }
+        }
+    """
+    And The headers of the response contains the following header "www-authenticate"
+
+  @skip     # will be fixed in LCORE-1366
+  Scenario: Check if query endpoint reports error when MCP invalid auth token is passed
+    Given The system is in default state
+    And I set the "MCP-HEADERS" header to
+    """
+    {"mcp-oauth": {"Authorization": "Bearer invalid-token"}}
+    """
+    When I use "query" to ask question with authorization header
+    """
+    {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"}
+    """
+    Then The status code of the response is 401
+    And The body of the response is the following
+    """
+        {
+            "detail": {
+                "response": "Missing or invalid credentials provided by client",
+                "cause": "MCP server at http://mock-mcp:3001 requires OAuth"
+            }
+        }
+    """
+    And The headers of the response contains the following header "www-authenticate"
+
+  @skip     # will be fixed in LCORE-1366
+  Scenario: Check if streaming_query endpoint reports error when MCP invalid auth token is passed
+    Given The system is in default state
+    And I set the "MCP-HEADERS" header to
+    """
+    {"mcp-oauth": {"Authorization": "Bearer invalid-token"}}
+    """
+    When I use "streaming_query" to ask question with authorization header
+    """
+    {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"}
+    """
+    Then The status code of the response is 401
+    And The body of the response is the following
+    """
+        {
+            "detail": {
+                "response": "Missing or invalid credentials provided by client",
+                "cause": "MCP server at http://mock-mcp:3001 requires OAuth"
+            }
+        }
+    """
+    And The headers of the response contains the following header "www-authenticate"