From 167695e8521edd374a13fd2722f3e286b2472f91 Mon Sep 17 00:00:00 2001 From: JR Boos Date: Wed, 18 Feb 2026 16:39:45 -0500 Subject: [PATCH 01/16] Refactor MCP mock server configuration and update tests for OAuth support - Removed the old `mcp-mock-server` service and replaced it with `mock-mcp` in both `docker-compose` files. - Updated health check and dependencies to reflect the new service name. - Modified E2E test configurations to use the new `mock-mcp` service URL. - Added a minimal mock MCP server implementation with OAuth support for testing. - Updated feature tests to check for OAuth authentication requirements and response headers. --- docker-compose-library.yaml | 35 +++--- docker-compose.yaml | 35 +++--- lightspeed-stack.yaml.backup | 38 +++++++ .../library-mode/lightspeed-stack.yaml | 17 +-- .../server-mode/lightspeed-stack.yaml | 19 +--- tests/e2e/features/info.feature | 22 ++-- tests/e2e/features/steps/common_http.py | 9 ++ tests/e2e/mock_mcp_server/Dockerfile | 5 + tests/e2e/mock_mcp_server/server.py | 107 ++++++++++++++++++ 9 files changed, 217 insertions(+), 70 deletions(-) create mode 100644 lightspeed-stack.yaml.backup create mode 100644 tests/e2e/mock_mcp_server/Dockerfile create mode 100644 tests/e2e/mock_mcp_server/server.py diff --git a/docker-compose-library.yaml b/docker-compose-library.yaml index 0a991e1cf..2200d7f2c 100644 --- a/docker-compose-library.yaml +++ b/docker-compose-library.yaml @@ -1,20 +1,4 @@ services: - # Mock MCP server for testing - mcp-mock-server: - build: - context: . - dockerfile: dev-tools/mcp-mock-server/Dockerfile - container_name: mcp-mock-server - ports: - - "3000:3000" - networks: - - lightspeednet - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:3000/"] - interval: 5s - timeout: 3s - retries: 3 - start_period: 5s # Lightspeed Stack with embedded llama-stack (library mode) lightspeed-stack: @@ -26,7 +10,7 @@ services: ports: - "8080:8080" depends_on: - mcp-mock-server: + mock-mcp: condition: service_healthy networks: - lightspeednet @@ -93,6 +77,23 @@ services: retries: 3 start_period: 2s + mock-mcp: + build: + context: ./tests/e2e/mock_mcp_server + dockerfile: Dockerfile + container_name: mock-mcp + ports: + - "3000:3000" + networks: + - lightspeednet + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3000/health')"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 2s + + networks: lightspeednet: driver: bridge \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 339e702ea..beccefa12 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,20 +1,4 @@ services: - # Mock MCP server for testing - mcp-mock-server: - build: - context: . - dockerfile: dev-tools/mcp-mock-server/Dockerfile - container_name: mcp-mock-server - ports: - - "3000:3000" - networks: - - lightspeednet - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:3000/"] - interval: 5s - timeout: 3s - retries: 3 - start_period: 5s # Red Hat llama-stack distribution with FAISS llama-stack: @@ -90,7 +74,7 @@ services: depends_on: llama-stack: condition: service_healthy - mcp-mock-server: + mock-mcp: condition: service_healthy networks: - lightspeednet @@ -118,6 +102,23 @@ services: retries: 3 start_period: 2s + mock-mcp: + build: + context: ./tests/e2e/mock_mcp_server + dockerfile: Dockerfile + container_name: mock-mcp + ports: + - "3000:3000" + networks: + - lightspeednet + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3000/health')"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 2s + + volumes: llama-storage: diff --git a/lightspeed-stack.yaml.backup b/lightspeed-stack.yaml.backup new file mode 100644 index 000000000..68e97ea1c --- /dev/null +++ b/lightspeed-stack.yaml.backup @@ -0,0 +1,38 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + base_url: http://localhost:8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Uses a remote llama-stack service + # The instance would have already been started with a llama-stack-run.yaml file + use_as_library_client: false + # Alternative for "as library use" + # use_as_library_client: true + # library_client_config_path: + url: http://localhost:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" + +# Conversation cache for storing Q&A history +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" # Persistent across requests, can be deleted between test runs + +authentication: + module: "noop" + + +# OKP Solr for supplementary RAG +solr: + enabled: false + offline: true \ No newline at end of file diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack.yaml index 118b917c5..2572a7c8c 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack.yaml @@ -18,19 +18,8 @@ user_data_collection: authentication: module: "noop" mcp_servers: - # Mock server with client-provided auth - should appear in mcp-auth/client-options response - - name: "github-api" + - name: "mcp-oauth" provider_id: "model-context-protocol" - url: "http://mcp-mock-server:3000" + url: "http://mock-mcp:3000" authorization_headers: - Authorization: "client" - # Mock server with client-provided auth (different header) - should appear in response - - name: "gitlab-api" - provider_id: "model-context-protocol" - url: "http://mcp-mock-server:3000" - authorization_headers: - X-API-Token: "client" - # Mock server with no auth - should NOT appear in response - - name: "public-api" - provider_id: "model-context-protocol" - url: "http://mcp-mock-server:3000" \ No newline at end of file + Authorization: "oauth" \ No newline at end of file diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack.yaml index 1dbef61cf..c62dcdc75 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack.yaml @@ -9,7 +9,7 @@ service: llama_stack: # Server mode - connects to separate llama-stack service use_as_library_client: false - url: http://llama-stack:8321 + url: http://localhost:8321 api_key: xyzzy user_data_collection: feedback_enabled: true @@ -19,19 +19,8 @@ user_data_collection: authentication: module: "noop" mcp_servers: - # Mock server with client-provided auth - should appear in mcp-auth/client-options response - - name: "github-api" + - name: "mcp-oauth" provider_id: "model-context-protocol" - url: "http://mcp-mock-server:3000" + url: "http://localhost:3000" authorization_headers: - Authorization: "client" - # Mock server with client-provided auth (different header) - should appear in response - - name: "gitlab-api" - provider_id: "model-context-protocol" - url: "http://mcp-mock-server:3000" - authorization_headers: - X-API-Token: "client" - # Mock server with no auth - should NOT appear in response - - name: "public-api" - provider_id: "model-context-protocol" - url: "http://mcp-mock-server:3000" \ No newline at end of file + Authorization: "oauth" \ No newline at end of file diff --git a/tests/e2e/features/info.feature b/tests/e2e/features/info.feature index d1c20937e..f02545403 100644 --- a/tests/e2e/features/info.feature +++ b/tests/e2e/features/info.feature @@ -124,16 +124,24 @@ Feature: Info tests {"detail": {"response": "Unable to connect to Llama Stack", "cause": "Connection error."}} """ + Scenario: Check if tools endpoint reports error when mcp requires authentication + Given The system is in default state + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3000 requires OAuth" + } + } + """ + And The headers of the response contains the following "www-authenticate" + Scenario: Check if metrics endpoint is working Given The system is in default state When I access endpoint "metrics" using HTTP GET method Then The status code of the response is 200 And The body of the response contains ls_provider_model_configuration - Scenario: Check if MCP client auth options endpoint is working - Given The system is in default state - When I access REST API endpoint "mcp-auth/client-options" using HTTP GET method - Then The status code of the response is 200 - And The body of the response has proper client auth options structure - And The response contains server "github-api" with client auth header "Authorization" - And The response contains server "gitlab-api" with client auth header "X-API-Token" diff --git a/tests/e2e/features/steps/common_http.py b/tests/e2e/features/steps/common_http.py index d35dc6e79..10c9dde36 100644 --- a/tests/e2e/features/steps/common_http.py +++ b/tests/e2e/features/steps/common_http.py @@ -188,6 +188,15 @@ def check_prediction_result(context: Context) -> None: assert result == expected_body, f"got:\n{result}\nwant:\n{expected_body}" +@then("The headers of the response contains {substring}") +def check_response_headers_contains(context: Context, substring: str) -> None: + """Check that response body contains a substring.""" + assert context.response is not None, "Request needs to be performed first" + assert ( + substring in context.response.headers + ), f"The response headers '{context.response.headers}' doesn't contain '{substring}'" + + @then('The body of the response, ignoring the "{field}" field, is the following') def check_prediction_result_ignoring_field(context: Context, field: str) -> None: """Check the content of the response to be exactly the same. diff --git a/tests/e2e/mock_mcp_server/Dockerfile b/tests/e2e/mock_mcp_server/Dockerfile new file mode 100644 index 000000000..c7b23d5fc --- /dev/null +++ b/tests/e2e/mock_mcp_server/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-slim +WORKDIR /app +COPY server.py . +EXPOSE 3000 +CMD ["python", "server.py"] diff --git a/tests/e2e/mock_mcp_server/server.py b/tests/e2e/mock_mcp_server/server.py new file mode 100644 index 000000000..a78cbeb79 --- /dev/null +++ b/tests/e2e/mock_mcp_server/server.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Minimal mock MCP server for E2E tests with OAuth support. + +Responds to GET (OAuth probe) with 401 and WWW-Authenticate. Accepts POST +(MCP JSON-RPC) when Authorization: Bearer is present; otherwise 401. +Uses only Python stdlib. +""" + +import json +from http.server import HTTPServer, BaseHTTPRequestHandler +from typing import Any + +# Standard OAuth-style challenge so the client can drive an OAuth flow +WWW_AUTHENTICATE = 'Bearer realm="mock-mcp", error="invalid_token"' + + +class Handler(BaseHTTPRequestHandler): + """HTTP handler: GET/POST without valid Bearer → 401; POST with Bearer → MCP.""" + + def _require_oauth(self) -> None: + """Send 401 with WWW-Authenticate.""" + self.send_response(401) + self.send_header("WWW-Authenticate", WWW_AUTHENTICATE) + self.send_header("Content-Type", "application/json") + body = b'{"error":"unauthorized"}' + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _parse_auth(self) -> str | None: + """Return Bearer token if present, else None.""" + auth = self.headers.get("Authorization") + if auth and auth.startswith("Bearer "): + return auth[7:].strip() + return None + + def _json_response(self, data: dict) -> None: + """Send JSON response.""" + body = json.dumps(data).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: # pylint: disable=invalid-name + """OAuth probe: always 401 with WWW-Authenticate.""" + if self.path == "/health": + self._json_response({"status": "ok"}) + else: + self._require_oauth() + + def do_POST(self) -> None: # pylint: disable=invalid-name + """MCP JSON-RPC: 401 without valid Bearer; 200 with minimal responses otherwise.""" + if self._parse_auth() is None: + self._require_oauth() + return + + length = int(self.headers.get("Content-Length", 0)) + raw = self.rfile.read(length) if length else b"{}" + try: + req = json.loads(raw.decode("utf-8")) + req_id = req.get("id", 1) + method = req.get("method", "") + except (json.JSONDecodeError, UnicodeDecodeError): + req_id = 1 + method = "" + + if method == "initialize": + self._json_response( + { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "mock-mcp-e2e", "version": "1.0.0"}, + }, + } + ) + elif method == "tools/list": + self._json_response( + { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "tools": [ + { + "name": "mock_tool", + "description": "Mock tool for E2E", + "inputSchema": {"type": "object"}, + } + ], + }, + } + ) + else: + self._json_response({"jsonrpc": "2.0", "id": req_id, "result": {}}) + + def log_message(self, format: str, *args: Any) -> None: + """Suppress request logging for minimal output.""" + + +if __name__ == "__main__": + server = HTTPServer(("0.0.0.0", 3000), Handler) + print("Mock MCP server on :3000") + server.serve_forever() From 29c08c35c713c727668008c92b4f08375a88e52f Mon Sep 17 00:00:00 2001 From: JR Boos Date: Wed, 18 Feb 2026 16:39:45 -0500 Subject: [PATCH 02/16] Refactor MCP mock server configuration and update tests for OAuth support - Removed the old `mcp-mock-server` service and replaced it with `mock-mcp` in both `docker-compose` files. - Updated health check and dependencies to reflect the new service name. - Modified E2E test configurations to use the new `mock-mcp` service URL. - Added a minimal mock MCP server implementation with OAuth support for testing. - Updated feature tests to check for OAuth authentication requirements and response headers. --- tests/e2e/configuration/server-mode/lightspeed-stack.yaml | 2 +- tests/e2e/mock_mcp_server/server.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack.yaml index c62dcdc75..c6e03bc8e 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack.yaml @@ -21,6 +21,6 @@ authentication: mcp_servers: - name: "mcp-oauth" provider_id: "model-context-protocol" - url: "http://localhost:3000" + url: "http://mock-mcp:3000" authorization_headers: Authorization: "oauth" \ No newline at end of file diff --git a/tests/e2e/mock_mcp_server/server.py b/tests/e2e/mock_mcp_server/server.py index a78cbeb79..b23119621 100644 --- a/tests/e2e/mock_mcp_server/server.py +++ b/tests/e2e/mock_mcp_server/server.py @@ -44,14 +44,14 @@ def _json_response(self, data: dict) -> None: self.wfile.write(body) def do_GET(self) -> None: # pylint: disable=invalid-name - """OAuth probe: always 401 with WWW-Authenticate.""" + """Handle GET requests.""" if self.path == "/health": self._json_response({"status": "ok"}) else: self._require_oauth() def do_POST(self) -> None: # pylint: disable=invalid-name - """MCP JSON-RPC: 401 without valid Bearer; 200 with minimal responses otherwise.""" + """Handle POST requests.""" if self._parse_auth() is None: self._require_oauth() return From 340f6151b7e2e2882c9091139372b72cc0c2927a Mon Sep 17 00:00:00 2001 From: JR Boos Date: Wed, 18 Feb 2026 17:09:00 -0500 Subject: [PATCH 03/16] Remove backup configuration file and update llama-stack service URL in E2E tests --- lightspeed-stack.yaml.backup | 38 ------------------- .../server-mode/lightspeed-stack.yaml | 2 +- 2 files changed, 1 insertion(+), 39 deletions(-) delete mode 100644 lightspeed-stack.yaml.backup diff --git a/lightspeed-stack.yaml.backup b/lightspeed-stack.yaml.backup deleted file mode 100644 index 68e97ea1c..000000000 --- a/lightspeed-stack.yaml.backup +++ /dev/null @@ -1,38 +0,0 @@ -name: Lightspeed Core Service (LCS) -service: - host: 0.0.0.0 - port: 8080 - base_url: http://localhost:8080 - auth_enabled: false - workers: 1 - color_log: true - access_log: true -llama_stack: - # Uses a remote llama-stack service - # The instance would have already been started with a llama-stack-run.yaml file - use_as_library_client: false - # Alternative for "as library use" - # use_as_library_client: true - # library_client_config_path: - url: http://localhost:8321 - api_key: xyzzy -user_data_collection: - feedback_enabled: true - feedback_storage: "/tmp/data/feedback" - transcripts_enabled: true - transcripts_storage: "/tmp/data/transcripts" - -# Conversation cache for storing Q&A history -conversation_cache: - type: "sqlite" - sqlite: - db_path: "/tmp/data/conversation-cache.db" # Persistent across requests, can be deleted between test runs - -authentication: - module: "noop" - - -# OKP Solr for supplementary RAG -solr: - enabled: false - offline: true \ No newline at end of file diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack.yaml index c6e03bc8e..fc5de2cd1 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack.yaml @@ -9,7 +9,7 @@ service: llama_stack: # Server mode - connects to separate llama-stack service use_as_library_client: false - url: http://localhost:8321 + url: http://llama-stack:8321 api_key: xyzzy user_data_collection: feedback_enabled: true From 4f789c7505bc64d056a0cc223249bd3ca9f08b93 Mon Sep 17 00:00:00 2001 From: JR Boos Date: Sun, 22 Feb 2026 14:55:28 -0500 Subject: [PATCH 04/16] fixed /tools in library mode and added e2e tests for query and streaming_query --- src/app/endpoints/tools.py | 4 ++-- tests/e2e/features/query.feature | 21 +++++++++++++++++++++ tests/e2e/features/steps/common_http.py | 2 +- tests/e2e/features/streaming_query.feature | 20 ++++++++++++++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/app/endpoints/tools.py b/src/app/endpoints/tools.py index 2021ab5ed..d272a7d66 100644 --- a/src/app/endpoints/tools.py +++ b/src/app/endpoints/tools.py @@ -4,6 +4,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request from llama_stack_client import APIConnectionError, BadRequestError, AuthenticationError +from llama_stack.core.datatypes import AuthenticationRequiredError from authentication import get_auth_dependency from authentication.interface import AuthTuple @@ -90,8 +91,7 @@ async def tools_endpoint_handler( # pylint: disable=too-many-locals,too-many-st except BadRequestError: logger.error("Toolgroup %s is not found", toolgroup.identifier) continue - except AuthenticationError as e: - logger.error("Authentication error: %s", e) + except (AuthenticationError, AuthenticationRequiredError) as e: if toolgroup.mcp_endpoint: await probe_mcp_oauth_and_raise_401( toolgroup.mcp_endpoint.uri, chain_from=e diff --git a/tests/e2e/features/query.feature b/tests/e2e/features/query.feature index ac43b786b..14dbd73a1 100644 --- a/tests/e2e/features/query.feature +++ b/tests/e2e/features/query.feature @@ -216,3 +216,24 @@ Scenario: Check if LLM responds for query request with error for missing query } """ Then The status code of the response is 200 + + Scenario: Check if LLM responds to sent question with error when not authenticated + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3000 requires OAuth" + } + } + """ + And The headers of the response contains the following "www-authenticate" + + diff --git a/tests/e2e/features/steps/common_http.py b/tests/e2e/features/steps/common_http.py index 10c9dde36..8ba039b5f 100644 --- a/tests/e2e/features/steps/common_http.py +++ b/tests/e2e/features/steps/common_http.py @@ -193,7 +193,7 @@ def check_response_headers_contains(context: Context, substring: str) -> None: """Check that response body contains a substring.""" assert context.response is not None, "Request needs to be performed first" assert ( - substring in context.response.headers + substring in context.response.headers.keys() ), f"The response headers '{context.response.headers}' doesn't contain '{substring}'" diff --git a/tests/e2e/features/streaming_query.feature b/tests/e2e/features/streaming_query.feature index 22b3255b9..039df5f14 100644 --- a/tests/e2e/features/streaming_query.feature +++ b/tests/e2e/features/streaming_query.feature @@ -178,3 +178,23 @@ Feature: streaming_query endpoint API tests } } """ + + Scenario: Check if LLM responds to sent question with error when not authenticated + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I use "streaming_query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3000 requires OAuth" + } + } + """ + And The headers of the response contains the following "www-authenticate" + From 936431d9adc07ac434d053e63cff5af22776fde6 Mon Sep 17 00:00:00 2001 From: JR Boos Date: Sun, 22 Feb 2026 15:26:48 -0500 Subject: [PATCH 05/16] Update E2E tests to standardize header checks - Modified feature files to change the wording for checking response headers from "contains the following" to "contains the following header". - Updated the corresponding step definition in common_http.py to reflect the new header check format. --- tests/e2e/features/info.feature | 2 +- tests/e2e/features/query.feature | 2 +- tests/e2e/features/steps/common_http.py | 10 +++++----- tests/e2e/features/streaming_query.feature | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/e2e/features/info.feature b/tests/e2e/features/info.feature index f02545403..8fd48c7f6 100644 --- a/tests/e2e/features/info.feature +++ b/tests/e2e/features/info.feature @@ -137,7 +137,7 @@ Feature: Info tests } } """ - And The headers of the response contains the following "www-authenticate" + And The headers of the response contains the following header "www-authenticate" Scenario: Check if metrics endpoint is working Given The system is in default state diff --git a/tests/e2e/features/query.feature b/tests/e2e/features/query.feature index 14dbd73a1..92967d059 100644 --- a/tests/e2e/features/query.feature +++ b/tests/e2e/features/query.feature @@ -234,6 +234,6 @@ Scenario: Check if LLM responds for query request with error for missing query } } """ - And The headers of the response contains the following "www-authenticate" + And The headers of the response contains the following header "www-authenticate" diff --git a/tests/e2e/features/steps/common_http.py b/tests/e2e/features/steps/common_http.py index 8ba039b5f..015830219 100644 --- a/tests/e2e/features/steps/common_http.py +++ b/tests/e2e/features/steps/common_http.py @@ -188,13 +188,13 @@ def check_prediction_result(context: Context) -> None: assert result == expected_body, f"got:\n{result}\nwant:\n{expected_body}" -@then("The headers of the response contains {substring}") -def check_response_headers_contains(context: Context, substring: str) -> None: - """Check that response body contains a substring.""" +@then('The headers of the response contains the following header "{header_name}"') +def check_response_headers_contains(context: Context, header_name: str) -> None: + """Check that response contains a header whose name matches.""" assert context.response is not None, "Request needs to be performed first" assert ( - substring in context.response.headers.keys() - ), f"The response headers '{context.response.headers}' doesn't contain '{substring}'" + header_name in context.response.headers.keys() + ), f"The response headers '{context.response.headers}' doesn't contain header '{header_name}'" @then('The body of the response, ignoring the "{field}" field, is the following') diff --git a/tests/e2e/features/streaming_query.feature b/tests/e2e/features/streaming_query.feature index 039df5f14..830de87ec 100644 --- a/tests/e2e/features/streaming_query.feature +++ b/tests/e2e/features/streaming_query.feature @@ -196,5 +196,5 @@ Feature: streaming_query endpoint API tests } } """ - And The headers of the response contains the following "www-authenticate" + And The headers of the response contains the following header "www-authenticate" From 8b2673fb7c79c4d010cfcf3fd648633ab01e34bf Mon Sep 17 00:00:00 2001 From: JR Boos Date: Mon, 23 Feb 2026 10:11:21 -0500 Subject: [PATCH 06/16] Placed MCP tests into new feature - Introduced new feature tests for MCP authentication scenarios, including checks for tools, query, and streaming_query endpoints. - Added MCP configuration files for both library and server modes. - Updated the environment setup to switch configurations based on feature tags. - Removed outdated authentication checks from existing feature files to streamline tests. --- .../library-mode/lightspeed-stack-mcp.yaml | 25 ++++++++ .../library-mode/lightspeed-stack.yaml | 8 +-- .../server-mode/lightspeed-stack-mcp.yaml | 26 ++++++++ .../server-mode/lightspeed-stack.yaml | 8 +-- tests/e2e/features/environment.py | 14 +++++ tests/e2e/features/info.feature | 16 ----- tests/e2e/features/mcp.feature | 59 +++++++++++++++++++ tests/e2e/features/query.feature | 23 +------- tests/e2e/features/streaming_query.feature | 22 +------ tests/e2e/test_list.txt | 1 + 10 files changed, 129 insertions(+), 73 deletions(-) create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml create mode 100644 tests/e2e/features/mcp.feature diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml new file mode 100644 index 000000000..2572a7c8c --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml @@ -0,0 +1,25 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds llama-stack as library + use_as_library_client: true + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-oauth" + provider_id: "model-context-protocol" + url: "http://mock-mcp:3000" + authorization_headers: + Authorization: "oauth" \ No newline at end of file diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack.yaml index 2572a7c8c..e6d02d3a6 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack.yaml @@ -16,10 +16,4 @@ user_data_collection: transcripts_enabled: true transcripts_storage: "/tmp/data/transcripts" authentication: - module: "noop" -mcp_servers: - - name: "mcp-oauth" - provider_id: "model-context-protocol" - url: "http://mock-mcp:3000" - authorization_headers: - Authorization: "oauth" \ No newline at end of file + module: "noop" \ No newline at end of file diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml new file mode 100644 index 000000000..fc5de2cd1 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml @@ -0,0 +1,26 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to separate llama-stack service + use_as_library_client: false + url: http://llama-stack:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-oauth" + provider_id: "model-context-protocol" + url: "http://mock-mcp:3000" + authorization_headers: + Authorization: "oauth" \ No newline at end of file diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack.yaml index fc5de2cd1..adc5b4829 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack.yaml @@ -17,10 +17,4 @@ user_data_collection: transcripts_enabled: true transcripts_storage: "/tmp/data/transcripts" authentication: - module: "noop" -mcp_servers: - - name: "mcp-oauth" - provider_id: "model-context-protocol" - url: "http://mock-mcp:3000" - authorization_headers: - Authorization: "oauth" \ No newline at end of file + module: "noop" \ No newline at end of file diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 3d6a4fdae..f1b609fc3 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -267,6 +267,15 @@ def before_feature(context: Context, feature: Feature) -> None: context.port = os.getenv("E2E_LSC_PORT", "8080") context.feedback_conversations = [] + if "MCP" in feature.tags: + mode_dir = "library-mode" if context.is_library_mode else "server-mode" + context.feature_config = ( + f"tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp.yaml" + ) + context.default_config_backup = create_config_backup("lightspeed-stack.yaml") + switch_config(context.feature_config) + restart_container("lightspeed-stack") + def after_feature(context: Context, feature: Feature) -> None: """Run after each feature file is exercised. @@ -295,3 +304,8 @@ def after_feature(context: Context, feature: Feature) -> None: headers = context.auth_headers if hasattr(context, "auth_headers") else {} response = requests.delete(url, headers=headers) assert response.status_code == 200, url + + if "MCP" in feature.tags: + switch_config(context.default_config_backup) + restart_container("lightspeed-stack") + remove_config_backup(context.default_config_backup) diff --git a/tests/e2e/features/info.feature b/tests/e2e/features/info.feature index 8fd48c7f6..014b9dbb4 100644 --- a/tests/e2e/features/info.feature +++ b/tests/e2e/features/info.feature @@ -124,24 +124,8 @@ Feature: Info tests {"detail": {"response": "Unable to connect to Llama Stack", "cause": "Connection error."}} """ - Scenario: Check if tools endpoint reports error when mcp requires authentication - Given The system is in default state - When I access REST API endpoint "tools" using HTTP GET method - Then The status code of the response is 401 - And The body of the response is the following - """ - { - "detail": { - "response": "Missing or invalid credentials provided by client", - "cause": "MCP server at http://mock-mcp:3000 requires OAuth" - } - } - """ - And The headers of the response contains the following header "www-authenticate" - Scenario: Check if metrics endpoint is working Given The system is in default state When I access endpoint "metrics" using HTTP GET method Then The status code of the response is 200 And The body of the response contains ls_provider_model_configuration - diff --git a/tests/e2e/features/mcp.feature b/tests/e2e/features/mcp.feature new file mode 100644 index 000000000..a132149c1 --- /dev/null +++ b/tests/e2e/features/mcp.feature @@ -0,0 +1,59 @@ +@MCP +Feature: MCP tests + + Background: + Given The service is started locally + And REST API service prefix is /v1 + + Scenario: Check if tools endpoint reports error when mcp requires authentication + Given The system is in default state + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3000 requires OAuth" + } + } + """ + And The headers of the response contains the following header "www-authenticate" + + Scenario: Check if query endpoint reports error error when mcp requires authentication + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3000 requires OAuth" + } + } + """ + And The headers of the response contains the following header "www-authenticate" + + Scenario: Check if streaming_query endpoint reports error error when mcp requires authentication + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I use "streaming_query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3000 requires OAuth" + } + } + """ + And The headers of the response contains the following header "www-authenticate" diff --git a/tests/e2e/features/query.feature b/tests/e2e/features/query.feature index 92967d059..298ae7251 100644 --- a/tests/e2e/features/query.feature +++ b/tests/e2e/features/query.feature @@ -215,25 +215,4 @@ Scenario: Check if LLM responds for query request with error for missing query "system_prompt": "You are a helpful assistant" } """ - Then The status code of the response is 200 - - Scenario: Check if LLM responds to sent question with error when not authenticated - Given The system is in default state - And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva - When I use "query" to ask question with authorization header - """ - {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 401 - And The body of the response is the following - """ - { - "detail": { - "response": "Missing or invalid credentials provided by client", - "cause": "MCP server at http://mock-mcp:3000 requires OAuth" - } - } - """ - And The headers of the response contains the following header "www-authenticate" - - + Then The status code of the response is 200 \ No newline at end of file diff --git a/tests/e2e/features/streaming_query.feature b/tests/e2e/features/streaming_query.feature index 830de87ec..867623850 100644 --- a/tests/e2e/features/streaming_query.feature +++ b/tests/e2e/features/streaming_query.feature @@ -177,24 +177,4 @@ Feature: streaming_query endpoint API tests "cause": "No Authorization header found" } } - """ - - Scenario: Check if LLM responds to sent question with error when not authenticated - Given The system is in default state - And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva - When I use "streaming_query" to ask question with authorization header - """ - {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 401 - And The body of the response is the following - """ - { - "detail": { - "response": "Missing or invalid credentials provided by client", - "cause": "MCP server at http://mock-mcp:3000 requires OAuth" - } - } - """ - And The headers of the response contains the following header "www-authenticate" - + """ \ No newline at end of file diff --git a/tests/e2e/test_list.txt b/tests/e2e/test_list.txt index 804e180cf..b39f68870 100644 --- a/tests/e2e/test_list.txt +++ b/tests/e2e/test_list.txt @@ -12,3 +12,4 @@ features/info.feature features/query.feature features/streaming_query.feature features/rest_api.feature +features/mcp.feature From 43517cdfda60ed9536f0531a256556629812fc80 Mon Sep 17 00:00:00 2001 From: JR Boos Date: Mon, 23 Feb 2026 10:57:14 -0500 Subject: [PATCH 07/16] Restored other mock mcp server --- docker-compose-library.yaml | 18 ++++++++++++++++++ docker-compose.yaml | 18 ++++++++++++++++++ .../library-mode/lightspeed-stack.yaml | 19 ++++++++++++++++++- .../server-mode/lightspeed-stack.yaml | 19 ++++++++++++++++++- tests/e2e/features/info.feature | 8 ++++++++ 5 files changed, 80 insertions(+), 2 deletions(-) diff --git a/docker-compose-library.yaml b/docker-compose-library.yaml index 2200d7f2c..01f5b8c8d 100644 --- a/docker-compose-library.yaml +++ b/docker-compose-library.yaml @@ -1,4 +1,20 @@ services: + # Mock MCP server for testing + mcp-mock-server: + build: + context: . + dockerfile: dev-tools/mcp-mock-server/Dockerfile + container_name: mcp-mock-server + ports: + - "3000:3000" + networks: + - lightspeednet + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 5s # Lightspeed Stack with embedded llama-stack (library mode) lightspeed-stack: @@ -10,6 +26,8 @@ services: ports: - "8080:8080" depends_on: + mcp-mock-server: + condition: service_healthy mock-mcp: condition: service_healthy networks: diff --git a/docker-compose.yaml b/docker-compose.yaml index beccefa12..bafe0700c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,4 +1,20 @@ services: + # Mock MCP server for testing + mcp-mock-server: + build: + context: . + dockerfile: dev-tools/mcp-mock-server/Dockerfile + container_name: mcp-mock-server + ports: + - "3000:3000" + networks: + - lightspeednet + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 5s # Red Hat llama-stack distribution with FAISS llama-stack: @@ -74,6 +90,8 @@ services: depends_on: llama-stack: condition: service_healthy + mcp-mock-server: + condition: service_healthy mock-mcp: condition: service_healthy networks: diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack.yaml index e6d02d3a6..118b917c5 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack.yaml @@ -16,4 +16,21 @@ user_data_collection: transcripts_enabled: true transcripts_storage: "/tmp/data/transcripts" authentication: - module: "noop" \ No newline at end of file + module: "noop" +mcp_servers: + # Mock server with client-provided auth - should appear in mcp-auth/client-options response + - name: "github-api" + provider_id: "model-context-protocol" + url: "http://mcp-mock-server:3000" + authorization_headers: + Authorization: "client" + # Mock server with client-provided auth (different header) - should appear in response + - name: "gitlab-api" + provider_id: "model-context-protocol" + url: "http://mcp-mock-server:3000" + authorization_headers: + X-API-Token: "client" + # Mock server with no auth - should NOT appear in response + - name: "public-api" + provider_id: "model-context-protocol" + url: "http://mcp-mock-server:3000" \ No newline at end of file diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack.yaml index adc5b4829..1dbef61cf 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack.yaml @@ -17,4 +17,21 @@ user_data_collection: transcripts_enabled: true transcripts_storage: "/tmp/data/transcripts" authentication: - module: "noop" \ No newline at end of file + module: "noop" +mcp_servers: + # Mock server with client-provided auth - should appear in mcp-auth/client-options response + - name: "github-api" + provider_id: "model-context-protocol" + url: "http://mcp-mock-server:3000" + authorization_headers: + Authorization: "client" + # Mock server with client-provided auth (different header) - should appear in response + - name: "gitlab-api" + provider_id: "model-context-protocol" + url: "http://mcp-mock-server:3000" + authorization_headers: + X-API-Token: "client" + # Mock server with no auth - should NOT appear in response + - name: "public-api" + provider_id: "model-context-protocol" + url: "http://mcp-mock-server:3000" \ No newline at end of file diff --git a/tests/e2e/features/info.feature b/tests/e2e/features/info.feature index 014b9dbb4..bfd172c67 100644 --- a/tests/e2e/features/info.feature +++ b/tests/e2e/features/info.feature @@ -129,3 +129,11 @@ Feature: Info tests When I access endpoint "metrics" using HTTP GET method Then The status code of the response is 200 And The body of the response contains ls_provider_model_configuration + + Scenario: Check if MCP client auth options endpoint is working + Given The system is in default state + When I access REST API endpoint "mcp-auth/client-options" using HTTP GET method + Then The status code of the response is 200 + And The body of the response has proper client auth options structure + And The response contains server "github-api" with client auth header "Authorization" + And The response contains server "gitlab-api" with client auth header "X-API-Token" \ No newline at end of file From 10d096ec1005c09bff3510ef8b1e7f50ab057cce Mon Sep 17 00:00:00 2001 From: JR Boos Date: Mon, 23 Feb 2026 11:03:28 -0500 Subject: [PATCH 08/16] Fixed already allocated port --- docker-compose-library.yaml | 4 ++-- docker-compose.yaml | 4 ++-- .../configuration/library-mode/lightspeed-stack-mcp.yaml | 2 +- .../e2e/configuration/server-mode/lightspeed-stack-mcp.yaml | 2 +- tests/e2e/features/mcp.feature | 6 +++--- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docker-compose-library.yaml b/docker-compose-library.yaml index 01f5b8c8d..3c198c0a8 100644 --- a/docker-compose-library.yaml +++ b/docker-compose-library.yaml @@ -101,11 +101,11 @@ services: dockerfile: Dockerfile container_name: mock-mcp ports: - - "3000:3000" + - "3001:3001" networks: - lightspeednet healthcheck: - test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3000/health')"] + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3001/health')"] interval: 5s timeout: 3s retries: 3 diff --git a/docker-compose.yaml b/docker-compose.yaml index bafe0700c..85a38bc62 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -126,11 +126,11 @@ services: dockerfile: Dockerfile container_name: mock-mcp ports: - - "3000:3000" + - "3001:3001" networks: - lightspeednet healthcheck: - test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3000/health')"] + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3001/health')"] interval: 5s timeout: 3s retries: 3 diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml index 2572a7c8c..0656aa87c 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml @@ -20,6 +20,6 @@ authentication: mcp_servers: - name: "mcp-oauth" provider_id: "model-context-protocol" - url: "http://mock-mcp:3000" + url: "http://mock-mcp:3001" authorization_headers: Authorization: "oauth" \ No newline at end of file diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml index fc5de2cd1..a598ce441 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml @@ -21,6 +21,6 @@ authentication: mcp_servers: - name: "mcp-oauth" provider_id: "model-context-protocol" - url: "http://mock-mcp:3000" + url: "http://mock-mcp:3001" authorization_headers: Authorization: "oauth" \ No newline at end of file diff --git a/tests/e2e/features/mcp.feature b/tests/e2e/features/mcp.feature index a132149c1..f0d65f547 100644 --- a/tests/e2e/features/mcp.feature +++ b/tests/e2e/features/mcp.feature @@ -14,7 +14,7 @@ Feature: MCP tests { "detail": { "response": "Missing or invalid credentials provided by client", - "cause": "MCP server at http://mock-mcp:3000 requires OAuth" + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" } } """ @@ -33,7 +33,7 @@ Feature: MCP tests { "detail": { "response": "Missing or invalid credentials provided by client", - "cause": "MCP server at http://mock-mcp:3000 requires OAuth" + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" } } """ @@ -52,7 +52,7 @@ Feature: MCP tests { "detail": { "response": "Missing or invalid credentials provided by client", - "cause": "MCP server at http://mock-mcp:3000 requires OAuth" + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" } } """ From bc00db3e8a13ad01ec1d83b744eb491ca29f6586 Mon Sep 17 00:00:00 2001 From: JR Boos Date: Mon, 23 Feb 2026 11:04:31 -0500 Subject: [PATCH 09/16] Fixed allocated port --- tests/e2e/mock_mcp_server/Dockerfile | 2 +- tests/e2e/mock_mcp_server/server.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e/mock_mcp_server/Dockerfile b/tests/e2e/mock_mcp_server/Dockerfile index c7b23d5fc..b641ca4c4 100644 --- a/tests/e2e/mock_mcp_server/Dockerfile +++ b/tests/e2e/mock_mcp_server/Dockerfile @@ -1,5 +1,5 @@ FROM python:3.12-slim WORKDIR /app COPY server.py . -EXPOSE 3000 +EXPOSE 3001 CMD ["python", "server.py"] diff --git a/tests/e2e/mock_mcp_server/server.py b/tests/e2e/mock_mcp_server/server.py index b23119621..d77ab6ea6 100644 --- a/tests/e2e/mock_mcp_server/server.py +++ b/tests/e2e/mock_mcp_server/server.py @@ -102,6 +102,6 @@ def log_message(self, format: str, *args: Any) -> None: if __name__ == "__main__": - server = HTTPServer(("0.0.0.0", 3000), Handler) - print("Mock MCP server on :3000") + server = HTTPServer(("0.0.0.0", 3001), Handler) + print("Mock MCP server on :3001") server.serve_forever() From 5614c155b7996aae5e7833c272d5d26cfc6c7473 Mon Sep 17 00:00:00 2001 From: JR Boos Date: Mon, 23 Feb 2026 14:47:17 -0500 Subject: [PATCH 10/16] Fixed `tools/list` for mock server and addressed coderabbit - Also removed authentication from `mcp.feature` --- tests/e2e/features/mcp.feature | 10 ++++------ tests/e2e/mock_mcp_server/server.py | 18 +++++++++++++----- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/e2e/features/mcp.feature b/tests/e2e/features/mcp.feature index f0d65f547..2b7849111 100644 --- a/tests/e2e/features/mcp.feature +++ b/tests/e2e/features/mcp.feature @@ -20,10 +20,9 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - Scenario: Check if query endpoint reports error error when mcp requires authentication + Scenario: Check if query endpoint reports error when mcp requires authentication Given The system is in default state - And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva - When I use "query" to ask question with authorization header + When I use "query" to ask question """ {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} """ @@ -39,10 +38,9 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - Scenario: Check if streaming_query endpoint reports error error when mcp requires authentication + Scenario: Check if streaming_query endpoint reports error when mcp requires authentication Given The system is in default state - And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva - When I use "streaming_query" to ask question with authorization header + When I use "streaming_query" to ask question """ {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} """ diff --git a/tests/e2e/mock_mcp_server/server.py b/tests/e2e/mock_mcp_server/server.py index d77ab6ea6..68d4105fa 100644 --- a/tests/e2e/mock_mcp_server/server.py +++ b/tests/e2e/mock_mcp_server/server.py @@ -85,11 +85,19 @@ def do_POST(self) -> None: # pylint: disable=invalid-name "id": req_id, "result": { "tools": [ - { - "name": "mock_tool", - "description": "Mock tool for E2E", - "inputSchema": {"type": "object"}, - } + { + "name": "mock_tool_e2e", + "description": "Mock tool for E2E", + "inputSchema": { + "type": "object", + "properties": { + "message": { + "type": "string", + "description": "Test message", + } + }, + }, + } ], }, } From cf27b93e5bdb81aa0999b529e6206fd66fcf4bff Mon Sep 17 00:00:00 2001 From: JR Boos Date: Mon, 23 Feb 2026 15:30:01 -0500 Subject: [PATCH 11/16] fixed black --- tests/e2e/mock_mcp_server/server.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/e2e/mock_mcp_server/server.py b/tests/e2e/mock_mcp_server/server.py index 68d4105fa..820e2f9d9 100644 --- a/tests/e2e/mock_mcp_server/server.py +++ b/tests/e2e/mock_mcp_server/server.py @@ -85,19 +85,19 @@ def do_POST(self) -> None: # pylint: disable=invalid-name "id": req_id, "result": { "tools": [ - { - "name": "mock_tool_e2e", - "description": "Mock tool for E2E", - "inputSchema": { - "type": "object", - "properties": { - "message": { - "type": "string", - "description": "Test message", - } + { + "name": "mock_tool_e2e", + "description": "Mock tool for E2E", + "inputSchema": { + "type": "object", + "properties": { + "message": { + "type": "string", + "description": "Test message", + } + }, }, - }, - } + } ], }, } From a4050c3a6fa45811cc5d65fb0e98c39486a02703 Mon Sep 17 00:00:00 2001 From: JR Boos Date: Tue, 24 Feb 2026 11:16:41 -0500 Subject: [PATCH 12/16] Add new end-to-end tests for MCP authentication scenarios - Implemented scenarios to verify successful and error responses for the 'tools', 'query', and 'streaming_query' endpoints when valid and invalid MCP auth tokens are provided. - Enhanced the common HTTP step definitions to support setting headers dynamically. - Updated the mock MCP server to handle invalid tokens appropriately. --- tests/e2e/features/mcp.feature | 147 ++++++++++++++++++ tests/e2e/features/steps/common_http.py | 9 ++ .../e2e/features/steps/llm_query_response.py | 1 + tests/e2e/mock_mcp_server/server.py | 2 +- 4 files changed, 158 insertions(+), 1 deletion(-) diff --git a/tests/e2e/features/mcp.feature b/tests/e2e/features/mcp.feature index 2b7849111..24ac7a7b4 100644 --- a/tests/e2e/features/mcp.feature +++ b/tests/e2e/features/mcp.feature @@ -55,3 +55,150 @@ Feature: MCP tests } """ And The headers of the response contains the following header "www-authenticate" + + @skip # will be fixed in LCORE-1368 + Scenario: Check if tools endpoint succeeds when mcp auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-oauth": {"Authorization": "Bearer test-token"}} + """ + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 200 + And The body of the response is the following + """ + { + "tools": [ + { + "identifier": "", + "description": "Insert documents into memory", + "parameters": [], + "provider_id": "", + "toolgroup_id": "builtin::rag", + "server_source": "builtin", + "type": "" + }, + { + "identifier": "", + "description": "Search for information in a database.", + "parameters": [], + "provider_id": "", + "toolgroup_id": "builtin::rag", + "server_source": "builtin", + "type": "" + }, + { + "identifier": "", + "description": "Mock tool for E2E", + "parameters": [], + "provider_id": "", + "toolgroup_id": "mcp-oauth", + "server_source": "http://localhost:3001", + "type": "" + } + ] + } + """ + + @skip # will be fixed in LCORE-1366 + Scenario: Check if query endpoint succeeds when mcp auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-oauth": {"Authorization": "Bearer test-token"}} + """ + And I capture the current token metrics + When I use "query" to ask question with headers + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response should contain following fragments + | Fragments in LLM response | + | hello | + And The token metrics should have increased + + @skip # will be fixed in LCORE-1366 + Scenario: Check if streaming_query endpoint succeeds when mcp auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-oauth": {"Authorization": "Bearer test-token"}} + """ + And I capture the current token metrics + When I use "streaming_query" to ask question with headers + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + When I wait for the response to be completed + Then The status code of the response is 200 + And The streamed response should contain following fragments + | Fragments in LLM response | + | hello | + And The token metrics should have increased + + @skip # will be fixed in LCORE-1368 + Scenario: Check if tools endpoint reports error when mcp invalid auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-oauth": {"Authorization": "Bearer invalid-token"}} + """ + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + And The headers of the response contains the following header "www-authenticate" + + @skip # will be fixed in LCORE-1366 + Scenario: Check if query endpoint reports error when mcp invalid auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-oauth": {"Authorization": "Bearer invalid-token"}} + """ + When I use "query" to ask question with headers + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + And The headers of the response contains the following header "www-authenticate" + + @skip # will be fixed in LCORE-1366 + Scenario: Check if streaming_query endpoint reports error when mcp invalid auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-oauth": {"Authorization": "Bearer invalid-token"}} + """ + When I use "streaming_query" to ask question with headers + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + And The headers of the response contains the following header "www-authenticate" diff --git a/tests/e2e/features/steps/common_http.py b/tests/e2e/features/steps/common_http.py index 015830219..fb0d58225 100644 --- a/tests/e2e/features/steps/common_http.py +++ b/tests/e2e/features/steps/common_http.py @@ -395,3 +395,12 @@ def check_response_partially(context: Context) -> None: json_str = replace_placeholders(context, context.text or "{}") expected = json.loads(json_str) validate_json_partially(body, expected) + +@given('I set the "{header_name}" header to') +def set_header(context: Context, header_name: str) -> None: + """Set a header in the request.""" + assert context.text is not None, "Header value needs to be specified" + + if not hasattr(context, "auth_headers"): + context.auth_headers = {} + context.auth_headers[header_name] = context.text diff --git a/tests/e2e/features/steps/llm_query_response.py b/tests/e2e/features/steps/llm_query_response.py index 732f6e291..c01756f2f 100644 --- a/tests/e2e/features/steps/llm_query_response.py +++ b/tests/e2e/features/steps/llm_query_response.py @@ -33,6 +33,7 @@ def ask_question(context: Context, endpoint: str) -> None: context.response = requests.post(url, json=data, timeout=DEFAULT_LLM_TIMEOUT) +@step('I use "{endpoint}" to ask question with headers') @step('I use "{endpoint}" to ask question with authorization header') def ask_question_authorized(context: Context, endpoint: str) -> None: """Call the service REST API endpoint with question.""" diff --git a/tests/e2e/mock_mcp_server/server.py b/tests/e2e/mock_mcp_server/server.py index 820e2f9d9..2d2768ed6 100644 --- a/tests/e2e/mock_mcp_server/server.py +++ b/tests/e2e/mock_mcp_server/server.py @@ -30,7 +30,7 @@ def _require_oauth(self) -> None: def _parse_auth(self) -> str | None: """Return Bearer token if present, else None.""" auth = self.headers.get("Authorization") - if auth and auth.startswith("Bearer "): + if auth and auth.startswith("Bearer ") and "invalid" not in auth: return auth[7:].strip() return None From ed459d6009b218e16fba5da58feb45c95f4d2bed Mon Sep 17 00:00:00 2001 From: JR Boos Date: Tue, 24 Feb 2026 11:18:53 -0500 Subject: [PATCH 13/16] fixed black and ruff --- tests/e2e/features/steps/common_http.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/e2e/features/steps/common_http.py b/tests/e2e/features/steps/common_http.py index fb0d58225..8e64fe5fc 100644 --- a/tests/e2e/features/steps/common_http.py +++ b/tests/e2e/features/steps/common_http.py @@ -3,7 +3,12 @@ import json import requests -from behave import then, when, step # pyright: ignore[reportAttributeAccessIssue] +from behave import ( + then, + when, + step, + given, +) # pyright: ignore[reportAttributeAccessIssue] from behave.runner import Context from tests.e2e.utils.utils import ( normalize_endpoint, @@ -396,6 +401,7 @@ def check_response_partially(context: Context) -> None: expected = json.loads(json_str) validate_json_partially(body, expected) + @given('I set the "{header_name}" header to') def set_header(context: Context, header_name: str) -> None: """Set a header in the request.""" From 48259f487f8955cf5bb5a82f90fdc83b43986f20 Mon Sep 17 00:00:00 2001 From: JR Boos Date: Tue, 24 Feb 2026 14:15:18 -0500 Subject: [PATCH 14/16] addresed comments --- tests/e2e/features/info.feature | 2 +- tests/e2e/features/mcp.feature | 20 ++++++++++---------- tests/e2e/features/query.feature | 2 +- tests/e2e/features/streaming_query.feature | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/e2e/features/info.feature b/tests/e2e/features/info.feature index f5951b485..38ce3ba39 100644 --- a/tests/e2e/features/info.feature +++ b/tests/e2e/features/info.feature @@ -118,4 +118,4 @@ Feature: Info tests Then The status code of the response is 200 And The body of the response has proper client auth options structure And The response contains server "github-api" with client auth header "Authorization" - And The response contains server "gitlab-api" with client auth header "X-API-Token" \ No newline at end of file + And The response contains server "gitlab-api" with client auth header "X-API-Token" diff --git a/tests/e2e/features/mcp.feature b/tests/e2e/features/mcp.feature index 24ac7a7b4..5cc0ecdae 100644 --- a/tests/e2e/features/mcp.feature +++ b/tests/e2e/features/mcp.feature @@ -5,7 +5,7 @@ Feature: MCP tests Given The service is started locally And REST API service prefix is /v1 - Scenario: Check if tools endpoint reports error when mcp requires authentication + Scenario: Check if tools endpoint reports error when MCP requires authentication Given The system is in default state When I access REST API endpoint "tools" using HTTP GET method Then The status code of the response is 401 @@ -20,7 +20,7 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - Scenario: Check if query endpoint reports error when mcp requires authentication + Scenario: Check if query endpoint reports error when MCP requires authentication Given The system is in default state When I use "query" to ask question """ @@ -38,7 +38,7 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - Scenario: Check if streaming_query endpoint reports error when mcp requires authentication + Scenario: Check if streaming_query endpoint reports error when MCP requires authentication Given The system is in default state When I use "streaming_query" to ask question """ @@ -57,7 +57,7 @@ Feature: MCP tests And The headers of the response contains the following header "www-authenticate" @skip # will be fixed in LCORE-1368 - Scenario: Check if tools endpoint succeeds when mcp auth token is passed + Scenario: Check if tools endpoint succeeds when MCP auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ @@ -101,7 +101,7 @@ Feature: MCP tests """ @skip # will be fixed in LCORE-1366 - Scenario: Check if query endpoint succeeds when mcp auth token is passed + Scenario: Check if query endpoint succeeds when MCP auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ @@ -119,7 +119,7 @@ Feature: MCP tests And The token metrics should have increased @skip # will be fixed in LCORE-1366 - Scenario: Check if streaming_query endpoint succeeds when mcp auth token is passed + Scenario: Check if streaming_query endpoint succeeds when MCP auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ @@ -134,11 +134,11 @@ Feature: MCP tests Then The status code of the response is 200 And The streamed response should contain following fragments | Fragments in LLM response | - | hello | + | hello | And The token metrics should have increased @skip # will be fixed in LCORE-1368 - Scenario: Check if tools endpoint reports error when mcp invalid auth token is passed + Scenario: Check if tools endpoint reports error when MCP invalid auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ @@ -158,7 +158,7 @@ Feature: MCP tests And The headers of the response contains the following header "www-authenticate" @skip # will be fixed in LCORE-1366 - Scenario: Check if query endpoint reports error when mcp invalid auth token is passed + Scenario: Check if query endpoint reports error when MCP invalid auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ @@ -181,7 +181,7 @@ Feature: MCP tests And The headers of the response contains the following header "www-authenticate" @skip # will be fixed in LCORE-1366 - Scenario: Check if streaming_query endpoint reports error when mcp invalid auth token is passed + Scenario: Check if streaming_query endpoint reports error when MCP invalid auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ diff --git a/tests/e2e/features/query.feature b/tests/e2e/features/query.feature index 298ae7251..ac43b786b 100644 --- a/tests/e2e/features/query.feature +++ b/tests/e2e/features/query.feature @@ -215,4 +215,4 @@ Scenario: Check if LLM responds for query request with error for missing query "system_prompt": "You are a helpful assistant" } """ - Then The status code of the response is 200 \ No newline at end of file + Then The status code of the response is 200 diff --git a/tests/e2e/features/streaming_query.feature b/tests/e2e/features/streaming_query.feature index 867623850..22b3255b9 100644 --- a/tests/e2e/features/streaming_query.feature +++ b/tests/e2e/features/streaming_query.feature @@ -177,4 +177,4 @@ Feature: streaming_query endpoint API tests "cause": "No Authorization header found" } } - """ \ No newline at end of file + """ From c5ebcd93df75094195129893576c052e9d97f2dc Mon Sep 17 00:00:00 2001 From: JR Boos Date: Wed, 25 Feb 2026 10:37:53 -0500 Subject: [PATCH 15/16] addressed comments --- tests/e2e/features/mcp.feature | 8 ++++---- tests/e2e/features/steps/llm_query_response.py | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/e2e/features/mcp.feature b/tests/e2e/features/mcp.feature index 5cc0ecdae..ae3da6ab0 100644 --- a/tests/e2e/features/mcp.feature +++ b/tests/e2e/features/mcp.feature @@ -108,7 +108,7 @@ Feature: MCP tests {"mcp-oauth": {"Authorization": "Bearer test-token"}} """ And I capture the current token metrics - When I use "query" to ask question with headers + When I use "query" to ask question with authorization header """ {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} """ @@ -126,7 +126,7 @@ Feature: MCP tests {"mcp-oauth": {"Authorization": "Bearer test-token"}} """ And I capture the current token metrics - When I use "streaming_query" to ask question with headers + When I use "streaming_query" to ask question with authorization header """ {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} """ @@ -164,7 +164,7 @@ Feature: MCP tests """ {"mcp-oauth": {"Authorization": "Bearer invalid-token"}} """ - When I use "query" to ask question with headers + When I use "query" to ask question with authorization header """ {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} """ @@ -187,7 +187,7 @@ Feature: MCP tests """ {"mcp-oauth": {"Authorization": "Bearer invalid-token"}} """ - When I use "streaming_query" to ask question with headers + When I use "streaming_query" to ask question with authorization header """ {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} """ diff --git a/tests/e2e/features/steps/llm_query_response.py b/tests/e2e/features/steps/llm_query_response.py index c01756f2f..732f6e291 100644 --- a/tests/e2e/features/steps/llm_query_response.py +++ b/tests/e2e/features/steps/llm_query_response.py @@ -33,7 +33,6 @@ def ask_question(context: Context, endpoint: str) -> None: context.response = requests.post(url, json=data, timeout=DEFAULT_LLM_TIMEOUT) -@step('I use "{endpoint}" to ask question with headers') @step('I use "{endpoint}" to ask question with authorization header') def ask_question_authorized(context: Context, endpoint: str) -> None: """Call the service REST API endpoint with question.""" From 4cab65a9f6b4415f6bbfcb7ea378c8fe6e35fb3d Mon Sep 17 00:00:00 2001 From: JR Boos Date: Wed, 25 Feb 2026 10:48:25 -0500 Subject: [PATCH 16/16] fixed merge error --- tests/e2e/features/environment.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 1f70ff20b..33d82fbc1 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -331,16 +331,10 @@ def after_feature(context: Context, feature: Feature) -> None: if "Feedback" in feature.tags: for conversation_id in context.feedback_conversations: url = f"http://{context.hostname}:{context.port}/v1/conversations/{conversation_id}" -<<<<<<< lcore-1247 - headers = context.auth_headers if hasattr(context, "auth_headers") else {} - response = requests.delete(url, headers=headers) - assert response.status_code == 200, url + response = requests.delete(url, timeout=10) + assert response.status_code == 200, f"{url} returned {response.status_code}" if "MCP" in feature.tags: switch_config(context.default_config_backup) restart_container("lightspeed-stack") remove_config_backup(context.default_config_backup) -======= - response = requests.delete(url, timeout=10) - assert response.status_code == 200, f"{url} returned {response.status_code}" ->>>>>>> main