From 9689664786ab48f385088407448614d901b8f4ac Mon Sep 17 00:00:00 2001 From: Shengqi Zhu Date: Fri, 14 Aug 2026 17:56:35 -0700 Subject: [PATCH 1/4] fix(detector): parse triage confidence case-insensitively --- Detection/guardrail/adr_agent/adr_baseline.py | 2 +- Detection/tests/test_adr_baseline.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Detection/guardrail/adr_agent/adr_baseline.py b/Detection/guardrail/adr_agent/adr_baseline.py index 3d9c71d..7a69666 100644 --- a/Detection/guardrail/adr_agent/adr_baseline.py +++ b/Detection/guardrail/adr_agent/adr_baseline.py @@ -361,7 +361,7 @@ def _parse_triage_result(self, result_text: str) -> 'TriageResult': confidence = 0.8 # Default if "confidence:" in result_lower: try: - conf_part = result_text.split("confidence:")[1].split()[0].strip() + conf_part = result_lower.split("confidence:", 1)[1].split()[0].strip() confidence = float(conf_part) except (IndexError, ValueError): confidence = 0.8 diff --git a/Detection/tests/test_adr_baseline.py b/Detection/tests/test_adr_baseline.py index 59d50c1..8d2ad20 100644 --- a/Detection/tests/test_adr_baseline.py +++ b/Detection/tests/test_adr_baseline.py @@ -34,6 +34,13 @@ def test_parse_benign_result(self): assert result.threat_tactic == "N/A" assert result.confidence == 0.2 + def test_parse_uppercase_confidence_from_prompt_format(self): + triage = TriageLLM(MagicMock(), ADSConfig()) + result = triage._parse_triage_result( + "CLASSIFICATION: BENIGN\nTHREAT_TACTIC: N/A\nREASONING: routine request\nCONFIDENCE: 0.99" + ) + assert result.confidence == 0.99 + def test_parse_suspicious_result(self): triage = TriageLLM(MagicMock(), ADSConfig()) result = triage._parse_triage_result( From 5a15a02f0dc7363bf9c43bf1cec92fa526f52213 Mon Sep 17 00:00:00 2001 From: Shengqi Zhu Date: Sat, 15 Aug 2026 23:58:30 -0700 Subject: [PATCH 2/4] fix(detector): robustly parse triage confidence --- Detection/guardrail/adr_agent/adr_baseline.py | 23 +++++++++++++----- Detection/tests/test_adr_baseline.py | 24 ++++++++++++++++--- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/Detection/guardrail/adr_agent/adr_baseline.py b/Detection/guardrail/adr_agent/adr_baseline.py index 7a69666..0c40b23 100644 --- a/Detection/guardrail/adr_agent/adr_baseline.py +++ b/Detection/guardrail/adr_agent/adr_baseline.py @@ -359,12 +359,23 @@ def _parse_triage_result(self, result_text: str) -> 'TriageResult': # Extract confidence confidence = 0.8 # Default - if "confidence:" in result_lower: - try: - conf_part = result_lower.split("confidence:", 1)[1].split()[0].strip() - confidence = float(conf_part) - except (IndexError, ValueError): - confidence = 0.8 + confidence_pattern = re.compile( + r"^\s*(?:[-*>]\s*)?\**confidence\s*:\**\s*" + r"\[?\**\s*(?P(?:\d+(?:\.\d*)?|\.\d+))" + r"\s*(?P%?)\s*\**\]?\s*[.,;:]?\s*$", + re.IGNORECASE, + ) + for line in result_text.splitlines(): + match = confidence_pattern.match(line) + if not match: + continue + + parsed_confidence = float(match.group("value")) + if match.group("percent"): + parsed_confidence /= 100 + if 0.0 <= parsed_confidence <= 1.0: + confidence = parsed_confidence + break # Extract reasoning (note: prompt says "REASONING:", not "REASON:") reason = "Fast triage assessment" # Default diff --git a/Detection/tests/test_adr_baseline.py b/Detection/tests/test_adr_baseline.py index 8d2ad20..ab0ba4d 100644 --- a/Detection/tests/test_adr_baseline.py +++ b/Detection/tests/test_adr_baseline.py @@ -2,6 +2,8 @@ from unittest.mock import MagicMock +import pytest + from guardrail.adr_agent.adr_baseline import ADSConfig, ReasoningAgent, TriageLLM, _safe_task_id_for_path @@ -34,12 +36,28 @@ def test_parse_benign_result(self): assert result.threat_tactic == "N/A" assert result.confidence == 0.2 - def test_parse_uppercase_confidence_from_prompt_format(self): + @pytest.mark.parametrize( + ("triage_output", "expected_confidence"), + [ + ("CONFIDENCE: 0.99", 0.99), + ("**CONFIDENCE:** 0.95", 0.95), + ("REASONING: low confidence: agent intent unclear\nCONFIDENCE: 0.30", 0.30), + ("CONFIDENCE: 0.75.", 0.75), + ("CONFIDENCE: 95%", 0.95), + ], + ) + def test_parse_common_confidence_formats(self, triage_output, expected_confidence): triage = TriageLLM(MagicMock(), ADSConfig()) result = triage._parse_triage_result( - "CLASSIFICATION: BENIGN\nTHREAT_TACTIC: N/A\nREASONING: routine request\nCONFIDENCE: 0.99" + f"CLASSIFICATION: BENIGN\nTHREAT_TACTIC: N/A\n{triage_output}" ) - assert result.confidence == 0.99 + assert result.confidence == expected_confidence + + @pytest.mark.parametrize("triage_output", ["CONFIDENCE: 1.1", "CONFIDENCE: 101%"]) + def test_out_of_range_confidence_uses_default(self, triage_output): + triage = TriageLLM(MagicMock(), ADSConfig()) + result = triage._parse_triage_result(triage_output) + assert result.confidence == 0.8 def test_parse_suspicious_result(self): triage = TriageLLM(MagicMock(), ADSConfig()) From ec42a44847baa4e906a8134db1ae5d992617478f Mon Sep 17 00:00:00 2001 From: Shengqi Zhu Date: Sun, 16 Aug 2026 00:02:19 -0700 Subject: [PATCH 3/4] refactor(detector): document confidence pattern --- Detection/guardrail/adr_agent/adr_baseline.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/Detection/guardrail/adr_agent/adr_baseline.py b/Detection/guardrail/adr_agent/adr_baseline.py index 0c40b23..6f52990 100644 --- a/Detection/guardrail/adr_agent/adr_baseline.py +++ b/Detection/guardrail/adr_agent/adr_baseline.py @@ -359,11 +359,23 @@ def _parse_triage_result(self, result_text: str) -> 'TriageResult': # Extract confidence confidence = 0.8 # Default + # Match only a dedicated confidence field, not incidental phrases such as + # "REASONING: low confidence: agent intent unclear". Models sometimes add + # Markdown decoration even though the prompt requests plain field labels. confidence_pattern = re.compile( - r"^\s*(?:[-*>]\s*)?\**confidence\s*:\**\s*" - r"\[?\**\s*(?P(?:\d+(?:\.\d*)?|\.\d+))" - r"\s*(?P%?)\s*\**\]?\s*[.,;:]?\s*$", - re.IGNORECASE, + r""" + ^\s* # Start of a line, allowing indentation. + (?:[-*>]\s*)? # Optional Markdown list/quote marker. + \**confidence\s*:\**\s* # Label, optionally wrapped in asterisks. + \[?\**\s* # Optional bracket/asterisks before value. + (?P # Decimal confidence value. + (?:\d+(?:\.\d*)?|\.\d+) + ) + \s*(?P%?) # Optional percentage notation. + \s*\**\]? # Optional closing asterisks/bracket. + \s*[.,;:]?\s*$ # Optional trailing punctuation. + """, + re.IGNORECASE | re.VERBOSE, ) for line in result_text.splitlines(): match = confidence_pattern.match(line) From ecbea23a83bea23934eea6fd7b242fe15622acc1 Mon Sep 17 00:00:00 2001 From: Shengqi Zhu Date: Sun, 16 Aug 2026 10:10:35 -0700 Subject: [PATCH 4/4] fix(detector): handle common confidence formats --- Detection/guardrail/adr_agent/adr_baseline.py | 14 ++++++++++--- Detection/tests/test_adr_baseline.py | 21 +++++++++++++++++-- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/Detection/guardrail/adr_agent/adr_baseline.py b/Detection/guardrail/adr_agent/adr_baseline.py index 6f52990..0720b54 100644 --- a/Detection/guardrail/adr_agent/adr_baseline.py +++ b/Detection/guardrail/adr_agent/adr_baseline.py @@ -366,14 +366,18 @@ def _parse_triage_result(self, result_text: str) -> 'TriageResult': r""" ^\s* # Start of a line, allowing indentation. (?:[-*>]\s*)? # Optional Markdown list/quote marker. - \**confidence\s*:\**\s* # Label, optionally wrapped in asterisks. + \** # Optional opening Markdown emphasis. + confidence + \**\s*: # Emphasis may end before the colon. + \**\s* # Or it may end after the colon. \[?\**\s* # Optional bracket/asterisks before value. (?P # Decimal confidence value. (?:\d+(?:\.\d*)?|\.\d+) ) \s*(?P%?) # Optional percentage notation. \s*\**\]? # Optional closing asterisks/bracket. - \s*[.,;:]?\s*$ # Optional trailing punctuation. + \s*[.,;:]? # Optional trailing punctuation. + (?=\s|$) # Allow commentary, but not numeric runoff. """, re.IGNORECASE | re.VERBOSE, ) @@ -383,7 +387,11 @@ def _parse_triage_result(self, result_text: str) -> 'TriageResult': continue parsed_confidence = float(match.group("value")) - if match.group("percent"): + # Special case: treat percentage values at or above 1 as a 0-100 + # scale ("1%" -> 0.01), but preserve values below 1 unchanged + # ("0.95%" -> 0.95). The latter intentionally favors the likely + # model intent: an already-normalized confidence with an extra "%". + if match.group("percent") and parsed_confidence >= 1.0: parsed_confidence /= 100 if 0.0 <= parsed_confidence <= 1.0: confidence = parsed_confidence diff --git a/Detection/tests/test_adr_baseline.py b/Detection/tests/test_adr_baseline.py index ab0ba4d..9ec4689 100644 --- a/Detection/tests/test_adr_baseline.py +++ b/Detection/tests/test_adr_baseline.py @@ -44,6 +44,14 @@ def test_parse_benign_result(self): ("REASONING: low confidence: agent intent unclear\nCONFIDENCE: 0.30", 0.30), ("CONFIDENCE: 0.75.", 0.75), ("CONFIDENCE: 95%", 0.95), + ("CONFIDENCE: 95.1%", 0.951), + ("CONFIDENCE: 0.95%", 0.95), + ("CONFIDENCE: 1%", 0.01), + ("confidence: 0.9 - agent behavior normal", 0.9), + ("CONFIDENCE: 0.9 (high)", 0.9), + ("**CONFIDENCE**: 0.95", 0.95), + ("*CONFIDENCE*: 0.9", 0.9), + ("> **Confidence**: 0.6", 0.6), ], ) def test_parse_common_confidence_formats(self, triage_output, expected_confidence): @@ -53,8 +61,17 @@ def test_parse_common_confidence_formats(self, triage_output, expected_confidenc ) assert result.confidence == expected_confidence - @pytest.mark.parametrize("triage_output", ["CONFIDENCE: 1.1", "CONFIDENCE: 101%"]) - def test_out_of_range_confidence_uses_default(self, triage_output): + @pytest.mark.parametrize( + "triage_output", + [ + "CONFIDENCE: 1.1", + "CONFIDENCE: 101%", + "CONFIDENCE: 0.95.2", + "CONFIDENCE: 0.95high", + "CLASSIFICATION: BENIGN | CONFIDENCE: 0.9", + ], + ) + def test_invalid_confidence_uses_default(self, triage_output): triage = TriageLLM(MagicMock(), ADSConfig()) result = triage._parse_triage_result(triage_output) assert result.confidence == 0.8