Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions engine/src/agent_control_engine/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,8 @@ async def evaluate_control(eval_task: _EvalTask) -> None:
error_msg = f"TimeoutError: Evaluator exceeded {timeout}s timeout"
logger.warning(
f"Evaluator timeout for control '{eval_task.item.name}' "
f"(evaluator: {eval_task.item.control.evaluator.name}): {error_msg}"
f"(evaluator: {eval_task.item.control.evaluator.name}): {error_msg}",
exc_info=True,
)
eval_task.result = EvaluatorResult(
matched=False,
Expand All @@ -222,9 +223,10 @@ async def evaluate_control(eval_task: _EvalTask) -> None:
# Evaluation error - fail open but mark as error
# The error field signals to callers that this was not a real evaluation
error_msg = f"{type(e).__name__}: {e}"
logger.warning(
logger.error(
f"Evaluator error for control '{eval_task.item.name}' "
f"(evaluator: {eval_task.item.control.evaluator.name}): {error_msg}"
f"(evaluator: {eval_task.item.control.evaluator.name}): {error_msg}",
exc_info=True,
)
eval_task.result = EvaluatorResult(
matched=False,
Expand Down
8 changes: 5 additions & 3 deletions server/src/agent_control_server/endpoints/agents.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
_DEFAULT_PAGINATION_OFFSET = 0
_DEFAULT_PAGINATION_LIMIT = 20
_MAX_PAGINATION_LIMIT = 100
_CORRUPTED_AGENT_DATA_MESSAGE = "Stored agent data is corrupted and cannot be parsed."

type StepKeyTuple = tuple[str, str]

Expand Down Expand Up @@ -441,7 +442,7 @@ async def init_agent(
# Parse existing data via AgentData Pydantic model
try:
data_model = AgentData.model_validate(existing.data)
except ValidationError as e:
except ValidationError:
if not request.force_replace:
_logger.error(
f"Failed to parse existing agent data for '{request.agent.agent_name}'",
Expand All @@ -459,14 +460,15 @@ async def init_agent(
resource="Agent",
field="data",
code="corrupted_data",
message=str(e),
message=_CORRUPTED_AGENT_DATA_MESSAGE,
)
],
)
# User explicitly requested replacement
_logger.warning(
f"Force-replacing corrupted data for agent '{request.agent.agent_name}' "
f"due to force_replace=true. Original error: {e}"
"due to force_replace=true.",
exc_info=True,
)
data_model = AgentData(agent_metadata={}, steps=[], evaluators=[])

Expand Down
32 changes: 23 additions & 9 deletions server/src/agent_control_server/endpoints/controls.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@
# Pagination constants
_DEFAULT_PAGINATION_LIMIT = 20
_MAX_PAGINATION_LIMIT = 100
_INVALID_PARAMETERS_MESSAGE = "Invalid config parameters for evaluator."
_CORRUPTED_CONTROL_DATA_MESSAGE = "Stored control data is corrupted and cannot be parsed."
_SCHEMA_VALIDATION_FAILED_MESSAGE = "Config does not satisfy the evaluator schema."

router = APIRouter(prefix="/controls", tags=["controls"])

Expand Down Expand Up @@ -127,7 +130,7 @@ async def _validate_control_definition(
validate_config_against_schema(
control_def.evaluator.config, evaluator.config_schema
)
except JSONSchemaValidationError as e:
except JSONSchemaValidationError:
raise APIValidationError(
error_code=ErrorCode.INVALID_CONFIG,
detail=f"Config validation failed for evaluator '{evaluator_ref}'",
Expand All @@ -138,7 +141,7 @@ async def _validate_control_definition(
resource="Control",
field="data.evaluator.config",
code="schema_validation_error",
message=e.message,
message=_SCHEMA_VALIDATION_FAILED_MESSAGE,
)
],
)
Expand Down Expand Up @@ -167,7 +170,12 @@ async def _validate_control_definition(
for err in e.errors()
],
)
except TypeError as e:
except TypeError:
_logger.warning(
"Config validation raised TypeError for evaluator '%s'",
parsed.name,
exc_info=True,
)
raise APIValidationError(
error_code=ErrorCode.INVALID_CONFIG,
detail=f"Invalid config parameters for evaluator '{parsed.name}'",
Expand All @@ -178,7 +186,7 @@ async def _validate_control_definition(
resource="Control",
field="data.evaluator.config",
code="invalid_parameters",
message=str(e),
message=_INVALID_PARAMETERS_MESSAGE,
)
],
)
Expand Down Expand Up @@ -279,13 +287,13 @@ async def get_control(
if control.data:
try:
control_data = ControlDefinition.model_validate(control.data)
except ValidationError as e:
except ValidationError:
# Data exists but is corrupted - log and return None
_logger.warning(
"Control '%s' (id=%s) has corrupted data that failed validation: %s",
"Control '%s' (id=%s) has corrupted data that failed validation",
control.name,
control_id,
str(e),
exc_info=True,
)
control_data = None

Expand Down Expand Up @@ -850,7 +858,13 @@ async def patch_control(
control.data = new_data
updated = True
current_enabled = request.enabled if updated else ctrl_def.enabled
except ValidationError as e:
except ValidationError:
_logger.error(
"Control '%s' (%s) has corrupted data in patch request",
control.name,
control_id,
exc_info=True,
)
raise APIValidationError(
error_code=ErrorCode.CORRUPTED_DATA,
detail=f"Control '{control.name}' has corrupted data",
Expand All @@ -861,7 +875,7 @@ async def patch_control(
resource="Control",
field="data",
code="corrupted_data",
message=str(e),
message=_CORRUPTED_CONTROL_DATA_MESSAGE,
)
],
)
Expand Down
61 changes: 58 additions & 3 deletions server/src/agent_control_server/endpoints/evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from agent_control_models import (
ControlDefinition,
ControlExecutionEvent,
ControlMatch,
EvaluationRequest,
EvaluationResponse,
)
Expand All @@ -33,6 +34,10 @@
# These are immediately recognizable as "not traced" and can be filtered in queries.
INVALID_TRACE_ID = "0" * 32 # 128-bit, 32 hex chars
INVALID_SPAN_ID = "0" * 16 # 64-bit, 16 hex chars
SAFE_EVALUATOR_ERROR = "Evaluation failed due to an internal evaluator error."
SAFE_EVALUATOR_TIMEOUT_ERROR = "Evaluation timed out before completion."
SAFE_INVALID_STEP_REGEX_ERROR = "Control configuration error: invalid step name regex."
SAFE_ENGINE_VALIDATION_MESSAGE = "Invalid evaluation request or control configuration."


class ControlAdapter:
Expand All @@ -44,6 +49,54 @@ def __init__(self, id: int, name: str, control: ControlDefinition):
self.control = control


def _sanitize_evaluator_error(error_message: str) -> str:
"""Convert evaluator runtime errors into safe client-facing text."""
if "invalid step_name_regex" in error_message.lower():
return SAFE_INVALID_STEP_REGEX_ERROR
if "timeout" in error_message.lower():
return SAFE_EVALUATOR_TIMEOUT_ERROR
return SAFE_EVALUATOR_ERROR


def _sanitize_control_match(match: ControlMatch) -> ControlMatch:
"""Redact internal evaluator error strings from a control match."""
if match.result.error is None:
return match

safe_error = _sanitize_evaluator_error(match.result.error)
safe_message = safe_error
sanitized_result = match.result.model_copy(
update={
"error": safe_error,
"message": safe_message,
}
)
return match.model_copy(update={"result": sanitized_result})


def _sanitize_evaluation_response(response: EvaluationResponse) -> EvaluationResponse:
"""Return a copy of the evaluation response with safe public error text."""
return response.model_copy(
update={
"matches": (
[_sanitize_control_match(match) for match in response.matches]
if response.matches
else None
),
"errors": (
[_sanitize_control_match(match) for match in response.errors]
if response.errors
else None
),
"non_matches": (
[_sanitize_control_match(match) for match in response.non_matches]
if response.non_matches
else None
),
}
)


@router.post(
"",
response_model=EvaluationResponse,
Expand Down Expand Up @@ -118,8 +171,8 @@ async def evaluate(
engine = ControlEngine(engine_controls)
try:
response = await engine.process(request)
except ValueError as e:
_logger.error(f"Evaluation failed: {e}")
except ValueError:
_logger.exception("Evaluation failed due to invalid configuration or input")
raise APIValidationError(
error_code=ErrorCode.EVALUATION_FAILED,
detail="Evaluation failed due to invalid configuration or input",
Expand All @@ -130,11 +183,13 @@ async def evaluate(
resource="Evaluation",
field=None,
code="evaluation_error",
message=str(e),
message=SAFE_ENGINE_VALIDATION_MESSAGE,
)
],
)

response = _sanitize_evaluation_response(response)

# Calculate total execution time
total_duration_ms = (time.perf_counter() - start_time) * 1000

Expand Down
10 changes: 8 additions & 2 deletions server/src/agent_control_server/endpoints/evaluator_configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
# Pagination constants
_DEFAULT_PAGINATION_LIMIT = 20
_MAX_PAGINATION_LIMIT = 100
_INVALID_PARAMETERS_MESSAGE = "Invalid config parameters for evaluator."

router = APIRouter(prefix="/evaluator-configs", tags=["evaluator-configs"])

Expand Down Expand Up @@ -100,14 +101,19 @@ def _validate_known_evaluator_config(evaluator: str, config: dict[str, Any]) ->
detail=f"Config validation failed for evaluator '{evaluator}'",
hint="Check the evaluator's config schema for required fields and types.",
)
except TypeError as e:
except TypeError:
_logger.warning(
"Evaluator config parameter validation raised TypeError for '%s'",
evaluator,
exc_info=True,
)
_raise_invalid_config(
[
ValidationErrorItem(
resource="EvaluatorConfig",
field="config",
code="invalid_parameters",
message=str(e),
message=_INVALID_PARAMETERS_MESSAGE,
)
],
detail=f"Invalid config parameters for evaluator '{evaluator}'",
Expand Down
9 changes: 6 additions & 3 deletions server/src/agent_control_server/endpoints/policies.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,11 +136,14 @@ async def add_control_to_policy(
)
await db.execute(stmt)
await db.commit()
except Exception as e:
except Exception:
await db.rollback()
_logger.error(
f"Failed to add control '{control.name}' ({control_id}) "
f"to policy '{policy.name}' ({policy_id}): {e}",
"Failed to add control '%s' (%s) to policy '%s' (%s)",
control.name,
control_id,
policy.name,
policy_id,
exc_info=True,
)
raise DatabaseError(
Expand Down
Loading
Loading