Index

Symbols

--audit-responses asqi-evaluate-score-cards command line option asqi-execute command line option --concurrent-tests asqi-execute command line option asqi-execute-tests command line option --container-config asqi-execute command line option asqi-execute-tests command line option --input-file asqi-evaluate-score-cards command line option --install-completion asqi command line option --manifests-dir asqi-validate command line option --max-failures asqi-execute command line option asqi-execute-tests command line option --output-file asqi-evaluate-score-cards command line option asqi-execute command line option asqi-execute-tests command line option --progress-interval asqi-execute command line option asqi-execute-tests command line option --score-card-config asqi-evaluate-score-cards command line option asqi-execute command line option --show-completion asqi command line option --skip-audit asqi-evaluate-score-cards command line option asqi-execute command line option --systems-config asqi-execute command line option asqi-execute-tests command line option asqi-validate command line option --test-ids asqi-execute-tests command line option

--test-suite-config asqi-execute command line option asqi-execute-tests command line option asqi-validate command line option --version asqi command line option -a asqi-evaluate-score-cards command line option asqi-execute command line option -c asqi-execute command line option asqi-execute-tests command line option -m asqi-execute command line option asqi-execute-tests command line option -o asqi-evaluate-score-cards command line option asqi-execute command line option asqi-execute-tests command line option -p asqi-execute command line option asqi-execute-tests command line option -r asqi-evaluate-score-cards command line option asqi-execute command line option -s asqi-execute command line option asqi-execute-tests command line option asqi-validate command line option -t asqi-execute command line option asqi-execute-tests command line option asqi-validate command line option -tids asqi-execute-tests command line option -V asqi command line option

A

add_score_cards_to_results() (in module src.asqi.workflow) ALLOWED_BOOL_OPS (src.asqi.metric_expression.MetricExpressionEvaluator attribute) ALLOWED_COMPARISONS (src.asqi.metric_expression.MetricExpressionEvaluator attribute) ALLOWED_FUNCTIONS (src.asqi.metric_expression.MetricExpressionEvaluator attribute) ALLOWED_OPS (src.asqi.metric_expression.MetricExpressionEvaluator attribute) api_key (src.asqi.schemas.LLMAPIParams attribute) app (in module src.asqi.main) apply_condition_to_value() (src.asqi.score_card_engine.ScoreCardEngine method) apply_to (src.asqi.schemas.ScoreCardIndicator attribute) asqi command line option --install-completion --show-completion --version -V asqi-evaluate-score-cards command line option --audit-responses --input-file --output-file --score-card-config --skip-audit -a -o -r asqi-execute command line option --audit-responses --concurrent-tests --container-config --max-failures --output-file --progress-interval --score-card-config --skip-audit --systems-config --test-suite-config -a -c -m -o -p -r -s -t

asqi-execute-tests command line option --concurrent-tests --container-config --max-failures --output-file --progress-interval --systems-config --test-ids --test-suite-config -c -m -o -p -s -t -tids asqi-validate command line option --manifests-dir --systems-config --test-suite-config -s -t asqi_logger (in module src.asqi.logging_config) assessment (src.asqi.schemas.AuditScoreCardIndicator attribute) (src.asqi.schemas.ScoreCardIndicator attribute) AssessmentRule (class in src.asqi.schemas) audit_indicators (src.asqi.errors.AuditResponsesRequiredError attribute) AuditAssessmentRule (class in src.asqi.schemas) AuditResponse (class in src.asqi.schemas) AuditResponses (class in src.asqi.schemas) AuditResponsesRequiredError AuditScoreCardIndicator (class in src.asqi.schemas)

B

base_url (src.asqi.schemas.LLMAPIParams attribute)

build_env_var_error_message() (in module src.asqi.validation)

C

check_images_availability() (in module src.asqi.container_manager) citations (src.asqi.rag_response_schema.RAGContext attribute) cleanup_force (src.asqi.config.ContainerConfig attribute) cleanup_on_finish (src.asqi.config.ContainerConfig attribute) computed_value (src.asqi.score_card_engine.ScoreCardEvaluationResult attribute) condition (src.asqi.schemas.AssessmentRule attribute) config (in module src.asqi.workflow) configure_logging() (in module src.asqi.logging_config) console (in module src.asqi.main) (in module src.asqi.validation) (in module src.asqi.workflow)

container_dict() (src.asqi.workflow.TestExecutionResult method) container_id (src.asqi.workflow.TestExecutionResult attribute) container_output (src.asqi.workflow.TestExecutionResult attribute) ContainerConfig (class in src.asqi.config) convert_test_results_to_objects() (in module src.asqi.workflow) create_container_logger() (in module src.asqi.logging_config) create_test_execution_plan() (in module src.asqi.validation) create_test_execution_progress() (in module src.asqi.output) create_workflow_summary() (in module src.asqi.output)

D

dbos_check_images_availability() (in module src.asqi.workflow) dbos_pull_images() (in module src.asqi.workflow) DEFAULT_CONCURRENT_TESTS (src.asqi.config.ExecutorConfig attribute) DEFAULT_RUN_PARAMS (src.asqi.config.ContainerConfig attribute) description (src.asqi.schemas.AssessmentRule attribute) (src.asqi.schemas.AuditAssessmentRule attribute) (src.asqi.schemas.EnvironmentVariable attribute) (src.asqi.schemas.InputParameter attribute) (src.asqi.schemas.Manifest attribute) (src.asqi.schemas.OutputArtifact attribute) (src.asqi.schemas.OutputMetric attribute) (src.asqi.schemas.SuiteConfig attribute) (src.asqi.schemas.SystemDefinition attribute) (src.asqi.schemas.SystemInput attribute) (src.asqi.schemas.TestDefinition attribute) (src.asqi.score_card_engine.ScoreCardEvaluationResult attribute)

details (src.asqi.score_card_engine.ScoreCardEvaluationResult attribute) docker_client() (in module src.asqi.container_manager) document_id (src.asqi.rag_response_schema.RAGCitation attribute) duplicate_dict (src.asqi.errors.DuplicateIDError attribute) DuplicateIDError

E

end_time (src.asqi.workflow.TestExecutionResult attribute) env_file (src.asqi.schemas.LLMAPIParams attribute) (src.asqi.schemas.TestDefinitionBase attribute) environment (src.asqi.schemas.TestDefinitionBase attribute) environment_variables (src.asqi.schemas.Manifest attribute) EnvironmentVariable (class in src.asqi.schemas) error (src.asqi.score_card_engine.ScoreCardEvaluationResult attribute) error_message (src.asqi.workflow.TestExecutionResult attribute) error_type (src.asqi.errors.ManifestExtractionError attribute) evaluate_audit_indicator() (src.asqi.score_card_engine.ScoreCardEngine method) evaluate_expression() (src.asqi.metric_expression.MetricExpressionEvaluator method) evaluate_indicator() (src.asqi.score_card_engine.ScoreCardEngine method) evaluate_score_card() (in module src.asqi.workflow)

evaluate_score_cards() (in module src.asqi.main) evaluate_score_cards_workflow() (in module src.asqi.workflow) evaluate_scorecard() (src.asqi.score_card_engine.ScoreCardEngine method) execute() (in module src.asqi.main) execute_single_test() (in module src.asqi.workflow) execute_tests() (in module src.asqi.main) execution_time (src.asqi.workflow.TestExecutionResult property) ExecutorConfig (class in src.asqi.config) exit_code (src.asqi.workflow.TestExecutionResult attribute) expression (src.asqi.schemas.MetricExpression attribute) extract_ids() (in module src.asqi.validation) extract_manifest_from_image() (in module src.asqi.container_manager) extract_manifest_from_image_step() (in module src.asqi.workflow) extract_metric_values() (src.asqi.score_card_engine.ScoreCardEngine method)

F

filter_results_by_test_id() (src.asqi.score_card_engine.ScoreCardEngine method) find_manifest_for_image() (in module src.asqi.validation)

format_execution_summary() (in module src.asqi.output) format_failure_summary() (in module src.asqi.output) from_run_params() (src.asqi.config.ContainerConfig class method)

G

GenericSystemConfig (class in src.asqi.schemas)

get_duplicate_ids() (in module src.asqi.validation) get_nested_value() (in module src.asqi.score_card_engine)

H

host_access (src.asqi.schemas.Manifest attribute)

I

id (src.asqi.schemas.AuditScoreCardIndicator attribute) (src.asqi.schemas.ScoreCardIndicator attribute) (src.asqi.schemas.TestDefinition attribute) IDsStringPattern (in module src.asqi.schemas) image (src.asqi.schemas.TestDefinition attribute) (src.asqi.workflow.TestExecutionResult attribute) indicator_id (src.asqi.schemas.AuditResponse attribute) (src.asqi.score_card_engine.ScoreCardEvaluationResult attribute)

indicator_name (src.asqi.score_card_engine.ScoreCardEvaluationResult attribute) indicators (src.asqi.schemas.ScoreCard attribute) INPUT_MOUNT_PATH (in module src.asqi.container_manager) input_schema (src.asqi.schemas.Manifest attribute) input_systems (src.asqi.schemas.Manifest attribute) InputParameter (class in src.asqi.schemas) interpolate_env_vars() (in module src.asqi.config)

L

LLMAPIConfig (class in src.asqi.schemas) LLMAPIParams (class in src.asqi.schemas) load_and_validate_plan() (in module src.asqi.main) load_audit_responses_file() (in module src.asqi.main) load_config_file() (in module src.asqi.config) load_from_yaml() (src.asqi.config.ContainerConfig class method)

load_score_card_file() (in module src.asqi.main) load_yaml_file() (in module src.asqi.main) logger (in module src.asqi.container_manager) (in module src.asqi.metric_expression) (in module src.asqi.score_card_engine) (in module src.asqi.validation)

M

Manifest (class in src.asqi.schemas) MANIFEST_PATH (src.asqi.config.ContainerConfig attribute) ManifestExtractionError MAX_FAILURES_DISPLAYED (src.asqi.config.ExecutorConfig attribute) merge_defaults_into_suite() (in module src.asqi.config) metric (src.asqi.schemas.ScoreCardIndicator attribute) metric_value (src.asqi.score_card_engine.ScoreCardEvaluationResult attribute) MetricExpression (class in src.asqi.schemas) MetricExpressionError MetricExpressionEvaluator (class in src.asqi.metric_expression) MissingIDFieldError MissingImageError model (src.asqi.schemas.LLMAPIParams attribute) module src src.asqi src.asqi.config src.asqi.container_manager src.asqi.errors src.asqi.evaluator src.asqi.logging_config src.asqi.main src.asqi.metric_expression src.asqi.output src.asqi.rag_response_schema src.asqi.schemas src.asqi.score_card_engine src.asqi.validation src.asqi.workflow

MountExtractionError

N

name (src.asqi.schemas.AuditScoreCardIndicator attribute) (src.asqi.schemas.EnvironmentVariable attribute) (src.asqi.schemas.InputParameter attribute) (src.asqi.schemas.Manifest attribute) (src.asqi.schemas.OutputArtifact attribute) (src.asqi.schemas.OutputMetric attribute) (src.asqi.schemas.ScoreCardIndicator attribute) (src.asqi.schemas.SystemInput attribute) (src.asqi.schemas.TestDefinition attribute)

notes (src.asqi.schemas.AuditResponse attribute) (src.asqi.score_card_engine.ScoreCardEvaluationResult attribute)

O

oltp_endpoint (in module src.asqi.workflow) original_error (src.asqi.errors.ManifestExtractionError attribute) outcome (src.asqi.schemas.AssessmentRule attribute) (src.asqi.schemas.AuditAssessmentRule attribute) (src.asqi.score_card_engine.ScoreCardEvaluationResult attribute)

output_artifacts (src.asqi.schemas.Manifest attribute) output_metrics (src.asqi.schemas.Manifest attribute) OUTPUT_MOUNT_PATH (in module src.asqi.container_manager) OutputArtifact (class in src.asqi.schemas) OutputMetric (class in src.asqi.schemas)

P

params (src.asqi.schemas.GenericSystemConfig attribute) (src.asqi.schemas.LLMAPIConfig attribute) (src.asqi.schemas.RAGAPIConfig attribute) (src.asqi.schemas.TestDefinitionBase attribute) parse_container_json_output() (in module src.asqi.output)

parse_expression() (src.asqi.metric_expression.MetricExpressionEvaluator method) parse_metric_path() (in module src.asqi.score_card_engine) path (src.asqi.schemas.OutputArtifact attribute) PROGRESS_UPDATE_INTERVAL (src.asqi.config.ExecutorConfig attribute) provider (src.asqi.schemas.SystemDefinition attribute) pull_images() (in module src.asqi.container_manager)

R

RAGAPIConfig (class in src.asqi.schemas) RAGCitation (class in src.asqi.rag_response_schema) RAGContext (class in src.asqi.rag_response_schema) required (src.asqi.schemas.EnvironmentVariable attribute) (src.asqi.schemas.InputParameter attribute) (src.asqi.schemas.SystemInput attribute) resolve_audit_options() (in module src.asqi.main)

resolve_metric_or_expression() (src.asqi.score_card_engine.ScoreCardEngine method) responses (src.asqi.schemas.AuditResponses attribute) result_dict() (src.asqi.workflow.TestExecutionResult method) retrieved_context (src.asqi.rag_response_schema.RAGCitation attribute) run_container_with_args() (in module src.asqi.container_manager) run_end_to_end_workflow() (in module src.asqi.workflow) run_params (src.asqi.config.ContainerConfig attribute) run_test_suite_workflow() (in module src.asqi.workflow)

S

save_container_results_to_file() (in module src.asqi.config) save_container_results_to_file_step() (in module src.asqi.workflow) save_results_to_file() (in module src.asqi.config) save_results_to_file_step() (in module src.asqi.workflow) score (src.asqi.rag_response_schema.RAGCitation attribute) score_card_name (src.asqi.errors.AuditResponsesRequiredError attribute) (src.asqi.schemas.ScoreCard attribute) ScoreCard (class in src.asqi.schemas) ScoreCardEngine (class in src.asqi.score_card_engine) ScoreCardEvaluationResult (class in src.asqi.score_card_engine) ScoreCardFilter (class in src.asqi.schemas) ScoreCardIndicator (class in src.asqi.schemas) selected_outcome (src.asqi.schemas.AuditResponse attribute) shutdown_containers() (in module src.asqi.container_manager) source_id (src.asqi.rag_response_schema.RAGCitation attribute) src module src.asqi module src.asqi.config module src.asqi.container_manager module src.asqi.errors module src.asqi.evaluator module src.asqi.logging_config module src.asqi.main module

src.asqi.metric_expression module src.asqi.output module src.asqi.rag_response_schema module src.asqi.schemas module src.asqi.score_card_engine module src.asqi.validation module src.asqi.workflow module start_score_card_evaluation() (in module src.asqi.workflow) start_test_execution() (in module src.asqi.workflow) start_time (src.asqi.workflow.TestExecutionResult attribute) stream_logs (src.asqi.config.ContainerConfig attribute) success (src.asqi.workflow.TestExecutionResult attribute) suite_name (src.asqi.schemas.SuiteConfig attribute) SuiteConfig (class in src.asqi.schemas) sut_name (src.asqi.schemas.AuditResponse attribute) (src.asqi.score_card_engine.ScoreCardEvaluationResult attribute) (src.asqi.workflow.TestExecutionResult attribute) system_database_url (in module src.asqi.workflow) SystemConfig (in module src.asqi.schemas) SystemDefinition (class in src.asqi.schemas) SystemInput (class in src.asqi.schemas) systems (src.asqi.schemas.SystemsConfig attribute) (src.asqi.schemas.TestDefinitionBase attribute) systems_under_test (src.asqi.schemas.TestDefinitionBase attribute) SystemsConfig (class in src.asqi.schemas)

T

tags (src.asqi.schemas.TestDefinitionBase attribute) test_id (src.asqi.schemas.ScoreCardFilter attribute) (src.asqi.score_card_engine.ScoreCardEvaluationResult attribute) (src.asqi.workflow.TestExecutionResult attribute) test_name (src.asqi.workflow.TestExecutionResult attribute) test_result_id (src.asqi.score_card_engine.ScoreCardEvaluationResult attribute) test_results (src.asqi.workflow.TestExecutionResult attribute) test_suite (src.asqi.schemas.SuiteConfig attribute) test_suite_default (src.asqi.schemas.SuiteConfig attribute) TestDefinition (class in src.asqi.schemas) TestDefinitionBase (class in src.asqi.schemas) TestExecutionResult (class in src.asqi.workflow)

TestSuiteDefault (class in src.asqi.schemas) threshold (src.asqi.schemas.AssessmentRule attribute) TIMEOUT_EXCEPTIONS (in module src.asqi.container_manager) timeout_seconds (src.asqi.config.ContainerConfig attribute) to_dict() (src.asqi.score_card_engine.ScoreCardEvaluationResult method) type (src.asqi.schemas.AuditScoreCardIndicator attribute) (src.asqi.schemas.GenericSystemConfig attribute) (src.asqi.schemas.InputParameter attribute) (src.asqi.schemas.LLMAPIConfig attribute) (src.asqi.schemas.OutputMetric attribute) (src.asqi.schemas.RAGAPIConfig attribute) (src.asqi.schemas.SystemInput attribute) typer_click_object (in module src.asqi.main)

V

validate() (in module src.asqi.main) validate_execution_inputs() (in module src.asqi.validation) validate_ids() (in module src.asqi.validation) validate_manifests_against_tests() (in module src.asqi.validation) validate_rag_response() (in module src.asqi.rag_response_schema) validate_score_card_inputs() (in module src.asqi.validation) validate_scorecard_test_ids() (src.asqi.score_card_engine.ScoreCardEngine method) validate_system_compatibility() (in module src.asqi.validation) validate_test_execution_inputs() (in module src.asqi.validation)

validate_test_parameters() (in module src.asqi.validation) validate_test_plan() (in module src.asqi.validation) (in module src.asqi.workflow) validate_test_volumes() (in module src.asqi.validation) validate_workflow_configurations() (in module src.asqi.validation) values (src.asqi.schemas.MetricExpression attribute) version (src.asqi.schemas.Manifest attribute) version_callback() (in module src.asqi.main) volumes (src.asqi.schemas.TestDefinitionBase attribute)

W

with_streaming() (src.asqi.config.ContainerConfig class method)