Index

A

actual (aiobs.evals.AssertionDetail attribute) add_label() (aiobs.collector.Collector method) ADDRESS (aiobs.evals.PIIType attribute) aiobs.classifier.base module aiobs.classifier.models.classification module aiobs.classifier.openai.classifier module aiobs.collector module aiobs.evals module aiobs.llm.anthropic module aiobs.llm.base module aiobs.llm.factory module aiobs.llm.gemini module

aiobs.llm.openai module aiobs.models.observability module aiobs.observe module aiobs.providers.base module aiobs.providers.gemini.provider module aiobs.providers.openai.provider module anthropic() (aiobs.llm.factory.LLM static method) AnthropicLLM (class in aiobs.llm.anthropic) api (aiobs.models.observability.Event attribute) (aiobs.models.observability.FunctionEvent attribute) args (aiobs.models.observability.FunctionEvent attribute) AssertionDetail (class in aiobs.evals) assertions (aiobs.evals.EvalResult attribute) auto_enhance_after (aiobs.models.observability.FunctionEvent attribute)

B

BAD (aiobs.classifier.models.classification.ClassificationVerdict attribute) BaseClassifier (class in aiobs.classifier.base) BaseEval (class in aiobs.evals)

BaseEvalConfig (class in aiobs.evals) BaseLLM (class in aiobs.llm.base) BaseProvider (class in aiobs.providers.base)

C

callsite (aiobs.models.observability.Event attribute) (aiobs.models.observability.FunctionEvent attribute) Callsite (class in aiobs.models.observability) case_sensitive (aiobs.evals.GroundTruthConfig attribute) (aiobs.evals.RegexAssertionConfig attribute) categories (aiobs.classifier.models.classification.ClassificationResult attribute) (aiobs.evals.ToxicityDetectionConfig attribute) check_against_context (aiobs.evals.HallucinationDetectionConfig attribute) check_against_input (aiobs.evals.HallucinationDetectionConfig attribute) check_input (aiobs.evals.PIIDetectionConfig attribute) (aiobs.evals.ToxicityDetectionConfig attribute) check_system_prompt (aiobs.evals.JailbreakDetectionConfig attribute) (aiobs.evals.PIIDetectionConfig attribute) classification_prompt (aiobs.classifier.models.classification.ClassificationConfig attribute) ClassificationConfig (class in aiobs.classifier.models.classification) ClassificationInput (class in aiobs.classifier.models.classification) ClassificationResult (class in aiobs.classifier.models.classification) ClassificationVerdict (class in aiobs.classifier.models.classification) classify() (aiobs.classifier.base.BaseClassifier method) (aiobs.classifier.openai.classifier.OpenAIClassifier method) classify_async() (aiobs.classifier.base.BaseClassifier method) (aiobs.classifier.openai.classifier.OpenAIClassifier method) classify_batch() (aiobs.classifier.base.BaseClassifier method) (aiobs.classifier.openai.classifier.OpenAIClassifier method) classify_batch_async() (aiobs.classifier.base.BaseClassifier method) (aiobs.classifier.openai.classifier.OpenAIClassifier method) coefficient_of_variation_threshold (aiobs.evals.LatencyConsistencyConfig attribute) Collector (class in aiobs.collector) complete() (aiobs.llm.anthropic.AnthropicLLM method) (aiobs.llm.base.BaseLLM method) (aiobs.llm.gemini.GeminiLLM method) (aiobs.llm.openai.OpenAILLM method)

complete_async() (aiobs.llm.anthropic.AnthropicLLM method) (aiobs.llm.base.BaseLLM method) (aiobs.llm.gemini.GeminiLLM method) (aiobs.llm.openai.OpenAILLM method) complete_messages() (aiobs.llm.anthropic.AnthropicLLM method) (aiobs.llm.base.BaseLLM method) (aiobs.llm.gemini.GeminiLLM method) (aiobs.llm.openai.OpenAILLM method) complete_messages_async() (aiobs.llm.base.BaseLLM method) confidence (aiobs.classifier.models.classification.ClassificationResult attribute) confidence_threshold (aiobs.classifier.models.classification.ClassificationConfig attribute) config_class (aiobs.evals.BaseEval attribute) (aiobs.evals.GroundTruthEval attribute) (aiobs.evals.HallucinationDetectionEval attribute) (aiobs.evals.JailbreakDetectionEval attribute) (aiobs.evals.LatencyConsistencyEval attribute) (aiobs.evals.PIIDetectionEval attribute) (aiobs.evals.RegexAssertion attribute) (aiobs.evals.SchemaAssertion attribute) (aiobs.evals.SQLQueryValidator attribute) (aiobs.evals.ToxicityDetectionEval attribute) CONTAINS (aiobs.evals.GroundTruthMatchMode attribute) contains() (aiobs.evals.GroundTruthEval class method) content (aiobs.llm.base.LLMMessage attribute) (aiobs.llm.base.LLMResponse attribute) context (aiobs.classifier.models.classification.ClassificationInput attribute) (aiobs.evals.EvalInput attribute) CREDIT_CARD (aiobs.evals.PIIType attribute) CUSTOM (aiobs.evals.PIIType attribute) custom_patterns (aiobs.evals.PIIDetectionConfig attribute) cwd (aiobs.models.observability.SessionMeta attribute)

D

DATE_OF_BIRTH (aiobs.evals.PIIType attribute) default() (aiobs.evals.PIIDetectionEval class method) DEFAULT_PATTERNS (aiobs.evals.PIIDetectionEval attribute) description (aiobs.evals.BaseEval attribute) (aiobs.evals.GroundTruthEval attribute) (aiobs.evals.HallucinationDetectionEval attribute) (aiobs.evals.JailbreakDetectionEval attribute) (aiobs.evals.LatencyConsistencyEval attribute) (aiobs.evals.PIIDetectionEval attribute) (aiobs.evals.RegexAssertion attribute) (aiobs.evals.SchemaAssertion attribute) (aiobs.evals.SQLQueryValidator attribute) (aiobs.evals.ToxicityDetectionEval attribute)

details (aiobs.evals.EvalResult attribute) detect_types (aiobs.evals.PIIDetectionConfig attribute) dialect (aiobs.evals.SQLQueryValidatorConfig attribute) duration_ms (aiobs.evals.EvalResult attribute) (aiobs.models.observability.Event attribute) (aiobs.models.observability.FunctionEvent attribute)

E

EMAIL (aiobs.evals.PIIType attribute) end() (aiobs.collector.Collector method) ended_at (aiobs.models.observability.Event attribute) (aiobs.models.observability.FunctionEvent attribute) (aiobs.models.observability.Session attribute) enh_prompt (aiobs.models.observability.FunctionEvent attribute) enh_prompt_id (aiobs.models.observability.FunctionEvent attribute) enh_prompt_traces (aiobs.models.observability.ObservabilityExport attribute) ERROR (aiobs.evals.EvalStatus attribute) error (aiobs.models.observability.Event attribute) (aiobs.models.observability.FunctionEvent attribute) error_result() (aiobs.evals.EvalResult class method) eval_name (aiobs.evals.BaseEval property) (aiobs.evals.EvalResult attribute) EvalInput (class in aiobs.evals) EvalResult (class in aiobs.evals) EvalStatus (class in aiobs.evals) evaluate() (aiobs.evals.BaseEval method) (aiobs.evals.GroundTruthEval method) (aiobs.evals.HallucinationDetectionEval method) (aiobs.evals.JailbreakDetectionEval method) (aiobs.evals.LatencyConsistencyEval method) (aiobs.evals.PIIDetectionEval method) (aiobs.evals.RegexAssertion method) (aiobs.evals.SchemaAssertion method) (aiobs.evals.SQLQueryValidator method) (aiobs.evals.ToxicityDetectionEval method)

evaluate_async() (aiobs.evals.BaseEval method) (aiobs.evals.HallucinationDetectionEval method) (aiobs.evals.JailbreakDetectionEval method) (aiobs.evals.ToxicityDetectionEval method) evaluate_batch() (aiobs.evals.BaseEval method) evaluate_batch_async() (aiobs.evals.BaseEval method) evaluated_at (aiobs.evals.EvalResult attribute) Event (class in aiobs.models.observability) events (aiobs.models.observability.ObservabilityExport attribute) EXACT (aiobs.evals.GroundTruthMatchMode attribute) exact() (aiobs.evals.GroundTruthEval class method) expected (aiobs.evals.AssertionDetail attribute) expected_output (aiobs.evals.EvalInput attribute) extract_claims (aiobs.evals.HallucinationDetectionConfig attribute) extract_json (aiobs.evals.SchemaAssertionConfig attribute)

F

fail_fast (aiobs.evals.BaseEvalConfig attribute) fail_on_detection (aiobs.evals.PIIDetectionConfig attribute) (aiobs.evals.ToxicityDetectionConfig attribute) fail_result() (aiobs.evals.EvalResult class method) failed (aiobs.evals.EvalResult property) FAILED (aiobs.evals.EvalStatus attribute) file (aiobs.models.observability.Callsite attribute)

flush() (aiobs.collector.Collector method) from_client() (aiobs.llm.factory.LLM static method) from_patterns() (aiobs.evals.RegexAssertion class method) from_schema() (aiobs.evals.SchemaAssertion class method) function (aiobs.models.observability.Callsite attribute) function_events (aiobs.models.observability.ObservabilityExport attribute) FunctionEvent (class in aiobs.models.observability)

G

gemini() (aiobs.llm.factory.LLM static method) GeminiLLM (class in aiobs.llm.gemini) GeminiProvider (class in aiobs.providers.gemini.provider) generated_at (aiobs.models.observability.ObservabilityExport attribute) get_compiled_negative_patterns() (aiobs.evals.RegexAssertionConfig method) get_compiled_patterns() (aiobs.evals.RegexAssertionConfig method)

get_current_span_id() (aiobs.collector.Collector method) get_labels() (aiobs.collector.Collector method) GOOD (aiobs.classifier.models.classification.ClassificationVerdict attribute) GroundTruthConfig (class in aiobs.evals) GroundTruthEval (class in aiobs.evals) GroundTruthMatchMode (class in aiobs.evals)

H

hallucination_threshold (aiobs.evals.HallucinationDetectionConfig attribute)

HallucinationDetectionConfig (class in aiobs.evals) HallucinationDetectionEval (class in aiobs.evals)

I

id (aiobs.models.observability.Session attribute) include_details (aiobs.evals.BaseEvalConfig attribute) install() (aiobs.providers.base.BaseProvider method) (aiobs.providers.gemini.provider.GeminiProvider method) (aiobs.providers.openai.provider.OpenAIProvider method) IP_ADDRESS (aiobs.evals.PIIType attribute) is_available() (aiobs.classifier.base.BaseClassifier class method) (aiobs.classifier.openai.classifier.OpenAIClassifier class method) (aiobs.evals.BaseEval class method) (aiobs.evals.SchemaAssertion class method) (aiobs.evals.SQLQueryValidator class method) (aiobs.providers.base.BaseProvider class method) (aiobs.providers.gemini.provider.GeminiProvider class method) (aiobs.providers.openai.provider.OpenAIProvider class method)

is_compatible() (aiobs.llm.anthropic.AnthropicLLM class method) (aiobs.llm.gemini.GeminiLLM class method) (aiobs.llm.openai.OpenAILLM class method)

J

JailbreakDetectionConfig (class in aiobs.evals)

JailbreakDetectionEval (class in aiobs.evals) json_schema (aiobs.evals.SchemaAssertionConfig attribute)

K

kwargs (aiobs.models.observability.FunctionEvent attribute)

L

labels (aiobs.models.observability.Session attribute) LatencyConsistencyConfig (class in aiobs.evals) LatencyConsistencyEval (class in aiobs.evals)

line (aiobs.models.observability.Callsite attribute) LLM (class in aiobs.llm.factory) LLMMessage (class in aiobs.llm.base) LLMResponse (class in aiobs.llm.base)

M

match_mode (aiobs.evals.GroundTruthConfig attribute) (aiobs.evals.RegexAssertionConfig attribute) max_claims (aiobs.evals.HallucinationDetectionConfig attribute) max_latency_ms (aiobs.evals.LatencyConsistencyConfig attribute) max_p95_ms (aiobs.evals.LatencyConsistencyConfig attribute) max_p99_ms (aiobs.evals.LatencyConsistencyConfig attribute) max_std_dev_ms (aiobs.evals.LatencyConsistencyConfig attribute) max_tokens (aiobs.classifier.models.classification.ClassificationConfig attribute) max_violations (aiobs.evals.JailbreakDetectionConfig attribute) message (aiobs.evals.AssertionDetail attribute) (aiobs.evals.EvalResult attribute) meta (aiobs.models.observability.Session attribute) metadata (aiobs.classifier.models.classification.ClassificationResult attribute) (aiobs.evals.EvalInput attribute) (aiobs.evals.EvalResult attribute) model (aiobs.classifier.models.classification.ClassificationConfig attribute) (aiobs.evals.HallucinationDetectionConfig attribute) (aiobs.evals.JailbreakDetectionConfig attribute) (aiobs.evals.ToxicityDetectionConfig attribute) (aiobs.llm.base.LLMResponse attribute) model_config (aiobs.classifier.models.classification.ClassificationConfig attribute) (aiobs.classifier.models.classification.ClassificationInput attribute) (aiobs.classifier.models.classification.ClassificationResult attribute) (aiobs.evals.AssertionDetail attribute) (aiobs.evals.BaseEvalConfig attribute) (aiobs.evals.EvalInput attribute) (aiobs.evals.EvalResult attribute) (aiobs.evals.GroundTruthConfig attribute) (aiobs.evals.HallucinationDetectionConfig attribute) (aiobs.evals.JailbreakDetectionConfig attribute) (aiobs.evals.LatencyConsistencyConfig attribute) (aiobs.evals.PIIDetectionConfig attribute) (aiobs.evals.RegexAssertionConfig attribute) (aiobs.evals.SchemaAssertionConfig attribute) (aiobs.evals.SQLQueryValidatorConfig attribute) (aiobs.evals.ToxicityDetectionConfig attribute) (aiobs.llm.base.LLMMessage attribute) (aiobs.llm.base.LLMResponse attribute) (aiobs.models.observability.Callsite attribute) (aiobs.models.observability.Event attribute) (aiobs.models.observability.FunctionEvent attribute) (aiobs.models.observability.ObservabilityExport attribute) (aiobs.models.observability.ObservedEvent attribute) (aiobs.models.observability.ObservedFunctionEvent attribute) (aiobs.models.observability.Session attribute) (aiobs.models.observability.SessionMeta attribute)

model_output (aiobs.classifier.models.classification.ClassificationInput attribute) (aiobs.evals.EvalInput attribute) module aiobs.classifier.base aiobs.classifier.models.classification aiobs.classifier.openai.classifier aiobs.collector aiobs.evals aiobs.llm.anthropic aiobs.llm.base aiobs.llm.factory aiobs.llm.gemini aiobs.llm.openai aiobs.models.observability aiobs.observe aiobs.providers.base aiobs.providers.gemini.provider aiobs.providers.openai.provider module (aiobs.models.observability.FunctionEvent attribute)

N

name (aiobs.classifier.base.BaseClassifier attribute) (aiobs.classifier.openai.classifier.OpenAIClassifier attribute) (aiobs.evals.AssertionDetail attribute) (aiobs.evals.BaseEval attribute) (aiobs.evals.BaseEvalConfig attribute) (aiobs.evals.GroundTruthEval attribute) (aiobs.evals.HallucinationDetectionEval attribute) (aiobs.evals.JailbreakDetectionEval attribute) (aiobs.evals.LatencyConsistencyEval attribute) (aiobs.evals.PIIDetectionEval attribute) NAME (aiobs.evals.PIIType attribute) name (aiobs.evals.RegexAssertion attribute) (aiobs.evals.SchemaAssertion attribute) (aiobs.evals.SQLQueryValidator attribute) (aiobs.evals.ToxicityDetectionEval attribute) (aiobs.models.observability.FunctionEvent attribute) (aiobs.models.observability.Session attribute) (aiobs.providers.base.BaseProvider attribute) (aiobs.providers.gemini.provider.GeminiProvider attribute) (aiobs.providers.openai.provider.OpenAIProvider attribute)

negative_patterns (aiobs.evals.RegexAssertionConfig attribute) normalize_whitespace (aiobs.evals.GroundTruthConfig attribute) NORMALIZED (aiobs.evals.GroundTruthMatchMode attribute) normalized() (aiobs.evals.GroundTruthEval class method)

O

ObservabilityExport (class in aiobs.models.observability) observe() (aiobs.collector.Collector method) (in module aiobs.observe) ObservedEvent (class in aiobs.models.observability)

ObservedFunctionEvent (class in aiobs.models.observability) openai() (aiobs.llm.factory.LLM static method) OpenAIClassifier (class in aiobs.classifier.openai.classifier) OpenAILLM (class in aiobs.llm.openai) OpenAIProvider (class in aiobs.providers.openai.provider)

P

parent_span_id (aiobs.models.observability.Event attribute) (aiobs.models.observability.FunctionEvent attribute) parse_json (aiobs.evals.SchemaAssertionConfig attribute) pass_result() (aiobs.evals.EvalResult class method) passed (aiobs.evals.AssertionDetail attribute) (aiobs.evals.EvalResult property) PASSED (aiobs.evals.EvalStatus attribute) patterns (aiobs.evals.RegexAssertionConfig attribute) PHONE (aiobs.evals.PIIType attribute)

pid (aiobs.models.observability.SessionMeta attribute) PIIDetectionConfig (class in aiobs.evals) PIIDetectionEval (class in aiobs.evals) PIIType (class in aiobs.evals) provider (aiobs.llm.anthropic.AnthropicLLM attribute) (aiobs.llm.base.BaseLLM attribute) (aiobs.llm.gemini.GeminiLLM attribute) (aiobs.llm.openai.OpenAILLM attribute) (aiobs.models.observability.Event attribute) (aiobs.models.observability.FunctionEvent attribute)

R

raw_response (aiobs.classifier.models.classification.ClassificationResult attribute) (aiobs.llm.base.LLMResponse attribute) reasoning (aiobs.classifier.models.classification.ClassificationResult attribute) redact (aiobs.evals.PIIDetectionConfig attribute) redact() (aiobs.evals.PIIDetectionEval method) REDACTION_MASKS (aiobs.evals.PIIDetectionEval attribute) RegexAssertion (class in aiobs.evals)

RegexAssertionConfig (class in aiobs.evals) remove_label() (aiobs.collector.Collector method) request (aiobs.models.observability.Event attribute) reset() (aiobs.collector.Collector method) reset_span_id() (aiobs.collector.Collector method) response (aiobs.models.observability.Event attribute) result (aiobs.models.observability.FunctionEvent attribute) role (aiobs.llm.base.LLMMessage attribute)

S

safety_threshold (aiobs.evals.JailbreakDetectionConfig attribute) scan() (aiobs.evals.PIIDetectionEval method) SchemaAssertion (class in aiobs.evals) SchemaAssertionConfig (class in aiobs.evals) score (aiobs.evals.EvalResult attribute) SEMANTIC (aiobs.evals.GroundTruthMatchMode attribute) Session (class in aiobs.models.observability) session_id (aiobs.models.observability.ObservedEvent attribute) (aiobs.models.observability.ObservedFunctionEvent attribute) SessionMeta (class in aiobs.models.observability) sessions (aiobs.models.observability.ObservabilityExport attribute) set_current_span_id() (aiobs.collector.Collector method) set_labels() (aiobs.collector.Collector method) similarity_threshold (aiobs.evals.GroundTruthConfig attribute) SKIPPED (aiobs.evals.EvalStatus attribute)

span_id (aiobs.models.observability.Event attribute) (aiobs.models.observability.FunctionEvent attribute) SQLQueryValidator (class in aiobs.evals) SQLQueryValidatorConfig (class in aiobs.evals) SSN (aiobs.evals.PIIType attribute) started_at (aiobs.models.observability.Event attribute) (aiobs.models.observability.FunctionEvent attribute) (aiobs.models.observability.Session attribute) status (aiobs.evals.EvalResult attribute) strict (aiobs.evals.HallucinationDetectionConfig attribute) (aiobs.evals.JailbreakDetectionConfig attribute) (aiobs.evals.SchemaAssertionConfig attribute) strict() (aiobs.evals.PIIDetectionEval class method) strip_punctuation (aiobs.evals.GroundTruthConfig attribute) system_prompt (aiobs.classifier.models.classification.ClassificationInput attribute) (aiobs.evals.EvalInput attribute)

T

tags (aiobs.evals.EvalInput attribute) temperature (aiobs.classifier.models.classification.ClassificationConfig attribute) (aiobs.evals.HallucinationDetectionConfig attribute) (aiobs.evals.JailbreakDetectionConfig attribute) (aiobs.evals.ToxicityDetectionConfig attribute)

toxicity_threshold (aiobs.evals.ToxicityDetectionConfig attribute) ToxicityDetectionConfig (class in aiobs.evals) ToxicityDetectionEval (class in aiobs.evals) trace_id (aiobs.models.observability.Event attribute) (aiobs.models.observability.FunctionEvent attribute) trace_tree (aiobs.models.observability.ObservabilityExport attribute)

U

UNCERTAIN (aiobs.classifier.models.classification.ClassificationVerdict attribute) usage (aiobs.llm.base.LLMResponse attribute)

user_input (aiobs.classifier.models.classification.ClassificationInput attribute) (aiobs.evals.EvalInput attribute)

V

validate_match_mode() (aiobs.evals.RegexAssertionConfig class method)

verdict (aiobs.classifier.models.classification.ClassificationResult attribute) version (aiobs.models.observability.ObservabilityExport attribute)

W

with_anthropic() (aiobs.evals.HallucinationDetectionEval class method) (aiobs.evals.JailbreakDetectionEval class method) (aiobs.evals.ToxicityDetectionEval class method) with_custom_patterns() (aiobs.evals.PIIDetectionEval class method) with_expected() (aiobs.evals.EvalInput method) with_gemini() (aiobs.evals.HallucinationDetectionEval class method) (aiobs.evals.JailbreakDetectionEval class method) (aiobs.evals.ToxicityDetectionEval class method)

with_metadata() (aiobs.evals.EvalInput method) with_openai() (aiobs.evals.HallucinationDetectionEval class method) (aiobs.evals.JailbreakDetectionEval class method) (aiobs.evals.ToxicityDetectionEval class method) with_thresholds() (aiobs.evals.LatencyConsistencyEval class method)