Schema-Aware Hallucination Entropy

The problem

AI models often provide incorrect information or fail to follow specific data formats, making it hard to trust their output. It is difficult to measure how much a response is deviating from both the facts and the required structure at once.

What it does

It calculates a score that measures how much an AI's response drifts away from both factual accuracy and the correct data format. It looks for linguistic errors and structural mistakes simultaneously.

Why it matters

It provides a clear way to measure how much an AI's output deviates from both the facts and the required structure.

Validation

It was run inside an isolated container with no network access. This is the exact command and the real output it produced — captured process output, not written by a model.

$ python3 schema_hallucination_entropy.py
Traceback (most recent call last):
  File "/work/schema_hallucination_entropy.py", line 72, in <module>
    data = json.loads(model_output.replace('\n', '').replace('$', ''))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/json/__init__.py", line 346, in loads
    return _default_decoder.decode(s)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/json/decoder.py", line 338, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/json/decoder.py", line 356, in raw_decode
    raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

A screenshot of that run.

A clean run proves this does what is shown above, in a CPU-only sandbox. It is a small research demo — not a production tool, and nothing here was published anywhere.

The code

All of it — 150 lines, one file, standard library only.

import json
import re
from typing import Dict, Any, List, Tuple

class SchemaAwareHallucinationEntropy:
    def __init__(self, schema: Dict[str, Any]):
        self.schema = schema
        self.linguistic_patterns = [
            r'uncertain|unsure|speculate|guess|assume',
            r'remember|know|fact|certain',
            r'image shows|visual context',
        ]
        self.structural_weights = {
            'missing_field': 0.3,
            'type_mismatch': 0.2,
            'nested_error': 0.5
        }

    def linguistic_error_score(self, text: str) -> float:
        matches = [re.findall(pattern, text) for pattern in self.linguistic_patterns]
        return len([m for lst in matches for m in lst]) / (len(text) + 1)

    def structural_validation_score(self, data: Dict[str, Any]) -> float:
        errors = []
        self._validate(data, self.schema, errors, "$")
        return sum(self.structural_weights[error['type']] for error in errors)

    def structural_validation_with_trace(self, data: Dict[str, Any]) -> Tuple[float, 'SchemaPathTrace']:
        errors = []
        self._validate(data, self.schema, errors, "$")
        score = sum(self.structural_weights[error['type']] for error in errors)
        trace = SchemaPathTrace(errors)
        return score, trace

    def _validate(self, data, schema, errors, path: str):
        if isinstance(schema, dict):
            for field in schema:
                if field not in data:
                    errors.append({'type': 'missing_field', 'path': path + '/' + field})
                else:
                    self._validate(data[field], schema[field], errors, path + '/' + field)
            return
        if type(data) != type(schema):
            errors.append({'type': 'type_mismatch', 'path': path})
        if isinstance(schema, list):
            for i, item in enumerate(data):
                self._validate(item, schema[0], errors, path + '[' + str(i) + ']')

    def calculate_entropy(self, text: str, data: Dict[str, Any]) -> float:
        linguistic_score = self.linguistic_error_score(text)
        structural_score = self.structural_validation_score(data)
        return 0.5 * linguistic_score + 0.5 * structural_score

    def calculate_entropy_with_trace(self, text: str, data: Dict[str, Any]) -> Tuple[float, 'SchemaPathTrace']:
        linguistic_score = self.linguistic_error_score(text)
        structural_score, trace = self.structural_validation_with_trace(data)
        entropy = 0.5 * linguistic_score + 0.5 * structural_score
        return entropy, trace


_WEIGHT_MAP = {'missing_field': 0.3, 'type_mismatch': 0.2, 'nested_error': 0.5}

class SchemaPathTrace:
    def __init__(self, errors: List[Dict[str, Any]]):
        self.errors = errors

    def each(self) -> List[Dict[str, Any]]:
        result = []
        for e in self.errors:
            result.append({
                'path': e['path'],
                'type': e['type'],
                'weight': _WEIGHT_MAP.get(e['type'], 0),
            })
        return result

    def total_weight(self) -> float:
        return sum(item['weight'] for item in self.each())

    def summary(self) -> Dict[str, Any]:
        type_counts = {}
        for e in self.errors:
            t = e['type']
            type_counts[t] = type_counts.get(t, 0) + 1
        return {
            'total_errors': len(self.errors),
            'by_type': type_counts,
            'affected_paths': sorted({e['path'] for e in self.errors}),
        }

    def __repr__(self):
        entries = self.each()
        if not entries:
            return 'Schema-Path Trace: no structural errors found.'
        total = sum(item['weight'] for item in entries)
        lines = [f'Schema-Path Trace (total weight: {total:.2f}):']
        for item in entries:
            lines.append(f'  {item["path"]}  ->  {item["type"]}  (weight: {item["weight"]:.2f})')
        return '\n'.join(lines)


# Example usage
if __name__ == '__main__':
    schema = {
        'type': 'object',
        'properties': {
            'title': {'type': 'string'},
            'description': {'type': 'string'},
            'price': {'type': 'number'},
            'features': {
                'type': 'array',
                'items': {'type': 'string'}
            }
        },
        'required': ['title', 'price']
    }

    print("=== Schema-Aware Hallucination Entropy v2 ===\n")

    correct_data = {
        'title': 'Amazing Product',
        'description': 'This product does everything you need.',
        'price': 99.99,
        'features': ['feature1', 'feature2', 'feature3']
    }

    incorrect_data = {
        'title': 'Widget',
        'description': 'A very fine widget indeed.',
        'price': 'not a number',
        'category': 'misc'
    }

    scorer = SchemaAwareHallucinationEntropy(schema)

    print("--- Test 1: Correct data (entropy + trace) ---")
    text1 = "I am certain this product has all the features listed."
    print(f'Entropy (v1):            {scorer.calculate_entropy(text1, correct_data):.4f}')
    entropy_v2, trace = scorer.calculate_entropy_with_trace(text1, correct_data)
    print(f'Entropy (v2 with trace): {entropy_v2:.4f}')
    print(trace)

    print("\n--- Test 2: Incorrect data (errors + trace) ---")
    text2 = 'I am uncertain about this, but I guess the price is $10 and the image shows features'
    print(f'Entropy (v1):            {scorer.calculate_entropy(text2, incorrect_data):.4f}')
    entropy_v2, trace = scorer.calculate_entropy_with_trace(text2, incorrect_data)
    print(f'Entropy (v2 with trace): {entropy_v2:.4f}')
    print(trace)
    print()
    print('Trace summary:', trace.summary())