Hallucination-Aware Memory Ranking

Validation

It was run inside an isolated container with no network access. This is the exact command and the real output it produced — captured process output, not written by a model.

$ python3 memory_ranking.py
Ranked Memories (Relevance/Hallucination Combined Score):
1. The capital of France is Paris, which is a verified fact. (Score: 0.9500)
2. Water boils at 100°C under standard conditions. (Score: 0.9200)
3. The moon is made of green cheese, an unclear statement. (Score: -0.7750)
4. Quantum physics explains everything, which is not fully confirmed. (Score: -0.7750)

A screenshot of that run.

A clean run proves this does what is shown above, in a CPU-only sandbox. It is a small research demo — not a production tool, and nothing here was published anywhere.

The code

All of it — 143 lines, one file, standard library only.

#!/usr/bin/env python3

KNOWN_FACTS = {
    "capital of france": {"relevance": 0.95, "hallucination": 0.0},
    "water boils": {"relevance": 0.92, "hallucination": 0.0},
}

HALLUCINATION_PATTERNS = [
    "green cheese",
    "quantum physics explains everything",
    "not fully confirmed",
]
HIGH_HALLUCINATION_PATTERNS = [
    "unclear",
    "green cheese",
    "not factual",
    "not fully confirmed",
]

AMBIGUOUS_TERMS = {
    "probably": 0.12,
    "maybe": 0.15,
    "might": 0.10,
    "could": 0.08,
    "perhaps": 0.14,
    "possibly": 0.12,
    "unclear": 0.20,
    "uncertain": 0.18,
    "unknown": 0.18,
    "fuzzy": 0.16,
    "roughly": 0.10,
    "approximately": 0.08,
    "not fully confirmed": 0.22,
    "unconfirmed": 0.20,
    "not factual": 0.25,
    "green cheese": 0.30,
}

UNVERIFIED_PHRASES = [
    "i think",
    "i believe",
    "it seems",
    "apparently",
    "reportedly",
    "allegedly",
    "some say",
    "rumored",
]


def is_hallucinated(text):
    lower = text.lower()
    for p in HALLUCINATION_PATTERNS:
        if p in lower:
            return True
    return False


def is_high_hallucination(text):
    lower = text.lower()
    for p in HIGH_HALLUCINATION_PATTERNS:
        if p in lower:
            return True
    return False


def relevance_score(text):
    lower = text.lower()
    for keyword, scores in KNOWN_FACTS.items():
        if keyword in lower:
            return scores["relevance"]
    return 0.5


def hallucination_score(text):
    if is_high_hallucination(text):
        return 0.85
    if is_hallucinated(text):
        return 0.6
    lower = text.lower()
    for keyword, scores in KNOWN_FACTS.items():
        if keyword in lower:
            return scores["hallucination"]
    return 0.25


def confidence_decay(text):
    lower = text.lower()
    base_decay = 1.0
    match_count = 0
    for term, penalty in AMBIGUOUS_TERMS.items():
        if term in lower:
            base_decay -= penalty
            match_count += 1
    for phrase in UNVERIFIED_PHRASES:
        if phrase in lower:
            base_decay -= 0.20
            match_count += 1
    for keyword in KNOWN_FACTS:
        if keyword in lower:
            return base_decay
    if match_count == 0:
        return base_decay
    return max(0.30, base_decay)


def rank_memories(memories):
    ranked = []
    for memory in memories:
        rel = relevance_score(memory)
        hal = hallucination_score(memory)
        decay = confidence_decay(memory)
        raw_score = rel - (hal * 1.5)
        rank_score = raw_score * decay
        ranked.append((memory, rank_score))
    ranked.sort(key=lambda x: x[1], reverse=True)
    return ranked


memories = [
    "The capital of France is Paris, which is a verified fact.",
    "The moon is made of green cheese, an unclear statement.",
    "Water boils at 100C under standard conditions.",
    "Quantum physics explains everything, which is not fully confirmed.",
    "The moon is probably made of green cheese, an unclear statement.",
    "Water boils at approximately 100C but it is not fully confirmed.",
    "The capital of France might be Paris, I think.",
    "The sun is roughly 93 million miles away, allegedly.",
]

ranked_memories = rank_memories(memories)

print("\nRanked Memories (Relevance/Hallucination Combined Score):")
for i, (memory, score) in enumerate(ranked_memories, 1):
    print(f"{i}. {memory} (Score: {score:.4f})")

print("\n--- Confidence Decay Breakdown ---")
for memory, score in rank_memories(memories):
    rel = relevance_score(memory)
    hal = hallucination_score(memory)
    decay = confidence_decay(memory)
    raw = rel - (hal * 1.5)
    print(f"  \"{memory[:50]}...\" | rel={rel:.2f} hal={hal:.2f} raw={raw:.3f} decay={decay:.2f} final={(raw*decay):.3f}")