Contextual-Saliency-Ranking

The problem

It is difficult to pinpoint which parts of a long text are most relevant to a specific question when information is scattered throughout a narrative.

What it does

It ranks pieces of text by measuring how many unique concepts they share with a query while accounting for where those pieces appear in the flow of the story.

Why it matters

It identifies the most relevant information by balancing the importance of the content with its position in the narrative.

Validation

It was run inside an isolated container with no network access. This is the exact command and the real output it produced — captured process output, not written by a model.

$ python3 contextual_saliency_ranker_v2.py
Contextual-Saliency Rankings
Score: 0.0031\nNatural language processing is a field of AI dealing with human-computer interaction.

Score: 0.0013\nTF-IDF is a numerical statistic that reflects how important a word is to a document in a collection or corpus.

Score: 0.0000\nThe quick brown fox jumps over the lazy dog.

A screenshot of that run.

A clean run proves this does what is shown above, in a CPU-only sandbox. It is a small research demo — not a production tool, and nothing here was published anywhere.

The code

All of it — 100 lines, one file, standard library only.

# contextual_saliency_ranker_v2.py
#!/usr/bin/env python3
import math
from collections import defaultdict
import re

def tokenize(text):
    """ Splits text into words, lowercases, and removes punctuation """
    return [word.lower() for word in re.findall(r'\w+', text)]

class ContextualSaliencyRanker:
    def __init__(self, segments, query):
        self.segments = segments
        self.query = query
        self.all_terms = defaultdict(int)
        self.segment_tf = []
        self.segment_token_positions = []  # Stores token positions
        self.query_tokens = tokenize(query)
        self._calculate_idf()
        self._calculate_tf()
        self._calculate_scores()
    
    def _calculate_idf(self):
        """ Calculates IDF (Inverse Document Frequency) for all terms """
        N = len(self.segments)
        # Count document frequencies for segment in self.segments:
        for segment in self.segments:
            tokens = set(tokenize(segment))
            for term in tokens:
                self.all_terms[term] += 1
        # Calculate IDF
        self.idf = {term: math.log(N / (count + 1)) for term, count in self.all_terms.items()}
    
    def _calculate_tf(self):
        """ Calculates TF (Term Frequency) for each segment """
        self.segment_tf = []
        for segment in self.segments:
            tokens = tokenize(segment)
            term_counts = defaultdict(int)
            token_positions = [(token, idx) for idx, token in enumerate(tokens)]
            for token, _ in token_positions:
                term_counts[token] += 1
            # TF as term count divided by total terms in segment
            tf = {token: count / len(tokens) for token, count in term_counts.items()}
            self.segment_tf.append(tf)
            self.segment_token_positions.append(token_positions)
    
    def _calculate_scores(self):
        """ Calculates relevance scores with positional decay and contextual weighting """
        self.scores = []
        for i, (segment, tf) in enumerate(zip(self.segments, self.segment_tf)):
            score = 0.0
            # Positional decay: 1/(position + 1)
            decay = 1 / (i + 1)
            
            # Collect all positions of query terms in this segment
            token_positions = self.segment_token_positions[i]
            query_term_positions = []
            for token, pos in token_positions:
                if token in self.query_tokens:
                    query_term_positions.append(pos)
            
            # Calculate proximity weight based on distances between query terms
            proximity_weight = 1.0
            if len(query_term_positions) > 1:
                # Sort the positions to get the order they appear in the segment
                query_term_positions.sort()
                # Sum the inverses of the distances between consecutive terms
                for j in range(len(query_term_positions) - 1):
                    distance = query_term_positions[j+1] - query_term_positions[j]
                    if distance > 0:
                        proximity_weight += 1 / distance  # Add inverse distance
            
            # Calculate base score
            for term in self.query_tokens:
                if term in tf:
                    tf_query = self.query_tokens.count(term) / len(self.query_tokens)
                    score += tf[term] * self.idf.get(term, 0) * tf_query
            
            # Apply decay and proximity weight
            score *= decay * proximity_weight
            self.scores.append(score)
    
    def get_rankings(self):
        """ Returns segments ranked by relevance score """
        ranked = sorted(enumerate(self.scores), key=lambda x: x[1], reverse=True)
        return [(self.segments[idx], score) for idx, score in ranked]

# Example usage with proximity demonstration
if __name__ == "__main__":
    segments = [
        "The importance of words in documents cannot be overstated.",
        "In documents, words are important, but context matters more for meaning.",
        "The document's importance is determined by its words and structure."
    ]
    query = "importance of words in documents"
    ranker = ContextualSaliencyRanker(segments, query)
    print("Contextual-Saliency Rankings with Proximity Weighting")
    for segment, score in ranker.get_rankings():
        print(f"Score: {score:.4f}\n{segment}\n")