Rule-Inferred Unit Test Coverage Score

The problem

It is difficult to know if your automated tests actually cover all the specific rules and requirements you wrote down in plain English.

What it does

It extracts natural language constraints from your system and checks them against your unit tests to generate a coverage score.

Why it matters

It measures how well your tests actually verify the specific rules of your system.

Validation

It was run in the sandbox and it failed. run output shows an error/traceback — the artifact does NOT run clean.

$ python3 rule_inferred_unit_test_coverage_v2.py
Inferred Invariants:
Rule 0: <lambda>
Test Coverage Score: 0.00%

No screenshot — there is nothing working to show. This is recorded as an unfinished sketch so the attempt stays visible instead of being quietly dropped.

The code

All of it — 104 lines, one file, standard library only.

import json
import os
import re
from typing import List, Callable, Tuple

...
def extract_rules(natural_language_rules: List[str]) -> List[Tuple[Callable[[dict], bool], str]]:
    """Extracts rule functions and condition strings from natural language constraints"""
    rules = []
    for rule_str in natural_language_rules:
        comparisons = re.findall(r'([a-zA-Z ]+) must be (greater than|less than|equal to) ([\d.]+)', rule_str)
        if comparisons:
            column, op, value = comparisons[0]
            condition_str = f'{column} {op} {value}'
            if op == 'greater than':
                rule = lambda x: x.get(column.strip(), 0) > float(value)
            elif op == 'less than':
                rule = lambda x: x.get(column.strip(), 0) < float(value)
            elif op == 'equal to':
                rule = lambda x: x.get(column.strip(), 0) == float(value)
            rules.append((rule, condition_str))
        # Add other rule types as needed
    return rules

def generate_test_cases(num_cases: int = 100) -> List[dict]:
    """Generates synthetic test cases"""
    test_cases = []
    for _ in range(num_cases):
        test_case = {
            'age': random.randint(10, 100),
            'email': f'user{random.randint(100, 999)}@example.com',
            'order_total': round(random.uniform(0, 1000), 2)
        }
        test_cases.append(test_case)
    return test_cases

def evaluate_coverage(rules: List[Tuple[Callable, str]], test_cases: List[dict]) -> dict:
    """Evaluates which rules are covered by test cases"""
    coverage = {i: False for i in range(len(rules))}
    for case in test_cases:
        for i, (rule, _) in enumerate(rules):
            if coverage[i]:
                continue
            try:
                if rule(case):
                    coverage[i] = True
            except:
                continue
    return coverage

def calculate_score(coverage: dict) -> float:
    """Calculates coverage percentage"""
    total = len(coverage)
    covered = sum(1 for v in coverage.values() if v)
    return (covered / total) * 100 if total > 0 else 0.0

def identify_coverage_gaps(rules: List[Tuple[Callable, str]], coverage: dict) -> list:
    """Identifies which specific conditions are missing coverage"""
    return [rules[i][1] for i, covered in coverage.items() if not covered]

import json

[... existing code ...]

if __name__ == "__main__":
    # ... existing code ...
    
    # Write results to output.json
    with open('output.json', 'w') as f:
        json.dump({
            'coverage_score': score,
            'coverage_gaps': coverage_gaps,
            'inferred_rules': list(rules_with_conditions)
        }, f)
    
    # 1. Extract rules with condition strings
    rules_with_conditions = extract_rules(constraints)
    rules, condition_strings = zip(*rules_with_conditions) if rules_with_conditions else ([], [])
    
    # 2. Generate test cases
    test_cases = generate_test_cases()
    
    # 3. Evaluate coverage
    coverage = evaluate_coverage(list(rules_with_conditions), test_cases)
    
    # 4. Calculate score
    score = calculate_score(coverage)
    
    # 5. Identify coverage gaps
    coverage_gaps = identify_coverage_gaps(rules_with_conditions, coverage)
    
    # Output results
    print('Inferred Invariants:')
    for i, (rule, cond) in enumerate(rules_with_conditions):
        print(f'Rule {i}: {cond}')
    
    print(f'\nTest Coverage Score: {score:.2f}%')
    
    if coverage_gaps:
        print('\nCoverage Gaps Found:')
        for gap in coverage_gaps:
            print(f'- {gap}')
    else:
        print('\nAll rules are covered by test cases.')