Skip to content

utils

utils

Utility functions.

load_json(path)

Load a JSON file.

Source code in autochecklist/utils/io.py 
 8
 9
10
11
def load_json(path: Union[str, Path]) -> Any:
    """Load a JSON file."""
    with open(path, "r", encoding="utf-8") as f:
        return json.load(f)

save_json(path, data, indent=2)

Save data to a JSON file.

Source code in autochecklist/utils/io.py 
14
15
16
17
def save_json(path: Union[str, Path], data: Any, indent: int = 2) -> None:
    """Save data to a JSON file."""
    with open(path, "w", encoding="utf-8") as f:
        json.dump(data, f, indent=indent, ensure_ascii=False)

load_jsonl(path)

Load a JSONL file as a list of dictionaries.

Source code in autochecklist/utils/io.py 
20
21
22
23
24
25
26
27
28
def load_jsonl(path: Union[str, Path]) -> List[Dict[str, Any]]:
    """Load a JSONL file as a list of dictionaries."""
    items = []
    with open(path, "r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if line:
                items.append(json.loads(line))
    return items

save_jsonl(path, data)

Save a list of dictionaries to a JSONL file.

Source code in autochecklist/utils/io.py 
40
41
42
43
44
def save_jsonl(path: Union[str, Path], data: List[Dict[str, Any]]) -> None:
    """Save a list of dictionaries to a JSONL file."""
    with open(path, "w", encoding="utf-8") as f:
        for item in data:
            f.write(json.dumps(item, ensure_ascii=False) + "\n")

parse_checklist_items(response, max_items=None)

Parse checklist items from LLM response.

Supports multiple formats: - [[item]] bracketed format (TICK, checklist-effectiveness-study) - Numbered list: "1. Question?" - Bulleted list: "- Question?" - RLCF weighted format: "- Question? (weight)"

Parameters:

Name Type Description Default
response str

Raw LLM response text

 required
max_items int

Maximum items to return

 None

Returns:

Type Description
List[ChecklistItem]

List of ChecklistItem objects
Source code in autochecklist/utils/parsing.py 
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
def parse_checklist_items(
    response: str,
    max_items: int = None,
) -> List[ChecklistItem]:
    """Parse checklist items from LLM response.

    Supports multiple formats:
    - [[item]] bracketed format (TICK, checklist-effectiveness-study)
    - Numbered list: "1. Question?"
    - Bulleted list: "- Question?"
    - RLCF weighted format: "- Question? (weight)"

    Args:
        response: Raw LLM response text
        max_items: Maximum items to return

    Returns:
        List of ChecklistItem objects
    """
    items: List[ChecklistItem] = []

    # Method 1: [[item]] bracketed format
    bracketed = re.findall(r"\[\[(.*?)\]\]", response, re.DOTALL)
    if bracketed:
        for text in bracketed:
            question = text.strip()
            if question and _looks_like_question(question):
                items.append(ChecklistItem(question=_ensure_question_mark(question)))

    # Method 2: RLCF weighted format "- Question? (weight)"
    if not items:
        weighted_pattern = r"[-•]\s*(.+?\?)\s*\((\d+)\)"
        for match in re.finditer(weighted_pattern, response):
            question = match.group(1).strip()
            weight = int(match.group(2))
            items.append(ChecklistItem(
                question=question,
                weight=float(weight)
            ))

    # Method 3: Numbered list "1. Question?"
    if not items:
        numbered = re.findall(r"^\s*\d+\.\s*(.+)$", response, re.MULTILINE)
        for text in numbered:
            question = text.strip()
            if _looks_like_question(question):
                items.append(ChecklistItem(question=_ensure_question_mark(question)))

    # Method 4: Bulleted list "- Question?"
    if not items:
        bulleted = re.findall(r"^\s*[-•]\s*(.+)$", response, re.MULTILINE)
        for text in bulleted:
            question = text.strip()
            if _looks_like_question(question):
                items.append(ChecklistItem(question=_ensure_question_mark(question)))

    # Limit to max_items
    if max_items:
        return items[:max_items]
    return items

get_embeddings(texts, model='text-embedding-3-large', api_key=None)

Get embeddings for a list of texts.

Uses OpenAI embeddings via direct API call.

Parameters:

Name Type Description Default
texts List[str]

List of text strings to embed

 required
model str

Embedding model to use (default: text-embedding-3-large)

 'text-embedding-3-large'
api_key Optional[str]

OpenAI API key (uses OPENAI_API_KEY env var if not provided)

 None

Returns:

Type Description
ndarray

numpy array of shape (len(texts), embedding_dim)
Source code in autochecklist/utils/embeddings.py 
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
def get_embeddings(
    texts: List[str],
    model: str = "text-embedding-3-large",
    api_key: Optional[str] = None,
) -> np.ndarray:
    """Get embeddings for a list of texts.

    Uses OpenAI embeddings via direct API call.

    Args:
        texts: List of text strings to embed
        model: Embedding model to use (default: text-embedding-3-large)
        api_key: OpenAI API key (uses OPENAI_API_KEY env var if not provided)

    Returns:
        numpy array of shape (len(texts), embedding_dim)
    """
    import httpx
    import os

    api_key = api_key or os.getenv("OPENAI_API_KEY")
    if not api_key:
        raise ValueError(
            "OpenAI API key required for embeddings. "
            "Set OPENAI_API_KEY environment variable or pass api_key parameter."
        )

    with httpx.Client(timeout=60.0) as client:
        response = client.post(
            "https://api.openai.com/v1/embeddings",
            headers={
                "Authorization": f"Bearer {api_key}",
                "Content-Type": "application/json",
            },
            json={
                "input": texts,
                "model": model,
            },
        )
        response.raise_for_status()
        data = response.json()

    # Extract embeddings in order
    np = _require_numpy()
    embeddings = [item["embedding"] for item in sorted(data["data"], key=lambda x: x["index"])]
    return np.array(embeddings)

cosine_similarity(embeddings)

Compute pairwise cosine similarity matrix.

Parameters:

Name Type Description Default
embeddings ndarray

numpy array of shape (n, embedding_dim)

 required

Returns:

Type Description
ndarray

numpy array of shape (n, n) with cosine similarities
Source code in autochecklist/utils/embeddings.py 
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
def cosine_similarity(embeddings: np.ndarray) -> np.ndarray:
    """Compute pairwise cosine similarity matrix.

    Args:
        embeddings: numpy array of shape (n, embedding_dim)

    Returns:
        numpy array of shape (n, n) with cosine similarities
    """
    np = _require_numpy()
    # Normalize embeddings
    norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
    normalized = embeddings / norms

    # Compute cosine similarity matrix
    return np.dot(normalized, normalized.T)

find_similar_pairs(similarity_matrix, threshold=0.85)

Find pairs of items with similarity above threshold.

Parameters:

Name Type Description Default
similarity_matrix ndarray

Pairwise similarity matrix

 required
threshold float

Minimum similarity to include

 0.85

Returns:

Type Description
List[Tuple[int, int, float]]

List of (i, j, similarity) tuples for similar pairs
Source code in autochecklist/utils/embeddings.py 
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
def find_similar_pairs(
    similarity_matrix: np.ndarray,
    threshold: float = 0.85,
) -> List[Tuple[int, int, float]]:
    """Find pairs of items with similarity above threshold.

    Args:
        similarity_matrix: Pairwise similarity matrix
        threshold: Minimum similarity to include

    Returns:
        List of (i, j, similarity) tuples for similar pairs
    """
    n = similarity_matrix.shape[0]
    similar_pairs = []

    for i in range(n):
        for j in range(i + 1, n):
            if similarity_matrix[i, j] >= threshold:
                similar_pairs.append((i, j, similarity_matrix[i, j]))

    return similar_pairs

build_similarity_graph(questions, threshold=0.85, embeddings=None, api_key=None)

Build similarity graph from questions.

Parameters:

Name Type Description Default
questions List[str]

List of question strings

 required
threshold float

Similarity threshold for edges

 0.85
embeddings Optional[ndarray]

Pre-computed embeddings (optional)

 None
api_key Optional[str]

OpenAI API key for embeddings

 None

Returns:

Type Description
Tuple[ndarray, ndarray, List[Tuple[int, int, float]]]

Tuple of (embeddings, similarity_matrix, similar_pairs)
Source code in autochecklist/utils/embeddings.py 
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
def build_similarity_graph(
    questions: List[str],
    threshold: float = 0.85,
    embeddings: Optional[np.ndarray] = None,
    api_key: Optional[str] = None,
) -> Tuple[np.ndarray, np.ndarray, List[Tuple[int, int, float]]]:
    """Build similarity graph from questions.

    Args:
        questions: List of question strings
        threshold: Similarity threshold for edges
        embeddings: Pre-computed embeddings (optional)
        api_key: OpenAI API key for embeddings

    Returns:
        Tuple of (embeddings, similarity_matrix, similar_pairs)
    """
    if embeddings is None:
        embeddings = get_embeddings(questions, api_key=api_key)

    similarity_matrix = cosine_similarity(embeddings)
    similar_pairs = find_similar_pairs(similarity_matrix, threshold)

    return embeddings, similarity_matrix, similar_pairs