cli

cli

Command-line interface for autochecklist.

cmd_run(args)

Run full pipeline: generate checklists + score targets.

Source code in autochecklist/cli.py

def cmd_run(args: argparse.Namespace) -> None:
    """Run full pipeline: generate checklists + score targets."""
    from autochecklist import pipeline

    _maybe_load_config(args)
    data = _load_jsonl(args.data, args.input_key, args.target_key)

    scorer_kwargs = {}
    if args.scorer_prompt:
        scorer_kwargs["custom_prompt"] = Path(args.scorer_prompt)

    pipe = pipeline(
        task=args.pipeline,
        generator_model=args.generator_model,
        scorer_model=args.scorer_model,
        scorer=args.scorer,
        provider=args.provider,
        base_url=args.base_url,
        api_key=args.api_key,
        api_format=args.api_format,
        custom_prompt=Path(args.generator_prompt) if args.generator_prompt else None,
        scorer_kwargs=scorer_kwargs or None,
    )

    result = pipe.run_batch(
        data=data,
        output_path=args.output,
        show_progress=True,
        overwrite=args.overwrite,
    )

    print(f"\n{'='*40}", file=sys.stderr)
    print(f"Examples:          {len(result.scores)}", file=sys.stderr)
    print(f"Mean score:        {result.mean_score:.4f}", file=sys.stderr)
    print(f"Macro pass rate:   {result.macro_pass_rate:.4f}", file=sys.stderr)
    print(f"Micro pass rate:   {result.micro_pass_rate:.4f}", file=sys.stderr)
    print(f"{'='*40}", file=sys.stderr)

    if args.output:
        print(f"Results written to: {args.output}", file=sys.stderr)

cmd_generate(args)

Generate checklists only (no scoring).

Source code in autochecklist/cli.py

def cmd_generate(args: argparse.Namespace) -> None:
    """Generate checklists only (no scoring)."""
    from autochecklist import pipeline

    _maybe_load_config(args)
    data = _load_jsonl(args.data, args.input_key, target_key="target")

    pipe = pipeline(
        task=args.pipeline,
        generator_model=args.generator_model,
        provider=args.provider,
        base_url=args.base_url,
        api_key=args.api_key,
        api_format=args.api_format,
        custom_prompt=Path(args.generator_prompt) if args.generator_prompt else None,
    )

    checklists = pipe.generate_batch(
        data=data,
        output_path=args.output,
        show_progress=True,
        overwrite=args.overwrite,
    )

    print(f"\nGenerated {len(checklists)} checklists", file=sys.stderr)
    if args.output:
        print(f"Written to: {args.output}", file=sys.stderr)

cmd_score(args)

Score targets against a pre-existing checklist.

Source code in autochecklist/cli.py

def cmd_score(args: argparse.Namespace) -> None:
    """Score targets against a pre-existing checklist."""
    from autochecklist import Checklist, get_scorer
    from autochecklist.pipeline import BatchResult

    checklist = Checklist.load(args.checklist)
    data = _load_jsonl(args.data, args.input_key, args.target_key)

    scorer_cls = get_scorer(args.scorer or "batch")
    scorer_kwargs: dict[str, Any] = {}
    if args.scorer_model:
        scorer_kwargs["model"] = args.scorer_model
    if args.provider:
        scorer_kwargs["provider"] = args.provider
    if args.base_url:
        scorer_kwargs["base_url"] = args.base_url
    if args.api_key:
        scorer_kwargs["api_key"] = args.api_key
    if args.api_format:
        scorer_kwargs["api_format"] = args.api_format
    if args.scorer_prompt:
        scorer_kwargs["custom_prompt"] = Path(args.scorer_prompt)

    scorer = scorer_cls(**scorer_kwargs)

    targets = [d["target"] for d in data]
    inputs = [d["input"] for d in data]
    scores = scorer.score_batch(checklist, targets, inputs)

    result = BatchResult(scores=scores, data=data, checklist=checklist)

    print(f"\n{'='*40}", file=sys.stderr)
    print(f"Examples:          {len(result.scores)}", file=sys.stderr)
    print(f"Mean score:        {result.mean_score:.4f}", file=sys.stderr)
    print(f"Macro pass rate:   {result.macro_pass_rate:.4f}", file=sys.stderr)
    print(f"Micro pass rate:   {result.micro_pass_rate:.4f}", file=sys.stderr)
    print(f"{'='*40}", file=sys.stderr)

    if args.output:
        result.to_jsonl(args.output)
        print(f"Results written to: {args.output}", file=sys.stderr)

cmd_list(args)

List available components.

Source code in autochecklist/cli.py

def cmd_list(args: argparse.Namespace) -> None:
    """List available components."""
    from autochecklist import (
        list_generators_with_info,
        list_refiners_with_info,
        list_scorers_with_info,
    )

    component = args.component

    if component == "generators":
        items = list_generators_with_info()
        print(f"{'Name':<25} {'Level':<10} {'Scorer':<12} Description")
        print("-" * 80)
        for g in items:
            scorer = g.get("default_scorer") or "-"
            print(
                f"{g['name']:<25} {g['level']:<10} "
                f"{scorer:<12} {g.get('description', '')}"
            )
    elif component == "scorers":
        items = list_scorers_with_info()
        print(f"{'Name':<20} {'Method':<15} Description")
        print("-" * 60)
        for s in items:
            print(f"{s['name']:<20} {s.get('method', '-'):<15} {s.get('description', '')}")
    elif component == "refiners":
        items = list_refiners_with_info()
        print(f"{'Name':<20} Description")
        print("-" * 50)
        for r in items:
            print(f"{r['name']:<20} {r.get('description', '')}")

cmd_ui(args)

Launch the AutoChecklist UI.

Source code in autochecklist/cli.py

def cmd_ui(args: argparse.Namespace) -> None:
    """Launch the AutoChecklist UI."""
    import os
    import subprocess

    repo_root = _find_repo_root()
    if repo_root is None:
        print("Error: could not find AutoChecklist source tree. "
              "The 'ui' command is only available from a source checkout.", file=sys.stderr)
        sys.exit(1)

    cmd = [str(repo_root / "ui" / "launch_ui.sh")]
    if args.dev:
        cmd.append("--dev")

    os.chdir(repo_root / "ui")
    subprocess.run(cmd)