mirror of
https://github.com/rizsotto/Bear.git
synced 2026-05-28 00:20:45 +02:00
requirements: link tests to requirements via source tags
Replace the `tests:` frontmatter list in each requirement file with a `Requirements: <id>` tag placed directly on the protecting test(s). This gives tests a single source of truth for the link, so renaming or deleting a test cannot silently orphan a requirement. - Drop `tests:` from the template and all 11 requirement files. - Document the new tag convention in requirements/CLAUDE.md and integration-tests/CLAUDE.md; remove the unused `test_req_<id>_<desc>` naming rule. - Tag the existing integration tests (compilation_output, config, exit_codes, intercept) with the requirements they protect. - Add `requirements/check-coverage.sh`, which scans implemented requirements and fails when any has zero tagged tests. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Laszlo Nagy
@@ -25,17 +25,39 @@ bear_test!(test_name, |env| {
|
||||
});
|
||||
```
|
||||
|
||||
## Naming convention
|
||||
## Linking tests to requirements
|
||||
|
||||
Tests that protect a specific requirement should reference it in the name:
|
||||
Tests that protect a requirement cite its ID with a `Requirements:` tag. This
|
||||
is the sole source of truth for the test-to-requirement link; requirement files
|
||||
do not list their tests.
|
||||
|
||||
```
|
||||
test_req_<requirement_id>_<description>
|
||||
Format:
|
||||
|
||||
```rust
|
||||
// Requirements: output-json-compilation-database, output-append
|
||||
#[test]
|
||||
fn append_works_as_expected() -> Result<()> { ... }
|
||||
```
|
||||
|
||||
Example: `test_req_output_001_json_format` protects requirement `output-001`.
|
||||
Rules:
|
||||
|
||||
This makes it possible to trace which requirements have test coverage.
|
||||
- Value is a comma-separated list of requirement IDs (filenames in
|
||||
`requirements/` without the `.md` extension).
|
||||
- Place the tag on the line(s) directly above `#[test]` (or the test macro).
|
||||
- If every test in a file covers the same requirement, a file-level
|
||||
`//! Requirements: <id>` near the top is sufficient. Test-level tags
|
||||
override file-level tags.
|
||||
- A test may cite multiple requirements when it legitimately exercises more
|
||||
than one.
|
||||
|
||||
To find tests for a requirement:
|
||||
|
||||
```bash
|
||||
grep -rn "Requirements:.*<requirement-id>" bear/ intercept-preload/ integration-tests/
|
||||
```
|
||||
|
||||
See `requirements/CLAUDE.md` for the coverage-check script that verifies every
|
||||
`implemented` requirement has at least one tagged test.
|
||||
|
||||
## Debugging
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ use serde_json::Value;
|
||||
|
||||
/// Test compilation with build script that calls compiler
|
||||
/// This generates events that the semantic analyzer can process
|
||||
// Requirements: output-json-compilation-database, output-compilation-entries, output-atomic-write, output-path-format
|
||||
#[test]
|
||||
#[cfg(all(has_executable_compiler_c, has_executable_shell))]
|
||||
fn simple_single_file_compilation() -> Result<()> {
|
||||
@@ -68,6 +69,7 @@ fn simple_single_file_compilation() -> Result<()> {
|
||||
|
||||
/// Test successful build with multiple sources (C and C++)
|
||||
/// Verifies Bear handles mixed compilation units
|
||||
// Requirements: output-json-compilation-database, output-compilation-entries
|
||||
#[test]
|
||||
#[cfg(all(has_executable_compiler_c, has_executable_compiler_cxx, has_executable_shell))]
|
||||
fn successful_build_multiple_sources() -> Result<()> {
|
||||
@@ -160,6 +162,7 @@ fn successful_build_multiple_sources() -> Result<()> {
|
||||
}
|
||||
|
||||
/// Test output is overwritten when no append flag
|
||||
// Requirements: output-append
|
||||
#[test]
|
||||
#[cfg(all(has_executable_compiler_c, has_executable_shell))]
|
||||
fn without_append_output_is_overwritten() -> Result<()> {
|
||||
@@ -210,6 +213,7 @@ fn without_append_output_is_overwritten() -> Result<()> {
|
||||
}
|
||||
|
||||
/// Test append functionality
|
||||
// Requirements: output-append
|
||||
#[test]
|
||||
#[cfg(all(has_executable_compiler_c, has_executable_shell))]
|
||||
fn append_works_as_expected() -> Result<()> {
|
||||
@@ -262,6 +266,7 @@ fn append_works_as_expected() -> Result<()> {
|
||||
|
||||
/// Test build with compilation failures - should still generate partial database
|
||||
/// Verifies Bear can handle partial build failures
|
||||
// Requirements: output-json-compilation-database
|
||||
#[test]
|
||||
#[cfg(all(has_executable_compiler_c, has_executable_shell))]
|
||||
fn broken_build_partial_success() -> Result<()> {
|
||||
@@ -331,6 +336,7 @@ fn broken_build_partial_success() -> Result<()> {
|
||||
|
||||
/// Test empty build - should generate empty compilation database
|
||||
/// Verifies Bear handles builds with no compilation commands
|
||||
// Requirements: output-json-compilation-database
|
||||
#[test]
|
||||
#[cfg(all(has_executable_true, has_executable_shell, has_executable_echo))]
|
||||
fn empty_build_generates_empty_database() -> Result<()> {
|
||||
@@ -359,6 +365,7 @@ fn empty_build_generates_empty_database() -> Result<()> {
|
||||
|
||||
/// Test compilation with multiple source files using single command
|
||||
/// Verifies Bear handles batch compilation commands
|
||||
// Requirements: output-json-compilation-database, output-compilation-entries
|
||||
#[test]
|
||||
#[cfg(all(has_executable_compiler_c, has_executable_shell))]
|
||||
fn multiple_sources_single_command() -> Result<()> {
|
||||
|
||||
@@ -205,6 +205,7 @@ sources:
|
||||
|
||||
/// Test path format configuration
|
||||
/// Verifies different path formatting options
|
||||
// Requirements: output-path-format
|
||||
#[test]
|
||||
#[cfg(has_preload_library)]
|
||||
#[cfg(all(has_executable_compiler_c, has_executable_shell))]
|
||||
@@ -357,6 +358,7 @@ sources:
|
||||
|
||||
/// Test duplicate filter configuration
|
||||
/// Verifies duplicate filtering options work
|
||||
// Requirements: output-duplicate-detection
|
||||
#[test]
|
||||
#[cfg(has_preload_library)]
|
||||
#[cfg(all(has_executable_compiler_c, has_executable_shell))]
|
||||
|
||||
@@ -68,6 +68,7 @@ fn exit_code_for_non_existing_command() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Requirements: interception-signal-forwarding
|
||||
#[test]
|
||||
#[cfg(has_executable_true)]
|
||||
fn exit_code_for_true() -> Result<()> {
|
||||
@@ -79,6 +80,7 @@ fn exit_code_for_true() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Requirements: interception-signal-forwarding
|
||||
#[test]
|
||||
#[cfg(has_executable_false)]
|
||||
fn exit_code_for_false() -> Result<()> {
|
||||
@@ -90,6 +92,7 @@ fn exit_code_for_false() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Requirements: interception-signal-forwarding
|
||||
#[test]
|
||||
#[cfg(has_executable_sleep)]
|
||||
fn exit_code_when_signaled() -> Result<()> {
|
||||
@@ -125,6 +128,7 @@ fn exit_code_when_signaled() -> Result<()> {
|
||||
// Intercept mode exit code tests
|
||||
|
||||
/// Test that intercept command returns 0 for successful interception
|
||||
// Requirements: interception-signal-forwarding
|
||||
#[test]
|
||||
#[cfg(has_executable_true)]
|
||||
fn intercept_exit_code_for_success() -> Result<()> {
|
||||
@@ -136,6 +140,7 @@ fn intercept_exit_code_for_success() -> Result<()> {
|
||||
}
|
||||
|
||||
/// Test that intercept command propagates command failure exit codes
|
||||
// Requirements: interception-signal-forwarding
|
||||
#[test]
|
||||
#[cfg(has_executable_false)]
|
||||
fn intercept_exit_code_for_failure() -> Result<()> {
|
||||
|
||||
@@ -12,6 +12,7 @@ use anyhow::Result;
|
||||
use encoding_rs;
|
||||
|
||||
/// Test basic command interception with preload mechanism
|
||||
// Requirements: interception-preload-mechanism
|
||||
#[test]
|
||||
#[cfg(target_family = "unix")]
|
||||
#[cfg(has_executable_compiler_c)]
|
||||
@@ -320,6 +321,7 @@ fn libtool_command_interception() -> Result<()> {
|
||||
}
|
||||
|
||||
/// Test wrapper-based interception
|
||||
// Requirements: interception-wrapper-mechanism
|
||||
#[test]
|
||||
#[cfg(target_family = "unix")]
|
||||
#[cfg(all(has_executable_compiler_c, has_executable_echo))]
|
||||
@@ -537,6 +539,7 @@ fn fakeroot_integration() -> Result<()> {
|
||||
/// ccache recursion (where the wrapper calls ccache which finds the wrapper
|
||||
/// again via PATH), we construct a PATH containing only the real compiler
|
||||
/// directory, excluding any ccache directories.
|
||||
// Requirements: interception-wrapper-mechanism
|
||||
#[test]
|
||||
#[cfg(all(has_executable_compiler_c, has_executable_shell))]
|
||||
fn wrapper_mode_resolves_cc_bare_name_via_path() -> Result<()> {
|
||||
|
||||
+72
-24
@@ -1,8 +1,8 @@
|
||||
## Requirements directory
|
||||
|
||||
This directory captures functional and non-functional requirements for Bear.
|
||||
Requirements are the source of truth for what Bear should do. Integration tests
|
||||
verify that implemented requirements work correctly.
|
||||
Requirements are the source of truth for what Bear should do. Tests (integration
|
||||
and unit) verify that implemented requirements work correctly.
|
||||
|
||||
## File naming
|
||||
|
||||
@@ -10,8 +10,9 @@ verify that implemented requirements work correctly.
|
||||
<area>-<short-name>.md
|
||||
```
|
||||
|
||||
The filename serves as the requirement's unique identifier. Use it for
|
||||
cross-references in other requirement files (e.g. "see `output-path-format`").
|
||||
The filename (without extension) is the requirement's unique identifier. Use it
|
||||
for cross-references in other requirement files and as the value tests cite in
|
||||
their `Requirements:` tag (see below).
|
||||
|
||||
Examples (see existing files in this directory):
|
||||
- `output-json-compilation-database.md`
|
||||
@@ -26,9 +27,6 @@ Every requirement file must follow this structure:
|
||||
---
|
||||
title: JSON compilation database format
|
||||
status: implemented
|
||||
tests:
|
||||
- test_basic_c_compilation
|
||||
- test_output_format_json
|
||||
---
|
||||
|
||||
## Intent
|
||||
@@ -45,6 +43,11 @@ What the user expects to happen, written from the user's perspective.
|
||||
Performance, platform support, backwards compatibility, etc.
|
||||
Only include if relevant.
|
||||
|
||||
## Testing
|
||||
|
||||
Given-When-Then scenarios that describe how the requirement should be verified.
|
||||
These are the canonical scenarios; tests implement them.
|
||||
|
||||
## Notes
|
||||
|
||||
Design decisions, trade-offs, links to issues or discussions.
|
||||
@@ -61,26 +64,69 @@ Design decisions, trade-offs, links to issues or discussions.
|
||||
| `deferred` | Accepted but postponed (add reason in Notes) |
|
||||
| `rejected` | Reviewed and declined (add reason in Notes) |
|
||||
|
||||
## Linking to tests
|
||||
## Linking tests to requirements
|
||||
|
||||
The `tests:` frontmatter field lists integration test function names that protect
|
||||
this requirement. When writing a new integration test for a requirement, add the
|
||||
test name here.
|
||||
Tests cite the requirements they protect using a `Requirements:` tag. The tag
|
||||
lives in the test source, not in this directory's frontmatter, so renaming or
|
||||
deleting a test updates the link in the same edit.
|
||||
|
||||
Format:
|
||||
|
||||
```rust
|
||||
// Requirements: output-json-compilation-database, output-append
|
||||
#[test]
|
||||
fn append_works_as_expected() -> Result<()> { ... }
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- Value is a comma-separated list of requirement IDs (filenames without `.md`).
|
||||
- Place the tag on the line(s) directly above `#[test]` (or the test macro).
|
||||
- For a whole file covering a single requirement, use `//! Requirements: <id>`
|
||||
at the top of the file. Test-level tags override file-level tags.
|
||||
- Unit tests in `bear/` and `intercept-preload/` use the same convention.
|
||||
|
||||
## Reverse lookup
|
||||
|
||||
To find every test that protects a requirement:
|
||||
|
||||
```bash
|
||||
grep -rn "Requirements:.*<requirement-id>" bear/ intercept-preload/ integration-tests/
|
||||
```
|
||||
|
||||
For example, to find tests for `output-append`:
|
||||
|
||||
```bash
|
||||
grep -rn "Requirements:.*output-append" bear/ intercept-preload/ integration-tests/
|
||||
```
|
||||
|
||||
## Coverage check
|
||||
|
||||
`requirements/check-coverage.sh` scans every requirement file and verifies that
|
||||
each `implemented` requirement has at least one `Requirements:` tag referencing
|
||||
it. Run it from the repo root:
|
||||
|
||||
```bash
|
||||
./requirements/check-coverage.sh
|
||||
```
|
||||
|
||||
The script exits non-zero if any `implemented` requirement lacks coverage.
|
||||
|
||||
## How agents should use this
|
||||
|
||||
1. **Before implementing a feature**: check if a requirement exists. If not, create one
|
||||
with status `proposed` and await approval before coding.
|
||||
2. **Before modifying behavior**: find the requirement that governs it. Read acceptance
|
||||
criteria to understand what must not break.
|
||||
3. **After implementing**: update status to `implemented`, list tests in frontmatter.
|
||||
4. **When fixing a bug**: check if the bug violates an existing requirement. If so,
|
||||
add a test that reproduces the bug and reference it in the requirement.
|
||||
1. **Before implementing a feature**: check if a requirement exists. If not,
|
||||
create one with status `proposed` and await approval before coding.
|
||||
2. **Before modifying behavior**: find the requirement that governs it. Read
|
||||
acceptance criteria to understand what must not break.
|
||||
3. **After implementing**: set status to `implemented` and add a
|
||||
`Requirements: <id>` tag to the test(s) that protect the requirement.
|
||||
4. **When fixing a bug**: check if the bug violates an existing requirement. If
|
||||
so, add a test that reproduces the bug and tag it with the requirement ID.
|
||||
|
||||
## Incubating new features
|
||||
|
||||
Features that are not yet ready for implementation stay at `proposed` or `accepted`.
|
||||
Use the requirement file to capture:
|
||||
Features that are not yet ready for implementation stay at `proposed` or
|
||||
`accepted`. Use the requirement file to capture:
|
||||
|
||||
- User-facing intent (what problem does this solve?)
|
||||
- Acceptance criteria (how do we know it works?)
|
||||
@@ -92,8 +138,10 @@ a requirement before it reaches `accepted`.
|
||||
|
||||
## Regression protection
|
||||
|
||||
The link between requirements and integration tests is the regression safety net:
|
||||
The link between requirements and tests is the regression safety net:
|
||||
|
||||
- Every `implemented` requirement must have at least one test in its `tests:` field
|
||||
- If a test is deleted or renamed, update the requirement's `tests:` field
|
||||
- Run `cargo test` to verify all listed tests still pass
|
||||
- Every `implemented` requirement must have at least one test tagged with its ID
|
||||
- When a test is renamed or deleted, the tag moves with it (or disappears), so
|
||||
the link cannot silently rot
|
||||
- `check-coverage.sh` catches `implemented` requirements that have drifted to
|
||||
zero test coverage
|
||||
|
||||
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env bash
|
||||
# Verify that every `implemented` requirement has at least one test
|
||||
# referencing it via a `Requirements: <id>` tag.
|
||||
#
|
||||
# Run from the repo root:
|
||||
# ./requirements/check-coverage.sh
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 - every implemented requirement has at least one test reference
|
||||
# 1 - at least one implemented requirement has zero references
|
||||
# 2 - invocation error (e.g. script run from wrong directory)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
repo_root="$(cd "${script_dir}/.." && pwd)"
|
||||
|
||||
if [ ! -d "${repo_root}/requirements" ]; then
|
||||
echo "error: requirements directory not found at ${repo_root}/requirements" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
search_roots=(
|
||||
"${repo_root}/bear"
|
||||
"${repo_root}/intercept-preload"
|
||||
"${repo_root}/integration-tests"
|
||||
)
|
||||
|
||||
missing=0
|
||||
checked=0
|
||||
|
||||
for file in "${repo_root}/requirements"/*.md; do
|
||||
[ -e "${file}" ] || continue
|
||||
base="$(basename "${file}" .md)"
|
||||
|
||||
# Skip the CLAUDE.md (not a requirement file)
|
||||
if [ "${base}" = "CLAUDE" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Extract status from YAML frontmatter (first match wins)
|
||||
status="$(awk '/^status:[[:space:]]*/ { sub(/^status:[[:space:]]*/, ""); print; exit }' "${file}")"
|
||||
|
||||
if [ "${status}" != "implemented" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
checked=$((checked + 1))
|
||||
|
||||
# Count matches across the search roots. A match is any line that contains
|
||||
# "Requirements:" followed (anywhere on the line) by the requirement ID.
|
||||
# Word-boundary on both sides prevents "output-path" from matching
|
||||
# "output-path-format".
|
||||
pattern="Requirements:.*\\b${base}\\b"
|
||||
|
||||
if grep -RnE "${pattern}" "${search_roots[@]}" >/dev/null 2>&1; then
|
||||
:
|
||||
else
|
||||
echo "MISSING: ${base} (status: implemented) has no test tagged with its ID"
|
||||
missing=$((missing + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "Checked ${checked} implemented requirement(s); ${missing} without coverage."
|
||||
|
||||
if [ "${missing}" -gt 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,7 +1,6 @@
|
||||
---
|
||||
title: Handle compiler env vars that contain flags
|
||||
status: proposed
|
||||
tests: []
|
||||
---
|
||||
|
||||
## Problem
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
---
|
||||
title: LD_PRELOAD-based command interception
|
||||
status: implemented
|
||||
tests:
|
||||
- basic_command_interception
|
||||
---
|
||||
|
||||
## Intent
|
||||
|
||||
@@ -1,12 +1,6 @@
|
||||
---
|
||||
title: Signal forwarding and exit-code propagation
|
||||
status: implemented
|
||||
tests:
|
||||
- exit_code_when_signaled
|
||||
- exit_code_for_true
|
||||
- exit_code_for_false
|
||||
- intercept_exit_code_for_success
|
||||
- intercept_exit_code_for_failure
|
||||
---
|
||||
|
||||
## Intent
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
---
|
||||
title: Wrapper-based command interception
|
||||
status: implemented
|
||||
tests:
|
||||
- wrapper_based_interception
|
||||
- wrapper_mode_resolves_cc_bare_name_via_path
|
||||
---
|
||||
|
||||
## Intent
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
---
|
||||
title: Prevent wrapper recursion with compiler wrappers
|
||||
status: proposed
|
||||
tests: []
|
||||
---
|
||||
|
||||
## Problem
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
---
|
||||
title: Append mode for compilation database
|
||||
status: implemented
|
||||
tests:
|
||||
- append_works_as_expected
|
||||
- without_append_output_is_overwritten
|
||||
---
|
||||
|
||||
## Intent
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
---
|
||||
title: Atomic file write for compilation database
|
||||
status: implemented
|
||||
tests:
|
||||
- simple_single_file_compilation
|
||||
---
|
||||
|
||||
## Intent
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
---
|
||||
title: Compilation entries from an intercepted invocation
|
||||
status: implemented
|
||||
tests:
|
||||
- multiple_sources_single_command
|
||||
- successful_build_multiple_sources
|
||||
- simple_single_file_compilation
|
||||
---
|
||||
|
||||
## Intent
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
---
|
||||
title: Duplicate entry detection and filtering
|
||||
status: implemented
|
||||
tests:
|
||||
- duplicate_filter_config
|
||||
---
|
||||
|
||||
## Intent
|
||||
|
||||
@@ -1,12 +1,6 @@
|
||||
---
|
||||
title: JSON compilation database output
|
||||
status: implemented
|
||||
tests:
|
||||
- simple_single_file_compilation
|
||||
- successful_build_multiple_sources
|
||||
- empty_build_generates_empty_database
|
||||
- multiple_sources_single_command
|
||||
- broken_build_partial_success
|
||||
---
|
||||
|
||||
## Intent
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
---
|
||||
title: Path format for file and directory fields
|
||||
status: implemented
|
||||
tests:
|
||||
- path_format_config
|
||||
- simple_single_file_compilation
|
||||
---
|
||||
|
||||
## Intent
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
---
|
||||
title: Source directory filtering
|
||||
status: implemented
|
||||
tests: []
|
||||
---
|
||||
|
||||
## Intent
|
||||
|
||||
Reference in New Issue
Block a user