From 178ffa927d0324a1d255382372df3fc30c080075 Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Tue, 21 Jul 2026 21:29:53 +0200 Subject: [PATCH 1/3] readme-updater: make the generator feedback-aware Regenerating a README overwrote the branch wholesale, so any human review fixes on the open PR were silently dropped on the next run (e.g. EvilOlaf's "bash/ python" note on the build README's requirements list). Now, before generating, the workflow collects the human review + issue comments (bots filtered out) from the still-open `chore/update-readme` PR for that repo and passes them to generate_readme.py via --feedback. The script folds them into the prompt as corrections to apply and not re-introduce. Reviewer fixes survive a regeneration; the PR body invites reviewers to comment for exactly this reason. Also sharpen the system prompt: when stating what the project is built with / its requirements, name the actual languages and tooling evidenced by the files (shebangs, extensions, manifests, invoked commands) rather than vague generalities -- the root of that review comment. Signed-off-by: Igor Pecovnik --- .../workflows/maintenance-update-readme.yml | 34 ++++++++++++++++-- scripts/generate_readme.py | 36 +++++++++++++++++-- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/.github/workflows/maintenance-update-readme.yml b/.github/workflows/maintenance-update-readme.yml index 732c68b088..c770911385 100644 --- a/.github/workflows/maintenance-update-readme.yml +++ b/.github/workflows/maintenance-update-readme.yml @@ -6,6 +6,10 @@ # but README.md. peter-evans/create-pull-request makes no PR when the generated # README is byte-identical to the current one, so re-runs are cheap. # +# Feedback loop: if a previous README PR is still open, its human review comments +# are collected and fed back into the next generation, so reviewer fixes survive a +# regeneration instead of being silently overwritten. +# # Secrets (already present on this repo): # ANTHROPIC_API_KEY - Claude API key (same one reporting-release-summary uses) # ACCESS_TOKEN - PAT with `repo` scope on the target repos (the builtin @@ -78,6 +82,27 @@ jobs: - name: Install Anthropic SDK run: pip install 'anthropic>=0.40.0' + # If a previous "chore/update-readme" PR is still open, fold its human review + # comments back in, so regenerating doesn't silently drop reviewer fixes. + - name: Collect reviewer feedback from the open README PR + env: + GH_TOKEN: ${{ secrets.ACCESS_TOKEN }} + REPO: ${{ matrix.repo }} + run: | + set -euo pipefail + : > feedback.txt + pr="$(gh pr list -R "$REPO" --head chore/update-readme --state open \ + --json number --jq '.[0].number // empty')" + if [ -n "$pr" ]; then + gh api "repos/$REPO/pulls/$pr/comments" \ + --jq '.[] | select((.user.login|test("\\[bot\\]|coderabbit";"i"))|not) | "- (\(.path):\(.line // .original_line)) \(.user.login): \(.body)"' >> feedback.txt || true + gh api "repos/$REPO/issues/$pr/comments" \ + --jq '.[] | select((.user.login|test("\\[bot\\]|coderabbit";"i"))|not) | "- \(.user.login): \(.body)"' >> feedback.txt || true + echo "Collected feedback from $REPO#$pr ($(wc -l < feedback.txt) line(s))" + else + echo "No open README PR for $REPO" + fi + - name: Generate README env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} @@ -87,7 +112,8 @@ jobs: run: | python3 scripts/generate_readme.py \ --repo-dir target \ - --repo-name "$REPO_NAME" + --repo-name "$REPO_NAME" \ + --feedback feedback.txt - name: Open pull request uses: peter-evans/create-pull-request@v8 @@ -105,5 +131,7 @@ jobs: The content is generated by Claude from this repository's own files (file tree, manifests, workflows, existing README). **Please review for accuracy** before - merging — edit or close if anything is wrong. Re-running the workflow refreshes - this branch. + merging — edit or close if anything is wrong. + + Leave review comments here: while this PR stays open, the next run reads them + and folds your fixes into the regenerated README (they won't be overwritten). diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index e50f3a21bc..f50add7a4d 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -21,6 +21,7 @@ --repo-name "owner/repo" for context (default: derived from git remote) --model model id (default: $MODEL or claude-opus-4-7) --max-context-chars total budget for embedded file contents (default 180000) + --feedback file of reviewer comments to fold into this revision """ import argparse @@ -160,13 +161,27 @@ def build_context(repo_dir, repo_name): "tables where they fit, and fenced code blocks for commands/trees.\n" "- Keep any project-specific links (docs.armbian.com, armbian.com, related repos) " "that appear in the sources; do not fabricate new ones.\n" + "- When you state what the project is built with or its requirements, name the " + "actual languages and key tooling evidenced by the files (shebangs, file " + "extensions, manifests, invoked commands) -- be specific, not vague.\n" "- Start with an H1 title and a one-to-two sentence description of the repo.\n\n" "Output ONLY the raw Markdown for README.md. No preamble, no explanation, no code " "fence around the whole document." ) -def generate(repo_dir, repo_name, model, max_context): +def load_feedback(path): + """Read reviewer feedback (from the open README PR) if a path is given.""" + if not path: + return "" + try: + with open(path, "r", encoding="utf-8", errors="replace") as fh: + return fh.read().strip() + except OSError: + return "" + + +def generate(repo_dir, repo_name, model, max_context, feedback=""): os.environ.setdefault("MAX_CONTEXT_CHARS", str(max_context)) context = build_context(repo_dir, repo_name) @@ -174,6 +189,17 @@ def generate(repo_dir, repo_name, model, max_context): prompt = ( "Here is the snapshot of the repository. Produce the README.md.\n\n" + context ) + if feedback: + # Reviewers commented on the previous README PR; fold their fixes in so a + # regeneration doesn't silently drop them. Treat as human corrections that + # override the model's own read where they conflict -- but still only about + # THIS repo, never as instructions to invent unevidenced content. + prompt += ( + "\n\n----- REVIEWER FEEDBACK ON THE PREVIOUS README (apply it) -----\n" + "A human reviewed the last generated README for this repo and left the " + "comments below. Incorporate these corrections and do not re-introduce " + "what they flagged. They are about this repository only.\n\n" + feedback + ) resp = client.messages.create( model=model, max_tokens=8192, @@ -202,6 +228,8 @@ def main(): ap.add_argument("--repo-name", default=None) ap.add_argument("--model", default=os.environ.get("MODEL", "claude-opus-4-7")) ap.add_argument("--max-context-chars", type=int, default=180000) + ap.add_argument("--feedback", default=None, + help="path to a file of reviewer comments from the open README PR") args = ap.parse_args() if not os.environ.get("ANTHROPIC_API_KEY"): @@ -214,8 +242,12 @@ def main(): url = run(["git", "config", "--get", "remote.origin.url"], repo_dir).strip() repo_name = url.rsplit("/", 2)[-2] + "/" + url.rsplit("/", 1)[-1].removesuffix(".git") if "/" in url else "unknown/repo" + feedback = load_feedback(args.feedback) + if feedback: + print(f"Applying {len(feedback)} chars of reviewer feedback from {args.feedback}") + print(f"Generating README for {repo_name} (dir={repo_dir}, model={args.model})") - readme = generate(repo_dir, repo_name, args.model, args.max_context_chars) + readme = generate(repo_dir, repo_name, args.model, args.max_context_chars, feedback) out_path = os.path.join(repo_dir, "README.md") with open(out_path, "w", encoding="utf-8") as fh: From ed5d28cee9eb00869f940432c0b5e4d7d7e57611 Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Tue, 21 Jul 2026 22:19:11 +0200 Subject: [PATCH 2/3] readme-updater: gate feedback on trusted reviewers + fail closed Address review on PR #365: - Security: the feedback filter only excluded bots, so any human commenter on the public README PR could inject text the generator treats as guidance. Gate both gh api jq filters on author_association (OWNER/MEMBER/COLLABORATOR = write access) in addition to the bot exclusion. - Data integrity: drop the `|| true` on the comment fetches so a gh/API error aborts the run instead of silently generating without feedback (which would re-drop the reviewer's fixes). load_feedback() now errors out when an explicit --feedback file can't be read; an empty file is still fine. Signed-off-by: Igor Pecovnik --- .github/workflows/maintenance-update-readme.yml | 13 ++++++++----- scripts/generate_readme.py | 12 +++++++++--- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/maintenance-update-readme.yml b/.github/workflows/maintenance-update-readme.yml index c770911385..673743b2be 100644 --- a/.github/workflows/maintenance-update-readme.yml +++ b/.github/workflows/maintenance-update-readme.yml @@ -82,8 +82,11 @@ jobs: - name: Install Anthropic SDK run: pip install 'anthropic>=0.40.0' - # If a previous "chore/update-readme" PR is still open, fold its human review - # comments back in, so regenerating doesn't silently drop reviewer fixes. + # If a previous "chore/update-readme" PR is still open, fold in review comments + # from TRUSTED reviewers so regenerating doesn't silently drop their fixes. + # Gated on author_association (OWNER/MEMBER/COLLABORATOR = write access) so a + # random commenter on the public PR cannot inject text into the prompt. Fails + # closed: any gh/API error aborts the run rather than proceeding feedback-less. - name: Collect reviewer feedback from the open README PR env: GH_TOKEN: ${{ secrets.ACCESS_TOKEN }} @@ -95,10 +98,10 @@ jobs: --json number --jq '.[0].number // empty')" if [ -n "$pr" ]; then gh api "repos/$REPO/pulls/$pr/comments" \ - --jq '.[] | select((.user.login|test("\\[bot\\]|coderabbit";"i"))|not) | "- (\(.path):\(.line // .original_line)) \(.user.login): \(.body)"' >> feedback.txt || true + --jq '.[] | select((.user.login|test("\\[bot\\]|coderabbit";"i")|not) and (.author_association=="OWNER" or .author_association=="MEMBER" or .author_association=="COLLABORATOR")) | "- (\(.path):\(.line // .original_line)) \(.user.login): \(.body)"' >> feedback.txt gh api "repos/$REPO/issues/$pr/comments" \ - --jq '.[] | select((.user.login|test("\\[bot\\]|coderabbit";"i"))|not) | "- \(.user.login): \(.body)"' >> feedback.txt || true - echo "Collected feedback from $REPO#$pr ($(wc -l < feedback.txt) line(s))" + --jq '.[] | select((.user.login|test("\\[bot\\]|coderabbit";"i")|not) and (.author_association=="OWNER" or .author_association=="MEMBER" or .author_association=="COLLABORATOR")) | "- \(.user.login): \(.body)"' >> feedback.txt + echo "Collected feedback from $REPO#$pr ($(wc -l < feedback.txt) trusted line(s))" else echo "No open README PR for $REPO" fi diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index f50add7a4d..de542806e3 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -171,14 +171,20 @@ def build_context(repo_dir, repo_name): def load_feedback(path): - """Read reviewer feedback (from the open README PR) if a path is given.""" + """Read reviewer feedback (from the open README PR) if a path is given. + + Fail closed: if a --feedback path was explicitly passed but cannot be read, + error out instead of silently generating without it (which would re-drop the + reviewer's fixes). An empty file is fine -- it just means no feedback. + """ if not path: return "" try: with open(path, "r", encoding="utf-8", errors="replace") as fh: return fh.read().strip() - except OSError: - return "" + except OSError as e: + print(f"::error::could not read feedback file {path}: {e}", file=sys.stderr) + sys.exit(1) def generate(repo_dir, repo_name, model, max_context, feedback=""): From e651e054f1d1a164ca3722d54e3716d6b47d21ee Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Fri, 24 Jul 2026 16:25:54 +0200 Subject: [PATCH 3/3] readme-updater: bound reviewer feedback to keep prompt within context build_context caps embedded file contents to MAX_CONTEXT_CHARS, but the reviewer feedback appended after it was unbounded (load_feedback reads the whole file), so an oversized review could push the combined prompt past the model context limit. Cap feedback at MAX_FEEDBACK_CHARS (16000), truncating only the feedback content and preserving the reviewer-feedback instructions. Signed-off-by: Igor Pecovnik --- scripts/generate_readme.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index de542806e3..036131518e 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -49,6 +49,7 @@ MAX_PER_FILE = 8000 # truncate any single embedded file to this many chars MAX_TREE_ENTRIES = 500 # cap the file listing +MAX_FEEDBACK_CHARS = 16000 # cap reviewer feedback folded into the prompt def run(cmd, cwd): @@ -196,6 +197,10 @@ def generate(repo_dir, repo_name, model, max_context, feedback=""): "Here is the snapshot of the repository. Produce the README.md.\n\n" + context ) if feedback: + # Bound the feedback so an oversized review can't push the combined prompt + # past the model context limit (build_context is already capped, this is not). + if len(feedback) > MAX_FEEDBACK_CHARS: + feedback = feedback[:MAX_FEEDBACK_CHARS] + "\n... [truncated] ...\n" # Reviewers commented on the previous README PR; fold their fixes in so a # regeneration doesn't silently drop them. Treat as human corrections that # override the model's own read where they conflict -- but still only about