diff --git a/build_docs.py b/build_docs.py index e44ffa0..1110335 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1251,7 +1251,8 @@ def build_docs(args: argparse.Namespace) -> int: build_sitemap(versions, languages, args.www_root, args.group) build_404(args.www_root, args.group) - copy_robots_txt( + build_robots_txt( + versions, args.www_root, args.group, args.skip_cache_invalidation, @@ -1338,20 +1339,23 @@ def build_404(www_root: Path, group: str) -> None: chgrp(not_found_file, group=group) -def copy_robots_txt( +def build_robots_txt( + versions: Versions, www_root: Path, group: str, skip_cache_invalidation: bool, http: urllib3.PoolManager, ) -> None: - """Copy robots.txt to www_root.""" + """Build robots.txt to www_root.""" if not www_root.exists(): - logging.info("Skipping copying robots.txt (www root does not even exist).") + logging.info("Skipping robots.txt generation (www root does not even exist).") return - logging.info("Copying robots.txt...") + logging.info("Starting robots.txt generation...") template_path = HERE / "templates" / "robots.txt" + template = jinja2.Template(template_path.read_text(encoding="UTF-8")) + rendered_template = template.render(versions=versions) robots_path = www_root / "robots.txt" - shutil.copyfile(template_path, robots_path) + robots_path.write_text(rendered_template, encoding="UTF-8") robots_path.chmod(0o775) chgrp(robots_path, group=group) if not skip_cache_invalidation: diff --git a/templates/robots.txt b/templates/robots.txt index a7de4a6..deb2211 100644 --- a/templates/robots.txt +++ b/templates/robots.txt @@ -13,14 +13,8 @@ Disallow: /2.2/ Disallow: /2.3/ Disallow: /2.4/ Disallow: /2.5/ -Disallow: /2.6/ -Disallow: /2.7/ -Disallow: /3.0/ -Disallow: /3.1/ -Disallow: /3.2/ -Disallow: /3.3/ -Disallow: /3.4/ -Disallow: /3.5/ -Disallow: /3.6/ -Disallow: /3.7/ -Disallow: /3.8/ +{% for version in versions -%} +{% if version.status == "EOL" -%} +Disallow: /{{ version.name }}/ +{% endif -%} +{% endfor %} diff --git a/tests/test_build_docs.py b/tests/test_build_docs.py index 028da90..8385ceb 100644 --- a/tests/test_build_docs.py +++ b/tests/test_build_docs.py @@ -1,6 +1,8 @@ +from unittest.mock import patch + import pytest -from build_docs import format_seconds +from build_docs import Version, Versions, build_robots_txt, format_seconds @pytest.mark.parametrize( @@ -24,3 +26,23 @@ ) def test_format_seconds(seconds: float, expected: str) -> None: assert format_seconds(seconds) == expected + + +@patch("build_docs.chgrp") +def test_build_robots_txt(mock_chgrp, tmp_path) -> None: + versions = Versions([ + Version(name="3.14", status="EOL", branch_or_tag="3.14"), + Version(name="3.15", status="EOL", branch_or_tag="3.15"), + Version(name="3.16", status="security-fixes", branch_or_tag="3.16"), + Version(name="3.17", status="stable", branch_or_tag="2.17"), + ]) + + build_robots_txt( + versions, tmp_path, group="", skip_cache_invalidation=True, http=None + ) + + result = (tmp_path / "robots.txt").read_text() + assert "Disallow: /3.14/" in result + assert "Disallow: /3.15/" in result + assert "/3.16/" not in result + assert "/3.17/" not in result