diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index 56bbe705..e1453239 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -16,7 +16,7 @@ jobs: name: "Build & Test" runs-on: ubuntu-latest container: - image: registry.fedoraproject.org/fedora:41 + image: registry.fedoraproject.org/fedora:43 options: --privileged env: # workaround for expired cert at source of indirect dependency @@ -24,6 +24,12 @@ jobs: GOPROXY: "https://proxy.golang.org,direct" steps: - uses: actions/checkout@v6 + - name: Install Golang + run: dnf install -y golang + - name: Vendor Go dependencies + run: go mod vendor + - name: Setup osbuild repository + run: python3 test/scripts/setup-osbuild-repo - name: Install dependencies run: | # we have the same build-deps as the images library @@ -31,6 +37,8 @@ jobs: curl -sL https://raw.githubusercontent.com/osbuild/images/refs/heads/main/test/scripts/install-dependencies | bash # we also need createrepo_c dnf install -y createrepo_c + - name: Check osbuild version + run: python3 test/scripts/check-osbuild-version - name: Build run: go build -v ./... - name: Test diff --git a/Schutzfile b/Schutzfile new file mode 100644 index 00000000..2646a482 --- /dev/null +++ b/Schutzfile @@ -0,0 +1,9 @@ +{ + "common": { + "dependencies": { + "osbuild": { + "commit": "6c34e00bedd8101293382f2d563d111825b46349" + } + } + } +} diff --git a/go.mod b/go.mod index 9e540b3b..832f84a5 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gobwas/glob v0.2.3 github.com/mattn/go-isatty v0.0.20 github.com/osbuild/blueprint v1.22.0 - github.com/osbuild/images v0.234.0 + github.com/osbuild/images v0.236.0 github.com/sirupsen/logrus v1.9.4 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 diff --git a/go.sum b/go.sum index 73d7b0ba..a10fcfb3 100644 --- a/go.sum +++ b/go.sum @@ -289,8 +289,8 @@ github.com/opencontainers/selinux v1.12.0 h1:6n5JV4Cf+4y0KNXW48TLj5DwfXpvWlxXplU github.com/opencontainers/selinux v1.12.0/go.mod h1:BTPX+bjVbWGXw7ZZWUbdENt8w0htPSrlgOOysQaU62U= github.com/osbuild/blueprint v1.22.0 h1:b3WicGjCFzEwOm/YwPH7w9YioCcehGejdOTkjJ3Fyz0= github.com/osbuild/blueprint v1.22.0/go.mod h1:HPlJzkEl7q5g8hzaGksUk7ifFAy9QFw9LmzhuFOAVm4= -github.com/osbuild/images v0.234.0 h1:8RrUzOxR2/rYk7ErWxiEJ5mTWZ0yEbjRXsbvT8hnPf0= -github.com/osbuild/images v0.234.0/go.mod h1:vjzHaL/8MDG6c3yjU8qgMKOIib89A1r2ql50Nronaw4= +github.com/osbuild/images v0.236.0 h1:roC5fEFCs2mFYFNDvnK8mRiN4XGJ28erx7kLpqI3dv8= +github.com/osbuild/images v0.236.0/go.mod h1:vjzHaL/8MDG6c3yjU8qgMKOIib89A1r2ql50Nronaw4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/image-builder.spec b/image-builder.spec index 8c70bc1c..fd80b951 100644 --- a/image-builder.spec +++ b/image-builder.spec @@ -3,7 +3,7 @@ # required. So if this needs backport to places where there is no # recent osbuild available we could simply make --use-librepo false # and go back to 129. -%global min_osbuild_version 167 +%global min_osbuild_version 170 %global goipath github.com/osbuild/image-builder-cli diff --git a/test/scripts/check-osbuild-version b/test/scripts/check-osbuild-version new file mode 100755 index 00000000..6a52f522 --- /dev/null +++ b/test/scripts/check-osbuild-version @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# pylint: disable=invalid-name +""" +Check that installed osbuild packages meet the minimum version requirement. + +This script reads the minimum osbuild version from the vendored osbuild/images +library and verifies that the installed osbuild and osbuild-depsolve-dnf packages +meet this requirement. + +Prerequisites: +- go mod vendor must have been run to populate the vendor directory +- osbuild and osbuild-depsolve-dnf packages must be installed +""" + +import os +import re +import subprocess +import sys + +# Path to the minimum version file in the vendored osbuild/images +MIN_VERSION_FILE = "vendor/github.com/osbuild/images/data/dependencies/osbuild" + +# Packages to check +PACKAGES = ["osbuild", "osbuild-depsolve-dnf"] + + +def read_min_version(): + """Read the minimum osbuild version from the vendored images library.""" + if not os.path.exists(MIN_VERSION_FILE): + print(f"ERROR: {MIN_VERSION_FILE} not found", file=sys.stderr) + print("Make sure to run 'go mod vendor' first.", file=sys.stderr) + sys.exit(1) + + with open(MIN_VERSION_FILE, encoding="utf-8") as f: + version = f.read().strip() + + if not version: + print(f"ERROR: {MIN_VERSION_FILE} is empty", file=sys.stderr) + sys.exit(1) + + return version + + +def get_installed_version(package): + """ + Get the installed version of an RPM package. + + Returns None if the package is not installed. + """ + try: + result = subprocess.run( + ["rpm", "-q", "--queryformat", "%{VERSION}", package], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + except subprocess.CalledProcessError: + return None + + +def parse_version(version_str): + """ + Parse a version string into a tuple of integers for comparison. + + Handles versions like "129", "129.1", "129.1.2". + """ + # Extract only the numeric parts (handle cases like "129~rc1") + match = re.match(r"^(\d+(?:\.\d+)*)", version_str) + if not match: + return (0,) + + parts = match.group(1).split(".") + return tuple(int(p) for p in parts) + + +def compare_versions(installed, minimum): + """ + Compare two version strings. + + Returns: + -1 if installed < minimum + 0 if installed == minimum + 1 if installed > minimum + """ + installed_tuple = parse_version(installed) + minimum_tuple = parse_version(minimum) + + # Pad shorter tuple with zeros for comparison + max_len = max(len(installed_tuple), len(minimum_tuple)) + installed_padded = installed_tuple + (0,) * (max_len - len(installed_tuple)) + minimum_padded = minimum_tuple + (0,) * (max_len - len(minimum_tuple)) + + if installed_padded < minimum_padded: + return -1 + if installed_padded > minimum_padded: + return 1 + return 0 + + +def main(): + # Read minimum version + min_version = read_min_version() + print(f"Minimum required osbuild version: {min_version}") + + errors = [] + + for package in PACKAGES: + installed_version = get_installed_version(package) + + if installed_version is None: + errors.append(f" {package}: NOT INSTALLED") + continue + + cmp_result = compare_versions(installed_version, min_version) + + if cmp_result < 0: + print(f" {package}: {installed_version} < {min_version} (TOO OLD)") + errors.append(f" {package}: {installed_version} < {min_version}") + else: + status = "==" if cmp_result == 0 else ">=" + print(f" {package}: {installed_version} {status} {min_version} (OK)") + + if errors: + print("\n" + "=" * 60, file=sys.stderr) + print("ERROR: Installed osbuild packages do not meet minimum requirements", file=sys.stderr) + print("=" * 60, file=sys.stderr) + print(f"\nThe vendored osbuild/images library requires osbuild >= {min_version}", file=sys.stderr) + print("\nProblems found:", file=sys.stderr) + for error in errors: + print(error, file=sys.stderr) + print("\nTo fix this, pin a specific osbuild commit in Schutzfile", file=sys.stderr) + return 1 + + print("\nAll osbuild packages meet minimum version requirements.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/scripts/setup-osbuild-repo b/test/scripts/setup-osbuild-repo new file mode 100755 index 00000000..48ec395f --- /dev/null +++ b/test/scripts/setup-osbuild-repo @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# pylint: disable=invalid-name +""" +Setup osbuild dnf repository from S3 bucket when a commit is pinned in Schutzfile. + +This script reads the Schutzfile and sets up a dnf repository pointing to pre-built +osbuild RPMs from the osbuild CI S3 bucket. If no commit is pinned, the script +exits successfully without making any changes (no-op). + +The Schutzfile supports both common and distro-specific configurations: +- Distro-specific: Schutzfile[distro]["dependencies"]["osbuild"]["commit"] +- Common fallback: Schutzfile["common"]["dependencies"]["osbuild"]["commit"] +""" + +import json +import os +import sys +import urllib.error +import urllib.request + +SCHUTZFILE = "Schutzfile" +OS_RELEASE_FILE = "/etc/os-release" +REPO_FILE = "/etc/yum.repos.d/osbuild.repo" + +URL_TEMPLATE = "http://osbuild-composer-repos.s3-website.us-east-2.amazonaws.com/osbuild/{distro}/{arch}/{commit}" + +REPO_TEMPLATE = """[osbuild] +name=osbuild {commit} +baseurl={baseurl} +enabled=1 +gpgcheck=0 +priority=10 +""" + + +def read_os_release(): + """ + Read /etc/os-release and return as a dictionary. + """ + osrelease = {} + with open(OS_RELEASE_FILE, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + key, value = line.split("=", 1) + osrelease[key] = value.strip('"') + return osrelease + + +def get_host_distro(): + """ + Get the host distro identifier (e.g., 'fedora-41'). + """ + osrelease = read_os_release() + return f"{osrelease['ID']}-{osrelease['VERSION_ID']}" + + +def get_osbuild_commit(schutzfile_data, distro): + """ + Get the osbuild commit from Schutzfile. + + Lookup order: + 1. Distro-specific: schutzfile_data[distro]["dependencies"]["osbuild"]["commit"] + 2. Common fallback: schutzfile_data["common"]["dependencies"]["osbuild"]["commit"] + + Returns None if no commit is specified or the commit is an empty string. + """ + # Try distro-specific first + commit = ( + schutzfile_data.get(distro, {}) + .get("dependencies", {}) + .get("osbuild", {}) + .get("commit") + ) + if commit: + return commit + + # Fall back to common + commit = ( + schutzfile_data.get("common", {}) + .get("dependencies", {}) + .get("osbuild", {}) + .get("commit") + ) + if commit: + return commit + + return None + + +def check_repo_url(url): + """ + Check if the repo URL is accessible. + """ + repomd_url = f"{url}/repodata/repomd.xml" + print(f"Checking URL: {repomd_url}") + try: + with urllib.request.urlopen(repomd_url, timeout=30) as resp: + print(f" {resp.status} ({resp.msg})") + return True + except urllib.error.HTTPError as e: + print(f" HTTP Error: {e.code} {e.reason}") + return False + except urllib.error.URLError as e: + print(f" URL Error: {e.reason}") + return False + + +def write_repo_file(commit, baseurl): + """ + Write the dnf repository configuration file. + """ + print(f"Writing repository configuration to {REPO_FILE}") + with open(REPO_FILE, "w", encoding="utf-8") as f: + f.write(REPO_TEMPLATE.format(commit=commit, baseurl=baseurl)) + print(" Done") + + +def main(): + if not os.path.exists(SCHUTZFILE): + print(f"No {SCHUTZFILE} found, skipping osbuild repo setup") + return + + with open(SCHUTZFILE, encoding="utf-8") as f: + schutzfile_data = json.load(f) + + distro = get_host_distro() + print(f"Host distro: {distro}") + + commit = get_osbuild_commit(schutzfile_data, distro) + if not commit: + print("No osbuild commit pinned in Schutzfile, skipping repo setup") + return + + print(f"Pinned osbuild commit: {commit}") + + arch = os.uname().machine + baseurl = URL_TEMPLATE.format(distro=distro, arch=arch, commit=commit) + print(f"Repository URL: {baseurl}") + + if not check_repo_url(baseurl): + print( + f"\nERROR: Failed to verify osbuild repo at {baseurl}/repodata/repomd.xml", + file=sys.stderr, + ) + print( + f"The commit {commit} may not have been built in osbuild CI.", + file=sys.stderr, + ) + sys.exit(1) + + write_repo_file(commit, baseurl) + + print(f"\nSuccessfully configured osbuild repo for commit {commit}") + return + + +if __name__ == "__main__": + main() diff --git a/test/test_build.py b/test/test_build.py index ce874a0d..306542cd 100644 --- a/test/test_build.py +++ b/test/test_build.py @@ -1,4 +1,5 @@ import os +import pathlib import platform import subprocess @@ -7,57 +8,74 @@ # put common podman run args in once place podman_run = ["podman", "run", "--rm", "--privileged"] +# Shared osbuild store to cache RPMs and intermediate artifacts between builds. +# This significantly reduces disk space usage and build time. +OSBUILD_TEST_STORE = "/var/tmp/osbuild-test-store" +# The container mount point matches the default --cache flag value in cmd/image-builder/main.go +OSBUILD_STORE_CONTAINER_PATH = "/var/cache/image-builder/store" + + +@pytest.fixture(name="shared_store", scope="session") +def shared_store_fixture(): + """Create and return a shared osbuild store directory.""" + pathlib.Path(OSBUILD_TEST_STORE).mkdir(exist_ok=True, parents=True) + return OSBUILD_TEST_STORE + @pytest.mark.parametrize("use_librepo", [False, True]) @pytest.mark.skipif(os.getuid() != 0, reason="needs root") -def test_build_builds_image(tmp_path, build_container, use_librepo): +def test_build_builds_image(tmp_path, build_container, shared_store, use_librepo): output_dir = tmp_path / "output" output_dir.mkdir() subprocess.check_call(podman_run + [ "-v", f"{output_dir}:/output", + "-v", f"{shared_store}:{OSBUILD_STORE_CONTAINER_PATH}", build_container, "build", - "minimal-raw", + "qcow2", "--distro", "centos-9", f"--use-librepo={use_librepo}", ]) arch = "x86_64" - basename = f"centos-9-minimal-raw-{arch}" - assert (output_dir / basename / f"{basename}.raw.xz").exists() + basename = f"centos-9-qcow2-{arch}" + assert (output_dir / basename / f"{basename}.qcow2").exists() # XXX: ensure no other leftover dirs dents = os.listdir(output_dir) assert len(dents) == 1, f"too many dentries in output dir: {dents}" @pytest.mark.skipif(os.getuid() != 0, reason="needs root") -def test_build_build_generates_manifest(tmp_path, build_container): +def test_build_build_generates_manifest(tmp_path, build_container, shared_store): output_dir = tmp_path / "output" output_dir.mkdir() subprocess.check_call(podman_run + [ "-v", f"{output_dir}:/output", + "-v", f"{shared_store}:{OSBUILD_STORE_CONTAINER_PATH}", build_container, "build", - "minimal-raw", + "qcow2", "--distro", "centos-9", "--with-manifest", ], stdout=subprocess.DEVNULL) arch = platform.machine() - fn = f"centos-9-minimal-raw-{arch}/centos-9-minimal-raw-{arch}.osbuild-manifest.json" + fn = f"centos-9-qcow2-{arch}/centos-9-qcow2-{arch}.osbuild-manifest.json" image_manifest_path = output_dir / fn assert image_manifest_path.exists() +# pylint: disable=too-many-arguments @pytest.mark.parametrize("progress,needle,forbidden", [ ("verbose", "osbuild-stdout-output", "[|]"), ("term", "[|]", "osbuild-stdout-output"), ]) @pytest.mark.skipif(os.getuid() != 0, reason="needs root") -def test_build_with_progress(tmp_path, build_fake_container, progress, needle, forbidden): +def test_build_with_progress(tmp_path, build_fake_container, shared_store, progress, needle, forbidden): output_dir = tmp_path / "output" output_dir.mkdir() output = subprocess.check_output(podman_run + [ "-t", "-v", f"{output_dir}:/output", + "-v", f"{shared_store}:{OSBUILD_STORE_CONTAINER_PATH}", build_fake_container, "build", "qcow2", @@ -69,7 +87,7 @@ def test_build_with_progress(tmp_path, build_fake_container, progress, needle, f assert forbidden not in output -def test_build_builds_bootc(tmp_path, build_container): +def test_build_builds_bootc(tmp_path, build_container, shared_store): bootc_ref = "quay.io/centos-bootc/centos-bootc:stream9" subprocess.check_call(["podman", "pull", bootc_ref]) @@ -77,6 +95,7 @@ def test_build_builds_bootc(tmp_path, build_container): output_dir.mkdir() subprocess.check_call(podman_run + [ "-v", f"{output_dir}:/output", + "-v", f"{shared_store}:{OSBUILD_STORE_CONTAINER_PATH}", build_container, "build", "qcow2",