Expose metadata file

Author: jobselko

pulp_python/app/serializers.py

@@ -22,6 +22,7 @@
 )
 from pulp_python.app.utils import (
     DIST_EXTENSIONS,
+    artifact_to_metadata_artifact,
     artifact_to_python_content_data,
     get_project_metadata_from_file,
     parse_project_metadata,
@@ -93,11 +94,35 @@ class Meta:
         model = python_models.PythonDistribution
 
 
+class PythonSingleContentArtifactField(core_serializers.SingleContentArtifactField):
+    """
+    Custom field with overridden get_attribute method. Meant to be used only in
+    PythonPackageContentSerializer to handle possible existence of metadata artifact.
+    """
+
+    def get_attribute(self, instance):
+        if instance._artifacts.count() == 0:
+            return None
+        elif instance._artifacts.count() == 1:
+            return instance._artifacts.all()[0]
+        else:
+            main_content_artifacts = instance.contentartifact_set.exclude(
+                relative_path__endswith=".metadata"
+            )
+            if main_content_artifacts.exists():
+                return main_content_artifacts.first().artifact
+            return instance._artifacts.all()[0]
+
+
 class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploadSerializer):
     """
     A Serializer for PythonPackageContent.
     """
 
+    artifact = PythonSingleContentArtifactField(
+        help_text=_("Artifact file representing the physical content"),
+    )
+
     # Core metadata
     # Version 1.0
     author = serializers.CharField(
@@ -386,8 +411,21 @@ def deferred_validate(self, data):
         if attestations := data.pop("attestations", None):
             data["provenance"] = self.handle_attestations(filename, data["sha256"], attestations)
 
+        # Create metadata artifact for wheel files
+        if filename.endswith(".whl"):
+            if metadata_artifact := artifact_to_metadata_artifact(filename, artifact):
+                data["metadata_artifact"] = metadata_artifact
+                data["metadata_sha256"] = metadata_artifact.sha256
+
         return data
 
+    def get_artifacts(self, validated_data):
+        artifacts = super().get_artifacts(validated_data)
+        if metadata_artifact := validated_data.pop("metadata_artifact", None):
+            relative_path = f"{validated_data['filename']}.metadata"
+            artifacts[relative_path] = metadata_artifact
+        return artifacts
+
     def retrieve(self, validated_data):
         content = python_models.PythonPackageContent.objects.filter(
             sha256=validated_data["sha256"], _pulp_domain=get_domain()

pulp_python/app/tasks/sync.py

@@ -229,11 +229,15 @@ async def create_content(self, pkg):
         create a Content Unit to put into the pipeline
         """
         declared_contents = {}
+        page = await aget_remote_simple_page(pkg.name, self.remote)
+        upstream_pkgs = {pkg.filename: pkg for pkg in page.packages}
+
         for version, dists in pkg.releases.items():
             for package in dists:
                 entry = parse_metadata(pkg.info, version, package)
                 url = entry.pop("url")
                 size = package["size"] or None
+                d_artifacts = []
 
                 artifact = Artifact(sha256=entry["sha256"], size=size)
                 package = PythonPackageContent(**entry)
@@ -245,11 +249,28 @@ async def create_content(self, pkg):
                     remote=self.remote,
                     deferred_download=self.deferred_download,
                 )
-                dc = DeclarativeContent(content=package, d_artifacts=[da])
+                d_artifacts.append(da)
+
+                if upstream_pkg := upstream_pkgs.get(entry["filename"]):
+                    if upstream_pkg.has_metadata:
+                        url = upstream_pkg.metadata_url
+                        md_sha256 = upstream_pkg.metadata_digests.get("sha256")
+                        artifact = Artifact(sha256=md_sha256)
+
+                        metadata_artifact = DeclarativeArtifact(
+                            artifact=artifact,
+                            url=url,
+                            relative_path=f"{entry['filename']}.metadata",
+                            remote=self.remote,
+                            deferred_download=self.deferred_download,
+                        )
+                        d_artifacts.append(metadata_artifact)
+
+                dc = DeclarativeContent(content=package, d_artifacts=d_artifacts)
                 declared_contents[entry["filename"]] = dc
                 await self.python_stage.put(dc)
 
-        if pkg.releases and (page := await aget_remote_simple_page(pkg.name, self.remote)):
+        if pkg.releases and page:
             if self.remote.provenance:
                 await self.sync_provenance(page, declared_contents)
 

pulp_python/app/tasks/upload.py

@@ -15,7 +15,7 @@
     Provenance,
     verify_provenance,
 )
-from pulp_python.app.utils import artifact_to_python_content_data
+from pulp_python.app.utils import artifact_to_metadata_artifact, artifact_to_python_content_data
 
 
 def upload(artifact_sha256, filename, attestations=None, repository_pk=None):
@@ -97,6 +97,11 @@ def create_content(artifact_sha256, filename, domain):
     def create():
         content = PythonPackageContent.objects.create(**data)
         ContentArtifact.objects.create(artifact=artifact, content=content, relative_path=filename)
+
+        if metadata_artifact := artifact_to_metadata_artifact(filename, artifact):
+            ContentArtifact.objects.create(
+                artifact=metadata_artifact, content=content, relative_path=f"{filename}.metadata"
+            )
         return content
 
     new_content = create()

pulp_python/app/utils.py

@@ -1,4 +1,5 @@
 import hashlib
+import logging
 import pkginfo
 import re
 import shutil
@@ -14,10 +15,13 @@
 from packaging.requirements import Requirement
 from packaging.version import parse, InvalidVersion
 from pypi_simple import ACCEPT_JSON_PREFERRED, ProjectPage
-from pulpcore.plugin.models import Remote
+from pulpcore.plugin.models import Artifact, Remote
 from pulpcore.plugin.exceptions import TimeoutException
 
 
+log = logging.getLogger(__name__)
+
+
 PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL"
 """TODO This serial constant is temporary until Python repositories implements serials"""
 PYPI_SERIAL_CONSTANT = 1000000000
@@ -206,25 +210,34 @@ def get_project_metadata_from_file(filename):
     return metadata
 
 
-def compute_metadata_sha256(filename: str) -> str | None:
+def extract_wheel_metadata(filename: str) -> bytes | None:
     """
-    Compute SHA256 hash of the metadata file from a Python package.
+    Extract the metadata file content from a wheel file.
 
-    Returns SHA256 hash or None if metadata cannot be extracted.
+    Returns the raw metadata content as bytes or None if metadata cannot be extracted.
     """
     if not filename.endswith(".whl"):
         return None
     try:
         with zipfile.ZipFile(filename, "r") as f:
             for file_path in f.namelist():
                 if file_path.endswith(".dist-info/METADATA"):
-                    metadata_content = f.read(file_path)
-                    return hashlib.sha256(metadata_content).hexdigest()
-    except (zipfile.BadZipFile, KeyError, OSError):
-        pass
+                    return f.read(file_path)
+    except (zipfile.BadZipFile, KeyError, OSError) as e:
+        log.warning(f"Failed to extract metadata file from {filename}: {e}")
     return None
 
 
+def compute_metadata_sha256(filename: str) -> str | None:
+    """
+    Compute SHA256 hash of the metadata file from a Python package.
+
+    Returns SHA256 hash or None if metadata cannot be extracted.
+    """
+    metadata_content = extract_wheel_metadata(filename)
+    return hashlib.sha256(metadata_content).hexdigest() if metadata_content else None
+
+
 def artifact_to_python_content_data(filename, artifact, domain=None):
     """
     Takes the artifact/filename and returns the metadata needed to create a PythonPackageContent.
@@ -233,6 +246,7 @@ def artifact_to_python_content_data(filename, artifact, domain=None):
     # because pkginfo validates that the filename has a valid extension before
     # reading it
     with tempfile.NamedTemporaryFile("wb", dir=".", suffix=filename) as temp_file:
+        artifact.file.seek(0)
         shutil.copyfileobj(artifact.file, temp_file)
         temp_file.flush()
         metadata = get_project_metadata_from_file(temp_file.name)
@@ -245,6 +259,28 @@ def artifact_to_python_content_data(filename, artifact, domain=None):
     return data
 
 
+def artifact_to_metadata_artifact(filename: str, artifact: Artifact) -> Artifact | None:
+    """
+    Creates artifact for metadata from the provided wheel artifact.
+    """
+    if not filename.endswith(".whl"):
+        return None
+
+    with tempfile.NamedTemporaryFile("wb", dir=".", suffix=filename) as temp_file:
+        artifact.file.seek(0)
+        shutil.copyfileobj(artifact.file, temp_file)
+        temp_file.flush()
+        metadata_content = extract_wheel_metadata(temp_file.name)
+        if not metadata_content:
+            return None
+        with tempfile.NamedTemporaryFile(suffix=".metadata") as metadata_temp:
+            metadata_temp.write(metadata_content)
+            metadata_temp.flush()
+            metadata_artifact = Artifact.init_and_validate(metadata_temp.name)
+            metadata_artifact.save()
+            return metadata_artifact
+
+
 def fetch_json_release_metadata(name: str, version: str, remotes: set[Remote]) -> dict:
     """
     Fetches metadata for a specific release from PyPI's JSON API. A release can contain
@@ -408,6 +444,7 @@ def find_artifact():
             _art = models.RemoteArtifact.objects.filter(content_artifact=content_artifact).first()
         return _art
 
+    # todo: fix .first()
     content_artifact = content.contentartifact_set.first()
     artifact = find_artifact()
     origin = settings.CONTENT_ORIGIN or settings.PYPI_API_HOSTNAME or ""

pulp_python/tests/functional/api/test_pypi_apis.py

@@ -10,9 +10,13 @@
     PYTHON_MD_PYPI_SUMMARY,
     PYTHON_EGG_FILENAME,
     PYTHON_EGG_SHA256,
+    PYTHON_WHEEL_FILENAME,
     PYTHON_WHEEL_SHA256,
+    PYTHON_WHEEL_URL,
+    PYTHON_XS_PROJECT_SPECIFIER,
     SHELF_PYTHON_JSON,
 )
+from pulpcore.pytest_plugin import pulp_content_url
 
 
 PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL"
@@ -137,6 +141,110 @@ def test_package_upload_simple(
     assert summary.added["python.python"]["count"] == 1
 
 
+# todo: tests + moving
+# PythonPackageSingleArtifactContentUploadViewSet - create
+def test_wheel_package_upload_with_metadata_1(
+    delete_orphans_pre,
+    pulp_content_url,
+    python_content_factory,
+    python_distribution_factory,
+    python_repo,
+):
+    # pdb.set_trace()
+    python_content_factory(
+        repository=python_repo, relative_path=PYTHON_WHEEL_FILENAME, url=PYTHON_WHEEL_URL
+    )
+    distro = python_distribution_factory(repository=python_repo)
+
+    # Test that metadata is accessible
+    relative_path = f"{distro.base_path}/{PYTHON_WHEEL_FILENAME}.metadata"
+    metadata_url = urljoin(pulp_content_url, relative_path)
+    metadata_response = requests.get(metadata_url)
+    assert metadata_response.status_code == 200
+    assert len(metadata_response.content) > 0
+    assert "Name: shelf-reader" in metadata_response.text
+
+
+# PythonPackageSingleArtifactContentUploadViewSet - upload
+def test_wheel_package_upload_with_metadata_2(
+    delete_orphans_pre,
+    download_python_file,
+    monitor_task,
+    pulp_content_url,
+    python_bindings,
+    python_distribution_factory,
+    python_repo,
+):
+    python_file = download_python_file(PYTHON_WHEEL_FILENAME, PYTHON_WHEEL_URL)
+    content_body = {"file": python_file}
+    content = python_bindings.ContentPackagesApi.upload(**content_body)
+
+    body = {"add_content_units": [content.pulp_href]}
+    monitor_task(python_bindings.RepositoriesPythonApi.modify(python_repo.pulp_href, body).task)
+    distro = python_distribution_factory(repository=python_repo)
+
+    # Test that metadata is accessible
+    relative_path = f"{distro.base_path}/{PYTHON_WHEEL_FILENAME}.metadata"
+    metadata_url = urljoin(pulp_content_url, relative_path)
+    metadata_response = requests.get(metadata_url)
+    assert metadata_response.status_code == 200
+    assert len(metadata_response.content) > 0
+    assert "Name: shelf-reader" in metadata_response.text
+
+
+# PythonRepositoryViewSet - sync
+def test_wheel_package_upload_with_metadata_3(
+    delete_orphans_pre,
+    pulp_content_url,
+    python_distribution_factory,
+    python_remote_factory,
+    python_repo_with_sync,
+):
+    remote = python_remote_factory(includes=PYTHON_XS_PROJECT_SPECIFIER)
+    repo = python_repo_with_sync(remote)
+    distro = python_distribution_factory(repository=repo)
+
+    # Test that metadata is accessible
+    relative_path = f"{distro.base_path}/{PYTHON_WHEEL_FILENAME}.metadata"
+    metadata_url = urljoin(pulp_content_url, relative_path)
+    metadata_response = requests.get(metadata_url)
+    assert metadata_response.status_code == 200
+    assert len(metadata_response.content) > 0
+    assert "Name: shelf-reader" in metadata_response.text
+
+
+# SimpleView - create
+def test_wheel_package_upload_with_metadata_4(
+    delete_orphans_pre,
+    monitor_task,
+    pulp_content_url,
+    python_content_summary,
+    python_empty_repo_distro,
+    python_package_dist_directory,
+):
+    repo, distro = python_empty_repo_distro()
+    url = urljoin(distro.base_url, "simple/")
+    dist_dir, egg_file, wheel_file = python_package_dist_directory
+    response = requests.post(
+        url,
+        data={"sha256_digest": PYTHON_WHEEL_SHA256},
+        files={"content": open(wheel_file, "rb")},
+        auth=("admin", "password"),
+    )
+    assert response.status_code == 202
+    monitor_task(response.json()["task"])
+    summary = python_content_summary(repository=repo)
+    assert summary.added["python.python"]["count"] == 1
+
+    # Test that metadata is accessible
+    relative_path = f"{distro.base_path}/{PYTHON_WHEEL_FILENAME}.metadata"
+    metadata_url = urljoin(pulp_content_url, relative_path)
+    metadata_response = requests.get(metadata_url)
+    assert metadata_response.status_code == 200
+    assert len(metadata_response.content) > 0
+    assert "Name: shelf-reader" in metadata_response.text
+
+
 @pytest.mark.parallel
 def test_twine_upload(
     pulpcore_bindings,