Add support for replace directive in go.mod files

src/packagedcode/go_mod.py

@@ -21,6 +21,7 @@ class GoModule(object):
     module = attr.ib(default=None)
     require = attr.ib(default=None)
     exclude = attr.ib(default=None)
+    local_replacements = attr.ib(default=None)
 
     def purl(self, include_version=True):
         version = None
@@ -50,6 +51,13 @@ def purl(self, include_version=True):
     r'(?P<version>(.*))'
 ).match
 
+parse_rep_link = re.compile(
+    r"(?P<ns_name>\S+)"
+    r"(?:\s+(?P<version>\S+))?"
+    r"\s*=>\s*"
+    r"(?P<replacement_ns_name>\S+)"
+    r"(?:\s+(?P<replacement_version>\S+))?"
+).match
 
 def preprocess(line):
     """
@@ -60,6 +68,60 @@ def preprocess(line):
     line = line.strip()
     return line
 
+def parse_replace_directive(line):
+    parsed_replace = parse_rep_link(line)
+    ns_name = parsed_replace.group("ns_name")
+    version = parsed_replace.group("version")
+    namespace, _, name = ns_name.rpartition("/")
+    original_module = {
+        "namespace": namespace,
+        "name": name,
+        "version": version
+    }
+
+    replacement_ns_name = parsed_replace.group("replacement_ns_name")
+    replacement_version = parsed_replace.group("replacement_version")
+    is_local = replacement_ns_name.startswith("./") or replacement_ns_name.startswith("../")
+
+    if is_local:
+        replacement_namespace = None
+        replacement_name = replacement_ns_name
+    else:
+        replacement_namespace, _, replacement_name = replacement_ns_name.rpartition("/")
+
+    replacement_module = {
+        "namespace": replacement_namespace,
+        "name": replacement_name,
+        "version": replacement_version,
+        "is_local": is_local,
+        "local_path": replacement_ns_name if is_local else None
+    }
+
+    return original_module, replacement_module
+
+def handle_replace_directive(line, require, exclude, local_replacements):
+    original, replacement = parse_replace_directive(line)
+    exclude.append(
+        GoModule(
+            namespace=original.get('namespace'),
+            name=original.get('name'),
+            version=original.get('version'),
+        )
+    )
+
+    if replacement.get('is_local'):
+        local_replacements.append({
+            'replaces': f"{original.get('namespace')}/{original.get('name')}",
+            'local_path': replacement.get('local_path')
+        })
+    else:
+        require.append(
+            GoModule(
+                namespace=replacement.get('namespace'),
+                name=replacement.get('name'),
+                version=replacement.get('version')
+            )
+        )
 
 def parse_gomod(location):
     """
@@ -120,6 +182,7 @@ def parse_gomod(location):
     gomods = GoModule()
     require = []
     exclude = []
+    local_replacements = []
 
     for i, line in enumerate(lines):
         line = preprocess(line)
@@ -158,6 +221,19 @@ def parse_gomod(location):
                     )
             continue
 
+        if 'replace' in line and '(' in line:
+            for rep in lines[i + 1:]:
+                rep = preprocess(rep)
+                if ')' in rep:
+                    break
+                handle_replace_directive(rep, require, exclude, local_replacements)
+            continue
+
+        if 'replace' in line and '=>' in line:
+            line = line.lstrip("replace").strip()
+            handle_replace_directive(line, require, exclude, local_replacements)
+            continue
+
         parsed_module_name = parse_module(line)
         if parsed_module_name:
             ns_name = parsed_module_name.group('ns_name')
@@ -188,6 +264,7 @@ def parse_gomod(location):
 
     gomods.require = require
     gomods.exclude = exclude
+    gomods.local_replacements = local_replacements
 
     return gomods
 
@@ -202,7 +279,6 @@ def parse_gomod(location):
     r'h1:(?P<checksum>[^\s]*)'
 ).match
 
-
 def parse_gosum(location):
     """
     Return a list of GoSum from parsing the go.sum file at `location`.

src/packagedcode/golang.py

@@ -7,6 +7,8 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import os
+import posixpath
 from packagedcode import go_mod
 from packagedcode import models
 
@@ -24,6 +26,27 @@
 # TODO: use the LICENSE file convention!
 # TODO: support "vendor" and "workspace" layouts
 
+# Tracing flags
+TRACE = False or os.environ.get("SCANCODE_DEBUG_PACKAGE", False)
+
+
+# Tracing flags
+def logger_debug(*args):
+    pass
+
+
+if TRACE:
+    import logging
+    import sys
+
+    logger = logging.getLogger(__name__)
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
+
+    def logger_debug(*args):
+        return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args))
+
+
 
 class BaseGoModuleHandler(models.DatafileHandler):
 
@@ -32,13 +55,93 @@ def assemble(cls, package_data, resource, codebase, package_adder):
         """
         Always use go.mod first then go.sum
         """
+
+        if not codebase.has_single_resource:
+            cls.resolve_local_replacements(
+                package_data=package_data,
+                resource=resource,
+                codebase=codebase,
+            )
+
+        resource.package_data[0] = package_data.to_dict()
+
         yield from cls.assemble_from_many_datafiles(
             datafile_name_patterns=('go.mod', 'go.sum',),
             directory=resource.parent(codebase),
             codebase=codebase,
             package_adder=package_adder,
         )
 
+    @classmethod
+    def resolve_local_replacements(cls, package_data, resource, codebase):
+        """
+        Resolve local paths present in replace directives
+        """
+
+        local_replacements = package_data.extra_data.get('local_replacements', [])
+        if not local_replacements: 
+            logger_debug(f"resolve_local_replacements: No local replacements found")
+            return
+
+        base_dir = resource.parent(codebase)
+        base_path = base_dir.path
+
+        for idx, replacement in enumerate(local_replacements):
+            local_path = replacement. get('local_path')
+            if not local_path:
+                logger_debug(f"resolve_local_replacements:  Skipping replacement {idx + 1} - no local_path found")
+                continue
+
+            full_path = posixpath.normpath(
+                posixpath.join(base_path, local_path)
+            )
+
+            local_resource = codebase.get_resource(full_path)
+            if not local_resource:
+                logger_debug(f"resolve_local_replacements:  Resource not found at {full_path}")
+                continue
+
+            local_gomod = None
+            for child in local_resource. children(codebase):
+                if child.name == 'go.mod':
+                    local_gomod = child
+                    break
+
+            if not local_gomod or not local_gomod. package_data:
+                logger_debug(f"resolve_local_replacements: No go.mod or package_data found in {full_path}")
+                continue
+
+            try:
+                local_pkg_dict = local_gomod.package_data[0]
+                local_pkg_data = models.PackageData.from_dict(local_pkg_dict)
+            except (IndexError, KeyError, TypeError) as e:
+                logger_debug(f"resolve_local_replacements: Failed to parse package data:  {e}")
+                continue
+
+            if not local_pkg_data. purl:
+                logger_debug(f"resolve_local_replacements: No purl found in local package data")
+                continue
+
+            resolved_dependency = models.DependentPackage(
+                purl=local_pkg_data.purl,
+                extracted_requirement=local_pkg_data.version or None,
+                resolved_package=local_pkg_data.to_dict(),
+                scope='require',
+                is_runtime=True,
+                is_optional=False,
+                extra_data={
+                    'replaces':  replacement.get('replaces'),
+                    'resolved_from_local':  True,
+                    'local_path': local_path,
+                    'local_resolved_path': full_path,
+                }
+            )
+
+            if not any(dep.purl == resolved_dependency.purl for dep in package_data.dependencies):
+                package_data.dependencies.append(resolved_dependency)
+                logger_debug(f"resolve_local_replacements: Added dependency: {resolved_dependency.purl}")
+            else:
+                logger_debug(f"resolve_local_replacements: Dependency already exists, skipping: {resolved_dependency. purl}")
 
 class GoModHandler(BaseGoModuleHandler):
     datasource_id = 'go_mod'
@@ -79,6 +182,10 @@ def parse(cls, location, package_only=False):
                 )
             )
 
+        extra_data = {
+            'local_replacements': gomods.local_replacements
+        }
+
         name = gomods.name
         namespace = gomods.namespace
 
@@ -98,6 +205,7 @@ def parse(cls, location, package_only=False):
             homepage_url=homepage_url,
             repository_homepage_url=repository_homepage_url,
             dependencies=dependencies,
+            extra_data=extra_data if gomods.local_replacements else {},
             primary_language=cls.default_primary_language,
         )
         yield models.PackageData.from_data(package_data, package_only)

tests/packagedcode/data/golang/gomod/gopls/go.mod

@@ -0,0 +1,30 @@
+module golang.org/x/tools/gopls
+
+go 1.18
+
+require (
+	github.com/google/go-cmp v0.5.9
+	github.com/jba/printsrc v0.2.2
+	github.com/jba/templatecheck v0.6.0
+	github.com/sergi/go-diff v1.1.0
+	golang.org/x/mod v0.12.0
+	golang.org/x/sync v0.3.0
+	golang.org/x/sys v0.11.0
+	golang.org/x/telemetry v0.0.0-20230808152233-a65b40c0fdb0
+	golang.org/x/text v0.12.0
+	golang.org/x/tools v0.6.0
+	golang.org/x/vuln v0.0.0-20230110180137-6ad3e3d07815
+	gopkg.in/yaml.v3 v3.0.1
+	honnef.co/go/tools v0.4.2
+	mvdan.cc/gofumpt v0.4.0
+	mvdan.cc/xurls/v2 v2.4.0
+)
+
+require (
+	github.com/BurntSushi/toml v1.2.1 // indirect
+	github.com/google/safehtml v0.1.0 // indirect
+	golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
+	golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338 // indirect
+)
+
+replace golang.org/x/tools => ../