diff --git a/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py b/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py index db7c56e35e..f3b009a76a 100644 --- a/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py +++ b/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py @@ -52,6 +52,7 @@ class AffiliationService(BaseService): MAX_CHUNK_SIZE = 5000 MAX_CONCURRENT_CHUNKS = 3 + MAX_FILE_SIZE_BYTES = 1_000_000 FILE_PICKER_PREVIEW_MAX_CHARS = 400 FILE_PICKER_BATCH_SIZE = 20 @@ -74,14 +75,19 @@ class AffiliationService(BaseService): ) @staticmethod - async def read_text_file(file_path: str) -> str: - async with aiofiles.open(file_path, "rb") as f: - return safe_decode(await f.read()) - - @staticmethod - def compute_file_hash(content: str) -> str: - """SHA-256 hex digest of UTF-8 file content (not a Git blob SHA).""" - return hashlib.sha256(content.encode("utf-8")).hexdigest() + async def compute_file_hash_from_path( + file_path: str, *, retain_content: bool = False + ) -> tuple[bytes | None, str]: + """SHA-256 of raw file bytes. Set retain_content to decode and parse under the size limit.""" + hasher = hashlib.sha256() + chunks: list[bytes] = [] + async with aiofiles.open(file_path, "rb") as affiliation_file: + while file_chunk := await affiliation_file.read(1024 * 1024): + hasher.update(file_chunk) + if retain_content: + chunks.append(file_chunk) + file_bytes = b"".join(chunks) if retain_content else None + return file_bytes, hasher.hexdigest() @classmethod def is_text_file_path(cls, relative_path: str) -> bool: @@ -955,15 +961,42 @@ async def process_affiliations( raise AffiliationFileNotFoundError(ai_cost=ai_cost) file_path_on_disk = os.path.join(batch_info.repo_path, latest_file_path) - content = await self.read_text_file(file_path_on_disk) - file_hash = self.compute_file_hash(content) - latest_file_hash = file_hash - - affiliations, parse_cost = await self.resolve_snapshot( - registry, - content, - file_hash, - ) + file_size_bytes = await aiofiles.os.path.getsize(file_path_on_disk) + # Too big for llm — mark unusable and move on. + if file_size_bytes > self.MAX_FILE_SIZE_BYTES: + # Steady state: already gave up on this file; getsize is enough. + if ( + registry + and registry.status == AffiliationRegistryStatus.UNUSABLE.value + and registry.file_path == latest_file_path + and registry.file_hash + ): + file_hash = registry.file_hash + latest_file_hash = file_hash + affiliations = [] + parse_cost = 0.0 + else: + # First hit: hash it so the registry can mark it as unusable. + _, file_hash = await self.compute_file_hash_from_path(file_path_on_disk) + latest_file_hash = file_hash + raise AffiliationAnalysisError( + retain_file_hash=True, + error_message=( + f"Affiliation file {latest_file_path!r} is too large for LLM parsing " + f"({file_size_bytes} bytes > {self.MAX_FILE_SIZE_BYTES} bytes)" + ), + ) + else: + file_bytes, file_hash = await self.compute_file_hash_from_path( + file_path_on_disk, retain_content=True + ) + content = safe_decode(file_bytes) + latest_file_hash = file_hash + affiliations, parse_cost = await self.resolve_snapshot( + registry, + content, + file_hash, + ) ai_cost += parse_cost if repository.parent_repo: diff --git a/services/libs/test-kit/src/factories/member.ts b/services/libs/test-kit/src/factories/member.ts new file mode 100644 index 0000000000..e69de29bb2