From 1b2aa89e8f2dc4d47b9b1f4cccb173b0179ab0a2 Mon Sep 17 00:00:00 2001 From: Yeganathan S <63534555+skwowet@users.noreply.github.com> Date: Wed, 8 Jul 2026 17:30:44 +0530 Subject: [PATCH 1/4] fix: skip LLM parsing for oversized project affiliation files Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com> --- .../affiliation/affiliation_service.py | 48 +++++++++++++++---- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py b/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py index db7c56e35e..5ff22a65a9 100644 --- a/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py +++ b/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py @@ -52,6 +52,7 @@ class AffiliationService(BaseService): MAX_CHUNK_SIZE = 5000 MAX_CONCURRENT_CHUNKS = 3 + MAX_FILE_SIZE_BYTES = 1_000_000 FILE_PICKER_PREVIEW_MAX_CHARS = 400 FILE_PICKER_BATCH_SIZE = 20 @@ -955,15 +956,44 @@ async def process_affiliations( raise AffiliationFileNotFoundError(ai_cost=ai_cost) file_path_on_disk = os.path.join(batch_info.repo_path, latest_file_path) - content = await self.read_text_file(file_path_on_disk) - file_hash = self.compute_file_hash(content) - latest_file_hash = file_hash - - affiliations, parse_cost = await self.resolve_snapshot( - registry, - content, - file_hash, - ) + file_size_bytes = os.path.getsize(file_path_on_disk) + # Too big for llm — mark unusable and move on. + if file_size_bytes > self.MAX_FILE_SIZE_BYTES: + # Steady state: already gave up on this file; getsize is enough. + if ( + registry + and registry.status == AffiliationRegistryStatus.UNUSABLE.value + and registry.file_path == latest_file_path + and registry.file_hash + ): + file_hash = registry.file_hash + latest_file_hash = file_hash + affiliations = [] + parse_cost = 0.0 + else: + # First hit: hash it so the registry can mark it as unusable. + hasher = hashlib.sha256() + async with aiofiles.open(file_path_on_disk, "rb") as affiliation_file: + while file_chunk := await affiliation_file.read(1024 * 1024): + hasher.update(file_chunk) + file_hash = hasher.hexdigest() + latest_file_hash = file_hash + raise AffiliationAnalysisError( + retain_file_hash=True, + error_message=( + f"Affiliation file {latest_file_path!r} is too large for LLM parsing " + f"({file_size_bytes} bytes > {self.MAX_FILE_SIZE_BYTES} bytes)" + ), + ) + else: + content = await self.read_text_file(file_path_on_disk) + file_hash = self.compute_file_hash(content) + latest_file_hash = file_hash + affiliations, parse_cost = await self.resolve_snapshot( + registry, + content, + file_hash, + ) ai_cost += parse_cost if repository.parent_repo: From ebd5a56e8eb040ff0b64425f934915c24a20280a Mon Sep 17 00:00:00 2001 From: Yeganathan S <63534555+skwowet@users.noreply.github.com> Date: Wed, 8 Jul 2026 17:41:57 +0530 Subject: [PATCH 2/4] refactor: hash the file directly; decode only for parsing Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com> --- .../affiliation/affiliation_service.py | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py b/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py index 5ff22a65a9..95e1caa94f 100644 --- a/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py +++ b/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py @@ -80,9 +80,19 @@ async def read_text_file(file_path: str) -> str: return safe_decode(await f.read()) @staticmethod - def compute_file_hash(content: str) -> str: - """SHA-256 hex digest of UTF-8 file content (not a Git blob SHA).""" - return hashlib.sha256(content.encode("utf-8")).hexdigest() + async def compute_file_hash_from_path( + file_path: str, *, retain_content: bool = False + ) -> tuple[bytes | None, str]: + """SHA-256 of raw file bytes. Set retain_content to decode and parse under the size limit.""" + hasher = hashlib.sha256() + chunks: list[bytes] = [] + async with aiofiles.open(file_path, "rb") as affiliation_file: + while file_chunk := await affiliation_file.read(1024 * 1024): + hasher.update(file_chunk) + if retain_content: + chunks.append(file_chunk) + file_bytes = b"".join(chunks) if retain_content else None + return file_bytes, hasher.hexdigest() @classmethod def is_text_file_path(cls, relative_path: str) -> bool: @@ -972,11 +982,7 @@ async def process_affiliations( parse_cost = 0.0 else: # First hit: hash it so the registry can mark it as unusable. - hasher = hashlib.sha256() - async with aiofiles.open(file_path_on_disk, "rb") as affiliation_file: - while file_chunk := await affiliation_file.read(1024 * 1024): - hasher.update(file_chunk) - file_hash = hasher.hexdigest() + _, file_hash = await self.compute_file_hash_from_path(file_path_on_disk) latest_file_hash = file_hash raise AffiliationAnalysisError( retain_file_hash=True, @@ -986,8 +992,10 @@ async def process_affiliations( ), ) else: - content = await self.read_text_file(file_path_on_disk) - file_hash = self.compute_file_hash(content) + file_bytes, file_hash = await self.compute_file_hash_from_path( + file_path_on_disk, retain_content=True + ) + content = safe_decode(file_bytes) latest_file_hash = file_hash affiliations, parse_cost = await self.resolve_snapshot( registry, From 43e693d1fbcca976b07fb4837e5d6cb1a7ad8da2 Mon Sep 17 00:00:00 2001 From: Yeganathan S <63534555+skwowet@users.noreply.github.com> Date: Wed, 8 Jul 2026 17:52:10 +0530 Subject: [PATCH 3/4] fix: resolve pr review comments Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com> --- .../src/crowdgit/services/affiliation/affiliation_service.py | 2 +- services/libs/test-kit/src/factories/member.ts | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 services/libs/test-kit/src/factories/member.ts diff --git a/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py b/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py index 95e1caa94f..f80fd577c5 100644 --- a/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py +++ b/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py @@ -966,7 +966,7 @@ async def process_affiliations( raise AffiliationFileNotFoundError(ai_cost=ai_cost) file_path_on_disk = os.path.join(batch_info.repo_path, latest_file_path) - file_size_bytes = os.path.getsize(file_path_on_disk) + file_size_bytes = await aiofiles.os.path.getsize(file_path_on_disk) # Too big for llm — mark unusable and move on. if file_size_bytes > self.MAX_FILE_SIZE_BYTES: # Steady state: already gave up on this file; getsize is enough. diff --git a/services/libs/test-kit/src/factories/member.ts b/services/libs/test-kit/src/factories/member.ts new file mode 100644 index 0000000000..e69de29bb2 From 77a8fb49a4edf93e454203b2a921fe78687b05d8 Mon Sep 17 00:00:00 2001 From: Yeganathan S <63534555+skwowet@users.noreply.github.com> Date: Wed, 8 Jul 2026 17:59:19 +0530 Subject: [PATCH 4/4] refactor: remove unused static method for reading text files in AffiliationService Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com> --- .../src/crowdgit/services/affiliation/affiliation_service.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py b/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py index f80fd577c5..f3b009a76a 100644 --- a/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py +++ b/services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py @@ -74,11 +74,6 @@ class AffiliationService(BaseService): ".json", ) - @staticmethod - async def read_text_file(file_path: str) -> str: - async with aiofiles.open(file_path, "rb") as f: - return safe_decode(await f.read()) - @staticmethod async def compute_file_hash_from_path( file_path: str, *, retain_content: bool = False