Add option to limit batch requests to Leetcode API

Leetcode responses got too large. As a result users are experiencing problem with running the script, because leetcode API fails to return such big results. This diff adds an ability to limit number of problems, downloaded in parallel and limits it to 1000 problems by default.

Author: fspv

generate.py

@@ -33,7 +33,13 @@ def parse_args() -> argparse.Namespace:
         "--start", type=int, help="Start generation from this problem", default=0
     )
     parser.add_argument(
-        "--stop", type=int, help="Stop generation on this problem", default=2 ** 64
+        "--stop", type=int, help="Stop generation on this problem", default=2**64
+    )
+    parser.add_argument(
+        "--page-size",
+        type=int,
+        help="Get at most this many problems (decrease if leetcode API times out)",
+        default=1000,
     )
 
     args = parser.parse_args()
@@ -90,7 +96,7 @@ async def generate_anki_note(
     )
 
 
-async def generate(start: int, stop: int) -> None:
+async def generate(start: int, stop: int, page_size: int) -> None:
     """
     Generate an Anki deck
     """
@@ -157,7 +163,7 @@ async def generate(start: int, stop: int) -> None:
     )
     leetcode_deck = genanki.Deck(LEETCODE_ANKI_DECK_ID, "leetcode")
 
-    leetcode_data = leetcode_anki.helpers.leetcode.LeetcodeData(start, stop)
+    leetcode_data = leetcode_anki.helpers.leetcode.LeetcodeData(start, stop, page_size)
 
     note_generators: List[Coroutine[Any, Any, LeetcodeNote]] = []
 
@@ -185,8 +191,8 @@ async def main() -> None:
     """
     args = parse_args()
 
-    start, stop = args.start, args.stop
-    await generate(start, stop)
+    start, stop, page_size = args.start, args.stop, args.page_size
+    await generate(start, stop, page_size)
 
 
 if __name__ == "__main__":

leetcode_anki/helpers/leetcode.py

@@ -83,7 +83,7 @@ class LeetcodeData:
     names.
     """
 
-    def __init__(self, start: int, stop: int) -> None:
+    def __init__(self, start: int, stop: int, page_size: int = 1000) -> None:
         """
         Initialize leetcode API and disk cache for API responses
         """
@@ -93,11 +93,15 @@ def __init__(self, start: int, stop: int) -> None:
         if stop < 0:
             raise ValueError(f"Stop must be non-negative: {start}")
 
+        if page_size < 0:
+            raise ValueError(f"Page size must be greater than 0: {page_size}")
+
         if start > stop:
             raise ValueError(f"Start (){start}) must be not greater than stop ({stop})")
 
         self._start = start
         self._stop = stop
+        self._page_size = page_size
 
     @cached_property
     def _api_instance(self) -> leetcode.api.default_api.DefaultApi:
@@ -214,7 +218,7 @@ def _get_problems_data(
         start = self._start
         stop = min(self._stop, problem_count)
 
-        page_size = min(3000, stop - start + 1)
+        page_size = min(self._page_size, stop - start + 1)
 
         problems: List[
             leetcode.models.graphql_question_detail.GraphqlQuestionDetail