Use USER_AGENT to try bypass being blocked. (#8486)

* Use USER_AGENT to try bypass being blocked.

* Use proxy

* updating test

Author: PaulRoms

electricitymap/contrib/parsers/CNDC.py

@@ -17,6 +17,8 @@
     TotalProductionList,
 )
 from electricitymap.contrib.lib.types import ZoneKey
+from electricitymap.contrib.parsers.lib.config import use_proxy
+from electricitymap.contrib.parsers.lib.exceptions import ParserException
 
 tz_bo = ZoneInfo("America/La_Paz")
 
@@ -28,6 +30,9 @@
 INDEX_URL = "https://www.cndc.bo/gene/index.php"
 DATA_URL = "https://www.cndc.bo/gene/dat/gene.php?fechag={0}"
 SOURCE = "cndc.bo"
+# User-Agent to avoid being blocked as a bot
+USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+REQUEST_TIMEOUT = 30  # seconds
 
 
 def extract_xsrf_token(html):
@@ -45,27 +50,84 @@ def get_datetime(query_date: datetime, hour: int) -> datetime:
     )
 
 
+def _check_response(response, context: str = ""):
+    """Check HTTP response and raise appropriate ParserException if needed."""
+    if response.status_code == 403:
+        raise ParserException(
+            "CNDC.py",
+            f"Access forbidden (403){context}. The server may be blocking requests.",
+            "BO",
+        )
+    elif response.status_code == 429:
+        raise ParserException("CNDC.py", f"Rate limit exceeded (429){context}.", "BO")
+    elif response.status_code >= 500:
+        raise ParserException(
+            "CNDC.py",
+            f"Server error ({response.status_code}){context}. The CNDC server may be down.",
+            "BO",
+        )
+    elif not response.ok:
+        raise ParserException("CNDC.py", f"HTTP {response.status_code}{context}", "BO")
+
+
 def fetch_data(
     session: Session | None = None, target_datetime: datetime | None = None
 ) -> tuple[list[dict], datetime]:
-    if session is None:
-        session = Session()
-
-    if target_datetime is None:
-        target_datetime = datetime.now()
-    target_datetime = target_datetime.astimezone(tz_bo)
-    # Define actual and previous day (for midnight data).
+    session = session or Session()
+    target_datetime = (target_datetime or datetime.now()).astimezone(tz_bo)
     formatted_dt = target_datetime.strftime("%Y-%m-%d")
 
-    # XSRF token for the initial request
-    xsrf_token = extract_xsrf_token(session.get(INDEX_URL).text)
-
-    resp = session.get(
-        DATA_URL.format(formatted_dt), headers={"x-csrf-token": xsrf_token}
-    )
-
-    hour_rows = json.loads(resp.text.replace("", ""))["data"]
-    return hour_rows, target_datetime
+    # Headers to mimic a browser and avoid being blocked
+    headers = {
+        "User-Agent": USER_AGENT,
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.5",
+        "Connection": "keep-alive",
+    }
+
+    try:
+        # Get XSRF token from index page
+        index_response = session.get(
+            INDEX_URL, headers=headers, timeout=REQUEST_TIMEOUT
+        )
+        _check_response(index_response)
+
+        try:
+            xsrf_token = extract_xsrf_token(index_response.text)
+        except (AttributeError, IndexError) as e:
+            raise ParserException(
+                "CNDC.py",
+                "Failed to extract XSRF token. Website structure may have changed.",
+                "BO",
+            ) from e
+
+        # Fetch data with XSRF token
+        headers["x-csrf-token"] = xsrf_token
+        data_response = session.get(
+            DATA_URL.format(formatted_dt), headers=headers, timeout=REQUEST_TIMEOUT
+        )
+        _check_response(data_response, " when fetching data")
+
+        # Parse JSON response
+        try:
+            hour_rows = json.loads(data_response.text.replace("", ""))["data"]
+        except (json.JSONDecodeError, KeyError) as e:
+            raise ParserException(
+                "CNDC.py",
+                f"Failed to parse JSON response. API format may have changed: {e}",
+                "BO",
+            ) from e
+
+        return hour_rows, target_datetime
+
+    except ParserException:
+        raise
+    except Exception as e:
+        raise ParserException(
+            "CNDC.py",
+            f"Unexpected error: {type(e).__name__}: {e}",
+            "BO",
+        ) from e
 
 
 def parse_generation_forecast(
@@ -105,6 +167,9 @@ def parser_production_breakdown(
         if total is None or None in modes_extracted:
             continue
 
+        unknown_value = round(total - thermo - hydro - solar - wind - bagasse, 3)
+        unknown_value = None if abs(unknown_value) < 0.05 else unknown_value
+
         result.append(
             zoneKey=zone_key,
             datetime=timestamp,
@@ -115,14 +180,15 @@ def parser_production_breakdown(
                 biomass=bagasse,
                 gas=round(thermo * gas_oil_ratio, 3),
                 oil=round(thermo * (1 - gas_oil_ratio), 3),
-                unknown=round(total - thermo - hydro - solar - wind - bagasse, 3),
+                unknown=unknown_value,
             ),
             source=SOURCE,
         )
 
     return result
 
 
+@use_proxy(country_code="BO")
 def fetch_production(
     zone_key: ZoneKey = ZoneKey("BO"),
     session: Session | None = None,
@@ -152,8 +218,8 @@ def fetch_generation_forecast(
 
 if __name__ == "__main__":
     """Main method, never used by the Electricity Map backend, but handy for testing."""
-    print("fetch_production() ->")
     print(fetch_production())
+    print("fetch_production() ->")
 
     # print("fetch_generation_forecast() ->")
     # print(fetch_generation_forecast())

electricitymap/contrib/parsers/tests/__snapshots__/test_CNDC.ambr

@@ -183,7 +183,7 @@
         'hydro': 384.86,
         'oil': 11.074,
         'solar': 0,
-        'unknown': -0.0,
+        'unknown': None,
         'wind': 45.39,
       }),
       'source': 'cndc.bo',
@@ -202,7 +202,7 @@
         'hydro': 402.78,
         'oil': 10.355,
         'solar': 0,
-        'unknown': -0.0,
+        'unknown': None,
         'wind': 47.16,
       }),
       'source': 'cndc.bo',
@@ -221,7 +221,7 @@
         'hydro': 406.37,
         'oil': 9.835,
         'solar': 0,
-        'unknown': -0.0,
+        'unknown': None,
         'wind': 48.76,
       }),
       'source': 'cndc.bo',
@@ -240,7 +240,7 @@
         'hydro': 380.18,
         'oil': 9.855,
         'solar': 0,
-        'unknown': -0.0,
+        'unknown': None,
         'wind': 47.33,
       }),
       'source': 'cndc.bo',
@@ -259,7 +259,7 @@
         'hydro': 379.26,
         'oil': 9.831,
         'solar': 0,
-        'unknown': 0.0,
+        'unknown': None,
         'wind': 51.32,
       }),
       'source': 'cndc.bo',
@@ -278,7 +278,7 @@
         'hydro': 350.42,
         'oil': 9.628,
         'solar': 0.39,
-        'unknown': -0.0,
+        'unknown': None,
         'wind': 50.26,
       }),
       'source': 'cndc.bo',
@@ -297,7 +297,7 @@
         'hydro': 422.66,
         'oil': 9.968,
         'solar': 13.45,
-        'unknown': -0.0,
+        'unknown': None,
         'wind': 41.49,
       }),
       'source': 'cndc.bo',
@@ -316,7 +316,7 @@
         'hydro': 500.35,
         'oil': 10.666,
         'solar': 47.85,
-        'unknown': 0.0,
+        'unknown': None,
         'wind': 45.95,
       }),
       'source': 'cndc.bo',
@@ -335,7 +335,7 @@
         'hydro': 505.62,
         'oil': 11.446,
         'solar': 90.01,
-        'unknown': -0.0,
+        'unknown': None,
         'wind': 46.62,
       }),
       'source': 'cndc.bo',
@@ -354,7 +354,7 @@
         'hydro': 523.88,
         'oil': 12.567,
         'solar': 113.07,
-        'unknown': -0.0,
+        'unknown': None,
         'wind': 32.52,
       }),
       'source': 'cndc.bo',
@@ -373,7 +373,7 @@
         'hydro': 511.72,
         'oil': 13.695,
         'solar': 124.97,
-        'unknown': 0.0,
+        'unknown': None,
         'wind': 24.76,
       }),
       'source': 'cndc.bo',