si3mshady · May 21, 2023 21:04 · May 21, 2023
diff --git a/http_requests.py b/http_requests.py
@@ -0,0 +1,152 @@
+"""
+This module provides functionality for making HTTP requests. It leverages the `aiohttp` 
+library for asynchronous HTTP requests, and the `backoff` library to implement exponential 
+backoff in case of failed requests.
+
+The module defines a child logger for logging purposes and implements two methods, `on_backoff` 
+and `on_giveup`, which log information about the retry attempts and when the retry attempts are 
+given up respectively.
+
+The `http_request` function is the primary function of the module, making an HTTP request with the 
+provided parameters. If the request fails due to an `aiohttp.ClientError`, the function will retry 
+the request using an exponential backoff strategy, up to a maximum of 5 times or a total of 60 
+seconds.
+
+The function logs the result of the HTTP request, either indicating success and the status code of 
+the response or raising an exception if the response cannot be parsed as JSON or if the response 
+status code indicates a client error.
+"""
+
+import json
+import logging
+import aiohttp
+import backoff
+
+# set up child logger for module
+logger = logging.getLogger(__name__)
+
+
+def on_backoff(backoff_event):
+    """
+    Logs a warning message whenever a backoff event occurs due to a failed HTTP request.
+
+    Parameters
+    ----------
+    backoff_event : dict
+        A dictionary containing detailed information about the backoff event.
+        It includes the following keys:
+        'wait' (the delay before the next retry),
+        'tries' (the number of attempts made),
+        'exception' (the exception that caused the backoff),
+        'target' (the function where the exception was raised),
+        'args' (the arguments passed to the target function),
+        'kwargs' (the keyword arguments passed to the target function),
+        and 'elapsed' (the time elapsed since the first attempt).
+    """
+    logger.warning(
+        "http backoff event",
+        extra={
+            "Retrying in, seconds": {backoff_event["wait"]},
+            "Attempt number": {backoff_event["tries"]},
+            "Exception": {backoff_event["exception"]},
+            "Target function": {backoff_event["target"].__name__},
+            "Args": {backoff_event["args"]},
+            "Kwargs": {backoff_event["kwargs"]},
+            "Elapsed time": {backoff_event["elapsed"]},
+        },
+    )
+
+
+def on_giveup(giveup_event):
+    """
+    Logs an error message when a series of HTTP requests fail and the retry attempts are given up.
+
+    Parameters
+    ----------
+    giveup_event : dict
+        A dictionary containing detailed information about the event when retries are given up.
+        It includes the following keys:
+        'tries' (the number of attempts made),
+        'exception' (the exception that caused the retries to be given up),
+        'target' (the function where the exception was raised),
+        'args' (the arguments passed to the target function),
+        'kwargs' (the keyword arguments passed to the target function),
+        and 'elapsed' (the time elapsed since the first attempt).
+    """
+    logger.error(
+        "http giveup event",
+        extra={
+            "Giving up after retries exceeded": giveup_event["tries"],
+            "Exception": giveup_event["exception"],
+            "Target function": giveup_event["target"].__name__,
+            "Args": giveup_event["args"],
+            "Kwargs": giveup_event["kwargs"],
+            "Elapsed time": giveup_event["elapsed"],
+        },
+    )
+
+
+@backoff.on_exception(
+    backoff.expo,
+    aiohttp.ClientError,
+    jitter=backoff.random_jitter,
+    max_tries=5,
+    max_time=60,
+    on_backoff=on_backoff,
+    on_giveup=on_giveup,
+)
+async def http_request(verb, url, query_params, headers, payload):
+    """
+    Performs an HTTP request, retrying on `aiohttp.ClientError` exceptions with an exponential
+    backoff strategy.
+
+    Parameters
+    ----------
+    verb : str
+        The HTTP method for the request, such as 'GET', 'POST', etc.
+    url : str
+        The URL for the HTTP request.
+    query_params : dict
+        The query parameters to be included in the request.
+    headers : dict
+        The headers to be included in the request.
+    payload : dict
+        The payload (body) of the request, which will be sent as JSON.
+
+    Returns
+    -------
+    None
+
+    Raises
+    ------
+    ValueError
+        If the response from the HTTP request cannot be parsed as JSON.
+    aiohttp.ClientResponseError
+        If the HTTP request returns a response with a status code indicating a client error.
+
+    Note
+    ----
+    The function will automatically retry the request if an `aiohttp.ClientError` is raised.
+    It uses an exponential backoff strategy with a maximum of 5 tries and a total retry duration
+    of 60 seconds. The 'on_backoff' function will be called after each failed attempt, and the
+    'on_giveup' function will be called if all retry attempts fail.
+    """
+    async with aiohttp.ClientSession() as session:
+        async with session.request(
+            method=verb, url=url, params=query_params, headers=headers, json=payload
+        ) as response:
+            if response.ok:
+                try:
+                    data = await response.json()
+                    return data
+                except json.JSONDecodeError as json_decode_error:
+                    raise ValueError(
+                        "Failed to parse response JSON"
+                    ) from json_decode_error
+            else:
+                raise aiohttp.ClientResponseError(
+                    response.request_info,
+                    response.history,
+                    status=response.status,
+                    message=response.reason,
+                )
No results found