Merge pull request #632 from DocNow/flattened-errors

Fix an edge case where a response from the API with only an errors block would be passed through unchanged.

Author: Sam Hames

test_twarc2.py

@@ -509,6 +509,14 @@ def test_ensure_flattened():
         twarc.expansions.ensure_flattened([[{"data": {"fake": "list_of_lists"}}]])
 
 
+def test_ensure_flattened_errors():
+    """
+    Test that ensure_flattened doesn't return tweets for API responses that only contain errors.
+    """
+    data = {"errors": ["fake error"]}
+    assert twarc.expansions.ensure_flattened(data) == []
+
+
 def test_ensure_user_id():
     """
     Test _ensure_user_id's ability to discriminate correctly between IDs and

twarc/client2.py

@@ -1178,7 +1178,7 @@ def _stream(self, url, params, event, record_keepalive, tries=30):
                     log.error(f"too many consecutive errors ({tries}). stopping")
                     return
                 else:
-                    secs = errors ** 2
+                    secs = errors**2
                     log.info("sleeping %s seconds before reconnecting", secs)
                     time.sleep(secs)
 

twarc/decorators2.py

@@ -91,7 +91,7 @@ def new_f(*args, **kwargs):
                 if errors > tries:
                     log.warning(f"too many errors ({tries}) from Twitter, giving up")
                     resp.raise_for_status()
-                seconds = errors ** 2
+                seconds = errors**2
                 log.warning(
                     "caught %s from Twitter API, sleeping %s", resp.status_code, seconds
                 )
@@ -142,7 +142,7 @@ def new_f(self, *args, **kwargs):
                 if errors > tries:
                     log.error(f"giving up, too many request exceptions: {tries}")
                     raise e
-                seconds = errors ** 2
+                seconds = errors**2
                 log.info("sleeping %s", seconds)
                 time.sleep(seconds)
                 self.connect()

twarc/expansions.py

@@ -242,6 +242,15 @@ def ensure_flattened(data):
         log.warning(f"Unable to expand dictionary without includes: {data}")
         return flatten(data)
 
+    # If it's just an object with errors return an empty list
+    elif (
+        isinstance(data, dict)
+        and "data" not in data
+        and "includes" not in data
+        and "errors" in data
+    ):
+        return []
+
     # If it's a single response and both "includes" and "data" are missing, it is already flattened
     elif isinstance(data, dict) and "data" not in data and "includes" not in data:
         return [data]