Fix handling undefined length in CdxRecord (#83)

Some CDX entries have `"-"` for their length, instead of a number. `WaybackClient.search()` used to raise an error in this situation, but now handles it gracefully by setting the resulting `CdxRecord` object's `length` property to `None`.

Author: 8W9aG

CONTRIBUTING.rst

@@ -79,10 +79,10 @@ Ready to contribute? Here's how to set up `wayback` for local development.
 5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox::
 
     $ flake8 wayback tests
-    $ python setup.py test
+    $ pytest
     $ tox
 
-   To get flake8 and tox, just pip install them into your virtualenv.
+   To get flake8, pytest and tox, just pip install them into your virtualenv using `pip install -r requirements-dev.txt`.
 
 6. Commit your changes and push your branch to GitHub::
 

README.rst

@@ -65,6 +65,7 @@ Thanks to the following people for their contributions and help on this package!
 
 - `Dan Allan <https://github.com/danielballan>`_ (Code, Tests, Documentation, Reviews)
 - `Rob Brackett <https://github.com/Mr0grog>`_ (Code, Tests, Documentation, Reviews)
+- `Will Sackfield <https://github.com/8W9aG>`_ (Code, Tests)
 - `Ed Summers <https://github.com/edsu>`_ (Code, Tests)
 - `Lion Szlagowski <https://github.com/LionSzl>`_ (Code, Tests)
 

wayback/_client.py

@@ -539,7 +539,7 @@ def search(self, url, *, matchType=None, limit=None, offset=None,
                     status_code = None
                 else:
                     status_code = int(data.status_code)
-                length = int(data.length)
+                length = None if data.length == '-' else int(data.length)
                 capture_time = _utils.parse_timestamp(data.timestamp)
             except Exception as err:
                 if 'RobotAccessControlException' in text:

wayback/tests/test_client.py

@@ -190,6 +190,34 @@ def test_search_removes_malformed_entries(requests_mock):
         assert 2 == len(list(records))
 
 
+def test_search_handles_no_length_cdx_records(requests_mock):
+    """
+    The CDX index can contain a "-" in lieu of an actual length, which can't be
+    parsed into an int. We should handle this.
+
+    Because these are rare and hard to get all in a single CDX query that isn't
+    *huge*, we use a made-up mock for this one instead of a VCR recording.
+    """
+    with open(Path(__file__).parent / 'test_files' / 'zero_length_cdx.txt') as f:
+        bad_cdx_data = f.read()
+
+    with WaybackClient() as client:
+        requests_mock.get('http://web.archive.org/cdx/search/cdx'
+                          '?url=www.cnn.com%2F%2A'
+                          '&matchType=domain&filter=statuscode%3A200'
+                          '&showResumeKey=true&resolveRevisits=true',
+                          [{'status_code': 200, 'text': bad_cdx_data}])
+        records = client.search('www.cnn.com/*',
+                                matchType="domain",
+                                filter_field="statuscode:200")
+
+        record_list = list(records)
+        assert 5 == len(record_list)
+        for record in record_list[:4]:
+            assert isinstance(record.length, int)
+        assert record_list[-1].length is None
+
+
 @ia_vcr.use_cassette()
 def test_get_memento():
     with WaybackClient() as client:

wayback/tests/test_files/zero_length_cdx.txt

@@ -0,0 +1,5 @@
+com,cnn)/ 20100215131836 http://www.cnn.com/ text/html 200 T6CDCO73TS77BY67HLKYFLHKT2APUM5Y 24186
+com,cnn)/ 20100217150314 http://www.cnn.com:80/ text/html 200 3Y4OQ7ETJXGA24RIDYJYKMTJKF525QS2 24028
+com,cnn)/ 20100217231950 http://www.cnn.com/ text/html 200 OSFLJPVDJVSRY4X4AGMRPUMVK7AEQCWI 24116
+com,cnn)/ 20100218000634 http://www.cnn.com/ text/html 200 73LNKJSLFZSDZ7T77JNZWXXNGDRSWTHO 24217
+com,cnn)/ 20100218025451 http://www.cnn.com/ text/html 200 JK7O4OWOMDJHBNQJ5U43X5OQRORF6FQB -