Merge pull request #171 from openstates/add-scrape-metadata

Add scrape metadata

Author: alexobaseki

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 6.22.0 - Apr 22, 2025
+* Add metadata to scraper output
+
 ## 6.21.6 - Apr 21, 2025
 * Add informal passage to bill action classification
 

openstates/scrape/base.py

@@ -164,7 +164,7 @@ def save_object(self, obj):
         Generally shouldn't be called directly.
         """
         clean_whitespace(obj)
-        obj.pre_save(self.jurisdiction.jurisdiction_id)
+        obj.pre_save(self.jurisdiction)
 
         filename = f"{obj._type}_{obj._id}.json".replace("/", "-")
         self.info(f"save {obj._type} {obj} as {filename}")
@@ -346,7 +346,7 @@ def validate(self, schema=None):
                 )
             )
 
-    def pre_save(self, jurisdiction_id):
+    def pre_save(self, jurisdiction):
         pass
 
     def as_dict(self):
@@ -366,6 +366,17 @@ def __setattr__(self, key, val):
             )
         super(BaseModel, self).__setattr__(key, val)
 
+    def add_scrape_metadata(self, jurisdiction):
+        """Add scrape metadata"""
+        self.jurisdiction = {
+            "id": jurisdiction.jurisdiction_id,
+            "name": jurisdiction.name,
+            "classification": jurisdiction.classification,
+            "division_id": jurisdiction.division_id,
+        }
+
+        self.scraped_at = utils.utcnow()
+
 
 class SourceMixin(object):
     def __init__(self):

openstates/scrape/bill.py

@@ -51,9 +51,10 @@ def __init__(
         self.versions = []
         self.citations = []
 
-    def pre_save(self, jurisdiction_id):
+    def pre_save(self, jurisdiction):
         # ensure subject is sorted for idempotent JSON output
         self.subject = sorted(self.subject)
+        self.add_scrape_metadata(jurisdiction)
 
     def add_action(
         self,
@@ -76,7 +77,14 @@ def add_action(
         return action
 
     def add_citation(
-        self, publication, citation, citation_type, *, effective=None, expires=None, url=None
+        self,
+        publication,
+        citation,
+        citation_type,
+        *,
+        effective=None,
+        expires=None,
+        url=None,
     ):
         self.citations.append(
             {

openstates/scrape/event.py

@@ -212,3 +212,6 @@ def add_document(
             date=date,
             classification=classification,
         )
+
+    def pre_save(self, jurisdiction):
+        self.add_scrape_metadata(jurisdiction)

openstates/scrape/schemas/bill.py

@@ -2,7 +2,13 @@
     Schema for bill objects.
 """
 import copy
-from .common import sources, extras, fuzzy_date_blank, fuzzy_datetime
+from .common import (
+    sources,
+    extras,
+    fuzzy_date_blank,
+    fuzzy_datetime,
+    jurisdiction_summary,
+)
 from ...data import common
 
 versions_or_documents = {
@@ -164,5 +170,7 @@
         },
         "sources": sources,
         "extras": extras,
+        "jurisdiction": jurisdiction_summary,
+        "scraped_at": fuzzy_datetime,
     },
 }

openstates/scrape/schemas/common.py

@@ -62,3 +62,14 @@
 }
 
 extras = {"type": "object"}
+
+jurisdiction_summary = {
+    "items": {
+        "properties": {
+            "id": {"type": "string", "minLength": 1},
+            "name": {"type": "string", "minLength": 1},
+            "classification": {"type": "string", "minLength": 1},
+            "division_id": {"type": "string", "minLength": 1},
+        },
+    },
+}

openstates/scrape/schemas/event.py

@@ -8,6 +8,7 @@
     fuzzy_date_blank,
     fuzzy_datetime,
     fuzzy_datetime_blank,
+    jurisdiction_summary,
 )
 
 media_schema = {
@@ -134,6 +135,8 @@
         "sources": sources,
         "extras": extras,
         "dedupe_key": {"type": ["string", "null"], "minLength": 1},
+        "jurisdiction": jurisdiction_summary,
+        "scraped_at": fuzzy_datetime,
     },
     "type": "object",
 }

openstates/scrape/tests/test_bill_scrape.py

@@ -1,11 +1,15 @@
 import pytest
 import warnings
 from datetime import date
-from openstates.scrape import Bill
+from openstates.scrape import Bill, State
 from openstates.utils.generic import get_pseudo_id
 from openstates.exceptions import ScrapeValueError
 
 
+class NewJersey(State):
+    pass
+
+
 def toy_bill():
     b = Bill(
         identifier="HB 2017",
@@ -311,10 +315,11 @@ def test_str():
 
 def test_pre_save():
     b = toy_bill()
+    j = NewJersey()
     b.add_subject("ZZZ")
     b.add_subject("AAA")
     b.add_subject("MMM")
-    b.pre_save(None)
+    b.pre_save(j)
     assert b.subject == ["AAA", "MMM", "ZZZ"]
 
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "openstates"
-version = "6.21.6"
+version = "6.22.0"
 description = "core infrastructure for the openstates project"
 authors = ["James Turk <[email protected]>"]
 license = "MIT"