Merge pull request #3 from openstates/build-docker-image

Build docker image

Author: alexobaseki

.dockerignore

@@ -0,0 +1,8 @@
+.cache/
+_data/
+logs/
+db.db
+*.pyc
+__pycache__/
+*.md
+.git/

.github/workflows/docker.yml

@@ -0,0 +1,44 @@
+name: Build and Push Docker Image
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - '*'
+jobs:
+  publish:
+    name: Build and Push to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and Push Docker Image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          tags: "openstates/scraper-audit:latest,openstates/scraper-audit:${{ github.sha }}"
+          push: true
+  publish-plural:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Authenticate with Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          project_id: civic-eagle-enview-prod
+          credentials_json: "${{ secrets.PLURAL_OPEN_REPO_PUSHER_KEY }}"
+      - name: Setup Google Cloud SDK
+        uses: "google-github-actions/setup-gcloud@v2"
+      - name: Docker Auth
+        run: gcloud auth configure-docker us-central1-docker.pkg.dev --quiet
+      - name: Build and Push Docker Image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          tags: "us-central1-docker.pkg.dev/civic-eagle-enview-prod/open-containers/scraper-audit:latest,us-central1-docker.pkg.dev/civic-eagle-enview-prod/open-containers/scraper-audit:${{ github.sha }}"
+          push: true

Dockerfile

@@ -0,0 +1,36 @@
+FROM python:3.11-slim
+
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONIOENCODING='utf-8'
+
+# set the workdir directory to the environment so bash can find it
+ENV WORKDIR='/app'
+WORKDIR /app
+
+# disable virtualenv creation to help save some space and slightly speedup startup
+RUN apt-get update -qq \
+  && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \
+  && pip install --disable-pip-version-check --no-cache-dir -q wheel \
+  && pip install --disable-pip-version-check --no-cache-dir -q poetry crcmod \
+  && poetry config virtualenvs.create false
+
+# Copy poetry files
+COPY pyproject.toml /app
+COPY poetry.lock /app
+
+
+# install dependencies before copying files in, this makes the install step *slightly* faster, and makes re-builds significantly faster
+RUN poetry install --no-root \
+    && rm -r /root/.cache/pypoetry/cache /root/.cache/pypoetry/artifacts/ \
+    && apt-get remove -y -qq \
+      build-essential \
+      libpq-dev \
+    && apt-get autoremove -y -qq \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy application code
+COPY . /app
+
+ENTRYPOINT ["poetry", "run", "python", "main.py"]

README.md

@@ -75,4 +75,29 @@ Updating virtual layer  ━━━━━━━━━━━━━━━━━━
 
 Audit failed:
  [WARNING] staged.bill: 'assert_bills_have_sponsor' audit error: 1179 rows failed. Learn more in logs: ~/scraper-audit/logs/sqlmesh_2025_05_06_20_58_30.log
-```
+```
+
+### Docker Usage
+
+To run the scraper-audit using Docker, you can build the image locally and execute it.
+Ensure your data directory is accessible to Docker.
+For example, if your JSON files are in a local `_data` folder:
+
+#### Build the Docker Image
+
+From the root of the project run
+```bash
+docker build -t scraper-audit .
+```
+
+#### Run the Container
+
+Assuming your JSON data is in a local _data directory run
+
+```bash
+docker run --rm -v ./_data:/app/_data scraper-audit --entity "bill"
+```
+
+Note: The `--entity` flag is optional.
+If provided, it can be set to `bill`, `event`, or `vote_event` to audit a specific entity.
+If omitted, audits will run for all entities.

main.py

@@ -1,6 +1,5 @@
 import argparse
 
-
 from sqlmesh_tasks import sqlmesh_plan
 
 if __name__ == "__main__":