add CI job

Author: vbarda

.github/workflows/eval.yml

@@ -0,0 +1,40 @@
+name: Eval
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+  workflow_dispatch:
+
+concurrency:
+  group: eval-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run_eval:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python + Poetry
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: "3.11"
+          poetry-version: "1.7.1"
+          cache-key: lint
+
+      - name: Install dependencies
+        run: poetry install --with dev
+
+      - name: Evaluate
+        env:
+          LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          WEAVIATE_URL: ${{ secrets.WEAVIATE_URL }}
+          WEAVIATE_API_KEY: ${{ secrets.WEAVIATE_API_KEY }}
+        run: poetry run pytest backend/tests/evals

.github/workflows/lint.yml

@@ -44,7 +44,7 @@ jobs:
         run: poetry lock --check
 
       - name: Install dependencies
-        run: poetry install --with lint
+        run: poetry install --with dev
 
       - name: Get .mypy_cache to speed up mypy
         uses: actions/cache@v3

poetry.lock

@@ -25,8 +25,8 @@ voyageai = "^0.1.4"
 pillow = "^10.2.0"
 psycopg2-binary = "^2.9.9"
 
-
-[tool.poetry.group.lint.dependencies]
+[tool.poetry.group.dev.dependencies]
+pytest = "^7.3.0"
 ruff = "^0.2.2"
 
 [build-system]