spectrum form no longer directly adds to public site (#215)

Author: tlambert03

backend/fpbase/templates/pages/contact.html

@@ -9,7 +9,8 @@ <h2>What's on your mind?</h2>
 
 <p class="small text-muted">A common reason people contact us is to request that new spectra be added to the database.
     While we will eventually try to accomodate these requests, please note that much of FPbase is publicly editable, and
-    you can <a href="{% url "proteins:submit-spectra" %}">add spectra to the database here</a>.</p>
+    you can <a href="{% url "proteins:submit-spectra" %}">add spectra to the database here</a>, and
+    <a href="{% url "proteins:submit" %}">submit new proteins here</a>.</p>
 <form action="{% url 'contact' %}" method="post">
     {% csrf_token %}
     {% crispy form %}

backend/proteins/admin.py

@@ -1,6 +1,6 @@
 from django import forms
 from django.contrib import admin
-from django.db.models import Count
+from django.db.models import Count, Prefetch
 from django.forms import TextInput
 from django.urls import reverse
 from django.utils.safestring import mark_safe
@@ -56,7 +56,8 @@ def __init__(self, *args, **kwargs):
     def spectra(self, obj):
         def _makelink(sp):
             url = reverse("admin:proteins_spectrum_change", args=(sp.pk,))
-            return f'<a href="{url}">{sp.get_subtype_display()}</a>'
+            pending = " (pending)" if sp.status == Spectrum.STATUS.pending else ""
+            return f'<a href="{url}">{sp.get_subtype_display()}{pending}</a>'
 
         links = []
         if isinstance(obj, Fluorophore):
@@ -67,13 +68,13 @@ def _makelink(sp):
 
     def get_queryset(self, request):
         qs = super().get_queryset(request)
-        return qs.prefetch_related("spectrum")
+        return qs.prefetch_related(Prefetch("spectrum", queryset=Spectrum.objects.all_objects()))
 
 
 class MultipleSpectraOwner(SpectrumOwner):
     def get_queryset(self, request):
         qs = super(SpectrumOwner, self).get_queryset(request)
-        return qs.prefetch_related("spectra")
+        return qs.prefetch_related(Prefetch("spectra", queryset=Spectrum.objects.all_objects()))
 
 
 class BleachInline(admin.TabularInline):
@@ -219,7 +220,7 @@ class SpectrumAdmin(VersionAdmin):
         "created_by",
     )
     list_display = ("__str__", "category", "subtype", "owner", "created_by")
-    list_filter = ("created", "category", "subtype")
+    list_filter = ("status", "created", "category", "subtype")
     readonly_fields = ("owner", "name", "created", "modified")
     search_fields = (
         "owner_state__protein__name",
@@ -253,6 +254,7 @@ def get_fields(self, request, obj=None):
             "solvent",
             "source",
             "reference",
+            "status",
             ("created", "created_by"),
             ("modified", "updated_by"),
         ]
@@ -267,9 +269,16 @@ def owner(self, obj):
         link = f'<a href="{url}">{obj.owner}</a>'
         return mark_safe(link)
 
-    # def get_queryset(self, request):
-    #     qs = super().get_queryset(request)
-    #     return qs.prefetch_related('owner_state__protein')
+    def get_queryset(self, request):
+        """
+        Return a QuerySet of all model instances that can be edited by the
+        admin site. This is used by changelist_view.
+        """
+        qs = Spectrum.objects.all_objects()
+        ordering = self.get_ordering(request)
+        if ordering:
+            qs = qs.order_by(*ordering)
+        return qs
 
 
 @admin.register(OSERMeasurement)
@@ -458,7 +467,7 @@ def save_model(self, request, obj, form, change):
 
 
 @admin.action(description="Mark selected proteins as approved")
-def make_approved(modeladmin, request, queryset):
+def approve_protein(modeladmin, request, queryset):
     # note, this will fail if the list is ordered by numproteins
     queryset.update(status=Protein.STATUS.approved)
 
@@ -487,7 +496,7 @@ class ProteinAdmin(CompareVersionAdmin):
     )
     prepopulated_fields = {"slug": ("name",)}
     inlines = (StateInline, StateTransitionInline, OSERInline, LineageInline)
-    actions = [make_approved]
+    actions = [approve_protein]
     fieldsets = [
         (
             None,

backend/proteins/forms/spectrum.py

@@ -160,14 +160,18 @@ def clean_owner_state(self):
         owner_state = self.cleaned_data.get("owner_state")
         stype = self.cleaned_data.get("subtype")
         if self.cleaned_data.get("category") == Spectrum.PROTEIN:
-            if owner_state.spectra.filter(subtype=stype).exists():
+            spectra = Spectrum.objects.all_objects().filter(owner_state=owner_state, subtype=stype)
+            if spectra.exists():
+                first = spectra.first()
                 self.add_error(
                     "owner_state",
                     forms.ValidationError(
-                        "%(owner)s already has a{} %(stype)s spectrum".format("n" if stype != Spectrum.TWOP else ""),
+                        "%(owner)s already has a%(n)s %(stype)s spectrum %(status)s",
                         params={
                             "owner": owner_state,
-                            "stype": owner_state.spectra.filter(subtype=stype).first().get_subtype_display().lower(),
+                            "stype": first.get_subtype_display().lower(),
+                            "n": "n" if stype != Spectrum.TWOP else "",
+                            "status": " (pending)" if first.status == Spectrum.STATUS.pending else "",
                         },
                         code="owner_exists",
                     ),

backend/proteins/migrations/0055_spectrum_status_spectrum_status_changed.py

@@ -0,0 +1,39 @@
+# Generated by Django 4.2.1 on 2024-09-20 14:26
+
+from django.db import migrations
+import django.utils.timezone
+import model_utils.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("proteins", "0054_microscope_cfg_calc_efficiency_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="spectrum",
+            name="status",
+            field=model_utils.fields.StatusField(
+                choices=[
+                    ("pending", "pending"),
+                    ("approved", "approved"),
+                    ("rejected", "rejected"),
+                ],
+                default="pending",
+                max_length=100,
+                no_check_for_status=True,
+                verbose_name="status",
+            ),
+        ),
+        migrations.AddField(
+            model_name="spectrum",
+            name="status_changed",
+            field=model_utils.fields.MonitorField(
+                default=django.utils.timezone.now,
+                monitor="status",
+                verbose_name="status changed",
+            ),
+        ),
+    ]

backend/proteins/migrations/0056_auto_20240920_1426.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.2.1 on 2024-09-20 14:26
+
+from contextlib import suppress
+from django.db import migrations
+from proteins.models import Spectrum
+
+def set_default_status(apps, schema_editor):
+    for spectrum in Spectrum.objects.all():
+        try:
+            spectrum.status = Spectrum.STATUS.approved
+            spectrum.save()
+        except Exception:
+            print(f"Failed to resave spectrum id: {spectrum.id}")
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("proteins", "0055_spectrum_status_spectrum_status_changed"),
+    ]
+
+    operations = [migrations.RunPython(set_default_status)]

backend/proteins/models/spectrum.py

@@ -1,5 +1,6 @@
 import ast
 import json
+import logging
 
 import numpy as np
 from django.contrib.postgres.fields import ArrayField
@@ -11,14 +12,17 @@
 from django.forms import CharField, Textarea
 from django.urls import reverse
 from django.utils.text import slugify
+from model_utils import Choices
 from model_utils.managers import QueryManager
-from model_utils.models import TimeStampedModel
+from model_utils.models import StatusModel, TimeStampedModel
 from references.models import Reference
 
 from ..util.helpers import wave_to_hex
 from ..util.spectra import interp_linear, interp_univar, norm2one, norm2P, step_size
 from .mixins import AdminURLMixin, Authorable, Product
 
+logger = logging.getLogger(__name__)
+
 
 class SpectrumOwner(Authorable, TimeStampedModel):
     name = models.CharField(max_length=100)  # required
@@ -74,6 +78,13 @@ def get_cached_spectra_info(timeout=60 * 60):
 
 
 class SpectrumManager(models.Manager):
+    def get_queryset(self):
+        # by default, only include approved spectra
+        return super().get_queryset().filter(status=Spectrum.STATUS.approved)
+
+    def all_objects(self):
+        return super().get_queryset()
+
     def state_slugs(self):
         L = (
             self.get_queryset()
@@ -294,7 +305,9 @@ def validate(self, value, model_instance):
                 raise ValidationError("All items in Spectrum list elements must be numbers")
 
 
-class Spectrum(Authorable, TimeStampedModel, AdminURLMixin):
+class Spectrum(Authorable, StatusModel, TimeStampedModel, AdminURLMixin):
+    STATUS = Choices("approved", "pending", "rejected")
+
     DYE = "d"
     PROTEIN = "p"
     LIGHT = "l"
@@ -419,12 +432,15 @@ def save(self, *args, **kwargs):
         super().save(*args, **kwargs)
 
     def _norm2one(self):
-        if self.subtype == self.TWOP:
-            y, self._peakval2p, maxi = norm2P(self.y)
-            self._peakwave2p = self.x[maxi]
-            self.change_y(y)
-        else:
-            self.change_y(norm2one(self.y))
+        try:
+            if self.subtype == self.TWOP:
+                y, self._peakval2p, maxi = norm2P(self.y)
+                self._peakwave2p = self.x[maxi]
+                self.change_y(y)
+            else:
+                self.change_y(norm2one(self.y))
+        except Exception:
+            logger.exception("Error normalizing spectrum data")
 
     def _interpolated_data(self, method=None, **kwargs):
         if not method or method.lower() == "linear":

backend/proteins/templates/spectrum_submitted.html

@@ -0,0 +1,12 @@
+{% extends "base.html" %} {% block content %}
+<div class="container">
+  <div class="row">
+    <div class="col-md-12">
+      <h1>Thank you!</h1>
+      <p>The {{spectrum_name}} spectrum has been submitted successfully. A moderator will verify it soon.</p>
+      <p>If you don't see it within a couple days, please <a href="{% url 'contact' %}">contact us</a>.</p>
+      <p>If you like, you may <a href="{% url 'proteins:submit-spectra' %}">add another spectrum.</a></p>
+    </div>
+  </div>
+</div>
+{% endblock content %}

backend/proteins/urls.py

@@ -65,6 +65,7 @@
         ),
         name="submit-spectra",
     ),
+    path("spectra/submitted/", views.spectrum_submitted, name="spectrum_submitted"),
     re_path(
         r"^spectra/submit/(?P<slug>[-\w]+)/$",
         (

backend/proteins/util/spectra.py

@@ -63,6 +63,8 @@ def norm2P(y):
     localmax = argrelextrema(y, np.greater, order=100)
     # can't be within first 10 points
     localmax = [i for i in localmax[0] if i > 10]
+    if not localmax:
+        return y, 0, 0
     maxind = localmax[np.argmax(y[localmax])]
     maxy = y[maxind]
     return [round(max(yy / maxy, 0), 4) for yy in y], maxy, maxind

backend/proteins/views/spectra.py

@@ -1,5 +1,6 @@
 import contextlib
 import json
+from textwrap import dedent
 
 # from django.views.decorators.cache import cache_page
 # from django.views.decorators.vary import vary_on_cookie
@@ -9,6 +10,7 @@
 from django.http import Http404, HttpResponse, JsonResponse
 from django.shortcuts import render
 from django.template.defaultfilters import slugify
+from django.urls import reverse_lazy
 from django.views.decorators.clickjacking import xframe_options_exempt
 from django.views.generic import CreateView
 from fpbase.util import is_ajax, uncache_protein_page
@@ -48,12 +50,17 @@ def protein_spectra_graph(request, slug=None):
     return render(request, "spectra_graph.html")
 
 
+def spectrum_submitted(request):
+    context = {"spectrum_name": request.session.get("spectrum_name", "")}
+    return render(request, "spectrum_submitted.html", context)
+
+
 class SpectrumCreateView(CreateView):
     model = Spectrum
     form_class = SpectrumForm
 
     def get_success_url(self, **kwargs):
-        return self.object.owner.get_absolute_url()
+        return reverse_lazy("proteins:spectrum_submitted")
 
     def get_initial(self):
         init = super().get_initial()
@@ -74,33 +81,44 @@ def get_form(self, form_class=None):
         form = super().get_form()
 
         if self.kwargs.get("slug", False):
-            try:
+            with contextlib.suppress(Exception):
                 form.fields["owner_state"] = forms.ModelChoiceField(
                     required=True,
                     label="Protein (state)",
                     empty_label=None,
                     queryset=State.objects.filter(protein=self.protein).select_related("protein"),
                 )
                 form.fields["category"].disabled = True
-            except Exception:
-                pass
         return form
 
     def form_valid(self, form):
+        # Set the status to "pending" before saving the form
+        if not self.request.user.is_staff:
+            form.instance.status = Spectrum.STATUS.pending
         # This method is called when valid form data has been POSTed.
         # It should return an HttpResponse.
-        i = super().form_valid(form)
+        response = super().form_valid(form)
         with contextlib.suppress(Exception):
             uncache_protein_page(self.object.owner_state.protein.slug, self.request)
 
         if not self.request.user.is_staff:
+            body = f"""
+            A new spectrum has been submitted to FPbase.
+
+            Admin URL: {self.request.build_absolute_uri(form.instance.get_admin_url())}
+            User: {self.request.user}
+            Data:
+
+            {form.cleaned_data}
+            """
             EmailMessage(
-                f'[FPbase] Spectrum submitted: {form.cleaned_data["owner"]}',
-                self.request.build_absolute_uri(form.instance.get_admin_url()),
+                subject=f'[FPbase] Spectrum needs validation: {form.cleaned_data["owner"]}',
+                body=dedent(body),
                 to=[a[1] for a in settings.ADMINS],
                 headers={"X-Mailgun-Track": "no"},
             ).send()
-        return i
+        self.request.session["spectrum_name"] = self.object.name
+        return response
 
     def get_context_data(self, **kwargs):
         # Call the base implementation first to get a context