Merge pull request #25 from mariya/DEVELOP-1099-instrument-specific-completed-marker-file

DEVELOP-1099 - use custom completed marker files for different instruments

Author: mariya

README.md

@@ -23,13 +23,19 @@ This means that the client (e.g. a workflow) is responsible for updating the sta
 
 **Try it out:**
 
-Install using pip:
+    # create venv
+    virtualenv -p python2.7 venv/   
 
-    pip install -r requirements/dev . # possible add -e if you're in development mode.
+    # activate venv
+    source venv/bin/activate
+
+    # install dependencies
+    pip install -e . -r ./requirements/dev
 
 Open up the `config/app.config` and specify the root directories that you want to monitor for runfolders. Then run:
 
-    runfolder-ws --port 9999 --configroot config/
+    # start the dev server
+    python server.py --debug --port=9999 --configroot='./config'
 
 This will star the runfolder service on port 9999, and the api dock will be available under `localhost:9999/api`.
 Try curl-ing to see what you can do with it:
@@ -47,7 +53,3 @@ Alternatively, you can run the same script against a remote server, specifying t
 
 Unit tests can be run with
     nosetests ./runfolder_tests/unit
-
-**Install in production**
-One way to install this as a daemon in a production environment
-can be seen at https://github.com/arteria-project/arteria-provisioning

config/app.config

@@ -9,17 +9,6 @@ monitored_directories:
 # also enables adding the runfolder-ready marker through the API.
 can_create_runfolder: False
 
-# An Illumina machine will write a RTAComplete.txt file when it has
-# finished the sequencing process. But when a user manually moves
-# the runfolder to the processing directory one can not guaranty
-# that all files have been moved when the RTAComplete.tx exist in
-# the new directory. The completed_marker_file can be changed to
-# another file, a file which for example will signal that the
-# transfer process has completed.
-completed_marker_file:
-    - RTAComplete.txt
-    - SequencingComplete.txt
-
 # It seems that sometimes the completed_marker_files are not a
 # reliable indicator that nothing more will be modified or written
 # in the sequencing runfolder. Therefore a grace period can be

runfolder/lib/__init__.py

@@ -0,0 +1,81 @@
+# Typically, an Illumina machine writes a RTAComplete.txt file when it has
+# finished the sequencing process. However, some instruments may use another
+# completed_marker_file, for instance one that indicates that files have
+# been transferred.
+
+import re
+
+
+class InstrumentFactory():
+    @staticmethod
+    def get_id(run_parameters):
+        run_parameters = run_parameters.get('RunParameters', None)
+
+        # HiSeq / HiSeq X instrument stored in Setup#ScannerID
+        val = run_parameters.get('Setup', {}).get('ScannerID')
+        if val:
+            return val
+
+        # Other instrument id's are at the top level of RunParameters
+        for key in ["InstrumentName", "InstrumentId", "ScannerID"]:
+            if key in run_parameters.viewkeys():
+                return run_parameters.get(key)
+
+    @staticmethod
+    def get_instrument(run_parameters):
+        if not run_parameters:
+            return Instrument()
+
+        instrument_id = InstrumentFactory.get_id(run_parameters)
+
+        if not instrument_id:
+            return Instrument()
+
+        if re.search(NovaSeq.ID_PATTERN, instrument_id):
+            return NovaSeq()
+        if re.search(ISeq.ID_PATTERN, instrument_id):
+            return ISeq()
+        if re.search(MiSeq.ID_PATTERN, instrument_id):
+            return MiSeq()
+        if re.search(HiSeq.ID_PATTERN, instrument_id):
+            return HiSeq()
+        if re.search(HiSeqX.ID_PATTERN, instrument_id):
+            return HiSeqX()
+        return Instrument()
+
+
+class Instrument():
+    COMPLETED_MARKER_FILE_RTA_COMPLETE = 'RTAComplete.txt'
+    COMPLETED_MARKER_FILE_COPY_COMPLETE = 'CopyComplete.txt'
+
+    @staticmethod
+    def completed_marker_file():
+        return Instrument.COMPLETED_MARKER_FILE_RTA_COMPLETE
+
+
+class NovaSeq(Instrument):
+    ID_PATTERN = '^A'
+
+    @staticmethod
+    def completed_marker_file():
+        return Instrument.COMPLETED_MARKER_FILE_COPY_COMPLETE
+
+
+class ISeq(Instrument):
+    ID_PATTERN = '^FS'
+
+    @staticmethod
+    def completed_marker_file():
+        return Instrument.COMPLETED_MARKER_FILE_COPY_COMPLETE
+
+
+class MiSeq(Instrument):
+    ID_PATTERN = '^M'
+
+
+class HiSeq(Instrument):
+    ID_PATTERN = '^D'
+
+
+class HiSeqX(HiSeq):
+    ID_PATTERN = '^ST-E'

runfolder/lib/instrument.py

@@ -7,6 +7,7 @@
 
 from arteria.web.state import State
 from arteria.web.state import validate_state
+from runfolder.lib.instrument import InstrumentFactory
 
 class RunfolderInfo:
     """
@@ -111,7 +112,8 @@ def create_runfolder(self, path):
                         'ReagentKitBarcode': 'AB1234567-123V1',
                         'RfidsInfo': {
                                 'LibraryTubeSerialBarcode': 'NV0012345-LIB'
-                            }
+                            },
+                        'ScannerID': 'M04499'
                     }
             }
 
@@ -186,13 +188,8 @@ def get_runfolder_state(self, runfolder):
         If the file .arteria/state exists, it will determine the state. If it doesn't
         exist, the existence of the marker file RTAComplete.txt determines the state.
         """
-        completed_marker_files = None
-        try:
-            completed_marker_files = self._configuration_svc["completed_marker_file"]
-            if isinstance(completed_marker_files, str):
-                completed_marker_files = [completed_marker_files]
-        except KeyError:
-            raise ConfigurationError("completed_marker_file must be set")
+        instrument = InstrumentFactory.get_instrument(self.read_run_parameters(runfolder))
+        completed_marker_file = instrument.completed_marker_file()
         completed_grace_minutes = None
         try:
             completed_grace_minutes = self._configuration_svc["completed_marker_grace_minutes"]
@@ -204,10 +201,9 @@ def get_runfolder_state(self, runfolder):
         state = self._get_runfolder_state_from_state_file(runfolder)
         if state == State.NONE:
             ready = True
-            for marker_file in completed_marker_files:
-                completed_marker = os.path.join(runfolder, marker_file)
-                if not self._file_exists_and_is_older_than(completed_marker, completed_grace_minutes):
-                    ready = False
+            completed_marker = os.path.join(runfolder, completed_marker_file)
+            if not self._file_exists_and_is_older_than(completed_marker, completed_grace_minutes):
+                ready = False
             if ready:
                 state = State.READY
         return state
@@ -293,6 +289,7 @@ def get_metadata(self, path):
             metadata['library_tube_barcode'] = library_tube_barcode
         return metadata
 
+
     def get_reagent_kit_barcode(self, path, run_parameters):
         try:
             barcode = run_parameters['RunParameters']['ReagentKitBarcode']

runfolder/services.py

@@ -0,0 +1,75 @@
+import unittest
+import logging
+import mock
+
+
+from runfolder.lib.instrument import InstrumentFactory, Instrument
+
+
+logger = logging.getLogger(__name__)
+
+class InstrumentTestCase(unittest.TestCase):
+    def test_get_instrument_by_id(self):
+        # HiSeqX
+        instrument_attr = {
+            'Setup': {
+                'ScannerID': 'ST-E123'
+            }
+        }
+        run_parameters = {
+            'RunParameters': instrument_attr
+        }
+
+        i = InstrumentFactory.get_instrument(run_parameters)
+        self.assertEqual(i.__class__.__name__, 'HiSeqX')
+        self.assertEqual(i.completed_marker_file(), 'RTAComplete.txt')
+
+        # HiSeq
+        run_parameters['RunParameters']['Setup']['ScannerID'] = 'D999'
+        i = InstrumentFactory.get_instrument(run_parameters)
+        self.assertEqual(i.__class__.__name__, 'HiSeq')
+        self.assertEqual(i.completed_marker_file(), 'RTAComplete.txt')
+
+        # NovaSeq
+        run_parameters = {
+            'RunParameters': {
+                'InstrumentName': 'ABC'
+            }
+        }
+        i = InstrumentFactory.get_instrument(run_parameters)
+        self.assertEqual(i.__class__.__name__, 'NovaSeq')
+        self.assertEqual(i.completed_marker_file(), 'CopyComplete.txt')
+
+        # iSeq
+        run_parameters = {
+            'RunParameters': {
+                'InstrumentId': 'FS1'
+            }
+        }
+        i = InstrumentFactory.get_instrument(run_parameters)
+        self.assertEqual(i.__class__.__name__, 'ISeq')
+        self.assertEqual(i.completed_marker_file(), 'CopyComplete.txt')
+
+        # MiSeq
+        run_parameters = {
+            'RunParameters': {
+                'ScannerID': 'M1'
+            }
+        }
+        i = InstrumentFactory.get_instrument(run_parameters)
+        self.assertEqual(i.__class__.__name__, 'MiSeq')
+        self.assertEqual(i.completed_marker_file(), 'RTAComplete.txt')
+
+        # Default
+        run_parameters = {
+            'RunParameters': {
+                'ScannerID': 'foo'
+            }
+        }
+        i = InstrumentFactory.get_instrument(run_parameters)
+        self.assertEqual(i.__class__.__name__, 'Instrument')
+        self.assertEqual(i.completed_marker_file(), 'RTAComplete.txt')
+
+
+if __name__ == '__main__':
+    unittest.main()

runfolder_tests/unit/instrument_tests.py

@@ -28,8 +28,7 @@ def test_list_available_runfolders(self):
             "monitored_directories": [
                 "/data/testarteria1/mon1",
                 "/data/testarteria1/mon2"
-            ],
-            "completed_marker_file": "RTAComplete.txt"
+            ]
         }
         runfolder_svc = RunfolderService(configuration_svc, logger)
 
@@ -53,8 +52,7 @@ def test_next_runfolder(self):
         configuration_svc = {
             "monitored_directories": [
                 "/data/testarteria1/mon1"
-            ],
-            "completed_marker_file": "RTAComplete.txt"
+            ]
         }
 
         # Since keys in configuration_svc can be directly indexed, we can mock it with a dict:

runfolder_tests/unit/runfolder_tests.py

@@ -0,0 +1,5 @@
+from runfolder.app import start
+
+
+if __name__ == '__main__':
+    start()