support general cohort analytics

Author: Honghan

autoimmune.py

@@ -69,4 +69,4 @@ def get_concepts(output_file):
 
 
 if __name__ == "__main__":
-    get_concepts('autoimmune_results_v2.csv')
+    get_concepts('autoimmune_results_v3.csv')

cohortanalysis.py

@@ -0,0 +1,62 @@
+import utils
+import sqldbutils as dutil
+import json
+import ontotextapi as oi
+
+# query concept sql
+autoimmune_concepts_sql = """
+select distinct concept_name from [SQLCRIS_User].[Kconnect].[ulms_concept_mapping]
+"""
+
+# query patient sql
+patients_sql = """
+select brcid, primary_diag, diagnosis_date, dob, gender_id, ethnicitycleaned from
+[SQLCRIS_User].[Kconnect].[cohorts]
+where patient_group='{}'
+"""
+
+# query concept freqs over patient
+concept_doc_freq_sql = """
+  select p.brcid, COUNT(distinct a.CN_Doc_ID) num
+  from [SQLCRIS_User].[Kconnect].[cohorts] c, [SQLCRIS_User].Kconnect.kconnect_annotations a, GateDB_Cris.dbo.gate d
+  where
+  a.inst_uri='{0}'
+  and a.CN_Doc_ID = d.CN_Doc_ID
+  and c.brcid = d.BrcId
+  and c.patient_group='{1}'
+  group by c.brcid
+"""
+
+
+def populate_patient_concept_table(cohort_name, concepts, out_file):
+    patients = []
+    dutil.query_data(patients_sql.format(cohort_name), patients)
+    id2p = {}
+    for p in patients:
+        id2p[p['brcid']] = p
+
+    non_empty_concepts = []
+    for c in concepts:
+        patient_concept_freq = []
+        dutil.query_data(concept_doc_freq_sql.format(c, cohort_name), patient_concept_freq)
+        if len(patient_concept_freq) > 0:
+            non_empty_concepts.append(c)
+            for pc in patient_concept_freq:
+                id2p[pc['brcid']][c] = pc['num']
+
+    label2cid = {}
+    concept_labels = []
+    for c in non_empty_concepts:
+        label2cid[oi.get_concept_label(c)] = c
+        concept_labels.append(c)
+    concept_labels = sorted(concept_labels)
+    s = '\t'.join(['brcid'] + non_empty_concepts) + '\n'
+    for p in patients:
+        s += '\t'.join([p['brcid']] + [p[label2cid[k]] for k in concept_labels]) + '\n'
+    utils.save_string(s, out_file)
+    print 'done'
+
+
+if __name__ == "__main__":
+    concepts = utils.load_json_data('./resources/cardiovascular_concepts.json')
+    populate_patient_concept_table('valproic acid patients', concepts, 'heart_cohorts.csv')

ontotextapi.py

@@ -23,6 +23,12 @@
 }}
 """
 
+subconcepts_only_query_tmp = """
+SELECT ?inst_full WHERE {{
+   <http://linkedlifedata.com/resource/umls/id/{}> <http://www.w3.org/2004/02/skos/core#narrower> ?inst_full .
+}}
+"""
+
 # mimir query template
 mimir_query_temp = """
 {{Mention sparql = "{}"}}
@@ -76,7 +82,8 @@ def generate_prospector_query(concept_id, sparql_only=None):
 
 # query to get all instances of a concept
 def query_instances(concept_id):
-    q = generate_prospector_query(concept_id, sparql_only=True)
+    # q = generate_prospector_query(concept_id, sparql_only=True)
+    q = subconcepts_only_query_tmp.format(concept_id)
     # print q
     ret = query_result(q)
     return [r['inst_full']['value'] for r in ret]
@@ -89,9 +96,10 @@ def get_all_instances(save_file):
         if concepts[c] == '':
             continue
         insts = query_instances(concepts[c])
+        insts = [concepts[c]] + insts
         print u'{}\t{}\t{}\t{}'.format(c, concepts[c], len(insts), json.dumps(insts))
         for cid in insts:
-            concpet2subconcepts_csv += u'"{}", "{}"\n'.format(c, cid[cid.rfind('/')+1:])
+            concpet2subconcepts_csv += u'{}, {}\n'.format(c, cid[cid.rfind('/')+1:])
     if save_file is not None:
         utils.save_string(concpet2subconcepts_csv, save_file)
 
@@ -110,9 +118,26 @@ def generate_all_queries():
     utils.save_json_array(concept2queries, './resources/mimir_queries.json')
 
 
+def get_concept_label(concept_id):
+    query2 = """
+    select ?label where {{
+        <http://linkedlifedata.com/resource/umls/id/{}> <http://www.w3.org/2008/05/skos-xl#prefLabel> ?labelObj .
+        ?labelObj <http://www.w3.org/2008/05/skos-xl#literalForm> ?label .
+        FILTER ( lang(?label) = "en" )
+    }}
+    """.format(concept_id)
+    ret = query_result(query2)
+    labels = [r['label']['value'] for r in ret]
+
+    if len(labels) > 0:
+        return labels[0]
+    else:
+        return None
+
 def main():
     # generate_all_queries()
-    get_all_instances('./resources/all_insts.csv')
+    # get_all_instances('./resources/all_insts.csv')
+    print get_concept_label('C0018799')
 
 if __name__ == "__main__":
     main()