From fe66cb1006b9263af84f118c3c2819c03c0a7b2f Mon Sep 17 00:00:00 2001 From: Zhouyang Lian Date: Sat, 10 Oct 2020 11:55:32 -0400 Subject: [PATCH 01/10] fixing https://github.com/KarrLab/datanator_rest_api/issues/133 --- datanator_query_python/query/query_protein.py | 59 +++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/datanator_query_python/query/query_protein.py b/datanator_query_python/query/query_protein.py index 2bc8413..8af7865 100644 --- a/datanator_query_python/query/query_protein.py +++ b/datanator_query_python/query/query_protein.py @@ -16,6 +16,7 @@ def __init__(self, username=None, password=None, server=None, authSource='admin' readPreference=readPreference, replicaSet=replicaSet) self.taxon_manager = query_taxon_tree.QueryTaxonTree(MongoDB=server, username=username, password=password, authSource=authSource, db=database, replicaSet=replicaSet) + self.taxon_col = self.db_obj['taxon_tree'] self.kegg_manager = query_kegg_orthology.QueryKO(username=username, password=password, server=server, authSource=authSource, replicaSet=replicaSet) self.file_manager = file_util.FileUtil() self.max_entries = max_entries @@ -936,7 +937,9 @@ def get_all_ortho(self, ko, anchor, max_distance): 'ancestor_taxon_id': 1, 'protein_name': 1, 'gene_name': 1, - 'modifications': 1 + 'modifications': 1, + "canon_anc_ids": 1, + "canon_anc_names": 1 } con_0 = {'orthodb_id': ko} con_1 = {'abundances': {'$exists': True}} @@ -944,17 +947,24 @@ def get_all_ortho(self, ko, anchor, max_distance): docs = self.collection.find(filter=query, projection=projection) queried = deque() names = {} + species_anc = {} + canon_anc_anchor = self.taxon_col.find_one({"tax_name": anchor})['canon_anc_names'] for doc in docs: doc = json.loads(json.dumps(doc, ignore_nan=True)) species = doc.get('species_name') - if species is None and species not in queried: + if species is None and species not in queried: # few documents don't have species_name field taxon_id = doc['ncbi_taxonomy_id'] - species = self.db_obj['taxon_tree'].find_one({"tax_id": taxon_id})['tax_name'] + tmp = self.taxon_col.find_one({"tax_id": taxon_id}) + species = tmp['tax_name'] + canon_anc_species = tmp["canon_anc_names"] queried.append(taxon_id) names[taxon_id] = species + species_anc[taxon_id] = canon_anc_species elif species is None and species in queried: species = names[doc['ncbi_taxonomy_id']] - obj = self.taxon_manager.get_canon_common_ancestor_fast(anchor, species, org_format='tax_name') + canon_anc_species = species_anc[doc['ncbi_taxonomy_id']] + obj = self._get_common_canon_anc(anchor, species, canon_anc_anchor, doc["canon_anc_names"]) + print(obj) distance = obj[anchor] if distance != -1 and distance <= max_distance: species_canon_ancestor = obj[species+'_canon_ancestors'] @@ -962,6 +972,47 @@ def get_all_ortho(self, ko, anchor, max_distance): result[distance-1]['documents'].append(doc) return result + def _get_common_canon_anc(self, org1, org2, canon_anc_1, canon_anc_2): + """Get canon common ancestors between species. + + Args: + org1(:obj:`str`): Name of species 1. + org2(:obj:`str`): Name of species 2. + canon_anc_1(:obj:`list` of `str`): List of canon ancestor names of species 1. + canon_anc_2(:obj:`list` of `str`): List of canon ancestor names of species 2. + + Return: + (:obj:`Obj`): {str(org1): distance1, str(org2): distance2, str(org1)+'_canon_ancestors':canon_anc_1, + str(org2)+'_canon_ancestors':canon_anc_2} + """ + if canon_anc_1 == canon_anc_2: + return {str(org1): 0, str(org1)+'_canon_ancestors':canon_anc_1, + str(org2): 0, str(org2)+'_canon_ancestors':canon_anc_1} + + if canon_anc_1[-1] == org2: + distance1 = 1 + distance2 = 0 + elif canon_anc_2[-1] == org1: + distance1 = 0 + distance2 = 1 + else: + distance1 = -1 + distance2 = -1 + ancestor = self.file_manager.get_common(canon_anc_1, canon_anc_2) + if ancestor == '': + return {str(org1): -1, str(org2): -1, 'reason': 'No common ancestor.'} + + idx_org1 = canon_anc_1.index(ancestor) + idx_org2 = canon_anc_2.index(ancestor) + + if distance1 == -1: + distance1 = len(canon_anc_1) - (idx_org1) + if distance2 == -1: + distance2 = len(canon_anc_2) - (idx_org2) + + return {str(org1): distance1, str(org2): distance2, str(org1)+'_canon_ancestors':canon_anc_1, + str(org2)+'_canon_ancestors':canon_anc_2} + def get_info_by_orthodb(self, orthodb): ''' Find all proteins with the same kegg orthology id. From 7c60b6c73a903e29e205bf28193c6344761ac04b Mon Sep 17 00:00:00 2001 From: Zhouyang Lian Date: Sat, 10 Oct 2020 14:25:06 -0400 Subject: [PATCH 02/10] more aggressive caching of distance obj --- datanator_query_python/query/query_protein.py | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/datanator_query_python/query/query_protein.py b/datanator_query_python/query/query_protein.py index 8af7865..1b9480a 100644 --- a/datanator_query_python/query/query_protein.py +++ b/datanator_query_python/query/query_protein.py @@ -928,13 +928,11 @@ def get_all_ortho(self, ko, anchor, max_distance): projection = { 'orthodb_id': 1, 'orthodb_name': 1, - 'ancestor_name': 1, 'ncbi_taxonomy_id': 1, 'abundances': 1, 'species_name': 1, 'uniprot_id': 1, '_id': 0, - 'ancestor_taxon_id': 1, 'protein_name': 1, 'gene_name': 1, 'modifications': 1, @@ -946,25 +944,34 @@ def get_all_ortho(self, ko, anchor, max_distance): query = {'$and': [con_0, con_1]} docs = self.collection.find(filter=query, projection=projection) queried = deque() + distances = {} names = {} species_anc = {} canon_anc_anchor = self.taxon_col.find_one({"tax_name": anchor})['canon_anc_names'] for doc in docs: doc = json.loads(json.dumps(doc, ignore_nan=True)) species = doc.get('species_name') - if species is None and species not in queried: # few documents don't have species_name field - taxon_id = doc['ncbi_taxonomy_id'] + taxon_id = doc['ncbi_taxonomy_id'] + if species is None and taxon_id not in queried: # few documents don't have species_name field tmp = self.taxon_col.find_one({"tax_id": taxon_id}) species = tmp['tax_name'] canon_anc_species = tmp["canon_anc_names"] queried.append(taxon_id) names[taxon_id] = species species_anc[taxon_id] = canon_anc_species - elif species is None and species in queried: - species = names[doc['ncbi_taxonomy_id']] - canon_anc_species = species_anc[doc['ncbi_taxonomy_id']] - obj = self._get_common_canon_anc(anchor, species, canon_anc_anchor, doc["canon_anc_names"]) - print(obj) + obj = self._get_common_canon_anc(anchor, species, canon_anc_anchor, doc["canon_anc_names"]) + distances[taxon_id] = obj + elif species is not None and taxon_id not in queried: + canon_anc_species = doc["canon_anc_names"] + queried.append(taxon_id) + names[taxon_id] = species + species_anc[taxon_id] = canon_anc_species + obj = self._get_common_canon_anc(anchor, species, canon_anc_anchor, doc["canon_anc_names"]) + distances[taxon_id] = obj + else: + species = names[taxon_id] + canon_anc_species = species_anc[taxon_id] + obj = distances[taxon_id] distance = obj[anchor] if distance != -1 and distance <= max_distance: species_canon_ancestor = obj[species+'_canon_ancestors'] From f1db7f6297f02f1689de1b8b696c24fcc2d0b0a7 Mon Sep 17 00:00:00 2001 From: Zhouyang Lian Date: Sat, 10 Oct 2020 18:41:44 -0400 Subject: [PATCH 03/10] tried to improve result of get_all_ortho by hinting compound sparse index https://github.com/KarrLab/datanator_rest_api/issues/133 --- datanator_query_python/query/query_protein.py | 5 ++++- tests/query/test_query_protein.py | 10 ++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/datanator_query_python/query/query_protein.py b/datanator_query_python/query/query_protein.py index 1b9480a..8f81dd9 100644 --- a/datanator_query_python/query/query_protein.py +++ b/datanator_query_python/query/query_protein.py @@ -1,6 +1,7 @@ from datanator_query_python.util import mongo_util, file_util from datanator_query_python.query import query_taxon_tree, query_kegg_orthology from pymongo.collation import Collation, CollationStrength +from pymongo import ASCENDING from collections import deque import simplejson as json @@ -942,7 +943,9 @@ def get_all_ortho(self, ko, anchor, max_distance): con_0 = {'orthodb_id': ko} con_1 = {'abundances': {'$exists': True}} query = {'$and': [con_0, con_1]} - docs = self.collection.find(filter=query, projection=projection) + docs = self.collection.find(filter=query, projection=projection, + hint=[("orthodb_id", ASCENDING), ("abundances", ASCENDING)], + batch_size=500) queried = deque() distances = {} names = {} diff --git a/tests/query/test_query_protein.py b/tests/query/test_query_protein.py index 214c48b..4303524 100644 --- a/tests/query/test_query_protein.py +++ b/tests/query/test_query_protein.py @@ -280,10 +280,12 @@ def test_get_all_kegg(self): result_0 = self.src_1.get_all_kegg('K00850','Escherichia coli', 10) print(result_0) - @unittest.skip("skipping") + # @unittest.skip("skipping") def test_get_all_ortho(self): - result_0 = self.src_2.get_all_ortho('494933at2759','Escherichia coli', 10) - print(result_0) + # result_0 = self.src_2.get_all_ortho('494933at2759','Escherichia coli', 10) + # print(result_0) + result_1 = self.src_2.get_all_ortho("1555708at2", "Bacillus subtilis subsp. subtilis", 40) + print(result_1) @unittest.skip("passed") def test_get_info_by_orthodb(self): @@ -292,4 +294,4 @@ def test_get_info_by_orthodb(self): def test_get_ortho_by_id(self): result = self.src_2.get_ortho_by_id("P53984") - print(result) \ No newline at end of file + print(result) From 4cbda8ba5f8c76b1ef09c2101ca7c516122daf25 Mon Sep 17 00:00:00 2001 From: Zhouyang Lian Date: Sat, 10 Oct 2020 18:47:47 -0400 Subject: [PATCH 04/10] another compound index hint --- datanator_query_python/query/query_protein.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datanator_query_python/query/query_protein.py b/datanator_query_python/query/query_protein.py index 8f81dd9..444374d 100644 --- a/datanator_query_python/query/query_protein.py +++ b/datanator_query_python/query/query_protein.py @@ -944,7 +944,7 @@ def get_all_ortho(self, ko, anchor, max_distance): con_1 = {'abundances': {'$exists': True}} query = {'$and': [con_0, con_1]} docs = self.collection.find(filter=query, projection=projection, - hint=[("orthodb_id", ASCENDING), ("abundances", ASCENDING)], + hint=[("abundances", ASCENDING), ("orthodb_id", ASCENDING)], batch_size=500) queried = deque() distances = {} From cfdf5471b15eaf3163a585dadbc86ff269772aae Mon Sep 17 00:00:00 2001 From: Zhouyang Lian Date: Mon, 12 Oct 2020 10:55:59 -0400 Subject: [PATCH 05/10] use new field https://github.com/KarrLab/datanator_query_python/issues/17 --- datanator_query_python/query/query_protein.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datanator_query_python/query/query_protein.py b/datanator_query_python/query/query_protein.py index 444374d..b3b0f1f 100644 --- a/datanator_query_python/query/query_protein.py +++ b/datanator_query_python/query/query_protein.py @@ -941,11 +941,11 @@ def get_all_ortho(self, ko, anchor, max_distance): "canon_anc_names": 1 } con_0 = {'orthodb_id': ko} - con_1 = {'abundances': {'$exists': True}} + con_1 = {'abu_exist': True} query = {'$and': [con_0, con_1]} docs = self.collection.find(filter=query, projection=projection, - hint=[("abundances", ASCENDING), ("orthodb_id", ASCENDING)], - batch_size=500) + hint=[("abu_exist", ASCENDING), ("orthodb_id", ASCENDING)], + batch_size=100) queried = deque() distances = {} names = {} @@ -980,7 +980,7 @@ def get_all_ortho(self, ko, anchor, max_distance): species_canon_ancestor = obj[species+'_canon_ancestors'] doc['canon_ancestors'] = species_canon_ancestor result[distance-1]['documents'].append(doc) - return result + return def _get_common_canon_anc(self, org1, org2, canon_anc_1, canon_anc_2): """Get canon common ancestors between species. From 736b2f89921a655576cac6f3c98d4a12bb670cbb Mon Sep 17 00:00:00 2001 From: Zhouyang Lian Date: Mon, 12 Oct 2020 11:00:01 -0400 Subject: [PATCH 06/10] return result --- datanator_query_python/query/query_protein.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datanator_query_python/query/query_protein.py b/datanator_query_python/query/query_protein.py index b3b0f1f..777849f 100644 --- a/datanator_query_python/query/query_protein.py +++ b/datanator_query_python/query/query_protein.py @@ -980,7 +980,7 @@ def get_all_ortho(self, ko, anchor, max_distance): species_canon_ancestor = obj[species+'_canon_ancestors'] doc['canon_ancestors'] = species_canon_ancestor result[distance-1]['documents'].append(doc) - return + return result def _get_common_canon_anc(self, org1, org2, canon_anc_1, canon_anc_2): """Get canon common ancestors between species. From 37a02ee667c9a1878c3cc8c94a3b7b5aa5b4906d Mon Sep 17 00:00:00 2001 From: Zhouyang Lian Date: Fri, 23 Oct 2020 11:27:52 -0400 Subject: [PATCH 07/10] test atlas on entity collection --- .../query_schema_2/ftx_search.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/datanator_query_python/query_schema_2/ftx_search.py b/datanator_query_python/query_schema_2/ftx_search.py index 1df4658..09de963 100644 --- a/datanator_query_python/query_schema_2/ftx_search.py +++ b/datanator_query_python/query_schema_2/ftx_search.py @@ -59,4 +59,60 @@ def search_taxon(self, ) for doc in docs: result.append(doc) + return result + + def search_entity(self, + msg, + path, + skip=0, + limit=10, + token_order="any", + db="datanator-demo"): + """Search for taxon names. + (https://docs.atlas.mongodb.com/reference/atlas-search) + + Args: + msg(:obj:`str`): query message. + path(:obj:`list`): fields to be queried. + skip(:obj:`int`, optional): number of records to skip. + limit(:obj:`int`, optional): max number of documents to return. + token_order(:obj:`str`, optional): token order, i.e. sequential or any. + db(:obj:`str`, optional): name of database in which the result resides. + + Return: + (:obj:`CommandCursor`): MongDB CommandCursor after aggregation. + """ + collection = self.client[db]["taxon_tree"] + result = [] + docs = collection.aggregate([ + { + "$search": { + "queryString": { + "path": path, + "query": msg, + "fuzzy": { + "maxEdits": 2, + "prefixLength": 1, + "maxExpansions": 100 + }, + "tokenOrder": token_order + } + } + }, + { + "$limit": limit + }, + { + "$skip": skip + }, + { + "$project": { + "_id": 0, + "tax_name": 1 + } + } + ] + ) + for doc in docs: + result.append(doc) return result \ No newline at end of file From 6166218913987f6cbf5fffe37c1b238afba5e13c Mon Sep 17 00:00:00 2001 From: Zhouyang Lian Date: Tue, 27 Oct 2020 11:53:27 -0400 Subject: [PATCH 08/10] switching to AWS Elasticsearch --- datanator_query_python/query_schema_2/ftx_search.py | 8 ++++---- tests/query_schema_2/test_ftx_search.py | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/datanator_query_python/query_schema_2/ftx_search.py b/datanator_query_python/query_schema_2/ftx_search.py index 09de963..f9ae855 100644 --- a/datanator_query_python/query_schema_2/ftx_search.py +++ b/datanator_query_python/query_schema_2/ftx_search.py @@ -73,7 +73,7 @@ def search_entity(self, Args: msg(:obj:`str`): query message. - path(:obj:`list`): fields to be queried. + path(:obj:`list` of :obj:`str`): fields to be queried. skip(:obj:`int`, optional): number of records to skip. limit(:obj:`int`, optional): max number of documents to return. token_order(:obj:`str`, optional): token order, i.e. sequential or any. @@ -82,13 +82,13 @@ def search_entity(self, Return: (:obj:`CommandCursor`): MongDB CommandCursor after aggregation. """ - collection = self.client[db]["taxon_tree"] + collection = self.client[db]["enitty"] result = [] docs = collection.aggregate([ { "$search": { "queryString": { - "path": path, + "defaultPath": path, "query": msg, "fuzzy": { "maxEdits": 2, @@ -108,7 +108,7 @@ def search_entity(self, { "$project": { "_id": 0, - "tax_name": 1 + "synonyms": 1 } } ] diff --git a/tests/query_schema_2/test_ftx_search.py b/tests/query_schema_2/test_ftx_search.py index 6dd8319..3d738fa 100644 --- a/tests/query_schema_2/test_ftx_search.py +++ b/tests/query_schema_2/test_ftx_search.py @@ -12,7 +12,14 @@ def setUpClass(cls): def tearDownClass(cls): cls.src.client.close() + @unittest.skip("not working") def test_search_taxon(self): docs = self.src.search_taxon("off", token_order="any") + print(docs) + + def test_search_entity(self): + msg = "2-Ketobutanoic acid" + path = "synonyms" + docs = self.src.search_entity(msg, path) print(docs) \ No newline at end of file From 3cf4ec86c75b6a7a7b5eac0ad82fa61d1d16b82f Mon Sep 17 00:00:00 2001 From: Zhouyang Lian Date: Tue, 27 Oct 2020 11:59:03 -0400 Subject: [PATCH 09/10] query AWS ftx works --- tests/query/test_full_text_query.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/query/test_full_text_query.py b/tests/query/test_full_text_query.py index 0899b56..37a41ef 100644 --- a/tests/query/test_full_text_query.py +++ b/tests/query/test_full_text_query.py @@ -181,4 +181,11 @@ def test_get_genes_orthodb_count(self): agg_field = "orthodb_id" query_message = '1398761at2759' result_0 = self.src.get_genes_orthodb_count(query_message, 15, agg_field=agg_field, size=10, fields=['*']) - print(result_0) \ No newline at end of file + print(result_0) + + def test_simple_query_string_entity(self): + msg = "2-Ketobutanoic acid" + index = "entity" + fields = "synonyms" + r = self.src.simple_query_string(msg, index, fileds=fields) + print(r) \ No newline at end of file From 980fa49afe82ffdebb1f910ccc6c02f7b7e2ba3b Mon Sep 17 00:00:00 2001 From: Zhouyang Lian Date: Tue, 27 Oct 2020 12:01:06 -0400 Subject: [PATCH 10/10] tests simple query string --- tests/query/test_full_text_query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/query/test_full_text_query.py b/tests/query/test_full_text_query.py index 37a41ef..6ff400f 100644 --- a/tests/query/test_full_text_query.py +++ b/tests/query/test_full_text_query.py @@ -188,4 +188,4 @@ def test_simple_query_string_entity(self): index = "entity" fields = "synonyms" r = self.src.simple_query_string(msg, index, fileds=fields) - print(r) \ No newline at end of file + self.assertEqual(r["hits"]["hits"][0]["_id"], "5efb9c4ca8c2d4a8d414d315") \ No newline at end of file