diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a7a20f..92d2a3d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,6 +46,9 @@ jobs: - run: name: Setup Python environment command: | + cd .. + git clone git@github.com:KarrLab/datanator_pattern_design.git + export WHERE=API_TEST if [[ -f .circleci/requirements.txt ]]; then while IFS="" read -r line || [ -n "$line" ]; do if [[ ! -z "$line" ]] && [[ ! "$line" =~ ^# ]]; then diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt index 848281d..f6bbf89 100644 --- a/.circleci/requirements.txt +++ b/.circleci/requirements.txt @@ -1,5 +1,4 @@ # Karr Lab packages git+https://github.com/KarrLab/pkg_utils.git#egg=pkg_utils git+https://github.com/KarrLab/wc_utils.git#egg=wc_utils - git+https://github.com/KarrLab/karr_lab_aws_manager.git#egg=karr_lab_aws_manager diff --git a/.karr_lab_build_utils.yml b/.karr_lab_build_utils.yml index 6e9a790..38876c3 100644 --- a/.karr_lab_build_utils.yml +++ b/.karr_lab_build_utils.yml @@ -1,5 +1,8 @@ static_analyses: ignore_files: [] + ignore_unused_requirements: [] + ignore_missing_requirements: + - configobj downstream_dependencies: - datanator - datanator_rest_api diff --git a/CODE_OF_CONDUCT.MD b/CODE_OF_CONDUCT.MD new file mode 100644 index 0000000..01aeb48 --- /dev/null +++ b/CODE_OF_CONDUCT.MD @@ -0,0 +1,123 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +[info@karrlab.org](mailto:info@karrlab.org). +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/version/2/0/code_of_conduct.html). + +The Community Impact Guidelines were inspired by [Mozilla's Code of Conduct Enforcement ladder](https://github.com/mozilla/diversity). + +Answers to common questions about this code of conduct are available at +https://www.contributor-covenant.org/faq. Translations of this code of conduct are available at +https://www.contributor-covenant.org/translations. diff --git a/CONTRIBUTING.MD b/CONTRIBUTING.MD new file mode 100644 index 0000000..15275dc --- /dev/null +++ b/CONTRIBUTING.MD @@ -0,0 +1,43 @@ +# Contributing to Datanator_query_python + +We enthusiastically welcome contributions to Datanator! + +## Coordinating contributions + +Before getting started, please contact the lead developers at [info@karrlab.org](mailto:info@karrlab.org) to coordinate your planned contributions with other ongoing efforts. Please also use GitHub issues to announce your plans to the community so that other developers can provide input into your plans and coordinate their own work. As the development community grows, we will institute additional infrastructure as needed such as a leadership committee and regular online meetings. + +## Repository organization + +Datanator_query_python follows standard Python conventions: + +* `datanator_query_python/`: source code +* `tests/`: tests +* `docs/`: documentation +* `setup.py`: installation script + +## Coding convention + +Datanator_query_python follows standard Python style conventions: + +* Module names: `lower_snake_case` +* Class names: `UpperCamelCase` +* Function names: `lower_snake_case` +* Variable names: `lower_snake_case` + +## Testing + +We strive to have complete test coverage of Datanator_query_python. As such, all contributions to Datanator should be tested. The tests are located in the `tests` subdirectory. The tests are implemented using the `unittest` module. The tests can be executed by running `pytest tests`. + +Upon each push to GitHub, GitHub will trigger CircleCI to execute all of the tests. + +## Documentation convention + +Datanator is documented using the napoleon Sphinx plugin. The documentation can be compiled by running `sphinx-build docs docs/_build/html`. + +## Submitting changes + +Please use GitHub pull requests to submit changes. Each request should include a brief description of the new and/or modified features. + +## Releasing and deploying new versions + +Contact [info@karrlab.org](mailto:info@karrlab.org) to request release and deployment of new changes. diff --git a/README.md b/README.md index e87dc18..7e3ab41 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[//]: # ( [![PyPI package](https://img.shields.io/pypi/v/datanator_query_python.svg)](https://pypi.python.org/pypi/datanator_query_python) ) +[![PyPI package](https://img.shields.io/pypi/v/datanator_query_python.svg)](https://pypi.python.org/pypi/datanator_query_python) [![Documentation](https://readthedocs.org/projects/datanator-query-python/badge/?version=latest)](https://docs.karrlab.org/datanator_query_python) [![Test results](https://circleci.com/gh/KarrLab/datanator_query_python.svg?style=shield)](https://circleci.com/gh/KarrLab/datanator_query_python) [![Test coverage](https://coveralls.io/repos/github/KarrLab/datanator_query_python/badge.svg)](https://coveralls.io/github/KarrLab/datanator_query_python) @@ -6,37 +6,130 @@ [![License](https://img.shields.io/github/license/KarrLab/datanator_query_python.svg)](LICENSE) ![Analytics](https://ga-beacon.appspot.com/UA-86759801-1/datanator_query_python/README.md?pixel) -# datanator_query_python +# Datanator-query-python +Python query interface of [Datanator](https://datanator.info), a toolkit for discovering the data needed for modeling the biochemistry of cells. -Write an overview +## Contents +* [Overview](#overview) +* [Usage and installation](#usage-and-installation) +* [License](#license) +* [Development team](#development-team) +* [Questions and comments](#questions-and-comments) -## Installation +## Overview +A central goal of synthetic biology is to rationally design organisms. Similarly, a central goal of precision medicine is to tailor therapy to each patient based on their unique genome. Many engineering fields use computer-aided design (CAD) tools driven by mechanistic models to efficiently design complex systems such as planes. Analogously, more comprehensive and more predictive models of biological systems, such as [whole-cell models](https://www.wholecell.org), are needed to help bioengineers and physicians design biomachines and medical therapy. -1. Install dependencies -2. Install the latest release from PyPI +One of the biggest bottlenecks to achieving such models is collecting and aggregating the large amount of data needed for model construction and verification. Due to advances in genomics and increased emphasis on data sharing, there is now extensive data about a wide range of biochemical entities and processes such as data about metabolite concentrations, RNA and protein abundances, and reaction rates. However, it remains difficult to utilize this information for modeling because the data scattered across numerous databases and publications; the data is described using different formats, identifiers, and units; and there are inadequate tools for finding relevant data for modeling a specific biological system in a specific environment. -``` -pip install datanator-query-python -``` +The *Datanator* toolkit seeks to address these problems for biochemical modeling by providing investigators an integrated database of molecular data and tools for discovering relevant data for modeling projects and other meta analyses. Please see the [About](https://datanator.info/about) page for more information about the goals and features of *Datanator*. -3. Install the latest revision from GitHub +The *Datanator* toolkit is composed of the following packages: +- [*Datanator*](https://github.com/KarrLab/datanator): Tools for aggregating and integrating diverse data from diverse sources into a single dataset and searching these datasets +- *Datanator-db*: MongoDB server for *Datanator-data* +- *Datanator-fulltext-db*: ElasticSearch server for *Datanator-data* +- *Datanator-query-python*: This package, tools for querying *Datanator-db* and *Datanator-fulltext-db* +- [*Datanator-rest-api*](https://github.com/KarrLab/datanator_rest_api): REST interface for *Datanator-query-python* +- [*Datanator-frontend*](https://github.com/KarrLab/datanator_frontend): A web-based graphical user interface to *Datanator-db*. -``` -pip install git+https://github.com/KarrLab/datanator_query_python.git#egg=datanator_query_python[all] -``` +This package provides a pythonic interface to programmatically access data stored in *Datanator-db* and *Datanator-fulltext-db*. -## Documentation +## Testing, installation, and usage -Please see the [API documentation](https://docs.karrlab.org/datanator_query_python). +### Users +We recommend using the [REST API](https://api.datanator.info) to programmatically access data stored in *Datanator-db* and *Datanator-fulltext-db*. -## License +### Developers + +#### Install *Datanator-query-python* + +1. Install Python >= 3.7 + ``` + apt-get install python3 + ``` + +2. Install `git` + ``` + apt-get install git + ``` + +3. Clone this repository + ``` + git clone https://github.com/karrlab/datanator_query_python + ``` + +4. Install this package + ``` + cd /path/to/datanator_query_python + pip install -r requirements.txt + pip install -e . + ``` + +5. Setup environment variables (credentials for accessing databases) + ``` + export USERNAME=XXXXXXX + export PASSWORD=XXXXXXX + export SERVER=XXXXXXX + export AUTHDB=XXXXXXX + export READ_PREFERENCE=XXXXXX + ``` -The package is released under the [MIT license](LICENSE). +#### Test *Datanator-query-python* +1. Run the unit tests + ``` + python -m pytest test-unit-coverage + ``` + +### File organization +This repository is organized as follows: + +- `datanator_query_python/`: + - `aggregate/`: aggregation pipelines for MongoDB + - `config/`: datanator_query_python configuration + - `query/`: collection-based query scripts + - `query_schema_2/`: data-type-based query scripts + - `util/`: utility tools +- `tests/` + - `aggregate/` - test scripts in `aggregate` directory + - `config/` - test scripts in `config` directory + - `query/` - test scripts in `query` directory + - `util/` - test scripts in `util` directory +- `CODE_OF_CONDUCT`: code of conduct +- `LICENSE`: license +- `requirements.txt`: package configuration +- `README`: overview of the application + + +## Development and deployment workflow +1. Create a Git new branch +2. Commit code to the new branch +3. Push the branch to GitHub +4. GitHub will automatically trigger [CircleCI](https://circleci.com/gh/KarrLab/datanator_query_python) to run the unit test, integration tests, and other static analyses +5. Use [Coveralls](https://coveralls.io/github/KarrLab/datanator_query_python) to review the coverage of the tests +6. As needed, add additional tests and fix any failing tests +7. Once the new code passes the tests and has high coverage, create a [pull request](https://github.com/KarrLab/datanator_query_python/compare) to merge the new branch into the master branch +7. One of the main developers will review the pull request and request changes as necessary +9. Once any necessary changes have been made, one of the main developers will approve the pull request +10. GitHub will automatically trigger [CircleCI](https://circleci.com/gh/KarrLab/datanator_query_python) to run the unit test, integration tests, and other static analyses for the master branch +11. Once the CircleCI build succeeds, downstream dependencies will automatically be tested and built + +## Contributing to *Datanator* +We welcome contributions to *Datanator* via GitHub pull requests. Please contact the developers to coordinate potential contributions, and please see above for information about how to submit pull requests. + +## License +This package is released under the [MIT license](LICENSE). ## Development team +This package was developed by the [Karr Lab](https://www.karrlab.org) at the Icahn School of Medicine at Mount Sinai in New York by the following individuals: -This package was developed by the [Karr Lab](https://www.karrlab.org) at the Icahn School of Medicine at Mount Sinai in New York, USA. +* [Yosef Roth](https://www.linkedin.com/in/yosef-roth-a80a378a) +* [Yang Lian](https://www.linkedin.com/in/zlian/) +* [Jonathan Karr](https://www.karrlab.org) +* [Saahith Pochiraju](https://www.linkedin.com/in/saahithpochiraju/) +* [Bilal Shaikh](https://www.linkedin.com/in/bilalshaikh42/) +* Balazs Szigeti ## Questions and comments +Please submit an [issue](https://github.com/KarrLab/datanator_query_python/issues/new), or contact the [Karr Lab](info@karrlab.org) with any questions or comments. -Please contact the [Karr Lab](mailto:info@karrlab.org) with any questions or comments. +## Acknowledgements +Datanator was developed with support from the [Center for Reproducible Biomedical Modeling](https://reproduciblebiomodels.org) from the National Institute of Bioimaging and Bioengineering and the National Institute of General Medical Sciences of the National Institutes of Health and the National Science Foundation (awards P41EB023912 and R35GM119771). diff --git a/datanator_query_python/__main__.py b/datanator_query_python/__main__.py index 376ce79..b46252e 100644 --- a/datanator_query_python/__main__.py +++ b/datanator_query_python/__main__.py @@ -7,8 +7,9 @@ """ import cement +from datanator_query_python.util import mongo_util +from datanator_query_python.config import config import datanator_query_python -import datanator_query_python.core class BaseController(cement.Controller): @@ -22,47 +23,83 @@ class Meta: version=datanator_query_python.__version__)), ] - @cement.ex(help='command_1 description') - def cmd1(self): - """ command_1 description """ - print('command_1 output') + @cement.ex(hide=True) + def _default(self): + self._parser.print_help() + - @cement.ex(help='command_2 description') - def cmd2(self): - """ command_2 description """ - print('command_2 output') +class DefineSchema(cement.Controller): + """Karrlab elasticsearch delete index. """ + + class Meta: + label = 'mongo-def-schema' + description = 'Define jsonschema of a collection' + stacked_on = 'base' + stacked_type = 'nested' + arguments = [ + (['db'], dict( + type=str, help='Name of the database in which the collection resides.')), + (['collection'], dict( + type=str, help='Name of the collection to be defined.')), + (['jsonschema'], dict( + type=str, help='Location of jsonschema')), + (['--config_name', '-cn'], dict( + type=str, default='TestConfig', + help='Config class to be used.')) + ] @cement.ex(hide=True) def _default(self): - self._parser.print_help() + ''' Delete elasticsearch index + Args: + index (:obj:`str`): name of index in es + _id (:obj:`int`): id of the doc in index (optional) + ''' + args = self.app.pargs + conf = getattr(config, args.config_name) + mongo_util.MongoUtil(MongoDB=conf.SERVER, + db=args.db, + username=conf.USERNAME, + password=conf.PASSWORD).define_schema(args.collection, args.jsonschema) + print("done") -class Command3WithArgumentsController(cement.Controller): - """ Command3 description """ + +class DefineSchema(cement.Controller): + """Karrlab elasticsearch delete index. """ class Meta: - label = 'command-3' - description = 'Command3 description' + label = 'mongo-def-schema' + description = 'Define jsonschema of a collection' stacked_on = 'base' stacked_type = 'nested' arguments = [ - (['arg_1'], dict( - type=str, help='Description of arg_1')), - (['arg_2'], dict( - type=str, help='Description of arg_2')), - (['--opt-arg-3'], dict( - type=str, default='default value of opt-arg-1', help='Description of opt-arg-3')), - (['--opt-arg-4'], dict( - type=float, default=float('nan'), help='Description of opt-arg-4')), + (['db'], dict( + type=str, help='Name of the database in which the collection resides.')), + (['collection'], dict( + type=str, help='Name of the collection to be defined.')), + (['jsonschema'], dict( + type=str, help='Location of jsonschema')), + (['--config_name', '-cn'], dict( + type=str, default='TestConfig', + help='Config class to be used.')) ] @cement.ex(hide=True) def _default(self): + ''' Delete elasticsearch index + + Args: + index (:obj:`str`): name of index in es + _id (:obj:`int`): id of the doc in index (optional) + ''' args = self.app.pargs - args.arg_1 - args.arg_2 - args.opt_arg_3 - args.opt_arg_4 + conf = getattr(config, args.config_name) + mongo_util.MongoUtil(MongoDB=conf.SERVER, + db=args.db, + username=conf.USERNAME, + password=conf.PASSWORD).define_schema(args.collection, args.jsonschema) + print("done") class App(cement.App): @@ -72,10 +109,13 @@ class Meta: base_controller = 'base' handlers = [ BaseController, - Command3WithArgumentsController, + DefineSchema ] def main(): with App() as app: app.run() + +if __name__=='__main__': + main() \ No newline at end of file diff --git a/datanator_query_python/aggregate/tabu.py b/datanator_query_python/aggregate/tabu.py index b2e9a70..1e92c5b 100644 --- a/datanator_query_python/aggregate/tabu.py +++ b/datanator_query_python/aggregate/tabu.py @@ -54,8 +54,8 @@ def taxon_dist(self, collection, field, match=None, unwind=None): def main(): db = 'datanator' conf = config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER src = Tabu(MongoDB=MongoDB, username=username, password=password, db=db, verbose=True, authSource='admin', diff --git a/datanator_query_python/config/config.py b/datanator_query_python/config/config.py index c70157b..e577e40 100644 --- a/datanator_query_python/config/config.py +++ b/datanator_query_python/config/config.py @@ -47,9 +47,9 @@ class TestConfig(Config): test user with read permission to 'datanator' and readWrite permission to 'test' ''' - MONGO_TEST_USERNAME = os.getenv("MONGO_TEST_USERNAME") - MONGO_TEST_PASSWORD = os.getenv("MONGO_TEST_PASSWORD") - MONGO_TEST_PASSWORD_READ_PREFERENCE = os.getenv("MONGO_TEST_READ_PREFERENCE") + USERNAME = os.getenv("MONGO_TEST_USERNAME") + PASSWORD = os.getenv("MONGO_TEST_PASSWORD") + READ_PREFERENCE = os.getenv("MONGO_TEST_READ_PREFERENCE") class UserAccountConfig(Config): @@ -70,7 +70,7 @@ class AtlasConfig: SERVER = os.getenv("MONGO_ATLAS_SERVER") PORT = os.getenv("MONGO_ATLAS_PORT") REPLSET = os.getenv("MONGO_ATLAS_REPL") - AUTHDB = os.getenv("MONGO_AUTHDB") + AUTHDB = os.getenv("MONGO_ATLAS_AUTHDB") READ_PREFERENCE = os.getenv("MONGO_ATLAS_READPREFERENCE") @@ -79,4 +79,18 @@ class FtxConfig(Config): """ FTX_AWS_PROFILE = os.getenv("FTX_AWS_PROFILE") TEST_FTX_PROFILE_NAME = os.getenv("TEST_FTX_PROFILE_NAME") - REST_FTX_AWS_PROFILE = os.getenv("REST_FTX_AWS_PROFILE") \ No newline at end of file + REST_FTX_AWS_PROFILE = os.getenv("REST_FTX_AWS_PROFILE") + + +class SchemaMigration(Config): + USERNAME=os.getenv('MONGO_DATANATOR_PROD') + PASSWORD=os.getenv('MONGO_DATANATOR_PROD_PASSWORD') + + +class DatanatorTest(Config): + USERNAME=os.getenv('MONGO_DATANATOR_TEST') + PASSWORD=os.getenv('MONGO_DATANATOR_TEST_PASSWORD') + +class DatanatorAdmin(Config): + USERNAME=os.getenv('MONGO_DATANATOR_ADMIN') + PASSWORD=os.getenv('MONGO_DATANATOR_ADMIN_PASSWORD') \ No newline at end of file diff --git a/datanator_query_python/config/motor_client_manager.py b/datanator_query_python/config/motor_client_manager.py new file mode 100644 index 0000000..a932591 --- /dev/null +++ b/datanator_query_python/config/motor_client_manager.py @@ -0,0 +1,17 @@ +from datanator_query_python.util import motor_util +import os +from dotenv import load_dotenv +from pathlib import Path, PurePath + + +home_path = PurePath(Path.home(), '.wc/datanator.env') +root_path = "/.wc/danatator.env" +if os.path.exists(home_path): + dotenv_path = home_path +else: + dotenv_path = root_path +load_dotenv(dotenv_path) + +where = os.getenv("WHERE") # API_TEST; API_PROD; MONGO_DATANATOR_TEST; MONGO_DATANATOR_PROD; +client = motor_util.MotorUtil(username=os.getenv(where), password=os.getenv("{}_PASSWORD".format(where)), + MongoDB=os.getenv("MONGO_ATLAS_SERVER")).client diff --git a/datanator_query_python/config/query_manager.py b/datanator_query_python/config/query_manager.py index d995cb5..cdfc25e 100644 --- a/datanator_query_python/config/query_manager.py +++ b/datanator_query_python/config/query_manager.py @@ -14,9 +14,9 @@ def __init__(self): self.read_preference = config.AtlasConfig.READ_PREFERENCE self.repl = config.AtlasConfig.REPLSET - def protein_manager(self): + def protein_manager(self, database="datanator"): return query_protein.QueryProtein(username=self.username, password=self.password, server=self.server, - authSource=self.authDB, readPreference=self.read_preference, replicaSet=self.repl) + authSource=self.authDB, readPreference=self.read_preference, replicaSet=self.repl, database=database) def metabolite_concentration_manager(self): return query_metabolite_concentrations.QueryMetaboliteConcentrations(MongoDB=self.server, db='datanator', @@ -71,10 +71,10 @@ def metabolites_meta_manager(): class RnaManager: - def rna_manager(self): + def rna_manager(self, db="datanator"): return query_rna_halflife.QueryRNA(username=config.AtlasConfig.USERNAME, password=config.AtlasConfig.PASSWORD, server=config.AtlasConfig.SERVER, authDB=config.AtlasConfig.AUTHDB, readPreference=config.AtlasConfig.READ_PREFERENCE, - db='datanator', collection_str='rna_halflife_new', replicaSet=config.AtlasConfig.REPLSET) + db=db, collection_str='rna_halflife_new', replicaSet=config.AtlasConfig.REPLSET) class KeggManager: diff --git a/datanator_query_python/config/query_schema_2_manager.py b/datanator_query_python/config/query_schema_2_manager.py new file mode 100644 index 0000000..1eba60b --- /dev/null +++ b/datanator_query_python/config/query_schema_2_manager.py @@ -0,0 +1,47 @@ +from datanator_query_python.config import config +from datanator_query_python.util import mongo_util +from pymongo import ReadPreference +from pymongo.collation import Collation, CollationStrength + + +class QM(mongo_util.MongoUtil): + + def __init__(self, + server=config.AtlasConfig.SERVER, + username=config.AtlasConfig.USERNAME, + password=config.AtlasConfig.PASSWORD, + authSource=config.AtlasConfig.AUTHDB, + replicaSet=config.AtlasConfig.REPLSET, + readPreference=config.AtlasConfig.READ_PREFERENCE + ): + super().__init__(MongoDB=server, replicaSet=replicaSet, + username=username, password=password, + authSource=authSource, readPreference=readPreference) + self.read_preference = self._convert_read_p(readPreference) + self.collation = Collation(locale='en', strength=CollationStrength.SECONDARY) + + def _convert_read_p(self, read_preference): + """Convert string read preference to pymongo + + Args: + read_preference(:obj:`str`): env variable in string. + + Return: + (:obj:pymongo.ReadPreference) + """ + if read_preference == 'nearest': + return ReadPreference.NEAREST + elif read_preference == 'primary': + return ReadPreference.PRIMARY + + def conn_protein(self, db): + """Establish connection with protein queries. + + Args: + db(:obj:`str`): name of database. + collection(:obj:`str`): name of collection. + + Return: + (:obj:`pymongo.database.Database`) + """ + return self.client.get_database(name=db, read_preference=self.read_preference) \ No newline at end of file diff --git a/datanator_query_python/core.py b/datanator_query_python/core.py deleted file mode 100644 index 3df9dea..0000000 --- a/datanator_query_python/core.py +++ /dev/null @@ -1,47 +0,0 @@ -""" datanator_query_python - -:Author: Name -:Date: 2019-8-26 -:Copyright: 2019, Karr Lab -:License: MIT -""" - - -class ExampleClass(object): - """ Descipton of ExampleClass - - Attributes: - attr_1 (:obj:`type of attr_1`): description of attr_1 - attr_2 (:obj:`type of attr_2`): description of attr_2 - ... - """ - - def __init__(self, arg_1, arg_2, kwarg_1=None, kwarg_2=None): - """ - Args: - arg_1 (:obj:`type of arg_1`): description of arg_1 - arg_2 (:obj:`type of arg_2`): description of arg_2 - kwarg_1 (:obj:`type of kwarg_1`, optional): description of kwarg_1 - kwarg_2 (:obj:`type of kwarg_2`, optional): description of kwarg_2 - ... - """ - self.attr_1 = arg_1 - self.attr_2 = arg_2 - - def method_1(self, arg_1, arg_2, kwarg_1=None, kwarg_2=None): - """ Description of method_1 - - Args: - arg_1 (:obj:`type of arg_1`): description of arg_1 - arg_2 (:obj:`type of arg_2`): description of arg_2 - kwarg_1 (:obj:`type of kwarg_1`, optional): description of kwarg_1 - kwarg_2 (:obj:`type of kwarg_2`, optional): description of kwarg_2 - ... - - Returns: - :obj:`type of return value`: description of return value - - Raises: - :obj:`type of raised exception(s)`: description of raised exceptions - """ - pass diff --git a/datanator_query_python/query/full_text_search.py b/datanator_query_python/query/full_text_search.py index d51a01c..f8c7df2 100644 --- a/datanator_query_python/query/full_text_search.py +++ b/datanator_query_python/query/full_text_search.py @@ -126,20 +126,29 @@ def get_num_source(self, q, q_index, index, fields=['name', 'synonyms'], iteration += 1 return result - def get_single_index_count(self, q, index, num, **kwargs): + def get_single_index_count(self, q, index, num, + excludes=[], includes=[], + **kwargs): """Get single index up to num hits Args: q (:obj:`str`): query message index (:obj:`str`): index in which query will be performed num (:obj:`int`): number of hits needed + includes(:obj:`list` of :obj:`str`): list of fields to be included in the data returned. + excludes(:obj:`list` of :obj:`str`): list of fields to be excluded from the data returned. Return: (:obj:`dict`): obj of index hits {'index': []} """ result = {} result[index] = [] - body = self.build_simple_query_string_body(q, **kwargs) + _source = {} + if excludes != []: + _source["includes"] = includes + if includes != []: + _source["excludes"] = excludes + body = self.build_simple_query_string_body(q, _source=_source, **kwargs) from_ = kwargs.get('from_', 0) r = self.build_es().search(index=index, body=body, size=num, from_=from_) hits = r['hits']['hits'] @@ -187,7 +196,7 @@ def get_index_ko_count(self, q, num, agg_field="frontend_gene_aggregate", index= "aggs": { "top_ko": { "top_hits": {'_source': {'includes': ['ko_number', 'ko_name', 'protein_name', 'definition', agg_field, - 'species_name']}, 'size': 1} + 'species_name', "orthodb_name", "orthodb_id", "uniprot_id"]}, 'size': 1} }, "top_hit" : { "max": { @@ -219,7 +228,7 @@ def get_index_ko_count(self, q, num, agg_field="frontend_gene_aggregate", index= # result[index].append(hit['_source']) # return result - def get_genes_ko_count(self, q, num, agg_field="frontend_gene_aggregate", **kwargs): + def get_genes_ko_count(self, q, num, agg_field="ko_number", **kwargs): """Get protein index with different ko_number field for up to num hits, provided at least one of the proteins under ko_number has abundance info. @@ -284,7 +293,7 @@ def get_genes_ko_count(self, q, num, agg_field="frontend_gene_aggregate", **kwar body['aggs'] = aggregation body['size'] = 0 r = self.build_es().search(index=index, body=body) - r_all = self.get_index_ko_count(q, num * 2, agg_field, index=index, **kwargs) + r_all = self.get_index_ko_count(q, num * 2, agg_field=agg_field, index=index, **kwargs) ko_abundance = set() ko_all = set() for i, s in enumerate(r['aggregations']['top_kos']['buckets']): @@ -330,4 +339,91 @@ def get_rxn_oi(self, query_message, minimum_should_match=0, from_=0, for hit in hits: hit['_source']['_score'] = hit['_score'] result['sabio_rk'].append(hit['_source']) - return result \ No newline at end of file + return result + + def get_genes_orthodb_count(self, q, num, agg_field="orthodb_id.keyword", **kwargs): + """Get protein index with different ko_number field for up to num hits, + provided at least one of the proteins under orthodb_id has abundance info. + + Args: + q (:obj:`str`): query message. + num (:obj:`int`): number of hits needed. + agg_field (:obj:`str`): field to be aggregated. + **from_ (:obj:`int`): starting offset (default: 0). + + Return: + (:obj:`dict`): obj of index hits {'index': []} + """ + result = {} + index = 'genes' + must_not = {"bool": { + "must_not": { + "exists": { + "field": agg_field + } + } + }} + aggregation = { + "top_kos": { + "terms": { + "field": agg_field, + "order": { + "top_hit": "desc" + }, + "missing": "N/A", + "size": kwargs.get('from_', 0) + num + }, + "aggs": { + "top_ko": { + "top_hits": {'_source': {'includes': ['orthodb_id', 'orthodb_name', 'protein_name', 'definition', agg_field, + 'species_name', "uniprot_id"]}, "size": 1} + }, + "top_hit" : { + "max": { + "script": { + "source": "_score" + } + } + }, + "score_bucket_sort": { + "bucket_sort":{ + "sort": [{"top_hit.value": {"order": "desc"}}], + "size": num, + "from": kwargs.get('from_', 0) + } + } + } + }, + "total_buckets": {'cardinality': {'field': agg_field, "missing": "N/A"}} + } + result[index] = [] + sqs_body = self.build_simple_query_string_body(q, **kwargs) + must = sqs_body['query'] + must = [must] + # must.append({"exists": {"field": "abundances"}}) + # body = self.build_bool_query_body(must=must, must_not=must_not) + body = self.build_bool_query_body(must=must) + body['aggs'] = aggregation + body['size'] = 0 + r = self.build_es().search(index=index, body=body) + r_all = self.get_index_ko_count(q, num * 2, agg_field=agg_field, index=index, **kwargs) + ko_abundance = set() + ko_all = set() + # for i, s in enumerate(r['aggregations']['top_kos']['buckets']): + # r['aggregations']['top_kos']['buckets'][i]['key'] = [s['key'][i:i+6] for i in range(0, len(s['key']), 6)] + for bucket_abundance in r['aggregations']['top_kos']['buckets']: + ko_abundance.add(bucket_abundance['top_ko']['hits']['hits'][0]['_source'].get(agg_field)) + + for bucket_all in r_all['top_kos']['buckets']: + ko_all.add(bucket_all['top_ko']['hits']['hits'][0]['_source'].get(agg_field)) + intersects = ko_abundance.intersection(ko_all) + for s in r['aggregations']['top_kos']['buckets']: + ko_str = s['top_ko']['hits']['hits'][0]['_source'].get(agg_field) # ko_str can be "K01234,K12345" + # if ko_str in intersects and ko_str != 'nan': + if ko_str is None: + # # s['top_ko']['hits']['hits'][0]['_source']['abundances'] = True + # s['top_ko']['hits']['hits'][0]['_source'][agg_field] = [ko_str[i:i+6] for i in range(0, len(ko_str), 6)] + # else: + # s['top_ko']['hits']['hits'][0]['_source']['abundances'] = False + s['top_ko']['hits']['hits'][0]['_source'][agg_field] = ["N/A"] + return r['aggregations'] \ No newline at end of file diff --git a/datanator_query_python/query/query_kegg_orthology.py b/datanator_query_python/query/query_kegg_orthology.py index 655dd17..886110a 100644 --- a/datanator_query_python/query/query_kegg_orthology.py +++ b/datanator_query_python/query/query_kegg_orthology.py @@ -14,6 +14,7 @@ def __init__(self, username=None, password=None, server=None, authSource='admin' self.max_entries = max_entries self.verbose = verbose self.client, self.db, self.collection = self.con_db('kegg_orthology') + self.ortho = self.db_obj["orthodb"] self.collation = Collation(locale='en', strength=CollationStrength.SECONDARY) def get_ko_by_name(self, name): @@ -97,6 +98,29 @@ def get_meta_by_kegg_ids(self, kegg_ids, projection={'_id': 0, 'gene_ortholog': count = self.collection.count_documents(query, collation=self.collation) return docs, count + def get_meta_by_ortho_ids(self, orthodb_ids, projection={'_id': 0, 'gene_ortholog': 0}, + limit=0): + """Get meta given kegg ids + + Args: + orthodb_ids (:obj:`list` of :obj:`str`): List of orthodb ids. + projection (:obj:`dict`): MongoDB result projection. + + Return: + (:obj:`tuple` of :obj:`pymongo.Cursor` and :obj:`int`): pymongo Cursor obj and number of documents found. + """ + projection['__order'] = 0 + query = {'orthodb_id': {'$in': orthodb_ids}} + pipeline = [ + {'$match': {'orthodb_id': {'$in': orthodb_ids}}}, + {'$addFields': {"__order": {'$indexOfArray': [orthodb_ids, "$orthodb_id" ]}}}, + {'$sort': {"__order": 1}}, + {"$project": projection} + ] + docs = self.ortho.aggregate(pipeline) + count = self.ortho.count_documents(query) + return docs, count + def get_meta_by_kegg_id(self, kegg_id): """Get meta information by kegg_id diff --git a/datanator_query_python/query/query_metabolite_concentrations.py b/datanator_query_python/query/query_metabolite_concentrations.py index 545f34a..2c6a8bc 100644 --- a/datanator_query_python/query/query_metabolite_concentrations.py +++ b/datanator_query_python/query/query_metabolite_concentrations.py @@ -27,32 +27,36 @@ def get_similar_concentrations(self, metabolite, threshold=0.6): (:obj:`list` of :obj:`Obj`): [{'inchikey': xxxx, 'similarity_score': ..., 'concentrations': []}] """ result = [] - meta_collection = self.db_obj['metabolites_meta'] + meta_collection = self.client.get_database("datanator-test")['metabolites_meta'] doc = meta_collection.find_one(filter={'InChI_Key': metabolite}, projection={'similar_compounds': 1}, collation=self.collation) if not doc: return result - obj = self.file_manager.merge_dict(doc.get('similar_compounds')) - inchikeys = list(obj.keys()) - inchikeys.reverse() - values = list(obj.values()) - values.reverse() - threshold_index = np.searchsorted(np.asarray(values), threshold, side='left') - r_inchikeys = inchikeys[threshold_index:] # relevant inchikeys + r_inchikeys = [] + scores = [] + for obj in doc["similar_compounds"]: + if obj["similarity_score"] >= threshold: + r_inchikeys.append(obj["inchikey"]) + scores.append(obj["similarity_score"]) + else: + break pipeline = [ {"$match": {"inchikey": {"$in": r_inchikeys}}}, {"$addFields": {"__order": {"$indexOfArray": [r_inchikeys, "$inchikey" ]}}}, {"$sort": {"__order": -1}} ] docs = self._collection.aggregate(pipeline) - for doc in docs: + if not docs: + return result + for i, doc in enumerate(docs): inchikey = doc['inchikey'] name = doc['metabolite'] result.append({'inchikey': inchikey, - 'similarity_score': obj[inchikey], + 'similarity_score': scores[i], 'metabolite': name, 'concentrations': doc['concentrations']}) + print(inchikey) return result def get_conc_count(self): @@ -65,4 +69,16 @@ def get_conc_count(self): pipeline = [project, group] docs = self._collection.aggregate(pipeline) for doc in docs: - return doc['total'] \ No newline at end of file + return doc['total'] + + def get_conc_by_taxon(self, _id): + """Get concentrations by ncbi taxonomy ID. + + Args: + _id(:obj:`int`): NCBI Taxonomy ID. + + Return: + (:obj:`Pymongo.Cursor`) + """ + query = {"concentrations.ncbi_taxonomy_id": _id} + return self._collection.find(filter=query) diff --git a/datanator_query_python/query/query_metabolites_meta.py b/datanator_query_python/query/query_metabolites_meta.py index ebc5bc9..063667a 100644 --- a/datanator_query_python/query/query_metabolites_meta.py +++ b/datanator_query_python/query/query_metabolites_meta.py @@ -17,7 +17,7 @@ def __init__(self, cache_dirname=None, MongoDB=None, replicaSet=None, db=None, verbose=verbose, max_entries=max_entries, username=username, password=password, authSource=authSource, readPreference=readPreference) - self._collection = self.db_obj[collection_str] + self._collection = self.client.get_database("datanator-test")[collection_str] self.e_client, self.e_db_obj, self.e_collection = self.con_db('ecmdb') self.y_client, self.y_db_obj, self.y_collection = self.con_db('ymdb') self.file_manager = file_util.FileUtil() @@ -188,62 +188,6 @@ def get_metabolite_name_by_hash(self, compounds): # result.append(['None']) return [x[-1] for x in result] - def get_metabolite_similar_compounds(self, compounds, num=0, threshold=0): - ''' Given a list of compound names - Return the top num number of similar compounds - with tanimoto score above threshold values - Args: - compounds: list of compound names - num: number of similar compounds to return - threshold: threshold tanimoto coefficient value - return_format: return dictionary key format, either - hashed inchi or name - Return: - result: list of similar compounds and their tanimoto score - [ {'compound1': score, 'compound2': score, ... 'compound_num': score}, - {'compound1': score, 'compound2': score, ... 'compound_num': score}, ...] - compound(1 - n) will be in name format - raw: list of similar compounds and their tanimoto score - [ {'compound1': score, 'compound2': score, ... 'compound_num': score}, - {'compound1': score, 'compound2': score, ... 'compound_num': score}, ...] - compound(1 - n) will be in hashed_inchi format - ''' - result = [] - raw = [] - hashed_inchi = self.get_metabolite_hashed_inchi(compounds) - projection = {'_id': 0, 'similar_compounds': 1} - - for item in hashed_inchi: - cursor = self._collection.find_one(filter={'InChI_Key': item}, - projection=projection) - if cursor is None: - continue - compounds = cursor['similar_compounds'] - scores = [list(dic.values()) for dic in compounds] - scores = self.file_manager.unpack_list(scores) - hashes = [list(dic.keys()) for dic in compounds] - hashes = self.file_manager.unpack_list(hashes) - names = self.get_metabolite_name_by_hash(hashes) - # convert to numpy object for faster calculations - scores_np = np.asarray(scores) - indices = np.nonzero(scores_np >= threshold) - size = indices[0].size - if size == 0: - raw.append({'raw': -1}) - result.append({'result': -1}) - elif 0 < size < num: - first_size = compounds[:size] - raw = first_size - replaced = self.file_manager.make_dict(names[:size], scores[:size]) - result.append(replaced) - else: - first_num = compounds[:num] - raw = first_num - replaced = self.file_manager.make_dict(names[:num], scores[:num]) - result.append(replaced) - - return raw, result - def get_unique_metabolites(self): """Get number of unique metabolites. diff --git a/datanator_query_python/query/query_protein.py b/datanator_query_python/query/query_protein.py index dbb679a..777849f 100644 --- a/datanator_query_python/query/query_protein.py +++ b/datanator_query_python/query/query_protein.py @@ -1,6 +1,7 @@ from datanator_query_python.util import mongo_util, file_util from datanator_query_python.query import query_taxon_tree, query_kegg_orthology from pymongo.collation import Collation, CollationStrength +from pymongo import ASCENDING from collections import deque import simplejson as json @@ -16,6 +17,7 @@ def __init__(self, username=None, password=None, server=None, authSource='admin' readPreference=readPreference, replicaSet=replicaSet) self.taxon_manager = query_taxon_tree.QueryTaxonTree(MongoDB=server, username=username, password=password, authSource=authSource, db=database, replicaSet=replicaSet) + self.taxon_col = self.db_obj['taxon_tree'] self.kegg_manager = query_kegg_orthology.QueryKO(username=username, password=password, server=server, authSource=authSource, replicaSet=replicaSet) self.file_manager = file_util.FileUtil() self.max_entries = max_entries @@ -64,6 +66,35 @@ def get_meta_by_id(self, _id): result.append(doc) return result + def get_ortho_by_id(self, _id): + ''' + Get protein's metadata given uniprot id + + Args: + _id (:obj:`str`): uniprot id. + + Returns: + (:obj:`list` of :obj:`dict`): list of information. + ''' + result = [] + query = {'uniprot_id': _id} + doc = self.collection.find_one(filter=query, projection={"_id": 0}) + if doc is None: + return {'uniprot_id': 'None', + 'entry_name': 'None', + 'gene_name': 'None', + 'protein_name': 'None', + 'canonical_sequence': 'None', + 'length': 99999999, + 'mass': '99999999', + 'abundances': [], + 'ncbi_taxonomy_id': 99999999, + 'species_name': '99999999'} + else: + doc = json.loads(json.dumps(doc, ignore_nan=True)) + result.append(doc) + return result + def get_meta_by_name_taxon(self, name, taxon_id): ''' Get protein's metadata given protein name @@ -869,4 +900,149 @@ def get_all_kegg(self, ko, anchor, max_distance): species_canon_ancestor = obj[species+'_canon_ancestors'] doc['canon_ancestors'] = species_canon_ancestor result[distance-1]['documents'].append(doc) + return result + + def get_all_ortho(self, ko, anchor, max_distance): + '''Get replacement abundance value by taxonomic distance + with the same OrthoDB group number. + + Args: + ko (:obj:`str`): OrthoDB group id to query for. + anchor (:obj:`str`): anchor species' name. + max_distance (:obj:`int`): max taxonomic distance from origin protein allowed for + proteins in results. + max_depth (:obj:`int`) max depth allowed from the common node. + + Returns: + (:obj:`list` of :obj:`dict`): list of result proteins and their info + [ + {'distance': 1, 'documents': [{}, {}, {} ...]}, + {'distance': 2, 'documents': [{}, {}, {} ...]}, ...]. + ''' + if max_distance <= 0: + return 'Please use get_abundance_by_id to check self abundance values' + + result = [] + for i in range(max_distance): + result.append({'distance': i + 1, 'documents': []}) + + projection = { + 'orthodb_id': 1, + 'orthodb_name': 1, + 'ncbi_taxonomy_id': 1, + 'abundances': 1, + 'species_name': 1, + 'uniprot_id': 1, + '_id': 0, + 'protein_name': 1, + 'gene_name': 1, + 'modifications': 1, + "canon_anc_ids": 1, + "canon_anc_names": 1 + } + con_0 = {'orthodb_id': ko} + con_1 = {'abu_exist': True} + query = {'$and': [con_0, con_1]} + docs = self.collection.find(filter=query, projection=projection, + hint=[("abu_exist", ASCENDING), ("orthodb_id", ASCENDING)], + batch_size=100) + queried = deque() + distances = {} + names = {} + species_anc = {} + canon_anc_anchor = self.taxon_col.find_one({"tax_name": anchor})['canon_anc_names'] + for doc in docs: + doc = json.loads(json.dumps(doc, ignore_nan=True)) + species = doc.get('species_name') + taxon_id = doc['ncbi_taxonomy_id'] + if species is None and taxon_id not in queried: # few documents don't have species_name field + tmp = self.taxon_col.find_one({"tax_id": taxon_id}) + species = tmp['tax_name'] + canon_anc_species = tmp["canon_anc_names"] + queried.append(taxon_id) + names[taxon_id] = species + species_anc[taxon_id] = canon_anc_species + obj = self._get_common_canon_anc(anchor, species, canon_anc_anchor, doc["canon_anc_names"]) + distances[taxon_id] = obj + elif species is not None and taxon_id not in queried: + canon_anc_species = doc["canon_anc_names"] + queried.append(taxon_id) + names[taxon_id] = species + species_anc[taxon_id] = canon_anc_species + obj = self._get_common_canon_anc(anchor, species, canon_anc_anchor, doc["canon_anc_names"]) + distances[taxon_id] = obj + else: + species = names[taxon_id] + canon_anc_species = species_anc[taxon_id] + obj = distances[taxon_id] + distance = obj[anchor] + if distance != -1 and distance <= max_distance: + species_canon_ancestor = obj[species+'_canon_ancestors'] + doc['canon_ancestors'] = species_canon_ancestor + result[distance-1]['documents'].append(doc) + return result + + def _get_common_canon_anc(self, org1, org2, canon_anc_1, canon_anc_2): + """Get canon common ancestors between species. + + Args: + org1(:obj:`str`): Name of species 1. + org2(:obj:`str`): Name of species 2. + canon_anc_1(:obj:`list` of `str`): List of canon ancestor names of species 1. + canon_anc_2(:obj:`list` of `str`): List of canon ancestor names of species 2. + + Return: + (:obj:`Obj`): {str(org1): distance1, str(org2): distance2, str(org1)+'_canon_ancestors':canon_anc_1, + str(org2)+'_canon_ancestors':canon_anc_2} + """ + if canon_anc_1 == canon_anc_2: + return {str(org1): 0, str(org1)+'_canon_ancestors':canon_anc_1, + str(org2): 0, str(org2)+'_canon_ancestors':canon_anc_1} + + if canon_anc_1[-1] == org2: + distance1 = 1 + distance2 = 0 + elif canon_anc_2[-1] == org1: + distance1 = 0 + distance2 = 1 + else: + distance1 = -1 + distance2 = -1 + ancestor = self.file_manager.get_common(canon_anc_1, canon_anc_2) + if ancestor == '': + return {str(org1): -1, str(org2): -1, 'reason': 'No common ancestor.'} + + idx_org1 = canon_anc_1.index(ancestor) + idx_org2 = canon_anc_2.index(ancestor) + + if distance1 == -1: + distance1 = len(canon_anc_1) - (idx_org1) + if distance2 == -1: + distance2 = len(canon_anc_2) - (idx_org2) + + return {str(org1): distance1, str(org2): distance2, str(org1)+'_canon_ancestors':canon_anc_1, + str(org2)+'_canon_ancestors':canon_anc_2} + + def get_info_by_orthodb(self, orthodb): + ''' + Find all proteins with the same kegg orthology id. + + Args: + orthodb(:obj:`str`): kegg orthology ID. + + Returns: + (:obj:`list` of :obj:`dict`): list of dictionary containing + protein's uniprot_id and kegg information + [{'orthodb_id': ... 'orthodb_name': ... 'uniprot_ids': []}, + {'orthodb_id': ... 'orthodb_name': ... 'uniprot_ids': []}]. + ''' + ko = orthodb.lower() + result = [{'orthodb_id': ko, 'uniprot_ids': []}] + query = {'orthodb_id': ko} + projection = {'uniprot_id': 1, '_id': 0, 'orthodb_name': 1, 'orthodb_id': 1} + docs = self.collection.find(filter=query, projection=projection) + + for doc in docs: + result[0]['orthodb_name'] = doc.get('orthodb_name', ['no name']) + result[0]['uniprot_ids'].append(doc.get('uniprot_id')) return result \ No newline at end of file diff --git a/datanator_query_python/query/query_rna_halflife.py b/datanator_query_python/query/query_rna_halflife.py index e002c32..f31786f 100644 --- a/datanator_query_python/query/query_rna_halflife.py +++ b/datanator_query_python/query/query_rna_halflife.py @@ -70,3 +70,24 @@ def get_doc_by_ko(self, ko_number, projection={'_id': 0}, skip=_from, limit=size) count = self.collection.count_documents(query) return docs, count + + def get_doc_by_orthodb(self, orthodb, projection={'_id': 0}, + _from=0, size=0): + """Get documents by orthodb group ID. + + Args: + orthodb (:obj:`str`): Orthodb group ID. + projection (:obj:`dict`, optional): mongodb query result + projection. Defaults to {'_id': 0}. + _from (:obj:`int`): first page (0-indexed). + size (:obj:`int`): number of items per page. + + Return: + (:obj:`tuple` of :obj:`Pymongo.Cursor` and :obj:`int`): + pymongo interable and number of documents. + """ + query = {'orthodb_id': orthodb} + docs = self.collection.find(filter=query, projection=projection, + skip=_from, limit=size) + count = self.collection.count_documents(query) + return docs, count \ No newline at end of file diff --git a/datanator_query_python/query/query_sabiork_old.py b/datanator_query_python/query/query_sabiork_old.py index d47f9d1..3654f61 100644 --- a/datanator_query_python/query/query_sabiork_old.py +++ b/datanator_query_python/query/query_sabiork_old.py @@ -20,6 +20,7 @@ def __init__(self, cache_dirname=None, MongoDB=None, replicaSet=None, db='datana replicaSet=replicaSet, db=db, verbose=verbose, max_entries=max_entries, username=username, password=password, authSource=authSource, readPreference=readPreference) + self.u = self.client["datanator-test"]["uniprot"] self.chem_manager = chem_util.ChemUtil() self.file_manager = file_util.FileUtil() self.collection = self.db_obj[collection_str] @@ -172,6 +173,79 @@ def get_kinlaw_by_rxn(self, substrates, products, dof=0, count = self.collection.count_documents(query) return count, docs + def get_kinlaw_by_rxn_ortho(self, substrates, products, dof=0, + projection={'kinlaw_id': 1, '_id': 0, "enzymes": 1}, + bound='loose', skip=0, limit=0): + ''' Find the kinlaw_id defined in sabio_rk using + rxn participants' inchikey + + Args: + substrates (:obj:`list`): list of substrates' inchikey + products (:obj:`list`): list of products' inchikey + dof (:obj:`int`, optional): degree of freedom allowed (number of parts of + inchikey to truncate); the default is 0 + projection (:obj:`dict`): pymongo query projection + bound (:obj:`str`): limit substrates/products to include only input values + + Return: + (:obj:`list` of :obj:`dict`): list of kinlaws that satisfy the condition + ''' + substrate = 'reaction_participant.substrate_aggregate' + product = 'reaction_participant.product_aggregate' + if dof == 0: + substrates = substrates + products = products + elif dof == 1: + substrates = [re.compile('^' + x[:-2]) for x in substrates] + products = [re.compile('^' + x[:-2]) for x in products] + else: + substrates = [re.compile('^' + x[:14]) for x in substrates] + products = [re.compile('^' + x[:14]) for x in products] + + if bound == 'loose': + constraint_0 = {substrate: {'$all': substrates}} + constraint_1 = {product: {'$all': products}} + constraint_2 = {"taxon_id": {"$ne": None}} + else: + constraint_0 = {substrate: substrates} + constraint_1 = {product: products} + constraint_2 = {"taxon_id": {"$ne": None}} + query = {'$and': [constraint_0, constraint_1, constraint_2]} + # lookup = lookups.Lookups().simple_lookup("kegg_orthology", "resource.id", "definition.ec_code", "kegg_meta") + # if limit > 0: + # pipeline = [{"$match": query}, {"$limit": limit}, {"$skip": skip}, lookup, {"$project": projection}] + # else: + # pipeline = [{"$match": query}, {"$skip": skip}, lookup, {"$project": projection}] + docs = self.collection.find(filter=query, + limit=limit, + skip=skip, + projection=projection) + cache = {} + result = [] + for doc in docs: + try: + u_id = doc["enzymes"][2]["subunit"][0]["uniprot_id"] + if u_id is not None: + if cache.get(u_id) is None: + x = self.u.find_one(filter={"uniprot_id": u_id}, + projection={"orthodb_id": 1, + "orthodb_name": 1}) + cache[u_id] = x + else: + x = cache.get(u_id) + doc["orthodb_id"] = x["orthodb_id"] + doc["orthodb_name"] = x["orthodb_name"] + else: + doc["orthodb_id"] = None + doc["orthodb_name"] = None + except: + doc["orthodb_id"] = None + doc["orthodb_name"] = None + doc.pop("enzymes", None) + result.append(doc) + count = self.collection.count_documents(query) + return count, result + def get_kinlaw_by_entryid(self, entry_id): """Find reactions by sabio entry id diff --git a/datanator_query_python/query/query_taxon_tree.py b/datanator_query_python/query/query_taxon_tree.py index 66387dc..a95ad0d 100644 --- a/datanator_query_python/query/query_taxon_tree.py +++ b/datanator_query_python/query/query_taxon_tree.py @@ -12,7 +12,7 @@ class QueryTaxonTree(mongo_util.MongoUtil): def __init__(self, cache_dirname=None, collection_str='taxon_tree', verbose=False, max_entries=float('inf'), username=None, MongoDB=None, - password=None, db='datanator', authSource='admin', readPreference='nearest', + password=None, db='datanator-test', authSource='admin', readPreference='nearest', replicaSet=None): self.collection_str = collection_str super().__init__(cache_dirname=cache_dirname, MongoDB=MongoDB, @@ -280,6 +280,8 @@ def get_canon_rank_distance_by_name(self, name, front_end=False): anc = self.collection.find_one(filter=query, projection={'anc_id':1, 'tax_name': 1, 'anc_name': 1}, collation=self.collation) result = [] + if anc is None: + return [{name: 0}] if front_end: result.append({anc['tax_name']: 0}) for i, tax_id in enumerate(reversed(anc['anc_id'])): @@ -413,6 +415,40 @@ def get_canon_common_ancestor_fast(self, org1, org2, org_format='tax_id'): Return: (:obj:`Obj`) ''' - anchor_org = self.collection.find_one({org_format: org1}, projection={'canon_anc_ids': 1, 'canon_anc_names': 1}) - pipeline = self.pipeline_manager.aggregate_common_canon_ancestors(anchor_org, org1, intersect_name="ancMatch") - return self.collection.aggregate(pipeline) \ No newline at end of file + if org1 is None or org2 is None: + return {'reason': 'Needs two organisms.'} + collection = self.client["datanator-test"]["taxon_tree"] + doc_1 = collection.find_one({org_format: org1}, projection={'canon_anc_ids': 1, 'canon_anc_names': 1}) + doc_2 = collection.find_one({org_format: org2}, projection={'canon_anc_ids': 1, 'canon_anc_names': 1}) + if doc_1 is None or doc_2 is None: + return {str(org1): -1, str(org2): -1, 'reason': 'No organism found.'} + + canon_anc_1 = doc_1["canon_anc_names"] + canon_anc_2 = doc_2["canon_anc_names"] + + if canon_anc_1 == canon_anc_2: + return {str(org1): 0, str(org1)+'_canon_ancestors':canon_anc_1} + + if canon_anc_1[-1] == org2: + distance1 = 1 + distance2 = 0 + elif canon_anc_2[-1] == org1: + distance1 = 0 + distance2 = 1 + else: + distance1 = -1 + distance2 = -1 + ancestor = self.file_manager.get_common(canon_anc_1, canon_anc_2) + if ancestor == '': + return {str(org1): -1, str(org2): -1, 'reason': 'No common ancestor.'} + + idx_org1 = canon_anc_1.index(ancestor) + idx_org2 = canon_anc_2.index(ancestor) + + if distance1 == -1: + distance1 = len(canon_anc_1) - (idx_org1) + if distance2 == -1: + distance2 = len(canon_anc_2) - (idx_org2) + + return {str(org1): distance1, str(org2): distance2, str(org1)+'_canon_ancestors':canon_anc_1, + str(org2)+'_canon_ancestors':canon_anc_2} \ No newline at end of file diff --git a/datanator_query_python/query_schema_2/__init__.py b/datanator_query_python/query_schema_2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/datanator_query_python/query_schema_2/ftx_search.py b/datanator_query_python/query_schema_2/ftx_search.py new file mode 100644 index 0000000..1df4658 --- /dev/null +++ b/datanator_query_python/query_schema_2/ftx_search.py @@ -0,0 +1,62 @@ +from datanator_query_python.config import query_schema_2_manager +from pymongo import MongoClient +from pymongo import TEXT + + +class FTX(query_schema_2_manager.QM): + def __init__(self): + super().__init__() + + def search_taxon(self, + msg, + skip=0, + limit=10, + token_order="any", + db="datanator-test"): + """Search for taxon names. + (https://docs.atlas.mongodb.com/reference/atlas-search) + + Args: + msg(:obj:`str`): query message. + skip(:obj:`int`, optional): number of records to skip. + limit(:obj:`int`, optional): max number of documents to return. + token_order(:obj:`str`, optional): token order, i.e. sequential or any. + db(:obj:`str`, optional): name of database in which the result resides. + + Return: + (:obj:`CommandCursor`): MongDB CommandCursor after aggregation. + """ + collection = self.client[db]["taxon_tree"] + result = [] + docs = collection.aggregate([ + { + "$search": { + "autocomplete": { + "path": "tax_name", + "query": msg, + "fuzzy": { + "maxEdits": 2, + "prefixLength": 1, + "maxExpansions": 100 + }, + "tokenOrder": token_order + } + } + }, + { + "$limit": limit + }, + { + "$skip": skip + }, + { + "$project": { + "_id": 0, + "tax_name": 1 + } + } + ] + ) + for doc in docs: + result.append(doc) + return result \ No newline at end of file diff --git a/datanator_query_python/query_schema_2/query_entity.py b/datanator_query_python/query_schema_2/query_entity.py new file mode 100644 index 0000000..1c60d7a --- /dev/null +++ b/datanator_query_python/query_schema_2/query_entity.py @@ -0,0 +1,41 @@ +from datanator_query_python.config import query_schema_2_manager + + +class QueryEn(query_schema_2_manager.QM): + def __init__(self, + db="datanator-demo"): + super().__init__() + self.db = db + + def query_entity(self, + identifier, + datatype="metabolite", + collection="entity", + limit=10, + skip=0, + projection={"_id": 0}): + """Get entity with identifier. + + Args: + identifier(:obj:`Obj`): identifier used for the entity. + datatype(:obj:`Obj`, optional): Datatype to be retrieved. + collection(:obj:`str`): name of the collection in which data resides. + limit(:obj:`int`, optional): number of results to return. + skip(:obj:`int`, optional): number of documents to skip. + projection(:obj:`Obj`, optional): MongoDB projection. + + Return: + (:obj:`list`): pymongo iterables. + """ + col = self.client[self.db][collection] + con_0 = {"identifiers": {"$elemMatch": identifier}} + con_1 = {"type": datatype} + query = {"$and": [con_0, con_1]} + result = [] + docs = col.find(filter=query, + limit=limit, + skip=skip, + projection=projection) + for doc in docs: + result.append(doc) + return result \ No newline at end of file diff --git a/datanator_query_python/query_schema_2/query_observation.py b/datanator_query_python/query_schema_2/query_observation.py new file mode 100644 index 0000000..a0241de --- /dev/null +++ b/datanator_query_python/query_schema_2/query_observation.py @@ -0,0 +1,52 @@ +from datanator_query_python.config import query_schema_2_manager +from pymongo import ASCENDING + + +class QueryObs(query_schema_2_manager.QM): + def __init__(self, + db="datanator-demo"): + super().__init__() + self.db = db + + def get_entity_datatype(self, + identifier, + entity="protein", + datatype="half-life", + collection="observation", + limit=10, + skip=0, + projection={"_id": 0}): + """Get entity datatype. + + Args: + identifier(:obj:`Obj`): identifier used for the entity. + entity(:obj:`Obj`, optional): entity type. i.e. "protein", "RNA", etc. + datatype(:obj:`Obj`, optional): Datatype to be retrieved. + collection(:obj:`str`, optional): name of collection in which values reside. + limit(:obj:`int`, optional): number of results to return. + skip(:obj:`int`, optional): number of documents to skip. + + Return: + (:obj:`list`): pymongo iterables. + """ + results = [] + col = self.client[self.db][collection] + con_0 = {"entity.type": entity} + con_1 = {} + if entity == "protein" and datatype != "localization": + con_1["values.type"] = datatype + elif entity == "protein" and datatype == "localization": + words = ["intramembrane_localization", "secretome location"] + con_1["values.type"] = {"$in": words} + elif entity == "RNA" and datatype == "localization": + con_1["values.type"] = "subcellular_localization" + query = {"$and": [{"identifier": identifier}, con_0, con_1]} + docs = col.find(filter=query, limit=limit, skip=skip, + collation=self.collation, + projection=projection, + hint=[("identifier", ASCENDING), + ("entity.type", ASCENDING), + ("values.type", ASCENDING)]) + for doc in docs: + results.append(doc) + return results diff --git a/datanator_query_python/query_schema_2/query_taxon_tree_v2.py b/datanator_query_python/query_schema_2/query_taxon_tree_v2.py new file mode 100644 index 0000000..bea2029 --- /dev/null +++ b/datanator_query_python/query_schema_2/query_taxon_tree_v2.py @@ -0,0 +1,68 @@ +from datanator_query_python.config import motor_client_manager +from collections import deque + + +class QTaxon: + + def __init__(self, + collection='taxon_tree', + db='datanator-test', + max_entries=float('inf')): + self.collection = motor_client_manager.client.get_database(db)[collection] + + async def get_canon_ancestor(self, _id, _format='tax_id'): + """Get organism's canon ancestor information. + + Args: + _id(:obj:`int` or :obj:`str`): identification for organism. + format(:obj:`str`, optional): identification format, i.e. tax_id or tax_name. + + Return: + (:obj:`tuple` of :obj:`list` of :obj:`int` and :obj:`list` of :obj:`str`): canon_anc_id, canon_anc_name + """ + query = {_format: _id} + doc = await self.collection.find_one(filter=query, + projection={"canon_anc_names": 1, + "canon_anc_ids": 1}) + if doc is None: + return [], [] + else: + return doc.get("canon_anc_ids", []), doc.get("canon_anc_names", []) + + async def aggregate_distance(self, docs, target_org, _format='tax_id', + name_field='species_name'): + """Aggregate distance information between organisms in documents and target + organism. + e.g. 1) org_0 has ancestors [a, b, c, d, e]; target_org has ancestors + [a, b, c, f, g, h] -> org_0_dist = 2 (e,d), target_org_dist = 3 (h,g,f). + 2) org_0 has ancestors [a, b, c, d, e]; target_org is d -> target_org_dist=0. + + Args: + docs(:obj:`list` of :obj:`Obj`): list of docs to be compared with target org. + target_org (:obj:`int` or :obj:`str`): identification of the target organism. + _format (:obj:`str`, optional): identification format. Defaults to 'tax_id'. + name_field (:obj:`str`, optional): Field where species name is in each doc. Defaults to 'species_name'. + + Return: + (:obj:`list` of :obj:`Obj`): Objects containing information on distance to target_org. + """ + # find target organism document + target_org_doc = await self.collection.find_one(filter={_format: target_org}, + projection={"canon_anc_ids": 1, "canon_anc_names": 1, + "tax_name": 1}) + if target_org_doc is None: + return docs + else: + target_org_name = target_org_doc['tax_name'] + target_org_anc_ids = target_org_doc['canon_anc_ids'] + target_org_anc_ids_set = set(target_org_anc_ids) + target_org_anc_names = target_org_doc['canon_anc_names'] + for doc in docs: + measured = doc['canon_anc_ids'] + intersect = len(set(measured).intersection(target_org_anc_ids_set)) + measured_dist = len(measured) - intersect + target_dist = len(target_org_anc_ids) - intersect + doc['taxon_distance'] = {doc[name_field]: measured_dist, + target_org_name: target_dist, + "{}_canon_ancestors".format(target_org_name): target_org_anc_names} + return docs \ No newline at end of file diff --git a/datanator_query_python/query_schema_2/query_uniprot_v2.py b/datanator_query_python/query_schema_2/query_uniprot_v2.py new file mode 100644 index 0000000..490521b --- /dev/null +++ b/datanator_query_python/query_schema_2/query_uniprot_v2.py @@ -0,0 +1,8 @@ +from datanator_query_python.config import query_schema_2_manager + + +class QueryUniprot: + + def __init__(self, db=None, collection=None): + self.db_obj = query_schema_2_manager.QM().conn_protein(db) + self.collection = self.db_obj[collection] \ No newline at end of file diff --git a/datanator_query_python/util/mongo_util.py b/datanator_query_python/util/mongo_util.py index 4650027..ba9ec46 100644 --- a/datanator_query_python/util/mongo_util.py +++ b/datanator_query_python/util/mongo_util.py @@ -1,6 +1,7 @@ import pymongo from pymongo.read_preferences import Primary, PrimaryPreferred, Secondary, SecondaryPreferred, Nearest import copy +import json from genson import SchemaBuilder @@ -77,4 +78,124 @@ def get_duplicates(self, collection_str, _key, **kwargs): if i == 0: num = count["total_return"] return num, collection.aggregate(pipeline, **kwargs) - \ No newline at end of file + + def define_schema(self, collection_str, json_schema): + """Define collection's $jsonSchema + (https://docs.mongodb.com/manual/reference/operator/query/jsonSchema/) + + Args: + collection_str (:obj:`str`): Name of the collection to be defined. + json_schema (:obj:`str` or :obj:`dict`): location of the jsonSchema definition. + """ + if isinstance(json_schema, str): + with open(json_schema) as j: + schema = json.load(j) + else: + schema = json_schema + self.db_obj.create_collection(collection_str, + validator={"$jsonSchema": schema}, + validationLevel="moderate") + + def update_observation(self, + obj, + source, + db="test", + op="update", + col="observation", + query={}): + """Update observation collection + + Args: + obj(:obj:`Obj`): obs object to be updated with. + source(:obj:`Obj`): one of the sources used for matching. + db(:obj:`Obj`): Name of database to be updated. + op(:obj:`str`): Operation to be done. + col(:obj:`str`): Name of collection to be updated. + query(:obj:`Obj`): Filter for updating operation. + """ + _set = {} + add_to_set = {} + for key, val in obj.items(): + if isinstance(val, list): + add_to_set[key] = {"$each": val} + else: + _set[key] = val + if query == {}: + query = {"$and": [{"identifier": _set["identifier"]}, + {"source": {"$elemMatch": source}}]} + _update = {"$set": _set, + "$addToSet": add_to_set} + if op == "update": + self.client[db][col].update_one(query, + _update, + upsert=True) + else: + return query, _update + + def update_entity(self, + obj, + match, + db="test", + col="entity", + op="update", + query={}): + """Update entity collection. + + Args: + obj(:obj:`Obj`): object to be updated with. + match(:obj:`Obj`): Identifier used to match existing document in collection. + db(:obj:`str`): Name of database to be updated. + col(:obj:`str`): Name of collection to be updated. + op(:obj:`str`): operation to be done, e.g. update or bulk. + query(:obj:`Obj`): Filter for updating operation. + """ + _set = {} + add_to_set = {} + for key, val in obj.items(): + if isinstance(val, list): + add_to_set[key] = {"$each": val} + else: + _set[key] = val + if query == {}: + query = {"identifiers": {"$elemMatch": match}} + _update = {"$set": _set, + "$addToSet": add_to_set} + if op == "update": + self.client[db][col].update_one(query, + _update, + upsert=True) + else: + return query, _update + + def build_taxon_object(self, _id, _format="tax_id"): + """Build taxon object from taxon_id. + (https://github.com/KarrLab/datanator_pattern_design/blob/master/components/taxon.json) + + Args: + _id (:obj:`int`): Organism's taxonomy ID. + + Return: + (:obj:`Obj`) + """ + if _format == "tax_name": + query = {"$or": [{"tax_name": _id}, {"name_txt": _id}]} + else: + query = {_format: _id} + obj = self.client["datanator-test"]["taxon_tree"].find_one(query, + projection={"canon_anc_ids": 1, + "canon_anc_names": 1, + "tax_name": 1, + "tax_id": 1}) + if obj is None: + return {} + else: + canon_ancestors = [] + canon_anc_ids = obj["canon_anc_ids"] + canon_anc_names = obj["canon_anc_names"] + for _id, name in zip(canon_anc_ids, canon_anc_names): + canon_ancestors.append({"ncbi_taxonomy_id": _id, + "name": name}) + return {"ncbi_taxonomy_id": obj["tax_id"], + "name": obj["tax_name"], + "canon_ancestors": canon_ancestors} + \ No newline at end of file diff --git a/datanator_query_python/util/motor_util.py b/datanator_query_python/util/motor_util.py new file mode 100644 index 0000000..552978e --- /dev/null +++ b/datanator_query_python/util/motor_util.py @@ -0,0 +1,10 @@ +import motor.motor_asyncio + + +class MotorUtil: + + def __init__(self, MongoDB=None, replicaSet=None, db='test', + verbose=False, max_entries=float('inf'), username=None, + password=None, authSource='admin', readPreference='nearest'): + string = "mongodb+srv://{}:{}@{}/{}?authSource={}&retryWrites=true&w=majority&readPreference={}".format(username, password, MongoDB, db, authSource, readPreference) + self.client = motor.motor_asyncio.AsyncIOMotorClient(string) \ No newline at end of file diff --git a/docs/taxon_distribution.json b/docs/taxon_distribution.json index 056de00..ddc1d95 100644 --- a/docs/taxon_distribution.json +++ b/docs/taxon_distribution.json @@ -1 +1,1032 @@ -{"9606": 51775, "10090": 41123, "3702": 14247, "10116": 12124, "511145": 11788, "559292": 11359, "272623": 10983, "188937": 10979, "83333": 8317, "562": 7875, "9913": 5814, "580240": 5492, "83332": 4973, "284812": 4602, "224308": 3925, "4932": 3658, "246196": 3644, "7227": 3600, "6239": 3553, "222523": 2948, "9031": 2463, "7955": 2222, "44689": 1998, "226900": 1817, "9823": 1705, "99287": 1363, "1423": 1275, "4577": 1231, "9986": 964, "208964": 910, "158878": 781, "9615": 632, "85962": 598, "1111708": 598, "1351": 529, "39947": 527, "632": 514, "3562": 510, "1360": 450, "287": 449, "9940": 415, "211586": 407, "301447": 400, "1582": 397, "64091": 381, "267671": 351, "90371": 341, "1396": 333, "283166": 330, "1422": 320, "449447": 303, "192222": 301, "1773": 296, "303": 290, "4113": 281, "10036": 264, "1280": 259, "5691": 258, "274": 252, "169963": 249, "8022": 244, "5693": 243, "4081": 237, "882": 231, "4513": 226, "3888": 219, "9796": 217, "3847": 215, "8364": 213, "1309": 206, "1359": 205, "122586": 205, "1314": 204, "40324": 203, "272624": 199, "5833": 199, "214684": 198, "4530": 195, "10141": 191, "5759": 186, "243159": 184, "226186": 181, "2190": 179, "550": 174, "3988": 167, "3055": 165, "1421": 165, "4565": 164, "593117": 161, "1590": 154, "2261": 153, "615": 151, "573": 143, "5664": 140, "12721": 138, "546414": 137, "817": 136, "7165": 126, "405212": 122, "2287": 121, "727": 120, "1409": 119, "34903": 115, "630": 112, "1313": 112, "5306": 111, "1358": 110, "5061": 110, "1402": 106, "11320": 103, "2336": 100, "294": 100, "293": 99, "8030": 99, "3879": 94, "45596": 92, "5811": 91, "1585": 91, "306": 90, "5141": 88, "1502": 86, "425011": 85, "2210": 85, "119602": 83, "1143": 81, "1718": 80, "266": 80, "45354": 80, "69488": 78, "210": 76, "145262": 76, "10298": 74, "4896": 73, "818": 73, "464": 71, "1506": 70, "10029": 70, "8839": 70, "4641": 69, "358": 69, "1833": 68, "1667": 67, "4097": 66, "6253": 66, "1245": 66, "37591": 66, "3708": 66, "56812": 65, "1580": 65, "12353": 65, "1931": 64, "5702": 61, "4550": 60, "39291": 60, "4558": 59, "157791": 58, "1583": 58, "37329": 58, "354": 57, "4228": 57, "5416": 56, "585": 56, "5476": 56, "4043": 56, "556": 56, "11520": 55, "1467": 55, "395598": 55, "5079": 55, "1392": 54, "1354": 54, "7460": 53, "1579": 52, "1352": 52, "5341": 52, "5518": 51, "426": 51, "1598": 49, "2234": 49, "582": 49, "311400": 49, "285": 49, "5689": 49, "56636": 48, "813": 48, "17": 48, "7225": 47, "1462": 47, "185431": 46, "28039": 46, "28985": 45, "1427": 45, "4547": 44, "548": 44, "8665": 44, "1398": 43, "106590": 43, "266265": 43, "435": 42, "51663": 42, "3704": 42, "192": 41, "27292": 41, "1901": 41, "48868": 40, "1488": 40, "5323": 40, "5270": 40, "1587": 39, "9544": 39, "5286": 39, "1061": 38, "1772": 38, "51453": 38, "3617": 38, "11788": 38, "3873": 37, "644": 37, "9542": 37, "588": 37, "33950": 37, "1390": 37, "2271": 37, "6183": 37, "8355": 36, "3870": 36, "161934": 36, "13689": 36, "1063": 36, "1394": 36, "44688": 36, "546": 35, "162425": 35, "292": 35, "3039": 34, "238": 34, "5908": 34, "542": 34, "2111": 34, "666": 34, "28116": 33, "32044": 33, "4540": 33, "1814": 33, "1831": 32, "1881": 32, "155077": 32, "487": 32, "53953": 31, "529": 31, "28450": 30, "82310": 30, "316": 30, "4039": 29, "1935": 29, "42879": 29, "10665": 29, "1393": 29, "4754": 29, "5692": 29, "1397": 29, "2246": 28, "480": 28, "353153": 27, "5482": 27, "87883": 27, "905": 27, "76772": 27, "106592": 27, "4227": 27, "47917": 27, "2024": 26, "1471": 26, "5507": 26, "2265": 26, "105351": 26, "203404": 26, "584": 26, "1167": 26, "1911": 25, "3885": 25, "9541": 25, "81473": 25, "33940": 25, "4058": 25, "8400": 25, "230624": 24, "5661": 24, "8078": 24, "3750": 24, "119219": 24, "56690": 23, "4498": 23, "3728": 23, "2783": 23, "203805": 23, "1721": 23, "4076": 23, "3710": 23, "452646": 23, "3649": 22, "4045": 22, "28038": 22, "6421": 22, "536": 22, "8932": 22, "1131": 21, "511": 21, "8403": 21, "4690": 21, "9103": 21, "339": 21, "1906": 21, "40674": 20, "574": 20, "36873": 20, "160400": 20, "5656": 20, "5331": 20, "86665": 19, "747": 19, "270110": 19, "1902": 19, "1492": 19, "7797": 19, "227859": 19, "32046": 19, "9685": 19, "93496": 19, "746128": 19, "3659": 19, "1591": 18, "139": 18, "9770": 18, "9521": 18, "8649": 18, "375": 18, "4120": 18, "7091": 18, "2104": 18, "1302": 18, "30326": 18, "47884": 18, "350": 17, "9925": 17, "29320": 17, "63559": 17, "2180": 17, "1752": 17, "5665": 17, "1629": 17, "1589": 17, "984": 17, "37632": 17, "158": 17, "15819": 17, "3916": 17, "3726": 17, "4102": 17, "39955": 16, "1270": 16, "38902": 16, "38313": 16, "1613": 16, "2102": 16, "33899": 16, "10368": 16, "3880": 16, "381": 16, "51936": 16, "2725": 15, "51288": 15, "7962": 15, "2714": 15, "5535": 15, "29438": 15, "1528": 15, "29760": 15, "1247": 15, "1305": 15, "8717": 15, "4026": 15, "1180": 15, "42684": 14, "2286": 14, "489": 14, "4079": 14, "671": 14, "6192": 14, "10376": 14, "2303": 14, "212035": 14, "2148": 14, "33008": 14, "1553": 14, "3767": 14, "5062": 13, "5807": 13, "202950": 13, "3818": 13, "3906": 13, "3570": 13, "35749": 13, "5722": 13, "13773": 13, "1428": 13, "28896": 13, "2208": 13, "8019": 13, "7159": 13, "2252": 13, "3886": 13, "5860": 13, "1085": 13, "93934": 13, "382": 13, "1308": 13, "2337": 13, "59899": 13, "879": 13, "109": 13, "2045": 13, "29158": 12, "46611": 12, "646": 12, "10117": 12, "329": 12, "7108": 12, "43306": 12, "9548": 12, "119072": 12, "161492": 12, "111955": 12, "4060": 12, "595605": 12, "55126": 12, "4808": 12, "150120": 12, "921": 12, "6850": 12, "1069": 12, "160": 12, "6210": 12, "42881": 12, "3860": 12, "228": 12, "1665": 11, "7998": 11, "80866": 11, "1404": 11, "837": 11, "5490": 11, "58172": 11, "73239": 11, "4472": 11, "1426": 11, "180675": 11, "1494": 11, "5063": 10, "4839": 10, "39350": 10, "76905": 10, "197": 10, "8467": 10, "5044": 10, "36599": 10, "3469": 10, "7067": 10, "28229": 10, "28901": 10, "43263": 10, "1520": 10, "438": 10, "7168": 10, "1076": 10, "8018": 10, "35889": 10, "1255": 10, "29363": 10, "111292": 10, "169388": 10, "3918": 10, "471": 10, "2162": 10, "4922": 10, "1765": 10, "3663": 10, "36903": 9, "1701": 9, "3627": 9, "1166": 9, "5666": 9, "2209": 9, "11176": 9, "33959": 9, "13335": 9, "3917": 9, "39534": 9, "262320": 9, "571": 9, "10506": 9, "33154": 9, "1681": 9, "5541": 9, "1829": 9, "3856": 9, "8933": 9, "29058": 9, "1496": 9, "1049": 9, "7602": 9, "847": 9, "1304": 9, "8017": 9, "52153": 9, "5076": 8, "3864": 8, "129134": 8, "91979": 8, "59405": 8, "85698": 8, "4108": 8, "225486": 8, "34029": 8, "1335": 8, "85549": 8, "183674": 8, "1328": 8, "2115": 8, "84158": 8, "68825": 8, "29370": 8, "150436": 8, "29494": 8, "235357": 8, "980415": 8, "6282": 8, "67780": 8, "2285": 8, "5544": 8, "34072": 8, "165200": 8, "3662": 8, "1888": 8, "298683": 8, "55127": 8, "1720": 8, "1544": 8, "7787": 8, "41984": 8, "669": 8, "46351": 7, "1299": 7, "319": 7, "3656": 7, "479": 7, "54254": 7, "7604": 7, "19": 7, "2203": 7, "3435": 7, "5671": 7, "9483": 7, "3352": 7, "20406": 7, "2097": 7, "4232": 7, "2214": 7, "730": 7, "5477": 7, "28446": 7, "659": 7, "85439": 7, "5400": 7, "3712": 7, "37931": 7, "1264": 7, "554": 7, "1708": 7, "1395": 7, "193462": 7, "850": 7, "2098": 6, "1769": 6, "5127": 6, "1517": 6, "103355": 6, "55588": 6, "334771": 6, "2262": 6, "6279": 6, "317": 6, "29883": 6, "78167": 6, "6941": 6, "520": 6, "1757": 6, "94694": 6, "204998": 6, "101473": 6, "5741": 6, "71721": 6, "1445": 6, "1640": 6, "408": 6, "280": 6, "99325": 6, "235": 6, "5111": 6, "59406": 6, "34573": 6, "305": 6, "3693": 6, "971": 6, "6622": 6, "1912": 6, "65409": 6, "186125": 6, "414": 6, "3349": 6, "1503": 6, "4072": 6, "6978": 6, "5763": 6, "154": 6, "4931": 6, "46429": 6, "5657": 6, "2224": 6, "4265": 6, "5791": 6, "9598": 5, "1311": 5, "5053": 5, "4233": 5, "6956": 5, "1728": 5, "90370": 5, "3991": 5, "4226": 5, "57043": 5, "27717": 5, "3442": 5, "42068": 5, "80865": 5, "14": 5, "9734": 5, "1599": 5, "46348": 5, "2762": 5, "77214": 5, "9543": 5, "1604": 5, "5858": 5, "5855": 5, "9378": 5, "37435": 5, "1164": 5, "1866": 5, "71712": 5, "290576": 5, "42043": 5, "33203": 5, "69367": 5, "70796": 5, "36330": 5, "6233": 5, "13489": 5, "40602": 5, "3332": 5, "29923": 5, "2095": 4, "4093": 4, "63363": 4, "362787": 4, "33119": 4, "70868": 4, "76114": 4, "1176": 4, "768495": 4, "43960": 4, "51351": 4, "474": 4, "97093": 4, "7782": 4, "5530": 4, "7959": 4, "9825": 4, "569628": 4, "47914": 4, "173": 4, "29850": 4, "9838": 4, "143350": 4, "84": 4, "1525": 4, "28564": 4, "5082": 4, "920": 4, "37334": 4, "1515": 4, "3486": 4, "3077": 4, "138853": 4, "2277": 4, "289672": 4, "2074": 4, "40384": 4, "89018": 4, "4442": 4, "33931": 4, "1172": 4, "11676": 4, "57975": 4, "3981": 4, "152480": 4, "11308": 4, "40549": 4, "1232": 4, "33903": 4, "1341": 4, "1922": 4, "1512": 4, "8697": 4, "3914": 4, "549": 4, "49338": 4, "29549": 4, "3112": 3, "1593": 3, "7375": 3, "40559": 3, "35798": 3, "5315": 3, "2746": 3, "7213": 3, "58923": 3, "1509": 3, "35525": 3, "11191": 3, "3625": 3, "65004": 3, "64609": 3, "82076": 3, "8023": 3, "83263": 3, "580": 3, "47881": 3, "373": 3, "8005": 3, "216465": 3, "587753": 3, "746889": 3, "29541": 3, "272636": 3, "7740": 3, "215440": 3, "60093": 3, "3884": 3, "2096": 3, "3565": 3, "668559": 3, "6715": 3, "20416": 3, "37734": 3, "49040": 3, "5051": 3, "633": 3, "1516": 3, "73492": 3, "40337": 3, "34305": 3, "1511": 3, "12917": 3, "5532": 3, "118562": 3, "87645": 3, "35833": 3, "1281": 3, "380174": 3, "213664": 3, "663": 3, "216816": 3, "29303": 3, "1006352": 3, "38080": 3, "63186": 3, "6565": 3, "359303": 3, "94850": 3, "2133": 2, "722438": 2, "50422": 2, "11104": 2, "188": 2, "272131": 2, "29780": 2, "28448": 2, "50053": 2, "107673": 2, "2021": 2, "34364": 2, "8782": 2, "33111": 2, "4075": 2, "120086": 2, "10245": 2, "3823": 2, "93385": 2, "3874": 2, "3827": 2, "46147": 2, "1932": 2, "51631": 2, "8056": 2, "29347": 2, "13676": 2, "88233": 2, "28442": 2, "1639": 2, "48935": 2, "9054": 2, "10879": 2, "82655": 2, "71647": 2, "985": 2, "6637": 2, "3654": 2, "644337": 2, "89184": 2, "55190": 2, "49450": 2, "8795": 2, "59788": 2, "1913": 2, "47519": 2, "90397": 2, "36778": 2, "1501": 2, "1455": 2, "130081": 2, "6182": 2, "1914": 2, "3583": 2, "11033": 2, "5821": 2, "6185": 2, "1677": 2, "29449": 2, "1306": 2, "41200": 2, "1267": 2, "37636": 2, "7010": 2, "33074": 2, "3707": 2, "83558": 2, "8239": 2, "31216": 2, "4924": 2, "79685": 2, "2334": 2, "1310": 2, "271": 2, "51953": 2, "13780": 2, "8267": 2, "29172": 2, "8386": 2, "2094": 2, "3747": 2, "143416": 2, "31958": 2, "6596": 2, "4846": 2, "41492": 2, "186153": 2, "43687": 2, "3467": 2, "57622": 2, "392815": 2, "114909": 2, "48386": 2, "31605": 2, "3711": 2, "187304": 2, "74111": 2, "139456": 2, "34": 2, "1303": 2, "39152": 2, "243964": 2, "6536": 2, "8801": 2, "2434": 2, "221569": 2, "4096": 2, "145263": 2, "1548": 2, "29301": 2, "195615": 2, "275": 2, "6550": 2, "3983": 2, "29377": 2, "4224": 2, "182262": 2, "4934": 2, "1685": 2, "3690": 2, "66219": 2, "5911": 1, "7160": 1, "7729": 1, "11866": 1, "1596": 1, "5690": 1, "12130": 1, "86029": 1, "29674": 1, "172714": 1, "7054": 1, "1408": 1, "2771": 1, "45465": 1, "3755": 1, "29523": 1, "9333": 1, "1497": 1, "1989": 1, "2107": 1, "41951": 1, "281": 1, "40353": 1, "5698": 1, "33040": 1, "1184": 1, "9327": 1, "58217": 1, "121769": 1, "881": 1, "47715": 1, "1836": 1, "3760": 1, "36841": 1, "12923": 1, "8139": 1, "40505": 1, "1153": 1, "121428": 1, "29139": 1, "102803": 1, "1588": 1, "33178": 1, "62337": 1, "104098": 1, "1818": 1, "206025": 1, "33996": 1, "116725": 1, "9316": 1, "8032": 1, "4555": 1, "198467": 1, "36095": 1, "135523": 1, "2181": 1, "1131482": 1, "146786": 1, "70864": 1, "1719": 1, "52862": 1, "89462": 1, "10255": 1, "332626": 1, "5699": 1, "169036": 1, "60961": 1, "92630": 1, "10030": 1, "388408": 1, "101201": 1, "33910": 1, "1491": 1, "9317": 1, "2001": 1, "9519": 1, "158080": 1, "114": 1, "223": 1, "2711": 1, "159": 1, "124763": 1, "34582": 1, "8790": 1, "2986": 1, "4556": 1, "55062": 1, "49545": 1, "13677": 1, "362597": 1, "1581": 1, "102642": 1, "74940": 1, "1468": 1, "55125": 1, "4567": 1, "1655": 1, "59729": 1, "644346": 1, "89585": 1, "29722": 1, "9337": 1, "3406": 1, "45834": 1, "2320": 1, "240818": 1, "1307": 1, "155680": 1, "36651": 1, "3660": 1, "3635": 1, "1079": 1, "38527": 1, "148305": 1, "5207": 1, "52861": 1, "137544": 1, "1231": 1, "1": 0} \ No newline at end of file +{ + "1": 0, + "14": 5, + "17": 48, + "19": 7, + "34": 2, + "84": 4, + "109": 13, + "114": 1, + "139": 18, + "154": 6, + "158": 17, + "159": 1, + "160": 12, + "173": 4, + "188": 2, + "192": 41, + "197": 10, + "210": 76, + "223": 1, + "228": 12, + "235": 6, + "238": 34, + "266": 80, + "271": 2, + "274": 252, + "275": 2, + "280": 6, + "281": 1, + "285": 49, + "287": 449, + "292": 35, + "293": 99, + "294": 100, + "303": 290, + "305": 6, + "306": 90, + "316": 30, + "317": 6, + "319": 7, + "329": 12, + "339": 21, + "350": 17, + "354": 57, + "358": 69, + "373": 3, + "375": 18, + "381": 16, + "382": 13, + "408": 6, + "414": 6, + "426": 51, + "435": 42, + "438": 10, + "464": 71, + "471": 10, + "474": 4, + "479": 7, + "480": 28, + "487": 32, + "489": 14, + "511": 21, + "520": 6, + "529": 31, + "536": 22, + "542": 34, + "546": 35, + "548": 44, + "549": 4, + "550": 174, + "554": 7, + "556": 56, + "562": 7875, + "571": 9, + "573": 143, + "574": 20, + "580": 3, + "582": 49, + "584": 26, + "585": 56, + "588": 37, + "615": 151, + "630": 112, + "632": 514, + "633": 3, + "644": 37, + "646": 12, + "659": 7, + "663": 3, + "666": 34, + "669": 8, + "671": 14, + "727": 120, + "730": 7, + "747": 19, + "813": 48, + "817": 136, + "818": 73, + "837": 11, + "847": 9, + "850": 7, + "879": 13, + "881": 1, + "882": 231, + "905": 27, + "920": 4, + "921": 12, + "971": 6, + "984": 17, + "985": 2, + "1049": 9, + "1061": 38, + "1063": 36, + "1069": 12, + "1076": 10, + "1079": 1, + "1085": 13, + "1131": 21, + "1143": 81, + "1153": 1, + "1164": 5, + "1166": 9, + "1167": 26, + "1172": 4, + "1176": 4, + "1180": 15, + "1184": 1, + "1231": 1, + "1232": 4, + "1245": 66, + "1247": 15, + "1255": 10, + "1264": 7, + "1267": 2, + "1270": 16, + "1280": 259, + "1281": 3, + "1299": 7, + "1302": 18, + "1303": 2, + "1304": 9, + "1305": 15, + "1306": 2, + "1307": 1, + "1308": 13, + "1309": 206, + "1310": 2, + "1311": 5, + "1313": 112, + "1314": 204, + "1328": 8, + "1335": 8, + "1341": 4, + "1351": 529, + "1352": 52, + "1354": 54, + "1358": 110, + "1359": 205, + "1360": 450, + "1390": 37, + "1392": 54, + "1393": 29, + "1394": 36, + "1395": 7, + "1396": 333, + "1397": 29, + "1398": 43, + "1402": 106, + "1404": 11, + "1408": 1, + "1409": 119, + "1421": 165, + "1422": 320, + "1423": 1275, + "1426": 11, + "1427": 45, + "1428": 13, + "1445": 6, + "1455": 2, + "1462": 47, + "1467": 55, + "1468": 1, + "1471": 26, + "1488": 40, + "1491": 1, + "1492": 19, + "1494": 11, + "1496": 9, + "1497": 1, + "1501": 2, + "1502": 86, + "1503": 6, + "1506": 70, + "1509": 3, + "1511": 3, + "1512": 4, + "1515": 4, + "1516": 3, + "1517": 6, + "1520": 10, + "1525": 4, + "1528": 15, + "1544": 8, + "1548": 2, + "1553": 14, + "1579": 52, + "1580": 65, + "1581": 1, + "1582": 397, + "1583": 58, + "1585": 91, + "1587": 39, + "1588": 1, + "1589": 17, + "1590": 154, + "1591": 18, + "1593": 3, + "1596": 1, + "1598": 49, + "1599": 5, + "1604": 5, + "1613": 16, + "1629": 17, + "1639": 2, + "1640": 6, + "1655": 1, + "1665": 11, + "1667": 67, + "1677": 2, + "1681": 9, + "1685": 2, + "1701": 9, + "1708": 7, + "1718": 80, + "1719": 1, + "1720": 8, + "1721": 23, + "1728": 5, + "1752": 17, + "1757": 6, + "1765": 10, + "1769": 6, + "1772": 38, + "1773": 296, + "1814": 33, + "1818": 1, + "1829": 9, + "1831": 32, + "1833": 68, + "1836": 1, + "1866": 5, + "1881": 32, + "1888": 8, + "1901": 41, + "1902": 19, + "1906": 21, + "1911": 25, + "1912": 6, + "1913": 2, + "1914": 2, + "1922": 4, + "1931": 64, + "1932": 2, + "1935": 29, + "1989": 1, + "2001": 1, + "2021": 2, + "2024": 26, + "2045": 13, + "2074": 4, + "2094": 2, + "2095": 4, + "2096": 3, + "2097": 7, + "2098": 6, + "2102": 16, + "2104": 18, + "2107": 1, + "2111": 34, + "2115": 8, + "2133": 2, + "2148": 14, + "2162": 10, + "2180": 17, + "2181": 1, + "2190": 179, + "2203": 7, + "2208": 13, + "2209": 9, + "2210": 85, + "2214": 7, + "2224": 6, + "2234": 49, + "2246": 28, + "2252": 13, + "2261": 153, + "2262": 6, + "2265": 26, + "2271": 37, + "2277": 4, + "2285": 8, + "2286": 14, + "2287": 121, + "2303": 14, + "2320": 1, + "2334": 2, + "2336": 100, + "2337": 13, + "2434": 2, + "2711": 1, + "2714": 15, + "2725": 15, + "2746": 3, + "2762": 5, + "2771": 1, + "2783": 23, + "2986": 1, + "3039": 34, + "3055": 165, + "3077": 4, + "3112": 3, + "3332": 5, + "3349": 6, + "3352": 7, + "3406": 1, + "3435": 7, + "3442": 5, + "3467": 2, + "3469": 10, + "3486": 4, + "3562": 510, + "3565": 3, + "3570": 13, + "3583": 2, + "3617": 38, + "3625": 3, + "3627": 9, + "3635": 1, + "3649": 22, + "3654": 2, + "3656": 7, + "3659": 19, + "3660": 1, + "3662": 8, + "3663": 10, + "3690": 2, + "3693": 6, + "3702": 14247, + "3704": 42, + "3707": 2, + "3708": 66, + "3710": 23, + "3711": 2, + "3712": 7, + "3726": 17, + "3728": 23, + "3747": 2, + "3750": 24, + "3755": 1, + "3760": 1, + "3767": 14, + "3818": 13, + "3823": 2, + "3827": 2, + "3847": 215, + "3856": 9, + "3860": 12, + "3864": 8, + "3870": 36, + "3873": 37, + "3874": 2, + "3879": 94, + "3880": 16, + "3884": 3, + "3885": 25, + "3886": 13, + "3888": 219, + "3906": 13, + "3914": 4, + "3916": 17, + "3917": 9, + "3918": 10, + "3981": 4, + "3983": 2, + "3988": 167, + "3991": 5, + "4026": 15, + "4039": 29, + "4043": 56, + "4045": 22, + "4058": 25, + "4060": 12, + "4072": 6, + "4075": 2, + "4076": 23, + "4079": 14, + "4081": 237, + "4093": 4, + "4096": 2, + "4097": 66, + "4102": 17, + "4108": 8, + "4113": 281, + "4120": 18, + "4224": 2, + "4226": 5, + "4227": 27, + "4228": 57, + "4232": 7, + "4233": 5, + "4265": 6, + "4442": 4, + "4472": 11, + "4498": 23, + "4513": 226, + "4530": 195, + "4540": 33, + "4547": 44, + "4550": 60, + "4555": 1, + "4556": 1, + "4558": 59, + "4565": 164, + "4567": 1, + "4577": 1231, + "4641": 69, + "4690": 21, + "4754": 29, + "4808": 12, + "4839": 10, + "4846": 2, + "4896": 73, + "4922": 10, + "4924": 2, + "4931": 6, + "4932": 3658, + "4934": 2, + "5044": 10, + "5051": 3, + "5053": 5, + "5061": 110, + "5062": 13, + "5063": 10, + "5076": 8, + "5079": 55, + "5082": 4, + "5111": 6, + "5127": 6, + "5141": 88, + "5207": 1, + "5270": 40, + "5286": 39, + "5306": 111, + "5315": 3, + "5323": 40, + "5331": 20, + "5341": 52, + "5400": 7, + "5416": 56, + "5476": 56, + "5477": 7, + "5482": 27, + "5490": 11, + "5507": 26, + "5518": 51, + "5530": 4, + "5532": 3, + "5535": 15, + "5541": 9, + "5544": 8, + "5656": 20, + "5657": 6, + "5661": 24, + "5664": 140, + "5665": 17, + "5666": 9, + "5671": 7, + "5689": 49, + "5690": 1, + "5691": 258, + "5692": 29, + "5693": 243, + "5698": 1, + "5699": 1, + "5702": 61, + "5722": 13, + "5741": 6, + "5759": 186, + "5763": 6, + "5791": 6, + "5807": 13, + "5811": 91, + "5821": 2, + "5833": 199, + "5855": 5, + "5858": 5, + "5860": 13, + "5908": 34, + "5911": 1, + "6182": 2, + "6183": 37, + "6185": 2, + "6192": 14, + "6210": 12, + "6233": 5, + "6239": 3553, + "6253": 66, + "6279": 6, + "6282": 8, + "6421": 22, + "6536": 2, + "6550": 2, + "6565": 3, + "6596": 2, + "6622": 6, + "6637": 2, + "6715": 3, + "6850": 12, + "6941": 6, + "6956": 5, + "6978": 6, + "7010": 2, + "7054": 1, + "7067": 10, + "7091": 18, + "7108": 12, + "7159": 13, + "7160": 1, + "7165": 126, + "7168": 10, + "7213": 3, + "7225": 47, + "7227": 3600, + "7375": 3, + "7460": 53, + "7602": 9, + "7604": 7, + "7729": 1, + "7740": 3, + "7782": 4, + "7787": 8, + "7797": 19, + "7955": 2222, + "7959": 4, + "7962": 15, + "7998": 11, + "8005": 3, + "8017": 9, + "8018": 10, + "8019": 13, + "8022": 244, + "8023": 3, + "8030": 99, + "8032": 1, + "8056": 2, + "8078": 24, + "8139": 1, + "8239": 2, + "8267": 2, + "8355": 36, + "8364": 213, + "8386": 2, + "8400": 25, + "8403": 21, + "8467": 10, + "8649": 18, + "8665": 44, + "8697": 4, + "8717": 15, + "8782": 2, + "8790": 1, + "8795": 2, + "8801": 2, + "8839": 70, + "8932": 22, + "8933": 9, + "9031": 2463, + "9054": 2, + "9103": 21, + "9316": 1, + "9317": 1, + "9327": 1, + "9333": 1, + "9337": 1, + "9378": 5, + "9483": 7, + "9519": 1, + "9521": 18, + "9541": 25, + "9542": 37, + "9543": 5, + "9544": 39, + "9548": 12, + "9598": 5, + "9606": 51775, + "9615": 632, + "9685": 19, + "9734": 5, + "9770": 18, + "9796": 217, + "9823": 1705, + "9825": 4, + "9838": 4, + "9913": 5814, + "9925": 17, + "9940": 415, + "9986": 964, + "10029": 70, + "10030": 1, + "10036": 264, + "10090": 41123, + "10116": 12124, + "10117": 12, + "10141": 191, + "10245": 2, + "10255": 1, + "10298": 74, + "10368": 16, + "10376": 14, + "10506": 9, + "10665": 29, + "10879": 2, + "11033": 2, + "11104": 2, + "11176": 9, + "11191": 3, + "11308": 4, + "11320": 103, + "11520": 55, + "11676": 4, + "11788": 38, + "11866": 1, + "12130": 1, + "12353": 65, + "12721": 138, + "12917": 3, + "12923": 1, + "13335": 9, + "13489": 5, + "13676": 2, + "13677": 1, + "13689": 36, + "13773": 13, + "13780": 2, + "15819": 17, + "20406": 7, + "20416": 3, + "27292": 41, + "27717": 5, + "28038": 22, + "28039": 46, + "28116": 33, + "28229": 10, + "28442": 2, + "28446": 7, + "28448": 2, + "28450": 30, + "28564": 4, + "28896": 13, + "28901": 10, + "28985": 45, + "29058": 9, + "29139": 1, + "29158": 12, + "29172": 2, + "29301": 2, + "29303": 3, + "29320": 17, + "29347": 2, + "29363": 10, + "29370": 8, + "29377": 2, + "29438": 15, + "29449": 2, + "29494": 8, + "29523": 1, + "29541": 3, + "29549": 4, + "29674": 1, + "29722": 1, + "29760": 15, + "29780": 2, + "29850": 4, + "29883": 6, + "29923": 5, + "30326": 18, + "31216": 2, + "31605": 2, + "31958": 2, + "32044": 33, + "32046": 19, + "33008": 14, + "33040": 1, + "33074": 2, + "33111": 2, + "33119": 4, + "33154": 9, + "33178": 1, + "33203": 5, + "33899": 16, + "33903": 4, + "33910": 1, + "33931": 4, + "33940": 25, + "33950": 37, + "33959": 9, + "33996": 1, + "34029": 8, + "34072": 8, + "34305": 3, + "34364": 2, + "34573": 6, + "34582": 1, + "34903": 115, + "35525": 3, + "35749": 13, + "35798": 3, + "35833": 3, + "35889": 10, + "36095": 1, + "36330": 5, + "36599": 10, + "36651": 1, + "36778": 2, + "36841": 1, + "36873": 20, + "36903": 9, + "37329": 58, + "37334": 4, + "37435": 5, + "37591": 66, + "37632": 17, + "37636": 2, + "37734": 3, + "37931": 7, + "38080": 3, + "38313": 16, + "38527": 1, + "38902": 16, + "39152": 2, + "39291": 60, + "39350": 10, + "39534": 9, + "39947": 527, + "39955": 16, + "40324": 203, + "40337": 3, + "40353": 1, + "40384": 4, + "40505": 1, + "40549": 4, + "40559": 3, + "40602": 5, + "40674": 20, + "41200": 2, + "41492": 2, + "41951": 1, + "41984": 8, + "42043": 5, + "42068": 5, + "42684": 14, + "42879": 29, + "42881": 12, + "43263": 10, + "43306": 12, + "43687": 2, + "43960": 4, + "44688": 36, + "44689": 1998, + "45354": 80, + "45465": 1, + "45596": 92, + "45834": 1, + "46147": 2, + "46348": 5, + "46351": 7, + "46429": 6, + "46611": 12, + "47519": 2, + "47715": 1, + "47881": 3, + "47884": 18, + "47914": 4, + "47917": 27, + "48386": 2, + "48868": 40, + "48935": 2, + "49040": 3, + "49338": 4, + "49450": 2, + "49545": 1, + "50053": 2, + "50422": 2, + "51288": 15, + "51351": 4, + "51453": 38, + "51631": 2, + "51663": 42, + "51936": 16, + "51953": 2, + "52153": 9, + "52861": 1, + "52862": 1, + "53953": 31, + "54254": 7, + "55062": 1, + "55125": 1, + "55126": 12, + "55127": 8, + "55190": 2, + "55588": 6, + "56636": 48, + "56690": 23, + "56812": 65, + "57043": 5, + "57622": 2, + "57975": 4, + "58172": 11, + "58217": 1, + "58923": 3, + "59405": 8, + "59406": 6, + "59729": 1, + "59788": 2, + "59899": 13, + "60093": 3, + "60961": 1, + "62337": 1, + "63186": 3, + "63363": 4, + "63559": 17, + "64091": 381, + "64609": 3, + "65004": 3, + "65409": 6, + "66219": 2, + "67780": 8, + "68825": 8, + "69367": 5, + "69488": 78, + "70796": 5, + "70864": 1, + "70868": 4, + "71647": 2, + "71712": 5, + "71721": 6, + "73239": 11, + "73492": 3, + "74111": 2, + "74940": 1, + "76114": 4, + "76772": 27, + "76905": 10, + "77214": 5, + "78167": 6, + "79685": 2, + "80865": 5, + "80866": 11, + "81473": 25, + "82076": 3, + "82310": 30, + "82655": 2, + "83263": 3, + "83332": 4973, + "83333": 8317, + "83558": 2, + "84158": 8, + "85439": 7, + "85549": 8, + "85698": 8, + "85962": 598, + "86029": 1, + "86665": 19, + "87645": 3, + "87883": 27, + "88233": 2, + "89018": 4, + "89184": 2, + "89462": 1, + "89585": 1, + "90370": 5, + "90371": 341, + "90397": 2, + "91979": 8, + "92630": 1, + "93385": 2, + "93496": 19, + "93934": 13, + "94694": 6, + "94850": 3, + "97093": 4, + "99287": 1363, + "99325": 6, + "101201": 1, + "101473": 6, + "102642": 1, + "102803": 1, + "103355": 6, + "104098": 1, + "105351": 26, + "106590": 43, + "106592": 27, + "107673": 2, + "111292": 10, + "111955": 12, + "114909": 2, + "116725": 1, + "118562": 3, + "119072": 12, + "119219": 24, + "119602": 83, + "120086": 2, + "121428": 1, + "121769": 1, + "122586": 205, + "124763": 1, + "129134": 8, + "130081": 2, + "135523": 1, + "137544": 1, + "138853": 4, + "139456": 2, + "143350": 4, + "143416": 2, + "145262": 76, + "145263": 2, + "146786": 1, + "148305": 1, + "150120": 12, + "150436": 8, + "152480": 4, + "155077": 32, + "155680": 1, + "157791": 58, + "158080": 1, + "158878": 781, + "160400": 20, + "161492": 12, + "161934": 36, + "162425": 35, + "165200": 8, + "169036": 1, + "169388": 10, + "169963": 249, + "172714": 1, + "180675": 11, + "182262": 2, + "183674": 8, + "185431": 46, + "186125": 6, + "186153": 2, + "187304": 2, + "188937": 10979, + "192222": 301, + "193462": 7, + "195615": 2, + "198467": 1, + "202950": 13, + "203404": 26, + "203805": 23, + "204998": 6, + "206025": 1, + "208964": 910, + "211586": 407, + "212035": 14, + "213664": 3, + "214684": 198, + "215440": 3, + "216465": 3, + "216816": 3, + "221569": 2, + "222523": 2948, + "224308": 3925, + "225486": 8, + "226186": 181, + "226900": 1817, + "227859": 19, + "230624": 24, + "235357": 8, + "240818": 1, + "243159": 184, + "243964": 2, + "246196": 3644, + "262320": 9, + "266265": 43, + "267671": 351, + "270110": 19, + "272131": 2, + "272623": 10983, + "272624": 199, + "272636": 3, + "283166": 330, + "284812": 4602, + "289672": 4, + "290576": 5, + "298683": 8, + "301447": 400, + "311400": 49, + "332626": 1, + "334771": 6, + "353153": 27, + "359303": 3, + "362597": 1, + "362787": 4, + "380174": 3, + "388408": 1, + "392815": 2, + "395598": 55, + "405212": 122, + "425011": 85, + "449447": 303, + "452646": 23, + "511145": 11788, + "546414": 137, + "559292": 11359, + "569628": 4, + "580240": 5492, + "587753": 3, + "593117": 161, + "595605": 12, + "644337": 2, + "644346": 1, + "668559": 3, + "722438": 2, + "746128": 19, + "746889": 3, + "768495": 4, + "980415": 8, + "1006352": 3, + "1111708": 598, + "1131482": 1 +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 0702551..f949128 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,22 +1,10 @@ -aniso8601 -attrs beautifulsoup4 cement -click -configparser genson -glob2 -itsdangerous -Jinja2 karr_lab_aws_manager -MarkupSafe numpy -pkg_utils -pymongo -pyrsistent -pytest +pymongo == 3.11.0 python_dotenv -pytz >= 2019.1 requests -requirements_parser -simplejson \ No newline at end of file +simplejson +motor \ No newline at end of file diff --git a/setup.py b/setup.py index e0fced1..a133349 100644 --- a/setup.py +++ b/setup.py @@ -46,6 +46,10 @@ ], entry_points={ 'console_scripts': [ + '{} = {}.__main__:main'.format(name, name), + '{} = {}.__main__:main'.format(name.replace('_', '-'), name), + '{}{:d} = {}.__main__:main'.format(name, sys.version_info[0], name), + '{}{:d} = {}.__main__:main'.format(name.replace('_', '-'), sys.version_info[0], name), ], }, ) diff --git a/tests/aggregate/test_tabu.py b/tests/aggregate/test_tabu.py index 8fc68e1..95e5444 100644 --- a/tests/aggregate/test_tabu.py +++ b/tests/aggregate/test_tabu.py @@ -9,8 +9,8 @@ class TestTabu(unittest.TestCase): def setUpClass(cls): db = 'datanator' conf = config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.src = tabu.Tabu(MongoDB=MongoDB, username=username, password=password, db=db, max_entries=20, verbose=True, authSource='admin', diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 8db3da1..8b6c145 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -28,8 +28,8 @@ def test_prod_config(self): def test_test_config(self): result = self.src_test - self.assertTrue(result.MONGO_TEST_USERNAME is not None) - self.assertTrue(result.MONGO_TEST_PASSWORD is not None) + self.assertTrue(result.USERNAME is not None) + self.assertTrue(result.PASSWORD is not None) def test_ftx_config(self): result = self.src_ftx diff --git a/tests/config/test_query_schema_2_manager.py b/tests/config/test_query_schema_2_manager.py new file mode 100644 index 0000000..526bb16 --- /dev/null +++ b/tests/config/test_query_schema_2_manager.py @@ -0,0 +1,18 @@ +import unittest +from datanator_query_python.config import query_schema_2_manager +from pymongo import ReadPreference + + +class TestQ(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.src = query_schema_2_manager.QM() + + @classmethod + def tearDownClass(cls): + cls.src.client.close() + + def test_conn_protein(self): + result = self.src.conn_protein('datanator-test') + self.assertEqual(result.read_preference, ReadPreference.NEAREST) \ No newline at end of file diff --git a/tests/query/test_full_text_query.py b/tests/query/test_full_text_query.py index 2629076..0899b56 100644 --- a/tests/query/test_full_text_query.py +++ b/tests/query/test_full_text_query.py @@ -152,8 +152,9 @@ def test_bool_query(self): } }} result = self.src.bool_query(query_message, index, must_not=must_not) - self.assertEqual(result['hits']['hits'][0]['_source']['ko_number'], 'K13954') + self.assertEqual(result['hits']['hits'][0]['_source']['ko_number'], 'K00001') + @unittest.skip("ko obsolete") def test_get_index_ko_count(self): agg_field = "frontend_gene_aggregate" query_message = 'alcohol dehydrogenase' @@ -170,8 +171,14 @@ def test_get_rxn_oi(self): result = self.src.get_rxn_oi(query_message) self.assertEqual(result['sabio_rk_total'], {'value': 10000, 'relation': 'gte'}) - def test_get_genes_ko_count(self): - agg_field = "frontend_gene_aggregate" - query_message = 'K14236' - result_0 = self.src.get_genes_ko_count(query_message, 15, agg_field=agg_field, size=10, fields=['*']) + # def test_get_genes_ko_count(self): + # agg_field = "ko_number" + # query_message = 'K14236' + # result_0 = self.src.get_genes_ko_count(query_message, 15, agg_field=agg_field, size=10, fields=['*']) + # print(result_0) + + def test_get_genes_orthodb_count(self): + agg_field = "orthodb_id" + query_message = '1398761at2759' + result_0 = self.src.get_genes_orthodb_count(query_message, 15, agg_field=agg_field, size=10, fields=['*']) print(result_0) \ No newline at end of file diff --git a/tests/query/test_query_corum.py b/tests/query/test_query_corum.py index 7df2aa1..2609d78 100644 --- a/tests/query/test_query_corum.py +++ b/tests/query/test_query_corum.py @@ -9,8 +9,8 @@ class TestQueryCorum(unittest.TestCase): def setUpClass(cls): cls.db = 'test' conf = query_config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.MongoDB = MongoDB cls.username = username diff --git a/tests/query/test_query_intact_complex.py b/tests/query/test_query_intact_complex.py index c78c715..ce1992f 100644 --- a/tests/query/test_query_intact_complex.py +++ b/tests/query/test_query_intact_complex.py @@ -10,8 +10,8 @@ class TestQueryCorum(unittest.TestCase): def setUpClass(cls): cls.db = 'test' conf = query_config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.MongoDB = MongoDB cls.username = username diff --git a/tests/query/test_query_kegg_organism_code.py b/tests/query/test_query_kegg_organism_code.py index 6211974..9c22380 100644 --- a/tests/query/test_query_kegg_organism_code.py +++ b/tests/query/test_query_kegg_organism_code.py @@ -9,8 +9,8 @@ class TestKOC(unittest.TestCase): def setUpClass(cls): db = 'datanator' conf = config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.src = query_kegg_organism_code.QueryKOC(username=username, password=password, server=MongoDB, database=db, collection_str='kegg_organism_code') diff --git a/tests/query/test_query_kegg_orthology.py b/tests/query/test_query_kegg_orthology.py index fba9569..3bac8bc 100644 --- a/tests/query/test_query_kegg_orthology.py +++ b/tests/query/test_query_kegg_orthology.py @@ -13,8 +13,8 @@ def setUpClass(cls): cls.cache_dirname = tempfile.mkdtemp() cls.db = 'datanator' conf = config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.MongoDB = MongoDB cls.username = username @@ -61,4 +61,10 @@ def test_get_meta_by_kegg_ids(self): def test_get_meta_by_kegg_id(self): kegg_id = 'k00001' doc = self.src.get_meta_by_kegg_id(kegg_id) - self.assertEqual(doc['gene_name'], ['E1.1.1.1', 'adh']) \ No newline at end of file + self.assertEqual(doc['gene_name'], ['E1.1.1.1', 'adh']) + + def test_get_meta_by_ortho_ids(self): + kegg_ids = ['643917at2', '567019at2'] + docs, count = self.src.get_meta_by_ortho_ids(kegg_ids) + for doc in docs: + print(doc) \ No newline at end of file diff --git a/tests/query/test_query_metabolite_concentrations.py b/tests/query/test_query_metabolite_concentrations.py index 11d8795..7db2221 100644 --- a/tests/query/test_query_metabolite_concentrations.py +++ b/tests/query/test_query_metabolite_concentrations.py @@ -8,8 +8,8 @@ class TestQMC(unittest.TestCase): @classmethod def setUpClass(cls): conf = config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.src = query_metabolite_concentrations.QueryMetaboliteConcentrations(MongoDB=MongoDB, password=password, username=username, db='datanator', collection_str='metabolite_concentrations') diff --git a/tests/query/test_query_metabolites.py b/tests/query/test_query_metabolites.py index ebcc2aa..c1da411 100644 --- a/tests/query/test_query_metabolites.py +++ b/tests/query/test_query_metabolites.py @@ -12,8 +12,8 @@ def setUpClass(cls): cls.cache_dirname = tempfile.mkdtemp() cls.db = 'datanator' conf = config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.src = query_metabolites.QueryMetabolites( cache_dirname=cls.cache_dirname, diff --git a/tests/query/test_query_metabolites_meta.py b/tests/query/test_query_metabolites_meta.py index c780c2d..f6fca61 100644 --- a/tests/query/test_query_metabolites_meta.py +++ b/tests/query/test_query_metabolites_meta.py @@ -12,8 +12,8 @@ def setUpClass(cls): cls.cache_dirname = tempfile.mkdtemp() cls.db = 'datanator' conf = config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.src = query_metabolites_meta.QueryMetabolitesMeta( cache_dirname=cls.cache_dirname, MongoDB=MongoDB, db=cls.db, @@ -86,24 +86,9 @@ def test_get_metabolite_hashed_inchi(self): hashed_inchi = self.src.get_metabolite_hashed_inchi(compound) self.assertEqual(hashed_inchi, ['DBXBTMSZEOQQDU-VKHMYHEASA-N']) - # @unittest.skip('passed') - def test_get_metabolite_similar_compounds(self): - - compound = ['Methyl-Pyruvic acid'] - raw1, result1 = self.src.get_metabolite_similar_compounds(compound, num = 3, threshold = 0.6) - self.assertTrue(list(raw1[0].keys())[0], 'FERIUCNNQQJTOY-UHFFFAOYSA-N') - raw2, result2 = self.src.get_metabolite_similar_compounds(compound, num = 10, threshold = 0.7) - self.assertEqual(len(raw2), 10) - raw3, result3 = self.src.get_metabolite_similar_compounds(compound, num = 10, threshold = 0.9) - # self.assertEqual(raw3[0]['raw'], -1) - # compound = ['β-D-Ribopyranose'] - # raw4, result4 = self.src.get_metabolite_similar_compounds(compound, num = 30, threshold = 0.6) - # self.assertTrue('d' not in list(result4[0].keys())) - # self.assertTrue('Auto inducer 2' in list(result4[0].keys())) - def test_get_unique_metabolites(self): result = self.src.get_unique_metabolites() - self.assertTrue(result in list(range(5222, 5226))) + self.assertTrue(isinstance(result, int)) def test_get_metabolites_meta(self): self.assertEqual(self.src.get_metabolites_meta('lafj;aj'), {}) diff --git a/tests/query/test_query_pax.py b/tests/query/test_query_pax.py index 9de6460..e258d4c 100644 --- a/tests/query/test_query_pax.py +++ b/tests/query/test_query_pax.py @@ -13,8 +13,8 @@ def setUpClass(cls): cls.cache_dirname = tempfile.mkdtemp() cls.db = 'datanator' conf = config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.MongoDB = MongoDB cls.username = username diff --git a/tests/query/test_query_protein.py b/tests/query/test_query_protein.py index 47598cc..4303524 100644 --- a/tests/query/test_query_protein.py +++ b/tests/query/test_query_protein.py @@ -11,8 +11,8 @@ class TestQueryProtein(unittest.TestCase): def setUpClass(cls): cls.db = 'test' conf = config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.MongoDB = MongoDB cls.username = username @@ -23,6 +23,9 @@ def setUpClass(cls): cls.src_1 = query_protein.QueryProtein(server=cls.MongoDB, database='datanator', verbose=True, username = cls.username, password = cls.password, readPreference='nearest') + cls.src_2 = query_protein.QueryProtein(server=cls.MongoDB, database='datanator-test', + verbose=True, username = cls.username, + password = cls.password, readPreference='nearest') cls.src.db_obj.drop_collection('test_query_protein') mock_doc_0 = {'uniprot_id': 'MOCK_0', 'ancestor_taxon_id': [105,104,103,102,101], @@ -88,13 +91,14 @@ def setUpClass(cls): cls.src.collection.insert_many([mock_doc_0, mock_doc_1, mock_doc_2, mock_doc_3, mock_doc_4,mock_doc_5,mock_doc_6]) cls.src.collection.insert_many([dic_0,dic_1,dic_2,dic_3,dic_4,dic_5,dic_6,dic_7,dic_8,dic_9,dic_10,dic_11,dic_12,dic_13,dic_14,dic_15,dic_16]) - cls.src.collection.create_index("uniprot_id", background=False, collation=cls.src.collation) - cls.src.collection.create_index([("protein_name", pymongo.TEXT)]) + # cls.src.collection.create_index("uniprot_id", background=False, collation=cls.src.collation) + # cls.src.collection.create_index([("protein_name", pymongo.TEXT)]) @classmethod def tearDownClass(cls): cls.src.db_obj.drop_collection('test_query_protein') cls.src_1.client.close() + cls.src_2.client.close() cls.src.client.close() def test_get_protein_meta(self): @@ -106,6 +110,7 @@ def test_get_protein_meta(self): self.assertEqual(result_0[1]['ko_number'], 'MOCK_0') self.assertTrue(isinstance(result_1, dict)) + @unittest.skip("takes too long.") def test_get_meta_by_name_taxon(self): name_0 = 'special name' taxon_id_0 = 0 @@ -115,17 +120,20 @@ def test_get_meta_by_name_taxon(self): result_1 = self.src.get_meta_by_name_taxon(name_0, taxon_id_1) self.assertEqual(result_1, []) + @unittest.skip("takes too long.") def test_get_meta_by_name_name(self): species_name_0 = 'escherichia coli' protein_name_0 = 'phosphofructokinase' result_0 = self.src_1.get_meta_by_name_name(protein_name_0, species_name_0) self.assertEqual(len(result_0), 6) + @unittest.skip("takes too long.") def test_get_info_by_text(self): name = 'special name' result = self.src.get_info_by_text(name) self.assertEqual(result[0]['ko_name'], ['no name']) + @unittest.skip("takes too long.") def test_get_info_by_text_abundances(self): name = 'special name' result = self.src.get_info_by_text_abundances(name) @@ -155,6 +163,7 @@ def test_get_info_by_ko_abundance(self): result = self.src.get_info_by_ko_abundance(ko) self.assertTrue(result[0]['uniprot_ids']['uniprot0']) + @unittest.skip("passed") def test_get_id_by_name(self): name = 'special name' result = self.src.get_id_by_name(name) @@ -167,13 +176,14 @@ def test_get_kinlaw_by_id(self): self.assertEqual(result_0[0]['similar_functions'], None) self.assertEqual(len(result_0[1]['similar_functions']), 2) + @unittest.skip("avoid building text index") def test_get_kinlaw_by_name(self): result_0 = self.src.get_kinlaw_by_name('special name one') self.assertEqual(result_0[0]['similar_functions'], [{'ncbi_taxonomy_id': 100, 'kinlaw_id': 1}, {'ncbi_taxonomy_id': 101, 'kinlaw_id': 2}]) result_1 = self.src.get_kinlaw_by_name('uniprot12') self.assertEqual(result_1, []) - # @unittest.skip('passed') + @unittest.skip('passed') def test_get_abundance_by_id(self): _id_0 = ['MOCK_0'] result_0 = self.src.get_abundance_by_id(_id_0) @@ -268,4 +278,20 @@ def test_get_unique_organism(self): @unittest.skip("skipping") def test_get_all_kegg(self): result_0 = self.src_1.get_all_kegg('K00850','Escherichia coli', 10) - print(result_0) \ No newline at end of file + print(result_0) + + # @unittest.skip("skipping") + def test_get_all_ortho(self): + # result_0 = self.src_2.get_all_ortho('494933at2759','Escherichia coli', 10) + # print(result_0) + result_1 = self.src_2.get_all_ortho("1555708at2", "Bacillus subtilis subsp. subtilis", 40) + print(result_1) + + @unittest.skip("passed") + def test_get_info_by_orthodb(self): + result = self.src_2.get_info_by_orthodb("643917at2") + print(result) + + def test_get_ortho_by_id(self): + result = self.src_2.get_ortho_by_id("P53984") + print(result) diff --git a/tests/query/test_query_rna_halflife.py b/tests/query/test_query_rna_halflife.py index df48156..bf19acf 100644 --- a/tests/query/test_query_rna_halflife.py +++ b/tests/query/test_query_rna_halflife.py @@ -9,8 +9,8 @@ class TestQueryRna(unittest.TestCase): def setUpClass(cls): db = 'datanator' conf = config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.col_str = 'rna_halflife_new' cls.src = query_rna_halflife.QueryRNA(server=MongoDB, username=username, password=password, diff --git a/tests/query/test_query_sabio_compound.py b/tests/query/test_query_sabio_compound.py index 7f5a6b7..ae82320 100644 --- a/tests/query/test_query_sabio_compound.py +++ b/tests/query/test_query_sabio_compound.py @@ -9,8 +9,8 @@ class TestSabioCompound(unittest.TestCase): def setUpClass(cls): cls.db = 'test' conf = config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.MongoDB = MongoDB cls.username = username @@ -27,6 +27,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): cls.src.db.drop_collection('test_query_sabio_compound') + cls.src.db.drop_collection('sabio_compound') cls.src.client.close() def test_query_sabio_compound(self): diff --git a/tests/query/test_query_sabio_reaction_entries.py b/tests/query/test_query_sabio_reaction_entries.py index f623f4c..04050eb 100644 --- a/tests/query/test_query_sabio_reaction_entries.py +++ b/tests/query/test_query_sabio_reaction_entries.py @@ -13,8 +13,8 @@ def setUpClass(cls): cls.cache_dirname = tempfile.mkdtemp() cls.db = 'datanator' conf = config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.MongoDB = MongoDB cls.username = username diff --git a/tests/query/test_query_sabiork.py b/tests/query/test_query_sabiork.py index 8393149..5e16703 100644 --- a/tests/query/test_query_sabiork.py +++ b/tests/query/test_query_sabiork.py @@ -12,8 +12,8 @@ def setUpClass(cls): cls.cache_dirname = tempfile.mkdtemp() cls.db = 'datanator' conf = config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.MongoDB = MongoDB cls.username = username diff --git a/tests/query/test_query_sabiork_old.py b/tests/query/test_query_sabiork_old.py index 0f2f6d1..b65d852 100644 --- a/tests/query/test_query_sabiork_old.py +++ b/tests/query/test_query_sabiork_old.py @@ -14,8 +14,8 @@ def setUpClass(cls): cls.cache_dirname = tempfile.mkdtemp() cls.db = 'datanator' conf = config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.MongoDB = MongoDB cls.username = username @@ -80,7 +80,7 @@ def test_get_kinlawid_by_rxn(self): finish = time.time() print('Time elapsed: {}s'.format(finish - start)) - @unittest.skip('collection not yet finished building') + # @unittest.skip('collection not yet finished building') def test_get_kinlaw_by_rxn(self): substrate_0 = 'XJLXINKUBYWONI-NNYOXOHSSA-N' substrate_1 = 'ODBLHEXUDAPZAU-UHFFFAOYSA-N' @@ -100,6 +100,8 @@ def test_get_kinlaw_by_rxn(self): self.assertTrue(count_2 >= 193) count_3, _ = self.src.get_kinlaw_by_rxn([substrate_0], [product_0, product_1], bound='tight') self.assertEqual(count_3, 0) + count_4, _ = self.src.get_kinlaw_by_rxn(["data"], [], bound='tight') + self.assertEqual(count_4, 0) # @unittest.skip('collection not yet finished building') def test_get_kinlaw_by_entryid(self): @@ -147,4 +149,12 @@ def test_get_rxn_with_prm(self): def test_get_reaction_by_subunit(self): _ids = ['P20932', 'P00803'] result = self.src.get_reaction_by_subunit(_ids) - self.assertTrue(result[-1]['kinlaw_id'] in [31611, 31609]) \ No newline at end of file + self.assertTrue(result[-1]['kinlaw_id'] in [31611, 31609]) + + def test_get_kinlaw_by_rxn_ortho(self): + substrate_0 = 'XJLXINKUBYWONI-NNYOXOHSSA-N' + substrate_1 = 'ODBLHEXUDAPZAU-UHFFFAOYSA-N' + product_0 = 'GPRLSGONYQIRFK-UHFFFAOYSA-N' + product_1 = 'KPGXRSRHYNQIFN-UHFFFAOYSA-N' + _, result = self.src.get_kinlaw_by_rxn_ortho([substrate_0, substrate_1], [product_0, product_1]) + print(result) \ No newline at end of file diff --git a/tests/query/test_query_taxon_tree.py b/tests/query/test_query_taxon_tree.py index 73e9006..6a9311b 100644 --- a/tests/query/test_query_taxon_tree.py +++ b/tests/query/test_query_taxon_tree.py @@ -13,8 +13,8 @@ def setUpClass(cls): cls.cache_dirname = tempfile.mkdtemp() cls.db = 'datanator' conf = config.TestConfig() - cls.username = conf.MONGO_TEST_USERNAME - cls.password = conf.MONGO_TEST_PASSWORD + cls.username = conf.USERNAME + cls.password = conf.PASSWORD cls.MongoDB = conf.SERVER cls.src = query_taxon_tree.QueryTaxonTree( cache_dirname=cls.cache_dirname, MongoDB=cls.MongoDB, db=cls.db, @@ -97,6 +97,9 @@ def test_get_canon_rank_distance_by_name(self): result_0 = self.src.get_canon_rank_distance_by_name(name, front_end=True) self.assertEqual(result_0[0], {'Homo sapiens': 0}) self.assertEqual(result_0[-1], {'cellular organisms': 30}) + name = 'about' + result = self.src.get_canon_rank_distance_by_name(name) + self.assertEqual(result, [{name: 0}]) def test_under_category(self): src_tax_id = 550690 @@ -135,10 +138,26 @@ def test_get_canon_common_ancestor(self): # print(result_0) def test_get_canon_common_ancestor_fast(self): - org_0 = 2160 - org_1 = 2161 - for doc in self.src.get_canon_common_ancestor_fast(org_0, org_1): - print(doc) + self.assertEqual(self.src.get_canon_common_ancestor_fast("Escherichia coli E1002", "Homo 1002", org_format="tax_name")["reason"], "No organism found.") + org_1 = 743725 + org_2 = 2107591 + result = self.src.get_canon_common_ancestor_fast(org_1, org_2) + self.assertEqual(result, {'2107591': 4, + '2107591_canon_ancestors': ['cellular organisms', 'Archaea', + 'Candidatus Diapherotrites', + 'Candidatus Forterrea', + 'Candidatus Forterrea multitransposorum'], + '743725': 1, + '743725_canon_ancestors': ['cellular organisms', 'Archaea']}) + self.assertEqual(self.src.get_canon_common_ancestor_fast("Escherichia coli E1002", "Escherichia coli", org_format="tax_name")["Escherichia coli"], 0) + result = self.src.get_canon_common_ancestor_fast('Escherichia coli', 'Escherichia coli', org_format='tax_name') + self.assertEqual(result['Escherichia coli'], 0) + org_3 = 9606 + org_4 = 4932 + result = self.src.get_canon_common_ancestor_fast(org_3, org_4) + self.assertEqual(result, {'9606': 7, '4932': 7, '9606_canon_ancestors': ['cellular organisms', 'Eukaryota', 'Metazoa', 'Chordata', 'Mammalia', 'Primates', 'Hominidae', 'Homo'], '4932_canon_ancestors': ['cellular organisms', 'Eukaryota', 'Fungi', 'Ascomycota', 'Saccharomycetes', 'Saccharomycetales', 'Saccharomycetaceae', 'Saccharomyces']}) + + class TestQueryTaxonTreeMock(unittest.TestCase): @@ -147,8 +166,8 @@ def setUpClass(cls): cls.cache_dirname = tempfile.mkdtemp() cls.db = 'test' conf = config.TestConfig() - cls.username = conf.MONGO_TEST_USERNAME - cls.password = conf.MONGO_TEST_PASSWORD + cls.username = conf.USERNAME + cls.password = conf.PASSWORD cls.MongoDB = conf.SERVER cls.src = query_taxon_tree.QueryTaxonTree( cache_dirname=cls.cache_dirname, MongoDB=cls.MongoDB, db=cls.db, collection_str='test_taxon_tree', diff --git a/tests/query/test_query_uniprot.py b/tests/query/test_query_uniprot.py index 2158c91..58fbbc3 100644 --- a/tests/query/test_query_uniprot.py +++ b/tests/query/test_query_uniprot.py @@ -9,8 +9,8 @@ class TestUniprot(unittest.TestCase): def setUpClass(cls): db = 'datanator' conf = config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.src = query_uniprot.QueryUniprot(username=username, password=password, server=MongoDB, database=db, collection_str='uniprot') diff --git a/tests/query/test_query_xmdb.py b/tests/query/test_query_xmdb.py index 4a7eac4..f1b366c 100644 --- a/tests/query/test_query_xmdb.py +++ b/tests/query/test_query_xmdb.py @@ -9,8 +9,8 @@ class TestQueryEcmdb(unittest.TestCase): def setUpClass(cls): cls.db = 'test' conf = query_config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.MongoDB = MongoDB cls.username = username diff --git a/tests/query_schema_2/test_ftx_search.py b/tests/query_schema_2/test_ftx_search.py new file mode 100644 index 0000000..6dd8319 --- /dev/null +++ b/tests/query_schema_2/test_ftx_search.py @@ -0,0 +1,18 @@ +from datanator_query_python.query_schema_2 import ftx_search +import unittest + + +class TestQEn(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.src = ftx_search.FTX() + + @classmethod + def tearDownClass(cls): + cls.src.client.close() + + def test_search_taxon(self): + docs = self.src.search_taxon("off", + token_order="any") + print(docs) \ No newline at end of file diff --git a/tests/query_schema_2/test_query_entity.py b/tests/query_schema_2/test_query_entity.py new file mode 100644 index 0000000..a37f7e2 --- /dev/null +++ b/tests/query_schema_2/test_query_entity.py @@ -0,0 +1,22 @@ +from datanator_query_python.query_schema_2 import query_entity +import unittest + + +class TestQEn(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.src = query_entity.QueryEn() + + @classmethod + def tearDownClass(cls): + cls.src.client.close() + + def test_query_entity(self): + _id = {"namespace": "inchikey", "value": "TYEYBOSBBBHJIV-UHFFFAOYSA-N"} + r = self.src.query_entity(_id) + self.assertEqual(len(r), 1) + self.assertEqual(r[0]["name"], "2-Ketobutyric acid") + _id = {"namespace": "inchikey", "value": "234wgadgas"} + r = self.src.query_entity(_id) + self.assertEqual(r, []) \ No newline at end of file diff --git a/tests/query_schema_2/test_query_observation.py b/tests/query_schema_2/test_query_observation.py new file mode 100644 index 0000000..7ca6c43 --- /dev/null +++ b/tests/query_schema_2/test_query_observation.py @@ -0,0 +1,27 @@ +from datanator_query_python.query_schema_2 import query_observation +import unittest + + +class TestQOb(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.src = query_observation.QueryObs() + + @classmethod + def tearDownClass(cls): + cls.src.client.close() + + def test_get_entity_datatype(self): + identfier = {"namespace": "gene_symbol", "value": "BAG1"} + results = self.src.get_entity_datatype(identfier) + self.assertEqual(results[0]["entity"]["name"], 'BCL2-associated athanogene.') + identfier = {"namespace": "gene_symbol", "value": "something"} + results = self.src.get_entity_datatype(identfier) + self.assertEqual(results, []) + identfier = {"namespace": "gene_name", "value": "EMC3"} + results = self.src.get_entity_datatype(identfier, datatype="localization") + self.assertEqual(results[0]["entity"]["name"], 'ER membrane protein complex subunit 3') + identfier = {"namespace": "gene_id", "value": "100003563"} + results = self.src.get_entity_datatype(identfier, entity="RNA", datatype="localization") + self.assertEqual(results[0]["genotype"]["taxon"]["name"], 'Danio rerio') \ No newline at end of file diff --git a/tests/query_schema_2/test_query_taxon_tree_v2.py b/tests/query_schema_2/test_query_taxon_tree_v2.py new file mode 100644 index 0000000..8d12b4b --- /dev/null +++ b/tests/query_schema_2/test_query_taxon_tree_v2.py @@ -0,0 +1,50 @@ +import os +os.environ["WHERE"] = "API_TEST" +import unittest +from datanator_query_python.config import config as query_config +from datanator_query_python.query_schema_2 import query_taxon_tree_v2 +import asyncio +from pprint import pprint + + + +class TestQTaxon(unittest.TestCase): + + @classmethod + def setUpClass(cls): + print(os.getenv("WHERE")) + cls.src = query_taxon_tree_v2.QTaxon() + + @classmethod + def tearDownClass(cls): + os.environ["WHERE"] = "" + + def test_get_canon_ancestor(self): + _id = 1915648 + loop = asyncio.get_event_loop() + ids, _ = loop.run_until_complete(self.src.get_canon_ancestor(_id, _format='tax_id')) + self.assertTrue(2283796 not in ids) + self.assertTrue(2157 in ids) + _id = "nonsense" + loop = asyncio.get_event_loop() + ids, _ = loop.run_until_complete(self.src.get_canon_ancestor(_id, _format='tax_name')) + self.assertEqual([], ids) + + # @unittest.skip("for now") + def test_aggregate_distance(self): + measured_0 = [{"canon_anc_ids": [131567, 2, 1224, 1236, 91347, 543, 590, 28901], + "tax_name": "Salmonella enterica subsp. enterica serovar Newport str. CFSAN000907"}] #tax_id1299189 + target_0 = 0 + target_1 = 1227178 + loop = asyncio.get_event_loop() + result_0 = loop.run_until_complete(self.src.aggregate_distance(measured_0, target_0, name_field='tax_name')) + self.assertEqual(result_0, measured_0) + result_1 = loop.run_until_complete(self.src.aggregate_distance(measured_0, target_1, name_field='tax_name')) + self.assertEqual(result_1[0]['taxon_distance']['Salmonella enterica subsp. enterica serovar Newport str. CFSAN001557'], 0) + # taget is measured's ancestor + measured_1 = [{"canon_anc_ids": [131567, 2759, 4751, 4890, 4891, 4892, 4893, 4930, 4932], + "tax_name": "Saccharomyces cerevisiae CAT-1"}] + target_2 = 4932 # Saccharomyces cerevisiae + result_2 = loop.run_until_complete(self.src.aggregate_distance(measured_1, target_2, name_field='tax_name')) + self.assertEqual(result_2[0]['taxon_distance']['Saccharomyces cerevisiae'], 0) + self.assertEqual(result_2[0]['taxon_distance']['Saccharomyces cerevisiae CAT-1'], 1) \ No newline at end of file diff --git a/tests/query_schema_2/test_query_uniprot_v2.py b/tests/query_schema_2/test_query_uniprot_v2.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_core.py b/tests/test_core.py deleted file mode 100644 index d54e9ae..0000000 --- a/tests/test_core.py +++ /dev/null @@ -1,20 +0,0 @@ -""" Test of datanator_query_python.core - -:Author: Name -:Date: 2019-8-26 -:Copyright: 2019, Karr Lab -:License: MIT -""" - -from datanator_query_python import core -from datanator_query_python import util -from datanator_query_python import query -from datanator_query_python import config -import unittest - - -class CoreTestCase(unittest.TestCase): - - def test_1(self): - # Example test - self.assertTrue(True) # example assertion diff --git a/tests/test_main.py b/tests/test_main.py index 131d62a..f8beaf4 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -7,6 +7,8 @@ """ from datanator_query_python import __main__ +from datanator_query_python.util import mongo_util +from datanator_query_python.config import config import datanator_query_python import capturer import mock @@ -44,44 +46,41 @@ def test_version(self): datanator_query_python.__version__) self.assertEqual(captured.stderr.get_text(), '') - def test_command_1(self): + def test_define_schema(self): with capturer.CaptureOutput(merged=False, relay=False) as captured: - with __main__.App(argv=['cmd1']) as app: + with __main__.App(argv=['mongo-def-schema', + 'test', + 'cli_test', + '../datanator_pattern_design/compiled/taxon_compiled.json']) as app: # run app app.run() - # test that the CLI produced the correct output - self.assertEqual(captured.stdout.get_text(), - 'command_1 output') - self.assertEqual(captured.stderr.get_text(), '') - - def test_command_1(self): - with capturer.CaptureOutput(merged=False, relay=False) as captured: - with __main__.App(argv=['cmd2']) as app: - # run app - app.run() + # test that the arguments to the CLI were correctly parsed + self.assertEqual(app.pargs.db, 'test') + self.assertTrue(app.pargs.collection, 'cli_test') # test that the CLI produced the correct output - self.assertEqual(captured.stdout.get_text(), - 'command_2 output') + self.assertEqual(captured.stdout.get_text(), 'done') self.assertEqual(captured.stderr.get_text(), '') - def test_command_3(self): + def test_define_schema(self): with capturer.CaptureOutput(merged=False, relay=False) as captured: - with __main__.App(argv=['command-3', - 'arg-1 value', - 'arg-2 value', - '--opt-arg-3', 'opt-arg-3 value', - '--opt-arg-4', '3.14']) as app: + with __main__.App(argv=['mongo-def-schema', + 'test', + 'cli_test', + '../datanator_pattern_design/compiled/taxon_compiled.json']) as app: # run app app.run() # test that the arguments to the CLI were correctly parsed - self.assertTrue(app.pargs.arg_1) - self.assertTrue(app.pargs.arg_2) - self.assertTrue(app.pargs.opt_arg_3) - self.assertTrue(app.pargs.opt_arg_4) + self.assertEqual(app.pargs.db, 'test') + self.assertTrue(app.pargs.collection, 'cli_test') # test that the CLI produced the correct output - self.assertEqual(captured.stdout.get_text(), '') + self.assertEqual(captured.stdout.get_text(), 'done') self.assertEqual(captured.stderr.get_text(), '') + conf = getattr(config, app.pargs.config_name) + mongo_util.MongoUtil(MongoDB=conf.SERVER, + db=app.pargs.db, + username=conf.USERNAME, + password=conf.PASSWORD).db_obj.drop_collection(app.pargs.collection) \ No newline at end of file diff --git a/tests/util/test_mongo_util.py b/tests/util/test_mongo_util.py index a28851a..4da17fb 100644 --- a/tests/util/test_mongo_util.py +++ b/tests/util/test_mongo_util.py @@ -1,4 +1,5 @@ import unittest +import pymongo from datanator_query_python.util import mongo_util from datanator_query_python.config import config import time @@ -14,8 +15,8 @@ def setUpClass(cls): cls.db = 'datanator' cls.duplicate = 'duplicate_test' conf = config.TestConfig() - username = conf.MONGO_TEST_USERNAME - password = conf.MONGO_TEST_PASSWORD + username = conf.USERNAME + password = conf.PASSWORD MongoDB = conf.SERVER cls.src = mongo_util.MongoUtil( cache_dirname=cls.cache_dirname, MongoDB=MongoDB, @@ -31,11 +32,13 @@ def setUpClass(cls): {"name": "john", "num": 2}, {"name": "mike", "num": 3}] cls.src_test.db_obj[cls.duplicate].insert_many(docs) + cls.schema_test = "schema_test" time.sleep(1) @classmethod def tearDownClass(cls): cls.src_test.db_obj.drop_collection(cls.duplicate) + cls.src_test.db_obj.drop_collection(cls.schema_test) shutil.rmtree(cls.cache_dirname) # @unittest.skip('passed') @@ -46,8 +49,8 @@ def test_list_all_collections(self): def test_con_db(self): client, db, col = self.src.con_db(self.db) self.assertNotEqual(self.src.con_db(self.db), 'Server not available') - self.assertEqual(str(self.src.client.read_preference), 'Nearest(tag_sets=None, max_staleness=-1)') - self.assertEqual(str(col.read_preference), 'Nearest(tag_sets=None, max_staleness=-1)') + self.assertEqual(str(self.src.client.read_preference), 'Nearest(tag_sets=None, max_staleness=-1, hedge=None)') + self.assertEqual(str(col.read_preference), 'Nearest(tag_sets=None, max_staleness=-1, hedge=None)') # @unittest.skip('passed') def test_print_schema(self): @@ -63,4 +66,237 @@ def test_get_duplicates(self): @unittest.skip('duplicate removed') def test_get_duplicates_real(self): num, results = self.src.get_duplicates('taxon_tree', 'tax_id', allowDiskUse=True) - self.assertEqual(num, 1) \ No newline at end of file + self.assertEqual(num, 1) + + def test_define_schema(self): + json_schema = "../datanator_pattern_design/compiled/taxon_compiled.json" + self.src_test.define_schema(self.schema_test, json_schema) + self.src_test.db_obj[self.schema_test].insert_one({"ncbi_taxonomy_id": 123, "name": "something", + "canon_ancestors": []}) + try: + self.src_test.db_obj[self.schema_test].insert_one({"ncbi_taxonomy_id": "123", "name": "something", + "canon_ancestors": []}) + except pymongo.errors.WriteError as e: + self.assertEqual(str(e), "Document failed validation, full error: {'index': 0, 'code': 121, 'errmsg': 'Document failed validation'}") + + def test_update_observation(self): + self.src_test.update_observation({"identifier": {"namespace": "something", "value": "a"}, + "something": []}, + {"namespace": "something", "value": "a"}, + op="test") + + def test_update_entity(self): + null = None + obj = { + "type": "protein", + "name": "Oryza sativa", + "synonyms": [], + "identifiers": [ + { + "namespace": "uniprot", + "value": "Q75IW1" + }, + { + "namespace": "gene_name_alt", + "value": null + }, + { + "namespace": "gene_name_orf", + "value": "OsJ_11271 OSJNBb0059G13.19" + }, + { + "namespace": "gene_name_oln", + "value": "Os03g0416300 LOC_Os03g30260" + }, + { + "namespace": "pro_id", + "value": "PR:000024921" + }, + { + "namespace": "pro_id", + "value": "PR:000036675" + }, + { + "namespace": "entrez_id", + "value": "4333115" + }, + { + "namespace": "entry_name", + "value": "COBL2_ORYSJ" + } + ], + "related": [ + { + "namespace": "ec", + "value": null + }, + { + "namespace": "gene_name", + "value": "BC1L2" + }, + { + "namespace": "ko_name", + "value": null + }, + { + "namespace": "ko_number", + "value": null + } + ], + "genotype": { + "taxon": { + "ncbi_taxonomy_id": 39947, + "obj": "Oryza sativa Japonica Group", + "canon_ancestors": [ + { + "ncbi_taxonomy_id": 131567, + "name": "cellular organisms" + }, + { + "ncbi_taxonomy_id": 2759, + "name": "Eukaryota" + }, + { + "ncbi_taxonomy_id": 33090, + "name": "Viridiplantae" + }, + { + "ncbi_taxonomy_id": 35493, + "name": "Streptophyta" + }, + { + "ncbi_taxonomy_id": 4447, + "name": "Liliopsida" + }, + { + "ncbi_taxonomy_id": 38820, + "name": "Poales" + }, + { + "ncbi_taxonomy_id": 4479, + "name": "Poaceae" + }, + { + "ncbi_taxonomy_id": 4527, + "name": "Oryza" + }, + { + "ncbi_taxonomy_id": 4530, + "name": "Oryza sativa" + } + ] + } + }, + "structures": [ + { + "format": "canonical_sequence", + "value": "MARFLLGAAAIALLAGVSSLLLMVPFAEAYDPLDPNGNITIKWDITQWTPDGYVAVVTIYNFQKYRHIQAPGWSLGWAWAKKEIIWSMAGGQATEQGDCSAFKANIPHCCKRDPRVVDLVPGAPYNMQFGNCCKGGVLTSWVQDPLNAVASFQITVGHSGTSNKTVKAPKNFTLKAPGPGYSCGLAQEVKPPTRFISLDGRRTTQAHVTWNVTCTYSQFVAQRAPTCCVSLSSFYNETIVNCPKCACGCQNKKPGSCVEGNSPYLASVVNGPGKGSLTPLVQCTPHMCPIRVHWHVKLNYRDYWRVKVTITNWNYRMNYSQWNLVVQHPNFENVSTVFSFNYKSLNPYGVINDTAMMWGVKYYNDLLMVAGPDGNVQSELLFRKDRSTFTFDKGWAFPRRIYFNGESCVMPSPDLYPWLPPSSTPRFRTVFLLMSFLVCGTLAFLHNHLVLDKNCGKC" + }, + { + "format": "processed_sequence_iubmb", + "value": null, + "molecular_weight": null, + "charge": null, + "formula": null, + "source": [ + { + "namespace": "pro_id", + "value": "PR:000024921", + "level": "secondary" + }, + { + "namespace": "doi", + "value": "10.1093/nar/gkw1075", + "level": "primary" + } + ] + }, + { + "format": "modified_sequence_abbreviated_bpforms", + "value": "SERFPNDVDPIETRDWLQAIESVIREEGVERAQYLIDQLLAEARKGGVNVAAGTGISNYINTIPVEEQPEYPGNLELERRIRSAIRWNAIMTVLRASKKDLELGGHMASFQSSATIYDVCFNHFFRARNEQDGGDLVYFQGHISPGVYARAFLEGRLTQEQLDNFRQEVHGNGLSSYPHPKLMPEFWQFPTVSMGLGPIGAIYQAKFLKYLEHRGLKDTSKQTVYAFLGDGEMDEPESKGAITIATREKLDNLVFVINCNLQRLDGPVTGNGKIINELEGIFEGAGWNVIKVMWGSRWDELLRKDTSGKLIQLMNETVDGDYQTFKSKDGAYVREHFFGKYPETAALVADWTDEQIWALNRGGHDPKKIYAAFKKAQETKGKATVILAHTIKGYGMGDAAEGKNIAHQVKKMNMDGVRHIRDRFNVPVSDADIEKLPYITFPEGSEEHTYLHAQRQKLHGYLPSRQPNFTEKLELPSLQDFGALLEEQSKEISTTIAFVRALNVMLKNKSIKDRLVPIIADEARTFGMEGLFRQIGIYSPNGQQYTPQDREQVAYYKEDEKGQILQEGINELGAGCSWLAAATSYSTNNLPMIPFYIYYSMFGFQRIGDLCWAAGDQQARGFLIGGTSGRTTLNGEGLQHEDGHSHIQSLTIPNCISYDPAYAYEVAVIMHDGLERMYGEKQENVYYYITTLNENYHMPAMPEGAEEGIRKGIY{AA0055}LETIEGSKGKVQLLGSGSILRHVREAAEILAKDYGVGSDVYSVTSFTELARDGQDCERWNMLHPLETPRVPYIAQVMNDAPAVASTDYMKLFAEQVRTYVPADDYRVLGTDGFGRSDSRENLRHHFEVDASYVVVAALGELAKRGEIDKKVVADAIAKFNIDADKVNPRLA", + "molecular_weight": 97709.46800000001, + "charge": 97, + "formula": "C4438H6966N1217O1217S27", + "modification": { + "description": "K --> MOD:00064 (716)", + "formula": "C2HO", + "weight": 41.028999999994994, + "charge": -1 + }, + "source": [ + { + "namespace": "pro_id", + "value": "PR:000024921", + "level": "secondary" + }, + { + "namespace": "doi", + "value": "10.1093/nar/gkw1075", + "level": "primary" + } + ] + }, + { + "format": "modified_sequence_bpforms", + "value": "SERFPNDVDPIETRDWLQAIESVIREEGVERAQYLIDQLLAEARKGGVNVAAGTGISNYINTIPVEEQPEYPGNLELERRIRSAIRWNAIMTVLRASKKDLELGGHMASFQSSATIYDVCFNHFFRARNEQDGGDLVYFQGHISPGVYARAFLEGRLTQEQLDNFRQEVHGNGLSSYPHPKLMPEFWQFPTVSMGLGPIGAIYQAKFLKYLEHRGLKDTSKQTVYAFLGDGEMDEPESKGAITIATREKLDNLVFVINCNLQRLDGPVTGNGKIINELEGIFEGAGWNVIKVMWGSRWDELLRKDTSGKLIQLMNETVDGDYQTFKSKDGAYVREHFFGKYPETAALVADWTDEQIWALNRGGHDPKKIYAAFKKAQETKGKATVILAHTIKGYGMGDAAEGKNIAHQVKKMNMDGVRHIRDRFNVPVSDADIEKLPYITFPEGSEEHTYLHAQRQKLHGYLPSRQPNFTEKLELPSLQDFGALLEEQSKEISTTIAFVRALNVMLKNKSIKDRLVPIIADEARTFGMEGLFRQIGIYSPNGQQYTPQDREQVAYYKEDEKGQILQEGINELGAGCSWLAAATSYSTNNLPMIPFYIYYSMFGFQRIGDLCWAAGDQQARGFLIGGTSGRTTLNGEGLQHEDGHSHIQSLTIPNCISYDPAYAYEVAVIMHDGLERMYGEKQENVYYYITTLNENYHMPAMPEGAEEGIRKGIY{AA0055}LETIEGSKGKVQLLGSGSILRHVREAAEILAKDYGVGSDVYSVTSFTELARDGQDCERWNMLHPLETPRVPYIAQVMNDAPAVASTDYMKLFAEQVRTYVPADDYRVLGTDGFGRSDSRENLRHHFEVDASYVVVAALGELAKRGEIDKKVVADAIAKFNIDADKVNPRLA" + }, + { + "format": "processed_sequence_iubmb", + "value": null, + "molecular_weight": null, + "charge": null, + "formula": null, + "source": [ + { + "namespace": "pro_id", + "value": "PR:000036675", + "level": "secondary" + }, + { + "namespace": "doi", + "value": "10.1093/nar/gkw1075", + "level": "primary" + } + ] + }, + { + "format": "modified_sequence_abbreviated_bpforms", + "value": "SERFPNDVDPIETRDWLQAIESVIREEGVERAQYLIDQLLAEARKGGVNVAAGTGISNYINTIPVEEQPEYPGNLELERRIRSAIRWNAIMTVLRASKKDLELGGHMASFQSSATIYDVCFNHFFRARNEQDGGDLVYFQGHISPGVYARAFLEGRLTQEQLDNFRQEVHGNGLSSYPHPKLMPEFWQFPTVSMGLGPIGAIYQAKFLKYLEHRGLKDTSKQTVYAFLGDGEMDEPESKGAITIATREKLDNLVFVINCNLQRLDGPVTGNGKIINELEGIFEGAGWNVIKVMWGSRWDELLRKDTSGKLIQLMNETVDGDYQTFKSKDGAYVREHFFGKYPETAALVADWTDEQIWALNRGGHDPKKIYAAFKKAQETKGKATVILAHTIKGYGMGDAAEGKNIAHQVKKMNMDGVRHIRDRFNVPVSDADIEKLPYITFPEGSEEHTYLHAQRQKLHGYLPSRQPNFTEKLELPSLQDFGALLEEQSKEISTTIAFVRALNVMLKNKSIKDRLVPIIADEARTFGMEGLFRQIGIYSPNGQQYTPQDREQVAYYKEDEKGQILQEGINELGAGCSWLAAATSYSTNNLPMIPFYIYYSMFGFQRIGDLCWAAGDQQARGFLIGGTSGRTTLNGEGLQHEDGHSHIQSLTIPNCISYDPAYAYEVAVIMHDGLERMYGEKQENVYYYITTLNENYHMPAMPEGAEEGIRKGIY{AA0055}LETIEGSKGKVQLLGSGSILRHVREAAEILAKDYGVGSDVYSVTSFTELARDGQDCERWNMLHPLETPRVPYIAQVMNDAPAVASTDYMKLFAEQVRTYVPADDYRVLGTDGFGRSDSRENLRHHFEVDASYVVVAALGELAKRGEIDKKVVADAIAKFNIDADKVNPRLA", + "molecular_weight": 97709.46800000001, + "charge": 97, + "formula": "C4438H6966N1217O1217S27", + "modification": { + "description": "K --> MOD:00064 (716)", + "formula": "C2HO", + "weight": 41.028999999994994, + "charge": -1 + }, + "source": [ + { + "namespace": "pro_id", + "value": "PR:000036675", + "level": "secondary" + }, + { + "namespace": "doi", + "value": "10.1093/nar/gkw1075", + "level": "primary" + } + ] + }, + { + "format": "modified_sequence_bpforms", + "value": "SERFPNDVDPIETRDWLQAIESVIREEGVERAQYLIDQLLAEARKGGVNVAAGTGISNYINTIPVEEQPEYPGNLELERRIRSAIRWNAIMTVLRASKKDLELGGHMASFQSSATIYDVCFNHFFRARNEQDGGDLVYFQGHISPGVYARAFLEGRLTQEQLDNFRQEVHGNGLSSYPHPKLMPEFWQFPTVSMGLGPIGAIYQAKFLKYLEHRGLKDTSKQTVYAFLGDGEMDEPESKGAITIATREKLDNLVFVINCNLQRLDGPVTGNGKIINELEGIFEGAGWNVIKVMWGSRWDELLRKDTSGKLIQLMNETVDGDYQTFKSKDGAYVREHFFGKYPETAALVADWTDEQIWALNRGGHDPKKIYAAFKKAQETKGKATVILAHTIKGYGMGDAAEGKNIAHQVKKMNMDGVRHIRDRFNVPVSDADIEKLPYITFPEGSEEHTYLHAQRQKLHGYLPSRQPNFTEKLELPSLQDFGALLEEQSKEISTTIAFVRALNVMLKNKSIKDRLVPIIADEARTFGMEGLFRQIGIYSPNGQQYTPQDREQVAYYKEDEKGQILQEGINELGAGCSWLAAATSYSTNNLPMIPFYIYYSMFGFQRIGDLCWAAGDQQARGFLIGGTSGRTTLNGEGLQHEDGHSHIQSLTIPNCISYDPAYAYEVAVIMHDGLERMYGEKQENVYYYITTLNENYHMPAMPEGAEEGIRKGIY{AA0055}LETIEGSKGKVQLLGSGSILRHVREAAEILAKDYGVGSDVYSVTSFTELARDGQDCERWNMLHPLETPRVPYIAQVMNDAPAVASTDYMKLFAEQVRTYVPADDYRVLGTDGFGRSDSRENLRHHFEVDASYVVVAALGELAKRGEIDKKVVADAIAKFNIDADKVNPRLA" + } + ], + "schema_version": "2.0" + } + x, y = self.src_test.update_entity(obj, + {"namespace": "uniprot", "value": "Q75IW1"}, + op="test") + print(y) + + def test_build_taxon_object(self): + self.assertEqual(self.src.build_taxon_object(0), {}) + _id = 9606 + self.assertEqual(self.src.build_taxon_object(_id)["canon_ancestors"][0]["name"], "cellular organisms") \ No newline at end of file diff --git a/tests/util/test_motor_util.py b/tests/util/test_motor_util.py new file mode 100644 index 0000000..0e9a5bc --- /dev/null +++ b/tests/util/test_motor_util.py @@ -0,0 +1,25 @@ +from datanator_query_python.util import motor_util +from datanator_query_python.config import config +import unittest + + +class TestMUtil(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.src = motor_util.MotorUtil(MongoDB=config.AtlasConfig.SERVER, + username=config.DatanatorTest.USERNAME, + password=config.DatanatorTest.PASSWORD, + authSource=config.AtlasConfig.AUTHDB, + replicaSet=config.AtlasConfig.REPLSET, + readPreference=config.AtlasConfig.READ_PREFERENCE ) + cls.test_collection = 'test_motor' + cls.test_database = "test" + + @classmethod + def tearDownClass(cls): + cls.src.client.close() + cls.src.client.get_database(cls.test_database).drop_collection(cls.test_collection) + + def test_client(self): + self.src.client.get_database(self.test_database)[self.test_collection].insert_one({"test": 1}) \ No newline at end of file