diff --git a/doc/API.rst b/doc/API.rst new file mode 100644 index 0000000..1817725 --- /dev/null +++ b/doc/API.rst @@ -0,0 +1,3 @@ +===== +API +===== diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000..1c19fc3 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = output + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/doc/_static/RUG_Logo.jpg b/doc/_static/RUG_Logo.jpg new file mode 100644 index 0000000..da3037b Binary files /dev/null and b/doc/_static/RUG_Logo.jpg differ diff --git a/doc/_static/custom.css b/doc/_static/custom.css new file mode 100644 index 0000000..819fc4b --- /dev/null +++ b/doc/_static/custom.css @@ -0,0 +1,16 @@ +/* override table width restrictions as found on https://github.com/getpelican/pelican/issues/1311 */ +.wy-table-responsive table td, .wy-table-responsive table th { + /* !important prevents the common CSS stylesheets from + overriding this as on RTD they are loaded after this stylesheet */ + white-space: normal !important; +} + +.wy-table-responsive { + overflow: visible !important; +} + +/* Add some more space between the function descriptions */ +dl.function, +dl.class { + margin-bottom: 1em; +} \ No newline at end of file diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 0000000..c432eea --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,241 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('../tusd/hooks')) + +# Django autodoc +sys.path.insert(0, os.path.abspath('../webservice')) +os.environ['DJANGO_SETTINGS_MODULE'] = 'webservice.settings' +import django +django.setup() + +# -- Project information ----------------------------------------------------- + +project = 'Synthea patient generator webservice' +copyright = '2020, Joshua Rubingh' +author = 'Joshua Rubingh' + +# The master toctree document. +master_doc = 'index' + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.napoleon', + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', + 'sphinx.ext.coverage', + 'sphinx_markdown_builder', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store','build/*'] + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +html_theme_options = { + 'logo': 'RUG_Logo.jpg', + 'logo_name' : True +} + +# -- Options for LaTeX output --------------------------------------------- +# Install Ubuntu/Debian package(s): texlive-latex-recommended, texlive-fonts-recommended, texlive-latex-extra, netpbm +latex_engine = 'pdflatex' +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + 'papersize': 'a4paper', + 'releasename':" ", + # Sonny, Lenny, Glenn, Conny, Rejne, Bjarne and Bjornstrup + # 'fncychap': '\\usepackage[Lenny]{fncychap}', + 'fncychap': '\\usepackage{fncychap}', + 'fontpkg': '\\usepackage{amsmath,amsfonts,amssymb,amsthm}', + + 'figure_align':'htbp', + # The font size ('10pt', '11pt' or '12pt'). + # + 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + 'preamble': r''' + %%%%%%%%%%%%%%%%%%%% Meher %%%%%%%%%%%%%%%%%% + %%%add number to subsubsection 2=subsection, 3=subsubsection + %%% below subsubsection is not good idea. + \setcounter{secnumdepth}{3} + % + %%%% Table of content upto 2=subsection, 3=subsubsection + \setcounter{tocdepth}{1} + + \usepackage{amsmath,amsfonts,amssymb,amsthm} + \usepackage{graphicx} + + %%% reduce spaces for Table of contents, figures and tables + %%% it is used "\addtocontents{toc}{\vskip -1.2cm}" etc. in the document + \usepackage[notlot,nottoc,notlof]{} + + \usepackage{color} + \usepackage{transparent} + \usepackage{eso-pic} + \usepackage{lipsum} + + \usepackage{footnotebackref} %%link at the footnote to go to the place of footnote in the text + + %% spacing between line + \usepackage{setspace} + %%%%\onehalfspacing + %%%%\doublespacing + \singlespacing + + + %%%%%%%%%%% datetime + \usepackage{datetime} + + \newdateformat{MonthYearFormat}{% + \monthname[\THEMONTH], \THEYEAR} + + + %% RO, LE will not work for 'oneside' layout. + %% Change oneside to twoside in document class + \usepackage{fancyhdr} + \pagestyle{fancy} + \fancyhf{} + + %%% Alternating Header for oneside + \fancyhead[L]{\ifthenelse{\isodd{\value{page}}}{ \small \nouppercase{\leftmark} }{}} + \fancyhead[R]{\ifthenelse{\isodd{\value{page}}}{}{ \small \nouppercase{\rightmark} }} + + %%% Alternating Header for two side + %\fancyhead[RO]{\small \nouppercase{\rightmark}} + %\fancyhead[LE]{\small \nouppercase{\leftmark}} + + %% for oneside: change footer at right side. If you want to use Left and right then use same as header defined above. + %% \fancyfoot[R]{\ifthenelse{\isodd{\value{page}}}{{\tiny Meher Krishna Patel} }{\href{http://pythondsp.readthedocs.io/en/latest/pythondsp/toc.html}{\tiny PythonDSP}}} + + %%% Alternating Footer for two side + %% %\fancyfoot[RO, RE]{\scriptsize Meher Krishna Patel (mekrip@gmail.com)} + + %%% page number + \fancyfoot[CO, CE]{\thepage} + + \renewcommand{\headrulewidth}{0.5pt} + \renewcommand{\footrulewidth}{0.5pt} + + \RequirePackage{tocbibind} %%% comment this to remove page number for following + \addto\captionsenglish{\renewcommand{\contentsname}{Table of contents}} + %% \addto\captionsenglish{\renewcommand{\listfigurename}{List of figures}} + %% \addto\captionsenglish{\renewcommand{\listtablename}{List of tables}} + %% % \addto\captionsenglish{\renewcommand{\chaptername}{Chapter}} + + + %%reduce spacing for itemize + \usepackage{enumitem} + \setlist{nosep} + + %%%%%%%%%%% Quote Styles at the top of chapter + %% \usepackage{epigraph} + %% \setlength{\epigraphwidth}{0.8\columnwidth} + %% \newcommand{\chapterquote}[2]{\epigraphhead[60]{\epigraph{\textit{#1}}{\textbf {\textit{--#2}}}}} + %%%%%%%%%%% Quote for all places except Chapter + %% \newcommand{\sectionquote}[2]{{\quote{\textit{``#1''}}{\textbf {\textit{--#2}}}}} + ''', + + + 'maketitle': r''' + \pagenumbering{Roman} %%% to avoid page 1 conflict with actual page 1 + + \begin{titlepage} + + \begingroup % for PDF information dictionary + \def\endgraf{ }\def\and{\& }% + \pdfstringdefDisableCommands{\def\\{, }}% overwrite hyperref setup + \hypersetup{pdfauthor={\@author}, pdftitle={\@title}}% + \endgroup + + \centering + + \vspace*{40mm} %%% * is used to give space from top + \textbf{\Huge {UMCG Synthea patient generator webservice} } + + \vspace{0mm} + \begin{figure}[!h] + \centering + \includegraphics[scale=0.4]{RUG_Logo.jpg} + \end{figure} + + \vspace{100mm} + \Large \textbf{{Joshua Rubingh}} + + + \small Created on : November, 2020 + + \vspace*{0mm} + \small Last updated : \MonthYearFormat\today + + + %% \vfill adds at the bottom + \vfill + %% \small \textit{More documents are freely available at }{\href{http://pythondsp.readthedocs.io/en/latest/pythondsp/toc.html}{PythonDSP}} + \end{titlepage} + + \clearpage + \pagenumbering{roman} + \tableofcontents + \listoffigures + \listoftables + \clearpage + \pagenumbering{arabic} + ''', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', + 'sphinxsetup': \ + 'hmargin={0.7in,0.7in}, vmargin={1in,1in}, \ + verbatimwithframe=true, \ + TitleColor={rgb}{0,0,0}, \ + HeaderFamily=\\rmfamily\\bfseries, \ + InnerLinkColor={rgb}{0,0,1}, \ + OuterLinkColor={rgb}{0,0,1}', + + 'tableofcontents':' ', +} + +latex_logo = '_static/RUG_Logo.jpg' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'documentation.tex', project, + author, 'report') +] \ No newline at end of file diff --git a/doc/documentation.pdf b/doc/documentation.pdf new file mode 100644 index 0000000..e003409 Binary files /dev/null and b/doc/documentation.pdf differ diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 0000000..137d30a --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,26 @@ +================================================== +Synthea patient generator webservice documentation +================================================== + +Here you can read more information about the UMCG Synthea patient generator webservice. + +This webservice uses `Django `_ for the front-end and `Synthea `_ for the patient generating. + +.. toctree:: + :caption: Table of Contents + :maxdepth: 2 + + install + models + views + API + utils + signals + +------------------ +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/doc/install.rst b/doc/install.rst new file mode 100644 index 0000000..dd258ba --- /dev/null +++ b/doc/install.rst @@ -0,0 +1,98 @@ +============ +Installation +============ + +In order to install this Virtual Research Environment project, we use the following packages / software. + +* Synthea +* NGINX +* Django + +First we need to checkout the code. + +.. code-block:: bash + + git clone https://git.web.rug.nl/P300021/synthea_webservice /opt/deploy/synthea_webservice + + +------- +Synthea +------- +TODO: Make clear setup for synthea. As we are using NL demographical data. + +----- +NGINX +----- +Install NGINX through the package manager. For Ubuntu this would be + +.. code-block:: console + + sudo apt install nginx + +Also configure SSL to make the connections secure. This is outside this installation scope. + +Setup +----- +After installation of the packages, create a symbolic link in the `/etc/nginx/sites-enabled` so that a new VHost is created. + +Important parts of the VHost configuration: + +.. literalinclude:: ../nginx/synthea_webservice.vhost.conf + :language: bash + +In order to test if NGINX is configured correctly run `nginx -t` and it should give an OK message: + +.. code-block:: bash + + nginx: the configuration file /etc/nginx/nginx.conf syntax is ok + nginx: configuration file /etc/nginx/nginx.conf test is successful + +------ +Django +------ +We install Django with standard settings. We could run it in Aync way, but then you need some more steps: https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ So for now, we keep it simple. + +Install +======= +Create a python virtual environment + +.. code-block:: bash + + cd /opt/deploy/synthea_webservice + python3 -m venv venv + source venv/bin/activate + +Finally we install the required Python modules + +.. code-block:: python + + pip install -r requirements + +This will install all the needed Python modules we need to run this Django project. + +Settings +-------- +The settings for Django are set in an `.env` file so that you can easily change the environment from production to testing. There is an `.env.example` file that could be used as a template. + +.. literalinclude:: ../webservice/webservice/.env.example + :language: jproperties + +Next we have to make the database structure. If you are using SQLite3 as a backend, make sure the database file **DOES** exist on disk. + +.. code-block:: console + + touch /opt/deploy/synthea_webservice/webservice/db.sqlite3 + +Then in the Python virtual environment we run the following commands: + +.. code-block:: console + + ./manage.py migrate + ./manage.py createsuperuser + ./manage.py collectstatic + +And finally you should be able to start the Django application + +.. code-block:: console + + ./manage.py runserver \ No newline at end of file diff --git a/doc/make.bat b/doc/make.bat new file mode 100644 index 0000000..2119f51 --- /dev/null +++ b/doc/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/doc/models.rst b/doc/models.rst new file mode 100644 index 0000000..e25f921 --- /dev/null +++ b/doc/models.rst @@ -0,0 +1,21 @@ +====== +Models +====== + +---- +Base +---- +.. automodule:: lib.models.base + :members: + +--- +API +--- +.. automodule:: apps.api.models + :members: + +------- +Synthea +------- +.. automodule:: apps.synthea.models + :members: diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 0000000..6263d83 --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,2 @@ +Sphinx +sphinx-markdown-builder diff --git a/doc/signals.rst b/doc/signals.rst new file mode 100644 index 0000000..dd82972 --- /dev/null +++ b/doc/signals.rst @@ -0,0 +1,9 @@ +======= +Signals +======= + +--- +API +--- +.. automodule:: apps.api.signals + :members: diff --git a/doc/utils.rst b/doc/utils.rst new file mode 100644 index 0000000..c3745cc --- /dev/null +++ b/doc/utils.rst @@ -0,0 +1,17 @@ +====== +Utils +====== + +------- +General +------- +.. automodule:: lib.utils.general + :members: +.. automodule:: lib.utils.emails + :members: + +------- +Synthea +------- +.. automodule:: apps.synthea.lib.utils + :members: \ No newline at end of file diff --git a/doc/views.rst b/doc/views.rst new file mode 100644 index 0000000..e5f0dce --- /dev/null +++ b/doc/views.rst @@ -0,0 +1,6 @@ +===== +Views +===== + +.. autoclass:: apps.api.views.Info + :members: diff --git a/nginx/synthea_webservice.vhost.conf b/nginx/synthea_webservice.vhost.conf new file mode 100644 index 0000000..34c9395 --- /dev/null +++ b/nginx/synthea_webservice.vhost.conf @@ -0,0 +1,73 @@ +## +# You should look at the following URL's in order to grasp a solid understanding +# of Nginx configuration files in order to fully unleash the power of Nginx. +# https://www.nginx.com/resources/wiki/start/ +# https://www.nginx.com/resources/wiki/start/topics/tutorials/config_pitfalls/ +# https://wiki.debian.org/Nginx/DirectoryStructure +# +# In most cases, administrators will remove this file from sites-enabled/ and +# leave it as reference inside of sites-available where it will continue to be +# updated by the nginx packaging team. +# +# This file will automatically load configuration files provided by other +# applications, such as Drupal or Wordpress. These applications will be made +# available underneath a path with that package name, such as /drupal8. +# +# Please see /usr/share/doc/nginx-doc/examples/ for more detailed examples. +## + +# Default server configuration +# + +server { + listen 80; + listen [::]:80; + + # SSL configuration + # + # listen 443 ssl default_server; + # listen [::]:443 ssl default_server; + # + # Note: You should disable gzip for SSL traffic. + # See: https://bugs.debian.org/773332 + # + # Read up on ssl_ciphers to ensure a secure configuration. + # See: https://bugs.debian.org/765782 + # + # Self signed certs generated by the ssl-cert package + # Don't use them in a production server! + # + # include snippets/snakeoil.conf; + + root /var/www/html; + + # Add index.php to the list if you are using PHP + index index.html index.htm index.nginx-debian.html; + + server_name localhost; + + access_log /var/log/nginx/synthea_webservice.access.log; + error_log /var/log/nginx/synthea_webservice.error.log; + + location /static { + alias /opt/deploy/synthea_webservice/webservice/static; + } + + location / { + # First attempt to serve request as file, then + # as directory, then fall back to displaying a 404. + # try_files $uri $uri/ =404; + + proxy_pass http://localhost:8000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_redirect off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $sent_http_host; + proxy_set_header X-Forwarded-Proto $scheme; + } +} diff --git a/webservice/apps/api/migrations/0001_initial.py b/webservice/apps/api/migrations/0001_initial.py index fd23d06..0cf399a 100644 --- a/webservice/apps/api/migrations/0001_initial.py +++ b/webservice/apps/api/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 3.0.8 on 2020-07-30 14:15 +# Generated by Django 3.1.3 on 2020-11-27 11:47 import apps.api.models from django.conf import settings diff --git a/webservice/apps/api/models.py b/webservice/apps/api/models.py index 3603aef..72f513e 100644 --- a/webservice/apps/api/models.py +++ b/webservice/apps/api/models.py @@ -55,7 +55,7 @@ class Token(MetaDataModel): """Boolean check if the token is belonging to a user with super user rights. Then this token is a super token. Returns: - bool: Returns true when the token belongs to a super user. + Boolean: Returns true when the token belongs to a super user. """ # TODO: Is it allowed to be a super user and researcher? Could give conflict of interests. With the API token you can read other researchers data... return self.user.is_superuser == True diff --git a/webservice/apps/api/signals.py b/webservice/apps/api/signals.py index e4e4c8a..3ac55a2 100644 --- a/webservice/apps/api/signals.py +++ b/webservice/apps/api/signals.py @@ -17,7 +17,7 @@ def create_user_token(sender, instance=None, created=False, **kwargs): instance: :attr:`~django.contrib.auth.models.User` The newly created user model data - created : boolean + created : Boolean Wether the object was created (True) or updated (False). """ if created: diff --git a/webservice/apps/synthea/apps.py b/webservice/apps/synthea/apps.py index bfe5f91..55eb0ed 100644 --- a/webservice/apps/synthea/apps.py +++ b/webservice/apps/synthea/apps.py @@ -27,21 +27,12 @@ class SyntheaConfig(AppConfig): # We only load this setting, if it is not available in the overall settings.py file settings.SYNTHEA_MODULE_DIR = settings.SYNTHEA_BASE_DIR / 'src/main/resources/modules/' - try: assert settings.SYNTHEA_RESOURCE_DIR except AttributeError: # We only load this setting, if it is not available in the overall settings.py file settings.SYNTHEA_RESOURCE_DIR = settings.SYNTHEA_BASE_DIR / 'src/main/resources/' - try: - assert settings.SYNTHEA_STATES_DIR - except AttributeError: - # We only load this setting, if it is not available in the overall settings.py file - settings.SYNTHEA_STATES_DIR = settings.SYNTHEA_BASE_DIR / 'src/main/resources/geography/' - - - try: assert settings.SYNTHEA_EXPORT_TYPE except AttributeError: diff --git a/webservice/apps/synthea/lib/utils.py b/webservice/apps/synthea/lib/utils.py index 57b1452..bb8a79a 100644 --- a/webservice/apps/synthea/lib/utils.py +++ b/webservice/apps/synthea/lib/utils.py @@ -8,20 +8,31 @@ import json from django.conf import settings from uuid import uuid4 - def available_states(): + """This method will return a sorted list of available states based on the 'geography/demographics.csv' in the `settings.SYNTHEA_RESOURCE_DIR` folder. + + Returns: + List: Sorted list on state name with dicts holding the id and name of the state. + """ + states = [] # Read the demographics.csv file from the Synthea resources and get all the unique state names # Important, the state name for synthea is case sensitive (field id) - df = pd.read_csv(settings.SYNTHEA_STATES_DIR / 'demographics.csv', index_col=False) + df = pd.read_csv(settings.SYNTHEA_RESOURCE_DIR / 'geography/demographics.csv', index_col=False) for state in df.STNAME.unique(): states.append({'id' : state , 'name' : state}) # Sort on name - states = sorted(states, key=lambda k: k['name'].lower()) + states = sorted(states, key=lambda s: s['name'].lower()) return states def available_modules(): + """This method will load all the available modules that are in the folder `settings.SYNTHEA_MODULE_DIR`. Only files ending on .json will be loaded. + + Returns: + List: Sorted list on module name with dicts holding the id and name of the module. + """ + # Assumption here: Only .json files in the main folder are modules. The rest are submodules... modules = [] for module in settings.SYNTHEA_MODULE_DIR.iterdir(): @@ -29,10 +40,29 @@ def available_modules(): data = json.loads(module.read_text()) modules.append({'id' : module.name.replace('.json',''), 'name' : data['name']}) - modules = sorted(modules, key=lambda k: k['name'].lower()) + modules = sorted(modules, key=lambda m: m['name'].lower()) return modules -def run_synthea(state = None, population = None, gender = None, age = None, module = None): +def run_synthea(state, population = 50, gender = None, age = None, module = None): + """This module will run the Synthea application on the background. This method expects Synthea to be installed on the `settings.SYNTHEA_BASE_DIR` location. + + The output will be written to a unique folder in `settings.SYNTHEA_OUTPUT_DIR` that will be zipped and returned. + + It will return the log and the zipfile location for futher processing. The zip file will not be deleted afterwards. So cleanup needs to be done manually. + + Args: + state (str, required): The state where to generate synthetic patient data for. + population (int, optional): The amount of patients to generate. Defaults to 50. + gender (str, optional): Either generate only male(m), only female(f), or None for both. Defaults to None. + age (str, optional): This is the age range of the generated patients. Input is always like [min_age]-[max_age]. Defaults to None. + module (str, optional): The module to use for generating patient data When None, all modules are used. Defaults to None. + + Raises: + Exception: When the Synthea run fails it will return an Exception witht he Java error in it. + + Returns: + (str,Path): The returning zipfile has the enabled options in the file name. + """ # Add a unique dir to the output, so multiple Synthea processes can run parallel temp_id = uuid4().hex output_folder = settings.SYNTHEA_OUTPUT_DIR / temp_id diff --git a/webservice/apps/synthea/migrations/0001_initial.py b/webservice/apps/synthea/migrations/0001_initial.py index 3221990..5373439 100644 --- a/webservice/apps/synthea/migrations/0001_initial.py +++ b/webservice/apps/synthea/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 3.1.3 on 2020-11-13 09:36 +# Generated by Django 3.1.3 on 2020-11-27 11:47 from django.db import migrations, models import uuid @@ -18,15 +18,16 @@ class Migration(migrations.Migration): ('created_at', models.DateTimeField(auto_now_add=True, help_text='The date and time this model has been created', verbose_name='Date created')), ('updated_at', models.DateTimeField(auto_now=True, help_text='The date and time this model has been updated', verbose_name='Date updated')), ('id', models.UUIDField(default=uuid.uuid4, editable=False, help_text='A unique id', primary_key=True, serialize=False, unique=True, verbose_name='ID')), - ('state', models.CharField(help_text='The state for which synthea generate data.', max_length=200, verbose_name='Stage')), + ('state', models.CharField(help_text='The state for which synthea generate data.', max_length=200, verbose_name='State')), ('population', models.PositiveSmallIntegerField(default=50, help_text='The size of the population', verbose_name='Population')), - ('gender', models.CharField(help_text='Select the gender type', max_length=1, verbose_name='Gender')), - ('age', models.CharField(help_text='Select the age range', max_length=10, verbose_name='Age range')), - ('module', models.CharField(help_text='Select the module', max_length=50, verbose_name='Mopdule')), + ('gender', models.CharField(blank=True, help_text='Select the gender type', max_length=1, verbose_name='Gender')), + ('age', models.CharField(blank=True, default='18-100', help_text='Select the age range. Enter [min age]-[max age]', max_length=10, verbose_name='Age range')), + ('module', models.CharField(blank=True, help_text='Select the module', max_length=50, verbose_name='Module')), + ('log', models.TextField(blank=True, help_text='Synthea logfile output', verbose_name='Log')), ], options={ - 'verbose_name': 'token', - 'verbose_name_plural': 'tokens', + 'verbose_name': 'synthea', + 'verbose_name_plural': 'synthea', }, ), ] diff --git a/webservice/apps/synthea/migrations/0002_auto_20201116_1457.py b/webservice/apps/synthea/migrations/0002_auto_20201116_1457.py deleted file mode 100644 index 09bc913..0000000 --- a/webservice/apps/synthea/migrations/0002_auto_20201116_1457.py +++ /dev/null @@ -1,38 +0,0 @@ -# Generated by Django 3.1.3 on 2020-11-16 13:57 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('synthea', '0001_initial'), - ] - - operations = [ - migrations.AddField( - model_name='synthea', - name='log', - field=models.TextField(blank=True, help_text='Synthea logfile output', verbose_name='Log'), - ), - migrations.AlterField( - model_name='synthea', - name='age', - field=models.CharField(blank=True, default='18-100', help_text='Select the age range. Enter [min age]-[max age]', max_length=10, verbose_name='Age range'), - ), - migrations.AlterField( - model_name='synthea', - name='gender', - field=models.CharField(blank=True, help_text='Select the gender type', max_length=1, verbose_name='Gender'), - ), - migrations.AlterField( - model_name='synthea', - name='module', - field=models.CharField(blank=True, help_text='Select the module', max_length=50, verbose_name='Module'), - ), - migrations.AlterField( - model_name='synthea', - name='state', - field=models.CharField(help_text='The state for which synthea generate data.', max_length=200, verbose_name='State'), - ), - ] diff --git a/webservice/apps/synthea/models.py b/webservice/apps/synthea/models.py index 755f071..6294def 100644 --- a/webservice/apps/synthea/models.py +++ b/webservice/apps/synthea/models.py @@ -13,10 +13,32 @@ import uuid # Create your models here. class Synthea(MetaDataModel): + """Synthea model that holds some iformation about a generated output. + + Attributes + ---------- + id : uuid + A unique ID for every Synthea run. Leave empty for auto generating a new value. + state : str + The state for which you want to generate Synthea patient data + population : int + The amount of patients you want to generate + gender : datetime + Generate only male (m), only female(f) or leave empty for male and female + age : str + The age range for the patients. Enter like [min_age]-[max_age] + module : str + The module to use for patient generating. Leave empty for random use by Synthea + log : str + The outcome of a single patient generating run. This can be read in the admin area. + + Returns: + Synthea: A new Synthea model + """ class Meta: - verbose_name = _('token') - verbose_name_plural = _('tokens') + verbose_name = _('synthea') + verbose_name_plural = _('synthea') id = models.UUIDField(_('ID'), primary_key=True, unique=True, default=uuid.uuid4, editable=False, help_text=_('A unique id')) state = models.CharField(_('State'), max_length=200, help_text=_('The state for which synthea generate data.')) @@ -26,8 +48,15 @@ class Synthea(MetaDataModel): module = models.CharField(_('Module'),blank=True, max_length=50, help_text=_('Select the module')) log = models.TextField(_('Log'),blank=True, help_text=_('Synthea logfile output')) - def generate(self): + """Run the patient generation. This will return a logfile and a zipfile location for download. + + The log will be stored in the model when done. This log can then be seen/readed in the admin section of Django + + Returns: + str: The zip file location on disk. + """ + # Start generating patient data. log,zip_file = run_synthea( self.state, self.population, @@ -35,10 +64,9 @@ class Synthea(MetaDataModel): self.age, self.module ) - + # Store the log from the Synthea run in the database. self.log = log - self.save() - + # Return the zip file locaton for download return zip_file diff --git a/webservice/lib/utils/emails.py b/webservice/lib/utils/emails.py index ac545e3..cea3c4f 100644 --- a/webservice/lib/utils/emails.py +++ b/webservice/lib/utils/emails.py @@ -8,15 +8,30 @@ from django.conf import settings # Source: https://djangosnippets.org/snippets/2215/ class EmailMultiRelated(EmailMultiAlternatives): - """ - A version of EmailMessage that makes it easy to send multipart/related + """A version of EmailMessage that makes it easy to send multipart/related messages. For example, including text and HTML versions with inline images. + + Returns: + EmailMultiAlternatives: EmailMultiAlternatives class """ + related_subtype = 'related' def __init__(self, subject='', body='', from_email=None, to=None, bcc=None, connection=None, attachments=None, headers=None, alternatives=None): - # self.related_ids = [] + """Create a new email object that can holds text and HTML content. + + Args: + subject (str, optional): The subject of the email. Defaults to ''. + body (str, optional): The body of the email. Defaults to ''. + from_email (str, optional): [description]. Defaults to None. + to ([type], optional): [description]. Defaults to None. + bcc ([type], optional): [description]. Defaults to None. + connection ([type], optional): [description]. Defaults to None. + attachments ([type], optional): [description]. Defaults to None. + headers ([type], optional): [description]. Defaults to None. + alternatives ([type], optional): [description]. Defaults to None. + """ self.related_attachments = [] super(EmailMultiRelated, self).__init__(subject, body, from_email, to, bcc, connection, attachments, headers, alternatives) @@ -27,7 +42,13 @@ class EmailMultiRelated(EmailMultiAlternatives): If the first parameter is a MIMEBase subclass it is inserted directly into the resulting message attachments. + + Args: + filename ([type], optional): [description]. Defaults to None. + content ([type], optional): [description]. Defaults to None. + mimetype ([type], optional): [description]. Defaults to None. """ + if isinstance(filename, MIMEBase): assert content == mimetype == None self.related_attachments.append(filename) diff --git a/webservice/lib/utils/general.py b/webservice/lib/utils/general.py index 05cce2c..60f9ccd 100644 --- a/webservice/lib/utils/general.py +++ b/webservice/lib/utils/general.py @@ -2,25 +2,67 @@ import re import random import string -def remove_html_tags(text): - """Remove html tags from a string""" - clean = re.compile('<.*?>') - return re.sub(clean, '', text) - -def get_random_int_value(length = 6): - return ''.join(list(map(lambda x: str(random.randint(1,9)), list(range(length))))) - -def get_random_string(length = 8): - return ''.join(random.choices(string.ascii_uppercase + string.digits + string.ascii_lowercase, k=length)) - def generate_encryption_key(length = 32): + """Generate a new encryption key of `length` chars. This is done by using the :func:`get_random_string` function with a default of 32 chars. + + Args: + length (int, optional): The length in chars of the encryption key. Defaults to 32. + + Returns: + str: A string of `length` chars. + """ return get_random_string(length) def get_ip_address(request): - """ use requestobject to fetch client machine's IP Address """ + """Get the IP address of the requesting viewer. This is done by looking into the following variables in the headers. + + 1. HTTP_X_FORWARDED_FOR + 2. REMOTE_ADDR + + Args: + request (BaseRequest): The Django request. + + Returns: + str: IP address of the request + """ x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR', None) if x_forwarded_for: ip = x_forwarded_for.split(',')[0] else: ip = request.META.get('REMOTE_ADDR') ### Real IP address of client Machine - return ip \ No newline at end of file + return ip + +def get_random_int_value(length = 6): + """Generate a random number of `length` length numbers. + + Args: + length (int, optional): The length of the random number in amount of numbers. Defaults to 6. + + Returns: + int: Returns a random number of 'length' numbers. + """ + return int(''.join(list(map(lambda x: str(random.randint(1,9)), list(range(length)))))) + +def get_random_string(length = 8): + """Generate a random string of `length` length characters. + + Args: + length (int, optional): The length of the random string in amount of characters. Defaults to 8. + + Returns: + str: Returns a random string of 'length' characters. + """ + return ''.join(random.choices(string.ascii_uppercase + string.digits + string.ascii_lowercase, k=length)) + +def remove_html_tags(text): + """Remove HTML tags and code from the input text + + Args: + text (str): Input text to be cleaned from HTML + + Returns: + str: Cleaned HTML. + """ + + clean = re.compile('<.*?>') + return re.sub(clean, '', text) \ No newline at end of file diff --git a/webservice/webservice/.env.example b/webservice/webservice/.env.example index 5c59f24..ea37971 100644 --- a/webservice/webservice/.env.example +++ b/webservice/webservice/.env.example @@ -1,8 +1,8 @@ # A uniquely secret key -SECRET_KEY=@wb=#(f4uc0l%e!5*eo+aoflnxb(@!l9!=c5w=4b+x$=!8&vy%' +SECRET_KEY=@wb=#(f4vc0l(e!5*eo+a@flnxb2@!l9!=c6w=4b+x$=!8&vy%' # Disable debug in production -DEBUG=False +DEBUG=True # Allowed hosts that Django does server. Take care when NGINX is proxying infront of Django ALLOWED_HOSTS=127.0.0.1,localhost @@ -11,7 +11,7 @@ ALLOWED_HOSTS=127.0.0.1,localhost INTERNAL_IPS=127.0.0.1 # Enter the database url connection: https://github.com/jacobian/dj-database-url -DATABASE_URL=sqlite:////opt/deploy/VRE/VirtualResearchEnvironment/db.sqlite3 +DATABASE_URL=sqlite:////opt/deploy/synthea_webservice/webservice/db.sqlite3 # The location on disk where the static files will be placed during deployment. Setting is required STATIC_ROOT= @@ -25,29 +25,32 @@ TIME_ZONE=Europe/Amsterdam EMAIL_HOST= # Email user name -EMAIL_HOST_USER= +EMAIL_HOST_USER=na # Email password -EMAIL_HOST_PASSWORD= +EMAIL_HOST_PASSWORD=na # Email server port number to use EMAIL_PORT=25 # Does the email server supports TLS? -EMAIL_USE_TLS= +EMAIL_USE_TLS=yes # The sender address. This needs to be one of the allowed domains due to SPF checks # The code will use a reply-to header to make sure that replies goes to the researcher and not this address EMAIL_FROM_ADDRESS=Do not reply -# What is the Dropoff hostname (webinterface) -DROPOFF_HOSTNAME=http://localhost:8000 +# The base folder where Synthea is installed. This folder should contain the file 'run_synthea.(|bat)' +settings.SYNTHEA_BASE_DIR = settings.BASE_DIR / '../synthea' -# What is the Dropoff Upload host -DROPOFF_UPLOAD_HOST=http://localhost +# The base output folder where Synthea will generate its output. This location will be appended with a unique folder name. +settings.SYNTHEA_OUTPUT_DIR = settings.BASE_DIR / '../synthea_output' -# Which file extensions are **NOT** allowed -DROPOFF_NOT_ALLOWED_EXTENSIONS=exe,com,bat,lnk,sh +# The location where the Synthea modules are located. +settings.SYNTHEA_MODULE_DIR = settings.SYNTHEA_BASE_DIR / 'src/main/resources/modules/' -# What is the full VRE Portal domains -VRE_BROKER_API=http://localhost:8000 \ No newline at end of file +# The location where the Synthea resources are located. This should also include the geography data. +settings.SYNTHEA_RESOURCE_DIR = settings.SYNTHEA_BASE_DIR / 'src/main/resources/' + +# The output type for Synthea. +settings.SYNTHEA_EXPORT_TYPE = 'fhir_stu3' \ No newline at end of file