P300021/synthea_webservice
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add documentation

Browse Source
This commit is contained in:
Joshua Rubingh
2020-11-27 12:49:03 +01:00
parent e4c51874dc
commit c9b94ed6c6
26 changed files with 743 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
webservice/apps/api/migrations/0001_initial.py
View File

@@ -1,4 +1,4 @@
# Generated by Django 3.0.8 on 2020-07-30 14:15
# Generated by Django 3.1.3 on 2020-11-27 11:47
import apps.api.models
from django.conf import settings

2
webservice/apps/api/models.py
View File

@@ -55,7 +55,7 @@ class Token(MetaDataModel):
"""Boolean check if the token is belonging to a user with super user rights. Then this token is a super token.
Returns:
bool: Returns true when the token belongs to a super user.
Boolean: Returns true when the token belongs to a super user.
"""
# TODO: Is it allowed to be a super user and researcher? Could give conflict of interests. With the API token you can read other researchers data...
return self.user.is_superuser == True

2
webservice/apps/api/signals.py
View File

@@ -17,7 +17,7 @@ def create_user_token(sender, instance=None, created=False, **kwargs):
instance: :attr:`~django.contrib.auth.models.User`
The newly created user model data
created : boolean
created : Boolean
Wether the object was created (True) or updated (False).
"""
if created:

9
webservice/apps/synthea/apps.py
View File

@@ -27,21 +27,12 @@ class SyntheaConfig(AppConfig):
# We only load this setting, if it is not available in the overall settings.py file
settings.SYNTHEA_MODULE_DIR = settings.SYNTHEA_BASE_DIR / 'src/main/resources/modules/'
try:
assert settings.SYNTHEA_RESOURCE_DIR
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.SYNTHEA_RESOURCE_DIR = settings.SYNTHEA_BASE_DIR / 'src/main/resources/'
try:
assert settings.SYNTHEA_STATES_DIR
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.SYNTHEA_STATES_DIR = settings.SYNTHEA_BASE_DIR / 'src/main/resources/geography/'
try:
assert settings.SYNTHEA_EXPORT_TYPE
except AttributeError:

40
webservice/apps/synthea/lib/utils.py
View File

@@ -8,20 +8,31 @@ import json
from django.conf import settings
from uuid import uuid4
def available_states():
"""This method will return a sorted list of available states based on the 'geography/demographics.csv' in the `settings.SYNTHEA_RESOURCE_DIR` folder.
Returns:
List: Sorted list on state name with dicts holding the id and name of the state.
"""
states = []
# Read the demographics.csv file from the Synthea resources and get all the unique state names
# Important, the state name for synthea is case sensitive (field id)
df = pd.read_csv(settings.SYNTHEA_STATES_DIR / 'demographics.csv', index_col=False)
df = pd.read_csv(settings.SYNTHEA_RESOURCE_DIR / 'geography/demographics.csv', index_col=False)
for state in df.STNAME.unique():
states.append({'id' : state , 'name' : state})
# Sort on name
states = sorted(states, key=lambda k: k['name'].lower())
states = sorted(states, key=lambda s: s['name'].lower())
return states
def available_modules():
"""This method will load all the available modules that are in the folder `settings.SYNTHEA_MODULE_DIR`. Only files ending on .json will be loaded.
Returns:
List: Sorted list on module name with dicts holding the id and name of the module.
"""
# Assumption here: Only .json files in the main folder are modules. The rest are submodules...
modules = []
for module in settings.SYNTHEA_MODULE_DIR.iterdir():
@@ -29,10 +40,29 @@ def available_modules():
data = json.loads(module.read_text())
modules.append({'id' : module.name.replace('.json',''), 'name' : data['name']})
modules = sorted(modules, key=lambda k: k['name'].lower())
modules = sorted(modules, key=lambda m: m['name'].lower())
return modules
def run_synthea(state = None, population = None, gender = None, age = None, module = None):
def run_synthea(state, population = 50, gender = None, age = None, module = None):
"""This module will run the Synthea application on the background. This method expects Synthea to be installed on the `settings.SYNTHEA_BASE_DIR` location.
The output will be written to a unique folder in `settings.SYNTHEA_OUTPUT_DIR` that will be zipped and returned.
It will return the log and the zipfile location for futher processing. The zip file will not be deleted afterwards. So cleanup needs to be done manually.
Args:
state (str, required): The state where to generate synthetic patient data for.
population (int, optional): The amount of patients to generate. Defaults to 50.
gender (str, optional): Either generate only male(m), only female(f), or None for both. Defaults to None.
age (str, optional): This is the age range of the generated patients. Input is always like [min_age]-[max_age]. Defaults to None.
module (str, optional): The module to use for generating patient data When None, all modules are used. Defaults to None.
Raises:
Exception: When the Synthea run fails it will return an Exception witht he Java error in it.
Returns:
(str,Path): The returning zipfile has the enabled options in the file name.
"""
# Add a unique dir to the output, so multiple Synthea processes can run parallel
temp_id = uuid4().hex
output_folder = settings.SYNTHEA_OUTPUT_DIR / temp_id

15
webservice/apps/synthea/migrations/0001_initial.py
View File

@@ -1,4 +1,4 @@
# Generated by Django 3.1.3 on 2020-11-13 09:36
# Generated by Django 3.1.3 on 2020-11-27 11:47
from django.db import migrations, models
import uuid
@@ -18,15 +18,16 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='The date and time this model has been created', verbose_name='Date created')),
('updated_at', models.DateTimeField(auto_now=True, help_text='The date and time this model has been updated', verbose_name='Date updated')),
('id', models.UUIDField(default=uuid.uuid4, editable=False, help_text='A unique id', primary_key=True, serialize=False, unique=True, verbose_name='ID')),
('state', models.CharField(help_text='The state for which synthea generate data.', max_length=200, verbose_name='Stage')),
('state', models.CharField(help_text='The state for which synthea generate data.', max_length=200, verbose_name='State')),
('population', models.PositiveSmallIntegerField(default=50, help_text='The size of the population', verbose_name='Population')),
('gender', models.CharField(help_text='Select the gender type', max_length=1, verbose_name='Gender')),
('age', models.CharField(help_text='Select the age range', max_length=10, verbose_name='Age range')),
('module', models.CharField(help_text='Select the module', max_length=50, verbose_name='Mopdule')),
('gender', models.CharField(blank=True, help_text='Select the gender type', max_length=1, verbose_name='Gender')),
('age', models.CharField(blank=True, default='18-100', help_text='Select the age range. Enter [min age]-[max age]', max_length=10, verbose_name='Age range')),
('module', models.CharField(blank=True, help_text='Select the module', max_length=50, verbose_name='Module')),
('log', models.TextField(blank=True, help_text='Synthea logfile output', verbose_name='Log')),
],
options={
'verbose_name': 'token',
'verbose_name_plural': 'tokens',
'verbose_name': 'synthea',
'verbose_name_plural': 'synthea',
},
),
]

38
webservice/apps/synthea/migrations/0002_auto_20201116_1457.py
View File

@@ -1,38 +0,0 @@
# Generated by Django 3.1.3 on 2020-11-16 13:57
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('synthea', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='synthea',
name='log',
field=models.TextField(blank=True, help_text='Synthea logfile output', verbose_name='Log'),
),
migrations.AlterField(
model_name='synthea',
name='age',
field=models.CharField(blank=True, default='18-100', help_text='Select the age range. Enter [min age]-[max age]', max_length=10, verbose_name='Age range'),
),
migrations.AlterField(
model_name='synthea',
name='gender',
field=models.CharField(blank=True, help_text='Select the gender type', max_length=1, verbose_name='Gender'),
),
migrations.AlterField(
model_name='synthea',
name='module',
field=models.CharField(blank=True, help_text='Select the module', max_length=50, verbose_name='Module'),
),
migrations.AlterField(
model_name='synthea',
name='state',
field=models.CharField(help_text='The state for which synthea generate data.', max_length=200, verbose_name='State'),
),
]

40
webservice/apps/synthea/models.py
View File

@@ -13,10 +13,32 @@ import uuid
# Create your models here.
class Synthea(MetaDataModel):
"""Synthea model that holds some iformation about a generated output.
Attributes
----------
id : uuid
A unique ID for every Synthea run. Leave empty for auto generating a new value.
state : str
The state for which you want to generate Synthea patient data
population : int
The amount of patients you want to generate
gender : datetime
Generate only male (m), only female(f) or leave empty for male and female
age : str
The age range for the patients. Enter like [min_age]-[max_age]
module : str
The module to use for patient generating. Leave empty for random use by Synthea
log : str
The outcome of a single patient generating run. This can be read in the admin area.
Returns:
Synthea: A new Synthea model
"""
class Meta:
verbose_name = _('token')
verbose_name_plural = _('tokens')
verbose_name = _('synthea')
verbose_name_plural = _('synthea')
id = models.UUIDField(_('ID'), primary_key=True, unique=True, default=uuid.uuid4, editable=False, help_text=_('A unique id'))
state = models.CharField(_('State'), max_length=200, help_text=_('The state for which synthea generate data.'))
@@ -26,8 +48,15 @@ class Synthea(MetaDataModel):
module = models.CharField(_('Module'),blank=True, max_length=50, help_text=_('Select the module'))
log = models.TextField(_('Log'),blank=True, help_text=_('Synthea logfile output'))
def generate(self):
"""Run the patient generation. This will return a logfile and a zipfile location for download.
The log will be stored in the model when done. This log can then be seen/readed in the admin section of Django
Returns:
str: The zip file location on disk.
"""
# Start generating patient data.
log,zip_file = run_synthea(
self.state,
self.population,
@@ -35,10 +64,9 @@ class Synthea(MetaDataModel):
self.age,
self.module
)
# Store the log from the Synthea run in the database.
self.log = log
self.save()
# Return the zip file locaton for download
return zip_file

27
webservice/lib/utils/emails.py
View File

@@ -8,15 +8,30 @@ from django.conf import settings
# Source: https://djangosnippets.org/snippets/2215/
class EmailMultiRelated(EmailMultiAlternatives):
"""
A version of EmailMessage that makes it easy to send multipart/related
"""A version of EmailMessage that makes it easy to send multipart/related
messages. For example, including text and HTML versions with inline images.
Returns:
EmailMultiAlternatives: EmailMultiAlternatives class
"""
related_subtype = 'related'
def __init__(self, subject='', body='', from_email=None, to=None, bcc=None,
connection=None, attachments=None, headers=None, alternatives=None):
# self.related_ids = []
"""Create a new email object that can holds text and HTML content.
Args:
subject (str, optional): The subject of the email. Defaults to ''.
body (str, optional): The body of the email. Defaults to ''.
from_email (str, optional): [description]. Defaults to None.
to ([type], optional): [description]. Defaults to None.
bcc ([type], optional): [description]. Defaults to None.
connection ([type], optional): [description]. Defaults to None.
attachments ([type], optional): [description]. Defaults to None.
headers ([type], optional): [description]. Defaults to None.
alternatives ([type], optional): [description]. Defaults to None.
"""
self.related_attachments = []
super(EmailMultiRelated, self).__init__(subject, body, from_email, to, bcc, connection, attachments, headers, alternatives)
@@ -27,7 +42,13 @@ class EmailMultiRelated(EmailMultiAlternatives):
If the first parameter is a MIMEBase subclass it is inserted directly
into the resulting message attachments.
Args:
filename ([type], optional): [description]. Defaults to None.
content ([type], optional): [description]. Defaults to None.
mimetype ([type], optional): [description]. Defaults to None.
"""
if isinstance(filename, MIMEBase):
assert content == mimetype == None
self.related_attachments.append(filename)

68
webservice/lib/utils/general.py
View File

@@ -2,25 +2,67 @@ import re
import random
import string
def remove_html_tags(text):
"""Remove html tags from a string"""
clean = re.compile('<.*?>')
return re.sub(clean, '', text)
def get_random_int_value(length = 6):
return ''.join(list(map(lambda x: str(random.randint(1,9)), list(range(length)))))
def get_random_string(length = 8):
return ''.join(random.choices(string.ascii_uppercase + string.digits + string.ascii_lowercase, k=length))
def generate_encryption_key(length = 32):
"""Generate a new encryption key of `length` chars. This is done by using the :func:`get_random_string` function with a default of 32 chars.
Args:
length (int, optional): The length in chars of the encryption key. Defaults to 32.
Returns:
str: A string of `length` chars.
"""
return get_random_string(length)
def get_ip_address(request):
""" use requestobject to fetch client machine's IP Address """
"""Get the IP address of the requesting viewer. This is done by looking into the following variables in the headers.
1. HTTP_X_FORWARDED_FOR
2. REMOTE_ADDR
Args:
request (BaseRequest): The Django request.
Returns:
str: IP address of the request
"""
x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR', None)
if x_forwarded_for:
ip = x_forwarded_for.split(',')[0]
else:
ip = request.META.get('REMOTE_ADDR') ### Real IP address of client Machine
return ip
return ip
def get_random_int_value(length = 6):
"""Generate a random number of `length` length numbers.
Args:
length (int, optional): The length of the random number in amount of numbers. Defaults to 6.
Returns:
int: Returns a random number of 'length' numbers.
"""
return int(''.join(list(map(lambda x: str(random.randint(1,9)), list(range(length))))))
def get_random_string(length = 8):
"""Generate a random string of `length` length characters.
Args:
length (int, optional): The length of the random string in amount of characters. Defaults to 8.
Returns:
str: Returns a random string of 'length' characters.
"""
return ''.join(random.choices(string.ascii_uppercase + string.digits + string.ascii_lowercase, k=length))
def remove_html_tags(text):
"""Remove HTML tags and code from the input text
Args:
text (str): Input text to be cleaned from HTML
Returns:
str: Cleaned HTML.
"""
clean = re.compile('<.*?>')
return re.sub(clean, '', text)

31
webservice/webservice/.env.example
View File

@@ -1,8 +1,8 @@
# A uniquely secret key
SECRET_KEY=@wb=#(f4uc0l%e!5*eo+aoflnxb(@!l9!=c5w=4b+x$=!8&vy%'
SECRET_KEY=@wb=#(f4vc0l(e!5*eo+a@flnxb2@!l9!=c6w=4b+x$=!8&vy%'
# Disable debug in production
DEBUG=False
DEBUG=True
# Allowed hosts that Django does server. Take care when NGINX is proxying infront of Django
ALLOWED_HOSTS=127.0.0.1,localhost
@@ -11,7 +11,7 @@ ALLOWED_HOSTS=127.0.0.1,localhost
INTERNAL_IPS=127.0.0.1
# Enter the database url connection: https://github.com/jacobian/dj-database-url
DATABASE_URL=sqlite:////opt/deploy/VRE/VirtualResearchEnvironment/db.sqlite3
DATABASE_URL=sqlite:////opt/deploy/synthea_webservice/webservice/db.sqlite3
# The location on disk where the static files will be placed during deployment. Setting is required
STATIC_ROOT=
@@ -25,29 +25,32 @@ TIME_ZONE=Europe/Amsterdam
EMAIL_HOST=
# Email user name
EMAIL_HOST_USER=
EMAIL_HOST_USER=na
# Email password
EMAIL_HOST_PASSWORD=
EMAIL_HOST_PASSWORD=na
# Email server port number to use
EMAIL_PORT=25
# Does the email server supports TLS?
EMAIL_USE_TLS=
EMAIL_USE_TLS=yes
# The sender address. This needs to be one of the allowed domains due to SPF checks
# The code will use a reply-to header to make sure that replies goes to the researcher and not this address
EMAIL_FROM_ADDRESS=Do not reply<no-reply@rug.nl>
# What is the Dropoff hostname (webinterface)
DROPOFF_HOSTNAME=http://localhost:8000
# The base folder where Synthea is installed. This folder should contain the file 'run_synthea.(|bat)'
settings.SYNTHEA_BASE_DIR = settings.BASE_DIR / '../synthea'
# What is the Dropoff Upload host
DROPOFF_UPLOAD_HOST=http://localhost
# The base output folder where Synthea will generate its output. This location will be appended with a unique folder name.
settings.SYNTHEA_OUTPUT_DIR = settings.BASE_DIR / '../synthea_output'
# Which file extensions are **NOT** allowed
DROPOFF_NOT_ALLOWED_EXTENSIONS=exe,com,bat,lnk,sh
# The location where the Synthea modules are located.
settings.SYNTHEA_MODULE_DIR = settings.SYNTHEA_BASE_DIR / 'src/main/resources/modules/'
# What is the full VRE Portal domains
VRE_BROKER_API=http://localhost:8000
# The location where the Synthea resources are located. This should also include the geography data.
settings.SYNTHEA_RESOURCE_DIR = settings.SYNTHEA_BASE_DIR / 'src/main/resources/'
# The output type for Synthea.
settings.SYNTHEA_EXPORT_TYPE = 'fhir_stu3'