P300021/synthea_webservice
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Another update

Browse Source
This commit is contained in:
Joshua Rubingh
2020-11-16 16:30:41 +01:00
parent 05491790df
commit 80711cd918
16 changed files with 283 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
webservice/apps/synthea/api/serializers.py Normal file
View File

@@ -0,0 +1,8 @@
from rest_framework import serializers
from apps.synthea.models import Synthea
class SyntheaSerializer(serializers.ModelSerializer):
class Meta:
model = Synthea
fields = ['state', 'population', 'gender', 'age', 'module']

41
webservice/apps/synthea/apps.py
View File

@@ -7,4 +7,43 @@ class SyntheaConfig(AppConfig):
name = 'apps.synthea'
label = 'synthea'
verbose_name = _('Synthea')
verbose_name_plural = _('Synthea')
verbose_name_plural = _('Synthea')
try:
assert settings.SYNTHEA_BASE_DIR
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.SYNTHEA_BASE_DIR = settings.BASE_DIR / '../synthea'
try:
assert settings.SYNTHEA_OUTPUT_DIR
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.SYNTHEA_OUTPUT_DIR = settings.BASE_DIR / '../synthea_output'
try:
assert settings.SYNTHEA_MODULE_DIR
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.SYNTHEA_MODULE_DIR = settings.SYNTHEA_BASE_DIR / 'src/main/resources/modules/'
try:
assert settings.SYNTHEA_RESOURCE_DIR
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.SYNTHEA_RESOURCE_DIR = settings.SYNTHEA_BASE_DIR / 'src/main/resources/'
try:
assert settings.SYNTHEA_STATES_DIR
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.SYNTHEA_STATES_DIR = settings.SYNTHEA_BASE_DIR / 'src/main/resources/geography/'
try:
assert settings.SYNTHEA_EXPORT_TYPE
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.SYNTHEA_EXPORT_TYPE = 'fhir_stu3'

12
webservice/apps/synthea/forms.py
View File

@@ -1,3 +1,5 @@
from django.utils.translation import gettext_lazy as _
from django.forms import ModelForm
from django import forms
@@ -12,16 +14,16 @@ class SyntheaForm(ModelForm):
fields = ['state', 'population', 'gender', 'age', 'module']
# This is loaded only once during startup. So changing the state data will not be picked up after a restart
state_options = [('','Any')]
state_options = [('',_('Select'))]
for item in available_states():
state_options.append((item,item))
state_options.append((item['id'],item['name']))
module_options = [('','Any')]
module_options = [('',_('Any'))]
for item in available_modules():
module_options.append((item['module'],item['name']))
module_options.append((item['id'],item['name']))
widgets = {
'state': forms.Select(choices=state_options),
'gender': forms.Select(choices=[('','Any'),('m','Male'),('f','Female')]),
'gender': forms.Select(choices=[('',_('Any')),('m',_('Male')),('f',_('Female'))]),
'module': forms.Select(choices=module_options)
}

53
webservice/apps/synthea/lib/utils.py
View File

@@ -5,36 +5,41 @@ import subprocess
from zipfile import ZipFile
import json
def available_states():
#TODO: Make a setting for this path
location = Path('/opt/development/synthea_webservice/synthea/src/main/resources/geography/')
from django.conf import settings
from uuid import uuid4
df = pd.read_csv(location / 'timezones.csv', index_col=False)
# The state information is expected in the first column
states = df[df.columns[0]].to_list()
states.sort()
def available_states():
states = []
# Read the timezones.csv file from the Synthea resources. This should give us all the 'state' on the first column
df = pd.read_csv(settings.SYNTHEA_STATES_DIR / 'timezones.csv', index_col=False)
for state in df[df.columns[0]].to_list():
states.append({'id' : state , 'name' : state})
#states = df[df.columns[0]].to_list()
# Sort on name
states = sorted(states, key=lambda k: k['name'].lower())
#states.sort()
return states
def available_modules():
#TODO: Make a setting for this path
location = Path('/opt/development/synthea_webservice/synthea/src/main/resources/modules/')
# Assumption here: A folder is a single module. And all .json in the main modules folder is a module.
# Assumption here: Only .json files in the main folder are modules. The rest are submodules...
modules = []
for module in location.iterdir():
for module in settings.SYNTHEA_MODULE_DIR.iterdir():
if module.is_file() and module.suffix == '.json':
data = json.loads(module.read_text())
modules.append({'module' : module.name.replace('.json',''), 'name' : data['name']})
modules.append({'id' : module.name.replace('.json',''), 'name' : data['name']})
modules = sorted(modules, key=lambda k: k['name'].lower())
return modules
def run_synthea(state = None, population = None, gender = None, age = None, module = None):
# TODO: Make synthea setting(s)
location = '/opt/development/synthea_webservice/synthea/'
synthea_cmd = ['/opt/development/synthea_webservice/synthea/run_synthea']
# Add a unique dir to the output, so multiple Synthea processes can run parallel
temp_id = uuid4().hex
output_folder = settings.SYNTHEA_OUTPUT_DIR / temp_id
synthea_cmd = [settings.SYNTHEA_BASE_DIR / 'run_synthea','--exporter.baseDirectory',output_folder]
zip_file = 'Synthea_'
zip_export = location
if population:
synthea_cmd.append('-p')
@@ -62,7 +67,7 @@ def run_synthea(state = None, population = None, gender = None, age = None, modu
process_ok = False
log = ''
with subprocess.Popen(synthea_cmd,cwd=location, stdout=subprocess.PIPE,stderr=subprocess.PIPE) as process:
with subprocess.Popen(synthea_cmd,cwd=settings.SYNTHEA_BASE_DIR, stdout=subprocess.PIPE,stderr=subprocess.PIPE) as process:
for line in process.stdout:
line = line.decode('utf8')
log += line
@@ -70,14 +75,10 @@ def run_synthea(state = None, population = None, gender = None, age = None, modu
process_ok = line.find('BUILD SUCCESSFUL') >= 0
if process_ok:
with ZipFile(f'{zip_export}/{zip_file}.zip', 'w') as export:
for file in Path(location + 'output/fhir_stu3').iterdir():
with ZipFile(f'{output_folder}/{zip_file}_{temp_id}.zip', 'w') as export:
for file in (output_folder / settings.SYNTHEA_EXPORT_TYPE).iterdir():
export.write(file,file.name)
return Path(f'{zip_export}/{zip_file}.zip')
return (log,Path(f'{output_folder}/{zip_file}_{temp_id}.zip'))
else:
raise Exception(log)
raise Exception(log)

38
webservice/apps/synthea/migrations/0002_auto_20201116_1457.py Normal file
View File

@@ -0,0 +1,38 @@
# Generated by Django 3.1.3 on 2020-11-16 13:57
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('synthea', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='synthea',
name='log',
field=models.TextField(blank=True, help_text='Synthea logfile output', verbose_name='Log'),
),
migrations.AlterField(
model_name='synthea',
name='age',
field=models.CharField(blank=True, default='18-100', help_text='Select the age range. Enter [min age]-[max age]', max_length=10, verbose_name='Age range'),
),
migrations.AlterField(
model_name='synthea',
name='gender',
field=models.CharField(blank=True, help_text='Select the gender type', max_length=1, verbose_name='Gender'),
),
migrations.AlterField(
model_name='synthea',
name='module',
field=models.CharField(blank=True, help_text='Select the module', max_length=50, verbose_name='Module'),
),
migrations.AlterField(
model_name='synthea',
name='state',
field=models.CharField(help_text='The state for which synthea generate data.', max_length=200, verbose_name='State'),
),
]

24
webservice/apps/synthea/models.py
View File

@@ -7,6 +7,8 @@ from django_cryptography.fields import encrypt
from lib.utils.general import get_random_string
from lib.models.base import MetaDataModel
from .lib.utils import run_synthea
import uuid
# Create your models here.
@@ -18,7 +20,25 @@ class Synthea(MetaDataModel):
id = models.UUIDField(_('ID'), primary_key=True, unique=True, default=uuid.uuid4, editable=False, help_text=_('A unique id'))
state = models.CharField(_('State'), max_length=200, help_text=_('The state for which synthea generate data.'))
population = models.PositiveSmallIntegerField(_('Population'), blank=True, default=50, help_text=_('The size of the population'))
population = models.PositiveSmallIntegerField(_('Population'), default=50, help_text=_('The size of the population'))
gender = models.CharField(_('Gender'), blank=True,max_length=1, help_text=_('Select the gender type'))
age = models.CharField(_('Age range'), blank=True,max_length=10, help_text=_('Select the age range'))
age = models.CharField(_('Age range'), blank=True,default='18-100', max_length=10, help_text=_('Select the age range. Enter [min age]-[max age]'))
module = models.CharField(_('Module'),blank=True, max_length=50, help_text=_('Select the module'))
log = models.TextField(_('Log'),blank=True, help_text=_('Synthea logfile output'))
def generate(self):
log,zip_file = run_synthea(
self.state,
self.population,
self.gender,
self.age,
self.module
)
self.log = log
self.save()
return zip_file

2
webservice/apps/synthea/templates/synthea/generator_form.html
View File

@@ -9,7 +9,7 @@
{% endblock %}
{% block content %}
<h1>Synthea generartor</h1>
<h1>Synthea Generator</h1>
<p>Enter the form fields and press submit. <small>U vraagt, wij draaien ;)</small></p>
<form method="POST" class="post-form">
{% csrf_token %}

1
webservice/apps/synthea/templates/synthea/index.html
View File

@@ -11,4 +11,5 @@
{% block content %}
<h1>dHealt Nederland</h1>
<p>Onder leiding van UMCG</p>
<p>Github branches: https://github.com/dHealthNL </p>
{% endblock %}

22
webservice/apps/synthea/views.py
View File

@@ -1,6 +1,7 @@
from django.shortcuts import render
from django.http import HttpResponse
from apps.synthea.forms import SyntheaForm
from apps.synthea.models import Synthea
from .lib.utils import run_synthea
@@ -21,24 +22,13 @@ def show_synthea_form(request):
# check whether it's valid:
if form.is_valid():
# process the data in form.cleaned_data as required
synthea = form.save()
zipfile = synthea.generate()
try:
zipfile = run_synthea(
form.cleaned_data['state'],
form.cleaned_data['population'],
form.cleaned_data['gender'],
form.cleaned_data['age'],
form.cleaned_data['module']
)
mime_type, _ = mimetypes.guess_type(zipfile)
response = HttpResponse(zipfile.open('rb'), content_type=mime_type)
response['Content-Disposition'] = f'attachment; filename={zipfile.name}'
response = HttpResponse(zipfile.open('rb'), content_type='application/zip')
response['Content-Disposition'] = f'attachment; filename={zipfile.name}'
return response
except Exception as ex:
print(ex)
return response
# if a GET (or any other method) we'll create a blank form
else: