P300021
/
synthea_webservice
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Another update

This commit is contained in:
Joshua Rubingh 2020-11-16 16:30:41 +01:00
parent 05491790df
commit 80711cd918
16 changed files with 283 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -7,3 +7,4 @@ doc/_build/
doc/output/
api_test.py
log/*
synthea_output/

2
requirements.txt
View File

@ -4,4 +4,6 @@ django_js_reverse
python-decouple
django-cryptography
djangorestframework
drf_yasg2
pandas

90
webservice/apps/api/apps.py
View File

@ -9,58 +9,58 @@ class ApiConfig(AppConfig):
verbose_name = _('API')
verbose_name_plural = _('APIs')
try:
assert settings.SWAGGER_SETTINGS
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.SWAGGER_SETTINGS = {
'SECURITY_DEFINITIONS': {
'Hawk': {
'type': 'apiKey',
'description': 'HTTP Holder-Of-Key Authentication Scheme, https://github.com/hapijs/hawk, https://hawkrest.readthedocs.io/en/latest/<br /><strong>Ex header:</strong><br />\'Authorization\': \'Hawk mac="F4+S9cu7yZiZEgdtqzMpOOdudvqcV2V2Yzk2WcphECc=", hash="+7fKUX+djeQolvnLTxr0X47e//UHKbkRlajwMw3tx3w=", id="7FI5JET4", ts="1592905433", nonce="DlV-fL"\'',
'name': 'Authorization',
'in': 'header'
}
}
}
# try:
# assert settings.SWAGGER_SETTINGS
# except AttributeError:
# # We only load this setting, if it is not available in the overall settings.py file
# settings.SWAGGER_SETTINGS = {
# 'SECURITY_DEFINITIONS': {
# 'Hawk': {
# 'type': 'apiKey',
# 'description': 'HTTP Holder-Of-Key Authentication Scheme, https://github.com/hapijs/hawk, https://hawkrest.readthedocs.io/en/latest/<br /><strong>Ex header:</strong><br />\'Authorization\': \'Hawk mac="F4+S9cu7yZiZEgdtqzMpOOdudvqcV2V2Yzk2WcphECc=", hash="+7fKUX+djeQolvnLTxr0X47e//UHKbkRlajwMw3tx3w=", id="7FI5JET4", ts="1592905433", nonce="DlV-fL"\'',
# 'name': 'Authorization',
# 'in': 'header'
# }
# }
# }
try:
assert settings.REST_FRAMEWORK
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
# To protect all API views with Hawk by default, put this in your settings:
# https://hawkrest.readthedocs.io/en/latest/usage.html#protecting-api-views-with-hawk
settings.REST_FRAMEWORK = {
# try:
# assert settings.REST_FRAMEWORK
# except AttributeError:
# # We only load this setting, if it is not available in the overall settings.py file
# # To protect all API views with Hawk by default, put this in your settings:
# # https://hawkrest.readthedocs.io/en/latest/usage.html#protecting-api-views-with-hawk
# settings.REST_FRAMEWORK = {
'DEFAULT_AUTHENTICATION_CLASSES': (
'apps.api.authentication.APIHawk',
),
# 'DEFAULT_AUTHENTICATION_CLASSES': (
# 'apps.api.authentication.APIHawk',
# ),
'DEFAULT_PERMISSION_CLASSES': (
'rest_framework.permissions.IsAuthenticated',
),
# 'DEFAULT_PERMISSION_CLASSES': (
# 'rest_framework.permissions.IsAuthenticated',
# ),
# 'DEFAULT_AUTHENTICATION_CLASSES': (
# 'rest_framework.authentication.TokenAuthentication',
# ),
# # 'DEFAULT_AUTHENTICATION_CLASSES': (
# # 'rest_framework.authentication.TokenAuthentication',
# # ),
# 'DEFAULT_PERMISSION_CLASSES': (
# 'rest_framework.permissions.IsAuthenticated', ),
# # 'DEFAULT_PERMISSION_CLASSES': (
# # 'rest_framework.permissions.IsAuthenticated', ),
# Use Django's standard `django.contrib.auth` permissions,
# or allow read-only access for unauthenticated users.
#'DEFAULT_PERMISSION_CLASSES': [
# 'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly'
#],
'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination',
'PAGE_SIZE': 10
}
# # Use Django's standard `django.contrib.auth` permissions,
# # or allow read-only access for unauthenticated users.
# #'DEFAULT_PERMISSION_CLASSES': [
# # 'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly'
# #],
# 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination',
# 'PAGE_SIZE': 10
# }
try:
assert settings.HAWK_MESSAGE_EXPIRATION
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.HAWK_MESSAGE_EXPIRATION = 60
# try:
# assert settings.HAWK_MESSAGE_EXPIRATION
# except AttributeError:
# # We only load this setting, if it is not available in the overall settings.py file
# settings.HAWK_MESSAGE_EXPIRATION = 60
def ready(self):
from . import signals

64
webservice/apps/api/urls.py
View File

@ -7,24 +7,24 @@ from drf_yasg2 import openapi
from . import views
from apps.dropoff.api.views import DatadropViewSet
from apps.invitation.api.views import InvitationViewSet
from apps.researcher.api.views import ResearcherViewSet
from apps.storage.api.views import StorageEngineViewSet, StorageLocationViewSet
from apps.study.api.views import StudyViewSet
from apps.virtual_machine.api.views import (VirtualMachineViewSet,
VirtualMachineOperatingSystemViewSet,
VirtualMachineProfileViewSet,
VirtualMachineMemoryViewSet,
VirtualMachineNetworkViewSet,
VirtualMachineStorageViewSet,
VirtualMachineGPUViewSet)
# from apps.dropoff.api.views import DatadropViewSet
# from apps.invitation.api.views import InvitationViewSet
# from apps.researcher.api.views import ResearcherViewSet
# from apps.storage.api.views import StorageEngineViewSet, StorageLocationViewSet
# from apps.study.api.views import StudyViewSet
# from apps.virtual_machine.api.views import (VirtualMachineViewSet,
# VirtualMachineOperatingSystemViewSet,
# VirtualMachineProfileViewSet,
# VirtualMachineMemoryViewSet,
# VirtualMachineNetworkViewSet,
# VirtualMachineStorageViewSet,
# VirtualMachineGPUViewSet)
schema_view = get_schema_view(
openapi.Info(
title="Virtual Research Environment API",
title="Synthea WebService API",
default_version='v1',
description="Here you can see a list of API endpoints and actions that are available to communicate with the VRE API",
description="Info about Synthea WebServer API",
terms_of_service="https://www.rug.nl",
contact=openapi.Contact(email="vre_team@rug.nl"),
license=openapi.License(name="MIT License"),
@ -33,27 +33,27 @@ schema_view = get_schema_view(
permission_classes=(permissions.AllowAny,),
)
api_router_v1 = routers.DefaultRouter()
#api_router_v1 = routers.DefaultRouter()
api_router_v1.register(r'researchers', ResearcherViewSet)
# api_router_v1.register(r'researchers', ResearcherViewSet)
api_router_v1.register(r'studies', StudyViewSet)
# api_router_v1.register(r'studies', StudyViewSet)
api_router_v1.register(r'dropoffs', DatadropViewSet)
# api_router_v1.register(r'dropoffs', DatadropViewSet)
api_router_v1.register(r'invitations', InvitationViewSet)
# api_router_v1.register(r'invitations', InvitationViewSet)
api_router_v1.register(r'storageengines', StorageEngineViewSet)
api_router_v1.register(r'storagelocations', StorageLocationViewSet)
# api_router_v1.register(r'storageengines', StorageEngineViewSet)
# api_router_v1.register(r'storagelocations', StorageLocationViewSet)
# Order is important for virtual machines. Longest match first
api_router_v1.register(r'virtualmachines/profiles', VirtualMachineProfileViewSet)
api_router_v1.register(r'virtualmachines/storage', VirtualMachineStorageViewSet)
api_router_v1.register(r'virtualmachines/memory', VirtualMachineMemoryViewSet)
api_router_v1.register(r'virtualmachines/network', VirtualMachineNetworkViewSet)
api_router_v1.register(r'virtualmachines/gpu', VirtualMachineGPUViewSet)
api_router_v1.register(r'virtualmachines/os', VirtualMachineOperatingSystemViewSet)
api_router_v1.register(r'virtualmachines', VirtualMachineViewSet)
# # Order is important for virtual machines. Longest match first
# api_router_v1.register(r'virtualmachines/profiles', VirtualMachineProfileViewSet)
# api_router_v1.register(r'virtualmachines/storage', VirtualMachineStorageViewSet)
# api_router_v1.register(r'virtualmachines/memory', VirtualMachineMemoryViewSet)
# api_router_v1.register(r'virtualmachines/network', VirtualMachineNetworkViewSet)
# api_router_v1.register(r'virtualmachines/gpu', VirtualMachineGPUViewSet)
# api_router_v1.register(r'virtualmachines/os', VirtualMachineOperatingSystemViewSet)
# api_router_v1.register(r'virtualmachines', VirtualMachineViewSet)
# Main namespace for the API urls
app_name = 'api'
@ -66,6 +66,10 @@ urlpatterns = [
# Also this will give the full url to the OpenAPI documentation
path('info/', views.Info.as_view(), name='info'),
path('states/', views.States.as_view(), name='states'),
path('modules/', views.Modules.as_view(), name='modules'),
path('generate/', views.Generate.as_view(), name='generate'),
# Add extra namespace for versioning the API
path('v1/', include((api_router_v1.urls,'api'),namespace='v1')),
#path('v1/', include((api_router_v1.urls,'api'),namespace='v1')),
]

47
webservice/apps/api/views.py
View File

@ -1,11 +1,23 @@
from rest_framework.views import APIView
from rest_framework.response import Response
from django.http import FileResponse
from django.views.decorators.csrf import csrf_exempt
from rest_framework.decorators import schema
from rest_framework import status
from django.urls import reverse
from lib.utils.general import get_ip_address
from apps.synthea.api.serializers import SyntheaSerializer
from apps.synthea.lib.utils import available_states, available_modules,run_synthea
import mimetypes
@schema(None)
class Info(APIView):
"""
@ -49,4 +61,37 @@ class Info(APIView):
except AttributeError:
pass
return Response(data)
return Response(data)
#@schema(None)
class States(APIView):
def get(self, request, format=None):
return Response(available_states())
class Modules(APIView):
def get(self, request, format=None):
return Response(available_modules())
class Generate(APIView):
def post(self, request, format=None):
print('Post data')
print(request.data)
serializer = SyntheaSerializer(data=request.data)
if serializer.is_valid():
synthea = serializer.save()
zipfile = synthea.generate()
print(zipfile)
response = FileResponse(zipfile.open('rb'), content_type='application/zip')
response['Content-Disposition'] = f'attachment; filename={zipfile.name}'
return response
#return Response(serializer.data, status=status.HTTP_201_CREATED)
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

8
webservice/apps/synthea/api/serializers.py Normal file
View File

@ -0,0 +1,8 @@
from rest_framework import serializers
from apps.synthea.models import Synthea
class SyntheaSerializer(serializers.ModelSerializer):
class Meta:
model = Synthea
fields = ['state', 'population', 'gender', 'age', 'module']

41
webservice/apps/synthea/apps.py
View File

@ -7,4 +7,43 @@ class SyntheaConfig(AppConfig):
name = 'apps.synthea'
label = 'synthea'
verbose_name = _('Synthea')
verbose_name_plural = _('Synthea')
verbose_name_plural = _('Synthea')
try:
assert settings.SYNTHEA_BASE_DIR
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.SYNTHEA_BASE_DIR = settings.BASE_DIR / '../synthea'
try:
assert settings.SYNTHEA_OUTPUT_DIR
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.SYNTHEA_OUTPUT_DIR = settings.BASE_DIR / '../synthea_output'
try:
assert settings.SYNTHEA_MODULE_DIR
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.SYNTHEA_MODULE_DIR = settings.SYNTHEA_BASE_DIR / 'src/main/resources/modules/'
try:
assert settings.SYNTHEA_RESOURCE_DIR
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.SYNTHEA_RESOURCE_DIR = settings.SYNTHEA_BASE_DIR / 'src/main/resources/'
try:
assert settings.SYNTHEA_STATES_DIR
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.SYNTHEA_STATES_DIR = settings.SYNTHEA_BASE_DIR / 'src/main/resources/geography/'
try:
assert settings.SYNTHEA_EXPORT_TYPE
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.SYNTHEA_EXPORT_TYPE = 'fhir_stu3'

12
webservice/apps/synthea/forms.py
View File

@ -1,3 +1,5 @@
from django.utils.translation import gettext_lazy as _
from django.forms import ModelForm
from django import forms
@ -12,16 +14,16 @@ class SyntheaForm(ModelForm):
fields = ['state', 'population', 'gender', 'age', 'module']
# This is loaded only once during startup. So changing the state data will not be picked up after a restart
state_options = [('','Any')]
state_options = [('',_('Select'))]
for item in available_states():
state_options.append((item,item))
state_options.append((item['id'],item['name']))
module_options = [('','Any')]
module_options = [('',_('Any'))]
for item in available_modules():
module_options.append((item['module'],item['name']))
module_options.append((item['id'],item['name']))
widgets = {
'state': forms.Select(choices=state_options),
'gender': forms.Select(choices=[('','Any'),('m','Male'),('f','Female')]),
'gender': forms.Select(choices=[('',_('Any')),('m',_('Male')),('f',_('Female'))]),
'module': forms.Select(choices=module_options)
}

53
webservice/apps/synthea/lib/utils.py
View File

@ -5,36 +5,41 @@ import subprocess
from zipfile import ZipFile
import json
def available_states():
#TODO: Make a setting for this path
location = Path('/opt/development/synthea_webservice/synthea/src/main/resources/geography/')
from django.conf import settings
from uuid import uuid4
df = pd.read_csv(location / 'timezones.csv', index_col=False)
# The state information is expected in the first column
states = df[df.columns[0]].to_list()
states.sort()
def available_states():
states = []
# Read the timezones.csv file from the Synthea resources. This should give us all the 'state' on the first column
df = pd.read_csv(settings.SYNTHEA_STATES_DIR / 'timezones.csv', index_col=False)
for state in df[df.columns[0]].to_list():
states.append({'id' : state , 'name' : state})
#states = df[df.columns[0]].to_list()
# Sort on name
states = sorted(states, key=lambda k: k['name'].lower())
#states.sort()
return states
def available_modules():
#TODO: Make a setting for this path
location = Path('/opt/development/synthea_webservice/synthea/src/main/resources/modules/')
# Assumption here: A folder is a single module. And all .json in the main modules folder is a module.
# Assumption here: Only .json files in the main folder are modules. The rest are submodules...
modules = []
for module in location.iterdir():
for module in settings.SYNTHEA_MODULE_DIR.iterdir():
if module.is_file() and module.suffix == '.json':
data = json.loads(module.read_text())
modules.append({'module' : module.name.replace('.json',''), 'name' : data['name']})
modules.append({'id' : module.name.replace('.json',''), 'name' : data['name']})
modules = sorted(modules, key=lambda k: k['name'].lower())
return modules
def run_synthea(state = None, population = None, gender = None, age = None, module = None):
# TODO: Make synthea setting(s)
location = '/opt/development/synthea_webservice/synthea/'
synthea_cmd = ['/opt/development/synthea_webservice/synthea/run_synthea']
# Add a unique dir to the output, so multiple Synthea processes can run parallel
temp_id = uuid4().hex
output_folder = settings.SYNTHEA_OUTPUT_DIR / temp_id
synthea_cmd = [settings.SYNTHEA_BASE_DIR / 'run_synthea','--exporter.baseDirectory',output_folder]
zip_file = 'Synthea_'
zip_export = location
if population:
synthea_cmd.append('-p')
@ -62,7 +67,7 @@ def run_synthea(state = None, population = None, gender = None, age = None, modu
process_ok = False
log = ''
with subprocess.Popen(synthea_cmd,cwd=location, stdout=subprocess.PIPE,stderr=subprocess.PIPE) as process:
with subprocess.Popen(synthea_cmd,cwd=settings.SYNTHEA_BASE_DIR, stdout=subprocess.PIPE,stderr=subprocess.PIPE) as process:
for line in process.stdout:
line = line.decode('utf8')
log += line
@ -70,14 +75,10 @@ def run_synthea(state = None, population = None, gender = None, age = None, modu
process_ok = line.find('BUILD SUCCESSFUL') >= 0
if process_ok:
with ZipFile(f'{zip_export}/{zip_file}.zip', 'w') as export:
for file in Path(location + 'output/fhir_stu3').iterdir():
with ZipFile(f'{output_folder}/{zip_file}_{temp_id}.zip', 'w') as export:
for file in (output_folder / settings.SYNTHEA_EXPORT_TYPE).iterdir():
export.write(file,file.name)
return Path(f'{zip_export}/{zip_file}.zip')
return (log,Path(f'{output_folder}/{zip_file}_{temp_id}.zip'))
else:
raise Exception(log)
raise Exception(log)

38
webservice/apps/synthea/migrations/0002_auto_20201116_1457.py Normal file
View File

@ -0,0 +1,38 @@
# Generated by Django 3.1.3 on 2020-11-16 13:57
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('synthea', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='synthea',
name='log',
field=models.TextField(blank=True, help_text='Synthea logfile output', verbose_name='Log'),
),
migrations.AlterField(
model_name='synthea',
name='age',
field=models.CharField(blank=True, default='18-100', help_text='Select the age range. Enter [min age]-[max age]', max_length=10, verbose_name='Age range'),
),
migrations.AlterField(
model_name='synthea',
name='gender',
field=models.CharField(blank=True, help_text='Select the gender type', max_length=1, verbose_name='Gender'),
),
migrations.AlterField(
model_name='synthea',
name='module',
field=models.CharField(blank=True, help_text='Select the module', max_length=50, verbose_name='Module'),
),
migrations.AlterField(
model_name='synthea',
name='state',
field=models.CharField(help_text='The state for which synthea generate data.', max_length=200, verbose_name='State'),
),
]

24
webservice/apps/synthea/models.py
View File

@ -7,6 +7,8 @@ from django_cryptography.fields import encrypt
from lib.utils.general import get_random_string
from lib.models.base import MetaDataModel
from .lib.utils import run_synthea
import uuid
# Create your models here.
@ -18,7 +20,25 @@ class Synthea(MetaDataModel):
id = models.UUIDField(_('ID'), primary_key=True, unique=True, default=uuid.uuid4, editable=False, help_text=_('A unique id'))
state = models.CharField(_('State'), max_length=200, help_text=_('The state for which synthea generate data.'))
population = models.PositiveSmallIntegerField(_('Population'), blank=True, default=50, help_text=_('The size of the population'))
population = models.PositiveSmallIntegerField(_('Population'), default=50, help_text=_('The size of the population'))
gender = models.CharField(_('Gender'), blank=True,max_length=1, help_text=_('Select the gender type'))
age = models.CharField(_('Age range'), blank=True,max_length=10, help_text=_('Select the age range'))
age = models.CharField(_('Age range'), blank=True,default='18-100', max_length=10, help_text=_('Select the age range. Enter [min age]-[max age]'))
module = models.CharField(_('Module'),blank=True, max_length=50, help_text=_('Select the module'))
log = models.TextField(_('Log'),blank=True, help_text=_('Synthea logfile output'))
def generate(self):
log,zip_file = run_synthea(
self.state,
self.population,
self.gender,
self.age,
self.module
)
self.log = log
self.save()
return zip_file

2
webservice/apps/synthea/templates/synthea/generator_form.html
View File

@ -9,7 +9,7 @@
{% endblock %}
{% block content %}
<h1>Synthea generartor</h1>
<h1>Synthea Generator</h1>
<p>Enter the form fields and press submit. <small>U vraagt, wij draaien ;)</small></p>
<form method="POST" class="post-form">
{% csrf_token %}

1
webservice/apps/synthea/templates/synthea/index.html
View File

@ -11,4 +11,5 @@
{% block content %}
<h1>dHealt Nederland</h1>
<p>Onder leiding van UMCG</p>
<p>Github branches: https://github.com/dHealthNL </p>
{% endblock %}

22
webservice/apps/synthea/views.py
View File

@ -1,6 +1,7 @@
from django.shortcuts import render
from django.http import HttpResponse
from apps.synthea.forms import SyntheaForm
from apps.synthea.models import Synthea
from .lib.utils import run_synthea
@ -21,24 +22,13 @@ def show_synthea_form(request):
# check whether it's valid:
if form.is_valid():
# process the data in form.cleaned_data as required
synthea = form.save()
zipfile = synthea.generate()
try:
zipfile = run_synthea(
form.cleaned_data['state'],
form.cleaned_data['population'],
form.cleaned_data['gender'],
form.cleaned_data['age'],
form.cleaned_data['module']
)
mime_type, _ = mimetypes.guess_type(zipfile)
response = HttpResponse(zipfile.open('rb'), content_type=mime_type)
response['Content-Disposition'] = f'attachment; filename={zipfile.name}'
response = HttpResponse(zipfile.open('rb'), content_type='application/zip')
response['Content-Disposition'] = f'attachment; filename={zipfile.name}'
return response
except Exception as ex:
print(ex)
return response
# if a GET (or any other method) we'll create a blank form
else:

2
webservice/webservice/settings.py
View File

@ -41,6 +41,8 @@ INSTALLED_APPS = [
'django.contrib.staticfiles',
'django_js_reverse',
'rest_framework',
'drf_yasg2',
'apps.api',
'apps.RUG_template',

3
webservice/webservice/urls.py
View File

@ -20,6 +20,9 @@ from django_js_reverse.views import urls_js
urlpatterns = [
path('admin/', admin.site.urls),
# Add API urls for all the known models
path('api/', include('apps.api.urls')),
path('', include('apps.synthea.urls')),