commit abbaaab98a45b0b4d35e8c2736eef62c55e1e40f Author: J.G. Rubingh Date: Thu Feb 17 14:35:54 2022 +0100 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fe7b0fd --- /dev/null +++ b/.gitignore @@ -0,0 +1,137 @@ +# Django # +*.log +*.pot +*.pyc +__pycache__ +db.sqlite3 +media + +# Backup files # +*.bak + +# If you are using PyCharm # +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# File-based project format +*.iws + +# IntelliJ +out/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Python # +*.py[cod] +*$py.class + +# Distribution / packaging +.Python build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +.pytest_cache/ +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery +celerybeat-schedule.* + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# Sublime Text # +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache +*.sublime-workspace +*.sublime-project + +# sftp configuration file +sftp-config.json + +# Package control specific files Package +Control.last-run +Control.ca-list +Control.ca-bundle +Control.system-ca-bundle +GitHub.sublime-settings + +# Visual Studio Code # +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +.history \ No newline at end of file diff --git a/Enquete/.pep8 b/Enquete/.pep8 new file mode 100644 index 0000000..2c0de05 --- /dev/null +++ b/Enquete/.pep8 @@ -0,0 +1,3 @@ +[pycodestyle] +max_line_length = 120 +ignore = E501 \ No newline at end of file diff --git a/Enquete/.vscode/launch.json b/Enquete/.vscode/launch.json new file mode 100644 index 0000000..11bebb2 --- /dev/null +++ b/Enquete/.vscode/launch.json @@ -0,0 +1,18 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Django", + "type": "python", + "request": "launch", + "program": "cd ${workspaceFolder}/enquete && ./manage.py", + "args": [ + "runserver" + ], + "django": true + } + ] +} \ No newline at end of file diff --git a/Enquete/enquete/apps/vragenlijst/__init__.py b/Enquete/enquete/apps/vragenlijst/__init__.py new file mode 100644 index 0000000..c70dfff --- /dev/null +++ b/Enquete/enquete/apps/vragenlijst/__init__.py @@ -0,0 +1 @@ +default_app_config = 'apps.vragenlijst.apps.VragenlijstConfig' diff --git a/Enquete/enquete/apps/vragenlijst/admin.py b/Enquete/enquete/apps/vragenlijst/admin.py new file mode 100644 index 0000000..e644f96 --- /dev/null +++ b/Enquete/enquete/apps/vragenlijst/admin.py @@ -0,0 +1,42 @@ +from django.contrib import admin +from .models import Questionnaire, QuestionnaireTopic, QuestionnaireQuestion, QuestionnaireResponse, QuestionnaireStorage + +# Register your models here. + + +@admin.register(Questionnaire) +class QuestionnaireAdmin(admin.ModelAdmin): + list_display = ('name', 'id',) + readonly_fields = ('created_at', 'updated_at') + + +@admin.register(QuestionnaireTopic) +class QuestionnaireTopicAdmin(admin.ModelAdmin): + list_display = ('name', 'order', 'questionnaire',) + ordering = ('order', ) + readonly_fields = ('created_at', 'updated_at') + + +@admin.register(QuestionnaireQuestion) +class QuestionnaireQuestionAdmin(admin.ModelAdmin): + list_display = ('name', 'order', 'type', 'topic', 'questionnaire') + ordering = ('order', ) + readonly_fields = ('created_at', 'updated_at') + + def questionnaire(self, item): + return item.topic.questionnaire + + +@admin.register(QuestionnaireResponse) +class QuestionnaireResponseAdmin(admin.ModelAdmin): + list_display = ('id', 'created_at', 'questionnaire') + ordering = ('-created_at', ) + readonly_fields = ('questionnaire','response','created_at', 'updated_at') + + +@admin.register(QuestionnaireStorage) +class QuestionnaireStorageAdmin(admin.ModelAdmin): + list_display = ('name', 'type', 'server') + ordering = ('name', ) + readonly_fields = ('created_at', 'updated_at') + diff --git a/Enquete/enquete/apps/vragenlijst/apps.py b/Enquete/enquete/apps/vragenlijst/apps.py new file mode 100644 index 0000000..08cbf47 --- /dev/null +++ b/Enquete/enquete/apps/vragenlijst/apps.py @@ -0,0 +1,11 @@ +from django.apps import AppConfig +from django.utils.translation import gettext_lazy as _ + + +class VragenlijstConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'apps.vragenlijst' + + label = 'vragenlijst' + verbose_name = _('Questionnaire') + verbose_name_plural = _('Questionnaire') diff --git a/Enquete/enquete/apps/vragenlijst/migrations/0001_initial.py b/Enquete/enquete/apps/vragenlijst/migrations/0001_initial.py new file mode 100644 index 0000000..c54063f --- /dev/null +++ b/Enquete/enquete/apps/vragenlijst/migrations/0001_initial.py @@ -0,0 +1,67 @@ +# Generated by Django 4.0.2 on 2022-02-03 10:59 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Questionnaire', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID')), + ('name', models.CharField(help_text='Name of the questionnaire.', max_length=200, verbose_name='Name')), + ('description', models.TextField(blank=True, help_text='Enter a short description for this questionnaire.', null=True, verbose_name='Description')), + ('created_at', models.DateTimeField(auto_now_add=True, help_text='The date and time this questionnaire has been created', verbose_name='Date created')), + ('updated_at', models.DateTimeField(auto_now=True, help_text='The date and time this questionnaire has been updated', verbose_name='Date updated')), + ], + options={ + 'verbose_name': 'Questionnaire', + 'verbose_name_plural': 'Questionnaires', + 'ordering': ['name'], + }, + ), + migrations.CreateModel( + name='QuestionnaireTopic', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID')), + ('name', models.CharField(help_text='Name of the questionnaire topic.', max_length=200, verbose_name='Name')), + ('description', models.TextField(blank=True, help_text='Enter a short description for this questionnaire topic.', null=True, verbose_name='Description')), + ('order', models.PositiveIntegerField(blank=True, verbose_name='Order')), + ('created_at', models.DateTimeField(auto_now_add=True, help_text='The date and time this questionnaire topic has been created', verbose_name='Date created')), + ('updated_at', models.DateTimeField(auto_now=True, help_text='The date and time this questionnaire topic has been updated', verbose_name='Date updated')), + ('questionnaire', models.ForeignKey(help_text='The questionnaire topic for this questionnaire.', on_delete=django.db.models.deletion.CASCADE, to='vragenlijst.questionnaire', verbose_name='Questionnaire')), + ], + options={ + 'verbose_name': 'Questionnaire topic', + 'verbose_name_plural': 'Questionnaire topics', + 'ordering': ['name'], + }, + ), + migrations.CreateModel( + name='QuestionnaireQuestion', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID')), + ('name', models.CharField(help_text='Name of the questionnaire topic.', max_length=200, verbose_name='Name')), + ('type', models.CharField(choices=[('DATE', 'Date field'), ('NUMBER', 'Number field'), ('MULTIPLE', 'Multi options field'), ('SINGLE', 'Single option field'), ('TEXT', 'Single text line')], default='SINGLE', help_text='Question type', max_length=15, verbose_name='Type')), + ('description', models.TextField(blank=True, help_text='Enter a short description for this questionnaire topic.', null=True, verbose_name='Description')), + ('order', models.PositiveIntegerField(blank=True, verbose_name='Order')), + ('choices', models.TextField(blank=True, help_text='Enter a short description for this questionnaire topic.', null=True, verbose_name='Choices')), + ('created_at', models.DateTimeField(auto_now_add=True, help_text='The date and time this questionnaire topic has been created', verbose_name='Date created')), + ('updated_at', models.DateTimeField(auto_now=True, help_text='The date and time this questionnaire topic has been updated', verbose_name='Date updated')), + ('topic', models.ForeignKey(help_text='The questionnaire topic for this questionnaire.', on_delete=django.db.models.deletion.CASCADE, to='vragenlijst.questionnairetopic', verbose_name='Questionnaire topic')), + ], + options={ + 'verbose_name': 'Questionnaire question', + 'verbose_name_plural': 'Questionnaire questions', + 'ordering': ['name'], + }, + ), + ] diff --git a/Enquete/enquete/apps/vragenlijst/migrations/0002_alter_questionnairetopic_questionnaire.py b/Enquete/enquete/apps/vragenlijst/migrations/0002_alter_questionnairetopic_questionnaire.py new file mode 100644 index 0000000..261bbd9 --- /dev/null +++ b/Enquete/enquete/apps/vragenlijst/migrations/0002_alter_questionnairetopic_questionnaire.py @@ -0,0 +1,19 @@ +# Generated by Django 4.0.2 on 2022-02-03 11:45 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('vragenlijst', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='questionnairetopic', + name='questionnaire', + field=models.ForeignKey(help_text='The questionnaire topic for this questionnaire.', on_delete=django.db.models.deletion.CASCADE, related_name='topics', to='vragenlijst.questionnaire', verbose_name='Questionnaire'), + ), + ] diff --git a/Enquete/enquete/apps/vragenlijst/migrations/0003_alter_questionnairequestion_topic.py b/Enquete/enquete/apps/vragenlijst/migrations/0003_alter_questionnairequestion_topic.py new file mode 100644 index 0000000..ad87b37 --- /dev/null +++ b/Enquete/enquete/apps/vragenlijst/migrations/0003_alter_questionnairequestion_topic.py @@ -0,0 +1,19 @@ +# Generated by Django 4.0.2 on 2022-02-03 12:21 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('vragenlijst', '0002_alter_questionnairetopic_questionnaire'), + ] + + operations = [ + migrations.AlterField( + model_name='questionnairequestion', + name='topic', + field=models.ForeignKey(help_text='The questionnaire topic for this questionnaire.', on_delete=django.db.models.deletion.CASCADE, related_name='questions', to='vragenlijst.questionnairetopic', verbose_name='Questionnaire topic'), + ), + ] diff --git a/Enquete/enquete/apps/vragenlijst/migrations/0004_questionnairestorage_questionnaireresponse.py b/Enquete/enquete/apps/vragenlijst/migrations/0004_questionnairestorage_questionnaireresponse.py new file mode 100644 index 0000000..5e3413e --- /dev/null +++ b/Enquete/enquete/apps/vragenlijst/migrations/0004_questionnairestorage_questionnaireresponse.py @@ -0,0 +1,49 @@ +# Generated by Django 4.0.2 on 2022-02-08 10:01 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('vragenlijst', '0003_alter_questionnairequestion_topic'), + ] + + operations = [ + migrations.CreateModel( + name='QuestionnaireStorage', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID')), + ('name', models.CharField(help_text='Name of the questionnaire storage.', max_length=200, verbose_name='Name')), + ('type', models.CharField(choices=[('WEBDAV', 'WebDAV')], default='WEBDAV', help_text='Storage type', max_length=15, verbose_name='Type')), + ('server', models.CharField(help_text='Server url', max_length=200, verbose_name='Server')), + ('username', models.CharField(help_text='Username', max_length=200, verbose_name='Username')), + ('password', models.CharField(help_text='Password', max_length=200, verbose_name='Password')), + ('path', models.CharField(help_text='Location on disk', max_length=200, verbose_name='Path')), + ('created_at', models.DateTimeField(auto_now_add=True, help_text='The date and time this questionnaire storage has been created', verbose_name='Date created')), + ('updated_at', models.DateTimeField(auto_now=True, help_text='The date and time this questionnaire storage has been updated', verbose_name='Date updated')), + ], + options={ + 'verbose_name': 'Questionnaire storage', + 'verbose_name_plural': 'Questionnaire storages', + 'ordering': ['name'], + }, + ), + migrations.CreateModel( + name='QuestionnaireResponse', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID')), + ('response', models.TextField(help_text='Questionaire response in CSV', verbose_name='Response')), + ('created_at', models.DateTimeField(auto_now_add=True, help_text='The date and time this questionnaire response has been created', verbose_name='Date created')), + ('updated_at', models.DateTimeField(auto_now=True, help_text='The date and time this questionnaire response has been updated', verbose_name='Date updated')), + ('questionnaire', models.ForeignKey(help_text='The questionnaire for this response.', on_delete=django.db.models.deletion.CASCADE, to='vragenlijst.questionnaire', verbose_name='Questionnaire')), + ], + options={ + 'verbose_name': 'Questionnaire response', + 'verbose_name_plural': 'Questionnaire responses', + 'ordering': ['-created_at'], + }, + ), + ] diff --git a/Enquete/enquete/apps/vragenlijst/migrations/__init__.py b/Enquete/enquete/apps/vragenlijst/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Enquete/enquete/apps/vragenlijst/models.py b/Enquete/enquete/apps/vragenlijst/models.py new file mode 100644 index 0000000..bc0e014 --- /dev/null +++ b/Enquete/enquete/apps/vragenlijst/models.py @@ -0,0 +1,134 @@ +from django.db import models +from django.db.models import Max, Value +from django.db.models.functions import Coalesce +from django.utils.translation import gettext_lazy as _ +import uuid +from encrypted_model_fields.fields import EncryptedCharField +# Create your models here. + + +class Questionnaire(models.Model): + + class Meta: + verbose_name = _('Questionnaire') + verbose_name_plural = _('Questionnaires') + ordering = ['name'] + + id = models.UUIDField(_('ID'), default=uuid.uuid4, editable=False, unique=True, primary_key=True) + name = models.CharField(_('Name'), max_length=200, help_text=_('Name of the questionnaire.')) + description = models.TextField(_('Description'), blank=True, null=True, help_text=_('Enter a short description for this questionnaire.')) + + created_at = models.DateTimeField(_('Date created'), auto_now_add=True, help_text=_('The date and time this questionnaire has been created')) + updated_at = models.DateTimeField(_('Date updated'), auto_now=True, help_text=_('The date and time this questionnaire has been updated')) + + + @property + def allQuestions(self): + return QuestionnaireQuestion.objects.filter(topic__questionnaire=self.id) + + def __str__(self): + """str: Returns a readable string.""" + return f'{self.name}' + + +class QuestionnaireTopic(models.Model): + + class Meta: + verbose_name = _('Questionnaire topic') + verbose_name_plural = _('Questionnaire topics') + ordering = ['name'] + + id = models.UUIDField(_('ID'), default=uuid.uuid4, editable=False, unique=True, primary_key=True) + name = models.CharField(_('Name'), max_length=200, help_text=_('Name of the questionnaire topic.')) + questionnaire = models.ForeignKey(Questionnaire, verbose_name=Questionnaire._meta.verbose_name, on_delete=models.CASCADE, help_text=_('The questionnaire topic for this questionnaire.'), related_name='topics') + description = models.TextField(_('Description'), blank=True, null=True, help_text=_('Enter a short description for this questionnaire topic.')) + + order = models.PositiveIntegerField(_('Order'), blank=True) + + created_at = models.DateTimeField(_('Date created'), auto_now_add=True, help_text=_('The date and time this questionnaire topic has been created')) + updated_at = models.DateTimeField(_('Date updated'), auto_now=True, help_text=_('The date and time this questionnaire topic has been updated')) + + def save(self, *args, **kwargs): + if self.order is None: + self.order = QuestionnaireTopic.objects.filter(questionnaire=self.questionnaire).aggregate(neworder=Coalesce(Max('order'), Value(0)))['neworder'] + 1 + super().save(*args, **kwargs) + + def __str__(self): + """str: Returns a readable string.""" + return f'{self.name}' + + +class QuestionnaireQuestionTypes(models.TextChoices): + + DATE = ('DATE', _('Date field')) + NUMBER = ('NUMBER', _('Number field')) + MULTIPLE = ('MULTIPLE', _('Multi options field')) + SINGLE = ('SINGLE', _('Single option field')) + TEXT = ('TEXT', _('Single text line')) + + +class QuestionnaireQuestion(models.Model): + + class Meta: + verbose_name = _('Questionnaire question') + verbose_name_plural = _('Questionnaire questions') + ordering = ['name'] + + id = models.UUIDField(_('ID'), default=uuid.uuid4, editable=False, unique=True, primary_key=True) + name = models.CharField(_('Name'), max_length=200, help_text=_('Name of the questionnaire topic.')) + type = models.CharField(_('Type'), max_length=15, choices=QuestionnaireQuestionTypes.choices, default=QuestionnaireQuestionTypes.SINGLE, help_text=_('Question type')) + description = models.TextField(_('Description'), blank=True, null=True, help_text=_('Enter a short description for this questionnaire topic.')) + order = models.PositiveIntegerField(_('Order'), blank=True,) + choices = models.TextField(_('Choices'), blank=True, null=True, help_text=_('Enter the choices 1 per line.
Use a format like \'= [Text]\' for an \'anders\' option.
Use format \'[Text]=[Value]\' for different value for each choice ')) + topic = models.ForeignKey(QuestionnaireTopic, verbose_name=QuestionnaireTopic._meta.verbose_name, on_delete=models.CASCADE, help_text=_('The questionnaire topic for this questionnaire.'), related_name='questions') + + created_at = models.DateTimeField(_('Date created'), auto_now_add=True, help_text=_('The date and time this questionnaire topic has been created')) + updated_at = models.DateTimeField(_('Date updated'), auto_now=True, help_text=_('The date and time this questionnaire topic has been updated')) + + def save(self, *args, **kwargs): + if self.order is None: + self.order = QuestionnaireQuestion.objects.filter(topic__questionnaire=self.topic.questionnaire).aggregate(neworder=Coalesce(Max('order'), Value(0)))['neworder'] + 1 + super().save(*args, **kwargs) + + def choices_list(self): + return [choice.strip() for choice in self.choices.strip("\n").split('\n')] + + def __str__(self): + """str: Returns a readable string.""" + return f'{self.name}' + + +class QuestionnaireResponse(models.Model): + class Meta: + verbose_name = _('Questionnaire response') + verbose_name_plural = _('Questionnaire responses') + ordering = ['-created_at'] + + id = models.UUIDField(_('ID'), default=uuid.uuid4, editable=False, unique=True, primary_key=True) + questionnaire = models.ForeignKey(Questionnaire, verbose_name=Questionnaire._meta.verbose_name, on_delete=models.CASCADE, help_text=_('The questionnaire for this response.')) + response = models.TextField(_('Response'), help_text=_('Questionaire response in CSV')) + + created_at = models.DateTimeField(_('Date created'), auto_now_add=True, help_text=_('The date and time this questionnaire response has been created')) + updated_at = models.DateTimeField(_('Date updated'), auto_now=True, help_text=_('The date and time this questionnaire response has been updated')) + + +class QuestionnaireStorageTypes(models.TextChoices): + + WEBDAV = ('WEBDAV', _('WebDAV')) + +class QuestionnaireStorage(models.Model): + class Meta: + verbose_name = _('Questionnaire storage') + verbose_name_plural = _('Questionnaire storages') + ordering = ['name'] + + id = models.UUIDField(_('ID'), default=uuid.uuid4, editable=False, unique=True, primary_key=True) + name = models.CharField(_('Name'), max_length=200, help_text=_('Name of the questionnaire storage.')) + type = models.CharField(_('Type'), max_length=15, choices=QuestionnaireStorageTypes.choices, default=QuestionnaireStorageTypes.WEBDAV, help_text=_('Storage type')) + server = models.CharField(_('Server'), max_length=200, help_text=_('Server url')) + username = EncryptedCharField(_('Username'), max_length=200, help_text=_('Username')) + password = EncryptedCharField(_('Password'), max_length=200, help_text=_('Password')) + path = models.CharField(_('Path'), max_length=200, help_text=_('Location on disk')) + + created_at = models.DateTimeField(_('Date created'), auto_now_add=True, help_text=_('The date and time this questionnaire storage has been created')) + updated_at = models.DateTimeField(_('Date updated'), auto_now=True, help_text=_('The date and time this questionnaire storage has been updated')) diff --git a/Enquete/enquete/apps/vragenlijst/templates/vragenlijst/form.html b/Enquete/enquete/apps/vragenlijst/templates/vragenlijst/form.html new file mode 100644 index 0000000..20bd373 --- /dev/null +++ b/Enquete/enquete/apps/vragenlijst/templates/vragenlijst/form.html @@ -0,0 +1,382 @@ +{% extends 'base.html' %} + +{% load replace %} +{% load choice_value %} +{% load i18n %} +{% load static %} + +{% block content %} +
+
+
+
+

Fill out the forms

+

Fill all form field to go to next step

+
+ {% csrf_token %} + +
    + {% for topic in questionnaire.topics.all|dictsort:"order" %} +
  • {{ topic.name }}
    • + {% endfor %} +
+
+
+
+
+ {% for topic in questionnaire.topics.all|dictsort:"order" %} +
+
+
+
+

{{topic.name}}:

+

{{topic.description}}

+
+
+

Step {{forloop.counter}} - {{ questionnaire.topics.all.count }}

+
+
+ {% for question in topic.questions.all|dictsort:"order" %} +

+
+ + {% if question.type == 'SINGLE' %} + {% for choice in question.choices_list %} + + + + {% if choice.strip|slice:"0:1" == "=" %} + + + {%endif%} + {% endfor %} + {%elif question.type == 'NUMBER' %} + + {%elif question.type == 'DATE' %} + + {%elif question.type == 'TEXT' %} + + {% endif %} + + {% endfor %} +
+ {% if forloop.counter > 1 %} + + {% endif %} + {% if forloop.last %} + + {%else %} + + + {%endif%} +
+ {% endfor %} +
+
+
+
+
+ + + + +{% endblock %} \ No newline at end of file diff --git a/Enquete/enquete/apps/vragenlijst/templates/vragenlijst/index.html b/Enquete/enquete/apps/vragenlijst/templates/vragenlijst/index.html new file mode 100644 index 0000000..5690ab7 --- /dev/null +++ b/Enquete/enquete/apps/vragenlijst/templates/vragenlijst/index.html @@ -0,0 +1,39 @@ +{% extends 'base.html' %} + +{% load i18n %} +{% load static %} + + + + +{% block content %} + +{% if messages %} +

+ {% for message in messages %} + {% if message.level == DEFAULT_MESSAGE_LEVELS.ERROR %}Important: {% endif %} + {{ message }}
+ {% endfor %} +{% endif %} +

+ +

Select the type of questionnaire you want to use

+ + + +{% if questionnaires %} + +{% for questionnaire in questionnaires %} + +{{ questionnaire.name }} + + +{% endfor %} + +{% else %} + +

No questionnaire are available.

+ +{% endif %} + +{% endblock %} \ No newline at end of file diff --git a/Enquete/enquete/apps/vragenlijst/templatetags/__init__.py b/Enquete/enquete/apps/vragenlijst/templatetags/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Enquete/enquete/apps/vragenlijst/templatetags/choice_value.py b/Enquete/enquete/apps/vragenlijst/templatetags/choice_value.py new file mode 100644 index 0000000..7d09df8 --- /dev/null +++ b/Enquete/enquete/apps/vragenlijst/templatetags/choice_value.py @@ -0,0 +1,9 @@ +from django import template +register = template.Library() + +@register.filter +def choice_value ( string ): + if string[0] != '=' and '=' in string: + return string.split('=')[1].strip() + + return string.strip() diff --git a/Enquete/enquete/apps/vragenlijst/templatetags/replace.py b/Enquete/enquete/apps/vragenlijst/templatetags/replace.py new file mode 100644 index 0000000..204915e --- /dev/null +++ b/Enquete/enquete/apps/vragenlijst/templatetags/replace.py @@ -0,0 +1,11 @@ +import re + +from django import template +register = template.Library() + +@register.filter +def replace ( string, args ): + search = args.split(args[0])[1] + replace = args.split(args[0])[2] + + return re.sub( search, replace, string ) \ No newline at end of file diff --git a/Enquete/enquete/apps/vragenlijst/tests.py b/Enquete/enquete/apps/vragenlijst/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/Enquete/enquete/apps/vragenlijst/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/Enquete/enquete/apps/vragenlijst/urls.py b/Enquete/enquete/apps/vragenlijst/urls.py new file mode 100644 index 0000000..6a4d102 --- /dev/null +++ b/Enquete/enquete/apps/vragenlijst/urls.py @@ -0,0 +1,8 @@ +from django.urls import path + +from . import views + +urlpatterns = [ + path('', views.index, name='index'), + path('/', views.questionnaire, name='questionnaire'), +] diff --git a/Enquete/enquete/apps/vragenlijst/views.py b/Enquete/enquete/apps/vragenlijst/views.py new file mode 100644 index 0000000..5a7bbbd --- /dev/null +++ b/Enquete/enquete/apps/vragenlijst/views.py @@ -0,0 +1,69 @@ +from django.http import HttpResponseRedirect +from django.shortcuts import render +from .models import Questionnaire, QuestionnaireResponse, QuestionnaireStorage +from django.conf import settings +from django.contrib import messages + +import csv +from uuid import uuid4 +from pathlib import Path + +from storage.storage import Storage +# Create your views here. + + +def index(request): + # latest_question_list = Question.objects.order_by('-pub_date')[:5] + questionnaires = Questionnaire.objects.order_by('name') + context = {'questionnaires': questionnaires} + return render(request, 'vragenlijst/index.html', context) + + +def questionnaire(request, questionnaire_id): + if request.method == 'POST': + + # As we should have a Django form, we could use te form validator. For now, everyhing is valid ;) + questionnaire = Questionnaire.objects.get(pk=questionnaire_id) + + # Get all the questions in one list with a single query + allQuestions = {} + for question in list(questionnaire.allQuestions.order_by('order').values('id','name')): + allQuestions[str(question['id'])] = question['name'] + + # Store the response as a ';' seperated CSV file in order of the questionaire questions order + try: + csv_file = Path(f'{settings.HANZE_TEMP_CSV_STORAGE}/{questionnaire.name}-{uuid4()}.csv') + with open(csv_file, 'w') as csvfile: + filewriter = csv.writer(csvfile, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL) + # Add headers + filewriter.writerow(['vraag', 'antwoord']) + + for question in allQuestions: + if request.POST.get(question): + filewriter.writerow([allQuestions[question], request.POST[question]]) + + # Store the data also in a database + QuestionnaireResponse(questionnaire=questionnaire, response=csv_file.read_text()).save() + storageSettings = QuestionnaireStorage.objects.first() + # Move data to external storage + webdav = Storage( + storage_type=storageSettings.type, + url=storageSettings.server, + username=storageSettings.username, + password=storageSettings.password + ) + + webdav.upload_file(source=csv_file, destination=f'{storageSettings.path}/{csv_file.name}') + + messages.success(request, 'Questionaire is saved to disk.') + + except Exception as ex: + messages.error(request, f'Could not save the questionaire. Error: {ex}') + + + # Redirect to starting point + return HttpResponseRedirect('/vragenlijst/') + + questionnaire = Questionnaire.objects.get(pk=questionnaire_id) + context = {'questionnaire': questionnaire, 'menu_width': 100 / questionnaire.topics.count()} + return render(request, 'vragenlijst/form.html', context) diff --git a/Enquete/enquete/enquete/__init__.py b/Enquete/enquete/enquete/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Enquete/enquete/enquete/asgi.py b/Enquete/enquete/enquete/asgi.py new file mode 100644 index 0000000..a8787bf --- /dev/null +++ b/Enquete/enquete/enquete/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for enquete project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'enquete.settings') + +application = get_asgi_application() diff --git a/Enquete/enquete/enquete/settings.py b/Enquete/enquete/enquete/settings.py new file mode 100644 index 0000000..6a2ffb5 --- /dev/null +++ b/Enquete/enquete/enquete/settings.py @@ -0,0 +1,135 @@ +""" +Django settings for enquete project. + +Generated by 'django-admin startproject' using Django 4.0.2. + +For more information on this file, see +https://docs.djangoproject.com/en/4.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.0/ref/settings/ +""" + +from pathlib import Path +from decouple import config +from dj_database_url import parse as db_url +import base64 + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/4.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = config('SECRET_KEY') + +FIELD_ENCRYPTION_KEY = config('FIELD_ENCRYPTION_KEY', str(base64.b64encode(SECRET_KEY[0:32].encode("utf-8")), "utf-8")) + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = config('DEBUG', default=False, cast=bool) +TEMPLATE_DEBUG = DEBUG + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'encrypted_model_fields', + + 'apps.vragenlijst' +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'enquete.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': ['templates'], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'enquete.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/4.0/ref/settings/#databases +DATABASES = { + 'default': config( + 'DATABASE_URL', + default='sqlite:///' + f'{BASE_DIR}' + '/db.sqlite3', + cast=db_url + ) +} + + +# Password validation +# https://docs.djangoproject.com/en/4.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'Europe/Amsterdam' + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.0/howto/static-files/ + +STATIC_URL = 'static/' + +# Default primary key field type +# https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + + +HANZE_TEMP_CSV_STORAGE = BASE_DIR / '../uitkomsten' +HANZE_TEMP_CSV_STORAGE.mkdir(exist_ok=True) diff --git a/Enquete/enquete/enquete/urls.py b/Enquete/enquete/enquete/urls.py new file mode 100644 index 0000000..d0ee962 --- /dev/null +++ b/Enquete/enquete/enquete/urls.py @@ -0,0 +1,22 @@ +"""enquete URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/4.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path, include + +urlpatterns = [ + path('vragenlijst/', include('apps.vragenlijst.urls')), + path('admin/', admin.site.urls), +] diff --git a/Enquete/enquete/enquete/wsgi.py b/Enquete/enquete/enquete/wsgi.py new file mode 100644 index 0000000..5bb31f0 --- /dev/null +++ b/Enquete/enquete/enquete/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for enquete project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'enquete.settings') + +application = get_wsgi_application() diff --git a/Enquete/enquete/manage.py b/Enquete/enquete/manage.py new file mode 100755 index 0000000..ddfafc7 --- /dev/null +++ b/Enquete/enquete/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'enquete.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/Enquete/enquete/requirements.txt b/Enquete/enquete/requirements.txt new file mode 100644 index 0000000..2b15177 --- /dev/null +++ b/Enquete/enquete/requirements.txt @@ -0,0 +1,9 @@ +dj-database-url==0.5.0 +Django==4.0.2 +django-encrypted-model-fields==0.6.1 +python-decouple==3.6 + +python-irodsclient==1.1.1 +giteapy==1.0.8 +webdavclient3==3.14.6 +PyGithub==1.55 \ No newline at end of file diff --git a/Enquete/enquete/storage/README.md b/Enquete/enquete/storage/README.md new file mode 100644 index 0000000..d04f76b --- /dev/null +++ b/Enquete/enquete/storage/README.md @@ -0,0 +1 @@ +A small questionnaire tool that stores the data trough a WebDAV connection. \ No newline at end of file diff --git a/Enquete/enquete/storage/__init__.py b/Enquete/enquete/storage/__init__.py new file mode 100644 index 0000000..4b4e3c6 --- /dev/null +++ b/Enquete/enquete/storage/__init__.py @@ -0,0 +1,8 @@ +import os +import logging +import logging.config + +if os.path.isfile('logging.custom.ini'): + logging.config.fileConfig('logging.custom.ini') +elif os.path.isfile('logging.ini'): + logging.config.fileConfig('logging.ini') diff --git a/Enquete/enquete/storage/engines/fs.py b/Enquete/enquete/storage/engines/fs.py new file mode 100644 index 0000000..165f2a2 --- /dev/null +++ b/Enquete/enquete/storage/engines/fs.py @@ -0,0 +1,28 @@ +import shutil +import os +from storage.storage import BaseStorage +import logging +logger = logging.getLogger(__name__) + + +class LocalStorage(BaseStorage): + + TYPE = 'fs' + + def file_exists(self, filepath): + return os.path.exists(filepath) and os.path.isfile(filepath) + + def directory_exists(self, filepath): + return os.path.exists(filepath) and os.path.isdir(filepath) + + def _make_folder_action(self, path): + os.makedirs(path) + return True + + def _upload_file_action(self, source, destination): + shutil.copy(source, destination) + return True + + def _download_file_action(self, source, destination): + shutil.copy(source, destination) + return True diff --git a/Enquete/enquete/storage/engines/gitea.py b/Enquete/enquete/storage/engines/gitea.py new file mode 100644 index 0000000..9beafae --- /dev/null +++ b/Enquete/enquete/storage/engines/gitea.py @@ -0,0 +1,89 @@ +from giteapy.rest import ApiException +import giteapy +import base64 +from storage.storage import BaseStorage +import logging +logger = logging.getLogger(__name__) + + +# Gitea Support - https://pypi.org/project/giteapy/ + + +class GiteaStorage(BaseStorage): + + TYPE = 'gitea' + + def __init__(self, url=None, username=None, password=None, source=None, destination=None, encryption_key=None, sender_name=None, sender_email=None): + # The repository is added to the url parameter. Use a '#' as seperator. The repository needs to be created first. + # Ex: https://git.web.rug.nl/api/v1#RepositoryName + (url, self.repository) = url.split('#') + destination = destination.strip('/') + + super().__init__(url, username, password, source, destination, encryption_key, sender_name, sender_email) + + # Create a commiter object when the data is uploaded through one of the invited accounts. + self.committer = None + if sender_name is not None or sender_email is not None: + self.committer = giteapy.Identity(name=sender_name, email=sender_email) + + def __connect(self): + try: + assert(self.client) + except AttributeError: + # Configuration for the GITEA connection + configuration = giteapy.Configuration() + # Overrule the host url....? + configuration.host = self.url + #configuration.debug = False + configuration.api_key['access_token'] = self.password + + # Create the client + self.client = giteapy.RepositoryApi(giteapy.ApiClient(configuration)) + logger.info(f'Created Gitea connection to url: {self.url}') + + def file_exists(self, filepath): + self.__connect() + try: + self.client.repo_get_contents(self.username, self.repository, filepath) + return True + except ApiException: + return False + + def directory_exists(self, filepath): + self.__connect() + return self.file_exists(filepath) + + def _make_folder_action(self, path): + # On GitHub you cannot create empty directories. So this actions will always succeed + return True + + def _upload_file_action(self, source, destination): + self.__connect() + try: + with open(source, 'rb') as datafile: + # This is a very big issue. Big files will be stored completely in memory :( + body = giteapy.CreateFileOptions(content=base64.b64encode(datafile.read()).decode(), + message=f'Upload from VRE DataDropOff\n Added file: {destination}', + committer=self.committer) + except Exception: + return False + + try: + # Create a file in a repository + api_response = self.client.repo_create_file(self.username, self.repository, destination, body) + return True + except ApiException as ex: + logger.exception(f'Exception when calling RepositoryApi->repo_create_file: {ex}') + + return True + + def _download_file_action(self, source, destination): + self.__connect() + with open(destination, 'wb') as destination_file: + try: + data = self.client.repo_get_contents(self.username, self.repository, source) + destination_file.write(base64.b64decode(data.content)) + except ApiException as ex: + logger.exception(f'Exception when calling RepositoryApi->repo_get_contents: {ex}') + + return True diff --git a/Enquete/enquete/storage/engines/github.py b/Enquete/enquete/storage/engines/github.py new file mode 100644 index 0000000..a4e09d2 --- /dev/null +++ b/Enquete/enquete/storage/engines/github.py @@ -0,0 +1,66 @@ +from github.GithubException import UnknownObjectException +from github import Github, InputGitAuthor, GithubObject +from storage.storage import BaseStorage +import os +import logging +logger = logging.getLogger(__name__) + + +# Github Support - https://pypi.org/project/PyGithub/ + + +class GithubStorage(BaseStorage): + + TYPE = 'github' + + def __init__(self, url=None, username=None, password=None, source=None, destination=None, encryption_key=None, sender_name=None, sender_email=None): + # The repository is added to the url parameter. Use a '#' as seperator. The repository needs to be created first. + # Ex: https://api.github.com/#RepositoryName + (url, self.repository) = url.split('#') + destination = destination.strip('/') + + super().__init__(url, username, password, source, destination, encryption_key, sender_name, sender_email) + + # Create a commiter object when the data is uploaded through one of the invited accounts. + self.committer = GithubObject.NotSet + if sender_name is not None or sender_email is not None: + self.committer = InputGitAuthor(name=sender_name, email=sender_email) + + def __connect(self): + try: + assert(self.repo) + except AttributeError: + client = Github(self.password) + self.repo = client.get_user().get_repo(self.repository) + logger.info('Created Github.com connection') + + def file_exists(self, filepath): + self.__connect() + try: + self.repo.get_contents(filepath) + return True + except UnknownObjectException: + return False + + def directory_exists(self, filepath): + return True + + def _make_folder_action(self, path): + # On GitHub you cannot create empty directories. So this actions will always succeed + return True + + def _upload_file_action(self, source, destination): + self.__connect() + # Read the file and post to Github. The library will convert to Base64 + with open(source, 'rb') as datafile: + self.repo.create_file(destination.strip('/'), f'Upload from VRE DataDropOff\n Added file: {destination}', datafile.read(), committer=self.committer) + + return True + + def _download_file_action(self, source, destination): + self.__connect() + download = self.repo.get_contents(source) + with open(destination, 'wb') as destination_file: + destination_file.write(download.decoded_content) + + return True diff --git a/Enquete/enquete/storage/engines/irods.py b/Enquete/enquete/storage/engines/irods.py new file mode 100644 index 0000000..d9b88e1 --- /dev/null +++ b/Enquete/enquete/storage/engines/irods.py @@ -0,0 +1,139 @@ +import atexit +from irods.session import iRODSSession +import irods +import storage.exceptions as StorageException +from storage.storage import BaseStorage +import logging +logger = logging.getLogger(__name__) + + +# iRods support - https://pypi.org/project/python-irodsclient/ + + +class iRODSStorage(BaseStorage): + + TYPE = 'irods' + + def __init__(self, url=None, username=None, password=None, source=None, destination=None, encryption_key=None, sender_name=None, sender_email=None): + # The iRODS zone is added to the url parameter. Use a '#' as seperator. This needs to be an Existing iRODS zone + # Ex: rdms-prod-icat.data.rug.nl#rug + (url, self.irods_zone) = url.split('#') + if destination: + destination = destination.strip('/') + + super().__init__(url, username, password, source, destination, encryption_key, sender_name, sender_email) + + # We need to clean up the iRODS session. Using atexit is the easiest way. + atexit.register(self.__close) + + def __connect(self): + try: + assert(self.client) + except AttributeError: + # Connect to the iRODS server + self.client = None + try: + self.client = iRODSSession(host=self.url, port=1247, user=self.username, password=self.password, zone=self.irods_zone) + # Need to make a call to validate the authentication. So by checking the version, we know if we can authenticate... + logger.debug(f'iRODS {self.client.server_version} connection through *native* authentication') + except irods.exception.CAT_INVALID_AUTHENTICATION: + # Authentication scheme is not native (default), so we try PAM here + try: + self.client = iRODSSession(host=self.url, port=1247, user=self.username, password=self.password, zone=self.irods_zone, irods_authentication_scheme='pam') + logger.debug(f'iRODS {self.client.server_version} connection through *PAM* authentication') + except irods.exception.CAT_INVALID_AUTHENTICATION: + # Authentication scheme is not PAM either last try: GIS + try: + self.client = iRODSSession(host=self.url, port=1247, user=self.username, password=self.password, zone=self.irods_zone, irods_authentication_scheme='gis') + logger.debug(f'iRODS {self.client.server_version} connection through *GIS* authentication') + except irods.exception.CAT_INVALID_AUTHENTICATION: + pass + + if self.client is None: + logger.error('Unable to login to the iRODS instance. Please check username and password combination!') + raise StorageException.InvalidAuthentication(self.username) + + logger.info('Created iRODS connection') + + def __close(self): + logger.debug('Closing iRODS storage connection and clean up') + self.client.cleanup() + + def _file_exists_action(self, path): + self.__connect() + try: + self.client.data_objects.get(f'/{self.irods_zone}/home/{self.username}/{path}') + except irods.exception.DataObjectDoesNotExist: + logger.debug(f'File \'{path}\' does NOT exists on the iRODS server') + return False + except irods.exception.CollectionDoesNotExist: + logger.debug(f'Parent folder of file \'{path}\' does NOT exists on the iRODS server') + return False + + return True + + def _directory_exists_action(self, path): + self.__connect() + try: + self.client.collections.get(f'/{self.irods_zone}/home/{self.username}/{path}') + logger.debug(f'Folder \'{path}\' exists on the iRODS server') + except irods.exception.CollectionDoesNotExist: + logger.debug(f'Folder \'{path}\' does NOT exists on the iRODS server') + return False + + return True + + def _make_folder_action(self, path): + self.__connect() + try: + self.client.collections.create(f'/{self.irods_zone}/home/{self.username}/{path}') + except irods.exception.CollectionDoesNotExist: + logger.debug(f'Parent folder of file \'{path}\' does NOT exists on the iRODS server') + return False + + return True + + def _upload_file_action(self, source, destination): + self.__connect() + # The upload path consists of a zone, username and path + destination = f'/{self.irods_zone}/home/{self.username}/{destination}' + logger.debug(f'Uploading to file: \'{destination}\'') + try: + obj = self.client.data_objects.create(destination) + logger.debug(f'Created file: \'{destination}\'') + # Open 'both' files and copy 4K data each time. + with obj.open('w') as irods_file, open(source, 'rb') as source_file_binary: + while True: + buf = source_file_binary.read(4096) + if buf: + irods_file.write(buf) + else: + break + + obj.metadata.add('source', f'Upload from VRE DataDropOff\n Added file: {destination} uploaded by: {self.sender_name}({self.sender_email})') + + except irods.exception.OVERWRITE_WITHOUT_FORCE_FLAG: + logger.warning('The uploaded file already exists. So we did NOT upload the new file!') + return False + + return True + + def _download_file_action(self, source, destination): + self.__connect() + logger.debug(f'Downloading file: \'{source}\' to \'{destination}\'') + try: + obj = self.client.data_objects.get(f'/{self.irods_zone}/home/{self.username}/{source}') + # Open 'both' files and copy 4K data each time. + with obj.open('r') as irods_source_file, open(destination, 'wb') as local_destination_file: + while True: + buf = irods_source_file.read(4096) + if buf: + local_destination_file.write(buf) + else: + break + + except irods.exception.DataObjectDoesNotExist: + logger.error(f'File: \'{source}\' does not exists on the iRODS server') + return False + + return True diff --git a/Enquete/enquete/storage/engines/webdav.py b/Enquete/enquete/storage/engines/webdav.py new file mode 100644 index 0000000..68a9bc4 --- /dev/null +++ b/Enquete/enquete/storage/engines/webdav.py @@ -0,0 +1,65 @@ +from webdav3.exceptions import WebDavException, ResponseErrorCode +from webdav3.client import Client +import storage.exceptions as StorageException +from storage.utils import human_filesize +from storage.storage import BaseStorage +import logging +logger = logging.getLogger(__name__) + + +# WebDAV Support - https://pypi.org/project/webdavclient3/ + + +class WebDAVStorage(BaseStorage): + + TYPE = 'webdav' + + def __connect(self): + # Connect to the external storage. This function can be run multiple times. It will check if it has already a connection to re-use + try: + # When this fails with an Attribute error, that means that the 'client' variable is not set and we need to make a new connection + assert(self.client) + except AttributeError: + # Because the 'client' variable is not known, the WebDAV connections is not created yet. So do it now! + self.client = Client({ + 'webdav_hostname': self.url, + 'webdav_login': self.username, + 'webdav_password': self.password, + }) + + try: + # Here we abuse the .free check to see if the login credentials do work + free_space = self.client.free() + logger.info(f'Created WebDAV connection to url: \'{self.url}\', with space left: {human_filesize(free_space)}') + except ResponseErrorCode as ex: + # Login went wrong, so delete the client variable for next run/try + del(self.client) + + # If there was an authentication error, raise exception and quit. + if 401 == ex.code: + raise StorageException.InvalidAuthentication(self.username) + + # TODO: More errors..... + + def _file_exists_action(self, path): + self.__connect() + return self.client.check(path) + + def _directory_exists_action(self, path): + self.__connect() + return self.client.check(path) + + def _make_folder_action(self, path): + self.__connect() + self.client.mkdir(path) + return True + + def _upload_file_action(self, source, destination): + self.__connect() + self.client.upload(local_path=source, remote_path=destination) + return True + + def _download_file_action(self, source, destination): + self.__connect() + self.client.download(source, destination) + return True diff --git a/Enquete/enquete/storage/exceptions.py b/Enquete/enquete/storage/exceptions.py new file mode 100644 index 0000000..272c288 --- /dev/null +++ b/Enquete/enquete/storage/exceptions.py @@ -0,0 +1,63 @@ + +class BaseStorageError(Exception): + pass + + +class StorageActionNotImplemented(Exception): + def __init__(self, storage, action, message='is not implemented'): + self.storage = storage + self.action = action + self.message = message + super().__init__(self.message) + + def __str__(self): + return f'{self.action} on class {self.storage} {self.message}' + + +class FileDoesNotExist(BaseStorageError): + def __init__(self, source, message='File does not exists on disk'): + self.source = source + self.message = message + super().__init__(self.message) + + def __str__(self): + return f'{self.source} -> {self.message}' + + +class InvalidLocation(BaseStorageError): + def __init__(self, location, message='Location does not exists or is not valid'): + self.location = location + self.message = message + super().__init__(self.message) + + def __str__(self): + return f'{self.location} -> {self.message}' + + +class InvalidAuthentication(BaseStorageError): + def __init__(self, user, message='Authentication failed'): + self.user = user + self.message = message + super().__init__(self.message) + + def __str__(self): + return f'{self.user} -> {self.message}' + + +class UnknownStorageEngine(BaseStorageError): + def __init__(self, engine, message='Storage engine is unknown, not available'): + self.engine = engine + self.message = message + super().__init__(self.message) + + def __str__(self): + return f'{self.engine} -> {self.message}' + + +class MissingEncryptionKey(BaseStorageError): + def __init__(self, message='The encryption keys are missing'): + self.message = message + super().__init__(self.message) + + def __str__(self): + return f'{self.message}' diff --git a/Enquete/enquete/storage/storage.py b/Enquete/enquete/storage/storage.py new file mode 100644 index 0000000..7bf9ccc --- /dev/null +++ b/Enquete/enquete/storage/storage.py @@ -0,0 +1,248 @@ +import shlex +import subprocess +import storage.exceptions as StorageException +import importlib +from pathlib import Path +import re +import glob +import os +import shutil +from datetime import datetime +import tempfile +import logging +logger = logging.getLogger(__name__) + + +class Storage(): + + CLASS_REGEX = re.compile(r'class\s+(?P[^\s\(]+)\s*\(\s*BaseStorage\s*\)\s*:') + + # This acts like a factory function. It will return a storage object from the requested engine + def __new__(self, storage_type, url=None, username=None, password=None, source=None, destination=None, encryption_key=None, sender_name=None, sender_email=None): + storage_type = storage_type.lower() + engines = Storage.__load_storage_engines() + logger.debug(f'Available storage engines({len(Storage.available_engines())}): {Storage.available_engines()}') + if storage_type not in engines: + raise StorageException.UnknownStorageEngine(storage_type) + + return engines[storage_type](url, username, password, source, destination, encryption_key, sender_name, sender_email) + + @staticmethod + def __load_storage_engines(): + loaded_engines = {} + engines = (Path(__file__)).parent.joinpath('engines') + + for engine in [x for x in engines.glob('*.py') if x.is_file()]: + with engine.open() as python_file: + data = python_file.read() + class_name = Storage.CLASS_REGEX.findall(data) + if len(class_name) == 1: + storage_engine_module = importlib.import_module('.{}' . format(engine.stem), package='storage.engines') + storage_engine_class = getattr(storage_engine_module, class_name[0]) + loaded_engines[storage_engine_class.TYPE.lower()] = storage_engine_class + + return loaded_engines + + @staticmethod + def available_engines(): + engines = list(Storage.__load_storage_engines().keys()) + engines.sort() + return engines + + +class BaseStorage(): + + ENCFS_XML = '.encfs6.xml' + ENCRYPT_CMD = '/usr/bin/encfs' + FUSER_MOUNT = '/bin/fusermount' + + TYPE = '' + + def __init__(self, url=None, username=None, password=None, source=None, destination=None, encryption_key=None, sender_name=None, sender_email=None): + if source is not None and not os.path.exists(source): + logger.error(f'Source file is not available on disk! It has vanished from: {source}') + raise StorageException.FileDoesNotExist(source) + + # if destination is None: + # logger.error(f'Destination is not valid: {destination}') + # raise StorageException.InvalidLocation(destination) + + self.source = source + self.destination_dir = None if destination is None else os.path.dirname(destination) + self.destination_file = None if destination is None else os.path.basename(destination) + self.encryption_key = encryption_key + self.encrypted = False + + self.url = url + self.username = username + self.password = password + self.sender_name = sender_name + self.sender_email = sender_email + + def encrypt_source(self): + if self.encryption_key is None: + logger.error(f'Cannot encrypt source file {self.source} due to missing encryption key!') + raise StorageException.MissingEncryptionKey() + + if self.encrypted: + logger.warning('File is already encrypted') + return True + + start_time = datetime.now() + logger.info(f'Encrypting new uploaded file: {self.source}') + + encrypted_dir = tempfile.mkdtemp() + logger.debug(f'Created encrypted source folder: {encrypted_dir}') + + decoded_dir = tempfile.mkdtemp() + logger.debug(f'Created decoded folder: {decoded_dir}') + + new_encryption_setup = True + existing_encfs_file = os.path.join(self.destination_dir, BaseStorage.ENCFS_XML) + logger.debug(f'Check for existing encryption key file \'{existing_encfs_file}\' on the destination location.') + if self.file_exists(existing_encfs_file): + logger.debug(f'Copying existing \'{BaseStorage.ENCFS_XML}\' file...') + self.download_file(existing_encfs_file, os.path.join(encrypted_dir, BaseStorage.ENCFS_XML)) + logger.info(f'Using existing \'{existing_encfs_file}\' from destination location.') + new_encryption_setup = False + + # Mounting part between source and encrypted folder + # TODO: Check what happens when there are spaces in the dir names... need some quotes I guess + cmd = f'{BaseStorage.ENCRYPT_CMD} --standard -S {encrypted_dir} {decoded_dir}' + logger.debug(f'Creating an encrypted EncFS mount point with command: {cmd}') + + process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + # # Send encryption password + logger.debug('Mounting in action. Sending encryption key...') + (output, error) = process.communicate(input=self.encryption_key.encode()) + + if process.wait(timeout=30) != 0: + output = output.decode().strip() + logger.error(f'Error creating an encrypted mount with EncFS. Error: \'{output}\'') + raise RuntimeError(f'Mounting error EncFS: {output}') + + logger.debug(f'Mountpoint is ready at path: {decoded_dir}') + + if new_encryption_setup: + logger.info(f'We have a new \'{BaseStorage.ENCFS_XML}\' file that needs to be moved to the same destination: {self.destination_dir}') + self.upload_file(os.path.join(encrypted_dir, BaseStorage.ENCFS_XML), existing_encfs_file, True) + + # Here we ignore the subdirectories on the destination. This will be fixed during the upload + destination_file = os.path.join(decoded_dir, self.destination_dir, self.destination_file) + logger.debug(f'Moving source file \'{self.source}\' to \'{destination_file}\' for encryption.') + os.makedirs(os.path.dirname(destination_file)) + shutil.move(self.source, destination_file) + + # Here we umount the decoded directory, so we only have the encypted data left + logger.debug(f'Encrypting is done, un-mounting decoded folder: {decoded_dir}') + cmd = f'{BaseStorage.FUSER_MOUNT} -u {decoded_dir}' + logger.debug(f'Umounting cmd: {cmd}') + process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + if process.wait() != 0: + # TODO: Better error handling... Add raise exception + logger.error(f'Error un-mounting mount point: {decoded_dir}') + raise RuntimeError(f'Un-mounting error EncFS: {process}') + + logger.debug(f'Cleanup temporary decoded dir: {decoded_dir}') + shutil.rmtree(decoded_dir) + + # Find the newly created encrypted file and move it back to the original source file + # We use the glob function so we can also support subdirectories in the encrypted storage + logger.debug(f'Finding newly created encrypted file in the encrypted source folder: {encrypted_dir}') + encrypted_listing = glob.glob(f'{encrypted_dir}/**', recursive=True) + logger.debug(f'Found encrypted file: {encrypted_listing[-1]}') + + # Source file is been changed to the new encrypted file name. So use that for the file upload process + self.source = os.path.join(os.path.dirname(self.source), os.path.basename(encrypted_listing[-1])) + self.destination_file = os.path.basename(self.source) + logger.debug(f'Moving encrypted file {encrypted_listing[-1]} back to original file: {self.source}') + logger.debug(f'Updated the destination file name based on the encrypted name: {self.destination_file}') + shutil.move(encrypted_listing[-1], self.source) + logger.info(f'Encrypted to \'{self.source}\' in {datetime.now() - start_time} (h:mm:ss.ms)') + + self.encrypted = True + return True + + def file_exists(self, path): + logger.debug(f'Check if file exists at path: \'{path}\' with engine: \'{self.TYPE}\'') + file_exists = self._file_exists_action(path) + exists = 'exist' if file_exists else 'does not exist' + logger.debug(f'File \'{path}\' {exists} on storage \'{self.TYPE}\'') + return file_exists + + def upload_file(self, source=None, destination=None, move=False): + source = self.source if source is None else source + destination = os.path.join(self.destination_dir, self.destination_file) if destination is None else destination + upload_ok = None + + if source is None or destination is None: + logger.error(f'Error uploading file. Either source: \'{source}\' or destination: \'{destination}\' is not set!') + + start_time = datetime.now() + logger.debug(f'Start uploading file: \'{source}\' to: \'{destination}\' with engine: \'{self.TYPE}\'') + + if not self.directory_exists(os.path.dirname(destination)): + self.create_directories(os.path.dirname(destination)) + + upload_ok = self._upload_file_action(source, destination) + if upload_ok: + logger.info(f'Uploaded \'{source}\' to: \'{destination}\' with engine: \'{self.TYPE}\' in {datetime.now() - start_time} (h:mm:ss.ms)') + if move or self.encrypted: + os.unlink(source) + logger.debug('Removed source file from disk!') + else: + logger.error(f'Error uploading \'{source}\' to: \'{destination}\' with engine: \'{self.TYPE}\' in {datetime.now() - start_time} (h:mm:ss.ms)') + + return upload_ok + + def directory_exists(self, path): + # logger.debug() + return self._directory_exists_action(path) + + def download_file(self, source=None, destination=None, move=False): + source = self.source if source is None else source + #destination = self.destination if destination is None else destination + destination = os.path.join(self.destination_dir, os.path.basename(self.destination_file)) if destination is None else destination + download_ok = None + + if source is None or destination is None: + logger.error(f'Error downloading file. Either source: {source} or destination: {destination} is not set!') + + start_time = datetime.now() + logger.debug('Downloading file: {source} to: {destination}') + download_ok = self._download_file_action(source, destination) + + if download_ok: + logger.info(f'Downloaded \'{source}\' to: \'{destination}\' in {datetime.now() - start_time} (h:mm:ss.ms)') + else: + logger.error(f'Downloading failed for \'{source}\' to: \'{destination}\' in {datetime.now() - start_time} (h:mm:ss.ms)') + + return download_ok + + def create_directories(self, path): + folders = [] + for folder in path.strip('/').split('/'): + # Store travelled path. We need this to make the directories on the remote servers + folders.append(folder) + if not self.directory_exists('/'.join(folders)): + logger.debug(f'Creating folder {folder} with full path: {"/".join(folders)}') + self._make_folder_action('/'.join(folders)) + else: + logger.debug(f'Folder \'{folder}\' already exists.') + + return True + + def _file_exists_action(self, path): + raise StorageException.StorageActionNotImplemented('BaseStorage', 'file_exists') + + def _directory_exists_action(self, path): + raise StorageException.StorageActionNotImplemented('BaseStorage', 'directory_exists') + + def _upload_file_action(self, source, destination): + raise StorageException.StorageActionNotImplemented('BaseStorage', '_upload_file') + + def _download_file_action(self, source, destination): + raise StorageException.StorageActionNotImplemented('BaseStorage', '_download_file') + + def _make_folder_action(self, path): + raise StorageException.StorageActionNotImplemented('BaseStorage', '_make_folder_action') diff --git a/Enquete/enquete/storage/utils.py b/Enquete/enquete/storage/utils.py new file mode 100644 index 0000000..091e434 --- /dev/null +++ b/Enquete/enquete/storage/utils.py @@ -0,0 +1,8 @@ +def human_filesize(nbytes): + suffixes = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'] + i = 0 + while nbytes >= 1024 and i < len(suffixes) - 1: + nbytes /= 1024. + i += 1 + f = ('%.2f' % nbytes).rstrip('0').rstrip('.') + return '%s %s' % (f, suffixes[i]) diff --git a/Enquete/enquete/templates/404.html b/Enquete/enquete/templates/404.html new file mode 100644 index 0000000..342deee --- /dev/null +++ b/Enquete/enquete/templates/404.html @@ -0,0 +1,19 @@ +{% extends 'base.html' %} +{% load i18n %} + +{% block title %}{% trans "Oops, sorry, page not found (404)" %}{% endblock %} +{% block pagetitle %}{% trans "Oops, sorry, page not found (404)" %}{% endblock %} +{% block content %} +
+ +
+ +{% endblock %} \ No newline at end of file diff --git a/Enquete/enquete/templates/base.html b/Enquete/enquete/templates/base.html new file mode 100644 index 0000000..a7f3e14 --- /dev/null +++ b/Enquete/enquete/templates/base.html @@ -0,0 +1,26 @@ +{% load static %} +{% load i18n %} + + + + + + + + + {% block title %}{% trans "Hanze enquete" %}{% endblock %} + + + + + + + +
+ + {% block content %} -=PageContent=- {% endblock %} +
+ + \ No newline at end of file diff --git a/Enquete/enquete/templates/index.html b/Enquete/enquete/templates/index.html new file mode 100644 index 0000000..a90b502 --- /dev/null +++ b/Enquete/enquete/templates/index.html @@ -0,0 +1,20 @@ +{% extends 'base.html' %} +{% load i18n %} + +{% block title %}{% trans "Virtual Research Environment" %}{% endblock %} +{% block pagetitle %}{% trans "Virtual Research Environment" %}{% endblock %} +{% block content %} +

+{% trans "Secure data drops to RUG Phd students" %} +

+

{% trans "Here you can securely upload files for Phd students and researchers." %}

+

+ {% trans "The following actions can be done:" %} +

    +
  1. {% trans "Study overview for grouping uploads for a single research study" %}
    2. +
  2. {% trans "Create a new study" %}
    3. +
  3. {% trans "Get a list of all the uploaded files for your studies" %}
    4. +
  4. {% trans "Logout" %}
    5. +
+

+{% endblock %} \ No newline at end of file diff --git a/Enquete/enquete/templates/menu.html b/Enquete/enquete/templates/menu.html new file mode 100644 index 0000000..fc2ba83 --- /dev/null +++ b/Enquete/enquete/templates/menu.html @@ -0,0 +1,44 @@ +{% load i18n %} + +
  • + {% trans "VRE" %} + +
    • diff --git a/Enquete/enquete/templates/singup.html b/Enquete/enquete/templates/singup.html new file mode 100644 index 0000000..a950e33 --- /dev/null +++ b/Enquete/enquete/templates/singup.html @@ -0,0 +1,12 @@ +{% extends 'base.html' %} +{% load i18n %} + +{% block title %}{% trans "Signup" %}{% endblock %} +{% block pagetitle %}{% trans "Signup" %}{% endblock %} +{% block content %} +

    +{% trans "Signup" %} +
    +{% blocktrans %}Please contact x@y.z{% endblocktrans %} +

    +{% endblock %} \ No newline at end of file diff --git a/Enquete/uitkomsten/.gitignore b/Enquete/uitkomsten/.gitignore new file mode 100644 index 0000000..2c27f21 --- /dev/null +++ b/Enquete/uitkomsten/.gitignore @@ -0,0 +1,2 @@ +* +!*.gitignore \ No newline at end of file