P300021/Hanze_Surfnet_Data_POC
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
Joshua Rubingh
2022-02-17 14:35:54 +01:00
commit abbaaab98a
43 changed files with 2079 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
Enquete/enquete/storage/engines/fs.py Normal file
View File

@ -0,0 +1,28 @@
import shutil
import os
from storage.storage import BaseStorage
import logging
logger = logging.getLogger(__name__)
class LocalStorage(BaseStorage):
TYPE = 'fs'
def file_exists(self, filepath):
return os.path.exists(filepath) and os.path.isfile(filepath)
def directory_exists(self, filepath):
return os.path.exists(filepath) and os.path.isdir(filepath)
def _make_folder_action(self, path):
os.makedirs(path)
return True
def _upload_file_action(self, source, destination):
shutil.copy(source, destination)
return True
def _download_file_action(self, source, destination):
shutil.copy(source, destination)
return True

89
Enquete/enquete/storage/engines/gitea.py Normal file
View File

@ -0,0 +1,89 @@
from giteapy.rest import ApiException
import giteapy
import base64
from storage.storage import BaseStorage
import logging
logger = logging.getLogger(__name__)
# Gitea Support - https://pypi.org/project/giteapy/
class GiteaStorage(BaseStorage):
TYPE = 'gitea'
def __init__(self, url=None, username=None, password=None, source=None, destination=None, encryption_key=None, sender_name=None, sender_email=None):
# The repository is added to the url parameter. Use a '#' as seperator. The repository needs to be created first.
# Ex: https://git.web.rug.nl/api/v1#RepositoryName
(url, self.repository) = url.split('#')
destination = destination.strip('/')
super().__init__(url, username, password, source, destination, encryption_key, sender_name, sender_email)
# Create a commiter object when the data is uploaded through one of the invited accounts.
self.committer = None
if sender_name is not None or sender_email is not None:
self.committer = giteapy.Identity(name=sender_name, email=sender_email)
def __connect(self):
try:
assert(self.client)
except AttributeError:
# Configuration for the GITEA connection
configuration = giteapy.Configuration()
# Overrule the host url....?
configuration.host = self.url
#configuration.debug = False
configuration.api_key['access_token'] = self.password
# Create the client
self.client = giteapy.RepositoryApi(giteapy.ApiClient(configuration))
logger.info(f'Created Gitea connection to url: {self.url}')
def file_exists(self, filepath):
self.__connect()
try:
self.client.repo_get_contents(self.username, self.repository, filepath)
return True
except ApiException:
return False
def directory_exists(self, filepath):
self.__connect()
return self.file_exists(filepath)
def _make_folder_action(self, path):
# On GitHub you cannot create empty directories. So this actions will always succeed
return True
def _upload_file_action(self, source, destination):
self.__connect()
try:
with open(source, 'rb') as datafile:
# This is a very big issue. Big files will be stored completely in memory :(
body = giteapy.CreateFileOptions(content=base64.b64encode(datafile.read()).decode(),
message=f'Upload from VRE DataDropOff\n Added file: {destination}',
committer=self.committer)
except Exception:
return False
try:
# Create a file in a repository
api_response = self.client.repo_create_file(self.username, self.repository, destination, body)
return True
except ApiException as ex:
logger.exception(f'Exception when calling RepositoryApi->repo_create_file: {ex}')
return True
def _download_file_action(self, source, destination):
self.__connect()
with open(destination, 'wb') as destination_file:
try:
data = self.client.repo_get_contents(self.username, self.repository, source)
destination_file.write(base64.b64decode(data.content))
except ApiException as ex:
logger.exception(f'Exception when calling RepositoryApi->repo_get_contents: {ex}')
return True

66
Enquete/enquete/storage/engines/github.py Normal file
View File

@ -0,0 +1,66 @@
from github.GithubException import UnknownObjectException
from github import Github, InputGitAuthor, GithubObject
from storage.storage import BaseStorage
import os
import logging
logger = logging.getLogger(__name__)
# Github Support - https://pypi.org/project/PyGithub/
class GithubStorage(BaseStorage):
TYPE = 'github'
def __init__(self, url=None, username=None, password=None, source=None, destination=None, encryption_key=None, sender_name=None, sender_email=None):
# The repository is added to the url parameter. Use a '#' as seperator. The repository needs to be created first.
# Ex: https://api.github.com/#RepositoryName
(url, self.repository) = url.split('#')
destination = destination.strip('/')
super().__init__(url, username, password, source, destination, encryption_key, sender_name, sender_email)
# Create a commiter object when the data is uploaded through one of the invited accounts.
self.committer = GithubObject.NotSet
if sender_name is not None or sender_email is not None:
self.committer = InputGitAuthor(name=sender_name, email=sender_email)
def __connect(self):
try:
assert(self.repo)
except AttributeError:
client = Github(self.password)
self.repo = client.get_user().get_repo(self.repository)
logger.info('Created Github.com connection')
def file_exists(self, filepath):
self.__connect()
try:
self.repo.get_contents(filepath)
return True
except UnknownObjectException:
return False
def directory_exists(self, filepath):
return True
def _make_folder_action(self, path):
# On GitHub you cannot create empty directories. So this actions will always succeed
return True
def _upload_file_action(self, source, destination):
self.__connect()
# Read the file and post to Github. The library will convert to Base64
with open(source, 'rb') as datafile:
self.repo.create_file(destination.strip('/'), f'Upload from VRE DataDropOff\n Added file: {destination}', datafile.read(), committer=self.committer)
return True
def _download_file_action(self, source, destination):
self.__connect()
download = self.repo.get_contents(source)
with open(destination, 'wb') as destination_file:
destination_file.write(download.decoded_content)
return True

139
Enquete/enquete/storage/engines/irods.py Normal file
View File

@ -0,0 +1,139 @@
import atexit
from irods.session import iRODSSession
import irods
import storage.exceptions as StorageException
from storage.storage import BaseStorage
import logging
logger = logging.getLogger(__name__)
# iRods support - https://pypi.org/project/python-irodsclient/
class iRODSStorage(BaseStorage):
TYPE = 'irods'
def __init__(self, url=None, username=None, password=None, source=None, destination=None, encryption_key=None, sender_name=None, sender_email=None):
# The iRODS zone is added to the url parameter. Use a '#' as seperator. This needs to be an Existing iRODS zone
# Ex: rdms-prod-icat.data.rug.nl#rug
(url, self.irods_zone) = url.split('#')
if destination:
destination = destination.strip('/')
super().__init__(url, username, password, source, destination, encryption_key, sender_name, sender_email)
# We need to clean up the iRODS session. Using atexit is the easiest way.
atexit.register(self.__close)
def __connect(self):
try:
assert(self.client)
except AttributeError:
# Connect to the iRODS server
self.client = None
try:
self.client = iRODSSession(host=self.url, port=1247, user=self.username, password=self.password, zone=self.irods_zone)
# Need to make a call to validate the authentication. So by checking the version, we know if we can authenticate...
logger.debug(f'iRODS {self.client.server_version} connection through *native* authentication')
except irods.exception.CAT_INVALID_AUTHENTICATION:
# Authentication scheme is not native (default), so we try PAM here
try:
self.client = iRODSSession(host=self.url, port=1247, user=self.username, password=self.password, zone=self.irods_zone, irods_authentication_scheme='pam')
logger.debug(f'iRODS {self.client.server_version} connection through *PAM* authentication')
except irods.exception.CAT_INVALID_AUTHENTICATION:
# Authentication scheme is not PAM either last try: GIS
try:
self.client = iRODSSession(host=self.url, port=1247, user=self.username, password=self.password, zone=self.irods_zone, irods_authentication_scheme='gis')
logger.debug(f'iRODS {self.client.server_version} connection through *GIS* authentication')
except irods.exception.CAT_INVALID_AUTHENTICATION:
pass
if self.client is None:
logger.error('Unable to login to the iRODS instance. Please check username and password combination!')
raise StorageException.InvalidAuthentication(self.username)
logger.info('Created iRODS connection')
def __close(self):
logger.debug('Closing iRODS storage connection and clean up')
self.client.cleanup()
def _file_exists_action(self, path):
self.__connect()
try:
self.client.data_objects.get(f'/{self.irods_zone}/home/{self.username}/{path}')
except irods.exception.DataObjectDoesNotExist:
logger.debug(f'File \'{path}\' does NOT exists on the iRODS server')
return False
except irods.exception.CollectionDoesNotExist:
logger.debug(f'Parent folder of file \'{path}\' does NOT exists on the iRODS server')
return False
return True
def _directory_exists_action(self, path):
self.__connect()
try:
self.client.collections.get(f'/{self.irods_zone}/home/{self.username}/{path}')
logger.debug(f'Folder \'{path}\' exists on the iRODS server')
except irods.exception.CollectionDoesNotExist:
logger.debug(f'Folder \'{path}\' does NOT exists on the iRODS server')
return False
return True
def _make_folder_action(self, path):
self.__connect()
try:
self.client.collections.create(f'/{self.irods_zone}/home/{self.username}/{path}')
except irods.exception.CollectionDoesNotExist:
logger.debug(f'Parent folder of file \'{path}\' does NOT exists on the iRODS server')
return False
return True
def _upload_file_action(self, source, destination):
self.__connect()
# The upload path consists of a zone, username and path
destination = f'/{self.irods_zone}/home/{self.username}/{destination}'
logger.debug(f'Uploading to file: \'{destination}\'')
try:
obj = self.client.data_objects.create(destination)
logger.debug(f'Created file: \'{destination}\'')
# Open 'both' files and copy 4K data each time.
with obj.open('w') as irods_file, open(source, 'rb') as source_file_binary:
while True:
buf = source_file_binary.read(4096)
if buf:
irods_file.write(buf)
else:
break
obj.metadata.add('source', f'Upload from VRE DataDropOff\n Added file: {destination} uploaded by: {self.sender_name}({self.sender_email})')
except irods.exception.OVERWRITE_WITHOUT_FORCE_FLAG:
logger.warning('The uploaded file already exists. So we did NOT upload the new file!')
return False
return True
def _download_file_action(self, source, destination):
self.__connect()
logger.debug(f'Downloading file: \'{source}\' to \'{destination}\'')
try:
obj = self.client.data_objects.get(f'/{self.irods_zone}/home/{self.username}/{source}')
# Open 'both' files and copy 4K data each time.
with obj.open('r') as irods_source_file, open(destination, 'wb') as local_destination_file:
while True:
buf = irods_source_file.read(4096)
if buf:
local_destination_file.write(buf)
else:
break
except irods.exception.DataObjectDoesNotExist:
logger.error(f'File: \'{source}\' does not exists on the iRODS server')
return False
return True

65
Enquete/enquete/storage/engines/webdav.py Normal file
View File

@ -0,0 +1,65 @@
from webdav3.exceptions import WebDavException, ResponseErrorCode
from webdav3.client import Client
import storage.exceptions as StorageException
from storage.utils import human_filesize
from storage.storage import BaseStorage
import logging
logger = logging.getLogger(__name__)
# WebDAV Support - https://pypi.org/project/webdavclient3/
class WebDAVStorage(BaseStorage):
TYPE = 'webdav'
def __connect(self):
# Connect to the external storage. This function can be run multiple times. It will check if it has already a connection to re-use
try:
# When this fails with an Attribute error, that means that the 'client' variable is not set and we need to make a new connection
assert(self.client)
except AttributeError:
# Because the 'client' variable is not known, the WebDAV connections is not created yet. So do it now!
self.client = Client({
'webdav_hostname': self.url,
'webdav_login': self.username,
'webdav_password': self.password,
})
try:
# Here we abuse the .free check to see if the login credentials do work
free_space = self.client.free()
logger.info(f'Created WebDAV connection to url: \'{self.url}\', with space left: {human_filesize(free_space)}')
except ResponseErrorCode as ex:
# Login went wrong, so delete the client variable for next run/try
del(self.client)
# If there was an authentication error, raise exception and quit.
if 401 == ex.code:
raise StorageException.InvalidAuthentication(self.username)
# TODO: More errors.....
def _file_exists_action(self, path):
self.__connect()
return self.client.check(path)
def _directory_exists_action(self, path):
self.__connect()
return self.client.check(path)
def _make_folder_action(self, path):
self.__connect()
self.client.mkdir(path)
return True
def _upload_file_action(self, source, destination):
self.__connect()
self.client.upload(local_path=source, remote_path=destination)
return True
def _download_file_action(self, source, destination):
self.__connect()
self.client.download(source, destination)
return True