import shlex import subprocess import storage.exceptions as StorageException import importlib from pathlib import Path import re import glob import os import shutil from datetime import datetime import tempfile import logging logger = logging.getLogger(__name__) class Storage(): CLASS_REGEX = re.compile(r'class\s+(?P[^\s\(]+)\s*\(\s*BaseStorage\s*\)\s*:') # This acts like a factory function. It will return a storage object from the requested engine def __new__(self, storage_type, url=None, username=None, password=None, source=None, destination=None, encryption_key=None, sender_name=None, sender_email=None): storage_type = storage_type.lower() engines = Storage.__load_storage_engines() logger.debug(f'Available storage engines({len(Storage.available_engines())}): {Storage.available_engines()}') if storage_type not in engines: raise StorageException.UnknownStorageEngine(storage_type) return engines[storage_type](url, username, password, source, destination, encryption_key, sender_name, sender_email) @staticmethod def __load_storage_engines(): loaded_engines = {} engines = (Path(__file__)).parent.joinpath('engines') for engine in [x for x in engines.glob('*.py') if x.is_file()]: with engine.open() as python_file: data = python_file.read() class_name = Storage.CLASS_REGEX.findall(data) if len(class_name) == 1: storage_engine_module = importlib.import_module('.{}' . format(engine.stem), package='storage.engines') storage_engine_class = getattr(storage_engine_module, class_name[0]) loaded_engines[storage_engine_class.TYPE.lower()] = storage_engine_class return loaded_engines @staticmethod def available_engines(): engines = list(Storage.__load_storage_engines().keys()) engines.sort() return engines class BaseStorage(): ENCFS_XML = '.encfs6.xml' ENCRYPT_CMD = '/usr/bin/encfs' FUSER_MOUNT = '/bin/fusermount' TYPE = '' def __init__(self, url=None, username=None, password=None, source=None, destination=None, encryption_key=None, sender_name=None, sender_email=None): if source is not None and not os.path.exists(source): logger.error(f'Source file is not available on disk! It has vanished from: {source}') raise StorageException.FileDoesNotExist(source) # if destination is None: # logger.error(f'Destination is not valid: {destination}') # raise StorageException.InvalidLocation(destination) self.source = source self.destination_dir = None if destination is None else os.path.dirname(destination) self.destination_file = None if destination is None else os.path.basename(destination) self.encryption_key = encryption_key self.encrypted = False self.url = url self.username = username self.password = password self.sender_name = sender_name self.sender_email = sender_email def encrypt_source(self): if self.encryption_key is None: logger.error(f'Cannot encrypt source file {self.source} due to missing encryption key!') raise StorageException.MissingEncryptionKey() if self.encrypted: logger.warning('File is already encrypted') return True start_time = datetime.now() logger.info(f'Encrypting new uploaded file: {self.source}') encrypted_dir = tempfile.mkdtemp() logger.debug(f'Created encrypted source folder: {encrypted_dir}') decoded_dir = tempfile.mkdtemp() logger.debug(f'Created decoded folder: {decoded_dir}') new_encryption_setup = True existing_encfs_file = os.path.join(self.destination_dir, BaseStorage.ENCFS_XML) logger.debug(f'Check for existing encryption key file \'{existing_encfs_file}\' on the destination location.') if self.file_exists(existing_encfs_file): logger.debug(f'Copying existing \'{BaseStorage.ENCFS_XML}\' file...') self.download_file(existing_encfs_file, os.path.join(encrypted_dir, BaseStorage.ENCFS_XML)) logger.info(f'Using existing \'{existing_encfs_file}\' from destination location.') new_encryption_setup = False # Mounting part between source and encrypted folder # TODO: Check what happens when there are spaces in the dir names... need some quotes I guess cmd = f'{BaseStorage.ENCRYPT_CMD} --standard -S {encrypted_dir} {decoded_dir}' logger.debug(f'Creating an encrypted EncFS mount point with command: {cmd}') process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) # # Send encryption password logger.debug('Mounting in action. Sending encryption key...') (output, error) = process.communicate(input=self.encryption_key.encode()) if process.wait(timeout=30) != 0: output = output.decode().strip() logger.error(f'Error creating an encrypted mount with EncFS. Error: \'{output}\'') raise RuntimeError(f'Mounting error EncFS: {output}') logger.debug(f'Mountpoint is ready at path: {decoded_dir}') if new_encryption_setup: logger.info(f'We have a new \'{BaseStorage.ENCFS_XML}\' file that needs to be moved to the same destination: {self.destination_dir}') self.upload_file(os.path.join(encrypted_dir, BaseStorage.ENCFS_XML), existing_encfs_file, True) # Here we ignore the subdirectories on the destination. This will be fixed during the upload destination_file = os.path.join(decoded_dir, self.destination_dir, self.destination_file) logger.debug(f'Moving source file \'{self.source}\' to \'{destination_file}\' for encryption.') os.makedirs(os.path.dirname(destination_file)) shutil.move(self.source, destination_file) # Here we umount the decoded directory, so we only have the encypted data left logger.debug(f'Encrypting is done, un-mounting decoded folder: {decoded_dir}') cmd = f'{BaseStorage.FUSER_MOUNT} -u {decoded_dir}' logger.debug(f'Umounting cmd: {cmd}') process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) if process.wait() != 0: # TODO: Better error handling... Add raise exception logger.error(f'Error un-mounting mount point: {decoded_dir}') raise RuntimeError(f'Un-mounting error EncFS: {process}') logger.debug(f'Cleanup temporary decoded dir: {decoded_dir}') shutil.rmtree(decoded_dir) # Find the newly created encrypted file and move it back to the original source file # We use the glob function so we can also support subdirectories in the encrypted storage logger.debug(f'Finding newly created encrypted file in the encrypted source folder: {encrypted_dir}') encrypted_listing = glob.glob(f'{encrypted_dir}/**', recursive=True) logger.debug(f'Found encrypted file: {encrypted_listing[-1]}') # Source file is been changed to the new encrypted file name. So use that for the file upload process self.source = os.path.join(os.path.dirname(self.source), os.path.basename(encrypted_listing[-1])) self.destination_file = os.path.basename(self.source) logger.debug(f'Moving encrypted file {encrypted_listing[-1]} back to original file: {self.source}') logger.debug(f'Updated the destination file name based on the encrypted name: {self.destination_file}') shutil.move(encrypted_listing[-1], self.source) logger.info(f'Encrypted to \'{self.source}\' in {datetime.now() - start_time} (h:mm:ss.ms)') self.encrypted = True return True def file_exists(self, path): logger.debug(f'Check if file exists at path: \'{path}\' with engine: \'{self.TYPE}\'') file_exists = self._file_exists_action(path) exists = 'exist' if file_exists else 'does not exist' logger.debug(f'File \'{path}\' {exists} on storage \'{self.TYPE}\'') return file_exists def upload_file(self, source=None, destination=None, move=False): source = self.source if source is None else source destination = os.path.join(self.destination_dir, self.destination_file) if destination is None else destination upload_ok = None if source is None or destination is None: logger.error(f'Error uploading file. Either source: \'{source}\' or destination: \'{destination}\' is not set!') start_time = datetime.now() logger.debug(f'Start uploading file: \'{source}\' to: \'{destination}\' with engine: \'{self.TYPE}\'') if not self.directory_exists(os.path.dirname(destination)): self.create_directories(os.path.dirname(destination)) upload_ok = self._upload_file_action(source, destination) if upload_ok: logger.info(f'Uploaded \'{source}\' to: \'{destination}\' with engine: \'{self.TYPE}\' in {datetime.now() - start_time} (h:mm:ss.ms)') if move or self.encrypted: os.unlink(source) logger.debug('Removed source file from disk!') else: logger.error(f'Error uploading \'{source}\' to: \'{destination}\' with engine: \'{self.TYPE}\' in {datetime.now() - start_time} (h:mm:ss.ms)') return upload_ok def directory_exists(self, path): # logger.debug() return self._directory_exists_action(path) def download_file(self, source=None, destination=None, move=False): source = self.source if source is None else source #destination = self.destination if destination is None else destination destination = os.path.join(self.destination_dir, os.path.basename(self.destination_file)) if destination is None else destination download_ok = None if source is None or destination is None: logger.error(f'Error downloading file. Either source: {source} or destination: {destination} is not set!') start_time = datetime.now() logger.debug('Downloading file: {source} to: {destination}') download_ok = self._download_file_action(source, destination) if download_ok: logger.info(f'Downloaded \'{source}\' to: \'{destination}\' in {datetime.now() - start_time} (h:mm:ss.ms)') else: logger.error(f'Downloading failed for \'{source}\' to: \'{destination}\' in {datetime.now() - start_time} (h:mm:ss.ms)') return download_ok def create_directories(self, path): folders = [] for folder in path.strip('/').split('/'): # Store travelled path. We need this to make the directories on the remote servers folders.append(folder) if not self.directory_exists('/'.join(folders)): logger.debug(f'Creating folder {folder} with full path: {"/".join(folders)}') self._make_folder_action('/'.join(folders)) else: logger.debug(f'Folder \'{folder}\' already exists.') return True def _file_exists_action(self, path): raise StorageException.StorageActionNotImplemented('BaseStorage', 'file_exists') def _directory_exists_action(self, path): raise StorageException.StorageActionNotImplemented('BaseStorage', 'directory_exists') def _upload_file_action(self, source, destination): raise StorageException.StorageActionNotImplemented('BaseStorage', '_upload_file') def _download_file_action(self, source, destination): raise StorageException.StorageActionNotImplemented('BaseStorage', '_download_file') def _make_folder_action(self, path): raise StorageException.StorageActionNotImplemented('BaseStorage', '_make_folder_action')