110 lines
4.3 KiB
Python
Executable File
110 lines
4.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import pandas as pd
|
|
import json
|
|
import re
|
|
|
|
FILE_NAME_REGEX = r'^\d+_\d+$' # Value of T3 in the annotations JSON
|
|
conversion_list = {'classification_id': [], 'filename':[], 'foglio':[], 'years': [], 'transcription':[], 'symbols': []}
|
|
|
|
source_files = []
|
|
source_files.append({
|
|
'filename' : 'source/transcribing-on-folio-level-classifications.csv',
|
|
'level' : 'folio'
|
|
})
|
|
source_files.append({
|
|
'filename' : 'source/transcribing-on-page-level-classifications.csv',
|
|
'level' : 'page'
|
|
})
|
|
|
|
FOLIO_REGEX = re.compile(r"\nfol[\. ,]", flags=re.IGNORECASE)
|
|
|
|
for source_file in source_files:
|
|
|
|
df = pd.read_csv(source_file['filename'], sep=',', quotechar='"')
|
|
|
|
for index, row in df.iterrows():
|
|
foglio = None
|
|
years = None
|
|
# Store the original classification_id
|
|
classification_id = row['classification_id']
|
|
transcription = ''
|
|
filename = ''
|
|
|
|
# Convert data from JSON to dict
|
|
tasks = json.loads(row['annotations'])
|
|
for task in tasks:
|
|
if 'folio' == source_file['level']:
|
|
if 'T1' == task['task']:
|
|
transcription = task['value'].strip()
|
|
elif 'T2' == task['task']:
|
|
symbols = False if 'no' == task['value'].lower() else True
|
|
|
|
elif 'T3' == task['task']:
|
|
filename = task['value'].strip()
|
|
|
|
if 'page' == source_file['level']:
|
|
if 'T0' == task['task']:
|
|
transcription = task['value'].strip()
|
|
elif 'T1' == task['task']:
|
|
symbols = False if 'no' == task['value'].lower() else True
|
|
|
|
|
|
if not re.match(FILE_NAME_REGEX, filename):
|
|
# Somebody mixedup the fields.... :(
|
|
data = re.search(r"(?P<filename>\d+_\d+)?(?P<rest>.*)",filename,re.MULTILINE | re.DOTALL)
|
|
|
|
if data:
|
|
filename = '' if not data.group('filename') else data.group('filename').strip()
|
|
if filename == transcription:
|
|
transcription = data.group('rest').strip()
|
|
else:
|
|
transcription += data.group('rest').strip()
|
|
|
|
if '' == filename:
|
|
# Final attempt, get it from the CSV column subject_data
|
|
subject_data = json.loads(row['subject_data'])
|
|
filename = subject_data[str(row['subject_ids'])]['Filename'].replace('.jpg','').strip()
|
|
|
|
# Somehow, sometimes, the transcription is starting with the filename. So remove that from the transcription
|
|
if transcription.startswith(filename):
|
|
transcription = transcription[len(filename):].strip()
|
|
|
|
if 'page' == source_file['level']:
|
|
# Here we split the transcription in multiple transcriptions due to the way of storing this data in the CSV
|
|
transcriptions = FOLIO_REGEX.split(transcription)
|
|
else:
|
|
# Only 1 transcription per line!
|
|
transcriptions = [transcription]
|
|
|
|
for index, transcription_part in enumerate(transcriptions):
|
|
|
|
if index == 0 and transcription.strip().lower().startswith('fol') or index > 0:
|
|
transcription_part = 'fol.' + transcription_part
|
|
|
|
transcription_part = transcription_part.strip().replace('\n','\\n')
|
|
|
|
# Get folio nummer.....
|
|
data = re.search(r"fol[\. ]+\d+[a-z]*",transcription_part,re.MULTILINE | re.IGNORECASE)
|
|
if data:
|
|
foglio = data.group(0)
|
|
|
|
# Get years
|
|
matches = re.finditer(r"1[5-8][0-9]{2}", transcription_part, re.MULTILINE | re.IGNORECASE)
|
|
if matches:
|
|
years = []
|
|
for matchNum, match in enumerate(matches, start=1):
|
|
years.append(match.group())
|
|
|
|
years = ','.join(sorted(list(set(years))))
|
|
|
|
conversion_list['classification_id'].append(classification_id)
|
|
conversion_list['filename'].append(filename)
|
|
conversion_list['foglio'].append(foglio)
|
|
conversion_list['years'].append(years)
|
|
conversion_list['transcription'].append(transcription_part)
|
|
conversion_list['symbols'].append(symbols)
|
|
|
|
convert = pd.DataFrame.from_dict(conversion_list)
|
|
convert.to_csv('converted.csv', encoding='utf-8', index=False)
|