376 lines
13 KiB
Python
376 lines
13 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""
|
|
Script permettant de générer des fichiers CSV à partir d'agenda Nextcloud
|
|
contenant un récapitulatif des heures de travail effectué
|
|
|
|
Pour Crealead
|
|
|
|
@author: Jordan
|
|
@date: 17/11/2022
|
|
|
|
@TODO: Améliorer la structure du code, pour l'instant c'est un peu moche
|
|
@TODO: Ajouter des paramètres pour le mois, et le verbose pour avoir des logs
|
|
@TOOD: README sur l'utilisation du script
|
|
"""
|
|
|
|
import sys
|
|
import caldav
|
|
import json
|
|
import csv
|
|
from datetime import date, datetime, time, timezone
|
|
from icalendar import Calendar
|
|
import pytz
|
|
from urllib.parse import unquote
|
|
import re
|
|
import traceback
|
|
from contextlib import contextmanager, redirect_stdout, redirect_stderr
|
|
from pathlib import Path
|
|
|
|
|
|
def utc_timestamp():
|
|
return datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
|
|
|
|
|
|
class TimestampedLog:
|
|
"""Prefix each log line, including lines written in several fragments."""
|
|
|
|
def __init__(self, log_file):
|
|
self.log_file = log_file
|
|
self.line_start = True
|
|
|
|
def write(self, value):
|
|
# Keep an event warning separate from an unfinished console message.
|
|
if value.startswith('[warning][event] - ') and not self.line_start:
|
|
self.log_file.write('\n')
|
|
self.line_start = True
|
|
for fragment in value.splitlines(keepends=True):
|
|
if self.line_start:
|
|
self.log_file.write('[{}]: '.format(utc_timestamp()))
|
|
if not fragment.startswith('[warning][event] - '):
|
|
self.log_file.write('[info] - ')
|
|
self.log_file.write(fragment)
|
|
self.line_start = fragment.endswith(('\n', '\r'))
|
|
return len(value)
|
|
|
|
def flush(self):
|
|
self.log_file.flush()
|
|
|
|
|
|
class TeeOutput:
|
|
"""Write to the original console stream and to the shared log file."""
|
|
|
|
def __init__(self, stream, log_file):
|
|
self.stream = stream
|
|
self.log_file = log_file
|
|
|
|
def write(self, value):
|
|
written = self.stream.write(value)
|
|
self.log_file.write(value)
|
|
self.log_file.flush()
|
|
return written
|
|
|
|
def flush(self):
|
|
self.stream.flush()
|
|
self.log_file.flush()
|
|
|
|
def __getattr__(self, name):
|
|
return getattr(self.stream, name)
|
|
|
|
|
|
@contextmanager
|
|
def log_output():
|
|
"""Append console output to logs/<script name>.log next to the script."""
|
|
script_path = Path(__file__).resolve()
|
|
log_path = script_path.parent / 'logs' / script_path.with_suffix('.log').name
|
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with log_path.open('a', encoding='utf-8') as log_file:
|
|
log_writer = TimestampedLog(log_file)
|
|
with redirect_stdout(TeeOutput(sys.stdout, log_writer)), \
|
|
redirect_stderr(TeeOutput(sys.stderr, log_writer)):
|
|
try:
|
|
yield
|
|
except BaseException as error:
|
|
# The interpreter prints the traceback after the streams are restored.
|
|
# Save a copy here while the log is still open, without duplicating it
|
|
# in the console. SystemExit normally has no traceback.
|
|
if not log_writer.line_start:
|
|
log_writer.write('\n')
|
|
if isinstance(error, SystemExit):
|
|
if error.code is not None and not isinstance(error.code, int):
|
|
log_writer.write(str(error.code) + '\n')
|
|
else:
|
|
traceback.print_exc(file=log_writer)
|
|
raise
|
|
finally:
|
|
if not log_writer.line_start:
|
|
log_writer.write('\n')
|
|
|
|
|
|
def recover_event_contexts(data):
|
|
"""Recover identifying fields even when the iCalendar parser rejects data."""
|
|
if isinstance(data, bytes):
|
|
data = data.decode('utf-8', errors='replace')
|
|
unfolded = re.sub(r'\r?\n[ \t]', '', str(data))
|
|
blocks = re.split(r'(?im)^BEGIN:VEVENT\s*\r?$', unfolded)
|
|
contexts = []
|
|
for block in blocks[1:] or [unfolded]:
|
|
context = {}
|
|
depth = 0
|
|
for line in block.splitlines():
|
|
name, separator, value = line.partition(':')
|
|
name = name.split(';', 1)[0].upper()
|
|
if name == 'BEGIN':
|
|
depth += 1
|
|
elif name == 'END':
|
|
if depth == 0:
|
|
break
|
|
depth -= 1
|
|
elif depth == 0 and separator and name in ('SUMMARY', 'DTSTART', 'DTEND', 'UID'):
|
|
context.setdefault(name, value)
|
|
contexts.append(context)
|
|
return contexts
|
|
|
|
|
|
def log_skipped_event(calendar_find, component, reason, fallback=None):
|
|
"""Every rejected event must have an identifiable warning in the log."""
|
|
fallback = fallback or {}
|
|
|
|
def field(name):
|
|
value = component.get(name)
|
|
if value is None:
|
|
return str(fallback.get(name, 'indisponible'))
|
|
decoded = getattr(value, 'dt', None)
|
|
if isinstance(decoded, datetime):
|
|
return local_datetime(decoded).astimezone(timezone.utc).isoformat().replace('+00:00', 'Z')
|
|
if isinstance(decoded, date):
|
|
return decoded.isoformat()
|
|
return str(value)
|
|
|
|
calendar_name = getattr(calendar_find.get('calendar_server'), 'name', None)
|
|
calendar_name = calendar_name or calendar_find.get('calendar_name', 'indisponible')
|
|
# repr keeps newlines in event metadata from breaking the warning format.
|
|
print('[warning][event] - Calendrier={!r} ; CSV={!r} ; titre={!r} ; '
|
|
'début={!r} ; fin={!r} ; UID={!r} ; événement non exporté : {}'.format(
|
|
str(calendar_name), calendar_find['csv_name'], field('SUMMARY'),
|
|
field('DTSTART'), field('DTEND'), field('UID'),
|
|
str(reason).replace('\r', '\\r').replace('\n', '\\n')), file=sys.stderr)
|
|
|
|
|
|
class EventRow():
|
|
|
|
def __init__(self):
|
|
self.title = ''
|
|
self.start_date = ''
|
|
self.start_time = ''
|
|
self.end_date = ''
|
|
self.end_time = ''
|
|
self.categories = ''
|
|
self.location = ''
|
|
self.description = ''
|
|
self.event_context = {}
|
|
|
|
def set_title(self, title):
|
|
self.title = title
|
|
|
|
def set_start(self, start):
|
|
self.start_date = start.date()
|
|
self.start_time = start.time()
|
|
|
|
def set_end(self, end):
|
|
self.end_date = end.date()
|
|
self.end_time = end.time()
|
|
|
|
def set_categories(self, categories):
|
|
self.categories = categories
|
|
|
|
def set_location(self, location):
|
|
self.location = location
|
|
|
|
def set_description(self, description):
|
|
self.description = description
|
|
|
|
def get_csv_row(self):
|
|
"""Return CSV row"""
|
|
return [self.title,self.start_date,self.start_time,self.end_date,self.end_time,self.categories,self.location,self.description]
|
|
|
|
def load_json_config(filename):
|
|
"""Return the config from JSON file"""
|
|
with open(filename, 'r') as f:
|
|
config_json = json.load(f)
|
|
return config_json
|
|
|
|
|
|
def init_connection(config):
|
|
"""Return all the calendars from the server"""
|
|
|
|
dav_client = caldav.DAVClient(url=config['caldav_url'], username=config['username'], password=config['password'])
|
|
|
|
try:
|
|
dav_connection = dav_client.principal()
|
|
except Exception as e:
|
|
print("Une erreur a eu lieu pendant la connexion : ", e)
|
|
sys.exit()
|
|
|
|
calendars_server = dav_connection.calendars()
|
|
|
|
if not calendars_server:
|
|
print("Aucun calendrier disponible sur le compte !")
|
|
sys.exit()
|
|
|
|
return calendars_server
|
|
|
|
|
|
def link_calendars(config, calendars_server):
|
|
"""Do the link between calendar we search (from the config)
|
|
and calendar available on the remote server"""
|
|
|
|
calendars_find = []
|
|
for calendar_config in config['calendars']:
|
|
cpt = 0
|
|
for calendar_server in calendars_server:
|
|
if calendar_config['calendar_name'] in calendar_server.name:
|
|
cpt += 1
|
|
calendar_find = calendar_config
|
|
calendar_find['calendar_server'] = calendar_server
|
|
calendars_find.append(calendar_find)
|
|
|
|
if not cpt:
|
|
print("Le calendrier nommé " + calendar_config['calendar_name'] + " pour le fichier " + calendar_config['csv_name'] + " n'est pas trouvé sur le serveur !")
|
|
sys.exit()
|
|
elif cpt >= 2:
|
|
print("Il y a plusieurs calendriers sur le serveur correspondant au calendrier nommé " + calendar_config['calendar_name'] + " pour le fichier " + calendar_config['csv_name'] + " !")
|
|
print(calendars_find)
|
|
sys.exit()
|
|
|
|
return calendars_find
|
|
|
|
|
|
def local_datetime(value):
|
|
"""Convert an iCalendar date or datetime to Europe/Paris."""
|
|
paris_tz = pytz.timezone('Europe/Paris')
|
|
if isinstance(value, datetime):
|
|
result = value
|
|
elif isinstance(value, date):
|
|
result = datetime.combine(value, time.min)
|
|
else:
|
|
raise ValueError("date absente ou invalide")
|
|
if result.tzinfo is None:
|
|
return paris_tz.localize(result)
|
|
return result.astimezone(paris_tz)
|
|
|
|
|
|
def get_all_events(config, calendars_find):
|
|
"""Return row with the specific information we need to
|
|
write it in CSV"""
|
|
|
|
calendars_to_write = []
|
|
|
|
for calendar_find in calendars_find:
|
|
print(calendar_find['csv_name'], "...")
|
|
events_fetched = calendar_find['calendar_server'].date_search(start=datetime(config['year_start'], config['month_start'], config['day_start']),
|
|
end=datetime(config['year_end'], config['month_end'], config['day_end']),
|
|
expand=True)
|
|
event_rows = []
|
|
|
|
for event in events_fetched:
|
|
contexts = recover_event_contexts(event.data)
|
|
# Parse parameters (TZID, VALUE=DATE), folded lines and escaped text.
|
|
try:
|
|
event_calendar = Calendar.from_ical(event.data)
|
|
except (ValueError, TypeError) as error:
|
|
for context in contexts:
|
|
log_skipped_event(calendar_find, {},
|
|
'document iCalendar illisible ({})'.format(error), context)
|
|
continue
|
|
|
|
components = event_calendar.walk('VEVENT')
|
|
if not components:
|
|
log_skipped_event(calendar_find, {}, 'aucun composant VEVENT', contexts[0])
|
|
|
|
for index, component in enumerate(components):
|
|
context = contexts[index] if index < len(contexts) else {}
|
|
# Fix problem #2023022110004936: ignore events without a summary.
|
|
if 'SUMMARY' not in component:
|
|
log_skipped_event(calendar_find, component, 'propriété SUMMARY absente', context)
|
|
continue
|
|
|
|
try:
|
|
start = component.decoded('DTSTART')
|
|
end = component.decoded('DTEND', None)
|
|
if end is None and 'DURATION' in component:
|
|
end = start + component.decoded('DURATION')
|
|
start = local_datetime(start)
|
|
end = local_datetime(end)
|
|
except (KeyError, ValueError, TypeError) as error:
|
|
log_skipped_event(calendar_find, component,
|
|
'dates de début/fin absentes ou invalides ({})'.format(error), context)
|
|
continue
|
|
|
|
event_row = EventRow()
|
|
event_row.event_context = component
|
|
event_row.set_title(str(component['SUMMARY']))
|
|
event_row.set_start(start)
|
|
event_row.set_end(end)
|
|
categories = component.get('CATEGORIES', [])
|
|
if not isinstance(categories, list):
|
|
categories = [categories]
|
|
event_row.set_categories(','.join(
|
|
str(category) for group in categories for category in group.cats))
|
|
event_row.set_location(str(component.get('LOCATION', '')))
|
|
event_row.set_description(clean_description(str(component.get('DESCRIPTION', ''))))
|
|
event_rows.append(event_row)
|
|
|
|
sorted_rows = sorted(event_rows, key=lambda row: (row.start_date, row.start_time))
|
|
calendars_to_write.append({'csv_name' : calendar_find['csv_name'],
|
|
'calendar' : calendar_find,
|
|
'line' : sorted_rows})
|
|
|
|
return calendars_to_write
|
|
|
|
def clean_description(value):
|
|
if value.startswith("text/html,"):
|
|
value = value[len("text/html,"):]
|
|
value = unquote(value)
|
|
value = re.sub(r'<[^>]+>', '', value)
|
|
return value.strip().strip('"')
|
|
|
|
def write_csv(filename, rows):
|
|
"""Write CSV file"""
|
|
with open(filename, 'w') as file:
|
|
writer = csv.writer(file, delimiter=';')
|
|
csv_title= ['Titre', 'Date debut', 'Heure debut', 'Date fin', 'Heure fin', 'Categories', 'Emplacement', 'Description']
|
|
writer.writerow(csv_title)
|
|
for row in rows:
|
|
writer.writerow(row.get_csv_row())
|
|
|
|
|
|
def main():
|
|
# Load config
|
|
config = load_json_config("./config.json")
|
|
|
|
# Init connection with server
|
|
calendars_server = init_connection(config)
|
|
|
|
# Link calendars in config with calendar from server
|
|
calendars_find = link_calendars(config, calendars_server)
|
|
|
|
# Get all events
|
|
calendars_to_write = get_all_events(config, calendars_find)
|
|
|
|
# Write CSV
|
|
for index, calendar_to_write in enumerate(calendars_to_write):
|
|
try:
|
|
write_csv(calendar_to_write['csv_name'], calendar_to_write['line'])
|
|
except Exception as error:
|
|
# The current CSV may be incomplete; later calendars were not written.
|
|
for pending in calendars_to_write[index:]:
|
|
for row in pending['line']:
|
|
log_skipped_event(pending['calendar'], row.event_context,
|
|
'export CSV interrompu, écriture non garantie ({})'.format(error))
|
|
raise
|
|
|
|
|
|
if __name__ == "__main__":
|
|
with log_output():
|
|
main()
|