Export caldav with logs

crealead-caldav.py

@@ -17,11 +17,144 @@ import sys
 import caldav
 import json
 import csv
-import calendar
-from datetime import date, datetime, timezone
+from datetime import date, datetime, time, timezone
+from icalendar import Calendar
 import pytz
 from urllib.parse import unquote
 import re
+import traceback
+from contextlib import contextmanager, redirect_stdout, redirect_stderr
+from pathlib import Path
+
+
+def utc_timestamp():
+  return datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+
+
+class TimestampedLog:
+  """Prefix each log line, including lines written in several fragments."""
+
+  def __init__(self, log_file):
+    self.log_file = log_file
+    self.line_start = True
+
+  def write(self, value):
+    # Keep an event warning separate from an unfinished console message.
+    if value.startswith('[warning][event] - ') and not self.line_start:
+      self.log_file.write('\n')
+      self.line_start = True
+    for fragment in value.splitlines(keepends=True):
+      if self.line_start:
+        self.log_file.write('[{}]: '.format(utc_timestamp()))
+        if not fragment.startswith('[warning][event] - '):
+          self.log_file.write('[info] - ')
+      self.log_file.write(fragment)
+      self.line_start = fragment.endswith(('\n', '\r'))
+    return len(value)
+
+  def flush(self):
+    self.log_file.flush()
+
+
+class TeeOutput:
+  """Write to the original console stream and to the shared log file."""
+
+  def __init__(self, stream, log_file):
+    self.stream = stream
+    self.log_file = log_file
+
+  def write(self, value):
+    written = self.stream.write(value)
+    self.log_file.write(value)
+    self.log_file.flush()
+    return written
+
+  def flush(self):
+    self.stream.flush()
+    self.log_file.flush()
+
+  def __getattr__(self, name):
+    return getattr(self.stream, name)
+
+
+@contextmanager
+def log_output():
+  """Append console output to logs/<script name>.log next to the script."""
+  script_path = Path(__file__).resolve()
+  log_path = script_path.parent / 'logs' / script_path.with_suffix('.log').name
+  log_path.parent.mkdir(parents=True, exist_ok=True)
+  with log_path.open('a', encoding='utf-8') as log_file:
+    log_writer = TimestampedLog(log_file)
+    with redirect_stdout(TeeOutput(sys.stdout, log_writer)), \
+         redirect_stderr(TeeOutput(sys.stderr, log_writer)):
+      try:
+        yield
+      except BaseException as error:
+        # The interpreter prints the traceback after the streams are restored.
+        # Save a copy here while the log is still open, without duplicating it
+        # in the console. SystemExit normally has no traceback.
+        if not log_writer.line_start:
+          log_writer.write('\n')
+        if isinstance(error, SystemExit):
+          if error.code is not None and not isinstance(error.code, int):
+            log_writer.write(str(error.code) + '\n')
+        else:
+          traceback.print_exc(file=log_writer)
+        raise
+      finally:
+        if not log_writer.line_start:
+          log_writer.write('\n')
+
+
+def recover_event_contexts(data):
+  """Recover identifying fields even when the iCalendar parser rejects data."""
+  if isinstance(data, bytes):
+    data = data.decode('utf-8', errors='replace')
+  unfolded = re.sub(r'\r?\n[ \t]', '', str(data))
+  blocks = re.split(r'(?im)^BEGIN:VEVENT\s*\r?$', unfolded)
+  contexts = []
+  for block in blocks[1:] or [unfolded]:
+    context = {}
+    depth = 0
+    for line in block.splitlines():
+      name, separator, value = line.partition(':')
+      name = name.split(';', 1)[0].upper()
+      if name == 'BEGIN':
+        depth += 1
+      elif name == 'END':
+        if depth == 0:
+          break
+        depth -= 1
+      elif depth == 0 and separator and name in ('SUMMARY', 'DTSTART', 'DTEND', 'UID'):
+        context.setdefault(name, value)
+    contexts.append(context)
+  return contexts
+
+
+def log_skipped_event(calendar_find, component, reason, fallback=None):
+  """Every rejected event must have an identifiable warning in the log."""
+  fallback = fallback or {}
+
+  def field(name):
+    value = component.get(name)
+    if value is None:
+      return str(fallback.get(name, 'indisponible'))
+    decoded = getattr(value, 'dt', None)
+    if isinstance(decoded, datetime):
+      return local_datetime(decoded).astimezone(timezone.utc).isoformat().replace('+00:00', 'Z')
+    if isinstance(decoded, date):
+      return decoded.isoformat()
+    return str(value)
+
+  calendar_name = getattr(calendar_find.get('calendar_server'), 'name', None)
+  calendar_name = calendar_name or calendar_find.get('calendar_name', 'indisponible')
+  # repr keeps newlines in event metadata from breaking the warning format.
+  print('[warning][event] - Calendrier={!r} ; CSV={!r} ; titre={!r} ; '
+        'début={!r} ; fin={!r} ; UID={!r} ; événement non exporté : {}'.format(
+    str(calendar_name), calendar_find['csv_name'], field('SUMMARY'),
+    field('DTSTART'), field('DTEND'), field('UID'),
+    str(reason).replace('\r', '\\r').replace('\n', '\\n')), file=sys.stderr)
+
 
 class EventRow():
 
@@ -34,6 +167,7 @@ class EventRow():
     self.categories = ''
     self.location = ''
     self.description = ''
+    self.event_context = {}
 
   def set_title(self, title):
     self.title = title
@@ -111,21 +245,18 @@ def link_calendars(config, calendars_server):
   return calendars_find
 
 
-def split_data(data):
-  """Return string splitted by ';' separator"""
-  return data.split(':')
-
-
-def datetime_format(date_str):
-  """Return date formated"""
-  try: # Nextcloud Format
-   date_formated = datetime.strptime(date_str, '%Y%m%dT%H%M%SZ')
-  except: # Zimbra format
-   date_formated = datetime.strptime(date_str, '%Y%m%d')
+def local_datetime(value):
+  """Convert an iCalendar date or datetime to Europe/Paris."""
   paris_tz = pytz.timezone('Europe/Paris')
-  date_formated = date_formated.replace(tzinfo=timezone.utc).astimezone(paris_tz)
-  #date_formated = date_formated.astimezone(paris_tz)
-  return date_formated
+  if isinstance(value, datetime):
+    result = value
+  elif isinstance(value, date):
+    result = datetime.combine(value, time.min)
+  else:
+    raise ValueError("date absente ou invalide")
+  if result.tzinfo is None:
+    return paris_tz.localize(result)
+  return result.astimezone(paris_tz)
 
 
 def get_all_events(config, calendars_find):
@@ -142,40 +273,56 @@ def get_all_events(config, calendars_find):
     event_rows = []
   
     for event in events_fetched:
-      event_fields = list(map(split_data, event.data.splitlines()))
+      contexts = recover_event_contexts(event.data)
+      # Parse parameters (TZID, VALUE=DATE), folded lines and escaped text.
+      try:
+        event_calendar = Calendar.from_ical(event.data)
+      except (ValueError, TypeError) as error:
+        for context in contexts:
+          log_skipped_event(calendar_find, {},
+            'document iCalendar illisible ({})'.format(error), context)
+        continue
 
-      # Fix problem #2023022110004936
-      has = False 
-      for sub in event_fields:
-          if 'SUMMARY' in sub:
-              has = True
-              break
-      if not has:
-          continue 
+      components = event_calendar.walk('VEVENT')
+      if not components:
+        log_skipped_event(calendar_find, {}, 'aucun composant VEVENT', contexts[0])
 
-      event_row = EventRow()
+      for index, component in enumerate(components):
+        context = contexts[index] if index < len(contexts) else {}
+        # Fix problem #2023022110004936: ignore events without a summary.
+        if 'SUMMARY' not in component:
+          log_skipped_event(calendar_find, component, 'propriété SUMMARY absente', context)
+          continue
 
-      for field in ['SUMMARY', 'DTSTART', 'DTEND', 'CATEGORIES', 'LOCATION', 'DESCRIPTION']:
-        for event_field in event_fields:
-          if field == event_field[0]:
-            if field == "SUMMARY":
-              event_row.set_title(event_field[1])
-            elif field == "DTSTART":
-              date = datetime_format(event_field[1])
-              event_row.set_start(date)
-            elif field == "DTEND":
-              date = datetime_format(event_field[1])
-              event_row.set_end(date)
-            elif field == "CATEGORIES":
-              event_row.set_categories(event_field[1])
-            elif field == "LOCATION":
-              event_row.set_location(event_field[1])
-            elif field == "DESCRIPTION":
-              event_row.set_description(clean_description(event_field[1]))
-      event_rows.append(event_row)
+        try:
+          start = component.decoded('DTSTART')
+          end = component.decoded('DTEND', None)
+          if end is None and 'DURATION' in component:
+            end = start + component.decoded('DURATION')
+          start = local_datetime(start)
+          end = local_datetime(end)
+        except (KeyError, ValueError, TypeError) as error:
+          log_skipped_event(calendar_find, component,
+            'dates de début/fin absentes ou invalides ({})'.format(error), context)
+          continue
 
-    sorted_rows = sorted(event_rows, key=lambda l: datetime.strptime(str(l.start_date)+'T'+str(l.start_time), "%Y-%m-%dT%H:%M:%S"))
+        event_row = EventRow()
+        event_row.event_context = component
+        event_row.set_title(str(component['SUMMARY']))
+        event_row.set_start(start)
+        event_row.set_end(end)
+        categories = component.get('CATEGORIES', [])
+        if not isinstance(categories, list):
+          categories = [categories]
+        event_row.set_categories(','.join(
+          str(category) for group in categories for category in group.cats))
+        event_row.set_location(str(component.get('LOCATION', '')))
+        event_row.set_description(clean_description(str(component.get('DESCRIPTION', ''))))
+        event_rows.append(event_row)
+
+    sorted_rows = sorted(event_rows, key=lambda row: (row.start_date, row.start_time))
     calendars_to_write.append({'csv_name' : calendar_find['csv_name'],
+                               'calendar' : calendar_find,
                                'line' : sorted_rows})
 
   return calendars_to_write
@@ -197,8 +344,7 @@ def write_csv(filename, rows):
       writer.writerow(row.get_csv_row())
 
 
-if __name__ == "__main__":
-
+def main():
   # Load config
   config = load_json_config("./config.json")
 
@@ -212,5 +358,18 @@ if __name__ == "__main__":
   calendars_to_write = get_all_events(config, calendars_find)
 
   # Write CSV
-  for calendar_to_write in calendars_to_write:
-    write_csv(calendar_to_write['csv_name'], calendar_to_write['line'])
+  for index, calendar_to_write in enumerate(calendars_to_write):
+    try:
+      write_csv(calendar_to_write['csv_name'], calendar_to_write['line'])
+    except Exception as error:
+      # The current CSV may be incomplete; later calendars were not written.
+      for pending in calendars_to_write[index:]:
+        for row in pending['line']:
+          log_skipped_event(pending['calendar'], row.event_context,
+            'export CSV interrompu, écriture non garantie ({})'.format(error))
+      raise
+
+
+if __name__ == "__main__":
+  with log_output():
+    main()