backend.workers.cleanup_tempfiles

  1"""
  2Delete old datasets
  3"""
  4import shutil
  5import re
  6import json
  7from datetime import datetime
  8from pathlib import Path
  9
 10from backend.lib.worker import BasicWorker
 11from common.lib.dataset import DataSet
 12from common.lib.exceptions import WorkerInterruptedException, DataSetException
 13
 14
 15class TempFileCleaner(BasicWorker):
 16    """
 17    Clean up discarded temporary files
 18
 19    If 4CAT crashes while processing something, it may result in staging
 20    folders that are never cleaned up. This worker checks for finished
 21    datasets with staging area folders and cleans them up.
 22
 23    Also cleans up orphaned result files for datasets that no longer exist.
 24    """
 25    type = "clean-temp-files"
 26    max_workers = 1
 27
 28    ensure_job = {"remote_id": "localhost", "interval": 10800}
 29
 30    # Use tracking file to delay deletion of files that may still be in use
 31    days_to_keep = 7
 32
 33    def work(self):
 34        """
 35        Go through result files, and for each one check if it should still
 36        exist
 37        :return:
 38        """
 39        # Load tracking file
 40        tracking_file = self.config.get('PATH_DATA').joinpath(".temp_file_cleaner")
 41        if not tracking_file.exists():
 42            tracked_files = {}
 43        else:
 44            tracked_files = json.loads(tracking_file.read_text())
 45
 46        result_files = Path(self.config.get('PATH_DATA')).glob("*")
 47        for file in result_files:
 48            if file.stem.startswith("."):
 49                # skip hidden files
 50                continue
 51
 52            if self.interrupted:
 53                tracking_file.write_text(json.dumps(tracked_files))
 54                raise WorkerInterruptedException("Interrupted while cleaning up orphaned result files")
 55
 56            # the key of the dataset files belong to can be extracted from the
 57            # file name in a predictable way.
 58            possible_keys = re.findall(r"[abcdef0-9]{32}", file.stem)
 59            if not possible_keys:
 60                self.log.warning("File %s does not seem to be a result file - clean up manually" % file)
 61                continue
 62
 63            # if for whatever reason there are multiple hashes in the filename,
 64            # the key would always be the last one
 65            key = possible_keys.pop()
 66
 67            try:
 68                dataset = DataSet(key=key, db=self.db)
 69            except DataSetException:
 70                # the dataset has been deleted since, but the result file still
 71                # exists - should be safe to clean up
 72                if file.name not in tracked_files:
 73                    self.log.info(f"No matching dataset with key {key} for file {file}; marking for deletion")
 74                    tracked_files[file.name] = datetime.now().timestamp() + (self.days_to_keep * 86400)
 75                elif tracked_files[file.name] < datetime.now().timestamp():
 76                    self.log.info(f"File {file} marked for deletion since {datetime.fromtimestamp(tracked_files[file.name]).strftime('%Y-%m-%d %H:%M:%S')}, deleting file")
 77                    if file.is_dir():
 78                        try:
 79                            shutil.rmtree(file)
 80                        except PermissionError:
 81                            self.log.info(f"Folder {file} does not belong to a dataset but cannot be deleted (no "
 82                                          f"permissions), skipping")
 83
 84                    else:
 85                        try:
 86                            file.unlink()
 87                        except FileNotFoundError:
 88                            # the file has been deleted since
 89                            pass
 90
 91                    # Remove from tracking
 92                    del tracked_files[file.name]
 93
 94                continue
 95
 96            if file.is_dir() and "-staging" in file.stem and dataset.is_finished():
 97                # staging area exists but dataset is marked as finished
 98                # if the dataset is finished, the staging area should have been
 99                # compressed into a zip file, or deleted, so this is also safe
100                # to clean up
101                if file.name not in tracked_files:
102                    self.log.info("Dataset %s is finished, but staging area remains at %s, marking for deletion" % (dataset.key, str(file)))
103                    tracked_files[file.name] = datetime.now().timestamp() + (self.days_to_keep * 86400)
104                elif tracked_files[file.name] < datetime.now().timestamp():
105                    self.log.info("Dataset %s is finished, but staging area remains at %s, deleting folder" % (dataset.key, str(file)))
106                    shutil.rmtree(file)
107
108        # Update tracked files
109        tracking_file.write_text(json.dumps(tracked_files))
110
111        self.job.finish()