backend.workers.cleanup_tempfiles

  1"""
  2Delete old datasets
  3"""
  4import shutil
  5import re
  6import json
  7from datetime import datetime
  8from pathlib import Path
  9
 10from backend.lib.worker import BasicWorker
 11from common.lib.dataset import DataSet
 12from common.lib.exceptions import WorkerInterruptedException, DataSetException
 13
 14
 15class TempFileCleaner(BasicWorker):
 16    """
 17    Clean up discarded temporary files
 18
 19    If 4CAT crashes while processing something, it may result in staging
 20    folders that are never cleaned up. This worker checks for finished
 21    datasets with staging area folders and cleans them up.
 22
 23    Also cleans up orphaned result files for datasets that no longer exist.
 24    """
 25    type = "clean-temp-files"
 26    max_workers = 1
 27
 28    # Use tracking file to delay deletion of files that may still be in use
 29    days_to_keep = 7
 30
 31    @classmethod
 32    def ensure_job(cls, config=None):
 33        """
 34        Ensure that the temp file cleaner is always running
 35
 36        This is used to ensure that the temp file cleaner is always running, and
 37        if it is not, it will be started by the WorkerManager.
 38
 39        :return:  Job parameters for the worker
 40        """
 41        return {"remote_id": "localhost", "interval": 10800}
 42
 43    def work(self):
 44        """
 45        Go through result files, and for each one check if it should still
 46        exist
 47        :return:
 48        """
 49        # Load tracking file
 50        tracking_file = self.config.get('PATH_DATA').joinpath(".temp_file_cleaner")
 51        if not tracking_file.exists():
 52            tracked_files = {}
 53        else:
 54            tracked_files = json.loads(tracking_file.read_text())
 55
 56        # Get 4CAT paths to avoid if they are mapped inside PATH_DATA
 57        fourcat_paths = [self.config.get(p) for p in self.config.get_all_setting_names() if p.startswith('PATH_')]
 58
 59        result_files = Path(self.config.get('PATH_DATA')).glob("*")
 60        for file in result_files:
 61            if file.stem.startswith("."):
 62                # skip hidden files
 63                continue
 64
 65            if self.interrupted:
 66                tracking_file.write_text(json.dumps(tracked_files))
 67                raise WorkerInterruptedException("Interrupted while cleaning up orphaned result files")
 68            
 69            # Check if the file is inside any of the 4CAT paths
 70            if any(file == fourcat_path or file.is_relative_to(fourcat_path) for fourcat_path in fourcat_paths):
 71                continue
 72
 73            # the key of the dataset files belong to can be extracted from the
 74            # file name in a predictable way.
 75            possible_keys = re.findall(r"[abcdef0-9]{32}", file.stem)
 76            if not possible_keys:
 77                self.log.warning("File %s does not seem to be a result file - clean up manually" % file)
 78                continue
 79
 80            # if for whatever reason there are multiple hashes in the filename,
 81            # the key would always be the last one
 82            key = possible_keys.pop()
 83
 84            try:
 85                dataset = DataSet(key=key, db=self.db, modules=self.modules)
 86            except DataSetException:
 87                # the dataset has been deleted since, but the result file still
 88                # exists - should be safe to clean up
 89                if file.name not in tracked_files:
 90                    self.log.info(f"No matching dataset with key {key} for file {file}; marking for deletion")
 91                    tracked_files[file.name] = datetime.now().timestamp() + (self.days_to_keep * 86400)
 92                elif tracked_files[file.name] < datetime.now().timestamp():
 93                    self.log.info(f"File {file} marked for deletion since {datetime.fromtimestamp(tracked_files[file.name]).strftime('%Y-%m-%d %H:%M:%S')}, deleting file")
 94                    if file.is_dir():
 95                        try:
 96                            shutil.rmtree(file)
 97                        except PermissionError:
 98                            self.log.info(f"Folder {file} does not belong to a dataset but cannot be deleted (no "
 99                                          f"permissions), skipping")
100
101                    else:
102                        try:
103                            file.unlink()
104                        except FileNotFoundError:
105                            # the file has been deleted since
106                            pass
107
108                    # Remove from tracking
109                    del tracked_files[file.name]
110
111                continue
112
113            if file.is_dir() and "-staging" in file.stem and dataset.is_finished():
114                # staging area exists but dataset is marked as finished
115                # if the dataset is finished, the staging area should have been
116                # compressed into a zip file, or deleted, so this is also safe
117                # to clean up
118                if file.name not in tracked_files:
119                    self.log.info("Dataset %s is finished, but staging area remains at %s, marking for deletion" % (dataset.key, str(file)))
120                    tracked_files[file.name] = datetime.now().timestamp() + (self.days_to_keep * 86400)
121                elif tracked_files[file.name] < datetime.now().timestamp():
122                    self.log.info("Dataset %s is finished, but staging area remains at %s, deleting folder" % (dataset.key, str(file)))
123                    shutil.rmtree(file)
124
125        # Update tracked files
126        tracking_file.write_text(json.dumps(tracked_files))
127
128        self.job.finish()