backend.workers.restart_4cat

  1"""
  2Restart 4CAT and optionally upgrade it to the latest release
  3"""
  4import subprocess
  5import requests
  6import hashlib
  7import oslex
  8import json
  9import time
 10import uuid
 11import sys
 12
 13from pathlib import Path
 14
 15from backend.lib.worker import BasicWorker
 16from common.lib.exceptions import WorkerInterruptedException
 17
 18
 19class FourcatRestarterAndUpgrader(BasicWorker):
 20    """
 21    Restart 4CAT and optionally upgrade it to the latest release
 22
 23    Why implement this as a worker? Trying to have 4CAT restart itself leads
 24    to an interesting conundrum: it will not be able to report the outcome of
 25    the restart, because whatever bit of code is keeping track of that will be
 26    interrupted by restarting 4CAT.
 27
 28    Using a worker has the benefit of it restarting after 4CAT restarts, so it
 29    can then figure out that 4CAT was just restarted and report the outcome. It
 30    then uses a log file to keep track of the results. The log file can then be
 31    used by other parts of 4CAT to see if the restart was successful.
 32
 33    It does lead to another conundrum - what if due to some error, 4CAT never
 34    restarts? Then this worker will not be run again to report its own failure.
 35    There seem to be no clean ways around this, so anything watching the
 36    outcome of the worker probably needs to implement some timeout after which
 37    it is assumed that the restart/upgrade process failed catastrophically.
 38    """
 39    type = "restart-4cat"
 40    max_workers = 1
 41
 42    def work(self):
 43        """
 44        Restart 4CAT and optionally upgrade it to the latest release
 45        """
 46        # figure out if we're starting the restart or checking the result
 47        # after 4cat has been restarted
 48        is_resuming = self.job.data["attempts"] > 0
 49
 50        # prevent multiple restarts running at the same time which could blow
 51        # up really fast
 52        lock_file = Path(self.config.get("PATH_ROOT")).joinpath("config/restart.lock")
 53
 54        # this file has the log of the restart worker itself and is checked by
 55        # the frontend to see how far we are
 56        log_file_restart = Path(self.config.get("PATH_ROOT")).joinpath(self.config.get("PATH_LOGS")).joinpath("restart.log")
 57        log_stream_restart = log_file_restart.open("a")
 58
 59        if not is_resuming:
 60            log_stream_restart.write("Initiating 4CAT restart worker\n")
 61            self.log.info("New restart initiated.")
 62
 63            # this lock file will ensure that people don't start two
 64            # simultaneous upgrades or something
 65            with lock_file.open("w") as outfile:
 66                hasher = hashlib.blake2b()
 67                hasher.update(str(uuid.uuid4()).encode("utf-8"))
 68                outfile.write(hasher.hexdigest())
 69
 70            # trigger a restart and/or upgrade
 71            # returns a JSON with a 'status' key and a message, the message
 72            # being the process output
 73
 74            if self.job.data["remote_id"].startswith("upgrade"):
 75                command = sys.executable + " helper-scripts/migrate.py --repository %s --yes --restart" % \
 76                          (oslex.quote(self.config.get("4cat.github_url")))
 77                if self.job.details and self.job.details.get("branch"):
 78                    # migrate to code in specific branch
 79                    command += f" --branch {oslex.quote(self.job.details['branch'])}"
 80                else:
 81                    # migrate to latest release
 82                    command += " --release"
 83
 84            else:
 85                command = sys.executable + " 4cat-daemon.py --no-version-check force-restart"
 86
 87            try:
 88                # flush any writes before the other process starts writing to
 89                # the stream
 90                self.log.info(f"Running command {command}")
 91                log_stream_restart.flush()
 92
 93                # the tricky part is that this command will interrupt the
 94                # daemon, i.e. this worker!
 95                # so we'll never get to actually send a response, if all goes
 96                # well. but the file descriptor that stdout is piped to remains
 97                # open, somehow, so we can use that to keep track of the output
 98                # stdin needs to be /dev/null here because else when 4CAT
 99                # restarts and we re-attempt to make a daemon, it will fail
100                # when trying to close the stdin file descriptor of the
101                # subprocess (man, that was a fun bug to hunt down)
102                process = subprocess.Popen(oslex.split(command), cwd=str(self.config.get("PATH_ROOT")),
103                                           stdout=log_stream_restart, stderr=log_stream_restart,
104                                           stdin=subprocess.DEVNULL)
105
106                while not self.interrupted:
107                    # basically wait for either the process to quit or 4CAT to
108                    # be restarted (hopefully the latter)
109                    try:
110                        # now see if the process is finished - if not a
111                        # TimeoutExpired will be raised
112                        process.wait(1)
113                        break
114                    except subprocess.TimeoutExpired:
115                        pass
116
117                if process.returncode is not None:
118                    # if we reach this, 4CAT was never restarted, and so the job failed
119                    log_stream_restart.write(
120                        f"\nUnexpected outcome of restart call ({process.returncode})\n")
121
122                    raise RuntimeError()
123                else:
124                    # interrupted before the process could finish (as it should)
125                    self.log.info("Restart triggered. Restarting 4CAT.\n")
126                    raise WorkerInterruptedException()
127
128            except (RuntimeError, subprocess.CalledProcessError) as e:
129                log_stream_restart.write(str(e))
130                log_stream_restart.write(
131                    "[Worker] Error while restarting 4CAT. The script returned a non-standard error code "
132                    "(see above). You may need to restart 4CAT manually.\n")
133                self.log.error(f"Error restarting 4CAT. See {log_stream_restart.name} for details.")
134                lock_file.unlink()
135                self.job.finish()
136
137            finally:
138                log_stream_restart.close()
139
140        else:
141            # 4CAT back-end was restarted - now check the results and make the
142            # front-end restart or upgrade too
143            self.log.info("Restart worker resumed after restarting 4CAT, restart successful.")
144            log_stream_restart.write("4CAT restarted.\n")
145            with Path(self.config.get("PATH_ROOT")).joinpath("config/.current-version").open() as infile:
146                log_stream_restart.write(f"4CAT is now running version {infile.readline().strip()}.\n")
147
148            # we're gonna use some specific Flask routes to trigger this, i.e.
149            # we're interacting with the front-end through HTTP
150            api_host = "https://" if self.config.get("flask.https") else "http://"
151            if self.config.get("USING_DOCKER"):
152                import os
153                docker_exposed_port = os.environ['PUBLIC_PORT']
154                api_host += f"host.docker.internal{':' + docker_exposed_port if docker_exposed_port != '80' else ''}"
155            else:
156                api_host += self.config.get("flask.server_name")
157
158            if self.job.data["remote_id"].startswith("upgrade") and self.config.get("USING_DOCKER"):
159                # when using Docker, the front-end needs to update separately
160                log_stream_restart.write("Telling front-end Docker container to upgrade...\n")
161                log_stream_restart.close()  # close, because front-end will be writing to it
162                upgrade_ok = False
163                upgrade_timeout = False
164                try:
165                    upgrade_url = api_host + "/admin/trigger-frontend-upgrade/"
166                    with lock_file.open() as infile:
167                        frontend_upgrade = requests.post(upgrade_url, data={"token": infile.read()}, timeout=(10 * 60))
168                    upgrade_ok = frontend_upgrade.json()["status"] == "OK"
169                except requests.RequestException:
170                    pass
171                except TimeoutError:
172                    upgrade_timeout = True
173
174                log_stream_restart = log_file_restart.open("a")
175                if not upgrade_ok:
176                    if upgrade_timeout:
177                        log_stream_restart.write("Upgrade timed out.")
178                    log_stream_restart.write("Error upgrading front-end container. You may need to upgrade and restart"
179                                             "containers manually.\n")
180                    lock_file.unlink()
181                    return self.job.finish()
182
183            # restart front-end
184            log_stream_restart.write("Asking front-end to restart itself...\n")
185            log_stream_restart.flush()
186            try:
187                restart_url = api_host + "/admin/trigger-frontend-restart/"
188                with lock_file.open() as infile:
189                    response = requests.post(restart_url, data={"token": infile.read()}, timeout=5).json()
190
191                if response.get("message"):
192                    log_stream_restart.write(response.get("message") + "\n")
193            except (json.JSONDecodeError, requests.RequestException):
194                # this may happen because the server restarts and interrupts
195                # the request
196                pass
197
198            # wait for front-end to come online after a restart
199            time.sleep(3)  # give some time for the restart to trigger
200            start_time = time.time()
201            frontend_ok = False
202            while time.time() < start_time + 60:
203                try:
204                    frontend = requests.get(api_host + "/", timeout=5)
205                    if frontend.status_code > 401:
206                        time.sleep(2)
207                        continue
208                    frontend_ok = True
209                    break
210                except requests.RequestException:
211                    time.sleep(1)
212                    continue
213
214            # too bad
215            if not frontend_ok:
216                log_stream_restart.write("Timed out waiting for front-end to restart. You may need to restart it "
217                                         "manually.\n")
218                self.log.error("Front-end did not come back online after restart")
219            else:
220                log_stream_restart.write("Front-end is available. Restart complete.")
221                self.log.info("Front-end is available. Restart complete.")
222
223            log_stream_restart.close()
224            lock_file.unlink()
225
226            self.job.finish()