common.lib.helpers

   1"""
   2Miscellaneous helper functions for the 4CAT backend
   3"""
   4import subprocess
   5import imagehash
   6import hashlib
   7import requests
   8import datetime
   9import smtplib
  10import fnmatch
  11import socket
  12import shlex
  13import copy
  14import time
  15import json
  16import math
  17import csv
  18import ssl
  19import re
  20import os
  21import io
  22
  23from pathlib import Path
  24from collections.abc import MutableMapping
  25from html.parser import HTMLParser
  26from urllib.parse import urlparse, urlunparse
  27from calendar import monthrange
  28from packaging import version
  29from PIL import Image
  30
  31from common.lib.user_input import UserInput
  32from common.config_manager import config
  33
  34
  35def init_datasource(database, logger, queue, name):
  36    """
  37    Initialize data source
  38
  39    Queues jobs to scrape the boards that were configured to be scraped in the
  40    4CAT configuration file. If none were configured, nothing happens.
  41
  42    :param Database database:  Database connection instance
  43    :param Logger logger:  Log handler
  44    :param JobQueue queue:  Job Queue instance
  45    :param string name:  ID of datasource that is being initialised
  46    """
  47    pass
  48
  49def strip_tags(html, convert_newlines=True):
  50    """
  51    Strip HTML from a string
  52
  53    :param html: HTML to strip
  54    :param convert_newlines: Convert <br> and </p> tags to \n before stripping
  55    :return: Stripped HTML
  56    """
  57    if not html:
  58        return ""
  59
  60    deduplicate_newlines = re.compile(r"\n+")
  61
  62    if convert_newlines:
  63        html = html.replace("<br>", "\n").replace("</p>", "</p>\n")
  64        html = deduplicate_newlines.sub("\n", html)
  65
  66    class HTMLStripper(HTMLParser):
  67        def __init__(self):
  68            super().__init__()
  69            self.reset()
  70            self.strict = False
  71            self.convert_charrefs = True
  72            self.fed = []
  73
  74        def handle_data(self, data):
  75            self.fed.append(data)
  76
  77        def get_data(self):
  78            return "".join(self.fed)
  79
  80    stripper = HTMLStripper()
  81    stripper.feed(html)
  82    return stripper.get_data()
  83
  84
  85def sniff_encoding(file):
  86    """
  87    Determine encoding from raw file bytes
  88
  89    Currently only distinguishes UTF-8 and UTF-8 with BOM
  90
  91    :param file:
  92    :return:
  93    """
  94    if type(file) == bytearray:
  95        maybe_bom = file[:3]
  96    elif hasattr(file, "getbuffer"):
  97        buffer = file.getbuffer()
  98        maybe_bom = buffer[:3].tobytes()
  99    elif hasattr(file, "peek"):
 100        buffer = file.peek(32)
 101        maybe_bom = buffer[:3]
 102    else:
 103        maybe_bom = False
 104
 105    return "utf-8-sig" if maybe_bom == b"\xef\xbb\xbf" else "utf-8"
 106
 107def sniff_csv_dialect(csv_input):
 108    """
 109    Determine CSV dialect for an input stream
 110
 111    :param csv_input:  Input stream
 112    :return tuple:  Tuple: Dialect object and a boolean representing whether
 113    the CSV file seems to have a header
 114    """
 115    encoding = sniff_encoding(csv_input)
 116    if type(csv_input) is io.TextIOWrapper:
 117        wrapped_input = csv_input
 118    else:
 119        wrapped_input = io.TextIOWrapper(csv_input, encoding=encoding)
 120    wrapped_input.seek(0)
 121    sample = wrapped_input.read(1024 * 1024)
 122    wrapped_input.seek(0)
 123    has_header = csv.Sniffer().has_header(sample)
 124    dialect = csv.Sniffer().sniff(sample, delimiters=(",", ";", "\t"))
 125
 126    return dialect, has_header
 127
 128
 129def get_git_branch():
 130    """
 131    Get current git branch
 132
 133    If the 4CAT root folder is a git repository, this function will return the
 134    name of the currently checked-out branch. If the folder is not a git
 135    repository or git is not installed an empty string is returned.
 136    """
 137    try:
 138        root_dir = str(config.get('PATH_ROOT').resolve())
 139        branch = subprocess.run(shlex.split(f"git -C {shlex.quote(root_dir)} branch --show-current"), stdout=subprocess.PIPE)
 140        if branch.returncode != 0:
 141            raise ValueError()
 142        branch_name = branch.stdout.decode("utf-8").strip()
 143        if not branch_name:
 144            # Check for detached HEAD state
 145            # Most likely occuring because of checking out release tags (which are not branches) or commits
 146            head_status = subprocess.run(shlex.split(f"git -C {shlex.quote(root_dir)} status"), stdout=subprocess.PIPE)
 147            if head_status.returncode == 0:
 148                for line in head_status.stdout.decode("utf-8").split("\n"):
 149                    if "HEAD detached at" in line:
 150                        branch_name = line.split("/")[-1] if "/" in line else line.split(" ")[-1]
 151                        return branch_name
 152    except (subprocess.SubprocessError, ValueError, FileNotFoundError):
 153        return ""
 154
 155
 156def get_software_commit(worker=None):
 157    """
 158    Get current 4CAT git commit hash
 159
 160    Use `get_software_version()` instead if you need the release version
 161    number rather than the precise commit hash.
 162
 163    If no version file is available, run `git show` to test if there is a git
 164    repository in the 4CAT root folder, and if so, what commit is currently
 165    checked out in it.
 166
 167    For extensions, get the repository information for that extension, or if
 168    the extension is not a git repository, return empty data.
 169
 170    :param BasicWorker processor:  Worker to get commit for. If not given, get
 171    version information for the main 4CAT installation.
 172
 173    :return tuple:  4CAT git commit hash, repository name
 174    """
 175    # try git command line within the 4CAT root folder
 176    # if it is a checked-out git repository, it will tell us the hash of
 177    # the currently checked-out commit
 178
 179    # path has no Path.relative()...
 180    relative_filepath = Path(re.sub(r"^[/\\]+", "", worker.filepath)).parent
 181    try:
 182        # if extension, go to the extension file's path
 183        # we will run git here - if it is not its own repository, we have no
 184        # useful version info (since the extension is by definition not in the
 185        # main 4CAT repository) and will return an empty value
 186        if worker and worker.is_extension:
 187            working_dir = str(config.get("PATH_ROOT").joinpath(relative_filepath).resolve())
 188            # check if we are in the extensions' own repo or 4CAT's
 189            git_cmd = f"git -C {shlex.quote(working_dir)} rev-parse --show-toplevel"
 190            repo_level = subprocess.run(shlex.split(git_cmd), stderr=subprocess.PIPE, stdout=subprocess.PIPE)
 191            if Path(repo_level.stdout.decode("utf-8")) == config.get("PATH_ROOT"):
 192                # not its own repository
 193                return ("", "")
 194
 195        else:
 196            working_dir = str(config.get("PATH_ROOT").resolve())
 197
 198        show = subprocess.run(shlex.split(f"git -C {shlex.quote(working_dir)} show"), stderr=subprocess.PIPE, stdout=subprocess.PIPE)
 199        if show.returncode != 0:
 200            raise ValueError()
 201        commit = show.stdout.decode("utf-8").split("\n")[0].split(" ")[1]
 202
 203        # now get the repository the commit belongs to, if we can
 204        origin = subprocess.run(shlex.split(f"git -C {shlex.quote(working_dir)} config --get remote.origin.url"), stderr=subprocess.PIPE, stdout=subprocess.PIPE)
 205        if origin.returncode != 0 or not origin.stdout:
 206            raise ValueError()
 207        repository = origin.stdout.decode("utf-8").strip()
 208        if repository.endswith(".git"):
 209            repository = repository[:-4]
 210
 211    except (subprocess.SubprocessError, IndexError, TypeError, ValueError, FileNotFoundError) as e:
 212        return ("", "")
 213
 214    return (commit, repository)
 215
 216def get_software_version():
 217    """
 218    Get current 4CAT version
 219
 220    This is the actual software version, i.e. not the commit hash (see
 221    `get_software_hash()` for that). The current version is stored in a file
 222    with a canonical location: if the file doesn't exist, an empty string is
 223    returned.
 224
 225    :return str:  Software version, for example `1.37`.
 226    """
 227    current_version_file = config.get("PATH_ROOT").joinpath("config/.current-version")
 228    if not current_version_file.exists():
 229        return ""
 230
 231    with current_version_file.open() as infile:
 232        return infile.readline().strip()
 233
 234def get_github_version(timeout=5):
 235    """
 236    Get latest release tag version from GitHub
 237
 238    Will raise a ValueError if it cannot retrieve information from GitHub.
 239
 240    :param int timeout:  Timeout in seconds for HTTP request
 241
 242    :return tuple:  Version, e.g. `1.26`, and release URL.
 243    """
 244    repo_url = config.get("4cat.github_url")
 245    if not repo_url.endswith("/"):
 246        repo_url += "/"
 247
 248    repo_id = re.sub(r"(\.git)?/?$", "", re.sub(r"^https?://(www\.)?github\.com/", "", repo_url))
 249
 250    api_url = "https://api.github.com/repos/%s/releases/latest" % repo_id
 251    response = requests.get(api_url, timeout=timeout)
 252    response = response.json()
 253    if response.get("message") == "Not Found":
 254        raise ValueError("Invalid GitHub URL or repository name")
 255
 256    latest_tag = response.get("tag_name", "unknown")
 257    if latest_tag.startswith("v"):
 258        latest_tag = re.sub(r"^v", "", latest_tag)
 259
 260    return (latest_tag, response.get("html_url"))
 261
 262def get_ffmpeg_version(ffmpeg_path):
 263    """
 264    Determine ffmpeg version
 265
 266    This can be necessary when using commands that change name between versions.
 267
 268    :param ffmpeg_path: ffmpeg executable path
 269    :return packaging.version:  Comparable ersion
 270    """
 271    command = [ffmpeg_path, "-version"]
 272    ffmpeg_version = subprocess.run(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
 273                                    stderr=subprocess.PIPE)
 274
 275    ffmpeg_version = ffmpeg_version.stdout.decode("utf-8").split("\n")[0].strip().split(" version ")[1]
 276    ffmpeg_version = re.split(r"[^0-9.]", ffmpeg_version)[0]
 277
 278    return version.parse(ffmpeg_version)
 279
 280
 281def find_extensions():
 282    """
 283    Find 4CAT extensions and load their metadata
 284
 285    Looks for subfolders of the extension folder, and loads additional metadata
 286    where available.
 287
 288    :return tuple:  A tuple with two items; the extensions, as an ID -> metadata
 289    dictionary, and a list of (str) errors encountered while loading
 290    """
 291    extension_path = config.get("PATH_ROOT").joinpath("extensions")
 292    errors = []
 293    if not extension_path.exists() or not extension_path.is_dir():
 294        return [], None
 295
 296    # each folder in the extensions folder is an extension
 297    extensions = {
 298        extension.name: {
 299            "name": extension.name,
 300            "version": "",
 301            "url": "",
 302            "git_url": "",
 303            "is_git": False
 304        } for extension in sorted(os.scandir(extension_path), key=lambda x: x.name) if extension.is_dir()
 305    }
 306
 307    # collect metadata for extensions
 308    allowed_metadata_keys = ("name", "version", "url")
 309    for extension in extensions:
 310        extension_folder = extension_path.joinpath(extension)
 311        metadata_file = extension_folder.joinpath("metadata.json")
 312        if metadata_file.exists():
 313            with metadata_file.open() as infile:
 314                try:
 315                    metadata = json.load(infile)
 316                    extensions[extension].update({k: metadata[k] for k in metadata if k in allowed_metadata_keys})
 317                except (TypeError, ValueError) as e:
 318                    errors.append(f"Error reading metadata file for extension '{extension}' ({e})")
 319                    continue
 320
 321        extensions[extension]["is_git"] = extension_folder.joinpath(".git/HEAD").exists()
 322        if extensions[extension]["is_git"]:
 323            # try to get remote URL
 324            try:
 325                extension_root = str(extension_folder.resolve())
 326                origin = subprocess.run(shlex.split(f"git -C {shlex.quote(extension_root)} config --get remote.origin.url"), stderr=subprocess.PIPE,
 327                                        stdout=subprocess.PIPE)
 328                if origin.returncode != 0 or not origin.stdout:
 329                    raise ValueError()
 330                repository = origin.stdout.decode("utf-8").strip()
 331                if repository.endswith(".git") and "github.com" in repository:
 332                    # use repo URL
 333                    repository = repository[:-4]
 334                extensions[extension]["git_url"] = repository
 335            except (subprocess.SubprocessError, IndexError, TypeError, ValueError, FileNotFoundError) as e:
 336                print(e)
 337                pass
 338
 339    return extensions, errors
 340
 341
 342def convert_to_int(value, default=0):
 343    """
 344    Convert a value to an integer, with a fallback
 345
 346    The fallback is used if an Error is thrown during converstion to int.
 347    This is a convenience function, but beats putting try-catches everywhere
 348    we're using user input as an integer.
 349
 350    :param value:  Value to convert
 351    :param int default:  Default value, if conversion not possible
 352    :return int:  Converted value
 353    """
 354    try:
 355        return int(value)
 356    except (ValueError, TypeError):
 357        return default
 358
 359
 360def timify_long(number):
 361    """
 362    Make a number look like an indication of time
 363
 364    :param number:  Number to convert. If the number is larger than the current
 365    UNIX timestamp, decrease by that amount
 366    :return str: A nice, string, for example `1 month, 3 weeks, 4 hours and 2 minutes`
 367    """
 368    number = int(number)
 369
 370    components = []
 371    if number > time.time():
 372        number = time.time() - number
 373
 374    month_length = 30.42 * 86400
 375    months = math.floor(number / month_length)
 376    if months:
 377        components.append("%i month%s" % (months, "s" if months != 1 else ""))
 378        number -= (months * month_length)
 379
 380    week_length = 7 * 86400
 381    weeks = math.floor(number / week_length)
 382    if weeks:
 383        components.append("%i week%s" % (weeks, "s" if weeks != 1 else ""))
 384        number -= (weeks * week_length)
 385
 386    day_length = 86400
 387    days = math.floor(number / day_length)
 388    if days:
 389        components.append("%i day%s" % (days, "s" if days != 1 else ""))
 390        number -= (days * day_length)
 391
 392    hour_length = 3600
 393    hours = math.floor(number / hour_length)
 394    if hours:
 395        components.append("%i hour%s" % (hours, "s" if hours != 1 else ""))
 396        number -= (hours * hour_length)
 397
 398    minute_length = 60
 399    minutes = math.floor(number / minute_length)
 400    if minutes:
 401        components.append("%i minute%s" % (minutes, "s" if minutes != 1 else ""))
 402
 403    if not components:
 404        components.append("less than a minute")
 405
 406    last_str = components.pop()
 407    time_str = ""
 408    if components:
 409        time_str = ", ".join(components)
 410        time_str += " and "
 411
 412    return time_str + last_str
 413
 414def andify(items):
 415    """
 416    Format a list of items for use in text
 417
 418    Returns a comma-separated list, the last item preceded by "and"
 419
 420    :param items:  Iterable list
 421    :return str:  Formatted string
 422    """
 423    if len(items) == 0:
 424        return ""
 425    elif len(items) == 1:
 426        return str(items[1])
 427
 428    result = f" and {items.pop()}"
 429    return ", ".join([str(item) for item in items]) + result
 430
 431
 432def hash_file(image_file, hash_type="file-hash"):
 433    """
 434    Generate an image hash
 435
 436    :param Path image_file:  Image file to hash
 437    :param str hash_type:  Hash type, one of `file-hash`, `colorhash`,
 438    `phash`, `average_hash`, `dhash`
 439    :return str:  Hexadecimal hash value
 440    """
 441    if not image_file.exists():
 442        raise FileNotFoundError()
 443
 444    if hash_type == "file-hash":
 445        hasher = hashlib.sha1()
 446
 447        # Open the file in binary mode
 448        with image_file.open("rb") as infile:
 449            # Read and update hash in chunks to handle large files
 450            while chunk := infile.read(1024):
 451                hasher.update(chunk)
 452
 453        return hasher.hexdigest()
 454
 455    elif hash_type in ("colorhash", "phash", "average_hash", "dhash"):
 456        image = Image.open(image_file)
 457
 458        return str(getattr(imagehash, hash_type)(image))
 459
 460    else:
 461        raise NotImplementedError(f"Unknown hash type '{hash_type}'")
 462
 463def get_yt_compatible_ids(yt_ids):
 464    """
 465    :param yt_ids list, a list of strings
 466    :returns list, a ist of joined strings in pairs of 50
 467
 468    Takes a list of IDs and returns list of joined strings
 469    in pairs of fifty. This should be done for the YouTube API
 470    that requires a comma-separated string and can only return
 471    max fifty results.
 472    """
 473
 474    # If there's only one item, return a single list item
 475    if isinstance(yt_ids, str):
 476        return [yt_ids]
 477
 478    ids = []
 479    last_i = 0
 480    for i, yt_id in enumerate(yt_ids):
 481
 482        # Add a joined string per fifty videos
 483        if i % 50 == 0 and i != 0:
 484            ids_string = ",".join(yt_ids[last_i:i])
 485            ids.append(ids_string)
 486            last_i = i
 487
 488        # If the end of the list is reached, add the last data
 489        elif i == (len(yt_ids) - 1):
 490            ids_string = ",".join(yt_ids[last_i:i])
 491            ids.append(ids_string)
 492
 493    return ids
 494
 495
 496def get_4cat_canvas(path, width, height, header=None, footer="made with 4CAT", fontsize_normal=None,
 497                    fontsize_small=None, fontsize_large=None):
 498    """
 499    Get a standard SVG canvas to draw 4CAT graphs to
 500
 501    Adds a border, footer, header, and some basic text styling
 502
 503    :param path:  The path where the SVG graph will be saved
 504    :param width:  Width of the canvas
 505    :param height:  Height of the canvas
 506    :param header:  Header, if necessary to draw
 507    :param footer:  Footer text, if necessary to draw. Defaults to shameless
 508    4CAT advertisement.
 509    :param fontsize_normal:  Font size of normal text
 510    :param fontsize_small:  Font size of small text (e.g. footer)
 511    :param fontsize_large:  Font size of large text (e.g. header)
 512    :return SVG:  SVG canvas (via svgwrite) that can be drawn to
 513    """
 514    from svgwrite.container import SVG, Hyperlink
 515    from svgwrite.drawing import Drawing
 516    from svgwrite.shapes import Rect
 517    from svgwrite.text import Text
 518
 519    if fontsize_normal is None:
 520        fontsize_normal = width / 75
 521
 522    if fontsize_small is None:
 523        fontsize_small = width / 100
 524
 525    if fontsize_large is None:
 526        fontsize_large = width / 50
 527
 528    # instantiate with border and white background
 529    canvas = Drawing(str(path), size=(width, height), style="font-family:monospace;font-size:%ipx" % fontsize_normal)
 530    canvas.add(Rect(insert=(0, 0), size=(width, height), stroke="#000", stroke_width=2, fill="#FFF"))
 531
 532    # header
 533    if header:
 534        header_shape = SVG(insert=(0, 0), size=("100%", fontsize_large * 2))
 535        header_shape.add(Rect(insert=(0, 0), size=("100%", "100%"), fill="#000"))
 536        header_shape.add(
 537            Text(insert=("50%", "50%"), text=header, dominant_baseline="middle", text_anchor="middle", fill="#FFF",
 538                 style="font-size:%ipx" % fontsize_large))
 539        canvas.add(header_shape)
 540
 541    # footer (i.e. 4cat banner)
 542    if footer:
 543        footersize = (fontsize_small * len(footer) * 0.7, fontsize_small * 2)
 544        footer_shape = SVG(insert=(width - footersize[0], height - footersize[1]), size=footersize)
 545        footer_shape.add(Rect(insert=(0, 0), size=("100%", "100%"), fill="#000"))
 546        link = Hyperlink(href="https://4cat.nl")
 547        link.add(
 548            Text(insert=("50%", "50%"), text=footer, dominant_baseline="middle", text_anchor="middle", fill="#FFF",
 549                 style="font-size:%ipx" % fontsize_small))
 550        footer_shape.add(link)
 551        canvas.add(footer_shape)
 552
 553    return canvas
 554
 555
 556def call_api(action, payload=None, wait_for_response=True):
 557    """
 558    Send message to server
 559
 560    Calls the internal API and returns interpreted response.
 561
 562    :param str action: API action
 563    :param payload: API payload
 564    :param bool wait_for_response:  Wait for response? If not close connection
 565    immediately after sending data.
 566
 567    :return: API response, or timeout message in case of timeout
 568    """
 569    connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 570    connection.settimeout(15)
 571    connection.connect((config.get('API_HOST'), config.get('API_PORT')))
 572
 573    msg = json.dumps({"request": action, "payload": payload})
 574    connection.sendall(msg.encode("ascii", "ignore"))
 575
 576    if wait_for_response:
 577        try:
 578            response = ""
 579            while True:
 580                bytes = connection.recv(2048)
 581                if not bytes:
 582                    break
 583
 584                response += bytes.decode("ascii", "ignore")
 585        except (socket.timeout, TimeoutError):
 586            response = "(Connection timed out)"
 587
 588    try:
 589        connection.shutdown(socket.SHUT_RDWR)
 590    except OSError:
 591        # already shut down automatically
 592        pass
 593    connection.close()
 594
 595    try:
 596        return json.loads(response) if wait_for_response else None
 597    except json.JSONDecodeError:
 598        return response
 599
 600
 601def get_interval_descriptor(item, interval):
 602    """
 603    Get interval descriptor based on timestamp
 604
 605    :param dict item:  Item to generate descriptor for, should have a
 606    "timestamp" key
 607    :param str interval:  Interval, one of "all", "overall", "year",
 608    "month", "week", "day"
 609    :return str:  Interval descriptor, e.g. "overall", "2020", "2020-08",
 610    "2020-43", "2020-08-01"
 611    """
 612    if interval in ("all", "overall"):
 613        return interval
 614
 615    if "timestamp" not in item:
 616        raise ValueError("No date available for item in dataset")
 617
 618    # Catch cases where a custom timestamp has an epoch integer as value.
 619    try:
 620        timestamp = int(item["timestamp"])
 621        try:
 622            timestamp = datetime.datetime.fromtimestamp(timestamp)
 623        except (ValueError, TypeError) as e:
 624            raise ValueError("Invalid timestamp '%s'" % str(item["timestamp"]))
 625    except:
 626        try:
 627            timestamp = datetime.datetime.strptime(item["timestamp"], "%Y-%m-%d %H:%M:%S")
 628        except (ValueError, TypeError) as e:
 629            raise ValueError("Invalid date '%s'" % str(item["timestamp"]))
 630
 631    if interval == "year":
 632        return str(timestamp.year)
 633    elif interval == "month":
 634        return str(timestamp.year) + "-" + str(timestamp.month).zfill(2)
 635    elif interval == "week":
 636        return str(timestamp.isocalendar()[0]) + "-" + str(timestamp.isocalendar()[1]).zfill(2)
 637    elif interval == "hour":
 638        return str(timestamp.year) + "-" + str(timestamp.month).zfill(2) + "-" + str(timestamp.day).zfill(
 639            2) + " " + str(timestamp.hour).zfill(2)
 640    elif interval == "minute":
 641        return str(timestamp.year) + "-" + str(timestamp.month).zfill(2) + "-" + str(timestamp.day).zfill(
 642            2) + " " + str(timestamp.hour).zfill(2) + ":" + str(timestamp.minute).zfill(2)
 643    else:
 644        return str(timestamp.year) + "-" + str(timestamp.month).zfill(2) + "-" + str(timestamp.day).zfill(2)
 645
 646
 647def pad_interval(intervals, first_interval=None, last_interval=None):
 648    """
 649    Pad an interval so all intermediate intervals are filled
 650
 651    Warning, ugly code (PRs very welcome)
 652
 653    :param dict intervals:  A dictionary, with dates (YYYY{-MM}{-DD}) as keys
 654    and a numerical value.
 655    :param first_interval:
 656    :param last_interval:
 657    :return:
 658    """
 659    missing = 0
 660    test_key = list(intervals.keys())[0]
 661
 662    # first determine the boundaries of the interval
 663    # these may be passed as parameters, or they can be inferred from the
 664    # interval given
 665    if first_interval:
 666        first_interval = str(first_interval)
 667        first_year = int(first_interval[0:4])
 668        if len(first_interval) > 4:
 669            first_month = int(first_interval[5:7])
 670        if len(first_interval) > 7:
 671            first_day = int(first_interval[8:10])
 672        if len(first_interval) > 10:
 673            first_hour = int(first_interval[11:13])
 674        if len(first_interval) > 13:
 675            first_minute = int(first_interval[14:16])
 676
 677    else:
 678        first_year = min([int(i[0:4]) for i in intervals])
 679        if len(test_key) > 4:
 680            first_month = min([int(i[5:7]) for i in intervals if int(i[0:4]) == first_year])
 681        if len(test_key) > 7:
 682            first_day = min(
 683                [int(i[8:10]) for i in intervals if int(i[0:4]) == first_year and int(i[5:7]) == first_month])
 684        if len(test_key) > 10:
 685            first_hour = min(
 686                [int(i[11:13]) for i in intervals if
 687                 int(i[0:4]) == first_year and int(i[5:7]) == first_month and int(i[8:10]) == first_day])
 688        if len(test_key) > 13:
 689            first_minute = min(
 690                [int(i[14:16]) for i in intervals if
 691                 int(i[0:4]) == first_year and int(i[5:7]) == first_month and int(i[8:10]) == first_day and int(
 692                     i[11:13]) == first_hour])
 693
 694    if last_interval:
 695        last_interval = str(last_interval)
 696        last_year = int(last_interval[0:4])
 697        if len(last_interval) > 4:
 698            last_month = int(last_interval[5:7])
 699        if len(last_interval) > 7:
 700            last_day = int(last_interval[8:10])
 701        if len(last_interval) > 10:
 702            last_hour = int(last_interval[11:13])
 703        if len(last_interval) > 13:
 704            last_minute = int(last_interval[14:16])
 705    else:
 706        last_year = max([int(i[0:4]) for i in intervals])
 707        if len(test_key) > 4:
 708            last_month = max([int(i[5:7]) for i in intervals if int(i[0:4]) == last_year])
 709        if len(test_key) > 7:
 710            last_day = max(
 711                [int(i[8:10]) for i in intervals if int(i[0:4]) == last_year and int(i[5:7]) == last_month])
 712        if len(test_key) > 10:
 713            last_hour = max(
 714                [int(i[11:13]) for i in intervals if
 715                 int(i[0:4]) == last_year and int(i[5:7]) == last_month and int(i[8:10]) == last_day])
 716        if len(test_key) > 13:
 717            last_minute = max(
 718                [int(i[14:16]) for i in intervals if
 719                 int(i[0:4]) == last_year and int(i[5:7]) == last_month and int(i[8:10]) == last_day and int(
 720                     i[11:13]) == last_hour])
 721
 722    has_month = re.match(r"^[0-9]{4}-[0-9]", test_key)
 723    has_day = re.match(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}", test_key)
 724    has_hour = re.match(r"^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}", test_key)
 725    has_minute = re.match(r"^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}", test_key)
 726
 727    all_intervals = []
 728    for year in range(first_year, last_year + 1):
 729        year_interval = str(year)
 730
 731        if not has_month:
 732            all_intervals.append(year_interval)
 733            continue
 734
 735        start_month = first_month if year == first_year else 1
 736        end_month = last_month if year == last_year else 12
 737        for month in range(start_month, end_month + 1):
 738            month_interval = year_interval + "-" + str(month).zfill(2)
 739
 740            if not has_day:
 741                all_intervals.append(month_interval)
 742                continue
 743
 744            start_day = first_day if all((year == first_year, month == first_month)) else 1
 745            end_day = last_day if all((year == last_year, month == last_month)) else monthrange(year, month)[1]
 746            for day in range(start_day, end_day + 1):
 747                day_interval = month_interval + "-" + str(day).zfill(2)
 748
 749                if not has_hour:
 750                    all_intervals.append(day_interval)
 751                    continue
 752
 753                start_hour = first_hour if all((year == first_year, month == first_month, day == first_day)) else 0
 754                end_hour = last_hour if all((year == last_year, month == last_month, day == last_day)) else 23
 755                for hour in range(start_hour, end_hour + 1):
 756                    hour_interval = day_interval + " " + str(hour).zfill(2)
 757
 758                    if not has_minute:
 759                        all_intervals.append(hour_interval)
 760                        continue
 761
 762                    start_minute = first_minute if all(
 763                        (year == first_year, month == first_month, day == first_day, hour == first_hour)) else 0
 764                    end_minute = last_minute if all(
 765                        (year == last_year, month == last_month, day == last_day, hour == last_hour)) else 59
 766
 767                    for minute in range(start_minute, end_minute + 1):
 768                        minute_interval = hour_interval + ":" + str(minute).zfill(2)
 769                        all_intervals.append(minute_interval)
 770
 771    for interval in all_intervals:
 772        if interval not in intervals:
 773            intervals[interval] = 0
 774            missing += 1
 775
 776    # sort while we're at it
 777    intervals = {key: intervals[key] for key in sorted(intervals)}
 778
 779    return missing, intervals
 780
 781
 782def remove_nuls(value):
 783    """
 784    Remove \0 from a value
 785
 786    The CSV library cries about a null byte when it encounters one :( :( :(
 787    poor little csv cannot handle a tiny little null byte
 788
 789    So remove them from the data because they should not occur in utf-8 data
 790    anyway.
 791
 792    :param value:  Value to remove nulls from. For dictionaries, sets, tuples
 793    and lists all items are parsed recursively.
 794    :return value:  Cleaned value
 795    """
 796    if type(value) is dict:
 797        for field in value:
 798            value[field] = remove_nuls(value[field])
 799    elif type(value) is list:
 800        value = [remove_nuls(item) for item in value]
 801    elif type(value) is tuple:
 802        value = tuple([remove_nuls(item) for item in value])
 803    elif type(value) is set:
 804        value = set([remove_nuls(item) for item in value])
 805    elif type(value) is str:
 806        value = value.replace("\0", "")
 807
 808    return value
 809
 810
 811class NullAwareTextIOWrapper(io.TextIOWrapper):
 812    """
 813    TextIOWrapper that skips null bytes
 814
 815    This can be used as a file reader that silently discards any null bytes it
 816    encounters.
 817    """
 818
 819    def __next__(self):
 820        value = super().__next__()
 821        return remove_nuls(value)
 822
 823
 824class HashCache:
 825    """
 826    Simple cache handler to cache hashed values
 827
 828    Avoids having to calculate a hash for values that have been hashed before
 829    """
 830
 831    def __init__(self, hasher):
 832        self.hash_cache = {}
 833        self.hasher = hasher
 834
 835    def update_cache(self, value):
 836        """
 837        Checks the hash_cache to see if the value has been cached previously,
 838        updates the hash_cache if needed, and returns the hashed value.
 839        """
 840        # value = str(value)
 841        if value not in self.hash_cache:
 842            author_hasher = self.hasher.copy()
 843            author_hasher.update(str(value).encode("utf-8"))
 844            self.hash_cache[value] = author_hasher.hexdigest()
 845            del author_hasher
 846        return self.hash_cache[value]
 847
 848
 849def dict_search_and_update(item, keyword_matches, function):
 850    """
 851    Filter fields in an object recursively
 852
 853    Apply a function to every item and sub item of a dictionary if the key
 854    contains one of the provided match terms.
 855
 856    Function loops through a dictionary or list and compares dictionary keys to
 857    the strings defined by keyword_matches. It then applies the change_function
 858    to corresponding values.
 859
 860    Note: if a matching term is found, all nested values will have the function
 861    applied to them. e.g., all these values would be changed even those with
 862    not_key_match:
 863
 864    {'key_match' : 'changed',
 865    'also_key_match' : {'not_key_match' : 'but_value_still_changed'},
 866    'another_key_match': ['this_is_changed', 'and_this', {'not_key_match' : 'even_this_is_changed'}]}
 867
 868    This is a comprehensive (and expensive) approach to updating a dictionary.
 869    IF a dictionary structure is known, a better solution would be to update
 870    using specific keys.
 871
 872    :param Dict/List item:  dictionary/list/json to loop through
 873    :param String keyword_matches:  list of strings that will be matched to
 874    dictionary keys. Can contain wildcards which are matched using fnmatch.
 875    :param Function function:  function appled to all values of any items
 876    nested under a matching key
 877
 878    :return Dict/List: Copy of original item, but filtered
 879    """
 880
 881    def loop_helper_function(d_or_l, match_terms, change_function):
 882        """
 883        Recursive helper function that updates item in place
 884        """
 885        if isinstance(d_or_l, dict):
 886            # Iterate through dictionary
 887            for key, value in iter(d_or_l.items()):
 888                if match_terms == 'True' or any([fnmatch.fnmatch(key, match_term) for match_term in match_terms]):
 889                    # Match found; apply function to all items and sub-items
 890                    if isinstance(value, (list, dict)):
 891                        # Pass item through again with match_terms = True
 892                        loop_helper_function(value, 'True', change_function)
 893                    elif value is None:
 894                        pass
 895                    else:
 896                        # Update the value
 897                        d_or_l[key] = change_function(value)
 898                elif isinstance(value, (list, dict)):
 899                    # Continue search
 900                    loop_helper_function(value, match_terms, change_function)
 901        elif isinstance(d_or_l, list):
 902            # Iterate through list
 903            for n, value in enumerate(d_or_l):
 904                if isinstance(value, (list, dict)):
 905                    # Continue search
 906                    loop_helper_function(value, match_terms, change_function)
 907                elif match_terms == 'True':
 908                    # List item nested in matching
 909                    d_or_l[n] = change_function(value)
 910        else:
 911            raise Exception('Must pass list or dictionary')
 912
 913    # Lowercase keyword_matches
 914    keyword_matches = [keyword.lower() for keyword in keyword_matches]
 915
 916    # Create deepcopy and return new item
 917    temp_item = copy.deepcopy(item)
 918    loop_helper_function(temp_item, keyword_matches, function)
 919    return temp_item
 920
 921
 922def get_last_line(filepath):
 923    """
 924    Seeks from end of file for '\n' and returns that line
 925
 926    :param str filepath:  path to file
 927    :return str: last line of file
 928    """
 929    with open(filepath, "rb") as file:
 930        try:
 931            # start at the end of file
 932            file.seek(-2, os.SEEK_END)
 933            # check if NOT endline i.e. '\n'
 934            while file.read(1) != b'\n':
 935                # if not '\n', back up two characters and check again
 936                file.seek(-2, os.SEEK_CUR)
 937        except OSError:
 938            file.seek(0)
 939        last_line = file.readline().decode()
 940    return last_line
 941
 942
 943def add_notification(db, user, notification, expires=None, allow_dismiss=True):
 944    db.insert("users_notifications", {
 945        "username": user,
 946        "notification": notification,
 947        "timestamp_expires": expires,
 948        "allow_dismiss": allow_dismiss
 949    }, safe=True)
 950
 951
 952def send_email(recipient, message):
 953    """
 954    Send an e-mail using the configured SMTP settings
 955
 956    Just a thin wrapper around smtplib, so we don't have to repeat ourselves.
 957    Exceptions are to be handled outside the function.
 958
 959    :param list recipient:  Recipient e-mail addresses
 960    :param MIMEMultipart message:  Message to send
 961    """
 962    # Create a secure SSL context
 963    context = ssl.create_default_context()
 964
 965    # Decide which connection type
 966    with smtplib.SMTP_SSL(config.get('mail.server'), port=config.get('mail.port', 0), context=context) if config.get(
 967            'mail.ssl') == 'ssl' else smtplib.SMTP(config.get('mail.server'),
 968                                                   port=config.get('mail.port', 0)) as server:
 969        if config.get('mail.ssl') == 'tls':
 970            # smtplib.SMTP adds TLS context here
 971            server.starttls(context=context)
 972
 973        # Log in
 974        if config.get('mail.username') and config.get('mail.password'):
 975            server.ehlo()
 976            server.login(config.get('mail.username'), config.get('mail.password'))
 977
 978        # Send message
 979        if type(message) == str:
 980            server.sendmail(config.get('mail.noreply'), recipient, message)
 981        else:
 982            server.sendmail(config.get('mail.noreply'), recipient, message.as_string())
 983
 984
 985def flatten_dict(d: MutableMapping, parent_key: str = '', sep: str = '.'):
 986    """
 987    Return a flattened dictionary where nested dictionary objects are given new
 988    keys using the partent key combined using the seperator with the child key.
 989
 990    Lists will be converted to json strings via json.dumps()
 991
 992    :param MutableMapping d:  Dictionary like object
 993    :param str partent_key: The original parent key prepending future nested keys
 994    :param str sep: A seperator string used to combine parent and child keys
 995    :return dict:  A new dictionary with the no nested values
 996    """
 997
 998    def _flatten_dict_gen(d, parent_key, sep):
 999        for k, v in d.items():
1000            new_key = parent_key + sep + k if parent_key else k
1001            if isinstance(v, MutableMapping):
1002                yield from flatten_dict(v, new_key, sep=sep).items()
1003            elif isinstance(v, (list, set)):
1004                yield new_key, json.dumps(
1005                    [flatten_dict(item, new_key, sep=sep) if isinstance(item, MutableMapping) else item for item in v])
1006            else:
1007                yield new_key, v
1008
1009    return dict(_flatten_dict_gen(d, parent_key, sep))
1010
1011
1012def sets_to_lists(d: MutableMapping):
1013    """
1014    Return a dictionary where all nested sets have been converted to lists.
1015
1016    :param MutableMapping d:  Dictionary like object
1017    :return dict:  A new dictionary with the no nested sets
1018    """
1019
1020    def _check_list(l):
1021        return [sets_to_lists(item) if isinstance(item, MutableMapping) else _check_list(item) if isinstance(item, (
1022        set, list)) else item for item in l]
1023
1024    def _sets_to_lists_gen(d):
1025        for k, v in d.items():
1026            if isinstance(v, MutableMapping):
1027                yield k, sets_to_lists(v)
1028            elif isinstance(v, (list, set)):
1029                yield k, _check_list(v)
1030            else:
1031                yield k, v
1032
1033    return dict(_sets_to_lists_gen(d))
1034
1035
1036def url_to_hash(url, remove_scheme=True, remove_www=True):
1037    """
1038    Convert a URL to a filename; some URLs are too long to be used as filenames, this keeps the domain and hashes the
1039    rest of the URL.
1040    """
1041    parsed_url = urlparse(url.lower())
1042    if parsed_url:
1043        if remove_scheme:
1044            parsed_url = parsed_url._replace(scheme="")
1045        if remove_www:
1046            netloc = re.sub(r"^www\.", "", parsed_url.netloc)
1047            parsed_url = parsed_url._replace(netloc=netloc)
1048
1049        url = re.sub(r"[^0-9a-z]+", "_", urlunparse(parsed_url).strip("/"))
1050    else:
1051        # Unable to parse URL; use regex
1052        if remove_scheme:
1053            url = re.sub(r"^https?://", "", url)
1054        if remove_www:
1055            if not remove_scheme:
1056                scheme = re.match(r"^https?://", url).group()
1057                temp_url = re.sub(r"^https?://", "", url)
1058                url = scheme + re.sub(r"^www\.", "", temp_url)
1059            else:
1060                url = re.sub(r"^www\.", "", url)
1061
1062        url = re.sub(r"[^0-9a-z]+", "_", url.lower().strip("/"))
1063
1064    return hashlib.blake2b(url.encode("utf-8"), digest_size=24).hexdigest()
1065
1066
1067def split_urls(url_string, allowed_schemes=None):
1068    """
1069    Split URL text by \n and commas.
1070
1071    4CAT allows users to input lists by either separating items with a newline or a comma. This function will split URLs
1072    and also check for commas within URLs using schemes.
1073
1074    Note: some urls may contain scheme (e.g., https://web.archive.org/web/20250000000000*/http://economist.com);
1075    this function will work so long as the inner scheme does not follow a comma (e.g., "http://,https://" would fail).
1076    """
1077    if allowed_schemes is None:
1078        allowed_schemes = ('http://', 'https://', 'ftp://', 'ftps://')
1079    potential_urls = []
1080    # Split the text by \n
1081    for line in url_string.split('\n'):
1082        # Handle commas that may exist within URLs
1083        parts = line.split(',')
1084        recombined_url = ""
1085        for part in parts:
1086            if part.startswith(allowed_schemes):  # Other schemes exist
1087                # New URL start detected
1088                if recombined_url:
1089                    # Already have a URL, add to list
1090                    potential_urls.append(recombined_url)
1091                # Start new URL
1092                recombined_url = part
1093            elif part:
1094                if recombined_url:
1095                    # Add to existing URL
1096                    recombined_url += "," + part
1097                else:
1098                    # No existing URL, start new
1099                    recombined_url = part
1100            else:
1101                # Ignore empty strings
1102                pass
1103        if recombined_url:
1104            # Add any remaining URL
1105            potential_urls.append(recombined_url)
1106    return potential_urls
1107
1108
1109def folder_size(path='.'):
1110    """
1111    Get the size of a folder using os.scandir for efficiency
1112    """
1113    total = 0
1114    for entry in os.scandir(path):
1115        if entry.is_file():
1116            total += entry.stat().st_size
1117        elif entry.is_dir():
1118            total += folder_size(entry.path)
1119    return total