Edit on GitHub

common.config_manager 

  1import itertools
  2import pickle
  3import time
  4import json
  5
  6from pathlib import Path
  7from common.lib.database import Database
  8
  9from common.lib.exceptions import ConfigException
 10from common.lib.config_definition import config_definition
 11from common.lib.user_input import UserInput
 12
 13import configparser
 14import os
 15
 16
 17class ConfigManager:
 18    db = None
 19    dbconn = None
 20    cache = {}
 21
 22    core_settings = {}
 23    config_definition = {}
 24    tag_context = []  # todo
 25
 26    def __init__(self, db=None):
 27        # ensure core settings (including database config) are loaded
 28        self.load_core_settings()
 29        self.load_user_settings()
 30
 31        # establish database connection if none available
 32        self.db = db
 33
 34    def with_db(self, db=None):
 35        """
 36        Initialise database
 37
 38        Not done on init, because something may need core settings before the
 39        database can be initialised
 40
 41        :param db:  Database object. If None, initialise it using the core config
 42        """
 43        if db or not self.db:
 44            # Replace w/ db if provided else only initialise if not already
 45            self.db = db if db else Database(logger=None, dbname=self.get("DB_NAME"), user=self.get("DB_USER"),
 46                                         password=self.get("DB_PASSWORD"), host=self.get("DB_HOST"),
 47                                         port=self.get("DB_PORT"), appname="config-reader")
 48        else:
 49            # self.db already initialized and no db provided
 50            pass
 51
 52    def load_user_settings(self):
 53        """
 54        Load settings configurable by the user
 55
 56        Does not load the settings themselves, but rather the definition so
 57        values can be validated, etc
 58        """
 59        # basic 4CAT settings
 60        self.config_definition.update(config_definition)
 61
 62        # module settings can't be loaded directly because modules need the
 63        # config manager to load, so that becomes circular
 64        # instead, this is cached on startup and then loaded here
 65        module_config_path = self.get("PATH_ROOT").joinpath("config/module_config.bin")
 66        if module_config_path.exists():
 67            try:
 68                with module_config_path.open("rb") as infile:
 69                    retries = 0
 70                    module_config = None
 71                    # if 4CAT is being run in two different containers
 72                    # (front-end and back-end) they might both be running this
 73                    # bit of code at the same time. If the file is half-written
 74                    # loading it will fail, so allow for a few retries
 75                    while retries < 3:
 76                        try:
 77                            module_config = pickle.load(infile)
 78                            break
 79                        except Exception:  # this can be a number of exceptions, all with the same recovery path
 80                            time.sleep(0.1)
 81                            retries += 1
 82                            continue
 83
 84                    if module_config is None:
 85                        # not really a way to gracefully recover from this, but
 86                        # we can at least describe the error
 87                        raise RuntimeError("Could not read module_config.bin. The 4CAT developers did a bad job of "
 88                                           "preventing this. Shame on them!")
 89
 90                    self.config_definition.update(module_config)
 91            except (ValueError, TypeError) as e:
 92                pass
 93
 94    def load_core_settings(self):
 95        """
 96        Load 4CAT core settings
 97
 98        These are (mostly) stored in config.ini and cannot be changed from the
 99        web interface.
100
101        :return:
102        """
103        config_file = Path(__file__).parent.parent.joinpath("config/config.ini")
104
105        config_reader = configparser.ConfigParser()
106        in_docker = False
107        if config_file.exists():
108            config_reader.read(config_file)
109            if config_reader["DOCKER"].getboolean("use_docker_config"):
110                # Can use throughtout 4CAT to know if Docker environment
111                in_docker = True
112        else:
113            # config should be created!
114            raise ConfigException("No config/config.ini file exists! Update and rename the config.ini-example file.")
115
116        self.core_settings.update({
117            "CONFIG_FILE": config_file.resolve(),
118            "USING_DOCKER": in_docker,
119            "DB_HOST": config_reader["DATABASE"].get("db_host"),
120            "DB_PORT": config_reader["DATABASE"].get("db_port"),
121            "DB_USER": config_reader["DATABASE"].get("db_user"),
122            "DB_NAME": config_reader["DATABASE"].get("db_name"),
123            "DB_PASSWORD": config_reader["DATABASE"].get("db_password"),
124
125            "API_HOST": config_reader["API"].get("api_host"),
126            "API_PORT": config_reader["API"].getint("api_port"),
127
128            "PATH_ROOT": Path(os.path.abspath(os.path.dirname(__file__))).joinpath(
129                "..").resolve(),  # better don"t change this
130            "PATH_LOGS": Path(config_reader["PATHS"].get("path_logs", "")),
131            "PATH_IMAGES": Path(config_reader["PATHS"].get("path_images", "")),
132            "PATH_DATA": Path(config_reader["PATHS"].get("path_data", "")),
133            "PATH_LOCKFILE": Path(config_reader["PATHS"].get("path_lockfile", "")),
134            "PATH_SESSIONS": Path(config_reader["PATHS"].get("path_sessions", "")),
135
136            "ANONYMISATION_SALT": config_reader["GENERATE"].get("anonymisation_salt"),
137            "SECRET_KEY": config_reader["GENERATE"].get("secret_key")
138        })
139
140    def ensure_database(self):
141        """
142        Ensure the database is in sync with the config definition
143
144        Deletes all stored settings not defined in 4CAT, and creates a global
145        setting for all settings not yet in the database.
146        """
147        self.with_db()
148
149        # create global values for known keys with the default
150        known_settings = self.get_all()
151        for setting, parameters in self.config_definition.items():
152            if setting in known_settings:
153                continue
154
155            self.db.log.debug(f"Creating setting: {setting} with default value {parameters.get('default', '')}")
156            self.set(setting, parameters.get("default", ""))
157
158        # make sure settings and user table are in sync
159        user_tags = list(set(itertools.chain(*[u["tags"] for u in self.db.fetchall("SELECT DISTINCT tags FROM users")])))
160        known_tags = [t["tag"] for t in self.db.fetchall("SELECT DISTINCT tag FROM settings")]
161        tag_order = self.get("flask.tag_order")
162
163        for tag in known_tags:
164            # add tags used by a setting to tag order
165            if tag and tag not in tag_order:
166                tag_order.append(tag)
167
168        for tag in user_tags:
169            # add tags used by a user to tag order
170            if tag and tag not in tag_order:
171                tag_order.append(tag)
172
173        # admin tag should always be first in order
174        if "admin" in tag_order:
175            tag_order.remove("admin")
176
177        tag_order.insert(0, "admin")
178
179        self.set("flask.tag_order", tag_order)
180        self.db.commit()
181
182    def get_all(self, is_json=False, user=None, tags=None):
183        """
184        Get all known settings
185
186        :param bool is_json:  if True, the value is returned as stored and not
187        interpreted as JSON if it comes from the database
188        :param user:  User object or name. Adds a tag `user:[username]` in
189        front of the tag list.
190        :param tags:  Tag or tags for the required setting. If a tag is
191        provided, the method checks if a special value for the setting exists
192        with the given tag, and returns that if one exists. First matching tag
193        wins.
194
195        :return dict: Setting value, as a dictionary with setting names as keys
196        and setting values as values.
197        """
198        return self.get(attribute_name=None, default=None, is_json=is_json, user=user, tags=tags)
199
200    def get(self, attribute_name, default=None, is_json=False, user=None, tags=None):
201        """
202        Get a setting's value from the database
203
204        If the setting does not exist, the provided fallback value is returned.
205
206        :param str|list|None attribute_name:  Setting to return. If a string,
207        return that setting's value. If a list, return a dictionary of values.
208        If none, return a dictionary with all settings.
209        :param default:  Value to return if setting does not exist
210        :param bool is_json:  if True, the value is returned as stored and not
211        interpreted as JSON if it comes from the database
212        :param user:  User object or name. Adds a tag `user:[username]` in
213        front of the tag list.
214        :param tags:  Tag or tags for the required setting. If a tag is
215        provided, the method checks if a special value for the setting exists
216        with the given tag, and returns that if one exists. First matching tag
217        wins.
218
219        :return:  Setting value, or the provided fallback, or `None`.
220        """
221        # core settings are not from the database
222        if type(attribute_name) is str:
223            if attribute_name in self.core_settings:
224                return self.core_settings[attribute_name]
225            else:
226                attribute_name = (attribute_name,)
227        elif type(attribute_name) in (set, str):
228            attribute_name = tuple(attribute_name)
229
230        # if trying to access a setting that's not a core setting, attempt to
231        # initialise the database connection
232        if not self.db:
233            self.with_db()
234
235        # get tags to look for
236        tags = self.get_active_tags(user, tags)
237
238        # query database for any values within the required tags
239        tags.append("")  # empty tag = default value
240        if attribute_name:
241            query = "SELECT * FROM settings WHERE name IN %s AND tag IN %s"
242            replacements = (tuple(attribute_name), tuple(tags))
243        else:
244            query = "SELECT * FROM settings WHERE tag IN %s"
245            replacements = (tuple(tags), )
246
247        settings = {setting: {} for setting in attribute_name} if attribute_name else {}
248
249        for setting in self.db.fetchall(query, replacements):
250            if setting["name"] not in settings:
251                settings[setting["name"]] = {}
252
253            settings[setting["name"]][setting["tag"]] = setting["value"]
254
255        final_settings = {}
256        for setting_name, setting in settings.items():
257            # return first matching setting with a required tag, in the order the
258            # tags were provided
259            value = None
260            if setting:
261                for tag in tags:
262                    if tag in setting:
263                        value = setting[tag]
264                        break
265
266            # no matching tags? try empty tag
267            if value is None and "" in setting:
268                value = setting[""]
269
270            if not is_json and value is not None:
271                value = json.loads(value)
272            # TODO: Which default should have priority? The provided default feels like it should be the highest priority, but I think that is an old implementation and perhaps should be removed. - Dale
273            elif value is None and setting_name in self.config_definition and "default" in self.config_definition[setting_name]:
274                value = self.config_definition[setting_name]["default"]
275            elif value is None and default is not None:
276                value = default
277
278            final_settings[setting_name] = value
279
280        if attribute_name is not None and len(attribute_name) == 1:
281            # Single attribute requests; provide only the highest priority result
282            # this works because attribute_name is converted to a tuple (else already returned)
283            # if attribute_name is None, return all settings
284            # print(f"{user}: {attribute_name[0]} = {list(final_settings.values())[0]}")
285            return list(final_settings.values())[0]
286        else:
287            # All settings requested (via get_all)
288            return final_settings
289
290    def get_active_tags(self, user=None, tags=None):
291        """
292        Get active tags for given user/tag list
293
294        Used internally to harmonize tag setting for various methods, but can
295        also be called directly to verify tag activation.
296
297        :param user:  User object or name. Adds a tag `user:[username]` in
298        front of the tag list.
299        :param tags:  Tag or tags for the required setting. If a tag is
300        provided, the method checks if a special value for the setting exists
301        with the given tag, and returns that if one exists. First matching tag
302        wins.
303        :return list:  List of tags
304        """
305        # be flexible about the input types here
306        if tags is None:
307            tags = []
308        elif type(tags) is str:
309            tags = [tags]
310
311        # can provide either a string or user object
312        if type(user) is not str:
313            if hasattr(user, "get_id"):
314                user = user.get_id()
315            elif user is not None:
316                raise TypeError("get() expects None, a User object or a string for argument 'user'")
317
318        # user-specific settings are just a special type of tag (which takes
319        # precedence), same goes for user groups
320        if user:
321            user_tags = self.db.fetchone("SELECT tags FROM users WHERE name = %s", (user,))
322            if user_tags:
323                try:
324                    tags.extend(user_tags["tags"])
325                except (TypeError, ValueError):
326                    # should be a JSON list, but isn't
327                    pass
328
329            tags.insert(0, f"user:{user}")
330
331        return tags
332
333    def set(self, attribute_name, value, is_json=False, tag="", overwrite_existing=True):
334        """
335        Insert OR set value for a setting
336
337        If overwrite_existing=True and the setting exists, the setting is updated; if overwrite_existing=False and the
338        setting exists the setting is not updated.
339
340        :param str attribute_name:  Attribute to set
341        :param value:  Value to set (will be serialised as JSON)
342        :param bool is_json:  True for a value that is already a serialised JSON string; False if value is object that needs to
343                          be serialised into a JSON string
344        :param bool overwrite_existing: True will overwrite existing setting, False will do nothing if setting exists
345        :param str tag:  Tag to write setting for
346
347        :return int: number of updated rows
348        """
349        # Check value is valid JSON
350        if is_json:
351            try:
352                json.dumps(json.loads(value))
353            except json.JSONDecodeError:
354                return None
355        else:
356            try:
357                value = json.dumps(value)
358            except json.JSONDecodeError:
359                return None
360
361        if attribute_name in self.config_definition and self.config_definition.get(attribute_name).get("global"):
362            tag = ""
363
364        if overwrite_existing:
365            query = "INSERT INTO settings (name, value, tag) VALUES (%s, %s, %s) ON CONFLICT (name, tag) DO UPDATE SET value = EXCLUDED.value"
366        else:
367            query = "INSERT INTO settings (name, value, tag) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING"
368
369        self.db.execute(query, (attribute_name, value, tag))
370        updated_rows = self.db.cursor.rowcount
371        self.db.log.debug(f"Updated setting for {attribute_name}: {value} (tag: {tag})")
372
373        return updated_rows
374
375    def delete_for_tag(self, attribute_name, tag):
376        """
377        Delete config override for a given tag
378
379        :param str attribute_name:
380        :param str tag:
381        :return int: number of deleted rows
382        """
383        self.db.delete("settings", where={"name": attribute_name, "tag": tag})
384        updated_rows = self.db.cursor.rowcount
385
386        return updated_rows
387
388    def __getattr__(self, attr):
389        """
390        Getter so we can directly request values
391
392        :param attr:  Config setting to get
393        :return:  Value
394        """
395
396        if attr in dir(self):
397            # an explicitly defined attribute should always be called in favour
398            # of this passthrough
399            attribute = getattr(self, attr)
400            return attribute
401        else:
402            return self.get(attr)
403
404
405class ConfigWrapper:
406    """
407    Wrapper for the config manager
408
409    Allows setting a default set of tags or user, so that all subsequent calls
410    to `get()` are done for those tags or that user. Can also adjust tags based
411    on the HTTP request, if used in a Flask context.
412    """
413    def __init__(self, config, user=None, tags=None, request=None):
414        """
415        Initialise config wrapper
416
417        :param ConfigManager config:  Initialised config manager
418        :param user:  User to get settings for
419        :param tags:  Tags to get settings for
420        :param request:  Request to get headers from. This can be used to set
421        a particular tag based on the HTTP headers of the request, e.g. to
422        serve 4CAT with a different configuration based on the proxy server
423        used.
424        """
425        self.config = config
426        self.user = user
427        self.tags = tags
428        self.request = request
429
430        # this ensures the user object in turn reads from the wrapper
431        if self.user:
432            self.user.with_config(self)
433
434
435    def set(self, *args, **kwargs):
436        """
437        Wrap `set()`
438
439        :param args:
440        :param kwargs:
441        :return:
442        """
443        if "tag" not in kwargs and self.tags:
444            tag = self.tags if type(self.tags) is str else self.tags[0]
445            kwargs["tag"] = self.tags
446
447        return self.config.set(*args, **kwargs)
448
449    def get_all(self, *args, **kwargs):
450        """
451        Wrap `get_all()`
452
453        Takes the `user`, `tags` and `request` given when initialised into
454        account. If `tags` is set explicitly, the HTTP header-based override
455        is not applied.
456
457        :param args:
458        :param kwargs:
459        :return:
460        """
461        if "user" not in kwargs and self.user:
462            kwargs["user"] = self.user
463
464        if "tags" not in kwargs:
465            kwargs["tags"] = self.tags if self.tags else []
466            kwargs["tags"] = self.request_override(kwargs["tags"])
467
468        return self.config.get_all(*args, **kwargs)
469
470    def get(self, *args, **kwargs):
471        """
472        Wrap `get()`
473
474        Takes the `user`, `tags` and `request` given when initialised into
475        account. If `tags` is set explicitly, the HTTP header-based override
476        is not applied.
477
478        :param args:
479        :param kwargs:
480        :return:
481        """
482        if "user" not in kwargs:
483            kwargs["user"] = self.user
484
485        if "tags" not in kwargs:
486            kwargs["tags"] = self.tags if self.tags else []
487            kwargs["tags"] = self.request_override(kwargs["tags"])
488
489        return self.config.get(*args, **kwargs)
490
491    def get_active_tags(self, user=None, tags=None):
492        """
493        Wrap `get_active_tags()`
494
495        Takes the `user`, `tags` and `request` given when initialised into
496        account. If `tags` is set explicitly, the HTTP header-based override
497        is not applied.
498
499        :param user:
500        :param tags:
501        :return list:
502        """
503        active_tags = self.config.get_active_tags(user, tags)
504        if not tags:
505            active_tags = self.request_override(active_tags)
506
507        return active_tags
508
509    def request_override(self, tags):
510        """
511        Force tag via HTTP request headers
512
513        To facilitate loading different configurations based on the HTTP
514        request, the request object can be passed to the ConfigWrapper and
515        if a certain request header is set, the value of that header will be
516        added to the list of tags to consider when retrieving settings.
517
518        See the flask.proxy_secret config setting; this is used to prevent
519        users from changing configuration by forging the header.
520
521        :param list|str tags:  List of tags to extend based on request
522        :return list:  Amended list of tags
523        """
524        if type(tags) is str:
525            tags = [tags]
526
527        if self.request and self.request.headers.get("X-4Cat-Config-Tag") and \
528            self.config.get("flask.proxy_secret") and \
529            self.request.headers.get("X-4Cat-Config-Via-Proxy") == self.config.get("flask.proxy_secret"):
530            # need to ensure not just anyone can add this header to their
531            # request!
532            # to this end, the second header must be set to the secret value;
533            # if it is not set, assume the headers are not being configured by
534            # the proxy server
535            if not tags:
536                tags = []
537
538            # can never set admin tag via headers (should always be user-based)
539            forbidden_overrides = ("admin",)
540            tags += [tag for tag in self.request.headers.get("X-4Cat-Config-Tag").split(",") if tag not in forbidden_overrides]
541
542        return tags
543
544    def __getattr__(self, item):
545        """
546        Generic wrapper
547
548        Just pipe everything through to the config object
549
550        :param item:
551        :return:
552        """
553        if hasattr(self.config, item):
554            return getattr(self.config, item)
555        elif hasattr(self, item):
556            return getattr(self, item)
557        else:
558            raise AttributeError(f"'{self.__name__}' object has no attribute '{item}'")
559
560class ConfigDummy:
561    """
562    Dummy class to use as initial value for class-based configs
563
564    The config manager in processor objects takes the owner of the dataset of
565    the processor into account. This is only available after the object has
566    been inititated, so until then use this dummy wrapper that throws an error
567    when used to access config variables
568    """
569    def __getattribute__(self, item):
570        """
571        Access class attribute
572
573        :param item:
574        :raises NotImplementedError:
575        """
576        raise NotImplementedError("Cannot call processor config object in a class or static method - call global "
577                                  "configuration manager instead.")
578
579
580config = ConfigManager()
class ConfigManager: 
 18class ConfigManager:
 19    db = None
 20    dbconn = None
 21    cache = {}
 22
 23    core_settings = {}
 24    config_definition = {}
 25    tag_context = []  # todo
 26
 27    def __init__(self, db=None):
 28        # ensure core settings (including database config) are loaded
 29        self.load_core_settings()
 30        self.load_user_settings()
 31
 32        # establish database connection if none available
 33        self.db = db
 34
 35    def with_db(self, db=None):
 36        """
 37        Initialise database
 38
 39        Not done on init, because something may need core settings before the
 40        database can be initialised
 41
 42        :param db:  Database object. If None, initialise it using the core config
 43        """
 44        if db or not self.db:
 45            # Replace w/ db if provided else only initialise if not already
 46            self.db = db if db else Database(logger=None, dbname=self.get("DB_NAME"), user=self.get("DB_USER"),
 47                                         password=self.get("DB_PASSWORD"), host=self.get("DB_HOST"),
 48                                         port=self.get("DB_PORT"), appname="config-reader")
 49        else:
 50            # self.db already initialized and no db provided
 51            pass
 52
 53    def load_user_settings(self):
 54        """
 55        Load settings configurable by the user
 56
 57        Does not load the settings themselves, but rather the definition so
 58        values can be validated, etc
 59        """
 60        # basic 4CAT settings
 61        self.config_definition.update(config_definition)
 62
 63        # module settings can't be loaded directly because modules need the
 64        # config manager to load, so that becomes circular
 65        # instead, this is cached on startup and then loaded here
 66        module_config_path = self.get("PATH_ROOT").joinpath("config/module_config.bin")
 67        if module_config_path.exists():
 68            try:
 69                with module_config_path.open("rb") as infile:
 70                    retries = 0
 71                    module_config = None
 72                    # if 4CAT is being run in two different containers
 73                    # (front-end and back-end) they might both be running this
 74                    # bit of code at the same time. If the file is half-written
 75                    # loading it will fail, so allow for a few retries
 76                    while retries < 3:
 77                        try:
 78                            module_config = pickle.load(infile)
 79                            break
 80                        except Exception:  # this can be a number of exceptions, all with the same recovery path
 81                            time.sleep(0.1)
 82                            retries += 1
 83                            continue
 84
 85                    if module_config is None:
 86                        # not really a way to gracefully recover from this, but
 87                        # we can at least describe the error
 88                        raise RuntimeError("Could not read module_config.bin. The 4CAT developers did a bad job of "
 89                                           "preventing this. Shame on them!")
 90
 91                    self.config_definition.update(module_config)
 92            except (ValueError, TypeError) as e:
 93                pass
 94
 95    def load_core_settings(self):
 96        """
 97        Load 4CAT core settings
 98
 99        These are (mostly) stored in config.ini and cannot be changed from the
100        web interface.
101
102        :return:
103        """
104        config_file = Path(__file__).parent.parent.joinpath("config/config.ini")
105
106        config_reader = configparser.ConfigParser()
107        in_docker = False
108        if config_file.exists():
109            config_reader.read(config_file)
110            if config_reader["DOCKER"].getboolean("use_docker_config"):
111                # Can use throughtout 4CAT to know if Docker environment
112                in_docker = True
113        else:
114            # config should be created!
115            raise ConfigException("No config/config.ini file exists! Update and rename the config.ini-example file.")
116
117        self.core_settings.update({
118            "CONFIG_FILE": config_file.resolve(),
119            "USING_DOCKER": in_docker,
120            "DB_HOST": config_reader["DATABASE"].get("db_host"),
121            "DB_PORT": config_reader["DATABASE"].get("db_port"),
122            "DB_USER": config_reader["DATABASE"].get("db_user"),
123            "DB_NAME": config_reader["DATABASE"].get("db_name"),
124            "DB_PASSWORD": config_reader["DATABASE"].get("db_password"),
125
126            "API_HOST": config_reader["API"].get("api_host"),
127            "API_PORT": config_reader["API"].getint("api_port"),
128
129            "PATH_ROOT": Path(os.path.abspath(os.path.dirname(__file__))).joinpath(
130                "..").resolve(),  # better don"t change this
131            "PATH_LOGS": Path(config_reader["PATHS"].get("path_logs", "")),
132            "PATH_IMAGES": Path(config_reader["PATHS"].get("path_images", "")),
133            "PATH_DATA": Path(config_reader["PATHS"].get("path_data", "")),
134            "PATH_LOCKFILE": Path(config_reader["PATHS"].get("path_lockfile", "")),
135            "PATH_SESSIONS": Path(config_reader["PATHS"].get("path_sessions", "")),
136
137            "ANONYMISATION_SALT": config_reader["GENERATE"].get("anonymisation_salt"),
138            "SECRET_KEY": config_reader["GENERATE"].get("secret_key")
139        })
140
141    def ensure_database(self):
142        """
143        Ensure the database is in sync with the config definition
144
145        Deletes all stored settings not defined in 4CAT, and creates a global
146        setting for all settings not yet in the database.
147        """
148        self.with_db()
149
150        # create global values for known keys with the default
151        known_settings = self.get_all()
152        for setting, parameters in self.config_definition.items():
153            if setting in known_settings:
154                continue
155
156            self.db.log.debug(f"Creating setting: {setting} with default value {parameters.get('default', '')}")
157            self.set(setting, parameters.get("default", ""))
158
159        # make sure settings and user table are in sync
160        user_tags = list(set(itertools.chain(*[u["tags"] for u in self.db.fetchall("SELECT DISTINCT tags FROM users")])))
161        known_tags = [t["tag"] for t in self.db.fetchall("SELECT DISTINCT tag FROM settings")]
162        tag_order = self.get("flask.tag_order")
163
164        for tag in known_tags:
165            # add tags used by a setting to tag order
166            if tag and tag not in tag_order:
167                tag_order.append(tag)
168
169        for tag in user_tags:
170            # add tags used by a user to tag order
171            if tag and tag not in tag_order:
172                tag_order.append(tag)
173
174        # admin tag should always be first in order
175        if "admin" in tag_order:
176            tag_order.remove("admin")
177
178        tag_order.insert(0, "admin")
179
180        self.set("flask.tag_order", tag_order)
181        self.db.commit()
182
183    def get_all(self, is_json=False, user=None, tags=None):
184        """
185        Get all known settings
186
187        :param bool is_json:  if True, the value is returned as stored and not
188        interpreted as JSON if it comes from the database
189        :param user:  User object or name. Adds a tag `user:[username]` in
190        front of the tag list.
191        :param tags:  Tag or tags for the required setting. If a tag is
192        provided, the method checks if a special value for the setting exists
193        with the given tag, and returns that if one exists. First matching tag
194        wins.
195
196        :return dict: Setting value, as a dictionary with setting names as keys
197        and setting values as values.
198        """
199        return self.get(attribute_name=None, default=None, is_json=is_json, user=user, tags=tags)
200
201    def get(self, attribute_name, default=None, is_json=False, user=None, tags=None):
202        """
203        Get a setting's value from the database
204
205        If the setting does not exist, the provided fallback value is returned.
206
207        :param str|list|None attribute_name:  Setting to return. If a string,
208        return that setting's value. If a list, return a dictionary of values.
209        If none, return a dictionary with all settings.
210        :param default:  Value to return if setting does not exist
211        :param bool is_json:  if True, the value is returned as stored and not
212        interpreted as JSON if it comes from the database
213        :param user:  User object or name. Adds a tag `user:[username]` in
214        front of the tag list.
215        :param tags:  Tag or tags for the required setting. If a tag is
216        provided, the method checks if a special value for the setting exists
217        with the given tag, and returns that if one exists. First matching tag
218        wins.
219
220        :return:  Setting value, or the provided fallback, or `None`.
221        """
222        # core settings are not from the database
223        if type(attribute_name) is str:
224            if attribute_name in self.core_settings:
225                return self.core_settings[attribute_name]
226            else:
227                attribute_name = (attribute_name,)
228        elif type(attribute_name) in (set, str):
229            attribute_name = tuple(attribute_name)
230
231        # if trying to access a setting that's not a core setting, attempt to
232        # initialise the database connection
233        if not self.db:
234            self.with_db()
235
236        # get tags to look for
237        tags = self.get_active_tags(user, tags)
238
239        # query database for any values within the required tags
240        tags.append("")  # empty tag = default value
241        if attribute_name:
242            query = "SELECT * FROM settings WHERE name IN %s AND tag IN %s"
243            replacements = (tuple(attribute_name), tuple(tags))
244        else:
245            query = "SELECT * FROM settings WHERE tag IN %s"
246            replacements = (tuple(tags), )
247
248        settings = {setting: {} for setting in attribute_name} if attribute_name else {}
249
250        for setting in self.db.fetchall(query, replacements):
251            if setting["name"] not in settings:
252                settings[setting["name"]] = {}
253
254            settings[setting["name"]][setting["tag"]] = setting["value"]
255
256        final_settings = {}
257        for setting_name, setting in settings.items():
258            # return first matching setting with a required tag, in the order the
259            # tags were provided
260            value = None
261            if setting:
262                for tag in tags:
263                    if tag in setting:
264                        value = setting[tag]
265                        break
266
267            # no matching tags? try empty tag
268            if value is None and "" in setting:
269                value = setting[""]
270
271            if not is_json and value is not None:
272                value = json.loads(value)
273            # TODO: Which default should have priority? The provided default feels like it should be the highest priority, but I think that is an old implementation and perhaps should be removed. - Dale
274            elif value is None and setting_name in self.config_definition and "default" in self.config_definition[setting_name]:
275                value = self.config_definition[setting_name]["default"]
276            elif value is None and default is not None:
277                value = default
278
279            final_settings[setting_name] = value
280
281        if attribute_name is not None and len(attribute_name) == 1:
282            # Single attribute requests; provide only the highest priority result
283            # this works because attribute_name is converted to a tuple (else already returned)
284            # if attribute_name is None, return all settings
285            # print(f"{user}: {attribute_name[0]} = {list(final_settings.values())[0]}")
286            return list(final_settings.values())[0]
287        else:
288            # All settings requested (via get_all)
289            return final_settings
290
291    def get_active_tags(self, user=None, tags=None):
292        """
293        Get active tags for given user/tag list
294
295        Used internally to harmonize tag setting for various methods, but can
296        also be called directly to verify tag activation.
297
298        :param user:  User object or name. Adds a tag `user:[username]` in
299        front of the tag list.
300        :param tags:  Tag or tags for the required setting. If a tag is
301        provided, the method checks if a special value for the setting exists
302        with the given tag, and returns that if one exists. First matching tag
303        wins.
304        :return list:  List of tags
305        """
306        # be flexible about the input types here
307        if tags is None:
308            tags = []
309        elif type(tags) is str:
310            tags = [tags]
311
312        # can provide either a string or user object
313        if type(user) is not str:
314            if hasattr(user, "get_id"):
315                user = user.get_id()
316            elif user is not None:
317                raise TypeError("get() expects None, a User object or a string for argument 'user'")
318
319        # user-specific settings are just a special type of tag (which takes
320        # precedence), same goes for user groups
321        if user:
322            user_tags = self.db.fetchone("SELECT tags FROM users WHERE name = %s", (user,))
323            if user_tags:
324                try:
325                    tags.extend(user_tags["tags"])
326                except (TypeError, ValueError):
327                    # should be a JSON list, but isn't
328                    pass
329
330            tags.insert(0, f"user:{user}")
331
332        return tags
333
334    def set(self, attribute_name, value, is_json=False, tag="", overwrite_existing=True):
335        """
336        Insert OR set value for a setting
337
338        If overwrite_existing=True and the setting exists, the setting is updated; if overwrite_existing=False and the
339        setting exists the setting is not updated.
340
341        :param str attribute_name:  Attribute to set
342        :param value:  Value to set (will be serialised as JSON)
343        :param bool is_json:  True for a value that is already a serialised JSON string; False if value is object that needs to
344                          be serialised into a JSON string
345        :param bool overwrite_existing: True will overwrite existing setting, False will do nothing if setting exists
346        :param str tag:  Tag to write setting for
347
348        :return int: number of updated rows
349        """
350        # Check value is valid JSON
351        if is_json:
352            try:
353                json.dumps(json.loads(value))
354            except json.JSONDecodeError:
355                return None
356        else:
357            try:
358                value = json.dumps(value)
359            except json.JSONDecodeError:
360                return None
361
362        if attribute_name in self.config_definition and self.config_definition.get(attribute_name).get("global"):
363            tag = ""
364
365        if overwrite_existing:
366            query = "INSERT INTO settings (name, value, tag) VALUES (%s, %s, %s) ON CONFLICT (name, tag) DO UPDATE SET value = EXCLUDED.value"
367        else:
368            query = "INSERT INTO settings (name, value, tag) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING"
369
370        self.db.execute(query, (attribute_name, value, tag))
371        updated_rows = self.db.cursor.rowcount
372        self.db.log.debug(f"Updated setting for {attribute_name}: {value} (tag: {tag})")
373
374        return updated_rows
375
376    def delete_for_tag(self, attribute_name, tag):
377        """
378        Delete config override for a given tag
379
380        :param str attribute_name:
381        :param str tag:
382        :return int: number of deleted rows
383        """
384        self.db.delete("settings", where={"name": attribute_name, "tag": tag})
385        updated_rows = self.db.cursor.rowcount
386
387        return updated_rows
388
389    def __getattr__(self, attr):
390        """
391        Getter so we can directly request values
392
393        :param attr:  Config setting to get
394        :return:  Value
395        """
396
397        if attr in dir(self):
398            # an explicitly defined attribute should always be called in favour
399            # of this passthrough
400            attribute = getattr(self, attr)
401            return attribute
402        else:
403            return self.get(attr)
ConfigManager(db=None) 
27    def __init__(self, db=None):
28        # ensure core settings (including database config) are loaded
29        self.load_core_settings()
30        self.load_user_settings()
31
32        # establish database connection if none available
33        self.db = db
db = None
dbconn = None
cache = {}
core_settings = {'CONFIG_FILE': PosixPath('/opt/docs-maker/4cat/config/config.ini'), 'USING_DOCKER': False, 'DB_HOST': 'localhost', 'DB_PORT': '5432', 'DB_USER': 'fourcat', 'DB_NAME': 'fourcat', 'DB_PASSWORD': 'supers3cr3t', 'API_HOST': 'localhost', 'API_PORT': 4444, 'PATH_ROOT': PosixPath('/opt/docs-maker/4cat'), 'PATH_LOGS': PosixPath('logs'), 'PATH_IMAGES': PosixPath('data'), 'PATH_DATA': PosixPath('data'), 'PATH_LOCKFILE': PosixPath('backend'), 'PATH_SESSIONS': PosixPath('sessions'), 'ANONYMISATION_SALT': 'REPLACE_THIS', 'SECRET_KEY': 'REPLACE_THIS'}
config_definition = {'datasources._intro': {'type': 'info', 'help': "Data sources enabled below will be offered to people on the 'Create Dataset' page. Additionally, people can upload datasets for these by for example exporting them with [Zeeschuimer](https://github.com/digitalmethodsinitiative/zeeschuimer) to this 4CAT instance.\n\nSome data sources offer further settings which may be configured on other tabs."}, 'datasources.enabled': {'type': 'datasources', 'default': ['ninegag', 'bsky', 'douban', 'douyin', 'imgur', 'upload', 'instagram', 'import_4cat', 'linkedin', 'media-import', 'telegram', 'tiktok', 'twitter', 'tiktok-comments', 'truthsocial', 'gab'], 'help': 'Data Sources', 'tooltip': 'A list of enabled data sources that people can choose from when creating a dataset page.'}, 'datasources._intro2': {'type': 'info', 'help': "*Warning:* changes take effect immediately. Datasets that would have expired under the new settings will be deleted. You can use the 'Dataset bulk management' module in the control panel to manage the expiration status of existing datasets."}, 'datasources.expiration': {'type': 'json', 'default': {'fourchan': {'enabled': False, 'allow_optout': False, 'timeout': 0}, 'eightchan': {'enabled': False, 'allow_optout': False, 'timeout': 0}, 'eightkun': {'enabled': False, 'allow_optout': False, 'timeout': 0}, 'ninegag': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'bitchute': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'bsky': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'dmi-tcat': {'enabled': False, 'allow_optout': False, 'timeout': 0}, 'dmi-tcatv2': {'enabled': False, 'allow_optout': False, 'timeout': 0}, 'douban': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'douyin': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'import_4cat': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'gab': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'imgur': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'upload': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'instagram': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'linkedin': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'media-import': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'parler': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'reddit': {'enabled': False, 'allow_optout': False, 'timeout': 0}, 'telegram': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'tiktok': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'tiktok-urls': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'truthsocial': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'tumblr': {'enabled': False, 'allow_optout': False, 'timeout': 0}, 'twitter': {'enabled': True, 'allow_optout': False, 'timeout': 0}, 'twitterv2': {'enabled': False, 'allow_optout': False, 'timeout': 0}, 'usenet': {'enabled': False, 'allow_optout': False, 'timeout': 0}, 'vk': {'enabled': False, 'allow_optout': False, 'timeout': 0}}, 'help': 'Data source-specific expiration', 'tooltip': "Allows setting expiration settings per datasource. Configured by proxy via the 'data sources' setting.", 'indirect': True}, '4cat.name': {'type': 'string', 'default': '4CAT', 'help': 'Short tool name', 'tooltip': "Configure short name for the tool in its web interface. The backend will always refer to '4CAT' - the name of the software, and a 'powered by 4CAT' notice may also show up in the web interface regardless of the value entered here."}, '4cat.name_long': {'type': 'string', 'default': '4CAT: Capture and Analysis Toolkit', 'help': 'Full tool name', 'tooltip': "Used in e.g. the interface header. The backend will always refer to '4CAT' - the name of the software, and a 'powered by 4CAT' notice may also show up in the web interface regardless of the value entered here."}, '4cat.about_this_server': {'type': 'textarea', 'default': '', 'help': 'Server information', 'tooltip': "Custom server information that is displayed on the 'About' page. Can for instance be used to show information about who maintains the tool or what its intended purpose is."}, '4cat.crash_message': {'type': 'textarea', 'default': "This processor has crashed; the crash has been logged. 4CAT will try again when it is restarted. Contact your server administrator if this error persists. You can also report issues via 4CAT's [GitHub repository](https://github.com/digitalmethodsinitiative/4cat/issues).", 'help': 'Crash message', 'tooltip': 'This message is shown to users in the interface when a processor crashes while processing their dataset. It can contain Markdown markup.'}, 'privileges.can_create_dataset': {'type': 'toggle', 'default': True, 'help': 'Can create dataset', 'tooltip': "Controls whether users can view and use the 'Create dataset' page. Does NOT control whether users can run processors (which also create datasets); this is a separate setting."}, 'privileges.can_run_processors': {'type': 'toggle', 'default': True, 'help': 'Can run processors', 'tooltip': 'Controls whether processors can be run. There may be processor-specific settings or dependencies that override this.'}, 'privileges.can_view_all_datasets': {'type': 'toggle', 'default': False, 'help': 'Can view global dataset index', 'tooltip': 'Controls whether users can see the global datasets overview, i.e. not just for their own user but for all other users as well.'}, 'privileges.can_view_private_datasets': {'type': 'toggle', 'default': False, 'help': 'Can view private datasets', 'tooltip': 'Controls whether users can see the datasets made private by their owners.'}, 'privileges.can_create_api_token': {'type': 'toggle', 'default': True, 'help': 'Can create API token', 'tooltip': "Controls whether users can create a token for authentication with 4CAT's Web API."}, 'privileges.can_use_explorer': {'type': 'toggle', 'default': True, 'help': 'Can use explorer', 'tooltip': 'Controls whether users can use the Explorer feature to navigate datasets.'}, 'privileges.can_export_datasets': {'type': 'toggle', 'default': True, 'help': 'Can export datasets', 'tooltip': 'Allows users to export datasets they own to other 4CAT instances.'}, 'privileges.admin.can_manage_users': {'type': 'toggle', 'default': False, 'help': 'Can manage users', 'tooltip': 'Controls whether users can add, edit and delete other users via the Control Panel'}, 'privileges.admin.can_manage_notifications': {'type': 'toggle', 'default': False, 'help': 'Can manage notifications', 'tooltip': 'Controls whether users can add, edit and delete notifications via the Control Panel'}, 'privileges.admin.can_manage_settings': {'type': 'toggle', 'default': False, 'help': 'Can manage settings', 'tooltip': 'Controls whether users can manipulate 4CAT settings via the Control Panel'}, 'privileges.admin.can_manipulate_all_datasets': {'type': 'toggle', 'default': False, 'help': 'Can manipulate all datasets', 'tooltip': 'Controls whether users can manipulate all datasets as if they were an owner, e.g. sharing it with others, running processors, et cetera.'}, 'privileges.admin.can_restart': {'type': 'toggle', 'default': False, 'help': 'Can restart/upgrade', 'tooltip': 'Controls whether users can restart, upgrade, and manage extensions 4CAT via the Control Panel'}, 'privileges.can_upgrade_to_dev': {'type': 'toggle', 'default': False, 'help': 'Can upgrade to development branch', 'tooltip': "Controls whether users can upgrade 4CAT to a development branch of the code via the Control Panel. This is an easy way to break 4CAT so it is recommended to not enable this unless you're really sure of what you're doing."}, 'privileges.admin.can_manage_tags': {'type': 'toggle', 'default': False, 'help': 'Can manage user tags', 'tooltip': 'Controls whether users can manipulate user tags via the Control Panel'}, 'privileges.admin.can_view_status': {'type': 'toggle', 'default': False, 'help': 'Can view worker status', 'tooltip': 'Controls whether users can view worker status via the Control Panel'}, '4cat.github_url': {'type': 'string', 'default': 'https://github.com/digitalmethodsinitiative/4cat', 'help': 'Repository URL', 'tooltip': 'URL to the github repository for this 4CAT instance', 'global': True}, '4cat.phone_home_url': {'type': 'string', 'default': 'https://ping.4cat.nl', 'help': 'Phone home URL', 'tooltip': 'This URL is called once - when 4CAT is installed. If the installing user consents, information is sent to this URL to help the 4CAT developers (the Digital Methods Initiative) keep track of how much it is used. There should be no need to change this URL after installation.', 'global': True}, '4cat.phone_home_asked': {'type': 'toggle', 'default': True, 'help': 'Shown phone home request?', 'tooltip': "Whether you've seen the 'phone home request'. Set to `False` to see the request again. There should be no need to change this manually.", 'global': True}, '4cat.layout_hue': {'type': 'hue', 'default': 356, 'help': 'Interface accent colour', 'saturation': 87, 'value': 81, 'min': 0, 'max': 360, 'coerce_type': <class 'int'>, 'global': True}, '4cat.layout_hue_secondary': {'type': 'hue', 'default': 86, 'help': 'Interface secondary colour', 'saturation': 87, 'value': 90, 'min': 0, 'max': 360, 'coerce_type': <class 'int'>, 'global': True}, '4cat.allow_access_request': {'type': 'toggle', 'default': True, 'help': 'Allow access requests', 'tooltip': 'When enabled, users can request a 4CAT account via the login page if they do not have one, provided e-mail settings are configured.'}, '4cat.sphinx_host': {'type': 'string', 'default': 'localhost', 'help': 'Sphinx host', 'tooltip': 'Sphinx is used for full-text search for collected datasources (e.g., 4chan, 8kun, 8chan) and requires additional setup (see 4CAT wiki on GitHub).', 'global': True}, 'logging.slack.level': {'type': 'choice', 'default': 'WARNING', 'options': {'DEBUG': 'Debug', 'INFO': 'Info', 'WARNING': 'Warning', 'ERROR': 'Error', 'CRITICAL': 'Critical'}, 'help': 'Slack alert level', 'tooltip': 'Level of alerts (or higher) to be sent to Slack. Only alerts above this level are sent to the Slack webhook', 'global': True}, 'logging.slack.webhook': {'type': 'string', 'default': '', 'help': 'Slack webhook URL', 'tooltip': 'Slack callback URL to use for alerts', 'global': True}, 'mail.admin_email': {'type': 'string', 'default': '', 'help': 'Admin e-mail', 'tooltip': 'E-mail of admin, to send account requests etc to', 'global': True}, 'mail.server': {'type': 'string', 'default': '', 'help': 'SMTP server', 'tooltip': 'SMTP server to connect to for sending e-mail alerts.', 'global': True}, 'mail.port': {'type': 'string', 'default': 0, 'coerce_type': <class 'int'>, 'help': 'SMTP port', 'tooltip': 'SMTP port to connect to for sending e-mail alerts. "0" defaults to "465" for SMTP_SSL or OS default for SMTP.', 'global': True}, 'mail.ssl': {'type': 'choice', 'default': 'ssl', 'options': {'ssl': 'SSL', 'tls': 'TLS', 'none': 'None'}, 'help': 'SMTP over SSL, TLS, or None', 'tooltip': 'Security scheme to use to connect to e-mail server', 'global': True}, 'mail.username': {'type': 'string', 'default': '', 'help': 'SMTP Username', 'tooltip': 'Only if your SMTP server requires login', 'global': True}, 'mail.password': {'type': 'string', 'default': '', 'help': 'SMTP Password', 'tooltip': 'Only if your SMTP server requires login', 'global': True}, 'mail.noreply': {'type': 'string', 'default': 'noreply@localhost', 'help': 'NoReply e-mail', 'global': True}, 'explorer.max_posts': {'type': 'string', 'default': 100000, 'help': 'Amount of posts', 'coerce_type': <class 'int'>, 'tooltip': 'Amount of posts to show in Explorer. The maximum allowed amount of rows (prevents timeouts and memory errors)'}, 'explorer.posts_per_page': {'type': 'string', 'default': 50, 'help': 'Posts per page', 'coerce_type': <class 'int'>, 'tooltip': 'Posts to display per page'}, 'flask.flask_app': {'type': 'string', 'default': 'webtool/fourcat', 'help': 'Flask App Name', 'tooltip': '', 'global': True}, 'flask.server_name': {'type': 'string', 'default': '4cat.local:5000', 'help': 'Host name', 'tooltip': 'e.g., my4CAT.com, localhost, 127.0.0.1. Default is localhost; when running 4CAT in Docker this setting is ignored as any domain/port binding should be handled outside of the Docker container; the Docker container itself will serve on any domain name on the port configured in the .env file.', 'global': True}, 'flask.autologin.hostnames': {'type': 'json', 'default': [], 'help': 'White-listed hostnames', 'tooltip': 'A list of host names or IP addresses to automatically log in. Docker should include localhost and Server Name', 'global': True}, 'flask.autologin.api': {'type': 'json', 'default': [], 'help': 'White-list for API', 'tooltip': 'A list of host names or IP addresses to allow access to API endpoints with no rate limiting. Docker should include localhost and Server Name', 'global': True}, 'flask.https': {'type': 'toggle', 'default': False, 'help': 'Use HTTPS', 'tooltip': "If your server is using 'https', set to True and 4CAT will use HTTPS links.", 'global': True}, 'flask.proxy_override': {'type': 'multi_select', 'default': [], 'options': {'x_for': 'X-Forwarded-For', 'x_proto': 'X-Forwarded-Proto', 'x_host': 'X-Forwarded-Host', 'x_port': 'X-Forwarded-Port', 'x_prefix': 'X-Forwarded-Prefix'}, 'help': 'Use proxy headers for URL', 'tooltip': 'These proxy headers will be taken into account when building URLs. For example, if X-Forwarded-Proto is enabled, the URL scheme (http/https) of the built URL will be based on the scheme defined by this header. Use when running 4CAT behind a reverse proxy. Requires a front-end restart to take effect.'}, 'flask.autologin.name': {'type': 'string', 'default': 'Automatic login', 'help': 'Auto-login name', 'tooltip': 'Username for whitelisted hosts (automatically logged in users see this name for themselves)'}, 'flask.secret_key': {'type': 'string', 'default': 'please change me... please...', 'help': 'Secret key', 'tooltip': 'Secret key for Flask, used for session cookies', 'global': True}, 'flask.max_form_parts': {'type': 'string', 'default': 1000, 'help': 'Max form parts per request', 'coerce_type': <class 'int'>, 'global': True, 'tooltip': 'Affects approximate number of files that can be uploaded at once'}, 'flask.tag_order': {'type': 'json', 'default': ['admin'], 'help': 'Tag priority', 'tooltip': "User tag priority order. This can be manipulated from the 'User tags' panel instead of directly.", 'global': True, 'indirect': True}, 'flask.proxy_secret': {'type': 'string', 'default': '', 'help': 'Proxy secret', 'tooltip': 'Secret value to authenticate proxy headers. If the value of the X-4CAT-Config-Via-Proxy header matches this value, the X-4CAT-Config-Tag header can be used to enable a given configuration tag. Leave empty to disable this functionality.'}, 'api.youtube.name': {'type': 'string', 'default': 'youtube', 'help': 'YouTube API Service', 'tooltip': "YouTube API 'service name', e.g. youtube, googleapis, etc.", 'global': True}, 'api.youtube.version': {'type': 'string', 'default': 'v3', 'help': 'YouTube API Version', 'tooltip': "e.g., ''v3'", 'global': True}, 'api.youtube.key': {'type': 'string', 'default': '', 'help': 'YouTube API Key', 'tooltip': 'The developer key from your API console'}, 'dmi-service-manager.aa_DSM-intro-1': {'type': 'info', 'help': "The [DMI Service Manager](https://github.com/digitalmethodsinitiative/dmi_service_manager#start-dmi-service-manager) is a support tool used to run some advanced processors. These processors generally require high CPU usage, a lot of RAM, or a dedicated GPU and thus do not fit within 4CAT's arcitecture. It is also possible for multiple 4CAT instances to use the same service manager. Please see the link for instructions on setting up your own instance of the DMI Service Manager."}, 'dmi-service-manager.ab_server_address': {'type': 'string', 'default': '', 'help': 'DMI Service Manager server/URL', 'tooltip': 'The URL of the DMI Service Manager server, e.g. http://localhost:5000', 'global': True}, 'dmi-service-manager.ac_local_or_remote': {'type': 'choice', 'default': 0, 'help': 'DMI Services Local or Remote', 'tooltip': 'Services have local access to 4CAT files or must be transferred from remote via DMI Service Manager', 'options': {'local': 'Local', 'remote': 'Remote'}, 'global': True}, 'ui.homepage': {'type': 'choice', 'options': {'about': "'About' page", 'create-dataset': "'Create dataset' page", 'datasets': 'Dataset overview'}, 'help': '4CAT home page', 'default': 'about'}, 'ui.inline_preview': {'type': 'toggle', 'help': 'Show inline preview', 'default': False, 'tooltip': "Show main dataset preview directly on dataset pages, instead of behind a 'preview' button"}, 'ui.offer_anonymisation': {'type': 'toggle', 'help': 'Offer anonymisation options', 'default': True, 'tooltip': 'Offer users the option to anonymise their datasets at the time of creation. It is strongly recommended to leave this enabled.'}, 'ui.advertise_install': {'type': 'toggle', 'help': 'Advertise local 4CAT', 'default': True, 'tooltip': 'In the login form, remind users of the possibility to install their own 4CAT server.'}, 'ui.show_datasource': {'type': 'toggle', 'help': 'Show data source', 'default': True, 'tooltip': 'Show data source for each dataset. Can be useful to disable if only one data source is enabled.'}, 'ui.nav_pages': {'type': 'multi_select', 'help': 'Pages in navigation', 'options': {'data-policy': 'Data Policy', 'citing': 'How to cite'}, 'default': [], 'tooltip': 'These pages will be included in the navigation bar at the top of the interface.'}, 'ui.prefer_mapped_preview': {'type': 'toggle', 'help': 'Prefer mapped preview', 'default': True, 'tooltip': 'If a dataset is a JSON file but it can be mapped to a CSV file, show the CSV in the preview insteadof the underlying JSON.'}, 'ui.offer_hashing': {'type': 'toggle', 'default': True, 'help': 'Offer pseudonymisation', 'tooltip': "Add a checkbox to the 'create dataset' forum to allow users to toggle pseudonymisation."}, 'ui.offer_private': {'type': 'toggle', 'default': True, 'help': 'Offer create as private', 'tooltip': "Add a checkbox to the 'create dataset' forum to allow users to make a dataset private."}, 'ui.option_email': {'type': 'choice', 'options': {'none': 'No Emails', 'processor_only': 'Processors only', 'datasources_only': 'Create Dataset only', 'both': 'Both datasets and processors'}, 'default': 'none', 'help': 'Show email when complete option', 'tooltip': 'If a mail server is set up, enabling this allow users to request emails when datasets and processors are completed.'}}
tag_context = []
def with_db(self, db=None): 
35    def with_db(self, db=None):
36        """
37        Initialise database
38
39        Not done on init, because something may need core settings before the
40        database can be initialised
41
42        :param db:  Database object. If None, initialise it using the core config
43        """
44        if db or not self.db:
45            # Replace w/ db if provided else only initialise if not already
46            self.db = db if db else Database(logger=None, dbname=self.get("DB_NAME"), user=self.get("DB_USER"),
47                                         password=self.get("DB_PASSWORD"), host=self.get("DB_HOST"),
48                                         port=self.get("DB_PORT"), appname="config-reader")
49        else:
50            # self.db already initialized and no db provided
51            pass

Initialise database

Not done on init, because something may need core settings before the database can be initialised

Parameters
  • db: Database object. If None, initialise it using the core config
def load_user_settings(self): 
53    def load_user_settings(self):
54        """
55        Load settings configurable by the user
56
57        Does not load the settings themselves, but rather the definition so
58        values can be validated, etc
59        """
60        # basic 4CAT settings
61        self.config_definition.update(config_definition)
62
63        # module settings can't be loaded directly because modules need the
64        # config manager to load, so that becomes circular
65        # instead, this is cached on startup and then loaded here
66        module_config_path = self.get("PATH_ROOT").joinpath("config/module_config.bin")
67        if module_config_path.exists():
68            try:
69                with module_config_path.open("rb") as infile:
70                    retries = 0
71                    module_config = None
72                    # if 4CAT is being run in two different containers
73                    # (front-end and back-end) they might both be running this
74                    # bit of code at the same time. If the file is half-written
75                    # loading it will fail, so allow for a few retries
76                    while retries < 3:
77                        try:
78                            module_config = pickle.load(infile)
79                            break
80                        except Exception:  # this can be a number of exceptions, all with the same recovery path
81                            time.sleep(0.1)
82                            retries += 1
83                            continue
84
85                    if module_config is None:
86                        # not really a way to gracefully recover from this, but
87                        # we can at least describe the error
88                        raise RuntimeError("Could not read module_config.bin. The 4CAT developers did a bad job of "
89                                           "preventing this. Shame on them!")
90
91                    self.config_definition.update(module_config)
92            except (ValueError, TypeError) as e:
93                pass

Load settings configurable by the user

Does not load the settings themselves, but rather the definition so values can be validated, etc

def load_core_settings(self): 
 95    def load_core_settings(self):
 96        """
 97        Load 4CAT core settings
 98
 99        These are (mostly) stored in config.ini and cannot be changed from the
100        web interface.
101
102        :return:
103        """
104        config_file = Path(__file__).parent.parent.joinpath("config/config.ini")
105
106        config_reader = configparser.ConfigParser()
107        in_docker = False
108        if config_file.exists():
109            config_reader.read(config_file)
110            if config_reader["DOCKER"].getboolean("use_docker_config"):
111                # Can use throughtout 4CAT to know if Docker environment
112                in_docker = True
113        else:
114            # config should be created!
115            raise ConfigException("No config/config.ini file exists! Update and rename the config.ini-example file.")
116
117        self.core_settings.update({
118            "CONFIG_FILE": config_file.resolve(),
119            "USING_DOCKER": in_docker,
120            "DB_HOST": config_reader["DATABASE"].get("db_host"),
121            "DB_PORT": config_reader["DATABASE"].get("db_port"),
122            "DB_USER": config_reader["DATABASE"].get("db_user"),
123            "DB_NAME": config_reader["DATABASE"].get("db_name"),
124            "DB_PASSWORD": config_reader["DATABASE"].get("db_password"),
125
126            "API_HOST": config_reader["API"].get("api_host"),
127            "API_PORT": config_reader["API"].getint("api_port"),
128
129            "PATH_ROOT": Path(os.path.abspath(os.path.dirname(__file__))).joinpath(
130                "..").resolve(),  # better don"t change this
131            "PATH_LOGS": Path(config_reader["PATHS"].get("path_logs", "")),
132            "PATH_IMAGES": Path(config_reader["PATHS"].get("path_images", "")),
133            "PATH_DATA": Path(config_reader["PATHS"].get("path_data", "")),
134            "PATH_LOCKFILE": Path(config_reader["PATHS"].get("path_lockfile", "")),
135            "PATH_SESSIONS": Path(config_reader["PATHS"].get("path_sessions", "")),
136
137            "ANONYMISATION_SALT": config_reader["GENERATE"].get("anonymisation_salt"),
138            "SECRET_KEY": config_reader["GENERATE"].get("secret_key")
139        })

Load 4CAT core settings

These are (mostly) stored in config.ini and cannot be changed from the web interface.

Returns
def ensure_database(self): 
141    def ensure_database(self):
142        """
143        Ensure the database is in sync with the config definition
144
145        Deletes all stored settings not defined in 4CAT, and creates a global
146        setting for all settings not yet in the database.
147        """
148        self.with_db()
149
150        # create global values for known keys with the default
151        known_settings = self.get_all()
152        for setting, parameters in self.config_definition.items():
153            if setting in known_settings:
154                continue
155
156            self.db.log.debug(f"Creating setting: {setting} with default value {parameters.get('default', '')}")
157            self.set(setting, parameters.get("default", ""))
158
159        # make sure settings and user table are in sync
160        user_tags = list(set(itertools.chain(*[u["tags"] for u in self.db.fetchall("SELECT DISTINCT tags FROM users")])))
161        known_tags = [t["tag"] for t in self.db.fetchall("SELECT DISTINCT tag FROM settings")]
162        tag_order = self.get("flask.tag_order")
163
164        for tag in known_tags:
165            # add tags used by a setting to tag order
166            if tag and tag not in tag_order:
167                tag_order.append(tag)
168
169        for tag in user_tags:
170            # add tags used by a user to tag order
171            if tag and tag not in tag_order:
172                tag_order.append(tag)
173
174        # admin tag should always be first in order
175        if "admin" in tag_order:
176            tag_order.remove("admin")
177
178        tag_order.insert(0, "admin")
179
180        self.set("flask.tag_order", tag_order)
181        self.db.commit()

Ensure the database is in sync with the config definition

Deletes all stored settings not defined in 4CAT, and creates a global setting for all settings not yet in the database.

def get_all(self, is_json=False, user=None, tags=None): 
183    def get_all(self, is_json=False, user=None, tags=None):
184        """
185        Get all known settings
186
187        :param bool is_json:  if True, the value is returned as stored and not
188        interpreted as JSON if it comes from the database
189        :param user:  User object or name. Adds a tag `user:[username]` in
190        front of the tag list.
191        :param tags:  Tag or tags for the required setting. If a tag is
192        provided, the method checks if a special value for the setting exists
193        with the given tag, and returns that if one exists. First matching tag
194        wins.
195
196        :return dict: Setting value, as a dictionary with setting names as keys
197        and setting values as values.
198        """
199        return self.get(attribute_name=None, default=None, is_json=is_json, user=user, tags=tags)

Get all known settings

Parameters
  • bool is_json: if True, the value is returned as stored and not interpreted as JSON if it comes from the database
  • **user: User object or name. Adds a tag user**: [username] in front of the tag list.
  • tags: Tag or tags for the required setting. If a tag is provided, the method checks if a special value for the setting exists with the given tag, and returns that if one exists. First matching tag wins.
Returns

Setting value, as a dictionary with setting names as keys and setting values as values.

def get( self, attribute_name, default=None, is_json=False, user=None, tags=None): 
201    def get(self, attribute_name, default=None, is_json=False, user=None, tags=None):
202        """
203        Get a setting's value from the database
204
205        If the setting does not exist, the provided fallback value is returned.
206
207        :param str|list|None attribute_name:  Setting to return. If a string,
208        return that setting's value. If a list, return a dictionary of values.
209        If none, return a dictionary with all settings.
210        :param default:  Value to return if setting does not exist
211        :param bool is_json:  if True, the value is returned as stored and not
212        interpreted as JSON if it comes from the database
213        :param user:  User object or name. Adds a tag `user:[username]` in
214        front of the tag list.
215        :param tags:  Tag or tags for the required setting. If a tag is
216        provided, the method checks if a special value for the setting exists
217        with the given tag, and returns that if one exists. First matching tag
218        wins.
219
220        :return:  Setting value, or the provided fallback, or `None`.
221        """
222        # core settings are not from the database
223        if type(attribute_name) is str:
224            if attribute_name in self.core_settings:
225                return self.core_settings[attribute_name]
226            else:
227                attribute_name = (attribute_name,)
228        elif type(attribute_name) in (set, str):
229            attribute_name = tuple(attribute_name)
230
231        # if trying to access a setting that's not a core setting, attempt to
232        # initialise the database connection
233        if not self.db:
234            self.with_db()
235
236        # get tags to look for
237        tags = self.get_active_tags(user, tags)
238
239        # query database for any values within the required tags
240        tags.append("")  # empty tag = default value
241        if attribute_name:
242            query = "SELECT * FROM settings WHERE name IN %s AND tag IN %s"
243            replacements = (tuple(attribute_name), tuple(tags))
244        else:
245            query = "SELECT * FROM settings WHERE tag IN %s"
246            replacements = (tuple(tags), )
247
248        settings = {setting: {} for setting in attribute_name} if attribute_name else {}
249
250        for setting in self.db.fetchall(query, replacements):
251            if setting["name"] not in settings:
252                settings[setting["name"]] = {}
253
254            settings[setting["name"]][setting["tag"]] = setting["value"]
255
256        final_settings = {}
257        for setting_name, setting in settings.items():
258            # return first matching setting with a required tag, in the order the
259            # tags were provided
260            value = None
261            if setting:
262                for tag in tags:
263                    if tag in setting:
264                        value = setting[tag]
265                        break
266
267            # no matching tags? try empty tag
268            if value is None and "" in setting:
269                value = setting[""]
270
271            if not is_json and value is not None:
272                value = json.loads(value)
273            # TODO: Which default should have priority? The provided default feels like it should be the highest priority, but I think that is an old implementation and perhaps should be removed. - Dale
274            elif value is None and setting_name in self.config_definition and "default" in self.config_definition[setting_name]:
275                value = self.config_definition[setting_name]["default"]
276            elif value is None and default is not None:
277                value = default
278
279            final_settings[setting_name] = value
280
281        if attribute_name is not None and len(attribute_name) == 1:
282            # Single attribute requests; provide only the highest priority result
283            # this works because attribute_name is converted to a tuple (else already returned)
284            # if attribute_name is None, return all settings
285            # print(f"{user}: {attribute_name[0]} = {list(final_settings.values())[0]}")
286            return list(final_settings.values())[0]
287        else:
288            # All settings requested (via get_all)
289            return final_settings

Get a setting's value from the database

If the setting does not exist, the provided fallback value is returned.

Parameters
  • str|list|None attribute_name: Setting to return. If a string, return that setting's value. If a list, return a dictionary of values. If none, return a dictionary with all settings.
  • default: Value to return if setting does not exist
  • bool is_json: if True, the value is returned as stored and not interpreted as JSON if it comes from the database
  • **user: User object or name. Adds a tag user**: [username] in front of the tag list.
  • tags: Tag or tags for the required setting. If a tag is provided, the method checks if a special value for the setting exists with the given tag, and returns that if one exists. First matching tag wins.
Returns

Setting value, or the provided fallback, or None.

def get_active_tags(self, user=None, tags=None): 
291    def get_active_tags(self, user=None, tags=None):
292        """
293        Get active tags for given user/tag list
294
295        Used internally to harmonize tag setting for various methods, but can
296        also be called directly to verify tag activation.
297
298        :param user:  User object or name. Adds a tag `user:[username]` in
299        front of the tag list.
300        :param tags:  Tag or tags for the required setting. If a tag is
301        provided, the method checks if a special value for the setting exists
302        with the given tag, and returns that if one exists. First matching tag
303        wins.
304        :return list:  List of tags
305        """
306        # be flexible about the input types here
307        if tags is None:
308            tags = []
309        elif type(tags) is str:
310            tags = [tags]
311
312        # can provide either a string or user object
313        if type(user) is not str:
314            if hasattr(user, "get_id"):
315                user = user.get_id()
316            elif user is not None:
317                raise TypeError("get() expects None, a User object or a string for argument 'user'")
318
319        # user-specific settings are just a special type of tag (which takes
320        # precedence), same goes for user groups
321        if user:
322            user_tags = self.db.fetchone("SELECT tags FROM users WHERE name = %s", (user,))
323            if user_tags:
324                try:
325                    tags.extend(user_tags["tags"])
326                except (TypeError, ValueError):
327                    # should be a JSON list, but isn't
328                    pass
329
330            tags.insert(0, f"user:{user}")
331
332        return tags

Get active tags for given user/tag list

Used internally to harmonize tag setting for various methods, but can also be called directly to verify tag activation.

Parameters
  • **user: User object or name. Adds a tag user**: [username] in front of the tag list.
  • tags: Tag or tags for the required setting. If a tag is provided, the method checks if a special value for the setting exists with the given tag, and returns that if one exists. First matching tag wins.
Returns

List of tags

def set( self, attribute_name, value, is_json=False, tag='', overwrite_existing=True): 
334    def set(self, attribute_name, value, is_json=False, tag="", overwrite_existing=True):
335        """
336        Insert OR set value for a setting
337
338        If overwrite_existing=True and the setting exists, the setting is updated; if overwrite_existing=False and the
339        setting exists the setting is not updated.
340
341        :param str attribute_name:  Attribute to set
342        :param value:  Value to set (will be serialised as JSON)
343        :param bool is_json:  True for a value that is already a serialised JSON string; False if value is object that needs to
344                          be serialised into a JSON string
345        :param bool overwrite_existing: True will overwrite existing setting, False will do nothing if setting exists
346        :param str tag:  Tag to write setting for
347
348        :return int: number of updated rows
349        """
350        # Check value is valid JSON
351        if is_json:
352            try:
353                json.dumps(json.loads(value))
354            except json.JSONDecodeError:
355                return None
356        else:
357            try:
358                value = json.dumps(value)
359            except json.JSONDecodeError:
360                return None
361
362        if attribute_name in self.config_definition and self.config_definition.get(attribute_name).get("global"):
363            tag = ""
364
365        if overwrite_existing:
366            query = "INSERT INTO settings (name, value, tag) VALUES (%s, %s, %s) ON CONFLICT (name, tag) DO UPDATE SET value = EXCLUDED.value"
367        else:
368            query = "INSERT INTO settings (name, value, tag) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING"
369
370        self.db.execute(query, (attribute_name, value, tag))
371        updated_rows = self.db.cursor.rowcount
372        self.db.log.debug(f"Updated setting for {attribute_name}: {value} (tag: {tag})")
373
374        return updated_rows

Insert OR set value for a setting

If overwrite_existing=True and the setting exists, the setting is updated; if overwrite_existing=False and the setting exists the setting is not updated.

Parameters
  • str attribute_name: Attribute to set
  • value: Value to set (will be serialised as JSON)
  • bool is_json: True for a value that is already a serialised JSON string; False if value is object that needs to be serialised into a JSON string
  • bool overwrite_existing: True will overwrite existing setting, False will do nothing if setting exists
  • str tag: Tag to write setting for
Returns

number of updated rows

def delete_for_tag(self, attribute_name, tag): 
376    def delete_for_tag(self, attribute_name, tag):
377        """
378        Delete config override for a given tag
379
380        :param str attribute_name:
381        :param str tag:
382        :return int: number of deleted rows
383        """
384        self.db.delete("settings", where={"name": attribute_name, "tag": tag})
385        updated_rows = self.db.cursor.rowcount
386
387        return updated_rows

Delete config override for a given tag

Parameters
  • str attribute_name:
  • str tag:
Returns

number of deleted rows

class ConfigWrapper: 
406class ConfigWrapper:
407    """
408    Wrapper for the config manager
409
410    Allows setting a default set of tags or user, so that all subsequent calls
411    to `get()` are done for those tags or that user. Can also adjust tags based
412    on the HTTP request, if used in a Flask context.
413    """
414    def __init__(self, config, user=None, tags=None, request=None):
415        """
416        Initialise config wrapper
417
418        :param ConfigManager config:  Initialised config manager
419        :param user:  User to get settings for
420        :param tags:  Tags to get settings for
421        :param request:  Request to get headers from. This can be used to set
422        a particular tag based on the HTTP headers of the request, e.g. to
423        serve 4CAT with a different configuration based on the proxy server
424        used.
425        """
426        self.config = config
427        self.user = user
428        self.tags = tags
429        self.request = request
430
431        # this ensures the user object in turn reads from the wrapper
432        if self.user:
433            self.user.with_config(self)
434
435
436    def set(self, *args, **kwargs):
437        """
438        Wrap `set()`
439
440        :param args:
441        :param kwargs:
442        :return:
443        """
444        if "tag" not in kwargs and self.tags:
445            tag = self.tags if type(self.tags) is str else self.tags[0]
446            kwargs["tag"] = self.tags
447
448        return self.config.set(*args, **kwargs)
449
450    def get_all(self, *args, **kwargs):
451        """
452        Wrap `get_all()`
453
454        Takes the `user`, `tags` and `request` given when initialised into
455        account. If `tags` is set explicitly, the HTTP header-based override
456        is not applied.
457
458        :param args:
459        :param kwargs:
460        :return:
461        """
462        if "user" not in kwargs and self.user:
463            kwargs["user"] = self.user
464
465        if "tags" not in kwargs:
466            kwargs["tags"] = self.tags if self.tags else []
467            kwargs["tags"] = self.request_override(kwargs["tags"])
468
469        return self.config.get_all(*args, **kwargs)
470
471    def get(self, *args, **kwargs):
472        """
473        Wrap `get()`
474
475        Takes the `user`, `tags` and `request` given when initialised into
476        account. If `tags` is set explicitly, the HTTP header-based override
477        is not applied.
478
479        :param args:
480        :param kwargs:
481        :return:
482        """
483        if "user" not in kwargs:
484            kwargs["user"] = self.user
485
486        if "tags" not in kwargs:
487            kwargs["tags"] = self.tags if self.tags else []
488            kwargs["tags"] = self.request_override(kwargs["tags"])
489
490        return self.config.get(*args, **kwargs)
491
492    def get_active_tags(self, user=None, tags=None):
493        """
494        Wrap `get_active_tags()`
495
496        Takes the `user`, `tags` and `request` given when initialised into
497        account. If `tags` is set explicitly, the HTTP header-based override
498        is not applied.
499
500        :param user:
501        :param tags:
502        :return list:
503        """
504        active_tags = self.config.get_active_tags(user, tags)
505        if not tags:
506            active_tags = self.request_override(active_tags)
507
508        return active_tags
509
510    def request_override(self, tags):
511        """
512        Force tag via HTTP request headers
513
514        To facilitate loading different configurations based on the HTTP
515        request, the request object can be passed to the ConfigWrapper and
516        if a certain request header is set, the value of that header will be
517        added to the list of tags to consider when retrieving settings.
518
519        See the flask.proxy_secret config setting; this is used to prevent
520        users from changing configuration by forging the header.
521
522        :param list|str tags:  List of tags to extend based on request
523        :return list:  Amended list of tags
524        """
525        if type(tags) is str:
526            tags = [tags]
527
528        if self.request and self.request.headers.get("X-4Cat-Config-Tag") and \
529            self.config.get("flask.proxy_secret") and \
530            self.request.headers.get("X-4Cat-Config-Via-Proxy") == self.config.get("flask.proxy_secret"):
531            # need to ensure not just anyone can add this header to their
532            # request!
533            # to this end, the second header must be set to the secret value;
534            # if it is not set, assume the headers are not being configured by
535            # the proxy server
536            if not tags:
537                tags = []
538
539            # can never set admin tag via headers (should always be user-based)
540            forbidden_overrides = ("admin",)
541            tags += [tag for tag in self.request.headers.get("X-4Cat-Config-Tag").split(",") if tag not in forbidden_overrides]
542
543        return tags
544
545    def __getattr__(self, item):
546        """
547        Generic wrapper
548
549        Just pipe everything through to the config object
550
551        :param item:
552        :return:
553        """
554        if hasattr(self.config, item):
555            return getattr(self.config, item)
556        elif hasattr(self, item):
557            return getattr(self, item)
558        else:
559            raise AttributeError(f"'{self.__name__}' object has no attribute '{item}'")

Wrapper for the config manager

Allows setting a default set of tags or user, so that all subsequent calls to get() are done for those tags or that user. Can also adjust tags based on the HTTP request, if used in a Flask context.

ConfigWrapper(config, user=None, tags=None, request=None) 
414    def __init__(self, config, user=None, tags=None, request=None):
415        """
416        Initialise config wrapper
417
418        :param ConfigManager config:  Initialised config manager
419        :param user:  User to get settings for
420        :param tags:  Tags to get settings for
421        :param request:  Request to get headers from. This can be used to set
422        a particular tag based on the HTTP headers of the request, e.g. to
423        serve 4CAT with a different configuration based on the proxy server
424        used.
425        """
426        self.config = config
427        self.user = user
428        self.tags = tags
429        self.request = request
430
431        # this ensures the user object in turn reads from the wrapper
432        if self.user:
433            self.user.with_config(self)

Initialise config wrapper

Parameters
  • ConfigManager config: Initialised config manager
  • user: User to get settings for
  • tags: Tags to get settings for
  • request: Request to get headers from. This can be used to set a particular tag based on the HTTP headers of the request, e.g. to serve 4CAT with a different configuration based on the proxy server used.
config
user
tags
request
def set(self, *args, **kwargs): 
436    def set(self, *args, **kwargs):
437        """
438        Wrap `set()`
439
440        :param args:
441        :param kwargs:
442        :return:
443        """
444        if "tag" not in kwargs and self.tags:
445            tag = self.tags if type(self.tags) is str else self.tags[0]
446            kwargs["tag"] = self.tags
447
448        return self.config.set(*args, **kwargs)

Wrap set()

Parameters
  • args:
  • kwargs:
Returns
def get_all(self, *args, **kwargs): 
450    def get_all(self, *args, **kwargs):
451        """
452        Wrap `get_all()`
453
454        Takes the `user`, `tags` and `request` given when initialised into
455        account. If `tags` is set explicitly, the HTTP header-based override
456        is not applied.
457
458        :param args:
459        :param kwargs:
460        :return:
461        """
462        if "user" not in kwargs and self.user:
463            kwargs["user"] = self.user
464
465        if "tags" not in kwargs:
466            kwargs["tags"] = self.tags if self.tags else []
467            kwargs["tags"] = self.request_override(kwargs["tags"])
468
469        return self.config.get_all(*args, **kwargs)

Wrap get_all()

Takes the user, tags and request given when initialised into account. If tags is set explicitly, the HTTP header-based override is not applied.

Parameters
  • args:
  • kwargs:
Returns
def get(self, *args, **kwargs): 
471    def get(self, *args, **kwargs):
472        """
473        Wrap `get()`
474
475        Takes the `user`, `tags` and `request` given when initialised into
476        account. If `tags` is set explicitly, the HTTP header-based override
477        is not applied.
478
479        :param args:
480        :param kwargs:
481        :return:
482        """
483        if "user" not in kwargs:
484            kwargs["user"] = self.user
485
486        if "tags" not in kwargs:
487            kwargs["tags"] = self.tags if self.tags else []
488            kwargs["tags"] = self.request_override(kwargs["tags"])
489
490        return self.config.get(*args, **kwargs)

Wrap get()

Takes the user, tags and request given when initialised into account. If tags is set explicitly, the HTTP header-based override is not applied.

Parameters
  • args:
  • kwargs:
Returns
def get_active_tags(self, user=None, tags=None): 
492    def get_active_tags(self, user=None, tags=None):
493        """
494        Wrap `get_active_tags()`
495
496        Takes the `user`, `tags` and `request` given when initialised into
497        account. If `tags` is set explicitly, the HTTP header-based override
498        is not applied.
499
500        :param user:
501        :param tags:
502        :return list:
503        """
504        active_tags = self.config.get_active_tags(user, tags)
505        if not tags:
506            active_tags = self.request_override(active_tags)
507
508        return active_tags

Wrap get_active_tags()

Takes the user, tags and request given when initialised into account. If tags is set explicitly, the HTTP header-based override is not applied.

Parameters
  • user:
  • tags:
Returns
def request_override(self, tags): 
510    def request_override(self, tags):
511        """
512        Force tag via HTTP request headers
513
514        To facilitate loading different configurations based on the HTTP
515        request, the request object can be passed to the ConfigWrapper and
516        if a certain request header is set, the value of that header will be
517        added to the list of tags to consider when retrieving settings.
518
519        See the flask.proxy_secret config setting; this is used to prevent
520        users from changing configuration by forging the header.
521
522        :param list|str tags:  List of tags to extend based on request
523        :return list:  Amended list of tags
524        """
525        if type(tags) is str:
526            tags = [tags]
527
528        if self.request and self.request.headers.get("X-4Cat-Config-Tag") and \
529            self.config.get("flask.proxy_secret") and \
530            self.request.headers.get("X-4Cat-Config-Via-Proxy") == self.config.get("flask.proxy_secret"):
531            # need to ensure not just anyone can add this header to their
532            # request!
533            # to this end, the second header must be set to the secret value;
534            # if it is not set, assume the headers are not being configured by
535            # the proxy server
536            if not tags:
537                tags = []
538
539            # can never set admin tag via headers (should always be user-based)
540            forbidden_overrides = ("admin",)
541            tags += [tag for tag in self.request.headers.get("X-4Cat-Config-Tag").split(",") if tag not in forbidden_overrides]
542
543        return tags

Force tag via HTTP request headers

To facilitate loading different configurations based on the HTTP request, the request object can be passed to the ConfigWrapper and if a certain request header is set, the value of that header will be added to the list of tags to consider when retrieving settings.

See the flask.proxy_secret config setting; this is used to prevent users from changing configuration by forging the header.

Parameters
  • list|str tags: List of tags to extend based on request
Returns

Amended list of tags

class ConfigDummy: 
561class ConfigDummy:
562    """
563    Dummy class to use as initial value for class-based configs
564
565    The config manager in processor objects takes the owner of the dataset of
566    the processor into account. This is only available after the object has
567    been inititated, so until then use this dummy wrapper that throws an error
568    when used to access config variables
569    """
570    def __getattribute__(self, item):
571        """
572        Access class attribute
573
574        :param item:
575        :raises NotImplementedError:
576        """
577        raise NotImplementedError("Cannot call processor config object in a class or static method - call global "
578                                  "configuration manager instead.")

Dummy class to use as initial value for class-based configs

The config manager in processor objects takes the owner of the dataset of the processor into account. This is only available after the object has been inititated, so until then use this dummy wrapper that throws an error when used to access config variables

config = <ConfigManager object>