common.lib.user_input

  1from dateutil.parser import parse as parse_datetime
  2from common.lib.exceptions import QueryParametersException
  3from werkzeug.datastructures import ImmutableMultiDict
  4import json
  5
  6import re
  7
  8class RequirementsNotMetException(Exception):
  9    """
 10    If this is raised while parsing, that option is not included in the parsed
 11    output. Used with the "requires" option setting.
 12    """
 13    pass
 14
 15class UserInput:
 16    """
 17    Class for handling user input
 18
 19    It is important to sanitise user input, as carelessly entered parameters
 20    may in e.g. requesting far more data than needed, or lead to undefined
 21    behaviour. This class offers a set of pre-defined value types that can be
 22    consistently rendered as form elements in an interface and parsed.
 23    """
 24    OPTION_TOGGLE = "toggle"  # boolean toggle (checkbox)
 25    OPTION_CHOICE = "choice"  # one choice out of a list (select)
 26    OPTION_TEXT = "string"  # simple string or integer (input text)
 27    OPTION_MULTI = "multi"  # multiple values out of a list (select multiple)
 28    OPTION_MULTI_SELECT = "multi_select"  # multiple values out of a dropdown list (select multiple)
 29    OPTION_INFO = "info"  # just a bit of text, not actual input
 30    OPTION_TEXT_LARGE = "textarea"  # longer text
 31    OPTION_TEXT_JSON = "json"  # text, but should be valid JSON
 32    OPTION_DATE = "date"  # a single date
 33    OPTION_DATERANGE = "daterange"  # a beginning and end date
 34    OPTION_DIVIDER = "divider"  # meta-option, divides related sets of options
 35    OPTION_FILE = "file"  # file upload
 36    OPTION_HUE = "hue"  # colour hue
 37    OPTION_DATASOURCES = "datasources"  # data source toggling
 38    OPTION_DATASOURCES_TABLE = "datasources_table"  # a table with settings per data source
 39    OPTION_ANNOTATION = "annotation"  # checkbox for whether to an annotation
 40    OPTION_ANNOTATIONS = "annotations"  # table for whether to write multiple annotations
 41
 42    OPTIONS_COSMETIC = (OPTION_INFO, OPTION_DIVIDER)
 43
 44    @staticmethod
 45    def parse_all(options, input, silently_correct=True):
 46        """
 47        Parse form input for the provided options
 48
 49        Ignores all input not belonging to any of the defined options: parses
 50        and sanitises the rest, and returns a dictionary with the sanitised
 51        options. If an option is *not* present in the input, the default value
 52        is used, and if that is absent, `None`.
 53
 54        In other words, this ensures a dictionary with 1) only white-listed
 55        keys, 2) a value of an expected type for each key.
 56
 57        :param dict options:  Options, as a name -> settings dictionary
 58        :param dict input:  Input, as a form field -> value dictionary
 59        :param bool silently_correct:  If true, replace invalid values with the
 60        given default value; else, raise a QueryParametersException if a value
 61        is invalid.
 62
 63        :return dict:  Sanitised form input
 64        """
 65
 66        from common.lib.helpers import convert_to_int
 67        parsed_input = {}
 68
 69        if type(input) is not dict and type(input) is not ImmutableMultiDict:
 70            raise TypeError("input must be a dictionary or ImmutableMultiDict")
 71
 72        if type(input) is ImmutableMultiDict:
 73            # we are not using to_dict, because that messes up multi-selects
 74            input = {key: input.getlist(key) for key in input}
 75            for key, value in input.items():
 76                if type(value) is list and len(value) == 1:
 77                    input[key] = value[0]
 78
 79        # all parameters are submitted as option-[parameter ID], this is an 
 80        # artifact of how the web interface works and we can simply remove the
 81        # prefix
 82        input = {re.sub(r"^option-", "", field): input[field] for field in input}
 83
 84        # re-order input so that the fields relying on the value of other
 85        # fields are parsed last
 86        options = {k: options[k] for k in sorted(options, key=lambda k: options[k].get("requires") is not None)}
 87
 88        for option, settings in options.items():
 89            if settings.get("indirect"):
 90                # these are settings that are derived from and set by other
 91                # settings
 92                continue
 93
 94            if settings.get("type") in UserInput.OPTIONS_COSMETIC:
 95                # these are structural form elements and never have a value
 96                continue
 97
 98            elif settings.get("type") == UserInput.OPTION_DATERANGE:
 99                # special case, since it combines two inputs
100                option_min = option + "-min"
101                option_max = option + "-max"
102
103                # normally this is taken care of client-side, but in case this
104                # didn't work, try to salvage it server-side
105                if option_min not in input or input.get(option_min) == "-1":
106                    option_min += "_proxy"
107
108                if option_max not in input or input.get(option_max) == "-1":
109                    option_max += "_proxy"
110
111                # save as a tuple of unix timestamps (or None)
112                try:
113                    after, before = (UserInput.parse_value(settings, input.get(option_min), parsed_input, silently_correct), UserInput.parse_value(settings, input.get(option_max), parsed_input, silently_correct))
114
115                    if before and after and after > before:
116                        if not silently_correct:
117                            raise QueryParametersException("End of date range must be after beginning of date range.")
118                        else:
119                            before = after
120
121                    parsed_input[option] = (after, before)
122                except RequirementsNotMetException:
123                    pass
124
125            elif settings.get("type") in (UserInput.OPTION_TOGGLE, UserInput.OPTION_ANNOTATION):
126                # special case too, since if a checkbox is unchecked, it simply
127                # does not show up in the input
128                try:
129                    if option in input:
130                        # Toggle needs to be parsed
131                        parsed_input[option] = UserInput.parse_value(settings, input[option], parsed_input, silently_correct)
132                    else:
133                        # Toggle was left blank
134                        parsed_input[option] = False
135                except RequirementsNotMetException:
136                    pass
137
138            elif settings.get("type") == UserInput.OPTION_DATASOURCES:
139                # special case, because this combines multiple inputs to
140                # configure data source availability and expiration
141                datasources = {datasource: {
142                    "enabled": f"{option}-enable-{datasource}" in input,
143                    "allow_optout": f"{option}-optout-{datasource}" in input,
144                    "timeout": convert_to_int(input[f"{option}-timeout-{datasource}"], 0)
145                } for datasource in input[option].split(",")}
146
147                parsed_input[option] = [datasource for datasource, v in datasources.items() if v["enabled"]]
148                parsed_input[option.split(".")[0] + ".expiration"] = datasources
149
150            elif settings.get("type") == UserInput.OPTION_DATASOURCES_TABLE:
151                # special case, parse table values to generate a dict
152                columns = list(settings["columns"].keys())
153                table_input = {}
154
155                for datasource in list(settings["default"].keys()):
156                    table_input[datasource] = {}
157                    for column in columns:
158
159                        choice = input.get(option + "-" + datasource + "-" + column, False)
160                        column_settings = settings["columns"][column]  # sub-settings per column
161                        table_input[datasource][column] = UserInput.parse_value(column_settings, choice, table_input, silently_correct=True)
162
163                parsed_input[option] = table_input
164
165            elif option not in input:
166                # not provided? use default
167                parsed_input[option] = settings.get("default", None)
168
169            else:
170                # normal parsing and sanitisation
171                try:
172                    parsed_input[option] = UserInput.parse_value(settings, input[option], parsed_input, silently_correct)
173                except RequirementsNotMetException:
174                    pass
175
176        return parsed_input
177
178    @staticmethod
179    def parse_value(settings, choice, other_input=None, silently_correct=True):
180        """
181        Filter user input
182
183        Makes sure user input for post-processors is valid and within the
184        parameters specified by the post-processor
185
186        :param obj settings:  Settings, including defaults and valid options
187        :param choice:  The chosen option, to be parsed
188        :param dict other_input:  Other input, as parsed so far
189        :param bool silently_correct:  If true, replace invalid values with the
190        given default value; else, raise a QueryParametersException if a value
191        is invalid.
192
193        :return:  Validated and parsed input
194        """
195        # short-circuit if there is a requirement for the field to be parsed
196        # and the requirement isn't met
197        if settings.get("requires"):
198            try:
199                field, operator, value = re.findall(r"([a-zA-Z0-9_-]+)([!=$~^]+)(.*)", settings.get("requires"))[0]
200            except IndexError:
201                # invalid condition, interpret as 'does the field with this name have a value'
202                field, operator, value = (choice, "!=", "")
203
204            if field not in other_input:
205                raise RequirementsNotMetException()
206
207            other_value = other_input.get(field)
208            if type(other_value) is bool:
209                # evalues to a boolean, i.e. checkboxes etc
210                if operator == "!=":
211                    if (other_value and value in ("", "false")) or (not other_value and value in ("true", "checked")):
212                        raise RequirementsNotMetException()
213                else:
214                    if (other_value and value not in ("true", "checked")) or (not other_value and value not in ("", "false")):
215                        raise RequirementsNotMetException()
216
217            else:
218                if type(other_value) in (tuple, list):
219                # iterables are a bit special
220                    if len(other_value) == 1:
221                        # treat one-item lists as "normal" values
222                        other_value = other_value[0]
223                    elif operator == "~=":  # interpret as 'is in list?'
224                        if value not in other_value:
225                            raise RequirementsNotMetException()
226                    else:
227                        # condition doesn't make sense for a list, so assume it's not True
228                        raise RequirementsNotMetException()
229
230                if operator == "^=" and not str(other_value).startswith(value):
231                    raise RequirementsNotMetException()
232                elif operator == "$=" and not str(other_value).endswith(value):
233                    raise RequirementsNotMetException()
234                elif operator == "~=" and value not in str(other_value):
235                    raise RequirementsNotMetException()
236                elif operator == "!=" and value == other_value:
237                    raise RequirementsNotMetException()
238                elif operator in ("==", "=") and value != other_value:
239                    raise RequirementsNotMetException()
240
241        input_type = settings.get("type", "")
242        if input_type in UserInput.OPTIONS_COSMETIC:
243            # these are structural form elements and can never return a value
244            return None
245
246        elif input_type in (UserInput.OPTION_TOGGLE, UserInput.OPTION_ANNOTATION):
247            # simple boolean toggle
248            if type(choice) is bool:
249                return choice
250            elif choice in ['false', 'False']:
251                # Sanitized options passed back to Flask can be converted to strings as 'false'
252                return False
253            elif choice in ['true', 'True', 'on']:
254                # Toggle will have value 'on', but may also becomes a string 'true'
255                return True
256            else:
257                raise QueryParametersException("Toggle invalid input")
258
259        elif input_type in (UserInput.OPTION_DATE, UserInput.OPTION_DATERANGE):
260            # parse either integers (unix timestamps) or try to guess the date
261            # format (the latter may be used for input if JavaScript is turned
262            # off in the front-end and the input comes from there)
263            value = None
264            try:
265                value = int(choice)
266            except ValueError:
267                parsed_choice = parse_datetime(choice)
268                value = int(parsed_choice.timestamp())
269            finally:
270                return value
271
272        elif input_type in (UserInput.OPTION_MULTI, UserInput.OPTION_ANNOTATIONS):
273            # any number of values out of a list of possible values
274            # comma-separated during input, returned as a list of valid options
275            if not choice:
276                return settings.get("default", [])
277
278            chosen = choice.split(",")
279            return [item for item in chosen if item in settings.get("options", [])]
280
281        elif input_type == UserInput.OPTION_MULTI_SELECT:
282            # multiple number of values out of a dropdown list of possible values
283            # comma-separated during input, returned as a list of valid options
284            if not choice:
285                return settings.get("default", [])
286
287            if type(choice) is str:
288                # should be a list if the form control was actually a multiselect
289                # but we have some client side UI helpers that may produce a string
290                # instead
291                choice = choice.split(",")
292
293            return [item for item in choice if item in settings.get("options", [])]
294
295        elif input_type == UserInput.OPTION_CHOICE:
296            # select box
297            # one out of multiple options
298            # return option if valid, or default
299            if choice not in settings.get("options"):
300                if not silently_correct:
301                    raise QueryParametersException(f"Invalid value selected; must be one of {', '.join(settings.get('options', {}).keys())}. {settings}")
302                else:
303                    return settings.get("default", "")
304            else:
305                return choice
306
307        elif input_type == UserInput.OPTION_TEXT_JSON:
308            # verify that this is actually json
309            try:
310                json.dumps(json.loads(choice))
311            except json.JSONDecodeError:
312                raise QueryParametersException("Invalid JSON value '%s'" % choice)
313
314            return json.loads(choice)
315
316        elif input_type in (UserInput.OPTION_TEXT, UserInput.OPTION_TEXT_LARGE, UserInput.OPTION_HUE):
317            # text string
318            # optionally clamp it as an integer; return default if not a valid
319            # integer (or float; inferred from default or made explicit via the
320            # coerce_type setting)
321            if settings.get("coerce_type"):
322                value_type = settings["coerce_type"]
323            else:
324                value_type = type(settings.get("default"))
325                if value_type not in (int, float):
326                    value_type = int
327
328            if "max" in settings:
329                try:
330                    choice = min(settings["max"], value_type(choice))
331                except (ValueError, TypeError):
332                    if not silently_correct:
333                        raise QueryParametersException("Provide a value of %s or lower." % str(settings["max"]))
334
335                    choice = settings.get("default")
336
337            if "min" in settings:
338                try:
339                    choice = max(settings["min"], value_type(choice))
340                except (ValueError, TypeError):
341                    if not silently_correct:
342                        raise QueryParametersException("Provide a value of %s or more." % str(settings["min"]))
343
344                    choice = settings.get("default")
345
346            if choice is None or choice == "":
347                choice = settings.get("default")
348
349            if choice is None:
350                choice = 0 if "min" in settings or "max" in settings else ""
351
352            if settings.get("coerce_type"):
353                try:
354                    return value_type(choice)
355                except (ValueError, TypeError):
356                    return settings.get("default")
357            else:
358                return choice
359
360        else:
361            # no filtering
362            return choice