common.lib.user_input
1from dateutil.parser import parse as parse_datetime 2from common.lib.exceptions import QueryParametersException 3from werkzeug.datastructures import ImmutableMultiDict 4import json 5 6import re 7 8class RequirementsNotMetException(Exception): 9 """ 10 If this is raised while parsing, that option is not included in the parsed 11 output. Used with the "requires" option setting. 12 """ 13 pass 14 15class UserInput: 16 """ 17 Class for handling user input 18 19 It is important to sanitise user input, as carelessly entered parameters 20 may in e.g. requesting far more data than needed, or lead to undefined 21 behaviour. This class offers a set of pre-defined value types that can be 22 consistently rendered as form elements in an interface and parsed. 23 """ 24 OPTION_TOGGLE = "toggle" # boolean toggle (checkbox) 25 OPTION_CHOICE = "choice" # one choice out of a list (select) 26 OPTION_TEXT = "string" # simple string or integer (input text) 27 OPTION_MULTI = "multi" # multiple values out of a list (select multiple) 28 OPTION_MULTI_SELECT = "multi_select" # multiple values out of a dropdown list (select multiple) 29 OPTION_INFO = "info" # just a bit of text, not actual input 30 OPTION_TEXT_LARGE = "textarea" # longer text 31 OPTION_TEXT_JSON = "json" # text, but should be valid JSON 32 OPTION_DATE = "date" # a single date 33 OPTION_DATERANGE = "daterange" # a beginning and end date 34 OPTION_DIVIDER = "divider" # meta-option, divides related sets of options 35 OPTION_FILE = "file" # file upload 36 OPTION_HUE = "hue" # colour hue 37 OPTION_DATASOURCES = "datasources" # data source toggling 38 OPTION_EXTENSIONS = "extensions" # extension toggling 39 OPTION_DATASOURCES_TABLE = "datasources_table" # a table with settings per data source 40 OPTION_ANNOTATION = "annotation" # checkbox for whether to an annotation 41 OPTION_ANNOTATIONS = "annotations" # table for whether to write multiple annotations 42 43 OPTIONS_COSMETIC = (OPTION_INFO, OPTION_DIVIDER) 44 45 @staticmethod 46 def parse_all(options, input, silently_correct=True): 47 """ 48 Parse form input for the provided options 49 50 Ignores all input not belonging to any of the defined options: parses 51 and sanitises the rest, and returns a dictionary with the sanitised 52 options. If an option is *not* present in the input, the default value 53 is used, and if that is absent, `None`. 54 55 In other words, this ensures a dictionary with 1) only white-listed 56 keys, 2) a value of an expected type for each key. 57 58 :param dict options: Options, as a name -> settings dictionary 59 :param dict input: Input, as a form field -> value dictionary 60 :param bool silently_correct: If true, replace invalid values with the 61 given default value; else, raise a QueryParametersException if a value 62 is invalid. 63 64 :return dict: Sanitised form input 65 """ 66 67 from common.lib.helpers import convert_to_int 68 parsed_input = {} 69 70 if type(input) is not dict and type(input) is not ImmutableMultiDict: 71 raise TypeError("input must be a dictionary or ImmutableMultiDict") 72 73 if type(input) is ImmutableMultiDict: 74 # we are not using to_dict, because that messes up multi-selects 75 input = {key: input.getlist(key) for key in input} 76 for key, value in input.items(): 77 if type(value) is list and len(value) == 1: 78 input[key] = value[0] 79 80 # all parameters are submitted as option-[parameter ID], this is an 81 # artifact of how the web interface works and we can simply remove the 82 # prefix 83 input = {re.sub(r"^option-", "", field): input[field] for field in input} 84 85 # fields can be 'delegated', i.e. they only show up under some condition 86 # or in a later stage of form input. here we determine if input was 87 # actually filled in or was only defined but never delegated 88 never_delegated = set([option for option in options if options[option].get("delegated")]) 89 never_delegated -= set(input.keys()) 90 91 # re-order input so that the fields relying on the value of other 92 # fields are parsed last 93 options = {k: options[k] for k in sorted(options, key=lambda k: options[k].get("requires") is not None)} 94 95 for option, settings in options.items(): 96 if settings.get("indirect"): 97 # these are settings that are derived from and set by other 98 # settings 99 continue 100 101 if settings.get("type") in UserInput.OPTIONS_COSMETIC: 102 # these are structural form elements and never have a value 103 continue 104 105 if option in never_delegated: 106 # these options were never actually part of the input because 107 # the required conditions were never met, so they can be 108 # ignored 109 continue 110 111 elif settings.get("type") == UserInput.OPTION_DATERANGE: 112 # special case, since it combines two inputs 113 option_min = option + "-min" 114 option_max = option + "-max" 115 116 # normally this is taken care of client-side, but in case this 117 # didn't work, try to salvage it server-side 118 if option_min not in input or input.get(option_min) == "-1": 119 option_min += "_proxy" 120 121 if option_max not in input or input.get(option_max) == "-1": 122 option_max += "_proxy" 123 124 # save as a tuple of unix timestamps (or None) 125 try: 126 after, before = (UserInput.parse_value(settings, input.get(option_min), parsed_input, silently_correct), UserInput.parse_value(settings, input.get(option_max), parsed_input, silently_correct)) 127 128 if before and after and after > before: 129 if not silently_correct: 130 raise QueryParametersException("End of date range must be after beginning of date range.") 131 else: 132 before = after 133 134 parsed_input[option] = (after, before) 135 except RequirementsNotMetException: 136 pass 137 138 elif settings.get("type") in (UserInput.OPTION_TOGGLE, UserInput.OPTION_ANNOTATION): 139 # special case too, since if a checkbox is unchecked, it simply 140 # does not show up in the input 141 try: 142 if option in input: 143 # Toggle needs to be parsed 144 parsed_input[option] = UserInput.parse_value(settings, input[option], parsed_input, silently_correct) 145 else: 146 # Toggle was left blank 147 parsed_input[option] = False 148 except RequirementsNotMetException: 149 pass 150 151 elif settings.get("type") == UserInput.OPTION_DATASOURCES: 152 # special case, because this combines multiple inputs to 153 # configure data source availability and expiration 154 datasources = {datasource: { 155 "enabled": f"{option}-enable-{datasource}" in input, 156 "allow_optout": f"{option}-optout-{datasource}" in input, 157 "timeout": convert_to_int(input[f"{option}-timeout-{datasource}"], 0) 158 } for datasource in input[option].split(",")} 159 160 parsed_input[option] = [datasource for datasource, v in datasources.items() if v["enabled"]] 161 parsed_input[option.split(".")[0] + ".expiration"] = datasources 162 163 elif settings.get("type") == UserInput.OPTION_EXTENSIONS: 164 # also a special case 165 parsed_input[option] = {extension: { 166 "enabled": f"{option}-enable-{extension}" in input 167 } for extension in input[option].split(",")} 168 169 elif settings.get("type") == UserInput.OPTION_DATASOURCES_TABLE: 170 # special case, parse table values to generate a dict 171 columns = list(settings["columns"].keys()) 172 table_input = {} 173 174 for datasource in list(settings["default"].keys()): 175 table_input[datasource] = {} 176 for column in columns: 177 178 choice = input.get(option + "-" + datasource + "-" + column, False) 179 column_settings = settings["columns"][column] # sub-settings per column 180 table_input[datasource][column] = UserInput.parse_value(column_settings, choice, table_input, silently_correct=True) 181 182 parsed_input[option] = table_input 183 184 elif option not in input: 185 # not provided? use default 186 parsed_input[option] = settings.get("default", None) 187 188 else: 189 # normal parsing and sanitisation 190 try: 191 parsed_input[option] = UserInput.parse_value(settings, input[option], parsed_input, silently_correct) 192 except RequirementsNotMetException: 193 pass 194 195 return parsed_input 196 197 @staticmethod 198 def parse_value(settings, choice, other_input=None, silently_correct=True): 199 """ 200 Filter user input 201 202 Makes sure user input for post-processors is valid and within the 203 parameters specified by the post-processor 204 205 :param obj settings: Settings, including defaults and valid options 206 :param choice: The chosen option, to be parsed 207 :param dict other_input: Other input, as parsed so far 208 :param bool silently_correct: If true, replace invalid values with the 209 given default value; else, raise a QueryParametersException if a value 210 is invalid. 211 212 :return: Validated and parsed input 213 """ 214 # short-circuit if there is a requirement for the field to be parsed 215 # and the requirement isn't met 216 if settings.get("requires"): 217 try: 218 field, operator, value = re.findall(r"([a-zA-Z0-9_-]+)([!=$~^]+)(.*)", settings.get("requires"))[0] 219 except IndexError: 220 # invalid condition, interpret as 'does the field with this name have a value' 221 field, operator, value = (choice, "!=", "") 222 223 if field not in other_input: 224 raise RequirementsNotMetException() 225 226 negated = operator.startswith("!") 227 if negated: 228 operator = operator[1:] 229 230 other_value = other_input.get(field) 231 if type(other_value) is bool: 232 # evalues to a boolean, i.e. checkboxes etc 233 if operator in ("==", "="): 234 if ((other_value and value in ("", "false")) or (not other_value and value in ("true", "checked"))) != negated: 235 raise RequirementsNotMetException() 236 else: 237 if ((other_value and value not in ("true", "checked")) or (not other_value and value not in ("", "false"))) != negated: 238 raise RequirementsNotMetException() 239 240 else: 241 if type(other_value) in (tuple, list): 242 # iterables are a bit special 243 if len(other_value) == 1: 244 # treat one-item lists as "normal" values 245 other_value = other_value[0] 246 elif operator == "~=": # interpret as 'is in list?' 247 if (value not in other_value) != negated: 248 raise RequirementsNotMetException() 249 elif not negated: 250 # condition doesn't make sense for a list, so assume it's not True 251 raise RequirementsNotMetException() 252 253 if operator == "^=" and str(other_value).startswith(value) == negated: 254 raise RequirementsNotMetException() 255 elif operator == "$=" and str(other_value).endswith(value) == negated: 256 raise RequirementsNotMetException() 257 elif operator == "~=" and (value in str(other_value)) == negated: 258 raise RequirementsNotMetException() 259 elif operator in ("==", "=") and (value == other_value) == negated: 260 raise RequirementsNotMetException() 261 262 input_type = settings.get("type", "") 263 if input_type in UserInput.OPTIONS_COSMETIC: 264 # these are structural form elements and can never return a value 265 return None 266 267 elif input_type in (UserInput.OPTION_TOGGLE, UserInput.OPTION_ANNOTATION): 268 # simple boolean toggle 269 if type(choice) is bool: 270 return choice 271 elif choice in ['false', 'False']: 272 # Sanitized options passed back to Flask can be converted to strings as 'false' 273 return False 274 elif choice in ['true', 'True', 'on']: 275 # Toggle will have value 'on', but may also becomes a string 'true' 276 return True 277 else: 278 raise QueryParametersException("Toggle invalid input") 279 280 elif input_type in (UserInput.OPTION_DATE, UserInput.OPTION_DATERANGE): 281 # parse either integers (unix timestamps) or try to guess the date 282 # format (the latter may be used for input if JavaScript is turned 283 # off in the front-end and the input comes from there) 284 value = None 285 try: 286 value = int(choice) 287 except ValueError: 288 parsed_choice = parse_datetime(choice) 289 value = int(parsed_choice.timestamp()) 290 finally: 291 return value 292 293 elif input_type in (UserInput.OPTION_MULTI, UserInput.OPTION_ANNOTATIONS): 294 # any number of values out of a list of possible values 295 # comma-separated during input, returned as a list of valid options 296 if not choice: 297 return settings.get("default", []) 298 299 chosen = choice.split(",") 300 return [item for item in chosen if item in settings.get("options", [])] 301 302 elif input_type == UserInput.OPTION_MULTI_SELECT: 303 # multiple number of values out of a dropdown list of possible values 304 # comma-separated during input, returned as a list of valid options 305 if not choice: 306 return settings.get("default", []) 307 308 if type(choice) is str: 309 # should be a list if the form control was actually a multiselect 310 # but we have some client side UI helpers that may produce a string 311 # instead 312 choice = choice.split(",") 313 314 return [item for item in choice if item in settings.get("options", [])] 315 316 elif input_type == UserInput.OPTION_CHOICE: 317 # select box 318 # one out of multiple options 319 # return option if valid, or default 320 if choice not in settings.get("options"): 321 if not silently_correct: 322 raise QueryParametersException(f"Invalid value selected; must be one of {', '.join(settings.get('options', {}).keys())}. {settings}") 323 else: 324 return settings.get("default", "") 325 else: 326 return choice 327 328 elif input_type == UserInput.OPTION_TEXT_JSON: 329 # verify that this is actually json 330 try: 331 json.dumps(json.loads(choice)) 332 except json.JSONDecodeError: 333 raise QueryParametersException("Invalid JSON value '%s'" % choice) 334 335 return json.loads(choice) 336 337 elif input_type in (UserInput.OPTION_TEXT, UserInput.OPTION_TEXT_LARGE, UserInput.OPTION_HUE): 338 # text string 339 # optionally clamp it as an integer; return default if not a valid 340 # integer (or float; inferred from default or made explicit via the 341 # coerce_type setting) 342 if settings.get("coerce_type"): 343 value_type = settings["coerce_type"] 344 else: 345 value_type = type(settings.get("default")) 346 if value_type not in (int, float): 347 value_type = int 348 349 if "max" in settings: 350 try: 351 choice = min(settings["max"], value_type(choice)) 352 except (ValueError, TypeError): 353 if not silently_correct: 354 raise QueryParametersException("Provide a value of %s or lower." % str(settings["max"])) 355 356 choice = settings.get("default") 357 358 if "min" in settings: 359 try: 360 choice = max(settings["min"], value_type(choice)) 361 except (ValueError, TypeError): 362 if not silently_correct: 363 raise QueryParametersException("Provide a value of %s or more." % str(settings["min"])) 364 365 choice = settings.get("default") 366 367 if choice is None or choice == "": 368 choice = settings.get("default") 369 370 if choice is None: 371 choice = 0 if "min" in settings or "max" in settings else "" 372 373 if settings.get("coerce_type"): 374 try: 375 return value_type(choice) 376 except (ValueError, TypeError): 377 return settings.get("default") 378 else: 379 return choice 380 381 else: 382 # no filtering 383 return choice
class
RequirementsNotMetException(builtins.Exception):
9class RequirementsNotMetException(Exception): 10 """ 11 If this is raised while parsing, that option is not included in the parsed 12 output. Used with the "requires" option setting. 13 """ 14 pass
If this is raised while parsing, that option is not included in the parsed output. Used with the "requires" option setting.
class
UserInput:
16class UserInput: 17 """ 18 Class for handling user input 19 20 It is important to sanitise user input, as carelessly entered parameters 21 may in e.g. requesting far more data than needed, or lead to undefined 22 behaviour. This class offers a set of pre-defined value types that can be 23 consistently rendered as form elements in an interface and parsed. 24 """ 25 OPTION_TOGGLE = "toggle" # boolean toggle (checkbox) 26 OPTION_CHOICE = "choice" # one choice out of a list (select) 27 OPTION_TEXT = "string" # simple string or integer (input text) 28 OPTION_MULTI = "multi" # multiple values out of a list (select multiple) 29 OPTION_MULTI_SELECT = "multi_select" # multiple values out of a dropdown list (select multiple) 30 OPTION_INFO = "info" # just a bit of text, not actual input 31 OPTION_TEXT_LARGE = "textarea" # longer text 32 OPTION_TEXT_JSON = "json" # text, but should be valid JSON 33 OPTION_DATE = "date" # a single date 34 OPTION_DATERANGE = "daterange" # a beginning and end date 35 OPTION_DIVIDER = "divider" # meta-option, divides related sets of options 36 OPTION_FILE = "file" # file upload 37 OPTION_HUE = "hue" # colour hue 38 OPTION_DATASOURCES = "datasources" # data source toggling 39 OPTION_EXTENSIONS = "extensions" # extension toggling 40 OPTION_DATASOURCES_TABLE = "datasources_table" # a table with settings per data source 41 OPTION_ANNOTATION = "annotation" # checkbox for whether to an annotation 42 OPTION_ANNOTATIONS = "annotations" # table for whether to write multiple annotations 43 44 OPTIONS_COSMETIC = (OPTION_INFO, OPTION_DIVIDER) 45 46 @staticmethod 47 def parse_all(options, input, silently_correct=True): 48 """ 49 Parse form input for the provided options 50 51 Ignores all input not belonging to any of the defined options: parses 52 and sanitises the rest, and returns a dictionary with the sanitised 53 options. If an option is *not* present in the input, the default value 54 is used, and if that is absent, `None`. 55 56 In other words, this ensures a dictionary with 1) only white-listed 57 keys, 2) a value of an expected type for each key. 58 59 :param dict options: Options, as a name -> settings dictionary 60 :param dict input: Input, as a form field -> value dictionary 61 :param bool silently_correct: If true, replace invalid values with the 62 given default value; else, raise a QueryParametersException if a value 63 is invalid. 64 65 :return dict: Sanitised form input 66 """ 67 68 from common.lib.helpers import convert_to_int 69 parsed_input = {} 70 71 if type(input) is not dict and type(input) is not ImmutableMultiDict: 72 raise TypeError("input must be a dictionary or ImmutableMultiDict") 73 74 if type(input) is ImmutableMultiDict: 75 # we are not using to_dict, because that messes up multi-selects 76 input = {key: input.getlist(key) for key in input} 77 for key, value in input.items(): 78 if type(value) is list and len(value) == 1: 79 input[key] = value[0] 80 81 # all parameters are submitted as option-[parameter ID], this is an 82 # artifact of how the web interface works and we can simply remove the 83 # prefix 84 input = {re.sub(r"^option-", "", field): input[field] for field in input} 85 86 # fields can be 'delegated', i.e. they only show up under some condition 87 # or in a later stage of form input. here we determine if input was 88 # actually filled in or was only defined but never delegated 89 never_delegated = set([option for option in options if options[option].get("delegated")]) 90 never_delegated -= set(input.keys()) 91 92 # re-order input so that the fields relying on the value of other 93 # fields are parsed last 94 options = {k: options[k] for k in sorted(options, key=lambda k: options[k].get("requires") is not None)} 95 96 for option, settings in options.items(): 97 if settings.get("indirect"): 98 # these are settings that are derived from and set by other 99 # settings 100 continue 101 102 if settings.get("type") in UserInput.OPTIONS_COSMETIC: 103 # these are structural form elements and never have a value 104 continue 105 106 if option in never_delegated: 107 # these options were never actually part of the input because 108 # the required conditions were never met, so they can be 109 # ignored 110 continue 111 112 elif settings.get("type") == UserInput.OPTION_DATERANGE: 113 # special case, since it combines two inputs 114 option_min = option + "-min" 115 option_max = option + "-max" 116 117 # normally this is taken care of client-side, but in case this 118 # didn't work, try to salvage it server-side 119 if option_min not in input or input.get(option_min) == "-1": 120 option_min += "_proxy" 121 122 if option_max not in input or input.get(option_max) == "-1": 123 option_max += "_proxy" 124 125 # save as a tuple of unix timestamps (or None) 126 try: 127 after, before = (UserInput.parse_value(settings, input.get(option_min), parsed_input, silently_correct), UserInput.parse_value(settings, input.get(option_max), parsed_input, silently_correct)) 128 129 if before and after and after > before: 130 if not silently_correct: 131 raise QueryParametersException("End of date range must be after beginning of date range.") 132 else: 133 before = after 134 135 parsed_input[option] = (after, before) 136 except RequirementsNotMetException: 137 pass 138 139 elif settings.get("type") in (UserInput.OPTION_TOGGLE, UserInput.OPTION_ANNOTATION): 140 # special case too, since if a checkbox is unchecked, it simply 141 # does not show up in the input 142 try: 143 if option in input: 144 # Toggle needs to be parsed 145 parsed_input[option] = UserInput.parse_value(settings, input[option], parsed_input, silently_correct) 146 else: 147 # Toggle was left blank 148 parsed_input[option] = False 149 except RequirementsNotMetException: 150 pass 151 152 elif settings.get("type") == UserInput.OPTION_DATASOURCES: 153 # special case, because this combines multiple inputs to 154 # configure data source availability and expiration 155 datasources = {datasource: { 156 "enabled": f"{option}-enable-{datasource}" in input, 157 "allow_optout": f"{option}-optout-{datasource}" in input, 158 "timeout": convert_to_int(input[f"{option}-timeout-{datasource}"], 0) 159 } for datasource in input[option].split(",")} 160 161 parsed_input[option] = [datasource for datasource, v in datasources.items() if v["enabled"]] 162 parsed_input[option.split(".")[0] + ".expiration"] = datasources 163 164 elif settings.get("type") == UserInput.OPTION_EXTENSIONS: 165 # also a special case 166 parsed_input[option] = {extension: { 167 "enabled": f"{option}-enable-{extension}" in input 168 } for extension in input[option].split(",")} 169 170 elif settings.get("type") == UserInput.OPTION_DATASOURCES_TABLE: 171 # special case, parse table values to generate a dict 172 columns = list(settings["columns"].keys()) 173 table_input = {} 174 175 for datasource in list(settings["default"].keys()): 176 table_input[datasource] = {} 177 for column in columns: 178 179 choice = input.get(option + "-" + datasource + "-" + column, False) 180 column_settings = settings["columns"][column] # sub-settings per column 181 table_input[datasource][column] = UserInput.parse_value(column_settings, choice, table_input, silently_correct=True) 182 183 parsed_input[option] = table_input 184 185 elif option not in input: 186 # not provided? use default 187 parsed_input[option] = settings.get("default", None) 188 189 else: 190 # normal parsing and sanitisation 191 try: 192 parsed_input[option] = UserInput.parse_value(settings, input[option], parsed_input, silently_correct) 193 except RequirementsNotMetException: 194 pass 195 196 return parsed_input 197 198 @staticmethod 199 def parse_value(settings, choice, other_input=None, silently_correct=True): 200 """ 201 Filter user input 202 203 Makes sure user input for post-processors is valid and within the 204 parameters specified by the post-processor 205 206 :param obj settings: Settings, including defaults and valid options 207 :param choice: The chosen option, to be parsed 208 :param dict other_input: Other input, as parsed so far 209 :param bool silently_correct: If true, replace invalid values with the 210 given default value; else, raise a QueryParametersException if a value 211 is invalid. 212 213 :return: Validated and parsed input 214 """ 215 # short-circuit if there is a requirement for the field to be parsed 216 # and the requirement isn't met 217 if settings.get("requires"): 218 try: 219 field, operator, value = re.findall(r"([a-zA-Z0-9_-]+)([!=$~^]+)(.*)", settings.get("requires"))[0] 220 except IndexError: 221 # invalid condition, interpret as 'does the field with this name have a value' 222 field, operator, value = (choice, "!=", "") 223 224 if field not in other_input: 225 raise RequirementsNotMetException() 226 227 negated = operator.startswith("!") 228 if negated: 229 operator = operator[1:] 230 231 other_value = other_input.get(field) 232 if type(other_value) is bool: 233 # evalues to a boolean, i.e. checkboxes etc 234 if operator in ("==", "="): 235 if ((other_value and value in ("", "false")) or (not other_value and value in ("true", "checked"))) != negated: 236 raise RequirementsNotMetException() 237 else: 238 if ((other_value and value not in ("true", "checked")) or (not other_value and value not in ("", "false"))) != negated: 239 raise RequirementsNotMetException() 240 241 else: 242 if type(other_value) in (tuple, list): 243 # iterables are a bit special 244 if len(other_value) == 1: 245 # treat one-item lists as "normal" values 246 other_value = other_value[0] 247 elif operator == "~=": # interpret as 'is in list?' 248 if (value not in other_value) != negated: 249 raise RequirementsNotMetException() 250 elif not negated: 251 # condition doesn't make sense for a list, so assume it's not True 252 raise RequirementsNotMetException() 253 254 if operator == "^=" and str(other_value).startswith(value) == negated: 255 raise RequirementsNotMetException() 256 elif operator == "$=" and str(other_value).endswith(value) == negated: 257 raise RequirementsNotMetException() 258 elif operator == "~=" and (value in str(other_value)) == negated: 259 raise RequirementsNotMetException() 260 elif operator in ("==", "=") and (value == other_value) == negated: 261 raise RequirementsNotMetException() 262 263 input_type = settings.get("type", "") 264 if input_type in UserInput.OPTIONS_COSMETIC: 265 # these are structural form elements and can never return a value 266 return None 267 268 elif input_type in (UserInput.OPTION_TOGGLE, UserInput.OPTION_ANNOTATION): 269 # simple boolean toggle 270 if type(choice) is bool: 271 return choice 272 elif choice in ['false', 'False']: 273 # Sanitized options passed back to Flask can be converted to strings as 'false' 274 return False 275 elif choice in ['true', 'True', 'on']: 276 # Toggle will have value 'on', but may also becomes a string 'true' 277 return True 278 else: 279 raise QueryParametersException("Toggle invalid input") 280 281 elif input_type in (UserInput.OPTION_DATE, UserInput.OPTION_DATERANGE): 282 # parse either integers (unix timestamps) or try to guess the date 283 # format (the latter may be used for input if JavaScript is turned 284 # off in the front-end and the input comes from there) 285 value = None 286 try: 287 value = int(choice) 288 except ValueError: 289 parsed_choice = parse_datetime(choice) 290 value = int(parsed_choice.timestamp()) 291 finally: 292 return value 293 294 elif input_type in (UserInput.OPTION_MULTI, UserInput.OPTION_ANNOTATIONS): 295 # any number of values out of a list of possible values 296 # comma-separated during input, returned as a list of valid options 297 if not choice: 298 return settings.get("default", []) 299 300 chosen = choice.split(",") 301 return [item for item in chosen if item in settings.get("options", [])] 302 303 elif input_type == UserInput.OPTION_MULTI_SELECT: 304 # multiple number of values out of a dropdown list of possible values 305 # comma-separated during input, returned as a list of valid options 306 if not choice: 307 return settings.get("default", []) 308 309 if type(choice) is str: 310 # should be a list if the form control was actually a multiselect 311 # but we have some client side UI helpers that may produce a string 312 # instead 313 choice = choice.split(",") 314 315 return [item for item in choice if item in settings.get("options", [])] 316 317 elif input_type == UserInput.OPTION_CHOICE: 318 # select box 319 # one out of multiple options 320 # return option if valid, or default 321 if choice not in settings.get("options"): 322 if not silently_correct: 323 raise QueryParametersException(f"Invalid value selected; must be one of {', '.join(settings.get('options', {}).keys())}. {settings}") 324 else: 325 return settings.get("default", "") 326 else: 327 return choice 328 329 elif input_type == UserInput.OPTION_TEXT_JSON: 330 # verify that this is actually json 331 try: 332 json.dumps(json.loads(choice)) 333 except json.JSONDecodeError: 334 raise QueryParametersException("Invalid JSON value '%s'" % choice) 335 336 return json.loads(choice) 337 338 elif input_type in (UserInput.OPTION_TEXT, UserInput.OPTION_TEXT_LARGE, UserInput.OPTION_HUE): 339 # text string 340 # optionally clamp it as an integer; return default if not a valid 341 # integer (or float; inferred from default or made explicit via the 342 # coerce_type setting) 343 if settings.get("coerce_type"): 344 value_type = settings["coerce_type"] 345 else: 346 value_type = type(settings.get("default")) 347 if value_type not in (int, float): 348 value_type = int 349 350 if "max" in settings: 351 try: 352 choice = min(settings["max"], value_type(choice)) 353 except (ValueError, TypeError): 354 if not silently_correct: 355 raise QueryParametersException("Provide a value of %s or lower." % str(settings["max"])) 356 357 choice = settings.get("default") 358 359 if "min" in settings: 360 try: 361 choice = max(settings["min"], value_type(choice)) 362 except (ValueError, TypeError): 363 if not silently_correct: 364 raise QueryParametersException("Provide a value of %s or more." % str(settings["min"])) 365 366 choice = settings.get("default") 367 368 if choice is None or choice == "": 369 choice = settings.get("default") 370 371 if choice is None: 372 choice = 0 if "min" in settings or "max" in settings else "" 373 374 if settings.get("coerce_type"): 375 try: 376 return value_type(choice) 377 except (ValueError, TypeError): 378 return settings.get("default") 379 else: 380 return choice 381 382 else: 383 # no filtering 384 return choice
Class for handling user input
It is important to sanitise user input, as carelessly entered parameters may in e.g. requesting far more data than needed, or lead to undefined behaviour. This class offers a set of pre-defined value types that can be consistently rendered as form elements in an interface and parsed.
@staticmethod
def
parse_all(options, input, silently_correct=True):
46 @staticmethod 47 def parse_all(options, input, silently_correct=True): 48 """ 49 Parse form input for the provided options 50 51 Ignores all input not belonging to any of the defined options: parses 52 and sanitises the rest, and returns a dictionary with the sanitised 53 options. If an option is *not* present in the input, the default value 54 is used, and if that is absent, `None`. 55 56 In other words, this ensures a dictionary with 1) only white-listed 57 keys, 2) a value of an expected type for each key. 58 59 :param dict options: Options, as a name -> settings dictionary 60 :param dict input: Input, as a form field -> value dictionary 61 :param bool silently_correct: If true, replace invalid values with the 62 given default value; else, raise a QueryParametersException if a value 63 is invalid. 64 65 :return dict: Sanitised form input 66 """ 67 68 from common.lib.helpers import convert_to_int 69 parsed_input = {} 70 71 if type(input) is not dict and type(input) is not ImmutableMultiDict: 72 raise TypeError("input must be a dictionary or ImmutableMultiDict") 73 74 if type(input) is ImmutableMultiDict: 75 # we are not using to_dict, because that messes up multi-selects 76 input = {key: input.getlist(key) for key in input} 77 for key, value in input.items(): 78 if type(value) is list and len(value) == 1: 79 input[key] = value[0] 80 81 # all parameters are submitted as option-[parameter ID], this is an 82 # artifact of how the web interface works and we can simply remove the 83 # prefix 84 input = {re.sub(r"^option-", "", field): input[field] for field in input} 85 86 # fields can be 'delegated', i.e. they only show up under some condition 87 # or in a later stage of form input. here we determine if input was 88 # actually filled in or was only defined but never delegated 89 never_delegated = set([option for option in options if options[option].get("delegated")]) 90 never_delegated -= set(input.keys()) 91 92 # re-order input so that the fields relying on the value of other 93 # fields are parsed last 94 options = {k: options[k] for k in sorted(options, key=lambda k: options[k].get("requires") is not None)} 95 96 for option, settings in options.items(): 97 if settings.get("indirect"): 98 # these are settings that are derived from and set by other 99 # settings 100 continue 101 102 if settings.get("type") in UserInput.OPTIONS_COSMETIC: 103 # these are structural form elements and never have a value 104 continue 105 106 if option in never_delegated: 107 # these options were never actually part of the input because 108 # the required conditions were never met, so they can be 109 # ignored 110 continue 111 112 elif settings.get("type") == UserInput.OPTION_DATERANGE: 113 # special case, since it combines two inputs 114 option_min = option + "-min" 115 option_max = option + "-max" 116 117 # normally this is taken care of client-side, but in case this 118 # didn't work, try to salvage it server-side 119 if option_min not in input or input.get(option_min) == "-1": 120 option_min += "_proxy" 121 122 if option_max not in input or input.get(option_max) == "-1": 123 option_max += "_proxy" 124 125 # save as a tuple of unix timestamps (or None) 126 try: 127 after, before = (UserInput.parse_value(settings, input.get(option_min), parsed_input, silently_correct), UserInput.parse_value(settings, input.get(option_max), parsed_input, silently_correct)) 128 129 if before and after and after > before: 130 if not silently_correct: 131 raise QueryParametersException("End of date range must be after beginning of date range.") 132 else: 133 before = after 134 135 parsed_input[option] = (after, before) 136 except RequirementsNotMetException: 137 pass 138 139 elif settings.get("type") in (UserInput.OPTION_TOGGLE, UserInput.OPTION_ANNOTATION): 140 # special case too, since if a checkbox is unchecked, it simply 141 # does not show up in the input 142 try: 143 if option in input: 144 # Toggle needs to be parsed 145 parsed_input[option] = UserInput.parse_value(settings, input[option], parsed_input, silently_correct) 146 else: 147 # Toggle was left blank 148 parsed_input[option] = False 149 except RequirementsNotMetException: 150 pass 151 152 elif settings.get("type") == UserInput.OPTION_DATASOURCES: 153 # special case, because this combines multiple inputs to 154 # configure data source availability and expiration 155 datasources = {datasource: { 156 "enabled": f"{option}-enable-{datasource}" in input, 157 "allow_optout": f"{option}-optout-{datasource}" in input, 158 "timeout": convert_to_int(input[f"{option}-timeout-{datasource}"], 0) 159 } for datasource in input[option].split(",")} 160 161 parsed_input[option] = [datasource for datasource, v in datasources.items() if v["enabled"]] 162 parsed_input[option.split(".")[0] + ".expiration"] = datasources 163 164 elif settings.get("type") == UserInput.OPTION_EXTENSIONS: 165 # also a special case 166 parsed_input[option] = {extension: { 167 "enabled": f"{option}-enable-{extension}" in input 168 } for extension in input[option].split(",")} 169 170 elif settings.get("type") == UserInput.OPTION_DATASOURCES_TABLE: 171 # special case, parse table values to generate a dict 172 columns = list(settings["columns"].keys()) 173 table_input = {} 174 175 for datasource in list(settings["default"].keys()): 176 table_input[datasource] = {} 177 for column in columns: 178 179 choice = input.get(option + "-" + datasource + "-" + column, False) 180 column_settings = settings["columns"][column] # sub-settings per column 181 table_input[datasource][column] = UserInput.parse_value(column_settings, choice, table_input, silently_correct=True) 182 183 parsed_input[option] = table_input 184 185 elif option not in input: 186 # not provided? use default 187 parsed_input[option] = settings.get("default", None) 188 189 else: 190 # normal parsing and sanitisation 191 try: 192 parsed_input[option] = UserInput.parse_value(settings, input[option], parsed_input, silently_correct) 193 except RequirementsNotMetException: 194 pass 195 196 return parsed_input
Parse form input for the provided options
Ignores all input not belonging to any of the defined options: parses
and sanitises the rest, and returns a dictionary with the sanitised
options. If an option is not present in the input, the default value
is used, and if that is absent, None.
In other words, this ensures a dictionary with 1) only white-listed keys, 2) a value of an expected type for each key.
Parameters
- dict options: Options, as a name -> settings dictionary
- dict input: Input, as a form field -> value dictionary
- bool silently_correct: If true, replace invalid values with the given default value; else, raise a QueryParametersException if a value is invalid.
Returns
Sanitised form input
@staticmethod
def
parse_value(settings, choice, other_input=None, silently_correct=True):
198 @staticmethod 199 def parse_value(settings, choice, other_input=None, silently_correct=True): 200 """ 201 Filter user input 202 203 Makes sure user input for post-processors is valid and within the 204 parameters specified by the post-processor 205 206 :param obj settings: Settings, including defaults and valid options 207 :param choice: The chosen option, to be parsed 208 :param dict other_input: Other input, as parsed so far 209 :param bool silently_correct: If true, replace invalid values with the 210 given default value; else, raise a QueryParametersException if a value 211 is invalid. 212 213 :return: Validated and parsed input 214 """ 215 # short-circuit if there is a requirement for the field to be parsed 216 # and the requirement isn't met 217 if settings.get("requires"): 218 try: 219 field, operator, value = re.findall(r"([a-zA-Z0-9_-]+)([!=$~^]+)(.*)", settings.get("requires"))[0] 220 except IndexError: 221 # invalid condition, interpret as 'does the field with this name have a value' 222 field, operator, value = (choice, "!=", "") 223 224 if field not in other_input: 225 raise RequirementsNotMetException() 226 227 negated = operator.startswith("!") 228 if negated: 229 operator = operator[1:] 230 231 other_value = other_input.get(field) 232 if type(other_value) is bool: 233 # evalues to a boolean, i.e. checkboxes etc 234 if operator in ("==", "="): 235 if ((other_value and value in ("", "false")) or (not other_value and value in ("true", "checked"))) != negated: 236 raise RequirementsNotMetException() 237 else: 238 if ((other_value and value not in ("true", "checked")) or (not other_value and value not in ("", "false"))) != negated: 239 raise RequirementsNotMetException() 240 241 else: 242 if type(other_value) in (tuple, list): 243 # iterables are a bit special 244 if len(other_value) == 1: 245 # treat one-item lists as "normal" values 246 other_value = other_value[0] 247 elif operator == "~=": # interpret as 'is in list?' 248 if (value not in other_value) != negated: 249 raise RequirementsNotMetException() 250 elif not negated: 251 # condition doesn't make sense for a list, so assume it's not True 252 raise RequirementsNotMetException() 253 254 if operator == "^=" and str(other_value).startswith(value) == negated: 255 raise RequirementsNotMetException() 256 elif operator == "$=" and str(other_value).endswith(value) == negated: 257 raise RequirementsNotMetException() 258 elif operator == "~=" and (value in str(other_value)) == negated: 259 raise RequirementsNotMetException() 260 elif operator in ("==", "=") and (value == other_value) == negated: 261 raise RequirementsNotMetException() 262 263 input_type = settings.get("type", "") 264 if input_type in UserInput.OPTIONS_COSMETIC: 265 # these are structural form elements and can never return a value 266 return None 267 268 elif input_type in (UserInput.OPTION_TOGGLE, UserInput.OPTION_ANNOTATION): 269 # simple boolean toggle 270 if type(choice) is bool: 271 return choice 272 elif choice in ['false', 'False']: 273 # Sanitized options passed back to Flask can be converted to strings as 'false' 274 return False 275 elif choice in ['true', 'True', 'on']: 276 # Toggle will have value 'on', but may also becomes a string 'true' 277 return True 278 else: 279 raise QueryParametersException("Toggle invalid input") 280 281 elif input_type in (UserInput.OPTION_DATE, UserInput.OPTION_DATERANGE): 282 # parse either integers (unix timestamps) or try to guess the date 283 # format (the latter may be used for input if JavaScript is turned 284 # off in the front-end and the input comes from there) 285 value = None 286 try: 287 value = int(choice) 288 except ValueError: 289 parsed_choice = parse_datetime(choice) 290 value = int(parsed_choice.timestamp()) 291 finally: 292 return value 293 294 elif input_type in (UserInput.OPTION_MULTI, UserInput.OPTION_ANNOTATIONS): 295 # any number of values out of a list of possible values 296 # comma-separated during input, returned as a list of valid options 297 if not choice: 298 return settings.get("default", []) 299 300 chosen = choice.split(",") 301 return [item for item in chosen if item in settings.get("options", [])] 302 303 elif input_type == UserInput.OPTION_MULTI_SELECT: 304 # multiple number of values out of a dropdown list of possible values 305 # comma-separated during input, returned as a list of valid options 306 if not choice: 307 return settings.get("default", []) 308 309 if type(choice) is str: 310 # should be a list if the form control was actually a multiselect 311 # but we have some client side UI helpers that may produce a string 312 # instead 313 choice = choice.split(",") 314 315 return [item for item in choice if item in settings.get("options", [])] 316 317 elif input_type == UserInput.OPTION_CHOICE: 318 # select box 319 # one out of multiple options 320 # return option if valid, or default 321 if choice not in settings.get("options"): 322 if not silently_correct: 323 raise QueryParametersException(f"Invalid value selected; must be one of {', '.join(settings.get('options', {}).keys())}. {settings}") 324 else: 325 return settings.get("default", "") 326 else: 327 return choice 328 329 elif input_type == UserInput.OPTION_TEXT_JSON: 330 # verify that this is actually json 331 try: 332 json.dumps(json.loads(choice)) 333 except json.JSONDecodeError: 334 raise QueryParametersException("Invalid JSON value '%s'" % choice) 335 336 return json.loads(choice) 337 338 elif input_type in (UserInput.OPTION_TEXT, UserInput.OPTION_TEXT_LARGE, UserInput.OPTION_HUE): 339 # text string 340 # optionally clamp it as an integer; return default if not a valid 341 # integer (or float; inferred from default or made explicit via the 342 # coerce_type setting) 343 if settings.get("coerce_type"): 344 value_type = settings["coerce_type"] 345 else: 346 value_type = type(settings.get("default")) 347 if value_type not in (int, float): 348 value_type = int 349 350 if "max" in settings: 351 try: 352 choice = min(settings["max"], value_type(choice)) 353 except (ValueError, TypeError): 354 if not silently_correct: 355 raise QueryParametersException("Provide a value of %s or lower." % str(settings["max"])) 356 357 choice = settings.get("default") 358 359 if "min" in settings: 360 try: 361 choice = max(settings["min"], value_type(choice)) 362 except (ValueError, TypeError): 363 if not silently_correct: 364 raise QueryParametersException("Provide a value of %s or more." % str(settings["min"])) 365 366 choice = settings.get("default") 367 368 if choice is None or choice == "": 369 choice = settings.get("default") 370 371 if choice is None: 372 choice = 0 if "min" in settings or "max" in settings else "" 373 374 if settings.get("coerce_type"): 375 try: 376 return value_type(choice) 377 except (ValueError, TypeError): 378 return settings.get("default") 379 else: 380 return choice 381 382 else: 383 # no filtering 384 return choice
Filter user input
Makes sure user input for post-processors is valid and within the parameters specified by the post-processor
Parameters
- obj settings: Settings, including defaults and valid options
- choice: The chosen option, to be parsed
- dict other_input: Other input, as parsed so far
- bool silently_correct: If true, replace invalid values with the given default value; else, raise a QueryParametersException if a value is invalid.
Returns
Validated and parsed input