datasources.fourcat_import.import_4cat

  1"""
  2Import datasets from other 4CATs
  3"""
  4import requests
  5import json
  6import time
  7import zipfile
  8from pathlib import Path
  9from psycopg2.errors import InFailedSqlTransaction
 10
 11from backend.lib.processor import BasicProcessor
 12from common.lib.exceptions import (QueryParametersException, FourcatException, ProcessorInterruptedException,
 13                                   DataSetException)
 14from common.lib.helpers import UserInput, get_software_version
 15from common.lib.dataset import DataSet
 16
 17
 18class FourcatImportException(FourcatException):
 19    pass
 20
 21
 22class SearchImportFromFourcat(BasicProcessor):
 23    type = "import_4cat-search"  # job ID
 24    category = "Search"  # category
 25    title = "Import 4CAT dataset and analyses"  # title displayed in UI
 26    description = "Import a dataset from another 4CAT server or from a zip file (exported from a 4CAT server)"  # description displayed in UI
 27    is_local = False  # Whether this datasource is locally scraped
 28    is_static = False  # Whether this datasource is still updated
 29
 30    max_workers = 1  # this cannot be more than 1, else things get VERY messy
 31
 32    created_datasets = None
 33    base = None
 34    remapped_keys = None
 35    dataset_owner = None
 36
 37    @classmethod
 38    def get_options(cls, parent_dataset=None, config=None) -> dict:
 39        """
 40        Get processor options
 41
 42        :param DataSet parent_dataset:  An object representing the dataset that
 43          the processor would or was run on
 44        :param ConfigManager|None config:  Configuration reader (context-aware)
 45        """
 46        return {
 47            "intro": {
 48                "type": UserInput.OPTION_INFO,
 49                "help": "Provide the URL of a dataset in another 4CAT server that you would like to copy to this one here. "
 50                        "\n\nTo import a dataset across servers, both servers need to be running the same version of 4CAT. "
 51                        "You can find the current version in the footer at the bottom of the interface."
 52            },
 53            "method": {
 54                "type": UserInput.OPTION_CHOICE,
 55                "help": "Import Type",
 56                "options": {
 57                    "zip": "Zip File",
 58                    "url": "4CAT URL",
 59                },
 60                "default": "url"
 61            },
 62            "url": {
 63                "type": UserInput.OPTION_TEXT,
 64                "help": "Dataset URL",
 65                "tooltip": "URL to the dataset's page, for example https://4cat.example/results/28da332f8918e6dc5aacd1c3b0170f01b80bd95f8ff9964ac646cecd33bfee49/.",
 66                "requires": "method^=url"
 67            },
 68            "intro2": {
 69                "type": UserInput.OPTION_INFO,
 70                "help": "You can create an API key via the 'API Access' item in 4CAT's navigation menu. Note that you need "
 71                        "an API key from **the server you are importing from**, not the one you are looking at right now. "
 72                        "Additionally, you need to have owner access to the dataset you want to import.",
 73                "requires": "method^=url"
 74            },
 75            "api-key": {
 76                "type": UserInput.OPTION_TEXT,
 77                "help": "4CAT API Key",
 78                "sensitive": True,
 79                "cache": True,
 80                "requires": "method^=url"
 81            },
 82            "data_upload": {
 83                "type": UserInput.OPTION_FILE,
 84                "help": "File",
 85                "tooltip": "Upload a ZIP file containing a dataset exported from a 4CAT server.",
 86                "requires": "method^=zip"
 87            },
 88
 89        }
 90
 91    def process(self):
 92        """
 93        Import 4CAT dataset either from another 4CAT server or from the uploaded zip file
 94        """
 95        self.created_datasets = set()  # keys of created datasets - may not be successful!
 96        self.remapped_keys = {}  # changed dataset keys
 97        self.dataset_owner = self.dataset.get_owners()[0]  # at this point it has 1 owner
 98        try:
 99            if self.parameters.get("method") == "zip":
100                self.process_zip()
101            else:
102                self.process_urls()
103        except Exception as e:
104            # Catch all exceptions and finish the job with an error
105            # Resuming is impossible because this dataset was overwritten with the importing dataset
106            # halt_and_catch_fire() will clean up and delete the datasets that were created
107            self.interrupted = True
108            try:
109                self.halt_and_catch_fire()
110            except ProcessorInterruptedException:
111                pass
112            except InFailedSqlTransaction:
113                # Catch database issue and retry
114                self.db.rollback()
115                try:
116                    self.halt_and_catch_fire()
117                except ProcessorInterruptedException:
118                    pass
119            # Reraise the original exception for logging
120            raise e
121
122    def after_create(query, dataset, request):
123        """
124        Hook to execute after the dataset for this source has been created
125
126        In this case, put the file in a temporary location so it can be
127        processed properly by the related Job later.
128
129        :param dict query:  Sanitised query parameters
130        :param DataSet dataset:  Dataset created for this query
131        :param request:  Flask request submitted for its creation
132        """
133        if query.get("method") == "zip":
134            file = request.files["option-data_upload"]
135            file.seek(0)
136            with dataset.get_results_path().with_suffix(".importing").open("wb") as outfile:
137                while True:
138                    chunk = file.read(1024)
139                    if len(chunk) == 0:
140                        break
141                    outfile.write(chunk)
142        else:
143            # nothing to do for URLs
144            pass
145
146
147    def process_zip(self):
148        """
149        Import 4CAT dataset from a ZIP file
150        """
151        self.dataset.update_status("Importing datasets and analyses from ZIP file.")
152        temp_file = self.dataset.get_results_path().with_suffix(".importing")
153
154        imported = []
155        processed_files = 1 # take into account the export.log file
156        failed_imports = []
157        primary_dataset_original_log = None
158        with zipfile.ZipFile(temp_file, "r") as zip_ref:
159            zip_contents = zip_ref.namelist()
160
161            # Get all metadata files and determine primary dataset
162            metadata_files = [file for file in zip_contents if file.endswith("_metadata.json")]
163            if not metadata_files:
164                self.dataset.finish_with_error("No metadata files found in ZIP file; is this a 4CAT export?")
165                return
166
167            # Get the primary dataset
168            primary_dataset_keys = set()
169            datasets = []
170            parent_child_mapping = {}
171            for file in metadata_files:
172                with zip_ref.open(file) as f:
173                    content = f.read().decode('utf-8')  # Decode the binary content using the desired encoding
174                    metadata = json.loads(content)
175                    if not metadata.get("key_parent"):
176                        primary_dataset_keys.add(metadata.get("key"))
177                        datasets.append(metadata)
178                    else:
179                        # Store the mapping of parent to child datasets
180                        parent_key = metadata.get("key_parent")
181                        if parent_key not in parent_child_mapping:
182                            parent_child_mapping[parent_key] = []
183                        parent_child_mapping[parent_key].append(metadata)
184
185            # Primary dataset will overwrite this dataset; we could address this to support multiple primary datasets
186            if len(primary_dataset_keys) != 1:
187                self.dataset.finish_with_error("ZIP file contains multiple primary datasets; only one is allowed.")
188                return
189
190            # Import datasets
191            while datasets:
192                self.halt_and_catch_fire()
193
194                # Create the datasets
195                metadata = datasets.pop(0)
196                dataset_key = metadata.get("key")
197                processed_metadata = self.process_metadata(metadata)
198                new_dataset = self.create_dataset(processed_metadata, dataset_key, dataset_key in primary_dataset_keys)
199                processed_files += 1
200
201                # Copy the log file
202                self.halt_and_catch_fire()
203                log_filename = Path(metadata["result_file"]).with_suffix(".log").name
204                if log_filename in zip_contents:
205                    self.dataset.update_status(f"Transferring log file for dataset {new_dataset.key}")
206                    with zip_ref.open(log_filename) as f:
207                        content = f.read().decode('utf-8')
208                        if new_dataset.key == self.dataset.key:
209                            # Hold the original log for the primary dataset and add at the end
210                            primary_dataset_original_log = content
211                        else:
212                            new_dataset.log("Original dataset log included below:")
213                            with new_dataset.get_log_path().open("a") as outfile:
214                                outfile.write(content)
215                    processed_files += 1
216                else:
217                    self.dataset.log(f"Log file not found for dataset {new_dataset.key} (original key {dataset_key}).")
218
219                # Copy the results
220                self.halt_and_catch_fire()
221                results_filename = metadata["result_file"]
222                if results_filename in zip_contents:
223                    self.dataset.update_status(f"Transferring data file for dataset {new_dataset.key}")
224                    with zip_ref.open(results_filename) as f:
225                        with new_dataset.get_results_path().open("wb") as outfile:
226                            outfile.write(f.read())
227                    processed_files += 1
228
229                    if not imported:
230                        # first dataset - use num rows as 'overall'
231                        num_rows = metadata["num_rows"]
232                else:
233                    self.dataset.log(f"Results file not found for dataset {new_dataset.key} (original key {dataset_key}).")
234                    new_dataset.finish_with_error(f"Results file not found for dataset {new_dataset.key} (original key {dataset_key}).")
235                    failed_imports.append(dataset_key)
236                    continue
237
238                # finally, the kids
239                self.halt_and_catch_fire()
240                if dataset_key in parent_child_mapping:
241                    datasets.extend(parent_child_mapping[dataset_key])
242                    self.dataset.log(f"Adding ({len(parent_child_mapping[dataset_key])}) child datasets to import queue")
243
244                # done - remember that we've imported this one
245                imported.append(new_dataset)
246                new_dataset.update_status(metadata["status"])
247
248                if new_dataset.key != self.dataset.key:
249                    # only finish if this is not the 'main' dataset, or the user
250                    # will think the whole import is done
251                    new_dataset.finish(metadata["num_rows"])
252
253            # Check that all files were processed
254            missed_files = []
255            if len(zip_contents) != processed_files:
256                for file in zip_contents:
257                    if file not in processed_files:
258                        missed_files.append(file)
259
260            # todo: this part needs updating if/when we support importing multiple datasets!
261            if failed_imports:
262                self.dataset.update_status(f"Dataset import finished, but not all data was imported properly. "
263                                           f"{len(failed_imports)} dataset(s) were not successfully imported. Check the "
264                                           f"dataset log file for details.", is_final=True)
265            elif missed_files:
266                self.dataset.log(f"ZIP file contained {len(missed_files)} files that were not processed: {missed_files}")
267                self.dataset.update_status(f"Dataset import finished, but not all files were processed. "
268                                           f"{len(missed_files)} files were not successfully imported. Check the "
269                                           f"dataset log file for details.", is_final=True)
270            else:
271                self.dataset.update_status(f"{len(imported)} dataset(s) succesfully imported.",
272                                           is_final=True)
273
274            if not self.dataset.is_finished():
275                # now all related datasets are imported, we can finish the 'main'
276                # dataset, and the user will be alerted that the full import is
277                # complete
278                self.dataset.finish(num_rows)
279
280            # Add the original log for the primary dataset
281            if primary_dataset_original_log:
282                self.dataset.log("Original dataset log included below:\n")
283                with self.dataset.get_log_path().open("a") as outfile:
284                    outfile.write(primary_dataset_original_log)
285
286
287    @staticmethod
288    def process_metadata(metadata):
289        """
290        Process metadata for import
291
292        :param dict metadata:  Metadata of import
293        :return:  Relevant metadata, or `None` if parsing error
294        """
295        # get rid of some keys that are server-specific and don't need to
296        # be stored (or don't correspond to database columns)
297        try:
298            metadata.pop("current_4CAT_version")
299            metadata.pop("id")
300            metadata.pop("job")
301            metadata.pop("is_private")
302            metadata.pop("is_finished")  # we'll finish it ourselves, thank you!!!
303
304            # extra params are stored as JSON...
305            metadata["parameters"] = json.loads(metadata["parameters"])
306            if "copied_from" in metadata["parameters"]:
307                metadata["parameters"].pop("copied_from")
308            metadata["parameters"] = json.dumps(metadata["parameters"])
309
310        except (ValueError, KeyError):
311            # we don't need all this metadata but it still should be present,
312            # otherwise something is wrong with the data format
313            return None
314
315        return metadata
316
317    def create_dataset(self, metadata, original_key, primary=False):
318        """
319        Create a new dataset
320        """
321        if primary:
322            self.dataset.update_status(f"Importing primary dataset {original_key}.")
323            # if this is the first dataset we're importing, make it the
324            # processor's "own" dataset. the key has already been set to
325            # the imported dataset's key via ensure_key() (or a new unqiue
326            # key if it already existed on this server)
327            # by making it the "own" dataset, the user initiating the
328            # import will see the imported dataset as the "result" of their
329            # import query in the interface, similar to the workflow for
330            # other data sources
331            new_dataset = self.dataset
332
333            # Update metadata and file
334            metadata.pop("key")  # key already OK (see above)
335            self.db.update("datasets", where={"key": new_dataset.key}, data=metadata)
336
337        else:
338            self.dataset.update_status(f"Importing child dataset {original_key}.")
339            # supernumerary datasets - handle on their own
340            # these include any children of imported datasets
341            try:
342                DataSet(key=metadata["key"], db=self.db, modules=self.modules)
343
344                # if we *haven't* thrown a DatasetException now, then the
345                # key is already in use, so create a "dummy" dataset and
346                # overwrite it with the metadata we have (except for the
347                # key). this ensures that a new unique key will be
348                # generated.
349                new_dataset = DataSet(parameters={}, type=self.type, db=self.db, modules=self.modules)
350                metadata.pop("key")
351                self.db.update("datasets", where={"key": new_dataset.key}, data=metadata)
352
353            except DataSetException:
354                # this is *good* since it means the key doesn't exist, so
355                # we can re-use the key of the imported dataset
356                self.db.insert("datasets", data=metadata)
357                new_dataset = DataSet(key=metadata["key"], db=self.db, modules=self.modules)
358
359        if new_dataset.key != original_key:
360            # could not use original key because it was already in use
361            # so update any references to use the new key
362            self.remapped_keys[original_key] = new_dataset.key
363            self.dataset.update_status(f"Cannot import with same key - already in use on this server. Using key "
364                                      f"{new_dataset.key} instead of key {original_key}!")
365
366        # refresh object, make sure it's in sync with the database
367        self.created_datasets.add(new_dataset.key)
368        new_dataset = DataSet(key=new_dataset.key, db=self.db, modules=self.modules)
369        current_log = None
370        if new_dataset.key == self.dataset.key:
371            # this ensures that the first imported dataset becomes the
372            # processor's "own" dataset, and that the import logs go to
373            # that dataset's log file. For later imports, this evaluates to
374            # False.
375
376            # Read the current log and store it; it needs to be after the result_file is updated (as it is used to determine the log file path)
377            current_log = self.dataset.get_log_path().read_text()
378            # Update the dataset
379            self.dataset = new_dataset
380
381        # if the key of the parent dataset was changed, change the
382        # reference to it that the child dataset has
383        if new_dataset.key_parent and new_dataset.key_parent in self.remapped_keys:
384            new_dataset.key_parent = self.remapped_keys[new_dataset.key_parent]
385
386        # update some attributes that should come from the new server, not
387        # the old
388        new_dataset.creator = self.dataset_owner
389        new_dataset.original_timestamp = new_dataset.timestamp
390        new_dataset.imported = True
391        new_dataset.timestamp = int(time.time())
392        new_dataset.db.commit()
393
394        # make sure the dataset path uses the new key and local dataset
395        # path settings. this also makes sure the log file is created in
396        # the right place (since it is derived from the results file path)
397        extension = metadata["result_file"].split(".")[-1]
398        updated = new_dataset.reserve_result_file(parameters=new_dataset.parameters, extension=extension)
399        if not updated:
400            self.dataset.log(f"Could not reserve result file for {new_dataset.key}!")
401
402        if current_log:
403            # Add the current log to the new dataset
404            with new_dataset.get_log_path().open("a") as outfile:
405                outfile.write(current_log)
406
407        return new_dataset
408
409
410    def process_urls(self):
411        """
412        Import 4CAT dataset from another 4CAT server
413
414        Interfaces with another 4CAT server to transfer a dataset's metadata,
415        data files and child datasets.
416        """
417        urls = [url.strip() for url in self.parameters.get("url").split(",")]
418        self.base = urls[0].split("/results/")[0]
419        keys = SearchImportFromFourcat.get_keys_from_urls(urls)
420        api_key = self.parameters.get("api-key")
421
422        imported = []  # successfully imported datasets
423        failed_imports = []  # keys that failed to import
424        num_rows = 0  # will be used later
425
426        # we can add support for multiple datasets later by removing
427        # this part!
428        keys = [keys[0]]
429
430        while keys:
431            dataset_key = keys.pop(0)
432
433            self.halt_and_catch_fire()
434            self.dataset.log(f"Importing dataset {dataset_key} from 4CAT server {self.base}.")
435
436            # first, metadata!
437            try:
438                metadata = SearchImportFromFourcat.fetch_from_4cat(self.base, dataset_key, api_key, "metadata")
439                metadata = metadata.json()
440            except FourcatImportException as e:
441                self.dataset.log(f"Error retrieving record for dataset {dataset_key}: {e}")
442                continue
443            except ValueError:
444                self.dataset.log(f"Could not read metadata for dataset {dataset_key}")
445                continue
446
447            # copying empty datasets doesn't really make sense
448            if metadata["num_rows"] == 0:
449                self.dataset.update_status(f"Skipping empty dataset {dataset_key}")
450                failed_imports.append(dataset_key)
451                continue
452
453            metadata = self.process_metadata(metadata)
454            if metadata is None:
455                self.dataset.update_status(f"Metadata for dataset {dataset_key} incomplete, skipping")
456                failed_imports.append(dataset_key)
457                continue
458
459            # create the new dataset
460            new_dataset = self.create_dataset(metadata, dataset_key, primary=True if not imported else False)
461
462            # then, the log
463            self.halt_and_catch_fire()
464            try:
465                self.dataset.update_status(f"Transferring log file for dataset {new_dataset.key}")
466                # TODO: for the primary, this ends up in the middle of the log as we are still adding to it...
467                log = SearchImportFromFourcat.fetch_from_4cat(self.base, dataset_key, api_key, "log")
468                logpath = new_dataset.get_log_path()
469                new_dataset.log("Original dataset log included below:")
470                with logpath.open("a") as outfile:
471                    outfile.write(log.text)
472            except FourcatImportException as e:
473                new_dataset.finish_with_error(f"Error retrieving log for dataset {new_dataset.key}: {e}")
474                failed_imports.append(dataset_key)
475                continue
476            except ValueError:
477                new_dataset.finish_with_error(f"Could not read log for dataset {new_dataset.key}: skipping dataset")
478                failed_imports.append(dataset_key)
479                continue
480
481            # then, the results
482            self.halt_and_catch_fire()
483            try:
484                self.dataset.update_status(f"Transferring data file for dataset {new_dataset.key}")
485                datapath = new_dataset.get_results_path()
486                SearchImportFromFourcat.fetch_from_4cat(self.base, dataset_key, api_key, "data", datapath)
487
488                if not imported:
489                    # first dataset - use num rows as 'overall'
490                    num_rows = metadata["num_rows"]
491
492            except FourcatImportException as e:
493                self.dataset.log(f"Dataset {new_dataset.key} unable to import: {e}, skipping import")
494                if new_dataset.key != self.dataset.key:
495                    new_dataset.delete()
496                continue
497
498            except ValueError:
499                new_dataset.finish_with_error(f"Could not read results for dataset {new_dataset.key}")
500                failed_imports.append(dataset_key)
501                continue
502
503            # finally, the kids
504            self.halt_and_catch_fire()
505            try:
506                self.dataset.update_status(f"Looking for child datasets to transfer for dataset {new_dataset.key}")
507                children = SearchImportFromFourcat.fetch_from_4cat(self.base, dataset_key, api_key, "children")
508                children = children.json()
509            except FourcatImportException as e:
510                self.dataset.update_status(f"Error retrieving children for dataset {new_dataset.key}: {e}")
511                failed_imports.append(dataset_key)
512                continue
513            except ValueError:
514                self.dataset.update_status(f"Could not collect children for dataset {new_dataset.key}")
515                failed_imports.append(dataset_key)
516                continue
517
518            for child in children:
519                keys.append(child)
520                self.dataset.log(f"Adding child dataset {child} to import queue")
521
522            # done - remember that we've imported this one
523            imported.append(new_dataset)
524            new_dataset.update_status(metadata["status"])
525
526            if new_dataset.key != self.dataset.key:
527                # only finish if this is not the 'main' dataset, or the user
528                # will think the whole import is done
529                new_dataset.finish(metadata["num_rows"])
530
531        # todo: this part needs updating if/when we support importing multiple datasets!
532        if failed_imports:
533            self.dataset.update_status(f"Dataset import finished, but not all data was imported properly. "
534                                       f"{len(failed_imports)} dataset(s) were not successfully imported. Check the "
535                                       f"dataset log file for details.", is_final=True)
536        else:
537            self.dataset.update_status(f"{len(imported)} dataset(s) succesfully imported from {self.base}.",
538                                       is_final=True)
539
540        if not self.dataset.is_finished():
541            # now all related datasets are imported, we can finish the 'main'
542            # dataset, and the user will be alerted that the full import is
543            # complete
544            self.dataset.finish(num_rows)
545
546    def halt_and_catch_fire(self):
547        """
548        Clean up on interrupt
549
550        There are multiple places in the code where we can bail out on an
551        interrupt, so abstract that away in its own function.
552        :return:
553        """
554        if self.interrupted:
555            # resuming is impossible because the original dataset (which
556            # has the list of URLs to import) has probably been
557            # overwritten by this point
558            deletables = [k for k in self.created_datasets if k != self.dataset.key]
559            for deletable in deletables:
560                DataSet(key=deletable, db=self.db, modules=self.modules).delete()
561
562            self.dataset.finish_with_error(f"Interrupted while importing datasets{' from '+self.base if self.base else ''}. Cannot resume - you "
563                                           f"will need to initiate the import again.")
564
565            raise ProcessorInterruptedException()
566
567    @staticmethod
568    def fetch_from_4cat(base, dataset_key, api_key, component, datapath=None):
569        """
570        Get dataset component from 4CAT export API
571
572        :param str base:  Server URL base to import from
573        :param str dataset_key:  Key of dataset to import
574        :param str api_key:  API authentication token
575        :param str component:  Component to retrieve
576        :return:  HTTP response object
577        """
578        try:
579            if component == "data" and datapath:
580                # Stream data
581                with requests.get(f"{base}/api/export-packed-dataset/{dataset_key}/{component}/", timeout=5, stream=True,
582                                  headers={
583                                            "User-Agent": "4cat/import",
584                                            "Authentication": api_key
585                                        }) as r:
586                    r.raise_for_status()
587                    with datapath.open("wb") as outfile:
588                        for chunk in r.iter_content(chunk_size=8192):
589                            outfile.write(chunk)
590                return r
591            else:
592                response = requests.get(f"{base}/api/export-packed-dataset/{dataset_key}/{component}/", timeout=5, headers={
593                    "User-Agent": "4cat/import",
594                    "Authentication": api_key
595                })
596        except requests.Timeout:
597            raise FourcatImportException(f"The 4CAT server at {base} took too long to respond. Make sure it is "
598                                         f"accessible to external connections and try again.")
599        except requests.RequestException as e:
600            raise FourcatImportException(f"Could not connect to the 4CAT server at {base} ({e}). Make sure it is "
601                                         f"accessible to external connections and try again.")
602
603        if response.status_code == 404:
604            raise FourcatImportException(
605                f"Dataset {dataset_key} not found at server {base} ({response.text}. Make sure all URLs point to "
606                f"a valid dataset.")
607        elif response.status_code in (401, 403):
608            raise FourcatImportException(
609                f"Dataset {dataset_key} not accessible at server {base}. Make sure you have access to this "
610                f"dataset and are using the correct API key.")
611        elif response.status_code != 200:
612            raise FourcatImportException(
613                f"Unexpected error while requesting {component} for dataset {dataset_key} from server {base}: {response.text}")
614
615        return response
616
617    @staticmethod
618    def validate_query(query, request, config):
619        """
620        Validate custom data input
621
622        Confirms that the uploaded file is a valid CSV or tab file and, if so, returns
623        some metadata.
624
625        :param dict query:  Query parameters, from client-side.
626        :param request:  Flask request
627        :param ConfigManager|None config:  Configuration reader (context-aware)
628        :return dict:  Safe query parameters
629        """
630        if query.get("method") == "zip":
631            filename = ""
632            if "option-data_upload-entries" in request.form:
633                # First pass sends list of files in the zip
634                pass
635            elif "option-data_upload" in request.files:
636                # Second pass sends the actual file
637                file = request.files["option-data_upload"]
638                if not file:
639                    raise QueryParametersException("No file uploaded.")
640
641                if not file.filename.endswith(".zip"):
642                    raise QueryParametersException("Uploaded file must be a ZIP file.")
643
644                filename = file.filename
645            else:
646                raise QueryParametersException("No file was offered for upload.")
647
648            return {
649                "method": "zip",
650                "filename": filename
651            }
652        elif query.get("method") == "url":
653            urls = query.get("url")
654            if not urls:
655                raise QueryParametersException("Provide at least one dataset URL.")
656
657            urls = urls.split(",")
658            bases = set([url.split("/results/")[0].lower() for url in urls])
659            keys = SearchImportFromFourcat.get_keys_from_urls(urls)
660
661            if len(keys) != 1:
662                # todo: change this to < 1 if we allow multiple datasets
663                raise QueryParametersException("You need to provide a single URL to a 4CAT dataset to import.")
664
665            if len(bases) != 1:
666                raise QueryParametersException("All URLs need to point to the same 4CAT server. You can only import from "
667                                                "one 4CAT server at a time.")
668
669            base = urls[0].split("/results/")[0]
670            try:
671                # test if API key is valid and server is reachable
672                test = SearchImportFromFourcat.fetch_from_4cat(base, keys[0], query.get("api-key"), "metadata")
673            except FourcatImportException as e:
674                raise QueryParametersException(str(e))
675
676            try:
677                # test if we get a response we can parse
678                metadata = test.json()
679            except ValueError:
680                raise QueryParametersException(f"Unexpected response when trying to fetch metadata for dataset {keys[0]}.")
681
682            version = get_software_version()
683
684            if metadata.get("current_4CAT_version") != version:
685                raise QueryParametersException(f"This 4CAT server is running a different version of 4CAT ({version}) than "
686                                               f"the one you are trying to import from ({metadata.get('current_4CAT_version')}). Make "
687                                               "sure both are running the same version of 4CAT and try again.")
688
689            # OK, we can import at least one dataset
690            return {
691                "url": ",".join(urls),
692                "api-key": query.get("api-key")
693            }
694        else:
695            raise QueryParametersException("Import method not yet implemented.")
696
697    @staticmethod
698    def get_keys_from_urls(urls):
699        """
700        Get dataset keys from 4CAT URLs
701
702        :param list urls:  List of URLs
703        :return list:  List of keys
704        """
705        return [url.split("/results/")[-1].split("/")[0].split("#")[0].split("?")[0] for url in urls]
706
707    @staticmethod
708    def ensure_key(query):
709        """
710        Determine key for dataset generated by this processor
711
712        When importing datasets, it's necessary to determine the key of the
713        dataset that is created before it is actually created, because we want
714        to keep the original key of the imported dataset if possible. Luckily,
715        we can deduce it from the URL we're importing the dataset from.
716
717        :param dict query:  Input from the user, through the front-end
718        :return str:  Desired dataset key
719        """
720        #TODO: Can this be done for the zip method as well? The original keys are in the zip file; we save them after
721        # this method is called via `after_create`. We could download here and also identify the primary dataset key...
722        urls = query.get("url", "").split(",")
723        keys = SearchImportFromFourcat.get_keys_from_urls(urls)
724        return keys[0]