datasources.ninegag.search_9gag

 1"""
 2Import scraped 9gag data
 3
 4It's prohibitively difficult to scrape data from 9gag within 4CAT itself due
 5to its aggressive rate limiting. Instead, import data collected elsewhere.
 6"""
 7from datetime import datetime
 8
 9from backend.lib.search import Search
10from common.lib.item_mapping import MappedItem
11from common.lib.helpers import normalize_url_encoding
12
13
14class SearchNineGag(Search):
15    """
16    Import scraped 9gag data
17    """
18    type = "ninegag-search"  # job ID
19    category = "Search"  # category
20    title = "Import scraped 9gag data"  # title displayed in UI
21    description = "Import 9gag data collected with an external tool such as Zeeschuimer."  # description displayed in UI
22    extension = "ndjson"  # extension of result file, used internally and in UI
23    is_from_zeeschuimer = True
24
25    # not available as a processor for existing datasets
26    accepts = [None]
27    references = [
28        "[Zeeschuimer browser extension](https://github.com/digitalmethodsinitiative/zeeschuimer)",
29        "[Worksheet: Capturing TikTok data with Zeeschuimer and 4CAT](https://tinyurl.com/nmrw-zeeschuimer-tiktok)"
30    ]
31
32    def get_items(self, query):
33        """
34        Run custom search
35
36        Not available for 9gag
37        """
38        raise NotImplementedError("9gag datasets can only be created by importing data from elsewhere")
39
40    @staticmethod
41    def map_item(post):
42        post_timestamp = datetime.fromtimestamp(post["creationTs"])
43
44        image = sorted([v for v in post["images"].values() if "hasAudio" not in v], key=lambda image: image["width"] * image["height"], reverse=True)[0]
45        video = sorted([v for v in post["images"].values() if "hasAudio" in v], key=lambda image: image["width"] * image["height"], reverse=True)
46
47        video_url = ""
48        if video:
49            # annoyingly, not all formats are always available
50            video = video[0]
51            if "av1Url" in video:
52                video_url = video["av1Url"]
53            elif "h265Url" in video:
54                video_url = video["h265Url"]
55            elif "vp9Url" in video:
56                video_url = video["vp9Url"]
57            elif "vp8Url" in video:
58                video_url = video["vp8Url"]
59
60        if not post["creator"]:
61            # anonymous posts exist
62            # they display as from the user '9GAGGER' on the website
63            post["creator"] = {
64                "username": "9GAGGER",
65                "fullName": "",
66                "emojiStatus": "",
67                "isVerifiedAccount": ""
68            }
69
70        return MappedItem({
71            "collected_from_url": normalize_url_encoding(post.get("__import_meta", {}).get("source_platform_url", "")),  # Zeeschuimer metadata
72            "id": post["id"],
73            "url": post["url"],
74            "subject": post["title"],
75            "body": post["description"],
76            "timestamp": post_timestamp.strftime("%Y-%m-%d %H:%M:%S"),
77            "author": post["creator"]["username"],
78            "author_name": post["creator"]["fullName"],
79            "author_status": post["creator"]["emojiStatus"],
80            "author_verified": "yes" if post["creator"]["isVerifiedAccount"] else "no",
81            "type": post["type"],
82            "image_url": image["url"],
83            "video_url": video_url,
84            "is_nsfw": "no" if post["nsfw"] == 0 else "yes",
85            "is_promoted": "no" if post["promoted"] == 0 else "yes",
86            "is_vote_masked": "no" if post["isVoteMasked"] == 0 else "yes",
87            "is_anonymous": "no" if not post["isAnonymous"] else "yes",
88            "source_domain": post["sourceDomain"],
89            "source_url": post["sourceUrl"],
90            "upvotes": post["upVoteCount"],
91            "downvotes": post["downVoteCount"],
92            "score": post["upVoteCount"] - post["downVoteCount"],
93            "comments": post["commentsCount"],
94            "tags": ",".join([tag["key"] for tag in post["tags"]]),
95            "tags_annotated": ",".join(post["annotationTags"]),
96            "unix_timestamp": int(post_timestamp.timestamp()),
97        })