datasources.ninegag.search_9gag

 1"""
 2Import scraped 9gag data
 3
 4It's prohibitively difficult to scrape data from 9gag within 4CAT itself due
 5to its aggressive rate limiting. Instead, import data collected elsewhere.
 6"""
 7from datetime import datetime
 8
 9from backend.lib.search import Search
10from common.lib.item_mapping import MappedItem
11
12
13class SearchNineGag(Search):
14    """
15    Import scraped 9gag data
16    """
17    type = "ninegag-search"  # job ID
18    category = "Search"  # category
19    title = "Import scraped 9gag data"  # title displayed in UI
20    description = "Import 9gag data collected with an external tool such as Zeeschuimer."  # description displayed in UI
21    extension = "ndjson"  # extension of result file, used internally and in UI
22    is_from_zeeschuimer = True
23
24    # not available as a processor for existing datasets
25    accepts = [None]
26    references = [
27        "[Zeeschuimer browser extension](https://github.com/digitalmethodsinitiative/zeeschuimer)",
28        "[Worksheet: Capturing TikTok data with Zeeschuimer and 4CAT](https://tinyurl.com/nmrw-zeeschuimer-tiktok)"
29    ]
30
31    def get_items(self, query):
32        """
33        Run custom search
34
35        Not available for 9gag
36        """
37        raise NotImplementedError("9gag datasets can only be created by importing data from elsewhere")
38
39    @staticmethod
40    def map_item(post):
41        post_timestamp = datetime.fromtimestamp(post["creationTs"])
42
43        image = sorted([v for v in post["images"].values() if not "hasAudio" in v], key=lambda image: image["width"] * image["height"], reverse=True)[0]
44        video = sorted([v for v in post["images"].values() if "hasAudio" in v], key=lambda image: image["width"] * image["height"], reverse=True)
45
46        video_url = ""
47        if video:
48            # annoyingly, not all formats are always available
49            video = video[0]
50            if "av1Url" in video:
51                video_url = video["av1Url"]
52            elif "h265Url" in video:
53                video_url = video["h265Url"]
54            elif "vp9Url" in video:
55                video_url = video["vp9Url"]
56            elif "vp8Url" in video:
57                video_url = video["vp8Url"]
58
59        if not post["creator"]:
60            # anonymous posts exist
61            # they display as from the user '9GAGGER' on the website
62            post["creator"] = {
63                "username": "9GAGGER",
64                "fullName": "",
65                "emojiStatus": "",
66                "isVerifiedAccount": ""
67            }
68
69        return MappedItem({
70            "id": post["id"],
71            "url": post["url"],
72            "subject": post["title"],
73            "body": post["description"],
74            "timestamp": post_timestamp.strftime("%Y-%m-%d %H:%M:%S"),
75            "author": post["creator"]["username"],
76            "author_name": post["creator"]["fullName"],
77            "author_status": post["creator"]["emojiStatus"],
78            "author_verified": "yes" if post["creator"]["isVerifiedAccount"] else "no",
79            "type": post["type"],
80            "image_url": image["url"],
81            "video_url": video_url,
82            "is_nsfw": "no" if post["nsfw"] == 0 else "yes",
83            "is_promoted": "no" if post["promoted"] == 0 else "yes",
84            "is_vote_masked": "no" if post["isVoteMasked"] == 0 else "yes",
85            "is_anonymous": "no" if not post["isAnonymous"] else "yes",
86            "source_domain": post["sourceDomain"],
87            "source_url": post["sourceUrl"],
88            "upvotes": post["upVoteCount"],
89            "downvotes": post["downVoteCount"],
90            "score": post["upVoteCount"] - post["downVoteCount"],
91            "comments": post["commentsCount"],
92            "tags": ",".join([tag["key"] for tag in post["tags"]]),
93            "tags_annotated": ",".join(post["annotationTags"]),
94            "unix_timestamp": int(post_timestamp.timestamp()),
95        })