datasources.truth.search_truth

  1"""
  2Import scraped Truth Social data
  3"""
  4import datetime
  5import re
  6
  7from backend.lib.search import Search
  8
  9
 10class SearchGab(Search):
 11    """
 12    Import scraped truth social data
 13    """
 14    type = "truthsocial-search"  # job ID
 15    category = "Search"  # category
 16    title = "Import scraped Truth Social data"  # title displayed in UI
 17    description = "Import Truth Social data collected with an external tool such as Zeeschuimer."  # description displayed in UI
 18    extension = "ndjson"  # extension of result file, used internally and in UI
 19    is_from_zeeschuimer = True
 20    fake = ""
 21
 22    # not available as a processor for existing datasets
 23    accepts = [None]
 24
 25    def get_items(self, query):
 26        """
 27        Run custom search
 28
 29        Not available for Truth Social
 30        """
 31        raise NotImplementedError("Truth Social datasets can only be created by importing data from elsewhere")
 32
 33    @staticmethod
 34    def map_item(post):
 35        """
 36        Parse Truth Social post
 37
 38        :param node:  Data as received from Truth Social
 39        :return dict:  Mapped item
 40        """
 41        
 42        post_time = datetime.datetime.strptime(post["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ")
 43        media_length = 0
 44        if "media_attachments" in post:
 45            media_length = len(post["media_attachments"])
 46
 47        if media_length > 0:
 48            mapped_item = {
 49                "id": post["id"],
 50                "created_at": post["created_at"],
 51                "body": post["content"],
 52                "url": post.get("url", None),
 53                "reblogs_count": post.get("reblogs_count", 0),
 54                "replies_count": post.get("replies_count", 0),
 55                
 56                "account_id": post["account"]["id"],
 57                "account_username": post["account"]["username"],
 58                "account_display_name": post["account"]["display_name"],
 59                "account_avatar": post["account"]["avatar"],
 60                "account_verified": post["account"]["verified"],
 61                "account_followers": post["account"]["followers_count"],
 62                "account_following": post["account"]["following_count"],
 63
 64                "media_id": post["media_attachments"][0].get("id", None),
 65                "media_type": post["media_attachments"][0].get("type", None),
 66                "media_url": post["media_attachments"][0].get("url", None),
 67                "media_preview_url": post["media_attachments"][0].get("preview_url", None),
 68
 69                #"group_id": post["group"].get("id", None),
 70                #"group_display_name": post["group"].get("display_name", None),
 71                #"group_avatar": post["group"].get("avatar", None),
 72                #"group_header": post["group"].get("header", None),
 73                #"group_members_count": post["group"].get("members_count", 0),
 74
 75                "thread_id": post["id"],
 76                "timestamp": post_time.strftime("%Y-%m-%d %H:%M:%S")
 77            }        
 78
 79        else:
 80            mapped_item = {
 81                "id": post["id"],
 82                "created_at": post["created_at"],
 83                "body": post["content"],
 84                "url": post.get("url", None),
 85                "reblogs_count": post.get("reblogs_count", 0),
 86                "replies_count": post.get("replies_count", 0),
 87                
 88                "account_id": post["account"]["id"],
 89                "account_username": post["account"]["username"],
 90                "account_display_name": post["account"]["display_name"],
 91                "account_avatar": post["account"]["avatar"],
 92                "account_verified": post["account"]["verified"],
 93                "account_followers": post["account"]["followers_count"],
 94                "account_following": post["account"]["following_count"],
 95
 96                #"group_id": post["group"].get("id", None),
 97                #"group_display_name": post["group"].get("display_name", None),
 98                #"group_avatar": post["group"].get("avatar", None),
 99                #"group_header": post["group"].get("header", None),
100                #"group_members_count": post["group"].get("members_count", 0),
101
102                "thread_id": post["id"],
103                "timestamp": post_time.strftime("%Y-%m-%d %H:%M:%S")
104            }       
105    
106        return mapped_item