datasources.truth.search_truth

  1"""
  2Import scraped Truth Social data
  3"""
  4import datetime
  5
  6from backend.lib.search import Search
  7from common.lib.item_mapping import MappedItem
  8
  9
 10class SearchGab(Search):
 11    """
 12    Import scraped truth social data
 13    """
 14    type = "truthsocial-search"  # job ID
 15    category = "Search"  # category
 16    title = "Import scraped Truth Social data"  # title displayed in UI
 17    description = "Import Truth Social data collected with an external tool such as Zeeschuimer."  # description displayed in UI
 18    extension = "ndjson"  # extension of result file, used internally and in UI
 19    is_from_zeeschuimer = True
 20    fake = ""
 21
 22    # not available as a processor for existing datasets
 23    accepts = [None]
 24
 25    def get_items(self, query):
 26        """
 27        Run custom search
 28
 29        Not available for Truth Social
 30        """
 31        raise NotImplementedError("Truth Social datasets can only be created by importing data from elsewhere")
 32
 33    @staticmethod
 34    def map_item(post):
 35        """
 36        Parse Truth Social post
 37
 38        :param post:  Data as received from Truth Social
 39        :return dict:  Mapped item
 40        """
 41        errors = []
 42        post_time = datetime.datetime.strptime(post["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ")
 43        images = []
 44        videos = []
 45        video_thumbs = []
 46        if "media_attachments" in post:
 47            for media in post["media_attachments"]:
 48                mtype = media.get("type")
 49                if mtype == "image":
 50                    images.append(media.get("url"))
 51                elif mtype == "video":
 52                    videos.append(media.get("url"))
 53                    video_thumbs.append(media.get("preview_url"))
 54                elif mtype == "tv":
 55                    # Truth social has "TV channels" with videos
 56                    # These do not have direct links to media
 57                    # url is a thumbnail
 58                    video_thumbs.append(media.get("url"))
 59                    # preview_url is a smaller thumb
 60                else:
 61                    errors.append(f"New media type: {mtype}")
 62
 63        group = post.get("group") if post.get("group") else {}
 64        
 65        if post.get("quote_id", None):
 66            thread_id = post.get("quote_id")
 67        elif post.get("in_reply_to", None):
 68            reply_to = post.get("in_reply_to")
 69            while reply_to:
 70                if reply_to.get("in_reply_to", None):
 71                    reply_to = reply_to.get("in_reply_to")
 72                else:
 73                    thread_id = reply_to.get("id")
 74                    break
 75        else:
 76            thread_id = post.get("id")
 77        
 78        mentions = [mention.get("username") for mention in (post.get("mentions") if post.get("mentions") else [])]
 79        hashtags = [tag.get("name") for tag in (post.get("tags") if post.get("tags") else [])]
 80    
 81        mapped_item = {
 82            "id": post["id"],
 83            "created_at": post["created_at"],
 84            "body": post["content"],
 85            "url": post.get("url", None),
 86            "reblogs_count": post.get("reblogs_count", 0),
 87            "replies_count": post.get("replies_count", 0),
 88
 89            "account_id": post["account"]["id"],
 90            "account_username": post["account"]["username"],
 91            "account_display_name": post["account"]["display_name"],
 92            "account_avatar": post["account"]["avatar"],
 93            "account_verified": post["account"]["verified"],
 94            "account_followers": post["account"]["followers_count"],
 95            "account_following": post["account"]["following_count"],
 96            
 97            "mentions": ",".join(mentions),
 98            "hashtags": ",".join(hashtags),
 99
100            # media
101            "images": ",".join(images),
102            "video_thumbs": ",".join(video_thumbs),
103            "video_urls": ",".join(videos),
104            
105            # group
106            "group_id": group.get("id", None),
107            "group_display_name": group.get("display_name", None),
108            "group_avatar": group.get("avatar", None),
109            "group_note": group.get("note", None),
110            "group_members_count": group.get("members_count", 0),
111
112            "thread_id": thread_id,
113            "timestamp": post_time.strftime("%Y-%m-%d %H:%M:%S")
114        }        
115
116        return MappedItem(mapped_item, message="; ".join(errors))