datasources.xiaohongshu_comments.search_rednote_comments

 1"""
 2Import scraped RedNote comments
 3
 4It's prohibitively difficult to scrape data from RedNote within 4CAT itself due
 5to its aggressive rate limiting. Instead, import data collected elsewhere.
 6"""
 7from datetime import datetime
 8
 9from backend.lib.search import Search
10from common.lib.item_mapping import MappedItem, MissingMappedField
11
12
13class SearchRedNoteComments(Search):
14    """
15    Import scraped RedNote/Xiaohongshu/XSH comment data
16    """
17    type = "xiaohongshu-comments-search"  # job ID
18    category = "Search"  # category
19    title = "Import scraped RedNote comment data"  # title displayed in UI
20    description = "Import RedNote comment data collected with an external tool such as Zeeschuimer."  # description displayed in UI
21    extension = "ndjson"  # extension of result file, used internally and in UI
22    is_from_zeeschuimer = True
23
24    # not available as a processor for existing datasets
25    accepts = [None]
26    references = [
27        "[Zeeschuimer browser extension](https://github.com/digitalmethodsinitiative/zeeschuimer)",
28        "[Worksheet: Capturing TikTok data with Zeeschuimer and 4CAT](https://tinyurl.com/nmrw-zeeschuimer-tiktok)"
29    ]
30
31    def get_items(self, query):
32        """
33        Run custom search
34
35        Not available for RedNote
36        """
37        raise NotImplementedError("RedNote/Xiaohongshu comment datasets can only be created by importing data from elsewhere")
38
39
40    @staticmethod
41    def map_item(item):
42        """
43        Map XSH comment object to 4CAT item
44
45        Depending on whether the object was captured from JSON or HTML, treat it
46        differently. A lot of data is missing from HTML objects.
47
48        :param item:
49        :return:
50        """
51
52        timestamp = datetime.fromtimestamp(int(item["create_time"]) / 1000).strftime("%Y-%m-%d %H:%M:%S")
53
54        return MappedItem({
55            "id": item["id"],
56            "thread_id": item["note_id"],
57            "url": f"https://www.xiaohongshu.com/explore/{item['note_id']}",
58            "body": item.get("content", ""),
59            "timestamp": timestamp,
60            "author": item["user_info"]["nickname"],
61            "author_avatar_url": item["user_info"]["image"],
62            "ip_location": item["ip_location"] if item.get("ip_location") else MissingMappedField(""),
63            "likes": item["like_count"],
64            "replies": item["sub_comment_count"],
65            "unix_timestamp": int(item["create_time"] / 1000)
66        })