backend.lib.scraper

  1"""
  2Basic scraper worker - should be inherited by workers to scrape specific types of content
  3"""
  4import collections
  5import requests
  6import random
  7import json
  8import abc
  9
 10from pathlib import Path
 11from backend.lib.worker import BasicWorker
 12
 13class BasicHTTPScraper(BasicWorker, metaclass=abc.ABCMeta):
 14	"""
 15	Abstract JSON scraper class
 16
 17	The job queue is continually checked for jobs of this scraper's type. If any are found,
 18	the URL for that job is scraped and the result is parsed as JSON. The parsed JSON is
 19	then passed to a processor method for further handling.
 20	"""
 21
 22	log_level = "warning"
 23	_logger_method = None
 24	category = "Collector"
 25
 26	def __init__(self, job, logger=None, manager=None, modules=None):
 27		"""
 28		Set up database connection - we need one to store the thread data
 29		"""
 30		super().__init__(logger=logger, manager=manager, job=job, modules=modules)
 31		self.prefix = self.type.split("-")[0]
 32		# Names were updated to be more consistent with the rest of the codebase, but we still need to support the old database
 33		# TODO: update database.sql names and create migrate script, then remove this
 34		self.prefix = {
 35			"fourchan": "4chan",
 36			"eightkun": "8kun",
 37			"eightchan": "8chan",
 38		}[self.prefix]
 39
 40		if not hasattr(logger, self.log_level):
 41			self.log_level = "warning"
 42
 43		self._logger_method = getattr(logger, self.log_level)
 44
 45	def work(self):
 46		"""
 47		Scrape something
 48
 49		This requests data according to the job's parameter - either from a
 50		local file or from a URL. The job is then either finished or released
 51		depending on whether that was successful, and the data is processed
 52		further if available.
 53		"""
 54		if "file" in self.job.details:
 55			# if the file is available locally, use that file
 56			id = self.job.details["file"]
 57			local_path = Path(self.job.details["file"])
 58			if not local_path.exists():
 59				self.job.finish()
 60				self.log.error("Scraper was told to use source file %s, but file does not exist, cancelling job." % self.job.details["file"])
 61				return
 62
 63			with local_path.open() as source:
 64				datafields = {
 65					"status_code": 200,
 66					"content": source.read()
 67				}
 68
 69				data = collections.namedtuple("object", datafields.keys())(*datafields.values())
 70		else:
 71			# if not, see what URL we need to request data from
 72			url = self.get_url()
 73			try:
 74				# see if any proxies were configured that would work for this URL
 75				protocol = url.split(":")[0]
 76				if protocol in self.config.get('SCRAPE_PROXIES', []) and self.config.get('SCRAPE_PROXIES')[protocol]:
 77					proxies = {protocol: random.choice(self.config.get('SCRAPE_PROXIES')[protocol])}
 78				else:
 79					proxies = None
 80
 81				# do the request!
 82				data = requests.get(url, timeout=self.config.get('SCRAPE_TIMEOUT', 60), proxies=proxies, headers={"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Safari/605.1.15"})
 83			except (requests.exceptions.RequestException, ConnectionRefusedError) as e:
 84				if self.job.data["attempts"] > 2:
 85					self.job.finish()
 86					self.log.error("Could not finish request for %s (%s), cancelling job" % (url, e))
 87				else:
 88					self.job.release(delay=random.randint(45,60))
 89					self.log.info("Could not finish request for %s (%s), releasing job" % (url, e))
 90				return
 91
 92			if "board" in self.job.details:
 93				id = self.job.details["board"] + "/" + self.job.data["remote_id"]
 94			else:
 95				id = self.job.data["remote_id"]
 96
 97		if data.status_code == 404 or data.status_code == 403:
 98			# this should be handled differently from an actually erroneous response
 99			# because it may indicate that the resource has been deleted
100			self.not_found()
101		else:
102			parsed_data = self.parse(data.content)
103			if parsed_data is None:
104				if self.job.data["attempts"] < 2:
105					self.log.info("Data for %s %s could not be parsed, retrying later" % (self.type, id))
106					self.job.release(delay=random.choice(range(15, 45)))  # try again later
107				else:
108					self._logger_method("Data for %s %s could not be parsed after %i attempts, aborting" % (
109					self.type, id, self.job.data["attempts"]))
110					self.job.finish()
111				return
112
113			# finally, pass it on
114			self.process(parsed_data)
115			self.after_process()
116
117	def after_process(self):
118		"""
119		After processing, declare job finished
120		"""
121		self.job.finish()
122
123	def not_found(self):
124		"""
125		Called if the job could not be completed because the request returned
126		a 404 response. This does not necessarily indicate failure.
127		"""
128		self.job.finish()
129
130	def parse(self, data):
131		"""
132		Parse incoming data
133
134		Can be overridden to, e.g., parse JSON data
135
136		:param data:  Body of HTTP request
137		:return:  Parsed data
138		"""
139		return data
140
141	@abc.abstractmethod
142	def process(self, data):
143		"""
144		Process scraped data
145
146		:param data:  Parsed JSON data
147		"""
148		pass
149
150	@abc.abstractmethod
151	def get_url(self):
152		"""
153		Get URL to scrape
154
155		:return string:  URL to scrape
156		"""
157		pass
158
159
160class BasicJSONScraper(BasicHTTPScraper, metaclass=abc.ABCMeta):
161	"""
162	Scraper for JSON-based data
163	"""
164
165	def parse(self, data):
166		"""
167		Parse data as JSON
168
169		:param str data:  Incoming JSON-encoded data
170		:return:  Decoded JSON object
171		"""
172		try:
173			return json.loads(data)
174		except json.JSONDecodeError:
175			return None