diff --git a/product_scraping/product_scraper.py b/product_scraping/product_scraper.py index aabaf44..45e7cd0 100644 --- a/product_scraping/product_scraper.py +++ b/product_scraping/product_scraper.py @@ -675,6 +675,95 @@ class Cost_Fetcher: except Exception as e: print(f"Error: {e}") + async def fetch_single(self): + product_source = self.product_sources.loc[index_product_source] + print(f'Product source: {product_source}') + index_row = product_source[TCG_Sole_Trader_Workbook_Container.NAME_COLUMN_INDEX_ROW] + source_name = product_source[TCG_Sole_Trader_Workbook_Container.NAME_COLUMN_SOURCE_NAME] + source_link = product_source[TCG_Sole_Trader_Workbook_Container.NAME_COLUMN_LINK] + index_domain = None + try: + index_domain = self.get_index_domain_from_name(source_name) + except: + continue + domain_details = self.domain_details[source_name] + self.workbook_container.clear_row_sourcing_sheet(index_row = index_row) + processed_count += 1 + Cost_Fetcher.log_processing_new_row( + index_row = index_row + , source_link = source_link + ) + + cost = None + price = None + active = None + if source_name == self.NAME_DOMAIN_CARD_MARKET: + while (time.time() - domain_details[self.ACCESSED_LAST_ON_FLAG] < random.uniform(30, 40)): + time.sleep(random.uniform(3, 5)) + if product_source[TCG_Sole_Trader_Workbook_Container.NAME_COLUMN_PRODUCT_IS_BOOSTER]: + price_quantity_pairs = self.product_scrapers[index_domain].scrape_prices_and_quantities_playwright_cardmarket(url = source_link, eur_to_gbp_rate = self.eur_to_gbp_rate) + if price_quantity_pairs: + active = True + max_quantity = 0 + updated_row_price = False + for price_quantity_pair in price_quantity_pairs: + eur_price = price_quantity_pair['price'] + quantity = price_quantity_pair['quantity'] + print(f" Found price: €{eur_price}") + print(f" Found quantity: {quantity}") + max_quantity = max(max_quantity, quantity) + if quantity >= 8: + if eur_price: + price = eur_price * self.eur_to_gbp_rate + print(f" Converted: €{eur_price:.2f} → £{price:.2f}") + # self.workbook_container.sourcing_sheet.cell(index_row, self.workbook_container.index_column_unit_price_sourcing).value = gbp_price + updated_count += 1 + updated_row_price = True + # print(f"output row: {index_row}, value: {self.workbook_container.sourcing_sheet.cell(index_row, self.workbook_container.index_column_unit_cost_sourcing).value}") + break + else: + print(f" Error: Could not parse price") + if not updated_row_price: + print("Offer with quantity >= 8 not found") + for price_quantity_pair in price_quantity_pairs: + eur_price = price_quantity_pair['price'] + quantity = price_quantity_pair['quantity'] + print(f" Found price: €{eur_price}") + print(f" Found quantity: {quantity}") + if max_quantity <= 2 or quantity == max_quantity: + if eur_price: + price = eur_price * self.eur_to_gbp_rate + print(f" Converted: €{eur_price:.2f} → £{price:.2f}") + # self.workbook_container.sourcing_sheet.cell(index_row, self.workbook_container.index_column_unit_cost_sourcing).value = gbp_price + updated_count += 1 + updated_row_price = True + # print(f"output row: {index_row}, value: {self.workbook_container.sourcing_sheet.cell(index_row, self.workbook_container.index_column_unit_cost_sourcing).value}") + break + else: + print(f" Error: Could not parse price") + cost, active = self.product_scrapers[index_domain].scrape_cost_and_active_playwright_cardmarket(url = source_link, eur_to_gbp_rate = self.eur_to_gbp_rate) + elif source_name == self.NAME_DOMAIN_CHAOS_CARDS: + while (time.time() - self.domain_details[source_name][self.ACCESSED_LAST_ON_FLAG] < random.uniform(30, 40)): + time.sleep(random.uniform(3, 5)) + cost, active = self.product_scrapers[index_domain].scrape_cost_and_active_playwright_chaoscards(url = source_link) + elif source_name == self.NAME_DOMAIN_GAMES_LORE: + while (time.time() - self.domain_details[source_name][self.ACCESSED_LAST_ON_FLAG] < random.uniform(10, 20)): + time.sleep(random.uniform(3, 5)) + cost, active = self.product_scrapers[index_domain].scrape_cost_and_active_playwright_gameslore(url = source_link) + elif source_name == self.NAME_DOMAIN_MAGIC_MADHOUSE: + while (time.time() - self.domain_details[source_name][self.ACCESSED_LAST_ON_FLAG] < random.uniform(10, 20)): + time.sleep(random.uniform(3, 5)) + cost, active = self.product_scrapers[index_domain].scrape_cost_and_active_playwright_magicmadhouse(url = source_link) + + self.domain_details[source_name][self.ACCESSED_LAST_ON_FLAG] = time.time() + + if ((cost is not None or price is not None) and active is not None): + print(f" Found cost: {cost}, price: {price}, active: {active}") + self.workbook_container.update_row_sourcing_sheet(index_row = index_row, unit_cost = cost, unit_price = price, active = active) + updated_count += 1 + else: + print(f" Error: Could not find cost on page") + @staticmethod def log_processing_new_row(index_row, source_link): print(f"\n{'='*60}")