admin/tcg_web_scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: MTG Commander Life Tracker webpage

Browse Source
This commit is contained in:
Teddy Middleton-Smith
2026-01-10 15:23:18 +00:00
parent 84e9ef92fe
commit 3bacc4616d
1 changed files with 89 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
product_scraping/product_scraper.py
View File

@@ -675,6 +675,95 @@ class Cost_Fetcher:
except Exception as e: except Exception as e:
print(f"Error: {e}") print(f"Error: {e}")
async def fetch_single(self):
product_source = self.product_sources.loc[index_product_source]
print(f'Product source: {product_source}')
index_row = product_source[TCG_Sole_Trader_Workbook_Container.NAME_COLUMN_INDEX_ROW]
source_name = product_source[TCG_Sole_Trader_Workbook_Container.NAME_COLUMN_SOURCE_NAME]
source_link = product_source[TCG_Sole_Trader_Workbook_Container.NAME_COLUMN_LINK]
index_domain = None
try:
index_domain = self.get_index_domain_from_name(source_name)
except:
continue
domain_details = self.domain_details[source_name]
self.workbook_container.clear_row_sourcing_sheet(index_row = index_row)
processed_count += 1
Cost_Fetcher.log_processing_new_row(
index_row = index_row
, source_link = source_link
)
cost = None
price = None
active = None
if source_name == self.NAME_DOMAIN_CARD_MARKET:
while (time.time() - domain_details[self.ACCESSED_LAST_ON_FLAG] < random.uniform(30, 40)):
time.sleep(random.uniform(3, 5))
if product_source[TCG_Sole_Trader_Workbook_Container.NAME_COLUMN_PRODUCT_IS_BOOSTER]:
price_quantity_pairs = self.product_scrapers[index_domain].scrape_prices_and_quantities_playwright_cardmarket(url = source_link, eur_to_gbp_rate = self.eur_to_gbp_rate)
if price_quantity_pairs:
active = True
max_quantity = 0
updated_row_price = False
for price_quantity_pair in price_quantity_pairs:
eur_price = price_quantity_pair['price']
quantity = price_quantity_pair['quantity']
print(f" Found price: €{eur_price}")
print(f" Found quantity: {quantity}")
max_quantity = max(max_quantity, quantity)
if quantity >= 8:
if eur_price:
price = eur_price * self.eur_to_gbp_rate
print(f" Converted: €{eur_price:.2f} → £{price:.2f}")
# self.workbook_container.sourcing_sheet.cell(index_row, self.workbook_container.index_column_unit_price_sourcing).value = gbp_price
updated_count += 1
updated_row_price = True
# print(f"output row: {index_row}, value: {self.workbook_container.sourcing_sheet.cell(index_row, self.workbook_container.index_column_unit_cost_sourcing).value}")
break
else:
print(f" Error: Could not parse price")
if not updated_row_price:
print("Offer with quantity >= 8 not found")
for price_quantity_pair in price_quantity_pairs:
eur_price = price_quantity_pair['price']
quantity = price_quantity_pair['quantity']
print(f" Found price: €{eur_price}")
print(f" Found quantity: {quantity}")
if max_quantity <= 2 or quantity == max_quantity:
if eur_price:
price = eur_price * self.eur_to_gbp_rate
print(f" Converted: €{eur_price:.2f} → £{price:.2f}")
# self.workbook_container.sourcing_sheet.cell(index_row, self.workbook_container.index_column_unit_cost_sourcing).value = gbp_price
updated_count += 1
updated_row_price = True
# print(f"output row: {index_row}, value: {self.workbook_container.sourcing_sheet.cell(index_row, self.workbook_container.index_column_unit_cost_sourcing).value}")
break
else:
print(f" Error: Could not parse price")
cost, active = self.product_scrapers[index_domain].scrape_cost_and_active_playwright_cardmarket(url = source_link, eur_to_gbp_rate = self.eur_to_gbp_rate)
elif source_name == self.NAME_DOMAIN_CHAOS_CARDS:
while (time.time() - self.domain_details[source_name][self.ACCESSED_LAST_ON_FLAG] < random.uniform(30, 40)):
time.sleep(random.uniform(3, 5))
cost, active = self.product_scrapers[index_domain].scrape_cost_and_active_playwright_chaoscards(url = source_link)
elif source_name == self.NAME_DOMAIN_GAMES_LORE:
while (time.time() - self.domain_details[source_name][self.ACCESSED_LAST_ON_FLAG] < random.uniform(10, 20)):
time.sleep(random.uniform(3, 5))
cost, active = self.product_scrapers[index_domain].scrape_cost_and_active_playwright_gameslore(url = source_link)
elif source_name == self.NAME_DOMAIN_MAGIC_MADHOUSE:
while (time.time() - self.domain_details[source_name][self.ACCESSED_LAST_ON_FLAG] < random.uniform(10, 20)):
time.sleep(random.uniform(3, 5))
cost, active = self.product_scrapers[index_domain].scrape_cost_and_active_playwright_magicmadhouse(url = source_link)
self.domain_details[source_name][self.ACCESSED_LAST_ON_FLAG] = time.time()
if ((cost is not None or price is not None) and active is not None):
print(f" Found cost: {cost}, price: {price}, active: {active}")
self.workbook_container.update_row_sourcing_sheet(index_row = index_row, unit_cost = cost, unit_price = price, active = active)
updated_count += 1
else:
print(f" Error: Could not find cost on page")
@staticmethod @staticmethod
def log_processing_new_row(index_row, source_link): def log_processing_new_row(index_row, source_link):
print(f"\n{'='*60}") print(f"\n{'='*60}")