source: phonelux_scrappers/scrappers/handy_scrapper.py@ 48f3030

Last change on this file since 48f3030 was b68ae8d, checked in by Marko <Marko@…>, 2 years ago

Created spring app, edited setec_scrapper

  • Property mode set to 100644
File size: 2.7 KB
Line 
1import unicodedata
2from datetime import datetime
3import psycopg2
4import config_read
5from bs4 import BeautifulSoup
6from selenium import webdriver
7import requests
8
9import sys
10
11file_path = 'outputfile.txt'
12sys.stdout = open(file_path, "w")
13
14# Call to read the configuration file and connect to database
15cinfo = config_read.get_databaseconfig("../postgresdb.config")
16db_connection = psycopg2.connect(
17 database=cinfo[0],
18 host=cinfo[1],
19 user=cinfo[2],
20 password=cinfo[3]
21)
22cur = db_connection.cursor()
23
24offer_shop = "Handy" # offer shop
25last_updated = datetime.now().date()
26is_validated = False
27
28handy_url = 'https://www.handy.mk/telefoni?page=6'
29
30response1 = requests.get(handy_url)
31soup1 = BeautifulSoup(response1.content, 'html.parser')
32
33phones = soup1.find_all('li', {'data-hook': 'product-list-grid-item'})
34
35for phone in phones:
36 offer_url = phone.find('a').get('href')
37 offer_name = phone.find('div', {'data-hook': 'not-image-container'})\
38 .find('h3', {'data-hook': 'product-item-name'}).get_text().strip()
39 brand = offer_name.split(' ')[0].capitalize()
40 price = int(float(phone.find('div', {'data-hook': 'not-image-container'}).find('div', {'data-hook': "product-item-product-details"})\
41 .find('span', {'data-hook': 'product-item-price-to-pay'}).get_text().strip().replace('ден', '').replace('.', '').replace(',', '.')))
42
43 response2 = requests.get(offer_url)
44 soup2 = BeautifulSoup(response2.text, 'html.parser')
45
46 color_section = soup2.find('section', {'data-hook': 'product-colors-title-section'})
47
48 color = None
49 if color_section is not None:
50 temp_colors = color_section.find('fieldset', {'class': 'ColorPickerbase3563640754__container'})\
51 .find_all('input', {'type': 'radio'})
52 colors_list = []
53 for temp_color in temp_colors:
54 colors_list.append(temp_color.get('aria-label'))
55 color = ','.join(colors_list)
56
57 rows = soup2.find('div', {'data-hook': 'info-section-description'}).find_all('li')
58
59 if len(rows) == 0:
60 rows = soup2.find('div', {'data-hook': 'info-section-description'}).find_all('tr')
61
62 specifications = []
63
64 for row in rows:
65 specifications.append(unicodedata.normalize('NFKD', row.get_text().strip()))
66
67 offer_description = '\n'.join(specifications)
68
69 insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name , price, offer_url, ' \
70 'offer_description, last_updated, is_validated)' \
71 ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s);'
72 insert_value = (offer_shop, brand, offer_name, price, offer_url, offer_description,
73 last_updated, is_validated)
74 cur.execute(insert_script, insert_value)
75 db_connection.commit()
76
77cur.close()
78db_connection.close()
Note: See TracBrowser for help on using the repository browser.