Index: phonelux_scrappers/classes/phone.py =================================================================== --- phonelux_scrappers/classes/phone.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) +++ phonelux_scrappers/classes/phone.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -0,0 +1,9 @@ +class Phone: + + def __init__(self, id, brand, model, image_url, total_offers, lowest_price): + self.id = id + self.brand = brand + self.model = model + self.image_url = image_url + self.total_offers = total_offers + self.lowest_price = lowest_price Index: phonelux_scrappers/classes/phoneoffer.py =================================================================== --- phonelux_scrappers/classes/phoneoffer.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) +++ phonelux_scrappers/classes/phoneoffer.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -0,0 +1,50 @@ +import json + + +class PhoneOffer: + + def __init__(self, *args): + if len(args) == 19: + self.offer_id = args[0] + self.offer_shop = args[1] + self.offer_name = args[2] + self.price = args[3] + self.ram_memory = args[4] + self.rom_memory = args[5] + self.color = args[6] + self.front_camera = args[7] + self.back_camera = args[8] + self.chipset = args[9] + self.battery = args[10] + self.operating_system = args[11] + self.cpu = args[12] + self.image_url = args[13] + self.offer_url = args[14] + self.last_updated = args[15] + self.is_validated = args[16] + self.offer_description = args[17] + self.offer_shop_code = args[18] + + if len(args) == 18: + self.offer_shop = args[0] + self.offer_name = args[1] + self.price = args[2] + self.ram_memory = args[3] + self.rom_memory = args[4] + self.color = args[5] + self.front_camera = args[6] + self.back_camera = args[7] + self.chipset = args[8] + self.battery = args[9] + self.operating_system = args[10] + self.cpu = args[11] + self.image_url = args[12] + self.offer_url = args[13] + self.last_updated = args[14] + self.is_validated = args[15] + self.offer_description = args[16] + self.offer_shop_code = args[17] + + def __str__(self): + return str(self.__dict__) + Index: phonelux_scrappers/lowest_price_setter.py =================================================================== --- phonelux_scrappers/lowest_price_setter.py (revision 48f3030e929e5d3717498b6e78a9813429162fde) +++ phonelux_scrappers/lowest_price_setter.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -1,6 +1,6 @@ - +import json import psycopg2 +import requests import config_read - import sys @@ -8,30 +8,17 @@ sys.stdout = open(file_path, "w") -# Call to read the configuration file and connect to database -cinfo = config_read.get_databaseconfig("postgresdb.config") -db_connection = psycopg2.connect( - database=cinfo[0], - host=cinfo[1], - user=cinfo[2], - password=cinfo[3] -) -cur = db_connection.cursor() - -cur.execute('SELECT id,model FROM phones ORDER BY id;') - -phones = cur.fetchall() +phones = json.loads(requests.get('http://localhost:8080/phones').text) for phone in phones: - phone_id = str(phone[0]) - print('id: '+phone_id) - cur.execute('SELECT offer_name,price FROM phone_offers WHERE phone_id=' + phone_id + ' ORDER BY price;') - details = cur.fetchone() - lowestPrice = None + phone_id = str(phone['id']) + print(phone['model']) - if details is not None: - lowestPrice = details[1] - cur.execute('UPDATE phones SET lowest_price='+str(lowestPrice)+' WHERE id='+phone_id+';') - db_connection.commit() + offers = list(json.loads(requests.get('http://localhost:8080/phones/offers/' + phone_id).text)) + offers.sort(key=lambda o: o['price']) + lowest_price = str(0) + if len(offers) > 0: + lowest_price = str(offers[0]['price']) + print(lowest_price) -cur.close() -db_connection.close() + # UPDATE DATABASE WITH NEW LOWEST PRICE + requests.put('http://localhost:8080/setlowestprice/' + phone_id + '/' + lowest_price) Index: phonelux_scrappers/outputfile.txt =================================================================== --- phonelux_scrappers/outputfile.txt (revision 48f3030e929e5d3717498b6e78a9813429162fde) +++ phonelux_scrappers/outputfile.txt (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -1,571 +1,588 @@ -phone id: 3 -https://admin.ledikom.mk/uploads/items/1500/1649755126honor-50-1-250x300-pad.jpg?v=1 -phone id: 4 -https://setec.mk/image/cache/catalog/Product/74172_0-228x228.jpg -phone id: 5 -https://admin.ledikom.mk/uploads/items/412/1641670848apple-iphone-13-1-250x300-pad.jpg?v=1 -phone id: 6 -https://admin.ledikom.mk/uploads/items/413/1641671221apple-iphone-13-pro-1-250x300-pad.jpg?v=1 -phone id: 7 -https://admin.ledikom.mk/uploads/items/411/1641668143apple-iphone-13-pro-max-1-250x300-pad.jpg?v=1 -phone id: 8 -https://admin.ledikom.mk/uploads/items/342/1641664916apple-iphone-12-1-250x300-pad.jpg?v=1 -phone id: 9 -https://admin.ledikom.mk/uploads/items/192/1641654006apple-iphone-11-1-250x300-pad.jpg?v=1 -phone id: 10 -https://mobitech.mk/wp-content/uploads/2022/01/apple-iphone-xr-new.jpg -phone id: 12 -https://admin.ledikom.mk/uploads/items/372/1642792043samsung-galaxy-a32-1-250x300-pad.jpg?v=1 -phone id: 13 -https://admin.ledikom.mk/uploads/items/1501/1649755462honor-50-lite-1-250x300-pad.jpg?v=1 -phone id: 14 -https://mobitech.mk/wp-content/uploads/2021/12/apple-iphone-7r4-1.jpg -phone id: 15 -https://mobitech.mk/wp-content/uploads/2021/12/apple-iphone-x-1.jpg -phone id: 16 -https://mobitech.mk/wp-content/uploads/2021/12/apple-iphone-xs-new.jpg -phone id: 18 -https://admin.ledikom.mk/uploads/items/83/1653051611nokia-105-2017-duos-1-250x300-pad.jpg?v=1 -phone id: 19 -https://admin.ledikom.mk/uploads/items/1478/1647544681samsung-galaxy-s22-5g-1-250x300-pad.jpg?v=1 -phone id: 20 -https://admin.ledikom.mk/uploads/items/392/1642792789samsung-galaxy-a22-5g-1-250x300-pad.jpg?v=1 -phone id: 21 -https://admin.ledikom.mk/uploads/items/1563/1651071492samsung-galaxy-a33-5g-1-250x300-pad.jpg?v=1 -phone id: 22 -https://www.neptun.mk/2022/04/07/23_e3fe9e55-914d-4ac3-89af-8aea5e659341.JPG?width=192 -phone id: 23 -https://admin.ledikom.mk/uploads/items/1497/1649754127samsung-galaxy-a13-1-250x300-pad.jpg?v=1 -phone id: 24 -https://mobelix.com.mk/storage/app/uploads/public/5eb/2bf/b67/thumb_2064_550_800_0_0_crop.jpg -phone id: 25 -https://admin.ledikom.mk/uploads/items/3036/1655459489xiaomi-redmi-10-1-250x300-pad.jpg?v=1 -phone id: 26 -https://admin.ledikom.mk/uploads/items/1499/1649754737samsung-galaxy-a53-5g-1-250x300-pad.jpg?v=1 -phone id: 27 -https://www.neptun.mk/2021/10/09/15571370a_e41b8951-dd67-4951-a899-104b37ff4f9d.jpg?width=192 -phone id: 28 -https://admin.ledikom.mk/uploads/items/344/1642708501samsung-galaxy-s21-5g-1-250x300-pad.jpg?v=1 -phone id: 30 -https://mobigo.mk/wp-content/uploads/2021/12/huawei-nova-9-5g.jpg -phone id: 32 -https://admin.ledikom.mk/uploads/items/406/1642794048samsung-a03s-1-250x300-pad.jpg?v=1 -phone id: 34 -https://mobigo.mk/wp-content/uploads/2021/07/xiaomi-redmi-note10-pro.jpg -phone id: 35 -https://mobigo.mk/wp-content/uploads/2021/01/xiaomi-mi-10t-5g-pro.jpg -phone id: 37 -https://www.a1.mk/documents/20126/2021612/POCO_X3_Pro.png -phone id: 38 -https://admin.ledikom.mk/uploads/items/340/1642705059samsung-galaxy-a02s-1-250x300-pad.jpg?v=1 -phone id: 40 -https://admin.ledikom.mk/uploads/items/390/mi10tlite5g-978742-250x300-pad.jpg?v=1 -phone id: 41 -https://mobigo.mk/wp-content/uploads/2020/07/samsung-galaxy-a10s.jpg -phone id: 42 -https://setec.mk/image/cache/catalog/Product/73862_0-228x228.jpg -phone id: 43 -https://admin.ledikom.mk/uploads/items/351/1653051682nokia-3310-2017-1-250x300-pad.jpg?v=1 -phone id: 44 -https://mobigo.mk/wp-content/uploads/2019/07/huawei-p30-pro.jpg -phone id: 45 -https://mobigo.mk/wp-content/uploads/2019/02/huawei-mate-20-pro-1.jpg -phone id: 46 -https://admin.ledikom.mk/uploads/items/364/1642789715samsung-galaxy-a72-1-250x300-pad.jpg?v=1 -phone id: 47 -https://mobelix.com.mk/storage/app/uploads/public/605/319/381/thumb_2496_550_800_0_0_crop.jpg -phone id: 48 -https://mobelix.com.mk/storage/app/uploads/public/605/31a/2d7/thumb_2499_550_800_0_0_crop.jpg -phone id: 54 -https://mobelix.com.mk/storage/app/uploads/public/622/360/d34/thumb_3692_550_800_0_0_crop.png -phone id: 55 -https://mobelix.com.mk/storage/app/uploads/public/5f8/037/ce9/thumb_2308_550_800_0_0_crop.png -phone id: 56 -https://admin.ledikom.mk/uploads/items/395/mi10lite5g-836560-250x300-pad.jpg?v=1 -phone id: 57 -https://mobelix.com.mk/storage/app/uploads/public/5f2/bc0/63d/thumb_2194_550_800_0_0_crop.jpg -phone id: 58 -https://mobelix.com.mk/storage/app/uploads/public/5dd/3c7/29c/thumb_1944_550_800_0_0_crop.jpg -phone id: 59 -https://mobelix.com.mk/storage/app/uploads/public/623/876/e71/thumb_3713_550_800_0_0_crop.png -phone id: 61 -https://mobelix.com.mk/storage/app/uploads/public/5de/4e7/f26/thumb_1978_550_800_0_0_crop.jpg -phone id: 62 -https://mobelix.com.mk/storage/app/uploads/public/5e5/3c4/5e6/thumb_2034_550_800_0_0_crop.png -phone id: 63 -https://mobelix.com.mk/storage/app/uploads/public/5de/4e7/49d/thumb_1976_550_800_0_0_crop.jpg -phone id: 64 -https://mobelix.com.mk/storage/app/uploads/public/5c9/0dc/413/thumb_1773_550_800_0_0_crop.jpg -phone id: 65 -https://mobelix.com.mk/storage/app/uploads/public/5cd/053/0ab/thumb_1811_550_800_0_0_crop.jpg -phone id: 68 -https://mobelix.com.mk/storage/app/uploads/public/5bc/877/975/thumb_686_550_800_0_0_crop.jpg -phone id: 69 -https://admin.ledikom.mk/uploads/items/436/1653051842nokia-6310-2021-1-250x300-pad.jpg?v=1 -phone id: 70 -https://admin.ledikom.mk/uploads/items/315/1644588871xiaomi-poco-x3-nfc-1-250x300-pad.jpg?v=1 -phone id: 71 -https://mobelix.com.mk/storage/app/uploads/public/5fd/343/c47/thumb_2416_550_800_0_0_crop.jpg -phone id: 73 -https://www.neptun.mk/2022/01/13/1150569a91-b4ec-4484-a977-da071903f48b_37266053-7f21-42d2-bde9-ebb29a1220c6.jpg?width=192 -phone id: 74 -https://mobelix.com.mk/storage/app/uploads/public/5ff/f06/b11/thumb_2451_550_800_0_0_crop.png -phone id: 75 -https://admin.ledikom.mk/uploads/items/345/1642711031samsung-galaxy-s21-5g-1-250x300-pad.jpg?v=1 -phone id: 76 -https://admin.ledikom.mk/uploads/items/362/1653383457asus-rog-phone-5-1-250x300-pad.jpg?v=1 -phone id: 77 -https://admin.ledikom.mk/uploads/items/343/1642708100samsung-galaxy-s21-ultra-5g-1-250x300-pad.jpg?v=1 -phone id: 79 -https://mobelix.com.mk/storage/app/uploads/public/606/eff/40a/thumb_2684_550_800_0_0_crop.png -phone id: 80 -https://mobelix.com.mk/storage/app/uploads/public/609/2af/b4b/thumb_3135_550_800_0_0_crop.png -phone id: 81 -https://mobelix.com.mk/storage/app/uploads/public/609/2af/c8d/thumb_3137_550_800_0_0_crop.png -phone id: 82 -https://admin.ledikom.mk/uploads/items/359/1642789645samsung-galaxy-a52-1-250x300-pad.jpg?v=1 -phone id: 83 -https://admin.ledikom.mk/uploads/items/379/1653381685xiaomi-redmi-note-10-pro-max-1-250x300-pad.jpg?v=1 -phone id: 85 -https://setec.mk/image/cache/catalog/Product/60645_0-228x228.jpg -phone id: 86 -https://admin.ledikom.mk/uploads/items/354/1642788228samsung-galaxy-m02s-1-250x300-pad.jpg?v=1 -phone id: 87 -https://mobelix.com.mk/storage/app/uploads/public/60d/ed0/0f3/thumb_3191_550_800_0_0_crop.png -phone id: 88 -https://mobelix.com.mk/storage/app/uploads/public/60d/eef/d6a/thumb_3193_550_800_0_0_crop.png -phone id: 89 -https://admin.ledikom.mk/uploads/items/398/1651156416xiaomi-mi-11-ultra-1-250x300-pad.jpg?v=1 -phone id: 90 -https://www.neptun.mk/2021/05/18/22222_28c3ee3c-d029-4ccd-a5a6-64aa1ab03fb2.jpg?width=192 -phone id: 91 -https://mobelix.com.mk/storage/app/uploads/public/5ee/0f2/e81/thumb_2131_550_800_0_0_crop.png -phone id: 98 -https://www.a1.mk/documents/20126/2346005/Honor-X7-Front.png -phone id: 102 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/22032022111538download(1).jpg -phone id: 106 -https://www.neptun.mk/2022/02/22/xiaomiredminote11azulestelar01l_367116a3-c5cb-46a0-af2b-b97401a4155f.jpg?width=192 -phone id: 119 -https://mobitech.mk/wp-content/uploads/2021/12/apple-iphone-xs-max-new1.jpg -phone id: 125 -https://admin.ledikom.mk/uploads/items/1498/1649754304samsung-galaxy-a23-1-250x300-pad.jpg?v=1 -phone id: 126 -https://admin.ledikom.mk/uploads/items/352/1642711564samsung-galaxy-a02-1-250x300-pad.jpg?v=1 -phone id: 129 -https://i2.wp.com/mobilezone.mk/wp-content/uploads/2021/08/12-pro-blue.png?resize=512%2C600&ssl=1 -phone id: 140 -https://www.a1.mk/documents/20126/2336887/Nokia+C21_Front_Blue.png -phone id: 153 -https://admin.ledikom.mk/uploads/items/429/1641673183apple-iphone-13-mini-1-250x300-pad.jpg?v=1 -phone id: 156 -https://admin.ledikom.mk/uploads/items/1477/1647544699samsung-galaxy-s21-fe-5g-1-250x300-pad.jpg?v=1 -phone id: 173 -https://www.a1.mk/documents/20126/2315371/Honor-8X-2022-Front-Blue.png -phone id: 184 -https://akcija.com.mk/app/img/products/gallery/2021/07/WIKVIEWBLAST_Wiko_View_4G_Black_32_+_3.webp -phone id: 194 -https://admin.ledikom.mk/uploads/items/1502/1649758047google-pixel-6-1-250x300-pad.jpg?v=1 -phone id: 195 -https://mobelix.com.mk/storage/app/uploads/public/612/e29/ab9/thumb_3301_550_800_0_0_crop.png -phone id: 196 -https://admin.ledikom.mk/uploads/items/408/galaxyzfold35g-874417-250x300-pad.jpg?v=1 -phone id: 199 -https://www.neptun.mk/2021/11/10/xiaomi11tproblancomedianoche04adl_dcdba358-c8be-4ed6-ada5-0dac8dd6a314.jpg?width=192 -phone id: 200 -https://mobelix.com.mk/storage/app/uploads/public/618/11f/35e/thumb_3446_550_800_0_0_crop.png -phone id: 201 -https://www.a1.mk/documents/20126/1900464/A1-Alpha-21.png -phone id: 202 -https://www.a1.mk/documents/20126/2152486/Motorola-MOTO-G60-Front.png -phone id: 203 -https://admin.ledikom.mk/uploads/items/1503/1649766652google-pixel-6-pro-1-250x300-pad.jpg?v=1 -phone id: 204 -https://mobelix.com.mk/storage/app/uploads/public/61a/0c6/65d/thumb_3478_550_800_0_0_crop.png -phone id: 205 -https://www.neptun.mk/2021/09/02/720_c7b32e05-41eb-47c4-b2e6-64b19d789d54.jpg?width=192 -phone id: 206 -https://admin.ledikom.mk/uploads/items/1485/1649682740xiaomi-redmi-note-11t-5g-1-250x300-pad.jpg?v=1 -phone id: 207 -https://mobelix.com.mk/storage/app/uploads/public/61f/26c/5d9/thumb_3564_550_800_0_0_crop.png -phone id: 208 -https://mobelix.com.mk/storage/app/uploads/public/5eb/2cd/447/thumb_2075_550_800_0_0_crop.png -phone id: 209 -https://www.a1.mk/documents/20126/1862420/Nokia-G10.png -phone id: 210 -https://admin.ledikom.mk/uploads/items/299/1642704279samsung-galaxy-note20-1-250x300-pad.jpg?v=1 -phone id: 211 -https://admin.ledikom.mk/uploads/items/341/1642707896samsung-galaxy-note20-ultra-5g-1-250x300-pad.jpg?v=1 -phone id: 214 -https://mobelix.com.mk/storage/app/uploads/public/621/a31/b32/thumb_3662_550_800_0_0_crop.png -phone id: 215 -https://www.neptun.mk/2022/07/21/54000_0a9e6bb1-320e-4b2e-a10a-0bfd4093b9fc.jpg?width=192 -phone id: 216 -https://admin.ledikom.mk/uploads/items/1492/1653141418xiaomi-redmi-note-11-pro-1-250x300-pad.jpg?v=1 -phone id: 217 -https://www.a1.mk/documents/20126/2336936/TCL-306_Atlantic-Blue_Front.png -phone id: 219 -https://admin.ledikom.mk/uploads/items/2798/1652776962xiaomi-poco-x4-pro-5g-1-250x300-pad.jpg?v=1 -phone id: 220 -https://admin.ledikom.mk/uploads/items/1480/1647546041samsung-galaxy-s22-ultra-5g-1-250x300-pad.jpg?v=1 -phone id: 221 -https://mobelix.com.mk/storage/app/uploads/public/5ec/92e/6d3/thumb_2119_550_800_0_0_crop.png -phone id: 222 -https://mobelix.com.mk/storage/app/uploads/public/624/5ca/928/thumb_3764_550_800_0_0_crop.png -phone id: 223 -https://admin.ledikom.mk/uploads/items/1566/1651355303oneplus-10-pro-1-250x300-pad.jpg?v=1 -phone id: 224 -https://admin.ledikom.mk/uploads/items/1564/1651071926samsung-galaxy-a73-5g-1-250x300-pad.jpg?v=1 -phone id: 225 -https://mobelix.com.mk/storage/app/uploads/public/628/ba6/995/thumb_3840_550_800_0_0_crop.png -phone id: 226 -https://mobelix.com.mk/storage/app/uploads/public/629/ddd/5ac/thumb_3862_550_800_0_0_crop.png -phone id: 227 -https://admin.ledikom.mk/uploads/items/3026/1655278093samsung-galaxy-m33-5g-1-250x300-pad.jpg?v=1 -phone id: 228 -https://mobelix.com.mk/storage/app/uploads/public/62c/d2d/bb7/thumb_3883_550_800_0_0_crop.png -phone id: 229 -https://mobelix.com.mk/storage/app/uploads/public/62d/574/ea8/thumb_3889_550_800_0_0_crop.png -phone id: 233 -https://www.neptun.mk/2019/12/31/06a36488-e86c-4599-8c59-7c7a4c588c80_8dbb8b4f-de84-4ae3-8392-5468909ee0e0.png?width=192 -phone id: 234 -https://akcija.com.mk/app/img/products/gallery/2022/03/Alcatel_1066G_Black.webp -phone id: 235 -https://www.neptun.mk/2018/08/14/Tiger_L5_Black_Special_v11.23_1c0a6cd6-8705-42ad-a6ba-db7c7b939b7c.jpg?width=192 -phone id: 237 -https://admin.ledikom.mk/uploads/items/1488/1649683643xiaomi-redmi-9c-nfc-1-250x300-pad.jpg?v=1 -phone id: 238 -https://admin.ledikom.mk/uploads/items/432/redmi9a-48378-250x300-pad.jpg?v=1 -phone id: 239 -https://www.neptun.mk/2022/03/21/realmec1120212gb32gb04azuladll_253ab3fd-0f10-4c00-97b3-062c3db73710.jpg?width=192 -phone id: 240 -https://akcija.com.mk/app/img/products/gallery/2022/03/Blackview_A80_Black_49835.jpg -phone id: 241 -https://akcija.com.mk/app/img/products/gallery/2021/07/WIKVIEMAXWP2ANTST_Wiko_View_Max_Anthracite.webp -phone id: 243 -https://akcija.com.mk/app/img/products/gallery/2021/07/WIKLENNYWK400ANTST_Wiko_Lenny5_WK400_Anthracit.webp -phone id: 244 -https://www.neptun.mk/2021/06/04/xiaomi-black-shark-4-600x600-600x600_78632f44-8eab-4436-8d92-4c30017fa7c5.jpg?width=192 -phone id: 245 -https://www.neptun.mk/2021/10/21/gsmarena004_415d078d-4c7e-43c8-b76c-857922a385b2.jpg?width=192 -phone id: 246 -https://www.a1.mk/documents/20126/2315411/Honor-Magic-4-lite-5G-Black-Front.png -phone id: 247 -https://www.a1.mk/documents/20126/2087663/POCO-M4_Pro-5G-black.png -phone id: 249 -https://admin.ledikom.mk/uploads/items/2800/1652862618xiaomi-poco-f4-gt-1-250x300-pad.jpg?v=1 -phone id: 250 -https://www.neptun.mk/2022/07/25/Capture1_39933e15-2e42-4fdd-ad50-b26daefc653d.JPG?width=192 -phone id: 251 -https://www.neptun.mk/2020/12/29/2.jpeg?width=192 -phone id: 252 -https://www.a1.mk/documents/20126/1482115/LG-K52.png -phone id: 253 -https://www.a1.mk/documents/20126/2097647/TCL-20-plus-Milky-Way-Grey.png -phone id: 255 -https://admin.ledikom.mk/uploads/items/400/1653051754nokia-6300-4g-1-250x300-pad.jpg?v=1 -phone id: 256 -https://setec.mk/image/cache/catalog/Product/73864_0-228x228.jpg -phone id: 257 -https://www.neptun.mk/2022/07/21/22.JPG?width=192 -phone id: 260 -https://www.neptun.mk/2021/11/24/white_542da90b-c215-4e75-b6a0-416d1cbb745c.JPG?width=192 -phone id: 261 -https://www.a1.mk/documents/20126/622141/CAT-S62-front.png -phone id: 262 -https://www.a1.mk/documents/20126/622141/CAT-B26.png -phone id: 263 -https://www.a1.mk/documents/20126/2014065/Alcatel-3082-4G.png -phone id: 264 -https://www.a1.mk/documents/20126/1384010/Doro_1370.png -phone id: 265 -https://www.a1.mk/documents/20126/1741700/Alcatel-3080.png -phone id: 266 -https://www.a1.mk/documents/20126/1384010/Doro_2404.png -phone id: 267 -https://www.a1.mk/documents/20126/2065640/Motorola-MOTO-E20.png -phone id: 270 -https://www.neptun.mk/2020/02/05/resizer.php_49f522a0-9f44-4a37-8212-e0e1b2995fa8.jpg?width=192 -phone id: 271 -https://setec.mk/image/cache/catalog/Product/48063_0-228x228.jpg -phone id: 272 -https://setec.mk/image/cache/catalog/Product/50397_0-228x228.jpg -phone id: 273 -https://setec.mk/image/cache/catalog/Product/52379_0-228x228.jpg -phone id: 274 -https://setec.mk/image/cache/catalog/Product/53519_0-228x228.jpg -phone id: 275 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/27062022151936download.jpg -phone id: 276 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/a60problack_1.jpg -phone id: 277 -https://setec.mk/image/cache/catalog/Product/49840_0-228x228.jpg -phone id: 278 -https://setec.mk/image/cache/catalog/Product/48542_0-228x228.jpg -phone id: 279 -https://setec.mk/image/cache/catalog/Product/52744_0-228x228.jpg -phone id: 280 -https://setec.mk/image/cache/catalog/Product/50394_0-228x228.jpg -phone id: 282 -https://www.neptun.mk/2021/01/27/111_3237af8a-8b15-4e3e-aff8-38a86a1bcb40.jpg?width=192 -phone id: 285 -https://www.neptun.mk/2022/03/21/C21Y-Black_3fe958bc-9eeb-4bc6-8cc5-5caa930a4369.jpg?width=192 -phone id: 286 -https://setec.mk/image/cache/catalog/Product/49553_0-228x228.jpg -phone id: 287 -https://setec.mk/image/cache/catalog/Product/74712_0-228x228.jpg -phone id: 288 -https://setec.mk/image/cache/catalog/Product/48099_0-228x228.jpg -phone id: 289 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/g10-auroragrey-front.jpg -phone id: 290 -https://setec.mk/image/cache/catalog/Product/52740_0-228x228.jpg -phone id: 291 -https://setec.mk/image/cache/catalog/Product/74998_0-228x228.jpg -phone id: 292 -https://setec.mk/image/cache/catalog/Product/52389_0-228x228.jpg -phone id: 293 -https://www.neptun.mk/2022/06/23/156755-1200-1200.png?width=192 -phone id: 294 -https://setec.mk/image/cache/catalog/Product/75793_0-228x228.jpg -phone id: 295 -https://www.neptun.mk/2022/06/23/156729-1200-1200.png?width=192 -phone id: 296 -https://setec.mk/image/cache/catalog/Product/74694_0-228x228.jpg -phone id: 297 -https://www.neptun.mk/2021/10/18/746380-1000x1000_4a0ec6bb-6131-439e-a4b4-406e57caf52e.jpg?width=192 -phone id: 298 -https://setec.mk/image/cache/catalog/Product/74996_0-228x228.jpg -phone id: 299 -https://www.neptun.mk/2022/03/21/2_50b1868c-30f0-4a9a-a8c5-180a3868aef5.JPG?width=192 -phone id: 300 -https://setec.mk/image//var/www/setec/image/catalog/Product/75706_0.jpg -phone id: 301 -https://setec.mk/image/cache/catalog/Product/74997_0-228x228.jpg -phone id: 304 -https://setec.mk/image/cache/catalog/Product/75805_0-228x228.jpg -phone id: 305 -https://www.neptun.mk/2022/03/22/33.JPG?width=192 -phone id: 306 -https://setec.mk/image/cache/catalog/Product/75703_0-228x228.jpg -phone id: 307 -https://setec.mk/image/cache/catalog/Product/74704_0-228x228.jpg -phone id: 308 -https://admin.ledikom.mk/uploads/items/368/1644923487xiaomi-mi-11-lite-1-250x300-pad.jpg?v=1 -phone id: 309 -https://www.neptun.mk/2022/06/23/156684-1600-1600.png?width=192 -phone id: 310 -https://admin.ledikom.mk/uploads/items/3042/1655899106oneplus-nord-ce-5g-1-250x300-pad.jpg?v=1 -phone id: 313 -https://www.neptun.mk/2022/03/21/11_e8705b0f-d153-4b36-b5f5-776e49163dbd.JPG?width=192 -phone id: 314 -https://www.neptun.mk/2022/05/13/xiaomi-11t-5g-8gb-128gb-dual-sim-blanco41319a91-e898-4122-9dad-0f3115643b9f_65cc4f17-4add-4eea-a64e-49b60f2d45b1.jpg?width=192 -phone id: 315 -https://setec.mk/image/cache/catalog/Product/75403_0-228x228.jpg -phone id: 316 -https://admin.ledikom.mk/uploads/items/2756/1651663563xiaomi-12x-1-250x300-pad.jpg?v=1 -phone id: 318 -https://admin.ledikom.mk/uploads/items/1562/1651071199xiaomi-12-1-250x300-pad.jpg?v=1 -phone id: 319 -https://admin.ledikom.mk/uploads/items/401/1642796375samsung-galaxy-z-flip3-5g-1-250x300-pad.jpg?v=1 -phone id: 321 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/11022022143748download.jpg -phone id: 323 -https://i2.wp.com/mobilezone.mk/wp-content/uploads/2021/08/12-pro-max-blue.png?resize=512%2C600&ssl=1 -phone id: 325 -https://i0.wp.com/mobilezone.mk/wp-content/uploads/2021/08/12-mini-blue.png?resize=512%2C600&ssl=1 -phone id: 326 -https://i2.wp.com/mobilezone.mk/wp-content/uploads/2021/08/11-pro-black.png?resize=512%2C600&ssl=1 -phone id: 327 -https://i0.wp.com/mobilezone.mk/wp-content/uploads/2021/08/11-pro-max-black.png?resize=512%2C600&ssl=1 -phone id: 328 -https://admin.ledikom.mk/uploads/items/1481/1649102984apple-iphone-se-2022-1-250x300-pad.jpg?v=1 -phone id: 329 -https://admin.ledikom.mk/uploads/items/1479/1647545279samsung-galaxy-s22-5g-1-250x300-pad.jpg?v=1 -phone id: 330 -https://admin.ledikom.mk/uploads/items/297/1642703362samsung-galaxy-a01-core-1-250x300-pad.jpg?v=1 -phone id: 331 -phone id: 332 -phone id: 333 -phone id: 334 -phone id: 336 -https://admin.ledikom.mk/uploads/items/319/1642704346samsung-galaxy-s20-fe-1-250x300-pad.jpg?v=1 -phone id: 337 -https://setec.mk/image/cache/catalog/Product/73860_0-228x228.jpg -phone id: 338 -https://admin.ledikom.mk/uploads/items/1486/1649685350xiaomi-redmi-9i-1-250x300-pad.jpg?v=1 -phone id: 340 -https://setec.mk/image/cache/catalog/Product/74714_0-228x228.jpg -phone id: 341 -https://www.neptun.mk/2021/06/14/xiaomiredminote105g00v2grisadl_a83a542a-c9d1-4475-8fc7-dae3521b9bc6.jpg?width=192 -phone id: 342 -https://www.neptun.mk/2022/07/19/51UhkQTNxmL.ACSL1000_37699529-bf74-498e-9d9b-ecc8e6b57976.jpg?width=192 -phone id: 343 -https://www.neptun.mk/2022/05/13/222_e9b3e0e4-4026-4d87-9b5f-44f05b306611.JPG?width=192 -phone id: 344 -https://admin.ledikom.mk/uploads/items/3044/1656151222xiaomi-12-pro-1-250x300-pad.jpg?v=1 -phone id: 345 -phone id: 346 -phone id: 348 -phone id: 349 -phone id: 350 -phone id: 351 -phone id: 352 -https://admin.ledikom.mk/uploads/items/356/1642789488samsung-galaxy-m62-1-250x300-pad.jpg?v=1 -phone id: 353 -phone id: 354 -phone id: 355 -phone id: 356 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/14102020175047ory-unlocked-global-872783_800x.jpg -phone id: 357 -https://mobelix.com.mk/storage/app/uploads/public/613/094/943/thumb_3309_550_800_0_0_crop.png -phone id: 358 -https://www.neptun.mk/2019/11/19/555.jpg_99d86467-c5a7-42e7-8896-2ff76ac4874d.PNG?width=192 -phone id: 359 -https://www.neptun.mk/2019/11/19/555.jpg_e9a27a11-7377-40e5-ba21-2c42a6216ed8.PNG?width=192 -phone id: 360 -https://www.neptun.mk/2021/10/25/20200514120927alcatel3025x256mbmetallicblue_5968b52f-10cf-408c-96ee-6ebb76a83d0e.jpeg?width=192 -phone id: 361 -https://www.neptun.mk/2021/10/25/71qaRJFLH2L.ACSS450_58ea1f14-bd9f-49a4-91e2-2af91b5826a0.jpg?width=192 -phone id: 362 -https://www.neptun.mk/2021/03/15/1122_3c24c189-3b4b-4b37-9d43-1232b082244b.png?width=192 -phone id: 363 -https://www.neptun.mk/2021/11/21/Alcatel-1S-2021513x599-1_6c7faa25-0939-4db9-b9ec-b80be480509f.png?width=192 -phone id: 364 -https://akcija.com.mk/app/img/products/gallery/2022/03/Alcatel_5024D_1S_3GB-32GB_LTE_Dual-Sim_Metallic_Black.webp -phone id: 365 -https://www.neptun.mk/2020/06/03/12_b9efc36f-7581-4ace-a550-007ac853a6cd.jpg?width=192 -phone id: 366 -https://www.a1.mk/documents/20126/2333741/Alcatel-1B_Front.png -phone id: 367 -phone id: 368 -phone id: 369 -https://www.neptun.mk/2020/08/14/1122_7fa6daf3-975e-4abe-9edb-197b70ffb150.jpeg?width=192 -phone id: 370 -https://www.a1.mk/documents/20126/2142278/Alcatel-1-2021.png -phone id: 371 -phone id: 372 -phone id: 373 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/a70problack.jpg -phone id: 374 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/c20purple.jpg -phone id: 375 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/doogee_x96_2gb_32gb_rojo_01_l.jpg -phone id: 376 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/doogee-x96-pro_arenamobiles.jpg -phone id: 377 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/10112021121646x95black_1.jpg -phone id: 378 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/x95black_1.jpg -phone id: 379 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/thumb_548333_default_big.jpg -phone id: 380 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/doogee_s58_pro_6gb_64gb_05_negro_ad_l.jpg -phone id: 381 -https://www.neptun.mk/2021/07/02/4_6eaa4b73-2e6d-40b3-86c2-6c5940433cbd.JPG?width=192 -phone id: 382 -https://www.neptun.mk/2020/11/17/77_55ded608-c4a7-45f6-84e2-1dd826f4db2c.jpg?width=192 -phone id: 383 -https://www.neptun.mk/2021/07/02/1_3c75a06b-29ee-459b-8260-3c065616cf65.JPG?width=192 -phone id: 384 -https://www.neptun.mk/2021/11/24/Motorola-Edge-20-Pro-128GB-Midnight-Blue-0840023220623-01102021-01-p_6099d602-c2c4-428c-9560-1dd79df3338d.jpg?width=192 -phone id: 385 -https://www.a1.mk/documents/20126/1706113/MOTO-G50.png -phone id: 386 -https://www.neptun.mk/2021/11/04/gsmarena004_6a7fd9b3-8102-4bb7-960b-0c72ca683936.jpg?width=192 -phone id: 387 -https://www.neptun.mk/2022/06/23/156709-1600-1600.png?width=192 -phone id: 388 -https://www.neptun.mk/2022/06/23/157221-1600-auto.png?width=192 -phone id: 389 -https://www.neptun.mk/2022/03/21/51u9-q9539L.SL1000_f04a7be5-ad53-445d-9bde-9a33c0ff3f33.jpg?width=192 -phone id: 390 -https://www.neptun.mk/2022/03/21/Realme-9-Pro-Plus-Green_d8eec8b7-5912-4dea-b36b-a3f66832abd4.jpg?width=192 -phone id: 391 -https://www.neptun.mk/2022/03/21/222_4c36f6d2-1a4b-4e91-a096-21d115330db2.JPG?width=192 -phone id: 392 -https://mobelix.com.mk/storage/app/uploads/public/619/e62/78d/thumb_3468_550_800_0_0_crop.png -phone id: 393 -https://www.neptun.mk/2022/03/21/523920-1000x1000_e0d86853-abff-4048-bd07-e6d6ca0915db.jpg?width=192 -phone id: 394 -https://www.neptun.mk/2022/03/21/6935117828220-003-1400Wx1400H_90483936-5a39-47cb-b1c8-f9e868d765c8.jpg?width=192 -phone id: 395 -https://www.neptun.mk/2019/10/16/hq_c86c98c2-729b-49d5-aab9-216be2fea777.jpg?width=192 -phone id: 396 -phone id: 397 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/nokia_106.jpg -phone id: 398 -phone id: 399 -phone id: 400 -https://www.neptun.mk/2022/07/08/45_7f5afee7-d808-4111-9853-738a7b5993a3.JPG?width=192 -phone id: 401 -https://www.neptun.mk/2022/07/21/1234_97e5321e-7ec0-438b-8c9b-0c89acb73b2f.JPG?width=192 -phone id: 402 -https://www.neptun.mk/2022/07/08/333_bda690bd-c2a4-4ee4-82c7-3e4001fd1655.JPG?width=192 -phone id: 403 -https://www.neptun.mk/2021/12/16/infinix-note-11-pro-8-128-zeleni-mobilni-6-95-quot-octa-core-mediatek-helio-g96-8gb-128gb-64mpx-13mpx-2mpx-dual-simZFNfn4_2d108fb3-756e-4d1b-a41a-afeae6104ac9.jpg?width=192 -phone id: 404 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/max10black_1.jpg -phone id: 405 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/flex50gtturnedred.jpg -phone id: 406 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/09142292_name.jpg -phone id: 407 -phone id: 408 -phone id: 409 -phone id: 410 -phone id: 411 -phone id: 412 -phone id: 413 -phone id: 414 -phone id: 415 -https://mobelix.com.mk/storage/app/uploads/public/5cd/051/f4c/thumb_1809_550_800_0_0_crop.jpg -phone id: 416 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/510095mub.jpg -phone id: 417 -phone id: 418 -phone id: 419 -phone id: 420 -phone id: 421 -https://i0.wp.com/mobilezone.mk/wp-content/uploads/2021/11/Honor-9x-lite-black.png?resize=512%2C600&ssl=1 -phone id: 422 -phone id: 423 -https://www.a1.mk/documents/20126/2117931/Wiko_Y62.png -phone id: 424 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/21122021152115ilt-in-camera-main-camera-03-mp.jpg -phone id: 425 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/21122021141255download.jpg -phone id: 426 -phone id: 427 -phone id: 428 -phone id: 429 -phone id: 430 -phone id: 431 -phone id: 432 -phone id: 433 -phone id: 434 -phone id: 435 -phone id: 436 -phone id: 437 -phone id: 438 -phone id: 439 -phone id: 440 -https://admin.ledikom.mk/uploads/items/3040/1655892316oppo-a94-5g-1-250x300-pad.jpg?v=1 -phone id: 441 -https://admin.ledikom.mk/uploads/items/3041/1655898755oppo-find-x5-lite-5g-1-250x300-pad.jpg?v=1 -phone id: 442 -phone id: 443 -phone id: 444 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/11012022150451veteranivplus_1.jpg -phone id: 445 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/seniorflipxl_1.jpg -phone id: 446 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/denver-bas-18300m-1.jpg -phone id: 447 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/23022022105342123.jpg -phone id: 448 -phone id: 449 -phone id: 450 -phone id: 451 -phone id: 452 -https://d3mrte3vpewnxc.cloudfront.net/img/products/full/09150922_name.jpg +Samsung Galaxy A13 +9990 +Apple iPhone 13 +47499 +Apple iPhone 13 Pro +62690 +Apple iPhone 12 +32690 +Samsung Galaxy S22 Ultra 5G +60000 +Apple iPhone 11 +23490 +Samsung Galaxy S22 5G +43000 +Samsung Galaxy S21 FE +33800 +Samsung Galaxy A33 5G +17490 +Apple iPhone 13 Pro Max +68890 +Samsung Galaxy A32 +13900 +Samsung Galaxy S22+ 5G +50150 +Apple iPhone 14 Pro +71890 +Xiaomi 12 +28999 +Samsung Galaxy A53 5G +20290 +Xiaomi Redmi Note 11 +11990 +Xiaomi Redmi 9A +6490 +Huawei Nova 9 +22900 +Samsung Galaxy A12 +9500 +Samsung Galaxy Z Flip 4 +58400 +Apple iPhone 14 +1 +Apple iPhone 13 Mini +42990 +Samsung Galaxy A52s +18500 +Xiaomi Redmi 10C +9990 +Apple iPhone SE +24600 +Samsung Galaxy A03s +6790 +Samsung Galaxy A22 5G +11700 +Xiaomi Mi 11 Lite +18500 +Samsung Galaxy A23 +11790 +Apple iPhone 14 Pro Max +99999 +Samsung Galaxy Z Fold 4 +98400 +Samsung Galaxy A03 +8490 +Huawei nova Y70 +13699 +Samsung Galaxy Z Flip3 5G +38990 +Huawei Nova 8i +19999 +Samsung Galaxy Z Fold 3 5G +69990 +Xiaomi 11 Lite 5G NE +24999 +Xiaomi Redmi Note 11 Pro +16590 +Xiaomi Mi 11T 5G +23400 +Huawei Nova 9 SE +22999 +Xiaomi Redmi 9 +9399 +Huawei nova Y90 +15999 +Apple iPhone SE (2022) +28290 +Xiaomi Poco X3 +13999 +OnePlus Nord N10 5G +18999 +Samsung Galaxy A52 +18190 +Xiaomi Redmi Note 10 +13999 +Blackview A70 Pro +7399 +Realme C11 +6999 +Honor 50 Lite +11990 +Xiaomi Poco F3 +21500 +Huawei P50 Pro +58400 +Xiaomi Redmi 9C +6990 +Xiaomi 12X +29900 +Blackview A55 +5999 +OnePlus Nord CE +26999 +Vivo Y72 5G +18999 +Xiaomi Mi 11T Pro 5G +33800 +Xiaomi Poco M3 +12290 +Vivo V21 5G +24999 +Vivo Y33s +15999 +Samsung Galaxy A02s +8890 +Motorola Moto G31 +12899 +Xiaomi Mi 11 Lite 5G NE +25999 +Alcatel 2019G +2290 +Samsung Galaxy S21+ 5G +41790 +Xiaomi Redmi Note 10 5G +13490 +Xiaomi 11T +31999 +Alcatel 1SE +6290 +Alcatel 1066D +1299 +Xiaomi Redmi Note 9 +9990 +Xiaomi Redmi Note 11 Pro+ 5G +27999 +Blackview A80 +6499 +Motorola Moto G22 +11299 +Xiaomi Poco M3 Pro 5G +12999 +Xiaomi Redmi Note 9S +11900 +Blackview A70 +7299 +Xiaomi Redmi 9AT +6999 +Doogee X95 +5999 +Xiaomi Redmi Note 10 Pro +15990 +Motorola Edge 20 +30999 +Motorola Moto G60 +16999 +Honor 50 +23400 +Apple iPhone 12 Pro Max +66500 +Apple iPhone 11 Pro Max +32690 +Nokia 105 (2017) +1490 +Samsung Galaxy S21 Ultra 5G +52290 +Motorola Moto E7 Power +8999 +Apple iPhone 11 Pro +27790 +Alcatel 1 +5499 +Xiaomi Poco X3 Pro +15400 +Xiaomi Redmi Note 9 Pro +15499 +Blackview A60 Pro +6299 +Blackview Oscal C20 +5249 +Realme C21Y +7999 +Vivo Y21 +11899 +Samsung Galaxy A20e +9699 +Vivo Y21s +13999 +Nokia 3310 +3399 +Xiaomi Mi 10 Lite 5G +15990 +Xiaomi Mi Note 10 Lite +19700 +Samsung Galaxy S21 5G +38999 +Xiaomi Mi 11 Ultra +53490 +Motorola Moto G10 +10299 +Alcatel 3026X +4499 +Samsung Galaxy S20 FE +22690 +Xiaomi Redmi Note 11S +19999 +Doogee X96 +7699 +Alcatel 2053D +2999 +Alcatel 3025X +3999 +Doogee X93 +5999 +Vivo Y20s +12499 +Infinix Hot 11 +10499 +Infinix Smart 6 +7999 +Panasonic KX-TU160 +3999 +Xiaomi 12 Pro +54150 +Xiaomi Mi 10T Pro 5G +29900 +Xiaomi Poco X4 Pro 5G +17200 +Apple iPhone 12 Pro +61590 +Google Pixel 6 Pro +61500 +Samsung Galaxy Note 20 +36900 +Xiaomi Poco M4 Pro 5G +15999 +OnePlus 10 Pro 5G +52290 +Wiko View +5990 +Samsung Galaxy A72 +23300 +Alcatel 2003D +2199 +Xiaomi Redmi Note 10 Pro Max +16490 +Trevi Max 10 +1599 +Honor X8 +16499 +Nokia 6310 +5490 +Xiaomi 11T Pro +47999 +Samsung Galaxy A02 +8000 +Samsung Galaxy A40 +11999 +Motorola Moto G71 +18999 +Samsung Galaxy Note 20 Ultra 5G +50390 +Samsung Galaxy A73 5G +27000 +Xiaomi Redmi Note 10S +14999 +Blackview A60 +4999 +Blackview A80 Plus +8999 +Realme C21 +8999 +Motorola Moto G51 +14999 +Xiaomi Mi 10T Lite 5G +15390 +OnePlus 9 Pro +46200 +Cubot P30 +9999 +Cubot Note 20 +8999 +Motorola Moto G30 +13999 +Asus ROG Phone 5 +46100 +Tesla Smartphone 3.4 +4999 +Oppo F11 Pro +16600 +Honor View 20 +36800 +Honor 70 +32999 +Realme 8i +15999 +OnePlus 9 +30800 +Samsung Galaxy M12 +9500 +Samsung Galaxy A31 +16999 +Xiaomi Redmi 9i +8890 +OnePlus 7T Pro McLaren Edition +36900 +OnePlus 7T +27700 +Oppo A9 +16600 +Nokia 8110 4G +4300 +Blackberry Porsche Design P9981 +19000 +Asus ROG Phone 2 +32000 +Xiaomi Redmi Note 9T 5G +12290 +Motorola Edge 20 Pro +44999 +Xiaomi Poco X3 GT +17200 +Nokia 105 (2019) +1999 +Apple iPhone 12 Mini +34999 +Samsung Galaxy Z Fold 2 +141499 +Samsung Galaxy M32 +14100 +Alcatel 1S +6390 +Doogee X95 Pro +7399 +Doogee S58 Pro +12499 +Motorola Moto G100 +30999 +Motorola Moto G200 +30999 +Realme 9i +16699 +Realme 9 Pro+ +27999 +Realme GT Master +20300 +Alcatel 1C +4499 +Trevi Flex 50 GT +2199 +Huawei Honor 8X Max +12300 +Huawei Honor 9X Lite +11000 +Oppo A94 5G +15390 +MeanIT Veteran I +1599 +MeanIT Senior Flip XL +3399 +Denver B242 +2999 +Asus ROG Phone 3 +36900 +Huawei Y Max +12300 +Apple iPhone X +15990 +Xiaomi Redmi Note 11T 5G +18500 +Motorola G60 +17999 +OnePlus 8 Pro +40000 +Samsung Galaxy M52 5G +21500 +Xiaomi Mi 12 Pro 5G +54100 +Alcatel 1066G +1190 +Google Pixel 6 +40000 +Wiko View Max +7990 +Xiaomi Black Shark 4 +29990 +Tcl 20L+ +15999 +Xiaomi Poco F4 GT +47999 +Samsung Galaxy A51 +18999 +Samsung Galaxy A50 +18999 +Xiaomi Mi A2 Lite +8999 +Xiaomi Mi Play +9999 +Xiaomi Mi 6 +9999 +Xiaomi Mi Max 2 +6999 +Xiaomi Mi A3 +8999 +Samsung Galaxy A30s +13999 +Samsung Galaxy A41 +16999 +Alcatel Pixi 4 Power Plus +5999 +Honor X7 +13999 +Energizer Energy E20 +1999 +Energizer Energy E241S +2299 +Energizer Hardcase H240S +3999 +Energizer Power Max P550S +4999 +Huawei Mate 10 Lite +9999 +Samsung Galaxy M02s +9500 +Xiaomi Mi 10T 5G +24500 +Xiaomi Redmi 8A +7400 +Huawei P30 Pro +29500 +Huawei P40 Lite E +9200 +Apple iPhone XS +17890 +Apple iPhone XS Max +21590 +Nokia C21 +8999 +Alcatel 1S (2021) +8999 +Huawei P50 Pocket +76900 +CAT B26 +5999 +Cubot J9 +5299 +Samsung Galaxy A21s +11199 +Xiaomi Mi A2 +13799 +Alcatel 1V +4399 +Trevi Phablet 5Q2 +4599 +Energizer Energy E10 +1299 +Huawei P20 +24999 +Honor 9 Lite +7999 +Huawei Honor 8X +16999 +Honor 10 Lite +8499 +Wiko Power U20 +8999 +Wiko Y62 +6999 +Vivax Pro M1 +4499 +Vivax Fun S10 +3999 +Vivax Fun S20 +4999 +Vivax Point X2 +5999 +Vivax Fly 5 Lite +7499 +Vivax Fly V1 +8999 +Vivax Point X502 +4499 +Vivax Point X1 +6999 +Vivax Fun S1 +2999 +Oppo A15s +8999 +Oppo Reno4 Z 5G +18999 +Oppo Reno5 5G +24999 +Oppo Reno4 Lite +15999 +Oppo Find X5 Lite +24590 +Denver B183 +1499 +Noa N2 +7499 +Noa Sprint 4G +5499 +Samsung Galaxy J7 +8999 +Manta MS1701 +1099 +Huawei P10 Lite +11999 +Xiaomi Redmi Note 10T 5G +12300 +Nokia 6300 4G +4000 +Tcl 306 +9999 +Oppo Find X3 Lite +16000 +Wiko Lenny 5 +4290 +Doro 2404 +4499 +AllCall Smooth 4G +5499 +Realme C25Y +10999 +Motorola Moto G41 +16999 +Honor Magic4 Lite +21999 +Huawei Honor 7A +5999 +Honor 20 Lite +11299 +Panasonic KX-TU155 +4699 +Vivax Fly 6 +5999 +Tesla Feature 3 +1599 +Noa Fresh 4G +2999 +Samsung Galaxy F12 +9200 +Nokia 106 +1999 +OnePlus Nord +22999 +Infinix Hot 12i +9999 +Doro 1370 +4999 +Apple iPhone XR +17290 +Doogee X96 Pro +8799 +Samsung Galaxy F62 +18400 +OnePlus 9R +27700 +Alcatel 3080G +4499 +Motorola Moto E20 +8999 +AllCall Brother 4G +5499 +Realme 8 +14999 +Huawei Honor 7S +7599 +Motorola Moto G50 +16999 +Nokia 110 +2299 +Motorola Edge 30 +31999 +Realme 9 Pro +22499 +Nokia 8 +14999 +OnePlus Nord N100 +10499 Index: phonelux_scrappers/phone_total_offers.py =================================================================== --- phonelux_scrappers/phone_total_offers.py (revision 48f3030e929e5d3717498b6e78a9813429162fde) +++ phonelux_scrappers/phone_total_offers.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -1,4 +1,6 @@ +import json import psycopg2 +import requests import config_read import sys @@ -7,27 +9,14 @@ sys.stdout = open(file_path, "w") -# Call to read the configuration file and connect to database -cinfo = config_read.get_databaseconfig("postgresdb.config") -db_connection = psycopg2.connect( - database=cinfo[0], - host=cinfo[1], - user=cinfo[2], - password=cinfo[3] -) -cur = db_connection.cursor() - -cur.execute('SELECT id, model FROM phones;') -phones = cur.fetchall() +phones = json.loads(requests.get('http://localhost:8080/phones').text) for phone in phones: - cur.execute('SELECT COUNT(*) FROM phone_offers WHERE phone_id='+str(phone[0])+';') - total_offers = cur.fetchone()[0] + phone_id = str(phone['id']) + totaloffers = requests.get('http://localhost:8080/totaloffers/'+str.join('*', phone['model'].split(' '))).text - cur.execute('UPDATE phones SET total_offers='+str(total_offers)+' WHERE id='+str(phone[0])+';') - db_connection.commit() + print(phone_id+' - '+totaloffers) - -cur.close() -db_connection.close() + # UPDATE DATABASE WITH NEW TOTAL OFFERS + requests.put('http://localhost:8080/settotaloffers/' + phone_id + '/' + totaloffers) Index: phonelux_scrappers/phones_image_setter.py =================================================================== --- phonelux_scrappers/phones_image_setter.py (revision 48f3030e929e5d3717498b6e78a9813429162fde) +++ phonelux_scrappers/phones_image_setter.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -1,45 +1,26 @@ -import unicodedata -from datetime import datetime +import json +import requests +import classes.phone +import sys -import psycopg2 -import config_read -from bs4 import BeautifulSoup -import requests - -import sys file_path = 'outputfile.txt' sys.stdout = open(file_path, "w") -# Call to read the configuration file and connect to database -cinfo = config_read.get_databaseconfig("postgresdb.config") -db_connection = psycopg2.connect( - database=cinfo[0], - host=cinfo[1], - user=cinfo[2], - password=cinfo[3] -) -cur = db_connection.cursor() +phones = json.loads(requests.get('http://localhost:8080/phones').text) +for phone in phones: + phone_id = str(phone['id']) + offers = list(json.loads(requests.get('http://localhost:8080/phones/offers/' + phone_id).text)) -cur.execute('SELECT * FROM PHONES ORDER BY id') + offers = list(filter(lambda offer: offer['image_url'] is not None, offers)) -db_connection.commit() -phones = cur.fetchall() + image_url = None -for phone in phones: - print(phone) - phone_id = phone[0] - cur.execute('SELECT phone_offers.image_url FROM phone_offers JOIN phones ON' - ' phone_offers.phone_id = phones.id WHERE ' - 'phones.id='+str(phone_id)+' AND phone_offers.image_url IS NOT NULL LIMIT 1') - tuple_image = cur.fetchone() - if tuple_image is not None: - print(tuple_image[0]) - cur.execute('UPDATE phones SET image_url = \''+tuple_image[0]+'\' WHERE id='+str(phone_id)); - db_connection.commit() - else: - print('None'); + if len(offers) > 0: + image_url = offers[0]['image_url'] + phone['image_url'] = image_url -cur.close() -db_connection.close() + # UPDATE DATABASE WITH NEW IMAGE URLS FOR PHONES + headers = {'Content-type': 'application/json'} + requests.put('http://localhost:8080/setimageurl/' + phone_id, headers=headers, data=phone['image_url']) Index: phonelux_scrappers/phones_model_setter.py =================================================================== --- phonelux_scrappers/phones_model_setter.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) +++ phonelux_scrappers/phones_model_setter.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -0,0 +1,26 @@ +import json +import unicodedata +import requests +import classes.phone +import sys + + +file_path = 'outputfile.txt' +sys.stdout = open(file_path, "w") + +phones = json.loads(requests.get('http://localhost:8080/phones').text) +offers = json.loads(requests.get('http://localhost:8080/alloffers').text) +phones.sort(key=lambda p: p['model'], reverse=True) +offers.sort(key=lambda o: o['offer_name']) + +for offer in offers: + flag = False + for phone in phones: + if phone['model'].lower() in offer['offer_name'].lower(): + flag = True + # Add phone model to offer + requests.put('http://localhost:8080/phoneoffer/'+str(offer['id'])+'/addphonemodel/'+str(phone['id'])) + break + if not flag: + # requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(offer['id'])) + print('delete offer id '+str(offer['id'])) Index: phonelux_scrappers/scrappers/a1_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/a1_scrapper.py (revision 48f3030e929e5d3717498b6e78a9813429162fde) +++ phonelux_scrappers/scrappers/a1_scrapper.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -1,28 +1,37 @@ import unicodedata from datetime import datetime - +import json import psycopg2 import config_read from bs4 import BeautifulSoup import requests +import sys +import unicodedata -import sys +from classes.phoneoffer import PhoneOffer file_path = 'outputfile.txt' sys.stdout = open(file_path, "w") -# Call to read the configuration file and connect to database -cinfo = config_read.get_databaseconfig("../postgresdb.config") -db_connection = psycopg2.connect( - database=cinfo[0], - host=cinfo[1], - user=cinfo[2], - password=cinfo[3] -) -cur = db_connection.cursor() - offer_shop = "A1" # offer shop last_updated = datetime.now().date() is_validated = False + +# A1 phone offers that are already in database + +offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/a1').text)) + +database_offers = [] + +for offer in offers: + phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'], + offer['ram_memory'], + offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'], + offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'], + offer['image_url'], + offer['offer_url'], offer['last_updated'], offer['is_validated'], + offer['offer_description'], + offer['offer_shop_code']) + database_offers.append(phoneOffer) a1_url = 'https://www.a1.mk/webshop/mk/phones' @@ -34,7 +43,9 @@ .find_all('div', {'class', 'dvc-idtfr by4'}) +new_offers = [] + for phone in phones: brand = phone.get('data-brand').strip() - offer_name = brand+" "+phone.get('data-model').strip() + offer_name = brand + " " + phone.get('data-model').strip() # if brand not in offer_name: @@ -77,4 +88,6 @@ battery = None front_camera = None + chipset = None + offer_description = None for row in table_rows: @@ -100,14 +113,49 @@ front_camera = row.get_text().replace('Предна камера', '').strip() - insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name, price, image_url, offer_url,' \ - 'ram_memory, rom_memory, battery, back_camera, front_camera, color, cpu, ' \ - 'operating_system, offer_shop_code, last_updated, is_validated)' \ - ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' - insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, ram_memory, rom_memory, - battery, back_camera, front_camera, color, cpu, operating_system, offer_shop_code, - last_updated, is_validated) - cur.execute(insert_script, insert_value) - db_connection.commit() + new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory, + color, front_camera, back_camera, chipset, battery, operating_system, cpu, image_url, + offer_url, last_updated, is_validated, offer_description, offer_shop_code)) -cur.close() -db_connection.close() +for new_offer in new_offers: + flag = False + flag_price = False + offer_id = None + + for old_offer in database_offers: + + if new_offer.offer_shop_code == old_offer.offer_shop_code: + flag = True + if new_offer.price != old_offer.price: + flag_price = True + offer_id = old_offer.offer_id + + if flag: + # print('ALREADY IN DATABASE') + # print(new_offer) + # if it's already in database, check PRICE and if it's changed, change it !!!!!! + if flag_price: + print('PRICE CHANGED!') # CHANGE PRICE + print('offer id: ' + str(offer_id)) + headers = {'Content-type': 'application/json'} + requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price), + headers=headers) + else: + print('ADDED') # ADD OFFER + print(new_offer) + headers = {'Content-type': 'application/json'} + requests.post('http://localhost:8080/phoneoffer/addoffer', headers=headers, data=json.dumps(new_offer.__dict__, + default=str)) + +print('------------------------------------') + +for old_offer in database_offers: + flag = False + for new_offer in new_offers: + if old_offer.offer_shop_code == new_offer.offer_shop_code: + flag = True + + if not flag: + print('OFFER DELETED') + print(old_offer) + # DELETE OFFER + requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id)) Index: phonelux_scrappers/scrappers/akcija_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/akcija_scrapper.py (revision 48f3030e929e5d3717498b6e78a9813429162fde) +++ phonelux_scrappers/scrappers/akcija_scrapper.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -1,2 +1,3 @@ +import json from datetime import datetime @@ -7,21 +8,32 @@ import unicodedata import sys +from classes.phoneoffer import PhoneOffer -# file_path = '../outputfile.txt' -# sys.stdout = open(file_path, "w") +file_path = 'outputfile.txt' +sys.stdout = open(file_path, "w") -# Call to read the configuration file and connect to database -cinfo = config_read.get_databaseconfig("../postgresdb.config") -db_connection = psycopg2.connect( - database=cinfo[0], - host=cinfo[1], - user=cinfo[2], - password=cinfo[3] -) -cur = db_connection.cursor() offer_shop = "Akcija" # offer shop last_updated = datetime.now().date() is_validated = False + +# Akcija phone offers that are already in database + +offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/akcija').text)) + +database_offers = [] + +for offer in offers: + phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'], + offer['ram_memory'], + offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'], + offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'], + offer['image_url'], + offer['offer_url'], offer['last_updated'], offer['is_validated'], + offer['offer_description'], + offer['offer_shop_code']) + database_offers.append(phoneOffer) + +new_offers = [] i = 0 @@ -51,4 +63,15 @@ soup2 = BeautifulSoup(response2.text, 'html.parser') + back_camera = None + operating_system = None + chipset = None + battery = None + ram_memory = None + rom_memory = None + cpu = None + front_camera = None + color = None + offer_shop_code = None + specifications = soup2.find('main', {'id': 'content'}) \ .find_all('div', {'class', 'container'})[1].find('div', {'class', 'mb-14'}) \ @@ -62,13 +85,51 @@ str(specification.get_text(separator='\n').strip())) + "\n" - insert_script = 'INSERT INTO phone_offers (offer_shop, brand,' \ - ' offer_name, price, image_url, offer_url, last_updated, is_validated, offer_description) ' \ - 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s);' - insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, - last_updated, is_validated, offer_description) - cur.execute(insert_script, insert_value) - db_connection.commit() + new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory, + color, front_camera, back_camera, chipset, battery, operating_system, cpu, + image_url, + offer_url, last_updated, is_validated, offer_description, offer_shop_code)) i += 20 -cur.close() -db_connection.close() +for new_offer in new_offers: + flag = False + flag_price = False + offer_id = None + + for old_offer in database_offers: + + if new_offer.offer_name == old_offer.offer_name: + flag = True + if new_offer.price != old_offer.price: + flag_price = True + offer_id = old_offer.offer_id + + if flag: + # print('ALREADY IN DATABASE') + # print(new_offer) + # if it's already in database, check PRICE and if it's changed, change it !!!!!! + if flag_price: + print('PRICE CHANGED!') # CHANGE PRICE + print('offer id: ' + str(offer_id)) + headers = {'Content-type': 'application/json'} + requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price), + headers=headers) + else: + print('ADDED') # ADD OFFER + print(new_offer) + headers = {'Content-type': 'application/json'} + requests.post('http://localhost:8080/phoneoffer/addoffer', + headers=headers, data=json.dumps(new_offer.__dict__, default=str)) + +print('------------------------------------') + +for old_offer in database_offers: + flag = False + for new_offer in new_offers: + if old_offer.offer_name == new_offer.offer_name: + flag = True + + if not flag: + print('OFFER DELETED') + print(old_offer) + # DELETE OFFER + requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id)) Index: onelux_scrappers/scrappers/anhoch_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/anhoch_scrapper.py (revision 48f3030e929e5d3717498b6e78a9813429162fde) +++ (revision ) @@ -1,90 +1,0 @@ -import time -from datetime import datetime -import psycopg2 -from selenium import webdriver -from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC - -import config_read -from bs4 import BeautifulSoup -import requests -import unicodedata -import sys - -file_path = '../outputfile.txt' -sys.stdout = open(file_path, "w") - -# Call to read the configuration file and connect to database -cinfo = config_read.get_databaseconfig("../postgresdb.config") -db_connection = psycopg2.connect( - database=cinfo[0], - host=cinfo[1], - user=cinfo[2], - password=cinfo[3] -) -cur = db_connection.cursor() - - -def scrape_function(driver1, i): - offer_shop = "Anhoch" # offer shop - last_updated = datetime.now().date() - is_validated = False - - anhoch_html = driver1.page_source - soup1 = BeautifulSoup(anhoch_html, 'html.parser') - active_li = soup1.find('div', {'class': 'adjust-elems pagination pagination-centered'}).find('li', {'class': 'active'}) - - li_element = int(active_li.get_text().strip()) - - print('page: '+str(li_element)) - - if int(active_li.get_text().strip()) == i: - phones = soup1.find('section', {'id': 'main'}).find('div', {'class': 'span8'}) \ - .find('div', {'class': 'products'}).find_all('li') - for phone in phones: - offer_url = phone.find('a').get('href') - image_url = phone.find('a').find('img').get('src') - offer_name = phone.find('div', {'class': 'product-name'}).find('a').get_text().strip() - price = int(phone.get('data-price')) - brand = phone.find('div', {'class': 'product-price'}).find_all('div')[2].find('strong').get_text().strip() - - response2 = requests.get(offer_url) - soup2 = BeautifulSoup(response2.content, 'html.parser') - offer_shop_code = soup2.find('div', {'class': 'product-desc'}).get_text().strip().split('\n')[3] - - offer_description = soup2.find('div', {'class': 'description'}) \ - .find('div', {'class': 'tab-content'}).find('pre').get_text().strip() - - print(offer_name) - print(brand) - print() - print() - - # insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name , price, image_url, offer_url,' \ - # 'offer_shop_code, offer_description, last_updated, is_validated)' \ - # ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' - # insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, - # offer_shop_code, offer_description, last_updated, is_validated) - # cur.execute(insert_script, insert_value) - # db_connection.commit() - else: - driver1.implicitly_wait(5) - scrape_function(driver1, i) - - -for i in range(1, 19): - anhoch_url = "https://www.anhoch.com/category/3017/smartfoni-i-mobilni-tel#page/"+str(i) - # print(anhoch_url) - - # selenium is used because of the dynamic content of the page - driver1 = webdriver.Safari(executable_path='/usr/bin/safaridriver') - driver1.get(anhoch_url) - - scrape_function(driver1, i) - # closing the driver so the safari instance can pair with another webdriver session - driver1.close() - - -cur.close() -db_connection.close() Index: phonelux_scrappers/scrappers/handy_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/handy_scrapper.py (revision 48f3030e929e5d3717498b6e78a9813429162fde) +++ phonelux_scrappers/scrappers/handy_scrapper.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -1,2 +1,3 @@ +import json import unicodedata from datetime import datetime @@ -9,20 +10,31 @@ import sys +from classes.phoneoffer import PhoneOffer + file_path = 'outputfile.txt' sys.stdout = open(file_path, "w") -# Call to read the configuration file and connect to database -cinfo = config_read.get_databaseconfig("../postgresdb.config") -db_connection = psycopg2.connect( - database=cinfo[0], - host=cinfo[1], - user=cinfo[2], - password=cinfo[3] -) -cur = db_connection.cursor() offer_shop = "Handy" # offer shop last_updated = datetime.now().date() is_validated = False + +# Handy phone offers that are already in database +offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/handy').text)) + +database_offers = [] + +for offer in offers: + phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'], + offer['ram_memory'], + offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'], + offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'], + offer['image_url'], + offer['offer_url'], offer['last_updated'], offer['is_validated'], + offer['offer_description'], + offer['offer_shop_code']) + database_offers.append(phoneOffer) + +new_offers = [] handy_url = 'https://www.handy.mk/telefoni?page=6' @@ -44,9 +56,19 @@ soup2 = BeautifulSoup(response2.text, 'html.parser') + back_camera = None + operating_system = None + chipset = None + battery = None + ram_memory = None + rom_memory = None + cpu = None + front_camera = None + offer_shop_code = None + color = None + image_url = None + color_section = soup2.find('section', {'data-hook': 'product-colors-title-section'}) - - color = None if color_section is not None: - temp_colors = color_section.find('fieldset', {'class': 'ColorPickerbase3563640754__container'})\ + temp_colors = color_section.find('fieldset', {'class': 'ColorPickerbase3548966286__container'})\ .find_all('input', {'type': 'radio'}) colors_list = [] @@ -67,12 +89,52 @@ offer_description = '\n'.join(specifications) - insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name , price, offer_url, ' \ - 'offer_description, last_updated, is_validated)' \ - ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s);' - insert_value = (offer_shop, brand, offer_name, price, offer_url, offer_description, - last_updated, is_validated) - cur.execute(insert_script, insert_value) - db_connection.commit() + new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory, + color, front_camera, back_camera, chipset, battery, operating_system, cpu, + image_url, + offer_url, last_updated, is_validated, offer_description, offer_shop_code)) -cur.close() -db_connection.close() +for new_offer in new_offers: + flag = False + flag_price = False + offer_id = None + + for old_offer in database_offers: + + if new_offer.offer_name == old_offer.offer_name: + flag = True + if new_offer.price != old_offer.price: + flag_price = True + offer_id = old_offer.offer_id + + if flag: + # print('ALREADY IN DATABASE') + # print(new_offer) + # if it's already in database, check PRICE and if it's changed, change it !!!!!! + if flag_price: + print('PRICE CHANGED!') # CHANGE PRICE + print('offer id: ' + str(offer_id)) + headers = {'Content-type': 'application/json'} + requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price), + headers=headers) + else: + print('ADDED') # ADD OFFER + print(new_offer) + headers = {'Content-type': 'application/json'} + requests.post('http://localhost:8080/phoneoffer/addoffer', + headers=headers, data=json.dumps(new_offer.__dict__, default=str)) + +print('------------------------------------') + +for old_offer in database_offers: + flag = False + for new_offer in new_offers: + if old_offer.offer_name == new_offer.offer_name: + flag = True + + if not flag: + print('OFFER DELETED') + print(old_offer) + # DELETE OFFER + requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id)) + + Index: phonelux_scrappers/scrappers/ledikom_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/ledikom_scrapper.py (revision 48f3030e929e5d3717498b6e78a9813429162fde) +++ phonelux_scrappers/scrappers/ledikom_scrapper.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -1,5 +1,5 @@ +import json import unicodedata from datetime import datetime - import psycopg2 import config_read @@ -10,20 +10,31 @@ import sys +from classes.phoneoffer import PhoneOffer + file_path = 'outputfile.txt' sys.stdout = open(file_path, "w") - -# Call to read the configuration file and connect to database -cinfo = config_read.get_databaseconfig("../postgresdb.config") -db_connection = psycopg2.connect( - database=cinfo[0], - host=cinfo[1], - user=cinfo[2], - password=cinfo[3] -) -cur = db_connection.cursor() offer_shop = "Ledikom" # offer shop last_updated = datetime.now().date() is_validated = False + +# Ledikom phone offers that are already in database + +offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/ledikom').text)) + +database_offers = [] + +for offer in offers: + phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'], + offer['ram_memory'], + offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'], + offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'], + offer['image_url'], + offer['offer_url'], offer['last_updated'], offer['is_validated'], + offer['offer_description'], + offer['offer_shop_code']) + database_offers.append(phoneOffer) + +new_offers = [] ledikom_phone_urls = [ @@ -65,5 +76,7 @@ offer_name = ' '.join(temp_offer_name.split()) brand = offer_name.split(' ')[0] - price = int(phone.find('span', {'class': 'price'}).get_text().replace('ден.', '').replace('.', '').strip()) + price = int(phone.find('span', {'class': 'price'}).get_text().replace('ден.', '') + .replace('ден', '') + .replace('.', '').strip()) driver1 = webdriver.Safari(executable_path='/usr/bin/safaridriver') @@ -82,4 +95,12 @@ rom_memory = None ram_memory = None + back_camera = None + operating_system = None + chipset = None + battery = None + cpu = None + front_camera = None + offer_shop_code = None + offer_description = None if len(specifications) != 0: @@ -114,12 +135,50 @@ color = temp - insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name, price, image_url, offer_url,' \ - 'ram_memory, rom_memory, color, last_updated, is_validated)' \ - ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' - insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, ram_memory, - rom_memory, color, last_updated, is_validated) - cur.execute(insert_script, insert_value) - db_connection.commit() + new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory, + color, front_camera, back_camera, chipset, battery, operating_system, cpu, + image_url, + offer_url, last_updated, is_validated, offer_description, offer_shop_code)) -cur.close() -db_connection.close() +for new_offer in new_offers: + flag = False + flag_price = False + offer_id = None + + for old_offer in database_offers: + + if new_offer.offer_name == old_offer.offer_name: + flag = True + if new_offer.price != old_offer.price: + flag_price = True + offer_id = old_offer.offer_id + + if flag: + # print('ALREADY IN DATABASE') + # print(new_offer) + # if it's already in database, check PRICE and if it's changed, change it !!!!!! + if flag_price: + print('PRICE CHANGED!') # CHANGE PRICE + print('offer id: ' + str(offer_id)) + headers = {'Content-type': 'application/json'} + requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price), + headers=headers) + else: + print('ADDED') # ADD OFFER + print(new_offer) + headers = {'Content-type': 'application/json'} + requests.post('http://localhost:8080/phoneoffer/addoffer', + headers=headers, data=json.dumps(new_offer.__dict__, default=str)) + +print('------------------------------------') + +for old_offer in database_offers: + flag = False + for new_offer in new_offers: + if old_offer.offer_name == new_offer.offer_name: + flag = True + + if not flag: + print('OFFER DELETED') + print(old_offer) + # DELETE OFFER + requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id)) Index: phonelux_scrappers/scrappers/mobelix_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/mobelix_scrapper.py (revision 48f3030e929e5d3717498b6e78a9813429162fde) +++ phonelux_scrappers/scrappers/mobelix_scrapper.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -1,2 +1,4 @@ +import json +import sys import unicodedata from datetime import datetime @@ -8,21 +10,31 @@ # import sys -# -# file_path = 'outputfile.txt' -# sys.stdout = open(file_path, "w") +from classes.phoneoffer import PhoneOffer -# Call to read the configuration file and connect to database -cinfo = config_read.get_databaseconfig("../postgresdb.config") -db_connection = psycopg2.connect( - database=cinfo[0], - host=cinfo[1], - user=cinfo[2], - password=cinfo[3] -) -cur = db_connection.cursor() +file_path = 'outputfile.txt' +sys.stdout = open(file_path, "w") offer_shop = "Mobelix" # offer shop last_updated = datetime.now().date() is_validated = False + +# Mobelix phone offers that are already in database + +offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/mobelix').text)) + +database_offers = [] + +for offer in offers: + phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'], + offer['ram_memory'], + offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'], + offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'], + offer['image_url'], + offer['offer_url'], offer['last_updated'], offer['is_validated'], + offer['offer_description'], + offer['offer_shop_code']) + database_offers.append(phoneOffer) + +new_offers = [] for i in range(1, 17): @@ -77,4 +89,6 @@ back_camera = '' cpu = None + offer_shop_code = None + offer_description = None for table in tables: @@ -120,14 +134,51 @@ back_camera = None - insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name, price, image_url, offer_url,' \ - 'ram_memory, rom_memory, battery, back_camera, front_camera, color, cpu, chipset, ' \ - 'operating_system, last_updated, is_validated)' \ - ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' - insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, ram_memory, rom_memory, - battery, back_camera, front_camera, color, cpu, chipset, operating_system, - last_updated, is_validated) - cur.execute(insert_script, insert_value) - db_connection.commit() + new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory, + color, front_camera, back_camera, chipset, battery, operating_system, cpu, + image_url, + offer_url, last_updated, is_validated, offer_description, offer_shop_code)) -cur.close() -db_connection.close() + +for new_offer in new_offers: + flag = False + flag_price = False + offer_id = None + + for old_offer in database_offers: + + if new_offer.offer_name == old_offer.offer_name: + flag = True + if new_offer.price != old_offer.price: + flag_price = True + offer_id = old_offer.offer_id + + if flag: + # print('ALREADY IN DATABASE') + # print(new_offer) + # if it's already in database, check PRICE and if it's changed, change it !!!!!! + if flag_price: + print('PRICE CHANGED!') # CHANGE PRICE + print('offer id: ' + str(offer_id)) + headers = {'Content-type': 'application/json'} + requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price), + headers=headers) + else: + print('ADDED') # ADD OFFER + print(new_offer) + headers = {'Content-type': 'application/json'} + requests.post('http://localhost:8080/phoneoffer/addoffer', + headers=headers, data=json.dumps(new_offer.__dict__, default=str)) + +print('------------------------------------') + +for old_offer in database_offers: + flag = False + for new_offer in new_offers: + if old_offer.offer_name == new_offer.offer_name: + flag = True + + if not flag: + print('OFFER DELETED') + print(old_offer) + # DELETE OFFER + requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id)) Index: phonelux_scrappers/scrappers/mobigo_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/mobigo_scrapper.py (revision 48f3030e929e5d3717498b6e78a9813429162fde) +++ phonelux_scrappers/scrappers/mobigo_scrapper.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -1,2 +1,4 @@ +import json +import unicodedata from datetime import datetime @@ -5,23 +7,35 @@ from bs4 import BeautifulSoup import requests +import sys -# import sys -# -# file_path = 'outputfile.txt' -# sys.stdout = open(file_path, "w") +from classes.phoneoffer import PhoneOffer -# Call to read the configuration file and connect to database -cinfo = config_read.get_databaseconfig("../postgresdb.config") -db_connection = psycopg2.connect( - database=cinfo[0], - host=cinfo[1], - user=cinfo[2], - password=cinfo[3] -) -cur = db_connection.cursor() +file_path = 'outputfile.txt' +sys.stdout = open(file_path, "w") offer_shop = "Mobi Go" # offer shop last_updated = datetime.now().date() is_validated = False + +# Mobi Go phone offers that are already in database + +offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/mobigo').text)) + +database_offers = [] + +for offer in offers: + phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'], + offer['ram_memory'], + offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'], + offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'], + offer['image_url'], + offer['offer_url'], offer['last_updated'], offer['is_validated'], + offer['offer_description'], + offer['offer_shop_code']) + database_offers.append(phoneOffer) + +new_offers = [] + + for i in range(1, 6): mobigo_url = "https://mobigo.mk/page/" + str(i) + "/" @@ -55,11 +69,15 @@ specifications = soup2.find('table', {'id': 'singlet'}).find_all('tr') - ram_memory = "" - rom_memory = "" - battery = "" - back_camera = "" - front_camera = "" - chipset = "" - operating_system = "" + ram_memory = None + rom_memory = None + battery = None + back_camera = None + front_camera = None + chipset = None + operating_system = None + cpu = None + offer_shop_code = None + offer_description = None + color = None for specification in specifications: @@ -111,12 +129,51 @@ battery = None - insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name, price, image_url, offer_url, ram_memory,' \ - ' rom_memory, battery, back_camera, front_camera, chipset, operating_system, last_updated, is_validated)' \ - ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' - insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, ram_memory, - rom_memory, battery, back_camera, front_camera, chipset, operating_system, last_updated, is_validated) - cur.execute(insert_script, insert_value) - db_connection.commit() + new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory, + color, front_camera, back_camera, chipset, battery, operating_system, cpu, + image_url, + offer_url, last_updated, is_validated, offer_description, offer_shop_code)) -cur.close() -db_connection.close() + +for new_offer in new_offers: + flag = False + flag_price = False + offer_id = None + + for old_offer in database_offers: + + if new_offer.offer_name == old_offer.offer_name: + flag = True + if new_offer.price != old_offer.price: + flag_price = True + offer_id = old_offer.offer_id + + if flag: + print('ALREADY IN DATABASE') + print(new_offer) + # if it's already in database, check PRICE and if it's changed, change it !!!!!! + if flag_price: + print('PRICE CHANGED!') # CHANGE PRICE + print('offer id: ' + str(offer_id)) + headers = {'Content-type': 'application/json'} + requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price), + headers=headers) + else: + print('ADDED') # ADD OFFER + print(new_offer) + headers = {'Content-type': 'application/json'} + requests.post('http://localhost:8080/phoneoffer/addoffer', + headers=headers, data=json.dumps(new_offer.__dict__, default=str)) + +print('------------------------------------') + +for old_offer in database_offers: + flag = False + for new_offer in new_offers: + if old_offer.offer_name == new_offer.offer_name: + flag = True + + if not flag: + print('OFFER DELETED') + print(old_offer) + # DELETE OFFER + requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id)) Index: phonelux_scrappers/scrappers/mobilezone_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/mobilezone_scrapper.py (revision 48f3030e929e5d3717498b6e78a9813429162fde) +++ phonelux_scrappers/scrappers/mobilezone_scrapper.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -1,2 +1,3 @@ +import json import unicodedata from datetime import datetime @@ -6,23 +7,33 @@ from selenium import webdriver import requests +import sys -import sys +from classes.phoneoffer import PhoneOffer file_path = 'outputfile.txt' sys.stdout = open(file_path, "w") -# Call to read the configuration file and connect to database -cinfo = config_read.get_databaseconfig("../postgresdb.config") -db_connection = psycopg2.connect( - database=cinfo[0], - host=cinfo[1], - user=cinfo[2], - password=cinfo[3] -) -cur = db_connection.cursor() - offer_shop = "Mobile Zone" # offer shop last_updated = datetime.now().date() is_validated = False + +# Mobile Zone phone offers that are already in database + +offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/mobilezone').text)) + +database_offers = [] + +for offer in offers: + phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'], + offer['ram_memory'], + offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'], + offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'], + offer['image_url'], + offer['offer_url'], offer['last_updated'], offer['is_validated'], + offer['offer_description'], + offer['offer_shop_code']) + database_offers.append(phoneOffer) + +new_offers = [] for i in range(1, 3): @@ -54,6 +65,13 @@ offer_name = brand + ' ' + offer_name - price = int(unicodedata.normalize('NFKD', phone.find('span', {'class': 'woocommerce-Price-amount amount'}) - .find('bdi').get_text().replace(',', '').replace('ден', '').strip())) + price_tag = phone.find('span', {'class': 'woocommerce-Price-amount amount'}) + price = None + + if price_tag is not None: + price = int(unicodedata.normalize('NFKD', price_tag.find('bdi').get_text() + .replace(',', '') + .replace('ден', '').strip())) + else: + continue response2 = requests.get(offer_url) @@ -65,4 +83,10 @@ front_camera = None rom_memory = None + ram_memory = None + operating_system = None + cpu = None + chipset = None + offer_description = None + offer_shop_code = None battery = None color = None @@ -84,14 +108,50 @@ color = specification.find('td').get_text().strip() + new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory, + color, front_camera, back_camera, chipset, battery, operating_system, cpu, + image_url, + offer_url, last_updated, is_validated, offer_description, offer_shop_code)) +for new_offer in new_offers: + flag = False + flag_price = False + offer_id = None - insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name , price, offer_url, image_url, ' \ - 'rom_memory, battery, color, front_camera, back_camera, last_updated, is_validated)' \ - ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' - insert_value = (offer_shop, brand, offer_name, price, offer_url, image_url, rom_memory, battery, color, - front_camera, back_camera, last_updated, is_validated) - cur.execute(insert_script, insert_value) - db_connection.commit() + for old_offer in database_offers: -cur.close() -db_connection.close() + if new_offer.offer_name == old_offer.offer_name: + flag = True + if new_offer.price != old_offer.price: + flag_price = True + offer_id = old_offer.offer_id + + if flag: + # print('ALREADY IN DATABASE') + # print(new_offer) + # if it's already in database, check PRICE and if it's changed, change it !!!!!! + if flag_price: + print('PRICE CHANGED!') # CHANGE PRICE + print('offer id: ' + str(offer_id)) + headers = {'Content-type': 'application/json'} + requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price), + headers=headers) + else: + print('ADDED') # ADD OFFER + print(new_offer) + headers = {'Content-type': 'application/json'} + requests.post('http://localhost:8080/phoneoffer/addoffer', + headers=headers, data=json.dumps(new_offer.__dict__, default=str)) + +print('------------------------------------') + +for old_offer in database_offers: + flag = False + for new_offer in new_offers: + if old_offer.offer_name == new_offer.offer_name: + flag = True + + if not flag: + print('OFFER DELETED') + print(old_offer) + # DELETE OFFER + requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id)) Index: phonelux_scrappers/scrappers/mobitech_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/mobitech_scrapper.py (revision 48f3030e929e5d3717498b6e78a9813429162fde) +++ phonelux_scrappers/scrappers/mobitech_scrapper.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -1,2 +1,4 @@ +import json +import unicodedata from datetime import datetime @@ -5,18 +7,11 @@ from bs4 import BeautifulSoup import requests +import sys -# import sys -# file_path = 'outputfile.txt' -# sys.stdout = open(file_path, "w") +from classes.phoneoffer import PhoneOffer -# Call to read the configuration file and connect to database -cinfo = config_read.get_databaseconfig("../postgresdb.config") -db_connection = psycopg2.connect( - database=cinfo[0], - host=cinfo[1], - user=cinfo[2], - password=cinfo[3] -) -cur = db_connection.cursor() +file_path = 'outputfile.txt' +sys.stdout = open(file_path, "w") + mobitech_url = "https://mobitech.mk/shop/" @@ -29,5 +24,25 @@ offer_shop = "Mobitech" # offer shop +last_updated = datetime.now().date() is_validated = False + +# Mobitech phone offers that are already in database + +offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/mobitech').text)) + +database_offers = [] + +for offer in offers: + phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'], + offer['ram_memory'], + offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'], + offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'], + offer['image_url'], + offer['offer_url'], offer['last_updated'], offer['is_validated'], + offer['offer_description'], + offer['offer_shop_code']) + database_offers.append(phoneOffer) + +new_offers = [] for phone in phones: @@ -40,5 +55,4 @@ temp_prices = phone.find('div', {'class': 'jet-woo-product-price'}).find_all('bdi') price = int(float(temp_prices[len(temp_prices) - 1].get_text().replace("ден", "").replace(",", "").strip())) # price - last_updated = datetime.now().date() # offer last_updated date response2 = requests.get(offer_url) @@ -47,9 +61,15 @@ specifications = soup2.find_all('h2', {'class': 'elementor-heading-title elementor-size-default'}) - ram_memory = "" - rom_memory = "" - battery = "" - back_camera = "" - operating_system = "" + ram_memory = None + rom_memory = None + battery = None + back_camera = None + front_camera = None + operating_system = None + chipset = None + color = None + offer_shop_code = None + cpu = None + offer_description = None for specification in specifications: @@ -62,5 +82,6 @@ # ram memory if specification.get_text().startswith("РАМ Меморија:"): - ram_memory = specification.get_text().split("РАМ Меморија:")[1].strip() + ram_memory = specification.get_text().split("РАМ Меморија:")[1].replace('RAM', '')\ + .replace('Ram', '').strip() if ram_memory == "Нема" or ram_memory == "/": ram_memory = None @@ -84,12 +105,51 @@ battery = None - insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name, price, image_url, offer_url, ram_memory,' \ - ' rom_memory, battery, back_camera, last_updated, operating_system, is_validated)' \ - ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' - insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, ram_memory, - rom_memory, battery, back_camera, last_updated, operating_system, is_validated) - cur.execute(insert_script, insert_value) - db_connection.commit() + new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory, + color, front_camera, back_camera, chipset, battery, operating_system, cpu, + image_url, + offer_url, last_updated, is_validated, offer_description, offer_shop_code)) -cur.close() -db_connection.close() +for new_offer in new_offers: + flag = False + flag_price = False + offer_id = None + + for old_offer in database_offers: + + if new_offer.offer_name == old_offer.offer_name: + flag = True + if new_offer.price != old_offer.price: + flag_price = True + offer_id = old_offer.offer_id + + if flag: + print('ALREADY IN DATABASE') + print(new_offer) + # if it's already in database, check PRICE and if it's changed, change it !!!!!! + if flag_price: + print('PRICE CHANGED!') # CHANGE PRICE + print('offer id: ' + str(offer_id)) + headers = {'Content-type': 'application/json'} + requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price), + headers=headers) + else: + print('ADDED') # ADD OFFER + print(new_offer) + headers = {'Content-type': 'application/json'} + requests.post('http://localhost:8080/phoneoffer/addoffer', + headers=headers, data=json.dumps(new_offer.__dict__, default=str)) + +print('------------------------------------') + +for old_offer in database_offers: + flag = False + for new_offer in new_offers: + if old_offer.offer_name == new_offer.offer_name: + flag = True + + if not flag: + print('OFFER DELETED') + print(old_offer) + # DELETE OFFER + requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id)) + Index: phonelux_scrappers/scrappers/neptun_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/neptun_scrapper.py (revision 48f3030e929e5d3717498b6e78a9813429162fde) +++ phonelux_scrappers/scrappers/neptun_scrapper.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -1,2 +1,3 @@ +import json import unicodedata from datetime import datetime @@ -9,16 +10,8 @@ import sys +from classes.phoneoffer import PhoneOffer + file_path = 'outputfile.txt' sys.stdout = open(file_path, "w") - -# Call to read the configuration file and connect to database -cinfo = config_read.get_databaseconfig("../postgresdb.config") -db_connection = psycopg2.connect( - database=cinfo[0], - host=cinfo[1], - user=cinfo[2], - password=cinfo[3] -) -cur = db_connection.cursor() offer_shop = "Neptun" # offer shop @@ -26,6 +19,25 @@ is_validated = False +# Neptun phone offers that are already in database + +offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/neptun').text)) + +database_offers = [] + +for offer in offers: + phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'], + offer['ram_memory'], + offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'], + offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'], + offer['image_url'], + offer['offer_url'], offer['last_updated'], offer['is_validated'], + offer['offer_description'], + offer['offer_shop_code']) + database_offers.append(phoneOffer) + +new_offers = [] + for i in range(1, 11): - neptun_url = 'https://www.neptun.mk/mobilni_telefoni.nspx?page='+str(i) + neptun_url = 'https://www.neptun.mk/mobilni_telefoni.nspx?page=' + str(i) # selenium is used because of the dynamic content of the page @@ -72,4 +84,5 @@ offer_description = specifications_table.get_text(separator='\n').strip() + back_camera = None operating_system = None chipset = None @@ -78,4 +91,7 @@ rom_memory = None cpu = None + front_camera = None + color = None + for specification in specifications: if 'Батерија:' in specification: @@ -105,14 +121,50 @@ operating_system = specification - insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name , price, image_url, offer_url,' \ - 'offer_shop_code, operating_system, battery, chipset, cpu, ram_memory, rom_memory, ' \ - 'offer_description, last_updated, is_validated)' \ - ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' - insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, - offer_shop_code, operating_system, battery, chipset, cpu, ram_memory, rom_memory, offer_description, - last_updated, is_validated) - cur.execute(insert_script, insert_value) - db_connection.commit() + new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory, + color, front_camera, back_camera, chipset, battery, operating_system, cpu, + image_url, + offer_url, last_updated, is_validated, offer_description, offer_shop_code)) -cur.close() -db_connection.close() +for new_offer in new_offers: + flag = False + flag_price = False + offer_id = None + + for old_offer in database_offers: + + if new_offer.offer_shop_code == old_offer.offer_shop_code: + flag = True + if new_offer.price != old_offer.price: + flag_price = True + offer_id = old_offer.offer_id + + if flag: + # print('ALREADY IN DATABASE') + # print(new_offer) + # if it's already in database, check PRICE and if it's changed, change it !!!!!! + if flag_price: + print('PRICE CHANGED!') # CHANGE PRICE + print('offer id: ' + str(offer_id)) + headers = {'Content-type': 'application/json'} + requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price), + headers=headers) + else: + print('ADDED') # ADD OFFER + print(new_offer) + headers = {'Content-type': 'application/json'} + requests.post('http://localhost:8080/phoneoffer/addoffer', + headers=headers, data=json.dumps(new_offer.__dict__, default=str)) + +print('------------------------------------') + +for old_offer in database_offers: + flag = False + for new_offer in new_offers: + if old_offer.offer_shop_code == new_offer.offer_shop_code: + flag = True + + if not flag: + print('OFFER DELETED') + print(old_offer) + # DELETE OFFER + requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id)) Index: phonelux_scrappers/scrappers/outputfile.txt =================================================================== --- phonelux_scrappers/scrappers/outputfile.txt (revision 48f3030e929e5d3717498b6e78a9813429162fde) +++ phonelux_scrappers/scrappers/outputfile.txt (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -0,0 +1,9 @@ +ADDED +{'offer_shop': 'Mobile Zone', 'offer_name': 'Apple iPhone 14 Pro', 'price': 95499, 'ram_memory': None, 'rom_memory': '128GB', 'color': 'Златна, Розева, Сива, Црна', 'front_camera': '12MP', 'back_camera': '48 Mp + 12 Mp + 12 Mp', 'chipset': None, 'battery': '3200mAh', 'operating_system': None, 'cpu': None, 'image_url': 'https://i0.wp.com/mobilezone.mk/wp-content/uploads/2022/09/14-pro-silver.png?resize=600%2C600&ssl=1', 'offer_url': 'https://mobilezone.mk/produkti/iphone-14-pro/', 'last_updated': datetime.date(2022, 10, 1), 'is_validated': False, 'offer_description': None, 'offer_shop_code': None} +------------------------------------ +OFFER DELETED +{'offer_id': 1179, 'offer_shop': 'Mobile Zone', 'offer_name': 'Samsung s20 FE', 'price': 24699, 'ram_memory': None, 'rom_memory': '128GB', 'color': 'Сина', 'front_camera': None, 'back_camera': None, 'chipset': None, 'battery': None, 'operating_system': None, 'cpu': None, 'image_url': 'https://i2.wp.com/mobilezone.mk/wp-content/uploads/2022/03/Samsung-Galaxy-S20-FE-blue.png?resize=512%2C600&ssl=1', 'offer_url': 'https://mobilezone.mk/produkti/samsung-s20-fe/', 'last_updated': '2022-07-29T22:00:00.000+00:00', 'is_validated': False, 'offer_description': None, 'offer_shop_code': None} +OFFER DELETED +{'offer_id': 1181, 'offer_shop': 'Mobile Zone', 'offer_name': 'Samsung Z Flip3 5G', 'price': 39999, 'ram_memory': None, 'rom_memory': '128GB', 'color': 'Црна', 'front_camera': None, 'back_camera': None, 'chipset': None, 'battery': None, 'operating_system': None, 'cpu': None, 'image_url': 'https://i2.wp.com/mobilezone.mk/wp-content/uploads/2022/03/11.png?resize=600%2C600&ssl=1', 'offer_url': 'https://mobilezone.mk/produkti/samsung-z-flip3-5g/', 'last_updated': '2022-07-29T22:00:00.000+00:00', 'is_validated': False, 'offer_description': None, 'offer_shop_code': None} +OFFER DELETED +{'offer_id': 1180, 'offer_shop': 'Mobile Zone', 'offer_name': 'Samsung S21 FE 5G', 'price': 30899, 'ram_memory': None, 'rom_memory': '128GB', 'color': 'Зелена, Црна', 'front_camera': None, 'back_camera': None, 'chipset': None, 'battery': None, 'operating_system': None, 'cpu': None, 'image_url': 'https://i1.wp.com/mobilezone.mk/wp-content/uploads/2022/03/5g.jpg?resize=600%2C600&ssl=1', 'offer_url': 'https://mobilezone.mk/produkti/samsung-s21-fe-5g/', 'last_updated': '2022-07-29T22:00:00.000+00:00', 'is_validated': False, 'offer_description': None, 'offer_shop_code': None} Index: phonelux_scrappers/scrappers/setec_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/setec_scrapper.py (revision 48f3030e929e5d3717498b6e78a9813429162fde) +++ phonelux_scrappers/scrappers/setec_scrapper.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -1,24 +1,15 @@ +import json import unicodedata from datetime import datetime - import psycopg2 import config_read from bs4 import BeautifulSoup import requests +import sys -import sys +from classes.phoneoffer import PhoneOffer file_path = 'outputfile.txt' sys.stdout = open(file_path, "w") - -# Call to read the configuration file and connect to database -cinfo = config_read.get_databaseconfig("../postgresdb.config") -db_connection = psycopg2.connect( - database=cinfo[0], - host=cinfo[1], - user=cinfo[2], - password=cinfo[3] -) -cur = db_connection.cursor() offer_shop = "Setec" # offer shop @@ -26,6 +17,25 @@ is_validated = False -for i in range(1, 7): - setec_url = 'https://setec.mk/index.php?route=product/category&path=10066_10067&page='+str(i) +# Setec phone offers that are already in database + +offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/setec').text)) + +database_offers = [] + +for offer in offers: + phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'], + offer['ram_memory'], + offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'], + offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'], + offer['image_url'], + offer['offer_url'], offer['last_updated'], offer['is_validated'], + offer['offer_description'], + offer['offer_shop_code']) + database_offers.append(phoneOffer) + +new_offers = [] + +for i in range(1, 9): + setec_url = 'https://setec.mk/index.php?route=product/category&path=10066_10067&page=' + str(i) response1 = requests.get(setec_url) @@ -41,4 +51,14 @@ brand = offer_name.split(' ')[0] + back_camera = None + operating_system = None + chipset = None + battery = None + ram_memory = None + rom_memory = None + cpu = None + front_camera = None + color = None + if 'Cable' in offer_name or 'AirTag' in offer_name: continue @@ -49,7 +69,13 @@ offer_shop_code = phone.find('div', {'class': 'right'}) \ .find('div', {'class': 'shifra'}).get_text().replace('Шифра:', '').strip() - price = int(phone.find('div', {'class': 'right'}).find('div', {'class': 'price'}). \ - find('div', {'class': 'category-price-redovna'}).find('span', {'class': 'price-old-new'}) \ - .get_text().replace('Ден.', '').replace(',', '').strip()) + + price_tag = phone.find('div', {'class': 'right'}).find('div', {'class': 'price'}). \ + find('div', {'class': 'category-price-redovna'}).find('span', {'class': 'price-old-new'}) + + if price_tag is None: + price_tag = phone.find('div', {'class': 'right'}).find('div', {'class': 'price'}). \ + find('div', {'class': 'category-price-redovna'}).find('span', {'class': 'cena_za_kesh'}) + + price = int(price_tag.get_text().replace('Ден.', '').replace(',', '').strip()) response2 = requests.get(offer_url) @@ -58,12 +84,50 @@ offer_description = soup2.find('div', {'id': 'tab-description'}).get_text(separator='\n') - insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name , price, image_url, offer_url,' \ - 'offer_shop_code, offer_description, last_updated, is_validated)' \ - ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' - insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, - offer_shop_code, offer_description, last_updated, is_validated) - cur.execute(insert_script, insert_value) - db_connection.commit() + new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory, + color, front_camera, back_camera, chipset, battery, operating_system, cpu, + image_url, + offer_url, last_updated, is_validated, offer_description, offer_shop_code)) -cur.close() -db_connection.close() +for new_offer in new_offers: + flag = False + flag_price = False + offer_id = None + + for old_offer in database_offers: + + if new_offer.offer_shop_code == old_offer.offer_shop_code: + flag = True + if new_offer.price != old_offer.price: + flag_price = True + offer_id = old_offer.offer_id + + if flag: + # print('ALREADY IN DATABASE') + # print(new_offer) + # if it's already in database, check PRICE and if it's changed, change it !!!!!! + if flag_price: + print('PRICE CHANGED!') # CHANGE PRICE + print('offer id: ' + str(offer_id)) + headers = {'Content-type': 'application/json'} + requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price), + headers=headers) + else: + print('ADDED') # ADD OFFER + print(new_offer) + headers = {'Content-type': 'application/json'} + requests.post('http://localhost:8080/phoneoffer/addoffer', + headers=headers, data=json.dumps(new_offer.__dict__, default=str)) + +print('------------------------------------') + +for old_offer in database_offers: + flag = False + for new_offer in new_offers: + if old_offer.offer_shop_code == new_offer.offer_shop_code: + flag = True + + if not flag: + print('OFFER DELETED') + print(old_offer) + # DELETE OFFER + requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id)) Index: phonelux_scrappers/scrappers/tehnomarket_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/tehnomarket_scrapper.py (revision 48f3030e929e5d3717498b6e78a9813429162fde) +++ phonelux_scrappers/scrappers/tehnomarket_scrapper.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -1,2 +1,3 @@ +import json import unicodedata from datetime import datetime @@ -6,22 +7,13 @@ from selenium import webdriver import requests +import sys -import sys +from classes.phoneoffer import PhoneOffer file_path = 'outputfile.txt' sys.stdout = open(file_path, "w") -# Call to read the configuration file and connect to database -cinfo = config_read.get_databaseconfig("../postgresdb.config") -db_connection = psycopg2.connect( - database=cinfo[0], - host=cinfo[1], - user=cinfo[2], - password=cinfo[3] -) -cur = db_connection.cursor() - -def scrape_function(driver1, i): +def scrape_function(driver1, i, new_offers): offer_shop = "Tehnomarket" # offer shop last_updated = datetime.now().date() @@ -59,26 +51,48 @@ offer_shop_code = details[4].strip() + back_camera = None + operating_system = None + chipset = None + battery = None + ram_memory = None + rom_memory = None + cpu = None + front_camera = None + color = None + specifications = [] for info in soup2.find_all('span', {'class': 'info'}): specifications.append(info.get_text()) - print(brand) - print(offer_name) - print() - print() - offer_description = '\n'.join(specifications) - insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name, price, image_url, offer_url,' \ - 'offer_description, offer_shop_code, last_updated, is_validated)' \ - ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' - insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, offer_description, - offer_shop_code, last_updated, is_validated) - cur.execute(insert_script, insert_value) - db_connection.commit() + new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory, + color, front_camera, back_camera, chipset, battery, operating_system, cpu, + image_url, + offer_url, last_updated, is_validated, offer_description, offer_shop_code)) else: driver1.implicitly_wait(30) - scrape_function(driver1, i) + scrape_function(driver1, i, new_offers) + +# Tehnomarket phone offers that are already in database + +offers = json.loads( + unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/tehnomarket').text)) + +database_offers = [] + +for offer in offers: + phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'], + offer['ram_memory'], + offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'], + offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'], + offer['image_url'], + offer['offer_url'], offer['last_updated'], offer['is_validated'], + offer['offer_description'], + offer['offer_shop_code']) + database_offers.append(phoneOffer) + +new_offers = [] for i in range(1, 6): @@ -90,8 +104,50 @@ driver1.get(tehnomarket_url) - scrape_function(driver1, i) + scrape_function(driver1, i, new_offers) + # closing the driver so the safari instance can pair with another webdriver session driver1.close() -cur.close() -db_connection.close() +for new_offer in new_offers: + flag = False + flag_price = False + offer_id = None + + for old_offer in database_offers: + + if new_offer.offer_shop_code == old_offer.offer_shop_code: + flag = True + if new_offer.price != old_offer.price: + flag_price = True + offer_id = old_offer.offer_id + + if flag: + # print('ALREADY IN DATABASE') + # print(new_offer) + # if it's already in database, check PRICE and if it's changed, change it !!!!!! + if flag_price: + print('PRICE CHANGED!') # CHANGE PRICE + print('offer id: ' + str(offer_id)) + headers = {'Content-type': 'application/json'} + requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price), + headers=headers) + else: + print('ADDED') # ADD OFFER + print(new_offer) + headers = {'Content-type': 'application/json'} + requests.post('http://localhost:8080/phoneoffer/addoffer', + headers=headers, data=json.dumps(new_offer.__dict__, default=str)) + +print('------------------------------------') + +for old_offer in database_offers: + flag = False + for new_offer in new_offers: + if old_offer.offer_shop_code == new_offer.offer_shop_code: + flag = True + + if not flag: + print('OFFER DELETED') + print(old_offer) + # DELETE OFFER + requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id)) Index: phonelux_scrappers/test.py =================================================================== --- phonelux_scrappers/test.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) +++ phonelux_scrappers/test.py (revision 895cd8764a7d97717d7e3eb6f4248a199d3623f5) @@ -0,0 +1,34 @@ +import requests +import sys +import unicodedata +import json +from classes.phone import Phone +from classes.phoneoffer import PhoneOffer + +file_path = 'outputfile.txt' +sys.stdout = open(file_path, "w") + +# testing connection with spring backend app +# print(unicodedata.normalize('NFKD', str(requests.get("http://localhost:8080/phoneoffer/7").text))) + +offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/a1').text)) + + +phoneOffers = [] + +for offer in offers: + phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'], offer['ram_memory'], + offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'], + offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'], offer['image_url'], + offer['offer_url'], offer['last_updated'], offer['is_validated'], offer['offer_description'], + offer['offer_shop_code']) + phoneOffers.append(phoneOffer) + +for offer in phoneOffers: + print(offer) + + + + + +