source: backend/GlobeGuru-backend/src/main/java/ScraperThread.java@ 53bad7e

Last change on this file since 53bad7e was 53bad7e, checked in by Kristijan <kristijanzafirovski26@…>, 6 days ago

dodadeno informacii za broj na lugje
  • Property mode set to 100644
File size: 13.6 KB
Line 
1import com.fasterxml.jackson.databind.JsonNode;
2import com.fasterxml.jackson.databind.ObjectMapper;
3import org.openqa.selenium.By;
4import org.openqa.selenium.WebDriver;
5import org.openqa.selenium.WebElement;
6import org.openqa.selenium.chrome.ChromeDriver;
7import org.openqa.selenium.chrome.ChromeOptions;
8import org.jsoup.Jsoup;
9import org.jsoup.nodes.Document;
10import org.jsoup.nodes.Element;
11import org.jsoup.select.Elements;
12import org.openqa.selenium.support.ui.ExpectedCondition;
13import org.openqa.selenium.support.ui.ExpectedConditions;
14import org.openqa.selenium.support.ui.WebDriverWait;
15
16import java.io.File;
17import java.io.IOException;
18import java.sql.Connection;
19import java.sql.DriverManager;
20import java.sql.PreparedStatement;
21import java.sql.SQLException;
22import java.text.ParseException;
23import java.text.SimpleDateFormat;
24import java.util.*;
25import java.util.concurrent.ConcurrentLinkedQueue;
26import java.util.concurrent.CountDownLatch;
27
28public class ScraperThread extends Thread {
29 private String url;
30 private ConcurrentLinkedQueue<Option> uniqueOptions;
31 private CountDownLatch latch;
32 private Set<Option> optionSet;
33
34 public ScraperThread(String url, ConcurrentLinkedQueue<Option> optionsQueue, CountDownLatch latch) {
35 this.url = url;
36 this.uniqueOptions = optionsQueue;
37 this.latch = latch;
38 this.optionSet = new HashSet<>();
39 }
40
41 private WebDriver driver;
42
43 private void initializeWebDriver() {
44 System.setProperty("webdriver.chrome.driver", "C:\\chromedriver-win64\\chromedriver.exe");
45 ChromeOptions options = new ChromeOptions();
46 options.setBinary("C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\brave.exe");
47 options.addArguments("--headless");
48 options.addArguments("--disable-gpu");
49 options.addArguments("--remote-allow-origins=*");
50 options.addArguments("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36");
51 driver = new ChromeDriver(options);
52 }
53
54 private void closeWebDriver() {
55 if (driver != null) {
56 driver.quit();
57 }
58 }
59
60 private void connectToWeb(String queryUrl, int numPeople) {
61 driver.get(queryUrl);
62
63 WebDriverWait wait = new WebDriverWait(driver, 40); // 40s timeout buffer
64 switch (url) {
65 case "https://booking.escapetravel.mk/":
66 wait.until(ExpectedConditions.presenceOfElementLocated(By.cssSelector("#hotels-container")));
67 try { Thread.sleep(10000);} catch (InterruptedException e) { e.printStackTrace(); }//price fetch
68 break;
69 case "https://magelantravel.mk/":
70 wait.until(ExpectedConditions.presenceOfElementLocated(By.cssSelector("div.sodrzina")));
71 break;
72 default:
73 System.out.println("URL not recognized for waiting condition.");
74 // Handle other URLs if needed
75 }
76
77 String pageSource = driver.getPageSource();
78 System.out.println("Connected to " + queryUrl);
79 Document doc = Jsoup.parse(pageSource);
80 Element parentDiv;
81 Elements childDivs;
82
83 switch (url) {
84 case "https://booking.escapetravel.mk/":
85 parentDiv = doc.selectFirst("#hotels-container");
86 if (parentDiv != null) {
87 childDivs = parentDiv.select("a.hotel-item");
88 for (Element div : childDivs) {
89 String data = div.outerHtml();
90 Option option = optionParser(data,numPeople);
91 if (option != null) {
92 Option existingOption = DatabaseUtil.findOption(option);
93 if (existingOption != null) {
94 if (existingOption.equals(option) || existingOption.getPrice() != option.getPrice()) {
95 option.setPriceChanged(true);
96 option.setNewPrice(option.getPrice());
97 }
98 DatabaseUtil.updateOptionInDatabase(option);
99 } else if (optionSet.add(option)) {
100 uniqueOptions.add(option);
101 DatabaseUtil.saveOptionToDatabase(option);
102 System.out.println("Parsed " + option);
103 }
104 }
105 }
106 } else {
107 System.out.println("Parent div not found");
108 }
109 break;
110 case "https://magelantravel.mk/":
111 parentDiv = doc.selectFirst("div.sodrzina");
112 if (parentDiv != null) {
113 childDivs = parentDiv.select("div.destinacija");
114 System.out.println(childDivs.size());
115 childDivs.removeIf(div -> div.attr("style").contains("display:none") || div.attr("style").contains("display: none"));
116 System.out.println("Filtered childDivs size: " + childDivs.size());
117 for (Element div : childDivs) {
118 String data = div.outerHtml();
119 Option newOption = optionParser(data,numPeople);
120 if (newOption != null) {
121 Option existingOption = DatabaseUtil.findOption(newOption);
122 if (existingOption != null) {
123 if (existingOption.equals(newOption) || existingOption.getPrice() != newOption.getPrice()) {
124 newOption.setPriceChanged(true);
125 newOption.setNewPrice(newOption.getPrice());
126 }
127 DatabaseUtil.updateOptionInDatabase(newOption);
128 } else if (optionSet.add(newOption)) {
129 uniqueOptions.add(newOption);
130 DatabaseUtil.saveOptionToDatabase(newOption);
131 System.out.println("Parsed " + newOption);
132 }
133 }
134 }
135
136 } else {
137 System.out.println("Parent div not found");
138 }
139 break;
140 default:
141 System.out.println("URL not recognized for parsing.");
142 }
143 }
144
145
146
147 private Option optionParser(String data, int numPeople) {
148 Document doc = Jsoup.parse(data);
149 Option created = new Option();
150 switch (url) {
151 case "https://magelantravel.mk/":
152 created = parseMagelan(doc);
153 created.setNumPeople(numPeople);
154 break;
155 case "https://booking.escapetravel.mk/":
156 created = parseEscapeTravel(doc);
157 created.setNumPeople(numPeople);
158 break;
159 default:
160 System.out.println("URL not recognized for parsing.");
161 break;
162 }
163 if (created.isEmpty()) {
164 System.out.println(created);
165 return null;
166 }
167 return created;
168 }
169
170 private Option parseMagelan(Document doc) {
171 Option created = new Option();
172 Element linkElement = doc.selectFirst("div.ponuda-sredina");
173 int id = Integer.parseInt(linkElement.attr("data-id"));
174 int turop = Integer.parseInt(linkElement.attr("data-turop"));
175 created.setLink("https://magelantravel.mk/ponudi.php?type=1&objektid=" + id + "&turop=" + turop);
176 Element imgElement = doc.selectFirst("div.imgLiquidFill.imgLiquid.ponuda-img.zoom");
177 created.setImgSrc(imgElement != null ? url + imgElement.attr("style")
178 .split("url\\(")[1].split("\\)")[0].replace("'", "").replace("./", "/") : null);
179 Element hotelNameElement = doc.selectFirst("div.ponuda-objekt");
180 created.setHotelName(hotelNameElement != null ? hotelNameElement.text() : null);
181 Element countryElement = doc.selectFirst("l.ponuda-lokacija");
182 created.setCountry(countryElement != null ? countryElement.text() : null);
183 Element priceElement = doc.selectFirst("div.ponuda-cena");
184 Element dateElement = doc.selectFirst("l.ponuda-opis.termin");
185 created.setDateRange(dateElement != null ? dateElement.text() : null);
186 float price = Float.parseFloat(priceElement != null ? priceElement.text().replaceAll("[^\\d.]", "") : "0");
187 created.setPrice(price);
188 return created;
189 }
190 private Option parseEscapeTravel(Document doc) {
191 Option created = new Option();
192 Element card = doc.selectFirst("a.hotel-item");
193 String link = card.attr("href");
194 created.setLink(link);
195 created.setImgSrc(card.attr("data-picture"));
196 created.setHotelName(card.attr("data-title"));
197 Element countryP = doc.selectFirst("p.text-info");
198 String country = countryP.text().replaceAll("leto hoteli", "");
199 created.setCountry(country);
200 Element priceElem = doc.selectFirst("span.hotel-price");
201 String priceText = priceElem.text();
202 float price = 0;
203 if(!priceText.isEmpty()) {
204 price = Float.parseFloat(priceText.replace("€", ""));
205 }
206 created.setPrice(price);
207 String[] queryParams = link.split("[?&]");
208 String startDateStr = null;
209 int nights = 0;
210 for (String param : queryParams) {
211 if (param.startsWith("Date=")) {
212 startDateStr = param.split("=")[1];
213 }
214 if (param.startsWith("Nights=")) {
215 nights = Integer.parseInt(param.split("=")[1]);
216 }
217 }
218 if (startDateStr != null && nights > 0)
219 {
220 SimpleDateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy");
221 try {
222 Date startDate = dateFormat.parse(startDateStr);
223
224 Calendar calendar = Calendar.getInstance();
225 calendar.setTime(startDate);
226 calendar.add(Calendar.DAY_OF_YEAR, nights);
227 Date endDate = calendar.getTime();
228 String dateRange = dateFormat.format(startDate) + " - " + dateFormat.format(endDate);
229 created.setDateRange(dateRange);
230 }catch (ParseException e){
231 e.printStackTrace();
232 }
233 }
234 return created;
235 }
236
237 @Override
238 public void run() {
239 System.out.println("Thread started for url: " + url);
240 initializeWebDriver();
241 if ("https://magelantravel.mk/".equals(url)) {
242 ObjectMapper mapper = new ObjectMapper();
243 try {
244 ClassLoader classLoader = getClass().getClassLoader();
245 JsonNode root = mapper.readTree(new File(classLoader.getResource("CountriesList.json").getFile()));
246 JsonNode countries = root.get("countries");
247 SimpleDateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy");
248 Calendar calendar = Calendar.getInstance();
249 calendar.add(Calendar.DAY_OF_YEAR, 1);
250
251 for (int i = 0; i < 90; i++) { // next three months
252 String date = dateFormat.format(calendar.getTime());
253 for (JsonNode countryNode : countries) {
254 String country = countryNode.asText();
255 for (int nokevanja = 2; nokevanja <= 10; nokevanja++) {
256 for(int lugje = 1; lugje <= 4; lugje++) {
257 String queryUrl = url + "/destinacii?ah_tip=1&iframe=&affiliate_code=&carter_id=0&carter_region=&carter_dataod=&carter_datado=&destinacija=" + country + "&oddatum=" + date + "&nokevanja=" + nokevanja + "&dodatum=&broj_vozrasni=" + lugje + "&broj_deca=0&spdete1=0&spdete2=0&spdete3=0&spdete4=0";
258 connectToWeb(queryUrl,lugje);
259 }
260 }
261 }
262 calendar.add(Calendar.DAY_OF_YEAR, 1); // next day
263 }
264
265 } catch (IOException e) {
266 e.printStackTrace();
267 }
268 } else if ("https://booking.escapetravel.mk/".equals(url)) {
269 ObjectMapper mapper = new ObjectMapper();
270 try {
271 ClassLoader classLoader = getClass().getClassLoader();
272 JsonNode root = mapper.readTree(new File(classLoader.getResource("CountriesList.json").getFile()));
273 JsonNode countries = root.get("countries");
274 SimpleDateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy");
275 Calendar calendar = Calendar.getInstance();
276 calendar.add(Calendar.DAY_OF_YEAR, 1);
277
278 for (int i = 0; i < 90; i++) { // next three months
279 String date = dateFormat.format(calendar.getTime());
280 for (JsonNode countryNode : countries) {
281 String country = countryNode.asText();
282 for(int nokevanja = 2; nokevanja <=10; nokevanja ++) {
283 for(int lugje = 1; lugje <= 4; lugje++) {
284 String queryUrl = url + "/hotels?Search=" + country + "&Date=" + date + "&Nights=" + nokevanja + "&Rooms=1&Adults=" + lugje;
285 connectToWeb(queryUrl,lugje);
286 }
287 }
288 }
289 calendar.add(Calendar.DAY_OF_YEAR, 1); // next day
290 }
291 } catch (IOException e) {
292 e.printStackTrace();
293 }
294 } else {
295 // Handle other URLs
296 }
297 closeWebDriver();
298 latch.countDown();
299 }
300
301}
Note: See TracBrowser for help on using the repository browser.