- Timestamp:
- 01/10/25 00:33:50 (6 days ago)
- Branches:
- master
- Children:
- 1c51912
- Parents:
- c164f8f
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
backend/GlobeGuru-backend/src/main/java/ScraperThread.java
rc164f8f r53bad7e 58 58 } 59 59 60 private void connectToWeb(String queryUrl ) {60 private void connectToWeb(String queryUrl, int numPeople) { 61 61 driver.get(queryUrl); 62 62 … … 65 65 case "https://booking.escapetravel.mk/": 66 66 wait.until(ExpectedConditions.presenceOfElementLocated(By.cssSelector("#hotels-container"))); 67 try { Thread.sleep( 5000);} catch (InterruptedException e) { e.printStackTrace(); }67 try { Thread.sleep(10000);} catch (InterruptedException e) { e.printStackTrace(); }//price fetch 68 68 break; 69 69 case "https://magelantravel.mk/": … … 82 82 83 83 switch (url) { 84 case "https://www.fibula.com.mk/":85 parentDiv = doc.selectFirst("div.flex.flex-col.gap-5");86 if (parentDiv != null) {87 childDivs = parentDiv.select("div");88 for (Element div : childDivs) {89 String data = div.html();90 Option option = optionParser(data);91 if (option != null && optionSet.add(option)) {92 uniqueOptions.add(option);93 System.out.println("Parsed " + option);94 }95 }96 } else {97 System.out.println("Parent div not found");98 }99 break;100 84 case "https://booking.escapetravel.mk/": 101 85 parentDiv = doc.selectFirst("#hotels-container"); … … 104 88 for (Element div : childDivs) { 105 89 String data = div.outerHtml(); 106 Option option = optionParser(data );90 Option option = optionParser(data,numPeople); 107 91 if (option != null) { 108 92 Option existingOption = DatabaseUtil.findOption(option); … … 133 117 for (Element div : childDivs) { 134 118 String data = div.outerHtml(); 135 Option newOption = optionParser(data );119 Option newOption = optionParser(data,numPeople); 136 120 if (newOption != null) { 137 121 Option existingOption = DatabaseUtil.findOption(newOption); … … 161 145 162 146 163 private Option optionParser(String data ) {147 private Option optionParser(String data, int numPeople) { 164 148 Document doc = Jsoup.parse(data); 165 149 Option created = new Option(); … … 167 151 case "https://magelantravel.mk/": 168 152 created = parseMagelan(doc); 153 created.setNumPeople(numPeople); 169 154 break; 170 155 case "https://booking.escapetravel.mk/": 171 156 created = parseEscapeTravel(doc); 157 created.setNumPeople(numPeople); 172 158 break; 173 159 default: … … 210 196 created.setHotelName(card.attr("data-title")); 211 197 Element countryP = doc.selectFirst("p.text-info"); 212 created.setCountry(countryP != null ? countryP.text() : null); 198 String country = countryP.text().replaceAll("leto hoteli", ""); 199 created.setCountry(country); 213 200 Element priceElem = doc.selectFirst("span.hotel-price"); 214 201 String priceText = priceElem.text(); … … 267 254 String country = countryNode.asText(); 268 255 for (int nokevanja = 2; nokevanja <= 10; nokevanja++) { 269 String queryUrl = url + "/destinacii?ah_tip=1&iframe=&affiliate_code=&carter_id=0&carter_region=&carter_dataod=&carter_datado=&destinacija=" + country + "&oddatum=" + date + "&nokevanja=" + nokevanja + "&dodatum=&broj_vozrasni=2&broj_deca=0&spdete1=0&spdete2=0&spdete3=0&spdete4=0"; 270 connectToWeb(queryUrl); 256 for(int lugje = 1; lugje <= 4; lugje++) { 257 String queryUrl = url + "/destinacii?ah_tip=1&iframe=&affiliate_code=&carter_id=0&carter_region=&carter_dataod=&carter_datado=&destinacija=" + country + "&oddatum=" + date + "&nokevanja=" + nokevanja + "&dodatum=&broj_vozrasni=" + lugje + "&broj_deca=0&spdete1=0&spdete2=0&spdete3=0&spdete4=0"; 258 connectToWeb(queryUrl,lugje); 259 } 271 260 } 272 261 } … … 282 271 ClassLoader classLoader = getClass().getClassLoader(); 283 272 JsonNode root = mapper.readTree(new File(classLoader.getResource("CountriesList.json").getFile())); 284 JsonNode countries = root.get("countries"); // Assuming "destinations" key in JSON273 JsonNode countries = root.get("countries"); 285 274 SimpleDateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy"); 286 275 Calendar calendar = Calendar.getInstance(); … … 292 281 String country = countryNode.asText(); 293 282 for(int nokevanja = 2; nokevanja <=10; nokevanja ++) { 294 String queryUrl = url + "/hotels?Search=" + country + "&Date=" + date + "&Nights=" + nokevanja + "&Rooms=1&Adults=2"; 295 connectToWeb(queryUrl); 283 for(int lugje = 1; lugje <= 4; lugje++) { 284 String queryUrl = url + "/hotels?Search=" + country + "&Date=" + date + "&Nights=" + nokevanja + "&Rooms=1&Adults=" + lugje; 285 connectToWeb(queryUrl,lugje); 286 } 296 287 } 297 288 }
Note:
See TracChangeset
for help on using the changeset viewer.