Changeset 1c51912 for backend/GlobeGuru-backend/src/main/java/ScraperThread.java

Timestamp:

01/10/25 19:07:51 (5 days ago)

Author:

Kristijan <kristijanzafirovski26@…>

Branches:

master

Children:

cd64b06

Parents:

53bad7e

Message:

Added details for magelan

File:

• backend/GlobeGuru-backend/src/main/java/ScraperThread.java (modified) (10 diffs)

backend/GlobeGuru-backend/src/main/java/ScraperThread.java

@@ -39 +39 @@
     }
 
-    private WebDriver driver;
+    public WebDriver driver;
 
     private void initializeWebDriver() {
@@ -70 +70 @@
                 wait.until(ExpectedConditions.presenceOfElementLocated(By.cssSelector("div.sodrzina")));
                 break;
-            default:
-                System.out.println("URL not recognized for waiting condition.");
-                // Handle other URLs if needed
         }
 
@@ -92 +89 @@
                             Option existingOption = DatabaseUtil.findOption(option);
                             if (existingOption != null) {
-                                if (existingOption.equals(option) || existingOption.getPrice() != option.getPrice()) {
+                                if (existingOption.equals(option)) {
                                     option.setPriceChanged(true);
                                     option.setNewPrice(option.getPrice());
@@ -112 +109 @@
                 if (parentDiv != null) {
                     childDivs = parentDiv.select("div.destinacija");
-                    System.out.println(childDivs.size());
                     childDivs.removeIf(div -> div.attr("style").contains("display:none") || div.attr("style").contains("display: none"));
                     System.out.println("Filtered childDivs size: " + childDivs.size());
@@ -119 +115 @@
                         Option newOption = optionParser(data,numPeople);
                         if (newOption != null) {
-                            Option existingOption = DatabaseUtil.findOption(newOption);
-                            if (existingOption != null) {
-                                if (existingOption.equals(newOption) || existingOption.getPrice() != newOption.getPrice()) {
-                                    newOption.setPriceChanged(true);
-                                    newOption.setNewPrice(newOption.getPrice());
-                                }
-                                DatabaseUtil.updateOptionInDatabase(newOption);
-                            } else if (optionSet.add(newOption)) {
+                            if (optionSet.add(newOption)) {
                                 uniqueOptions.add(newOption);
-                                DatabaseUtil.saveOptionToDatabase(newOption);
+
+                                newOption.setId(DatabaseUtil.saveOptionToDatabase(newOption));
+                                scrapeOptionInfo(newOption);
                                 System.out.println("Parsed " + newOption);
                             }
@@ -142 +133 @@
         }
     }
-
-
-
-    private Option optionParser(String data, int numPeople) {
+    private void scrapeOptionInfo(Option option) {
+        String url = option.getLink();
+        if(url.contains("magelantravel.mk")) {
+            System.out.println("Scraping info for " + option.getHotelName());
+         String[] dates = option.getDateRange().split(" - ");
+         url += "&checkin=" + dates[0] + "&checkout=" + dates[1] + "&adult=" + option.getNumPeople();
+
+         driver.get(url);
+         try { Thread.sleep(5000); } catch (InterruptedException e) { e.printStackTrace(); } //data fetch
+         String pageSource = driver.getPageSource();
+         Document doc = Jsoup.parse(pageSource);
+         Elements roomOptions = doc.select(".tblroom > tbody > tr");
+         for (Element roomOption : roomOptions) {
+             String type = roomOption.select("a.tblroom-type").text();
+
+             String board = roomOption.select(".rezervacija-objekt").text();
+             if(board.length() > 2){
+                 board = board.substring(0,2);
+             }
+             if(board.isEmpty() || type.isEmpty()){
+                 continue;
+             }
+             Elements amenityElement = roomOption.select(".objekt-opis");
+             String amenity = (amenityElement != null ? amenityElement.text() : "");
+             System.out.println(amenity + " " + board + " " + type );
+             String priceText = roomOption.select(".tbl-cena").text().replace("€", "").trim();
+             float price;
+             if (!priceText.isEmpty()) {
+                 price = Float.parseFloat(priceText);
+             }else continue;
+
+             DatabaseUtil.saveOptionDetails(option.getId(), type,board,amenity, price);
+         }
+        }
+    }
+    private Option optionParser(String data, int numPeople){
         Document doc = Jsoup.parse(data);
         Option created = new Option();
@@ -162 +185 @@
         }
         if (created.isEmpty()) {
-            System.out.println(created);
             return null;
         }
+        //scrapeOptionInfo(created);
         return created;
     }
@@ -181 +204 @@
         Element countryElement = doc.selectFirst("l.ponuda-lokacija");
         created.setCountry(countryElement != null ? countryElement.text() : null);
-        Element priceElement = doc.selectFirst("div.ponuda-cena");
+        //Element priceElement = doc.selectFirst("div.ponuda-cena");
         Element dateElement = doc.selectFirst("l.ponuda-opis.termin");
         created.setDateRange(dateElement != null ? dateElement.text() : null);
-        float price = Float.parseFloat(priceElement != null ? priceElement.text().replaceAll("[^\\d.]", "") : "0");
-        created.setPrice(price);
+        /*float price = Float.parseFloat(priceElement != null ? priceElement.text().replaceAll("[^\\d.]", "") : "0");
+        created.setPrice(price);*/
         return created;
     }
@@ -198 +221 @@
         String country = countryP.text().replaceAll("leto hoteli", "");
         created.setCountry(country);
-        Element priceElem = doc.selectFirst("span.hotel-price");
+        /*Element priceElem = doc.selectFirst("span.hotel-price");
         String priceText = priceElem.text();
         float price = 0;
@@ -204 +227 @@
             price = Float.parseFloat(priceText.replace("€", ""));
         }
-        created.setPrice(price);
+        created.setPrice(price);*/
         String[] queryParams = link.split("[?&]");
         String startDateStr = null;