Changeset c164f8f for backend/GlobeGuru-backend/src

Timestamp:

01/09/25 18:31:38 (6 days ago)

Author:

Kristijan <kristijanzafirovski26@…>

Branches:

master

Children:

53bad7e

Parents:

d4d8f61

Message:

pred-finalna

Location:

backend/GlobeGuru-backend/src/main

Files:

• java/AccountHandler.java (added)
• java/DatabaseUtil.java (added)
• java/FrontendHandler.java (added)
• java/Option.java (modified) (3 diffs)
• java/Scraper.java (modified) (2 diffs)
• java/ScraperServer.java (deleted)
• java/ScraperThread.java (modified) (4 diffs)
• java/Server.java (added)
• resources/CountriesList.json (added)
• resources/URLsJSON.json (moved) (moved from backend/GlobeGuru-backend/src/main/java/URLsJSON.json ) (1 diff)
• resources/lastUpdateTime.json (added)

backend/GlobeGuru-backend/src/main/java/Option.java

@@ +1 @@
+import java.util.Objects;
+
 public class Option {
+    private int id;
     private String hotelName;
     private String country;
-    private String price;
-    private String link; // Ensure to add a field for link
+    private float price;
+    private String link;
     private String imgSrc;
+
+    //Price changing
+    private float newPrice = 0;
+    private boolean isPriceChanged = false;
+    private String dateRange;
     // Constructor
-    public Option(){}
+    public Option(){
+        price = 0;
+    }
+
+    public void setDateRange(String dateRange) {
+        this.dateRange = dateRange;
+    }
+
+    public String getDateRange() {
+        return dateRange;
+    }
 
     public boolean isEmpty(){
-        return (hotelName == null || country == null || price == null || link == null || imgSrc == null);
+        return (hotelName == null || country == null || price == 0 || link == null || imgSrc == null);
     }
-    // Getters and setters (ensure you have these methods)
     public String getHotelName() {
         return hotelName;
@@ -28 +45 @@
     }
 
-    public String getPrice() {
+    public float getPrice() {
         return price;
     }
 
-    public void setPrice(String price) {
+    public void setPrice(float price) {
         this.price = price;
     }
@@ -52 +69 @@
     }
 
-    // toString method (for debugging purposes)
+    @Override
+    public boolean equals(Object obj) {
+        if(this==obj) return true;
+        if(obj == null || getClass() != obj.getClass()) return false;
+        Option option = (Option) obj;
+        return Float.compare(option.price, price) == 0
+                && Objects.equals(hotelName, option.hotelName)
+                && Objects.equals(country, option.country)
+                && Objects.equals(link, option.link);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(hotelName,country,price,link);
+    }
+
+    public int getId() {
+        return id;
+    }
+
+    public void setId(int id) {
+        this.id = id;
+    }
+
+    //debug
     @Override
     public String toString() {
         return "Option{" +
+                "id='" + id + '\'' +
+                "dateRange='" + dateRange + '\'' +
                 "hotelName='" + hotelName + '\'' +
                 ", country='" + country + '\'' +
                 ", price='" + price + '\'' +
                 ", link='" + link + '\'' +
+                ", image='" + imgSrc +
                 '}';
     }
+
+    public void setPriceChanged(boolean a){
+        isPriceChanged = a;
+    }
+    public void setNewPrice(float a){
+        newPrice = a;
+    }
+
+    public boolean isPriceChanged() {
+        return isPriceChanged;
+    }
+
+    public float getNewPrice() {
+        return newPrice;
+    }
 }

backend/GlobeGuru-backend/src/main/java/Scraper.java

@@ -7 +7 @@
 import java.util.Iterator;
 import java.util.List;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.CountDownLatch;
 
-public class Scraper extends Thread {
+public class Scraper implements Callable<Void> {
+
     private List<String> urls;
-    private String destination;
-    private String departureDate;
-    private int numberOfPeople;
     private ConcurrentLinkedQueue<Option> optionsQueue;
     private CountDownLatch latch;
 
-    public Scraper(String destination, String departureDate, int numberOfPeople) {
+    public Scraper() {
         urls = new ArrayList<>();
         this.optionsQueue = new ConcurrentLinkedQueue<>();
         ObjectMapper mapper = new ObjectMapper();
         try {
-            JsonNode root = mapper.readTree(new File("src/main/java/URLsJSON.json"));
+            ClassLoader classLoader = getClass().getClassLoader();
+            JsonNode root = mapper.readTree(new File(classLoader.getResource("URLsJSON.json").getFile()));
+
             JsonNode urlNode = root.get("agencyurls");
             if (urlNode.isArray()) {
@@ -36 +37 @@
             throw new RuntimeException(e);
         }
-        this.destination = destination;
-        this.departureDate = departureDate;
-        this.numberOfPeople = numberOfPeople;
         this.latch = new CountDownLatch(urls.size());
     }
 
-    @Override
-    public void run() {
+
+    public Void call() {
         System.out.println("Scraper has started ");
         for (String url : urls) {
-            new ScraperThread(url, destination, departureDate, numberOfPeople, optionsQueue, latch).start();
+            new ScraperThread(url, optionsQueue, latch).start();
         }
-    }
-    public List<Option> getOptions() {
-        try {
-            latch.await();  // Wait for all threads to finish
-        } catch (InterruptedException e) {
-            e.printStackTrace();
-        }
-        return new ArrayList<>(optionsQueue);
+        return null;
     }
 }

backend/GlobeGuru-backend/src/main/java/ScraperThread.java

@@ +1 @@
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.openqa.selenium.By;
 import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.WebElement;
 import org.openqa.selenium.chrome.ChromeDriver;
 import org.openqa.selenium.chrome.ChromeOptions;
@@ -6 +10 @@
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
-
+import org.openqa.selenium.support.ui.ExpectedCondition;
+import org.openqa.selenium.support.ui.ExpectedConditions;
+import org.openqa.selenium.support.ui.WebDriverWait;
+
+import java.io.File;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.CountDownLatch;
@@ -12 +28 @@
 public class ScraperThread extends Thread {
     private String url;
-    private String destination;
-    private String departureDate;
-    private int numberOfPeople;
     private ConcurrentLinkedQueue<Option> uniqueOptions;
     private CountDownLatch latch;
-
-    public ScraperThread(String url, String destination, String departureDate, int numberOfPeople, ConcurrentLinkedQueue<Option> optionsQueue, CountDownLatch latch) {
+    private Set<Option> optionSet;
+
+    public ScraperThread(String url, ConcurrentLinkedQueue<Option> optionsQueue, CountDownLatch latch) {
         this.url = url;
-        this.destination = destination;
-        this.departureDate = departureDate;
-        this.numberOfPeople = numberOfPeople;
         this.uniqueOptions = optionsQueue;
         this.latch = latch;
+        this.optionSet = new HashSet<>();
+    }
+
+    private WebDriver driver;
+
+    private void initializeWebDriver() {
+        System.setProperty("webdriver.chrome.driver", "C:\\chromedriver-win64\\chromedriver.exe");
+        ChromeOptions options = new ChromeOptions();
+        options.setBinary("C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\brave.exe");
+        options.addArguments("--headless");
+        options.addArguments("--disable-gpu");
+        options.addArguments("--remote-allow-origins=*");
+        options.addArguments("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36");
+        driver = new ChromeDriver(options);
+    }
+
+    private void closeWebDriver() {
+        if (driver != null) {
+            driver.quit();
+        }
     }
 
     private void connectToWeb(String queryUrl) {
-        // Selenium
-        ChromeOptions options = new ChromeOptions();
-        options.setBinary("C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\brave.exe"); // Path to Brave, remove for Chrome compatibility
-        options.addArguments("--headless");  // Run in headless mode
-        options.addArguments("--disable-gpu");
-        options.addArguments("--window-size=1920,1080");
-        options.addArguments("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"); // User-Agent
-
-        // chromeDriver
-        System.setProperty("webdriver.chrome.driver", "C:\\drivers\\chromedriver.exe");
-        System.setProperty("webdriver.http.factory", "jdk-http-client");
-        WebDriver driver = new ChromeDriver(options);
-        try {
-            // Navigate to URL
-            driver.get(queryUrl);
-            Thread.sleep(10000); // Sleep to fetch all data
-
-            // Get page source
-            String pageSource = driver.getPageSource();
-            System.out.println("Thread " + Thread.currentThread().getId() + " connected to " + queryUrl);
-
-            // Get only options
-            Document doc = Jsoup.parse(pageSource);
-            Element parentDiv;
-            Elements childDivs;
-            switch (url) {
-                case "https://www.fibula.com.mk/":
-                    parentDiv = doc.selectFirst("div.flex.flex-col.gap-5");
-                    if (parentDiv != null) {
-                        childDivs = parentDiv.select("div");
-                        for (Element div : childDivs) {
-                            String data = div.html();
-                            Option option = optionParser(data);
-                            if (option != null) {
-                                if (uniqueOptions.add(option)) {
-                                    System.out.println("Parsed Option: " + option);
+        driver.get(queryUrl);
+
+        WebDriverWait wait = new WebDriverWait(driver, 40); // 40s timeout buffer
+        switch (url) {
+            case "https://booking.escapetravel.mk/":
+                wait.until(ExpectedConditions.presenceOfElementLocated(By.cssSelector("#hotels-container")));
+                try { Thread.sleep(5000);} catch (InterruptedException e) { e.printStackTrace(); }
+                break;
+            case "https://magelantravel.mk/":
+                wait.until(ExpectedConditions.presenceOfElementLocated(By.cssSelector("div.sodrzina")));
+                break;
+            default:
+                System.out.println("URL not recognized for waiting condition.");
+                // Handle other URLs if needed
+        }
+
+        String pageSource = driver.getPageSource();
+        System.out.println("Connected to " + queryUrl);
+        Document doc = Jsoup.parse(pageSource);
+        Element parentDiv;
+        Elements childDivs;
+
+        switch (url) {
+            case "https://www.fibula.com.mk/":
+                parentDiv = doc.selectFirst("div.flex.flex-col.gap-5");
+                if (parentDiv != null) {
+                    childDivs = parentDiv.select("div");
+                    for (Element div : childDivs) {
+                        String data = div.html();
+                        Option option = optionParser(data);
+                        if (option != null && optionSet.add(option)) {
+                            uniqueOptions.add(option);
+                            System.out.println("Parsed " + option);
+                        }
+                    }
+                } else {
+                    System.out.println("Parent div not found");
+                }
+                break;
+            case "https://booking.escapetravel.mk/":
+                parentDiv = doc.selectFirst("#hotels-container");
+                if (parentDiv != null) {
+                    childDivs = parentDiv.select("a.hotel-item");
+                    for (Element div : childDivs) {
+                        String data = div.outerHtml();
+                        Option option = optionParser(data);
+                        if (option != null) {
+                            Option existingOption = DatabaseUtil.findOption(option);
+                            if (existingOption != null) {
+                                if (existingOption.equals(option) || existingOption.getPrice() != option.getPrice()) {
+                                    option.setPriceChanged(true);
+                                    option.setNewPrice(option.getPrice());
                                 }
+                                DatabaseUtil.updateOptionInDatabase(option);
+                            } else if (optionSet.add(option)) {
+                                uniqueOptions.add(option);
+                                DatabaseUtil.saveOptionToDatabase(option);
+                                System.out.println("Parsed " + option);
                             }
                         }
-                    } else {
-                        System.out.println("Parent div not found");
-                    }
-                    break;
-                case "https://booking.escapetravel.mk/":
-                    parentDiv = doc.selectFirst("div.container.pt-4.pt-md-6.scroll-into-view");
-                    Element subParent;
-                    System.out.println(parentDiv);
-                    if(parentDiv != null) {
-                         subParent = parentDiv.selectFirst("div.row");
-                    }else{
-                        System.out.println("Parent div not found");
-                        break;
-                    }
-
-                    if (subParent != null) {
-                        childDivs = subParent.select("div.col-md-3");
-
-                        for (Element div : childDivs) {
-                            String data = div.html();
-                            Option option = optionParser(data);
-                            if (option != null) {
-                                if (uniqueOptions.add(option)) {
-                                    System.out.println("Parsed option: " + option);
+                    }
+                } else {
+                    System.out.println("Parent div not found");
+                }
+                break;
+            case "https://magelantravel.mk/":
+                parentDiv = doc.selectFirst("div.sodrzina");
+                if (parentDiv != null) {
+                    childDivs = parentDiv.select("div.destinacija");
+                    System.out.println(childDivs.size());
+                    childDivs.removeIf(div -> div.attr("style").contains("display:none") || div.attr("style").contains("display: none"));
+                    System.out.println("Filtered childDivs size: " + childDivs.size());
+                    for (Element div : childDivs) {
+                        String data = div.outerHtml();
+                        Option newOption = optionParser(data);
+                        if (newOption != null) {
+                            Option existingOption = DatabaseUtil.findOption(newOption);
+                            if (existingOption != null) {
+                                if (existingOption.equals(newOption) || existingOption.getPrice() != newOption.getPrice()) {
+                                    newOption.setPriceChanged(true);
+                                    newOption.setNewPrice(newOption.getPrice());
                                 }
+                                DatabaseUtil.updateOptionInDatabase(newOption);
+                            } else if (optionSet.add(newOption)) {
+                                uniqueOptions.add(newOption);
+                                DatabaseUtil.saveOptionToDatabase(newOption);
+                                System.out.println("Parsed " + newOption);
                             }
                         }
-                    }else {
-                        System.out.println("subparent div not found");
-                    }
-                    break;
-            }
-        } catch (InterruptedException e) {
-            e.printStackTrace();
-        } finally {
-            driver.quit();
-            latch.countDown();
-        }
-    }
+                    }
+
+        } else {
+                    System.out.println("Parent div not found");
+                }
+                break;
+            default:
+                System.out.println("URL not recognized for parsing.");
+        }
+    }
+
+
 
     private Option optionParser(String data) {
         Document doc = Jsoup.parse(data);
         Option created = new Option();
-
         switch (url) {
-            case "https://www.fibula.com.mk/":
-                created = parseFibula(doc);
+            case "https://magelantravel.mk/":
+                created = parseMagelan(doc);
                 break;
             case "https://booking.escapetravel.mk/":
@@ -122 +175 @@
                 break;
         }
-
         if (created.isEmpty()) {
+            System.out.println(created);
             return null;
         }
-
         return created;
     }
 
-    private Option parseFibula(Document doc) {
+    private Option parseMagelan(Document doc) {
         Option created = new Option();
-
-        Element linkElement = doc.selectFirst("a[target='_blank']");
-        created.setLink(linkElement != null ? url + linkElement.attr("href") : null);
-
-        Element imgElement = doc.selectFirst("div.md\\:aspect-none img");
-        created.setImgSrc(imgElement != null ? imgElement.attr("src") : null);
-
-        Element hotelNameElement = doc.selectFirst("h5.text-md");
+        Element linkElement = doc.selectFirst("div.ponuda-sredina");
+        int id = Integer.parseInt(linkElement.attr("data-id"));
+        int turop = Integer.parseInt(linkElement.attr("data-turop"));
+        created.setLink("https://magelantravel.mk/ponudi.php?type=1&objektid=" + id + "&turop=" + turop);
+        Element imgElement = doc.selectFirst("div.imgLiquidFill.imgLiquid.ponuda-img.zoom");
+        created.setImgSrc(imgElement != null ? url + imgElement.attr("style")
+                .split("url\\(")[1].split("\\)")[0].replace("'", "").replace("./", "/") : null);
+        Element hotelNameElement = doc.selectFirst("div.ponuda-objekt");
         created.setHotelName(hotelNameElement != null ? hotelNameElement.text() : null);
-
-        Element countryElement = doc.selectFirst("small.text-navy");
+        Element countryElement = doc.selectFirst("l.ponuda-lokacija");
         created.setCountry(countryElement != null ? countryElement.text() : null);
-
-        Element priceElement = doc.selectFirst("small.line-through");
-        String price = priceElement != null ? priceElement.text().replaceAll("[^\\d.]", "") : "0";
+        Element priceElement = doc.selectFirst("div.ponuda-cena");
+        Element dateElement = doc.selectFirst("l.ponuda-opis.termin");
+        created.setDateRange(dateElement != null ? dateElement.text() : null);
+        float price = Float.parseFloat(priceElement != null ? priceElement.text().replaceAll("[^\\d.]", "") : "0");
         created.setPrice(price);
-
         return created;
     }
-
     private Option parseEscapeTravel(Document doc) {
         Option created = new Option();
-
-        // Extract link
-        Element linkElement = doc.selectFirst("a[target='_blank']");
-        created.setLink(linkElement != null ? linkElement.attr("href") : null);
-
-        // Extract image source
-        Element imgElement = doc.selectFirst("img.card-img-top");
-        created.setImgSrc(imgElement != null ? imgElement.attr("src") : null);
-
-        // Extract hotel name
-        Element hotelNameElement = doc.selectFirst("h3.fw-bold.text-body.mb-2");
-        created.setHotelName(hotelNameElement != null ? hotelNameElement.text() : null);
-
-        // Extract country/location
-        Element countryElement = doc.selectFirst("h5.fw-light.text-primary.mb-1");
-        created.setCountry(countryElement != null ? countryElement.text() : null);
-
-        // Extract price
-        Element priceElement = doc.selectFirst("h4.fw-light.text-success.mb-0");
-        String price = priceElement != null ? priceElement.text().replaceAll("[^\\d.]", "") : "0";
+        Element card = doc.selectFirst("a.hotel-item");
+        String link = card.attr("href");
+        created.setLink(link);
+        created.setImgSrc(card.attr("data-picture"));
+        created.setHotelName(card.attr("data-title"));
+        Element countryP = doc.selectFirst("p.text-info");
+        created.setCountry(countryP != null ? countryP.text() : null);
+        Element priceElem = doc.selectFirst("span.hotel-price");
+        String priceText = priceElem.text();
+        float price = 0;
+        if(!priceText.isEmpty()) {
+            price = Float.parseFloat(priceText.replace("€", ""));
+        }
         created.setPrice(price);
-
+        String[] queryParams = link.split("[?&]");
+        String startDateStr = null;
+        int nights = 0;
+        for (String param : queryParams) {
+            if (param.startsWith("Date=")) {
+                startDateStr = param.split("=")[1];
+            }
+            if (param.startsWith("Nights=")) {
+                nights = Integer.parseInt(param.split("=")[1]);
+            }
+        }
+        if (startDateStr != null && nights > 0)
+        {
+            SimpleDateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy");
+            try {
+                Date startDate = dateFormat.parse(startDateStr);
+
+                Calendar calendar = Calendar.getInstance();
+                calendar.setTime(startDate);
+                calendar.add(Calendar.DAY_OF_YEAR, nights);
+                Date endDate = calendar.getTime();
+                String dateRange = dateFormat.format(startDate) + " - " + dateFormat.format(endDate);
+                created.setDateRange(dateRange);
+            }catch (ParseException e){
+                e.printStackTrace();
+            }
+        }
         return created;
     }
 
-
     @Override
-        public void run() {
-            System.out.println("Thread started for url: " + url);
-            StringBuilder builder = new StringBuilder();
-            builder.append(url);
-            String queryUrl;
-            switch (url) {
-                case "https://www.fibula.com.mk/":
-                    builder.append("search?productType=2&"); // search for hotels
-                    for (int i = 0; i < numberOfPeople; i++) { // add all passengers (default adults)
-                        builder.append("passengers=1993-01-01&");
-                    }
-                    queryUrl = builder.toString();
-                    System.out.println(queryUrl);
-                    connectToWeb(queryUrl);
-                    break;
-                case "https://booking.escapetravel.mk/":
-                    builder.append("destinations?Category=&Search=&DateFrom=");
-                    builder.append(departureDate);
-                    builder.append("&Rooms=1&Adults=");
-                    builder.append(numberOfPeople);
-                    queryUrl = builder.toString();
-                    System.out.println(queryUrl);
-                    connectToWeb(queryUrl);
-                    break;
-                default:
-                    System.out.println("Not available for current url");
-                    latch.countDown();
-                    break;
-            }
-        }
-    }
+    public void run() {
+        System.out.println("Thread started for url: " + url);
+        initializeWebDriver();
+        if ("https://magelantravel.mk/".equals(url)) {
+            ObjectMapper mapper = new ObjectMapper();
+            try {
+                ClassLoader classLoader = getClass().getClassLoader();
+                JsonNode root = mapper.readTree(new File(classLoader.getResource("CountriesList.json").getFile()));
+                JsonNode countries = root.get("countries");
+                SimpleDateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy");
+                Calendar calendar = Calendar.getInstance();
+                calendar.add(Calendar.DAY_OF_YEAR, 1);
+
+                for (int i = 0; i < 90; i++) { // next three months
+                    String date = dateFormat.format(calendar.getTime());
+                    for (JsonNode countryNode : countries) {
+                        String country = countryNode.asText();
+                        for (int nokevanja = 2; nokevanja <= 10; nokevanja++) {
+                            String queryUrl = url + "/destinacii?ah_tip=1&iframe=&affiliate_code=&carter_id=0&carter_region=&carter_dataod=&carter_datado=&destinacija=" + country + "&oddatum=" + date + "&nokevanja=" + nokevanja + "&dodatum=&broj_vozrasni=2&broj_deca=0&spdete1=0&spdete2=0&spdete3=0&spdete4=0";
+                            connectToWeb(queryUrl);
+                        }
+                    }
+                    calendar.add(Calendar.DAY_OF_YEAR, 1); // next day
+                }
+
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        } else if ("https://booking.escapetravel.mk/".equals(url)) {
+            ObjectMapper mapper = new ObjectMapper();
+            try {
+                ClassLoader classLoader = getClass().getClassLoader();
+                JsonNode root = mapper.readTree(new File(classLoader.getResource("CountriesList.json").getFile()));
+                JsonNode countries = root.get("countries"); // Assuming "destinations" key in JSON
+                SimpleDateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy");
+                Calendar calendar = Calendar.getInstance();
+                calendar.add(Calendar.DAY_OF_YEAR, 1);
+
+                for (int i = 0; i < 90; i++) { // next three months
+                    String date = dateFormat.format(calendar.getTime());
+                    for (JsonNode countryNode : countries) {
+                        String country = countryNode.asText();
+                        for(int nokevanja = 2; nokevanja <=10; nokevanja ++) {
+                            String queryUrl = url + "/hotels?Search=" + country + "&Date=" + date + "&Nights=" + nokevanja + "&Rooms=1&Adults=2";
+                            connectToWeb(queryUrl);
+                        }
+                    }
+                    calendar.add(Calendar.DAY_OF_YEAR, 1); // next day
+                }
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        } else {
+            // Handle other URLs
+        }
+        closeWebDriver();
+        latch.countDown();
+    }
+
+}

backend/GlobeGuru-backend/src/main/resources/URLsJSON.json

@@ -2 +2 @@
   "agencyurls":
           [
-            "https://www.fibula.com.mk/",
+            "https://magelantravel.mk/",
             "https://booking.escapetravel.mk/"
           ]