source: backend/GlobeGuru-backend/src/main/java/ScraperThread.java@ 1c51912

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.support.ui.ExpectedCondition;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;

import java.io.File;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CountDownLatch;

public class ScraperThread extends Thread {
    private String url;
    private ConcurrentLinkedQueue<Option> uniqueOptions;
    private CountDownLatch latch;
    private Set<Option> optionSet;

    public ScraperThread(String url, ConcurrentLinkedQueue<Option> optionsQueue, CountDownLatch latch) {
        this.url = url;
        this.uniqueOptions = optionsQueue;
        this.latch = latch;
        this.optionSet = new HashSet<>();
    }

    public WebDriver driver;

    private void initializeWebDriver() {
        System.setProperty("webdriver.chrome.driver", "C:\\chromedriver-win64\\chromedriver.exe");
        ChromeOptions options = new ChromeOptions();
        options.setBinary("C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\brave.exe");
        options.addArguments("--headless");
        options.addArguments("--disable-gpu");
        options.addArguments("--remote-allow-origins=*");
        options.addArguments("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36");
        driver = new ChromeDriver(options);
    }

    private void closeWebDriver() {
        if (driver != null) {
            driver.quit();
        }
    }

    private void connectToWeb(String queryUrl, int numPeople) {
        driver.get(queryUrl);

        WebDriverWait wait = new WebDriverWait(driver, 40); // 40s timeout buffer
        switch (url) {
            case "https://booking.escapetravel.mk/":
                wait.until(ExpectedConditions.presenceOfElementLocated(By.cssSelector("#hotels-container")));
                try { Thread.sleep(10000);} catch (InterruptedException e) { e.printStackTrace(); }//price fetch
                break;
            case "https://magelantravel.mk/":
                wait.until(ExpectedConditions.presenceOfElementLocated(By.cssSelector("div.sodrzina")));
                break;
        }

        String pageSource = driver.getPageSource();
        System.out.println("Connected to " + queryUrl);
        Document doc = Jsoup.parse(pageSource);
        Element parentDiv;
        Elements childDivs;

        switch (url) {
            case "https://booking.escapetravel.mk/":
                parentDiv = doc.selectFirst("#hotels-container");
                if (parentDiv != null) {
                    childDivs = parentDiv.select("a.hotel-item");
                    for (Element div : childDivs) {
                        String data = div.outerHtml();
                        Option option = optionParser(data,numPeople);
                        if (option != null) {
                            Option existingOption = DatabaseUtil.findOption(option);
                            if (existingOption != null) {
                                if (existingOption.equals(option)) {
                                    option.setPriceChanged(true);
                                    option.setNewPrice(option.getPrice());
                                }
                                DatabaseUtil.updateOptionInDatabase(option);
                            } else if (optionSet.add(option)) {
                                uniqueOptions.add(option);
                                DatabaseUtil.saveOptionToDatabase(option);
                                System.out.println("Parsed " + option);
                            }
                        }
                    }
                } else {
                    System.out.println("Parent div not found");
                }
                break;
            case "https://magelantravel.mk/":
                parentDiv = doc.selectFirst("div.sodrzina");
                if (parentDiv != null) {
                    childDivs = parentDiv.select("div.destinacija");
                    childDivs.removeIf(div -> div.attr("style").contains("display:none") || div.attr("style").contains("display: none"));
                    System.out.println("Filtered childDivs size: " + childDivs.size());
                    for (Element div : childDivs) {
                        String data = div.outerHtml();
                        Option newOption = optionParser(data,numPeople);
                        if (newOption != null) {
                            if (optionSet.add(newOption)) {
                                uniqueOptions.add(newOption);

                                newOption.setId(DatabaseUtil.saveOptionToDatabase(newOption));
                                scrapeOptionInfo(newOption);
                                System.out.println("Parsed " + newOption);
                            }
                        }
                    }

        } else {
                    System.out.println("Parent div not found");
                }
                break;
            default:
                System.out.println("URL not recognized for parsing.");
        }
    }
    private void scrapeOptionInfo(Option option) {
        String url = option.getLink();
        if(url.contains("magelantravel.mk")) {
            System.out.println("Scraping info for " + option.getHotelName());
         String[] dates = option.getDateRange().split(" - ");
         url += "&checkin=" + dates[0] + "&checkout=" + dates[1] + "&adult=" + option.getNumPeople();

         driver.get(url);
         try { Thread.sleep(5000); } catch (InterruptedException e) { e.printStackTrace(); } //data fetch
         String pageSource = driver.getPageSource();
         Document doc = Jsoup.parse(pageSource);
         Elements roomOptions = doc.select(".tblroom > tbody > tr");
         for (Element roomOption : roomOptions) {
             String type = roomOption.select("a.tblroom-type").text();

             String board = roomOption.select(".rezervacija-objekt").text();
             if(board.length() > 2){
                 board = board.substring(0,2);
             }
             if(board.isEmpty() || type.isEmpty()){
                 continue;
             }
             Elements amenityElement = roomOption.select(".objekt-opis");
             String amenity = (amenityElement != null ? amenityElement.text() : "");
             System.out.println(amenity + " " + board + " " + type );
             String priceText = roomOption.select(".tbl-cena").text().replace("€", "").trim();
             float price;
             if (!priceText.isEmpty()) {
                 price = Float.parseFloat(priceText);
             }else continue;

             DatabaseUtil.saveOptionDetails(option.getId(), type,board,amenity, price);
         }
        }
    }
    private Option optionParser(String data, int numPeople){
        Document doc = Jsoup.parse(data);
        Option created = new Option();
        switch (url) {
            case "https://magelantravel.mk/":
                created = parseMagelan(doc);
                created.setNumPeople(numPeople);
                break;
            case "https://booking.escapetravel.mk/":
                created = parseEscapeTravel(doc);
                created.setNumPeople(numPeople);
                break;
            default:
                System.out.println("URL not recognized for parsing.");
                break;
        }
        if (created.isEmpty()) {
            return null;
        }
        //scrapeOptionInfo(created);
        return created;
    }

    private Option parseMagelan(Document doc) {
        Option created = new Option();
        Element linkElement = doc.selectFirst("div.ponuda-sredina");
        int id = Integer.parseInt(linkElement.attr("data-id"));
        int turop = Integer.parseInt(linkElement.attr("data-turop"));
        created.setLink("https://magelantravel.mk/ponudi.php?type=1&objektid=" + id + "&turop=" + turop);
        Element imgElement = doc.selectFirst("div.imgLiquidFill.imgLiquid.ponuda-img.zoom");
        created.setImgSrc(imgElement != null ? url + imgElement.attr("style")
                .split("url\\(")[1].split("\\)")[0].replace("'", "").replace("./", "/") : null);
        Element hotelNameElement = doc.selectFirst("div.ponuda-objekt");
        created.setHotelName(hotelNameElement != null ? hotelNameElement.text() : null);
        Element countryElement = doc.selectFirst("l.ponuda-lokacija");
        created.setCountry(countryElement != null ? countryElement.text() : null);
        //Element priceElement = doc.selectFirst("div.ponuda-cena");
        Element dateElement = doc.selectFirst("l.ponuda-opis.termin");
        created.setDateRange(dateElement != null ? dateElement.text() : null);
        /*float price = Float.parseFloat(priceElement != null ? priceElement.text().replaceAll("[^\\d.]", "") : "0");
        created.setPrice(price);*/
        return created;
    }
    private Option parseEscapeTravel(Document doc) {
        Option created = new Option();
        Element card = doc.selectFirst("a.hotel-item");
        String link = card.attr("href");
        created.setLink(link);
        created.setImgSrc(card.attr("data-picture"));
        created.setHotelName(card.attr("data-title"));
        Element countryP = doc.selectFirst("p.text-info");
        String country = countryP.text().replaceAll("leto hoteli", "");
        created.setCountry(country);
        /*Element priceElem = doc.selectFirst("span.hotel-price");
        String priceText = priceElem.text();
        float price = 0;
        if(!priceText.isEmpty()) {
            price = Float.parseFloat(priceText.replace("€", ""));
        }
        created.setPrice(price);*/
        String[] queryParams = link.split("[?&]");
        String startDateStr = null;
        int nights = 0;
        for (String param : queryParams) {
            if (param.startsWith("Date=")) {
                startDateStr = param.split("=")[1];
            }
            if (param.startsWith("Nights=")) {
                nights = Integer.parseInt(param.split("=")[1]);
            }
        }
        if (startDateStr != null && nights > 0)
        {
            SimpleDateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy");
            try {
                Date startDate = dateFormat.parse(startDateStr);

                Calendar calendar = Calendar.getInstance();
                calendar.setTime(startDate);
                calendar.add(Calendar.DAY_OF_YEAR, nights);
                Date endDate = calendar.getTime();
                String dateRange = dateFormat.format(startDate) + " - " + dateFormat.format(endDate);
                created.setDateRange(dateRange);
            }catch (ParseException e){
                e.printStackTrace();
            }
        }
        return created;
    }

    @Override
    public void run() {
        System.out.println("Thread started for url: " + url);
        initializeWebDriver();
        if ("https://magelantravel.mk/".equals(url)) {
            ObjectMapper mapper = new ObjectMapper();
            try {
                ClassLoader classLoader = getClass().getClassLoader();
                JsonNode root = mapper.readTree(new File(classLoader.getResource("CountriesList.json").getFile()));
                JsonNode countries = root.get("countries");
                SimpleDateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy");
                Calendar calendar = Calendar.getInstance();
                calendar.add(Calendar.DAY_OF_YEAR, 1);

                for (int i = 0; i < 90; i++) { // next three months
                    String date = dateFormat.format(calendar.getTime());
                    for (JsonNode countryNode : countries) {
                        String country = countryNode.asText();
                        for (int nokevanja = 2; nokevanja <= 10; nokevanja++) {
                            for(int lugje = 1; lugje <= 4; lugje++) {
                                String queryUrl = url + "/destinacii?ah_tip=1&iframe=&affiliate_code=&carter_id=0&carter_region=&carter_dataod=&carter_datado=&destinacija=" + country + "&oddatum=" + date + "&nokevanja=" + nokevanja + "&dodatum=&broj_vozrasni=" + lugje + "&broj_deca=0&spdete1=0&spdete2=0&spdete3=0&spdete4=0";
                                connectToWeb(queryUrl,lugje);
                            }
                        }
                    }
                    calendar.add(Calendar.DAY_OF_YEAR, 1); // next day
                }

            } catch (IOException e) {
                e.printStackTrace();
            }
        } else if ("https://booking.escapetravel.mk/".equals(url)) {
            ObjectMapper mapper = new ObjectMapper();
            try {
                ClassLoader classLoader = getClass().getClassLoader();
                JsonNode root = mapper.readTree(new File(classLoader.getResource("CountriesList.json").getFile()));
                JsonNode countries = root.get("countries");
                SimpleDateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy");
                Calendar calendar = Calendar.getInstance();
                calendar.add(Calendar.DAY_OF_YEAR, 1);

                for (int i = 0; i < 90; i++) { // next three months
                    String date = dateFormat.format(calendar.getTime());
                    for (JsonNode countryNode : countries) {
                        String country = countryNode.asText();
                        for(int nokevanja = 2; nokevanja <=10; nokevanja ++) {
                            for(int lugje = 1; lugje <= 4; lugje++) {
                                String queryUrl = url + "/hotels?Search=" + country + "&Date=" + date + "&Nights=" + nokevanja + "&Rooms=1&Adults=" + lugje;
                                connectToWeb(queryUrl,lugje);
                            }
                        }
                    }
                    calendar.add(Calendar.DAY_OF_YEAR, 1); // next day
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        } else {
            // Handle other URLs
        }
        closeWebDriver();
        latch.countDown();
    }

}