Changeset cd64b06 for backend/GlobeGuru-backend

Timestamp:

01/11/25 01:23:10 (5 days ago)

Author:

Kristijan <kristijanzafirovski26@…>

Branches:

master

Children:

0a7426e

Parents:

1c51912

Message:

Added info scraping for escape travel

Location:

backend/GlobeGuru-backend

Files:

• src/main/java/DatabaseUtil.java (modified) (2 diffs)
• src/main/java/FrontendHandler.java (modified) (1 diff)
• src/main/java/ScraperThread.java (modified) (3 diffs)
• src/main/java/Server.java (modified) (1 diff)
• src/main/resources/URLsJSON.json (modified) (1 diff)
• target/classes/DatabaseUtil.class (modified) ( previous)
• target/classes/FrontendHandler.class (modified) ( previous)
• target/classes/ScraperThread.class (modified) ( previous)
• target/classes/Server.class (modified) ( previous)
• target/classes/URLsJSON.json (modified) (1 diff)
• target/classes/lastUpdateTime.json (modified) (1 diff)

backend/GlobeGuru-backend/src/main/java/DatabaseUtil.java

@@ -151 +151 @@
                 "FROM options o " +
                 "LEFT JOIN optionDetails od ON o.id = od.optionId " +
-                "WHERE (o.country LIKE ? OR o.hotelName LIKE ?)";
+                "WHERE (o.country LIKE ? OR o.hotelName LIKE ?) " +
+                "AND od.type IS NOT NULL " +
+                "AND od.board IS NOT NULL " +
+                "AND od.price > 0";
 
         if (dateQuery != null && !dateQuery.isEmpty() && !dateFlag) {
@@ -200 +203 @@
         return options;
     }
+
 
 

backend/GlobeGuru-backend/src/main/java/FrontendHandler.java

@@ -130 +130 @@
             return;
         }
-
         Server.sendResponse(exchange, 200, mapper.writeValueAsString(options));
     }

backend/GlobeGuru-backend/src/main/java/ScraperThread.java

@@ -96 +96 @@
                             } else if (optionSet.add(option)) {
                                 uniqueOptions.add(option);
-                                DatabaseUtil.saveOptionToDatabase(option);
+                                option.setId(DatabaseUtil.saveOptionToDatabase(option));
+                                scrapeOptionInfo(option);
                                 System.out.println("Parsed " + option);
                             }
@@ -149 +150 @@
 
              String board = roomOption.select(".rezervacija-objekt").text();
-             if(board.length() > 2){
-                 board = board.substring(0,2);
+             if(board.length() > 2) {
+                 board = board.substring(0, 2);
              }
-             if(board.isEmpty() || type.isEmpty()){
-                 continue;
-             }
+             if(board.isEmpty() || type.isEmpty()) continue;
+
              Elements amenityElement = roomOption.select(".objekt-opis");
              String amenity = (amenityElement != null ? amenityElement.text() : "");
@@ -166 +166 @@
              DatabaseUtil.saveOptionDetails(option.getId(), type,board,amenity, price);
          }
+        }
+        else if(url.contains("booking.escapetravel.mk")){
+            System.out.println("Scraping info for " + url);
+
+            driver.get(url);
+            try { Thread.sleep(5000); } catch (InterruptedException e) { e.printStackTrace(); } //data fetch
+            String pageSource = driver.getPageSource();
+            Document doc = Jsoup.parse(pageSource);
+            Elements roomOptions = doc.select("#hotel-rooms-container .hotel-room-row");
+            for(Element roomOption : roomOptions){
+                String type = roomOption.select("td.align-middle").first().text();
+                String board = roomOption.select("td.align-middle.text-primary.lead").text();
+                if (board.isEmpty() || type.isEmpty())  continue;
+                String priceText = roomOption.select("td.align-middle.text-end .text-success.d-block.lead").text().replace("€", "").trim();
+                float price;
+                if (!priceText.isEmpty()) {
+                    price = Float.parseFloat(priceText.replace(",", ""));
+                } else continue;
+
+                Elements amenityElements = doc.select("div.row > div.col-6.col-md-3.col-xl-2");
+                StringBuilder amenities = new StringBuilder();
+                for (Element amenityElement : amenityElements) {
+                    amenities.append(amenityElement.text()).append(", ");
+                }
+                if (!amenities.isEmpty()) {
+                    amenities.setLength(amenities.length() - 2);
+                }
+                System.out.println(type + board + price + amenities);
+                DatabaseUtil.saveOptionDetails(option.getId(), type, board, amenities.toString(), price);
+            }
+
         }
     }

backend/GlobeGuru-backend/src/main/java/Server.java

@@ -196 +196 @@
 
     public static void sendResponse(HttpExchange exchange, int statusCode, String response) throws IOException {
+        byte[] responseBytes = response.getBytes(StandardCharsets.UTF_8);
         exchange.getResponseHeaders().add("Cross-Origin-Opener-Policy", "unsafe-none");
         exchange.getResponseHeaders().add("Cross-Origin-Embedder-Policy", "require-corp");
         exchange.getResponseHeaders().add("Content-Type", "application/json");
-        exchange.sendResponseHeaders(statusCode, response.length());
-        exchange.getResponseBody().write(response.getBytes());
-        exchange.getResponseBody().close();
-    }
-
+        exchange.getResponseHeaders().add("Transfer-Encoding", "chunked");
+        exchange.sendResponseHeaders(statusCode, 0);
+        try (OutputStream os = exchange.getResponseBody()) {
+            os.write(responseBytes);
+        }
+    }
 }
 

backend/GlobeGuru-backend/src/main/resources/URLsJSON.json

@@ -2 +2 @@
   "agencyurls":
           [
-            "https://magelantravel.mk/"
-
+            "https://magelantravel.mk/",
+            "https://booking.escapetravel.mk/"
           ]
 }

backend/GlobeGuru-backend/target/classes/URLsJSON.json

@@ -2 +2 @@
   "agencyurls":
           [
-            "https://magelantravel.mk/"
-
+            "https://booking.escapetravel.mk/"
           ]
 }

backend/GlobeGuru-backend/target/classes/lastUpdateTime.json

@@ -1 +1 @@
 {
-  "lastUpdateTime" : "2025-01-10T18:31:22.019011800"
+  "lastUpdateTime" : "2025-01-11T00:07:21.694581100"
 }