Index: phonelux-backend/.gitignore =================================================================== --- phonelux-backend/.gitignore (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux-backend/.gitignore (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,33 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ Index: phonelux-backend/.mvn/wrapper/maven-wrapper.properties =================================================================== --- phonelux-backend/.mvn/wrapper/maven-wrapper.properties (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux-backend/.mvn/wrapper/maven-wrapper.properties (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,2 @@ +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.6/apache-maven-3.8.6-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar Index: phonelux-backend/mvnw =================================================================== --- phonelux-backend/mvnw (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux-backend/mvnw (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,316 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /usr/local/etc/mavenrc ] ; then + . /usr/local/etc/mavenrc + fi + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`\\unset -f command; \\command -v java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found .mvn/wrapper/maven-wrapper.jar" + fi +else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." + fi + if [ -n "$MVNW_REPOURL" ]; then + jarUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar" + else + jarUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar" + fi + while IFS="=" read key value; do + case "$key" in (wrapperUrl) jarUrl="$value"; break ;; + esac + done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" + if [ "$MVNW_VERBOSE" = true ]; then + echo "Downloading from: $jarUrl" + fi + wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" + if $cygwin; then + wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` + fi + + if command -v wget > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found wget ... using wget" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget "$jarUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + else + wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found curl ... using curl" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl -o "$wrapperJarPath" "$jarUrl" -f + else + curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f + fi + + else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Falling back to using Java to download" + fi + javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaClass=`cygpath --path --windows "$javaClass"` + fi + if [ -e "$javaClass" ]; then + if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Compiling MavenWrapperDownloader.java ..." + fi + # Compiling the Java class + ("$JAVA_HOME/bin/javac" "$javaClass") + fi + if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + # Running the downloader + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Running MavenWrapperDownloader.java ..." + fi + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +if [ "$MVNW_VERBOSE" = true ]; then + echo $MAVEN_PROJECTBASEDIR +fi +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + $MAVEN_DEBUG_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" \ + "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" Index: phonelux-backend/mvnw.cmd =================================================================== --- phonelux-backend/mvnw.cmd (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux-backend/mvnw.cmd (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,188 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %* +if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %* +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar" + +FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + if "%MVNW_VERBOSE%" == "true" ( + echo Found %WRAPPER_JAR% + ) +) else ( + if not "%MVNW_REPOURL%" == "" ( + SET DOWNLOAD_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar" + ) + if "%MVNW_VERBOSE%" == "true" ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %DOWNLOAD_URL% + ) + + powershell -Command "&{"^ + "$webclient = new-object System.Net.WebClient;"^ + "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ + "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ + "}"^ + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ + "}" + if "%MVNW_VERBOSE%" == "true" ( + echo Finished downloading %WRAPPER_JAR% + ) +) +@REM End of extension + +@REM Provide a "standardized" way to retrieve the CLI args that will +@REM work with both Windows and non-Windows executions. +set MAVEN_CMD_LINE_ARGS=%* + +%MAVEN_JAVA_EXE% ^ + %JVM_CONFIG_MAVEN_PROPS% ^ + %MAVEN_OPTS% ^ + %MAVEN_DEBUG_OPTS% ^ + -classpath %WRAPPER_JAR% ^ + "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^ + %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat" +if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%"=="on" pause + +if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE% + +cmd /C exit /B %ERROR_CODE% Index: phonelux-backend/pom.xml =================================================================== --- phonelux-backend/pom.xml (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux-backend/pom.xml (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,72 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 2.7.3 + + + finki.it + phonelux-backend + 0.0.1-SNAPSHOT + phonelux-backend + Backend application for project Phonelux + + 17 + + + + org.springframework.boot + spring-boot-starter-data-jpa + + + + + + + org.springframework.boot + spring-boot-starter-web + + + + org.postgresql + postgresql + runtime + + + org.projectlombok + lombok + true + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.projectlombok + lombok + + + + + + + + Index: phonelux-backend/src/main/java/finki/it/phoneluxbackend/PhoneluxBackendApplication.java =================================================================== --- phonelux-backend/src/main/java/finki/it/phoneluxbackend/PhoneluxBackendApplication.java (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux-backend/src/main/java/finki/it/phoneluxbackend/PhoneluxBackendApplication.java (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,13 @@ +package finki.it.phoneluxbackend; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class PhoneluxBackendApplication { + + public static void main(String[] args) { + SpringApplication.run(PhoneluxBackendApplication.class, args); + } + +} Index: phonelux-backend/src/main/java/finki/it/phoneluxbackend/controllers/PhoneController.java =================================================================== --- phonelux-backend/src/main/java/finki/it/phoneluxbackend/controllers/PhoneController.java (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux-backend/src/main/java/finki/it/phoneluxbackend/controllers/PhoneController.java (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,31 @@ +package finki.it.phoneluxbackend.controllers; + +import finki.it.phoneluxbackend.entities.Phone; +import finki.it.phoneluxbackend.entities.PhoneOffer; +import finki.it.phoneluxbackend.services.PhoneService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import java.util.List; + +@RestController +@RequestMapping(path = "/") +public class PhoneController { + private final PhoneService phoneService; + + @Autowired + public PhoneController(PhoneService phoneService) { + this.phoneService = phoneService; + } + + @GetMapping + public List getPhones(){ + return phoneService.getPhones(); + } + + @PostMapping(path = "{phoneId}") + public List getOffersForPhone(@PathVariable("phoneId") Long phoneId){ + return phoneService.getOffersForPhone(phoneId); + } + +} Index: phonelux-backend/src/main/java/finki/it/phoneluxbackend/entities/Phone.java =================================================================== --- phonelux-backend/src/main/java/finki/it/phoneluxbackend/entities/Phone.java (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux-backend/src/main/java/finki/it/phoneluxbackend/entities/Phone.java (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,37 @@ +package finki.it.phoneluxbackend.entities; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.*; + +import javax.persistence.*; +import java.util.List; + +@AllArgsConstructor +@NoArgsConstructor +@Getter +@Setter +@ToString +@Entity(name = "Phone") +@Table(name = "phones") +public class Phone { + @Id + @Column(name = "id") + private Long id; + + @Column(name = "brand") + private String brand; + + @Column(name = "model") + private String model; + + @OneToMany(fetch = FetchType.LAZY, mappedBy = "phone") + @JsonIgnore + private List phoneOffers; + + public Phone(String brand, String model) { + this.brand = brand; + this.model = model; + } + + +} Index: phonelux-backend/src/main/java/finki/it/phoneluxbackend/entities/PhoneOffer.java =================================================================== --- phonelux-backend/src/main/java/finki/it/phoneluxbackend/entities/PhoneOffer.java (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux-backend/src/main/java/finki/it/phoneluxbackend/entities/PhoneOffer.java (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,79 @@ +package finki.it.phoneluxbackend.entities; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.*; + +import javax.persistence.*; +import java.util.Date; + +@AllArgsConstructor +@NoArgsConstructor +@Getter +@Setter +@ToString +@Entity(name = "PhoneOffer") +@Table(name = "phone_offers") +public class PhoneOffer { + @Id + @Column(name = "offer_id") + private Long id; + + @Column(name = "offer_shop") + private String offer_shop; + + @Column(name = "offer_name") + private String offer_name; + + @Column(name = "price") + private Integer price; + + @Column(name = "ram_memory") + private String ram_memory; + + @Column(name = "rom_memory") + private String rom_memory; + + @Column(name = "color") + private String color; + + @Column(name = "front_camera") + private String front_camera; + + @Column(name = "back_camera") + private String back_camera; + + @Column(name = "chipset") + private String chipset; + + @Column(name = "battery") + private String battery; + + @Column(name = "operating_system") + private String operating_system; + + @Column(name = "cpu") + private String cpu; + + @Column(name = "image_url") + private String image_url; + + @Column(name = "offer_url") + private String offer_url; + + @Column(name = "last_updated") + private Date last_updated; + + @Column(name = "is_validated") + private Boolean is_validated; + + @Column(name = "offer_description") + private String offer_description; + + @Column(name = "offer_shop_code") + private String offer_shop_code; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "phone_id", referencedColumnName = "id") + @JsonIgnore + private Phone phone; +} Index: phonelux-backend/src/main/java/finki/it/phoneluxbackend/repositories/PhoneRepository.java =================================================================== --- phonelux-backend/src/main/java/finki/it/phoneluxbackend/repositories/PhoneRepository.java (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux-backend/src/main/java/finki/it/phoneluxbackend/repositories/PhoneRepository.java (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,10 @@ +package finki.it.phoneluxbackend.repositories; + +import finki.it.phoneluxbackend.entities.Phone; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface PhoneRepository extends JpaRepository { + +} Index: phonelux-backend/src/main/java/finki/it/phoneluxbackend/services/PhoneService.java =================================================================== --- phonelux-backend/src/main/java/finki/it/phoneluxbackend/services/PhoneService.java (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux-backend/src/main/java/finki/it/phoneluxbackend/services/PhoneService.java (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,30 @@ +package finki.it.phoneluxbackend.services; + +import finki.it.phoneluxbackend.entities.Phone; +import finki.it.phoneluxbackend.entities.PhoneOffer; +import finki.it.phoneluxbackend.repositories.PhoneRepository; +import org.springframework.data.domain.Sort; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class PhoneService { + private final PhoneRepository phoneRepository; + + public PhoneService(PhoneRepository phoneRepository){ + this.phoneRepository = phoneRepository; + } + + public List getPhones(){ + return phoneRepository.findAll(); + } + + public List getOffersForPhone(Long phoneId) { + boolean exists = phoneRepository.existsById(phoneId); + if(!exists) + throw new IllegalStateException("Phone with id "+phoneId+" does not exist"); + + return phoneRepository.findById(phoneId).get().getPhoneOffers(); + } +} Index: phonelux-backend/src/main/resources/application.properties =================================================================== --- phonelux-backend/src/main/resources/application.properties (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux-backend/src/main/resources/application.properties (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,7 @@ +spring.datasource.url=jdbc:postgresql://localhost:5432/phonelux +spring.datasource.username= +spring.datasource.password= +spring.jpa.hibernate.ddl-auto=update +spring.jpa.show-sql=true +spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQL9Dialect +spring.jpa.properties.hibernate.format_sql=true Index: phonelux-backend/src/test/java/finki/it/phoneluxbackend/PhoneluxBackendApplicationTests.java =================================================================== --- phonelux-backend/src/test/java/finki/it/phoneluxbackend/PhoneluxBackendApplicationTests.java (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux-backend/src/test/java/finki/it/phoneluxbackend/PhoneluxBackendApplicationTests.java (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,13 @@ +package finki.it.phoneluxbackend; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +class PhoneluxBackendApplicationTests { + + @Test + void contextLoads() { + } + +} Index: phonelux_scrappers/.idea/.gitignore =================================================================== --- phonelux_scrappers/.idea/.gitignore (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/.idea/.gitignore (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml Index: phonelux_scrappers/.idea/inspectionProfiles/profiles_settings.xml =================================================================== --- phonelux_scrappers/.idea/inspectionProfiles/profiles_settings.xml (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/.idea/inspectionProfiles/profiles_settings.xml (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,6 @@ + + + + Index: phonelux_scrappers/.idea/misc.xml =================================================================== --- phonelux_scrappers/.idea/misc.xml (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/.idea/misc.xml (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,4 @@ + + + + Index: phonelux_scrappers/.idea/modules.xml =================================================================== --- phonelux_scrappers/.idea/modules.xml (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/.idea/modules.xml (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,8 @@ + + + + + + + + Index: phonelux_scrappers/.idea/phonelux_scrappers.iml =================================================================== --- phonelux_scrappers/.idea/phonelux_scrappers.iml (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/.idea/phonelux_scrappers.iml (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,10 @@ + + + + + + + + + + Index: phonelux_scrappers/config_read.py =================================================================== --- phonelux_scrappers/config_read.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/config_read.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,11 @@ +import configparser + +def get_databaseconfig(filename): + cf = configparser.ConfigParser() + cf.read(filename) # Read configuration file + # Read corresponding file parameters + _database = cf.get("Database1", "database") + _host = cf.get("Database1", "hostname") + _username = cf.get("Database1", "username") + _pwd = cf.get("Database1", "pwd") + return _database, _host, _username, _pwd # return required parameters Index: phonelux_scrappers/outputfile.txt =================================================================== --- phonelux_scrappers/outputfile.txt (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/outputfile.txt (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,767 @@ +page: 1 +Alcatel 1066G Black +Alcatel + + +Alcatel 1066D Dual-Sim Black +Alcatel + + +Alcatel 1066D Dual-Sim White +Alcatel + + +Mobile Phone MeanIT F26 Red +MeanIT + + +Mobile Phone MeanIT F26 Black +MeanIT + + +Mobile Phone MeanIT Veteran I White +MeanIT + + +Nokia 105 2019 Dual-Sim Black +Nokia + + +Mobile Phone MeanIT Veteran I Red +MeanIT + + +Mobile Phone MeanIT Veteran I Black +MeanIT + + +Mobile Phone Trevi MAX 10 Silver +Trevi + + +Mobile Phone Trevi MAX 10 Black +Trevi + + +Mobile Phone Denver B183 Black +Denver + + +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 +page: 1 Index: phonelux_scrappers/scrappers/a1_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/a1_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/scrappers/a1_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,113 @@ +import unicodedata +from datetime import datetime + +import psycopg2 +import config_read +from bs4 import BeautifulSoup +import requests + +import sys + +file_path = 'outputfile.txt' +sys.stdout = open(file_path, "w") + +# Call to read the configuration file and connect to database +cinfo = config_read.get_databaseconfig("../postgresdb.config") +db_connection = psycopg2.connect( + database=cinfo[0], + host=cinfo[1], + user=cinfo[2], + password=cinfo[3] +) +cur = db_connection.cursor() + +offer_shop = "A1" # offer shop +last_updated = datetime.now().date() +is_validated = False + +a1_url = 'https://www.a1.mk/webshop/mk/phones' + +response1 = requests.get(a1_url) +soup1 = BeautifulSoup(response1.content, 'html.parser') + +phones = soup1.find('main', {'class', 'gsm-advisor-grid phones'}).find('div', {'class', 'd-flex'}) \ + .find_all('div', {'class', 'dvc-idtfr by4'}) + +for phone in phones: + brand = phone.get('data-brand').strip() + offer_name = brand+" "+phone.get('data-model').strip() + + # if brand not in offer_name: + # offer_name = brand+" "+offer_name + + offer_shop_code = phone.get('data-productid').strip() + offer_url = phone.find('a', {'class', 'device-link'}).get('href') + image_url = phone.get('data-image') + + response2 = requests.get(offer_url) + soup2 = BeautifulSoup(response2.content, 'html.parser') + + temp_prices = soup2.find('div', {'class': 'ured-tabs-content'}) \ + .find('div', {'class': 'cenovnik-secondary d-flex justify-content-between'}).find_all('div') + + # offer price + price = None + for temp_price in temp_prices: + if 'Цена само за уред' in temp_price.get_text().strip(): + price = int(temp_price.get_text().replace('Цена само за уред', '') + .replace('Одбери', '').replace('денари', '').replace('.', '').strip()) + + colors_section = soup2.find('div', {'id': 'hero'}).find('div', {'class': 'widget'}).find_all('label') + + temp_colors = [] + for color_section in colors_section: + temp_colors.append(color_section.get('data-content')) + + color = ','.join(temp_colors) # colors available for the offer + + phone_description = soup2.find('div', {'class': 'desc section'}).find('p').get_text().strip() + + table_rows = soup2.find('table', {'class': 'table karakteristiki'}).find_all('tr') + + back_camera = None + operating_system = None + cpu = None + rom_memory = None + ram_memory = None + battery = None + front_camera = None + + for row in table_rows: + if 'Камера' in row.get_text().strip(): + back_camera = row.get_text().replace('Камера', '').strip() + + if 'Оперативен систем' in row.get_text().strip(): + operating_system = row.get_text().replace('Оперативен систем', '').strip() + + if 'CPU' in row.get_text().strip(): + cpu = row.get_text().replace('CPU', '').strip() + + if 'Вградена меморија' in row.get_text().strip(): + rom_memory = row.get_text().replace('Вградена меморија', '').strip() + + if 'RAM меморија' in row.get_text().strip(): + ram_memory = row.get_text().replace('RAM меморија', '').strip() + + if 'Батерија' in row.get_text().strip(): + battery = row.get_text().replace('Батерија', '').strip() + + if 'Предна камера' in row.get_text().strip(): + front_camera = row.get_text().replace('Предна камера', '').strip() + + insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name, price, image_url, offer_url,' \ + 'ram_memory, rom_memory, battery, back_camera, front_camera, color, cpu, ' \ + 'operating_system, offer_shop_code, last_updated, is_validated)' \ + ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' + insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, ram_memory, rom_memory, + battery, back_camera, front_camera, color, cpu, operating_system, offer_shop_code, + last_updated, is_validated) + cur.execute(insert_script, insert_value) + db_connection.commit() + +cur.close() +db_connection.close() Index: phonelux_scrappers/scrappers/akcija_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/akcija_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/scrappers/akcija_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,74 @@ +from datetime import datetime + +import psycopg2 +import config_read +from bs4 import BeautifulSoup +import requests +import unicodedata +import sys + +# file_path = '../outputfile.txt' +# sys.stdout = open(file_path, "w") + +# Call to read the configuration file and connect to database +cinfo = config_read.get_databaseconfig("../postgresdb.config") +db_connection = psycopg2.connect( + database=cinfo[0], + host=cinfo[1], + user=cinfo[2], + password=cinfo[3] +) +cur = db_connection.cursor() + +offer_shop = "Akcija" # offer shop +last_updated = datetime.now().date() +is_validated = False + +i = 0 +while i <= 20: + akcija_url = "https://akcija.com.mk/listing/" + str(i) + "?category=mobilnitelefoni" + response1 = requests.get(akcija_url) + response1.encoding = 'utf-8' + soup1 = BeautifulSoup(response1.text, 'html.parser') + + phones = soup1.find_all('div', {'class', 'product-item__body pb-xl-2'}) + + for phone in phones: + offer_name = phone.find('h5', {'class': 'mb-1 product-item__title'}).find('a') \ + .get_text().replace('Паметен телефон', '').strip() + brand = offer_name.split(' ')[0] + + if brand not in offer_name: + offer_name = brand + " " + offer_name + + offer_url = phone.find('h5', {'class': 'mb-1 product-item__title'}).find('a').get('href') + image_url = phone.find('div', {'class', 'mb-2'}).find('img').get('src') + price = int(phone.find('div', {'class', 'flex-center-between mb-1 pt-xl-2'}) \ + .find('ins').get_text().split(' ')[0].strip()) + + response2 = requests.get(offer_url) + response2.encoding = 'utf-8' + soup2 = BeautifulSoup(response2.text, 'html.parser') + + specifications = soup2.find('main', {'id': 'content'}) \ + .find_all('div', {'class', 'container'})[1].find('div', {'class', 'mb-14'}) \ + .find('div', {'class', 'col-md-6 col-lg-4 col-xl-4 mb-md-6 mb-lg-0'}).find_all('p') + + offer_description = '' + for specification in specifications: + if 'Код за нарачка' in str(specification.get_text(separator='\n').replace('NBSP', '').strip()): + continue + offer_description += unicodedata.normalize('NFKD', + str(specification.get_text(separator='\n').strip())) + "\n" + + insert_script = 'INSERT INTO phone_offers (offer_shop, brand,' \ + ' offer_name, price, image_url, offer_url, last_updated, is_validated, offer_description) ' \ + 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s);' + insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, + last_updated, is_validated, offer_description) + cur.execute(insert_script, insert_value) + db_connection.commit() + i += 20 + +cur.close() +db_connection.close() Index: phonelux_scrappers/scrappers/anhoch_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/anhoch_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/scrappers/anhoch_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,90 @@ +import time +from datetime import datetime +import psycopg2 +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC + +import config_read +from bs4 import BeautifulSoup +import requests +import unicodedata +import sys + +file_path = '../outputfile.txt' +sys.stdout = open(file_path, "w") + +# Call to read the configuration file and connect to database +cinfo = config_read.get_databaseconfig("../postgresdb.config") +db_connection = psycopg2.connect( + database=cinfo[0], + host=cinfo[1], + user=cinfo[2], + password=cinfo[3] +) +cur = db_connection.cursor() + + +def scrape_function(driver1, i): + offer_shop = "Anhoch" # offer shop + last_updated = datetime.now().date() + is_validated = False + + anhoch_html = driver1.page_source + soup1 = BeautifulSoup(anhoch_html, 'html.parser') + active_li = soup1.find('div', {'class': 'adjust-elems pagination pagination-centered'}).find('li', {'class': 'active'}) + + li_element = int(active_li.get_text().strip()) + + print('page: '+str(li_element)) + + if int(active_li.get_text().strip()) == i: + phones = soup1.find('section', {'id': 'main'}).find('div', {'class': 'span8'}) \ + .find('div', {'class': 'products'}).find_all('li') + for phone in phones: + offer_url = phone.find('a').get('href') + image_url = phone.find('a').find('img').get('src') + offer_name = phone.find('div', {'class': 'product-name'}).find('a').get_text().strip() + price = int(phone.get('data-price')) + brand = phone.find('div', {'class': 'product-price'}).find_all('div')[2].find('strong').get_text().strip() + + response2 = requests.get(offer_url) + soup2 = BeautifulSoup(response2.content, 'html.parser') + offer_shop_code = soup2.find('div', {'class': 'product-desc'}).get_text().strip().split('\n')[3] + + offer_description = soup2.find('div', {'class': 'description'}) \ + .find('div', {'class': 'tab-content'}).find('pre').get_text().strip() + + print(offer_name) + print(brand) + print() + print() + + # insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name , price, image_url, offer_url,' \ + # 'offer_shop_code, offer_description, last_updated, is_validated)' \ + # ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' + # insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, + # offer_shop_code, offer_description, last_updated, is_validated) + # cur.execute(insert_script, insert_value) + # db_connection.commit() + else: + driver1.implicitly_wait(5) + scrape_function(driver1, i) + + +for i in range(1, 19): + anhoch_url = "https://www.anhoch.com/category/3017/smartfoni-i-mobilni-tel#page/"+str(i) + # print(anhoch_url) + + # selenium is used because of the dynamic content of the page + driver1 = webdriver.Safari(executable_path='/usr/bin/safaridriver') + driver1.get(anhoch_url) + + scrape_function(driver1, i) + # closing the driver so the safari instance can pair with another webdriver session + driver1.close() + + +cur.close() +db_connection.close() Index: phonelux_scrappers/scrappers/handy_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/handy_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/scrappers/handy_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,78 @@ +import unicodedata +from datetime import datetime +import psycopg2 +import config_read +from bs4 import BeautifulSoup +from selenium import webdriver +import requests + +import sys + +file_path = 'outputfile.txt' +sys.stdout = open(file_path, "w") + +# Call to read the configuration file and connect to database +cinfo = config_read.get_databaseconfig("../postgresdb.config") +db_connection = psycopg2.connect( + database=cinfo[0], + host=cinfo[1], + user=cinfo[2], + password=cinfo[3] +) +cur = db_connection.cursor() + +offer_shop = "Handy" # offer shop +last_updated = datetime.now().date() +is_validated = False + +handy_url = 'https://www.handy.mk/telefoni?page=6' + +response1 = requests.get(handy_url) +soup1 = BeautifulSoup(response1.content, 'html.parser') + +phones = soup1.find_all('li', {'data-hook': 'product-list-grid-item'}) + +for phone in phones: + offer_url = phone.find('a').get('href') + offer_name = phone.find('div', {'data-hook': 'not-image-container'})\ + .find('h3', {'data-hook': 'product-item-name'}).get_text().strip() + brand = offer_name.split(' ')[0].capitalize() + price = int(float(phone.find('div', {'data-hook': 'not-image-container'}).find('div', {'data-hook': "product-item-product-details"})\ + .find('span', {'data-hook': 'product-item-price-to-pay'}).get_text().strip().replace('ден', '').replace('.', '').replace(',', '.'))) + + response2 = requests.get(offer_url) + soup2 = BeautifulSoup(response2.text, 'html.parser') + + color_section = soup2.find('section', {'data-hook': 'product-colors-title-section'}) + + color = None + if color_section is not None: + temp_colors = color_section.find('fieldset', {'class': 'ColorPickerbase3563640754__container'})\ + .find_all('input', {'type': 'radio'}) + colors_list = [] + for temp_color in temp_colors: + colors_list.append(temp_color.get('aria-label')) + color = ','.join(colors_list) + + rows = soup2.find('div', {'data-hook': 'info-section-description'}).find_all('li') + + if len(rows) == 0: + rows = soup2.find('div', {'data-hook': 'info-section-description'}).find_all('tr') + + specifications = [] + + for row in rows: + specifications.append(unicodedata.normalize('NFKD', row.get_text().strip())) + + offer_description = '\n'.join(specifications) + + insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name , price, offer_url, ' \ + 'offer_description, last_updated, is_validated)' \ + ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s);' + insert_value = (offer_shop, brand, offer_name, price, offer_url, offer_description, + last_updated, is_validated) + cur.execute(insert_script, insert_value) + db_connection.commit() + +cur.close() +db_connection.close() Index: phonelux_scrappers/scrappers/ledikom_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/ledikom_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/scrappers/ledikom_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,125 @@ +import unicodedata +from datetime import datetime + +import psycopg2 +import config_read +from bs4 import BeautifulSoup +from selenium import webdriver +import requests + +import sys + +file_path = 'outputfile.txt' +sys.stdout = open(file_path, "w") + +# Call to read the configuration file and connect to database +cinfo = config_read.get_databaseconfig("../postgresdb.config") +db_connection = psycopg2.connect( + database=cinfo[0], + host=cinfo[1], + user=cinfo[2], + password=cinfo[3] +) +cur = db_connection.cursor() + +offer_shop = "Ledikom" # offer shop +last_updated = datetime.now().date() +is_validated = False + +ledikom_phone_urls = [ + 'https://ledikom.mk/c/416/uredi/apple/iphone?limit=96', + 'https://ledikom.mk/c/421/uredi/samsung/telefoni?limit=96', + 'https://ledikom.mk/c/424/mobilni-telefoni/xiaomi/telefoni?limit=96', + 'https://ledikom.mk/c/430/uredi/huawei/telefoni?limit=96', + 'https://ledikom.mk/c/441/uredi/oneplus/telefoni?limit=96', + 'https://ledikom.mk/c/413/uredi/google/telefoni?limit=96', + 'https://ledikom.mk/c/411/uredi/honor/telefoni?limit=96', + 'https://ledikom.mk/c/460/uredi/nokia/telefoni?limit=96', + 'https://ledikom.mk/c/461/uredi/asus/telefoni?limit=96', + 'https://ledikom.mk/c/488/proizvodi/oppo/telefoni?limit=96' +] + +for ledikom_url in ledikom_phone_urls: + + # selenium is used because of the dynamic content of the page + driver1 = webdriver.Safari(executable_path='/usr/bin/safaridriver') + driver1.get(ledikom_url) + ledikom_html = driver1.page_source + + # closing the driver so the safari instance can pair with another webdriver session + driver1.close() + + soup1 = BeautifulSoup(ledikom_html, 'html.parser') + + phones = soup1.find('div', {'id': 'content'}) \ + .find('div', {'class': 'container'}).find('div', {'class': 'row'}).find('div', {'class': 'item-display'}) \ + .find_all('div', {'class': 'item-in-grid'}) + + if len(phones) == 0: + continue + + for phone in phones: + offer_url = 'https://ledikom.mk' + phone.find('a').get('href') + image_url = phone.find('a').find('img').get('src') + temp_offer_name = phone.find('div', {'class': 'item-name'}).find('a').get_text().strip() + offer_name = ' '.join(temp_offer_name.split()) + brand = offer_name.split(' ')[0] + price = int(phone.find('span', {'class': 'price'}).get_text().replace('ден.', '').replace('.', '').strip()) + + driver1 = webdriver.Safari(executable_path='/usr/bin/safaridriver') + driver1.get(offer_url) + # getting offer page html + offer_html = driver1.page_source + driver1.close() + + soup2 = BeautifulSoup(offer_html, 'html.parser') + + specifications = soup2.find('div', {'id': 'content'}).find('section', {'class': 'padding-section'}) \ + .find_all('div', {'class': 'container'})[1].find('div', {'class': 'col-md-7'}) \ + .find_all('div', {'class': 'row'}) + + color = None + rom_memory = None + ram_memory = None + + if len(specifications) != 0: + colors_tags = specifications[0].find('div', {'class': 'col-md-12 col-xs-12'}).find_all('a') + temp_colors = [] + for color_tag in colors_tags: + temp_colors.append(color_tag.get_text().strip()) + color = ','.join(temp_colors) + + if len(specifications) >= 2: + temp_rom = specifications[1].find('div', {'class': 'col-md-12 col-xs-12'}).find_all('a') + rom_list = [] + for rom in temp_rom: + rom_list.append(rom.get('title')) + rom_memory = ','.join(rom_list) + + if len(specifications) >= 3: + temp_ram = specifications[2].find('div', {'class': 'col-md-12 col-xs-12'}).find_all('a') + ram_list = [] + for ram in temp_ram: + ram_list.append(ram.get('title')) + + ram_memory = ','.join(ram_list) + + if 'Xiaomi' in brand: + temp = color + color = rom_memory + rom_memory = temp + + temp = ram_memory + ram_memory = color + color = temp + + insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name, price, image_url, offer_url,' \ + 'ram_memory, rom_memory, color, last_updated, is_validated)' \ + ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' + insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, ram_memory, + rom_memory, color, last_updated, is_validated) + cur.execute(insert_script, insert_value) + db_connection.commit() + +cur.close() +db_connection.close() Index: phonelux_scrappers/scrappers/mobelix_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/mobelix_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/scrappers/mobelix_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,133 @@ +import unicodedata +from datetime import datetime + +import psycopg2 +import config_read +from bs4 import BeautifulSoup +import requests + +# import sys +# +# file_path = 'outputfile.txt' +# sys.stdout = open(file_path, "w") + +# Call to read the configuration file and connect to database +cinfo = config_read.get_databaseconfig("../postgresdb.config") +db_connection = psycopg2.connect( + database=cinfo[0], + host=cinfo[1], + user=cinfo[2], + password=cinfo[3] +) +cur = db_connection.cursor() + +offer_shop = "Mobelix" # offer shop +last_updated = datetime.now().date() +is_validated = False + +for i in range(1, 17): + mobelix_url = "https://mobelix.com.mk/mk/mobilni-telefoni?page=" + str(i) + + response1 = requests.get(mobelix_url) + soup1 = BeautifulSoup(response1.content, 'html.parser') + + phones = soup1.find_all('div', {'class': 'p-2 rounded text-dark bg-white d-flex w-100'}) + + for phone in phones: + offer_url = phone.find('a').get('href') + image_url = phone.find_all('div', {'class': 'col-12'})[0].find('img').get('src') + brand = phone.find_all('div', {'class': 'col-12'})[1].find('h5', {'class': 'mb-0'}).get_text().strip() + offer_name = phone.find_all('div', {'class': 'col-12'})[1] \ + .find('h3', {'class': 'h5 font-weight-normal'}).get_text().strip() + + if 'Watch' in offer_name or 'Pad' in offer_name or 'Tab' in offer_name or 'Pods' in offer_name or 'Buds' in offer_name or 'HomePod' in offer_name: + continue + + if brand not in offer_name: + offer_name = brand + " " + offer_name + + temp_prices = phone.find_all('div', {'class': 'col-12'})[1] \ + .find('p', {'class': 'h5 price'}).get_text(separator='/').strip() + + if len(temp_prices.split('/')) > 1: + price = int(float(temp_prices.split('/')[1].replace(',', '').replace('ден', '').strip())) + else: + price = int(float(temp_prices.split('/')[0].replace(',', '').replace('ден', '').strip())) + + response2 = requests.get(offer_url) + soup2 = BeautifulSoup(response2.content, 'html.parser') + + colors_divs = soup2.find('div', {'class': 'color-wrapper mt-2 mb-1'}) \ + .find_all('div', {'class': 'color-box d-inline-block'}) # color div tags + + temp_colors = [] + for div in colors_divs: + temp_colors.append(div.get('title')) + + color = ",".join(temp_colors) # available colors for offer + + tables = soup2.find('div', {'class': 'mobelix-specs table-white bordered-table'}).find_all('table') + + operating_system = None + chipset = None + battery = None + ram_memory = None + rom_memory = None + front_camera = '' + back_camera = '' + cpu = None + + for table in tables: + for cell in table.find_all('td'): + if cell.get('data-spec') is None: + continue + + if cell.get('data-spec') == 'os': + operating_system = unicodedata.normalize('NFKD', cell.get_text().strip()) + + if cell.get('data-spec') == 'chipset': + chipset = unicodedata.normalize('NFKD', cell.get_text().strip()) + + if cell.get('data-spec') == 'cpu': + cpu = unicodedata.normalize('NFKD', cell.get_text().strip()) + + if cell.get('data-spec') == 'internalmemory': + temp_rom = [] + temp_ram = [] + temp_internalmemory = unicodedata.normalize('NFKD', cell.get_text().strip()) + for internalmemory in temp_internalmemory.split(','): + temp_rom.append(internalmemory.strip().split(' ')[0]) + if len(internalmemory.strip().split(' ')) > 1: + temp_ram.append(internalmemory.strip().split(' ')[1]) + rom_memory = ','.join(temp_rom) + ram_memory = ','.join(temp_ram) + + if cell.get('data-spec') == 'cam1modules' or cell.get('data-spec') == 'cam1features' or cell.get( + 'data-spec') == 'cam1video': + back_camera += unicodedata.normalize('NFKD', cell.get_text().strip()) + '\n' + + if cell.get('data-spec') == 'cam2modules' or cell.get('data-spec') == 'cam2features' or cell.get( + 'data-spec') == 'cam2video': + front_camera += unicodedata.normalize('NFKD', cell.get_text().strip()) + '\n' + + if cell.get('data-spec') == 'batdescription1': + battery = unicodedata.normalize('NFKD', cell.get_text().strip()) + + if front_camera == 'No': + front_camera = None + + if back_camera == 'No': + back_camera = None + + insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name, price, image_url, offer_url,' \ + 'ram_memory, rom_memory, battery, back_camera, front_camera, color, cpu, chipset, ' \ + 'operating_system, last_updated, is_validated)' \ + ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' + insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, ram_memory, rom_memory, + battery, back_camera, front_camera, color, cpu, chipset, operating_system, + last_updated, is_validated) + cur.execute(insert_script, insert_value) + db_connection.commit() + +cur.close() +db_connection.close() Index: phonelux_scrappers/scrappers/mobigo_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/mobigo_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/scrappers/mobigo_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,122 @@ +from datetime import datetime + +import psycopg2 +import config_read +from bs4 import BeautifulSoup +import requests + +# import sys +# +# file_path = 'outputfile.txt' +# sys.stdout = open(file_path, "w") + +# Call to read the configuration file and connect to database +cinfo = config_read.get_databaseconfig("../postgresdb.config") +db_connection = psycopg2.connect( + database=cinfo[0], + host=cinfo[1], + user=cinfo[2], + password=cinfo[3] +) +cur = db_connection.cursor() + +offer_shop = "Mobi Go" # offer shop +last_updated = datetime.now().date() +is_validated = False +for i in range(1, 6): + mobigo_url = "https://mobigo.mk/page/" + str(i) + "/" + + response1 = requests.get(mobigo_url) + + soup1 = BeautifulSoup(response1.content, 'html.parser') + + phone_sections = soup1.find_all('ul', {'class': 'recent-posts'}) + phones = phone_sections[len(phone_sections) - 1].find_all('li') + + for phone in phones: + offer_url = phone.find('div', {'class', 'post-thumb'}).find('a').get('href') # offer url + image_url = phone.find('div', {'class', 'post-thumb'}).find('a').find('img').get('src') # image url + offer_name = phone.find('div', {'class', 'post-content'}).find_all('h2')[0].get_text().strip() # offer_name + + if "Watch" in offer_name or "Tab" in offer_name: # if the product is watch or tablet, continue + continue + + price = int(float(phone.find('div', {'class', 'post-content'}).find_all('h2')[1] \ + .get_text().replace('ден.', '').replace('.', '').strip())) # price + + response2 = requests.get(offer_url) + soup2 = BeautifulSoup(response2.content, 'html.parser') + + brand = soup2.find('a', {'rel': 'category tag'}).get_text().strip() # brand + + if brand not in offer_name: + offer_name = brand + " " + offer_name + + specifications = soup2.find('table', {'id': 'singlet'}).find_all('tr') + + ram_memory = "" + rom_memory = "" + battery = "" + back_camera = "" + front_camera = "" + chipset = "" + operating_system = "" + + for specification in specifications: + if specification.find('td') == None: + continue + + # operating system + if specification.find('td').get_text() == "Платформа": + if specification.find('i').get_text() != "/": + operating_system = specification.find('i').get_text().strip() + else: + operating_system = None + + # chipset + if specification.find('td').get_text() == "Chipset": + if specification.find('i').get_text() != "/": + chipset = specification.find('i').get_text().strip() + else: + chipset = None + + # ram and rom memory + if specification.find('td').get_text() == "Меморија": + if specification.find('i').get_text() != "/": + rom_memory = specification.find('i').get_text().replace(',', '').split(' ')[0].strip() + ram_memory = specification.find('i').get_text().replace(',', '').split(' ')[1].strip() + else: + rom_memory = None + ram_memory = None + + # back camera + if specification.find('td').get_text() == "Главна Камера": + if specification.find('i').get_text() != "/": + back_camera = specification.find('i').get_text().strip() + else: + back_camera = None + + # front camera + if specification.find('td').get_text() == "Селфи Камера": + if specification.find('i').get_text() != "/": + front_camera = specification.find('i').get_text().strip() + else: + front_camera = None + + # battery + if specification.find('td').get_text() == "Батерија": + if specification.find('i').get_text() != "/": + battery = specification.find('i').get_text().strip() + else: + battery = None + + insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name, price, image_url, offer_url, ram_memory,' \ + ' rom_memory, battery, back_camera, front_camera, chipset, operating_system, last_updated, is_validated)' \ + ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' + insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, ram_memory, + rom_memory, battery, back_camera, front_camera, chipset, operating_system, last_updated, is_validated) + cur.execute(insert_script, insert_value) + db_connection.commit() + +cur.close() +db_connection.close() Index: phonelux_scrappers/scrappers/mobilezone_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/mobilezone_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/scrappers/mobilezone_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,97 @@ +import unicodedata +from datetime import datetime +import psycopg2 +import config_read +from bs4 import BeautifulSoup +from selenium import webdriver +import requests + +import sys + +file_path = 'outputfile.txt' +sys.stdout = open(file_path, "w") + +# Call to read the configuration file and connect to database +cinfo = config_read.get_databaseconfig("../postgresdb.config") +db_connection = psycopg2.connect( + database=cinfo[0], + host=cinfo[1], + user=cinfo[2], + password=cinfo[3] +) +cur = db_connection.cursor() + +offer_shop = "Mobile Zone" # offer shop +last_updated = datetime.now().date() +is_validated = False + +for i in range(1, 3): + mobilezone_url = 'https://mobilezone.mk/produkt-kategorija/telefoni/novi-telefoni/page/' + str(i) + '/' + + response1 = requests.get(mobilezone_url) + soup1 = BeautifulSoup(response1.content, 'html.parser') + + phones = soup1.find('ul', { + 'class': 'products columns-tablet-2 columns-mobile-2 --skin-proto rey-wcGap-default rey-wcGrid-default ' + '--paginated columns-4'}).find_all('li') + + for phone in phones: + offer_url = phone.find('a', {'class': 'woocommerce-LoopProduct-link woocommerce-loop-product__link'}).get( + 'href') + image_url = phone.find('a', {'class': 'woocommerce-LoopProduct-link woocommerce-loop-product__link'}) \ + .find('img').get('data-lazy-src') + + brand_section = phone.find('div', {'class': 'rey-productInner'}).find('div', {'class': 'rey-brandLink'}) + + if brand_section is not None: + brand = brand_section.find('a').get_text().strip() + else: + brand = None + + offer_name = phone.find('h2', {'class': 'woocommerce-loop-product__title'}).find('a').get_text().strip() + + if brand is not None and brand not in offer_name: + offer_name = brand + ' ' + offer_name + + price = int(unicodedata.normalize('NFKD', phone.find('span', {'class': 'woocommerce-Price-amount amount'}) + .find('bdi').get_text().replace(',', '').replace('ден', '').strip())) + + response2 = requests.get(offer_url) + soup2 = BeautifulSoup(response2.text, 'html.parser') + + specifications = soup2.find('table', {'class': 'woocommerce-product-attributes shop_attributes'}).find_all('tr') + + back_camera = None + front_camera = None + rom_memory = None + battery = None + color = None + + for specification in specifications: + if 'Главна камера' in specification.find('th').get_text(): + back_camera = specification.find('td').get_text().strip() + + if 'Селфи камера' in specification.find('th').get_text(): + front_camera = specification.find('td').get_text().strip() + + if 'Батерија' in specification.find('th').get_text(): + battery = specification.find('td').get_text().strip() + + if 'Меморија' in specification.find('th').get_text(): + rom_memory = specification.find('td').get_text().strip() + + if 'Боја' in specification.find('th').get_text(): + color = specification.find('td').get_text().strip() + + + + insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name , price, offer_url, image_url, ' \ + 'rom_memory, battery, color, front_camera, back_camera, last_updated, is_validated)' \ + ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' + insert_value = (offer_shop, brand, offer_name, price, offer_url, image_url, rom_memory, battery, color, + front_camera, back_camera, last_updated, is_validated) + cur.execute(insert_script, insert_value) + db_connection.commit() + +cur.close() +db_connection.close() Index: phonelux_scrappers/scrappers/mobitech_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/mobitech_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/scrappers/mobitech_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,95 @@ +from datetime import datetime + +import psycopg2 +import config_read +from bs4 import BeautifulSoup +import requests + +# import sys +# file_path = 'outputfile.txt' +# sys.stdout = open(file_path, "w") + +# Call to read the configuration file and connect to database +cinfo = config_read.get_databaseconfig("../postgresdb.config") +db_connection = psycopg2.connect( + database=cinfo[0], + host=cinfo[1], + user=cinfo[2], + password=cinfo[3] +) +cur = db_connection.cursor() + +mobitech_url = "https://mobitech.mk/shop/" + +response1 = requests.get(mobitech_url) + +soup1 = BeautifulSoup(response1.content, 'html.parser') + +phones = soup1.find_all('div', {'class': 'jet-woo-products__inner-box'}) + +offer_shop = "Mobitech" # offer shop +is_validated = False + +for phone in phones: + offer_url = phone.find('h5', {'class': 'jet-woo-product-title'}).find('a').get('href') # url + image_url = phone.find('div', {'class': 'jet-woo-product-thumbnail'}).find('img').get('src') # image + brand = phone.find_next('div', {'class': 'jet-woo-product-categories'}).find('a').get_text().strip() # brand + offer_name = phone.find('h5', {'class': 'jet-woo-product-title'}).find('a').get_text().strip() # offer_name + if brand not in offer_name: + offer_name = brand+" "+offer_name + temp_prices = phone.find('div', {'class': 'jet-woo-product-price'}).find_all('bdi') + price = int(float(temp_prices[len(temp_prices) - 1].get_text().replace("ден", "").replace(",", "").strip())) # price + last_updated = datetime.now().date() # offer last_updated date + + response2 = requests.get(offer_url) + soup2 = BeautifulSoup(response2.content, 'html.parser') + + specifications = soup2.find_all('h2', {'class': 'elementor-heading-title elementor-size-default'}) + + ram_memory = "" + rom_memory = "" + battery = "" + back_camera = "" + operating_system = "" + + for specification in specifications: + # rom memory + if specification.get_text().startswith("Меморија:"): + rom_memory = specification.get_text().split("Меморија:")[1].strip() + if rom_memory == "Нема" or rom_memory == "/": + rom_memory = None + + # ram memory + if specification.get_text().startswith("РАМ Меморија:"): + ram_memory = specification.get_text().split("РАМ Меморија:")[1].strip() + if ram_memory == "Нема" or ram_memory == "/": + ram_memory = None + + # camera + if specification.get_text().startswith("Камера:"): + back_camera = specification.get_text().split("Камера:")[1].strip() + if back_camera == "Нема": + back_camera = None + + # operating system + if specification.get_text().startswith("Оперативен систем:"): + operating_system = specification.get_text().split("Оперативен систем:")[1].split(",")[0].strip() + if operating_system == "Нема": + operating_system = None + + # battery + if specification.get_text().startswith("Батерија:"): + battery = specification.get_text().split("Батерија:")[1].strip() + if battery == "Нема": + battery = None + + insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name, price, image_url, offer_url, ram_memory,' \ + ' rom_memory, battery, back_camera, last_updated, operating_system, is_validated)' \ + ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' + insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, ram_memory, + rom_memory, battery, back_camera, last_updated, operating_system, is_validated) + cur.execute(insert_script, insert_value) + db_connection.commit() + +cur.close() +db_connection.close() Index: phonelux_scrappers/scrappers/neptun_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/neptun_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/scrappers/neptun_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,118 @@ +import unicodedata +from datetime import datetime +import psycopg2 +import config_read +from bs4 import BeautifulSoup +from selenium import webdriver +import requests + +import sys + +file_path = 'outputfile.txt' +sys.stdout = open(file_path, "w") + +# Call to read the configuration file and connect to database +cinfo = config_read.get_databaseconfig("../postgresdb.config") +db_connection = psycopg2.connect( + database=cinfo[0], + host=cinfo[1], + user=cinfo[2], + password=cinfo[3] +) +cur = db_connection.cursor() + +offer_shop = "Neptun" # offer shop +last_updated = datetime.now().date() +is_validated = False + +for i in range(1, 11): + neptun_url = 'https://www.neptun.mk/mobilni_telefoni.nspx?page='+str(i) + + # selenium is used because of the dynamic content of the page + driver1 = webdriver.Safari(executable_path='/usr/bin/safaridriver') + driver1.get(neptun_url) + neptun_html = driver1.page_source + + # closing the driver so the safari instance can pair with another webdriver session + driver1.close() + + # response1 = requests.get(neptun_url) + soup1 = BeautifulSoup(neptun_html, 'html.parser') + + phones = soup1.find('div', {'id': 'mainContainer'}).find('div', + {'class': 'col-lg-9 col-md-9 col-sm-8 col-fix-main'}) \ + .find_all('div', {'class': 'ng-scope product-list-item-grid'}) + + for phone in phones: + offer_url = 'https://www.neptun.mk' + phone.find('a').get('href') + offer_name = phone.find('a').find('h2').get_text().replace('MOB.TEL.', '').strip() + brand = offer_name.split(' ')[0].strip().capitalize() + image_url = 'https://www.neptun.mk' + phone.find('a').find('div', {'class': 'row'}).find('img').get('src') + price = int( + phone.find('div', {'class': 'col-sm-12 static'}).find('div', {'class': 'product-list-item__prices pt35'}) + .find('div', {'class': 'row'}).find('div', {'class': 'newPriceModel'}) \ + .find('span', {'class': 'product-price__amount--value ng-binding'}).get_text().replace('.', '')) + + driver1 = webdriver.Safari(executable_path='/usr/bin/safaridriver') + driver1.get(offer_url) + offer_html = driver1.page_source + # closing the driver so the safari instance can pair with another webdriver session + driver1.close() + + soup2 = BeautifulSoup(offer_html, 'html.parser') + + offer_shop_code = soup2.find('div', {'ng-if': 'showProductDetails'}) \ + .find('div', {'class': 'product-details-first-row'}).find('span', { + 'ng-bind': 'model.CodeNumber'}).get_text().strip() + + specifications_table = \ + soup2.find('div', {'id': 'mainContainer'}).find('div', {'ng-if': 'showProductDetails'}).find_all('ul')[-1] + specifications = specifications_table.get_text(separator='\n').strip().split("\n") + + offer_description = specifications_table.get_text(separator='\n').strip() + + operating_system = None + chipset = None + battery = None + ram_memory = None + rom_memory = None + cpu = None + for specification in specifications: + if 'Батерија:' in specification: + battery = specification.split('Батерија:')[1] + + if 'CPU:' in specification: + cpu = specification.split('CPU:')[1] + + if 'Chipset:' in specification: + chipset = specification.split('Chipset:')[1] + + if 'RAM Меморија:' in specification: + ram_memory = specification.split('RAM Меморија:')[1] + continue + + if 'ROM Меморија:' in specification: + rom_memory = specification.split('ROM Меморија:')[1] + continue + + if 'ROM:' in specification: + rom_memory = specification.split('ROM:')[1] + + if 'RAM:' in specification: + ram_memory = specification.split('RAM:')[1] + + if 'iOS' in specification or 'Android' in specification: + operating_system = specification + + insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name , price, image_url, offer_url,' \ + 'offer_shop_code, operating_system, battery, chipset, cpu, ram_memory, rom_memory, ' \ + 'offer_description, last_updated, is_validated)' \ + ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' + insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, + offer_shop_code, operating_system, battery, chipset, cpu, ram_memory, rom_memory, offer_description, + last_updated, is_validated) + cur.execute(insert_script, insert_value) + db_connection.commit() + +cur.close() +db_connection.close() Index: phonelux_scrappers/scrappers/setec_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/setec_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/scrappers/setec_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,69 @@ +import unicodedata +from datetime import datetime + +import psycopg2 +import config_read +from bs4 import BeautifulSoup +import requests + +import sys + +file_path = 'outputfile.txt' +sys.stdout = open(file_path, "w") + +# Call to read the configuration file and connect to database +cinfo = config_read.get_databaseconfig("../postgresdb.config") +db_connection = psycopg2.connect( + database=cinfo[0], + host=cinfo[1], + user=cinfo[2], + password=cinfo[3] +) +cur = db_connection.cursor() + +offer_shop = "Setec" # offer shop +last_updated = datetime.now().date() +is_validated = False + +for i in range(1, 7): + setec_url = 'https://setec.mk/index.php?route=product/category&path=10066_10067&page='+str(i) + + response1 = requests.get(setec_url) + soup1 = BeautifulSoup(response1.content, 'html.parser') + + phones = soup1.find('div', {'id': 'mfilter-content-container'}) \ + .find_all('div', {'class': 'col-sm-4 col-xs-6'}) + + for phone in phones: + offer_url = phone.find('div', {'class': 'left'}).find('a').get('href') + image_url = phone.find('div', {'class': 'left'}).find('a').find('img').get('src') + offer_name = phone.find('div', {'class': 'right'}).find('div', {'class': 'name'}).find('a').get_text().strip() + brand = offer_name.split(' ')[0] + + if 'Cable' in offer_name or 'AirTag' in offer_name: + continue + + if brand not in offer_name: + offer_name = brand + " " + offer_name + + offer_shop_code = phone.find('div', {'class': 'right'}) \ + .find('div', {'class': 'shifra'}).get_text().replace('Шифра:', '').strip() + price = int(phone.find('div', {'class': 'right'}).find('div', {'class': 'price'}). \ + find('div', {'class': 'category-price-redovna'}).find('span', {'class': 'price-old-new'}) \ + .get_text().replace('Ден.', '').replace(',', '').strip()) + + response2 = requests.get(offer_url) + soup2 = BeautifulSoup(response2.content, 'html.parser') + + offer_description = soup2.find('div', {'id': 'tab-description'}).get_text(separator='\n') + + insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name , price, image_url, offer_url,' \ + 'offer_shop_code, offer_description, last_updated, is_validated)' \ + ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' + insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, + offer_shop_code, offer_description, last_updated, is_validated) + cur.execute(insert_script, insert_value) + db_connection.commit() + +cur.close() +db_connection.close() Index: phonelux_scrappers/scrappers/tehnomarket_scrapper.py =================================================================== --- phonelux_scrappers/scrappers/tehnomarket_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) +++ phonelux_scrappers/scrappers/tehnomarket_scrapper.py (revision b68ae8d3cc34d054d169fb34943b9fcd77bff7c2) @@ -0,0 +1,97 @@ +import unicodedata +from datetime import datetime +import psycopg2 +import config_read +from bs4 import BeautifulSoup +from selenium import webdriver +import requests + +import sys + +file_path = 'outputfile.txt' +sys.stdout = open(file_path, "w") + +# Call to read the configuration file and connect to database +cinfo = config_read.get_databaseconfig("../postgresdb.config") +db_connection = psycopg2.connect( + database=cinfo[0], + host=cinfo[1], + user=cinfo[2], + password=cinfo[3] +) +cur = db_connection.cursor() + + +def scrape_function(driver1, i): + offer_shop = "Tehnomarket" # offer shop + last_updated = datetime.now().date() + is_validated = False + + tehnomarket_html = driver1.page_source + soup1 = BeautifulSoup(tehnomarket_html, 'html.parser') + active_li = soup1.find('div', {'class': 'adjust-elems pagination pagination-centered'}).find('li', + {'class': 'active'}) + + print('page: ' + active_li.get_text()) + + if int(active_li.get_text().strip()) == i: + phones = soup1.find('ul', {'class': 'products products-display-grid thumbnails'}).find_all('li', { + 'class': 'span4 product-fix'}) + + for phone in phones: + offer_url = phone.find('a').get('href') + offer_name = phone.find('div', {'class': 'product-name'}).get_text().strip() + price = int(phone.find('div', {'class': 'product-price clearfix'}).find('strong') \ + .get_text().replace('ден.', '').replace(',', '').strip()) + + response2 = requests.get(offer_url) + soup2 = BeautifulSoup(response2.content, 'html.parser') + + image = soup2.find('div', {'id': 'product_gallery'}).find('img') + + image_url = None + if image is not None: + image_url = image.get('src') + + details = soup2.find('div', {'class': 'product-desc'}).get_text().split('\n') + + brand = details[2].strip().capitalize() + offer_shop_code = details[4].strip() + + specifications = [] + for info in soup2.find_all('span', {'class': 'info'}): + specifications.append(info.get_text()) + + print(brand) + print(offer_name) + print() + print() + + offer_description = '\n'.join(specifications) + + insert_script = 'INSERT INTO phone_offers (offer_shop, brand, offer_name, price, image_url, offer_url,' \ + 'offer_description, offer_shop_code, last_updated, is_validated)' \ + ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' + insert_value = (offer_shop, brand, offer_name, price, image_url, offer_url, offer_description, + offer_shop_code, last_updated, is_validated) + cur.execute(insert_script, insert_value) + db_connection.commit() + else: + driver1.implicitly_wait(30) + scrape_function(driver1, i) + + +for i in range(1, 6): + tehnomarket_url = 'https://tehnomarket.com.mk/category/4109/mobilni-telefoni#page/' + str(i) + # print(anhoch_url) + + # selenium is used because of the dynamic content of the page + driver1 = webdriver.Safari(executable_path='/usr/bin/safaridriver') + driver1.get(tehnomarket_url) + + scrape_function(driver1, i) + # closing the driver so the safari instance can pair with another webdriver session + driver1.close() + +cur.close() +db_connection.close()