Changeset 47f4eaf


Ignore:
Timestamp:
11/20/22 16:34:52 (18 months ago)
Author:
Marko <Marko@…>
Branches:
master
Parents:
ffd50db
Message:

Final features implemented

Files:
87 added
34 edited

Legend:

Unmodified
Added
Removed

  • phonelux-backend/pom.xml

    rffd50db r47f4eaf  
    6969                </dependency>
    7070
     71                <dependency>
     72                        <groupId>org.springframework.boot</groupId>
     73                        <artifactId>spring-boot-starter-oauth2-client</artifactId>
     74                </dependency>
     75
    7176        </dependencies>
    7277

  • phonelux-backend/src/main/java/finki/it/phoneluxbackend/entities/User.java

    rffd50db r47f4eaf  
    7878    }
    7979
     80    public User(String firstName, String lastName, String email, UserRole userRole) {
     81        this.firstName = firstName;
     82        this.lastName = lastName;
     83        this.email = email;
     84        this.userRole = userRole;
     85    }
     86
    8087    @Override
    8188    public Collection<? extends GrantedAuthority> getAuthorities() {

  • phonelux-backend/src/main/java/finki/it/phoneluxbackend/security/configs/WebSecurityConfig.java

    rffd50db r47f4eaf  
    55import finki.it.phoneluxbackend.services.UserService;
    66import lombok.AllArgsConstructor;
     7import org.springframework.beans.factory.annotation.Autowired;
    78import org.springframework.context.annotation.Bean;
    89import org.springframework.context.annotation.Configuration;
    … …  
    1718import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
    1819
    19 import static org.springframework.http.HttpMethod.GET;
    20 
    2120@Configuration
    2221@AllArgsConstructor
    … …  
    2625    private final UserService userService;
    2726    private final BCryptPasswordEncoder bCryptPasswordEncoder;
    28 
    2927    @Override
    3028    protected void configure(HttpSecurity http) throws Exception {
    … …  
    4745                .antMatchers("/admin/**")
    4846                .hasAnyAuthority("ADMIN","SUPERADMIN")
     47                .and()
     48                .authorizeRequests()
     49                .antMatchers("/offerreport/**")
     50                .hasAnyAuthority("USER", "ADMIN", "SUPERADMIN")
     51                .and()
     52                .authorizeRequests()
     53                .antMatchers("/scrapperinfo/**")
     54                .hasAnyAuthority("SUPERADMIN")
     55                .and()
     56                .authorizeRequests()
    4957                .anyRequest().permitAll();
    50 
    5158
    5259
    … …  
    5562
    5663    }
     64
    5765
    5866    @Override

  • phonelux-backend/src/main/java/finki/it/phoneluxbackend/services/RegistrationService.java

    rffd50db r47f4eaf  
    11package finki.it.phoneluxbackend.services;
    22
    3 import com.fasterxml.jackson.core.JsonParser;
    4 import com.fasterxml.jackson.core.io.JsonStringEncoder;
    5 import com.fasterxml.jackson.databind.ObjectMapper;
    6 import com.fasterxml.jackson.databind.util.JSONPObject;
    73import finki.it.phoneluxbackend.data.RegistrationRequest;
    84import finki.it.phoneluxbackend.data.UserRole;
    … …  
    128import finki.it.phoneluxbackend.security.email.EmailValidator;
    139import lombok.AllArgsConstructor;
    14 import org.apache.coyote.Response;
    15 import org.apache.tomcat.util.json.JSONParser;
    1610import org.springframework.http.HttpStatus;
    1711import org.springframework.http.ResponseEntity;
    … …  
    2014
    2115import java.time.LocalDateTime;
    22 import java.util.HashMap;
    2316
    2417@Service

  • phonelux-backend/src/main/java/finki/it/phoneluxbackend/services/UserService.java

    rffd50db r47f4eaf  
    4141    }
    4242
     43
     44
    4345    public ResponseEntity<Object> signUpUser(User user)
    4446    {

  • phonelux-frontend/package-lock.json

    rffd50db r47f4eaf  
    2121        "@testing-library/user-event": "^13.5.0",
    2222        "@tippyjs/react": "^4.2.6",
     23        "antd": "^4.23.6",
    2324        "axios": "^0.27.2",
    2425        "jwt-decode": "^3.1.2",
    … …  
    4748      "engines": {
    4849        "node": ">=6.0.0"
     50      }
     51    },
     52    "node_modules/@ant-design/colors": {
     53      "version": "6.0.0",
     54      "resolved": "https://registry.npmjs.org/@ant-design/colors/-/colors-6.0.0.tgz",
     55      "integrity": "sha512-qAZRvPzfdWHtfameEGP2Qvuf838NhergR35o+EuVyB5XvSA98xod5r4utvi4TJ3ywmevm290g9nsCG5MryrdWQ==",
     56      "dependencies": {
     57        "@ctrl/tinycolor": "^3.4.0"
     58      }
     59    },
     60    "node_modules/@ant-design/icons": {
     61      "version": "4.7.0",
     62      "resolved": "https://registry.npmjs.org/@ant-design/icons/-/icons-4.7.0.tgz",
     63      "integrity": "sha512-aoB4Z7JA431rt6d4u+8xcNPPCrdufSRMUOpxa1ab6mz1JCQZOEVolj2WVs/tDFmN62zzK30mNelEsprLYsSF3g==",
     64      "dependencies": {
     65        "@ant-design/colors": "^6.0.0",
     66        "@ant-design/icons-svg": "^4.2.1",
     67        "@babel/runtime": "^7.11.2",
     68        "classnames": "^2.2.6",
     69        "rc-util": "^5.9.4"
     70      },
     71      "engines": {
     72        "node": ">=8"
     73      },
     74      "peerDependencies": {
     75        "react": ">=16.0.0",
     76        "react-dom": ">=16.0.0"
     77      }
     78    },
     79    "node_modules/@ant-design/icons-svg": {
     80      "version": "4.2.1",
     81      "resolved": "https://registry.npmjs.org/@ant-design/icons-svg/-/icons-svg-4.2.1.tgz",
     82      "integrity": "sha512-EB0iwlKDGpG93hW8f85CTJTs4SvMX7tt5ceupvhALp1IF44SeUFOMhKUOYqpsoYWQKAOuTRDMqn75rEaKDp0Xw=="
     83    },
     84    "node_modules/@ant-design/react-slick": {
     85      "version": "0.29.2",
     86      "resolved": "https://registry.npmjs.org/@ant-design/react-slick/-/react-slick-0.29.2.tgz",
     87      "integrity": "sha512-kgjtKmkGHa19FW21lHnAfyyH9AAoh35pBdcJ53rHmQ3O+cfFHGHnUbj/HFrRNJ5vIts09FKJVAD8RpaC+RaWfA==",
     88      "dependencies": {
     89        "@babel/runtime": "^7.10.4",
     90        "classnames": "^2.2.5",
     91        "json2mq": "^0.2.0",
     92        "lodash": "^4.17.21",
     93        "resize-observer-polyfill": "^1.5.1"
     94      },
     95      "peerDependencies": {
     96        "react": ">=16.9.0"
    4997      }
    5098    },
    … …  
    21692217      }
    21702218    },
     2219    "node_modules/@ctrl/tinycolor": {
     2220      "version": "3.4.1",
     2221      "resolved": "https://registry.npmjs.org/@ctrl/tinycolor/-/tinycolor-3.4.1.tgz",
     2222      "integrity": "sha512-ej5oVy6lykXsvieQtqZxCOaLT+xD4+QNarq78cIYISHmZXshCvROLudpQN3lfL8G0NL7plMSSK+zlyvCaIJ4Iw==",
     2223      "engines": {
     2224        "node": ">=10"
     2225      }
     2226    },
    21712227    "node_modules/@emotion/babel-plugin": {
    21722228      "version": "11.10.2",
    … …  
    52535309      }
    52545310    },
     5311    "node_modules/antd": {
     5312      "version": "4.23.6",
     5313      "resolved": "https://registry.npmjs.org/antd/-/antd-4.23.6.tgz",
     5314      "integrity": "sha512-AYH57cWBDe1ChtbnvG8i9dpKG4WnjE3AG0zIKpXByFNnxsr4saV6/19ihE8/ImSGpohN4E2zTXmo7R5/MyVRKQ==",
     5315      "dependencies": {
     5316        "@ant-design/colors": "^6.0.0",
     5317        "@ant-design/icons": "^4.7.0",
     5318        "@ant-design/react-slick": "~0.29.1",
     5319        "@babel/runtime": "^7.18.3",
     5320        "@ctrl/tinycolor": "^3.4.0",
     5321        "classnames": "^2.2.6",
     5322        "copy-to-clipboard": "^3.2.0",
     5323        "lodash": "^4.17.21",
     5324        "memoize-one": "^6.0.0",
     5325        "moment": "^2.29.2",
     5326        "rc-cascader": "~3.7.0",
     5327        "rc-checkbox": "~2.3.0",
     5328        "rc-collapse": "~3.3.0",
     5329        "rc-dialog": "~8.9.0",
     5330        "rc-drawer": "~5.1.0",
     5331        "rc-dropdown": "~4.0.0",
     5332        "rc-field-form": "~1.27.0",
     5333        "rc-image": "~5.7.0",
     5334        "rc-input": "~0.1.2",
     5335        "rc-input-number": "~7.3.9",
     5336        "rc-mentions": "~1.10.0",
     5337        "rc-menu": "~9.6.3",
     5338        "rc-motion": "^2.6.1",
     5339        "rc-notification": "~4.6.0",
     5340        "rc-pagination": "~3.1.17",
     5341        "rc-picker": "~2.6.11",
     5342        "rc-progress": "~3.3.2",
     5343        "rc-rate": "~2.9.0",
     5344        "rc-resize-observer": "^1.2.0",
     5345        "rc-segmented": "~2.1.0",
     5346        "rc-select": "~14.1.13",
     5347        "rc-slider": "~10.0.0",
     5348        "rc-steps": "~4.1.0",
     5349        "rc-switch": "~3.2.0",
     5350        "rc-table": "~7.26.0",
     5351        "rc-tabs": "~12.2.0",
     5352        "rc-textarea": "~0.4.5",
     5353        "rc-tooltip": "~5.2.0",
     5354        "rc-tree": "~5.7.0",
     5355        "rc-tree-select": "~5.5.0",
     5356        "rc-trigger": "^5.2.10",
     5357        "rc-upload": "~4.3.0",
     5358        "rc-util": "^5.22.5",
     5359        "scroll-into-view-if-needed": "^2.2.25"
     5360      },
     5361      "funding": {
     5362        "type": "opencollective",
     5363        "url": "https://opencollective.com/ant-design"
     5364      },
     5365      "peerDependencies": {
     5366        "react": ">=16.9.0",
     5367        "react-dom": ">=16.9.0"
     5368      }
     5369    },
    52555370    "node_modules/anymatch": {
    52565371      "version": "3.1.2",
    … …  
    53135428      }
    53145429    },
     5430    "node_modules/array-tree-filter": {
     5431      "version": "2.1.0",
     5432      "resolved": "https://registry.npmjs.org/array-tree-filter/-/array-tree-filter-2.1.0.tgz",
     5433      "integrity": "sha512-4ROwICNlNw/Hqa9v+rk5h22KjmzB1JGTMVKP2AKJBOCgb0yL0ASf0+YvCcLNNwquOHNX48jkeZIJ3a+oOQqKcw=="
     5434    },
    53155435    "node_modules/array-union": {
    53165436      "version": "2.1.0",
    … …  
    53875507      "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz",
    53885508      "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ=="
     5509    },
     5510    "node_modules/async-validator": {
     5511      "version": "4.2.5",
     5512      "resolved": "https://registry.npmjs.org/async-validator/-/async-validator-4.2.5.tgz",
     5513      "integrity": "sha512-7HhHjtERjqlNbZtqNqy2rckN/SpOOlmDliet+lP7k+eKZEjPk3DgyeU9lIXLdeLz0uBbbVp+9Qdow9wJWgwwfg=="
    53895514    },
    53905515    "node_modules/asynckit": {
    … …  
    61196244      "integrity": "sha512-cOU9usZw8/dXIXKtwa8pM0OTJQuJkxMN6w30csNRUerHfeQ5R6U3kkU/FtJeIf3M202OHfY2U8ccInBG7/xogA=="
    61206245    },
     6246    "node_modules/classnames": {
     6247      "version": "2.3.2",
     6248      "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz",
     6249      "integrity": "sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw=="
     6250    },
    61216251    "node_modules/clean-css": {
    61226252      "version": "5.3.1",
    … …  
    62836413      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
    62846414      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
     6415    },
     6416    "node_modules/compute-scroll-into-view": {
     6417      "version": "1.0.17",
     6418      "resolved": "https://registry.npmjs.org/compute-scroll-into-view/-/compute-scroll-into-view-1.0.17.tgz",
     6419      "integrity": "sha512-j4dx+Fb0URmzbwwMUrhqWM2BEWHdFGx+qZ9qqASHRPqvTYdqvWnHg0H1hIbcyLnvgnoNAVMlwkepyqM3DaIFUg=="
    62856420    },
    62866421    "node_modules/concat-map": {
    … …  
    63606495      "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
    63616496      "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ=="
     6497    },
     6498    "node_modules/copy-to-clipboard": {
     6499      "version": "3.3.2",
     6500      "resolved": "https://registry.npmjs.org/copy-to-clipboard/-/copy-to-clipboard-3.3.2.tgz",
     6501      "integrity": "sha512-Vme1Z6RUDzrb6xAI7EZlVZ5uvOk2F//GaxKUxajDqm9LhOVM1inxNAD2vy+UZDYsd0uyA9s7b3/FVZPSxqrCfg==",
     6502      "dependencies": {
     6503        "toggle-selection": "^1.0.6"
     6504      }
    63626505    },
    63636506    "node_modules/core-js": {
    … …  
    68506993      }
    68516994    },
     6995    "node_modules/date-fns": {
     6996      "version": "2.29.3",
     6997      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.29.3.tgz",
     6998      "integrity": "sha512-dDCnyH2WnnKusqvZZ6+jA1O51Ibt8ZMRNkDZdyAyK4YfbDwa/cEmuztzG5pk6hqlp9aSBPYcjOlktquahGwGeA==",
     6999      "engines": {
     7000        "node": ">=0.11"
     7001      },
     7002      "funding": {
     7003        "type": "opencollective",
     7004        "url": "https://opencollective.com/date-fns"
     7005      }
     7006    },
     7007    "node_modules/dayjs": {
     7008      "version": "1.11.6",
     7009      "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.6.tgz",
     7010      "integrity": "sha512-zZbY5giJAinCG+7AGaw0wIhNZ6J8AhWuSXKvuc1KAyMiRsvGQWqh4L+MomvhdAYjN+lqvVCMq1I41e3YHvXkyQ=="
     7011    },
    68527012    "node_modules/debug": {
    68537013      "version": "4.3.4",
    … …  
    70717231      "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.14.tgz",
    70727232      "integrity": "sha512-NMt+m9zFMPZe0JcY9gN224Qvk6qLIdqex29clBvc/y75ZBX9YA9wNK3frsYvu2DI1xcCIwxwnX+TlsJ2DSOADg=="
     7233    },
     7234    "node_modules/dom-align": {
     7235      "version": "1.12.3",
     7236      "resolved": "https://registry.npmjs.org/dom-align/-/dom-align-1.12.3.tgz",
     7237      "integrity": "sha512-Gj9hZN3a07cbR6zviMUBOMPdWxYhbMI+x+WS0NAIu2zFZmbK8ys9R79g+iG9qLnlCwpFoaB+fKy8Pdv470GsPA=="
    70737238    },
    70747239    "node_modules/dom-converter": {
    … …  
    1189112056      "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw=="
    1189212057    },
     12058    "node_modules/json2mq": {
     12059      "version": "0.2.0",
     12060      "resolved": "https://registry.npmjs.org/json2mq/-/json2mq-0.2.0.tgz",
     12061      "integrity": "sha512-SzoRg7ux5DWTII9J2qkrZrqV1gt+rTaoufMxEzXbS26Uid0NwaJd123HcoB80TgubEppxxIGdNxCx50fEoEWQA==",
     12062      "dependencies": {
     12063        "string-convert": "^0.2.0"
     12064      }
     12065    },
    1189312066    "node_modules/json5": {
    1189412067      "version": "2.2.1",
    … …  
    1217312346      }
    1217412347    },
     12348    "node_modules/memoize-one": {
     12349      "version": "6.0.0",
     12350      "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-6.0.0.tgz",
     12351      "integrity": "sha512-rkpe71W0N0c0Xz6QD0eJETuWAJGnJ9afsl1srmwPrI+yBCkge5EycXXbYRyvL29zZVUWQCY7InPRCv3GDXuZNw=="
     12352    },
    1217512353    "node_modules/merge-descriptors": {
    1217612354      "version": "1.0.1",
    … …  
    1235412532      "bin": {
    1235512533        "mkdirp": "bin/cmd.js"
     12534      }
     12535    },
     12536    "node_modules/moment": {
     12537      "version": "2.29.4",
     12538      "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz",
     12539      "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==",
     12540      "engines": {
     12541        "node": "*"
    1235612542      }
    1235712543    },
    … …  
    1440014586      }
    1440114587    },
     14588    "node_modules/rc-align": {
     14589      "version": "4.0.12",
     14590      "resolved": "https://registry.npmjs.org/rc-align/-/rc-align-4.0.12.tgz",
     14591      "integrity": "sha512-3DuwSJp8iC/dgHzwreOQl52soj40LchlfUHtgACOUtwGuoFIOVh6n/sCpfqCU8kO5+iz6qR0YKvjgB8iPdE3aQ==",
     14592      "dependencies": {
     14593        "@babel/runtime": "^7.10.1",
     14594        "classnames": "2.x",
     14595        "dom-align": "^1.7.0",
     14596        "lodash": "^4.17.21",
     14597        "rc-util": "^5.3.0",
     14598        "resize-observer-polyfill": "^1.5.1"
     14599      },
     14600      "peerDependencies": {
     14601        "react": ">=16.9.0",
     14602        "react-dom": ">=16.9.0"
     14603      }
     14604    },
     14605    "node_modules/rc-cascader": {
     14606      "version": "3.7.0",
     14607      "resolved": "https://registry.npmjs.org/rc-cascader/-/rc-cascader-3.7.0.tgz",
     14608      "integrity": "sha512-SFtGpwmYN7RaWEAGTS4Rkc62ZV/qmQGg/tajr/7mfIkleuu8ro9Hlk6J+aA0x1YS4zlaZBtTcSaXM01QMiEV/A==",
     14609      "dependencies": {
     14610        "@babel/runtime": "^7.12.5",
     14611        "array-tree-filter": "^2.1.0",
     14612        "classnames": "^2.3.1",
     14613        "rc-select": "~14.1.0",
     14614        "rc-tree": "~5.7.0",
     14615        "rc-util": "^5.6.1"
     14616      },
     14617      "peerDependencies": {
     14618        "react": ">=16.9.0",
     14619        "react-dom": ">=16.9.0"
     14620      }
     14621    },
     14622    "node_modules/rc-checkbox": {
     14623      "version": "2.3.2",
     14624      "resolved": "https://registry.npmjs.org/rc-checkbox/-/rc-checkbox-2.3.2.tgz",
     14625      "integrity": "sha512-afVi1FYiGv1U0JlpNH/UaEXdh6WUJjcWokj/nUN2TgG80bfG+MDdbfHKlLcNNba94mbjy2/SXJ1HDgrOkXGAjg==",
     14626      "dependencies": {
     14627        "@babel/runtime": "^7.10.1",
     14628        "classnames": "^2.2.1"
     14629      },
     14630      "peerDependencies": {
     14631        "react": ">=16.9.0",
     14632        "react-dom": ">=16.9.0"
     14633      }
     14634    },
     14635    "node_modules/rc-collapse": {
     14636      "version": "3.3.1",
     14637      "resolved": "https://registry.npmjs.org/rc-collapse/-/rc-collapse-3.3.1.tgz",
     14638      "integrity": "sha512-cOJfcSe3R8vocrF8T+PgaHDrgeA1tX+lwfhwSj60NX9QVRidsILIbRNDLD6nAzmcvVC5PWiIRiR4S1OobxdhCg==",
     14639      "dependencies": {
     14640        "@babel/runtime": "^7.10.1",
     14641        "classnames": "2.x",
     14642        "rc-motion": "^2.3.4",
     14643        "rc-util": "^5.2.1",
     14644        "shallowequal": "^1.1.0"
     14645      },
     14646      "peerDependencies": {
     14647        "react": ">=16.9.0",
     14648        "react-dom": ">=16.9.0"
     14649      }
     14650    },
     14651    "node_modules/rc-dialog": {
     14652      "version": "8.9.0",
     14653      "resolved": "https://registry.npmjs.org/rc-dialog/-/rc-dialog-8.9.0.tgz",
     14654      "integrity": "sha512-Cp0tbJnrvPchJfnwIvOMWmJ4yjX3HWFatO6oBFD1jx8QkgsQCR0p8nUWAKdd3seLJhEC39/v56kZaEjwp9muoQ==",
     14655      "dependencies": {
     14656        "@babel/runtime": "^7.10.1",
     14657        "classnames": "^2.2.6",
     14658        "rc-motion": "^2.3.0",
     14659        "rc-util": "^5.21.0"
     14660      },
     14661      "peerDependencies": {
     14662        "react": ">=16.9.0",
     14663        "react-dom": ">=16.9.0"
     14664      }
     14665    },
     14666    "node_modules/rc-drawer": {
     14667      "version": "5.1.0",
     14668      "resolved": "https://registry.npmjs.org/rc-drawer/-/rc-drawer-5.1.0.tgz",
     14669      "integrity": "sha512-pU3Tsn99pxGdYowXehzZbdDVE+4lDXSGb7p8vA9mSmr569oc2Izh4Zw5vLKSe/Xxn2p5MSNbLVqD4tz+pK6SOw==",
     14670      "dependencies": {
     14671        "@babel/runtime": "^7.10.1",
     14672        "classnames": "^2.2.6",
     14673        "rc-motion": "^2.6.1",
     14674        "rc-util": "^5.21.2"
     14675      },
     14676      "peerDependencies": {
     14677        "react": ">=16.9.0",
     14678        "react-dom": ">=16.9.0"
     14679      }
     14680    },
     14681    "node_modules/rc-dropdown": {
     14682      "version": "4.0.1",
     14683      "resolved": "https://registry.npmjs.org/rc-dropdown/-/rc-dropdown-4.0.1.tgz",
     14684      "integrity": "sha512-OdpXuOcme1rm45cR0Jzgfl1otzmU4vuBVb+etXM8vcaULGokAKVpKlw8p6xzspG7jGd/XxShvq+N3VNEfk/l5g==",
     14685      "dependencies": {
     14686        "@babel/runtime": "^7.18.3",
     14687        "classnames": "^2.2.6",
     14688        "rc-trigger": "^5.3.1",
     14689        "rc-util": "^5.17.0"
     14690      },
     14691      "peerDependencies": {
     14692        "react": ">=16.11.0",
     14693        "react-dom": ">=16.11.0"
     14694      }
     14695    },
     14696    "node_modules/rc-field-form": {
     14697      "version": "1.27.2",
     14698      "resolved": "https://registry.npmjs.org/rc-field-form/-/rc-field-form-1.27.2.tgz",
     14699      "integrity": "sha512-NaTjkSB8JsHRgm52wkDorsDzFf2HH6GmCQ2AqkwO8zo+zIqybw8K1lkzDBMDJI8nw1pFuD46U5QsYZv4blYvdw==",
     14700      "dependencies": {
     14701        "@babel/runtime": "^7.18.0",
     14702        "async-validator": "^4.1.0",
     14703        "rc-util": "^5.8.0"
     14704      },
     14705      "engines": {
     14706        "node": ">=8.x"
     14707      },
     14708      "peerDependencies": {
     14709        "react": ">=16.9.0",
     14710        "react-dom": ">=16.9.0"
     14711      }
     14712    },
     14713    "node_modules/rc-image": {
     14714      "version": "5.7.1",
     14715      "resolved": "https://registry.npmjs.org/rc-image/-/rc-image-5.7.1.tgz",
     14716      "integrity": "sha512-QyMfdhoUfb5W14plqXSisaYwpdstcLYnB0MjX5ccIK2rydQM9sDPuekQWu500DDGR2dBaIF5vx9XbWkNFK17Fg==",
     14717      "dependencies": {
     14718        "@babel/runtime": "^7.11.2",
     14719        "classnames": "^2.2.6",
     14720        "rc-dialog": "~8.9.0",
     14721        "rc-util": "^5.0.6"
     14722      },
     14723      "peerDependencies": {
     14724        "react": ">=16.9.0",
     14725        "react-dom": ">=16.9.0"
     14726      }
     14727    },
     14728    "node_modules/rc-input": {
     14729      "version": "0.1.2",
     14730      "resolved": "https://registry.npmjs.org/rc-input/-/rc-input-0.1.2.tgz",
     14731      "integrity": "sha512-ZPmwcFspgfYpUfbSx3KnLk9gImBcLOrlQCr4oTJ4jBoIXgJLTfm26yelzRgBJewhkvD8uJbgX0sQ/yOzuOHnJg==",
     14732      "dependencies": {
     14733        "@babel/runtime": "^7.11.1",
     14734        "classnames": "^2.2.1",
     14735        "rc-util": "^5.18.1"
     14736      },
     14737      "peerDependencies": {
     14738        "react": ">=16.0.0",
     14739        "react-dom": ">=16.0.0"
     14740      }
     14741    },
     14742    "node_modules/rc-input-number": {
     14743      "version": "7.3.9",
     14744      "resolved": "https://registry.npmjs.org/rc-input-number/-/rc-input-number-7.3.9.tgz",
     14745      "integrity": "sha512-u0+miS+SATdb6DtssYei2JJ1WuZME+nXaG6XGtR8maNyW5uGDytfDu60OTWLQEb0Anv/AcCzehldV8CKmKyQfA==",
     14746      "dependencies": {
     14747        "@babel/runtime": "^7.10.1",
     14748        "classnames": "^2.2.5",
     14749        "rc-util": "^5.23.0"
     14750      },
     14751      "peerDependencies": {
     14752        "react": ">=16.9.0",
     14753        "react-dom": ">=16.9.0"
     14754      }
     14755    },
     14756    "node_modules/rc-mentions": {
     14757      "version": "1.10.0",
     14758      "resolved": "https://registry.npmjs.org/rc-mentions/-/rc-mentions-1.10.0.tgz",
     14759      "integrity": "sha512-oMlYWnwXSxP2NQVlgxOTzuG/u9BUc3ySY78K3/t7MNhJWpZzXTao+/Bic6tyZLuNCO89//hVQJBdaR2rnFQl6Q==",
     14760      "dependencies": {
     14761        "@babel/runtime": "^7.10.1",
     14762        "classnames": "^2.2.6",
     14763        "rc-menu": "~9.6.0",
     14764        "rc-textarea": "^0.4.0",
     14765        "rc-trigger": "^5.0.4",
     14766        "rc-util": "^5.22.5"
     14767      },
     14768      "peerDependencies": {
     14769        "react": ">=16.9.0",
     14770        "react-dom": ">=16.9.0"
     14771      }
     14772    },
     14773    "node_modules/rc-menu": {
     14774      "version": "9.6.4",
     14775      "resolved": "https://registry.npmjs.org/rc-menu/-/rc-menu-9.6.4.tgz",
     14776      "integrity": "sha512-6DiNAjxjVIPLZXHffXxxcyE15d4isRL7iQ1ru4MqYDH2Cqc5bW96wZOdMydFtGLyDdnmEQ9jVvdCE9yliGvzkw==",
     14777      "dependencies": {
     14778        "@babel/runtime": "^7.10.1",
     14779        "classnames": "2.x",
     14780        "rc-motion": "^2.4.3",
     14781        "rc-overflow": "^1.2.0",
     14782        "rc-trigger": "^5.1.2",
     14783        "rc-util": "^5.12.0",
     14784        "shallowequal": "^1.1.0"
     14785      },
     14786      "peerDependencies": {
     14787        "react": ">=16.9.0",
     14788        "react-dom": ">=16.9.0"
     14789      }
     14790    },
     14791    "node_modules/rc-motion": {
     14792      "version": "2.6.2",
     14793      "resolved": "https://registry.npmjs.org/rc-motion/-/rc-motion-2.6.2.tgz",
     14794      "integrity": "sha512-4w1FaX3dtV749P8GwfS4fYnFG4Rb9pxvCYPc/b2fw1cmlHJWNNgOFIz7ysiD+eOrzJSvnLJWlNQQncpNMXwwpg==",
     14795      "dependencies": {
     14796        "@babel/runtime": "^7.11.1",
     14797        "classnames": "^2.2.1",
     14798        "rc-util": "^5.21.0"
     14799      },
     14800      "peerDependencies": {
     14801        "react": ">=16.9.0",
     14802        "react-dom": ">=16.9.0"
     14803      }
     14804    },
     14805    "node_modules/rc-notification": {
     14806      "version": "4.6.0",
     14807      "resolved": "https://registry.npmjs.org/rc-notification/-/rc-notification-4.6.0.tgz",
     14808      "integrity": "sha512-xF3MKgIoynzjQAO4lqsoraiFo3UXNYlBfpHs0VWvwF+4pimen9/H1DYLN2mfRWhHovW6gRpla73m2nmyIqAMZQ==",
     14809      "dependencies": {
     14810        "@babel/runtime": "^7.10.1",
     14811        "classnames": "2.x",
     14812        "rc-motion": "^2.2.0",
     14813        "rc-util": "^5.20.1"
     14814      },
     14815      "engines": {
     14816        "node": ">=8.x"
     14817      },
     14818      "peerDependencies": {
     14819        "react": ">=16.9.0",
     14820        "react-dom": ">=16.9.0"
     14821      }
     14822    },
     14823    "node_modules/rc-overflow": {
     14824      "version": "1.2.8",
     14825      "resolved": "https://registry.npmjs.org/rc-overflow/-/rc-overflow-1.2.8.tgz",
     14826      "integrity": "sha512-QJ0UItckWPQ37ZL1dMEBAdY1dhfTXFL9k6oTTcyydVwoUNMnMqCGqnRNA98axSr/OeDKqR6DVFyi8eA5RQI/uQ==",
     14827      "dependencies": {
     14828        "@babel/runtime": "^7.11.1",
     14829        "classnames": "^2.2.1",
     14830        "rc-resize-observer": "^1.0.0",
     14831        "rc-util": "^5.19.2"
     14832      },
     14833      "peerDependencies": {
     14834        "react": ">=16.9.0",
     14835        "react-dom": ">=16.9.0"
     14836      }
     14837    },
     14838    "node_modules/rc-pagination": {
     14839      "version": "3.1.17",
     14840      "resolved": "https://registry.npmjs.org/rc-pagination/-/rc-pagination-3.1.17.tgz",
     14841      "integrity": "sha512-/BQ5UxcBnW28vFAcP2hfh+Xg15W0QZn8TWYwdCApchMH1H0CxiaUUcULP8uXcFM1TygcdKWdt3JqsL9cTAfdkQ==",
     14842      "dependencies": {
     14843        "@babel/runtime": "^7.10.1",
     14844        "classnames": "^2.2.1"
     14845      },
     14846      "peerDependencies": {
     14847        "react": ">=16.9.0",
     14848        "react-dom": ">=16.9.0"
     14849      }
     14850    },
     14851    "node_modules/rc-picker": {
     14852      "version": "2.6.11",
     14853      "resolved": "https://registry.npmjs.org/rc-picker/-/rc-picker-2.6.11.tgz",
     14854      "integrity": "sha512-INJ7ULu+Kj4UgqbcqE8Q+QpMw55xFf9kkyLBHJFk0ihjJpAV4glialRfqHE7k4KX2BWYPQfpILwhwR14x2EiRQ==",
     14855      "dependencies": {
     14856        "@babel/runtime": "^7.10.1",
     14857        "classnames": "^2.2.1",
     14858        "date-fns": "2.x",
     14859        "dayjs": "1.x",
     14860        "moment": "^2.24.0",
     14861        "rc-trigger": "^5.0.4",
     14862        "rc-util": "^5.4.0",
     14863        "shallowequal": "^1.1.0"
     14864      },
     14865      "engines": {
     14866        "node": ">=8.x"
     14867      },
     14868      "peerDependencies": {
     14869        "react": ">=16.9.0",
     14870        "react-dom": ">=16.9.0"
     14871      }
     14872    },
     14873    "node_modules/rc-progress": {
     14874      "version": "3.3.3",
     14875      "resolved": "https://registry.npmjs.org/rc-progress/-/rc-progress-3.3.3.tgz",
     14876      "integrity": "sha512-MDVNVHzGanYtRy2KKraEaWeZLri2ZHWIRyaE1a9MQ2MuJ09m+Wxj5cfcaoaR6z5iRpHpA59YeUxAlpML8N4PJw==",
     14877      "dependencies": {
     14878        "@babel/runtime": "^7.10.1",
     14879        "classnames": "^2.2.6",
     14880        "rc-util": "^5.16.1"
     14881      },
     14882      "peerDependencies": {
     14883        "react": ">=16.9.0",
     14884        "react-dom": ">=16.9.0"
     14885      }
     14886    },
     14887    "node_modules/rc-rate": {
     14888      "version": "2.9.2",
     14889      "resolved": "https://registry.npmjs.org/rc-rate/-/rc-rate-2.9.2.tgz",
     14890      "integrity": "sha512-SaiZFyN8pe0Fgphv8t3+kidlej+cq/EALkAJAc3A0w0XcPaH2L1aggM8bhe1u6GAGuQNAoFvTLjw4qLPGRKV5g==",
     14891      "dependencies": {
     14892        "@babel/runtime": "^7.10.1",
     14893        "classnames": "^2.2.5",
     14894        "rc-util": "^5.0.1"
     14895      },
     14896      "engines": {
     14897        "node": ">=8.x"
     14898      },
     14899      "peerDependencies": {
     14900        "react": ">=16.9.0",
     14901        "react-dom": ">=16.9.0"
     14902      }
     14903    },
     14904    "node_modules/rc-resize-observer": {
     14905      "version": "1.2.0",
     14906      "resolved": "https://registry.npmjs.org/rc-resize-observer/-/rc-resize-observer-1.2.0.tgz",
     14907      "integrity": "sha512-6W+UzT3PyDM0wVCEHfoW3qTHPTvbdSgiA43buiy8PzmeMnfgnDeb9NjdimMXMl3/TcrvvWl5RRVdp+NqcR47pQ==",
     14908      "dependencies": {
     14909        "@babel/runtime": "^7.10.1",
     14910        "classnames": "^2.2.1",
     14911        "rc-util": "^5.15.0",
     14912        "resize-observer-polyfill": "^1.5.1"
     14913      },
     14914      "peerDependencies": {
     14915        "react": ">=16.9.0",
     14916        "react-dom": ">=16.9.0"
     14917      }
     14918    },
     14919    "node_modules/rc-segmented": {
     14920      "version": "2.1.0",
     14921      "resolved": "https://registry.npmjs.org/rc-segmented/-/rc-segmented-2.1.0.tgz",
     14922      "integrity": "sha512-hUlonro+pYoZcwrH6Vm56B2ftLfQh046hrwif/VwLIw1j3zGt52p5mREBwmeVzXnSwgnagpOpfafspzs1asjGw==",
     14923      "dependencies": {
     14924        "@babel/runtime": "^7.11.1",
     14925        "classnames": "^2.2.1",
     14926        "rc-motion": "^2.4.4",
     14927        "rc-util": "^5.17.0"
     14928      },
     14929      "peerDependencies": {
     14930        "react": ">=16.0.0",
     14931        "react-dom": ">=16.0.0"
     14932      }
     14933    },
     14934    "node_modules/rc-select": {
     14935      "version": "14.1.13",
     14936      "resolved": "https://registry.npmjs.org/rc-select/-/rc-select-14.1.13.tgz",
     14937      "integrity": "sha512-WMEsC3gTwA1dbzWOdVIXDmWyidYNLq68AwvvUlRROw790uGUly0/vmqDozXrIr0QvN/A3CEULx12o+WtLCAefg==",
     14938      "dependencies": {
     14939        "@babel/runtime": "^7.10.1",
     14940        "classnames": "2.x",
     14941        "rc-motion": "^2.0.1",
     14942        "rc-overflow": "^1.0.0",
     14943        "rc-trigger": "^5.0.4",
     14944        "rc-util": "^5.16.1",
     14945        "rc-virtual-list": "^3.2.0"
     14946      },
     14947      "engines": {
     14948        "node": ">=8.x"
     14949      },
     14950      "peerDependencies": {
     14951        "react": "*",
     14952        "react-dom": "*"
     14953      }
     14954    },
     14955    "node_modules/rc-slider": {
     14956      "version": "10.0.1",
     14957      "resolved": "https://registry.npmjs.org/rc-slider/-/rc-slider-10.0.1.tgz",
     14958      "integrity": "sha512-igTKF3zBet7oS/3yNiIlmU8KnZ45npmrmHlUUio8PNbIhzMcsh+oE/r2UD42Y6YD2D/s+kzCQkzQrPD6RY435Q==",
     14959      "dependencies": {
     14960        "@babel/runtime": "^7.10.1",
     14961        "classnames": "^2.2.5",
     14962        "rc-util": "^5.18.1",
     14963        "shallowequal": "^1.1.0"
     14964      },
     14965      "engines": {
     14966        "node": ">=8.x"
     14967      },
     14968      "peerDependencies": {
     14969        "react": ">=16.9.0",
     14970        "react-dom": ">=16.9.0"
     14971      }
     14972    },
     14973    "node_modules/rc-steps": {
     14974      "version": "4.1.4",
     14975      "resolved": "https://registry.npmjs.org/rc-steps/-/rc-steps-4.1.4.tgz",
     14976      "integrity": "sha512-qoCqKZWSpkh/b03ASGx1WhpKnuZcRWmvuW+ZUu4mvMdfvFzVxblTwUM+9aBd0mlEUFmt6GW8FXhMpHkK3Uzp3w==",
     14977      "dependencies": {
     14978        "@babel/runtime": "^7.10.2",
     14979        "classnames": "^2.2.3",
     14980        "rc-util": "^5.0.1"
     14981      },
     14982      "engines": {
     14983        "node": ">=8.x"
     14984      },
     14985      "peerDependencies": {
     14986        "react": ">=16.9.0",
     14987        "react-dom": ">=16.9.0"
     14988      }
     14989    },
     14990    "node_modules/rc-switch": {
     14991      "version": "3.2.2",
     14992      "resolved": "https://registry.npmjs.org/rc-switch/-/rc-switch-3.2.2.tgz",
     14993      "integrity": "sha512-+gUJClsZZzvAHGy1vZfnwySxj+MjLlGRyXKXScrtCTcmiYNPzxDFOxdQ/3pK1Kt/0POvwJ/6ALOR8gwdXGhs+A==",
     14994      "dependencies": {
     14995        "@babel/runtime": "^7.10.1",
     14996        "classnames": "^2.2.1",
     14997        "rc-util": "^5.0.1"
     14998      },
     14999      "peerDependencies": {
     15000        "react": ">=16.9.0",
     15001        "react-dom": ">=16.9.0"
     15002      }
     15003    },
     15004    "node_modules/rc-table": {
     15005      "version": "7.26.0",
     15006      "resolved": "https://registry.npmjs.org/rc-table/-/rc-table-7.26.0.tgz",
     15007      "integrity": "sha512-0cD8e6S+DTGAt5nBZQIPFYEaIukn17sfa5uFL98faHlH/whZzD8ii3dbFL4wmUDEL4BLybhYop+QUfZJ4CPvNQ==",
     15008      "dependencies": {
     15009        "@babel/runtime": "^7.10.1",
     15010        "classnames": "^2.2.5",
     15011        "rc-resize-observer": "^1.1.0",
     15012        "rc-util": "^5.22.5",
     15013        "shallowequal": "^1.1.0"
     15014      },
     15015      "engines": {
     15016        "node": ">=8.x"
     15017      },
     15018      "peerDependencies": {
     15019        "react": ">=16.9.0",
     15020        "react-dom": ">=16.9.0"
     15021      }
     15022    },
     15023    "node_modules/rc-tabs": {
     15024      "version": "12.2.1",
     15025      "resolved": "https://registry.npmjs.org/rc-tabs/-/rc-tabs-12.2.1.tgz",
     15026      "integrity": "sha512-09pVv4kN8VFqp6THceEmxOW8PAShQC08hrroeVYP4Y8YBFaP1PIWdyFL01czcbyz5YZFj9flZ7aljMaAl0jLVg==",
     15027      "dependencies": {
     15028        "@babel/runtime": "^7.11.2",
     15029        "classnames": "2.x",
     15030        "rc-dropdown": "~4.0.0",
     15031        "rc-menu": "~9.6.0",
     15032        "rc-motion": "^2.6.2",
     15033        "rc-resize-observer": "^1.0.0",
     15034        "rc-util": "^5.5.0"
     15035      },
     15036      "engines": {
     15037        "node": ">=8.x"
     15038      },
     15039      "peerDependencies": {
     15040        "react": ">=16.9.0",
     15041        "react-dom": ">=16.9.0"
     15042      }
     15043    },
     15044    "node_modules/rc-textarea": {
     15045      "version": "0.4.5",
     15046      "resolved": "https://registry.npmjs.org/rc-textarea/-/rc-textarea-0.4.5.tgz",
     15047      "integrity": "sha512-WHeJRgUlloNyVgTsItMrIXwMhU6P3NmrUDkxX+JRwEpJjECsKtZNlNcXe9pHNLCaYQ3Z1cVCfsClhgDDgJ2kFQ==",
     15048      "dependencies": {
     15049        "@babel/runtime": "^7.10.1",
     15050        "classnames": "^2.2.1",
     15051        "rc-resize-observer": "^1.0.0",
     15052        "rc-util": "^5.7.0",
     15053        "shallowequal": "^1.1.0"
     15054      },
     15055      "peerDependencies": {
     15056        "react": ">=16.9.0",
     15057        "react-dom": ">=16.9.0"
     15058      }
     15059    },
     15060    "node_modules/rc-tooltip": {
     15061      "version": "5.2.2",
     15062      "resolved": "https://registry.npmjs.org/rc-tooltip/-/rc-tooltip-5.2.2.tgz",
     15063      "integrity": "sha512-jtQzU/18S6EI3lhSGoDYhPqNpWajMtS5VV/ld1LwyfrDByQpYmw/LW6U7oFXXLukjfDHQ7Ju705A82PRNFWYhg==",
     15064      "dependencies": {
     15065        "@babel/runtime": "^7.11.2",
     15066        "classnames": "^2.3.1",
     15067        "rc-trigger": "^5.0.0"
     15068      },
     15069      "peerDependencies": {
     15070        "react": ">=16.9.0",
     15071        "react-dom": ">=16.9.0"
     15072      }
     15073    },
     15074    "node_modules/rc-tree": {
     15075      "version": "5.7.0",
     15076      "resolved": "https://registry.npmjs.org/rc-tree/-/rc-tree-5.7.0.tgz",
     15077      "integrity": "sha512-F+Ewkv/UcutshnVBMISP+lPdHDlcsL+YH/MQDVWbk+QdkfID7vXiwrHMEZn31+2Rbbm21z/HPceGS8PXGMmnQg==",
     15078      "dependencies": {
     15079        "@babel/runtime": "^7.10.1",
     15080        "classnames": "2.x",
     15081        "rc-motion": "^2.0.1",
     15082        "rc-util": "^5.16.1",
     15083        "rc-virtual-list": "^3.4.8"
     15084      },
     15085      "engines": {
     15086        "node": ">=10.x"
     15087      },
     15088      "peerDependencies": {
     15089        "react": "*",
     15090        "react-dom": "*"
     15091      }
     15092    },
     15093    "node_modules/rc-tree-select": {
     15094      "version": "5.5.2",
     15095      "resolved": "https://registry.npmjs.org/rc-tree-select/-/rc-tree-select-5.5.2.tgz",
     15096      "integrity": "sha512-toxkzVhkWQ2wvInOhHwmyEpCCi9osnoRQmN0trKvdvyzLattCw63F2T+V/dS2d/xUkrw6Zr1Y2J0/xP57x/jYQ==",
     15097      "dependencies": {
     15098        "@babel/runtime": "^7.10.1",
     15099        "classnames": "2.x",
     15100        "rc-select": "~14.1.0",
     15101        "rc-tree": "~5.7.0",
     15102        "rc-util": "^5.16.1"
     15103      },
     15104      "peerDependencies": {
     15105        "react": "*",
     15106        "react-dom": "*"
     15107      }
     15108    },
     15109    "node_modules/rc-trigger": {
     15110      "version": "5.3.1",
     15111      "resolved": "https://registry.npmjs.org/rc-trigger/-/rc-trigger-5.3.1.tgz",
     15112      "integrity": "sha512-5gaFbDkYSefZ14j2AdzucXzlWgU2ri5uEjkHvsf1ynRhdJbKxNOnw4PBZ9+FVULNGFiDzzlVF8RJnR9P/xrnKQ==",
     15113      "dependencies": {
     15114        "@babel/runtime": "^7.18.3",
     15115        "classnames": "^2.2.6",
     15116        "rc-align": "^4.0.0",
     15117        "rc-motion": "^2.0.0",
     15118        "rc-util": "^5.19.2"
     15119      },
     15120      "engines": {
     15121        "node": ">=8.x"
     15122      },
     15123      "peerDependencies": {
     15124        "react": ">=16.9.0",
     15125        "react-dom": ">=16.9.0"
     15126      }
     15127    },
     15128    "node_modules/rc-upload": {
     15129      "version": "4.3.4",
     15130      "resolved": "https://registry.npmjs.org/rc-upload/-/rc-upload-4.3.4.tgz",
     15131      "integrity": "sha512-uVbtHFGNjHG/RyAfm9fluXB6pvArAGyAx8z7XzXXyorEgVIWj6mOlriuDm0XowDHYz4ycNK0nE0oP3cbFnzxiQ==",
     15132      "dependencies": {
     15133        "@babel/runtime": "^7.18.3",
     15134        "classnames": "^2.2.5",
     15135        "rc-util": "^5.2.0"
     15136      },
     15137      "peerDependencies": {
     15138        "react": ">=16.9.0",
     15139        "react-dom": ">=16.9.0"
     15140      }
     15141    },
     15142    "node_modules/rc-util": {
     15143      "version": "5.24.4",
     15144      "resolved": "https://registry.npmjs.org/rc-util/-/rc-util-5.24.4.tgz",
     15145      "integrity": "sha512-2a4RQnycV9eV7lVZPEJ7QwJRPlZNc06J7CwcwZo4vIHr3PfUqtYgl1EkUV9ETAc6VRRi8XZOMFhYG63whlIC9Q==",
     15146      "dependencies": {
     15147        "@babel/runtime": "^7.18.3",
     15148        "react-is": "^16.12.0",
     15149        "shallowequal": "^1.1.0"
     15150      },
     15151      "peerDependencies": {
     15152        "react": ">=16.9.0",
     15153        "react-dom": ">=16.9.0"
     15154      }
     15155    },
     15156    "node_modules/rc-util/node_modules/react-is": {
     15157      "version": "16.13.1",
     15158      "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
     15159      "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="
     15160    },
     15161    "node_modules/rc-virtual-list": {
     15162      "version": "3.4.10",
     15163      "resolved": "https://registry.npmjs.org/rc-virtual-list/-/rc-virtual-list-3.4.10.tgz",
     15164      "integrity": "sha512-Jv0cgJxJ+8F/YViW8WGs/jQF2rmT8RUcJ5uDJs5MOFLTYLAvCpM/xU+Zu6EpCun50fmovhXiItQctcfE2UY3Aw==",
     15165      "dependencies": {
     15166        "classnames": "^2.2.6",
     15167        "rc-resize-observer": "^1.0.0",
     15168        "rc-util": "^5.15.0"
     15169      },
     15170      "engines": {
     15171        "node": ">=8.x"
     15172      },
     15173      "peerDependencies": {
     15174        "react": "*",
     15175        "react-dom": "*"
     15176      }
     15177    },
    1440215178    "node_modules/react": {
    1440315179      "version": "18.2.0",
    … …  
    1489515671      "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ=="
    1489615672    },
     15673    "node_modules/resize-observer-polyfill": {
     15674      "version": "1.5.1",
     15675      "resolved": "https://registry.npmjs.org/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz",
     15676      "integrity": "sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg=="
     15677    },
    1489715678    "node_modules/resolve": {
    1489815679      "version": "1.22.1",
    … …  
    1520615987        "type": "opencollective",
    1520715988        "url": "https://opencollective.com/webpack"
     15989      }
     15990    },
     15991    "node_modules/scroll-into-view-if-needed": {
     15992      "version": "2.2.29",
     15993      "resolved": "https://registry.npmjs.org/scroll-into-view-if-needed/-/scroll-into-view-if-needed-2.2.29.tgz",
     15994      "integrity": "sha512-hxpAR6AN+Gh53AdAimHM6C8oTN1ppwVZITihix+WqalywBeFcQ6LdQP5ABNl26nX8GTEL7VT+b8lKpdqq65wXg==",
     15995      "dependencies": {
     15996        "compute-scroll-into-view": "^1.0.17"
    1520815997      }
    1520915998    },
    … …  
    1537616165      "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
    1537716166    },
     16167    "node_modules/shallowequal": {
     16168      "version": "1.1.0",
     16169      "resolved": "https://registry.npmjs.org/shallowequal/-/shallowequal-1.1.0.tgz",
     16170      "integrity": "sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ=="
     16171    },
    1537816172    "node_modules/shebang-command": {
    1537916173      "version": "2.0.0",
    … …  
    1560116395        }
    1560216396      ]
     16397    },
     16398    "node_modules/string-convert": {
     16399      "version": "0.2.1",
     16400      "resolved": "https://registry.npmjs.org/string-convert/-/string-convert-0.2.1.tgz",
     16401      "integrity": "sha512-u/1tdPl4yQnPBjnVrmdLo9gtuLvELKsAoRapekWggdiQNvvvum+jYF329d84NAa660KQw7pB2n36KrIKVoXa3A=="
    1560316402    },
    1560416403    "node_modules/string-length": {
    … …  
    1614616945        "node": ">=8.0"
    1614716946      }
     16947    },
     16948    "node_modules/toggle-selection": {
     16949      "version": "1.0.6",
     16950      "resolved": "https://registry.npmjs.org/toggle-selection/-/toggle-selection-1.0.6.tgz",
     16951      "integrity": "sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ=="
    1614816952    },
    1614916953    "node_modules/toidentifier": {
    … …  
    1744218246        "@jridgewell/gen-mapping": "^0.1.0",
    1744318247        "@jridgewell/trace-mapping": "^0.3.9"
     18248      }
     18249    },
     18250    "@ant-design/colors": {
     18251      "version": "6.0.0",
     18252      "resolved": "https://registry.npmjs.org/@ant-design/colors/-/colors-6.0.0.tgz",
     18253      "integrity": "sha512-qAZRvPzfdWHtfameEGP2Qvuf838NhergR35o+EuVyB5XvSA98xod5r4utvi4TJ3ywmevm290g9nsCG5MryrdWQ==",
     18254      "requires": {
     18255        "@ctrl/tinycolor": "^3.4.0"
     18256      }
     18257    },
     18258    "@ant-design/icons": {
     18259      "version": "4.7.0",
     18260      "resolved": "https://registry.npmjs.org/@ant-design/icons/-/icons-4.7.0.tgz",
     18261      "integrity": "sha512-aoB4Z7JA431rt6d4u+8xcNPPCrdufSRMUOpxa1ab6mz1JCQZOEVolj2WVs/tDFmN62zzK30mNelEsprLYsSF3g==",
     18262      "requires": {
     18263        "@ant-design/colors": "^6.0.0",
     18264        "@ant-design/icons-svg": "^4.2.1",
     18265        "@babel/runtime": "^7.11.2",
     18266        "classnames": "^2.2.6",
     18267        "rc-util": "^5.9.4"
     18268      }
     18269    },
     18270    "@ant-design/icons-svg": {
     18271      "version": "4.2.1",
     18272      "resolved": "https://registry.npmjs.org/@ant-design/icons-svg/-/icons-svg-4.2.1.tgz",
     18273      "integrity": "sha512-EB0iwlKDGpG93hW8f85CTJTs4SvMX7tt5ceupvhALp1IF44SeUFOMhKUOYqpsoYWQKAOuTRDMqn75rEaKDp0Xw=="
     18274    },
     18275    "@ant-design/react-slick": {
     18276      "version": "0.29.2",
     18277      "resolved": "https://registry.npmjs.org/@ant-design/react-slick/-/react-slick-0.29.2.tgz",
     18278      "integrity": "sha512-kgjtKmkGHa19FW21lHnAfyyH9AAoh35pBdcJ53rHmQ3O+cfFHGHnUbj/HFrRNJ5vIts09FKJVAD8RpaC+RaWfA==",
     18279      "requires": {
     18280        "@babel/runtime": "^7.10.4",
     18281        "classnames": "^2.2.5",
     18282        "json2mq": "^0.2.0",
     18283        "lodash": "^4.17.21",
     18284        "resize-observer-polyfill": "^1.5.1"
    1744418285      }
    1744518286    },
    … …  
    1881319654      "requires": {}
    1881419655    },
     19656    "@ctrl/tinycolor": {
     19657      "version": "3.4.1",
     19658      "resolved": "https://registry.npmjs.org/@ctrl/tinycolor/-/tinycolor-3.4.1.tgz",
     19659      "integrity": "sha512-ej5oVy6lykXsvieQtqZxCOaLT+xD4+QNarq78cIYISHmZXshCvROLudpQN3lfL8G0NL7plMSSK+zlyvCaIJ4Iw=="
     19660    },
    1881519661    "@emotion/babel-plugin": {
    1881619662      "version": "11.10.2",
    … …  
    2103221878      }
    2103321879    },
     21880    "antd": {
     21881      "version": "4.23.6",
     21882      "resolved": "https://registry.npmjs.org/antd/-/antd-4.23.6.tgz",
     21883      "integrity": "sha512-AYH57cWBDe1ChtbnvG8i9dpKG4WnjE3AG0zIKpXByFNnxsr4saV6/19ihE8/ImSGpohN4E2zTXmo7R5/MyVRKQ==",
     21884      "requires": {
     21885        "@ant-design/colors": "^6.0.0",
     21886        "@ant-design/icons": "^4.7.0",
     21887        "@ant-design/react-slick": "~0.29.1",
     21888        "@babel/runtime": "^7.18.3",
     21889        "@ctrl/tinycolor": "^3.4.0",
     21890        "classnames": "^2.2.6",
     21891        "copy-to-clipboard": "^3.2.0",
     21892        "lodash": "^4.17.21",
     21893        "memoize-one": "^6.0.0",
     21894        "moment": "^2.29.2",
     21895        "rc-cascader": "~3.7.0",
     21896        "rc-checkbox": "~2.3.0",
     21897        "rc-collapse": "~3.3.0",
     21898        "rc-dialog": "~8.9.0",
     21899        "rc-drawer": "~5.1.0",
     21900        "rc-dropdown": "~4.0.0",
     21901        "rc-field-form": "~1.27.0",
     21902        "rc-image": "~5.7.0",
     21903        "rc-input": "~0.1.2",
     21904        "rc-input-number": "~7.3.9",
     21905        "rc-mentions": "~1.10.0",
     21906        "rc-menu": "~9.6.3",
     21907        "rc-motion": "^2.6.1",
     21908        "rc-notification": "~4.6.0",
     21909        "rc-pagination": "~3.1.17",
     21910        "rc-picker": "~2.6.11",
     21911        "rc-progress": "~3.3.2",
     21912        "rc-rate": "~2.9.0",
     21913        "rc-resize-observer": "^1.2.0",
     21914        "rc-segmented": "~2.1.0",
     21915        "rc-select": "~14.1.13",
     21916        "rc-slider": "~10.0.0",
     21917        "rc-steps": "~4.1.0",
     21918        "rc-switch": "~3.2.0",
     21919        "rc-table": "~7.26.0",
     21920        "rc-tabs": "~12.2.0",
     21921        "rc-textarea": "~0.4.5",
     21922        "rc-tooltip": "~5.2.0",
     21923        "rc-tree": "~5.7.0",
     21924        "rc-tree-select": "~5.5.0",
     21925        "rc-trigger": "^5.2.10",
     21926        "rc-upload": "~4.3.0",
     21927        "rc-util": "^5.22.5",
     21928        "scroll-into-view-if-needed": "^2.2.25"
     21929      }
     21930    },
    2103421931    "anymatch": {
    2103521932      "version": "3.1.2",
    … …  
    2108021977      }
    2108121978    },
     21979    "array-tree-filter": {
     21980      "version": "2.1.0",
     21981      "resolved": "https://registry.npmjs.org/array-tree-filter/-/array-tree-filter-2.1.0.tgz",
     21982      "integrity": "sha512-4ROwICNlNw/Hqa9v+rk5h22KjmzB1JGTMVKP2AKJBOCgb0yL0ASf0+YvCcLNNwquOHNX48jkeZIJ3a+oOQqKcw=="
     21983    },
    2108221984    "array-union": {
    2108321985      "version": "2.1.0",
    … …  
    2113322035      "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz",
    2113422036      "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ=="
     22037    },
     22038    "async-validator": {
     22039      "version": "4.2.5",
     22040      "resolved": "https://registry.npmjs.org/async-validator/-/async-validator-4.2.5.tgz",
     22041      "integrity": "sha512-7HhHjtERjqlNbZtqNqy2rckN/SpOOlmDliet+lP7k+eKZEjPk3DgyeU9lIXLdeLz0uBbbVp+9Qdow9wJWgwwfg=="
    2113522042    },
    2113622043    "asynckit": {
    … …  
    2167722584      "integrity": "sha512-cOU9usZw8/dXIXKtwa8pM0OTJQuJkxMN6w30csNRUerHfeQ5R6U3kkU/FtJeIf3M202OHfY2U8ccInBG7/xogA=="
    2167822585    },
     22586    "classnames": {
     22587      "version": "2.3.2",
     22588      "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz",
     22589      "integrity": "sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw=="
     22590    },
    2167922591    "clean-css": {
    2168022592      "version": "5.3.1",
    … …  
    2181522727      }
    2181622728    },
     22729    "compute-scroll-into-view": {
     22730      "version": "1.0.17",
     22731      "resolved": "https://registry.npmjs.org/compute-scroll-into-view/-/compute-scroll-into-view-1.0.17.tgz",
     22732      "integrity": "sha512-j4dx+Fb0URmzbwwMUrhqWM2BEWHdFGx+qZ9qqASHRPqvTYdqvWnHg0H1hIbcyLnvgnoNAVMlwkepyqM3DaIFUg=="
     22733    },
    2181722734    "concat-map": {
    2181822735      "version": "0.0.1",
    … …  
    2186722784      "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
    2186822785      "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ=="
     22786    },
     22787    "copy-to-clipboard": {
     22788      "version": "3.3.2",
     22789      "resolved": "https://registry.npmjs.org/copy-to-clipboard/-/copy-to-clipboard-3.3.2.tgz",
     22790      "integrity": "sha512-Vme1Z6RUDzrb6xAI7EZlVZ5uvOk2F//GaxKUxajDqm9LhOVM1inxNAD2vy+UZDYsd0uyA9s7b3/FVZPSxqrCfg==",
     22791      "requires": {
     22792        "toggle-selection": "^1.0.6"
     22793      }
    2186922794    },
    2187022795    "core-js": {
    … …  
    2219823123      }
    2219923124    },
     23125    "date-fns": {
     23126      "version": "2.29.3",
     23127      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.29.3.tgz",
     23128      "integrity": "sha512-dDCnyH2WnnKusqvZZ6+jA1O51Ibt8ZMRNkDZdyAyK4YfbDwa/cEmuztzG5pk6hqlp9aSBPYcjOlktquahGwGeA=="
     23129    },
     23130    "dayjs": {
     23131      "version": "1.11.6",
     23132      "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.6.tgz",
     23133      "integrity": "sha512-zZbY5giJAinCG+7AGaw0wIhNZ6J8AhWuSXKvuc1KAyMiRsvGQWqh4L+MomvhdAYjN+lqvVCMq1I41e3YHvXkyQ=="
     23134    },
    2220023135    "debug": {
    2220123136      "version": "4.3.4",
    … …  
    2236023295      "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.14.tgz",
    2236123296      "integrity": "sha512-NMt+m9zFMPZe0JcY9gN224Qvk6qLIdqex29clBvc/y75ZBX9YA9wNK3frsYvu2DI1xcCIwxwnX+TlsJ2DSOADg=="
     23297    },
     23298    "dom-align": {
     23299      "version": "1.12.3",
     23300      "resolved": "https://registry.npmjs.org/dom-align/-/dom-align-1.12.3.tgz",
     23301      "integrity": "sha512-Gj9hZN3a07cbR6zviMUBOMPdWxYhbMI+x+WS0NAIu2zFZmbK8ys9R79g+iG9qLnlCwpFoaB+fKy8Pdv470GsPA=="
    2236223302    },
    2236323303    "dom-converter": {
    … …  
    2585126791      "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw=="
    2585226792    },
     26793    "json2mq": {
     26794      "version": "0.2.0",
     26795      "resolved": "https://registry.npmjs.org/json2mq/-/json2mq-0.2.0.tgz",
     26796      "integrity": "sha512-SzoRg7ux5DWTII9J2qkrZrqV1gt+rTaoufMxEzXbS26Uid0NwaJd123HcoB80TgubEppxxIGdNxCx50fEoEWQA==",
     26797      "requires": {
     26798        "string-convert": "^0.2.0"
     26799      }
     26800    },
    2585326801    "json5": {
    2585426802      "version": "2.2.1",
    … …  
    2606627014        "fs-monkey": "^1.0.3"
    2606727015      }
     27016    },
     27017    "memoize-one": {
     27018      "version": "6.0.0",
     27019      "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-6.0.0.tgz",
     27020      "integrity": "sha512-rkpe71W0N0c0Xz6QD0eJETuWAJGnJ9afsl1srmwPrI+yBCkge5EycXXbYRyvL29zZVUWQCY7InPRCv3GDXuZNw=="
    2606827021    },
    2606927022    "merge-descriptors": {
    … …  
    2619427147        "minimist": "^1.2.6"
    2619527148      }
     27149    },
     27150    "moment": {
     27151      "version": "2.29.4",
     27152      "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz",
     27153      "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w=="
    2619627154    },
    2619727155    "ms": {
    … …  
    2747928437      }
    2748028438    },
     28439    "rc-align": {
     28440      "version": "4.0.12",
     28441      "resolved": "https://registry.npmjs.org/rc-align/-/rc-align-4.0.12.tgz",
     28442      "integrity": "sha512-3DuwSJp8iC/dgHzwreOQl52soj40LchlfUHtgACOUtwGuoFIOVh6n/sCpfqCU8kO5+iz6qR0YKvjgB8iPdE3aQ==",
     28443      "requires": {
     28444        "@babel/runtime": "^7.10.1",
     28445        "classnames": "2.x",
     28446        "dom-align": "^1.7.0",
     28447        "lodash": "^4.17.21",
     28448        "rc-util": "^5.3.0",
     28449        "resize-observer-polyfill": "^1.5.1"
     28450      }
     28451    },
     28452    "rc-cascader": {
     28453      "version": "3.7.0",
     28454      "resolved": "https://registry.npmjs.org/rc-cascader/-/rc-cascader-3.7.0.tgz",
     28455      "integrity": "sha512-SFtGpwmYN7RaWEAGTS4Rkc62ZV/qmQGg/tajr/7mfIkleuu8ro9Hlk6J+aA0x1YS4zlaZBtTcSaXM01QMiEV/A==",
     28456      "requires": {
     28457        "@babel/runtime": "^7.12.5",
     28458        "array-tree-filter": "^2.1.0",
     28459        "classnames": "^2.3.1",
     28460        "rc-select": "~14.1.0",
     28461        "rc-tree": "~5.7.0",
     28462        "rc-util": "^5.6.1"
     28463      }
     28464    },
     28465    "rc-checkbox": {
     28466      "version": "2.3.2",
     28467      "resolved": "https://registry.npmjs.org/rc-checkbox/-/rc-checkbox-2.3.2.tgz",
     28468      "integrity": "sha512-afVi1FYiGv1U0JlpNH/UaEXdh6WUJjcWokj/nUN2TgG80bfG+MDdbfHKlLcNNba94mbjy2/SXJ1HDgrOkXGAjg==",
     28469      "requires": {
     28470        "@babel/runtime": "^7.10.1",
     28471        "classnames": "^2.2.1"
     28472      }
     28473    },
     28474    "rc-collapse": {
     28475      "version": "3.3.1",
     28476      "resolved": "https://registry.npmjs.org/rc-collapse/-/rc-collapse-3.3.1.tgz",
     28477      "integrity": "sha512-cOJfcSe3R8vocrF8T+PgaHDrgeA1tX+lwfhwSj60NX9QVRidsILIbRNDLD6nAzmcvVC5PWiIRiR4S1OobxdhCg==",
     28478      "requires": {
     28479        "@babel/runtime": "^7.10.1",
     28480        "classnames": "2.x",
     28481        "rc-motion": "^2.3.4",
     28482        "rc-util": "^5.2.1",
     28483        "shallowequal": "^1.1.0"
     28484      }
     28485    },
     28486    "rc-dialog": {
     28487      "version": "8.9.0",
     28488      "resolved": "https://registry.npmjs.org/rc-dialog/-/rc-dialog-8.9.0.tgz",
     28489      "integrity": "sha512-Cp0tbJnrvPchJfnwIvOMWmJ4yjX3HWFatO6oBFD1jx8QkgsQCR0p8nUWAKdd3seLJhEC39/v56kZaEjwp9muoQ==",
     28490      "requires": {
     28491        "@babel/runtime": "^7.10.1",
     28492        "classnames": "^2.2.6",
     28493        "rc-motion": "^2.3.0",
     28494        "rc-util": "^5.21.0"
     28495      }
     28496    },
     28497    "rc-drawer": {
     28498      "version": "5.1.0",
     28499      "resolved": "https://registry.npmjs.org/rc-drawer/-/rc-drawer-5.1.0.tgz",
     28500      "integrity": "sha512-pU3Tsn99pxGdYowXehzZbdDVE+4lDXSGb7p8vA9mSmr569oc2Izh4Zw5vLKSe/Xxn2p5MSNbLVqD4tz+pK6SOw==",
     28501      "requires": {
     28502        "@babel/runtime": "^7.10.1",
     28503        "classnames": "^2.2.6",
     28504        "rc-motion": "^2.6.1",
     28505        "rc-util": "^5.21.2"
     28506      }
     28507    },
     28508    "rc-dropdown": {
     28509      "version": "4.0.1",
     28510      "resolved": "https://registry.npmjs.org/rc-dropdown/-/rc-dropdown-4.0.1.tgz",
     28511      "integrity": "sha512-OdpXuOcme1rm45cR0Jzgfl1otzmU4vuBVb+etXM8vcaULGokAKVpKlw8p6xzspG7jGd/XxShvq+N3VNEfk/l5g==",
     28512      "requires": {
     28513        "@babel/runtime": "^7.18.3",
     28514        "classnames": "^2.2.6",
     28515        "rc-trigger": "^5.3.1",
     28516        "rc-util": "^5.17.0"
     28517      }
     28518    },
     28519    "rc-field-form": {
     28520      "version": "1.27.2",
     28521      "resolved": "https://registry.npmjs.org/rc-field-form/-/rc-field-form-1.27.2.tgz",
     28522      "integrity": "sha512-NaTjkSB8JsHRgm52wkDorsDzFf2HH6GmCQ2AqkwO8zo+zIqybw8K1lkzDBMDJI8nw1pFuD46U5QsYZv4blYvdw==",
     28523      "requires": {
     28524        "@babel/runtime": "^7.18.0",
     28525        "async-validator": "^4.1.0",
     28526        "rc-util": "^5.8.0"
     28527      }
     28528    },
     28529    "rc-image": {
     28530      "version": "5.7.1",
     28531      "resolved": "https://registry.npmjs.org/rc-image/-/rc-image-5.7.1.tgz",
     28532      "integrity": "sha512-QyMfdhoUfb5W14plqXSisaYwpdstcLYnB0MjX5ccIK2rydQM9sDPuekQWu500DDGR2dBaIF5vx9XbWkNFK17Fg==",
     28533      "requires": {
     28534        "@babel/runtime": "^7.11.2",
     28535        "classnames": "^2.2.6",
     28536        "rc-dialog": "~8.9.0",
     28537        "rc-util": "^5.0.6"
     28538      }
     28539    },
     28540    "rc-input": {
     28541      "version": "0.1.2",
     28542      "resolved": "https://registry.npmjs.org/rc-input/-/rc-input-0.1.2.tgz",
     28543      "integrity": "sha512-ZPmwcFspgfYpUfbSx3KnLk9gImBcLOrlQCr4oTJ4jBoIXgJLTfm26yelzRgBJewhkvD8uJbgX0sQ/yOzuOHnJg==",
     28544      "requires": {
     28545        "@babel/runtime": "^7.11.1",
     28546        "classnames": "^2.2.1",
     28547        "rc-util": "^5.18.1"
     28548      }
     28549    },
     28550    "rc-input-number": {
     28551      "version": "7.3.9",
     28552      "resolved": "https://registry.npmjs.org/rc-input-number/-/rc-input-number-7.3.9.tgz",
     28553      "integrity": "sha512-u0+miS+SATdb6DtssYei2JJ1WuZME+nXaG6XGtR8maNyW5uGDytfDu60OTWLQEb0Anv/AcCzehldV8CKmKyQfA==",
     28554      "requires": {
     28555        "@babel/runtime": "^7.10.1",
     28556        "classnames": "^2.2.5",
     28557        "rc-util": "^5.23.0"
     28558      }
     28559    },
     28560    "rc-mentions": {
     28561      "version": "1.10.0",
     28562      "resolved": "https://registry.npmjs.org/rc-mentions/-/rc-mentions-1.10.0.tgz",
     28563      "integrity": "sha512-oMlYWnwXSxP2NQVlgxOTzuG/u9BUc3ySY78K3/t7MNhJWpZzXTao+/Bic6tyZLuNCO89//hVQJBdaR2rnFQl6Q==",
     28564      "requires": {
     28565        "@babel/runtime": "^7.10.1",
     28566        "classnames": "^2.2.6",
     28567        "rc-menu": "~9.6.0",
     28568        "rc-textarea": "^0.4.0",
     28569        "rc-trigger": "^5.0.4",
     28570        "rc-util": "^5.22.5"
     28571      }
     28572    },
     28573    "rc-menu": {
     28574      "version": "9.6.4",
     28575      "resolved": "https://registry.npmjs.org/rc-menu/-/rc-menu-9.6.4.tgz",
     28576      "integrity": "sha512-6DiNAjxjVIPLZXHffXxxcyE15d4isRL7iQ1ru4MqYDH2Cqc5bW96wZOdMydFtGLyDdnmEQ9jVvdCE9yliGvzkw==",
     28577      "requires": {
     28578        "@babel/runtime": "^7.10.1",
     28579        "classnames": "2.x",
     28580        "rc-motion": "^2.4.3",
     28581        "rc-overflow": "^1.2.0",
     28582        "rc-trigger": "^5.1.2",
     28583        "rc-util": "^5.12.0",
     28584        "shallowequal": "^1.1.0"
     28585      }
     28586    },
     28587    "rc-motion": {
     28588      "version": "2.6.2",
     28589      "resolved": "https://registry.npmjs.org/rc-motion/-/rc-motion-2.6.2.tgz",
     28590      "integrity": "sha512-4w1FaX3dtV749P8GwfS4fYnFG4Rb9pxvCYPc/b2fw1cmlHJWNNgOFIz7ysiD+eOrzJSvnLJWlNQQncpNMXwwpg==",
     28591      "requires": {
     28592        "@babel/runtime": "^7.11.1",
     28593        "classnames": "^2.2.1",
     28594        "rc-util": "^5.21.0"
     28595      }
     28596    },
     28597    "rc-notification": {
     28598      "version": "4.6.0",
     28599      "resolved": "https://registry.npmjs.org/rc-notification/-/rc-notification-4.6.0.tgz",
     28600      "integrity": "sha512-xF3MKgIoynzjQAO4lqsoraiFo3UXNYlBfpHs0VWvwF+4pimen9/H1DYLN2mfRWhHovW6gRpla73m2nmyIqAMZQ==",
     28601      "requires": {
     28602        "@babel/runtime": "^7.10.1",
     28603        "classnames": "2.x",
     28604        "rc-motion": "^2.2.0",
     28605        "rc-util": "^5.20.1"
     28606      }
     28607    },
     28608    "rc-overflow": {
     28609      "version": "1.2.8",
     28610      "resolved": "https://registry.npmjs.org/rc-overflow/-/rc-overflow-1.2.8.tgz",
     28611      "integrity": "sha512-QJ0UItckWPQ37ZL1dMEBAdY1dhfTXFL9k6oTTcyydVwoUNMnMqCGqnRNA98axSr/OeDKqR6DVFyi8eA5RQI/uQ==",
     28612      "requires": {
     28613        "@babel/runtime": "^7.11.1",
     28614        "classnames": "^2.2.1",
     28615        "rc-resize-observer": "^1.0.0",
     28616        "rc-util": "^5.19.2"
     28617      }
     28618    },
     28619    "rc-pagination": {
     28620      "version": "3.1.17",
     28621      "resolved": "https://registry.npmjs.org/rc-pagination/-/rc-pagination-3.1.17.tgz",
     28622      "integrity": "sha512-/BQ5UxcBnW28vFAcP2hfh+Xg15W0QZn8TWYwdCApchMH1H0CxiaUUcULP8uXcFM1TygcdKWdt3JqsL9cTAfdkQ==",
     28623      "requires": {
     28624        "@babel/runtime": "^7.10.1",
     28625        "classnames": "^2.2.1"
     28626      }
     28627    },
     28628    "rc-picker": {
     28629      "version": "2.6.11",
     28630      "resolved": "https://registry.npmjs.org/rc-picker/-/rc-picker-2.6.11.tgz",
     28631      "integrity": "sha512-INJ7ULu+Kj4UgqbcqE8Q+QpMw55xFf9kkyLBHJFk0ihjJpAV4glialRfqHE7k4KX2BWYPQfpILwhwR14x2EiRQ==",
     28632      "requires": {
     28633        "@babel/runtime": "^7.10.1",
     28634        "classnames": "^2.2.1",
     28635        "date-fns": "2.x",
     28636        "dayjs": "1.x",
     28637        "moment": "^2.24.0",
     28638        "rc-trigger": "^5.0.4",
     28639        "rc-util": "^5.4.0",
     28640        "shallowequal": "^1.1.0"
     28641      }
     28642    },
     28643    "rc-progress": {
     28644      "version": "3.3.3",
     28645      "resolved": "https://registry.npmjs.org/rc-progress/-/rc-progress-3.3.3.tgz",
     28646      "integrity": "sha512-MDVNVHzGanYtRy2KKraEaWeZLri2ZHWIRyaE1a9MQ2MuJ09m+Wxj5cfcaoaR6z5iRpHpA59YeUxAlpML8N4PJw==",
     28647      "requires": {
     28648        "@babel/runtime": "^7.10.1",
     28649        "classnames": "^2.2.6",
     28650        "rc-util": "^5.16.1"
     28651      }
     28652    },
     28653    "rc-rate": {
     28654      "version": "2.9.2",
     28655      "resolved": "https://registry.npmjs.org/rc-rate/-/rc-rate-2.9.2.tgz",
     28656      "integrity": "sha512-SaiZFyN8pe0Fgphv8t3+kidlej+cq/EALkAJAc3A0w0XcPaH2L1aggM8bhe1u6GAGuQNAoFvTLjw4qLPGRKV5g==",
     28657      "requires": {
     28658        "@babel/runtime": "^7.10.1",
     28659        "classnames": "^2.2.5",
     28660        "rc-util": "^5.0.1"
     28661      }
     28662    },
     28663    "rc-resize-observer": {
     28664      "version": "1.2.0",
     28665      "resolved": "https://registry.npmjs.org/rc-resize-observer/-/rc-resize-observer-1.2.0.tgz",
     28666      "integrity": "sha512-6W+UzT3PyDM0wVCEHfoW3qTHPTvbdSgiA43buiy8PzmeMnfgnDeb9NjdimMXMl3/TcrvvWl5RRVdp+NqcR47pQ==",
     28667      "requires": {
     28668        "@babel/runtime": "^7.10.1",
     28669        "classnames": "^2.2.1",
     28670        "rc-util": "^5.15.0",
     28671        "resize-observer-polyfill": "^1.5.1"
     28672      }
     28673    },
     28674    "rc-segmented": {
     28675      "version": "2.1.0",
     28676      "resolved": "https://registry.npmjs.org/rc-segmented/-/rc-segmented-2.1.0.tgz",
     28677      "integrity": "sha512-hUlonro+pYoZcwrH6Vm56B2ftLfQh046hrwif/VwLIw1j3zGt52p5mREBwmeVzXnSwgnagpOpfafspzs1asjGw==",
     28678      "requires": {
     28679        "@babel/runtime": "^7.11.1",
     28680        "classnames": "^2.2.1",
     28681        "rc-motion": "^2.4.4",
     28682        "rc-util": "^5.17.0"
     28683      }
     28684    },
     28685    "rc-select": {
     28686      "version": "14.1.13",
     28687      "resolved": "https://registry.npmjs.org/rc-select/-/rc-select-14.1.13.tgz",
     28688      "integrity": "sha512-WMEsC3gTwA1dbzWOdVIXDmWyidYNLq68AwvvUlRROw790uGUly0/vmqDozXrIr0QvN/A3CEULx12o+WtLCAefg==",
     28689      "requires": {
     28690        "@babel/runtime": "^7.10.1",
     28691        "classnames": "2.x",
     28692        "rc-motion": "^2.0.1",
     28693        "rc-overflow": "^1.0.0",
     28694        "rc-trigger": "^5.0.4",
     28695        "rc-util": "^5.16.1",
     28696        "rc-virtual-list": "^3.2.0"
     28697      }
     28698    },
     28699    "rc-slider": {
     28700      "version": "10.0.1",
     28701      "resolved": "https://registry.npmjs.org/rc-slider/-/rc-slider-10.0.1.tgz",
     28702      "integrity": "sha512-igTKF3zBet7oS/3yNiIlmU8KnZ45npmrmHlUUio8PNbIhzMcsh+oE/r2UD42Y6YD2D/s+kzCQkzQrPD6RY435Q==",
     28703      "requires": {
     28704        "@babel/runtime": "^7.10.1",
     28705        "classnames": "^2.2.5",
     28706        "rc-util": "^5.18.1",
     28707        "shallowequal": "^1.1.0"
     28708      }
     28709    },
     28710    "rc-steps": {
     28711      "version": "4.1.4",
     28712      "resolved": "https://registry.npmjs.org/rc-steps/-/rc-steps-4.1.4.tgz",
     28713      "integrity": "sha512-qoCqKZWSpkh/b03ASGx1WhpKnuZcRWmvuW+ZUu4mvMdfvFzVxblTwUM+9aBd0mlEUFmt6GW8FXhMpHkK3Uzp3w==",
     28714      "requires": {
     28715        "@babel/runtime": "^7.10.2",
     28716        "classnames": "^2.2.3",
     28717        "rc-util": "^5.0.1"
     28718      }
     28719    },
     28720    "rc-switch": {
     28721      "version": "3.2.2",
     28722      "resolved": "https://registry.npmjs.org/rc-switch/-/rc-switch-3.2.2.tgz",
     28723      "integrity": "sha512-+gUJClsZZzvAHGy1vZfnwySxj+MjLlGRyXKXScrtCTcmiYNPzxDFOxdQ/3pK1Kt/0POvwJ/6ALOR8gwdXGhs+A==",
     28724      "requires": {
     28725        "@babel/runtime": "^7.10.1",
     28726        "classnames": "^2.2.1",
     28727        "rc-util": "^5.0.1"
     28728      }
     28729    },
     28730    "rc-table": {
     28731      "version": "7.26.0",
     28732      "resolved": "https://registry.npmjs.org/rc-table/-/rc-table-7.26.0.tgz",
     28733      "integrity": "sha512-0cD8e6S+DTGAt5nBZQIPFYEaIukn17sfa5uFL98faHlH/whZzD8ii3dbFL4wmUDEL4BLybhYop+QUfZJ4CPvNQ==",
     28734      "requires": {
     28735        "@babel/runtime": "^7.10.1",
     28736        "classnames": "^2.2.5",
     28737        "rc-resize-observer": "^1.1.0",
     28738        "rc-util": "^5.22.5",
     28739        "shallowequal": "^1.1.0"
     28740      }
     28741    },
     28742    "rc-tabs": {
     28743      "version": "12.2.1",
     28744      "resolved": "https://registry.npmjs.org/rc-tabs/-/rc-tabs-12.2.1.tgz",
     28745      "integrity": "sha512-09pVv4kN8VFqp6THceEmxOW8PAShQC08hrroeVYP4Y8YBFaP1PIWdyFL01czcbyz5YZFj9flZ7aljMaAl0jLVg==",
     28746      "requires": {
     28747        "@babel/runtime": "^7.11.2",
     28748        "classnames": "2.x",
     28749        "rc-dropdown": "~4.0.0",
     28750        "rc-menu": "~9.6.0",
     28751        "rc-motion": "^2.6.2",
     28752        "rc-resize-observer": "^1.0.0",
     28753        "rc-util": "^5.5.0"
     28754      }
     28755    },
     28756    "rc-textarea": {
     28757      "version": "0.4.5",
     28758      "resolved": "https://registry.npmjs.org/rc-textarea/-/rc-textarea-0.4.5.tgz",
     28759      "integrity": "sha512-WHeJRgUlloNyVgTsItMrIXwMhU6P3NmrUDkxX+JRwEpJjECsKtZNlNcXe9pHNLCaYQ3Z1cVCfsClhgDDgJ2kFQ==",
     28760      "requires": {
     28761        "@babel/runtime": "^7.10.1",
     28762        "classnames": "^2.2.1",
     28763        "rc-resize-observer": "^1.0.0",
     28764        "rc-util": "^5.7.0",
     28765        "shallowequal": "^1.1.0"
     28766      }
     28767    },
     28768    "rc-tooltip": {
     28769      "version": "5.2.2",
     28770      "resolved": "https://registry.npmjs.org/rc-tooltip/-/rc-tooltip-5.2.2.tgz",
     28771      "integrity": "sha512-jtQzU/18S6EI3lhSGoDYhPqNpWajMtS5VV/ld1LwyfrDByQpYmw/LW6U7oFXXLukjfDHQ7Ju705A82PRNFWYhg==",
     28772      "requires": {
     28773        "@babel/runtime": "^7.11.2",
     28774        "classnames": "^2.3.1",
     28775        "rc-trigger": "^5.0.0"
     28776      }
     28777    },
     28778    "rc-tree": {
     28779      "version": "5.7.0",
     28780      "resolved": "https://registry.npmjs.org/rc-tree/-/rc-tree-5.7.0.tgz",
     28781      "integrity": "sha512-F+Ewkv/UcutshnVBMISP+lPdHDlcsL+YH/MQDVWbk+QdkfID7vXiwrHMEZn31+2Rbbm21z/HPceGS8PXGMmnQg==",
     28782      "requires": {
     28783        "@babel/runtime": "^7.10.1",
     28784        "classnames": "2.x",
     28785        "rc-motion": "^2.0.1",
     28786        "rc-util": "^5.16.1",
     28787        "rc-virtual-list": "^3.4.8"
     28788      }
     28789    },
     28790    "rc-tree-select": {
     28791      "version": "5.5.2",
     28792      "resolved": "https://registry.npmjs.org/rc-tree-select/-/rc-tree-select-5.5.2.tgz",
     28793      "integrity": "sha512-toxkzVhkWQ2wvInOhHwmyEpCCi9osnoRQmN0trKvdvyzLattCw63F2T+V/dS2d/xUkrw6Zr1Y2J0/xP57x/jYQ==",
     28794      "requires": {
     28795        "@babel/runtime": "^7.10.1",
     28796        "classnames": "2.x",
     28797        "rc-select": "~14.1.0",
     28798        "rc-tree": "~5.7.0",
     28799        "rc-util": "^5.16.1"
     28800      }
     28801    },
     28802    "rc-trigger": {
     28803      "version": "5.3.1",
     28804      "resolved": "https://registry.npmjs.org/rc-trigger/-/rc-trigger-5.3.1.tgz",
     28805      "integrity": "sha512-5gaFbDkYSefZ14j2AdzucXzlWgU2ri5uEjkHvsf1ynRhdJbKxNOnw4PBZ9+FVULNGFiDzzlVF8RJnR9P/xrnKQ==",
     28806      "requires": {
     28807        "@babel/runtime": "^7.18.3",
     28808        "classnames": "^2.2.6",
     28809        "rc-align": "^4.0.0",
     28810        "rc-motion": "^2.0.0",
     28811        "rc-util": "^5.19.2"
     28812      }
     28813    },
     28814    "rc-upload": {
     28815      "version": "4.3.4",
     28816      "resolved": "https://registry.npmjs.org/rc-upload/-/rc-upload-4.3.4.tgz",
     28817      "integrity": "sha512-uVbtHFGNjHG/RyAfm9fluXB6pvArAGyAx8z7XzXXyorEgVIWj6mOlriuDm0XowDHYz4ycNK0nE0oP3cbFnzxiQ==",
     28818      "requires": {
     28819        "@babel/runtime": "^7.18.3",
     28820        "classnames": "^2.2.5",
     28821        "rc-util": "^5.2.0"
     28822      }
     28823    },
     28824    "rc-util": {
     28825      "version": "5.24.4",
     28826      "resolved": "https://registry.npmjs.org/rc-util/-/rc-util-5.24.4.tgz",
     28827      "integrity": "sha512-2a4RQnycV9eV7lVZPEJ7QwJRPlZNc06J7CwcwZo4vIHr3PfUqtYgl1EkUV9ETAc6VRRi8XZOMFhYG63whlIC9Q==",
     28828      "requires": {
     28829        "@babel/runtime": "^7.18.3",
     28830        "react-is": "^16.12.0",
     28831        "shallowequal": "^1.1.0"
     28832      },
     28833      "dependencies": {
     28834        "react-is": {
     28835          "version": "16.13.1",
     28836          "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
     28837          "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="
     28838        }
     28839      }
     28840    },
     28841    "rc-virtual-list": {
     28842      "version": "3.4.10",
     28843      "resolved": "https://registry.npmjs.org/rc-virtual-list/-/rc-virtual-list-3.4.10.tgz",
     28844      "integrity": "sha512-Jv0cgJxJ+8F/YViW8WGs/jQF2rmT8RUcJ5uDJs5MOFLTYLAvCpM/xU+Zu6EpCun50fmovhXiItQctcfE2UY3Aw==",
     28845      "requires": {
     28846        "classnames": "^2.2.6",
     28847        "rc-resize-observer": "^1.0.0",
     28848        "rc-util": "^5.15.0"
     28849      }
     28850    },
    2748128851    "react": {
    2748228852      "version": "18.2.0",
    … …  
    2785929229      "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ=="
    2786029230    },
     29231    "resize-observer-polyfill": {
     29232      "version": "1.5.1",
     29233      "resolved": "https://registry.npmjs.org/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz",
     29234      "integrity": "sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg=="
     29235    },
    2786129236    "resolve": {
    2786229237      "version": "1.22.1",
    … …  
    2805129426        "ajv": "^6.12.5",
    2805229427        "ajv-keywords": "^3.5.2"
     29428      }
     29429    },
     29430    "scroll-into-view-if-needed": {
     29431      "version": "2.2.29",
     29432      "resolved": "https://registry.npmjs.org/scroll-into-view-if-needed/-/scroll-into-view-if-needed-2.2.29.tgz",
     29433      "integrity": "sha512-hxpAR6AN+Gh53AdAimHM6C8oTN1ppwVZITihix+WqalywBeFcQ6LdQP5ABNl26nX8GTEL7VT+b8lKpdqq65wXg==",
     29434      "requires": {
     29435        "compute-scroll-into-view": "^1.0.17"
    2805329436      }
    2805429437    },
    … …  
    2820029583      "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
    2820129584    },
     29585    "shallowequal": {
     29586      "version": "1.1.0",
     29587      "resolved": "https://registry.npmjs.org/shallowequal/-/shallowequal-1.1.0.tgz",
     29588      "integrity": "sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ=="
     29589    },
    2820229590    "shebang-command": {
    2820329591      "version": "2.0.0",
    … …  
    2837329761        }
    2837429762      }
     29763    },
     29764    "string-convert": {
     29765      "version": "0.2.1",
     29766      "resolved": "https://registry.npmjs.org/string-convert/-/string-convert-0.2.1.tgz",
     29767      "integrity": "sha512-u/1tdPl4yQnPBjnVrmdLo9gtuLvELKsAoRapekWggdiQNvvvum+jYF329d84NAa660KQw7pB2n36KrIKVoXa3A=="
    2837529768    },
    2837629769    "string-length": {
    … …  
    2877730170        "is-number": "^7.0.0"
    2877830171      }
     30172    },
     30173    "toggle-selection": {
     30174      "version": "1.0.6",
     30175      "resolved": "https://registry.npmjs.org/toggle-selection/-/toggle-selection-1.0.6.tgz",
     30176      "integrity": "sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ=="
    2877930177    },
    2878030178    "toidentifier": {

  • phonelux-frontend/package.json

    rffd50db r47f4eaf  
    1616    "@testing-library/user-event": "^13.5.0",
    1717    "@tippyjs/react": "^4.2.6",
     18    "antd": "^4.23.6",
    1819    "axios": "^0.27.2",
    1920    "jwt-decode": "^3.1.2",

  • phonelux-frontend/src/App.js

    rffd50db r47f4eaf  
    1212import CompareOffersComponent from "./components/CompareOffersComponent/CompareOffersComponent";
    1313import SpecificationsFilterComponent from "./components/FiltersComponents/SpecificationsFilterComponent";
     14import OfferReportsComponent from "./components/OfferReportsComponent/OfferReportsComponent";
     15import ScrappersComponent from "./components/ScrappersComponent/ScrappersComponent";
    1416
    1517
    … …  
    2931        <Route path="/admin/editoffer/:offerId" element={<EditOfferComponent/>}/>
    3032        <Route path="/compareoffers" element={<CompareOffersComponent/>}/>
     33        <Route path="/offerreport/reports" element={<OfferReportsComponent/>}/>
     34        <Route path="/scrapperinfo" element={<ScrappersComponent/>}/>
    3135      </Routes>
    3236    </BrowserRouter>

  • phonelux-frontend/src/components/EditOfferComponent/EditOfferComponent.js

    rffd50db r47f4eaf  
    1717
    1818    componentDidMount(){
    19         // if(!localStorage.getItem('token'))
    20         // {
    21         //     window.location.href = "/"
    22         // }
    23 
    24         // if(this.context.role != 'ADMIN' && this.context.role != 'SUPERADMIN')
    25         // {
    26         //     window.location.href = "/"
    27         // }
    28 
    2919        var config = {
    3020            method: 'get',

  • phonelux-frontend/src/components/HomepageComponent.js

    rffd50db r47f4eaf  
    220220  render() {
    221221    // console.log(this.context)
    222     // console.log(localStorage.getItem('token'))
     222    console.log(localStorage.getItem('token'))
    223223    // console.log(this.state)
    224224    return (

  • phonelux-frontend/src/components/LoginRegisterComponents/LoginFormComponent.css

    rffd50db r47f4eaf  
    162162  background-color: rgb(251, 224, 224);
    163163 }
     164
     165 .login-with-facebook-image{
     166  width: 40%;
     167 }
     168
     169.login-with-facebook-image:hover{
     170  cursor: pointer;
     171}
     172
     173 .login-with-facebook-wrapper{
     174  display: flex;
     175  justify-content: center;
     176 }

  • phonelux-frontend/src/components/LoginRegisterComponents/LoginFormComponent.js

    rffd50db r47f4eaf  
    66import { Link } from 'react-router-dom';
    77import axios from 'axios';
     8import FacebookLogin from 'react-facebook-login'
    89
    910export class LoginFormComponent extends Component {
    … …  
    1819    }
    1920  }
    20  
    21   changeHandler = (e) =>{
    22       this.setState({[e.target.name] : e.target.value})
    23   }
     21
    2422
    2523  submitHandler = (e) => {

  • phonelux-frontend/src/components/NavbarComponent/NavbarComponent.js

    rffd50db r47f4eaf  
    1212import SupervisorAccountIcon from '@mui/icons-material/SupervisorAccount';
    1313import CompareIcon from '@mui/icons-material/Compare';
     14import BugReportIcon from '@mui/icons-material/BugReport';
     15import ReportIcon from '@mui/icons-material/Report';
     16import UpdateIcon from '@mui/icons-material/Update';
    1417
    1518export class NavbarComponent extends Component {
    … …  
    3639            <Link style={{color: 'black'}} to={"/management/users"}>
    3740              <SupervisorAccountIcon style={{fontSize: '40px', marginTop: '10px', marginRight: '10px' }} className='navbar-superadmin-icon'/>
     41            </Link>
     42          </Tippy> : <></>
     43        }
     44         {
     45          localStorage.getItem('token') && this.context.role == 'SUPERADMIN' ?
     46          <Tippy placement='bottom' content='Преземање на содржина'>
     47            <Link style={{color: 'black'}} to={"/scrapperinfo"}>
     48              <UpdateIcon style={{fontSize: '40px', marginTop: '10px', marginRight: '10px' }} className='navbar-superadmin-icon'/>
     49            </Link>
     50          </Tippy> : <></>
     51        }
     52         {
     53          localStorage.getItem('token') && (this.context.role == 'SUPERADMIN' || this.context.role == 'ADMIN') ?
     54          <Tippy placement='bottom' content='Пријавени понуди'>
     55            <Link style={{color: 'black'}} to={"/offerreport/reports"}>
     56              <BugReportIcon style={{fontSize: '40px', marginTop: '10px', marginRight: '10px' }} className='navbar-offerreports-icon'/>
    3857            </Link>
    3958          </Tippy> : <></>

  • phonelux-frontend/src/components/PhoneOfferDetailsComponent/PhoneOfferDetailsComponent.css

    rffd50db r47f4eaf  
    140140}
    141141
     142.offerdetails-report-icon{
     143    cursor: pointer;
     144}
    142145
    143146
     147

  • phonelux-frontend/src/components/PhoneOfferDetailsComponent/PhoneOfferDetailsComponent.js

    rffd50db r47f4eaf  
    99import VisibilityIcon from '@mui/icons-material/Visibility';
    1010import VisibilityOffIcon from '@mui/icons-material/VisibilityOff';
     11import ReportIcon from '@mui/icons-material/Report';
    1112
    1213
    … …  
    6162  }
    6263
     64    reportOffer = () =>{
     65      alert('Пратена е пријава до администратор за невалидност на спецификациите на понудата!')
     66      var config = {
     67        method: 'post',
     68        url: '/offerreport/'+this.state.offerId+'/'+this.context.userId,
     69        headers: {
     70          'Authorization': 'Bearer '+localStorage.getItem('token')
     71        }
     72      };
     73     
     74      axios(config)
     75      .then(function (response) {
     76        console.log(JSON.stringify(response.data));
     77      })
     78      .catch(function (error) {
     79        console.log(error);
     80      });
     81    }
     82
    6383  render() {
    6484    console.log(this.state)
    … …  
    7191          {
    7292            localStorage.getItem('token') ?
     93            <Tippy placement='bottom' content='Пријави понуда за неточни спецификации'>
     94            <ReportIcon onClick={this.reportOffer} className='offerdetails-report-icon' style={{fontSize: '45px'}}/>
     95         </Tippy> : <></>
     96          }
     97          {
     98            localStorage.getItem('token') ?
    7399            this.state.showAllSpecs ?
    74100              <Tippy placement='bottom' content='Прикажи ги избраните спецификации'>
    … …  
    103129
    104130          })()}
     131
    105132        </div>
    106133        <div className='phone-offer-details-last-updated-wrapper'></div>
    … …  
    109136        <table className='phone-offer-details-table'>
    110137          <thead>
    111           <tr><th colSpan={2}>Детали за понудата</th></tr>
     138          <tr><th colSpan={2}>
     139            Детали за понудата
     140          </th></tr>
    112141          </thead>
    113142          <tbody>

  • phonelux-frontend/src/components/SuperAdminComponent/SuperAdminComponent.js

    rffd50db r47f4eaf  
    6969
    7070    componentDidMount(){
    71       // if(!localStorage.getItem('token'))
    72       // {
    73       //     window.location.href = "/"
    74       // }
    75 
    76       // if(this.context.role != 'SUPERADMIN')
    77       // {
    78       //     window.location.href = "/"
    79       // }
    8071      this.getUsers()
    8172    }

  • phonelux_scrappers/.idea/misc.xml

    rffd50db r47f4eaf  
    11<?xml version="1.0" encoding="UTF-8"?>
    22<project version="4">
    3   <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8 (phonelux_scrappers) (2)" project-jdk-type="Python SDK" />
     3  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (phonelux_scrappers)" project-jdk-type="Python SDK" />
    44</project>

  • phonelux_scrappers/.idea/phonelux_scrappers.iml

    rffd50db r47f4eaf  
    55      <excludeFolder url="file://$MODULE_DIR$/venv" />
    66    </content>
    7     <orderEntry type="inheritedJdk" />
     7    <orderEntry type="jdk" jdkName="Python 3.9 (phonelux_scrappers)" jdkType="Python SDK" />
    88    <orderEntry type="sourceFolder" forTests="false" />
    99  </component>

  • phonelux_scrappers/outputfile.txt

    rffd50db r47f4eaf  
    1 Samsung Galaxy A13
    2 9990
    3 Apple iPhone 13
    4 47499
    5 Apple iPhone 13 Pro
    6 62690
    7 Apple iPhone 12
    8 32690
    9 Samsung Galaxy S22 Ultra 5G
    10 60000
    11 Apple iPhone 11
    12 23490
    13 Samsung Galaxy S22 5G
    14 43000
    15 Samsung Galaxy S21 FE
    16 33800
    17 Samsung Galaxy A33 5G
    18 17490
    19 Apple iPhone 13 Pro Max
    20 68890
    21 Samsung Galaxy A32
    22 13900
    23 Samsung Galaxy S22+ 5G
    24 50150
    25 Apple iPhone 14 Pro
    26 71890
    27 Xiaomi 12
    28 28999
    29 Samsung Galaxy A53 5G
    30 20290
    31 Xiaomi Redmi Note 11
    32 11990
    33 Xiaomi Redmi 9A
    34 6490
    35 Huawei Nova 9
    36 22900
    37 Samsung Galaxy A12
    38 9500
    39 Samsung Galaxy Z Flip 4
    40 58400
    41 Apple iPhone 14
    42 1
    43 Apple iPhone 13 Mini
    44 42990
    45 Samsung Galaxy A52s
    46 18500
    47 Xiaomi Redmi 10C
    48 9990
    49 Apple iPhone SE
    50 24600
    51 Samsung Galaxy A03s
    52 6790
    53 Samsung Galaxy A22 5G
    54 11700
    55 Xiaomi Mi 11 Lite
    56 18500
    57 Samsung Galaxy A23
    58 11790
    59 Apple iPhone 14 Pro Max
    60 99999
    61 Samsung Galaxy Z Fold 4
    62 98400
    63 Samsung Galaxy A03
    64 8490
    65 Huawei nova Y70
    66 13699
    67 Samsung Galaxy Z Flip3 5G
    68 38990
    69 Huawei Nova 8i
    70 19999
    71 Samsung Galaxy Z Fold 3 5G
    72 69990
    73 Xiaomi 11 Lite 5G NE
    74 24999
    75 Xiaomi Redmi Note 11 Pro
    76 16590
    77 Xiaomi Mi 11T 5G
    78 23400
    79 Huawei Nova 9 SE
    80 22999
    81 Xiaomi Redmi 9
    82 9399
    83 Huawei nova Y90
    84 15999
    85 Apple iPhone SE (2022)
    86 28290
    87 Xiaomi Poco X3
    88 13999
    89 OnePlus Nord N10 5G
    90 18999
    91 Samsung Galaxy A52
    92 18190
    93 Xiaomi Redmi Note 10
    94 13999
    95 Blackview A70 Pro
    96 7399
    97 Realme C11
    98 6999
    99 Honor 50 Lite
    100 11990
    101 Xiaomi Poco F3
    102 21500
    103 Huawei P50 Pro
    104 58400
    105 Xiaomi Redmi 9C
    106 6990
    107 Xiaomi 12X
    108 29900
    109 Blackview A55
    110 5999
    111 OnePlus Nord CE
    112 26999
    113 Vivo Y72 5G
    114 18999
    115 Xiaomi Mi 11T Pro 5G
    116 33800
    117 Xiaomi Poco M3
    118 12290
    119 Vivo V21 5G
    120 24999
    121 Vivo Y33s
    122 15999
    123 Samsung Galaxy A02s
    124 8890
    125 Motorola Moto G31
    126 12899
    127 Xiaomi Mi 11 Lite 5G NE
    128 25999
    129 Alcatel 2019G
    130 2290
    131 Samsung Galaxy S21+ 5G
    132 41790
    133 Xiaomi Redmi Note 10 5G
    134 13490
    135 Xiaomi 11T
    136 31999
    137 Alcatel 1SE
    138 6290
    139 Alcatel 1066D
    140 1299
    141 Xiaomi Redmi Note 9
    142 9990
    143 Xiaomi Redmi Note 11 Pro+ 5G
    144 27999
    145 Blackview A80
    146 6499
    147 Motorola Moto G22
    148 11299
    149 Xiaomi Poco M3 Pro 5G
    150 12999
    151 Xiaomi Redmi Note 9S
    152 11900
    153 Blackview A70
    154 7299
    155 Xiaomi Redmi 9AT
    156 6999
    157 Doogee X95
    158 5999
    159 Xiaomi Redmi Note 10 Pro
    160 15990
    161 Motorola Edge 20
    162 30999
    163 Motorola Moto G60
    164 16999
    165 Honor 50
    166 23400
    167 Apple iPhone 12 Pro Max
    168 66500
    169 Apple iPhone 11 Pro Max
    170 32690
    171 Nokia 105 (2017)
    172 1490
    173 Samsung Galaxy S21 Ultra 5G
    174 52290
    175 Motorola Moto E7 Power
    176 8999
    177 Apple iPhone 11 Pro
    178 27790
    179 Alcatel 1
    180 5499
    181 Xiaomi Poco X3 Pro
    182 15400
    183 Xiaomi Redmi Note 9 Pro
    184 15499
    185 Blackview A60 Pro
    186 6299
    187 Blackview Oscal C20
    188 5249
    189 Realme C21Y
    190 7999
    191 Vivo Y21
    192 11899
    193 Samsung Galaxy A20e
    194 9699
    195 Vivo Y21s
    196 13999
    197 Nokia 3310
    198 3399
    199 Xiaomi Mi 10 Lite 5G
    200 15990
    201 Xiaomi Mi Note 10 Lite
    202 19700
    203 Samsung Galaxy S21 5G
    204 38999
    205 Xiaomi Mi 11 Ultra
    206 53490
    207 Motorola Moto G10
    208 10299
    209 Alcatel 3026X
    210 4499
    211 Samsung Galaxy S20 FE
    212 22690
    213 Xiaomi Redmi Note 11S
    214 19999
    215 Doogee X96
    216 7699
    217 Alcatel 2053D
    218 2999
    219 Alcatel 3025X
    220 3999
    221 Doogee X93
    222 5999
    223 Vivo Y20s
    224 12499
    225 Infinix Hot 11
    226 10499
    227 Infinix Smart 6
    228 7999
    229 Panasonic KX-TU160
    230 3999
    231 Xiaomi 12 Pro
    232 54150
    233 Xiaomi Mi 10T Pro 5G
    234 29900
    235 Xiaomi Poco X4 Pro 5G
    236 17200
    237 Apple iPhone 12 Pro
    238 61590
    239 Google Pixel 6 Pro
    240 61500
    241 Samsung Galaxy Note 20
    242 36900
    243 Xiaomi Poco M4 Pro 5G
    244 15999
    245 OnePlus 10 Pro 5G
    246 52290
    247 Wiko View
    248 5990
    249 Samsung Galaxy A72
    250 23300
    251 Alcatel 2003D
    252 2199
    253 Xiaomi Redmi Note 10 Pro Max
    254 16490
    255 Trevi Max 10
    256 1599
    257 Honor X8
    258 16499
    259 Nokia 6310
    260 5490
    261 Xiaomi 11T Pro
    262 47999
    263 Samsung Galaxy A02
    264 8000
    265 Samsung Galaxy A40
    266 11999
    267 Motorola Moto G71
    268 18999
    269 Samsung Galaxy Note 20 Ultra 5G
    270 50390
    271 Samsung Galaxy A73 5G
    272 27000
    273 Xiaomi Redmi Note 10S
    274 14999
    275 Blackview A60
    276 4999
    277 Blackview A80 Plus
    278 8999
    279 Realme C21
    280 8999
    281 Motorola Moto G51
    282 14999
    283 Xiaomi Mi 10T Lite 5G
    284 15390
    285 OnePlus 9 Pro
    286 46200
    287 Cubot P30
    288 9999
    289 Cubot Note 20
    290 8999
    291 Motorola Moto G30
    292 13999
    293 Asus ROG Phone 5
    294 46100
    295 Tesla Smartphone 3.4
    296 4999
    297 Oppo F11 Pro
    298 16600
    299 Honor View 20
    300 36800
    301 Honor 70
    302 32999
    303 Realme 8i
    304 15999
    305 OnePlus 9
    306 30800
    307 Samsung Galaxy M12
    308 9500
    309 Samsung Galaxy A31
    310 16999
    311 Xiaomi Redmi 9i
    312 8890
    313 OnePlus 7T Pro McLaren Edition
    314 36900
    315 OnePlus 7T
    316 27700
    317 Oppo A9
    318 16600
    319 Nokia 8110 4G
    320 4300
    321 Blackberry Porsche Design P9981
    322 19000
    323 Asus ROG Phone 2
    324 32000
    325 Xiaomi Redmi Note 9T 5G
    326 12290
    327 Motorola Edge 20 Pro
    328 44999
    329 Xiaomi Poco X3 GT
    330 17200
    331 Nokia 105 (2019)
    332 1999
    333 Apple iPhone 12 Mini
    334 34999
    335 Samsung Galaxy Z Fold 2
    336 141499
    337 Samsung Galaxy M32
    338 14100
    339 Alcatel 1S
    340 6390
    341 Doogee X95 Pro
    342 7399
    343 Doogee S58 Pro
    344 12499
    345 Motorola Moto G100
    346 30999
    347 Motorola Moto G200
    348 30999
    349 Realme 9i
    350 16699
    351 Realme 9 Pro+
    352 27999
    353 Realme GT Master
    354 20300
    355 Alcatel 1C
    356 4499
    357 Trevi Flex 50 GT
    358 2199
    359 Huawei Honor 8X Max
    360 12300
    361 Huawei Honor 9X Lite
    362 11000
    363 Oppo A94 5G
    364 15390
    365 MeanIT Veteran I
    366 1599
    367 MeanIT Senior Flip XL
    368 3399
    369 Denver B242
    370 2999
    371 Asus ROG Phone 3
    372 36900
    373 Huawei Y Max
    374 12300
    375 Apple iPhone X
    376 15990
    377 Xiaomi Redmi Note 11T 5G
    378 18500
    379 Motorola G60
    380 17999
    381 OnePlus 8 Pro
    382 40000
    383 Samsung Galaxy M52 5G
    384 21500
    385 Xiaomi Mi 12 Pro 5G
    386 54100
    387 Alcatel 1066G
    388 1190
    389 Google Pixel 6
    390 40000
    391 Wiko View Max
    392 7990
    393 Xiaomi Black Shark 4
    394 29990
    395 Tcl 20L+
    396 15999
    397 Xiaomi Poco F4 GT
    398 47999
    399 Samsung Galaxy A51
    400 18999
    401 Samsung Galaxy A50
    402 18999
    403 Xiaomi Mi A2 Lite
    404 8999
    405 Xiaomi Mi Play
    406 9999
    407 Xiaomi Mi 6
    408 9999
    409 Xiaomi Mi Max 2
    410 6999
    411 Xiaomi Mi A3
    412 8999
    413 Samsung Galaxy A30s
    414 13999
    415 Samsung Galaxy A41
    416 16999
    417 Alcatel Pixi 4 Power Plus
    418 5999
    419 Honor X7
    420 13999
    421 Energizer Energy E20
    422 1999
    423 Energizer Energy E241S
    424 2299
    425 Energizer Hardcase H240S
    426 3999
    427 Energizer Power Max P550S
    428 4999
    429 Huawei Mate 10 Lite
    430 9999
    431 Samsung Galaxy M02s
    432 9500
    433 Xiaomi Mi 10T 5G
    434 24500
    435 Xiaomi Redmi 8A
    436 7400
    437 Huawei P30 Pro
    438 29500
    439 Huawei P40 Lite E
    440 9200
    441 Apple iPhone XS
    442 17890
    443 Apple iPhone XS Max
    444 21590
    445 Nokia C21
    446 8999
    447 Alcatel 1S (2021)
    448 8999
    449 Huawei P50 Pocket
    450 76900
    451 CAT B26
    452 5999
    453 Cubot J9
    454 5299
    455 Samsung Galaxy A21s
    456 11199
    457 Xiaomi Mi A2
    458 13799
    459 Alcatel 1V
    460 4399
    461 Trevi Phablet 5Q2
    462 4599
    463 Energizer Energy E10
    464 1299
    465 Huawei P20
    466 24999
    467 Honor 9 Lite
    468 7999
    469 Huawei Honor 8X
    470 16999
    471 Honor 10 Lite
    472 8499
    473 Wiko Power U20
    474 8999
    475 Wiko Y62
    476 6999
    477 Vivax Pro M1
    478 4499
    479 Vivax Fun S10
    480 3999
    481 Vivax Fun S20
    482 4999
    483 Vivax Point X2
    484 5999
    485 Vivax Fly 5 Lite
    486 7499
    487 Vivax Fly V1
    488 8999
    489 Vivax Point X502
    490 4499
    491 Vivax Point X1
    492 6999
    493 Vivax Fun S1
    494 2999
    495 Oppo A15s
    496 8999
    497 Oppo Reno4 Z 5G
    498 18999
    499 Oppo Reno5 5G
    500 24999
    501 Oppo Reno4 Lite
    502 15999
    503 Oppo Find X5 Lite
    504 24590
    505 Denver B183
    506 1499
    507 Noa N2
    508 7499
    509 Noa Sprint 4G
    510 5499
    511 Samsung Galaxy J7
    512 8999
    513 Manta MS1701
    514 1099
    515 Huawei P10 Lite
    516 11999
    517 Xiaomi Redmi Note 10T 5G
    518 12300
    519 Nokia 6300 4G
    520 4000
    521 Tcl 306
    522 9999
    523 Oppo Find X3 Lite
    524 16000
    525 Wiko Lenny 5
    526 4290
    527 Doro 2404
    528 4499
    529 AllCall Smooth 4G
    530 5499
    531 Realme C25Y
    532 10999
    533 Motorola Moto G41
    534 16999
    535 Honor Magic4 Lite
    536 21999
    537 Huawei Honor 7A
    538 5999
    539 Honor 20 Lite
    540 11299
    541 Panasonic KX-TU155
    542 4699
    543 Vivax Fly 6
    544 5999
    545 Tesla Feature 3
    546 1599
    547 Noa Fresh 4G
    548 2999
    549 Samsung Galaxy F12
    550 9200
    551 Nokia 106
    552 1999
    553 OnePlus Nord
    554 22999
    555 Infinix Hot 12i
    556 9999
    557 Doro 1370
    558 4999
    559 Apple iPhone XR
    560 17290
    561 Doogee X96 Pro
    562 8799
    563 Samsung Galaxy F62
    564 18400
    565 OnePlus 9R
    566 27700
    567 Alcatel 3080G
    568 4499
    569 Motorola Moto E20
    570 8999
    571 AllCall Brother 4G
    572 5499
    573 Realme 8
    574 14999
    575 Huawei Honor 7S
    576 7599
    577 Motorola Moto G50
    578 16999
    579 Nokia 110
    580 2299
    581 Motorola Edge 30
    582 31999
    583 Realme 9 Pro
    584 22499
    585 Nokia 8
    586 14999
    587 OnePlus Nord N100
    588 10499

  • phonelux_scrappers/phones_image_setter.py

    rffd50db r47f4eaf  
    1313    offers = list(json.loads(requests.get('http://localhost:8080/phones/offers/' + phone_id).text))
    1414
    15     offers = list(filter(lambda offer: offer['image_url'] is not None, offers))
     15# remove mobitech condition
     16    offers = list(filter(lambda offer: offer['image_url'] is not None and 'mobitech' not in offer['image_url']
     17                         , offers))
    1618
    1719    image_url = None
    1820
     21    if len(offers) > 1:
     22        image_url = offers[1]['image_url']
     23
    1924    if len(offers) > 0:
    2025        image_url = offers[0]['image_url']
     26
     27    # print('phone :'+str(phone))
     28    # print(image_url)
     29    # print('------------------------------')
    2130
    2231    phone['image_url'] = image_url

  • phonelux_scrappers/phones_model_setter.py

    rffd50db r47f4eaf  
    2323            break
    2424    if not flag:
    25         # requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(offer['id']))
    26         print('delete offer id '+str(offer['id']))
     25        requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(offer['id']))

  • phonelux_scrappers/scrappers/a1_scrapper.py

    rffd50db r47f4eaf  
     1import traceback
    12import unicodedata
    23from datetime import datetime
    … …  
    1819is_validated = False
    1920
    20 # A1 phone offers that are already in database
     21# Call to read the configuration file and connect to database
     22cinfo = config_read.get_databaseconfig("../postgresdb.config")
     23db_connection = psycopg2.connect(
     24    database=cinfo[0],
     25    host=cinfo[1],
     26    user=cinfo[2],
     27    password=cinfo[3]
     28)
     29cur = db_connection.cursor()
    2130
    22 offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/a1').text))
     31try:
     32    # A1 phone offers that are already in database
     33    offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/a1').text))
    2334
    24 database_offers = []
     35    database_offers = []
    2536
    26 for offer in offers:
    27     phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
    28                             offer['ram_memory'],
    29                             offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
    30                             offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
    31                             offer['image_url'],
    32                             offer['offer_url'], offer['last_updated'], offer['is_validated'],
    33                             offer['offer_description'],
    34                             offer['offer_shop_code'])
    35     database_offers.append(phoneOffer)
     37    for offer in offers:
     38        phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
     39                                offer['ram_memory'],
     40                                offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
     41                                offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
     42                                offer['image_url'],
     43                                offer['offer_url'], offer['last_updated'], offer['is_validated'],
     44                                offer['offer_description'],
     45                                offer['offer_shop_code'])
     46        database_offers.append(phoneOffer)
    3647
    37 a1_url = 'https://www.a1.mk/webshop/mk/phones'
     48    a1_url = 'https://www.a1.mk/webshop/mk/phones'
    3849
    39 response1 = requests.get(a1_url)
    40 soup1 = BeautifulSoup(response1.content, 'html.parser')
     50    response1 = requests.get(a1_url)
     51    soup1 = BeautifulSoup(response1.content, 'html.parser')
    4152
    42 phones = soup1.find('main', {'class', 'gsm-advisor-grid phones'}).find('div', {'class', 'd-flex'}) \
    43     .find_all('div', {'class', 'dvc-idtfr by4'})
     53    phones = soup1.find('main', {'class', 'gsm-advisor-grid phones'}).find('div', {'class', 'd-flex'}) \
     54        .find_all('div', {'class', 'dvc-idtfr by4'})
    4455
    45 new_offers = []
     56    new_offers = []
    4657
    47 for phone in phones:
    48     brand = phone.get('data-brand').strip()
    49     offer_name = brand + " " + phone.get('data-model').strip()
     58    for phone in phones:
     59        brand = phone.get('data-brand').strip()
     60        offer_name = brand + " " + phone.get('data-model').strip()
    5061
    51     # if brand not in offer_name:
    52     #     offer_name = brand+" "+offer_name
     62        # if brand not in offer_name:
     63        #     offer_name = brand+" "+offer_name
    5364
    54     offer_shop_code = phone.get('data-productid').strip()
    55     offer_url = phone.find('a', {'class', 'device-link'}).get('href')
    56     image_url = phone.get('data-image')
     65        offer_shop_code = phone.get('data-productid').strip()
     66        offer_url = phone.find('a', {'class', 'device-link'}).get('href')
     67        image_url = phone.get('data-image')
    5768
    58     response2 = requests.get(offer_url)
    59     soup2 = BeautifulSoup(response2.content, 'html.parser')
     69        response2 = requests.get(offer_url)
     70        soup2 = BeautifulSoup(response2.content, 'html.parser')
    6071
    61     temp_prices = soup2.find('div', {'class': 'ured-tabs-content'}) \
    62         .find('div', {'class': 'cenovnik-secondary d-flex justify-content-between'}).find_all('div')
     72        temp_prices = soup2.find('div', {'class': 'ured-tabs-content'}) \
     73            .find('div', {'class': 'cenovnik-secondary d-flex justify-content-between'}).find_all('div')
    6374
    64     # offer price
    65     price = None
    66     for temp_price in temp_prices:
    67         if 'Цена само за уред' in temp_price.get_text().strip():
    68             price = int(temp_price.get_text().replace('Цена само за уред', '')
    69                         .replace('Одбери', '').replace('денари', '').replace('.', '').strip())
     75        # offer price
     76        price = None
     77        for temp_price in temp_prices:
     78            if 'Цена само за уред' in temp_price.get_text().strip():
     79                price = int(temp_price.get_text().replace('Цена само за уред', '')
     80                            .replace('Одбери', '').replace('денари', '').replace('.', '').strip())
    7081
    71     colors_section = soup2.find('div', {'id': 'hero'}).find('div', {'class': 'widget'}).find_all('label')
     82        colors_section = soup2.find('div', {'id': 'hero'}).find('div', {'class': 'widget'}).find_all('label')
    7283
    73     temp_colors = []
    74     for color_section in colors_section:
    75         temp_colors.append(color_section.get('data-content'))
     84        temp_colors = []
     85        for color_section in colors_section:
     86            temp_colors.append(color_section.get('data-content'))
    7687
    77     color = ','.join(temp_colors)  # colors available for the offer
     88        color = ','.join(temp_colors)  # colors available for the offer
    7889
    79     phone_description = soup2.find('div', {'class': 'desc section'}).find('p').get_text().strip()
     90        phone_description = soup2.find('div', {'class': 'desc section'}).find('p').get_text().strip()
    8091
    81     table_rows = soup2.find('table', {'class': 'table karakteristiki'}).find_all('tr')
     92        table_rows = soup2.find('table', {'class': 'table karakteristiki'}).find_all('tr')
    8293
    83     back_camera = None
    84     operating_system = None
    85     cpu = None
    86     rom_memory = None
    87     ram_memory = None
    88     battery = None
    89     front_camera = None
    90     chipset = None
    91     offer_description = None
     94        back_camera = None
     95        operating_system = None
     96        cpu = None
     97        rom_memory = None
     98        ram_memory = None
     99        battery = None
     100        front_camera = None
     101        chipset = None
     102        offer_description = None
    92103
    93     for row in table_rows:
    94         if 'Камера' in row.get_text().strip():
    95             back_camera = row.get_text().replace('Камера', '').strip()
     104        for row in table_rows:
     105            if 'Камера' in row.get_text().strip():
     106                back_camera = row.get_text().replace('Камера', '').strip()
    96107
    97         if 'Оперативен систем' in row.get_text().strip():
    98             operating_system = row.get_text().replace('Оперативен систем', '').strip()
     108            if 'Оперативен систем' in row.get_text().strip():
     109                operating_system = row.get_text().replace('Оперативен систем', '').strip()
    99110
    100         if 'CPU' in row.get_text().strip():
    101             cpu = row.get_text().replace('CPU', '').strip()
     111            if 'CPU' in row.get_text().strip():
     112                cpu = row.get_text().replace('CPU', '').strip()
    102113
    103         if 'Вградена меморија' in row.get_text().strip():
    104             rom_memory = row.get_text().replace('Вградена меморија', '').strip()
     114            if 'Вградена меморија' in row.get_text().strip():
     115                rom_memory = row.get_text().replace('Вградена меморија', '').strip()
    105116
    106         if 'RAM меморија' in row.get_text().strip():
    107             ram_memory = row.get_text().replace('RAM меморија', '').strip()
     117            if 'RAM меморија' in row.get_text().strip():
     118                ram_memory = row.get_text().replace('RAM меморија', '').strip()
    108119
    109         if 'Батерија' in row.get_text().strip():
    110             battery = row.get_text().replace('Батерија', '').strip()
     120            if 'Батерија' in row.get_text().strip():
     121                battery = row.get_text().replace('Батерија', '').strip()
    111122
    112         if 'Предна камера' in row.get_text().strip():
    113             front_camera = row.get_text().replace('Предна камера', '').strip()
     123            if 'Предна камера' in row.get_text().strip():
     124                front_camera = row.get_text().replace('Предна камера', '').strip()
    114125
    115     new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
    116                                  color, front_camera, back_camera, chipset, battery, operating_system, cpu, image_url,
    117                                  offer_url, last_updated, is_validated, offer_description, offer_shop_code))
     126        new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
     127                                     color, front_camera, back_camera, chipset, battery, operating_system, cpu,
     128                                     image_url,
     129                                     offer_url, last_updated, is_validated, offer_description, offer_shop_code))
    118130
    119 for new_offer in new_offers:
    120     flag = False
    121     flag_price = False
    122     offer_id = None
     131    for new_offer in new_offers:
     132        flag = False
     133        flag_price = False
     134        offer_id = None
     135
     136        for old_offer in database_offers:
     137
     138            if new_offer.offer_shop_code == old_offer.offer_shop_code:
     139                flag = True
     140                if new_offer.price != old_offer.price:
     141                    flag_price = True
     142                    offer_id = old_offer.offer_id
     143
     144        if flag:
     145            # print('ALREADY IN DATABASE')
     146            # print(new_offer)
     147            # if it's already in database, check PRICE and if it's changed, change it !!!!!!
     148            if flag_price:
     149                print('PRICE CHANGED!')  # CHANGE PRICE
     150                print('offer id: ' + str(offer_id))
     151                headers = {'Content-type': 'application/json'}
     152                requests.put(
     153                    'http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
     154                    headers=headers)
     155        else:
     156            print('ADDED')  # ADD OFFER
     157            print(new_offer)
     158            headers = {'Content-type': 'application/json'}
     159            requests.post('http://localhost:8080/phoneoffer/addoffer', headers=headers,
     160                          data=json.dumps(new_offer.__dict__,
     161                                          default=str))
     162
     163    print('------------------------------------')
    123164
    124165    for old_offer in database_offers:
     166        flag = False
     167        for new_offer in new_offers:
     168            if old_offer.offer_shop_code == new_offer.offer_shop_code:
     169                flag = True
    125170
    126         if new_offer.offer_shop_code == old_offer.offer_shop_code:
    127             flag = True
    128             if new_offer.price != old_offer.price:
    129                 flag_price = True
    130                 offer_id = old_offer.offer_id
    131 
    132     if flag:
    133         # print('ALREADY IN DATABASE')
    134         # print(new_offer)
    135         # if it's already in database, check PRICE and if it's changed, change it !!!!!!
    136         if flag_price:
    137             print('PRICE CHANGED!')  # CHANGE PRICE
    138             print('offer id: ' + str(offer_id))
    139             headers = {'Content-type': 'application/json'}
    140             requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
    141                          headers=headers)
    142     else:
    143         print('ADDED')  # ADD OFFER
    144         print(new_offer)
    145         headers = {'Content-type': 'application/json'}
    146         requests.post('http://localhost:8080/phoneoffer/addoffer', headers=headers, data=json.dumps(new_offer.__dict__,
    147                                                                                                     default=str))
    148 
    149 print('------------------------------------')
    150 
    151 for old_offer in database_offers:
    152     flag = False
    153     for new_offer in new_offers:
    154         if old_offer.offer_shop_code == new_offer.offer_shop_code:
    155             flag = True
    156 
    157     if not flag:
    158         print('OFFER DELETED')
    159         print(old_offer)
    160         # DELETE OFFER
    161         requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     171        if not flag:
     172            print('OFFER DELETED')
     173            print(old_offer)
     174            # DELETE OFFER
     175            requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     176except Exception:
     177    traceback.print_exc()
     178    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     179                    ' VALUES (%s, %s, %s);'
     180    insert_value = (offer_shop, last_updated, 'failed')
     181    cur.execute(insert_script, insert_value)
     182    db_connection.commit()
     183    cur.close()
     184    db_connection.close()
     185else:
     186    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     187                    ' VALUES (%s, %s, %s);'
     188    insert_value = (offer_shop, last_updated, 'success')
     189    cur.execute(insert_script, insert_value)
     190    db_connection.commit()
     191    cur.close()
     192    db_connection.close()

  • phonelux_scrappers/scrappers/akcija_scrapper.py

    rffd50db r47f4eaf  
    11import json
     2import traceback
    23from datetime import datetime
    34
    … …  
    1819is_validated = False
    1920
    20 # Akcija phone offers that are already in database
     21# Call to read the configuration file and connect to database
     22cinfo = config_read.get_databaseconfig("../postgresdb.config")
     23db_connection = psycopg2.connect(
     24    database=cinfo[0],
     25    host=cinfo[1],
     26    user=cinfo[2],
     27    password=cinfo[3]
     28)
     29cur = db_connection.cursor()
    2130
    22 offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/akcija').text))
     31try:
     32    # Akcija phone offers that are already in database
     33    offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/akcija').text))
    2334
    24 database_offers = []
     35    database_offers = []
    2536
    26 for offer in offers:
    27     phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
    28                             offer['ram_memory'],
    29                             offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
    30                             offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
    31                             offer['image_url'],
    32                             offer['offer_url'], offer['last_updated'], offer['is_validated'],
    33                             offer['offer_description'],
    34                             offer['offer_shop_code'])
    35     database_offers.append(phoneOffer)
     37    for offer in offers:
     38        phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
     39                                offer['ram_memory'],
     40                                offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
     41                                offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
     42                                offer['image_url'],
     43                                offer['offer_url'], offer['last_updated'], offer['is_validated'],
     44                                offer['offer_description'],
     45                                offer['offer_shop_code'])
     46        database_offers.append(phoneOffer)
    3647
    37 new_offers = []
     48    new_offers = []
    3849
    39 i = 0
    40 while i <= 20:
    41     akcija_url = "https://akcija.com.mk/listing/" + str(i) + "?category=mobilnitelefoni"
    42     response1 = requests.get(akcija_url)
    43     response1.encoding = 'utf-8'
    44     soup1 = BeautifulSoup(response1.text, 'html.parser')
     50    i = 0
     51    while i <= 20:
     52        akcija_url = "https://akcija.com.mk/listing/" + str(i) + "?category=mobilnitelefoni"
     53        response1 = requests.get(akcija_url)
     54        response1.encoding = 'utf-8'
     55        soup1 = BeautifulSoup(response1.text, 'html.parser')
    4556
    46     phones = soup1.find_all('div', {'class', 'product-item__body pb-xl-2'})
     57        phones = soup1.find_all('div', {'class', 'product-item__body pb-xl-2'})
    4758
    48     for phone in phones:
    49         offer_name = phone.find('h5', {'class': 'mb-1 product-item__title'}).find('a') \
    50             .get_text().replace('Паметен телефон', '').strip()
    51         brand = offer_name.split(' ')[0]
     59        for phone in phones:
     60            offer_name = phone.find('h5', {'class': 'mb-1 product-item__title'}).find('a') \
     61                .get_text().replace('Паметен телефон', '').strip()
     62            brand = offer_name.split(' ')[0]
    5263
    53         if brand not in offer_name:
    54             offer_name = brand + " " + offer_name
     64            if brand not in offer_name:
     65                offer_name = brand + " " + offer_name
    5566
    56         offer_url = phone.find('h5', {'class': 'mb-1 product-item__title'}).find('a').get('href')
    57         image_url = phone.find('div', {'class', 'mb-2'}).find('img').get('src')
    58         price = int(phone.find('div', {'class', 'flex-center-between mb-1 pt-xl-2'}) \
    59                     .find('ins').get_text().split(' ')[0].strip())
     67            offer_url = phone.find('h5', {'class': 'mb-1 product-item__title'}).find('a').get('href')
     68            image_url = phone.find('div', {'class', 'mb-2'}).find('img').get('src')
     69            price = int(phone.find('div', {'class', 'flex-center-between mb-1 pt-xl-2'}) \
     70                        .find('ins').get_text().split(' ')[0].strip())
    6071
    61         response2 = requests.get(offer_url)
    62         response2.encoding = 'utf-8'
    63         soup2 = BeautifulSoup(response2.text, 'html.parser')
     72            response2 = requests.get(offer_url)
     73            response2.encoding = 'utf-8'
     74            soup2 = BeautifulSoup(response2.text, 'html.parser')
    6475
    65         back_camera = None
    66         operating_system = None
    67         chipset = None
    68         battery = None
    69         ram_memory = None
    70         rom_memory = None
    71         cpu = None
    72         front_camera = None
    73         color = None
    74         offer_shop_code = None
     76            back_camera = None
     77            operating_system = None
     78            chipset = None
     79            battery = None
     80            ram_memory = None
     81            rom_memory = None
     82            cpu = None
     83            front_camera = None
     84            color = None
     85            offer_shop_code = None
    7586
    76         specifications = soup2.find('main', {'id': 'content'}) \
    77             .find_all('div', {'class', 'container'})[1].find('div', {'class', 'mb-14'}) \
    78             .find('div', {'class', 'col-md-6 col-lg-4 col-xl-4 mb-md-6 mb-lg-0'}).find_all('p')
     87            specifications = soup2.find('main', {'id': 'content'}) \
     88                .find_all('div', {'class', 'container'})[1].find('div', {'class', 'mb-14'}) \
     89                .find('div', {'class', 'col-md-6 col-lg-4 col-xl-4 mb-md-6 mb-lg-0'}).find_all('p')
    7990
    80         offer_description = ''
    81         for specification in specifications:
    82             if 'Код за нарачка' in str(specification.get_text(separator='\n').replace('NBSP', '').strip()):
    83                 continue
    84             offer_description += unicodedata.normalize('NFKD',
    85                                                        str(specification.get_text(separator='\n').strip())) + "\n"
     91            offer_description = ''
     92            for specification in specifications:
     93                if 'Код за нарачка' in str(specification.get_text(separator='\n').replace('NBSP', '').strip()):
     94                    continue
     95                offer_description += unicodedata.normalize('NFKD',
     96                                                           str(specification.get_text(separator='\n').strip())) + "\n"
    8697
    87         new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
    88                                      color, front_camera, back_camera, chipset, battery, operating_system, cpu,
    89                                      image_url,
    90                                      offer_url, last_updated, is_validated, offer_description, offer_shop_code))
    91     i += 20
     98            new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
     99                                         color, front_camera, back_camera, chipset, battery, operating_system, cpu,
     100                                         image_url,
     101                                         offer_url, last_updated, is_validated, offer_description, offer_shop_code))
     102        i += 20
    92103
    93 for new_offer in new_offers:
    94     flag = False
    95     flag_price = False
    96     offer_id = None
     104    for new_offer in new_offers:
     105        flag = False
     106        flag_price = False
     107        offer_id = None
     108
     109        for old_offer in database_offers:
     110
     111            if new_offer.offer_name == old_offer.offer_name:
     112                flag = True
     113                if new_offer.price != old_offer.price:
     114                    flag_price = True
     115                    offer_id = old_offer.offer_id
     116
     117        if flag:
     118            # print('ALREADY IN DATABASE')
     119            # print(new_offer)
     120            # if it's already in database, check PRICE and if it's changed, change it !!!!!!
     121            if flag_price:
     122                print('PRICE CHANGED!')  # CHANGE PRICE
     123                print('offer id: ' + str(offer_id))
     124                headers = {'Content-type': 'application/json'}
     125                requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
     126                             headers=headers)
     127        else:
     128            print('ADDED')  # ADD OFFER
     129            print(new_offer)
     130            headers = {'Content-type': 'application/json'}
     131            requests.post('http://localhost:8080/phoneoffer/addoffer',
     132                          headers=headers, data=json.dumps(new_offer.__dict__, default=str))
     133
     134    print('------------------------------------')
    97135
    98136    for old_offer in database_offers:
     137        flag = False
     138        for new_offer in new_offers:
     139            if old_offer.offer_name == new_offer.offer_name:
     140                flag = True
    99141
    100         if new_offer.offer_name == old_offer.offer_name:
    101             flag = True
    102             if new_offer.price != old_offer.price:
    103                 flag_price = True
    104                 offer_id = old_offer.offer_id
     142        if not flag:
     143            print('OFFER DELETED')
     144            print(old_offer)
     145            # DELETE OFFER
     146            requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     147except Exception:
     148    traceback.print_exc()
     149    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     150                    ' VALUES (%s, %s, %s);'
     151    insert_value = (offer_shop, last_updated, 'failed')
     152    cur.execute(insert_script, insert_value)
     153    db_connection.commit()
     154    cur.close()
     155    db_connection.close()
     156else:
     157    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     158                    ' VALUES (%s, %s, %s);'
     159    insert_value = (offer_shop, last_updated, 'success')
     160    cur.execute(insert_script, insert_value)
     161    db_connection.commit()
     162    cur.close()
     163    db_connection.close()
    105164
    106     if flag:
    107         # print('ALREADY IN DATABASE')
    108         # print(new_offer)
    109         # if it's already in database, check PRICE and if it's changed, change it !!!!!!
    110         if flag_price:
    111             print('PRICE CHANGED!')  # CHANGE PRICE
    112             print('offer id: ' + str(offer_id))
    113             headers = {'Content-type': 'application/json'}
    114             requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
    115                          headers=headers)
    116     else:
    117         print('ADDED')  # ADD OFFER
    118         print(new_offer)
    119         headers = {'Content-type': 'application/json'}
    120         requests.post('http://localhost:8080/phoneoffer/addoffer',
    121                       headers=headers, data=json.dumps(new_offer.__dict__, default=str))
    122 
    123 print('------------------------------------')
    124 
    125 for old_offer in database_offers:
    126     flag = False
    127     for new_offer in new_offers:
    128         if old_offer.offer_name == new_offer.offer_name:
    129             flag = True
    130 
    131     if not flag:
    132         print('OFFER DELETED')
    133         print(old_offer)
    134         # DELETE OFFER
    135         requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))

  • phonelux_scrappers/scrappers/handy_scrapper.py

    rffd50db r47f4eaf  
    11import json
     2import traceback
    23import unicodedata
    34from datetime import datetime
    … …  
    2021is_validated = False
    2122
    22 # Handy phone offers that are already in database
    23 offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/handy').text))
     23# Call to read the configuration file and connect to database
     24cinfo = config_read.get_databaseconfig("../postgresdb.config")
     25db_connection = psycopg2.connect(
     26    database=cinfo[0],
     27    host=cinfo[1],
     28    user=cinfo[2],
     29    password=cinfo[3]
     30)
     31cur = db_connection.cursor()
    2432
    25 database_offers = []
     33try:
     34    # Handy phone offers that are already in database
     35    offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/handy').text))
    2636
    27 for offer in offers:
    28     phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
    29                             offer['ram_memory'],
    30                             offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
    31                             offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
    32                             offer['image_url'],
    33                             offer['offer_url'], offer['last_updated'], offer['is_validated'],
    34                             offer['offer_description'],
    35                             offer['offer_shop_code'])
    36     database_offers.append(phoneOffer)
     37    database_offers = []
    3738
    38 new_offers = []
     39    for offer in offers:
     40        phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
     41                                offer['ram_memory'],
     42                                offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
     43                                offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
     44                                offer['image_url'],
     45                                offer['offer_url'], offer['last_updated'], offer['is_validated'],
     46                                offer['offer_description'],
     47                                offer['offer_shop_code'])
     48        database_offers.append(phoneOffer)
    3949
    40 handy_url = 'https://www.handy.mk/telefoni?page=6'
     50    new_offers = []
    4151
    42 response1 = requests.get(handy_url)
    43 soup1 = BeautifulSoup(response1.content, 'html.parser')
     52    handy_url = 'https://www.handy.mk/telefoni?page=6'
    4453
    45 phones = soup1.find_all('li', {'data-hook': 'product-list-grid-item'})
     54    response1 = requests.get(handy_url)
     55    soup1 = BeautifulSoup(response1.content, 'html.parser')
    4656
    47 for phone in phones:
    48     offer_url = phone.find('a').get('href')
    49     offer_name = phone.find('div', {'data-hook': 'not-image-container'})\
    50         .find('h3', {'data-hook': 'product-item-name'}).get_text().strip()
    51     brand = offer_name.split(' ')[0].capitalize()
    52     price = int(float(phone.find('div', {'data-hook': 'not-image-container'}).find('div', {'data-hook': "product-item-product-details"})\
    53         .find('span', {'data-hook': 'product-item-price-to-pay'}).get_text().strip().replace('ден', '').replace('.', '').replace(',', '.')))
     57    phones = soup1.find_all('li', {'data-hook': 'product-list-grid-item'})
    5458
    55     response2 = requests.get(offer_url)
    56     soup2 = BeautifulSoup(response2.text, 'html.parser')
     59    for phone in phones:
     60        offer_url = phone.find('a').get('href')
     61        offer_name = phone.find('div', {'data-hook': 'not-image-container'})\
     62            .find('h3', {'data-hook': 'product-item-name'}).get_text().strip()
     63        brand = offer_name.split(' ')[0].capitalize()
     64        price = int(float(phone.find('div', {'data-hook': 'not-image-container'}).find('div', {'data-hook': "product-item-product-details"})\
     65            .find('span', {'data-hook': 'product-item-price-to-pay'}).get_text().strip().replace('ден', '').replace('.', '').replace(',', '.')))
    5766
    58     back_camera = None
    59     operating_system = None
    60     chipset = None
    61     battery = None
    62     ram_memory = None
    63     rom_memory = None
    64     cpu = None
    65     front_camera = None
    66     offer_shop_code = None
    67     color = None
    68     image_url = None
     67        response2 = requests.get(offer_url)
     68        soup2 = BeautifulSoup(response2.text, 'html.parser')
    6969
    70     color_section = soup2.find('section', {'data-hook': 'product-colors-title-section'})
    71     if color_section is not None:
    72         temp_colors = color_section.find('fieldset', {'class': 'ColorPickerbase3548966286__container'})\
    73             .find_all('input', {'type': 'radio'})
    74         colors_list = []
    75         for temp_color in temp_colors:
    76             colors_list.append(temp_color.get('aria-label'))
    77         color = ','.join(colors_list)
     70        back_camera = None
     71        operating_system = None
     72        chipset = None
     73        battery = None
     74        ram_memory = None
     75        rom_memory = None
     76        cpu = None
     77        front_camera = None
     78        offer_shop_code = None
     79        color = None
     80        image_url = None
    7881
    79     rows = soup2.find('div', {'data-hook': 'info-section-description'}).find_all('li')
     82        color_section = soup2.find('section', {'data-hook': 'product-colors-title-section'})
     83        if color_section is not None:
     84            temp_colors = color_section.find('fieldset', {'class': 'ColorPickerbase3548966286__container'})\
     85                .find_all('input', {'type': 'radio'})
     86            colors_list = []
     87            for temp_color in temp_colors:
     88                colors_list.append(temp_color.get('aria-label'))
     89            color = ','.join(colors_list)
    8090
    81     if len(rows) == 0:
    82         rows = soup2.find('div', {'data-hook': 'info-section-description'}).find_all('tr')
     91        rows = soup2.find('div', {'data-hook': 'info-section-description'}).find_all('li')
    8392
    84     specifications = []
     93        if len(rows) == 0:
     94            rows = soup2.find('div', {'data-hook': 'info-section-description'}).find_all('tr')
    8595
    86     for row in rows:
    87         specifications.append(unicodedata.normalize('NFKD', row.get_text().strip()))
     96        specifications = []
    8897
    89     offer_description = '\n'.join(specifications)
     98        for row in rows:
     99            specifications.append(unicodedata.normalize('NFKD', row.get_text().strip()))
    90100
    91     new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
    92                                  color, front_camera, back_camera, chipset, battery, operating_system, cpu,
    93                                  image_url,
    94                                  offer_url, last_updated, is_validated, offer_description, offer_shop_code))
     101        offer_description = '\n'.join(specifications)
    95102
    96 for new_offer in new_offers:
    97     flag = False
    98     flag_price = False
    99     offer_id = None
     103        new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
     104                                     color, front_camera, back_camera, chipset, battery, operating_system, cpu,
     105                                     image_url,
     106                                     offer_url, last_updated, is_validated, offer_description, offer_shop_code))
     107
     108    for new_offer in new_offers:
     109        flag = False
     110        flag_price = False
     111        offer_id = None
     112
     113        for old_offer in database_offers:
     114
     115            if new_offer.offer_name == old_offer.offer_name:
     116                flag = True
     117                if new_offer.price != old_offer.price:
     118                    flag_price = True
     119                    offer_id = old_offer.offer_id
     120
     121        if flag:
     122            # print('ALREADY IN DATABASE')
     123            # print(new_offer)
     124            # if it's already in database, check PRICE and if it's changed, change it !!!!!!
     125            if flag_price:
     126                print('PRICE CHANGED!')  # CHANGE PRICE
     127                print('offer id: ' + str(offer_id))
     128                headers = {'Content-type': 'application/json'}
     129                requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
     130                             headers=headers)
     131        else:
     132            print('ADDED')  # ADD OFFER
     133            print(new_offer)
     134            headers = {'Content-type': 'application/json'}
     135            requests.post('http://localhost:8080/phoneoffer/addoffer',
     136                          headers=headers, data=json.dumps(new_offer.__dict__, default=str))
     137
     138    print('------------------------------------')
    100139
    101140    for old_offer in database_offers:
     141        flag = False
     142        for new_offer in new_offers:
     143            if old_offer.offer_name == new_offer.offer_name:
     144                flag = True
    102145
    103         if new_offer.offer_name == old_offer.offer_name:
    104             flag = True
    105             if new_offer.price != old_offer.price:
    106                 flag_price = True
    107                 offer_id = old_offer.offer_id
    108 
    109     if flag:
    110         # print('ALREADY IN DATABASE')
    111         # print(new_offer)
    112         # if it's already in database, check PRICE and if it's changed, change it !!!!!!
    113         if flag_price:
    114             print('PRICE CHANGED!')  # CHANGE PRICE
    115             print('offer id: ' + str(offer_id))
    116             headers = {'Content-type': 'application/json'}
    117             requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
    118                          headers=headers)
    119     else:
    120         print('ADDED')  # ADD OFFER
    121         print(new_offer)
    122         headers = {'Content-type': 'application/json'}
    123         requests.post('http://localhost:8080/phoneoffer/addoffer',
    124                       headers=headers, data=json.dumps(new_offer.__dict__, default=str))
    125 
    126 print('------------------------------------')
    127 
    128 for old_offer in database_offers:
    129     flag = False
    130     for new_offer in new_offers:
    131         if old_offer.offer_name == new_offer.offer_name:
    132             flag = True
    133 
    134     if not flag:
    135         print('OFFER DELETED')
    136         print(old_offer)
    137         # DELETE OFFER
    138         requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     146        if not flag:
     147            print('OFFER DELETED')
     148            print(old_offer)
     149            # DELETE OFFER
     150            requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     151except Exception:
     152    traceback.print_exc()
     153    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     154                    ' VALUES (%s, %s, %s);'
     155    insert_value = (offer_shop, last_updated, 'failed')
     156    cur.execute(insert_script, insert_value)
     157    db_connection.commit()
     158    cur.close()
     159    db_connection.close()
     160else:
     161    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     162                    ' VALUES (%s, %s, %s);'
     163    insert_value = (offer_shop, last_updated, 'success')
     164    cur.execute(insert_script, insert_value)
     165    db_connection.commit()
     166    cur.close()
     167    db_connection.close()
    139168
    140169
     170

  • phonelux_scrappers/scrappers/ledikom_scrapper.py

    rffd50db r47f4eaf  
    11import json
     2import traceback
    23import unicodedata
    34from datetime import datetime
    … …  
    1920is_validated = False
    2021
    21 # Ledikom phone offers that are already in database
    22 
    23 offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/ledikom').text))
    24 
    25 database_offers = []
    26 
    27 for offer in offers:
    28     phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
    29                             offer['ram_memory'],
    30                             offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
    31                             offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
    32                             offer['image_url'],
    33                             offer['offer_url'], offer['last_updated'], offer['is_validated'],
    34                             offer['offer_description'],
    35                             offer['offer_shop_code'])
    36     database_offers.append(phoneOffer)
    37 
    38 new_offers = []
    39 
    40 ledikom_phone_urls = [
    41     'https://ledikom.mk/c/416/uredi/apple/iphone?limit=96',
    42     'https://ledikom.mk/c/421/uredi/samsung/telefoni?limit=96',
    43     'https://ledikom.mk/c/424/mobilni-telefoni/xiaomi/telefoni?limit=96',
    44     'https://ledikom.mk/c/430/uredi/huawei/telefoni?limit=96',
    45     'https://ledikom.mk/c/441/uredi/oneplus/telefoni?limit=96',
    46     'https://ledikom.mk/c/413/uredi/google/telefoni?limit=96',
    47     'https://ledikom.mk/c/411/uredi/honor/telefoni?limit=96',
    48     'https://ledikom.mk/c/460/uredi/nokia/telefoni?limit=96',
    49     'https://ledikom.mk/c/461/uredi/asus/telefoni?limit=96',
    50     'https://ledikom.mk/c/488/proizvodi/oppo/telefoni?limit=96'
    51 ]
    52 
    53 for ledikom_url in ledikom_phone_urls:
    54 
    55     # selenium is used because of the dynamic content of the page
    56     driver1 = webdriver.Safari(executable_path='/usr/bin/safaridriver')
    57     driver1.get(ledikom_url)
    58     ledikom_html = driver1.page_source
    59 
    60     # closing the driver so the safari instance can pair with another webdriver session
    61     driver1.close()
    62 
    63     soup1 = BeautifulSoup(ledikom_html, 'html.parser')
    64 
    65     phones = soup1.find('div', {'id': 'content'}) \
    66         .find('div', {'class': 'container'}).find('div', {'class': 'row'}).find('div', {'class': 'item-display'}) \
    67         .find_all('div', {'class': 'item-in-grid'})
    68 
    69     if len(phones) == 0:
    70         continue
    71 
    72     for phone in phones:
    73         offer_url = 'https://ledikom.mk' + phone.find('a').get('href')
    74         image_url = phone.find('a').find('img').get('src')
    75         temp_offer_name = phone.find('div', {'class': 'item-name'}).find('a').get_text().strip()
    76         offer_name = ' '.join(temp_offer_name.split())
    77         brand = offer_name.split(' ')[0]
    78         price = int(phone.find('span', {'class': 'price'}).get_text().replace('ден.', '')
    79                     .replace('ден', '')
    80                     .replace('.', '').strip())
    81 
     22# Call to read the configuration file and connect to database
     23cinfo = config_read.get_databaseconfig("../postgresdb.config")
     24db_connection = psycopg2.connect(
     25    database=cinfo[0],
     26    host=cinfo[1],
     27    user=cinfo[2],
     28    password=cinfo[3]
     29)
     30cur = db_connection.cursor()
     31
     32try:
     33    # Ledikom phone offers that are already in database
     34    offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/ledikom').text))
     35
     36    database_offers = []
     37
     38    for offer in offers:
     39        phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
     40                                offer['ram_memory'],
     41                                offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
     42                                offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
     43                                offer['image_url'],
     44                                offer['offer_url'], offer['last_updated'], offer['is_validated'],
     45                                offer['offer_description'],
     46                                offer['offer_shop_code'])
     47        database_offers.append(phoneOffer)
     48
     49    new_offers = []
     50
     51    ledikom_phone_urls = [
     52        'https://ledikom.mk/c/416/uredi/apple/iphone?limit=96',
     53        'https://ledikom.mk/c/421/uredi/samsung/telefoni?limit=96',
     54        'https://ledikom.mk/c/424/mobilni-telefoni/xiaomi/telefoni?limit=96',
     55        'https://ledikom.mk/c/430/uredi/huawei/telefoni?limit=96',
     56        'https://ledikom.mk/c/441/uredi/oneplus/telefoni?limit=96',
     57        'https://ledikom.mk/c/413/uredi/google/telefoni?limit=96',
     58        'https://ledikom.mk/c/411/uredi/honor/telefoni?limit=96',
     59        'https://ledikom.mk/c/460/uredi/nokia/telefoni?limit=96',
     60        'https://ledikom.mk/c/461/uredi/asus/telefoni?limit=96',
     61        'https://ledikom.mk/c/488/proizvodi/oppo/telefoni?limit=96'
     62    ]
     63
     64    for ledikom_url in ledikom_phone_urls:
     65
     66        # selenium is used because of the dynamic content of the page
    8267        driver1 = webdriver.Safari(executable_path='/usr/bin/safaridriver')
    83         driver1.get(offer_url)
    84         # getting offer page html
    85         offer_html = driver1.page_source
     68        driver1.get(ledikom_url)
     69        ledikom_html = driver1.page_source
     70
     71        # closing the driver so the safari instance can pair with another webdriver session
    8672        driver1.close()
    8773
    88         soup2 = BeautifulSoup(offer_html, 'html.parser')
    89 
    90         specifications = soup2.find('div', {'id': 'content'}).find('section', {'class': 'padding-section'}) \
    91             .find_all('div', {'class': 'container'})[1].find('div', {'class': 'col-md-7'}) \
    92             .find_all('div', {'class': 'row'})
    93 
    94         color = None
    95         rom_memory = None
    96         ram_memory = None
    97         back_camera = None
    98         operating_system = None
    99         chipset = None
    100         battery = None
    101         cpu = None
    102         front_camera = None
    103         offer_shop_code = None
    104         offer_description = None
    105 
    106         if len(specifications) != 0:
    107             colors_tags = specifications[0].find('div', {'class': 'col-md-12 col-xs-12'}).find_all('a')
    108             temp_colors = []
    109             for color_tag in colors_tags:
    110                 temp_colors.append(color_tag.get_text().strip())
    111             color = ','.join(temp_colors)
    112 
    113         if len(specifications) >= 2:
    114             temp_rom = specifications[1].find('div', {'class': 'col-md-12 col-xs-12'}).find_all('a')
    115             rom_list = []
    116             for rom in temp_rom:
    117                 rom_list.append(rom.get('title'))
    118             rom_memory = ','.join(rom_list)
    119 
    120         if len(specifications) >= 3:
    121             temp_ram = specifications[2].find('div', {'class': 'col-md-12 col-xs-12'}).find_all('a')
    122             ram_list = []
    123             for ram in temp_ram:
    124                 ram_list.append(ram.get('title'))
    125 
    126             ram_memory = ','.join(ram_list)
    127 
    128         if 'Xiaomi' in brand:
    129             temp = color
    130             color = rom_memory
    131             rom_memory = temp
    132 
    133             temp = ram_memory
    134             ram_memory = color
    135             color = temp
    136 
    137         new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
    138                                      color, front_camera, back_camera, chipset, battery, operating_system, cpu,
    139                                      image_url,
    140                                      offer_url, last_updated, is_validated, offer_description, offer_shop_code))
    141 
    142 for new_offer in new_offers:
    143     flag = False
    144     flag_price = False
    145     offer_id = None
     74        soup1 = BeautifulSoup(ledikom_html, 'html.parser')
     75
     76        phones = soup1.find('div', {'id': 'content'}) \
     77            .find('div', {'class': 'container'}).find('div', {'class': 'row'}).find('div', {'class': 'item-display'}) \
     78            .find_all('div', {'class': 'item-in-grid'})
     79
     80        if len(phones) == 0:
     81            continue
     82
     83        for phone in phones:
     84            offer_url = 'https://ledikom.mk' + phone.find('a').get('href')
     85            image_url = phone.find('a').find('img').get('src')
     86            temp_offer_name = phone.find('div', {'class': 'item-name'}).find('a').get_text().strip()
     87            offer_name = ' '.join(temp_offer_name.split())
     88            brand = offer_name.split(' ')[0]
     89            price = int(phone.find('span', {'class': 'price'}).get_text().replace('ден.', '')
     90                        .replace('ден', '')
     91                        .replace('.', '').strip())
     92
     93            driver1 = webdriver.Safari(executable_path='/usr/bin/safaridriver')
     94            driver1.get(offer_url)
     95            # getting offer page html
     96            offer_html = driver1.page_source
     97            driver1.close()
     98
     99            soup2 = BeautifulSoup(offer_html, 'html.parser')
     100
     101            specifications = soup2.find('div', {'id': 'content'}).find('section', {'class': 'padding-section'}) \
     102                .find_all('div', {'class': 'container'})[1].find('div', {'class': 'col-md-7'}) \
     103                .find_all('div', {'class': 'row'})
     104
     105            color = None
     106            rom_memory = None
     107            ram_memory = None
     108            back_camera = None
     109            operating_system = None
     110            chipset = None
     111            battery = None
     112            cpu = None
     113            front_camera = None
     114            offer_shop_code = None
     115            offer_description = None
     116
     117            if len(specifications) != 0:
     118                colors_tags = specifications[0].find('div', {'class': 'col-md-12 col-xs-12'}).find_all('a')
     119                temp_colors = []
     120                for color_tag in colors_tags:
     121                    temp_colors.append(color_tag.get_text().strip())
     122                color = ','.join(temp_colors)
     123
     124            if len(specifications) >= 2:
     125                temp_rom = specifications[1].find('div', {'class': 'col-md-12 col-xs-12'}).find_all('a')
     126                rom_list = []
     127                for rom in temp_rom:
     128                    rom_list.append(rom.get('title'))
     129                rom_memory = ','.join(rom_list)
     130
     131            if len(specifications) >= 3:
     132                temp_ram = specifications[2].find('div', {'class': 'col-md-12 col-xs-12'}).find_all('a')
     133                ram_list = []
     134                for ram in temp_ram:
     135                    ram_list.append(ram.get('title'))
     136
     137                ram_memory = ','.join(ram_list)
     138
     139            if 'Xiaomi' in brand:
     140                temp = color
     141                color = rom_memory
     142                rom_memory = temp
     143
     144                temp = ram_memory
     145                ram_memory = color
     146                color = temp
     147
     148            new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
     149                                         color, front_camera, back_camera, chipset, battery, operating_system, cpu,
     150                                         image_url,
     151                                         offer_url, last_updated, is_validated, offer_description, offer_shop_code))
     152
     153    for new_offer in new_offers:
     154        flag = False
     155        flag_price = False
     156        offer_id = None
     157
     158        for old_offer in database_offers:
     159
     160            if new_offer.offer_name == old_offer.offer_name:
     161                flag = True
     162                if new_offer.price != old_offer.price:
     163                    flag_price = True
     164                    offer_id = old_offer.offer_id
     165
     166        if flag:
     167            # print('ALREADY IN DATABASE')
     168            # print(new_offer)
     169            # if it's already in database, check PRICE and if it's changed, change it !!!!!!
     170            if flag_price:
     171                print('PRICE CHANGED!')  # CHANGE PRICE
     172                print('offer id: ' + str(offer_id))
     173                headers = {'Content-type': 'application/json'}
     174                requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
     175                             headers=headers)
     176        else:
     177            print('ADDED')  # ADD OFFER
     178            print(new_offer)
     179            headers = {'Content-type': 'application/json'}
     180            requests.post('http://localhost:8080/phoneoffer/addoffer',
     181                          headers=headers, data=json.dumps(new_offer.__dict__, default=str))
     182
     183    print('------------------------------------')
    146184
    147185    for old_offer in database_offers:
    148 
    149         if new_offer.offer_name == old_offer.offer_name:
    150             flag = True
    151             if new_offer.price != old_offer.price:
    152                 flag_price = True
    153                 offer_id = old_offer.offer_id
    154 
    155     if flag:
    156         # print('ALREADY IN DATABASE')
    157         # print(new_offer)
    158         # if it's already in database, check PRICE and if it's changed, change it !!!!!!
    159         if flag_price:
    160             print('PRICE CHANGED!')  # CHANGE PRICE
    161             print('offer id: ' + str(offer_id))
    162             headers = {'Content-type': 'application/json'}
    163             requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
    164                          headers=headers)
    165     else:
    166         print('ADDED')  # ADD OFFER
    167         print(new_offer)
    168         headers = {'Content-type': 'application/json'}
    169         requests.post('http://localhost:8080/phoneoffer/addoffer',
    170                       headers=headers, data=json.dumps(new_offer.__dict__, default=str))
    171 
    172 print('------------------------------------')
    173 
    174 for old_offer in database_offers:
    175     flag = False
    176     for new_offer in new_offers:
    177         if old_offer.offer_name == new_offer.offer_name:
    178             flag = True
    179 
    180     if not flag:
    181         print('OFFER DELETED')
    182         print(old_offer)
    183         # DELETE OFFER
    184         requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     186        flag = False
     187        for new_offer in new_offers:
     188            if old_offer.offer_name == new_offer.offer_name:
     189                flag = True
     190
     191        if not flag:
     192            print('OFFER DELETED')
     193            print(old_offer)
     194            # DELETE OFFER
     195            requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     196except Exception:
     197    traceback.print_exc()
     198    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     199                    ' VALUES (%s, %s, %s);'
     200    insert_value = (offer_shop, last_updated, 'failed')
     201    cur.execute(insert_script, insert_value)
     202    db_connection.commit()
     203    cur.close()
     204    db_connection.close()
     205else:
     206    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     207                    ' VALUES (%s, %s, %s);'
     208    insert_value = (offer_shop, last_updated, 'success')
     209    cur.execute(insert_script, insert_value)
     210    db_connection.commit()
     211    cur.close()
     212    db_connection.close()
     213

  • phonelux_scrappers/scrappers/mobelix_scrapper.py

    rffd50db r47f4eaf  
    33import unicodedata
    44from datetime import datetime
    5 
     5import traceback
    66import psycopg2
    77import config_read
    … …  
    1919is_validated = False
    2020
    21 # Mobelix phone offers that are already in database
    22 
    23 offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/mobelix').text))
    24 
    25 database_offers = []
    26 
    27 for offer in offers:
    28     phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
    29                             offer['ram_memory'],
    30                             offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
    31                             offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
    32                             offer['image_url'],
    33                             offer['offer_url'], offer['last_updated'], offer['is_validated'],
    34                             offer['offer_description'],
    35                             offer['offer_shop_code'])
    36     database_offers.append(phoneOffer)
    37 
    38 new_offers = []
    39 
    40 for i in range(1, 17):
    41     mobelix_url = "https://mobelix.com.mk/mk/mobilni-telefoni?page=" + str(i)
    42 
    43     response1 = requests.get(mobelix_url)
    44     soup1 = BeautifulSoup(response1.content, 'html.parser')
    45 
    46     phones = soup1.find_all('div', {'class': 'p-2 rounded text-dark bg-white d-flex w-100'})
    47 
    48     for phone in phones:
    49         offer_url = phone.find('a').get('href')
    50         image_url = phone.find_all('div', {'class': 'col-12'})[0].find('img').get('src')
    51         brand = phone.find_all('div', {'class': 'col-12'})[1].find('h5', {'class': 'mb-0'}).get_text().strip()
    52         offer_name = phone.find_all('div', {'class': 'col-12'})[1] \
    53             .find('h3', {'class': 'h5 font-weight-normal'}).get_text().strip()
    54 
    55         if 'Watch' in offer_name or 'Pad' in offer_name or 'Tab' in offer_name or 'Pods' in offer_name or 'Buds' in offer_name or 'HomePod' in offer_name:
    56             continue
    57 
    58         if brand not in offer_name:
    59             offer_name = brand + " " + offer_name
    60 
    61         temp_prices = phone.find_all('div', {'class': 'col-12'})[1] \
    62             .find('p', {'class': 'h5 price'}).get_text(separator='/').strip()
    63 
    64         if len(temp_prices.split('/')) > 1:
    65             price = int(float(temp_prices.split('/')[1].replace(',', '').replace('ден', '').strip()))
     21# Call to read the configuration file and connect to database
     22cinfo = config_read.get_databaseconfig("../postgresdb.config")
     23db_connection = psycopg2.connect(
     24    database=cinfo[0],
     25    host=cinfo[1],
     26    user=cinfo[2],
     27    password=cinfo[3]
     28)
     29cur = db_connection.cursor()
     30
     31try:
     32    # Mobelix phone offers that are already in database
     33    offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/mobelix').text))
     34
     35    database_offers = []
     36
     37    for offer in offers:
     38        phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
     39                                    offer['ram_memory'],
     40                                    offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
     41                                    offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
     42                                    offer['image_url'],
     43                                    offer['offer_url'], offer['last_updated'], offer['is_validated'],
     44                                    offer['offer_description'],
     45                                    offer['offer_shop_code'])
     46        database_offers.append(phoneOffer)
     47
     48    new_offers = []
     49
     50    for i in range(1, 17):
     51        mobelix_url = "https://mobelix.com.mk/mk/mobilni-telefoni?page=" + str(i)
     52
     53        response1 = requests.get(mobelix_url)
     54        soup1 = BeautifulSoup(response1.content, 'html.parser')
     55
     56        phones = soup1.find_all('div', {'class': 'p-2 rounded text-dark bg-white d-flex w-100'})
     57
     58        for phone in phones:
     59            offer_url = phone.find('a').get('href')
     60            image_url = phone.find_all('div', {'class': 'col-12'})[0].find('img').get('src')
     61            brand = phone.find_all('div', {'class': 'col-12'})[1].find('h5', {'class': 'mb-0'}).get_text().strip()
     62            offer_name = phone.find_all('div', {'class': 'col-12'})[1] \
     63                .find('h3', {'class': 'h5 font-weight-normal'}).get_text().strip()
     64
     65            if 'Watch' in offer_name or 'Pad' in offer_name or 'Tab' in offer_name or 'Pods' in offer_name or 'Buds' in offer_name or 'HomePod' in offer_name:
     66                continue
     67
     68            if brand not in offer_name:
     69                offer_name = brand + " " + offer_name
     70
     71            temp_prices = phone.find_all('div', {'class': 'col-12'})[1] \
     72                .find('p', {'class': 'h5 price'}).get_text(separator='/').strip()
     73
     74            if len(temp_prices.split('/')) > 1:
     75                price = int(float(temp_prices.split('/')[1].replace(',', '').replace('ден', '').strip()))
     76            else:
     77                price = int(float(temp_prices.split('/')[0].replace(',', '').replace('ден', '').strip()))
     78
     79            response2 = requests.get(offer_url)
     80            soup2 = BeautifulSoup(response2.content, 'html.parser')
     81
     82            colors_divs = soup2.find('div', {'class': 'color-wrapper mt-2 mb-1'}) \
     83                .find_all('div', {'class': 'color-box d-inline-block'})  # color div tags
     84
     85            temp_colors = []
     86            for div in colors_divs:
     87                temp_colors.append(div.get('title'))
     88
     89            color = ",".join(temp_colors)  # available colors for offer
     90
     91            tables = soup2.find('div', {'class': 'mobelix-specs table-white bordered-table'}).find_all('table')
     92
     93            operating_system = None
     94            chipset = None
     95            battery = None
     96            ram_memory = None
     97            rom_memory = None
     98            front_camera = ''
     99            back_camera = ''
     100            cpu = None
     101            offer_shop_code = None
     102            offer_description = None
     103
     104            for table in tables:
     105                for cell in table.find_all('td'):
     106                    if cell.get('data-spec') is None:
     107                        continue
     108
     109                    if cell.get('data-spec') == 'os':
     110                        operating_system = unicodedata.normalize('NFKD', cell.get_text().strip())
     111
     112                    if cell.get('data-spec') == 'chipset':
     113                        chipset = unicodedata.normalize('NFKD', cell.get_text().strip())
     114
     115                    if cell.get('data-spec') == 'cpu':
     116                        cpu = unicodedata.normalize('NFKD', cell.get_text().strip())
     117
     118                    if cell.get('data-spec') == 'internalmemory':
     119                        temp_rom = []
     120                        temp_ram = []
     121                        temp_internalmemory = unicodedata.normalize('NFKD', cell.get_text().strip())
     122                        for internalmemory in temp_internalmemory.split(','):
     123                            temp_rom.append(internalmemory.strip().split(' ')[0])
     124                            if len(internalmemory.strip().split(' ')) > 1:
     125                                temp_ram.append(internalmemory.strip().split(' ')[1])
     126                        rom_memory = ','.join(temp_rom)
     127                        ram_memory = ','.join(temp_ram)
     128
     129                    if cell.get('data-spec') == 'cam1modules' or cell.get('data-spec') == 'cam1features' or cell.get(
     130                                'data-spec') == 'cam1video':
     131                        back_camera += unicodedata.normalize('NFKD', cell.get_text().strip()) + '\n'
     132
     133                    if cell.get('data-spec') == 'cam2modules' or cell.get('data-spec') == 'cam2features' or cell.get(
     134                                'data-spec') == 'cam2video':
     135                        front_camera += unicodedata.normalize('NFKD', cell.get_text().strip()) + '\n'
     136
     137                    if cell.get('data-spec') == 'batdescription1':
     138                        battery = unicodedata.normalize('NFKD', cell.get_text().strip())
     139
     140            if front_camera == 'No':
     141                front_camera = None
     142
     143            if back_camera == 'No':
     144                back_camera = None
     145
     146            new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
     147                                            color, front_camera, back_camera, chipset, battery, operating_system, cpu,
     148                                            image_url,
     149                                            offer_url, last_updated, is_validated, offer_description, offer_shop_code))
     150
     151
     152    for new_offer in new_offers:
     153        flag = False
     154        flag_price = False
     155        offer_id = None
     156
     157        for old_offer in database_offers:
     158
     159            if new_offer.offer_name == old_offer.offer_name:
     160                flag = True
     161                if new_offer.price != old_offer.price:
     162                    flag_price = True
     163                    offer_id = old_offer.offer_id
     164
     165        if flag:
     166            # print('ALREADY IN DATABASE')
     167            # print(new_offer)
     168            # if it's already in database, check PRICE and if it's changed, change it !!!!!!
     169            if flag_price:
     170                print('PRICE CHANGED!')  # CHANGE PRICE
     171                print('offer id: ' + str(offer_id))
     172                headers = {'Content-type': 'application/json'}
     173                requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
     174                        headers=headers)
    66175        else:
    67             price = int(float(temp_prices.split('/')[0].replace(',', '').replace('ден', '').strip()))
    68 
    69         response2 = requests.get(offer_url)
    70         soup2 = BeautifulSoup(response2.content, 'html.parser')
    71 
    72         colors_divs = soup2.find('div', {'class': 'color-wrapper mt-2 mb-1'}) \
    73             .find_all('div', {'class': 'color-box d-inline-block'})  # color div tags
    74 
    75         temp_colors = []
    76         for div in colors_divs:
    77             temp_colors.append(div.get('title'))
    78 
    79         color = ",".join(temp_colors)  # available colors for offer
    80 
    81         tables = soup2.find('div', {'class': 'mobelix-specs table-white bordered-table'}).find_all('table')
    82 
    83         operating_system = None
    84         chipset = None
    85         battery = None
    86         ram_memory = None
    87         rom_memory = None
    88         front_camera = ''
    89         back_camera = ''
    90         cpu = None
    91         offer_shop_code = None
    92         offer_description = None
    93 
    94         for table in tables:
    95             for cell in table.find_all('td'):
    96                 if cell.get('data-spec') is None:
    97                     continue
    98 
    99                 if cell.get('data-spec') == 'os':
    100                     operating_system = unicodedata.normalize('NFKD', cell.get_text().strip())
    101 
    102                 if cell.get('data-spec') == 'chipset':
    103                     chipset = unicodedata.normalize('NFKD', cell.get_text().strip())
    104 
    105                 if cell.get('data-spec') == 'cpu':
    106                     cpu = unicodedata.normalize('NFKD', cell.get_text().strip())
    107 
    108                 if cell.get('data-spec') == 'internalmemory':
    109                     temp_rom = []
    110                     temp_ram = []
    111                     temp_internalmemory = unicodedata.normalize('NFKD', cell.get_text().strip())
    112                     for internalmemory in temp_internalmemory.split(','):
    113                         temp_rom.append(internalmemory.strip().split(' ')[0])
    114                         if len(internalmemory.strip().split(' ')) > 1:
    115                             temp_ram.append(internalmemory.strip().split(' ')[1])
    116                     rom_memory = ','.join(temp_rom)
    117                     ram_memory = ','.join(temp_ram)
    118 
    119                 if cell.get('data-spec') == 'cam1modules' or cell.get('data-spec') == 'cam1features' or cell.get(
    120                         'data-spec') == 'cam1video':
    121                     back_camera += unicodedata.normalize('NFKD', cell.get_text().strip()) + '\n'
    122 
    123                 if cell.get('data-spec') == 'cam2modules' or cell.get('data-spec') == 'cam2features' or cell.get(
    124                         'data-spec') == 'cam2video':
    125                     front_camera += unicodedata.normalize('NFKD', cell.get_text().strip()) + '\n'
    126 
    127                 if cell.get('data-spec') == 'batdescription1':
    128                     battery = unicodedata.normalize('NFKD', cell.get_text().strip())
    129 
    130         if front_camera == 'No':
    131             front_camera = None
    132 
    133         if back_camera == 'No':
    134             back_camera = None
    135 
    136         new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
    137                                      color, front_camera, back_camera, chipset, battery, operating_system, cpu,
    138                                      image_url,
    139                                      offer_url, last_updated, is_validated, offer_description, offer_shop_code))
    140 
    141 
    142 for new_offer in new_offers:
    143     flag = False
    144     flag_price = False
    145     offer_id = None
     176            print('ADDED')  # ADD OFFER
     177            print(new_offer)
     178            headers = {'Content-type': 'application/json'}
     179            requests.post('http://localhost:8080/phoneoffer/addoffer',
     180                        headers=headers, data=json.dumps(new_offer.__dict__, default=str))
     181
     182    print('------------------------------------')
    146183
    147184    for old_offer in database_offers:
    148 
    149         if new_offer.offer_name == old_offer.offer_name:
    150             flag = True
    151             if new_offer.price != old_offer.price:
    152                 flag_price = True
    153                 offer_id = old_offer.offer_id
    154 
    155     if flag:
    156         # print('ALREADY IN DATABASE')
    157         # print(new_offer)
    158         # if it's already in database, check PRICE and if it's changed, change it !!!!!!
    159         if flag_price:
    160             print('PRICE CHANGED!')  # CHANGE PRICE
    161             print('offer id: ' + str(offer_id))
    162             headers = {'Content-type': 'application/json'}
    163             requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
    164                          headers=headers)
    165     else:
    166         print('ADDED')  # ADD OFFER
    167         print(new_offer)
    168         headers = {'Content-type': 'application/json'}
    169         requests.post('http://localhost:8080/phoneoffer/addoffer',
    170                       headers=headers, data=json.dumps(new_offer.__dict__, default=str))
    171 
    172 print('------------------------------------')
    173 
    174 for old_offer in database_offers:
    175     flag = False
    176     for new_offer in new_offers:
    177         if old_offer.offer_name == new_offer.offer_name:
    178             flag = True
    179 
    180     if not flag:
    181         print('OFFER DELETED')
    182         print(old_offer)
    183         # DELETE OFFER
    184         requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     185        flag = False
     186        for new_offer in new_offers:
     187            if old_offer.offer_name == new_offer.offer_name:
     188                flag = True
     189
     190        if not flag:
     191            print('OFFER DELETED')
     192            print(old_offer)
     193            # DELETE OFFER
     194            requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     195except Exception:
     196    traceback.print_exc()
     197    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     198                    ' VALUES (%s, %s, %s);'
     199    insert_value = (offer_shop, last_updated, 'failed')
     200    cur.execute(insert_script, insert_value)
     201    db_connection.commit()
     202    cur.close()
     203    db_connection.close()
     204else:
     205    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     206                    ' VALUES (%s, %s, %s);'
     207    insert_value = (offer_shop, last_updated, 'success')
     208    cur.execute(insert_script, insert_value)
     209    db_connection.commit()
     210    cur.close()
     211    db_connection.close()
     212

  • phonelux_scrappers/scrappers/mobigo_scrapper.py

    rffd50db r47f4eaf  
    11import json
     2import traceback
    23import unicodedata
    34from datetime import datetime
    … …  
    1819is_validated = False
    1920
    20 # Mobi Go phone offers that are already in database
    21 
    22 offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/mobigo').text))
    23 
    24 database_offers = []
    25 
    26 for offer in offers:
    27     phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
    28                             offer['ram_memory'],
    29                             offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
    30                             offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
    31                             offer['image_url'],
    32                             offer['offer_url'], offer['last_updated'], offer['is_validated'],
    33                             offer['offer_description'],
    34                             offer['offer_shop_code'])
    35     database_offers.append(phoneOffer)
    36 
    37 new_offers = []
    38 
    39 
    40 for i in range(1, 6):
    41     mobigo_url = "https://mobigo.mk/page/" + str(i) + "/"
    42 
    43     response1 = requests.get(mobigo_url)
    44 
    45     soup1 = BeautifulSoup(response1.content, 'html.parser')
    46 
    47     phone_sections = soup1.find_all('ul', {'class': 'recent-posts'})
    48     phones = phone_sections[len(phone_sections) - 1].find_all('li')
    49 
    50     for phone in phones:
    51         offer_url = phone.find('div', {'class', 'post-thumb'}).find('a').get('href')  # offer url
    52         image_url = phone.find('div', {'class', 'post-thumb'}).find('a').find('img').get('src')  # image url
    53         offer_name = phone.find('div', {'class', 'post-content'}).find_all('h2')[0].get_text().strip()  # offer_name
    54 
    55         if "Watch" in offer_name or "Tab" in offer_name:  # if the product is watch or tablet, continue
    56             continue
    57 
    58         price = int(float(phone.find('div', {'class', 'post-content'}).find_all('h2')[1] \
    59                           .get_text().replace('ден.', '').replace('.', '').strip()))  # price
    60 
    61         response2 = requests.get(offer_url)
    62         soup2 = BeautifulSoup(response2.content, 'html.parser')
    63 
    64         brand = soup2.find('a', {'rel': 'category tag'}).get_text().strip()  # brand
    65 
    66         if brand not in offer_name:
    67             offer_name = brand + " " + offer_name
    68 
    69         specifications = soup2.find('table', {'id': 'singlet'}).find_all('tr')
    70 
    71         ram_memory = None
    72         rom_memory = None
    73         battery = None
    74         back_camera = None
    75         front_camera = None
    76         chipset = None
    77         operating_system = None
    78         cpu = None
    79         offer_shop_code = None
    80         offer_description = None
    81         color = None
    82 
    83         for specification in specifications:
    84             if specification.find('td') == None:
     21# Call to read the configuration file and connect to database
     22cinfo = config_read.get_databaseconfig("../postgresdb.config")
     23db_connection = psycopg2.connect(
     24    database=cinfo[0],
     25    host=cinfo[1],
     26    user=cinfo[2],
     27    password=cinfo[3]
     28)
     29cur = db_connection.cursor()
     30
     31try:
     32    # Mobi Go phone offers that are already in database
     33    offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/mobigo').text))
     34
     35    database_offers = []
     36
     37    for offer in offers:
     38        phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
     39                                offer['ram_memory'],
     40                                offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
     41                                offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
     42                                offer['image_url'],
     43                                offer['offer_url'], offer['last_updated'], offer['is_validated'],
     44                                offer['offer_description'],
     45                                offer['offer_shop_code'])
     46        database_offers.append(phoneOffer)
     47
     48    new_offers = []
     49
     50
     51    for i in range(1, 6):
     52        mobigo_url = "https://mobigo.mk/page/" + str(i) + "/"
     53
     54        response1 = requests.get(mobigo_url)
     55
     56        soup1 = BeautifulSoup(response1.content, 'html.parser')
     57
     58        phone_sections = soup1.find_all('ul', {'class': 'recent-posts'})
     59        phones = phone_sections[len(phone_sections) - 1].find_all('li')
     60
     61        for phone in phones:
     62            offer_url = phone.find('div', {'class', 'post-thumb'}).find('a').get('href')  # offer url
     63            image_url = phone.find('div', {'class', 'post-thumb'}).find('a').find('img').get('src')  # image url
     64            offer_name = phone.find('div', {'class', 'post-content'}).find_all('h2')[0].get_text().strip()  # offer_name
     65
     66            if "Watch" in offer_name or "Tab" in offer_name:  # if the product is watch or tablet, continue
    8567                continue
    8668
    87             # operating system
    88             if specification.find('td').get_text() == "Платформа":
    89                 if specification.find('i').get_text() != "/":
    90                     operating_system = specification.find('i').get_text().strip()
    91                 else:
    92                     operating_system = None
    93 
    94             # chipset
    95             if specification.find('td').get_text() == "Chipset":
    96                 if specification.find('i').get_text() != "/":
    97                     chipset = specification.find('i').get_text().strip()
    98                 else:
    99                     chipset = None
    100 
    101             # ram and rom memory
    102             if specification.find('td').get_text() == "Меморија":
    103                 if specification.find('i').get_text() != "/":
    104                     rom_memory = specification.find('i').get_text().replace(',', '').split(' ')[0].strip()
    105                     ram_memory = specification.find('i').get_text().replace(',', '').split(' ')[1].strip()
    106                 else:
    107                     rom_memory = None
    108                     ram_memory = None
    109 
    110             # back camera
    111             if specification.find('td').get_text() == "Главна Камера":
    112                 if specification.find('i').get_text() != "/":
    113                     back_camera = specification.find('i').get_text().strip()
    114                 else:
    115                     back_camera = None
    116 
    117             # front camera
    118             if specification.find('td').get_text() == "Селфи Камера":
    119                 if specification.find('i').get_text() != "/":
    120                     front_camera = specification.find('i').get_text().strip()
    121                 else:
    122                     front_camera = None
    123 
    124             # battery
    125             if specification.find('td').get_text() == "Батерија":
    126                 if specification.find('i').get_text() != "/":
    127                     battery = specification.find('i').get_text().strip()
    128                 else:
    129                     battery = None
    130 
    131         new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
    132                                      color, front_camera, back_camera, chipset, battery, operating_system, cpu,
    133                                      image_url,
    134                                      offer_url, last_updated, is_validated, offer_description, offer_shop_code))
    135 
    136 
    137 for new_offer in new_offers:
    138     flag = False
    139     flag_price = False
    140     offer_id = None
     69            price = int(float(phone.find('div', {'class', 'post-content'}).find_all('h2')[1] \
     70                              .get_text().replace('ден.', '').replace('.', '').strip()))  # price
     71
     72            response2 = requests.get(offer_url)
     73            soup2 = BeautifulSoup(response2.content, 'html.parser')
     74
     75            brand = soup2.find('a', {'rel': 'category tag'}).get_text().strip()  # brand
     76
     77            if brand not in offer_name:
     78                offer_name = brand + " " + offer_name
     79
     80            specifications = soup2.find('table', {'id': 'singlet'}).find_all('tr')
     81
     82            ram_memory = None
     83            rom_memory = None
     84            battery = None
     85            back_camera = None
     86            front_camera = None
     87            chipset = None
     88            operating_system = None
     89            cpu = None
     90            offer_shop_code = None
     91            offer_description = None
     92            color = None
     93
     94            for specification in specifications:
     95                if specification.find('td') == None:
     96                    continue
     97
     98                # operating system
     99                if specification.find('td').get_text() == "Платформа":
     100                    if specification.find('i').get_text() != "/":
     101                        operating_system = specification.find('i').get_text().strip()
     102                    else:
     103                        operating_system = None
     104
     105                # chipset
     106                if specification.find('td').get_text() == "Chipset":
     107                    if specification.find('i').get_text() != "/":
     108                        chipset = specification.find('i').get_text().strip()
     109                    else:
     110                        chipset = None
     111
     112                # ram and rom memory
     113                if specification.find('td').get_text() == "Меморија":
     114                    if specification.find('i').get_text() != "/":
     115                        rom_memory = specification.find('i').get_text().replace(',', '').split(' ')[0].strip()
     116                        ram_memory = specification.find('i').get_text().replace(',', '').split(' ')[1].strip()
     117                    else:
     118                        rom_memory = None
     119                        ram_memory = None
     120
     121                # back camera
     122                if specification.find('td').get_text() == "Главна Камера":
     123                    if specification.find('i').get_text() != "/":
     124                        back_camera = specification.find('i').get_text().strip()
     125                    else:
     126                        back_camera = None
     127
     128                # front camera
     129                if specification.find('td').get_text() == "Селфи Камера":
     130                    if specification.find('i').get_text() != "/":
     131                        front_camera = specification.find('i').get_text().strip()
     132                    else:
     133                        front_camera = None
     134
     135                # battery
     136                if specification.find('td').get_text() == "Батерија":
     137                    if specification.find('i').get_text() != "/":
     138                        battery = specification.find('i').get_text().strip()
     139                    else:
     140                        battery = None
     141
     142            new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
     143                                         color, front_camera, back_camera, chipset, battery, operating_system, cpu,
     144                                         image_url,
     145                                         offer_url, last_updated, is_validated, offer_description, offer_shop_code))
     146
     147
     148    for new_offer in new_offers:
     149        flag = False
     150        flag_price = False
     151        offer_id = None
     152
     153        for old_offer in database_offers:
     154
     155            if new_offer.offer_name == old_offer.offer_name:
     156                flag = True
     157                if new_offer.price != old_offer.price:
     158                    flag_price = True
     159                    offer_id = old_offer.offer_id
     160
     161        if flag:
     162            print('ALREADY IN DATABASE')
     163            print(new_offer)
     164            # if it's already in database, check PRICE and if it's changed, change it !!!!!!
     165            if flag_price:
     166                print('PRICE CHANGED!')  # CHANGE PRICE
     167                print('offer id: ' + str(offer_id))
     168                headers = {'Content-type': 'application/json'}
     169                requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
     170                             headers=headers)
     171        else:
     172            print('ADDED')  # ADD OFFER
     173            print(new_offer)
     174            headers = {'Content-type': 'application/json'}
     175            requests.post('http://localhost:8080/phoneoffer/addoffer',
     176                          headers=headers, data=json.dumps(new_offer.__dict__, default=str))
     177
     178    print('------------------------------------')
    141179
    142180    for old_offer in database_offers:
    143 
    144         if new_offer.offer_name == old_offer.offer_name:
    145             flag = True
    146             if new_offer.price != old_offer.price:
    147                 flag_price = True
    148                 offer_id = old_offer.offer_id
    149 
    150     if flag:
    151         print('ALREADY IN DATABASE')
    152         print(new_offer)
    153         # if it's already in database, check PRICE and if it's changed, change it !!!!!!
    154         if flag_price:
    155             print('PRICE CHANGED!')  # CHANGE PRICE
    156             print('offer id: ' + str(offer_id))
    157             headers = {'Content-type': 'application/json'}
    158             requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
    159                          headers=headers)
    160     else:
    161         print('ADDED')  # ADD OFFER
    162         print(new_offer)
    163         headers = {'Content-type': 'application/json'}
    164         requests.post('http://localhost:8080/phoneoffer/addoffer',
    165                       headers=headers, data=json.dumps(new_offer.__dict__, default=str))
    166 
    167 print('------------------------------------')
    168 
    169 for old_offer in database_offers:
    170     flag = False
    171     for new_offer in new_offers:
    172         if old_offer.offer_name == new_offer.offer_name:
    173             flag = True
    174 
    175     if not flag:
    176         print('OFFER DELETED')
    177         print(old_offer)
    178         # DELETE OFFER
    179         requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     181        flag = False
     182        for new_offer in new_offers:
     183            if old_offer.offer_name == new_offer.offer_name:
     184                flag = True
     185
     186        if not flag:
     187            print('OFFER DELETED')
     188            print(old_offer)
     189            # DELETE OFFER
     190            requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     191except Exception:
     192    traceback.print_exc()
     193    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     194                    ' VALUES (%s, %s, %s);'
     195    insert_value = (offer_shop, last_updated, 'failed')
     196    cur.execute(insert_script, insert_value)
     197    db_connection.commit()
     198    cur.close()
     199    db_connection.close()
     200else:
     201    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     202                    ' VALUES (%s, %s, %s);'
     203    insert_value = (offer_shop, last_updated, 'success')
     204    cur.execute(insert_script, insert_value)
     205    db_connection.commit()
     206    cur.close()
     207    db_connection.close()

  • phonelux_scrappers/scrappers/mobilezone_scrapper.py

    rffd50db r47f4eaf  
    11import json
     2import traceback
    23import unicodedata
    34from datetime import datetime
    … …  
    1819is_validated = False
    1920
    20 # Mobile Zone phone offers that are already in database
     21# Call to read the configuration file and connect to database
     22cinfo = config_read.get_databaseconfig("../postgresdb.config")
     23db_connection = psycopg2.connect(
     24    database=cinfo[0],
     25    host=cinfo[1],
     26    user=cinfo[2],
     27    password=cinfo[3]
     28)
     29cur = db_connection.cursor()
    2130
    22 offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/mobilezone').text))
     31try:
     32    # Mobile Zone phone offers that are already in database
     33    offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/mobilezone').text))
    2334
    24 database_offers = []
     35    database_offers = []
    2536
    26 for offer in offers:
    27     phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
    28                             offer['ram_memory'],
    29                             offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
    30                             offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
    31                             offer['image_url'],
    32                             offer['offer_url'], offer['last_updated'], offer['is_validated'],
    33                             offer['offer_description'],
    34                             offer['offer_shop_code'])
    35     database_offers.append(phoneOffer)
     37    for offer in offers:
     38        phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
     39                                offer['ram_memory'],
     40                                offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
     41                                offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
     42                                offer['image_url'],
     43                                offer['offer_url'], offer['last_updated'], offer['is_validated'],
     44                                offer['offer_description'],
     45                                offer['offer_shop_code'])
     46        database_offers.append(phoneOffer)
    3647
    37 new_offers = []
     48    new_offers = []
    3849
    39 for i in range(1, 3):
    40     mobilezone_url = 'https://mobilezone.mk/produkt-kategorija/telefoni/novi-telefoni/page/' + str(i) + '/'
     50    for i in range(1, 3):
     51        mobilezone_url = 'https://mobilezone.mk/produkt-kategorija/telefoni/novi-telefoni/page/' + str(i) + '/'
    4152
    42     response1 = requests.get(mobilezone_url)
    43     soup1 = BeautifulSoup(response1.content, 'html.parser')
     53        response1 = requests.get(mobilezone_url)
     54        soup1 = BeautifulSoup(response1.content, 'html.parser')
    4455
    45     phones = soup1.find('ul', {
    46         'class': 'products columns-tablet-2 columns-mobile-2 --skin-proto rey-wcGap-default rey-wcGrid-default '
    47                  '--paginated columns-4'}).find_all('li')
     56        phones = soup1.find('ul', {
     57            'class': 'products columns-tablet-2 columns-mobile-2 --skin-proto rey-wcGap-default rey-wcGrid-default '
     58                     '--paginated columns-4'}).find_all('li')
    4859
    49     for phone in phones:
    50         offer_url = phone.find('a', {'class': 'woocommerce-LoopProduct-link woocommerce-loop-product__link'}).get(
    51             'href')
    52         image_url = phone.find('a', {'class': 'woocommerce-LoopProduct-link woocommerce-loop-product__link'}) \
    53             .find('img').get('data-lazy-src')
     60        for phone in phones:
     61            offer_url = phone.find('a', {'class': 'woocommerce-LoopProduct-link woocommerce-loop-product__link'}).get(
     62                'href')
     63            image_url = phone.find('a', {'class': 'woocommerce-LoopProduct-link woocommerce-loop-product__link'}) \
     64                .find('img').get('data-lazy-src')
    5465
    55         brand_section = phone.find('div', {'class': 'rey-productInner'}).find('div', {'class': 'rey-brandLink'})
     66            brand_section = phone.find('div', {'class': 'rey-productInner'}).find('div', {'class': 'rey-brandLink'})
    5667
    57         if brand_section is not None:
    58             brand = brand_section.find('a').get_text().strip()
     68            if brand_section is not None:
     69                brand = brand_section.find('a').get_text().strip()
     70            else:
     71                brand = None
     72
     73            offer_name = phone.find('h2', {'class': 'woocommerce-loop-product__title'}).find('a').get_text().strip()
     74
     75            if brand is not None and brand not in offer_name:
     76                offer_name = brand + ' ' + offer_name
     77
     78            price_tag = phone.find('span', {'class': 'woocommerce-Price-amount amount'})
     79            price = None
     80
     81            if price_tag is not None:
     82                price = int(unicodedata.normalize('NFKD', price_tag.find('bdi').get_text()
     83                                              .replace(',', '')
     84                                              .replace('ден', '').strip()))
     85            else:
     86                continue
     87
     88            response2 = requests.get(offer_url)
     89            soup2 = BeautifulSoup(response2.text, 'html.parser')
     90
     91            specifications = soup2.find('table', {'class': 'woocommerce-product-attributes shop_attributes'}).find_all('tr')
     92
     93            back_camera = None
     94            front_camera = None
     95            rom_memory = None
     96            ram_memory = None
     97            operating_system = None
     98            cpu = None
     99            chipset = None
     100            offer_description = None
     101            offer_shop_code = None
     102            battery = None
     103            color = None
     104
     105            for specification in specifications:
     106                if 'Главна камера' in specification.find('th').get_text():
     107                    back_camera = specification.find('td').get_text().strip()
     108
     109                if 'Селфи камера' in specification.find('th').get_text():
     110                    front_camera = specification.find('td').get_text().strip()
     111
     112                if 'Батерија' in specification.find('th').get_text():
     113                    battery = specification.find('td').get_text().strip()
     114
     115                if 'Меморија' in specification.find('th').get_text():
     116                    rom_memory = specification.find('td').get_text().strip()
     117
     118                if 'Боја' in specification.find('th').get_text():
     119                    color = specification.find('td').get_text().strip()
     120
     121            new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
     122                                         color, front_camera, back_camera, chipset, battery, operating_system, cpu,
     123                                         image_url,
     124                                         offer_url, last_updated, is_validated, offer_description, offer_shop_code))
     125
     126    for new_offer in new_offers:
     127        flag = False
     128        flag_price = False
     129        offer_id = None
     130
     131        for old_offer in database_offers:
     132
     133            if new_offer.offer_name == old_offer.offer_name:
     134                flag = True
     135                if new_offer.price != old_offer.price:
     136                    flag_price = True
     137                    offer_id = old_offer.offer_id
     138
     139        if flag:
     140            # print('ALREADY IN DATABASE')
     141            # print(new_offer)
     142            # if it's already in database, check PRICE and if it's changed, change it !!!!!!
     143            if flag_price:
     144                print('PRICE CHANGED!')  # CHANGE PRICE
     145                print('offer id: ' + str(offer_id))
     146                headers = {'Content-type': 'application/json'}
     147                requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
     148                             headers=headers)
    59149        else:
    60             brand = None
     150            print('ADDED')  # ADD OFFER
     151            print(new_offer)
     152            headers = {'Content-type': 'application/json'}
     153            requests.post('http://localhost:8080/phoneoffer/addoffer',
     154                          headers=headers, data=json.dumps(new_offer.__dict__, default=str))
    61155
    62         offer_name = phone.find('h2', {'class': 'woocommerce-loop-product__title'}).find('a').get_text().strip()
    63 
    64         if brand is not None and brand not in offer_name:
    65             offer_name = brand + ' ' + offer_name
    66 
    67         price_tag = phone.find('span', {'class': 'woocommerce-Price-amount amount'})
    68         price = None
    69 
    70         if price_tag is not None:
    71             price = int(unicodedata.normalize('NFKD', price_tag.find('bdi').get_text()
    72                                           .replace(',', '')
    73                                           .replace('ден', '').strip()))
    74         else:
    75             continue
    76 
    77         response2 = requests.get(offer_url)
    78         soup2 = BeautifulSoup(response2.text, 'html.parser')
    79 
    80         specifications = soup2.find('table', {'class': 'woocommerce-product-attributes shop_attributes'}).find_all('tr')
    81 
    82         back_camera = None
    83         front_camera = None
    84         rom_memory = None
    85         ram_memory = None
    86         operating_system = None
    87         cpu = None
    88         chipset = None
    89         offer_description = None
    90         offer_shop_code = None
    91         battery = None
    92         color = None
    93 
    94         for specification in specifications:
    95             if 'Главна камера' in specification.find('th').get_text():
    96                 back_camera = specification.find('td').get_text().strip()
    97 
    98             if 'Селфи камера' in specification.find('th').get_text():
    99                 front_camera = specification.find('td').get_text().strip()
    100 
    101             if 'Батерија' in specification.find('th').get_text():
    102                 battery = specification.find('td').get_text().strip()
    103 
    104             if 'Меморија' in specification.find('th').get_text():
    105                 rom_memory = specification.find('td').get_text().strip()
    106 
    107             if 'Боја' in specification.find('th').get_text():
    108                 color = specification.find('td').get_text().strip()
    109 
    110         new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
    111                                      color, front_camera, back_camera, chipset, battery, operating_system, cpu,
    112                                      image_url,
    113                                      offer_url, last_updated, is_validated, offer_description, offer_shop_code))
    114 
    115 for new_offer in new_offers:
    116     flag = False
    117     flag_price = False
    118     offer_id = None
     156    print('------------------------------------')
    119157
    120158    for old_offer in database_offers:
     159        flag = False
     160        for new_offer in new_offers:
     161            if old_offer.offer_name == new_offer.offer_name:
     162                flag = True
    121163
    122         if new_offer.offer_name == old_offer.offer_name:
    123             flag = True
    124             if new_offer.price != old_offer.price:
    125                 flag_price = True
    126                 offer_id = old_offer.offer_id
    127 
    128     if flag:
    129         # print('ALREADY IN DATABASE')
    130         # print(new_offer)
    131         # if it's already in database, check PRICE and if it's changed, change it !!!!!!
    132         if flag_price:
    133             print('PRICE CHANGED!')  # CHANGE PRICE
    134             print('offer id: ' + str(offer_id))
    135             headers = {'Content-type': 'application/json'}
    136             requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
    137                          headers=headers)
    138     else:
    139         print('ADDED')  # ADD OFFER
    140         print(new_offer)
    141         headers = {'Content-type': 'application/json'}
    142         requests.post('http://localhost:8080/phoneoffer/addoffer',
    143                       headers=headers, data=json.dumps(new_offer.__dict__, default=str))
    144 
    145 print('------------------------------------')
    146 
    147 for old_offer in database_offers:
    148     flag = False
    149     for new_offer in new_offers:
    150         if old_offer.offer_name == new_offer.offer_name:
    151             flag = True
    152 
    153     if not flag:
    154         print('OFFER DELETED')
    155         print(old_offer)
    156         # DELETE OFFER
    157         requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     164        if not flag:
     165            print('OFFER DELETED')
     166            print(old_offer)
     167            # DELETE OFFER
     168            requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     169except Exception:
     170    traceback.print_exc()
     171    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     172                    ' VALUES (%s, %s, %s);'
     173    insert_value = (offer_shop, last_updated, 'failed')
     174    cur.execute(insert_script, insert_value)
     175    db_connection.commit()
     176    cur.close()
     177    db_connection.close()
     178else:
     179    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     180                    ' VALUES (%s, %s, %s);'
     181    insert_value = (offer_shop, last_updated, 'success')
     182    cur.execute(insert_script, insert_value)
     183    db_connection.commit()
     184    cur.close()
     185    db_connection.close()

  • phonelux_scrappers/scrappers/mobitech_scrapper.py

    rffd50db r47f4eaf  
    11import json
     2import traceback
    23import unicodedata
    34from datetime import datetime
    … …  
    1415sys.stdout = open(file_path, "w")
    1516
    16 
    17 mobitech_url = "https://mobitech.mk/shop/"
    18 
    19 response1 = requests.get(mobitech_url)
    20 
    21 soup1 = BeautifulSoup(response1.content, 'html.parser')
    22 
    23 phones = soup1.find_all('div', {'class': 'jet-woo-products__inner-box'})
    24 
    2517offer_shop = "Mobitech"  # offer shop
    2618last_updated = datetime.now().date()
    2719is_validated = False
    2820
    29 # Mobitech phone offers that are already in database
     21# Call to read the configuration file and connect to database
     22cinfo = config_read.get_databaseconfig("../postgresdb.config")
     23db_connection = psycopg2.connect(
     24    database=cinfo[0],
     25    host=cinfo[1],
     26    user=cinfo[2],
     27    password=cinfo[3]
     28)
     29cur = db_connection.cursor()
    3030
    31 offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/mobitech').text))
     31try:
     32    mobitech_url = "https://mobitech.mk/shop/"
    3233
    33 database_offers = []
     34    response1 = requests.get(mobitech_url)
    3435
    35 for offer in offers:
    36     phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
    37                             offer['ram_memory'],
    38                             offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
    39                             offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
    40                             offer['image_url'],
    41                             offer['offer_url'], offer['last_updated'], offer['is_validated'],
    42                             offer['offer_description'],
    43                             offer['offer_shop_code'])
    44     database_offers.append(phoneOffer)
     36    soup1 = BeautifulSoup(response1.content, 'html.parser')
    4537
    46 new_offers = []
     38    phones = soup1.find_all('div', {'class': 'jet-woo-products__inner-box'})
    4739
    48 for phone in phones:
    49     offer_url = phone.find('h5', {'class': 'jet-woo-product-title'}).find('a').get('href')  # url
    50     image_url = phone.find('div', {'class': 'jet-woo-product-thumbnail'}).find('img').get('src')  # image
    51     brand = phone.find_next('div', {'class': 'jet-woo-product-categories'}).find('a').get_text().strip()  # brand
    52     offer_name = phone.find('h5', {'class': 'jet-woo-product-title'}).find('a').get_text().strip()  # offer_name
    53     if brand not in offer_name:
    54         offer_name = brand+" "+offer_name
    55     temp_prices = phone.find('div', {'class': 'jet-woo-product-price'}).find_all('bdi')
    56     price = int(float(temp_prices[len(temp_prices) - 1].get_text().replace("ден", "").replace(",", "").strip())) # price
     40    # Mobitech phone offers that are already in database
     41    offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/mobitech').text))
    5742
    58     response2 = requests.get(offer_url)
    59     soup2 = BeautifulSoup(response2.content, 'html.parser')
     43    database_offers = []
    6044
    61     specifications = soup2.find_all('h2', {'class': 'elementor-heading-title elementor-size-default'})
     45    for offer in offers:
     46        phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
     47                                offer['ram_memory'],
     48                                offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
     49                                offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
     50                                offer['image_url'],
     51                                offer['offer_url'], offer['last_updated'], offer['is_validated'],
     52                                offer['offer_description'],
     53                                offer['offer_shop_code'])
     54        database_offers.append(phoneOffer)
    6255
    63     ram_memory = None
    64     rom_memory = None
    65     battery = None
    66     back_camera = None
    67     front_camera = None
    68     operating_system = None
    69     chipset = None
    70     color = None
    71     offer_shop_code = None
    72     cpu = None
    73     offer_description = None
     56    new_offers = []
    7457
    75     for specification in specifications:
    76         # rom memory
    77         if specification.get_text().startswith("Меморија:"):
    78             rom_memory = specification.get_text().split("Меморија:")[1].strip()
    79             if rom_memory == "Нема" or rom_memory == "/":
    80                 rom_memory = None
     58    for phone in phones:
     59        offer_url = phone.find('h5', {'class': 'jet-woo-product-title'}).find('a').get('href')  # url
     60        image_url = phone.find('div', {'class': 'jet-woo-product-thumbnail'}).find('img').get('src')  # image
     61        brand = phone.find_next('div', {'class': 'jet-woo-product-categories'}).find('a').get_text().strip()  # brand
     62        offer_name = phone.find('h5', {'class': 'jet-woo-product-title'}).find('a').get_text().strip()  # offer_name
     63        if brand not in offer_name:
     64            offer_name = brand+" "+offer_name
     65        temp_prices = phone.find('div', {'class': 'jet-woo-product-price'}).find_all('bdi')
     66        price = int(float(temp_prices[len(temp_prices) - 1].get_text().replace("ден", "").replace(",", "").strip())) # price
    8167
    82         # ram memory
    83         if specification.get_text().startswith("РАМ Меморија:"):
    84             ram_memory = specification.get_text().split("РАМ Меморија:")[1].replace('RAM', '')\
    85                 .replace('Ram', '').strip()
    86             if ram_memory == "Нема" or ram_memory == "/":
    87                 ram_memory = None
     68        response2 = requests.get(offer_url)
     69        soup2 = BeautifulSoup(response2.content, 'html.parser')
    8870
    89         # camera
    90         if specification.get_text().startswith("Камера:"):
    91             back_camera = specification.get_text().split("Камера:")[1].strip()
    92             if back_camera == "Нема":
    93                 back_camera = None
     71        specifications = soup2.find_all('h2', {'class': 'elementor-heading-title elementor-size-default'})
    9472
    95         # operating system
    96         if specification.get_text().startswith("Оперативен систем:"):
    97             operating_system = specification.get_text().split("Оперативен систем:")[1].split(",")[0].strip()
    98             if operating_system == "Нема":
    99                 operating_system = None
     73        ram_memory = None
     74        rom_memory = None
     75        battery = None
     76        back_camera = None
     77        front_camera = None
     78        operating_system = None
     79        chipset = None
     80        color = None
     81        offer_shop_code = None
     82        cpu = None
     83        offer_description = None
    10084
    101         # battery
    102         if specification.get_text().startswith("Батерија:"):
    103             battery = specification.get_text().split("Батерија:")[1].strip()
    104             if battery == "Нема":
    105                 battery = None
     85        for specification in specifications:
     86            # rom memory
     87            if specification.get_text().startswith("Меморија:"):
     88                rom_memory = specification.get_text().split("Меморија:")[1].strip()
     89                if rom_memory == "Нема" or rom_memory == "/":
     90                    rom_memory = None
    10691
    107     new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
    108                                  color, front_camera, back_camera, chipset, battery, operating_system, cpu,
    109                                  image_url,
    110                                  offer_url, last_updated, is_validated, offer_description, offer_shop_code))
     92            # ram memory
     93            if specification.get_text().startswith("РАМ Меморија:"):
     94                ram_memory = specification.get_text().split("РАМ Меморија:")[1].replace('RAM', '')\
     95                    .replace('Ram', '').strip()
     96                if ram_memory == "Нема" or ram_memory == "/":
     97                    ram_memory = None
    11198
    112 for new_offer in new_offers:
    113     flag = False
    114     flag_price = False
    115     offer_id = None
     99            # camera
     100            if specification.get_text().startswith("Камера:"):
     101                back_camera = specification.get_text().split("Камера:")[1].strip()
     102                if back_camera == "Нема":
     103                    back_camera = None
     104
     105            # operating system
     106            if specification.get_text().startswith("Оперативен систем:"):
     107                operating_system = specification.get_text().split("Оперативен систем:")[1].split(",")[0].strip()
     108                if operating_system == "Нема":
     109                    operating_system = None
     110
     111            # battery
     112            if specification.get_text().startswith("Батерија:"):
     113                battery = specification.get_text().split("Батерија:")[1].strip()
     114                if battery == "Нема":
     115                    battery = None
     116
     117        new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
     118                                     color, front_camera, back_camera, chipset, battery, operating_system, cpu,
     119                                     image_url,
     120                                     offer_url, last_updated, is_validated, offer_description, offer_shop_code))
     121
     122    for new_offer in new_offers:
     123        flag = False
     124        flag_price = False
     125        offer_id = None
     126
     127        for old_offer in database_offers:
     128
     129            if new_offer.offer_name == old_offer.offer_name:
     130                flag = True
     131                if new_offer.price != old_offer.price:
     132                    flag_price = True
     133                    offer_id = old_offer.offer_id
     134
     135        if flag:
     136            print('ALREADY IN DATABASE')
     137            print(new_offer)
     138            # if it's already in database, check PRICE and if it's changed, change it !!!!!!
     139            if flag_price:
     140                print('PRICE CHANGED!')  # CHANGE PRICE
     141                print('offer id: ' + str(offer_id))
     142                headers = {'Content-type': 'application/json'}
     143                requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
     144                             headers=headers)
     145        else:
     146            print('ADDED')  # ADD OFFER
     147            print(new_offer)
     148            headers = {'Content-type': 'application/json'}
     149            requests.post('http://localhost:8080/phoneoffer/addoffer',
     150                          headers=headers, data=json.dumps(new_offer.__dict__, default=str))
     151
     152    print('------------------------------------')
    116153
    117154    for old_offer in database_offers:
     155        flag = False
     156        for new_offer in new_offers:
     157            if old_offer.offer_name == new_offer.offer_name:
     158                flag = True
    118159
    119         if new_offer.offer_name == old_offer.offer_name:
    120             flag = True
    121             if new_offer.price != old_offer.price:
    122                 flag_price = True
    123                 offer_id = old_offer.offer_id
     160        if not flag:
     161            print('OFFER DELETED')
     162            print(old_offer)
     163            # DELETE OFFER
     164            requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     165except Exception:
     166    traceback.print_exc()
     167    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     168                    ' VALUES (%s, %s, %s);'
     169    insert_value = (offer_shop, last_updated, 'failed')
     170    cur.execute(insert_script, insert_value)
     171    db_connection.commit()
     172    cur.close()
     173    db_connection.close()
     174else:
     175    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     176                    ' VALUES (%s, %s, %s);'
     177    insert_value = (offer_shop, last_updated, 'success')
     178    cur.execute(insert_script, insert_value)
     179    db_connection.commit()
     180    cur.close()
     181    db_connection.close()
    124182
    125     if flag:
    126         print('ALREADY IN DATABASE')
    127         print(new_offer)
    128         # if it's already in database, check PRICE and if it's changed, change it !!!!!!
    129         if flag_price:
    130             print('PRICE CHANGED!')  # CHANGE PRICE
    131             print('offer id: ' + str(offer_id))
    132             headers = {'Content-type': 'application/json'}
    133             requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
    134                          headers=headers)
    135     else:
    136         print('ADDED')  # ADD OFFER
    137         print(new_offer)
    138         headers = {'Content-type': 'application/json'}
    139         requests.post('http://localhost:8080/phoneoffer/addoffer',
    140                       headers=headers, data=json.dumps(new_offer.__dict__, default=str))
    141 
    142 print('------------------------------------')
    143 
    144 for old_offer in database_offers:
    145     flag = False
    146     for new_offer in new_offers:
    147         if old_offer.offer_name == new_offer.offer_name:
    148             flag = True
    149 
    150     if not flag:
    151         print('OFFER DELETED')
    152         print(old_offer)
    153         # DELETE OFFER
    154         requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
    155 

  • phonelux_scrappers/scrappers/neptun_scrapper.py

    rffd50db r47f4eaf  
    11import json
     2import traceback
    23import unicodedata
    34from datetime import datetime
    … …  
    1920is_validated = False
    2021
    21 # Neptun phone offers that are already in database
     22# Call to read the configuration file and connect to database
     23cinfo = config_read.get_databaseconfig("../postgresdb.config")
     24db_connection = psycopg2.connect(
     25    database=cinfo[0],
     26    host=cinfo[1],
     27    user=cinfo[2],
     28    password=cinfo[3]
     29)
     30cur = db_connection.cursor()
    2231
    23 offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/neptun').text))
     32try:
     33    # Neptun phone offers that are already in database
     34    offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/neptun').text))
    2435
    25 database_offers = []
     36    database_offers = []
    2637
    27 for offer in offers:
    28     phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
    29                             offer['ram_memory'],
    30                             offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
    31                             offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
    32                             offer['image_url'],
    33                             offer['offer_url'], offer['last_updated'], offer['is_validated'],
    34                             offer['offer_description'],
    35                             offer['offer_shop_code'])
    36     database_offers.append(phoneOffer)
     38    for offer in offers:
     39        phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
     40                                offer['ram_memory'],
     41                                offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
     42                                offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
     43                                offer['image_url'],
     44                                offer['offer_url'], offer['last_updated'], offer['is_validated'],
     45                                offer['offer_description'],
     46                                offer['offer_shop_code'])
     47        database_offers.append(phoneOffer)
    3748
    38 new_offers = []
     49    new_offers = []
    3950
    40 for i in range(1, 11):
    41     neptun_url = 'https://www.neptun.mk/mobilni_telefoni.nspx?page=' + str(i)
     51    for i in range(1, 11):
     52        neptun_url = 'https://www.neptun.mk/mobilni_telefoni.nspx?page=' + str(i)
    4253
    43     # selenium is used because of the dynamic content of the page
    44     driver1 = webdriver.Safari(executable_path='/usr/bin/safaridriver')
    45     driver1.get(neptun_url)
    46     neptun_html = driver1.page_source
     54        # selenium is used because of the dynamic content of the page
     55        driver1 = webdriver.Safari(executable_path='/usr/bin/safaridriver')
     56        driver1.get(neptun_url)
     57        neptun_html = driver1.page_source
    4758
    48     # closing the driver so the safari instance can pair with another webdriver session
    49     driver1.close()
    50 
    51     # response1 = requests.get(neptun_url)
    52     soup1 = BeautifulSoup(neptun_html, 'html.parser')
    53 
    54     phones = soup1.find('div', {'id': 'mainContainer'}).find('div',
    55                                                              {'class': 'col-lg-9 col-md-9 col-sm-8 col-fix-main'}) \
    56         .find_all('div', {'class': 'ng-scope product-list-item-grid'})
    57 
    58     for phone in phones:
    59         offer_url = 'https://www.neptun.mk' + phone.find('a').get('href')
    60         offer_name = phone.find('a').find('h2').get_text().replace('MOB.TEL.', '').strip()
    61         brand = offer_name.split(' ')[0].strip().capitalize()
    62         image_url = 'https://www.neptun.mk' + phone.find('a').find('div', {'class': 'row'}).find('img').get('src')
    63         price = int(
    64             phone.find('div', {'class': 'col-sm-12 static'}).find('div', {'class': 'product-list-item__prices pt35'})
    65             .find('div', {'class': 'row'}).find('div', {'class': 'newPriceModel'}) \
    66             .find('span', {'class': 'product-price__amount--value ng-binding'}).get_text().replace('.', ''))
    67 
    68         driver1 = webdriver.Safari(executable_path='/usr/bin/safaridriver')
    69         driver1.get(offer_url)
    70         offer_html = driver1.page_source
    7159        # closing the driver so the safari instance can pair with another webdriver session
    7260        driver1.close()
    7361
    74         soup2 = BeautifulSoup(offer_html, 'html.parser')
     62        # response1 = requests.get(neptun_url)
     63        soup1 = BeautifulSoup(neptun_html, 'html.parser')
    7564
    76         offer_shop_code = soup2.find('div', {'ng-if': 'showProductDetails'}) \
    77             .find('div', {'class': 'product-details-first-row'}).find('span', {
    78             'ng-bind': 'model.CodeNumber'}).get_text().strip()
     65        phones = soup1.find('div', {'id': 'mainContainer'}).find('div',
     66                                                                 {'class': 'col-lg-9 col-md-9 col-sm-8 col-fix-main'}) \
     67            .find_all('div', {'class': 'ng-scope product-list-item-grid'})
    7968
    80         specifications_table = \
    81             soup2.find('div', {'id': 'mainContainer'}).find('div', {'ng-if': 'showProductDetails'}).find_all('ul')[-1]
    82         specifications = specifications_table.get_text(separator='\n').strip().split("\n")
     69        for phone in phones:
     70            offer_url = 'https://www.neptun.mk' + phone.find('a').get('href')
     71            offer_name = phone.find('a').find('h2').get_text().replace('MOB.TEL.', '').strip()
     72            brand = offer_name.split(' ')[0].strip().capitalize()
     73            image_url = 'https://www.neptun.mk' + phone.find('a').find('div', {'class': 'row'}).find('img').get('src')
     74            price = int(
     75                phone.find('div', {'class': 'col-sm-12 static'}).find('div', {'class': 'product-list-item__prices pt35'})
     76                .find('div', {'class': 'row'}).find('div', {'class': 'newPriceModel'}) \
     77                .find('span', {'class': 'product-price__amount--value ng-binding'}).get_text().replace('.', ''))
    8378
    84         offer_description = specifications_table.get_text(separator='\n').strip()
     79            driver1 = webdriver.Safari(executable_path='/usr/bin/safaridriver')
     80            driver1.get(offer_url)
     81            offer_html = driver1.page_source
     82            # closing the driver so the safari instance can pair with another webdriver session
     83            driver1.close()
    8584
    86         back_camera = None
    87         operating_system = None
    88         chipset = None
    89         battery = None
    90         ram_memory = None
    91         rom_memory = None
    92         cpu = None
    93         front_camera = None
    94         color = None
     85            soup2 = BeautifulSoup(offer_html, 'html.parser')
    9586
    96         for specification in specifications:
    97             if 'Батерија:' in specification:
    98                 battery = specification.split('Батерија:')[1]
     87            offer_shop_code = soup2.find('div', {'ng-if': 'showProductDetails'}) \
     88                .find('div', {'class': 'product-details-first-row'}).find('span', {
     89                'ng-bind': 'model.CodeNumber'}).get_text().strip()
    9990
    100             if 'CPU:' in specification:
    101                 cpu = specification.split('CPU:')[1]
     91            specifications_table = \
     92                soup2.find('div', {'id': 'mainContainer'}).find('div', {'ng-if': 'showProductDetails'}).find_all('ul')[-1]
     93            specifications = specifications_table.get_text(separator='\n').strip().split("\n")
    10294
    103             if 'Chipset:' in specification:
    104                 chipset = specification.split('Chipset:')[1]
     95            offer_description = specifications_table.get_text(separator='\n').strip()
    10596
    106             if 'RAM Меморија:' in specification:
    107                 ram_memory = specification.split('RAM Меморија:')[1]
    108                 continue
     97            back_camera = None
     98            operating_system = None
     99            chipset = None
     100            battery = None
     101            ram_memory = None
     102            rom_memory = None
     103            cpu = None
     104            front_camera = None
     105            color = None
    109106
    110             if 'ROM Меморија:' in specification:
    111                 rom_memory = specification.split('ROM Меморија:')[1]
    112                 continue
     107            for specification in specifications:
     108                if 'Батерија:' in specification:
     109                    battery = specification.split('Батерија:')[1]
    113110
    114             if 'ROM:' in specification:
    115                 rom_memory = specification.split('ROM:')[1]
     111                if 'CPU:' in specification:
     112                    cpu = specification.split('CPU:')[1]
    116113
    117             if 'RAM:' in specification:
    118                 ram_memory = specification.split('RAM:')[1]
     114                if 'Chipset:' in specification:
     115                    chipset = specification.split('Chipset:')[1]
    119116
    120             if 'iOS' in specification or 'Android' in specification:
    121                 operating_system = specification
     117                if 'RAM Меморија:' in specification:
     118                    ram_memory = specification.split('RAM Меморија:')[1]
     119                    continue
    122120
    123         new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
    124                                      color, front_camera, back_camera, chipset, battery, operating_system, cpu,
    125                                      image_url,
    126                                      offer_url, last_updated, is_validated, offer_description, offer_shop_code))
     121                if 'ROM Меморија:' in specification:
     122                    rom_memory = specification.split('ROM Меморија:')[1]
     123                    continue
    127124
    128 for new_offer in new_offers:
    129     flag = False
    130     flag_price = False
    131     offer_id = None
     125                if 'ROM:' in specification:
     126                    rom_memory = specification.split('ROM:')[1]
     127
     128                if 'RAM:' in specification:
     129                    ram_memory = specification.split('RAM:')[1]
     130
     131                if 'iOS' in specification or 'Android' in specification:
     132                    operating_system = specification
     133
     134            new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
     135                                         color, front_camera, back_camera, chipset, battery, operating_system, cpu,
     136                                         image_url,
     137                                         offer_url, last_updated, is_validated, offer_description, offer_shop_code))
     138
     139    for new_offer in new_offers:
     140        flag = False
     141        flag_price = False
     142        offer_id = None
     143
     144        for old_offer in database_offers:
     145
     146            if new_offer.offer_shop_code == old_offer.offer_shop_code:
     147                flag = True
     148                if new_offer.price != old_offer.price:
     149                    flag_price = True
     150                    offer_id = old_offer.offer_id
     151
     152        if flag:
     153            # print('ALREADY IN DATABASE')
     154            # print(new_offer)
     155            # if it's already in database, check PRICE and if it's changed, change it !!!!!!
     156            if flag_price:
     157                print('PRICE CHANGED!')  # CHANGE PRICE
     158                print('offer id: ' + str(offer_id))
     159                headers = {'Content-type': 'application/json'}
     160                requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
     161                             headers=headers)
     162        else:
     163            print('ADDED')  # ADD OFFER
     164            print(new_offer)
     165            headers = {'Content-type': 'application/json'}
     166            requests.post('http://localhost:8080/phoneoffer/addoffer',
     167                          headers=headers, data=json.dumps(new_offer.__dict__, default=str))
     168
     169    print('------------------------------------')
    132170
    133171    for old_offer in database_offers:
     172        flag = False
     173        for new_offer in new_offers:
     174            if old_offer.offer_shop_code == new_offer.offer_shop_code:
     175                flag = True
    134176
    135         if new_offer.offer_shop_code == old_offer.offer_shop_code:
    136             flag = True
    137             if new_offer.price != old_offer.price:
    138                 flag_price = True
    139                 offer_id = old_offer.offer_id
    140 
    141     if flag:
    142         # print('ALREADY IN DATABASE')
    143         # print(new_offer)
    144         # if it's already in database, check PRICE and if it's changed, change it !!!!!!
    145         if flag_price:
    146             print('PRICE CHANGED!')  # CHANGE PRICE
    147             print('offer id: ' + str(offer_id))
    148             headers = {'Content-type': 'application/json'}
    149             requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
    150                          headers=headers)
    151     else:
    152         print('ADDED')  # ADD OFFER
    153         print(new_offer)
    154         headers = {'Content-type': 'application/json'}
    155         requests.post('http://localhost:8080/phoneoffer/addoffer',
    156                       headers=headers, data=json.dumps(new_offer.__dict__, default=str))
    157 
    158 print('------------------------------------')
    159 
    160 for old_offer in database_offers:
    161     flag = False
    162     for new_offer in new_offers:
    163         if old_offer.offer_shop_code == new_offer.offer_shop_code:
    164             flag = True
    165 
    166     if not flag:
    167         print('OFFER DELETED')
    168         print(old_offer)
    169         # DELETE OFFER
    170         requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     177        if not flag:
     178            print('OFFER DELETED')
     179            print(old_offer)
     180            # DELETE OFFER
     181            requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     182except Exception:
     183    traceback.print_exc()
     184    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     185                    ' VALUES (%s, %s, %s);'
     186    insert_value = (offer_shop, last_updated, 'failed')
     187    cur.execute(insert_script, insert_value)
     188    db_connection.commit()
     189    cur.close()
     190    db_connection.close()
     191else:
     192    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     193                    ' VALUES (%s, %s, %s);'
     194    insert_value = (offer_shop, last_updated, 'success')
     195    cur.execute(insert_script, insert_value)
     196    db_connection.commit()
     197    cur.close()
     198    db_connection.close()

  • phonelux_scrappers/scrappers/outputfile.txt

    rffd50db r47f4eaf  
    1 ADDED
    2 {'offer_shop': 'Mobile Zone', 'offer_name': 'Apple iPhone 14 Pro', 'price': 95499, 'ram_memory': None, 'rom_memory': '128GB', 'color': 'Златна, Розева, Сива, Црна', 'front_camera': '12MP', 'back_camera': '48 Mp + 12 Mp + 12 Mp', 'chipset': None, 'battery': '3200mAh', 'operating_system': None, 'cpu': None, 'image_url': 'https://i0.wp.com/mobilezone.mk/wp-content/uploads/2022/09/14-pro-silver.png?resize=600%2C600&ssl=1', 'offer_url': 'https://mobilezone.mk/produkti/iphone-14-pro/', 'last_updated': datetime.date(2022, 10, 1), 'is_validated': False, 'offer_description': None, 'offer_shop_code': None}
    3 ------------------------------------
    4 OFFER DELETED
    5 {'offer_id': 1179, 'offer_shop': 'Mobile Zone', 'offer_name': 'Samsung s20 FE', 'price': 24699, 'ram_memory': None, 'rom_memory': '128GB', 'color': 'Сина', 'front_camera': None, 'back_camera': None, 'chipset': None, 'battery': None, 'operating_system': None, 'cpu': None, 'image_url': 'https://i2.wp.com/mobilezone.mk/wp-content/uploads/2022/03/Samsung-Galaxy-S20-FE-blue.png?resize=512%2C600&ssl=1', 'offer_url': 'https://mobilezone.mk/produkti/samsung-s20-fe/', 'last_updated': '2022-07-29T22:00:00.000+00:00', 'is_validated': False, 'offer_description': None, 'offer_shop_code': None}
    6 OFFER DELETED
    7 {'offer_id': 1181, 'offer_shop': 'Mobile Zone', 'offer_name': 'Samsung Z Flip3 5G', 'price': 39999, 'ram_memory': None, 'rom_memory': '128GB', 'color': 'Црна', 'front_camera': None, 'back_camera': None, 'chipset': None, 'battery': None, 'operating_system': None, 'cpu': None, 'image_url': 'https://i2.wp.com/mobilezone.mk/wp-content/uploads/2022/03/11.png?resize=600%2C600&ssl=1', 'offer_url': 'https://mobilezone.mk/produkti/samsung-z-flip3-5g/', 'last_updated': '2022-07-29T22:00:00.000+00:00', 'is_validated': False, 'offer_description': None, 'offer_shop_code': None}
    8 OFFER DELETED
    9 {'offer_id': 1180, 'offer_shop': 'Mobile Zone', 'offer_name': 'Samsung S21 FE 5G', 'price': 30899, 'ram_memory': None, 'rom_memory': '128GB', 'color': 'Зелена, Црна', 'front_camera': None, 'back_camera': None, 'chipset': None, 'battery': None, 'operating_system': None, 'cpu': None, 'image_url': 'https://i1.wp.com/mobilezone.mk/wp-content/uploads/2022/03/5g.jpg?resize=600%2C600&ssl=1', 'offer_url': 'https://mobilezone.mk/produkti/samsung-s21-fe-5g/', 'last_updated': '2022-07-29T22:00:00.000+00:00', 'is_validated': False, 'offer_description': None, 'offer_shop_code': None}

  • phonelux_scrappers/scrappers/setec_scrapper.py

    rffd50db r47f4eaf  
    11import json
     2import traceback
    23import unicodedata
    34from datetime import datetime
    … …  
    1718is_validated = False
    1819
    19 # Setec phone offers that are already in database
     20# Call to read the configuration file and connect to database
     21cinfo = config_read.get_databaseconfig("../postgresdb.config")
     22db_connection = psycopg2.connect(
     23    database=cinfo[0],
     24    host=cinfo[1],
     25    user=cinfo[2],
     26    password=cinfo[3]
     27)
     28cur = db_connection.cursor()
    2029
    21 offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/setec').text))
     30try:
     31    # Setec phone offers that are already in database
     32    offers = json.loads(unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/setec').text))
    2233
    23 database_offers = []
     34    database_offers = []
    2435
    25 for offer in offers:
    26     phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
    27                             offer['ram_memory'],
    28                             offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
    29                             offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
    30                             offer['image_url'],
    31                             offer['offer_url'], offer['last_updated'], offer['is_validated'],
    32                             offer['offer_description'],
    33                             offer['offer_shop_code'])
    34     database_offers.append(phoneOffer)
     36    for offer in offers:
     37        phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
     38                                offer['ram_memory'],
     39                                offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
     40                                offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
     41                                offer['image_url'],
     42                                offer['offer_url'], offer['last_updated'], offer['is_validated'],
     43                                offer['offer_description'],
     44                                offer['offer_shop_code'])
     45        database_offers.append(phoneOffer)
    3546
    36 new_offers = []
     47    new_offers = []
    3748
    38 for i in range(1, 9):
    39     setec_url = 'https://setec.mk/index.php?route=product/category&path=10066_10067&page=' + str(i)
     49    for i in range(1, 9):
     50        setec_url = 'https://setec.mk/index.php?route=product/category&path=10066_10067&page=' + str(i)
    4051
    41     response1 = requests.get(setec_url)
    42     soup1 = BeautifulSoup(response1.content, 'html.parser')
     52        response1 = requests.get(setec_url)
     53        soup1 = BeautifulSoup(response1.content, 'html.parser')
    4354
    44     phones = soup1.find('div', {'id': 'mfilter-content-container'}) \
    45         .find_all('div', {'class': 'col-sm-4 col-xs-6'})
     55        phones = soup1.find('div', {'id': 'mfilter-content-container'}) \
     56            .find_all('div', {'class': 'col-sm-4 col-xs-6'})
    4657
    47     for phone in phones:
    48         offer_url = phone.find('div', {'class': 'left'}).find('a').get('href')
    49         image_url = phone.find('div', {'class': 'left'}).find('a').find('img').get('src')
    50         offer_name = phone.find('div', {'class': 'right'}).find('div', {'class': 'name'}).find('a').get_text().strip()
    51         brand = offer_name.split(' ')[0]
     58        for phone in phones:
     59            offer_url = phone.find('div', {'class': 'left'}).find('a').get('href')
     60            image_url = phone.find('div', {'class': 'left'}).find('a').find('img').get('src')
     61            offer_name = phone.find('div', {'class': 'right'}).find('div', {'class': 'name'}).find('a').get_text().strip()
     62            brand = offer_name.split(' ')[0]
    5263
    53         back_camera = None
    54         operating_system = None
    55         chipset = None
    56         battery = None
    57         ram_memory = None
    58         rom_memory = None
    59         cpu = None
    60         front_camera = None
    61         color = None
     64            back_camera = None
     65            operating_system = None
     66            chipset = None
     67            battery = None
     68            ram_memory = None
     69            rom_memory = None
     70            cpu = None
     71            front_camera = None
     72            color = None
    6273
    63         if 'Cable' in offer_name or 'AirTag' in offer_name:
    64             continue
     74            if 'Cable' in offer_name or 'AirTag' in offer_name:
     75                continue
    6576
    66         if brand not in offer_name:
    67             offer_name = brand + " " + offer_name
     77            if brand not in offer_name:
     78                offer_name = brand + " " + offer_name
    6879
    69         offer_shop_code = phone.find('div', {'class': 'right'}) \
    70             .find('div', {'class': 'shifra'}).get_text().replace('Шифра:', '').strip()
     80            offer_shop_code = phone.find('div', {'class': 'right'}) \
     81                .find('div', {'class': 'shifra'}).get_text().replace('Шифра:', '').strip()
    7182
    72         price_tag = phone.find('div', {'class': 'right'}).find('div', {'class': 'price'}). \
    73             find('div', {'class': 'category-price-redovna'}).find('span', {'class': 'price-old-new'})
     83            price_tag = phone.find('div', {'class': 'right'}).find('div', {'class': 'price'}). \
     84                find('div', {'class': 'category-price-redovna'}).find('span', {'class': 'price-old-new'})
    7485
    75         if price_tag is None:
    76             price_tag = phone.find('div', {'class': 'right'}).find('div', {'class': 'price'}). \
    77                 find('div', {'class': 'category-price-redovna'}).find('span', {'class': 'cena_za_kesh'})
     86            if price_tag is None:
     87                price_tag = phone.find('div', {'class': 'right'}).find('div', {'class': 'price'}). \
     88                    find('div', {'class': 'category-price-redovna'}).find('span', {'class': 'cena_za_kesh'})
    7889
    79         price = int(price_tag.get_text().replace('Ден.', '').replace(',', '').strip())
     90            price = int(price_tag.get_text().replace('Ден.', '').replace(',', '').strip())
    8091
    81         response2 = requests.get(offer_url)
    82         soup2 = BeautifulSoup(response2.content, 'html.parser')
     92            response2 = requests.get(offer_url)
     93            soup2 = BeautifulSoup(response2.content, 'html.parser')
    8394
    84         offer_description = soup2.find('div', {'id': 'tab-description'}).get_text(separator='\n')
     95            offer_description = soup2.find('div', {'id': 'tab-description'}).get_text(separator='\n')
    8596
    86         new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
    87                                      color, front_camera, back_camera, chipset, battery, operating_system, cpu,
    88                                      image_url,
    89                                      offer_url, last_updated, is_validated, offer_description, offer_shop_code))
     97            new_offers.append(PhoneOffer(offer_shop, offer_name, price, ram_memory, rom_memory,
     98                                         color, front_camera, back_camera, chipset, battery, operating_system, cpu,
     99                                         image_url,
     100                                         offer_url, last_updated, is_validated, offer_description, offer_shop_code))
    90101
    91 for new_offer in new_offers:
    92     flag = False
    93     flag_price = False
    94     offer_id = None
     102    for new_offer in new_offers:
     103        flag = False
     104        flag_price = False
     105        offer_id = None
     106
     107        for old_offer in database_offers:
     108
     109            if new_offer.offer_shop_code == old_offer.offer_shop_code:
     110                flag = True
     111                if new_offer.price != old_offer.price:
     112                    flag_price = True
     113                    offer_id = old_offer.offer_id
     114
     115        if flag:
     116            # print('ALREADY IN DATABASE')
     117            # print(new_offer)
     118            # if it's already in database, check PRICE and if it's changed, change it !!!!!!
     119            if flag_price:
     120                print('PRICE CHANGED!')  # CHANGE PRICE
     121                print('offer id: ' + str(offer_id))
     122                headers = {'Content-type': 'application/json'}
     123                requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
     124                             headers=headers)
     125        else:
     126            print('ADDED')  # ADD OFFER
     127            print(new_offer)
     128            headers = {'Content-type': 'application/json'}
     129            requests.post('http://localhost:8080/phoneoffer/addoffer',
     130                          headers=headers, data=json.dumps(new_offer.__dict__, default=str))
     131
     132    print('------------------------------------')
    95133
    96134    for old_offer in database_offers:
     135        flag = False
     136        for new_offer in new_offers:
     137            if old_offer.offer_shop_code == new_offer.offer_shop_code:
     138                flag = True
    97139
    98         if new_offer.offer_shop_code == old_offer.offer_shop_code:
    99             flag = True
    100             if new_offer.price != old_offer.price:
    101                 flag_price = True
    102                 offer_id = old_offer.offer_id
    103 
    104     if flag:
    105         # print('ALREADY IN DATABASE')
    106         # print(new_offer)
    107         # if it's already in database, check PRICE and if it's changed, change it !!!!!!
    108         if flag_price:
    109             print('PRICE CHANGED!')  # CHANGE PRICE
    110             print('offer id: ' + str(offer_id))
    111             headers = {'Content-type': 'application/json'}
    112             requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
    113                          headers=headers)
    114     else:
    115         print('ADDED')  # ADD OFFER
    116         print(new_offer)
    117         headers = {'Content-type': 'application/json'}
    118         requests.post('http://localhost:8080/phoneoffer/addoffer',
    119                       headers=headers, data=json.dumps(new_offer.__dict__, default=str))
    120 
    121 print('------------------------------------')
    122 
    123 for old_offer in database_offers:
    124     flag = False
    125     for new_offer in new_offers:
    126         if old_offer.offer_shop_code == new_offer.offer_shop_code:
    127             flag = True
    128 
    129     if not flag:
    130         print('OFFER DELETED')
    131         print(old_offer)
    132         # DELETE OFFER
    133         requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     140        if not flag:
     141            print('OFFER DELETED')
     142            print(old_offer)
     143            # DELETE OFFER
     144            requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     145except Exception:
     146    traceback.print_exc()
     147    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     148                    ' VALUES (%s, %s, %s);'
     149    insert_value = (offer_shop, last_updated, 'failed')
     150    cur.execute(insert_script, insert_value)
     151    db_connection.commit()
     152    cur.close()
     153    db_connection.close()
     154else:
     155    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     156                    ' VALUES (%s, %s, %s);'
     157    insert_value = (offer_shop, last_updated, 'success')
     158    cur.execute(insert_script, insert_value)
     159    db_connection.commit()
     160    cur.close()
     161    db_connection.close()

  • phonelux_scrappers/scrappers/tehnomarket_scrapper.py

    rffd50db r47f4eaf  
    11import json
     2import traceback
    23import unicodedata
    34from datetime import datetime
    … …  
    7677
    7778
    78 # Tehnomarket phone offers that are already in database
     79# Call to read the configuration file and connect to database
     80cinfo = config_read.get_databaseconfig("../postgresdb.config")
     81db_connection = psycopg2.connect(
     82    database=cinfo[0],
     83    host=cinfo[1],
     84    user=cinfo[2],
     85    password=cinfo[3]
     86)
     87cur = db_connection.cursor()
    7988
    80 offers = json.loads(
    81     unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/tehnomarket').text))
     89try:
     90    # Tehnomarket phone offers that are already in database
     91    offers = json.loads(
     92        unicodedata.normalize('NFKD', requests.get('http://localhost:8080/phoneoffer/shop/tehnomarket').text))
    8293
    83 database_offers = []
     94    database_offers = []
    8495
    85 for offer in offers:
    86     phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
    87                             offer['ram_memory'],
    88                             offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
    89                             offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
    90                             offer['image_url'],
    91                             offer['offer_url'], offer['last_updated'], offer['is_validated'],
    92                             offer['offer_description'],
    93                             offer['offer_shop_code'])
    94     database_offers.append(phoneOffer)
     96    for offer in offers:
     97        phoneOffer = PhoneOffer(offer['id'], offer['offer_shop'], offer['offer_name'], offer['price'],
     98                                offer['ram_memory'],
     99                                offer['rom_memory'], offer['color'], offer['front_camera'], offer['back_camera'],
     100                                offer['chipset'], offer['battery'], offer['operating_system'], offer['cpu'],
     101                                offer['image_url'],
     102                                offer['offer_url'], offer['last_updated'], offer['is_validated'],
     103                                offer['offer_description'],
     104                                offer['offer_shop_code'])
     105        database_offers.append(phoneOffer)
    95106
    96 new_offers = []
     107    new_offers = []
    97108
    98 for i in range(1, 6):
    99     tehnomarket_url = 'https://tehnomarket.com.mk/category/4109/mobilni-telefoni#page/' + str(i)
    100     # print(anhoch_url)
     109    for i in range(1, 6):
     110        tehnomarket_url = 'https://tehnomarket.com.mk/category/4109/mobilni-telefoni#page/' + str(i)
     111        # print(anhoch_url)
    101112
    102     # selenium is used because of the dynamic content of the page
    103     driver1 = webdriver.Safari(executable_path='/usr/bin/safaridriver')
    104     driver1.get(tehnomarket_url)
     113        # selenium is used because of the dynamic content of the page
     114        driver1 = webdriver.Safari(executable_path='/usr/bin/safaridriver')
     115        driver1.get(tehnomarket_url)
    105116
    106     scrape_function(driver1, i, new_offers)
     117        scrape_function(driver1, i, new_offers)
    107118
    108     # closing the driver so the safari instance can pair with another webdriver session
    109     driver1.close()
     119        # closing the driver so the safari instance can pair with another webdriver session
     120        driver1.close()
    110121
    111 for new_offer in new_offers:
    112     flag = False
    113     flag_price = False
    114     offer_id = None
     122    for new_offer in new_offers:
     123        flag = False
     124        flag_price = False
     125        offer_id = None
     126
     127        for old_offer in database_offers:
     128
     129            if new_offer.offer_shop_code == old_offer.offer_shop_code:
     130                flag = True
     131                if new_offer.price != old_offer.price:
     132                    flag_price = True
     133                    offer_id = old_offer.offer_id
     134
     135        if flag:
     136            # print('ALREADY IN DATABASE')
     137            # print(new_offer)
     138            # if it's already in database, check PRICE and if it's changed, change it !!!!!!
     139            if flag_price:
     140                print('PRICE CHANGED!')  # CHANGE PRICE
     141                print('offer id: ' + str(offer_id))
     142                headers = {'Content-type': 'application/json'}
     143                requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
     144                             headers=headers)
     145        else:
     146            print('ADDED')  # ADD OFFER
     147            print(new_offer)
     148            headers = {'Content-type': 'application/json'}
     149            requests.post('http://localhost:8080/phoneoffer/addoffer',
     150                          headers=headers, data=json.dumps(new_offer.__dict__, default=str))
     151
     152    print('------------------------------------')
    115153
    116154    for old_offer in database_offers:
     155        flag = False
     156        for new_offer in new_offers:
     157            if old_offer.offer_shop_code == new_offer.offer_shop_code:
     158                flag = True
    117159
    118         if new_offer.offer_shop_code == old_offer.offer_shop_code:
    119             flag = True
    120             if new_offer.price != old_offer.price:
    121                 flag_price = True
    122                 offer_id = old_offer.offer_id
    123 
    124     if flag:
    125         # print('ALREADY IN DATABASE')
    126         # print(new_offer)
    127         # if it's already in database, check PRICE and if it's changed, change it !!!!!!
    128         if flag_price:
    129             print('PRICE CHANGED!')  # CHANGE PRICE
    130             print('offer id: ' + str(offer_id))
    131             headers = {'Content-type': 'application/json'}
    132             requests.put('http://localhost:8080/phoneoffer/' + str(offer_id) + '/changeprice/' + str(new_offer.price),
    133                          headers=headers)
    134     else:
    135         print('ADDED')  # ADD OFFER
    136         print(new_offer)
    137         headers = {'Content-type': 'application/json'}
    138         requests.post('http://localhost:8080/phoneoffer/addoffer',
    139                       headers=headers, data=json.dumps(new_offer.__dict__, default=str))
    140 
    141 print('------------------------------------')
    142 
    143 for old_offer in database_offers:
    144     flag = False
    145     for new_offer in new_offers:
    146         if old_offer.offer_shop_code == new_offer.offer_shop_code:
    147             flag = True
    148 
    149     if not flag:
    150         print('OFFER DELETED')
    151         print(old_offer)
    152         # DELETE OFFER
    153         requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     160        if not flag:
     161            print('OFFER DELETED')
     162            print(old_offer)
     163            # DELETE OFFER
     164            requests.delete('http://localhost:8080/phoneoffer/deleteoffer/' + str(old_offer.offer_id))
     165except Exception:
     166    traceback.print_exc()
     167    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     168                    ' VALUES (%s, %s, %s);'
     169    insert_value = ('Tehnomarket', datetime.now().date(), 'failed')
     170    cur.execute(insert_script, insert_value)
     171    db_connection.commit()
     172    cur.close()
     173    db_connection.close()
     174else:
     175    insert_script = 'INSERT INTO scrapper_info (store, recieved_at, status)' \
     176                    ' VALUES (%s, %s, %s);'
     177    insert_value = ('Tehnomarket', datetime.now().date(), 'success')
     178    cur.execute(insert_script, insert_value)
     179    db_connection.commit()
     180    cur.close()
     181    db_connection.close()
Note: See TracChangeset for help on using the changeset viewer.