source: backend/GlobeGuru-backend/src/main/java/Scraper.java@ 1c51912

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CountDownLatch;

public class Scraper implements Callable<Void> {

    private List<String> urls;
    private ConcurrentLinkedQueue<Option> optionsQueue;
    private CountDownLatch latch;

    public Scraper() {
        urls = new ArrayList<>();
        this.optionsQueue = new ConcurrentLinkedQueue<>();
        ObjectMapper mapper = new ObjectMapper();
        try {
            ClassLoader classLoader = getClass().getClassLoader();
            JsonNode root = mapper.readTree(new File(classLoader.getResource("URLsJSON.json").getFile()));

            JsonNode urlNode = root.get("agencyurls");
            if (urlNode.isArray()) {
                Iterator<JsonNode> elements = urlNode.elements();
                while (elements.hasNext()) {
                    JsonNode next = elements.next();
                    urls.add(next.asText());
                }
            }
            System.out.println("Loaded " + urls.size() + " urls");
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        this.latch = new CountDownLatch(urls.size());
    }


    public Void call() {
        System.out.println("Scraper has started ");
        for (String url : urls) {
            new ScraperThread(url, optionsQueue, latch).start();
        }
        return null;
    }
}