Java应用开源框架实现简易web搜索引擎

package webCrawler.Spider;import java.io.IOException;import java.util.ArrayList;import java.util.HashSet;import java.util.Scanner;import org.jsoup.Jsoup;import org.jsoup.nodes.Document;import org.jsoup.nodes.Element;import org.jsoup.select.Elements;import webCrawler.Index.BuildIndex;public class Spider {  ArrayList<String> URLs;  private String startURL;  private int digLevel;    public Spider(String startURL, int digLevel){    this.startURL = startURL;    this.digLevel = digLevel;    this.URLs = new ArrayList<>();  }    public ArrayList<String> getLevelURLs(int level, ArrayList<String> arrayList)       throws IOException{    ArrayList<String> total = null;    if(level>0){            total = new ArrayList<>();      for(String url: arrayList){                for(String each: getBareLinks(url)){          total.add(each);        }      }            HashSet<String> hashSet = new HashSet<>(total);      total = new ArrayList<>(hashSet);    }    return total;  }    public void getAll() throws IOException{    ArrayList<String> newURLs;    ArrayList<String> currentURLs = new ArrayList<>();        currentURLs.add(startURL);    for(int i=digLevel; i>0; i--){            System.out.println("Dig into level: " + (digLevel-i+1));      newURLs = getLevelURLs(i, currentURLs);      for(String each: currentURLs){        URLs.add(each);      }      currentURLs = newURLs;    }    for(String each:currentURLs){      URLs.add(each);    }    HashSet<String> hashSet = new HashSet<>(URLs);    URLs = new ArrayList<>(hashSet);  }    public void storeURLsAndInfo(String path) throws IOException{    BuildIndex build = new BuildIndex(path);        for(String each:URLs){      String text = getLinkText(each);      if(text!=null){        build.addField("url", each);        build.addField("text", text);                build.pushIndex();      }    }    build.close();  }    public String getLinkText(String url) throws IOException{    Document document = null;    try {            document = Jsoup.connect(url).timeout(3000).get();    } catch (Exception e) {      System.out.println("[TIMEOUT]Get title of url:"+url);      return null;    }    String title = document.title();    return title;  }    public ArrayList<String> getBareLinks(String url) throws IOException{    ArrayList<String> linksList = new ArrayList<>();    Document document;    try {      document = Jsoup.connect(url).timeout(2000).get();    } catch (Exception e) {      return linksList;    }        Elements links = document.select("body").select("a[href]");    for(Element link: links){            String href = link.attr("abs:href").replaceAll("#", "");            if(href.contains("zju.edu.cn")){        if (href.endsWith("/")){          href = href.substring(0, href.length()-1);        }        linksList.add(href);      }    }    HashSet<String> hashSet = new HashSet<>(linksList);    ArrayList<String> arrayList = new ArrayList<>(hashSet);    return arrayList;  }  public static void main(String[] args) {    Scanner in = new Scanner(System.in);    System.out.println("Enter url:");    String url = in.nextLine().trim();    while(!url.startsWith("http://")){      System.out.println("http:// is needed!");      System.out.println("Enter url:");      url = in.nextLine().trim();    }    System.out.println("Enter depth to dig more urls[<=3 recommended]：");    int depth = in.nextInt();    Spider spider = new Spider(url, depth);    System.out.println("Enter path you want to save[default=d:/index-spider]:");    String path = in.nextLine().trim();    if(path.length()==0){      path = "d:/index-spider";    }    try {      System.out.println("Start fetching...");      spider.getAll();      System.out.println("Urls got success!");      spider.storeURLsAndInfo(path);      System.out.println("Stored success!");    } catch (IOException e) {      e.printStackTrace();    }  } }