分享一个简单的java爬虫框架

package DownloadPackage;import java.io.BufferedReader;import java.io.IOException;import java.io.InputStreamReader;import org.apache.http.HttpEntity;import org.apache.http.HttpResponse;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClients;public class HtmlDownloader {RequestSet requestset = null;public HtmlDownloader(RequestSet requestset){this.requestset = requestset;}public String downloadhtml(String url){String html = null;//创建一个客户端//创建一个读取流从entity读取htmlBufferedReader reader = null;CloseableHttpClient httpclient = HttpClients.createDefault();HttpResponse response = null;try {response = httpclient.execute(requestset.getMethod(url));HttpEntity entity = response.getEntity();reader = new BufferedReader(new InputStreamReader(entity.getContent()));StringBuilder sb = new StringBuilder();while((html = reader.readLine()) != null){sb.append(html);}html = sb.toString();System.out.println("一个html页面获取成功");}catch (IOException e) {System.out.println(url+"连接失败");}finally{if(reader != null){try {reader.close();httpclient.close();}catch (IOException e) {// TODO Auto-generated catch blocke.printStackTrace();}}}return html;}}