文章详情

短信预约-IT技能 免费直播动态提醒

请输入下面的图形验证码

提交验证

短信预约提醒成功

Java HttpClient爬虫请求

2023-08-17 08:13

关注

**本项目采用spring-boot构建, maven工程

添加依赖

pom文件

  4.0.0  org.test  testDome  0.0.1-SNAPSHOT      org.apache.httpcomponents  httpclient  4.5.5      com.alibaba  fastjson  1.2.47    

GET 无参形式

package testDemo;import org.apache.http.Header;import org.apache.http.HttpHost;import org.apache.http.client.config.RequestConfig;import org.apache.http.client.methods.CloseableHttpResponse;import org.apache.http.client.methods.HttpGet;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClients;import org.apache.http.util.EntityUtils;public class DoGET {    public static void main(String[] args) throws Exception {//        RequestConfig config = RequestConfig.custom().setRedirectsEnabled(false).build();//不允许重定向//        CloseableHttpClient httpClient = HttpClients.custom().setDefaultRequestConfig(config).build();//        proxyHost -- 代理ip; proxyPort -- 端口号//        int proxyPort = 8000;//        String proxyHost = "192.10.2.125";//        HttpHost proxy = new HttpHost(proxyHost, proxyPort, "HTTP");        //创建Httpclient对象        CloseableHttpClient httpclient = HttpClients.createDefault();        //get请求(忽略SSL证书),获取结果        // TODO: 2020/4/27  忽略SSL证书        //创建http GET请求        HttpGet get = new HttpGet("http://www.baidu.com");//        CloseableHttpResponse response = httpclient.execute(proxy, get);        CloseableHttpResponse response = httpclient.execute(get);        try {            // 执行请求            response = httpclient.execute(get);            // 判断返回状态是否为200            if (response.getStatusLine().getStatusCode() == 200) {                //请求体内容    String content = EntityUtils.toString(response.getEntity(), "UTF-8");                //内容                System.out.println("<<" + content + ">>");                System.out.println("内容长度:" + content.length());//                Header[] cookie =  response.getHeaders("Set-Cookie");            }        } finally {            if (response != null) {                response.close();            }            //相当于关闭浏览器            httpclient.close();        }    }}

GET带参请求

package testDemo;import java.io.File;import java.net.URI;import org.apache.http.client.methods.CloseableHttpResponse;import org.apache.http.client.methods.HttpGet;import org.apache.http.client.utils.URIBuilder;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClients;import org.apache.http.util.EntityUtils;public class DoGETParam {    public static void main(String[] args) throws Exception {        // 创建Httpclient对象        CloseableHttpClient httpclient = HttpClients.createDefault();        // 定义请求的参数        URI uri = new URIBuilder("http://www.baidu.com/s").setParameter("wd", "java").build();        // 创建http GET请求        HttpGet httpGet = new HttpGet(uri);        //response 对象        CloseableHttpResponse response = null;        try {            // 执行http get请求            response = httpclient.execute(httpGet);            // 判断返回状态是否为200            if (response.getStatusLine().getStatusCode() == 200) {                String content = EntityUtils.toString(response.getEntity(), "UTF-8");                //内容                System.out.println("内容长度:" + content.length());                System.out.println("内容<<:" + content);            }        } finally {            if (response != null) {                response.close();            }            httpclient.close();        }    }}

POST无参请求

package testDemo;import org.apache.http.client.methods.CloseableHttpResponse;import org.apache.http.client.methods.HttpPost;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClients;import org.apache.http.util.EntityUtils;public class DoPOST {    public static void main(String[] args) throws Exception {        // 创建Httpclient对象        CloseableHttpClient httpclient = HttpClients.createDefault();        // 创建http POST请求        HttpPost httpPost = new HttpPost("http://www.oschina.net/");        //伪装浏览器请求        httpPost.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");        CloseableHttpResponse response = null;        try {            // 执行请求            response = httpclient.execute(httpPost);            // 判断返回状态是否为200            if (response.getStatusLine().getStatusCode() == 200) {                String content = EntityUtils.toString(response.getEntity(), "UTF-8");                //内容写入文件                System.out.println(">>" + content);                System.out.println("内容长度:" + content.length());            } else {                System.out.println(response.getStatusLine().getStatusCode());                String content = EntityUtils.toString(response.getEntity(), "UTF-8");                System.out.println(">>" + content);            }        } finally {            if (response != null) {                response.close();            }            httpclient.close();        }    }}

POST带参请求

package testDemo;import java.io.File;import java.util.ArrayList;import java.util.List;import org.apache.http.NameValuePair;import org.apache.http.client.entity.UrlEncodedFormEntity;import org.apache.http.client.methods.CloseableHttpResponse;import org.apache.http.client.methods.HttpPost;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClients;import org.apache.http.message.BasicNameValuePair;import org.apache.http.util.EntityUtils;public class DoPOSTParam {    public static void main(String[] args) throws Exception {        // 创建Httpclient对象        CloseableHttpClient httpclient = HttpClients.createDefault();        // 创建http POST请求        HttpPost httpPost = new HttpPost("http://www.bcia.com.cn/bcia/FAQ/search");        // 设置2个post参数        List parameters = new ArrayList(0);        parameters.add(new BasicNameValuePair("lang", "cn"));        parameters.add(new BasicNameValuePair("pageNum", "1"));        // 构造一个form表单式的实体        UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(parameters);        // 将请求实体设置到httpPost对象中        httpPost.setEntity(formEntity);        //伪装浏览器        httpPost.setHeader("Referer", "http://www.bcia.com.cn/cjwt.html");        httpPost.setHeader("Host", "www.bcia.com.cn");        httpPost.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36");        CloseableHttpResponse response = null;        try {            // 执行请求            response = httpclient.execute(httpPost);            // 判断返回状态是否为200            if (response.getStatusLine().getStatusCode() == 200) {                String content = EntityUtils.toString(response.getEntity(), "UTF-8");                System.out.println("内容" + content);                System.out.println("内容长度:" + content.length());            } else                System.out.println("内容111" + response.getStatusLine().getStatusCode());        } finally {            if (response != null)                response.close();        }        httpclient.close();    }}

来源地址:https://blog.csdn.net/qq_41369057/article/details/131222505

阅读原文内容投诉

免责声明:

① 本站未注明“稿件来源”的信息均来自网络整理。其文字、图片和音视频稿件的所属权归原作者所有。本站收集整理出于非商业性的教育和科研之目的,并不意味着本站赞同其观点或证实其内容的真实性。仅作为临时的测试数据,供内部测试之用。本站并未授权任何人以任何方式主动获取本站任何信息。

② 本站未注明“稿件来源”的临时测试数据将在测试完成后最终做删除处理。有问题或投稿请发送至: 邮箱/279061341@qq.com QQ/279061341

软考中级精品资料免费领

  • 历年真题答案解析
  • 备考技巧名师总结
  • 高频考点精准押题
  • 2024年上半年信息系统项目管理师第二批次真题及答案解析(完整版)

    难度     813人已做
    查看
  • 【考后总结】2024年5月26日信息系统项目管理师第2批次考情分析

    难度     354人已做
    查看
  • 【考后总结】2024年5月25日信息系统项目管理师第1批次考情分析

    难度     318人已做
    查看
  • 2024年上半年软考高项第一、二批次真题考点汇总(完整版)

    难度     435人已做
    查看
  • 2024年上半年系统架构设计师考试综合知识真题

    难度     224人已做
    查看

相关文章

发现更多好内容

猜你喜欢

AI推送时光机
位置:首页-资讯-后端开发
咦!没有更多了?去看看其它编程学习网 内容吧
首页课程
资料下载
问答资讯