java 实现通过 post 方式提交json参数操作
由于所爬取的网站需要验证码,通过网页的开发人员工具【F12】及在线httppost,get接口测试请求工具(http://coolaf.com/)发现访问时加上请求头header信息时可以跳过验证码校验。
而且该网站只接受post请求,对提交的参数也只接受json格式,否则请求失败。
现将通过post方式提交json参数的方法记录如下:
importjava.io.UnsupportedEncodingException; importjava.net.URI; importjava.net.URLDecoder; importjava.util.ArrayList; importjava.util.List; importorg.apache.http.HttpEntity; importorg.apache.http.HttpResponse; importorg.apache.http.client.HttpClient; importorg.apache.http.client.config.RequestConfig; importorg.apache.http.client.methods.HttpPost; importorg.apache.http.client.methods.HttpRequestBase; importorg.apache.http.client.utils.URIBuilder; importorg.apache.http.entity.StringEntity; importorg.apache.http.impl.client.CloseableHttpClient; importorg.apache.http.impl.client.HttpClientBuilder; importorg.apache.http.impl.client.HttpClients; importorg.apache.http.util.EntityUtils; importcom.alibaba.fastjson.JSONArray; importcom.alibaba.fastjson.JSONObject; /** *@PostJsonParamsTest.java
*@version1.0 *@authorzxk *@Date2018-3-3 */ publicclassPostJsonParamsTest{ //超时时间 privatestaticfinalintRUN_TIME=10000; //爬取初始页数 privateStringpage; publicstaticvoidmain(String[]args)throwsException{ PostJsonParamsTestcrawl=newPostJsonParamsTest(); //请求的url地址 Stringurl="http://www.gzcredit.gov.cn/Service/CreditService.asmx/searchOrgWithPage"; //设置起始访问页码 crawl.setPage("1"); StringisStop=""; //设置请求 HttpRequestBaserequest=null; request=newHttpPost(url); try{ //设置config RequestConfigrequestConfig=RequestConfig.custom() .setSocketTimeout(RUN_TIME) .setConnectTimeout(RUN_TIME) .setConnectionRequestTimeout(RUN_TIME) .build(); request.setConfig(requestConfig); //json格式的post参数 StringpostParams="{\"condition\":{\"qymc\":\"%%%%\",\"cydw\":\"\"},\"pageNo\":"+crawl.getPage()+",\"pageSize\":100,count:2709846}"; System.out.println(postParams); HttpEntityhttpEntity=newStringEntity(postParams); ((HttpPost)request).setEntity(httpEntity); //添加请求头,可以绕过验证码 request.addHeader("Accept","application/json,text/javascript,*/*"); request.addHeader("Accept-Encoding","gzip,deflate"); request.addHeader("Accept-Language","zh-CN,zh;q=0.8"); request.addHeader("Connection","keep-alive"); request.addHeader("Host","www.gzcredit.gov.cn"); request.addHeader("Content-Type","application/json;charset=UTF-8"); URIBuilderbuilder=newURIBuilder(url); URIuri=builder.build(); uri=newURI(URLDecoder.decode(uri.toString(),"UTF-8")); request.setURI(uri); while(!isStop.equals("停止")||isStop.equals("重跑")){ isStop=crawl.crawlList(request); if(isStop.equals("爬取")){ crawl.setPage(String.valueOf(Integer.parseInt(crawl.getPage())+1)); } //if("2713".equals(crawl.getPage()))break; if("2".equals(crawl.getPage())){ break; } } }catch(NumberFormatExceptione){ e.printStackTrace(); thrownewNumberFormatException("数字格式错误"); }catch(UnsupportedEncodingExceptione){ e.printStackTrace(); thrownewUnsupportedEncodingException("不支持的编码集"); } } /** *爬取搜索列表 *@parampage *@return */ privateStringcrawlList(HttpRequestBaserequest){ intstatusCode=0; //下面两种方式都可以用来创建客户端连接,相当于打开了一个浏览器 CloseableHttpClienthttpClient=HttpClients.createDefault(); //HttpClienthttpClient=HttpClientBuilder.create().build(); HttpEntityhttpEntity=null; HttpResponseresponse=null; try{ try{ response=httpClient.execute(request); }catch(Exceptione){ e.printStackTrace(); EntityUtils.consumeQuietly(httpEntity); return"重跑"; } //打印状态 statusCode=response.getStatusLine().getStatusCode(); if(statusCode!=200){ EntityUtils.consumeQuietly(httpEntity); return"重跑"; } //实体 httpEntity=response.getEntity(); StringsearchListStr=EntityUtils.toString(httpEntity,"GBK").replaceAll("\\\\米","米"); StringallData=(String)JSONObject.parseObject(searchListStr).get("d"); //字符串值中间含双引号的替换处理 Strings=allData.replaceAll("\\{\"","{'") .replaceAll("\":\"","':'") .replaceAll("\",\"","','") .replaceAll("\":","':") .replaceAll(",\"",",'") .replaceAll("\"\\}","'}") .replaceAll("\"","") .replaceAll("'","\"") .replaceAll("
","") .replaceAll("\t","") .replaceAll("\\\\","?"); JSONObjectjsonData=JSONObject.parseObject(s); JSONArrayjsonContent=jsonData.getJSONArray("orgList"); searchListStr=null; allData=null; s=null; if(jsonContent==null||jsonContent.size()<1){ return"重跑"; } System.out.println(jsonContent.toJSONString()); return"爬取"; }catch(Exceptione){ e.printStackTrace(); return"重跑"; }finally{ EntityUtils.consumeQuietly(httpEntity); } } privateStringgetPage(){ returnpage; } privatevoidsetPage(Stringpage){ this.page=page; } }
补充知识:JAVA利用HttpClient发送post请求,将请求数据放到body里
我就废话不多说了,大家还是直接看代码吧~
/** *post请求,请求数据放到body里 *@paramurl请求地址 *@parambodyData参数 *@authorwangyj *@date2019年4月20日 */ publicstaticStringdoPostBodyData(Stringurl,StringbodyData)throwsException{ Stringresult=""; CloseableHttpClienthttpClient=null; CloseableHttpResponseresponse=null; try{ HttpPosthttpPost=getHttpPost(url,null);//请求地址 httpPost.setEntity(newStringEntity(bodyData,Encoding)); httpClient=getHttpClient(); //得到返回的response response=httpClient.execute(httpPost); HttpEntityentity=response.getEntity(); result=getResult(entity,Encoding); }catch(Exceptione){ throwe; }finally{ //关闭httpClient if(null!=httpClient){ httpClient.close(); } //关闭response if(null!=response){ EntityUtils.consume(response.getEntity());//会自动释放连接 response.close(); } } returnresult; }
以上这篇java实现通过post方式提交json参数操作就是小编分享给大家的全部内容了,希望能给大家一个参考,也希望大家多多支持毛票票。
声明:本文内容来源于网络,版权归原作者所有,内容由互联网用户自发贡献自行上传,本网站不拥有所有权,未作人工编辑处理,也不承担相关法律责任。如果您发现有涉嫌版权的内容,欢迎发送邮件至:czq8825#qq.com(发邮件时,请将#更换为@)进行举报,并提供相关证据,一经查实,本站将立刻删除涉嫌侵权内容。