抓取网站信息源码.docx

1、抓取网站信息源码用HttpURLConnection抓取登陆网站信息的源码：供参考。一共2个类。一个测试类package dev.smart.craw.util;import java.io.BufferedReader;import java.io.BufferedWriter;import java.io.IOException;import java.io.InputStream;import java.io.InputStreamReader;import java.io.OutputStream;import java.io.OutputStreamWriter;import .Ht

2、tpURLConnection;import .ProtocolException;import .URL;import java.util.List;import java.util.Map;/* * * author royoky * */public class CrawWeb /* * 获得一个HttpURLConnection对象 * * param cookie * param referer * 上一个引用页面 * param url * 请求的URL地址 * param method * 方法类型（POST GET） * return * throws IOException

3、* throws ProtocolException */ public HttpURLConnection getHttpURLConnection(HeaderEntiy entity) throws IOException, ProtocolException URL cookieUrl = new URL(entity.getUrl(); HttpURLConnection cookiecon = (HttpURLConnection) cookieUrl .openConnection(); cookiecon.setInstanceFollowRedirects(false); i

4、f (entity.getHost() != null & !.equals(entity.getHost() cookiecon.setRequestProperty(Host, entity.getHost(); if (entity.getUser_Agent() != null & !.equals(entity.getUser_Agent() cookiecon.setRequestProperty(User-Agent, entity.getUser_Agent(); if (entity.getAccept() != null & !.equals(entity.getAccep

5、t() cookiecon.setRequestProperty(Accept, entity.getAccept(); if (entity.getAccept_Language() != null & !.equals(entity.getAccept_Language() cookiecon.setRequestProperty(Accept-Language, entity .getAccept_Language(); if (entity.getAccept_Encoding() != null & !.equals(entity.getAccept_Encoding() cooki

6、econ.setRequestProperty(Accept-Encoding, entity .getAccept_Encoding(); if (entity.getContent_Type() != null & !.equals(entity.getContent_Type() cookiecon.setRequestProperty(Content-Type, entity .getContent_Type(); if (entity.getContent_Length() != null & !.equals(entity.getContent_Length() cookiecon

7、.setRequestProperty(Content-Length, entity .getContent_Length(); / map.put(Accept-Charset, x-gbk,utf-8;q=0.7,*;q=0.7); if (entity.getAccept_Charset() != null & !.equals(entity.getAccept_Charset() cookiecon.setRequestProperty(Accept-Charset, entity .getAccept_Charset(); if (entity.getConnection() !=

8、null & !.equals(entity.getConnection() cookiecon.setRequestProperty(Connection, entity.getConnection(); if (entity.getCache_Control() != null & !.equals(entity.getCache_Control() cookiecon.setRequestProperty(Cache-Control, entity .getCache_Control(); if (entity.getReferer() != null & !.equals(entity

9、.getReferer() cookiecon.setRequestProperty(Referer, entity.getReferer(); if (entity.getCookie() != null & !.equals(entity.getCookie() cookiecon.setRequestProperty(Cookie, entity.getCookie(); cookiecon.setRequestMethod(entity.getFormWay(); cookiecon.setConnectTimeout(90000); cookiecon.setReadTimeout(

10、120000); cookiecon.setDoOutput(true); cookiecon.setDoInput(true); cookiecon.setUseCaches(false); return cookiecon; /* * POST 做参数传递 * * param con * param param * throws IOException */ public void sendParameter(HttpURLConnection con, HeaderEntiy entity) throws IOException con.connect(); OutputStream o

11、ut = con.getOutputStream(); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out); if (entity.getParamPost() != null) bw.write(entity.getParamPost(); bw.flush(); bw.close(); out.close(); /* * 获得请求返回的文本信息 * * param con * param encode * return * throws Exception */ public String getText(H

12、ttpURLConnection con, String encode) throws Exception / String host=; / String / url= / String / parm=startDate=+startDate+&endDate=+endDate+&payType=all&orderStatus=&hostType=D&tripType=all&isGroup=all&userid=DZC001&orgID=TUAIR&officeCode=SZX348&iataNo=08017074&grp=SZXYYB&CanViewOrhersRight=true&do

13、wntype=b2b_pay&fileType=csv; return this.doString(con, con.getInputStream(), encode); public String getCookie(HttpURLConnection con) List cookie = null; String session = ; MapString, List map = con.getHeaderFields(); if (map != null) cookie = map.get(Set-Cookie); if (cookie != null) for (String s :

14、cookie) session += s.replace(path=/, ); return session; /* * 得到请求返回的状态 200表示请求成功 * * param con * return */ public int getResponseState(HttpURLConnection con) try if (con != null) return con.getResponseCode(); catch (IOException e) / TODO Auto-generated catch block e.printStackTrace(); return 0; priv

15、ate String doString(HttpURLConnection con, InputStream in, String encode) throws Exception String temp = null; String _viewstate = ; if (encode = null | .equals(encode) encode = UTF-8; BufferedReader br = new BufferedReader( new InputStreamReader(in, encode); while (temp = br.readLine() != null) / S

16、ystem.out.println(temp); _viewstate += temp + rn; con.disconnect(); br.close(); in.close(); return _viewstate; package dev.smart.craw.util;public class HeaderEntiy / map.put(Accept, image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/msword, application/vnd.ms

17、-excel, application/vnd.ms-powerpoint, */*);/ map.put(Accept-Language, zh-cn);/ map.put(Accept-Encoding, GBK);/ map.put(User-Agent, Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727);/ map.put(Host, www.cococ.cc);/ map.put(Connection, Keep-Alive);/ map.put(Content-Ty

18、pe, application/x-www-form-urlencoded);/ map.put(Cache-Control, no-cache);/ map.put(Accept-Charset, x-gbk,utf-8;q=0.7,*;q=0.7);/ map.put(Referer, http:/www.cococ.cc/member/?act=login&jumpUrl=http:/www.cococ.cc/); private String Host; private String User_Agent; private String Accept; private String A

19、ccept_Language; private String Accept_Encoding; private String Content_Type; private String Content_Length; private String Accept_Charset; private String Connection; private String Cache_Control; private String Referer; private String Cookie; private String Url; private String FormWay=GET; private S

20、tring ParamPost; public String getAccept() return Accept; public void setAccept(String accept) Accept = accept; public String getAccept_Charset() return Accept_Charset; public void setAccept_Charset(String accept_Charset) Accept_Charset = accept_Charset; public String getAccept_Encoding() return Acc

21、ept_Encoding; public void setAccept_Encoding(String accept_Encoding) Accept_Encoding = accept_Encoding; public String getAccept_Language() return Accept_Language; public void setAccept_Language(String accept_Language) Accept_Language = accept_Language; public String getCache_Control() return Cache_C

22、ontrol; public void setCache_Control(String cache_Control) Cache_Control = cache_Control; public String getConnection() return Connection; public void setConnection(String connection) Connection = connection; public String getContent_Length() return Content_Length; public void setContent_Length(Stri

23、ng content_Length) Content_Length = content_Length; public String getContent_Type() return Content_Type; public void setContent_Type(String content_Type) Content_Type = content_Type; public String getCookie() return Cookie; public void setCookie(String cookie) Cookie = cookie; public String getHost(

24、) return Host; public void setHost(String host) Host = host; public String getReferer() return Referer; public void setReferer(String referer) Referer = referer; public String getUser_Agent() return User_Agent; public void setUser_Agent(String user_Agent) User_Agent = user_Agent; public String getPa

25、ramPost() return ParamPost; public void setParamPost(String paramPost) ParamPost = paramPost; public String getUrl() return Url; public void setUrl(String url) Url = url; public String getFormWay() return FormWay; public void setFormWay(String formWay) FormWay = formWay; package dev.smart.craw.util;

26、import .HttpURLConnection;public class Test extends CrawWeb public String getInfo() throws Exception HeaderEntiy vo=new HeaderEntiy(); vo.setAccept( image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/msword, application/vnd.ms-excel, application/vnd.ms-powerp

27、oint, */*); vo.setAccept_Charset(x-gbk,utf-8;q=0.7,*;q=0.7); vo.setAccept_Encoding(GBK); vo.setAccept_Language(zh-cn); vo.setCache_Control(no-cache); vo.setConnection(Keep-Alive); vo.setContent_Length(); vo.setContent_Type(application/x-www-form-urlencoded); vo.setCookie(); vo.setHost(www.cococ.cc);

28、 vo.setReferer(http:/www.cococ.cc/member/?act=login&jumpUrl=http:/www.cococ.cc/); vo.setUser_Agent(Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727); vo.setUrl(http:/www.cococ.cc/member/?act=login); vo.setParamPost(username=royoks&password=123456&+%B5%C7+%C2%BD+=+&handler=login&jumpurl=http%3A%2F%2Fwww.cococ.cc%2FPOST /member/?act=login); vo.setFormWay(POST); HttpURLConnection con=super.getHttpURLConnection(vo