上代码

package com.fzb.common.util;


import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentSkipListSet;

import jodd.io.FileUtil;

import org.apache.http.Header;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.DataNode;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Node;

import com.fzb.common.io.CompressUtil;

public class FetchWebPage{

	private static Logger log=Logger.getLogger(FetchWebPage.class);
	private String recordStr;
	private OutputStream socketOut;
	private boolean userOne=true ;
	private CloseableHttpClient  httpclient;
	private File file;
	public FetchWebPage(String index,String key,String savePath) {
		this.index=index;
		this.key=key;
		this.savePath=savePath+"/"+key+"/";
	}
	
	public FetchWebPage(String index,String key,String savePath,String cookie) {
		this.index=index;
		this.key=key;
		this.savePath=savePath+"/"+key+"/";
		this.cookie=cookie;
		 
	}
	
	public FetchWebPage(String index,String key,String savePath,String cookie,OutputStream out) {
		this.index=index;
		this.key=key;
		this.savePath=savePath+"/"+key+"/";
		this.cookie=cookie;
		this.socketOut=out;


	}

	public String goWithInZip(){
		go();
		// inZip
		String zipFile= "e:/temp/"+ParseTools.getRandomFileName("zip");
		CompressUtil.zip(file.toString(),zipFile, true, null);
		return zipFile;
	
	}
	
	public void go() {
		try {
			file=new File(savePath);
			FilesManageUtil.delFiler(file.toString());
			file.mkdirs();
			
			long start=System.currentTimeMillis();
			String htmlStr = getResponseBody(index);
			Document doc=Jsoup.parse(htmlStr);
			List<Node> nodes=doc.childNodes();
			findHref(nodes,index);
			aHref.remove(index);
			bHref.add(index);
			while(aHref.size()!=0){
				for (String url : aHref) {
					aHref.remove(url);
					if(bHref.contains(url)){
						continue;
					}
					try{
						htmlStr=getResponseBody(url);
					}
					catch (Exception e) {
						
					}
					 
					bHref.add(url);
					if(htmlStr!=null){
						Document tdoc=Jsoup.parse(htmlStr);
						List<Node> tnodes=tdoc.childNodes();
						findHref(tnodes,url);
					}
				}
			}
			
			recordStr+="总耗时  "+(System.currentTimeMillis()-start);
			System.out.println(recordStr);
		} catch (ClientProtocolException e1) {
			e1.printStackTrace();
		} catch (Exception e1) {
			e1.printStackTrace();
		}
		
	}
	
	public String getResponseBody(String url) throws ClientProtocolException, Exception{
		 
		log.info("get page " +url);
		long start=System.currentTimeMillis();
		String fileStr=url;
		fileStr=fileStr.replace("http://", "");
		//fileStr=fileStr.replace(":8080", "");
		fileStr=fileStr.replace(":", "");
		if(fileStr.contains("?")){
			fileStr=fileStr.substring(0,fileStr.lastIndexOf("?"));
		}
		fileStr=fileStr.replace("?", "_");
		//fileStr=fileStr.replace(".asp", ".jsp");
		if(!fileStr.contains(".htm") && fileStr.endsWith("/")){
			File file=new File(savePath+fileStr.substring(0,fileStr.length()-1));
			file.delete();
			fileStr+="/index.html";
		}
		 
		//fileStr=fileStr.substring(url.indexOf("/"));
		File file=new File(savePath+fileStr);
		if(file.exists()){
			return "";
		}
		if(file.isDirectory()){
			file=new File(file+"/index.html");
		}
		if(!userOne){
			httpclient=HttpClients.createSystem();
		}
		 
		file.getParentFile().mkdirs();
		HttpGet httpget=new HttpGet(url);
		
        httpget.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");    
        
        httpget.setHeader("Accept-Charset", "GB2312,utf-8;q=0.7,*;q=0.7");    
    
        httpget.setHeader("Accept-Encoding", "gzip, deflate");    
    
        httpget.setHeader("Accept-Language", "zh-cn,zh;q=0.5");    
    
        httpget.setHeader("Connection", "keep-alive");    
        if(cookie!=null){
        	httpget.setHeader("Cookie", cookie);    
        }
    
        httpget.setHeader("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:32.0) Gecko/20100101 Firefox/32.0");  
        CloseableHttpResponse response = httpclient.execute(httpget);
		Header[] header=response.getAllHeaders();
		//关闭http的301 自动跳转
		httpget.getParams().setParameter("http.protocol.handle-redirects",false);
		for (Header header2 : header) {
			System.out.println(header2.getValue());
			if(header2.getName().equals("Location")){
				log.info("this page " + url+" forward to--> "+ header2.getValue());
				aHref.add(header2.getValue());
				return "this page hava a 3XX";
			}
		}
		InputStream reader=(response.getEntity().getContent());
		/*String tmp=null;
		StringBuffer sb=new StringBuffer();*/
		byte b[]=new byte[1024*32];
		int len;
		if(!file.exists()){
			File tempfile=file;
			while(!(tempfile=tempfile.getParentFile()).exists()){
				 
			}
			tempfile.getParentFile().delete();
			file.getParentFile().mkdirs();
			//file.createNewFile();
		}
		FileOutputStream out=new FileOutputStream(file);
		while((len=reader.read(b))!=-1){
			out.write(b,0,len);
			out.flush();
		}
		 
		/*while((tmp=reader.)!=null){
			sb.append(tmp+"\n");
		}
		*/
		out.close();
		recordStr+="处理：---〉"+url+" 用时 " + ((System.currentTimeMillis()-start)/1000.0)+"\n";
		if(socketOut!=null){
			socketOut.write(("处理：---〉"+url+" 用时 " +((System.currentTimeMillis()-start)/1000.0)).getBytes());
		}
		System.out.println("处理：---〉"+url+" 用时 " +((System.currentTimeMillis()-start)/1000.0));
		if(file.length()>1024*1024*2){
			return "this is file";
		}
		FileInputStream in=new FileInputStream(file);
		byte r[]=new byte[(int) file.length()];
		in.read(r);
		in.close();
		// 如果网页存在CSS文件的处理
		if(url.contains(".css")){
			aHref.addAll(CssParseTool.getImgsUrlByFilePath(file.toString(), url,index));
		}
		String htmlStr=new String(r);
		//FileUtil.writeString(utf8FilePath, FileUtil.readString(javaGbkFile, "GBK"), "UTF-8");
		/*if(htmlStr.contains(" content=\"text/html; charset=gb2312\"")){
			FileUtil.writeString(file, FileUtil.readString(file, "GBK").replace("gb2312", "utf-8"), "UTF-8");
			//FileUtils.writeLines(file, "UTF-8", FileUtils.readLines(file));  
		}*/
		/*if(htmlStr.contains("<html>")){
			 FileUtil.writeString(file, FileUtil.readString(file, "utf-8").replace(".asp", ".jsp").replace("/hostadm/", "").replace("<html>", "<%@ page language=\"java\" import=\"java.util.*\" pageEncoding=\"utf-8\"%>\n<%String path = request.getContextPath();\nString basePath = request.getScheme()+\"://\"+request.getServerName()+\":\"+request.getServerPort()+path+\"/\";%>\n<html>"), "UTF-8");
		}*/
		
		return htmlStr;
		
	}
 
	private  String index;
	private  String key;
	private String savePath;
	private String cookie;
	private  Set<String> aHref=new ConcurrentSkipListSet<String>();
	private  Set<String> bHref=new ConcurrentSkipListSet<String>();
	private void findHref(List<Node> nodes,String url){
		for (Node node : nodes) {
			dispose(url, node.attr("href"));
			dispose(url, node.attr("src"));
			dispose(url, node.attr("background"));
			if("style".equals(node.nodeName())){
				String nowStr=node.outerHtml().replace("<style>", "").replace("</style>", "");
				try{
					Set<String> urls=CssParseTool.getImgsUrlByStr(nowStr, url,index);
					for(String urlt:urls){
						aHref.add(urlt);
					}
					continue;
				}
				catch(Exception e){
					e.printStackTrace();
				}
				continue;
			}
			else{
				if(node.attr("style").contains("background:url")){
					System.out.println(node.attr("style"));
					dispose(url,node.attr("style").substring(node.attr("style").indexOf("(")+1, node.attr("style").indexOf(")")));
				}
				findHref(node.childNodes(),url);
			}
		}
	}

	private void dispose(String url,String context){
		if(context.contains("javascript")) return;
		// 去掉网址的瞄文本链接
		if(context.indexOf("#")!=-1){
			context=context.substring(0, context.indexOf("#"));
		}
		//网址的绝对路径
		if(context.startsWith("/") && !context.contains("http://")){
			context=index+context;
		}
		//  网址中只一个形如 src="1.jpg"的图片路径 || 用于拼接网址中的相对路径
		if(!context.contains("/") || (!context.startsWith("/") && !context.contains("http://") &&  !context.contains("https://"))){
			context=url.substring(0, url.lastIndexOf("/"))+"/"+context;
		}
		if(!bHref.contains(context) && context.contains(getKey()))
		aHref.add(context);
	}
 
	public void setIndex(String index) {
		this.index = index;
	}

	public String getKey() {
		return key;
	}

	public void setKey(String key) {
		this.key = key;
	}
	
	public static void main(String[] args) {
		String savePath="";
		if(File.separatorChar=='/'){
			savePath="/home/zcc/fecth_temp";
		}
		else{
			savePath="/e:/home/zcc/fecth_temp";
		}
		
		FetchWebPage fwp=new FetchWebPage("http://www.xxxx.net/","www.xxxx.net",savePath);
		fwp.userOne=false;
		
		fwp.go();
		
	}
	
}