前几周在弄爬虫相关的东西，因为在实习，所以在闲来无事的时候，自己通过java api实现了一个基于多线程的简单广度网络爬虫，由于这个爬虫没有使用任何jar包，所以功能也相对简单，只是根据深度提取了网页的url和title，给新人一些灵感。闲话不多说了直接上代码
package com.rimi.crawl;import java.io.ByteArrayOutputStream;import java.io.FileNotFoundException;import java.io.IOException;import java.io.InputStream;import java.io.PrintWriter;import java.net.HttpURLConnection;import java.net.URL;import java.util.Collections;import java.util.HashSet;import java.util.List;import java.util.Set;import java.util.concurrent.BlockingQueue;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.LinkedBlockingQueue;import java.util.concurrent.atomic.AtomicInteger;import java.util.regex.Matcher;import java.util.regex.Pattern;public class Crawl implements Runnable{ /** * 需要爬取的url */ public static BlockingQueue crawlQueue; /** * 爬取后的内容 */ public static BlockingQueue saveQueue; /** * 已经爬取过的url */ public static Set visitedSet; /** * url提取正则 */ public static Pattern urlPattern = Pattern.compile("href\\s*=\\s*\"?(.*?)[\"|>]"); /** * title提取正则 */ public static Pattern titlePattern = Pattern.compile(" .* "); /** * 开始url */ public static String StartUrl = "http://www.duowan.com"; /** * 爬取深度 */ public static AtomicInteger crawlSize = new AtomicInteger(50); static{ crawlQueue = new LinkedBlockingQueue (); saveQueue = new LinkedBlockingQueue (); crawlQueue.add(StartUrl); visitedSet = Collections.synchronizedSet( new HashSet () ); } public void crawl(){ String urlStr = null; String pageStr = null; try { while( true ){ urlStr = crawlQueue.take(); visitedSet.add(urlStr); pageStr = downLoadPage(urlStr); if( pageStr!=null ) pageHandel(pageStr,urlStr); crawlSize.decrementAndGet(); } } catch (Exception e) { e.printStackTrace(); } } /** * 页面下载 * @param urlStr * @return * @throws Exception */ private String downLoadPage(String urlStr) throws Exception{ if( urlStr==null ) return null; URL url = new URL(urlStr); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); connHandel(conn); conn.setDoInput(true); conn.connect(); String pageStr = null; int status = conn.getResponseCode(); //这里只对200做处理 if( status==200 ){ //这里只对HTML做处理 if( isHtml(conn) ){ String charset = getCharset(conn); InputStream is = conn.getInputStream(); pageStr = stream2String(is,charset); } } return pageStr; } /** * 判断是否是html页面 * @param conn * @return */ private boolean isHtml(HttpURLConnection conn){ boolean b = false; String tc = getContentType(conn); if( tc!=null&&tc.contains("html") ) b = true; return b; } /** * 获取字符编码 * @param conn * @return */ private String getCharset(HttpURLConnection conn){ String charset = "utf-8"; String tc = getContentType(conn); if( tc!=null ){ String[] charsets = tc.split("="); if( charset!=null&&charsets.length>=2 ){ charset = charsets[1]; } } return charset; } /** * 获取ContentType * @param conn * @return */ private String getContentType(HttpURLConnection conn){ List tc = conn.getHeaderFields().get("Content-Type"); if( tc!=null&&!tc.isEmpty() ) return tc.get(0); else return null; } /** * 页面处理 * @param pageStr * @param urlStr * @throws InterruptedException */ private void pageHandel(String pageStr,String urlStr) throws InterruptedException{ if( crawlSize.get()>0 ) urlHandel(pageStr,urlStr); titleHandel(pageStr,urlStr); //这里可以进行其他操作，如页面内容提取等 } /** * 标题处理 * @param pageStr * @param urlStr * @throws InterruptedException */ private void titleHandel(String pageStr,String urlStr) throws InterruptedException{ Matcher titleMatcher = titlePattern.matcher(pageStr); while( titleMatcher.find() ){ String titleStr = titleStrHandel(titleMatcher.group()); System.out.println(Thread.currentThread().getName()+" "+crawlQueue.size()+" "+saveQueue.size()+" "+titleStr); String saveStr = titleStr+" "+urlStr; saveQueue.put(saveStr); } } /** * url处理 * @param pageStr * @param pUrlStr * @throws InterruptedException */ private void urlHandel(String pageStr,String pUrlStr) throws InterruptedException{ Matcher urlM = urlPattern.matcher(pageStr); String cUtlStr = null; while( urlM.find() ){ cUtlStr = urlM.group().replace("href=", "").replace("\"", ""); if( cUtlStr.startsWith("http://")||cUtlStr.startsWith("https://") ){ if( !visitedSet.contains(cUtlStr) ){ crawlQueue.put(cUtlStr); } }else if( cUtlStr.startsWith("/") ){ cUtlStr = childUrlHandel(cUtlStr,pUrlStr); if( !visitedSet.contains(cUtlStr) ){ crawlQueue.put(cUtlStr); } } } } /** * 子url处理 * @param cUrlStr * @param pUrlStr * @return */ private String childUrlHandel(String cUrlStr,String pUrlStr){ String nameSpace = getUrlNameSpace(pUrlStr); if( nameSpace!=null ){ return nameSpace+cUrlStr; }else return null; } /** * 获取一级域名 * @param urlStr * @return */ private String getUrlNameSpace(String urlStr){ String nameSpace = null; String[] strs = null; String protocol = "http://"; if( urlStr.startsWith("http://") ){ strs = urlStr.replace("http://", "").split("/"); }else if( urlStr.startsWith("https://") ){ strs = urlStr.replace("https://", "").split("/"); protocol = "https://"; }else if( urlStr.startsWith("www") ){ strs = urlStr.split("/"); } if( strs.length>=1 ){ nameSpace = protocol+strs[0]; } return nameSpace; } /** * 标题处理 * @param titleStr * @return */ private String titleStrHandel(String titleStr){ return titleStr.replace(" ", "").replace(" ", ""); } /** * 流转字符串 * @param is * @param charset * @return * @throws IOException */ private String stream2String(InputStream is,String charset) throws IOException{ ByteArrayOutputStream bos = new ByteArrayOutputStream(1000); if( is!=null ){ byte[] buff = new byte[1024]; int len = 0; while( (len = is.read(buff))!=-1 ){ bos.write(buff, 0, len); } } return bos.toString(charset); } /** * 请求连接处理 * @param conn */ private void connHandel(HttpURLConnection conn){ conn.setReadTimeout(6000); conn.setConnectTimeout(6000); //TODO 设置请求头 conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.6)"); conn.setRequestProperty("Connection", "Keep-Alive"); } @Override public void run() { this.crawl(); } /** * 保存类 * @author rimi * */ public static class WritePage implements Runnable{ private String fileName; private PrintWriter pw; public WritePage(String fileName) { this.fileName = fileName; this.init(); } private void init(){ try { pw = new PrintWriter(fileName); } catch (FileNotFoundException e) { e.printStackTrace(); } } @Override public void run() { String str = null; while( true ){ try { str = saveQueue.take(); pw.println(str); } catch (InterruptedException e) { e.printStackTrace(); } } } } public static void main(String[] args) { ExecutorService es = Executors.newFixedThreadPool(12); for( int i=0;i<10;i++ ){ es.execute(new Crawl()); } for( int i=0;i<2;i++ ){ es.execute(new WritePage("file"+i+".txt")); } es.shutdown(); }} 这段代码主要是，开启10个爬取线程，取爬取网页，开启2个写线程，对爬取的内容进行保存（这里是通过文件的方式保存），我写的代码不是很优雅，新手向，大神不要喷我。