网络爬虫之httpclient的使用

原文地址：http://blog.csdn.net/qy20115549/article/details/52201843

HttpClient 是 Apache Jakarta Common 下的子项目，可以用来提供高效的、最新的、功能丰富的支持 HTTP 协议的客户端编程工具包，并且它支持 HTTP 协议最新的版本和建议。
如下面java程序，为我写的一个简单的httpclient的测试程序：

package crawlerTest;
import java.io.IOException;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.HttpVersion;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicHttpResponse;
import org.apache.http.util.EntityUtils;
public class httpTest {
    public static void main(String[] args) throws IOException {
        HttpClient client = new DefaultHttpClient();   //初始化httpclient
        String personalUrl="http://www.autohome.com.cn/beijing/";     //指定一个种子url
        HttpGet postMethod = new HttpGet(personalUrl);   //post方法请求
        System.out.println("postMethod====="+ postMethod+"=====");
        HttpGet getMethod = new HttpGet(personalUrl);        //    get方法请求
        System.out.println("Method======"+getMethod+"=====");
        HttpResponse response = new BasicHttpResponse(HttpVersion.HTTP_1_1,
                HttpStatus.SC_OK, "OK");                        //初始化response        
        response = client.execute(getMethod);                    //执行响应
        System.out.println("response========"+response+"=====");
        int StatusCode = response.getStatusLine().getStatusCode(); //获取响应状态码
        System.out.println("StatusCode===="+StatusCode+"=====");
        System.out.println(response);
        if(StatusCode == 200){                          //状态码200表示响应成功
            //获取实体内容
            String entity = EntityUtils.toString (response.getEntity(),"utf-8");
            //输出实体内容
            System.out.println(entity);
            EntityUtils.consume(response.getEntity());       //消耗实体
        }else {
            //关闭HttpEntity的流实体
            EntityUtils.consume(response.getEntity());        //消耗实体
        }
    }
}