SpringBoot集成okhttp3爬虫_okhttp3需要引入的pom

一.引入依赖

pom依赖如下:

        <dependency>
		    <groupId>com.squareup.okhttp3</groupId>
		    <artifactId>okhttp</artifactId>
		    <version>3.10.0</version>
		</dependency>
1
2
3
4
5

二.在resources目录下配置application.properties

#  配置okhttp3
ok.http.connect-timeout=30
ok.http.read-timeout=30
ok.http.write-timeout=30
# 连接池中整体的空闲连接的最大数量
ok.http.max-idle-connections=200
# 连接空闲时间最多为 300 秒
ok.http.keep-alive-duration=300
1
2
3
4
5
6
7
8

三.创建conf包,在该包下创建OkHttpConfiguration配置类

import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.concurrent.TimeUnit;

import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;

import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import okhttp3.ConnectionPool;
import okhttp3.OkHttpClient;


@Configuration
public class OkHttpConfiguration {
	
	 @Value("${ok.http.connect-timeout}")
	    private Integer connectTimeout;

	    @Value("${ok.http.read-timeout}")
	    private Integer readTimeout;

	    @Value("${ok.http.write-timeout}")
	    private Integer writeTimeout;

	    @Value("${ok.http.max-idle-connections}")
	    private Integer maxIdleConnections;

	    @Value("${ok.http.keep-alive-duration}")
	    private Long keepAliveDuration;

	    @Bean
	    public OkHttpClient okHttpClient() {
	        return new OkHttpClient.Builder()
	                .sslSocketFactory(sslSocketFactory(), x509TrustManager())
	                // 是否开启缓存
	                .retryOnConnectionFailure(false)
	                .connectionPool(pool())
	                .connectTimeout(connectTimeout, TimeUnit.SECONDS)
	                .readTimeout(readTimeout, TimeUnit.SECONDS)
	                .writeTimeout(writeTimeout,TimeUnit.SECONDS)
	            	.hostnameVerifier((hostname, session) -> true)
	            	// 设置代理
//	            	.proxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress("127.0.0.1", 8888)))
	                // 拦截器
//	                .addInterceptor()
	                .build();
	    }

	    @Bean
	    public X509TrustManager x509TrustManager() {
	        return new X509TrustManager() {
	            @Override
	            public void checkClientTrusted(X509Certificate[] chain, String authType)
	                    throws CertificateException {
	            }
	            @Override
	            public void checkServerTrusted(X509Certificate[] chain, String authType)
	                    throws CertificateException {
	            }
	            @Override
	            public X509Certificate[] getAcceptedIssuers() {
	                return new X509Certificate[0];
	            }
	        };
	    }

	    @Bean
	    public SSLSocketFactory sslSocketFactory() {
	        try {
	            // 信任任何链接
	            SSLContext sslContext = SSLContext.getInstance("TLS");
	            sslContext.init(null, new TrustManager[]{x509TrustManager()}, new SecureRandom());
	            return sslContext.getSocketFactory();
	        } catch (NoSuchAlgorithmException | KeyManagementException e) {
	            e.printStackTrace();
	        }
	        return null;
	    }

	    @Bean
	    public ConnectionPool pool() {
	        return new ConnectionPool(maxIdleConnections, keepAliveDuration, TimeUnit.SECONDS);
	    }

}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

四.使用工具类快速完成解析

package com.okhttpcilentofspringbootl.springbootinit.utils;

import java.io.IOException;

import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;

public class OkHttpUtils {
    public static String OkGetArt(String url){
        String html = null;
        OkHttpClient client = new OkHttpClient();
        Request requestObj = new Request.Builder().url(url).build();
        try (Response responseObj = client.newCall(requestObj).execute()){
            html = responseObj.body().string();
        } catch (IOException e) {
            e.printStackTrace();
        }
    return html;

    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23