Downloader.java
1.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
package com.canrd.webmagic.processor.config;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;
import java.util.Random;
/**
* @author: xms
* @description: TODO
* @date: 2024/4/9 10:37
* @version: 1.0
*/
@Slf4j
@Component
public class Downloader {
private static RedisTemplate redisTemplate;
@Autowired
Downloader(RedisTemplate redisTemplate) {
Downloader.redisTemplate = redisTemplate;
}
/**
*
* @return
*/
public static HttpClientDownloader newIpDownloader() {
HttpClientDownloader downloader = new HttpClientDownloader() {
@Override
protected void onError(Request request) {
String[] ips = newIp();
setProxyProvider(SimpleProxyProvider.from(new Proxy(ips[0], Integer.parseInt(ips[1]))));
}
};
return downloader;
}
static String[] newIp() {
Long size = redisTemplate.opsForList().size("ip");
String ip = redisTemplate.opsForList().index("ip", new Random().nextInt(size.intValue())).toString();
log.info("获取ip===========>" + ip);
String[] ips = ip.split(":");
return ips;
}
}