Agent.java
3.09 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
package com.canrd.webmagic.processor.config;
import cn.hutool.core.io.resource.ClassPathResource;
import lombok.extern.slf4j.Slf4j;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* @author: xms
* @description: TODO
* @date: 2024/4/9 10:28
* @version: 1.0
*/
@Slf4j
public class Agent {
private static final String AGENT_FILE_PATH = "user-agent/User-Agents.txt";
private static ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
private static List<String> agents;
/**
* @return
*/
public static String getRandom() {
String random = getRandom(null);
log.info("Agent======================>" + random);
return random;
}
/**
* @param agent
* @return
*/
private static String getRandom(String agent) {
try {
lock.readLock().lock();
int size = agents.size();
if (size == 0) {
return "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36";
}
Random random = new Random();
if (null != agent) {
return agent;
} else {
return agents.get(random.nextInt(size));
}
} catch (Exception e) {
e.printStackTrace();
return "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36";
} finally {
lock.readLock().unlock();
}
}
static {
agents = new ArrayList<>();
InputStream resourceAsStream = null;
InputStreamReader inputStreamReader = null;
BufferedReader bufferedReader = null;
try {
resourceAsStream = new ClassPathResource(AGENT_FILE_PATH).getStream();
inputStreamReader = new InputStreamReader(resourceAsStream);
bufferedReader = new BufferedReader(inputStreamReader);
String len;
while ((len = bufferedReader.readLine()) != null) {
if (!len.matches("^#.*")) {
agents.add(len.trim());
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (null != bufferedReader) {
try {
bufferedReader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (null != inputStreamReader) {
try {
inputStreamReader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (null != resourceAsStream) {
try {
resourceAsStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}