feat: 配置userAgent和IP代理池

谢茂盛
1 parent 836066f9
Showing 11 changed files with 348 additions and 6 deletions
src/main/java/com/canrd/webmagic/controller/NatureArticleController.java
src/main/java/com/canrd/webmagic/processor/NatureSearchPageProcessor.java
src/main/java/com/canrd/webmagic/processor/config/Agent.java
src/main/java/com/canrd/webmagic/processor/config/Downloader.java
src/main/java/com/canrd/webmagic/processor/config/UpdateIp.java
src/main/java/com/canrd/webmagic/processor/NatureArticlePipeline.java → src/main/java/com/canrd/webmagic/processor/pipeline/NatureArticlePipeline.java
src/main/resources/log4j2-dev.xml
src/main/resources/log4j2-prod.xml
src/main/resources/user-agent/User-Agents.txt
src/test/java/com/canrd/webmagic/BaseTest.java
src/test/java/com/canrd/webmagic/utils/DateTimeUtilTest.java
@@ -4,7 +4,7 @@ import com.canrd.webmagic.common.constant.ServerResult;
 import com.canrd.webmagic.common.jsr303.OperateGroup;
 import com.canrd.webmagic.domain.vo.NatureArticleQueryVO;
 import com.canrd.webmagic.domain.vo.NatureArticleVO;
-import com.canrd.webmagic.processor.NatureArticlePipeline;
+import com.canrd.webmagic.processor.pipeline.NatureArticlePipeline;
 import com.canrd.webmagic.processor.NatureSearchPageProcessor;
 import com.canrd.webmagic.service.NatureArticleService;
 import org.springframework.validation.annotation.Validated;
@@ -4,6 +4,8 @@ import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.canrd.webmagic.common.utils.StringUtils;
 import com.canrd.webmagic.domain.dto.NatureArticleDO;
+import com.canrd.webmagic.processor.config.Agent;
+import com.canrd.webmagic.processor.pipeline.NatureArticlePipeline;
 import org.springframework.stereotype.Component;
 import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.Site;
@@ -24,9 +26,10 @@ import java.util.Objects;
  */
 @Component
 public class NatureSearchPageProcessor implements PageProcessor {
+    private String agent = Agent.getRandom();
     // 抓取网站的相关配置，包括编码、抓取间隔、重试次数等
-    private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
+    private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setUserAgent(agent);
     /**
      * 定制爬虫逻辑的核心接口，在这里编写抽取逻辑
+package com.canrd.webmagic.processor.config;
+
+import cn.hutool.core.io.resource.ClassPathResource;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * @author: xms
+ * @description: TODO
+ * @date: 2024/4/9 10:28
+ * @version: 1.0
+ */
+@Slf4j
+public class Agent {
+    private static final String AGENT_FILE_PATH = "user-agent/User-Agents.txt";
+    private static ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+    private static List<String> agents;
+
+    /**
+     * @return
+     */
+    public static String getRandom() {
+        String random = getRandom(null);
+        log.info("Agent======================>" + random);
+        return random;
+    }
+
+    /**
+     * @param agent
+     * @return
+     */
+    private static String getRandom(String agent) {
+        try {
+            lock.readLock().lock();
+            int size = agents.size();
+            if (size == 0) {
+                return "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36";
+            }
+            Random random = new Random();
+            if (null != agent) {
+                return agent;
+            } else {
+                return agents.get(random.nextInt(size));
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            return "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36";
+        } finally {
+            lock.readLock().unlock();
+        }
+    }
+
+    static {
+        agents = new ArrayList<>();
+        InputStream resourceAsStream = null;
+        InputStreamReader inputStreamReader = null;
+        BufferedReader bufferedReader = null;
+        try {
+            resourceAsStream = new ClassPathResource(AGENT_FILE_PATH).getStream();
+            inputStreamReader = new InputStreamReader(resourceAsStream);
+            bufferedReader = new BufferedReader(inputStreamReader);
+            String len;
+            while ((len = bufferedReader.readLine()) != null) {
+                if (!len.matches("^#.*")) {
+                    agents.add(len.trim());
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            if (null != bufferedReader) {
+                try {
+                    bufferedReader.close();
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+            if (null != inputStreamReader) {
+                try {
+                    inputStreamReader.close();
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+            if (null != resourceAsStream) {
+                try {
+                    resourceAsStream.close();
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+    }
+}
+package com.canrd.webmagic.processor.config;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.downloader.HttpClientDownloader;
+import us.codecraft.webmagic.proxy.Proxy;
+import us.codecraft.webmagic.proxy.SimpleProxyProvider;
+
+import java.util.Random;
+
+/**
+ * @author: xms
+ * @description: TODO
+ * @date: 2024/4/9 10:37
+ * @version: 1.0
+ */
+@Slf4j
+@Component
+public class Downloader {
+    private static RedisTemplate redisTemplate;
+
+    @Autowired
+    Downloader(RedisTemplate redisTemplate) {
+        Downloader.redisTemplate = redisTemplate;
+    }
+
+    /**
+     *
+     * @return
+     */
+    public static HttpClientDownloader newIpDownloader() {
+        HttpClientDownloader downloader = new HttpClientDownloader() {
+            @Override
+            protected void onError(Request request) {
+                String[] ips = newIp();
+                setProxyProvider(SimpleProxyProvider.from(new Proxy(ips[0], Integer.parseInt(ips[1]))));
+            }
+        };
+        return downloader;
+    }
+
+    static String[] newIp() {
+        Long size = redisTemplate.opsForList().size("ip");
+        String ip = redisTemplate.opsForList().index("ip", new Random().nextInt(size.intValue())).toString();
+        log.info("获取ip===========>" + ip);
+        String[] ips = ip.split(":");
+        return ips;
+    }
+}
+package com.canrd.webmagic.processor.config;
+
+import com.baomidou.mybatisplus.core.toolkit.StringUtils;
+import org.apache.commons.io.IOUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetSocketAddress;
+import java.net.Proxy;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.List;
+
+/**
+ * @author: xms
+ * @description: TODO
+ * @date: 2024/4/9 10:35
+ * @version: 1.0
+ */
+@Component
+public class UpdateIp {
+
+    @Autowired
+    private RedisTemplate redisTemplate;
+
+    @Scheduled(cron = "*/20 * * * * ?")
+    void update() {
+        List<String> range = redisTemplate.opsForList().range("ip", 0, -1);
+        for (String ip : range) {
+            if (ifUseless(ip)) {
+                System.err.println(ip + "  从redis移除");
+                redisTemplate.opsForList().remove("ip", 0, ip);
+            }
+        }
+    }
+
+    @Scheduled(cron = "*/15 * * * * ?")
+    void ips() {
+        String string = null;
+        try {
+            Document document = Jsoup.connect("https://www.xicidaili.com/nn").timeout(3000).get();
+            Elements tags = document.select("#ip_list > tbody > tr");
+            for (Element element : tags) {
+                //取得ip地址节点
+                Elements tdChilds = element.select("tr > td:nth-child(2)");
+                //取得端口号节点
+                Elements tcpd = element.select("tr > td:nth-child(3)");
+                if (StringUtils.isNotBlank(tdChilds.text()) && StringUtils.isNotBlank(tcpd.text())) {
+                    string = tdChilds.text() + ":" + tcpd.text();
+                    if (!ifUseless(string)) {
+                        List<String> range = redisTemplate.opsForList().range("ip", 0, -1);
+                        if (!range.contains(string)) {
+                            System.err.println(string + "  存进redis");
+                            if (redisTemplate.opsForList().size("ip") > 100) {
+                                redisTemplate.opsForList().rightPopAndLeftPush("ip", string);
+                            } else {
+                                redisTemplate.opsForList().leftPush("ip", string);
+                            }
+                        }
+                    }
+                }
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 无效的ip 返回true 有效的ip返回false
+     *
+     * @param ip
+     * @return
+     */
+    boolean ifUseless(String ip) {
+        String[] split = ip.split(":");
+        URL url = null;
+        try {
+            url = new URL("http://www.baidu.com");
+            InetSocketAddress addr = new InetSocketAddress(split[0], Integer.parseInt(split[1]));
+            Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
+            InputStream in = null;
+            try {
+                URLConnection conn = url.openConnection(proxy);
+                conn.setConnectTimeout(2000);
+                in = conn.getInputStream();
+            } catch (Exception e) {
+                return true;
+            }
+            String s = IOUtils.toString(in);
+            if (s.indexOf("baidu") > 0) {
+                return false;
+            }
+            return true;
+        } catch (Exception e) {
+            return true;
+        }
+    }
+}
-package com.canrd.webmagic.processor;
+package com.canrd.webmagic.processor.pipeline;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.canrd.webmagic.domain.dto.NatureArticleDO;
@@ -66,7 +66,7 @@
             <ThresholdFilter level="debug"/>
             <appender-ref ref="RollingFileInfo" />
         </logger>
-        <logger name="com.canrd.shop" level="DEBUG" >
+        <logger name="com.canrd.webmagic" level="DEBUG" >
             <ThresholdFilter level="debug"/>
             <appender-ref ref="RollingFileInfo" />
         </logger>
@@ -30,7 +30,7 @@
         <!--</console>-->
         <!-- 这个会打印出所有的info及以下级别的信息，每次大小超过size，则这size大小的日志会自动存入按年份-月份建立的文件夹下面并进行压缩，作为存档 -->
-        <RollingFile name="RollingFileInfo" fileName="${sys:logging.path}/logs/overtime.log" filePattern="${sys:logging.path}/logs/$${date:yyyy-MM-dd}/info-%d{yyyy-MM-dd}-%i.log">
+        <RollingFile name="RollingFileInfo" fileName="${sys:logging.path}/logs/webmagic.log" filePattern="${sys:logging.path}/logs/$${date:yyyy-MM-dd}/info-%d{yyyy-MM-dd}-%i.log">
             <!--控制台只输出level及以上级别的信息（onMatch），其他的直接拒绝（onMismatch） -->
             <ThresholdFilter level="info" onMatch="ACCEPT" onMismatch="DENY" />
             <!--<Filters>-->
@@ -65,7 +65,7 @@
             <ThresholdFilter level="info"/>
             <appender-ref ref="RollingFileInfo" />
         </logger>
-        <logger name="com.canrd.shop" level="info" >
+        <logger name="com.canrd.webmagic" level="info" >
             <ThresholdFilter level="info"/>
             <appender-ref ref="RollingFileInfo" />
         </logger>
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60
+Opera/8.0 (Windows NT 5.1; U; en)
+Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50
+Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50
+Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0
+Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11
+Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36
+Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER
+Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)
+Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)"
+Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)
+Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)
+Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0
+Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.3.4000 Chrome/30.0.1599.101 Safari/537.36
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36
 \ No newline at end of file
+package com.canrd.webmagic;
+
+import org.junit.runner.RunWith;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.transaction.annotation.Transactional;
+
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(classes = Application.class)
+//主动回滚测试产生的数据
+@Transactional
+public class BaseTest {
+
+
+}
+package com.canrd.webmagic.utils;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.canrd.webmagic.BaseTest;
+import com.canrd.webmagic.domain.dto.NatureArticleDO;
+import com.canrd.webmagic.service.NatureArticleService;
+import org.junit.Test;
+
+import javax.annotation.Resource;
+import java.util.List;
+
+/**
+ * @author: xms
+ * @description: TODO
+ * @date: 2023/2/10 14:09
+ * @version: 1.0
+ */
+public class DateTimeUtilTest extends BaseTest {
+
+    @Resource
+    private NatureArticleService natureArticleService;
+
+    @Test
+    public void export() {
+        List<NatureArticleDO> articleDOList = natureArticleService.list();
+        JSONArray array = new JSONArray();
+        for (NatureArticleDO articleDO : articleDOList) {
+            JSONArray jsonArray = JSONArray.parseArray(articleDO.getEmailInfo());
+            array.addAll(jsonArray);
+        }
+        System.out.println("联系作者,邮箱");
+        for (Object o : array) {
+            JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(o));
+            String authorEmailName = jsonObject.getString("authorEmailName");
+            String email = jsonObject.getString("email");
+            System.out.println(authorEmailName + "," + email);
+        }
+    }
+}