添加ip

PurelzMgnead
1 parent 965b4941
Showing 28 changed files with 443 additions and 36 deletions
src/main/java/com/canrd/webmagic/config/MybatisPlusConfig.java
src/main/java/com/canrd/webmagic/config/MybatisplusObjectMetaHandler.java
src/main/java/com/canrd/webmagic/controller/IPUqiDataController.java
src/main/java/com/canrd/webmagic/controller/IpCheckController.java
src/main/java/com/canrd/webmagic/controller/SciengineController.java
src/main/java/com/canrd/webmagic/domain/dto/IpDO.java
src/main/java/com/canrd/webmagic/mapper/IpMapper.java
src/main/java/com/canrd/webmagic/processor/IPUqiDataProcessor.java
src/main/java/com/canrd/webmagic/processor/download/SeleniumDownloader.java
src/main/java/com/canrd/webmagic/processor/pipeline/IpPipeline.java
src/main/java/com/canrd/webmagic/service/IpService.java
src/main/java/com/canrd/webmagic/service/impl/IpServiceImpl.java
src/main/java/com/canrd/webmagic/util/PingUtil.java
src/main/resources/Mapper/IpMapper.xml
target/classes/Mapper/IpMapper.xml
target/classes/com/canrd/webmagic/config/MybatisPlusConfig.class
target/classes/com/canrd/webmagic/controller/IPUqiDataController.class
target/classes/com/canrd/webmagic/controller/IpCheckController.class
target/classes/com/canrd/webmagic/controller/SciengineController.class
target/classes/com/canrd/webmagic/domain/dto/IpDO.class
 package com.canrd.webmagic.config;
  
+import com.baomidou.mybatisplus.annotation.DbType;
+import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor;
+import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor;
+import org.mybatis.spring.annotation.MapperScan;
+import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.transaction.annotation.EnableTransactionManagement;
  
-
+@EnableTransactionManagement
 @Configuration
+@MapperScan("com.canrd.webmagic.mapper")
 public class MybatisPlusConfig {
-
-}
+    /**
+     *
+     * @return
+     */
+    @Bean
+    public MybatisPlusInterceptor addPaginationInnerInterceptor(){
+        MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor();
+        PaginationInnerInterceptor innerInterceptor = new PaginationInnerInterceptor(DbType.MYSQL);
+        interceptor.addInnerInterceptor(innerInterceptor);
+        return interceptor;
+    }
+}
 \ No newline at end of file
-package com.gec.tra.system.config;
-
-import com.baomidou.mybatisplus.core.handlers.MetaObjectHandler;
-import org.apache.ibatis.reflection.MetaObject;
-import org.springframework.stereotype.Component;
-
-import java.util.Date;
-
-
-// 做自动填充  工具类
-// mp 讲过 大家可以翻之前视频
-@Component
-public class MybatisplusObjectMetaHandler implements MetaObjectHandler {
-
-
-    // mp 在执行添加的时候，这个方法会被执行
-    @Override
-    public void insertFill(MetaObject metaObject) {
-
-        this.setFieldValByName("createDate",new Date(),metaObject);
-
-    }
-
-    // mp 在执行修改的时候，这个方法会执行
-    @Override
-    public void updateFill(MetaObject metaObject) {
-        this.setFieldValByName("updateDate",new Date(),metaObject);
-
-    }
-}
 \ No newline at end of file
+//package com.canrd.webmagic.config;
+//
+//import com.baomidou.mybatisplus.core.handlers.MetaObjectHandler;
+//import org.apache.ibatis.reflection.MetaObject;
+//import org.springframework.stereotype.Component;
+//
+//import java.util.Date;
+//
+//
+//@Component
+//public class MybatisplusObjectMetaHandler implements MetaObjectHandler {
+//    @Override
+//    public void insertFill(MetaObject metaObject) {
+//        this.setFieldValByName("createDate",new Date(),metaObject);
+//    }
+//
+//    @Override
+//    public void updateFill(MetaObject metaObject) {
+//        this.setFieldValByName("updateDate",new Date(),metaObject);
+//    }
+//}
 \ No newline at end of file
+package com.canrd.webmagic.controller;
+
+import com.canrd.webmagic.common.constant.ServerResult;
+import com.canrd.webmagic.processor.IPUqiDataProcessor;
+import com.canrd.webmagic.processor.ScienginePcoessor;
+import com.canrd.webmagic.processor.download.SeleniumDownloader;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.logging.log4j.core.util.UuidUtil;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Spider;
+
+import javax.annotation.Resource;
+@RestController
+@RequestMapping("/uqidata/ip")
+@Api("IP-Uqidata")
+public class IPUqiDataController {
+    @Resource
+    private IPUqiDataProcessor ipUqiDataProcessor;
+
+    @Resource
+    private SeleniumDownloader seleniumDownloader;
+
+
+    @GetMapping("/start")
+    @ApiOperation("start")
+    public ServerResult start()  {
+        Spider.create(ipUqiDataProcessor)
+                .setUUID(UuidUtil.getTimeBasedUuid().toString())
+                .addRequest(new Request("https://ip.uqidata.com/freeip-0.html"))
+                .setDownloader(seleniumDownloader)
+                .thread(1).run();
+        return ServerResult.success();
+    }
+}
+package com.canrd.webmagic.controller;
+
+import com.canrd.webmagic.common.constant.ServerResult;
+import com.canrd.webmagic.domain.dto.IpDO;
+import com.canrd.webmagic.service.IpService;
+import com.canrd.webmagic.util.PingUtil;
+import io.swagger.annotations.Api;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import javax.annotation.Resource;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+@RestController
+@RequestMapping("/ip")
+@Api("Ip-Check")
+public class IpCheckController {
+    @Resource
+    private IpService ipService;
+
+    @GetMapping("/getIpById/{id}")
+    public ServerResult getIpById(@PathVariable int id){
+        IpDO ip = ipService.getById(id);
+        return ServerResult.success(ip);
+    }
+
+    @GetMapping("/getIps")
+    public ServerResult getIps(){
+        List<IpDO> ipList = ipService.findAll();
+        return ServerResult.success(ipList);
+    }
+
+    @GetMapping("/checkIps")
+    public ServerResult checkIps(){
+        List<IpDO> ipList = ipService.findAll();
+        AtomicInteger error= new AtomicInteger();
+        AtomicInteger success= new AtomicInteger();
+        ipList.forEach(ip-> {
+            if (!PingUtil.ping(ip.getIp(),4,3000)) {
+                ip.setIsUse(1);
+                ip.setDetectionTime(new Date());
+                ipService.updateById(ip);
+                error.addAndGet(1);
+            }else {
+                success.addAndGet(1);
+            }
+        });
+        String mes ="本次检测,存活ip数为"+success+",失活ip数为"+error;
+        return ServerResult.success(mes);
+    }
+
+    @GetMapping("/randomIp")
+    public ServerResult randomIp(){
+        List<IpDO> ip = ipService.findAll();
+        int size = ip.size();
+        int ran = (int) (Math.random() * size-1);
+        IpDO ipDO = ip.get(ran);
+        return ServerResult.success(ipDO);
+    }
+}
@@ -35,7 +35,7 @@ public class SciengineController {
                 .setUUID(UuidUtil.getTimeBasedUuid().toString())
                 .addRequest(new Request("https://www.sciengine.com/plat/search?queryField_a=battery"))
                 .addPipeline(articlePipeline)
-                .thread(20).run();
+                .thread(60).run();
         return ServerResult.success();
     }
 }
+package com.canrd.webmagic.domain.dto;
+
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.*;
+import lombok.experimental.SuperBuilder;
+
+import java.util.Date;
+
+@TableName("ip")
+@Data
+public class IpDO{
+    private int id;
+    private String ip;
+    private int port;
+    private String type;
+    private String anonymity;
+    private String region;
+    private String speed;
+    private Date detectionTime;
+    private int isUse;
+}
+package com.canrd.webmagic.mapper;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.canrd.webmagic.domain.dto.IpDO;
+
+import java.util.List;
+
+public interface IpMapper extends BaseMapper<IpDO> {
+    List<IpDO> findAll();
+}
+package com.canrd.webmagic.processor;
+
+import com.canrd.webmagic.domain.dto.IpDO;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.processor.PageProcessor;
+import us.codecraft.webmagic.selector.Html;
+import us.codecraft.webmagic.selector.Selectable;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+@Component
+@Slf4j
+public class IPUqiDataProcessor implements PageProcessor {
+    private static int index = 0;
+
+    @Override
+    public void process(Page page) {
+        String url = page.getUrl().get();
+        if (url.contains("https://ip.uqidata.com/freeip-") && index == 0) {
+            getMaxPage(page);
+        } else if (url.contains("https://ip.uqidata.com/freeip-") && index == 1) {
+            everyPage(page);
+        } else if (url.contains("https://ip.uqidata.com/freeip/")) {
+            getPageDetail(page);
+        }
+    }
+
+    @Override
+    public Site getSite() {
+        return PageProcessor.super.getSite();
+    }
+
+    void getMaxPage(Page page) {
+        index = 1;
+        Html html = page.getHtml();
+        List<String> pageList = html.xpath("//div[@class='wp-pagenavi pd25']/a/text()").all();
+        for (int i = 1; i <= Integer.parseInt(pageList.get(pageList.size() - 1)); i++) {
+            page.addTargetRequest("https://ip.uqidata.com/freeip-" + i + ".html");
+        }
+    }
+
+    void everyPage(Page page) {
+        Html html = page.getHtml();
+        log.info(page.getUrl().get());
+        List<String> hrefList = html.xpath("//div[@class='entry entry-content freeip']/h2/a/@href").all();
+        hrefList.forEach(href->page.addTargetRequest("https:"+href));
+    }
+
+    void getPageDetail(Page page) {
+        Html html = page.getHtml();
+        String ips = html.xpath("//div[@class='content']/p[@class!='pd25']/text()").get();
+        String[] ipArray = ips.split(" ");
+        List ipDoList = new ArrayList<>();
+        for (int i = 0; i < ipArray.length; i += 3) {
+            IpDO ipDO = new IpDO();
+            String[] ip_port = ipArray[i].split(":");
+            String ip = ip_port[0];
+            int port = Integer.parseInt(ip_port[1]);
+            String anonymity = ipArray[i+1];
+            StringBuffer region = new StringBuffer();
+            char[] chars = ipArray[i + 2].toCharArray();
+            for (int j = 0; j < chars.length; j++) {
+                if (!Character.isDigit(chars[j])){
+                    region.append(chars[j]);
+                }
+            }
+            ipDO.setIp(ip);
+            ipDO.setPort(port);
+            ipDO.setAnonymity(anonymity);
+            ipDO.setRegion(String.valueOf(region));
+            ipDoList.add(ipDO);
+        }
+        page.putField("ips",ipDoList);
+    }
+}
@@ -29,7 +29,7 @@ import java.util.regex.Pattern;
 @Slf4j
 @Component
 public class SeleniumDownloader extends AbstractDownloader {
-    private int sleepTime = 3000000;
+    private int sleepTime = 10000;
  
     @Resource
     private SeleniumConfig config;
@@ -98,7 +98,7 @@ public class SeleniumDownloader extends AbstractDownloader {
                     String pageSource = webDriver.getPageSource();
                     log.info(pageSource);
 //                    WebElement element = webDriver.findElement(By.xpath("//div[@class='form-row']/from/div[@id='jsEnabled']/input"));
-                    webDriver.findElement(By.xpath("//a[@class='ot-sdk-show-settings cookie anchor-text']")).click();
+//                    webDriver.findElement(By.xpath("//a[@class='ot-sdk-show-settings cookie anchor-text']")).click();
                     webDriver.findElement(By.xpath("//input[@id='bdd-email']")).getAttribute("1187551704@qq.com");
                     webDriver.findElement(By.xpath("//button[@id='bdd-els-searchBtn']")).submit();
 //                    String text = element.getText();
@@ -137,6 +137,7 @@ public class SeleniumDownloader extends AbstractDownloader {
             page.setHtml(new Html(content, request.getUrl()));
             page.setUrl(new PlainText(request.getUrl()));
             page.setRequest(request);
+            log.info("onSuccess");
             onSuccess(request, task);
         } catch (Exception e) {
             log.warn("download page {} error", request.getUrl(), e);
+package com.canrd.webmagic.processor.pipeline;
+
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
+import com.canrd.webmagic.domain.dto.ArticleDO;
+import com.canrd.webmagic.domain.dto.IpDO;
+import com.canrd.webmagic.service.IpService;
+import us.codecraft.webmagic.ResultItems;
+import us.codecraft.webmagic.Task;
+import us.codecraft.webmagic.pipeline.Pipeline;
+
+import javax.annotation.Resource;
+import java.util.List;
+import java.util.Objects;
+
+public class IpPipeline implements Pipeline {
+    @Resource
+    private IpService ipService;
+
+    @Override
+    public void process(ResultItems resultItems, Task task) {
+        List<IpDO> ips= resultItems.get("ips");
+        if (CollectionUtils.isNotEmpty(ips)) {
+            for (IpDO ipDO : ips) {
+                QueryWrapper<IpDO> ipDOQueryWrapper = new QueryWrapper<IpDO>()
+                        .eq("ip",ipDO.getIp())
+                        .eq("port",ipDO.getPort())
+                        .eq("anonymity",ipDO.getAnonymity())
+                        .eq("region",ipDO.getRegion());
+                List<IpDO> ipList = ipService.list(ipDOQueryWrapper);
+                if (CollectionUtils.isNotEmpty(ipList)) {
+                    return;
+                }
+                ipService.save(ipDO);
+            }
+        }
+    }
+}
+package com.canrd.webmagic.service;
+
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.service.IService;
+import com.canrd.webmagic.domain.dto.IpDO;
+import com.canrd.webmagic.mapper.IpMapper;
+
+import java.util.List;
+
+public interface IpService extends IService<IpDO> {
+
+    List<IpDO> findAll();
+}
+package com.canrd.webmagic.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.canrd.webmagic.domain.dto.IpDO;
+import com.canrd.webmagic.mapper.IpMapper;
+import com.canrd.webmagic.service.IpService;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+@Service
+public class IpServiceImpl extends ServiceImpl<IpMapper,IpDO> implements IpService {
+
+    @Override
+    public List<IpDO> findAll() {
+        return this.baseMapper.findAll();
+    }
+}
+package com.canrd.webmagic.util;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.InetAddress;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+ 
+public class PingUtil {
+    
+    public static boolean ping(String ipAddress) throws Exception {
+        int  timeOut =  3000 ;
+        boolean status = InetAddress.getByName(ipAddress).isReachable(timeOut);
+        return status;
+    }
+    
+    public static void ping02(String ipAddress) throws Exception {
+        String line = null;
+        try {
+            Process pro = Runtime.getRuntime().exec("ping " + ipAddress);
+            BufferedReader buf = new BufferedReader(new InputStreamReader(
+                    pro.getInputStream()));
+            while ((line = buf.readLine()) != null)
+                System.out.println(line);
+        } catch (Exception ex) {
+            System.out.println(ex.getMessage());
+        }
+    }
+    
+    public static boolean ping(String ipAddress, int pingTimes, int timeOut) {  
+        BufferedReader in = null;  
+        Runtime r = Runtime.getRuntime();  // 将要执行的ping命令,此命令是windows格式的命令  
+        String pingCommand = "ping " + ipAddress + " -n " + pingTimes    + " -w " + timeOut;  
+        try {   // 执行命令并获取输出  
+            System.out.println(pingCommand);   
+            Process p = r.exec(pingCommand);   
+            if (p == null) {    
+                return false;   
+            }
+            in = new BufferedReader(new InputStreamReader(p.getInputStream()));   // 逐行检查输出,计算类似出现=23ms TTL=62字样的次数  
+            int connectedCount = 0;   
+            String line = null;   
+            while ((line = in.readLine()) != null) {    
+                connectedCount += getCheckResult(line);   
+            }   // 如果出现类似=23ms TTL=62这样的字样,出现的次数=测试次数则返回真  
+            return connectedCount == pingTimes;  
+        } catch (Exception ex) {   
+            ex.printStackTrace();   // 出现异常则返回假  
+            return false;  
+        } finally {   
+            try {    
+                in.close();   
+            } catch (IOException e) {    
+                e.printStackTrace();   
+            }  
+        }
+    }
+    //若line含有=18ms TTL=16字样,说明已经ping通,返回1,否則返回0.
+    private static int getCheckResult(String line) {  // System.out.println("控制台输出的结果为:"+line);  
+        Pattern pattern = Pattern.compile("(\\d+ms)(\\s+)(TTL=\\d+)",Pattern.CASE_INSENSITIVE);
+        Matcher matcher = pattern.matcher(line);  
+        while (matcher.find()) {
+            return 1;
+        }
+        return 0; 
+    }
+    public static void main(String[] args) throws Exception {
+        String ipAddress = "127.0.0.1";
+        System.out.println(ping(ipAddress));
+        ping02(ipAddress);
+        System.out.println(ping(ipAddress, 5, 5000));
+    }
+}
 \ No newline at end of file
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.canrd.webmagic.mapper.IpMapper">
+
+    <resultMap id="IpMap" type="com.canrd.webmagic.domain.dto.IpDO">
+    </resultMap>
+
+    <sql id="columns">
+        id,ip,port,type,anonymity,region，speed,detection_time,is_use
+    </sql>
+    <select id="findAll" resultMap="IpMap">
+        select <include refid="columns"></include>
+        from ip
+        <where>
+            is_use = 0
+        </where>
+    </select>
+
+</mapper>
 \ No newline at end of file
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.canrd.webmagic.mapper.IpMapper">
+
+    <resultMap id="IpMap" type="com.canrd.webmagic.domain.dto.IpDO">
+    </resultMap>
+
+    <sql id="columns">
+        id,ip,port,type,anonymity,region，speed,detection_time,is_use
+    </sql>
+    <select id="findAll" resultMap="IpMap">
+        select <include refid="columns"></include>
+        from ip
+        <where>
+            is_use = 0
+        </where>
+    </select>
+
+</mapper>
 \ No newline at end of file