添加其他网站的爬取方法

PurelzMgnead
1 parent 9f5b8d9b
Showing 16 changed files with 559 additions and 5 deletions
.idea/inspectionProfiles/Project_Default.xml
pom.xml
src/main/java/com/canrd/webmagic/controller/ChemicalController.java
src/main/java/com/canrd/webmagic/controller/MatterController.java
src/main/java/com/canrd/webmagic/domain/ArticleTypeEnum.java
src/main/java/com/canrd/webmagic/processor/AdvancedEnergyMaterialPcoessor.java
src/main/java/com/canrd/webmagic/processor/ChemicalPcoessor.java
src/main/java/com/canrd/webmagic/processor/ChemicalsciencePcoessor.java
src/main/java/com/canrd/webmagic/processor/download/SeleniumDownloader.java
target/classes/com/canrd/webmagic/controller/ChemicalController.class
target/classes/com/canrd/webmagic/controller/MatterController.class
target/classes/com/canrd/webmagic/domain/ArticleTypeEnum.class
target/classes/com/canrd/webmagic/processor/AdvancedEnergyMaterialPcoessor.class
target/classes/com/canrd/webmagic/processor/ChemicalPcoessor.class
target/classes/com/canrd/webmagic/processor/ChemicalsciencePcoessor.class
target/classes/com/canrd/webmagic/processor/download/SeleniumDownloader.class
 <component name="InspectionProjectProfileManager">
   <profile version="1.0">
     <option name="myName" value="Project Default" />
+    <inspection_tool class="AutoCloseableResource" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="METHOD_MATCHER_CONFIG" value="java.util.Formatter,format,java.io.Writer,append,com.google.common.base.Preconditions,checkNotNull,org.hibernate.Session,close,java.io.PrintWriter,printf,java.io.PrintStream,printf,okhttp3.Call,execute" />
+    </inspection_tool>
     <inspection_tool class="NonSerializableWithSerialVersionUIDField" enabled="true" level="WARNING" enabled_by_default="true" />
     <inspection_tool class="SpringJavaInjectionPointsAutowiringInspection" enabled="false" level="ERROR" enabled_by_default="false" />
   </profile>
@@ -77,6 +77,12 @@
             <version>2.1.8</version>
         </dependency>
+        <dependency>
+            <groupId>com.squareup.okhttp3</groupId>
+            <artifactId>okhttp</artifactId>
+            <version>3.8.1</version>
+        </dependency>
+
         <!-- webmagic核心库 -->
         <dependency>
             <groupId>us.codecraft</groupId>
 package com.canrd.webmagic.controller;
+import com.canrd.webmagic.common.constant.ServerResult;
+import com.canrd.webmagic.processor.ChemicalPcoessor;
+import com.canrd.webmagic.processor.ChemicalsciencePcoessor;
+import com.canrd.webmagic.processor.MatterPagePcoessor;
+import com.canrd.webmagic.processor.download.SeleniumDownloader;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
 import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.logging.log4j.core.util.UuidUtil;
+import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.scheduler.RedisScheduler;
+
+import javax.annotation.Resource;
 @RestController
-@RequestMapping("/nature/article")
-@Api("Nature")
+@RequestMapping("/chemical/article")
+@Api("Chemical")
 public class ChemicalController {
+    @Resource
+    private ChemicalPcoessor chemicalPcoessor;
+
+    @Resource
+    private SeleniumDownloader seleniumDownloader;
+
+    @Resource
+    private ArticlePipeline articlePipeline;
+    @GetMapping("/start")
+    @ApiOperation("start")
+    public ServerResult start()  {
+        Spider.create(chemicalPcoessor)
+                .setUUID(UuidUtil.getTimeBasedUuid().toString())
+                .addRequest(new Request("https://pubs.rsc.org/en/results?searchtext=battery"))
+//                .setScheduler(new RedisScheduler("127.0.0.1"))
+                .addPipeline(articlePipeline)
+//                .setDownloader(seleniumDownloader)
+                .thread(1).run();
+        return ServerResult.success();
+    }
 }
@@ -26,13 +26,14 @@ public class MatterController {
     @GetMapping("/start")
     @ApiOperation("start")
     public ServerResult start()  {
+
         Spider.create(matterPragePcoessor)
                 // 添加这个Spider要爬取的网页地址
                 .addUrl("https://www.cell.com/matter/home")
                 .setUUID(UuidUtil.getTimeBasedUuid().toString())
                 .setDownloader(seleniumDownloader)
                 // 开启5个线程执行，并开始爬取
-                .thread(5).run();
+                .thread(60).start();
         return ServerResult.success();
     }
 }
@@ -25,6 +25,7 @@ public enum ArticleTypeEnum {
     NATURE_COMMUNICATIONS("nature-communications","nuture网站-communications"),
     NATURE_COMPUTATIONAL_SCIENCE("nature-computational|science","nuture网站-computational|science"),
     NATURE_METHODS("nature-methods","nuture网站-methods"),
+    Chemical("chemical","chemical网站")
     ;
     private String type;
     private String desc;
@@ -4,10 +4,14 @@ import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.Site;
 import us.codecraft.webmagic.processor.PageProcessor;
+import java.util.ArrayList;
+import java.util.List;
+
 public class AdvancedEnergyMaterialPcoessor implements PageProcessor {
     @Override
     public void process(Page page) {
-
+        String url = page.getUrl().get();
+        if (url.equals("https://techxplore.com/journals/advanced-energy-materials/")){}
     }
     @Override
@@ -15,5 +19,5 @@ public class AdvancedEnergyMaterialPcoessor implements PageProcessor {
         return PageProcessor.super.getSite();
     }
-    
+
 }
+package com.canrd.webmagic.processor;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
+import com.canrd.webmagic.common.utils.DateUtil;
+import com.canrd.webmagic.common.utils.StringUtils;
+import com.canrd.webmagic.domain.ArticleTypeEnum;
+import com.canrd.webmagic.domain.dto.ArticleDO;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import com.google.common.net.MediaType;
+import lombok.extern.slf4j.Slf4j;
+import okhttp3.MultipartBody;
+import okhttp3.OkHttpClient;
+import okhttp3.Response;
+import okhttp3.ResponseBody;
+import org.springframework.stereotype.Component;
+import org.springframework.web.bind.annotation.RequestBody;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.model.HttpRequestBody;
+import us.codecraft.webmagic.processor.PageProcessor;
+import us.codecraft.webmagic.selector.Html;
+import us.codecraft.webmagic.selector.Selectable;
+import us.codecraft.webmagic.utils.HttpConstant;
+
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Component
+public class ChemicalPcoessor implements PageProcessor {
+    private final Site request = Site.me().setTimeOut(300000);
+
+    private int index;
+    private String substring;
+
+    private HttpRequestBody httpRequestBody;
+
+    private ConcurrentHashMap<String, Object> map = new ConcurrentHashMap<>();
+
+    @Override
+    public void process(Page page) {
+//        synchronized (this) {
+        String url = page.getUrl().get();
+        if (url.equals("https://pubs.rsc.org/en/results?searchtext=battery")) {
+            try {
+                getMaxPages(page);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+//            }else if (url.equals("https://pubs.rsc.org/en/search/journalresult")&&index==0) {
+//                getResultMax(page);
+        } else if (url.equals("https://pubs.rsc.org/en/search/journalresult")) {
+            eveyPage(page);
+        } else if (url.contains("https://pubs.rsc.org/en/content")) {
+            getDetil(page);
+        }
+//        }
+    }
+
+    void getMaxPages(Page page) throws IOException {
+        index = 0;
+        Html html = page.getHtml();
+        String script = html.css("script").regex(".*var searchResultCounts = (.*?);").get();
+//        log.info(script);
+//        String replace = script.replace("[", "");
+//        String replace1 = replace.replace("]", "");
+//        String[] split = replace1.split(",");
+//        int i1 = split[1].indexOf(":");
+//        int i2 = split[1].indexOf("}");
+//        substring = split[1].substring(i1 + 1, i1);
+//        if (!StringUtils.isEmpty(substring)) {
+        for (int i = 1; i <= 2118; i++) {
+            String baseUrl = "https://pubs.rsc.org/en/search/journalresult";
+            map.put("searchterm", "PD94bWwgdmVyc2lvbj0iMS4wIj8+DQo8U2VhcmNoVGVybSB4" +
+                    "bWxuczp4c2Q9Imh0dHA6Ly93d3cudzMub3JnLzIwMDEvWE1MU2NoZW1hIiB4bW" +
+                    "xuczp4c2k9Imh0dHA6Ly93d3cudzMub3JnLzIwMDEvWE1MU2NoZW1hLWluc3Rh" +
+                    "bmNlIj4NCiAgPENhdGVnb3J5PmpvdXJuYWw8L0NhdGVnb3J5Pg0KICA8U3ViQ2" +
+                    "F0ZWdvcnkgLz4NCiAgPENvbnRlbnRUeXBlPmpvdXJuYWw8L0NvbnRlbnRUeXBl" +
+                    "Pg0KICA8Q3JpdGVyaWFzPg0KICAgIDxOYW1lVmFsdWU+DQogICAgICA8TmFtZT" +
+                    "5mcmVldGV4dDwvTmFtZT4NCiAgICAgIDxWYWx1ZT5iYXR0ZXJ5PC9WYWx1ZT4N" +
+                    "CiAgICA8L05hbWVWYWx1ZT4NCiAgICA8TmFtZVZhbHVlPg0KICAgICAgPE5hbW" +
+                    "U+T3JpZ2luYWxGcmVlVGV4dDwvTmFtZT4NCiAgICAgIDxWYWx1ZT5iYXR0ZXJ5" +
+                    "PC9WYWx1ZT4NCiAgICA8L05hbWVWYWx1ZT4NCiAgPC9Dcml0ZXJpYXM+DQogID" +
+                    "xGYWNldHMgLz4NCiAgPFJlcXVlc3RUaW1lPjAwMDEtMDEtMDFUMDA6MDA6MDA8" +
+                    "L1JlcXVlc3RUaW1lPg0KICA8QXV0aG9yQ3JpdGVyaWEgLz4NCiAgPFB1YmxpY2" +
+                    "F0aW9uRGF0ZT4NCiAgICA8SXNTZWxlY3RlZERhdGU+ZmFsc2U8L0lzU2VsZWN0" +
+                    "ZWREYXRlPg0KICA8L1B1YmxpY2F0aW9uRGF0ZT4NCiAgPEV4Y2x1ZGVzIC8+DQ" +
+                    "o8L1NlYXJjaFRlcm0+");
+            map.put("resultcount", 52942);
+            map.put("pageno", i);
+            httpRequestBody = HttpRequestBody.form(map, "UTF-8");
+            Request request = new Request(baseUrl).setMethod(HttpConstant.Method.POST)
+                    .addHeader("Content-Type", "application/x-www-form-urlencoded")
+                    .addHeader("Connection", "keep-alive")
+                    .addHeader("Host", "pubs.rsc.org")
+                    .addHeader("Accept-Encoding", "gzip, deflate, br")
+                    .addHeader("Accept", "*/*")
+                    .addHeader("Cookie", "X-Mapping-hhmaobcf=3C9897E0250B6117CDBA3EC8F724E9A9;_PubsBFCleared=1;ASP.NET_SessionId=bang00oiyj1p1v45jkuk2cgt;ShowEUCookieLawBanner=true")
+                    .setCharset("UTF-8");
+//            log.info(map.toString());
+            request.setRequestBody(httpRequestBody);
+            page.addTargetRequest(request);
+
+//        OkHttpClient client = new OkHttpClient().newBuilder()
+//                .build();
+//        MediaType mediaType = MediaType.parse("text/plain");
+//        for (int i = 1; i <= 3; i++) {
+//            MultipartBody body = new MultipartBody.Builder().setType(MultipartBody.FORM)
+//                    .addFormDataPart("searchterm", "PD94bWwgdmVyc2lvbj0iMS4wIj8+DQo8U2VhcmNoVGVybSB4bWxuczp4c2Q9Imh0dHA6Ly93d3cudzMub3JnLzIwMDEvWE1MU2NoZW1hIiB4bWxuczp4c2k9Imh0dHA6Ly93d3cudzMub3JnLzIwMDEvWE1MU2NoZW1hLWluc3RhbmNlIj4NCiAgPENhdGVnb3J5PmpvdXJuYWw8L0NhdGVnb3J5Pg0KICA8U3ViQ2F0ZWdvcnkgLz4NCiAgPENvbnRlbnRUeXBlPmpvdXJuYWw8L0NvbnRlbnRUeXBlPg0KICA8Q3JpdGVyaWFzPg0KICAgIDxOYW1lVmFsdWU+DQogICAgICA8TmFtZT5mcmVldGV4dDwvTmFtZT4NCiAgICAgIDxWYWx1ZT5iYXR0ZXJ5PC9WYWx1ZT4NCiAgICA8L05hbWVWYWx1ZT4NCiAgICA8TmFtZVZhbHVlPg0KICAgICAgPE5hbWU+T3JpZ2luYWxGcmVlVGV4dDwvTmFtZT4NCiAgICAgIDxWYWx1ZT5iYXR0ZXJ5PC9WYWx1ZT4NCiAgICA8L05hbWVWYWx1ZT4NCiAgPC9Dcml0ZXJpYXM+DQogIDxGYWNldHMgLz4NCiAgPFJlcXVlc3RUaW1lPjAwMDEtMDEtMDFUMDA6MDA6MDA8L1JlcXVlc3RUaW1lPg0KICA8QXV0aG9yQ3JpdGVyaWEgLz4NCiAgPFB1YmxpY2F0aW9uRGF0ZT4NCiAgICA8SXNTZWxlY3RlZERhdGU+ZmFsc2U8L0lzU2VsZWN0ZWREYXRlPg0KICA8L1B1YmxpY2F0aW9uRGF0ZT4NCiAgPEV4Y2x1ZGVzIC8+DQo8L1NlYXJjaFRlcm0+")
+//                    .addFormDataPart("resultcount", "25")
+////                    .addFormDataPart("category", "journal")
+//                    .addFormDataPart("pageno", String.valueOf(i))
+//                    .build();
+//            okhttp3.Request okrequest = new okhttp3.Request.Builder()
+//                    .url("https://pubs.rsc.org/en/search/journalresult")
+//                    .method("POST", body)
+//                    .addHeader("Cookie", "X-Mapping-hhmaobcf=3C9897E0250B6117CDBA3EC8F724E9A9;_PubsBFCleared=1;ASP.NET_SessionId=bang00oiyj1p1v45jkuk2cgt;ShowEUCookieLawBanner=true")
+//                    .build();
+//            Response response = client.newCall(okrequest).execute();
+//            String responseBody = response.body().string();
+//            log.info(String.valueOf(responseBody));
+        }
+    }
+//        }else{
+//            throw new RuntimeException();
+//        }
+//    }
+
+    void getResultMax(Page page) {
+        index = 1;
+//        .xpath("//span[@class='paging--label']/text()")
+        page.putField("html", page.getHtml());
+        String ever = page.getHtml().xpath("//div[@class='fixpadv--l']/strong/text()").get();
+        int total = Integer.parseInt(substring);
+        int everReq = Integer.parseInt(ever.split(" ")[0]);
+        int pageNo = total / everReq;
+        if ((pageNo * everReq) % total < everReq && (pageNo * everReq) % total == 0) {
+            pageNo = pageNo + 1;
+        }
+        log.info(String.valueOf(pageNo));
+//        String s = page.getHtml().xpath("//span[@class='paging--label']/text()").get();
+//        String[] pageTotal = s.split(" ");
+//        int now = Integer.parseInt(pageTotal[4]);
+//        int total = Integer.parseInt(pageTotal[6]);
+        for (int i = 1; i <= total; i++) {
+            String baseUrl = "https://pubs.rsc.org/en/search/journalresult";
+            Map<String, Object> map = new HashMap<>();
+            map.put("searchterm", "PD94bWwgdmVyc2lvbj0iMS4wIj8+DQo8U2VhcmNoVGVybSB4" +
+                    "bWxuczp4c2Q9Imh0dHA6Ly93d3cudzMub3JnLzIwMDEvWE1MU2NoZW1hIiB4bW" +
+                    "xuczp4c2k9Imh0dHA6Ly93d3cudzMub3JnLzIwMDEvWE1MU2NoZW1hLWluc3Rh" +
+                    "bmNlIj4NCiAgPENhdGVnb3J5PmpvdXJuYWw8L0NhdGVnb3J5Pg0KICA8U3ViQ2" +
+                    "F0ZWdvcnkgLz4NCiAgPENvbnRlbnRUeXBlPmpvdXJuYWw8L0NvbnRlbnRUeXBl" +
+                    "Pg0KICA8Q3JpdGVyaWFzPg0KICAgIDxOYW1lVmFsdWU+DQogICAgICA8TmFtZT" +
+                    "5mcmVldGV4dDwvTmFtZT4NCiAgICAgIDxWYWx1ZT5iYXR0ZXJ5PC9WYWx1ZT4N" +
+                    "CiAgICA8L05hbWVWYWx1ZT4NCiAgICA8TmFtZVZhbHVlPg0KICAgICAgPE5hbW" +
+                    "U+T3JpZ2luYWxGcmVlVGV4dDwvTmFtZT4NCiAgICAgIDxWYWx1ZT5iYXR0ZXJ5" +
+                    "PC9WYWx1ZT4NCiAgICA8L05hbWVWYWx1ZT4NCiAgPC9Dcml0ZXJpYXM+DQogID" +
+                    "xGYWNldHMgLz4NCiAgPFJlcXVlc3RUaW1lPjAwMDEtMDEtMDFUMDA6MDA6MDA8" +
+                    "L1JlcXVlc3RUaW1lPg0KICA8QXV0aG9yQ3JpdGVyaWEgLz4NCiAgPFB1YmxpY2" +
+                    "F0aW9uRGF0ZT4NCiAgICA8SXNTZWxlY3RlZERhdGU+ZmFsc2U8L0lzU2VsZWN0" +
+                    "ZWREYXRlPg0KICA8L1B1YmxpY2F0aW9uRGF0ZT4NCiAgPEV4Y2x1ZGVzIC8+DQ" +
+                    "o8L1NlYXJjaFRlcm0+");
+            map.put("resultcount", 25);
+            map.put("pageno", i);
+            HttpRequestBody httpRequestBody = HttpRequestBody.form(map, "UTF-8");
+            Request request = new Request(baseUrl).setMethod(HttpConstant.Method.POST)
+                    .addHeader("Content-Type", "application/x-www-form-urlencoded;charset=utf-8")
+                    .addHeader("Connection", "keep-alive")
+                    .addHeader("Host", "pubs.rsc.org")
+                    .addHeader("Accept-Encoding", "gzip, deflate, br, zstd")
+                    .addHeader("Accept", "text/html, */*; q=0.01")
+                    .setCharset("UTF-8");
+            request.setRequestBody(httpRequestBody);
+            page.addTargetRequest(request);
+        }
+    }
+
+    void eveyPage(Page page) {
+//        <span class="paging--label"> - Showing page 2 of 4000</span>
+//        log.info(page.getHtml().get());
+        List<String> hrefList = page.getHtml().xpath("//a[@class='capsule__action']/@href").all();
+        for (String herf : hrefList) {
+            log.info("https://pubs.rsc.org" + herf);
+            String url = "https://pubs.rsc.org" + herf;
+            page.addTargetRequest(url);
+        }
+    }
+
+    void getDetil(Page page) {
+        Html html = page.getHtml();
+        //文章链接
+        String articleCode = page.getUrl().get();
+        //文章标题
+        String title = html.xpath("//div[@class='article__title']/h2/font/text()").get();
+        if (StringUtils.isBlank(title)) {
+            title = html.xpath("//div[@class='article__title']/h2/text()").get();
+        }
+
+        //文章内容
+        String articleDesc = html.xpath("//div[@class='capsule__text']/p/text()").get();
+
+        //时间
+        String publishTime;
+        Date publishTimeDateTime = null;
+        List<String> all = html.xpath("//div[@class='c fixpadt--l']/dd[@class='c__10']").all();
+        AtomicInteger timeIndex = new AtomicInteger(0);
+        all.stream().filter(s -> {
+            timeIndex.getAndIncrement();
+            return s.equals("test2");
+        }).findFirst();
+        publishTime = html.xpath("//div[@class='c fixpadt--l']/dd[@class='c__14']/text()").all().get(timeIndex.get());
+        SimpleDateFormat formatter = new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH);
+        try {
+            publishTimeDateTime = formatter.parse(publishTime);
+        } catch (ParseException e) {
+            e.printStackTrace();
+        }
+
+        //作者名字
+        List<String> authorList = html.xpath("//div[@class='article__authors']/span[@class='article__author-link']/a/text()").all();
+        List<String> collect = authorList.stream().filter(a -> !a.equals("†")).collect(Collectors.toList());
+
+//        collect.forEach(log::info);
+        StringBuffer stringBuffer = new StringBuffer();
+        collect.forEach(stringBuffer::append);
+
+        StringBuffer authorName = new StringBuffer();
+        authorList.forEach(authorName::append);
+        HashMap<Object, Object> authorHashMap = new HashMap<>();
+        List<String> cupList = html.xpath("//div[@class='article__authors']/span/span/sup/i/text()").all();
+        if (collect.size() == cupList.size()) {
+            for (int i = 0; i < collect.size(); i++) {
+                authorHashMap.put(collect.get(i), cupList.get(i));
+            }
+        }
+//        单位地址 邮箱
+        JSONArray authorAddress = new JSONArray();
+        JSONArray authorMail = new JSONArray();
+        List<Selectable> addressMailList = html.xpath("//p[@class='article__author-affiliation']").nodes();
+        HashMap<Object, Object> addressMap = new HashMap<>();
+        HashMap<Object, Object> mailMap = new HashMap<>();
+        if (CollectionUtils.isNotEmpty(addressMailList)) {
+            for (Selectable selectable : addressMailList) {
+                List<Selectable> nodes = selectable.xpath("//span").nodes();
+                if (CollectionUtils.isNotEmpty(nodes)) {
+                    Selectable keyXpath = nodes.get(0);
+                    Selectable valueXpath = nodes.get(1);
+                    String key = keyXpath.xpath("//sup/text()").get();
+                    String address = valueXpath.xpath("//span/text()").get();
+                    String mail = valueXpath.xpath("//span/a/text()").get();
+                    if (!StringUtils.isBlank(key)) {
+                        if (!StringUtils.isBlank(address)) {
+                            addressMap.put(key, address);
+                        } else {
+                            addressMap.put(key, null);
+                        }
+                        if (!StringUtils.isBlank(mail)) {
+                            mailMap.put(key, mail);
+                        } else {
+                            mailMap.put(key, null);
+                        }
+                    } else {
+                        if (!StringUtils.isBlank(address)) {
+                            addressMap.put("*", address);
+                        } else {
+                            addressMap.put("*", null);
+                        }
+                        if (!StringUtils.isBlank(mail)) {
+                            mailMap.put("*", mail);
+                        } else {
+                            mailMap.put("*", null);
+                        }
+                    }
+                }
+            }
+            Object[] objects = authorHashMap.keySet().stream().toArray();
+//            log.info(Arrays.toString(objects));
+            for (int i = 0; i < objects.length; i++) {
+                JSONObject addressObj = new JSONObject();
+                JSONObject mailObj = new JSONObject();
+                Object point = authorHashMap.get(String.valueOf(objects[i]));
+                Object address = addressMap.get(point);
+                Object mail = mailMap.get(point);
+                addressObj.put("address", address);
+                addressObj.put("authorNames", objects[i]);
+                mailObj.put("authorEmailName", objects[i]);
+                mailObj.put("email", mail);
+
+                authorAddress.add(addressObj);
+                authorMail.add(mailObj);
+            }
+        }
+
+        JSONArray references = new JSONArray();
+        List<Selectable> referenceNodeList = html.xpath("//div[@class='ref-list']/ol/li").nodes();
+        if (CollectionUtils.isNotEmpty(referenceNodeList)) {
+            for (Selectable reference : referenceNodeList) {
+                List<String> herfOpList = reference.xpath("//a/@href").all();
+                ArrayList<Object> herfList = new ArrayList<>();
+                for (String herf : herfOpList) {
+                    if (herf.startsWith("https") || herf.startsWith("http")) {
+                        String before = "https://pubs.rsc.org";
+                        herfList.add(before + herf);
+                    } else {
+                        herfList.add(herf);
+                    }
+                }
+                StringBuffer referenceTitle = new StringBuffer();
+                List<String> spanList = reference.xpath("//span/text()").all();
+                if (CollectionUtils.isNotEmpty(spanList)) {
+                    for (int i = 0; i < spanList.size(); i++) {
+                        referenceTitle.append(spanList.get(i));
+                        if (i < spanList.size() - 1) {
+                            referenceTitle.append(",");
+                        }
+                    }
+                }
+                String em = reference.xpath("//em/text()").get();
+                String strong = reference.xpath("//strong/text()").get();
+                referenceTitle.append(em);
+                referenceTitle.append(strong);
+                JSONObject referencesObj = new JSONObject();
+                referencesObj.put("links", herfList);
+                referencesObj.put("referenceTitle", referenceTitle);
+                references.add(referencesObj);
+            }
+        }
+        log.info("文章链接：{},发布时间:{},标题:{},作者:{},邮箱信息:{}", articleCode, publishTime, title, authorName, authorMail.toJSONString());
+        page.putField("article", ArticleDO.builder()
+                .articleType(ArticleTypeEnum.Chemical.getType())
+                .articleCode(articleCode)
+                .authorName(authorName.toString())
+                .title(title)
+                .publishTime(Objects.isNull(publishTimeDateTime) ? publishTime : DateUtil.format(publishTimeDateTime, DateUtil.DATE))
+                .emailInfo(authorMail.toJSONString())
+                .articleDesc(articleDesc)
+                .authorAddress(authorAddress.toJSONString())
+                .referenceInfo(references.toJSONString()).build());
+    }
+
+    @Override
+    public Site getSite() {
+        return request;
+    }
+
+    public static void main(String[] args) {
+        Spider.create(new ChemicalPcoessor())
+                .addUrl("https://pubs.rsc.org/en/search/journalresult")
+                .addPipeline(new ArticlePipeline())
+                .thread(5).run();
+    }
+}
+package com.canrd.webmagic.processor;
+
+import com.canrd.webmagic.common.utils.StringUtils;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.logging.log4j.core.util.UuidUtil;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.model.HttpRequestBody;
+import us.codecraft.webmagic.processor.PageProcessor;
+import us.codecraft.webmagic.selector.*;
+import us.codecraft.webmagic.utils.HttpConstant;
+
+import java.util.*;
+
+@Slf4j
+@Component
+public class ChemicalsciencePcoessor implements PageProcessor {
+
+    @Autowired
+    private ChemicalPcoessor journalResultPcoessor;
+    private final Site base = Site.me().setDomain("base").setTimeOut(200000);
+
+    @Override
+    public Site getSite() {
+        return base;
+    }
+
+    private String substring;
+
+    @Override
+    public void process(Page page) {
+        String url = page.getUrl().get();
+        if (url.equals("https://pubs.rsc.org/en/results?searchtext=battery")) {
+            getMaxPages(page);
+        } else if (url.equals("https://pubs.rsc.org/en/search/journalresult")) {
+            getResultMax(page);
+        }
+    }
+
+    void getMaxPages(Page page) {
+        Html html = page.getHtml();
+        String script = page.getHtml().css("script").regex(".*var searchResultCounts = (.*?);").get();
+        String replace = script.replace("[", "");
+        String replace1 = replace.replace("]", "");
+        String[] split = replace1.split(",");
+        int i = split[1].indexOf(":");
+        int i1 = split[1].indexOf("}");
+        substring = split[1].substring(i + 1, i1);
+        if (!StringUtils.isEmpty(substring)) {
+            String baseUrl = "https://pubs.rsc.org/en/search/journalresult";
+            Map<String, Object> map = new HashMap<>();
+            map.put("searchterm", "PD94bWwgdmVyc2lvbj0iMS4wIj8+DQo8U2VhcmNoVGVybSB4" +
+                    "bWxuczp4c2Q9Imh0dHA6Ly93d3cudzMub3JnLzIwMDEvWE1MU2NoZW1hIiB4bW" +
+                    "xuczp4c2k9Imh0dHA6Ly93d3cudzMub3JnLzIwMDEvWE1MU2NoZW1hLWluc3Rh" +
+                    "bmNlIj4NCiAgPENhdGVnb3J5PmpvdXJuYWw8L0NhdGVnb3J5Pg0KICA8U3ViQ2" +
+                    "F0ZWdvcnkgLz4NCiAgPENvbnRlbnRUeXBlPmpvdXJuYWw8L0NvbnRlbnRUeXBl" +
+                    "Pg0KICA8Q3JpdGVyaWFzPg0KICAgIDxOYW1lVmFsdWU+DQogICAgICA8TmFtZT" +
+                    "5mcmVldGV4dDwvTmFtZT4NCiAgICAgIDxWYWx1ZT5iYXR0ZXJ5PC9WYWx1ZT4N" +
+                    "CiAgICA8L05hbWVWYWx1ZT4NCiAgICA8TmFtZVZhbHVlPg0KICAgICAgPE5hbW" +
+                    "U+T3JpZ2luYWxGcmVlVGV4dDwvTmFtZT4NCiAgICAgIDxWYWx1ZT5iYXR0ZXJ5" +
+                    "PC9WYWx1ZT4NCiAgICA8L05hbWVWYWx1ZT4NCiAgPC9Dcml0ZXJpYXM+DQogID" +
+                    "xGYWNldHMgLz4NCiAgPFJlcXVlc3RUaW1lPjAwMDEtMDEtMDFUMDA6MDA6MDA8" +
+                    "L1JlcXVlc3RUaW1lPg0KICA8QXV0aG9yQ3JpdGVyaWEgLz4NCiAgPFB1YmxpY2" +
+                    "F0aW9uRGF0ZT4NCiAgICA8SXNTZWxlY3RlZERhdGU+ZmFsc2U8L0lzU2VsZWN0" +
+                    "ZWREYXRlPg0KICA8L1B1YmxpY2F0aW9uRGF0ZT4NCiAgPEV4Y2x1ZGVzIC8+DQ" +
+                    "o8L1NlYXJjaFRlcm0+");
+            log.info(substring);
+            map.put("resultcount",25);
+            map.put("pageno",2);
+            HttpRequestBody httpRequestBody = HttpRequestBody.form(map, "UTF-8");
+            Request request = new Request(baseUrl).setMethod(HttpConstant.Method.POST)
+                    .addHeader("Content-Type", "application/x-www-form-urlencoded;charset=utf-8")
+                    .addHeader("Connection", "keep-alive")
+                    .addHeader("Host", "pubs.rsc.org")
+                    .addHeader("Accept-Encoding", "gzip, deflate, br, zstd")
+                    .addHeader("Accept", "text/html, */*; q=0.01")
+                    .setCharset("UTF-8");
+            request.setRequestBody(httpRequestBody);
+            page.addTargetRequest(request);
+        } else {
+            throw new RuntimeException();
+        }
+    }
+
+    void getResultMax(Page page) {
+//        .xpath("//span[@class='paging--label']/text()")
+        page.putField("html", page.getHtml());
+        String ever = page.getHtml().xpath("//div[@class='fixpadv--l']/strong/text()").get();
+        int total = Integer.parseInt(substring);
+        int everReq = Integer.parseInt(ever.split(" ")[0]);
+        int pageNo=total/everReq;
+        if ((pageNo*everReq)%total<everReq&&(pageNo*everReq)%total==0){
+            pageNo=pageNo+1;
+        }
+        log.info(String.valueOf(pageNo));
+//        String s = page.getHtml().xpath("//span[@class='paging--label']/text()").get();
+//        String[] pageTotal = s.split(" ");
+//        int now = Integer.parseInt(pageTotal[4]);
+//        int total = Integer.parseInt(pageTotal[6]);
+        for (int i = 1; i <= total; i++) {
+        String baseUrl = "https://pubs.rsc.org/en/search/journalresult";
+        Map<String, Object> map = new HashMap<>();
+        map.put("searchterm", "PD94bWwgdmVyc2lvbj0iMS4wIj8+DQo8U2VhcmNoVGVybSB4" +
+                "bWxuczp4c2Q9Imh0dHA6Ly93d3cudzMub3JnLzIwMDEvWE1MU2NoZW1hIiB4bW" +
+                "xuczp4c2k9Imh0dHA6Ly93d3cudzMub3JnLzIwMDEvWE1MU2NoZW1hLWluc3Rh" +
+                "bmNlIj4NCiAgPENhdGVnb3J5PmpvdXJuYWw8L0NhdGVnb3J5Pg0KICA8U3ViQ2" +
+                "F0ZWdvcnkgLz4NCiAgPENvbnRlbnRUeXBlPmpvdXJuYWw8L0NvbnRlbnRUeXBl" +
+                "Pg0KICA8Q3JpdGVyaWFzPg0KICAgIDxOYW1lVmFsdWU+DQogICAgICA8TmFtZT" +
+                "5mcmVldGV4dDwvTmFtZT4NCiAgICAgIDxWYWx1ZT5iYXR0ZXJ5PC9WYWx1ZT4N" +
+                "CiAgICA8L05hbWVWYWx1ZT4NCiAgICA8TmFtZVZhbHVlPg0KICAgICAgPE5hbW" +
+                "U+T3JpZ2luYWxGcmVlVGV4dDwvTmFtZT4NCiAgICAgIDxWYWx1ZT5iYXR0ZXJ5" +
+                "PC9WYWx1ZT4NCiAgICA8L05hbWVWYWx1ZT4NCiAgPC9Dcml0ZXJpYXM+DQogID" +
+                "xGYWNldHMgLz4NCiAgPFJlcXVlc3RUaW1lPjAwMDEtMDEtMDFUMDA6MDA6MDA8" +
+                "L1JlcXVlc3RUaW1lPg0KICA8QXV0aG9yQ3JpdGVyaWEgLz4NCiAgPFB1YmxpY2" +
+                "F0aW9uRGF0ZT4NCiAgICA8SXNTZWxlY3RlZERhdGU+ZmFsc2U8L0lzU2VsZWN0" +
+                "ZWREYXRlPg0KICA8L1B1YmxpY2F0aW9uRGF0ZT4NCiAgPEV4Y2x1ZGVzIC8+DQ" +
+                "o8L1NlYXJjaFRlcm0+");
+        map.put("resultcount",25);
+        map.put("pageno",i);
+//        map.put("pageno",1);
+        HttpRequestBody httpRequestBody = HttpRequestBody.form(map, "UTF-8");
+        Request request = new Request(baseUrl).setMethod(HttpConstant.Method.POST)
+                .addHeader("Content-Type", "application/x-www-form-urlencoded;charset=utf-8")
+                .addHeader("Connection", "keep-alive")
+                .addHeader("Host", "pubs.rsc.org")
+                .addHeader("Accept-Encoding", "gzip, deflate, br, zstd")
+                .addHeader("Accept", "text/html, */*; q=0.01")
+                .setCharset("UTF-8");
+        request.setRequestBody(httpRequestBody);
+//        page.addTargetRequest(request);
+        Spider.create(journalResultPcoessor)
+                .setUUID(UuidUtil.getTimeBasedUuid().toString())
+                .addRequest(request)
+                .addPipeline(new ArticlePipeline())
+                .thread(1).start();
+    }
+    }
+}
@@ -71,6 +71,7 @@ public class SeleniumDownloader extends AbstractDownloader {
                 element.submit();
                 WebDriverWait wait = new WebDriverWait(webDriver, 30);
                 wait.until(ExpectedConditions.urlContains("https://www.cell.com/action/doSearch?"));
+
 //                wait.until(ExpectedConditions.visibilityOfElementLocated(By.xpath("//div[@class='cb-c']")));
 //                WebElement cloudFlare = webDriver.findElement(By.xpath("//div[@class='cb-c']/label/input"));
 //                if (cloudFlare!=null){