Blame view

src/main/java/com/canrd/webmagic/controller/ChemicalController.java 1.66 KB
凌世锦 authored
1
2
3
package com.canrd.webmagic.controller;
4
5
6
7
8
9
import com.canrd.webmagic.common.constant.ServerResult;
import com.canrd.webmagic.processor.ChemicalPcoessor;
import com.canrd.webmagic.processor.ChemicalsciencePcoessor;
import com.canrd.webmagic.processor.MatterPagePcoessor;
import com.canrd.webmagic.processor.download.SeleniumDownloader;
import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
凌世锦 authored
10
import io.swagger.annotations.Api;
11
12
13
import io.swagger.annotations.ApiOperation;
import org.apache.logging.log4j.core.util.UuidUtil;
import org.springframework.web.bind.annotation.GetMapping;
凌世锦 authored
14
15
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
16
17
18
19
20
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.scheduler.RedisScheduler;

import javax.annotation.Resource;
凌世锦 authored
21
22

@RestController
23
24
@RequestMapping("/chemical/article")
@Api("Chemical")
凌世锦 authored
25
public class ChemicalController {
26
27
28
29
30
31
32
33
    @Resource
    private ChemicalPcoessor chemicalPcoessor;

    @Resource
    private SeleniumDownloader seleniumDownloader;

    @Resource
    private ArticlePipeline articlePipeline;
凌世锦 authored
34
35
36
37
38
39
40
41
42
43
    @GetMapping("/start")
    @ApiOperation("start")
    public ServerResult start()  {
        Spider.create(chemicalPcoessor)
                .setUUID(UuidUtil.getTimeBasedUuid().toString())
                .addRequest(new Request("https://pubs.rsc.org/en/results?searchtext=battery"))
//                .setScheduler(new RedisScheduler("127.0.0.1"))
                .addPipeline(articlePipeline)
//                .setDownloader(seleniumDownloader)
凌世锦 authored
44
                .thread(100).run();
45
46
        return ServerResult.success();
    }
凌世锦 authored
47
}