trush / webmagic-canrd-service · Files

package com.canrd.webmagic.processor.download;

import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;

import java.util.Random;

/**
 * @author: xms
 * @description: TODO
 * @date: 2024/4/9 10:37
 * @version: 1.0
 */
@Slf4j
@Component
public class Downloader {
    private static RedisTemplate redisTemplate;

    @Autowired
    Downloader(RedisTemplate redisTemplate) {
        Downloader.redisTemplate = redisTemplate;
    }

    /**
     * @return
     */

    public HttpClientDownloader newIpDownloader() {

        HttpClientDownloader downloader = new HttpClientDownloader() {
            @Override
            protected void onError(Request request) {
                String[] ips = newIp();
                setProxyProvider(SimpleProxyProvider.from(new Proxy(ips[0], Integer.parseInt(ips[1]))));
            }
        };

        String[] ips = newIp();
        downloader.setProxyProvider(SimpleProxyProvider.from(new Proxy(ips[0], Integer.parseInt(ips[1]))));

        return downloader;
    }

    private String[] newIp() {

        Long size = redisTemplate.opsForList().size("ip");
        String ip = redisTemplate.opsForList().index("ip", new Random().nextInt(size.intValue())).toString();
        log.info("获取ip===========>" + ip);
        String[] ips = ip.split(":");
        return ips;
    }
}