[Fix] #698 修复使用Redis,Request丢失附加信息问题

[jianyun8023]

[Fix] #698 修复使用Redis,Request丢失附加信息问题

[jianyun8023]

ci报JAVA_HOME找不到可以考虑使用openjdk7
我今天看你使用的ci,也去使用了它,参考你使用的oraclejdk7,也报同样的问题,换成openjdk7就好了
.travis.yml

[code4craft]

已合！另外，方法名不要大写，CheckForAdditionalInfo这个也改一下？

[jianyun8023]

尴尬了,手残了,尴尬,你直接修改吧

[code4craft]

改掉了，另外pool.returnResource(jedis);也被废弃了，我改成了新的jedis.close()

[jianyun8023]

基于spring data redis 弄得

另外我用你的爬虫+springboot+spring boot admin整合了一个简单的分布式爬虫监控

package cc.yihy.spider; import com.alibaba.fastjson.JSON; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.redis.core.HashOperations; import org.springframework.data.redis.core.ListOperations; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.data.redis.core.SetOperations; import org.springframework.stereotype.Component; import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Task; import us.codecraft.webmagic.scheduler.DuplicateRemovedScheduler; import us.codecraft.webmagic.scheduler.MonitorableScheduler; import us.codecraft.webmagic.scheduler.component.DuplicateRemover; import javax.annotation.Resource; @Component public class RedisDataScheduler extends DuplicateRemovedScheduler implements MonitorableScheduler, DuplicateRemover { private static final String QUEUE_PREFIX = "queue_"; private static final String SET_PREFIX = "set_"; private static final String ITEM_PREFIX = "item_"; private static final String HEADER_PREFIX = "header_"; // inject the actual template @Autowired private RedisTemplate<String, String> redisTemplate; // inject the template as ListOperations @Resource(name = "redisTemplate") private SetOperations<String, String> setOps; @Resource(name = "redisTemplate") private ListOperations<String, String> listOps; @Resource(name = "redisTemplate") private HashOperations<String, String, String> hashOps; public RedisDataScheduler() { setDuplicateRemover(this); } /**  * 添加集合元素  *  * @param request  * @param task  * @return  */ @Override public boolean isDuplicate(Request request, Task task) { return setOps.add(getSetKey(task), request.getUrl()).intValue() == 0; } @Override protected void pushWhenNoDuplicate(Request request, Task task) { listOps.rightPush(getQueueKey(task), request.getUrl()); if (checkForAdditionalInfo(request)) { String field = DigestUtils.shaHex(request.getUrl()); String value = JSON.toJSONString(request); hashOps.put(getItemKey(task), field, value); } } private boolean checkForAdditionalInfo(Request request) { if (request == null) { return false; } if (!request.getHeaders().isEmpty() || !request.getCookies().isEmpty()) { return true; } if (StringUtils.isNotBlank(request.getCharset()) || StringUtils.isNotBlank(request.getMethod())) { return true; } if (request.isBinaryContent() || request.getRequestBody() != null) { return true; } if (request.getExtras() != null && !request.getExtras().isEmpty()) { return true; } if (request.getPriority() != 0L) { return true; } return false; } /**  * 删除一个  *  * @param task  */ @Override public void resetDuplicateCheck(Task task) { setOps.remove(getSetKey(task)); } /**  * 获取一个url,  *  * @param task  * @return  */ @Override public Request poll(Task task) { String url = listOps.leftPop(getQueueKey(task)); if (url == null) { return null; } String key = getItemKey(task); String field = DigestUtils.shaHex(url); String value = hashOps.get(key, field); if (value != null) { Request o = JSON.parseObject(value, Request.class); return o; } Request request = new Request(url); return request; } @Override public int getTotalRequestsCount(Task task) { Long size = setOps.size(getSetKey(task)); return size.intValue(); } /**  * 获取队列中的请求数量  *  * @param task  * @return  */ @Override public int getLeftRequestsCount(Task task) { Long size = listOps.size(getQueueKey(task)); return size.intValue(); } protected String getSetKey(Task task) { return SET_PREFIX + task.getUUID(); } protected String getQueueKey(Task task) { return QUEUE_PREFIX + task.getUUID(); } protected String getItemKey(Task task) { return ITEM_PREFIX + task.getUUID(); } }