diff --git a/.gitignore b/.gitignore index 9c0418d..0bbf947 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ target/ \.DS_Store + +*.iml diff --git a/pom.xml b/pom.xml index 29e76e5..b942a7d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,11 +6,12 @@ com.irrigation common-lib - 1.0-SNAPSHOT + 1.2.0 UTF-8 1.8 + 1.5.10.RELEASE 2.7.0 1.16.20 1.11 @@ -19,6 +20,8 @@ 1.2.46 4.3.14.RELEASE 4.2.4.RELEASE + 2.6.11 + @@ -83,6 +86,36 @@ ${spring-security-core.version} provided + + net.sf.ehcache + ehcache-core + ${ehcache.version} + + + org.slf4j + slf4j-api + + + + + + org.springframework.boot + spring-boot-starter-cache + ${spring-boot.version} + + + + org.springframework.boot + spring-boot-starter-data-redis + ${spring-boot.version} + + + + org.springframework.boot + spring-boot-configuration-processor + ${spring-boot.version} + + diff --git a/src/main/java/com/irrigation/icl/cache/Command.java b/src/main/java/com/irrigation/icl/cache/Command.java new file mode 100644 index 0000000..6e12468 --- /dev/null +++ b/src/main/java/com/irrigation/icl/cache/Command.java @@ -0,0 +1,68 @@ +package com.irrigation.icl.cache; + +import lombok.Data; + +import java.io.Serializable; + +/** + * Command + */ +@Data +public class Command implements Serializable { + + private static final long serialVersionUID = 7126530485423286910L; + + // 设置本地缓存 + public final static byte OPT_SET = 0x01; + // 删除本地缓存Key + public final static byte OPT_DEL = 0x02; + // 删除本地缓存 + public final static byte OPT_REM = 0x03; + + public byte oper; + public String name; + public String key; + public String src; + + public Command() { + } + + public Command(String src, byte oper, String name, String key) { + this.src = src; + this.oper = oper; + this.name = name; + this.key = key; + } + + /** + * 更新本地缓存 + * + * @param cacheName + * @param key + * @return + */ + public static Command set(String src, String cacheName, String key) { + return new Command(src, OPT_SET, cacheName, key); + } + + /** + * 删除本地缓存Key + * + * @param cacheName + * @param key + * @return + */ + public static Command del(String src, String cacheName, String key) { + return new Command(src, OPT_DEL, cacheName, key); + } + + /** + * 删除本地缓存 + * + * @param cacheName + * @return + */ + public static Command rem(String src, String cacheName) { + return new Command(src, OPT_REM, cacheName, null); + } +} \ No newline at end of file diff --git a/src/main/java/com/irrigation/icl/cache/LayeringCacheAutoFactory.java b/src/main/java/com/irrigation/icl/cache/LayeringCacheAutoFactory.java new file mode 100644 index 0000000..aed50d9 --- /dev/null +++ b/src/main/java/com/irrigation/icl/cache/LayeringCacheAutoFactory.java @@ -0,0 +1,77 @@ +package com.irrigation.icl.cache; + +import com.irrigation.icl.cache.config.CacheConfig; +import com.irrigation.icl.cache.core.LayeringCacheLoadCondition; +import com.irrigation.icl.cache.core.LayeringCacheManager; +import com.irrigation.icl.cache.core.LayeringCacheProperties; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cache.CacheManager; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.connection.Message; +import org.springframework.data.redis.connection.MessageListener; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.listener.PatternTopic; +import org.springframework.data.redis.listener.RedisMessageListenerContainer; +import org.springframework.data.redis.listener.adapter.MessageListenerAdapter; + +/** + * @Author: boni + * @Date: 2018/8/24-下午2:37 + */ +@Configuration +@Conditional(LayeringCacheLoadCondition.class) +@EnableConfigurationProperties(LayeringCacheProperties.class) +@Slf4j +public class LayeringCacheAutoFactory { + @Bean + public LayeringCacheManager cacheManager(RedisTemplate redisTemplate, + LayeringCacheProperties layeringCacheProperties) { + LayeringCacheManager cacheManager = new LayeringCacheManager(redisTemplate, layeringCacheProperties); + cacheManager.setUsePrefix(true); + cacheManager.setDefaultExpiration(90); + return cacheManager; + } + + @Bean + RedisMessageListenerContainer container(RedisConnectionFactory connectionFactory, + LayeringCacheProperties layeringCacheProperties, + MessageListenerAdapter listenerAdapter) { + RedisMessageListenerContainer container = new RedisMessageListenerContainer(); + container.setConnectionFactory(connectionFactory); + container.addMessageListener(listenerAdapter, new PatternTopic(layeringCacheProperties.getCacheL1().getTopic())); + + return container; + } + + @Bean + LayeringCacheSyncCommandHandler layeringCacheSyncCommandHandler(CacheManager cacheManager) { + LayeringCacheSyncCommandHandler handler = new LayeringCacheSyncCommandHandler(cacheManager); + return handler; + } + + @Bean + MessageListenerAdapter listenerAdapter(LayeringCacheSyncCommandHandler handler, + RedisTemplate redisTemplate) { + + return new MessageListenerAdapter(new MessageListener() { + @Override + public void onMessage(Message message, byte[] pattern) { + byte[] content = message.getBody(); + if (content != null) { + Command command = (Command) redisTemplate.getValueSerializer().deserialize(content); + if (command != null) { + log.info(command.toString()); + handler.handle(command); + } + } + } + + }); + } +} \ No newline at end of file diff --git a/src/main/java/com/irrigation/icl/cache/LayeringCacheSyncCommandHandler.java b/src/main/java/com/irrigation/icl/cache/LayeringCacheSyncCommandHandler.java new file mode 100644 index 0000000..a786937 --- /dev/null +++ b/src/main/java/com/irrigation/icl/cache/LayeringCacheSyncCommandHandler.java @@ -0,0 +1,30 @@ +package com.irrigation.icl.cache; + +import com.irrigation.icl.cache.core.LayeringCache; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cache.CacheManager; + +/** + * @Author: boni + * @Date: 2018/8/29-下午5:44 + */ +@Slf4j +@Getter +public class LayeringCacheSyncCommandHandler { + CacheManager cacheManager; + + public LayeringCacheSyncCommandHandler(CacheManager cacheManager) { + this.cacheManager = cacheManager; + } + + public void handle(Command command) { + if (cacheManager != null) { + LayeringCache layeringCache = (LayeringCache) cacheManager.getCache(command.getName()); + if (layeringCache == null) { + return; + } + layeringCache.updateL1Cache(command); + } + } +} diff --git a/src/main/java/com/irrigation/icl/cache/config/CacheConfig.java b/src/main/java/com/irrigation/icl/cache/config/CacheConfig.java new file mode 100644 index 0000000..de36ec4 --- /dev/null +++ b/src/main/java/com/irrigation/icl/cache/config/CacheConfig.java @@ -0,0 +1,43 @@ +package com.irrigation.icl.cache.config; + +import com.alibaba.fastjson.support.spring.GenericFastJsonRedisSerializer; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.serializer.StringRedisSerializer; + +/** + * @Author: boni + * @Date: 2018/8/24-下午2:03 + */ +@Configuration +public class CacheConfig { + + @Bean + public RedisTemplate redisTemplate(RedisConnectionFactory connectionFactory) { + RedisTemplate template = new RedisTemplate<>(); + template.setConnectionFactory(connectionFactory); + + //使用Jackson2JsonRedisSerializer来序列化和反序列化redis的value值 +// Jackson2JsonRedisSerializer serializer = new Jackson2JsonRedisSerializer(Object.class); + + GenericFastJsonRedisSerializer serializer = new GenericFastJsonRedisSerializer(); + +// ObjectMapper mapper = new ObjectMapper(); +// mapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY); +// mapper.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL); +// serializer.setObjectMapper(mapper); + + template.setValueSerializer(serializer); + //使用StringRedisSerializer来序列化和反序列化redis的key值 + template.setKeySerializer(new StringRedisSerializer()); + template.afterPropertiesSet(); + return template; + } + + +} diff --git a/src/main/java/com/irrigation/icl/cache/core/LayeringCache.java b/src/main/java/com/irrigation/icl/cache/core/LayeringCache.java new file mode 100644 index 0000000..f9886b6 --- /dev/null +++ b/src/main/java/com/irrigation/icl/cache/core/LayeringCache.java @@ -0,0 +1,221 @@ +package com.irrigation.icl.cache.core; + +import com.irrigation.icl.cache.Command; +import lombok.extern.slf4j.Slf4j; +import net.sf.ehcache.CacheException; +import net.sf.ehcache.CacheManager; +import net.sf.ehcache.Ehcache; +import net.sf.ehcache.Element; +import net.sf.ehcache.config.CacheConfiguration; +import net.sf.ehcache.config.Configuration; +import net.sf.ehcache.config.DiskStoreConfiguration; +import net.sf.ehcache.config.PersistenceConfiguration; +import net.sf.ehcache.store.MemoryStoreEvictionPolicy; +import org.springframework.cache.Cache; +import org.springframework.cache.support.SimpleValueWrapper; +import org.springframework.data.redis.cache.RedisCache; +import org.springframework.data.redis.core.RedisOperations; +import org.springframework.util.StopWatch; + +import java.sql.Time; +import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.util.Date; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; + +/** + * @Author: boni + * @Date: 2018/8/24-下午2:27 + */ +@Slf4j +public class LayeringCache extends RedisCache { + private LayeringCacheProperties layeringCacheProperties; + private LayeringCacheManager layeringCacheManager; + private CacheManager layeringL1CacheManager; + private final ConcurrentHashMap> layeringL1CacheContainer = new ConcurrentHashMap<>(); + + private boolean layerL1Enable; + private String ID; +// private String HOST; +// private String CACHE_STORE; +// private String CACHE_STORE_SYNC; + + + public LayeringCache(LayeringCacheManager layeringCacheManager, String name, byte[] prefix, + RedisOperations redisOperations, long expiration) { + super(name, prefix, redisOperations, expiration); + this.layeringCacheManager = layeringCacheManager; + this.layeringCacheProperties = layeringCacheManager.getLayeringCacheProperties(); + this.layerL1Enable = layeringCacheProperties.getCacheL1().isEnable(); + initEhcache(); + } + + @Override + public Cache.ValueWrapper get(Object key) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + Ehcache ehCache = null; + if (layerL1Enable) { + ehCache = getL1Cache(super.getName()); + Element value = ehCache.get(key); + if (value != null) { + log.info("Hit Cache L1 (ehcache) :{}={}", key, value); + return (value != null ? new SimpleValueWrapper(value.getObjectValue()) : null); + } + } + Cache.ValueWrapper wrapper = super.get(key); + if (layerL1Enable && ehCache != null && wrapper != null) { + ehCache.put(new Element(key, wrapper.get())); + log.info("Hit Cache L2 (redis) :{}={}", key, wrapper); + } + stopWatch.stop(); + log.info("get use {} ms", stopWatch.getTotalTimeMillis()); + return wrapper; + } + + @Override + public void put(final Object key, final Object value) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + // 先更新1级然后再更新2级;允许短时间内的数据不一致 + if (layerL1Enable) { + Ehcache ehCache = getL1Cache(super.getName()); + ehCache.put(new Element(key, value, false, layeringCacheProperties.getCacheL1().getLocalTimeToIdleSeconds(), layeringCacheProperties.getCacheL1().getLocalTimeToLiveSeconds())); + layeringCacheManager.publishMessage(Command.set(this.ID, super.getName(), String.valueOf(key))); + } + super.put(key, value); + stopWatch.stop(); + log.info("put use {} ms", stopWatch.getTotalTimeMillis()); + } + + @Override + public void evict(Object key) { + // 先删除2级缓存，然后再删除1级缓存; + super.evict(key); + if (layerL1Enable) { + Ehcache ehCache = getL1Cache(super.getName()); + ehCache.remove(key); + layeringCacheManager.publishMessage(Command.del(this.ID, super.getName(), String.valueOf(key))); + } + } + + @Override + public Cache.ValueWrapper putIfAbsent(Object key, final Object value) { + if (layerL1Enable) { + Ehcache ehCache = getL1Cache(super.getName()); + ehCache.putIfAbsent(new Element(key, value, false, layeringCacheProperties.getCacheL1().getLocalTimeToIdleSeconds(), layeringCacheProperties.getCacheL1().getLocalTimeToLiveSeconds())); + layeringCacheManager.publishMessage(Command.set(this.ID, super.getName(), String.valueOf(key))); + } + Cache.ValueWrapper wrapper = super.putIfAbsent(key, value); + return wrapper; + } + + public void updateL1Cache(Command command) { + if (layerL1Enable && command != null && !this.ID.equals(command.getSrc())) { + switch (command.getOper()) { + case Command.OPT_DEL: + case Command.OPT_REM: + getL1Cache(command.getName()).remove(command.getKey()); + log.info("Clear Cache L1 {} - {}", command.getName(), command.getKey()); + break; + case Command.OPT_SET: + Cache.ValueWrapper wrapper = super.get(command.getKey()); + if (wrapper != null) { + Ehcache ehCache = getL1Cache(super.getName()); + ehCache.put(new Element(command.getKey(), wrapper.get(), false, layeringCacheProperties.getCacheL1().getLocalTimeToIdleSeconds(), layeringCacheProperties.getCacheL1().getLocalTimeToLiveSeconds())); + log.info("Update Cache L1 {} - {}", command.getKey(), wrapper.get()); + } + break; + default: + break; + } + } + } + + /** + * 创建本地缓存 + */ + private Ehcache getL1Cache(final String name) { + Future future = this.layeringL1CacheContainer.get(name); + if (future == null) { + Callable callable = () -> { + Ehcache cache = layeringL1CacheManager.getEhcache(name); + if (cache == null) { + layeringL1CacheManager.addCache(name); + cache = layeringL1CacheManager.getEhcache(name); + } + return cache; + }; + FutureTask task = new FutureTask<>(callable); + future = this.layeringL1CacheContainer.putIfAbsent(name, task); + if (future == null) { + future = task; + task.run(); + } + } + try { + return future.get(); + } catch (Exception e) { + this.layeringL1CacheContainer.remove(name); + throw new CacheException(e); + } + + } + + private String getDate() { + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMddHHmmssSSS"); + + return simpleDateFormat.format(new Date()); + } + + private void initEhcache() { + if (layerL1Enable) { + this.ID = layeringCacheProperties.getCacheL1().getKey() + "." + getDate(); +// this.HOST = Utils.getLocalHostIP(); +// this.CACHE_STORE = layeringCacheProperties.getCacheL1().getKey() +// + layeringCacheProperties.getCacheL1().getSeparator() +// + "cache" +// + layeringCacheProperties.getCacheL1().getSeparator() +// + "store"; +// this.CACHE_STORE_SYNC = this.CACHE_STORE + layeringCacheProperties.getCacheL1().getSeparator() + "sync"; + + Configuration configuration = new Configuration(); + configuration.setName(this.ID); + configuration.setMaxBytesLocalHeap(layeringCacheProperties.getCacheL1().getLocalMaxBytesLocalHeap()); + configuration.setMaxBytesLocalDisk(layeringCacheProperties.getCacheL1().getLocalMaxBytesLocalDisk()); + // DiskStore + // 每次启动设置新的文件地址,以避免重启期间一级缓存未同步,以及单机多应用启动造成EhcacheManager重复的问题. + DiskStoreConfiguration dsc = new DiskStoreConfiguration(); + dsc.setPath(layeringCacheProperties.getCacheL1().getLocalStoreLocation() + this.ID); + configuration.diskStore(dsc); + + PersistenceConfiguration persistenceConfiguration = new PersistenceConfiguration(); + persistenceConfiguration.setStrategy(PersistenceConfiguration.Strategy.NONE.name()); + + // Cache + CacheConfiguration cacheConfiguration = new CacheConfiguration(); + cacheConfiguration.setEternal(false); + + cacheConfiguration.persistence(persistenceConfiguration); + cacheConfiguration.memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU); + cacheConfiguration.setDiskExpiryThreadIntervalSeconds(layeringCacheProperties.getCacheL1().getLocalDiskExpiryThreadIntervalSeconds()); + // 默认false,使用引用.设置为true,避免外部代码修改了缓存对象.造成EhCache的缓存对象也随之改变 + // 但是设置为true后,将引起element的tti不自动刷新.如果直接新建element去覆盖原值.则本地ttl和远程ttl会产生一定的误差. + // 因此,使用时放弃手动覆盖方式刷新本地tti,当本地tti过期后,自动从Redis中再获取即可. + cacheConfiguration.copyOnRead(true); + cacheConfiguration.copyOnWrite(true); + + cacheConfiguration.setTimeToIdleSeconds(layeringCacheProperties.getCacheL1().getLocalTimeToIdleSeconds()); + cacheConfiguration.setTimeToLiveSeconds(layeringCacheProperties.getCacheL1().getLocalTimeToLiveSeconds()); + + configuration.setDefaultCacheConfiguration(cacheConfiguration); + configuration.setDynamicConfig(false); + configuration.setUpdateCheck(false); + + layeringL1CacheManager = new CacheManager(configuration); + } + } +} diff --git a/src/main/java/com/irrigation/icl/cache/core/LayeringCacheLoadCondition.java b/src/main/java/com/irrigation/icl/cache/core/LayeringCacheLoadCondition.java new file mode 100644 index 0000000..a8cfbe6 --- /dev/null +++ b/src/main/java/com/irrigation/icl/cache/core/LayeringCacheLoadCondition.java @@ -0,0 +1,25 @@ +package com.irrigation.icl.cache.core; + +import org.springframework.boot.bind.RelaxedPropertyResolver; +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.type.AnnotatedTypeMetadata; + +/** + * @Author: boni + * @Date: 2018/8/24-下午2:52 + */ +public class LayeringCacheLoadCondition implements Condition { + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + RelaxedPropertyResolver resolver = new RelaxedPropertyResolver( + context.getEnvironment(), "springext.cache."); + + Boolean enable = resolver.getProperty("enable",Boolean.class); + if(enable == null){ + return false; + } + return enable.booleanValue(); + + } +} diff --git a/src/main/java/com/irrigation/icl/cache/core/LayeringCacheManager.java b/src/main/java/com/irrigation/icl/cache/core/LayeringCacheManager.java new file mode 100644 index 0000000..dd7fedd --- /dev/null +++ b/src/main/java/com/irrigation/icl/cache/core/LayeringCacheManager.java @@ -0,0 +1,48 @@ +package com.irrigation.icl.cache.core; + +import com.irrigation.icl.cache.Command; +import org.springframework.data.redis.cache.RedisCache; +import org.springframework.data.redis.cache.RedisCacheManager; +import org.springframework.data.redis.core.RedisOperations; + +/** + * @Author: boni + * @Date: 2018/8/24-下午2:29 + */ +public class LayeringCacheManager extends RedisCacheManager { + + private LayeringCacheProperties layeringCacheProperties; + + public LayeringCacheProperties getLayeringCacheProperties() { + return layeringCacheProperties; + } + + public void setLayeringCacheProperties(LayeringCacheProperties layeringCacheProperties) { + this.layeringCacheProperties = layeringCacheProperties; + } + + public LayeringCacheManager(RedisOperations redisOperations, LayeringCacheProperties layeringCacheProperties) { + super(redisOperations); + this.layeringCacheProperties = layeringCacheProperties; + } + + @SuppressWarnings("unchecked") + @Override + protected RedisCache createCache(String cacheName) { + long expiration = computeExpiration(cacheName); + return new LayeringCache( + this, + cacheName, + (this.isUsePrefix() ? this.getCachePrefix().prefix(cacheName) : null), + this.getRedisOperations(), + expiration); + } + + + //notify other redis clent to update L1 + public void publishMessage(Command command) { + this.getRedisOperations().convertAndSend(this.getLayeringCacheProperties().getCacheL1().getTopic(), command); + } + + +} diff --git a/src/main/java/com/irrigation/icl/cache/core/LayeringCacheProperties.java b/src/main/java/com/irrigation/icl/cache/core/LayeringCacheProperties.java new file mode 100644 index 0000000..611d8ea --- /dev/null +++ b/src/main/java/com/irrigation/icl/cache/core/LayeringCacheProperties.java @@ -0,0 +1,67 @@ +package com.irrigation.icl.cache.core; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * @Author: boni + * @Date: 2018/8/24-下午2:59 + */ + +@Data +@ConfigurationProperties(prefix = "springext.cache") +public class LayeringCacheProperties { + + /** + * L1缓存配置 + */ + private L1 cacheL1; + private boolean enable = true; + + @Data + // ehcache设置 + public static class L1 { + + /** + * 是否启用L1缓存 + */ + private boolean enable = false; + /** + * 启用L1缓存后同步的topic + */ + private String topic = "default_L1_sync_topic"; + + /** + * cache中使用的key前缀 + */ + private String key = "L1"; + + /** + * key分隔符 + */ + private String separator = ":"; + + /** + * L1缓存存储磁盘位置 + */ + private String localStoreLocation = "./cache/"; + /** + * 本地缓存最大内存大小 + */ + private String localMaxBytesLocalHeap = "128M"; + /** + * 本地缓存最大磁盘大小 + */ + private String localMaxBytesLocalDisk = "1024M"; + + /** + * 本地缓存过期时间，单位秒，默认10分钟 + */ + private int localTimeToLiveSeconds = 1 * 60; + private int localTimeToIdleSeconds = 0; + /** + * 本地缓存清理周期，单位秒，默认3分钟 + */ + private int localDiskExpiryThreadIntervalSeconds = 90; + } +}