devbox
爱喝兽奶帝天荒
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

Spring Boot如何压缩Json并写入redis？

作者：爱喝兽奶帝天荒 | 2024-08-28 06:09:17

赞

踩

Spring Boot如何压缩Json并写入redis？

1.为什么需要压缩json？

由于业务需要，存入redis中的缓存数据过大，占用了10+G的内存，内存作为重要资源，需要优化一下大对象缓存，采用gzip压缩存储，可以将 redis 的 kv 对大小缩小大约 7-8 倍，加快存储、读取速度

2.环境搭建

详建redis模块的docker目录

  1. version: '3'
  2. services:
  3.   redis:
  4.     image: registry.cn-hangzhou.aliyuncs.com/zhengqing/redis:6.0.8                   
  5.     container_name: redis                                                             
  6.     restart: unless-stopped                                                                  
  7.     command: redis-server /etc/redis/redis.conf --requirepass 123456 --appendonly no
  8. #    command: redis-server --requirepass 123456 --appendonly yes 
  9.     environment:                        
  10.       TZ: Asia/Shanghai
  11.       LANG: en_US.UTF-8
  12.     volumes:                           
  13.       - "./redis/data:/data"
  14.       - "./redis/config/redis.conf:/etc/redis/redis.conf"  
  15.     ports:                              
  16.       - "6379:6379"

3.代码工程

 实验目标

实验存入redis的json数据压缩和解压缩

pom.xml

  1. <?xml version="1.0" encoding="UTF-8"?>
  2. <project xmlns="http://maven.apache.org/POM/4.0.0"
  3.          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  4.          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  5.     <parent>
  6.         <artifactId>springboot-demo</artifactId>
  7.         <groupId>com.et</groupId>
  8.         <version>1.0-SNAPSHOT</version>
  9.     </parent>
  10.     <modelVersion>4.0.0</modelVersion>
  11.  
  12.     <artifactId>gzip</artifactId>
  13.  
  14.     <properties>
  15.         <maven.compiler.source>8</maven.compiler.source>
  16.         <maven.compiler.target>8</maven.compiler.target>
  17.     </properties>
  18.     <dependencies>
  19.         <dependency>
  20.             <groupId>org.springframework.boot</groupId>
  21.             <artifactId>spring-boot-starter-web</artifactId>
  22.         </dependency>
  23.  
  24.         <dependency>
  25.             <groupId>org.springframework.boot</groupId>
  26.             <artifactId>spring-boot-autoconfigure</artifactId>
  27.         </dependency>
  28.         <dependency>
  29.             <groupId>org.springframework.boot</groupId>
  30.             <artifactId>spring-boot-starter-test</artifactId>
  31.             <scope>test</scope>
  32.         </dependency>
  33.         <dependency>
  34.             <groupId>org.projectlombok</groupId>
  35.             <artifactId>lombok</artifactId>
  36.         </dependency>
  37.         <dependency>
  38.             <groupId>org.springframework.boot</groupId>
  39.             <artifactId>spring-boot-starter-data-redis</artifactId>
  40.         </dependency>
  41.         <dependency>
  42.             <groupId>org.apache.commons</groupId>
  43.             <artifactId>commons-pool2</artifactId>
  44.             <version>2.9.0</version>
  45.         </dependency>
  46.  
  47.     </dependencies>
  48. </project>

controller

  1. package com.et.gzip.controller;
  2.  
  3. import com.et.gzip.model.User;
  4. import lombok.extern.slf4j.Slf4j;
  5. import org.springframework.beans.factory.annotation.Autowired;
  6. import org.springframework.data.redis.core.RedisTemplate;
  7. import org.springframework.web.bind.annotation.PostMapping;
  8. import org.springframework.web.bind.annotation.RequestBody;
  9. import org.springframework.web.bind.annotation.RequestMapping;
  10. import org.springframework.web.bind.annotation.RestController;
  11.  
  12. import java.util.HashMap;
  13. import java.util.Map;
  14.  
  15. @RestController
  16. @Slf4j
  17. public class HelloWorldController {
  18.     @Autowired
  19.     private RedisTemplate redisTemplateWithJackson;
  20.  
  21.     @PostMapping("/hello")
  22.     public User showHelloWorld(@RequestBody User user){
  23.         log.info("user:"+ user);
  24.  
  25.         return user;
  26.     }
  27.     @PostMapping("/redis")
  28.     public User redis(@RequestBody User user){
  29.         log.info("user:"+ user);
  30.         redisTemplateWithJackson.opsForValue().set("user",user);
  31.         User redisUser = (User) redisTemplateWithJackson.opsForValue().get("user");
  32.         return redisUser;
  33.     }
  34. }

redis压缩和解压缩配置

压缩类

  1. package com.et.gzip.config;
  2.  
  3. import com.et.gzip.model.User;
  4.  
  5. import com.fasterxml.jackson.databind.ObjectMapper;
  6. import com.fasterxml.jackson.databind.annotation.JsonSerialize;
  7. import lombok.extern.slf4j.Slf4j;
  8. import org.apache.tomcat.util.http.fileupload.IOUtils;
  9.  
  10. import org.springframework.data.redis.serializer.JdkSerializationRedisSerializer;
  11. import org.springframework.data.redis.serializer.SerializationException;
  12. import sun.misc.BASE64Encoder;
  13.  
  14. import java.io.ByteArrayInputStream;
  15. import java.io.ByteArrayOutputStream;
  16. import java.text.SimpleDateFormat;
  17. import java.util.zip.GZIPInputStream;
  18. import java.util.zip.GZIPOutputStream;
  19.  
  20. @Slf4j
  21. public class CompressRedis extends JdkSerializationRedisSerializer {
  22.  
  23.     public static final int BUFFER_SIZE = 4096;
  24.  
  25.     private JacksonRedisSerializer<User>  jacksonRedisSerializer;
  26.     public CompressRedis() {
  27.         this.jacksonRedisSerializer = getValueSerializer();
  28.     }
  29.  
  30.     @Override
  31.     public byte[] serialize(Object graph) throws SerializationException {
  32.         if (graph == null) {
  33.             return new byte[0];
  34.         }
  35.         ByteArrayOutputStream bos = null;
  36.         GZIPOutputStream gzip = null;
  37.         try {
  38.             // serialize
  39.             byte[] bytes = jacksonRedisSerializer.serialize(graph);
  40.             log.info("bytes size{}",bytes.length);
  41.             bos = new ByteArrayOutputStream();
  42.             gzip = new GZIPOutputStream(bos);
  43.  
  44.             // compress
  45.             gzip.write(bytes);
  46.             gzip.finish();
  47.             byte[] result = bos.toByteArray();
  48.  
  49.             log.info("result size{}",result.length);
  50.             //return result;
  51.             return new BASE64Encoder().encode(result).getBytes();
  52.         } catch (Exception e) {
  53.             throw new SerializationException("Gzip Serialization Error", e);
  54.         } finally {
  55.             IOUtils.closeQuietly(bos);
  56.             IOUtils.closeQuietly(gzip);
  57.         }
  58.     }
  59.  
  60.     @Override
  61.     public Object deserialize(byte[] bytes) throws SerializationException {
  62.         if (bytes == null || bytes.length == 0) {
  63.             return null;
  64.         }
  65.         ByteArrayOutputStream bos = null;
  66.         ByteArrayInputStream bis = null;
  67.         GZIPInputStream gzip = null;
  68.         try {
  69.             bos = new ByteArrayOutputStream();
  70.             byte[] compressed = new sun.misc.BASE64Decoder().decodeBuffer( new String(bytes));;
  71.             bis = new ByteArrayInputStream(compressed);
  72.             gzip = new GZIPInputStream(bis);
  73.             byte[] buff = new byte[BUFFER_SIZE];
  74.             int n;
  75.  
  76.  
  77.             // uncompress
  78.             while ((n = gzip.read(buff, 0, BUFFER_SIZE)) > 0) {
  79.                 bos.write(buff, 0, n);
  80.             }
  81.             //deserialize
  82.             Object result = jacksonRedisSerializer.deserialize(bos.toByteArray());
  83.             return result;
  84.         } catch (Exception e) {
  85.             throw new SerializationException("Gzip deserizelie error", e);
  86.         } finally {
  87.             IOUtils.closeQuietly(bos);
  88.             IOUtils.closeQuietly(bis);
  89.             IOUtils.closeQuietly(gzip);
  90.         }
  91.     }
  92.  
  93.     private static JacksonRedisSerializer<User> getValueSerializer() {
  94.         JacksonRedisSerializer<User> jackson2JsonRedisSerializer = new JacksonRedisSerializer<>(User.class);
  95.         ObjectMapper mapper=new ObjectMapper();
  96.         jackson2JsonRedisSerializer.setObjectMapper(mapper);
  97.         return jackson2JsonRedisSerializer;
  98.     }
  99.  
  100. }

java序列化

  1.  
  2. package com.et.gzip.config;
  3.  
  4.  
  5. import com.fasterxml.jackson.databind.JavaType;
  6. import com.fasterxml.jackson.databind.ObjectMapper;
  7. import com.fasterxml.jackson.databind.type.TypeFactory;
  8. import lombok.extern.slf4j.Slf4j;
  9.  
  10. import org.springframework.data.redis.serializer.RedisSerializer;
  11. import org.springframework.data.redis.serializer.SerializationException;
  12. import org.springframework.lang.Nullable;
  13. import org.springframework.util.Assert;
  14.  
  15. import java.nio.charset.Charset;
  16. import java.nio.charset.StandardCharsets;
  17.  
  18. @Slf4j
  19. public class JacksonRedisSerializer<T> implements RedisSerializer<T> {
  20.     public static final Charset DEFAULT_CHARSET;
  21.     private final JavaType javaType;
  22.     private ObjectMapper objectMapper = new ObjectMapper();
  23.  
  24.     public JacksonRedisSerializer(Class<T> type) {
  25.         this.javaType = this.getJavaType(type);
  26.     }
  27.  
  28.     public JacksonRedisSerializer(JavaType javaType) {
  29.         this.javaType = javaType;
  30.     }
  31.  
  32.     public T deserialize(@Nullable byte[] bytes) throws SerializationException {
  33.         if (bytes == null || bytes.length == 0) {
  34.             return null;
  35.         } else {
  36.             try {
  37.                 return this.objectMapper.readValue(bytes, 0, bytes.length, this.javaType);
  38.  
  39.             } catch (Exception var3) {
  40.                 throw new SerializationException("Could not read JSON: " + var3.getMessage(), var3);
  41.             }
  42.         }
  43.     }
  44.  
  45.     public byte[] serialize(@Nullable Object t) throws SerializationException {
  46.         if (t == null) {
  47.             return  new byte[0];
  48.         } else {
  49.             try {
  50.                 return this.objectMapper.writeValueAsBytes(t);
  51.             } catch (Exception var3) {
  52.                 throw new SerializationException("Could not write JSON: " + var3.getMessage(), var3);
  53.             }
  54.         }
  55.     }
  56.  
  57.     public void setObjectMapper(ObjectMapper objectMapper) {
  58.         Assert.notNull(objectMapper, "'objectMapper' must not be null");
  59.         this.objectMapper = objectMapper;
  60.     }
  61.  
  62.     protected JavaType getJavaType(Class<?> clazz) {
  63.         return TypeFactory.defaultInstance().constructType(clazz);
  64.     }
  65.  
  66.     static {
  67.         DEFAULT_CHARSET = StandardCharsets.UTF_8;
  68.     }
  69. }

redis序列化

  1. package com.et.gzip.config;
  2.  
  3. import com.et.gzip.model.User;
  4. import org.springframework.context.annotation.Bean;
  5. import org.springframework.context.annotation.Configuration;
  6. import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
  7. import org.springframework.data.redis.core.RedisTemplate;
  8. import org.springframework.data.redis.serializer.RedisSerializer;
  9. import org.springframework.data.redis.serializer.StringRedisSerializer;
  10.  
  11.  
  12. @Configuration
  13. public class RedisWithJacksonConfig {
  14.  
  15.  
  16.     @Bean(name="redisTemplateWithJackson")
  17.     public RedisTemplate<String, User> redisTemplate(LettuceConnectionFactory lettuceConnectionFactory) {
  18.  
  19.         CompressRedis  compressRedis =  new CompressRedis();
  20.         //redisTemplate
  21.         RedisTemplate<String, User> redisTemplate = new RedisTemplate<>();
  22.         redisTemplate.setConnectionFactory(lettuceConnectionFactory);
  23.         RedisSerializer<?> stringSerializer = new StringRedisSerializer();
  24.         redisTemplate.setKeySerializer(stringSerializer);
  25.         redisTemplate.setValueSerializer(compressRedis);
  26.         redisTemplate.setHashKeySerializer(stringSerializer);
  27.         redisTemplate.setHashValueSerializer(compressRedis);
  28.         redisTemplate.afterPropertiesSet();
  29.         return redisTemplate;
  30.     }
  31. }

application.yaml

  1. spring:
  2.   redis:
  3.     host: 127.0.0.1
  4.     port: 6379
  5.     database: 10
  6.     password: 123456
  7.     timeout: 10s
  8.     lettuce:
  9.       pool:
  10.         min-idle: 0
  11.         max-idle: 8
  12.         max-active: 8
  13.         max-wait: -1ms
  14. server:
  15.   port: 8088
  16.   compression:
  17.     enabled: true
  18.     mime-types: application/json,application/xml,text/html,text/plain,text/css,application/x-javascript

以上只是一些关键代码，所有代码请参见下面代码仓库

代码仓库

4.测试

  1. 启动spring boot应用
  2. 用postman访问http://127.0.0.1:8088/redis

3

可以看到redis里面存储的是gzip压缩的内容

4

查看控制台日志

  1. 2024-08-26 14:37:56.445 INFO 43832 --- [nio-8088-exec-5] com.et.gzip.config.CompressRedis : bytes size371
  2. 2024-08-26 14:37:56.445 INFO 43832 --- [nio-8088-exec-5] com.et.gzip.config.CompressRedis : result size58

JSON经过gzip压缩，371-->58, 数据大小减少7-8倍

5.引用

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/爱喝兽奶帝天荒/article/detail/1021488
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号