【发布时间】：2018-04-12 00:01:31
【问题描述】：
我们有大量数据存储在 Redis 中。实际上，我们有大量 keys 存储在 Redis 中，以及与每个 key 相关联的 微小 数据量。密钥为 8 个字节长，数据为 8 个字节长（一个长值）。有 10 亿 个键（是的，十亿）。
鉴于 Redis 存储的结构，据我所知（https://redislabs.com/blog/redis-ram-ramifications-part-i/ 和 https://github.com/antirez/sds/blob/master/README.md）给定 8 个字节的密钥，标头的开销为 8 个字节，结尾处的 null 为 1 个字节钥匙。那是 17 个字节。假设这四舍五入到至少 24 个字节，加上 8 个字节的 long 值得到 32 个字节。
十亿个密钥将是 32GB。实测使用量为 158GB。当然会有开销，但 5:1 的比例似乎很大。任何人都可以解释这一点或指出减少内存使用的方法。
我已经包含了我基于 Jedis 的测试程序。
import java.security.SecureRandom;
import java.text.DecimalFormat;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.exceptions.JedisClusterMaxRedirectionsException;
public class Test8byteKeys {
protected static JedisCluster cluster = null;
protected static final ExecutorService executor;
protected static volatile boolean shuttingDown = false;
private static final int AVAILABLE_PROCESSORS = Runtime.getRuntime().availableProcessors();
static {
final int cores = Math.max(4, (AVAILABLE_PROCESSORS * 3) / 4);
executor = new ThreadPoolExecutor(cores, cores, //
15, TimeUnit.SECONDS, //
new LinkedBlockingQueue<>(cores), //
new ThreadPoolExecutor.CallerRunsPolicy());
System.out.println("Running with " + cores + " threads");
}
static private GenericObjectPoolConfig getPoolConfiguration() {
GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig();
poolConfig.setLifo(true);
poolConfig.setTestOnBorrow(true);
poolConfig.setTestOnReturn(false);
poolConfig.setBlockWhenExhausted(true);
poolConfig.setMinIdle(1);
poolConfig.setMaxTotal(101);
poolConfig.setTestWhileIdle(false);
poolConfig.setSoftMinEvictableIdleTimeMillis(3000L);
poolConfig.setNumTestsPerEvictionRun(5);
poolConfig.setTimeBetweenEvictionRunsMillis(5000L);
poolConfig.setJmxEnabled(true);
return poolConfig;
}
private static void connectToCluster() {
try {
Set<HostAndPort> nodes = new HashSet<>();
String hap /* host and port */ = System.getProperty("hap", null);
if (hap == null) {
System.err.println("You must supply the host and port of a master in the cluster on the command line");
System.err.println("java -Dhap=<host:port> -jar <jar> ");
System.exit(1);
}
String[] parts = hap.split(":"); // assume ipv4 address
nodes.add(new HostAndPort(parts[0].trim(), Integer.valueOf(parts[1].trim())));
System.out.println("Connecting to " + hap);
cluster = new JedisCluster(nodes, getPoolConfiguration());
}
catch (Exception e) {
System.err.println("Could not connect to redis -- " + e.getMessage());
System.exit(1);
}
}
private static final Thread shutdown = new Thread(new Runnable() {
// Clean up at exit
@Override
public void run() {
shuttingDown = true;
System.out.println((new Date()).toString() + "\t" + "Executor shutdown in progress");
try {
executor.shutdown();
executor.awaitTermination(10L, TimeUnit.SECONDS);
}
catch (Exception e) {
// ignore
}
finally {
try {
if (!executor.isShutdown()) {
executor.shutdownNow();
}
}
catch (Exception e) {
//ignore
}
}
try {
cluster.close();
}
catch (Exception e) {
System.err.println("cluster disconnection failure: " + e);
}
finally {
//
}
System.out.println((new Date()).toString() + "\t" + "shutdown complete.");
}
});
final static char[] CHARACTERS = { //
'0', '1', '2', '3', '4', '5', //
'6', '7', '8', '9', 'a', 'b', //
'c', 'd', 'e', 'f', 'g', 'h', //
'i', 'j', 'k', 'l', 'm', 'n', //
'o', 'p', 'q', 'r', 's', 't', //
'u', 'v', 'w', 'x', 'y', 'z', //
'A', 'B', 'C', 'D', 'E', 'F', //
'G', 'H', 'I', 'J', 'K', 'L', //
'M', 'N', 'O', 'P', 'Q', 'R', //
'S', 'T', 'U', 'V', 'W', 'X', //
'Y', 'Z', '#', '@' //
};
protected final static byte[] KEY_EXISTS_MARKER = { '1' };
static class Runner implements Runnable {
private byte[] key = null;
public Runner(byte[] key) {
this.key = key;
}
@Override
public void run() {
if (!shuttingDown) {
try {
cluster.set(key, KEY_EXISTS_MARKER);
cluster.expire(key, 60 * 60 * 48); // for test purposes, only keep around for 2 days
}
catch (JedisClusterMaxRedirectionsException e) {
System.err.println(
(new Date()).toString() + "\tIGNORING\t" + e + "\t" + "For key " + new String(key));
}
catch (Exception e) {
System.err.println((new Date()).toString() + "\t" + e + "\t" + "For key " + new String(key));
e.printStackTrace();
System.exit(1);
}
}
}
}
public static void main(String[] args) {
SecureRandom random = new SecureRandom();
DecimalFormat decimal = new DecimalFormat("#,##0");
final byte[] randomBytes = new byte[8];
connectToCluster();
Runtime.getRuntime().addShutdownHook(shutdown);
System.out.println((new Date()) + " Starting test");
for (int i = 0; i < 1000000000; i++) {
random.nextBytes(randomBytes);
final byte[] key = new byte[8];
for (int j = 0; j < randomBytes.length; j++)
key[j] = (byte) (CHARACTERS[((randomBytes[j] & 0xFF)) % CHARACTERS.length] & 0xFF);
try {
if (shuttingDown) {
System.err.println((new Date()).toString() + "\t" + "Main loop terminating due to shutdown");
break;
}
if (i % 1000000 == 0)
System.out.println((new Date()).toString() + "\t" + decimal.format(i));
try {
executor.submit(new Runner(key));
}
catch (Exception e) {
System.err.println((new Date()).toString() + "\t" + e);
}
}
catch (Exception e) {
System.err.println("Failed to set key " + new String(key) + " -- " + e);
}
}
if (!shuttingDown) {
System.out.println((new Date()) + " Done");
System.exit(0);
}
}
}
【问题讨论】：