1、CityHash简史
Google 2011年发布了 CityHash 系列字符串散列算法 。今天发布的有两种算法：CityHash64 与 CityHash128 。它们分别根据字串计算 64 和 128 位的散列值。这些算法不适用于加密，但适合用在散列表等处。
Google 一直在根据其数据中心常用的 CPU 对算法进行优化，结果发现对大多数个人计算机与笔记本同样有效益。尤其是在 64 位寄存器、指令集级的并行，以及快速非对其内存存取方面。
该算法的开发受到了前人在散列算法方面的巨大启发，尤其是 Austin Appleby 的 MurmurHash 。但 CityHash 的主要优点是大部分步骤包含了至少两步独立的数学运算。现代 CPU 通常能从这种代码获得最佳性能。
但 CityHash 也有其缺点：代码较同类流行算法复杂。Google 希望为速度而不是为了简单而优化，因此没有照顾较短输入的特例。
总体而言，CityHash64 与 CityHash128 是解决经典问题的全新算法。在实际应用中，Google 预计 CityHash64 在速度方面至少能提高30% ，并有望提高多达两倍 。此外，这些算法的统计特性也很完备。
2、CityHash介绍（字符串的哈希函数家族）
CityHash为字符串提供哈希函数。这些功能混合了彻底地输入比特，但不适用于密码学。看见下面的“哈希质量”，了解有关CityHash如何测试等的详细信息。
我们提供了C++中的参考实现，并提供了友好的MIT许可证。
CityHash32（）返回32位哈希。
CityHash64（）和类似的函数返回64位哈希。
CityHash128（）和类似的函数返回128位哈希，并针对至少几百字节的字符串。取决于编译器在足够长的时间内，它可能比CityHash64（）更快串。在较短的字符串上，它比所需的要慢，但我们预计这种情况相对来说并不重要。
CityHashCrc128（）和类似的是CityHash128（）的变体在_mm_crc32_u64（）上，编译为crc32指令的内部函数在一些CPU上。然而，我们提供的功能都不是CRC。
CityHashCrc256（）是CityHashCrc128（）的变体在_mm_crc32_u64（）上。它返回256位哈希。
CityHash家族的所有成员在设计时都非常依赖奥斯汀·阿普比、鲍勃·詹金斯和其他人之前的工作。
例如，CityHash32与Murmur3a有许多相似之处。
长字符串性能：64位CPU
CityHash64（）及其变体在短字符串，但长字符串也很有趣。
CityHash的目标是快速，但它的哈希非常好对于具有CRC32指令的CPU，CRC是快速的，但CRC不是设计为哈希函数，不应作为一个函数使用。CityHashCrc128（）不是CRC，但它使用CRC32机制。
在2.67GHz Intel Xeon X5550的单核上，CityHashCrc256的峰值约为5至5.5字节/周期。其他CityHashCrc函数是包装器CityHashCrc256和应该在长字符串上具有类似的性能。（v1.0.3中的CityHashCrc256甚至更快，但我们认为它不够彻底CityHash128的峰值约为4.3字节/周期。最快的该硬件上的Murmur变体Murmur3F的峰值约为2.4字节/周期。
我们预计CityHash128的峰值速度将主导CityHash64更倾向于短字符串或在哈希表中使用。
对于长字符串，Bob Jenkins的一个新函数SpookyHash只是在Intel x86-64 CPU上比CityHash128稍慢，但明显在AMD x86-64 CPU上速度更快。用于在AMD CPU上哈希长字符串和/或没有CRC指令的CPU，SpookyHash可能与比任何CityHash变体都好或更好。
短字符串性能：64位CPU
对于短字符串，例如大多数哈希表键，CityHash64比CityHash128，这取决于串长度的混合。这里有一些结果同样的硬件，我们（不切实际地）在再次：
哈希结果
CityHash64 v1.0.3 7ns表示1字节，或6ns表示8字节，或9ns表示64字节Murmur2（64位）6ns表示1字节，或6ns表示8字节，或15ns表示64字节1字节为14ns，8字节为15ns，64字节为23ns我们没有1.1版的CityHash64基准测试结果，但我们希望数字相似。
性能：32位CPU
CityHash32是CityHash的最新变体。其目的是通常是32位硬件，但主要是在x86上测试的。我们的基准暗示Murmur3是x86上CityHash32最接近的竞争对手。
我们不知道有什么速度更快的产品质量可以媲美。速度排名在我们的测试中：CityHash32＞Murmur3f＞Murmur3a（用于长字符串），以CityHash32>Murmur3a>Murmur3f（短字符串）。
3、CityHash64 与 CityHash128 的源程序
改编自：
Java版cityHash64 与cityHash128算法的实现
https://blog.csdn.net/aA518189/article/details/107867147参考了：
Clickhouse cityHash64 Java 实现https://blog.csdn.net/lvxueshi/article/details/122505870
using System;
namespace Legalsoft.Truffer.Encryption
{
/// <summary>
/// Google CityHash 64，128位 算法
/// https://blog.csdn.net/lvxueshi/article/details/122505870
/// https://blog.csdn.net/aA518189/article/details/107867147
/// </summary>
public static class CityHash
{
private const long k0 = unchecked((long)0xc3a5c85c97cb3127L);
private const long k1 = unchecked((long)0xb492b66fbe98f273L);
private const long k2 = unchecked((long)0x9ae16a3b2f90404fL);
private const long k3 = unchecked((long)0xc949d7c7509e6557L);
/// <summary>
///
/// </summary>
/// <param name="b"></param>
/// <param name="i"></param>
/// <returns></returns>
private static long toLongLE(byte[] b, int i)
{
return (((long)b[i + 7] << 56) +
((long)(b[i + 6] & 255) << 48) +
((long)(b[i + 5] & 255) << 40) +
((long)(b[i + 4] & 255) << 32) +
((long)(b[i + 3] & 255) << 24) +
((b[i + 2] & 255) << 16) +
((b[i + 1] & 255) << 8) +
((b[i + 0] & 255) << 0));
}
/// <summary>
///
/// </summary>
/// <param name="b"></param>
/// <param name="i"></param>
/// <returns></returns>
private static long toIntLE(byte[] b, int i)
{
return (((b[i + 3] & 255L) << 24) +
((b[i + 2] & 255L) << 16) +
((b[i + 1] & 255L) << 8) +
((b[i + 0] & 255L) << 0));
}
/// <summary>
///
/// </summary>
/// <param name="s"></param>
/// <param name="pos"></param>
/// <returns></returns>
private static long fetch64(byte[] s, int pos)
{
return toLongLE(s, pos);
}
/// <summary>
///
/// </summary>
/// <param name="s"></param>
/// <param name="pos"></param>
/// <returns></returns>
private static long fetch32(byte[] s, int pos)
{
return toIntLE(s, pos);
}
/// <summary>
///
/// </summary>
/// <param name="b"></param>
/// <returns></returns>
private static int staticCastToInt(byte b)
{
return b & 0xFF;
}
/// <summary>
///
/// </summary>
/// <param name="val"></param>
/// <param name="shift"></param>
/// <returns></returns>
private static long rotate(long val, int shift)
{
// Avoid shifting by 64: doing so yields an undefined result.
return shift == 0 ? val : ((long)((ulong)val >> shift)) | (val << (64 - shift));
}
/// <summary>
/// On x86-64, and probably others, it's possible for
/// this to compileEquivalent to Rotate(), but requires the second arg to be non-zero.
/// to a single instruction if both args are already in registers.
/// </summary>
/// <param name="val"></param>
/// <param name="shift"></param>
/// <returns></returns>
private static long rotateByAtLeast1(long val, int shift)
{
return ((long)((ulong)val >> shift)) | (val << (64 - shift));
}
/// <summary>
///
/// </summary>
/// <param name="val"></param>
/// <returns></returns>
private static long shiftMix(long val)
{
//return val ^ (val >>> 47);
return val ^ ((long)((ulong)val >> 47));
}
/// <summary>
///
/// </summary>
private const long kMul = unchecked((long)0x9ddfea08eb382d69L);
/// <summary>
///
/// </summary>
/// <param name="u"></param>
/// <param name="v"></param>
/// <returns></returns>
private static long hash128to64(long u, long v)
{
long a = (u ^ v) * kMul;
a ^= ((long)((ulong)a >> 47));
// long b = (u ^ a) * kMul;
long b = (v ^ a) * kMul;
b ^= ((long)((ulong)b >> 47));
b *= kMul;
return b;
}
/// <summary>
///
/// </summary>
/// <param name="u"></param>
/// <param name="v"></param>
/// <returns></returns>
private static long hashLen16(long u, long v)
{
return hash128to64(u, v);
}
private static long hashLen16(long u, long v, long kmul)
{
long a = (u ^ v) * kmul;
a ^= (a >> 47);
long b = (v ^ a) * kmul;
b ^= (b >> 47);
b *= kmul;
return b;
}
/// <summary>
///
/// </summary>
/// <param name="s"></param>
/// <param name="pos"></param>
/// <param name="len"></param>
/// <returns></returns>
private static long hashLen0to16(byte[] s, int pos, int len)
{
if (len >= 8)
{
/*
long a = fetch64(s, pos);
long b = fetch64(s, pos + len - 8);
return hashLen16(a, rotateByAtLeast1(b + len, len)) ^ b;
*/
long kmul = k2 + len * 2;
long a = fetch64(s, pos) + k2;
long b = fetch64(s, pos + len - 8);
long c = rotate(b, 37) * kmul + a;
long d = (rotate(a, 25) + b) * kmul;
return hashLen16(c, d, kmul);
}
if (len >= 4)
{
/*
long a = fetch32(s, pos);
return hashLen16(len + (a << 3), fetch32(s, pos + len - 4));
*/
long kmul = k2 + len * 2;
long a = fetch32(s, pos + 0);
return hashLen16((a << 3) + len, fetch32(s, pos + len - 4), kmul);
}
if (len > 0)
{
byte a = s[pos];
byte b = s[pos + ((int)((uint)len >> 1))];
byte c = s[pos + len - 1];
int y = staticCastToInt(a) + (staticCastToInt(b) << 8);
int z = len + (staticCastToInt(c) << 2);
return shiftMix(y * k2 ^ z * k3) * k2;
}
return k2;
}
/// <summary>
/// This probably works well for 16-byte strings as well,
/// but it may be overkill in that case.
/// </summary>
/// <param name="s"></param>
/// <param name="pos"></param>
/// <param name="len"></param>
/// <returns></returns>
private static long hashLen17to32(byte[] s, int pos, int len)
{
long a = fetch64(s, pos) * k1;
long b = fetch64(s, pos + 8);
long c = fetch64(s, pos + len - 8) * k2;
long d = fetch64(s, pos + len - 16) * k0;
return hashLen16(rotate(a - b, 43) + rotate(c, 30) + d, a + rotate(b ^ k3, 20) - c + len);
}
public static long reversalByte(long l)
{
/*
ByteBuffer buffer = ByteBuffer.allocate(8);
sbyte[] array = buffer.putLong(0, l).array();
sbyte[] newArr = new sbyte[array.Length];
for (int i = array.Length - 1; i >= 0; i--)
{
newArr[array.Length - i - 1] = array[i];
}
ByteBuffer buffer2 = ByteBuffer.wrap(newArr, 0, 8);
//if(littleEndian){
// ByteBuffer.order(ByteOrder) 方法指定字节序,即大小端模式(BIG_ENDIAN/LITTLE_ENDIAN)
// ByteBuffer 默认为大端(BIG_ENDIAN)模式
//buffer.order(ByteOrder.LITTLE_ENDIAN);
//}
buffer.order(ByteOrder.LITTLE_ENDIAN);
return buffer.getLong();
*/
byte[] buf = BitConverter.GetBytes(l);
byte[] newary = new byte[buf.Length];
for (int i = buf.Length - 1; i >= 0; i--) newary[i] = buf[i];
byte[] buf2 = (byte[])newary.Clone();
return BitConverter.ToInt64(buf2, 0);
}
/*
/// <summary>
///
/// </summary>
/// <param name="s"></param>
/// <param name="pos"></param>
/// <param name="len"></param>
/// <returns></returns>
private static long hashLen33to64(byte[] s, int pos, int len)
{
long z = fetch64(s, pos + 24);
long a = fetch64(s, pos) + (len + fetch64(s, pos + len - 16)) * k0;
long b = rotate(a + z, 52);
long c = rotate(a, 37);
a += fetch64(s, pos + 8);
c += rotate(a, 7);
a += fetch64(s, pos + 16);
long vf = a + z;
long vs = b + rotate(a, 31) + c;
a = fetch64(s, pos + 16) + fetch64(s, len - 32);
z = fetch64(s, len - 8);
b = rotate(a + z, 52);
c = rotate(a, 37);
a += fetch64(s, len - 24);
c += rotate(a, 7);
a += fetch64(s, len - 16);
long wf = a + z;
long ws = b + rotate(a, 31) + c;
long r = shiftMix((vf + ws) * k2 + (wf + vs) * k0);
return shiftMix(r * k0 + vs) * k2;
}
*/
private static long hashLen33to64(byte[] s, int pos, int len)
{
long mul = k2 + len * 2;
long a = fetch64(s, pos) * k2;
long b = fetch64(s, pos + 8);
long c = fetch64(s, pos + len - 24);
long d = fetch64(s, pos + len - 32);
long e = fetch64(s, pos + 16) * k2;
long f = fetch64(s, pos + 24) * 9;
long g = fetch64(s, pos + len - 8);
long h = fetch64(s, pos + len - 16) * mul;
long u = rotate(a + g, 43) + (rotate(b, 30) + c) * 9;
long v = ((a + g) ^ d) + f + 1;
long w = reversalByte((u + v) * mul) + h;
long x = rotate(e + f, 42) + c;
long y = (reversalByte((v + w) * mul) + g) * mul;
long z = e + f + c;
a = reversalByte((x + z) * mul + y) + b;
b = shiftMix((z + a) * mul + d + h) * mul;
return b + x;
}
/// <summary>
///
/// </summary>
/// <param name="s"></param>
/// <param name="pos"></param>
/// <param name="len"></param>
/// <returns></returns>
public static long cityHash64(byte[] s, int pos, int len)
{
if (len <= 32)
{
if (len <= 16)
{
return hashLen0to16(s, pos, len);
}
else
{
return hashLen17to32(s, pos, len);
}
}
else if (len <= 64)
{
return hashLen33to64(s, pos, len);
}
// For strings over 64 bytes we hash the end first, and then as we
// loop we keep 56 bytes of state: v, w, x, y, and z.
long x = fetch64(s, pos);
long y = fetch64(s, pos + len - 16) ^ k1;
long z = fetch64(s, pos + len - 56) ^ k0;
long[] v = weakHashLen32WithSeeds(s, pos + len - 64, len, y);
long[] w = weakHashLen32WithSeeds(s, pos + len - 32, len * k1, k0);
z += shiftMix(v[1]) * k1;
x = rotate(z + x, 39) * k1;
y = rotate(y, 33) * k1;
// Decrease len to the nearest multiple of 64, and operate on 64-byte chunks.
len = (len - 1) & ~staticCastToInt((byte)63);
do
{
x = rotate(x + y + v[0] + fetch64(s, pos + 16), 37) * k1;
y = rotate(y + v[1] + fetch64(s, pos + 48), 42) * k1;
x ^= w[1];
y ^= v[0];
z = rotate(z ^ w[0], 33);
v = weakHashLen32WithSeeds(s, pos, v[1] * k1, x + w[0]);
w = weakHashLen32WithSeeds(s, pos + 32, z + w[1], y);
long tmp = x;
x = z;
z = tmp;
pos += 64;
len -= 64;
} while (len != 0);
return hashLen16(hashLen16(v[0], w[0]) + shiftMix(y) * k1 + z, hashLen16(v[1], w[1]) + x);
}
/// <summary>
///
/// </summary>
/// <param name="w"></param>
/// <param name="x"></param>
/// <param name="y"></param>
/// <param name="z"></param>
/// <param name="a"></param>
/// <param name="b"></param>
/// <returns></returns>
private static long[] weakHashLen32WithSeeds(long w, long x, long y, long z, long a, long b)
{
a += w;
b = rotate(b + a + z, 21);
long c = a;
a += x;
a += y;
b += rotate(a, 44);
return new long[] { a + z, b + c };
}
/// <summary>
/// Return a 16-byte hash for s[0] ... s[31], a, and b.
/// Quick and dirty.
/// </summary>
/// <param name="s"></param>
/// <param name="pos"></param>
/// <param name="a"></param>
/// <param name="b"></param>
/// <returns></returns>
private static long[] weakHashLen32WithSeeds(byte[] s, int pos, long a, long b)
{
return weakHashLen32WithSeeds(fetch64(s, pos + 0),
fetch64(s, pos + 8),
fetch64(s, pos + 16),
fetch64(s, pos + 24),
a, b);
}
private static long[] cityMurmur(byte[] s, int pos, int len, long seed0, long seed1)
{
long a = seed0;
long b = seed1;
long c = 0;
long d = 0;
int l = len - 16;
if (l <= 0)
{
a = shiftMix(a * k1) * k1;
c = b * k1 + hashLen0to16(s, pos, len);
d = shiftMix(a + (len >= 8 ? fetch64(s, pos + 0) : c));
}
else
{
c = hashLen16(fetch64(s, pos + len - 8) + k1, a);
d = hashLen16(b + len, c + fetch64(s, pos + len - 16));
a += d;
do
{
a ^= shiftMix(fetch64(s, pos + 0) * k1) * k1;
a *= k1;
b ^= a;
c ^= shiftMix(fetch64(s, pos + 8) * k1) * k1;
c *= k1;
d ^= c;
pos += 16;
l -= 16;
} while (l > 0);
}
a = hashLen16(a, c);
b = hashLen16(d, b);
return new long[] { a ^ b, hashLen16(b, a) };
}
private static long[] cityHash128WithSeed(byte[] s, int pos, int len, long seed0, long seed1)
{
if (len < 128)
{
return cityMurmur(s, pos, len, seed0, seed1);
}
long[] v = new long[2], w = new long[2];
long x = seed0;
long y = seed1;
long z = k1 * len;
v[0] = rotate(y ^ k1, 49) * k1 + fetch64(s, pos);
v[1] = rotate(v[0], 42) * k1 + fetch64(s, pos + 8);
w[0] = rotate(y + z, 35) * k1 + x;
w[1] = rotate(x + fetch64(s, pos + 88), 53) * k1;
// This is the same inner loop as CityHash64(), manually unrolled.
do
{
x = rotate(x + y + v[0] + fetch64(s, pos + 16), 37) * k1;
y = rotate(y + v[1] + fetch64(s, pos + 48), 42) * k1;
x ^= w[1];
y ^= v[0];
z = rotate(z ^ w[0], 33);
v = weakHashLen32WithSeeds(s, pos, v[1] * k1, x + w[0]);
w = weakHashLen32WithSeeds(s, pos + 32, z + w[1], y);
{
long swap = z;
z = x;
x = swap;
}
pos += 64;
x = rotate(x + y + v[0] + fetch64(s, pos + 16), 37) * k1;
y = rotate(y + v[1] + fetch64(s, pos + 48), 42) * k1;
x ^= w[1];
y ^= v[0];
z = rotate(z ^ w[0], 33);
v = weakHashLen32WithSeeds(s, pos, v[1] * k1, x + w[0]);
w = weakHashLen32WithSeeds(s, pos + 32, z + w[1], y);
{
long swap = z;
z = x;
x = swap;
}
pos += 64;
len -= 128;
} while (len >= 128);
y += rotate(w[0], 37) * k0 + z;
x += rotate(v[0] + z, 49) * k0;
// If 0 < len < 128, hash up to 4 chunks of 32 bytes each from the end of s.
for (int tail_done = 0; tail_done < len;)
{
tail_done += 32;
y = rotate(y - x, 42) * k0 + v[1];
w[0] += fetch64(s, pos + len - tail_done + 16);
x = rotate(x, 49) * k0 + w[0];
w[0] += v[0];
v = weakHashLen32WithSeeds(s, pos + len - tail_done, v[0], v[1]);
}
// At this point our 48 bytes of state should contain more than
// enough information for a strong 128-bit hash. We use two
// different 48-byte-to-8-byte hashes to get a 16-byte final result.
x = hashLen16(x, v[0]);
y = hashLen16(y, w[0]);
return new long[] { hashLen16(x + v[1], w[1]) + y, hashLen16(x + w[1], y + v[1]) };
}
internal static long[] cityHash128(byte[] s, int pos, int len)
{
if (len >= 16)
{
return cityHash128WithSeed(s, pos + 16, len - 16, fetch64(s, pos) ^ k3, fetch64(s, pos + 8));
}
else if (len >= 8)
{
return cityHash128WithSeed(new byte[0], 0, 0, fetch64(s, pos) ^ (len * k0), fetch64(s, pos + len - 8) ^ k1);
}
else
{
return cityHash128WithSeed(s, pos, len, k0, k1);
}
}
}
}
4、调用方法的演示代码
private void button1_Click(object sender, EventArgs e)
{
string str = "马列主义";
byte[] buf = Encoding.Default.GetBytes(str);
long hash64 = CityHash.cityHash64(buf, 0, buf.Length);
long[] hash128 = CityHash.cityHash128(buf, 0, buf.Length);
label16.Text = hash64 + " " + hash128[0] + "." + hash128[1];
}
