crypto/aes: dedicated asm version of AES-GCM

The existing implementation didn't use the CLMUL instructions for fast
and constant time binary-field multiplication. With this change, amd64
CPUs that support both AES and CLMUL instructions will use an optimised
asm implementation.

benchmark                 old ns/op     new ns/op     delta
BenchmarkAESGCMSeal8K     91723         3200          -96.51%
BenchmarkAESGCMOpen8K     91487         3324          -96.37%
BenchmarkAESGCMSeal1K     11873         546           -95.40%
BenchmarkAESGCMOpen1K     11833         594           -94.98%

benchmark                 old MB/s     new MB/s     speedup
BenchmarkAESGCMSeal8K     89.31        2559.62      28.66x
BenchmarkAESGCMOpen8K     89.54        2463.78      27.52x
BenchmarkAESGCMSeal1K     86.24        1872.49      21.71x
BenchmarkAESGCMOpen1K     86.53        1721.78      19.90x

Change-Id: Idd63233098356d8b353d16624747b74d0c3f193e
Reviewed-on: https://go-review.googlesource.com/10484
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Adam Langley <agl@golang.org>

3 files changed, 90 insertions, 9 deletions

diff --git a/src/crypto/cipher/benchmark_test.go b/src/crypto/cipher/benchmark_test.go
index 027b248510..93c40d0f46 100644
--- a/src/crypto/cipher/benchmark_test.go
+++ b/src/crypto/cipher/benchmark_test.go
@@ -10,42 +10,58 @@ import (
 	"testing"
 )
 
-func BenchmarkAESGCMSeal1K(b *testing.B) {
-	buf := make([]byte, 1024)
+func benchmarkAESGCMSeal(b *testing.B, buf []byte) {
 	b.SetBytes(int64(len(buf)))
 
 	var key [16]byte
 	var nonce [12]byte
+	var ad [13]byte
 	aes, _ := aes.NewCipher(key[:])
 	aesgcm, _ := cipher.NewGCM(aes)
 	var out []byte
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		out = aesgcm.Seal(out[:0], nonce[:], buf, nonce[:])
+		out = aesgcm.Seal(out[:0], nonce[:], buf, ad[:])
 	}
 }
 
-func BenchmarkAESGCMOpen1K(b *testing.B) {
-	buf := make([]byte, 1024)
+func benchmarkAESGCMOpen(b *testing.B, buf []byte) {
 	b.SetBytes(int64(len(buf)))
 
 	var key [16]byte
 	var nonce [12]byte
+	var ad [13]byte
 	aes, _ := aes.NewCipher(key[:])
 	aesgcm, _ := cipher.NewGCM(aes)
 	var out []byte
-	out = aesgcm.Seal(out[:0], nonce[:], buf, nonce[:])
+	out = aesgcm.Seal(out[:0], nonce[:], buf, ad[:])
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		_, err := aesgcm.Open(buf[:0], nonce[:], out, nonce[:])
+		_, err := aesgcm.Open(buf[:0], nonce[:], out, ad[:])
 		if err != nil {
 			b.Errorf("Open: %v", err)
 		}
 	}
 }
 
+func BenchmarkAESGCMSeal1K(b *testing.B) {
+	benchmarkAESGCMSeal(b, make([]byte, 1024))
+}
+
+func BenchmarkAESGCMOpen1K(b *testing.B) {
+	benchmarkAESGCMOpen(b, make([]byte, 1024))
+}
+
+func BenchmarkAESGCMSeal8K(b *testing.B) {
+	benchmarkAESGCMSeal(b, make([]byte, 8*1024))
+}
+
+func BenchmarkAESGCMOpen8K(b *testing.B) {
+	benchmarkAESGCMOpen(b, make([]byte, 8*1024))
+}
+
 // If we test exactly 1K blocks, we would generate exact multiples of
 // the cipher's block size, and the cipher stream fragments would
 // always be wordsize aligned, whereas non-aligned is a more typical
diff --git a/src/crypto/cipher/gcm.go b/src/crypto/cipher/gcm.go
index bbdf9f5d3d..3a0474f502 100644
--- a/src/crypto/cipher/gcm.go
+++ b/src/crypto/cipher/gcm.go
@@ -38,6 +38,13 @@ type AEAD interface {
 	Open(dst, nonce, ciphertext, data []byte) ([]byte, error)
 }
 
+// gcmAble is an interface implemented by ciphers that have a specific optimized
+// implementation of GCM, like crypto/aes. NewGCM will check for this interface
+// and return the specific AEAD if found.
+type gcmAble interface {
+	NewGCM(int) (AEAD, error)
+}
+
 // gcmFieldElement represents a value in GF(2¹²⁸). In order to reflect the GCM
 // standard and make getUint64 suitable for marshaling these values, the bits
 // are stored backwards. For example:
@@ -72,6 +79,10 @@ func NewGCM(cipher Block) (AEAD, error) {
 // cryptosystem that uses non-standard nonce lengths. All other users should use
 // NewGCM, which is faster and more resistant to misuse.
 func NewGCMWithNonceSize(cipher Block, size int) (AEAD, error) {
+	if cipher, ok := cipher.(gcmAble); ok {
+		return cipher.NewGCM(size)
+	}
+
 	if cipher.BlockSize() != gcmBlockSize {
 		return nil, errors.New("cipher: NewGCM requires 128-bit block cipher")
 	}
diff --git a/src/crypto/cipher/gcm_test.go b/src/crypto/cipher/gcm_test.go
index 81b9aa2419..904091ed5d 100644
--- a/src/crypto/cipher/gcm_test.go
+++ b/src/crypto/cipher/gcm_test.go
@@ -12,8 +12,6 @@ import (
 	"testing"
 )
 
-// AES-GCM test vectors taken from gcmEncryptExtIV128.rsp from
-// http://csrc.nist.gov/groups/STM/cavp/index.html.
 var aesGCMTests = []struct {
 	key, nonce, plaintext, ad, result string
 }{
@@ -32,6 +30,27 @@ var aesGCMTests = []struct {
 		"60d20404af527d248d893ae495707d1a",
 	},
 	{
+		"fbe3467cc254f81be8e78d765a2e6333",
+		"c6697351ff4aec29cdbaabf2",
+		"",
+		"67",
+		"3659cdc25288bf499ac736c03bfc1159",
+	},
+	{
+		"8a7f9d80d08ad0bd5a20fb689c88f9fc",
+		"88b7b27d800937fda4f47301",
+		"",
+		"50edd0503e0d7b8c91608eb5a1",
+		"ed6f65322a4740011f91d2aae22dd44e",
+	},
+	{
+		"051758e95ed4abb2cdc69bb454110e82",
+		"c99a66320db73158a35a255d",
+		"",
+		"67c6697351ff4aec29cdbaabf2fbe3467cc254f81be8e78d765a2e63339f",
+		"6ce77f1a5616c505b6aec09420234036",
+	},
+	{
 		"77be63708971c4e240d1cb79e8d77feb",
 		"e0e00f19fed7ba0136a797f3",
 		"",
@@ -130,6 +149,41 @@ var aesGCMTests = []struct {
 		"8d8c7ffc55086d539b5a8f0d1232654c",
 		"0d803ec309482f35b8e6226f2b56303239298e06b281c2d51aaba3c125",
 	},
+	{
+		"0e18a844ac5bf38e4cd72d9b0942e506",
+		"0870d4b28a2954489a0abcd5",
+		"67c6697351ff4aec29cdbaabf2fbe3467cc254f81be8e78d765a2e63339fc99a66320db73158a35a255d051758e95ed4abb2cdc69bb454110e827441213ddc8770e93ea141e1fc673e017e97eadc6b968f385c2aecb03bfb32af3c54ec18db5c021afe43fbfaaa3afb29d1e6053c7c9475d8be6189f95cbba8990f95b1ebf1b3",
+		"05eff700e9a13ae5ca0bcbd0484764bd1f231ea81c7b64c514735ac55e4b79633b706424119e09dcaad4acf21b10af3b33cde3504847155cbb6f2219ba9b7df50be11a1c7f23f829f8a41b13b5ca4ee8983238e0794d3d34bc5f4e77facb6c05ac86212baa1a55a2be70b5733b045cd33694b3afe2f0e49e4f321549fd824ea9",
+		"cace28f4976afd72e3c5128167eb788fbf6634dda0a2f53148d00f6fa557f5e9e8f736c12e450894af56cb67f7d99e1027258c8571bd91ee3b7360e0d508aa1f382411a16115f9c05251cc326d4016f62e0eb8151c048465b0c6c8ff12558d43310e18b2cb1889eec91557ce21ba05955cf4c1d4847aadfb1b0a83f3a3b82b7efa62a5f03c5d6eda381a85dd78dbc55c",
+	},
+	{
+		"1f6c3a3bc0542aabba4ef8f6c7169e73",
+		"f3584606472b260e0dd2ebb2",
+		"67c6697351ff4aec29cdbaabf2fbe3467cc254f81be8e78d765a2e63339fc99a66320db73158a35a255d051758e95ed4abb2cdc69bb454110e827441213ddc8770e93ea141e1fc673e017e97eadc6b968f385c2aecb03bfb32af3c54ec18db5c021afe43fbfaaa3afb29d1e6053c7c9475d8be6189f95cbba8990f95b1ebf1b305eff700e9a13ae5ca0bcbd0484764bd1f231ea81c7b64c514735ac55e4b79633b706424119e09dcaad4acf21b10af3b33cde3504847155cbb6f2219ba9b7df50be11a1c7f23f829f8a41b13b5ca4ee8983238e0794d3d34bc5f4e77facb6c05ac86212baa1a55a2be70b5733b045cd33694b3afe2f0e49e4f321549fd824ea90870d4b28a2954489a0abcd50e18a844ac5bf38e4cd72d9b0942e506c433afcda3847f2dadd47647de321cec4ac430f62023856cfbb20704f4ec0bb920ba86c33e05f1ecd96733b79950a3e314d3d934f75ea0f210a8f6059401beb4bc4478fa4969e623d01ada696a7e4c7e5125b34884533a94fb319990325744ee9bbce9e525cf08f5e9e25e5360aad2b2d085fa54d835e8d466826498d9a8877565705a8a3f62802944de7ca5894e5759d351adac869580ec17e485f18c0c66f17cc07cbb22fce466da610b63af62bc83b4692f3affaf271693ac071fb86d11342d8def4f89d4b66335c1c7e4248367d8ed9612ec453902d8e50af89d7709d1a596c1f41f",
+		"95aa82ca6c49ae90cd1668baac7aa6f2b4a8ca99b2c2372acb08cf61c9c3805e6e0328da4cd76a19edd2d3994c798b0022569ad418d1fee4d9cd45a391c601ffc92ad91501432fee150287617c13629e69fc7281cd7165a63eab49cf714bce3a75a74f76ea7e64ff81eb61fdfec39b67bf0de98c7e4e32bdf97c8c6ac75ba43c02f4b2ed7216ecf3014df000108b67cf99505b179f8ed4980a6103d1bca70dbe9bbfab0ed59801d6e5f2d6f67d3ec5168e212e2daf02c6b963c98a1f7097de0c56891a2b211b01070dd8fd8b16c2a1a4e3cfd292d2984b3561d555d16c33ddc2bcf7edde13efe520c7e2abdda44d81881c531aeeeb66244c3b791ea8acfb6a68",
+		"55864065117e07650ca650a0f0d9ef4b02aee7c58928462fddb49045bf85355b4653fa26158210a7f3ef5b3ca48612e8b7adf5c025c1b821960af770d935df1c9a1dd25077d6b1c7f937b2e20ce981b07980880214698f3fad72fa370b3b7da257ce1d0cf352bc5304fada3e0f8927bd4e5c1abbffa563bdedcb567daa64faaed748cb361732200ba3506836a3c1c82aafa14c76dc07f6c4277ff2c61325f91fdbd6c1883e745fcaadd5a6d692eeaa5ad56eead6a9d74a595d22757ed89532a4b8831e2b9e2315baea70a9b95d228f09d491a5ed5ab7076766703457e3159bbb9b17b329525669863153079448c68cd2f200c0be9d43061a60639cb59d50993d276c05caaa565db8ce633b2673e4012bebbca02b1a64d779d04066f3e949ece173825885ec816468c819a8129007cc05d8785c48077d09eb1abcba14508dde85a6f16a744bc95faef24888d53a8020515ab20307efaecbdf143a26563c67989bceedc2d6d2bb9699bb6c615d93767e4158c1124e3b6c723aaa47796e59a60d3696cd85adfae9a62f2c02c22009f80ed494bdc587f31dd892c253b5c6d6b7db078fa72d23474ee54f8144d6561182d71c862941dbc0b2cb37a4d4b23cbad5637e6be901cc73f16d5aec39c60dddee631511e57b47520b61ae1892d2d1bd2b486e30faec892f171b6de98d96108016fac805604761f8e74742b3bb7dc8a290a46bf697c3e4446e6e65832cbae7cf1aaad1",
+	},
+	{
+		"0795d80bc7f40f4d41c280271a2e4f7f",
+		"ff824c906594aff365d3cb1f",
+		"1ad4e74d127f935beee57cff920665babe7ce56227377afe570ba786193ded3412d4812453157f42fafc418c02a746c1232c234a639d49baa8f041c12e2ef540027764568ce49886e0d913e28059a3a485c6eee96337a30b28e4cd5612c2961539fa6bc5de034cbedc5fa15db844013e0bef276e27ca7a4faf47a5c1093bd643354108144454d221b3737e6cb87faac36ed131959babe44af2890cfcc4e23ffa24470e689ce0894f5407bb0c8665cff536008ad2ac6f1c9ef8289abd0bd9b72f21c597bda5210cf928c805af2dd4a464d52e36819d521f967bba5386930ab5b4cf4c71746d7e6e964673457348e9d71d170d9eb560bd4bdb779e610ba816bf776231ebd0af5966f5cdab6815944032ab4dd060ad8dab880549e910f1ffcf6862005432afad",
+		"98a47a430d8fd74dc1829a91e3481f8ed024d8ba34c9b903321b04864db333e558ae28653dffb2",
+		"3b8f91443480e647473a0a0b03d571c622b7e70e4309a02c9bb7980053010d865e6aec161354dc9f481b2cd5213e09432b57ec4e58fbd0a8549dd15c8c4e74a6529f75fad0ce5a9e20e2beeb2f91eb638bf88999968de438d2f1cedbfb0a1c81f9e8e7362c738e0fddd963692a4f4df9276b7f040979ce874cf6fa3de26da0713784bdb25e4efcb840554ef5b38b5fe8380549a496bd8e423a7456df6f4ae78a07ebe2276a8e22fc2243ec4f78abe0c99c733fd67c8c492699fa5ee2289cdd0a8d469bf883520ee74efb854bfadc7366a49ee65ca4e894e3335e2b672618d362eee12a577dd8dc2ba55c49c1fc3ad68180e9b112d0234d4aa28f5661f1e036450ca6f18be0166676bd80f8a4890c6ddea306fabb7ff3cb2860aa32a827e3a312912a2dfa70f6bc1c07de238448f2d751bd0cf15bf7",
+	},
+	{
+		"e2e001a36c60d2bf40d69ff5b2b1161ea218db263be16a4e",
+		"84230643130d05425826641e",
+		"adb034f3f4a7ca45e2993812d113a9821d50df151af978bccc6d3bc113e15bc0918fb385377dca1916022ce816d56a332649484043c0fc0f2d37d040182b00a9bbb42ef231f80b48fb3730110d9a4433e38c73264c703579a705b9c031b969ec6d98de9f90e9e78b21179c2eb1e061946cd4bbb844f031ecf6eaac27a4151311adf1b03eda97c9fbae66295f468af4b35faf6ba39f9d8f95873bbc2b51cf3dfec0ed3c9b850696336cc093b24a8765a936d14dd56edc6bf518272169f75e67b74ba452d0aae90416a997c8f31e2e9d54ffea296dc69462debc8347b3e1af6a2d53bdfdfda601134f98db42b609df0a08c9347590c8d86e845bb6373d65a26ab85f67b50569c85401a396b8ad76c2b53ff62bcfbf033e435ef47b9b591d05117c6dc681d68e",
+		"d5d7316b8fdee152942148bff007c22e4b2022c6bc7be3c18c5f2e52e004e0b5dc12206bf002bd",
+		"f2c39423ee630dfe961da81909159dba018ce09b1073a12a477108316af5b7a31f86be6a0548b572d604bd115ea737dde899e0bd7f7ac9b23e38910dc457551ecc15c814a9f46d8432a1a36097dc1afe2712d1ba0838fa88cb55d9f65a2e9bece0dbf8999562503989041a2c87d7eb80ef649769d2f4978ce5cf9664f2bd0849646aa81cb976e45e1ade2f17a8126219e917aadbb4bae5e2c4b3f57bbc7f13fcc807df7842d9727a1b389e0b749e5191482adacabd812627c6eae2c7a30caf0844ad2a22e08f39edddf0ae10413e47db433dfe3febbb5a5cec9ade21fbba1e548247579395880b747669a8eb7e2ec0c1bff7fed2defdb92b07a14edf07b1bde29c31ab052ff1214e6b5ebbefcb8f21b5d6f8f6e07ee57ad6e14d4e142cb3f51bb465ab3a28a2a12f01b7514ad0463f2bde0d71d221",
+	},
+	{
+		"5394e890d37ba55ec9d5f327f15680f6a63ef5279c79331643ad0af6d2623525",
+		"815e840b7aca7af3b324583f",
+		"8e63067cd15359f796b43c68f093f55fdf3589fc5f2fdfad5f9d156668a617f7091d73da71cdd207810e6f71a165d0809a597df9885ca6e8f9bb4e616166586b83cc45f49917fc1a256b8bc7d05c476ab5c4633e20092619c4747b26dad3915e9fd65238ee4e5213badeda8a3a22f5efe6582d0762532026c89b4ca26fdd000eb45347a2a199b55b7790e6b1b2dba19833ce9f9522c0bcea5b088ccae68dd99ae0203c81b9f1dd3181c3e2339e83ccd1526b67742b235e872bea5111772aab574ae7d904d9b6355a79178e179b5ae8edc54f61f172bf789ea9c9af21f45b783e4251421b077776808f04972a5e801723cf781442378ce0e0568f014aea7a882dcbcb48d342be53d1c2ebfb206b12443a8a587cc1e55ca23beca385d61d0d03e9d84cbc1b0a",
+		"0feccdfae8ed65fa31a0858a1c466f79e8aa658c2f3ba93c3f92158b4e30955e1c62580450beff",
+		"b69a7e17bb5af688883274550a4ded0d1aff49a0b18343f4b382f745c163f7f714c9206a32a1ff012427e19431951edd0a755e5f491b0eedfd7df68bbc6085dd2888607a2f998c3e881eb1694109250db28291e71f4ad344a125624fb92e16ea9815047cd1111cabfdc9cb8c3b4b0f40aa91d31774009781231400789ed545404af6c3f76d07ddc984a7bd8f52728159782832e298cc4d529be96d17be898efd83e44dc7b0e2efc645849fd2bba61fef0ae7be0dcab233cc4e2b7ba4e887de9c64b97f2a1818aa54371a8d629dae37975f7784e5e3cc77055ed6e975b1e5f55e6bbacdc9f295ce4ada2c16113cd5b323cf78b7dde39f4a87aa8c141a31174e3584ccbd380cf5ec6d1dba539928b084fa9683e9c0953acf47cc3ac384a2c38914f1da01fb2cfd78905c2b58d36b2574b9df15535d82",
+	},
 }
 
 func TestAESGCM(t *testing.T) {