You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
336 lines
10 KiB
C
336 lines
10 KiB
C
/* fat-arm64.c
|
|
|
|
Copyright (C) 2021 Mamone Tarsha
|
|
|
|
This file is part of GNU Nettle.
|
|
|
|
GNU Nettle is free software: you can redistribute it and/or
|
|
modify it under the terms of either:
|
|
|
|
* the GNU Lesser General Public License as published by the Free
|
|
Software Foundation; either version 3 of the License, or (at your
|
|
option) any later version.
|
|
|
|
or
|
|
|
|
* the GNU General Public License as published by the Free
|
|
Software Foundation; either version 2 of the License, or (at your
|
|
option) any later version.
|
|
|
|
or both in parallel, as here.
|
|
|
|
GNU Nettle is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
General Public License for more details.
|
|
|
|
You should have received copies of the GNU General Public License and
|
|
the GNU Lesser General Public License along with this program. If
|
|
not, see http://www.gnu.org/licenses/.
|
|
*/
|
|
|
|
#define _GNU_SOURCE
|
|
|
|
#if HAVE_CONFIG_H
|
|
# include "config.h"
|
|
#endif
|
|
|
|
#include <assert.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#if defined(__linux__) && defined(__GLIBC__) && defined(__GLIBC_PREREQ)
|
|
# if __GLIBC_PREREQ(2, 16)
|
|
# define USE_GETAUXVAL 1
|
|
# endif
|
|
#elif __ANDROID_API__ >= 18
|
|
# define USE_GETAUXVAL 1
|
|
#endif
|
|
|
|
#if USE_GETAUXVAL
|
|
# include <asm/hwcap.h>
|
|
# include <sys/auxv.h>
|
|
#elif defined(HAVE_ELF_AUX_INFO)
|
|
# include <sys/auxv.h>
|
|
#elif defined(__OpenBSD__)
|
|
# include <sys/sysctl.h>
|
|
# include <machine/cpu.h>
|
|
# include <machine/armreg.h>
|
|
#elif defined(__APPLE__)
|
|
# include <sys/sysctl.h>
|
|
#endif
|
|
|
|
#include "nettle-types.h"
|
|
|
|
#include "aes.h"
|
|
#include "ghash-internal.h"
|
|
#include "fat-setup.h"
|
|
|
|
/* Defines from arch/arm64/include/uapi/asm/hwcap.h in Linux kernel */
|
|
#ifndef HWCAP_ASIMD
|
|
#define HWCAP_ASIMD (1 << 1)
|
|
#endif
|
|
#ifndef HWCAP_AES
|
|
#define HWCAP_AES (1 << 3)
|
|
#endif
|
|
#ifndef HWCAP_PMULL
|
|
#define HWCAP_PMULL (1 << 4)
|
|
#endif
|
|
#ifndef HWCAP_SHA1
|
|
#define HWCAP_SHA1 (1 << 5)
|
|
#endif
|
|
#ifndef HWCAP_SHA2
|
|
#define HWCAP_SHA2 (1 << 6)
|
|
#endif
|
|
|
|
struct arm64_features
|
|
{
|
|
int have_aes;
|
|
int have_pmull;
|
|
int have_sha1;
|
|
int have_sha2;
|
|
};
|
|
|
|
#define MATCH(s, slen, literal, llen) \
|
|
((slen) == (llen) && memcmp ((s), (literal), llen) == 0)
|
|
|
|
#if defined(__APPLE__)
|
|
static int
|
|
check_sysctlbyname(const char* name)
|
|
{
|
|
int val;
|
|
size_t s = sizeof(val);
|
|
return sysctlbyname(name, &val, &s, NULL, 0) ? 0 : val;
|
|
}
|
|
#endif
|
|
|
|
static void
|
|
get_arm64_features (struct arm64_features *features)
|
|
{
|
|
const char *s;
|
|
features->have_aes = 0;
|
|
features->have_pmull = 0;
|
|
features->have_sha1 = 0;
|
|
features->have_sha2 = 0;
|
|
|
|
s = secure_getenv (ENV_OVERRIDE);
|
|
if (s)
|
|
for (;;)
|
|
{
|
|
const char *sep = strchr (s, ',');
|
|
size_t length = sep ? (size_t) (sep - s) : strlen(s);
|
|
|
|
if (MATCH (s, length, "aes", 3))
|
|
features->have_aes = 1;
|
|
else if (MATCH (s, length, "pmull", 5))
|
|
features->have_pmull = 1;
|
|
else if (MATCH (s, length, "sha1", 4))
|
|
features->have_sha1 = 1;
|
|
else if (MATCH (s, length, "sha2", 4))
|
|
features->have_sha2 = 1;
|
|
if (!sep)
|
|
break;
|
|
s = sep + 1;
|
|
}
|
|
else
|
|
{
|
|
#if USE_GETAUXVAL || defined(HAVE_ELF_AUX_INFO)
|
|
unsigned long hwcap = 0;
|
|
|
|
#if USE_GETAUXVAL
|
|
hwcap = getauxval(AT_HWCAP);
|
|
#elif defined(HAVE_ELF_AUX_INFO)
|
|
elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap));
|
|
#endif
|
|
|
|
features->have_aes
|
|
= ((hwcap & (HWCAP_ASIMD | HWCAP_AES)) == (HWCAP_ASIMD | HWCAP_AES));
|
|
features->have_pmull
|
|
= ((hwcap & (HWCAP_ASIMD | HWCAP_PMULL)) == (HWCAP_ASIMD | HWCAP_PMULL));
|
|
features->have_sha1
|
|
= ((hwcap & (HWCAP_ASIMD | HWCAP_SHA1)) == (HWCAP_ASIMD | HWCAP_SHA1));
|
|
features->have_sha2
|
|
= ((hwcap & (HWCAP_ASIMD | HWCAP_SHA2)) == (HWCAP_ASIMD | HWCAP_SHA2));
|
|
#elif defined(__OpenBSD__)
|
|
const int isar0_mib[] = { CTL_MACHDEP, CPU_ID_AA64ISAR0 };
|
|
uint64_t isar0;
|
|
size_t len = sizeof(isar0);
|
|
|
|
if (sysctl(isar0_mib, 2, &isar0, &len, NULL, 0) < 0)
|
|
return;
|
|
features->have_aes
|
|
= (ID_AA64ISAR0_AES(isar0) >= ID_AA64ISAR0_AES_BASE);
|
|
features->have_pmull
|
|
= (ID_AA64ISAR0_AES(isar0) >= ID_AA64ISAR0_AES_PMULL);
|
|
features->have_sha1
|
|
= (ID_AA64ISAR0_SHA1(isar0) >= ID_AA64ISAR0_SHA1_BASE);
|
|
features->have_sha2
|
|
= (ID_AA64ISAR0_SHA2(isar0) >= ID_AA64ISAR0_SHA2_BASE);
|
|
#elif defined(__APPLE__)
|
|
/* See https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics */
|
|
features->have_aes = check_sysctlbyname("hw.optional.arm.FEAT_AES");
|
|
features->have_pmull = check_sysctlbyname("hw.optional.arm.FEAT_PMULL");
|
|
features->have_sha1 = check_sysctlbyname("hw.optional.arm.FEAT_SHA1");
|
|
features->have_sha2 = check_sysctlbyname("hw.optional.arm.FEAT_SHA256");
|
|
#endif
|
|
}
|
|
}
|
|
|
|
DECLARE_FAT_FUNC(nettle_aes128_encrypt, aes128_crypt_func)
|
|
DECLARE_FAT_FUNC_VAR(aes128_encrypt, aes128_crypt_func, c)
|
|
DECLARE_FAT_FUNC_VAR(aes128_encrypt, aes128_crypt_func, arm64)
|
|
DECLARE_FAT_FUNC(nettle_aes128_decrypt, aes128_crypt_func)
|
|
DECLARE_FAT_FUNC_VAR(aes128_decrypt, aes128_crypt_func, c)
|
|
DECLARE_FAT_FUNC_VAR(aes128_decrypt, aes128_crypt_func, arm64)
|
|
|
|
DECLARE_FAT_FUNC(nettle_aes192_encrypt, aes192_crypt_func)
|
|
DECLARE_FAT_FUNC_VAR(aes192_encrypt, aes192_crypt_func, c)
|
|
DECLARE_FAT_FUNC_VAR(aes192_encrypt, aes192_crypt_func, arm64)
|
|
DECLARE_FAT_FUNC(nettle_aes192_decrypt, aes192_crypt_func)
|
|
DECLARE_FAT_FUNC_VAR(aes192_decrypt, aes192_crypt_func, c)
|
|
DECLARE_FAT_FUNC_VAR(aes192_decrypt, aes192_crypt_func, arm64)
|
|
|
|
DECLARE_FAT_FUNC(nettle_aes256_encrypt, aes256_crypt_func)
|
|
DECLARE_FAT_FUNC_VAR(aes256_encrypt, aes256_crypt_func, c)
|
|
DECLARE_FAT_FUNC_VAR(aes256_encrypt, aes256_crypt_func, arm64)
|
|
DECLARE_FAT_FUNC(nettle_aes256_decrypt, aes256_crypt_func)
|
|
DECLARE_FAT_FUNC_VAR(aes256_decrypt, aes256_crypt_func, c)
|
|
DECLARE_FAT_FUNC_VAR(aes256_decrypt, aes256_crypt_func, arm64)
|
|
|
|
DECLARE_FAT_FUNC(_nettle_ghash_set_key, ghash_set_key_func)
|
|
DECLARE_FAT_FUNC_VAR(ghash_set_key, ghash_set_key_func, c)
|
|
DECLARE_FAT_FUNC_VAR(ghash_set_key, ghash_set_key_func, arm64)
|
|
|
|
DECLARE_FAT_FUNC(_nettle_ghash_update, ghash_update_func)
|
|
DECLARE_FAT_FUNC_VAR(ghash_update, ghash_update_func, c)
|
|
DECLARE_FAT_FUNC_VAR(ghash_update, ghash_update_func, arm64)
|
|
|
|
DECLARE_FAT_FUNC(nettle_sha1_compress, sha1_compress_func)
|
|
DECLARE_FAT_FUNC_VAR(sha1_compress, sha1_compress_func, c)
|
|
DECLARE_FAT_FUNC_VAR(sha1_compress, sha1_compress_func, arm64)
|
|
|
|
DECLARE_FAT_FUNC(_nettle_sha256_compress_n, sha256_compress_n_func)
|
|
DECLARE_FAT_FUNC_VAR(sha256_compress_n, sha256_compress_n_func, c)
|
|
DECLARE_FAT_FUNC_VAR(sha256_compress_n, sha256_compress_n_func, arm64)
|
|
|
|
static void CONSTRUCTOR
|
|
fat_init (void)
|
|
{
|
|
struct arm64_features features;
|
|
int verbose;
|
|
|
|
get_arm64_features (&features);
|
|
|
|
verbose = getenv (ENV_VERBOSE) != NULL;
|
|
if (verbose)
|
|
fprintf (stderr, "libnettle: cpu features:%s%s%s%s\n",
|
|
features.have_aes ? " aes instructions" : "",
|
|
features.have_pmull ? " polynomial multiply long instructions (PMULL/PMULL2)" : "",
|
|
features.have_sha1 ? " sha1 instructions" : "",
|
|
features.have_sha2 ? " sha2 instructions" : "");
|
|
|
|
if (features.have_aes)
|
|
{
|
|
if (verbose)
|
|
fprintf (stderr, "libnettle: enabling hardware accelerated AES encrypt/decrypt code.\n");
|
|
nettle_aes128_encrypt_vec = _nettle_aes128_encrypt_arm64;
|
|
nettle_aes128_decrypt_vec = _nettle_aes128_decrypt_arm64;
|
|
nettle_aes192_encrypt_vec = _nettle_aes192_encrypt_arm64;
|
|
nettle_aes192_decrypt_vec = _nettle_aes192_decrypt_arm64;
|
|
nettle_aes256_encrypt_vec = _nettle_aes256_encrypt_arm64;
|
|
nettle_aes256_decrypt_vec = _nettle_aes256_decrypt_arm64;
|
|
}
|
|
else
|
|
{
|
|
nettle_aes128_encrypt_vec = _nettle_aes128_encrypt_c;
|
|
nettle_aes128_decrypt_vec = _nettle_aes128_decrypt_c;
|
|
nettle_aes192_encrypt_vec = _nettle_aes192_encrypt_c;
|
|
nettle_aes192_decrypt_vec = _nettle_aes192_decrypt_c;
|
|
nettle_aes256_encrypt_vec = _nettle_aes256_encrypt_c;
|
|
nettle_aes256_decrypt_vec = _nettle_aes256_decrypt_c;
|
|
}
|
|
|
|
if (features.have_pmull)
|
|
{
|
|
if (verbose)
|
|
fprintf (stderr, "libnettle: enabling hardware-accelerated polynomial multiply code.\n");
|
|
|
|
/* Make sure _nettle_ghash_set_key_vec function is compatible
|
|
with _nettle_ghash_update_vec function e.g. _nettle_ghash_key_c()
|
|
fills gcm_key table with values that are incompatible with
|
|
_nettle_ghash_update_arm64() */
|
|
_nettle_ghash_set_key_vec = _nettle_ghash_set_key_arm64;
|
|
_nettle_ghash_update_vec = _nettle_ghash_update_arm64;
|
|
}
|
|
else
|
|
{
|
|
_nettle_ghash_set_key_vec = _nettle_ghash_set_key_c;
|
|
_nettle_ghash_update_vec = _nettle_ghash_update_c;
|
|
}
|
|
if (features.have_sha1)
|
|
{
|
|
if (verbose)
|
|
fprintf (stderr, "libnettle: enabling hardware-accelerated sha1 compress code.\n");
|
|
nettle_sha1_compress_vec = _nettle_sha1_compress_arm64;
|
|
}
|
|
else
|
|
{
|
|
nettle_sha1_compress_vec = _nettle_sha1_compress_c;
|
|
}
|
|
if (features.have_sha2)
|
|
{
|
|
if (verbose)
|
|
fprintf (stderr, "libnettle: enabling hardware-accelerated sha256 compress code.\n");
|
|
_nettle_sha256_compress_n_vec = _nettle_sha256_compress_n_arm64;
|
|
}
|
|
else
|
|
{
|
|
_nettle_sha256_compress_n_vec = _nettle_sha256_compress_n_c;
|
|
}
|
|
}
|
|
|
|
DEFINE_FAT_FUNC(nettle_aes128_encrypt, void,
|
|
(const struct aes128_ctx *ctx, size_t length,
|
|
uint8_t *dst,const uint8_t *src),
|
|
(ctx, length, dst, src))
|
|
DEFINE_FAT_FUNC(nettle_aes128_decrypt, void,
|
|
(const struct aes128_ctx *ctx, size_t length,
|
|
uint8_t *dst,const uint8_t *src),
|
|
(ctx, length, dst, src))
|
|
|
|
DEFINE_FAT_FUNC(nettle_aes192_encrypt, void,
|
|
(const struct aes192_ctx *ctx, size_t length,
|
|
uint8_t *dst,const uint8_t *src),
|
|
(ctx, length, dst, src))
|
|
DEFINE_FAT_FUNC(nettle_aes192_decrypt, void,
|
|
(const struct aes192_ctx *ctx, size_t length,
|
|
uint8_t *dst,const uint8_t *src),
|
|
(ctx, length, dst, src))
|
|
|
|
DEFINE_FAT_FUNC(nettle_aes256_encrypt, void,
|
|
(const struct aes256_ctx *ctx, size_t length,
|
|
uint8_t *dst,const uint8_t *src),
|
|
(ctx, length, dst, src))
|
|
DEFINE_FAT_FUNC(nettle_aes256_decrypt, void,
|
|
(const struct aes256_ctx *ctx, size_t length,
|
|
uint8_t *dst,const uint8_t *src),
|
|
(ctx, length, dst, src))
|
|
|
|
DEFINE_FAT_FUNC(_nettle_ghash_set_key, void,
|
|
(struct gcm_key *ctx, const union nettle_block16 *key),
|
|
(ctx, key))
|
|
DEFINE_FAT_FUNC(_nettle_ghash_update, const uint8_t *,
|
|
(const struct gcm_key *ctx, union nettle_block16 *state,
|
|
size_t blocks, const uint8_t *data),
|
|
(ctx, state, blocks, data))
|
|
|
|
DEFINE_FAT_FUNC(nettle_sha1_compress, void,
|
|
(uint32_t *state, const uint8_t *input),
|
|
(state, input))
|
|
|
|
DEFINE_FAT_FUNC(_nettle_sha256_compress_n, const uint8_t *,
|
|
(uint32_t *state, const uint32_t *k,
|
|
size_t blocks, const uint8_t *input),
|
|
(state, k, blocks, input))
|