Move SHA2 routines to a new generic API layer for crypto hashes

Two new routines to allocate a hash context and to free it are created,
as these become necessary for the goal behind this refactoring: switch
the all cryptohash implementations for OpenSSL to use EVP (for FIPS and
also because upstream does not recommend the use of low-level cryptohash
functions for 20 years).  Note that OpenSSL hides the internals of
cryptohash contexts since 1.1.0, so it is necessary to leave the
allocation to OpenSSL itself, explaining the need for those two new
routines.  This part is going to require more work to properly track
hash contexts with resource owners, but this not introduced here.
Still, this refactoring makes the move possible.

This reduces the number of routines for all SHA2 implementations from
twelve (SHA{224,256,386,512} with init, update and final calls) to five
(create, free, init, update and final calls) by incorporating the hash
type directly into the hash context data.

The new cryptohash routines are moved to a new file, called cryptohash.c
for the fallback implementations, with SHA2 specifics becoming a part
internal to src/common/.  OpenSSL specifics are part of
cryptohash_openssl.c.  This infrastructure is usable for more hash
types, like MD5 or HMAC.

Any code paths using the internal SHA2 routines are adapted to report
correctly errors, which are most of the changes of this commit.  The
zones mostly impacted are checksum manifests, libpq and SCRAM.

Note that e21cbb4 was a first attempt to switch SHA2 to EVP, but it
lacked the refactoring needed for libpq, as done here.

This patch has been tested on Linux and Windows, with and without
OpenSSL, and down to 1.0.1, the oldest version supported on HEAD.

Author: Michael Paquier
Reviewed-by: Daniel Gustafsson
Discussion: https://postgr.es/m/20200924025314.GE7405@paquier.xyz

4 files changed, 59 insertions, 100 deletions

diff --git a/src/include/common/checksum_helper.h b/src/include/common/checksum_helper.h
index 48b0745dadd..b07a34e7e49 100644
--- a/src/include/common/checksum_helper.h
+++ b/src/include/common/checksum_helper.h
@@ -14,6 +14,7 @@
 #ifndef CHECKSUM_HELPER_H
 #define CHECKSUM_HELPER_H
 
+#include "common/cryptohash.h"
 #include "common/sha2.h"
 #include "port/pg_crc32c.h"
 
@@ -41,10 +42,10 @@ typedef enum pg_checksum_type
 typedef union pg_checksum_raw_context
 {
 	pg_crc32c	c_crc32c;
-	pg_sha224_ctx c_sha224;
-	pg_sha256_ctx c_sha256;
-	pg_sha384_ctx c_sha384;
-	pg_sha512_ctx c_sha512;
+	pg_cryptohash_ctx *c_sha224;
+	pg_cryptohash_ctx *c_sha256;
+	pg_cryptohash_ctx *c_sha384;
+	pg_cryptohash_ctx *c_sha512;
 } pg_checksum_raw_context;
 
 /*
@@ -66,8 +67,8 @@ typedef struct pg_checksum_context
 extern bool pg_checksum_parse_type(char *name, pg_checksum_type *);
 extern char *pg_checksum_type_name(pg_checksum_type);
 
-extern void pg_checksum_init(pg_checksum_context *, pg_checksum_type);
-extern void pg_checksum_update(pg_checksum_context *, const uint8 *input,
+extern int	pg_checksum_init(pg_checksum_context *, pg_checksum_type);
+extern int	pg_checksum_update(pg_checksum_context *, const uint8 *input,
 							   size_t len);
 extern int	pg_checksum_final(pg_checksum_context *, uint8 *output);
 
diff --git a/src/include/common/cryptohash.h b/src/include/common/cryptohash.h
new file mode 100644
index 00000000000..0e4a6631a38
--- /dev/null
+++ b/src/include/common/cryptohash.h
@@ -0,0 +1,40 @@
+/*-------------------------------------------------------------------------
+ *
+ * cryptohash.h
+ *	  Generic headers for cryptographic hash functions.
+ *
+ * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ *		  src/include/common/cryptohash.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef PG_CRYPTOHASH_H
+#define PG_CRYPTOHASH_H
+
+/* Context Structures for each hash function */
+typedef enum
+{
+	PG_SHA224 = 0,
+	PG_SHA256,
+	PG_SHA384,
+	PG_SHA512
+} pg_cryptohash_type;
+
+typedef struct pg_cryptohash_ctx
+{
+	pg_cryptohash_type type;
+	/* private area used by each hash implementation */
+	void	   *data;
+} pg_cryptohash_ctx;
+
+extern pg_cryptohash_ctx *pg_cryptohash_create(pg_cryptohash_type type);
+extern int	pg_cryptohash_init(pg_cryptohash_ctx *ctx);
+extern int	pg_cryptohash_update(pg_cryptohash_ctx *ctx, const uint8 *data, size_t len);
+extern int	pg_cryptohash_final(pg_cryptohash_ctx *ctx, uint8 *dest);
+extern void pg_cryptohash_free(pg_cryptohash_ctx *ctx);
+
+#endif							/* PG_CRYPTOHASH_H */
diff --git a/src/include/common/scram-common.h b/src/include/common/scram-common.h
index 2edae2dd3c0..f4a7c60725b 100644
--- a/src/include/common/scram-common.h
+++ b/src/include/common/scram-common.h
@@ -13,6 +13,7 @@
 #ifndef SCRAM_COMMON_H
 #define SCRAM_COMMON_H
 
+#include "common/cryptohash.h"
 #include "common/sha2.h"
 
 /* Name of SCRAM mechanisms per IANA */
@@ -50,19 +51,19 @@
  */
 typedef struct
 {
-	pg_sha256_ctx sha256ctx;
+	pg_cryptohash_ctx *sha256ctx;
 	uint8		k_opad[SHA256_HMAC_B];
 } scram_HMAC_ctx;
 
-extern void scram_HMAC_init(scram_HMAC_ctx *ctx, const uint8 *key, int keylen);
-extern void scram_HMAC_update(scram_HMAC_ctx *ctx, const char *str, int slen);
-extern void scram_HMAC_final(uint8 *result, scram_HMAC_ctx *ctx);
+extern int	scram_HMAC_init(scram_HMAC_ctx *ctx, const uint8 *key, int keylen);
+extern int	scram_HMAC_update(scram_HMAC_ctx *ctx, const char *str, int slen);
+extern int	scram_HMAC_final(uint8 *result, scram_HMAC_ctx *ctx);
 
-extern void scram_SaltedPassword(const char *password, const char *salt,
+extern int	scram_SaltedPassword(const char *password, const char *salt,
 								 int saltlen, int iterations, uint8 *result);
-extern void scram_H(const uint8 *str, int len, uint8 *result);
-extern void scram_ClientKey(const uint8 *salted_password, uint8 *result);
-extern void scram_ServerKey(const uint8 *salted_password, uint8 *result);
+extern int	scram_H(const uint8 *str, int len, uint8 *result);
+extern int	scram_ClientKey(const uint8 *salted_password, uint8 *result);
+extern int	scram_ServerKey(const uint8 *salted_password, uint8 *result);
 
 extern char *scram_build_secret(const char *salt, int saltlen, int iterations,
 								const char *password);
diff --git a/src/include/common/sha2.h b/src/include/common/sha2.h
index 9c4abf777d4..c8b9096043f 100644
--- a/src/include/common/sha2.h
+++ b/src/include/common/sha2.h
@@ -1,9 +1,10 @@
 /*-------------------------------------------------------------------------
  *
  * sha2.h
- *	  Generic headers for SHA224, 256, 384 AND 512 functions of PostgreSQL.
+ *	  Constants related to SHA224, 256, 384 AND 512.
  *
- * Portions Copyright (c) 2016-2020, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
  *		  src/include/common/sha2.h
@@ -11,49 +12,9 @@
  *-------------------------------------------------------------------------
  */
 
-/* $OpenBSD: sha2.h,v 1.2 2004/04/28 23:11:57 millert Exp $ */
-
-/*
- * FILE:	sha2.h
- * AUTHOR:	Aaron D. Gifford <me@aarongifford.com>
- *
- * Copyright (c) 2000-2001, Aaron D. Gifford
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *	  notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *	  notice, this list of conditions and the following disclaimer in the
- *	  documentation and/or other materials provided with the distribution.
- * 3. Neither the name of the copyright holder nor the names of contributors
- *	  may be used to endorse or promote products derived from this software
- *	  without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTOR(S) ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTOR(S) BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $From: sha2.h,v 1.1 2001/11/08 00:02:01 adg Exp adg $
- */
-
 #ifndef _PG_SHA2_H_
 #define _PG_SHA2_H_
 
-#ifdef USE_OPENSSL
-#include <openssl/sha.h>
-#endif
-
 /*** SHA224/256/384/512 Various Length Definitions ***********************/
 #define PG_SHA224_BLOCK_LENGTH			64
 #define PG_SHA224_DIGEST_LENGTH			28
@@ -68,48 +29,4 @@
 #define PG_SHA512_DIGEST_LENGTH			64
 #define PG_SHA512_DIGEST_STRING_LENGTH	(PG_SHA512_DIGEST_LENGTH * 2 + 1)
 
-/* Context Structures for SHA224/256/384/512 */
-#ifdef USE_OPENSSL
-typedef SHA256_CTX pg_sha256_ctx;
-typedef SHA512_CTX pg_sha512_ctx;
-typedef SHA256_CTX pg_sha224_ctx;
-typedef SHA512_CTX pg_sha384_ctx;
-#else
-typedef struct pg_sha256_ctx
-{
-	uint32		state[8];
-	uint64		bitcount;
-	uint8		buffer[PG_SHA256_BLOCK_LENGTH];
-} pg_sha256_ctx;
-typedef struct pg_sha512_ctx
-{
-	uint64		state[8];
-	uint64		bitcount[2];
-	uint8		buffer[PG_SHA512_BLOCK_LENGTH];
-} pg_sha512_ctx;
-typedef struct pg_sha256_ctx pg_sha224_ctx;
-typedef struct pg_sha512_ctx pg_sha384_ctx;
-#endif							/* USE_OPENSSL */
-
-/* Interface routines for SHA224/256/384/512 */
-extern void pg_sha224_init(pg_sha224_ctx *ctx);
-extern void pg_sha224_update(pg_sha224_ctx *ctx, const uint8 *input0,
-							 size_t len);
-extern void pg_sha224_final(pg_sha224_ctx *ctx, uint8 *dest);
-
-extern void pg_sha256_init(pg_sha256_ctx *ctx);
-extern void pg_sha256_update(pg_sha256_ctx *ctx, const uint8 *input0,
-							 size_t len);
-extern void pg_sha256_final(pg_sha256_ctx *ctx, uint8 *dest);
-
-extern void pg_sha384_init(pg_sha384_ctx *ctx);
-extern void pg_sha384_update(pg_sha384_ctx *ctx,
-							 const uint8 *, size_t len);
-extern void pg_sha384_final(pg_sha384_ctx *ctx, uint8 *dest);
-
-extern void pg_sha512_init(pg_sha512_ctx *ctx);
-extern void pg_sha512_update(pg_sha512_ctx *ctx, const uint8 *input0,
-							 size_t len);
-extern void pg_sha512_final(pg_sha512_ctx *ctx, uint8 *dest);
-
 #endif							/* _PG_SHA2_H_ */