rsa: add support of padding pss

We add the support of the padding pss for rsa signature.
This new padding is often recommended instead of pkcs-1.5.

Signed-off-by: Philippe Reynes <philippe.reynes@softathome.com>
Reviewed-by: Simon Glass <sjg@chromium.org>

2 files changed, 200 insertions, 0 deletions

diff --git a/lib/rsa/rsa-sign.c b/lib/rsa/rsa-sign.c
index 6aa0e2ab5d..fb5e07b56d 100644
--- a/lib/rsa/rsa-sign.c
+++ b/lib/rsa/rsa-sign.c
@@ -438,6 +438,16 @@ static int rsa_sign_with_key(RSA *rsa, struct padding_algo *padding_algo,
 		goto err_sign;
 	}
 
+#ifdef CONFIG_FIT_ENABLE_RSASSA_PSS_SUPPORT
+	if (padding_algo && !strcmp(padding_algo->name, "pss")) {
+		if (EVP_PKEY_CTX_set_rsa_padding(ckey,
+						 RSA_PKCS1_PSS_PADDING) <= 0) {
+			ret = rsa_err("Signer padding setup failed");
+			goto err_sign;
+		}
+	}
+#endif /* CONFIG_FIT_ENABLE_RSASSA_PSS_SUPPORT */
+
 	for (i = 0; i < region_count; i++) {
 		if (!EVP_DigestSignUpdate(context, region[i].data,
 					  region[i].size)) {
diff --git a/lib/rsa/rsa-verify.c b/lib/rsa/rsa-verify.c
index 279a9ba456..9734f6d3bd 100644
--- a/lib/rsa/rsa-verify.c
+++ b/lib/rsa/rsa-verify.c
@@ -80,6 +80,196 @@ int padding_pkcs_15_verify(struct image_sign_info *info,
 	return 0;
 }
 
+#ifdef CONFIG_FIT_ENABLE_RSASSA_PSS_SUPPORT
+static void u32_i2osp(uint32_t val, uint8_t *buf)
+{
+	buf[0] = (uint8_t)((val >> 24) & 0xff);
+	buf[1] = (uint8_t)((val >> 16) & 0xff);
+	buf[2] = (uint8_t)((val >>  8) & 0xff);
+	buf[3] = (uint8_t)((val >>  0) & 0xff);
+}
+
+/**
+ * mask_generation_function1() - generate an octet string
+ *
+ * Generate an octet string used to check rsa signature.
+ * It use an input octet string and a hash function.
+ *
+ * @checksum:	A Hash function
+ * @seed:	Specifies an input variable octet string
+ * @seed_len:	Size of the input octet string
+ * @output:	Specifies the output octet string
+ * @output_len:	Size of the output octet string
+ * @return 0 if the octet string was correctly generated, others on error
+ */
+static int mask_generation_function1(struct checksum_algo *checksum,
+				     uint8_t *seed, int seed_len,
+				     uint8_t *output, int output_len)
+{
+	struct image_region region[2];
+	int ret = 0, i, i_output = 0, region_count = 2;
+	uint32_t counter = 0;
+	uint8_t buf_counter[4], *tmp;
+	int hash_len = checksum->checksum_len;
+
+	memset(output, 0, output_len);
+
+	region[0].data = seed;
+	region[0].size = seed_len;
+	region[1].data = &buf_counter[0];
+	region[1].size = 4;
+
+	tmp = malloc(hash_len);
+	if (!tmp) {
+		debug("%s: can't allocate array tmp\n", __func__);
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	while (i_output < output_len) {
+		u32_i2osp(counter, &buf_counter[0]);
+
+		ret = checksum->calculate(checksum->name,
+					  region, region_count,
+					  tmp);
+		if (ret < 0) {
+			debug("%s: Error in checksum calculation\n", __func__);
+			goto out;
+		}
+
+		i = 0;
+		while ((i_output < output_len) && (i < hash_len)) {
+			output[i_output] = tmp[i];
+			i_output++;
+			i++;
+		}
+
+		counter++;
+	}
+
+out:
+	free(tmp);
+
+	return ret;
+}
+
+static int compute_hash_prime(struct checksum_algo *checksum,
+			      uint8_t *pad, int pad_len,
+			      uint8_t *hash, int hash_len,
+			      uint8_t *salt, int salt_len,
+			      uint8_t *hprime)
+{
+	struct image_region region[3];
+	int ret, region_count = 3;
+
+	region[0].data = pad;
+	region[0].size = pad_len;
+	region[1].data = hash;
+	region[1].size = hash_len;
+	region[2].data = salt;
+	region[2].size = salt_len;
+
+	ret = checksum->calculate(checksum->name, region, region_count, hprime);
+	if (ret < 0) {
+		debug("%s: Error in checksum calculation\n", __func__);
+		goto out;
+	}
+
+out:
+	return ret;
+}
+
+int padding_pss_verify(struct image_sign_info *info,
+		       uint8_t *msg, int msg_len,
+		       const uint8_t *hash, int hash_len)
+{
+	uint8_t *masked_db = NULL;
+	int masked_db_len = msg_len - hash_len - 1;
+	uint8_t *h = NULL, *hprime = NULL;
+	int h_len = hash_len;
+	uint8_t *db_mask = NULL;
+	int db_mask_len = masked_db_len;
+	uint8_t *db = NULL, *salt = NULL;
+	int db_len = masked_db_len, salt_len = msg_len - hash_len - 2;
+	uint8_t pad_zero[8] = { 0 };
+	int ret, i, leftmost_bits = 1;
+	uint8_t leftmost_mask;
+	struct checksum_algo *checksum = info->checksum;
+
+	/* first, allocate everything */
+	masked_db = malloc(masked_db_len);
+	h = malloc(h_len);
+	db_mask = malloc(db_mask_len);
+	db = malloc(db_len);
+	salt = malloc(salt_len);
+	hprime = malloc(hash_len);
+	if (!masked_db || !h || !db_mask || !db || !salt || !hprime) {
+		printf("%s: can't allocate some buffer\n", __func__);
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	/* step 4: check if the last byte is 0xbc */
+	if (msg[msg_len - 1] != 0xbc) {
+		printf("%s: invalid pss padding (0xbc is missing)\n", __func__);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	/* step 5 */
+	memcpy(masked_db, msg, masked_db_len);
+	memcpy(h, msg + masked_db_len, h_len);
+
+	/* step 6 */
+	leftmost_mask = (0xff >> (8 - leftmost_bits)) << (8 - leftmost_bits);
+	if (masked_db[0] & leftmost_mask) {
+		printf("%s: invalid pss padding ", __func__);
+		printf("(leftmost bit of maskedDB not zero)\n");
+		ret = -EINVAL;
+		goto out;
+	}
+
+	/* step 7 */
+	mask_generation_function1(checksum, h, h_len, db_mask, db_mask_len);
+
+	/* step 8 */
+	for (i = 0; i < db_len; i++)
+		db[i] = masked_db[i] ^ db_mask[i];
+
+	/* step 9 */
+	db[0] &= 0xff >> leftmost_bits;
+
+	/* step 10 */
+	if (db[0] != 0x01) {
+		printf("%s: invalid pss padding ", __func__);
+		printf("(leftmost byte of db isn't 0x01)\n");
+		ret = EINVAL;
+		goto out;
+	}
+
+	/* step 11 */
+	memcpy(salt, &db[1], salt_len);
+
+	/* step 12 & 13 */
+	compute_hash_prime(checksum, pad_zero, 8,
+			   (uint8_t *)hash, hash_len,
+			   salt, salt_len, hprime);
+
+	/* step 14 */
+	ret = memcmp(h, hprime, hash_len);
+
+out:
+	free(hprime);
+	free(salt);
+	free(db);
+	free(db_mask);
+	free(h);
+	free(masked_db);
+
+	return ret;
+}
+#endif
+
 /**
  * rsa_verify_key() - Verify a signature against some data using RSA Key
  *