Add checks for saltlen and trailerfield to rsa key writer.

Fixes openssl#18168.

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/18615)

providers/common/der/der_rsa_key.c

@@ -305,6 +305,15 @@ int ossl_DER_w_RSASSA_PSS_params(WPACKET *pkt, int tag,
     saltlen = ossl_rsa_pss_params_30_saltlen(pss);
     trailerfield = ossl_rsa_pss_params_30_trailerfield(pss);
 
+    if (saltlen < 0) {
+        ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_SALT_LENGTH);
+        return 0;
+    }
+    if (trailerfield != 1) {
+        ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_TRAILER);
+        return 0;
+    }
+
     /* Getting default values */
     default_hashalg_nid = ossl_rsa_pss_params_30_hashalg(NULL);
     default_saltlen = ossl_rsa_pss_params_30_saltlen(NULL);

test/recipes/15-test_rsapss.t

@@ -11,12 +11,12 @@ use strict;
 use warnings;
 
 use File::Spec;
-use OpenSSL::Test qw/:DEFAULT with srctop_file/;
+use OpenSSL::Test qw/:DEFAULT with srctop_file data_file/;
 use OpenSSL::Test::Utils;
 
 setup("test_rsapss");
 
-plan tests => 9;
+plan tests => 10;
 
 #using test/testrsa.pem which happens to be a 512 bit RSA
 ok(run(app(['openssl', 'dgst', '-sign', srctop_file('test', 'testrsa.pem'), '-sha1',
@@ -74,3 +74,7 @@ ok(run(app(['openssl', 'dgst', '-prverify', srctop_file('test', 'testrsa.pem'),
    ok(run(app(['openssl', 'rsa', '-check',
                '-in', $rsapss])));
 }
+
+ok(!run(app([ 'openssl', 'rsa',
+             '-in' => data_file('negativesaltlen.pem')],
+             '-out' => 'badout')));

test/recipes/15-test_rsapss_data/negativesaltlen.pem

@@ -0,0 +1,29 @@
+-----BEGIN PRIVATE KEY-----
+MIIE7gIB0DA9BgkqhkiG9w0BAQowMKANMAsGCWCGSAFlAwQCA6EaMBgGCSqGSIb3
+DQEBCDALBglghkgBZQMEAgOiAwIB6ASCBKgwggSkAgEAAoIBAQDdiLMYj8fgrXKB
+dEC704hcfmeJebCyaZbYHBE/1YthJOptbhisBbNk4onKMITO6hkYOoH12rNxqwY5
+d9J1Ray6SJETVHxYCKftJ1LlrUJGqpyRCAAff1LYjjGRyqcMzVItWffy2iCgKGud
+uUqs9Og3wsVxUeXfTSGnLo1UevVc1qTKZJuDRWD2EItuwnFt7GA89IgGx8/liLsg
+cdlnm81gGdDmNKxNGi3VeOaJqFWnP9CpL8iXybG7F32U9mgEdE+EYt8GhQfNLzjL
+j17xfLl5K0SMqL8q+phas6Md0OmTl3Xg8Tupdoo/okAoYGXrv/sHDiV1YBSkXD4i
+dbV42aUfAgMBAAECggEAEyEJrfZEYR85Avqh2FYksS/tCs7qNg2uC80opCVxWbsQ
+bxCRqtD3M5/oHABih2dpcVEkBbGzyv3klLPHBX9VseQwOsYR0pw0u+KoYtK6JVX4
+HQHe2Nlqsu5cU2V3VUCpducM5Ph21r2GxWDJlPO01ZPI7scOnWCQpln7tC7F3xU0
+jNQ0SnFZ6SO4FrrBxOMjnIFiNMexxZt0fU7khy/dGck9aN4DtmQENcQkGdXj5xRv
+lInh92mQ16yMCbEU8cslWaAwqRF/k/5QxoIwTXr8PqaWshH9TIAht0rvTilWpHPg
+zpW6Pog/wGzVat3NeU3vBDYIUayHc6n3gbfJZDNxmQKBgQD41lAkxNsA89mYY7S9
+5NkDJ1N1hKNwg+iEyCZJkjxUk+SymdO7U/iD27Hgn/XyXm4RC5aHYpXJSnuiOk7R
+Z1Az1jjqLzPxsP72sWLORzGq82smYrK+iV2rhozWNlfVyazDkBcRRz2bLSESzgvO
+JWD3K3pjvj8U9ZSUhz+zXo4sUwKBgQDj6TBTKGDb8Au8sUOC916GrIrUEq5SkMDT
+A4CiD4fmvbdNs90AhD/mmqBw/dP3TbCPNmP8tGMUT0BDev6BoRKYOt+1XGYXt2de
+P38teVU/ZUcAO2RGdMNSdWT5o9BCWQZ18qSoOR/QanckOnkhKCgU/wqSdIvBBRMQ
+5e4qdI0qhQKBgB2MJTxYfADi88WaoU2jLPmo48oik926bBPISHOX/73zScbDaVbn
+I61UmwyXMfczq1Iu1BMDa9HZHFEpJ07KO8XL/DoinMJoR/43Fgp0fbtU6DZIpfzm
+Bs9lTLfrAAcMyYz3QSX2FaSleTXobZJu8dKnwQKzBn6QorH4VWIRKkStAoGBAIYL
+M1nlaLpSf4S2OT/A376Ton9CkXaMHmy9JZ2rRsHmGPZBcB0Kq06k6PIrx8wuzEYe
+tkX9jjx2tBQ8NY3mPzp7ffF766vNOaWL8O+86e+EUHMJe1uY9vv7gaz1tNog5BTg
+5gjuuBBrXbFYFr/yj0hyDDTBCSU4J9OLeD1OGWzFAoGBAMGc9h8oLyA3rQEjIuVA
+CuzgvZxOFPbtODFPcL4EQgAKLiKS+oZK0jONfCHaQB1AhIq8/nT/4suw7tWqYoKp
+KGH/+8tKNodKZfZLjVp0k8gsehyMDz1002/RLMJyFRIJWa1BqEJs7v7XgWW3RcmC
+PWznhdpNx3BYDSao5Ibl7I5E
+-----END PRIVATE KEY-----