libnm/crypto: rework _nm_crypto_verify_cert() to return boolean

Rename _nm_crypto_verify_cert() to _nm_crypto_verify_x509().
Also, don't let it return a NMCryptoFileFormat result. This
function only checks for a particular format, hence it
should only return true/false.

Also, fix setting error output argument when the function fails.

libnm-core/nm-crypto-gnutls.c

@@ -268,8 +268,8 @@ out:
 	return output;
 }
 
-NMCryptoFileFormat
+gboolean
-_nm_crypto_verify_cert (const unsigned char *data,
+_nm_crypto_verify_x509 (const unsigned char *data,
                         gsize len,
                         GError **error)
 {
@@ -278,7 +278,7 @@ _nm_crypto_verify_cert (const unsigned char *data,
 	int err;
 
 	if (!_nm_crypto_init (error))
-		return NM_CRYPTO_FILE_FORMAT_UNKNOWN;
+		return FALSE;
 
 	err = gnutls_x509_crt_init (&der);
 	if (err < 0) {
@@ -286,7 +286,7 @@ _nm_crypto_verify_cert (const unsigned char *data,
 		             NM_CRYPTO_ERROR_INVALID_DATA,
 		             _("Error initializing certificate data: %s"),
 		             gnutls_strerror (err));
-		return NM_CRYPTO_FILE_FORMAT_UNKNOWN;
+		return FALSE;
 	}
 
 	/* Try DER first */
@@ -295,20 +295,20 @@ _nm_crypto_verify_cert (const unsigned char *data,
 	err = gnutls_x509_crt_import (der, &dt, GNUTLS_X509_FMT_DER);
 	if (err == GNUTLS_E_SUCCESS) {
 		gnutls_x509_crt_deinit (der);
-		return NM_CRYPTO_FILE_FORMAT_X509;
+		return TRUE;
 	}
 
 	/* And PEM next */
 	err = gnutls_x509_crt_import (der, &dt, GNUTLS_X509_FMT_PEM);
 	gnutls_x509_crt_deinit (der);
 	if (err == GNUTLS_E_SUCCESS)
-		return NM_CRYPTO_FILE_FORMAT_X509;
+		return TRUE;
 
 	g_set_error (error, NM_CRYPTO_ERROR,
 	             NM_CRYPTO_ERROR_INVALID_DATA,
 	             _("Couldn't decode certificate: %s"),
 	             gnutls_strerror (err));
-	return NM_CRYPTO_FILE_FORMAT_UNKNOWN;
+	return FALSE;
 }
 
 gboolean

libnm-core/nm-crypto-impl.h

@@ -55,9 +55,9 @@ char *_nm_crypto_decrypt (const char *cipher,
                           gsize *out_len,
                           GError **error);
 
-NMCryptoFileFormat _nm_crypto_verify_cert (const guint8 *data,
+gboolean _nm_crypto_verify_x509 (const guint8 *data,
-                                           gsize len,
+                                 gsize len,
-                                           GError **error);
+                                 GError **error);
 
 gboolean _nm_crypto_verify_pkcs12 (const guint8 *data,
                                    gsize data_len,

libnm-core/nm-crypto-nss.c

@@ -370,15 +370,15 @@ out:
 	return (char *) output;
 }
 
-NMCryptoFileFormat
+gboolean
-_nm_crypto_verify_cert (const unsigned char *data,
+_nm_crypto_verify_x509 (const unsigned char *data,
                         gsize len,
                         GError **error)
 {
 	CERTCertificate *cert;
 
 	if (!_nm_crypto_init (error))
-		return NM_CRYPTO_FILE_FORMAT_UNKNOWN;
+		return FALSE;
 
 	/* Try DER/PEM first */
 	cert = CERT_DecodeCertFromPackage ((char *) data, len);
@@ -387,11 +387,11 @@ _nm_crypto_verify_cert (const unsigned char *data,
 		             NM_CRYPTO_ERROR_INVALID_DATA,
 		             _("Couldn't decode certificate: %d"),
 		             PORT_GetError());
-		return NM_CRYPTO_FILE_FORMAT_UNKNOWN;
+		return FALSE;
 	}
 
 	CERT_DestroyCertificate (cert);
-	return NM_CRYPTO_FILE_FORMAT_X509;
+	return TRUE;
 }
 
 gboolean

libnm-core/nm-crypto.c

@@ -654,13 +654,19 @@ nm_crypto_load_and_verify_certificate (const char *file,
 	g_return_val_if_fail (file != NULL, NULL);
 	g_return_val_if_fail (out_file_format != NULL, NULL);
 
-	*out_file_format = NM_CRYPTO_FILE_FORMAT_UNKNOWN;
-
 	if (!_nm_crypto_init (error))
-		return NULL;
+		goto out;
 
 	if (!file_read_contents (file, &contents, error))
-		return NULL;
+		goto out;
+
+	if (contents.len == 0) {
+		g_set_error (error,
+		             NM_CRYPTO_ERROR,
+		             NM_CRYPTO_ERROR_INVALID_DATA,
+		             _("Certificate file is empty"));
+		goto out;
+	}
 
 	/* Check for PKCS#12 */
 	if (nm_crypto_is_pkcs12_data (contents.bin, contents.len, NULL)) {
@@ -670,20 +676,29 @@ nm_crypto_load_and_verify_certificate (const char *file,
 
 	/* Check for plain DER format */
 	if (contents.len > 2 && contents.bin[0] == 0x30 && contents.bin[1] == 0x82) {
-		*out_file_format = _nm_crypto_verify_cert (contents.bin, contents.len, error);
+		if (_nm_crypto_verify_x509 (contents.bin, contents.len, NULL)) {
+			*out_file_format = NM_CRYPTO_FILE_FORMAT_X509;
+			return to_gbyte_array_mem (contents.bin, contents.len);
+		}
 	} else {
 		nm_auto_clear_secret_ptr NMSecretPtr pem_cert = { 0 };
 
-		if (!extract_pem_cert_data (contents.bin, contents.len, &pem_cert, error))
+		if (extract_pem_cert_data (contents.bin, contents.len, &pem_cert, NULL)) {
-			return NULL;
+			if (_nm_crypto_verify_x509 (pem_cert.bin, pem_cert.len, NULL)) {
-
+				*out_file_format = NM_CRYPTO_FILE_FORMAT_X509;
-		*out_file_format = _nm_crypto_verify_cert (pem_cert.bin, pem_cert.len, error);
+				return to_gbyte_array_mem (contents.bin, contents.len);
+			}
+		}
 	}
 
-	if (*out_file_format != NM_CRYPTO_FILE_FORMAT_X509)
+	g_set_error (error,
-		return NULL;
+	             NM_CRYPTO_ERROR,
+	             NM_CRYPTO_ERROR_INVALID_DATA,
+	             _("Failed to recognize certificate"));
 
-	return to_gbyte_array_mem (contents.bin, contents.len);
+out:
+	*out_file_format = NM_CRYPTO_FILE_FORMAT_UNKNOWN;
+	return NULL;
 }
 
 gboolean
@@ -694,8 +709,13 @@ nm_crypto_is_pkcs12_data (const guint8 *data,
 	GError *local = NULL;
 	gboolean success;
 
-	if (!data_len)
+	if (!data_len) {
+		g_set_error (error,
+		             NM_CRYPTO_ERROR,
+		             NM_CRYPTO_ERROR_INVALID_DATA,
+		             _("Certificate file is empty"));
 		return FALSE;
+	}
 
 	g_return_val_if_fail (data != NULL, FALSE);