diff --git a/sublime/adapters/adapter_base.py b/sublime/adapters/adapter_base.py
index b2f95b6..56e538f 100644
--- a/sublime/adapters/adapter_base.py
+++ b/sublime/adapters/adapter_base.py
@@ -1,4 +1,5 @@
 import abc
+import copy
 import hashlib
 import uuid
 from dataclasses import dataclass
@@ -174,18 +175,47 @@ class ConfigurationStore(dict):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+        self._changed_secrets_store = {}
 
     def __repr__(self) -> str:
         values = ", ".join(f"{k}={v!r}" for k, v in sorted(self.items()))
         return f"ConfigurationStore({values})"
 
+    def clone(self) -> "ConfigurationStore":
+        configuration_store = ConfigurationStore(**copy.deepcopy(self))
+        configuration_store._changed_secrets_store = copy.deepcopy(
+            self._changed_secrets_store
+        )
+        return configuration_store
+
+    def persist_secrets(self):
+        if not keyring_imported or ConfigurationStore.MOCK:
+            return
+
+        for key, secret in self._changed_secrets_store.items():
+            try:
+                password_id = None
+                if password_type_and_id := self.get(key):
+                    if cast(List[str], password_type_and_id)[0] == "keyring":
+                        password_id = password_type_and_id[1]
+
+                if password_id is None:
+                    password_id = str(uuid.uuid4())
+
+                keyring.set_password(KEYRING_APP_NAME, password_id, secret)
+                self[key] = ["keyring", password_id]
+            except Exception:
+                return
+
     def get_secret(self, key: str) -> Optional[str]:
         """
-        Get the secret value in the store with the given key. If the key doesn't exist
-        in the store, return the default. This will retrieve the secret from whatever is
-        configured as the underlying secret storage mechanism so you don't have to deal
-        with secret storage yourself.
+        Get the secret value in the store with the given key. This will retrieve the
+        secret from whatever is configured as the underlying secret storage mechanism so
+        you don't have to deal with secret storage yourself.
         """
+        if secret := self._changed_secrets_store.get(key):
+            return secret
+
         value = self.get(key)
         if not isinstance(value, list) or len(value) != 2:
             return None
@@ -199,26 +229,11 @@ class ConfigurationStore(dict):
     def set_secret(self, key: str, value: str = None):
         """
         Set the secret value of the given key in the store. This should be used for
-        things such as passwords or API tokens. This will store the secret in whatever
-        is configured as the underlying secret storage mechanism so you don't have to
-        deal with secret storage yourself.
+        things such as passwords or API tokens. When :class:`persist_secrets` is called,
+        the secrets will be stored in whatever is configured as the underlying secret
+        storage mechanism so you don't have to deal with secret storage yourself.
         """
-        if keyring_imported and not ConfigurationStore.MOCK:
-            try:
-                password_id = None
-                if password_type_and_id := self.get(key):
-                    if cast(List[str], password_type_and_id)[0] == "keyring":
-                        password_id = password_type_and_id[1]
-
-                if password_id is None:
-                    password_id = str(uuid.uuid4())
-
-                keyring.set_password(KEYRING_APP_NAME, password_id, value)
-                self[key] = ["keyring", password_id]
-                return
-            except Exception:
-                pass
-
+        self._changed_secrets_store[key] = value
         self[key] = ["plaintext", value]
 
 
diff --git a/sublime/app.py b/sublime/app.py
index d10c2ab..9412e5d 100644
--- a/sublime/app.py
+++ b/sublime/app.py
@@ -594,7 +594,7 @@ class SublimeMusicApp(Gtk.Application):
         self.show_configure_servers_dialog()
 
     def on_edit_current_music_provider(self, *args):
-        self.show_configure_servers_dialog(self.app_config.provider)
+        self.show_configure_servers_dialog(self.app_config.provider.clone())
 
     def on_switch_music_provider(self, _, provider_id: GLib.Variant):
         if self.app_config.state.playing:
@@ -983,7 +983,9 @@ class SublimeMusicApp(Gtk.Application):
         if result == Gtk.ResponseType.APPLY:
             assert dialog.provider_config is not None
             provider_id = dialog.provider_config.id
+            dialog.provider_config.persist_secrets()
             self.app_config.providers[provider_id] = dialog.provider_config
+            self.app_config.save()
 
             if provider_id == self.app_config.current_provider_id:
                 # Just update the window.
diff --git a/sublime/config.py b/sublime/config.py
index e601378..b93442e 100644
--- a/sublime/config.py
+++ b/sublime/config.py
@@ -47,6 +47,25 @@ class ProviderConfiguration:
         if self.caching_adapter_type:
             self.caching_adapter_type.migrate_configuration(self.caching_adapter_config)
 
+    def clone(self) -> "ProviderConfiguration":
+        return ProviderConfiguration(
+            self.id,
+            self.name,
+            self.ground_truth_adapter_type,
+            self.ground_truth_adapter_config.clone(),
+            self.caching_adapter_type,
+            (
+                self.caching_adapter_config.clone()
+                if self.caching_adapter_config
+                else None
+            ),
+        )
+
+    def persist_secrets(self):
+        self.ground_truth_adapter_config.persist_secrets()
+        if self.caching_adapter_config:
+            self.caching_adapter_config.persist_secrets()
+
 
 def encode_providers(
     providers_dict: Dict[str, Dict[str, Any]]