diff --git a/modules/meson.build b/modules/meson.build
index 2662b3e7..69234a77 100644
--- a/modules/meson.build
+++ b/modules/meson.build
@@ -3,17 +3,6 @@ common_c_args = [
   '-DG_LOG_USE_STRUCTURED',
 ]
 
-shared_library(
-  'wireplumber-module-client-permissions',
-  [
-    'module-client-permissions.c'
-  ],
-  c_args : [common_c_args, '-DG_LOG_DOMAIN="m-client-permissions"'],
-  install : true,
-  install_dir : wireplumber_module_dir,
-  dependencies : [wp_dep, pipewire_dep],
-)
-
 shared_library(
   'wireplumber-module-metadata',
   [
diff --git a/modules/module-client-permissions.c b/modules/module-client-permissions.c
deleted file mode 100644
index 2d0d53eb..00000000
--- a/modules/module-client-permissions.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/* WirePlumber
- *
- * Copyright © 2019 Collabora Ltd.
- *    @author George Kiagiadakis <george.kiagiadakis@collabora.com>
- *
- * SPDX-License-Identifier: MIT
- */
-
-#include <wp/wp.h>
-#include <pipewire/pipewire.h>
-
-struct _WpClientPermissions
-{
-  WpPlugin parent;
-  WpObjectManager *om;
-};
-
-G_DECLARE_FINAL_TYPE (WpClientPermissions, wp_client_permissions,
-                      WP, CLIENT_PERMISSIONS, WpPlugin)
-G_DEFINE_TYPE (WpClientPermissions, wp_client_permissions, WP_TYPE_PLUGIN)
-
-static void
-wp_client_permissions_init (WpClientPermissions * self)
-{
-}
-
-static void
-client_added (WpObjectManager * om, WpClient *client, WpClientPermissions * self)
-{
-  guint32 id = wp_proxy_get_bound_id (WP_PROXY (client));
-  const char *access = wp_pipewire_object_get_property (
-      WP_PIPEWIRE_OBJECT (client), PW_KEY_ACCESS);
-
-  wp_debug_object (self, "Client added: %d, access: %s", id, access);
-
-  if (!g_strcmp0 (access, "flatpak") || !g_strcmp0 (access, "restricted")) {
-    wp_debug_object (self, "Granting full access to client %d", id);
-    wp_client_update_permissions (client, 1, -1, PW_PERM_RWX);
-  }
-}
-
-static void
-wp_client_permissions_enable (WpPlugin * plugin, WpTransition * transition)
-{
-  WpClientPermissions * self = WP_CLIENT_PERMISSIONS (plugin);
-  g_autoptr (WpCore) core = wp_object_get_core (WP_OBJECT (plugin));
-
-  g_return_if_fail (core);
-
-  self->om = wp_object_manager_new ();
-  wp_object_manager_add_interest (self->om, WP_TYPE_CLIENT, NULL);
-  wp_object_manager_request_object_features (self->om, WP_TYPE_CLIENT,
-      WP_PIPEWIRE_OBJECT_FEATURES_MINIMAL);
-  g_signal_connect (self->om, "object-added", (GCallback) client_added, self);
-  wp_core_install_object_manager (core, self->om);
-
-  wp_object_update_features (WP_OBJECT (self), WP_PLUGIN_FEATURE_ENABLED, 0);
-}
-
-static void
-wp_client_permissions_disable (WpPlugin * plugin)
-{
-  WpClientPermissions * self = WP_CLIENT_PERMISSIONS (plugin);
-
-  g_clear_object (&self->om);
-}
-
-static void
-wp_client_permissions_class_init (WpClientPermissionsClass * klass)
-{
-  WpPluginClass *plugin_class = (WpPluginClass *) klass;
-
-  plugin_class->enable = wp_client_permissions_enable;
-  plugin_class->disable = wp_client_permissions_disable;
-}
-
-WP_PLUGIN_EXPORT gboolean
-wireplumber__module_init (WpCore * core, GVariant * args, GError ** error)
-{
-  wp_plugin_register (g_object_new (wp_client_permissions_get_type (),
-          "name", "client-permissions",
-          "core", core,
-          NULL));
-  return TRUE;
-}
diff --git a/src/config/config.lua b/src/config/config.lua
index 14a5a34b..ae82b415 100644
--- a/src/config/config.lua
+++ b/src/config/config.lua
@@ -34,6 +34,10 @@ function load_monitor(s, a)
   load_script("monitors/monitor-" .. s .. ".lua", a)
 end
 
+function load_access(s, a)
+  load_script("access/access-" .. s .. ".lua", a)
+end
+
 -- Session item factories, building blocks for the session management graph
 -- Do not disable these unless you really know what you are doing
 load_module("si-adapter")
@@ -62,9 +66,10 @@ load_script("suspend-node.lua")
 -- Automatically sets device profiles to 'On'
 load_module("device-activation")
 
--- Grants access to security confined clients
-load_module("client-permissions")
-
+function enable_access()
+  -- Flatpak access
+  load_access("flatpak")
+end
 
 function enable_audio()
   -- Enables functionality to save and restore default device profiles
@@ -104,6 +109,7 @@ end
 -- split these calls into .lua files in config.lua.d/
 -- to get a similar effect as the 'with-audio', 'with-pusleaudio', etc
 -- flag files that ship with pipewire-media-session
+enable_access()
 enable_audio()
 enable_bluetooth()
 enable_endpoints()
diff --git a/src/scripts/access/access-flatpak.lua b/src/scripts/access/access-flatpak.lua
new file mode 100644
index 00000000..83d81dc5
--- /dev/null
+++ b/src/scripts/access/access-flatpak.lua
@@ -0,0 +1,13 @@
+ID_ALL = 0xffffffff
+
+clients_om = ObjectManager { Interest { type = "client",
+  Constraint { "pipewire.access", "=", "flatpak" },
+} }
+
+clients_om:connect("object-added", function (om, client)
+  local id = client["bound-id"]
+  Log.info(client, "Granting RX access to client " .. id)
+  client:update_permissions ({[ID_ALL] = "rx" })
+end)
+
+clients_om:activate()