Refactor service to use external profile JSONs

- Store only profile key/metadata into service.

- Map profiles by (context, id), context being either provider
or host.

- Initialize cache with a placeholder profile, lazily load full
profile (e.g. after opening profile).

- Only serialize non-placeholder profiles (opened once).

- Do not load full profiles for organizer listing

WARNING: always load active profile as non-placeholder.

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - Host parameters are read-only if there isn't an original configuration to revert to.
+- Overall serialization performance.
 
 ## 1.0 beta 1084 (2018-10-24)
 

Passepartout-iOS/Scenes/Organizer/OrganizerViewController.swift

@@ -30,9 +30,9 @@ import UIKit
 class OrganizerViewController: UITableViewController, TableModelHost {
     private let service = TransientStore.shared.service
     
-    private var providerProfiles: [ProviderConnectionProfile] = []
+    private var providers: [String] = []
 
-    private var hostProfiles: [HostConnectionProfile] = []
+    private var hosts: [String] = []
     
     private var availableProviderNames: [Infrastructure.Name]?
 
@@ -56,29 +56,16 @@ class OrganizerViewController: UITableViewController, TableModelHost {
     }()
     
     func reloadModel() {
-        providerProfiles.removeAll()
+        providers = service.ids(forContext: .provider).sorted()
-        hostProfiles.removeAll()
+        hosts = service.ids(forContext: .host).sorted()
         
-        service.profileIds().forEach {
+        var providerRows = [RowType](repeating: .profile, count: providers.count)
-            let profile = service.profile(withId: $0)
+        var hostRows = [RowType](repeating: .profile, count: hosts.count)
-            if let p = profile as? ProviderConnectionProfile {
+        providerRows.append(.addProvider)
-                providerProfiles.append(p)
+        hostRows.append(.addHost)
-            } else if let p = profile as? HostConnectionProfile {
-                hostProfiles.append(p)
-            } else {
-                fatalError("Unexpected profile type \(type(of: profile))")
-            }
-        }
-        providerProfiles.sort { $0.name.rawValue < $1.name.rawValue }
-        hostProfiles.sort { $0.title < $1.title }
         
-        var providers = [RowType](repeating: .profile, count: providerProfiles.count)
+        model.set(providerRows, in: .providers)
-        var hosts = [RowType](repeating: .profile, count: hostProfiles.count)
+        model.set(hostRows, in: .hosts)
-        providers.append(.addProvider)
-        hosts.append(.addHost)
-        
-        model.set(providers, in: .providers)
-        model.set(hosts, in: .hosts)
     }
     
     // MARK: UIViewController
@@ -163,7 +150,13 @@ class OrganizerViewController: UITableViewController, TableModelHost {
     
     private func addNewProvider() {
         var names = Set(InfrastructureFactory.shared.allNames)
-        let createdNames = providerProfiles.map { $0.name }
+        var createdNames: [Infrastructure.Name] = []
+        providers.forEach {
+            guard let name = Infrastructure.Name(rawValue: $0) else {
+                return
+            }
+            createdNames.append(name)
+        }
         names.formSymmetricDifference(createdNames)
 
         guard !names.isEmpty else {
@@ -191,13 +184,13 @@ class OrganizerViewController: UITableViewController, TableModelHost {
 
     private func removeProfile(at indexPath: IndexPath) {
         let sectionObject = model.section(for: indexPath.section)
-        let rowProfile = profile(at: indexPath)
+        let rowProfile = profileKey(at: indexPath)
         switch sectionObject {
         case .providers:
-            providerProfiles.remove(at: indexPath.row)
+            providers.remove(at: indexPath.row)
             
         case .hosts:
-            hostProfiles.remove(at: indexPath.row)
+            hosts.remove(at: indexPath.row)
             
         default:
             return
@@ -205,7 +198,7 @@ class OrganizerViewController: UITableViewController, TableModelHost {
         
 //        var fallbackSection: SectionType?
         
-        let total = providerProfiles.count + hostProfiles.count
+        let total = providers.count + hosts.count
         
         // removed all profiles
         if total == 0 {
@@ -280,14 +273,19 @@ extension OrganizerViewController {
     }
     
     private var selectedIndexPath: IndexPath? {
-        guard let active = service.activeProfile?.id else {
+        guard let active = service.activeProfileKey else {
             return nil
         }
-        if let row = providerProfiles.index(where: { $0.id == active }) {
+        switch active.context {
-            return IndexPath(row: row, section: 0)
+        case .provider:
-        }
+            if let row = providers.index(where: { $0 == active.id }) {
-        if let row = hostProfiles.index(where: { $0.id == active }) {
+                return IndexPath(row: row, section: 0)
-            return IndexPath(row: row, section: 1)
+            }
+
+        case .host:
+            if let row = hosts.index(where: { $0 == active.id }) {
+                return IndexPath(row: row, section: 1)
+            }
         }
         return nil
     }
@@ -312,7 +310,7 @@ extension OrganizerViewController {
         switch model.row(at: indexPath) {
         case .profile:
             let cell = Cells.setting.dequeue(from: tableView, for: indexPath)
-            let rowProfile = profile(at: indexPath)
+            let rowProfile = profileKey(at: indexPath)
             cell.leftText = rowProfile.id
             cell.rightText = service.isActiveProfile(rowProfile) ? L10n.Organizer.Cells.Profile.Value.current : nil
             return cell
@@ -375,27 +373,57 @@ extension OrganizerViewController {
     
     // MARK: Helpers
     
-    private func sectionProfiles(at indexPath: IndexPath) -> [ConnectionProfile] {
+    private func sectionProfiles(at indexPath: IndexPath) -> [String] {
-        let sectionProfiles: [ConnectionProfile]
+        let ids: [String]
         let sectionObject = model.section(for: indexPath.section)
         switch sectionObject {
         case .providers:
-            sectionProfiles = providerProfiles
+            ids = providers
             
         case .hosts:
-            sectionProfiles = hostProfiles
+            ids = hosts
             
         default:
             fatalError("Unexpected section: \(sectionObject)")
         }
-        guard indexPath.row < sectionProfiles.count else {
+        guard indexPath.row < ids.count else {
             fatalError("No profile found at \(indexPath), is it an add cell?")
         }
-        return sectionProfiles
+        return ids
     }
     
+    private func profileKey(at indexPath: IndexPath) -> ConnectionService.ProfileKey {
+        let section = model.section(for: indexPath.section)
+        switch section {
+        case .providers:
+            return ConnectionService.ProfileKey(.provider, providers[indexPath.row])
+            
+        case .hosts:
+            return ConnectionService.ProfileKey(.host, hosts[indexPath.row])
+            
+        default:
+            fatalError("Profile found in unexpected section: \(section)")
+        }
+    }
+
     private func profile(at indexPath: IndexPath) -> ConnectionProfile {
-        return sectionProfiles(at: indexPath)[indexPath.row]
+        let id = sectionProfiles(at: indexPath)[indexPath.row]
+        let section = model.section(for: indexPath.section)
+        let context: ConnectionService.ProfileKey.Context
+        switch section {
+        case .providers:
+            context = .provider
+            
+        case .hosts:
+            context = .host
+            
+        default:
+            fatalError("Profile found in unexpected section: \(section)")
+        }
+        guard let found = service.profile(withContext: context, id: id) else {
+            fatalError("Profile (\(context), \(id)) could not be found, why was it returned?")
+        }
+        return found
     }
 }
 

Passepartout-iOS/Scenes/Organizer/WizardHostViewController.swift

@@ -44,17 +44,8 @@ class WizardHostViewController: UITableViewController, TableModelHost, Wizard {
 
     @IBOutlet private weak var itemNext: UIBarButtonItem!
     
-    private let existingHosts: [HostConnectionProfile] = {
+    private let existingHosts: [String] = {
-        var hosts: [HostConnectionProfile] = []
+        return TransientStore.shared.service.ids(forContext: .host).sorted()
-        let service = TransientStore.shared.service
-        let ids = service.profileIds()
-        for id in ids {
-            guard let host = service.profile(withId: id) as? HostConnectionProfile else {
-                continue
-            }
-            hosts.append(host)
-        }
-        return hosts.sorted { $0.title < $1.title }
     }()
     
     private var parsedFile: ParsedFile? {
@@ -231,10 +222,9 @@ extension WizardHostViewController {
             return cell
             
         case .existingHost:
-            let profile = existingHosts[indexPath.row]
+            let hostTitle = existingHosts[indexPath.row]
-            
             let cell = Cells.setting.dequeue(from: tableView, for: indexPath)
-            cell.leftText = profile.title
+            cell.leftText = hostTitle
             cell.accessoryType = .none
             cell.isTappable = true
             return cell
@@ -247,9 +237,9 @@ extension WizardHostViewController {
             guard let titleIndexPath = model.indexPath(row: .titleInput, section: .meta) else {
                 fatalError("Could not found title cell?")
             }
-            let profile = existingHosts[indexPath.row]
+            let hostTitle = existingHosts[indexPath.row]
             let cellTitle = tableView.cellForRow(at: titleIndexPath) as? FieldTableViewCell
-            cellTitle?.field.text = profile.title
+            cellTitle?.field.text = hostTitle
             tableView.deselectRow(at: indexPath, animated: true)
             
         default:

Passepartout/Sources/Model/ConnectionService+Migration.swift

@@ -160,6 +160,13 @@ extension ConnectionService {
                 try data.write(to: url)
             }
         }
+        
+        guard let activeProfileId = json["activeProfileId"] else {
+            return
+        }
+        json["activeProfileKey"] = activeProfileId
+        json.removeValue(forKey: "activeProfileId")
+        json.removeValue(forKey: "profiles")
     }
     
     // MARK: Helpers

Passepartout/Sources/Model/ConnectionService.swift

@@ -44,13 +44,85 @@ class ConnectionService: Codable {
         
         case baseConfiguration
         
-        case profiles
+        case activeProfileKey
-        
-        case activeProfileId
         
         case preferences
     }
 
+    struct ProfileKey: RawRepresentable, Hashable, Codable {
+        enum Context: String {
+            case provider
+            
+            case host
+        }
+        
+        let context: Context
+        
+        let id: String
+
+        init(_ context: Context, _ id: String) {
+            self.context = context
+            self.id = id
+        }
+
+        init(_ profile: ConnectionProfile) {
+            if let _ = profile as? ProviderConnectionProfile {
+                context = .provider
+            } else if let _ = profile as? HostConnectionProfile {
+                context = .host
+            } else if let placeholder = profile as? PlaceholderConnectionProfile {
+                context = placeholder.context
+            } else {
+                fatalError("Unexpected profile type: \(type(of: profile))")
+            }
+            id = profile.id
+        }
+        
+        fileprivate func profileURL(in service: ConnectionService) -> URL {
+            let contextURL: URL
+            switch context {
+            case .provider:
+                contextURL = service.providersURL
+                
+            case .host:
+                contextURL = service.hostsURL
+            }
+            return ConnectionService.url(in: contextURL, forProfileId: id)
+        }
+
+        fileprivate func profileData(in service: ConnectionService) throws -> Data {
+            return try Data(contentsOf: profileURL(in: service))
+        }
+
+        // MARK: RawRepresentable
+        
+        var rawValue: String {
+            return "\(context).\(id)"
+        }
+        
+        init?(rawValue: String) {
+            let comps = rawValue.components(separatedBy: ".")
+            guard comps.count == 2 else {
+                return nil
+            }
+            guard let context = Context(rawValue: comps[0]) else {
+                return nil
+            }
+            self.context = context
+            id = comps[1]
+        }
+    }
+
+    lazy var directory = FileManager.default.userURL(for: .documentDirectory, appending: nil)
+    
+    private var providersURL: URL {
+        return directory.appendingPathComponent(AppConstants.Store.providersDirectory)
+    }
+
+    private var hostsURL: URL {
+        return directory.appendingPathComponent(AppConstants.Store.hostsDirectory)
+    }
+    
     private var build: Int
     
     private let appGroup: String
@@ -61,9 +133,11 @@ class ConnectionService: Codable {
     
     var baseConfiguration: TunnelKitProvider.Configuration
     
-    private var profiles: [String: ConnectionProfile]
+    private var cache: [ProfileKey: ConnectionProfile]
     
-    private var activeProfileId: String? {
+    private var pendingRemoval: Set<ProfileKey>
+    
+    private(set) var activeProfileKey: ProfileKey? {
         willSet {
             if let oldProfile = activeProfile {
                 delegate?.connectionService(didDeactivate: oldProfile)
@@ -77,10 +151,15 @@ class ConnectionService: Codable {
     }
     
     var activeProfile: ConnectionProfile? {
-        guard let id = activeProfileId else {
+        guard let id = activeProfileKey else {
             return nil
         }
-        return profiles[id]
+        var hit = cache[id]
+        if let placeholder = hit as? PlaceholderConnectionProfile {
+            hit = profile(withContext: placeholder.context, id: placeholder.id)
+            cache[id] = hit
+        }
+        return hit
     }
     
     let preferences: EditablePreferences
@@ -97,9 +176,11 @@ class ConnectionService: Codable {
         keychain = Keychain(group: appGroup)
 
         self.baseConfiguration = baseConfiguration
-        profiles = [:]
+        activeProfileKey = nil
-        activeProfileId = nil
         preferences = EditablePreferences()
+
+        cache = [:]
+        pendingRemoval = []
     }
     
     // MARK: Codable
@@ -116,17 +197,11 @@ class ConnectionService: Codable {
         keychain = Keychain(group: appGroup)
 
         baseConfiguration = try container.decode(TunnelKitProvider.Configuration.self, forKey: .baseConfiguration)
-        let profilesArray = try container.decode([ConnectionProfileHolder].self, forKey: .profiles).map { $0.contained }
+        activeProfileKey = try container.decodeIfPresent(ProfileKey.self, forKey: .activeProfileKey)
-        var profiles: [String: ConnectionProfile] = [:]
-        profilesArray.forEach {
-            guard let p = $0 else {
-                return
-            }
-            profiles[p.id] = p
-        }
-        self.profiles = profiles
-        activeProfileId = try container.decodeIfPresent(String.self, forKey: .activeProfileId)
         preferences = try container.decode(EditablePreferences.self, forKey: .preferences)
+
+        cache = [:]
+        pendingRemoval = []
     }
     
     func encode(to encoder: Encoder) throws {
@@ -136,23 +211,126 @@ class ConnectionService: Codable {
         try container.encode(build, forKey: .build)
         try container.encode(appGroup, forKey: .appGroup)
         try container.encode(baseConfiguration, forKey: .baseConfiguration)
-        try container.encode(profiles.map { ConnectionProfileHolder($0.value) }, forKey: .profiles)
+        try container.encodeIfPresent(activeProfileKey, forKey: .activeProfileKey)
-        try container.encodeIfPresent(activeProfileId, forKey: .activeProfileId)
         try container.encode(preferences, forKey: .preferences)
     }
     
+    // MARK: Serialization
+    
+    func loadProfiles() {
+        let fm = FileManager.default
+        try? fm.createDirectory(at: providersURL, withIntermediateDirectories: false, attributes: nil)
+        try? fm.createDirectory(at: hostsURL, withIntermediateDirectories: false, attributes: nil)
+        
+        do {
+            let files = try fm.contentsOfDirectory(at: providersURL, includingPropertiesForKeys: nil, options: [])
+//            log.debug("Found \(files.count) provider files: \(files)")
+            for entry in files {
+                guard let id = ConnectionService.profileId(fromURL: entry) else {
+                    return
+                }
+                let key = ProfileKey(.provider, id)
+                cache[key] = PlaceholderConnectionProfile(key)
+            }
+        } catch let e {
+            log.warning("Could not list provider contents: \(e) (\(providersURL))")
+        }
+        do {
+            let files = try fm.contentsOfDirectory(at: hostsURL, includingPropertiesForKeys: nil, options: [])
+//            log.debug("Found \(files.count) host files: \(files)")
+            for entry in files {
+                guard let id = ConnectionService.profileId(fromURL: entry) else {
+                    continue
+                }
+                let key = ProfileKey(.host, id)
+                cache[key] = PlaceholderConnectionProfile(key)
+            }
+        } catch let e {
+            log.warning("Could not list host contents: \(e) (\(hostsURL))")
+        }
+    }
+    
+    func saveProfiles() throws {
+        let encoder = JSONEncoder()
+
+        let fm = FileManager.default
+        try? fm.createDirectory(at: providersURL, withIntermediateDirectories: false, attributes: nil)
+        try? fm.createDirectory(at: hostsURL, withIntermediateDirectories: false, attributes: nil)
+
+        for key in pendingRemoval {
+            let url = key.profileURL(in: self)
+            try? fm.removeItem(at: url)
+        }
+        for entry in cache.values {
+            if let profile = entry as? ProviderConnectionProfile {
+                do {
+                    let url = ConnectionService.url(in: providersURL, forProfileId: entry.id)
+                    let data = try encoder.encode(profile)
+                    try data.write(to: url)
+                    log.debug("Saved provider '\(profile.id)'")
+                } catch let e {
+                    log.warning("Could not save provider '\(profile.id)': \(e)")
+                    continue
+                }
+            } else if let profile = entry as? HostConnectionProfile {
+                do {
+                    let url = ConnectionService.url(in: hostsURL, forProfileId: entry.id)
+                    let data = try encoder.encode(profile)
+                    try data.write(to: url)
+                    log.debug("Saved host '\(profile.id)'")
+                } catch let e {
+                    log.warning("Could not save host '\(profile.id)': \(e)")
+                    continue
+                }
+            } else if let placeholder = entry as? PlaceholderConnectionProfile {
+                log.debug("Skipped \(placeholder.context) '\(placeholder.id)'")
+            }
+        }
+    }
+    
+    func profile(withContext context: ProfileKey.Context, id: String) -> ConnectionProfile? {
+        let key = ProfileKey(context, id)
+        var profile = cache[key]
+        if let _ = profile as? PlaceholderConnectionProfile {
+            let decoder = JSONDecoder()
+            do {
+                let data = try key.profileData(in: self)
+                switch context {
+                case .provider:
+                    profile = try decoder.decode(ProviderConnectionProfile.self, from: data)
+                    
+                case .host:
+                    profile = try decoder.decode(HostConnectionProfile.self, from: data)
+                }
+                cache[key] = profile
+            } catch let e {
+                log.warning("Could not decode profile JSON: \(e)")
+                return nil
+            }
+        }
+        return profile
+    }
+    
+    func ids(forContext context: ProfileKey.Context) -> [String] {
+        return cache.keys.filter { $0.context == context }.map { $0.id }
+    }
+    
+    private static func profileId(fromURL url: URL) -> String? {
+        let filename = url.lastPathComponent
+        guard let extRange = filename.range(of: ".json") else {
+            return nil
+        }
+        return String(filename[filename.startIndex..<extRange.lowerBound])
+    }
+    
+    private static func url(in directory: URL, forProfileId profileId: String) -> URL {
+        return directory.appendingPathComponent("\(profileId).json")
+    }
+    
     // MARK: Profiles
-    
+
-    func profileIds() -> [String] {
-        return Array(profiles.keys)
-    }
-    
-    func profile(withId id: String) -> ConnectionProfile? {
-        return profiles[id]
-    }
-    
     func addProfile(_ profile: ConnectionProfile, credentials: Credentials?) -> Bool {
-        guard profiles.index(forKey: profile.id) == nil else {
+        guard cache.index(forKey: ProfileKey(profile)) == nil else {
             return false
         }
         addOrReplaceProfile(profile, credentials: credentials)
@@ -160,37 +338,51 @@ class ConnectionService: Codable {
     }
     
     func addOrReplaceProfile(_ profile: ConnectionProfile, credentials: Credentials?) {
-        profiles[profile.id] = profile
+        let key = ProfileKey(profile)
+        cache[key] = profile
+        pendingRemoval.remove(key)
         try? setCredentials(credentials, for: profile)
-        if profiles.count == 1 {
+        if cache.count == 1 {
-            activeProfileId = profile.id
+            activeProfileKey = key
         }
+
+        // serialize immediately
+        try? saveProfiles()
     }
     
-    func removeProfile(_ profile: ConnectionProfile) {
+    func removeProfile(_ key: ProfileKey) {
-        guard let i = profiles.index(forKey: profile.id) else {
+        guard let i = cache.index(forKey: key) else {
             return
         }
-        profiles.remove(at: i)
+        cache.remove(at: i)
-        if profiles.isEmpty {
+        pendingRemoval.insert(key)
-            activeProfileId = nil
+        if cache.isEmpty {
+            activeProfileKey = nil
         }
     }
     
+    func containsProfile(_ key: ProfileKey) -> Bool {
+        return cache.index(forKey: key) != nil
+    }
+
     func containsProfile(_ profile: ConnectionProfile) -> Bool {
-        return profiles.index(forKey: profile.id) != nil
+        return containsProfile(ProfileKey(profile))
     }
-
+    
     func hasActiveProfile() -> Bool {
-        return activeProfileId != nil
+        return activeProfileKey != nil
     }
 
+    func isActiveProfile(_ key: ProfileKey) -> Bool {
+        return key == activeProfileKey
+    }
+    
     func isActiveProfile(_ profile: ConnectionProfile) -> Bool {
-        return profile.id == activeProfileId
+        return isActiveProfile(ProfileKey(profile))
     }
     
     func activateProfile(_ profile: ConnectionProfile) {
-        activeProfileId = profile.id
+        activeProfileKey = ProfileKey(profile)
     }
     
     // MARK: Credentials
@@ -291,3 +483,34 @@ class ConnectionService: Codable {
 //        defaults.removeObject(forKey: Keys.vpnLog)
 //    }
 }
+
+private class PlaceholderConnectionProfile: ConnectionProfile {
+    let id: String
+    
+    var username: String?
+    
+    var requiresCredentials: Bool = false
+    
+    func generate(from configuration: TunnelKitProvider.Configuration, preferences: Preferences) throws -> TunnelKitProvider.Configuration {
+        fatalError("Generating configuration from a PlaceholderConnectionProfile")
+    }
+    
+    var mainAddress: String = ""
+    
+    var addresses: [String] = []
+    
+    var protocols: [TunnelKitProvider.EndpointProtocol] = []
+    
+    var canCustomizeEndpoint: Bool = false
+    
+    var customAddress: String?
+    
+    var customProtocol: TunnelKitProvider.EndpointProtocol?
+
+    let context: ConnectionService.ProfileKey.Context
+    
+    init(_ key: ConnectionService.ProfileKey) {
+        self.context = key.context
+        self.id = key.id
+    }
+}

Passepartout/Sources/Model/TransientStore.swift

@@ -35,10 +35,12 @@ class TransientStore {
     
     static let shared = TransientStore()
     
-    private let servicePath: URL
+    private let rootURL: URL
+
+    private let serviceURL: URL
     
     let service: ConnectionService
-    
+
     var didHandleSubreddit: Bool {
         get {
             return UserDefaults.standard.bool(forKey: Keys.didHandleSubreddit)
@@ -49,27 +51,29 @@ class TransientStore {
     }
 
     private init() {
-        servicePath = FileManager.default.userURL(
+        rootURL = FileManager.default.userURL(for: .documentDirectory, appending: nil)
-            for: .documentDirectory,
+        serviceURL = rootURL.appendingPathComponent(AppConstants.Store.serviceFilename)
-            appending: AppConstants.Store.serviceFilename
+
-        )
         let cfg = AppConstants.VPN.baseConfiguration()
         do {
-            ConnectionService.migrateJSON(at: servicePath, to: servicePath)
+            ConnectionService.migrateJSON(at: serviceURL, to: serviceURL)
             
-            let data = try Data(contentsOf: servicePath)
+            let data = try Data(contentsOf: serviceURL)
             if let content = String(data: data, encoding: .utf8) {
                 log.verbose("Service JSON:")
                 log.verbose(content)
             }
             service = try JSONDecoder().decode(ConnectionService.self, from: data)
+            service.directory = rootURL
             service.baseConfiguration = cfg
+            service.loadProfiles()
         } catch let e {
             log.error("Could not decode service: \(e)")
             service = ConnectionService(
                 withAppGroup: GroupConstants.App.appGroup,
                 baseConfiguration: cfg
             )
+            service.directory = rootURL
 
 //            // hardcoded loading
 //            _ = service.addProfile(ProviderConnectionProfile(name: .pia), credentials: nil)
@@ -79,6 +83,7 @@ class TransientStore {
     }
     
     func serialize() {
-        try? JSONEncoder().encode(service).write(to: servicePath)
+        try? JSONEncoder().encode(service).write(to: serviceURL)
+        try? service.saveProfiles()
     }
 }