Merge pull request #12 from keeshux/split-profiles-serialization

Split profiles serialization

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - Host parameters are read-only if there isn't an original configuration to revert to.
+- Overall serialization performance.
 
 ## 1.0 beta 1084 (2018-10-24)
 

Passepartout-iOS/Cells/FieldTableViewCell.swift

@@ -60,6 +60,14 @@ class FieldTableViewCell: UITableViewCell {
         }
     }
     
+    var allowedCharset: CharacterSet? {
+        didSet {
+            illegalCharset = allowedCharset?.inverted
+        }
+    }
+
+    private var illegalCharset: CharacterSet?
+
     private(set) lazy var field = UITextField()
     
     weak var delegate: FieldTableViewCellDelegate?
@@ -96,6 +104,16 @@ class FieldTableViewCell: UITableViewCell {
 }
 
 extension FieldTableViewCell: UITextFieldDelegate {
+    func textField(_ textField: UITextField, shouldChangeCharactersIn range: NSRange, replacementString string: String) -> Bool {
+        guard let illegalCharset = illegalCharset else {
+            return true
+        }
+        guard string.rangeOfCharacter(from: illegalCharset) == nil else {
+            return false
+        }
+        return true
+    }
+    
     func textFieldDidEndEditing(_ textField: UITextField) {
         delegate?.fieldCellDidEdit(self)
     }

Passepartout-iOS/Scenes/Organizer/OrganizerViewController.swift

@@ -30,9 +30,9 @@ import UIKit
 class OrganizerViewController: UITableViewController, TableModelHost {
     private let service = TransientStore.shared.service
     
-    private var providerProfiles: [ProviderConnectionProfile] = []
+    private var providers: [String] = []
 
-    private var hostProfiles: [HostConnectionProfile] = []
+    private var hosts: [String] = []
     
     private var availableProviderNames: [Infrastructure.Name]?
 
@@ -56,29 +56,16 @@ class OrganizerViewController: UITableViewController, TableModelHost {
     }()
     
     func reloadModel() {
-        providerProfiles.removeAll()
+        providers = service.ids(forContext: .provider).sorted()
-        hostProfiles.removeAll()
+        hosts = service.ids(forContext: .host).sorted()
         
-        service.profileIds().forEach {
+        var providerRows = [RowType](repeating: .profile, count: providers.count)
-            let profile = service.profile(withId: $0)
+        var hostRows = [RowType](repeating: .profile, count: hosts.count)
-            if let p = profile as? ProviderConnectionProfile {
+        providerRows.append(.addProvider)
-                providerProfiles.append(p)
+        hostRows.append(.addHost)
-            } else if let p = profile as? HostConnectionProfile {
-                hostProfiles.append(p)
-            } else {
-                fatalError("Unexpected profile type \(type(of: profile))")
-            }
-        }
-        providerProfiles.sort { $0.name.rawValue < $1.name.rawValue }
-        hostProfiles.sort { $0.title < $1.title }
         
-        var providers = [RowType](repeating: .profile, count: providerProfiles.count)
+        model.set(providerRows, in: .providers)
-        var hosts = [RowType](repeating: .profile, count: hostProfiles.count)
+        model.set(hostRows, in: .hosts)
-        providers.append(.addProvider)
-        hosts.append(.addHost)
-        
-        model.set(providers, in: .providers)
-        model.set(hosts, in: .hosts)
     }
     
     // MARK: UIViewController
@@ -163,7 +150,13 @@ class OrganizerViewController: UITableViewController, TableModelHost {
     
     private func addNewProvider() {
         var names = Set(InfrastructureFactory.shared.allNames)
-        let createdNames = providerProfiles.map { $0.name }
+        var createdNames: [Infrastructure.Name] = []
+        providers.forEach {
+            guard let name = Infrastructure.Name(rawValue: $0) else {
+                return
+            }
+            createdNames.append(name)
+        }
         names.formSymmetricDifference(createdNames)
 
         guard !names.isEmpty else {
@@ -191,13 +184,13 @@ class OrganizerViewController: UITableViewController, TableModelHost {
 
     private func removeProfile(at indexPath: IndexPath) {
         let sectionObject = model.section(for: indexPath.section)
-        let rowProfile = profile(at: indexPath)
+        let rowProfile = profileKey(at: indexPath)
         switch sectionObject {
         case .providers:
-            providerProfiles.remove(at: indexPath.row)
+            providers.remove(at: indexPath.row)
             
         case .hosts:
-            hostProfiles.remove(at: indexPath.row)
+            hosts.remove(at: indexPath.row)
             
         default:
             return
@@ -205,7 +198,7 @@ class OrganizerViewController: UITableViewController, TableModelHost {
         
 //        var fallbackSection: SectionType?
         
-        let total = providerProfiles.count + hostProfiles.count
+        let total = providers.count + hosts.count
         
         // removed all profiles
         if total == 0 {
@@ -280,15 +273,20 @@ extension OrganizerViewController {
     }
     
     private var selectedIndexPath: IndexPath? {
-        guard let active = service.activeProfile?.id else {
+        guard let active = service.activeProfileKey else {
             return nil
         }
-        if let row = providerProfiles.index(where: { $0.id == active }) {
+        switch active.context {
+        case .provider:
+            if let row = providers.index(where: { $0 == active.id }) {
                 return IndexPath(row: row, section: 0)
             }
-        if let row = hostProfiles.index(where: { $0.id == active }) {
+
+        case .host:
+            if let row = hosts.index(where: { $0 == active.id }) {
                 return IndexPath(row: row, section: 1)
             }
+        }
         return nil
     }
 
@@ -312,8 +310,8 @@ extension OrganizerViewController {
         switch model.row(at: indexPath) {
         case .profile:
             let cell = Cells.setting.dequeue(from: tableView, for: indexPath)
-            let rowProfile = profile(at: indexPath)
+            let rowProfile = profileKey(at: indexPath)
-            cell.leftText = rowProfile.title
+            cell.leftText = rowProfile.id
             cell.rightText = service.isActiveProfile(rowProfile) ? L10n.Organizer.Cells.Profile.Value.current : nil
             return cell
 
@@ -375,27 +373,57 @@ extension OrganizerViewController {
     
     // MARK: Helpers
     
-    private func sectionProfiles(at indexPath: IndexPath) -> [ConnectionProfile] {
+    private func sectionProfiles(at indexPath: IndexPath) -> [String] {
-        let sectionProfiles: [ConnectionProfile]
+        let ids: [String]
         let sectionObject = model.section(for: indexPath.section)
         switch sectionObject {
         case .providers:
-            sectionProfiles = providerProfiles
+            ids = providers
             
         case .hosts:
-            sectionProfiles = hostProfiles
+            ids = hosts
             
         default:
             fatalError("Unexpected section: \(sectionObject)")
         }
-        guard indexPath.row < sectionProfiles.count else {
+        guard indexPath.row < ids.count else {
             fatalError("No profile found at \(indexPath), is it an add cell?")
         }
-        return sectionProfiles
+        return ids
+    }
+    
+    private func profileKey(at indexPath: IndexPath) -> ConnectionService.ProfileKey {
+        let section = model.section(for: indexPath.section)
+        switch section {
+        case .providers:
+            return ConnectionService.ProfileKey(.provider, providers[indexPath.row])
+            
+        case .hosts:
+            return ConnectionService.ProfileKey(.host, hosts[indexPath.row])
+            
+        default:
+            fatalError("Profile found in unexpected section: \(section)")
+        }
     }
 
     private func profile(at indexPath: IndexPath) -> ConnectionProfile {
-        return sectionProfiles(at: indexPath)[indexPath.row]
+        let id = sectionProfiles(at: indexPath)[indexPath.row]
+        let section = model.section(for: indexPath.section)
+        let context: Context
+        switch section {
+        case .providers:
+            context = .provider
+            
+        case .hosts:
+            context = .host
+            
+        default:
+            fatalError("Profile found in unexpected section: \(section)")
+        }
+        guard let found = service.profile(withContext: context, id: id) else {
+            fatalError("Profile (\(context), \(id)) could not be found, why was it returned?")
+        }
+        return found
     }
 }
 

Passepartout-iOS/Scenes/Organizer/WizardHostViewController.swift

@@ -34,7 +34,8 @@ class WizardHostViewController: UITableViewController, TableModelHost, Wizard {
         let url: URL
 
         var filename: String {
-            return url.deletingPathExtension().lastPathComponent
+            let raw = url.deletingPathExtension().lastPathComponent
+            return raw.components(separatedBy: AppConstants.Store.filenameCharset.inverted).joined(separator: "_")
         }
 
         let hostname: String
@@ -44,17 +45,8 @@ class WizardHostViewController: UITableViewController, TableModelHost, Wizard {
 
     @IBOutlet private weak var itemNext: UIBarButtonItem!
     
-    private let existingHosts: [HostConnectionProfile] = {
+    private let existingHosts: [String] = {
-        var hosts: [HostConnectionProfile] = []
+        return TransientStore.shared.service.ids(forContext: .host).sorted()
-        let service = TransientStore.shared.service
-        let ids = service.profileIds()
-        for id in ids {
-            guard let host = service.profile(withId: id) as? HostConnectionProfile else {
-                continue
-            }
-            hosts.append(host)
-        }
-        return hosts.sorted { $0.title < $1.title }
     }()
     
     private var parsedFile: ParsedFile? {
@@ -224,6 +216,7 @@ extension WizardHostViewController {
             let cell = Cells.field.dequeue(from: tableView, for: indexPath)
             cell.caption = L10n.Wizards.Host.Cells.TitleInput.caption
             cell.captionWidth = 100.0
+            cell.allowedCharset = AppConstants.Store.filenameCharset
             cell.field.placeholder = L10n.Wizards.Host.Cells.TitleInput.placeholder
             cell.field.clearButtonMode = .always
             cell.field.returnKeyType = .done
@@ -231,10 +224,9 @@ extension WizardHostViewController {
             return cell
             
         case .existingHost:
-            let profile = existingHosts[indexPath.row]
+            let hostTitle = existingHosts[indexPath.row]
-            
             let cell = Cells.setting.dequeue(from: tableView, for: indexPath)
-            cell.leftText = profile.title
+            cell.leftText = hostTitle
             cell.accessoryType = .none
             cell.isTappable = true
             return cell
@@ -247,9 +239,9 @@ extension WizardHostViewController {
             guard let titleIndexPath = model.indexPath(row: .titleInput, section: .meta) else {
                 fatalError("Could not found title cell?")
             }
-            let profile = existingHosts[indexPath.row]
+            let hostTitle = existingHosts[indexPath.row]
             let cellTitle = tableView.cellForRow(at: titleIndexPath) as? FieldTableViewCell
-            cellTitle?.field.text = profile.title
+            cellTitle?.field.text = hostTitle
             tableView.deselectRow(at: indexPath, animated: true)
             
         default:

Passepartout-iOS/Scenes/ServiceViewController.swift

@@ -37,7 +37,7 @@ class ServiceViewController: UIViewController, TableModelHost {
     
     var profile: ConnectionProfile? {
         didSet {
-            title = profile?.title
+            title = profile?.id
             reloadModel()
             updateViewsIfNeeded()
         }
@@ -78,7 +78,7 @@ class ServiceViewController: UIViewController, TableModelHost {
             lastInfrastructureUpdate = InfrastructureFactory.shared.modificationDate(for: providerProfile.name)
         }
 
-        title = profile?.title
+        title = profile?.id
         navigationItem.leftBarButtonItem = splitViewController?.displayModeButtonItem
         navigationItem.leftItemsSupplementBackButton = true
 

Passepartout.xcodeproj/project.pbxproj

@@ -58,7 +58,6 @@
 		0EBE3AA1213DC1A100BFA2F5 /* ConnectionService.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0EBE3A9F213DC1A100BFA2F5 /* ConnectionService.swift */; };
 		0EBE3AA5213DC1B000BFA2F5 /* HostConnectionProfile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0EBE3AA3213DC1B000BFA2F5 /* HostConnectionProfile.swift */; };
 		0EBE3AA6213DC1B000BFA2F5 /* ProviderConnectionProfile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0EBE3AA4213DC1B000BFA2F5 /* ProviderConnectionProfile.swift */; };
-		0EBE3AAC213DEB8800BFA2F5 /* ConnectionProfileHolder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0EBE3AAB213DEB8800BFA2F5 /* ConnectionProfileHolder.swift */; };
 		0EC7F20520E24308004EA58E /* DebugLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0EC7F20420E24308004EA58E /* DebugLog.swift */; };
 		0ECEE44E20E1122200A6BB43 /* TableModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0ECEE44D20E1122200A6BB43 /* TableModel.swift */; };
 		0ECEE45020E1182E00A6BB43 /* Theme+Cells.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0ECEE44F20E1182E00A6BB43 /* Theme+Cells.swift */; };
@@ -178,7 +177,6 @@
 		0EBE3A9F213DC1A100BFA2F5 /* ConnectionService.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ConnectionService.swift; sourceTree = "<group>"; };
 		0EBE3AA3213DC1B000BFA2F5 /* HostConnectionProfile.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = HostConnectionProfile.swift; sourceTree = "<group>"; };
 		0EBE3AA4213DC1B000BFA2F5 /* ProviderConnectionProfile.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ProviderConnectionProfile.swift; sourceTree = "<group>"; };
-		0EBE3AAB213DEB8800BFA2F5 /* ConnectionProfileHolder.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ConnectionProfileHolder.swift; sourceTree = "<group>"; };
 		0EC7F20420E24308004EA58E /* DebugLog.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DebugLog.swift; sourceTree = "<group>"; };
 		0ECEE44D20E1122200A6BB43 /* TableModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TableModel.swift; sourceTree = "<group>"; };
 		0ECEE44F20E1182E00A6BB43 /* Theme+Cells.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Theme+Cells.swift"; sourceTree = "<group>"; };
@@ -383,7 +381,6 @@
 			children = (
 				0EBE3AA2213DC1B000BFA2F5 /* Profiles */,
 				0EBE3A9E213DC1A100BFA2F5 /* ConnectionProfile.swift */,
-				0EBE3AAB213DEB8800BFA2F5 /* ConnectionProfileHolder.swift */,
 				0EBE3A9F213DC1A100BFA2F5 /* ConnectionService.swift */,
 				0EBBE8F42182361700106008 /* ConnectionService+Migration.swift */,
 				0EDE8DE620C93945004C739C /* Credentials.swift */,
@@ -851,7 +848,6 @@
 				0E89DFC8213E8FC500741BA1 /* SessionProxy+Communication.swift in Sources */,
 				0ED38AEA214054A50004D387 /* OptionViewController.swift in Sources */,
 				0EFD943E215BE10800529B64 /* IssueReporter.swift in Sources */,
-				0EBE3AAC213DEB8800BFA2F5 /* ConnectionProfileHolder.swift in Sources */,
 				0EB60FDA2111136E00AD27F3 /* UITextView+Search.swift in Sources */,
 				0E57F63E20C83FC5008323CF /* ServiceViewController.swift in Sources */,
 				0E39BCF0214B9EF10035E9DE /* WebServices.swift in Sources */,

Passepartout/Sources/AppConstants.swift

@@ -37,7 +37,19 @@ class AppConstants {
         
         static let infrastructureCacheDirectory = "Infrastructures"
 
-        static let profileConfigurationsDirectory = "Configurations"
+        static let providersDirectory = "Providers"
+
+        static let hostsDirectory = "Hosts"
+        
+        static let filenameCharset: CharacterSet = {
+            var chars: CharacterSet = .decimalDigits
+            let english = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+            let symbols = "-_"
+            chars.formUnion(CharacterSet(charactersIn: english))
+            chars.formUnion(CharacterSet(charactersIn: english.lowercased()))
+            chars.formUnion(CharacterSet(charactersIn: symbols))
+            return chars
+        }()
     }
     
     class VPN {

Passepartout/Sources/Model/ConnectionProfile.swift

@@ -27,10 +27,16 @@ import Foundation
 import TunnelKit
 import NetworkExtension
 
-protocol ConnectionProfile: class, EndpointDataSource {
+enum Context: String, Codable {
-    var id: String { get }
+    case provider
     
-    var title: String { get }
+    case host
+}
+
+protocol ConnectionProfile: class, EndpointDataSource {
+    var context: Context { get }
+    
+    var id: String { get }
     
     var username: String? { get set }
     
@@ -44,7 +50,7 @@ extension ConnectionProfile {
         guard let username = username else {
             return nil
         }
-        return "\(Bundle.main.bundleIdentifier!).\(id).\(username)"
+        return "\(Bundle.main.bundleIdentifier!).\(context.rawValue).\(id).\(username)"
     }
 
     func password(in keychain: Keychain) -> String? {

Passepartout/Sources/Model/ConnectionProfileHolder.swift

@@ -1,59 +0,0 @@
-//
-//  ConnectionProfileHolder.swift
-//  Passepartout
-//
-//  Created by Davide De Rosa on 9/3/18.
-//  Copyright (c) 2018 Davide De Rosa. All rights reserved.
-//
-//  https://github.com/keeshux
-//
-//  This file is part of Passepartout.
-//
-//  Passepartout is free software: you can redistribute it and/or modify
-//  it under the terms of the GNU General Public License as published by
-//  the Free Software Foundation, either version 3 of the License, or
-//  (at your option) any later version.
-//
-//  Passepartout is distributed in the hope that it will be useful,
-//  but WITHOUT ANY WARRANTY; without even the implied warranty of
-//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-//  GNU General Public License for more details.
-//
-//  You should have received a copy of the GNU General Public License
-//  along with Passepartout.  If not, see <http://www.gnu.org/licenses/>.
-//
-
-import Foundation
-import TunnelKit
-
-class ConnectionProfileHolder: Codable {
-    private let provider: ProviderConnectionProfile?
-    
-    private let host: HostConnectionProfile?
-    
-    convenience init(_ profile: ConnectionProfile) {
-        if let p = profile as? ProviderConnectionProfile {
-            self.init(p)
-        } else if let p = profile as? HostConnectionProfile {
-            self.init(p)
-        } else {
-            fatalError("Unexpected ConnectionProfile subtype: \(type(of: profile))")
-        }
-    }
-    
-    init(_ provider: ProviderConnectionProfile) {
-        self.provider = provider
-        host = nil
-    }
-
-    init(_ host: HostConnectionProfile) {
-        provider = nil
-        self.host = host
-    }
-
-    var contained: ConnectionProfile? {
-        let found: ConnectionProfile? = provider ?? host
-        assert(found != nil, "Either provider or host must be non-nil")
-        return found
-    }
-}

Passepartout/Sources/Model/ConnectionService+Migration.swift

@@ -46,19 +46,25 @@ extension ConnectionService {
         }
 
         // replace migration logic here
+        // TODO: remove this code after 1.0 release
         let build = json["build"] as? Int ?? 0
         if build <= 1084 {
             try migrateToWrappedSessionConfiguration(&json)
             try migrateToBaseConfiguration(&json)
             try migrateToBuildNumber(&json)
+            try migrateHostProfileConfigurations()
+            try migrateSplitProfileSerialization(&json)
         }
 
         return try JSONSerialization.data(withJSONObject: json, options: [])
     }
     
+    // MARK: Atomic migrations
+
     static func migrateToWrappedSessionConfiguration(_ json: inout [String: Any]) throws {
         guard let profiles = json["profiles"] as? [[String: Any]] else {
-            throw ApplicationError.migration
+            // migrated
+            return
         }
         var newProfiles: [[String: Any]] = []
         for var container in profiles {
@@ -83,6 +89,7 @@ extension ConnectionService {
     
     static func migrateToBaseConfiguration(_ json: inout [String: Any]) throws {
         guard var baseConfiguration = json["tunnelConfiguration"] as? [String: Any] else {
+            // migrated
             return
         }
         migrateSessionConfiguration(in: &baseConfiguration)
@@ -94,6 +101,74 @@ extension ConnectionService {
         json["build"] = GroupConstants.App.buildNumber
     }
 
+    static func migrateHostProfileConfigurations() throws {
+        let fm = FileManager.default
+        let oldDirectory = fm.userURL(for: .documentDirectory, appending: "Configurations")
+        guard fm.fileExists(atPath: oldDirectory.path) else {
+            return
+        }
+        
+        let newDirectory = fm.userURL(for: .documentDirectory, appending: AppConstants.Store.hostsDirectory)
+        try fm.moveItem(at: oldDirectory, to: newDirectory)
+        let list = try fm.contentsOfDirectory(at: newDirectory, includingPropertiesForKeys: nil, options: [])
+        let prefix = "host."
+        for url in list {
+            let filename = url.lastPathComponent
+            guard filename.hasPrefix(prefix) else {
+                continue
+            }
+            let postPrefixIndex = filename.index(filename.startIndex, offsetBy: prefix.count)
+            let newFilename = String(filename[postPrefixIndex..<filename.endIndex])
+            var newURL = url
+            newURL.deleteLastPathComponent()
+            newURL.appendPathComponent(newFilename)
+            try fm.moveItem(at: url, to: newURL)
+        }
+    }
+    
+    static func migrateSplitProfileSerialization(_ json: inout [String: Any]) throws {
+        guard let profiles = json["profiles"] as? [[String: Any]] else {
+            return
+        }
+
+        let fm = FileManager.default
+        let providersParentURL = fm.userURL(for: .documentDirectory, appending: AppConstants.Store.providersDirectory)
+        let hostsParentURL = fm.userURL(for: .documentDirectory, appending: AppConstants.Store.hostsDirectory)
+        try? fm.createDirectory(at: providersParentURL, withIntermediateDirectories: false, attributes: nil)
+        try? fm.createDirectory(at: hostsParentURL, withIntermediateDirectories: false, attributes: nil)
+
+        for p in profiles {
+            if var provider = p["provider"] as? [String: Any] {
+                guard let id = provider["name"] as? String else {
+                    continue
+                }
+//                provider["id"] = id
+//                provider.removeValue(forKey: "name")
+        
+                let url = providersParentURL.appendingPathComponent("\(id).json")
+                let data = try JSONSerialization.data(withJSONObject: provider, options: [])
+                try data.write(to: url)
+            } else if var host = p["host"] as? [String: Any] {
+                guard let id = host["title"] as? String else {
+                    continue
+                }
+//                host["id"] = id
+//                host.removeValue(forKey: "title")
+
+                let url = hostsParentURL.appendingPathComponent("\(id).json")
+                let data = try JSONSerialization.data(withJSONObject: host, options: [])
+                try data.write(to: url)
+            }
+        }
+        
+        guard let activeProfileId = json["activeProfileId"] else {
+            return
+        }
+        json["activeProfileKey"] = activeProfileId
+        json.removeValue(forKey: "activeProfileId")
+        json.removeValue(forKey: "profiles")
+    }
+    
     // MARK: Helpers
     
     private static func migrateSessionConfiguration(in map: inout [String: Any]) {
@@ -124,5 +199,8 @@ extension ConnectionService {
             sessionConfiguration["renegotiatesAfter"] = value
         }
         map["sessionConfiguration"] = sessionConfiguration
+
+        map.removeValue(forKey: "debugLogKey")
+        map.removeValue(forKey: "lastErrorKey")
     }
 }

Passepartout/Sources/Model/ConnectionService.swift

@@ -44,13 +44,71 @@ class ConnectionService: Codable {
         
         case baseConfiguration
         
-        case profiles
+        case activeProfileKey
-        
-        case activeProfileId
         
         case preferences
     }
 
+    struct ProfileKey: RawRepresentable, Hashable, Codable {
+        let context: Context
+        
+        let id: String
+
+        init(_ context: Context, _ id: String) {
+            self.context = context
+            self.id = id
+        }
+
+        init(_ profile: ConnectionProfile) {
+            context = profile.context
+            id = profile.id
+        }
+        
+        fileprivate func profileURL(in service: ConnectionService) -> URL {
+            let contextURL: URL
+            switch context {
+            case .provider:
+                contextURL = service.providersURL
+                
+            case .host:
+                contextURL = service.hostsURL
+            }
+            return ConnectionService.url(in: contextURL, forProfileId: id)
+        }
+
+        fileprivate func profileData(in service: ConnectionService) throws -> Data {
+            return try Data(contentsOf: profileURL(in: service))
+        }
+
+        // MARK: RawRepresentable
+        
+        var rawValue: String {
+            return "\(context).\(id)"
+        }
+        
+        init?(rawValue: String) {
+            let comps = rawValue.components(separatedBy: ".")
+            guard comps.count == 2 else {
+                return nil
+            }
+            guard let context = Context(rawValue: comps[0]) else {
+                return nil
+            }
+            self.context = context
+            id = comps[1]
+        }
+    }
+
+    lazy var directory = FileManager.default.userURL(for: .documentDirectory, appending: nil)
+    
+    private var providersURL: URL {
+        return directory.appendingPathComponent(AppConstants.Store.providersDirectory)
+    }
+
+    private var hostsURL: URL {
+        return directory.appendingPathComponent(AppConstants.Store.hostsDirectory)
+    }
+    
     private var build: Int
     
     private let appGroup: String
@@ -61,9 +119,11 @@ class ConnectionService: Codable {
     
     var baseConfiguration: TunnelKitProvider.Configuration
     
-    private var profiles: [String: ConnectionProfile]
+    private var cache: [ProfileKey: ConnectionProfile]
     
-    private var activeProfileId: String? {
+    private var pendingRemoval: Set<ProfileKey>
+    
+    private(set) var activeProfileKey: ProfileKey? {
         willSet {
             if let oldProfile = activeProfile {
                 delegate?.connectionService(didDeactivate: oldProfile)
@@ -77,10 +137,15 @@ class ConnectionService: Codable {
     }
     
     var activeProfile: ConnectionProfile? {
-        guard let id = activeProfileId else {
+        guard let id = activeProfileKey else {
             return nil
         }
-        return profiles[id]
+        var hit = cache[id]
+        if let placeholder = hit as? PlaceholderConnectionProfile {
+            hit = profile(withContext: placeholder.context, id: placeholder.id)
+            cache[id] = hit
+        }
+        return hit
     }
     
     let preferences: EditablePreferences
@@ -97,9 +162,11 @@ class ConnectionService: Codable {
         keychain = Keychain(group: appGroup)
 
         self.baseConfiguration = baseConfiguration
-        profiles = [:]
+        activeProfileKey = nil
-        activeProfileId = nil
         preferences = EditablePreferences()
+
+        cache = [:]
+        pendingRemoval = []
     }
     
     // MARK: Codable
@@ -116,17 +183,11 @@ class ConnectionService: Codable {
         keychain = Keychain(group: appGroup)
 
         baseConfiguration = try container.decode(TunnelKitProvider.Configuration.self, forKey: .baseConfiguration)
-        let profilesArray = try container.decode([ConnectionProfileHolder].self, forKey: .profiles).map { $0.contained }
+        activeProfileKey = try container.decodeIfPresent(ProfileKey.self, forKey: .activeProfileKey)
-        var profiles: [String: ConnectionProfile] = [:]
-        profilesArray.forEach {
-            guard let p = $0 else {
-                return
-            }
-            profiles[p.id] = p
-        }
-        self.profiles = profiles
-        activeProfileId = try container.decodeIfPresent(String.self, forKey: .activeProfileId)
         preferences = try container.decode(EditablePreferences.self, forKey: .preferences)
+
+        cache = [:]
+        pendingRemoval = []
     }
     
     func encode(to encoder: Encoder) throws {
@@ -136,23 +197,126 @@ class ConnectionService: Codable {
         try container.encode(build, forKey: .build)
         try container.encode(appGroup, forKey: .appGroup)
         try container.encode(baseConfiguration, forKey: .baseConfiguration)
-        try container.encode(profiles.map { ConnectionProfileHolder($0.value) }, forKey: .profiles)
+        try container.encodeIfPresent(activeProfileKey, forKey: .activeProfileKey)
-        try container.encodeIfPresent(activeProfileId, forKey: .activeProfileId)
         try container.encode(preferences, forKey: .preferences)
     }
     
+    // MARK: Serialization
+    
+    func loadProfiles() {
+        let fm = FileManager.default
+        try? fm.createDirectory(at: providersURL, withIntermediateDirectories: false, attributes: nil)
+        try? fm.createDirectory(at: hostsURL, withIntermediateDirectories: false, attributes: nil)
+        
+        do {
+            let files = try fm.contentsOfDirectory(at: providersURL, includingPropertiesForKeys: nil, options: [])
+//            log.debug("Found \(files.count) provider files: \(files)")
+            for entry in files {
+                guard let id = ConnectionService.profileId(fromURL: entry) else {
+                    return
+                }
+                let key = ProfileKey(.provider, id)
+                cache[key] = PlaceholderConnectionProfile(key)
+            }
+        } catch let e {
+            log.warning("Could not list provider contents: \(e) (\(providersURL))")
+        }
+        do {
+            let files = try fm.contentsOfDirectory(at: hostsURL, includingPropertiesForKeys: nil, options: [])
+//            log.debug("Found \(files.count) host files: \(files)")
+            for entry in files {
+                guard let id = ConnectionService.profileId(fromURL: entry) else {
+                    continue
+                }
+                let key = ProfileKey(.host, id)
+                cache[key] = PlaceholderConnectionProfile(key)
+            }
+        } catch let e {
+            log.warning("Could not list host contents: \(e) (\(hostsURL))")
+        }
+    }
+    
+    func saveProfiles() throws {
+        let encoder = JSONEncoder()
+
+        let fm = FileManager.default
+        try? fm.createDirectory(at: providersURL, withIntermediateDirectories: false, attributes: nil)
+        try? fm.createDirectory(at: hostsURL, withIntermediateDirectories: false, attributes: nil)
+
+        for key in pendingRemoval {
+            let url = key.profileURL(in: self)
+            try? fm.removeItem(at: url)
+        }
+        for entry in cache.values {
+            if let profile = entry as? ProviderConnectionProfile {
+                do {
+                    let url = ConnectionService.url(in: providersURL, forProfileId: entry.id)
+                    let data = try encoder.encode(profile)
+                    try data.write(to: url)
+                    log.debug("Saved provider '\(profile.id)'")
+                } catch let e {
+                    log.warning("Could not save provider '\(profile.id)': \(e)")
+                    continue
+                }
+            } else if let profile = entry as? HostConnectionProfile {
+                do {
+                    let url = ConnectionService.url(in: hostsURL, forProfileId: entry.id)
+                    let data = try encoder.encode(profile)
+                    try data.write(to: url)
+                    log.debug("Saved host '\(profile.id)'")
+                } catch let e {
+                    log.warning("Could not save host '\(profile.id)': \(e)")
+                    continue
+                }
+            } else if let placeholder = entry as? PlaceholderConnectionProfile {
+                log.debug("Skipped \(placeholder.context) '\(placeholder.id)'")
+            }
+        }
+    }
+    
+    func profile(withContext context: Context, id: String) -> ConnectionProfile? {
+        let key = ProfileKey(context, id)
+        var profile = cache[key]
+        if let _ = profile as? PlaceholderConnectionProfile {
+            let decoder = JSONDecoder()
+            do {
+                let data = try key.profileData(in: self)
+                switch context {
+                case .provider:
+                    profile = try decoder.decode(ProviderConnectionProfile.self, from: data)
+                    
+                case .host:
+                    profile = try decoder.decode(HostConnectionProfile.self, from: data)
+                }
+                cache[key] = profile
+            } catch let e {
+                log.warning("Could not decode profile JSON: \(e)")
+                return nil
+            }
+        }
+        return profile
+    }
+    
+    func ids(forContext context: Context) -> [String] {
+        return cache.keys.filter { $0.context == context }.map { $0.id }
+    }
+    
+    private static func profileId(fromURL url: URL) -> String? {
+        let filename = url.lastPathComponent
+        guard let extRange = filename.range(of: ".json") else {
+            return nil
+        }
+        return String(filename[filename.startIndex..<extRange.lowerBound])
+    }
+    
+    private static func url(in directory: URL, forProfileId profileId: String) -> URL {
+        return directory.appendingPathComponent("\(profileId).json")
+    }
+    
     // MARK: Profiles
 
-    func profileIds() -> [String] {
-        return Array(profiles.keys)
-    }
-    
-    func profile(withId id: String) -> ConnectionProfile? {
-        return profiles[id]
-    }
-    
     func addProfile(_ profile: ConnectionProfile, credentials: Credentials?) -> Bool {
-        guard profiles.index(forKey: profile.id) == nil else {
+        guard cache.index(forKey: ProfileKey(profile)) == nil else {
             return false
         }
         addOrReplaceProfile(profile, credentials: credentials)
@@ -160,37 +324,51 @@ class ConnectionService: Codable {
     }
     
     func addOrReplaceProfile(_ profile: ConnectionProfile, credentials: Credentials?) {
-        profiles[profile.id] = profile
+        let key = ProfileKey(profile)
+        cache[key] = profile
+        pendingRemoval.remove(key)
         try? setCredentials(credentials, for: profile)
-        if profiles.count == 1 {
+        if cache.count == 1 {
-            activeProfileId = profile.id
+            activeProfileKey = key
+        }
+
+        // serialize immediately
+        try? saveProfiles()
+    }
+    
+    func removeProfile(_ key: ProfileKey) {
+        guard let i = cache.index(forKey: key) else {
+            return
+        }
+        cache.remove(at: i)
+        pendingRemoval.insert(key)
+        if cache.isEmpty {
+            activeProfileKey = nil
         }
     }
     
-    func removeProfile(_ profile: ConnectionProfile) {
+    func containsProfile(_ key: ProfileKey) -> Bool {
-        guard let i = profiles.index(forKey: profile.id) else {
+        return cache.index(forKey: key) != nil
-            return
-        }
-        profiles.remove(at: i)
-        if profiles.isEmpty {
-            activeProfileId = nil
-        }
     }
 
     func containsProfile(_ profile: ConnectionProfile) -> Bool {
-        return profiles.index(forKey: profile.id) != nil
+        return containsProfile(ProfileKey(profile))
     }
     
     func hasActiveProfile() -> Bool {
-        return activeProfileId != nil
+        return activeProfileKey != nil
+    }
+
+    func isActiveProfile(_ key: ProfileKey) -> Bool {
+        return key == activeProfileKey
     }
     
     func isActiveProfile(_ profile: ConnectionProfile) -> Bool {
-        return profile.id == activeProfileId
+        return isActiveProfile(ProfileKey(profile))
     }
     
     func activateProfile(_ profile: ConnectionProfile) {
-        activeProfileId = profile.id
+        activeProfileKey = ProfileKey(profile)
     }
     
     // MARK: Credentials
@@ -291,3 +469,34 @@ class ConnectionService: Codable {
 //        defaults.removeObject(forKey: Keys.vpnLog)
 //    }
 }
+
+private class PlaceholderConnectionProfile: ConnectionProfile {
+    let context: Context
+    
+    let id: String
+    
+    var username: String?
+    
+    var requiresCredentials: Bool = false
+    
+    func generate(from configuration: TunnelKitProvider.Configuration, preferences: Preferences) throws -> TunnelKitProvider.Configuration {
+        fatalError("Generating configuration from a PlaceholderConnectionProfile")
+    }
+    
+    var mainAddress: String = ""
+    
+    var addresses: [String] = []
+    
+    var protocols: [TunnelKitProvider.EndpointProtocol] = []
+    
+    var canCustomizeEndpoint: Bool = false
+    
+    var customAddress: String?
+    
+    var customProtocol: TunnelKitProvider.EndpointProtocol?
+
+    init(_ key: ConnectionService.ProfileKey) {
+        self.context = key.context
+        self.id = key.id
+    }
+}

Passepartout/Sources/Model/ProfileConfigurationFactory.swift

@@ -25,28 +25,50 @@
 
 import Foundation
 
-class ProfileConfigurationFactory {
+protocol ProfileConfigurationSource {
-    static let shared = ProfileConfigurationFactory(withDirectory: AppConstants.Store.profileConfigurationsDirectory)
+    var id: String { get }
 
-    private let cachePath: URL
+    var profileDirectory: String { get }
+}
+
+extension ProfileConfigurationSource {
+    var profileConfigurationPath: String {
+        return "\(profileDirectory)/\(id).ovpn"
+    }
+}
+
+extension ProviderConnectionProfile: ProfileConfigurationSource {
+    var profileDirectory: String {
+        return AppConstants.Store.providersDirectory
+    }
+}
+
+extension HostConnectionProfile: ProfileConfigurationSource {
+    var profileDirectory: String {
+        return AppConstants.Store.hostsDirectory
+    }
+}
+
+class ProfileConfigurationFactory {
+    static let shared = ProfileConfigurationFactory()
     
     private let configurationsPath: URL
 
-    private init(withDirectory directory: String) {
+    private init() {
         let fm = FileManager.default
-        cachePath = fm.userURL(for: .cachesDirectory, appending: directory)
+        configurationsPath = fm.userURL(for: .documentDirectory, appending: nil)
-        configurationsPath = fm.userURL(for: .documentDirectory, appending: directory)
-        try? fm.createDirectory(at: cachePath, withIntermediateDirectories: false, attributes: nil)
         try? fm.createDirectory(at: configurationsPath, withIntermediateDirectories: false, attributes: nil)
     }
 
-    func save(url: URL, for profile: ConnectionProfile) throws -> URL {
+    func save(url: URL, for profile: ProfileConfigurationSource) throws -> URL {
         let savedUrl = targetConfigurationURL(for: profile)
-        try FileManager.default.copyItem(at: url, to: savedUrl)
+        let fm = FileManager.default
+        try? fm.removeItem(at: savedUrl)
+        try fm.copyItem(at: url, to: savedUrl)
         return savedUrl
     }
     
-    func configurationURL(for profile: ConnectionProfile) -> URL? {
+    func configurationURL(for profile: ProfileConfigurationSource) -> URL? {
         let url = targetConfigurationURL(for: profile)
         guard FileManager.default.fileExists(atPath: url.path) else {
             return nil
@@ -54,8 +76,7 @@ class ProfileConfigurationFactory {
         return url
     }
 
-    private func targetConfigurationURL(for profile: ConnectionProfile) -> URL {
+    private func targetConfigurationURL(for profile: ProfileConfigurationSource) -> URL {
-        let filename = "\(profile.id).ovpn"
+        return configurationsPath.appendingPathComponent(profile.profileConfigurationPath)
-        return configurationsPath.appendingPathComponent(filename)
     }
 }

Passepartout/Sources/Model/Profiles/HostConnectionProfile.swift

@@ -27,6 +27,8 @@ import Foundation
 import TunnelKit
 
 class HostConnectionProfile: ConnectionProfile, Codable, Equatable {
+    var title: String
+
     let hostname: String
     
     var parameters: TunnelKitProvider.Configuration
@@ -40,11 +42,11 @@ class HostConnectionProfile: ConnectionProfile, Codable, Equatable {
     
     // MARK: ConnectionProfile
     
-    var id: String {
+    let context: Context = .host
-        return "host.\(title)"
-    }
     
-    var title: String
+    var id: String {
+        return title
+    }
     
     var username: String?
     

Passepartout/Sources/Model/Profiles/ProviderConnectionProfile.swift

@@ -66,7 +66,6 @@ class ProviderConnectionProfile: ConnectionProfile, Codable, Equatable {
         poolId = ""
         presetId = ""
 
-        id = "provider.\(name.rawValue)"
         username = nil
 
         poolId = infrastructure.defaults.pool
@@ -93,9 +92,9 @@ class ProviderConnectionProfile: ConnectionProfile, Codable, Equatable {
     
     // MARK: ConnectionProfile
     
-    let id: String
+    let context: Context = .provider
 
-    var title: String {
+    var id: String {
         return name.rawValue
     }
     

Passepartout/Sources/Model/TransientStore.swift

@@ -35,7 +35,9 @@ class TransientStore {
     
     static let shared = TransientStore()
     
-    private let servicePath: URL
+    private let rootURL: URL
+
+    private let serviceURL: URL
     
     let service: ConnectionService
 
@@ -49,27 +51,29 @@ class TransientStore {
     }
 
     private init() {
-        servicePath = FileManager.default.userURL(
+        rootURL = FileManager.default.userURL(for: .documentDirectory, appending: nil)
-            for: .documentDirectory,
+        serviceURL = rootURL.appendingPathComponent(AppConstants.Store.serviceFilename)
-            appending: AppConstants.Store.serviceFilename
+
-        )
         let cfg = AppConstants.VPN.baseConfiguration()
         do {
-            ConnectionService.migrateJSON(at: servicePath, to: servicePath)
+            ConnectionService.migrateJSON(at: serviceURL, to: serviceURL)
             
-            let data = try Data(contentsOf: servicePath)
+            let data = try Data(contentsOf: serviceURL)
             if let content = String(data: data, encoding: .utf8) {
                 log.verbose("Service JSON:")
                 log.verbose(content)
             }
             service = try JSONDecoder().decode(ConnectionService.self, from: data)
+            service.directory = rootURL
             service.baseConfiguration = cfg
+            service.loadProfiles()
         } catch let e {
             log.error("Could not decode service: \(e)")
             service = ConnectionService(
                 withAppGroup: GroupConstants.App.appGroup,
                 baseConfiguration: cfg
             )
+            service.directory = rootURL
 
 //            // hardcoded loading
 //            _ = service.addProfile(ProviderConnectionProfile(name: .pia), credentials: nil)
@@ -79,6 +83,7 @@ class TransientStore {
     }
     
     func serialize() {
-        try? JSONEncoder().encode(service).write(to: servicePath)
+        try? JSONEncoder().encode(service).write(to: serviceURL)
+        try? service.saveProfiles()
     }
 }

Podfile

@@ -3,7 +3,7 @@ use_frameworks!
 
 def shared_pods
     #pod 'TunnelKit', '~> 1.1.2'
-    pod 'TunnelKit', :git => 'https://github.com/keeshux/tunnelkit', :commit => 'd94733f'
+    pod 'TunnelKit', :git => 'https://github.com/keeshux/tunnelkit', :commit => '3447128'
     #pod 'TunnelKit', :path => '../tunnelkit'
 end
 

Podfile.lock

@@ -14,7 +14,7 @@ PODS:
 
 DEPENDENCIES:
   - MBProgressHUD
-  - TunnelKit (from `https://github.com/keeshux/tunnelkit`, commit `d94733f`)
+  - TunnelKit (from `https://github.com/keeshux/tunnelkit`, commit `3447128`)
 
 SPEC REPOS:
   https://github.com/cocoapods/specs.git:
@@ -24,12 +24,12 @@ SPEC REPOS:
 
 EXTERNAL SOURCES:
   TunnelKit:
-    :commit: d94733f
+    :commit: '3447128'
     :git: https://github.com/keeshux/tunnelkit
 
 CHECKOUT OPTIONS:
   TunnelKit:
-    :commit: d94733f
+    :commit: '3447128'
     :git: https://github.com/keeshux/tunnelkit
 
 SPEC CHECKSUMS:
@@ -38,6 +38,6 @@ SPEC CHECKSUMS:
   SwiftyBeaver: ccfcdf85a04d429f1633f668650b0ce8020bda3a
   TunnelKit: 8e747cac28959ebfdfa4eeab589c933f1856c0fb
 
-PODFILE CHECKSUM: 38237684ab2fdb5e262da936fd6932218abca0b4
+PODFILE CHECKSUM: 2e3ddf964a7da5d6afc6d39c26218d9af992c770
 
 COCOAPODS: 1.6.0.beta.2