Fastest way to save structs iOS / Swift

Question

I have structs like

struct RGBA: Codable {
        
   var r: UInt8
   var g: UInt8
   var b: UInt8
   var a: UInt8 
}

I want save large amount of this structs (>1_000_000)

Decode

guard let history = try? JSONDecoder().decode(HistoryRGBA.self, from: data) else { return }

Encode

guard let jsonData = try? encoder.encode(dataForSave) else { return false }

How can I improve encoding/decoding time and amount of RAM memory?

Bouilli

Do you need this to be JSON? There are dramatically more efficient ways to encode an array of 32-bit words. Also, when you say "save," how are you writing these to disk? Does the whole array change, or could you update things with random-access? Or is the data immutable once written? There are many performance enhancements available depending on the nature of the data and its access. – Sweeten

@napier I need any way to save and load large amount of structures. Save format is not important – Bouilli

The data is saved to disk. Data can be overwritten – Bouilli

Answer 1

Considering that all your properties are UInt8 (bytes) you can make your struct conform to ContiguousBytes and save its raw bytes:

struct RGBA {
   let r, g, b, a: UInt8
}

---

extension RGBA: ContiguousBytes {
    func withUnsafeBytes<R>(_ body: (UnsafeRawBufferPointer) throws -> R) rethrows -> R {
        try Swift.withUnsafeBytes(of: self) { try body($0) }
    }
}

---

extension ContiguousBytes {
    init<T: ContiguousBytes>(_ bytes: T) {
        self = bytes.withUnsafeBytes { $0.load(as: Self.self) }
    }
}

---

extension RGBA: ExpressibleByArrayLiteral {
    typealias ArrayLiteralElement = UInt8
    init(arrayLiteral elements: UInt8...) {
        self.init(elements)
    }
}

---

extension Array {
    var bytes: [UInt8] { withUnsafeBytes { .init($0) } }
    var data: Data { withUnsafeBytes { .init($0) } }
}

---

extension ContiguousBytes {
    var bytes: [UInt8] { withUnsafeBytes { .init($0) } }
    var data: Data { withUnsafeBytes { .init($0) } }
}

---

extension ContiguousBytes {
    func object<T>() -> T { withUnsafeBytes { $0.load(as: T.self) } }
    func objects<T>() -> [T] { withUnsafeBytes { .init($0.bindMemory(to: T.self)) } }
}

---

extension ContiguousBytes {
    var rgba: RGBA { object() }
    var rgbaCollection: [RGBA] { objects() }
}

---

extension UIColor {
    convenience init<T: Collection>(_ bytes: T) where T.Index == Int, T.Element == UInt8 {
        self.init(red:   CGFloat(bytes[0])/255,
                  green: CGFloat(bytes[1])/255,
                  blue:  CGFloat(bytes[2])/255,
                  alpha: CGFloat(bytes[3])/255)
    }
}

---

extension RGBA {
    var color: UIColor { .init(bytes) }
}

---

let red: RGBA = [255, 0, 0, 255]
let green: RGBA = [0, 255, 0, 255]
let blue: RGBA = [0, 0, 255, 255]

let redBytes = red.bytes            // [255, 0, 0, 255]
let redData = red.data              // 4 bytes
let rgbaFromBytes = redBytes.rgba    // RGBA
let rgbaFromData = redData.rgba      // RGBA
let colorFromRGBA = red.color       // r 1.0 g 0.0 b 0.0 a 1.0
let rgba: RGBA = [255,255,0,255]    // RGBA yellow
let yellow = rgba.color             // r 1.0 g 1.0 b 0.0 a 1.0

let colors = [red, green, blue]      // [{r 255, g 0, b 0, a 255}, {r 0, g 255, b 0, a 255}, {r 0, g 0, b 255, a 255}]
let colorsData = colors.data          // 12 bytes
let colorsFromData = colorsData.rgbaCollection // [{r 255, g 0, b 0, a 255}, {r 0, g 255, b 0, a 255}, {r 0, g 0, b 255, a 255}]

edit/update:

struct LayerRGBA {
    var canvas: [[RGBA]]
}

---

extension LayerRGBA {
    var data: Data { canvas.data }
    init(_ data: Data) { canvas = data.objects() }
}

---

struct AnimationRGBA {
    var layers: [LayerRGBA]
}

---

extension AnimationRGBA {
    var data: Data { layers.data }
    init(_ data: Data) {
        layers = data.objects()
    }
}

---

struct HistoryRGBA {
    var layers: [LayerRGBA] = []
    var animations: [AnimationRGBA] = []
}

---

extension HistoryRGBA {
    var data: Data {
        let layersData = layers.data
        return layersData.count.data + layersData + animations.data
    }
    init(data: Data)  {
        let index = Int(Data(data.prefix(8))).advanced(by: 8)
        self.init(layers: data.subdata(in: 8..<index).objects(),
                  animations: data.subdata(in: index..<data.endIndex).objects())
    }
}

---

extension Numeric {
    var data: Data {
        var bytes = self
        return .init(bytes: &bytes, count: MemoryLayout<Self>.size)
    }
}

---

extension Numeric {
    init<D: DataProtocol>(_ data: D) {
        var value: Self = .zero
        let _ = withUnsafeMutableBytes(of: &value, { data.copyBytes(to: $0)} )
        self = value
    }
}

---

Playground testing:

let layer1: LayerRGBA = .init(canvas: [colors,[red],[green, blue]])
let layer2: LayerRGBA = .init(canvas: [[red],[green, rgba]])
let loaded: LayerRGBA = .init(layer1.data)
loaded.canvas[0]
loaded.canvas[1]
loaded.canvas[2]

let animationRGBA: AnimationRGBA = .init(layers: [layer1,layer2])
let loadedAnimation: AnimationRGBA = .init(animationRGBA.data)
loadedAnimation.layers.count // 2
loadedAnimation.layers[0].canvas[0]
loadedAnimation.layers[0].canvas[1]
loadedAnimation.layers[0].canvas[2]
loadedAnimation.layers[1].canvas[0]
loadedAnimation.layers[1].canvas[1]

let hRGBA: HistoryRGBA = .init(layers: [loaded], animations: [animationRGBA])
let loadedHistory: HistoryRGBA = .init(data: hRGBA.data)
loadedHistory.layers[0].canvas[0]
loadedHistory.layers[0].canvas[1]
loadedHistory.layers[0].canvas[2]

loadedHistory.animations[0].layers[0].canvas[0]
loadedHistory.animations[0].layers[0].canvas[1]
loadedHistory.animations[0].layers[0].canvas[2]
loadedHistory.animations[0].layers[1].canvas[0]
loadedHistory.animations[0].layers[1].canvas[1]

Tupelo

If I has struct LayerRGBA: Codable { var canvas: [[RGBA]] } struct AnimationRGBA: Codable, LayersDrawable { var layers: [LayerRGBA] } struct HistoryRGBA: Codable, LayersDrawable { var layers: [LayerRGBA] = [], var animations: [AnimationRGBA] = [] } How I can encode to Data and back with your method? – Bouilli

Answer 2

The performance of JSONEncoder/Decoder performance is...not great. ZippyJSON is a drop-in replacement that is supposedly about 4 times faster than Foundation's implmenetation, and if you're going for better performance and lower memory usage, you'll probably want to Google for some kind of streaming JSON decoder library.

However, you said in the comments that you don't need the JSON format. That's great, because we can store the data much more efficiently as just an array of raw bytes rather than a text-based format such as JSON:

extension RGBA {
    static let size = 4 // the size of a (packed) RGBA structure
}

// encoding
var data = Data(count: history.rgba.count * RGBA.size)
for i in 0..<history.rgba.count {
    let rgba = history.rgba[i]
    data[i*RGBA.size] = rgba.r
    data[i*RGBA.size+1] = rgba.g
    data[i*RGBA.size+2] = rgba.b
    data[i*RGBA.size+3] = rgba.a
}


// decoding
guard data.count % RGBA.size == 0 else {
    // data is incomplete, handle error
    return
}
let rgbaCount = data.count / RGBA.size
var result = [RGBA]()
result.reserveCapacity(rgbaCount)
for i in 0..<rgbaCount {
    result.append(RGBA(r: data[i*RGBA.size],
                       g: data[i*RGBA.size+1],
                       b: data[i*RGBA.size+2],
                       a: data[i*RGBA.size+3]))
}

This is already about 50 times faster than using JSONEncoder on my machine (~100ms instead of ~5 seconds).

You can get even faster by bypassing some of Swift's safety checks and memory management and dropping down to raw pointers:

// encoding
let byteCount = history.rgba.count * RGBA.size
let rawBuf = malloc(byteCount)!
let buf = rawBuf.bindMemory(to: UInt8.self, capacity: byteCount)

for i in 0..<history.rgba.count {
    let rgba = history.rgba[i]
    buf[i*RGBA.size] = rgba.r
    buf[i*RGBA.size+1] = rgba.g
    buf[i*RGBA.size+2] = rgba.b
    buf[i*RGBA.size+3] = rgba.a
}
let data = Data(bytesNoCopy: rawBuf, count: byteCount, deallocator: .free)


// decoding
guard data.count % RGBA.size == 0 else {
    // data is incomplete, handle error
    return
}
let result: [RGBA] = data.withUnsafeBytes { rawBuf in
    let buf = rawBuf.bindMemory(to: UInt8.self)
    let rgbaCount = buf.count / RGBA.size
    return [RGBA](unsafeUninitializedCapacity: rgbaCount) { resultBuf, initializedCount in
        for i in 0..<rgbaCount {
            resultBuf[i] = RGBA(r: data[i*RGBA.size],
                                g: data[i*RGBA.size+1],
                                b: data[i*RGBA.size+2],
                                a: data[i*RGBA.size+3])
        }
    }
}

Benchmark results on my machine (I did not test ZippyJSON):

JSON:
Encode: 4967.0ms; 32280478 bytes
Decode: 5673.0ms

Data:
Encode: 96.0ms; 4000000 bytes
Decode: 19.0ms

Pointers:
Encode: 1.0ms; 4000000 bytes
Decode: 18.0ms

You could probably get even faster by just writing your array directly from memory to disk without serializing it at all, although I haven't tested that either. And of course, when you're testing performance, be sure you're testing in Release mode.

Linguistician

Answer 3

If anyone else like me was wondering if using PropertyListEncoder/Decoder or writing custom encoding/decoding methods for Codable structs can make any difference in performance then I made some tests to check it and the answer is that they can improve it a little bit compared to standard JSONEncoder/Decoder but not much. I can't really recommended this as there are far faster ways of doing it in other answers but I think it might be useful in some cases so I'm putting the results here. Using unkeyedContainer for encoding/decoding Codable made encoding about 2x faster in my tests but it had minimal impact on decoding and using PropertyListEncoder/Decoder made only minimal difference as pasted below. Test code:

struct RGBA1: Codable {
   var r: UInt8
   var g: UInt8
   var b: UInt8
   var a: UInt8
}

struct RGBA2 {
   var r: UInt8
   var g: UInt8
   var b: UInt8
   var a: UInt8
}
extension RGBA2: Codable {
    func encode(to encoder: Encoder) throws {
        var container = encoder.unkeyedContainer()
        try container.encode(r)
        try container.encode(g)
        try container.encode(b)
        try container.encode(a)
    }
    init(from decoder: Decoder) throws {
        var container = try decoder.unkeyedContainer()
        r = try container.decode(UInt8.self)
        g = try container.decode(UInt8.self)
        b = try container.decode(UInt8.self)
        a = try container.decode(UInt8.self)
    }
}

class PerformanceTests: XCTestCase {
    var rgba1: [RGBA1] = {
        var rgba1: [RGBA1] = []
        for i in 0..<1_000_000 {
            rgba1.append(RGBA1(r: UInt8(i % 256), g: UInt8(i % 256), b: UInt8(i % 256), a: UInt8(i % 256)))
        }
        return rgba1
    }()
    var rgba2: [RGBA2] = {
        var rgba2: [RGBA2] = []
        for i in 0..<1_000_000 {
            rgba2.append(RGBA2(r: UInt8(i % 256), g: UInt8(i % 256), b: UInt8(i % 256), a: UInt8(i % 256)))
        }
        return rgba2
    }()

    func testRgba1JsonEncoding() throws {
        var result: Data?
        self.measure { result = try? JSONEncoder().encode(rgba1) }
        print("rgba1 json size: \(result?.count ?? 0)")
    }
    func testRgba1JsonDecoding() throws {
        let result = try? JSONEncoder().encode(rgba1)
        self.measure { _ = try? JSONDecoder().decode([RGBA1].self, from: result!) }
    }
    func testRgba1PlistEncoding() throws {
        var result: Data?
        self.measure { result = try? PropertyListEncoder().encode(rgba1) }
        print("rgba1 plist size: \(result?.count ?? 0)")
    }
    func testRgba1PlistDecoding() throws {
        let result = try? PropertyListEncoder().encode(rgba1)
        self.measure { _ = try? PropertyListDecoder().decode([RGBA1].self, from: result!) }
    }
    
    func testRgba2JsonEncoding() throws {
        var result: Data?
        self.measure { result = try? JSONEncoder().encode(rgba2) }
        print("rgba2 json size: \(result?.count ?? 0)")
    }
    func testRgba2JsonDecoding() throws {
        let result = try? JSONEncoder().encode(rgba2)
        self.measure { _ = try? JSONDecoder().decode([RGBA2].self, from: result!) }
    }
    func testRgba2PlistEncoding() throws {
        var result: Data?
        self.measure { result = try? PropertyListEncoder().encode(rgba2) }
        print("rgba2 plist size: \(result?.count ?? 0)")
    }
    func testRgba2PlistDecoding() throws {
        let result = try? PropertyListEncoder().encode(rgba2)
        self.measure { _ = try? PropertyListDecoder().decode([RGBA2].self, from: result!) }
    }
}

Results on my device:

testRgba1JsonEncoding average 5.251 sec 32281065 bytes
testRgba1JsonDecoding average 7.749 sec
testRgba1PlistEncoding average 4.811 sec 41001610 bytes
testRgba1PlistDecoding average 7.529 sec

testRgba2JsonEncoding average 2.546 sec 16281065 bytes
testRgba2JsonDecoding average 7.906 sec
testRgba2PlistEncoding average 2.710 sec 25001586 bytes
testRgba2PlistDecoding average 6.432 sec

Osugi