Kotlinx.serialization: Decoding nested generics data class.

[hkhanz]

Hi 👋

I'm trying to decode the following Json using kotlinx.serialization and I am running into some issues.

The JSON:

{
    "fill": 0.0
}

Can also be represented as:

{
    "fill": [0.0, 0.0, 0.0, 0.0]
}

The data classes:

/// Fill.kt

@Serializable(FillSerializer::class)
public data class Fill<T>(@SerialName("fill") val insets: Inset<T>) {
    companion object
}

@Serializer(forClass = Fill::class)
class FillSerializer<T>(private val fillSerializer: KSerializer<T>): KSerializer<Fill<T>> {

    override val descriptor: SerialDescriptor = object : SerialClassDescImpl("FillSerializer") {
        init {
            addElement("fill")
        }
    }

    override fun deserialize(decoder: Decoder): Fill<T> {
        val inp = decoder.beginStructure(descriptor)
        val key = inp.decodeStringElement(descriptor, 0)
        val insets = inp.decodeSerializableElement(descriptor, 0, fillSerializer)
        inp.endStructure(descriptor)
        return Fill(insets)
    }

    override fun serialize(encoder: Encoder, obj: Fill<T>) {
        print(obj)
    }
}

/// Inset.kt

@Serializable(InsetSerializer::class)
data class Inset<T>(val start: T,
                    val end: T,
                    val top: T,
                    val bottom: T): WithDefault {

    constructor(array: Array<T>) : this(array[0], array[1], array[2], array[3])

}

@Serializer(forClass = Inset::class)
class InsetSerializer<T>(private val serializer: KSerializer<T>): KSerializer<Inset<T>> {

    override val descriptor: SerialDescriptor
        get() = StringDescriptor.withName("InsetSerializer")

    override fun deserialize(decoder: Decoder): Inset<T> {

        val decoded = decoder.decode(serializer)
            .guard {
                throw Exception("Some exception sonny.")
            }

        return when (decoded) {
            is List<*> -> {
                Inset(decoded.toTypedArray() as Array<T>)
            }
            is Float -> {
                Inset(decoded, decoded, decoded, decoded)
            }
            else -> throw Exception("WTF.")
        }
    }

    override fun serialize(encoder: Encoder, obj: Inset<T>) {

    }
}

I wrote the above following code to get something workable, Inset<T> by itself parses fine. The following tests pass:

final class InsetTests {

    val insetString = "0.0"
    val insetArrayString = "[0.0, 0.0, 0.0, 0.0]"
    val variedInsetArrayString = "[0.0, 12.0, 24.0, 18]"

    @Test
    fun testInsetString() {
        val json = Json(JsonConfiguration.Stable)
        val insets = json.parse(Inset.serializer(FloatSerializer), insetString)
        assert(insets == Inset(0.0f, 0.0f, 0.0f, 0.0f))
    }

    @Test
    fun testInsetArrayString() {
        val json = Json(JsonConfiguration.Stable)
        val insets = json.parse(Inset.serializer(ArrayListSerializer(FloatSerializer)), insetArrayString)
        assert(insets == Inset(0.0f, 0.0f, 0.0f, 0.0f))
    }

    @Test
    fun testVariedInsetArrayString() {
        val json = Json(JsonConfiguration.Stable)
        val insets = json.parse(Inset.serializer(ArrayListSerializer(FloatSerializer)), variedInsetArrayString)
        assert(insets == Inset(0.0f, 12.0f, 24.0f, 18.0f))
    }

}

Tests for Fill fail however:

final class FillTests {

    val fillJSONExample1: String = """
        {
            "fill": 0.0
        }
    """.trimIndent()

    val fillJSONExample2: String = """
        {
            "fill": [0.0, 0.0, 0.0, 0.0] 
        }
    """.trimIndent()


    @Test
    fun testParsingExample1() {
        val json = Json(JsonConfiguration.Stable)
        val parsedObject = json.parse(
            Fill.serializer(
                Inset.serializer(FloatSerializer)
            ),
            fillJSONExample1
        )
        print(parsedObject)
    }

}

With the following exception:

kotlinx.serialization.json.JsonParsingException: Invalid JSON at 12: Expected string or non-null literal

    at kotlinx.serialization.json.internal.JsonReader.takeString(JsonReader.kt:337)
    at kotlinx.serialization.json.internal.StreamingJsonInput.decodeString(StreamingJsonInput.kt:111)
    at kotlinx.serialization.ElementValueDecoder.decodeStringElement(ElementWise.kt:139)
    at com.ancestry.layoutcomposer.values.FillSerializer.deserialize(Layout.Fill.kt:48)
    at com.ancestry.layoutcomposer.values.FillSerializer.deserialize(Layout.Fill.kt:36)
    at kotlinx.serialization.json.internal.PolymorphicKt.decodeSerializableValuePolymorphic(Polymorphic.kt:33)
    at kotlinx.serialization.json.internal.StreamingJsonInput.decodeSerializableValue(StreamingJsonInput.kt:29)
    at kotlinx.serialization.CoreKt.decode(Core.kt:79)
    at kotlinx.serialization.json.Json.parse(Json.kt:152)
    at com.ancestry.layoutcomposer.FillTests.testParsingExample1(Fill.Tests.kt:29)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
    at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
    at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
    at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
    at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
    at com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)
    at com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:47)
    at com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:242)
    at com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70)

What is the correct way to parse that dynamic json?

[pdvrieze]

The challenge here is that handling this in the best way is an interaction between the format/decoder and the serializer (which is format independent). First of all it means that the deserializer used in decoding the fill value is determined dynamically. The general way to do so (as many formats would be supported) would be to have the deserializer first try one (hardcoded) deserializer (say for floats) and then the other (for lists of floats). Note that this requires that the parser will be in a valid state (and position) if parsing fails - I have not verified this and it relies on implementation details.

It would look like:

@Serializer(forClass = Inset::class)
class InsetSerializer<T>(private val serializer: KSerializer<T>): KSerializer<Inset<T>> {
    private val listSerializer = serializer.list

    override val descriptor: SerialDescriptor
        get() = StringDescriptor.withName("InsetSerializer")

    override fun deserialize(decoder: Decoder): Inset<T> {
        try {
            val singleInset = decoder.decode(serializer)
            return Inset(singleInset, singleInset, singleInset, singleInset)
        } catch (e: TheSpecificDecodingException) { /*just ignore this exception */ }
        val insetList = decoder.decode(listSerializer)
        // you want to check the array length here or in the Inset constructor
        return Inset(insetList.toTypedArray())
    }

    override fun serialize(encoder: Encoder, obj: Inset<T>) {
     // not implemented
    }
}

[hkhanz]

@pdvrieze thanks for the example however my specific issue is with the Fill data class, Inset parsing is working ok for me with the following code:

@Serializer(forClass = Inset::class)
class InsetSerializer<T>(private val serializer: KSerializer<T>): KSerializer<Inset<T>> {

    override val descriptor: SerialDescriptor
        get() = StringDescriptor.withName("InsetSerializer")

    override fun deserialize(decoder: Decoder): Inset<T> {

        val decoded = decoder.decode(serializer)
            .guard {
                throw Exception("Some exception sonny.")
            }

        return when (decoded) {
            is List<*> -> {
                Inset(decoded.toTypedArray() as Array<T>)
            }
            is Float -> {
                Inset(decoded, decoded, decoded, decoded)
            }
            else -> throw Exception("WTF.")
        }
    }

    override fun serialize(encoder: Encoder, obj: Inset<T>) {

    }
}

[hkhanz]

In Swift this is fairly easy to accomplish:

struct Fill<T> {
    let insets: Inset<T>
}

extension Fill: Codable {

    private enum RootKeys: String, CodingKey {
        case fill
    }

    init(from decoder: Decoder) throws {
       let container = try decoder.container(keyedBy: RootKeys.self)
       self.insets = try container.decode(Inset<T>.self, forKey: .fill)
   }

}

struct Inset<T> {
    let start: T
    let end: T 
    let top: T
    let bottom: T 
}

extension Inset: Codable where T: HasDefaultValue {

    init(from decoder: Decoder) throws {
        let singleValueContainer = try decoder.singleValueContainer()

        // If single value initialize with all 
        if let singleValue = try? singleValueContainer.decode(T.self) {
           self.start = singleValue
           self.end = singleValue
           self.top = singleValue
           self.bottom = singleValue
       } else if let array = try? singleValueContainer.decode([T].self),
          array.count >= 4 {
          self.start = array[0]
          self.end = array[1]
          self.top = array[2]
          self.bottom = array[3]
      } else {
          self.start = T.defaultValue
          self.end = T.defaultValue
          self.top = T.defaultValue
          self.bottom = T.defaultValue
      }
   }

}

let json1 = """
{
    "fill": 0.0
}
"""
let json2 = """
{
    "fill": [0.0, 0.0, 0.0, 0.0]
}
"""

let decoder = JSONDecoder()
guard 
    let fill1 = try? decoder.decode(Fill<Float>.self, for: json1),
    let fill2 = try? decoder.decode(Fill<Float>.self, for: json2)
    else { return } 
assert(fill1 == Fill(Inset<Float>(start: 0.0, end: 0.0, top: 0.0, bottom: 0.0))
assert(fill2 == Fill(Inset<Float>(start: 0.0, end: 0.0, top: 0.0, bottom: 0.0))

Why is this problem space so difficult in Kotlin? The needless abstractions make this library needlessly difficult to utilize.

[pdvrieze]

@hkhanz I'm not sure why you say it works with the InsetSerializer, because it can't.

The question is what does: decoder.decode(serializer) do?
To answer this we first need to realise that serializer will be DoubleSerializer, decoder will be a JsonInput child of some sort. What it does is call serializer.decode(this). DoubleSerializer implements this through decoder.readDouble. As such it will never return a list.

The problem is that what you want to do is dynamically determine the input based upon the actual content of the value. In JSON (or XML) this is possible, but in other formats this is not. For example Protobuf or the format android uses for Parcelable. Binary formats don't come with delimiters or default markers, values just follow on from each other directly.

It is important to keep in mind that kotlin serialization is a general, format independent, serialization framework. There is no general way in which probing the underlying data is always valid. You can special case a format (just check that the decoder is an instance of JsonInput), and perhaps allow further flexibility for that case.

To make a container with an inset work you will need to make sure that the custom serializer is used. Btw. if I look at what you are doing in SWIFT you are doing exactly what I suggested. When parsing the inset you first try parsing a float, then try parsing a list of floats. This should work with a format where it is possible to throw an exception when you try to read a float, but which isn't a float.

The most robust solution would be to use a custom (de)serializer together with a custom decoder (possibly extending the default one). The custom decoder would have the ability to probe or decode based upon particular values. The custom deserializer would detect this decoder and call that special probing/opportunistic decoding function and fall back to standard decoding for other decoders.

I had a quick look, it is not possible to access the underlying json parser from a StreamingJsonInput (it is an internal property). What you can do is call decodeJson which will give you a JsonElement that you can then do whatever you want with (including using serialization on again).

[hkhanz]

@pdvrieze I understand what your point is now. You are pointing out the fact that I have to utilize two different types of serializers to accomplish my parsing:

// 1.
val jsonParser = Json(JsonConfiguration.Stable)
val parsed = jsonParser.parse(
    Fill.serializer(
        Inset.serializer(FloatSerializer)
    ),
    jsonString1
)

// 2.
val parsed2 = jsonParser.parse(
    Fill.serializer(
        Inset.serializer(ArrayListSerializer(FloatSerializer))
    ),
    jsonString2
)

And what you are advocating for is using just 1 approach for both Array and Float:

val jsonParser = Json(JsonConfiguration.Stable)
val parsed1 = jsonParser.parse(
    Fill.serializer(SomeSpecialCustomInsetSerializer),
    jsonString1
)

val parsed2 = jsonParser.parse(
    Fill.serializer(SomeSpecialCustomInsetSeriliazer),
    jsonString2
)

Am I correct in my understanding?

[sandwwraith]

I believe it would easier to use cast in deserializer to Json-specific JsonInput to accomplish you goal. Because, as @pdvrieze correctly mentioned, kotlinx.serialization is a format-agnostic framework and catching exceptions may break the internal state of some parsers. JsonInput has specific decodeJson method to read json into abstract tree. See this example with Either: https://github.com/kotlin/kotlinx.serialization/blob/45aa7c7ab97f25f205408b46a7069886408bce6b/runtime/commonTest/src/kotlinx/serialization/json/JsonTreeAndMapperTest.kt#L36

So, your deserializer will look like this:

val input = decoder as? JsonInput ?: throw SerializationException("This class can be loaded only by Json")
val tree = input.decodeJson()
when(tree) {
  is JsonPrimitive -> // work with tree.content: String
  is JsonArray -> // do other stuff
}

[qoomon]

That is my implementation

import kotlinx.serialization.*
import kotlinx.serialization.builtins.ListSerializer
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonInput

@Serializer(forClass = List::class)
class SingleElementListSerializer<T : Any>(private val dataSerializer: KSerializer<T>) : KSerializer<List<T>> {

    override val descriptor: SerialDescriptor = SerialDescriptor(serialName = "SingleElementListSerializer")

    private val dataListSerializer = ListSerializer(dataSerializer)

    override fun deserialize(decoder: Decoder): List<T> {
        decoder as? JsonInput ?: throw IllegalStateException(
                "This serializer can be used only with Json format." +
                    "Expected Decoder to be JsonInput, got ${this::class}"
            )
        return when (val jsonElement = decoder.decodeJson()) {
            is JsonArray -> decoder.json.fromJson(dataListSerializer, jsonElement)
            else -> listOf(decoder.json.fromJson(dataSerializer, jsonElement))
        }
    }

    override fun serialize(encoder: Encoder, value: List<T>) {
        return when (value.size) {
            1 -> encoder.encode(dataSerializer, value.single())
            else -> encoder.encode(dataListSerializer, value)
        }
    }
}