Advanced JSON parsing techniques using Moshi and Kotlin

The example model and JSON file

class Person(val id: Long, val name: String, val age: Int = -1)
[
{
"id": 1,
"name": "John",
"age": 38
},
{
"id": 8,
"name": "Lisa",
"age": 23
},
{
"id": 23,
"name": "Karen"
}
]

1. Fully manual parsing

class ManualParser {
fun parse(reader: JsonReader): List<Person> {
val result = mutableListOf<Person>()

reader.beginArray()
while (reader.hasNext()) {
var id: Long = -1L
var name: String = ""
var
age: Int = -1

reader.beginObject()
while (reader.hasNext()) {
when (reader.nextName()) {
"id" -> id = reader.nextLong()
"name" -> name = reader.nextString()
"age" -> age = reader.nextInt()
else -> reader.skipValue()
}
}
reader.endObject()

if (id == -1L || name == "") {
throw JsonDataException("Missing required field")
}
val person = Person(id, name, age)
result.add(person)
}
reader.endArray()

return result
}
}

Introducing selectName() for performance

class ManualParser {
companion object {
val NAMES = JsonReader.Options.of("id", "name", "age")
}

fun parse(reader: JsonReader): List<Person> {
val result = mutableListOf<Person>()

reader.beginArray()
while (reader.hasNext()) {
var id: Long = -1L
var name: String = ""
var
age: Int = -1

reader.beginObject()
while (reader.hasNext()) {
when (reader.selectName(NAMES)) {
0 -> id = reader.nextLong()
1 -> name = reader.nextString()
2 -> age = reader.nextInt()
else -> {
reader.skipName()
reader.skipValue()
}
}
}
reader.endObject()

if (id == -1L || name == "") {
throw JsonDataException("Missing required field")
}
val person = Person(id, name, age)
result.add(person)
}
reader.endArray()

return result
}
}

Reducing boilerplate code using extension functions

fun JsonReader.skipNameAndValue() {
skipName()
skipValue()
}
inline fun JsonReader.readObject(body: () -> Unit) {
beginObject()
while (hasNext()) {
body()
}
endObject()
}

inline fun JsonReader.readArray(body: () -> Unit) {
beginArray()
while (hasNext()) {
body()
}
endArray()
}
inline fun <T : Any> JsonReader.readArrayToList(body: () -> T?): List<T> {
val result = mutableListOf<T>()
beginArray()
while (hasNext()) {
body()?.let { result.add(it) }
}
endArray()
return result
}
class ManualParser {
companion object {
val NAMES = JsonReader.Options.of("id", "name", "age")
}

fun parse(reader: JsonReader): List<Person> {
return reader.readArrayToList {
var
id: Long = -1L
var name: String = ""
var
age: Int = -1

reader.readObject {
when
(reader.selectName(NAMES)) {
0 -> id = reader.nextLong()
1 -> name = reader.nextString()
2 -> age = reader.nextInt()
else -> reader.skipNameAndValue()
}
}

if
(id == -1L || name == "") {
throw JsonDataException("Missing required field")
}
Person(id, name, age)
}
}
}

Immutability, default values and mandatory fields

var id: Long = -1L
var name: String = ""
var
age: Int = -1
  • Variables for optional fields (age) need to be assigned a default value matching the default value of the field.
  • Variables for mandatory fields (id, name) are assigned an arbitrary default value outside of the range of valid values. null can also be used as default value for non-null mandatory fields. Then before instantiating the model object, we need to check that these variables have been reassigned valid values, otherwise throw an Exception or skip the item:
if (id == -1L || name == "") {
throw JsonDataException("Missing required field")
}
class Person {
var id: Long = 0L
var name: String = ""
var age
: Int = -1
}
  • Less lines of code in the parser;
  • We can rely on the model default values instead of duplicating them in the parser code.
  • We lose all the benefits of immutability: coherent states, no side effects, thread safety;
  • All fields are now declared as optional. We need to document which ones are in practice mandatory and the parser still needs to manually check if they have been assigned a valid value.

2. Moshi’s Kotlin Code Gen

  • Using reflection, via the moshi-kotlin artifact. Adapters will be generated at runtime when needed, then reused. The main downside of this solution is that in order to understand metadata like nullability and default values of Kotlin properties, this artifact depends on the kotlin-reflect library, a 2.5 MiB jar file which will make your application code size grow significantly.
  • Since Moshi 1.6, adapters can also be generated at compile time using the moshi-kotlin-codegen annotation processor. This is a much better solution because it provides better performance while adding no extra dependency to your application. The only limitation of that solution compared to kotlin-reflect is that it is unable to initialize private or protected fields, but the same limitations apply when writing your parsers manually.

Setup moshi-kotlin-codegen

apply plugin: 'kotlin-kapt'
dependencies {
kapt 'com.squareup.moshi:moshi-kotlin-codegen:1.6.0'
}
@JsonClass(generateAdapter = true)
class Person(val id: Long, val name: String, val age: Int = -1)

A look at the generated code

class PersonJsonAdapter(moshi: Moshi) : JsonAdapter<Person>() {
private val options: JsonReader.Options = JsonReader.Options.of("id", "name", "age")

private val longAdapter: JsonAdapter<Long> = moshi.adapter(Long::class.java).nonNull()

private val stringAdapter: JsonAdapter<String> = moshi.adapter(String::class.java).nonNull()

private val intAdapter: JsonAdapter<Int> = moshi.adapter(Int::class.java).nonNull()

override fun toString(): String = "GeneratedJsonAdapter(Person)"

override fun
fromJson(reader: JsonReader): Person {
var id: Long? = null
var
name: String? = null
var
age: Int? = null
reader.beginObject()
while (reader.hasNext()) {
when (reader.selectName(options)) {
0 -> id = longAdapter.fromJson(reader) ?: throw JsonDataException("Non-null value 'id' was null at ${reader.path}")
1 -> name = stringAdapter.fromJson(reader) ?: throw JsonDataException("Non-null value 'name' was null at ${reader.path}")
2 -> age = intAdapter.fromJson(reader) ?: throw JsonDataException("Non-null value 'age' was null at ${reader.path}")
-1 -> {
// Unknown name, skip it.
reader.skipName()
reader.skipValue()
}
}
}
reader.endObject()
var result = Person(
id = id
?: throw JsonDataException("Required property 'id' missing at ${reader.path}"),
name = name
?: throw JsonDataException("Required property 'name' missing at ${reader.path}"))
result = Person(
id = id ?: result.id,
name = name ?: result.name,
age = age ?: result.age)
return result
}

override fun toJson(writer: JsonWriter, value: Person?) {
// Removed for brevity
}
}
  • Nullable variables are used to store values of non-null primitive types (Long, Int) and generic adapters are used to read them ( longAdapter, intAdapter), which means unnecessary boxing will occur when reading these values.
  • For each item, two instances of Person are created instead of one. The reason behind this is to allow reading back all the default values of optional fields from the first instance when creating the second, final instance.
    This should not cause any performance issue, unless your model class includes heavy initialization code that you want to avoid doing twice.
    This also means that when all fields are mandatory, only one instance will be created by the adapter.

Using the generated adapters

val adapter: JsonAdapter<Person> = moshi.adapter(Person::class.java)
val listType = Types.newParameterizedType(List::class.java, Person::class.java)
val adapter: JsonAdapter<List<Person>> = moshi.adapter(listType)
val result = adapter.fromJson(reader)
class HybridParser(moshi: Moshi) {
private val personAdapter: JsonAdapter<Person> = moshi.adapter(Person::class.java)

fun parse(reader: JsonReader): List<Person> {
return reader.readArrayToList {
personAdapter
.fromJson(reader)?.takeIf { it.age >= 18 }
}
}
}

3. Coroutines magic for big data sources

class LazyParser(moshi: Moshi) {
private val personAdapter: JsonAdapter<Person> = moshi.adapter(Person::class.java)

fun parse(reader: JsonReader): Sequence<Person> {
return sequence {
reader.readArray {
yield(personAdapter.fromJson(reader)!!)
}
}
}
}
val sequence = LazyParser(moshi).parse(reader)
val filteredList = sequence.filter { it.age >= 18 }.toList()

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store