Issue #134: support multiple DTs in BootV2 and VendorBoot

https://source.android.com/docs/core/architecture/dto/multiple

.github/workflows/main.yml

@@ -31,7 +31,7 @@ jobs:
           java-version: 17
 
       - name: apt
-        run: sudo apt install device-tree-compiler p7zip-full android-sdk-libsparse-utils
+        run: sudo apt install device-tree-compiler p7zip-full android-sdk-libsparse-utils erofs-utils
 
       # Runs a single command using the runners shell
       - name: Unit Test

README.md

@@ -315,6 +315,20 @@ Refer to Issue https://github.com/cfig/Android_boot_image_editor/issues/120
 
 </details>
 
+<details>
+  <summary>How to work with vendor_dlkm.img</summary>
+
+```bash
+cp <your_vendor_dlkm.img> vendor_dlkm.img
+cp <your_vbmeta_image> vbmeta.img
+./gradlew unpack
+# replace your .ko
+./gradlew pack
+```
+Then flash `vbmeta.img.signed` and `vendor_dlkm.img.signed` to the device.
+
+</details>
+
 ## boot.img layout
 Read [boot layout](doc/layout.md) of Android boot.img and vendor\_boot.img.
 Read [misc layout](doc/misc_image_layout.md) of misc\.img

bbootimg/build.gradle.kts

@@ -15,7 +15,7 @@
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
 plugins {
-    kotlin("jvm") version "1.9.20"
+    kotlin("jvm") version "1.9.22"
     application
 }
 

bbootimg/src/main/kotlin/bootimg/v2/BootV2.kt

@@ -44,7 +44,7 @@ data class BootV2(
     var ramdisk: RamdiskArgs = RamdiskArgs(),
     var secondBootloader: CommArgs? = null,
     var recoveryDtbo: CommArgsLong? = null,
-    var dtb: CommArgsLong? = null,
+    var dtb: DtbArgsLong? = null,
 ) {
     data class MiscInfo(
         var output: String = "",
@@ -85,6 +85,14 @@ data class BootV2(
         var loadOffset: Long = 0,
     )
 
+    data class DtbArgsLong(
+        var file: String? = null,
+        var position: Long = 0,
+        var size: Int = 0,
+        var loadOffset: Long = 0,
+        var dtbList: MutableList<DTC.DtbEntry> = mutableListOf(),
+    )
+
     companion object {
         private val log = LoggerFactory.getLogger(BootV2::class.java)
         private val workDir = Helper.prop("workDir")
@@ -148,7 +156,7 @@ data class BootV2(
                     ret.recoveryDtbo!!.position = ret.getRecoveryDtboPosition()
                 }
                 if (bh2.dtbLength > 0) {
-                    ret.dtb = CommArgsLong()
+                    ret.dtb = DtbArgsLong()
                     ret.dtb!!.size = bh2.dtbLength
                     ret.dtb!!.loadOffset = bh2.dtbOffset //Q
                     ret.dtb!!.file = "${workDir}dtb"
@@ -243,6 +251,13 @@ data class BootV2(
         //dtb
         this.dtb?.let { _ ->
             Common.dumpDtb(Helper.Slice(info.output, dtb!!.position.toInt(), dtb!!.size, dtb!!.file!!))
+            this.dtb!!.dtbList = DTC.parseMultiple(dtb!!.file!!)
+            log.info("dtb sz = " + this.dtb!!.dtbList.size)
+            //dump info again
+            mapper.writerWithDefaultPrettyPrinter().writeValue(File(workDir + info.json), this)
+
+            //dump dtb items
+            DTC.extractMultiple(dtb!!.file!!, this.dtb!!.dtbList)
         }
 
         return this
@@ -340,12 +355,13 @@ data class BootV2(
             //dtb
             this.dtb?.let { theDtb ->
                 if (theDtb.size > 0) {
+                    val dtbCount = this.dtb!!.dtbList.size
                     it.addRule()
                     it.addRow("dtb", theDtb.file)
                     prints.add(Pair("dtb", theDtb.file.toString()))
                     if (File(theDtb.file + ".${dtsSuffix}").exists()) {
-                        it.addRow("\\-- decompiled dts", theDtb.file + ".${dtsSuffix}")
+                        it.addRow("\\-- decompiled dts [$dtbCount]", theDtb.file + ".*.${dtsSuffix}")
-                        prints.add(Pair("\\-- decompiled dts", theDtb.file + ".${dtsSuffix}"))
+                        prints.add(Pair("\\-- decompiled dts [$dtbCount]", theDtb.file + ".*.${dtsSuffix}"))
                     }
                 }
             }
@@ -441,7 +457,7 @@ data class BootV2(
         //refresh dtb size
         dtb?.let { theDtb ->
             if (File(theDtb.file!! + ".${dtsSuffix}").exists()) {
-                check(DTC().compile(theDtb.file!! + ".${dtsSuffix}", theDtb.file!!)) { "fail to compile dts" }
+                DTC.packMultiple(theDtb.file!!, theDtb.dtbList)
             }
             theDtb.size = File(theDtb.file!!).length().toInt()
         }

bbootimg/src/main/kotlin/bootimg/v3/VendorBoot.kt

@@ -39,7 +39,7 @@ import cfig.bootimg.Common as C
 data class VendorBoot(
     var info: MiscInfo = MiscInfo(),
     var ramdisk: RamdiskArgs = RamdiskArgs(),
-    var dtb: CommArgs = CommArgs(),
+    var dtb: DtbArgs = DtbArgs(),
     var ramdisk_table: Vrt = Vrt(),
     var bootconfig: CommArgs = CommArgs(),
 ) {
@@ -50,6 +50,14 @@ data class VendorBoot(
         var loadAddr: Long = 0,
     )
 
+    data class DtbArgs(
+        var file: String = "",
+        var position: Long = 0,
+        var size: Int = 0,
+        var loadAddr: Long = 0,
+        var dtbList: MutableList<DTC.DtbEntry> = mutableListOf(),
+    )
+
     data class RamdiskArgs(
         var file: String = "",
         var position: Long = 0,
@@ -251,7 +259,7 @@ data class VendorBoot(
         }
         //update dtb
         if (File(this.dtb.file + ".${dtsSuffix}").exists()) {
-            check(DTC().compile(this.dtb.file + ".${dtsSuffix}", this.dtb.file)) { "fail to compile dts" }
+            DTC.packMultiple(this.dtb.file, this.dtb.dtbList)
         }
         this.dtb.size = File(this.dtb.file).length().toInt()
         //header
@@ -359,7 +367,13 @@ data class VendorBoot(
             this.ramdisk.xzFlags = checkType
         }
         //dtb
-        C.dumpDtb(Helper.Slice(info.output, dtb.position.toInt(), dtb.size, dtb.file))
+        run {
+            C.dumpDtb(Helper.Slice(info.output, dtb.position.toInt(), dtb.size, dtb.file))
+            if (dtb.size > 0) {
+                dtb.dtbList = DTC.parseMultiple(dtb.file)
+                DTC.extractMultiple(dtb.file, dtb.dtbList)
+            }
+        }
         //vrt
         this.ramdisk_table.ramdidks.forEachIndexed { index, it ->
             log.info("dumping vendor ramdisk ${index + 1}/${this.ramdisk_table.ramdidks.size} ...")
@@ -425,8 +439,8 @@ data class VendorBoot(
                 it.addRow("dtb", this.dtb.file)
                 prints.add(Pair("dtb", this.dtb.file))
                 if (File(this.dtb.file + ".${dtsSuffix}").exists()) {
-                    it.addRow("\\-- decompiled dts", dtb.file + ".${dtsSuffix}")
+                    it.addRow("\\-- decompiled dts [${dtb.dtbList.size}]", dtb.file + "*.${dtsSuffix}")
-                    prints.add(Pair("\\-- decompiled dts", dtb.file + ".${dtsSuffix}"))
+                    prints.add(Pair("\\-- decompiled dts [${dtb.dtbList.size}]", dtb.file + "*.${dtsSuffix}"))
                 }
             } else {
                 it.addRow("dtb", "-")

bbootimg/src/main/kotlin/utils/DTC.kt

@@ -14,13 +14,67 @@
 
 package cfig.utils
 
+import cc.cfig.io.Struct
+import cfig.helper.Dumpling
+import cfig.helper.Helper
 import org.apache.commons.exec.CommandLine
 import org.apache.commons.exec.DefaultExecutor
 import org.slf4j.LoggerFactory
+import java.io.File
+import java.io.FileOutputStream
+import java.io.InputStream
 
 class DTC {
     private val log = LoggerFactory.getLogger(DTC::class.java)
 
+    data class DtbEntry(
+        var seqNo: Int = 0,
+        var offset: Long = 0,
+        var header: FdtHeader = FdtHeader(),
+    )
+
+    data class FdtHeader(
+        var magic: Int = 0,
+        val totalsize: Int = 0,
+        val offDtStruct: Int = 0,
+        val offDtStrings: Int = 0,
+        val offMemRsvmap: Int = 0,
+        val version: Int = 0,
+        val lastCompVersion: Int = 0,
+        val bootCpuidPhys: Int = 0,
+        val sizeDtStrings: Int = 0,
+        val sizeDtStruct: Int = 0
+    ) {
+        companion object {
+            private const val MAGIC = 0xd00dfeedu
+            const val FORMAT_STRING = ">10i"
+            const val SIZE = 40
+
+            init {
+                check(Struct(FORMAT_STRING).calcSize() == SIZE)
+            }
+
+            @Throws(IllegalStateException::class)
+            fun parse(iS: InputStream): FdtHeader {
+                val info = Struct(FORMAT_STRING).unpack(iS)
+                val ret = FdtHeader(
+                    info[0] as Int,
+                    info[1] as Int,
+                    info[2] as Int,
+                    info[3] as Int,
+                    info[4] as Int,
+                    info[5] as Int,
+                    info[6] as Int,
+                    info[7] as Int,
+                    info[8] as Int,
+                    info[9] as Int
+                )
+                check(ret.magic.toUInt() == MAGIC) { "bad magic: ${ret.magic}" }
+                return ret
+            }
+        }
+    }
+
     fun decompile(dtbFile: String, outFile: String): Boolean {
         log.info("parsing DTB: $dtbFile")
         //CommandLine.parse("fdtdump").let {
@@ -75,4 +129,63 @@ class DTC {
         }
         return true
     }
+
+    companion object {
+        private val log = LoggerFactory.getLogger(DTC::class.java)
+        fun parseMultiple(fileName: String): MutableList<DtbEntry> {
+            val ret = mutableListOf<DtbEntry>()
+            var seqNo = 0
+            while (true) {
+                try {
+                    val index = ret.sumOf { it.header.totalsize.toLong() }
+                    val data = Dumpling(fileName).readFully(Pair(index, FdtHeader.SIZE))
+                    val header = FdtHeader.parse(data.inputStream())
+                    log.info("Found FDT header: #${seqNo} $header")
+                    ret.add(DtbEntry(seqNo, index, header))
+                    seqNo++
+                } catch (e: IllegalStateException) {
+                    log.info("no more FDT header")
+                    break
+                }
+            }
+            val remainder = File(fileName).length() - ret.sumOf { it.header.totalsize }.toLong()
+            if (remainder == 0L) {
+                log.info("Successfully parsed ${ret.size} FDT headers")
+            } else {
+                log.warn("Successfully parsed ${ret.size} FDT headers, remainder: $remainder bytes")
+            }
+            return ret
+        }
+
+        fun extractMultiple(fileStem: String, entries: List<DtbEntry>) {
+            entries.forEach {
+                val slice = Helper.Slice(
+                    fileStem,
+                    it.offset.toInt(), it.header.totalsize, "${fileStem}.${it.seqNo}"
+                )
+                Helper.extractFile(slice.srcFile, slice.dumpFile, slice.offset.toLong(), slice.length)
+                if (EnvironmentVerifier().hasDtc) {
+                    DTC().decompile(slice.dumpFile, slice.dumpFile + "." + Helper.prop("config.dts_suffix"))
+                }
+            }
+        }
+
+        fun packMultiple(fileStem: String, entries: List<DtbEntry>) {
+            if (EnvironmentVerifier().hasDtc) {
+                entries.forEach {
+                    DTC().compile(
+                        fileStem + ".${it.seqNo}." + Helper.prop("config.dts_suffix"),
+                        fileStem + "." + it.seqNo
+                    )
+                }
+                FileOutputStream(fileStem).use { outFile ->
+                    entries.indices.forEach {
+                        log.info("Appending ${fileStem}.${it} ...")
+                        outFile.write(File("$fileStem.$it").readBytes())
+                    }
+                    log.info("Appended ${entries.size} DTBs to ${fileStem}")
+                }
+            }
+        }
+    }
 }

helper/build.gradle.kts

@@ -15,7 +15,7 @@
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
 plugins {
-    kotlin("jvm") version "1.9.20"
+    kotlin("jvm") version "1.9.22"
     `java-library`
     application
 }

lazybox/build.gradle.kts

@@ -7,7 +7,7 @@
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
 plugins {
-    kotlin("jvm") version "1.9.20"
+    kotlin("jvm") version "1.9.22"
     application
 }
 

src/integrationTest/resources

@@ -1 +1 @@
-Subproject commit d683034c3e83818b3a1264331b281df650fc2e6b
+Subproject commit c2e5850a225bd765fa45b6c628106a82aee2a66f

src/integrationTest/resources_2

@@ -1 +1 @@
-Subproject commit 517ca7a72425c6b8b913feea0b505f07879549c9
+Subproject commit b5906e9ac69e0de09d8380b3f3409ed336017262