From 0681a5e0c95f6aa15b0476f77783b8173c575e79 Mon Sep 17 00:00:00 2001 From: cfig Date: Thu, 28 Jul 2022 21:31:05 +0800 Subject: [PATCH] routine update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ● avbtool: update to asop master, we see the new flag AvbHashtreeDescriptor.FLAGS_CHECK_AT_MOST_ONCE ● For China mainland users: Using domestic gradle+maven mirrors can speed up the build. ● gradle: 7.5 ● kotlin: 1.7.10 --- aosp/avb/avbtool.v1.2.py | 120 ++++++++++++------ bbootimg/build.gradle.kts | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- helper/build.gradle.kts | 2 +- .../src/main/kotlin/cfig/helper/Launcher.kt | 5 +- tools/work_from_China.diff | 63 +++++++++ 6 files changed, 153 insertions(+), 41 deletions(-) create mode 100644 tools/work_from_China.diff diff --git a/aosp/avb/avbtool.v1.2.py b/aosp/avb/avbtool.v1.2.py index 8647b29..f944af4 100755 --- a/aosp/avb/avbtool.v1.2.py +++ b/aosp/avb/avbtool.v1.2.py @@ -342,6 +342,7 @@ class RSAPublicKey(object): exponent: The key exponent. modulus: The key modulus. num_bits: The key size. + key_path: The path to a key file. """ MODULUS_PREFIX = b'modulus=' @@ -900,7 +901,7 @@ class ImageHandler(object): Arguments: num_bytes: Size in number of bytes of the DONT_CARE chunk. - Raises + Raises: OSError: If ImageHandler was initialized in read-only mode. """ assert num_bytes % self.block_size == 0 @@ -937,7 +938,7 @@ class ImageHandler(object): Arguments: data: Data to append as bytes. - Raises + Raises: OSError: If ImageHandler was initialized in read-only mode. """ assert len(data) % self.block_size == 0 @@ -974,7 +975,7 @@ class ImageHandler(object): fill_data: Fill data to append - must be four bytes. size: Number of chunk - must be a multiple of four and the block size. - Raises + Raises: OSError: If ImageHandler was initialized in read-only mode. """ assert len(fill_data) == 4 @@ -1259,7 +1260,7 @@ class AvbPropertyDescriptor(AvbDescriptor): Raises: LookupError: If the given descriptor is malformed. """ - super(AvbPropertyDescriptor, self).__init__(None) + super().__init__(None) assert struct.calcsize(self.FORMAT_STRING) == self.SIZE if data: @@ -1273,7 +1274,8 @@ class AvbPropertyDescriptor(AvbDescriptor): try: self.key = data[self.SIZE:(self.SIZE + key_size)].decode('utf-8') except UnicodeDecodeError as e: - raise LookupError('Key cannot be decoded as UTF-8: {}.'.format(e)) + raise LookupError('Key cannot be decoded as UTF-8: {}.' + .format(e)) from e self.value = data[(self.SIZE + key_size + 1):(self.SIZE + key_size + 1 + value_size)] else: @@ -1377,6 +1379,9 @@ class AvbHashtreeDescriptor(AvbDescriptor): 'L' + # flags str(RESERVED) + 's') # reserved + FLAGS_DO_NOT_USE_AB = (1 << 0) + FLAGS_CHECK_AT_MOST_ONCE = (1 << 1) + def __init__(self, data=None): """Initializes a new hashtree descriptor. @@ -1386,7 +1391,7 @@ class AvbHashtreeDescriptor(AvbDescriptor): Raises: LookupError: If the given descriptor is malformed. """ - super(AvbHashtreeDescriptor, self).__init__(None) + super().__init__(None) assert struct.calcsize(self.FORMAT_STRING) == self.SIZE if data: @@ -1410,7 +1415,7 @@ class AvbHashtreeDescriptor(AvbDescriptor): ].decode('utf-8') except UnicodeDecodeError as e: raise LookupError('Partition name cannot be decoded as UTF-8: {}.' - .format(e)) + .format(e)) from e o += partition_name_len self.salt = data[(self.SIZE + o):(self.SIZE + o + salt_len)] o += salt_len @@ -1582,7 +1587,7 @@ class AvbHashDescriptor(AvbDescriptor): Raises: LookupError: If the given descriptor is malformed. """ - super(AvbHashDescriptor, self).__init__(None) + super().__init__(None) assert struct.calcsize(self.FORMAT_STRING) == self.SIZE if data: @@ -1603,7 +1608,7 @@ class AvbHashDescriptor(AvbDescriptor): ].decode('utf-8') except UnicodeDecodeError as e: raise LookupError('Partition name cannot be decoded as UTF-8: {}.' - .format(e)) + .format(e)) from e o += partition_name_len self.salt = data[(self.SIZE + o):(self.SIZE + o + salt_len)] o += salt_len @@ -1720,7 +1725,7 @@ class AvbKernelCmdlineDescriptor(AvbDescriptor): Raises: LookupError: If the given descriptor is malformed. """ - super(AvbKernelCmdlineDescriptor, self).__init__(None) + super().__init__(None) assert struct.calcsize(self.FORMAT_STRING) == self.SIZE if data: @@ -1737,7 +1742,7 @@ class AvbKernelCmdlineDescriptor(AvbDescriptor): self.SIZE:(self.SIZE + kernel_cmdline_length)].decode('utf-8') except UnicodeDecodeError as e: raise LookupError('Kernel command-line cannot be decoded as UTF-8: {}.' - .format(e)) + .format(e)) from e else: self.flags = 0 self.kernel_cmdline = '' @@ -1835,7 +1840,7 @@ class AvbChainPartitionDescriptor(AvbDescriptor): ].decode('utf-8') except UnicodeDecodeError as e: raise LookupError('Partition name cannot be decoded as UTF-8: {}.' - .format(e)) + .format(e)) from e o += partition_name_len self.public_key = data[(self.SIZE + o):(self.SIZE + o + public_key_len)] @@ -2468,8 +2473,8 @@ class Avb(object): o.write(' Metadata version: {}\n'.format(version)) def print_atx_certificate(cert): - version, public_key, subject, usage, key_version, _signature = \ - struct.unpack(' 0: - c += ' 10' # number of optional args + if ht.flags & AvbHashtreeDescriptor.FLAGS_CHECK_AT_MOST_ONCE: + c += ' 11' # number of optional args + c += ' check_at_most_once' + else: + c += ' 10' # number of optional args c += ' $(ANDROID_VERITY_MODE)' c += ' ignore_zero_blocks' c += ' use_fec_from_device PARTUUID=$(ANDROID_SYSTEM_PARTUUID)' @@ -2843,7 +2852,11 @@ class Avb(object): c += ' fec_blocks {}'.format(ht.fec_offset // ht.data_block_size) c += ' fec_start {}'.format(ht.fec_offset // ht.data_block_size) else: - c += ' 2' # number of optional args + if ht.flags & AvbHashtreeDescriptor.FLAGS_CHECK_AT_MOST_ONCE: + c += ' 3' # number of optional args + c += ' check_at_most_once' + else: + c += ' 2' # number of optional args c += ' $(ANDROID_VERITY_MODE)' c += ' ignore_zero_blocks' c += '" root=/dev/dm-0' @@ -3025,8 +3038,9 @@ class Avb(object): """ try: alg = ALGORITHMS[algorithm_name] - except KeyError: - raise AvbError('Unknown algorithm with name {}'.format(algorithm_name)) + except KeyError as e: + raise AvbError('Unknown algorithm with name {}' + .format(algorithm_name)) from e if not descriptors: descriptors = [] @@ -3313,9 +3327,10 @@ class Avb(object): except Exception as e: # Truncate back to original size, then re-raise. image.truncate(original_image_size) - raise AvbError('Appending VBMeta image failed: {}.'.format(e)) + raise AvbError('Appending VBMeta image failed: {}.'.format(e)) from e - def add_hash_footer(self, image_filename, partition_size, partition_name, + def add_hash_footer(self, image_filename, partition_size, + dynamic_partition_size, partition_name, hash_algorithm, salt, chain_partitions, algorithm_name, key_path, public_key_metadata_path, rollback_index, flags, @@ -3333,6 +3348,7 @@ class Avb(object): Arguments: image_filename: File to add the footer to. partition_size: Size of partition. + dynamic_partition_size: Calculate partition size based on image size. partition_name: Name of partition (without A/B suffix). hash_algorithm: Hash algorithm to use. salt: Salt to use as a hexadecimal string or None to use /dev/urandom. @@ -3366,6 +3382,14 @@ class Avb(object): Raises: AvbError: If an argument is incorrect of if adding of hash_footer failed. """ + if not partition_size and not dynamic_partition_size: + raise AvbError('--dynamic_partition_size required when not specifying a ' + 'partition size') + + if dynamic_partition_size and calc_max_image_size: + raise AvbError('--calc_max_image_size not supported with ' + '--dynamic_partition_size') + required_libavb_version_minor = 0 if use_persistent_digest or do_not_use_ab: required_libavb_version_minor = 1 @@ -3381,24 +3405,18 @@ class Avb(object): # this size + metadata (footer + vbmeta struct) fits in # |partition_size|. max_metadata_size = self.MAX_VBMETA_SIZE + self.MAX_FOOTER_SIZE - if partition_size < max_metadata_size: + if not dynamic_partition_size and partition_size < max_metadata_size: raise AvbError('Parition size of {} is too small. ' 'Needs to be at least {}'.format( partition_size, max_metadata_size)) - max_image_size = partition_size - max_metadata_size # If we're asked to only calculate the maximum image size, we're done. if calc_max_image_size: - print('{}'.format(max_image_size)) + print('{}'.format(partition_size - max_metadata_size)) return image = ImageHandler(image_filename) - if partition_size % image.block_size != 0: - raise AvbError('Partition size of {} is not a multiple of the image ' - 'block size {}.'.format(partition_size, - image.block_size)) - # If there's already a footer, truncate the image to its original # size. This way 'avbtool add_hash_footer' is idempotent (modulo # salts). @@ -3415,6 +3433,16 @@ class Avb(object): # Image size is too small to possibly contain a footer. original_image_size = image.image_size + if dynamic_partition_size: + partition_size = round_to_multiple( + original_image_size + max_metadata_size, image.block_size) + + max_image_size = partition_size - max_metadata_size + if partition_size % image.block_size != 0: + raise AvbError('Partition size of {} is not a multiple of the image ' + 'block size {}.'.format(partition_size, + image.block_size)) + # If anything goes wrong from here-on, restore the image back to # its original size. try: @@ -3514,7 +3542,7 @@ class Avb(object): except Exception as e: # Truncate back to original size, then re-raise. image.truncate(original_image_size) - raise AvbError('Adding hash_footer failed: {}.'.format(e)) + raise AvbError('Adding hash_footer failed: {}.'.format(e)) from e def add_hashtree_footer(self, image_filename, partition_size, partition_name, generate_fec, fec_num_roots, hash_algorithm, @@ -3532,7 +3560,7 @@ class Avb(object): output_vbmeta_image, do_not_append_vbmeta_image, print_required_libavb_version, use_persistent_root_digest, do_not_use_ab, - no_hashtree): + no_hashtree, check_at_most_once): """Implements the 'add_hashtree_footer' command. See https://gitlab.com/cryptsetup/cryptsetup/wikis/DMVerity for @@ -3576,13 +3604,15 @@ class Avb(object): use_persistent_root_digest: Use a persistent root digest on device. do_not_use_ab: The partition does not use A/B. no_hashtree: Do not append hashtree. Set size in descriptor as zero. + check_at_most_once: Set to verify data blocks only the first time they + are read from the data device. Raises: AvbError: If an argument is incorrect or adding the hashtree footer failed. """ required_libavb_version_minor = 0 - if use_persistent_root_digest or do_not_use_ab: + if use_persistent_root_digest or do_not_use_ab or check_at_most_once: required_libavb_version_minor = 1 if rollback_index_location > 0: required_libavb_version_minor = 2 @@ -3713,9 +3743,11 @@ class Avb(object): ht_desc.partition_name = partition_name ht_desc.salt = salt if do_not_use_ab: - ht_desc.flags |= 1 # AVB_HASHTREE_DESCRIPTOR_FLAGS_DO_NOT_USE_AB + ht_desc.flags |= AvbHashtreeDescriptor.FLAGS_DO_NOT_USE_AB if not use_persistent_root_digest: ht_desc.root_digest = root_digest + if check_at_most_once: + ht_desc.flags |= AvbHashtreeDescriptor.FLAGS_CHECK_AT_MOST_ONCE # Write the hash tree padding_needed = (round_to_multiple(len(hash_tree), image.block_size) - @@ -3788,7 +3820,7 @@ class Avb(object): except Exception as e: # Truncate back to original size, then re-raise. image.truncate(original_image_size) - raise AvbError('Adding hashtree_footer failed: {}.'.format(e)) + raise AvbError('Adding hashtree_footer failed: {}.'.format(e)) from e def make_atx_certificate(self, output, authority_key_path, subject_key_path, subject_key_version, subject, @@ -4040,7 +4072,8 @@ def generate_fec_data(image_filename, num_roots): fec_tmpfile.name], stderr=open(os.devnull, 'wb')) except subprocess.CalledProcessError as e: - raise ValueError('Execution of \'fec\' tool failed: {}.'.format(e)) + raise ValueError('Execution of \'fec\' tool failed: {}.' + .format(e)) from e fec_data = fec_tmpfile.read() footer_size = struct.calcsize(FEC_FOOTER_FORMAT) @@ -4074,6 +4107,14 @@ def generate_hash_tree(image, image_size, block_size, hash_alg_name, salt, hash_src_offset = 0 hash_src_size = image_size level_num = 0 + + # If there is only one block, returns the top-level hash directly. + if hash_src_size == block_size: + hasher = create_avb_hashtree_hasher(hash_alg_name, salt) + image.seek(0) + hasher.update(image.read(block_size)) + return hasher.digest(), bytes(hash_ret) + while hash_src_size > block_size: level_output_list = [] remaining = hash_src_size @@ -4304,6 +4345,9 @@ class AvbTool(object): sub_parser.add_argument('--partition_size', help='Partition size', type=parse_number) + sub_parser.add_argument('--dynamic_partition_size', + help='Calculate partition size based on image size', + action='store_true') sub_parser.add_argument('--partition_name', help='Partition name', default=None) @@ -4404,6 +4448,9 @@ class AvbTool(object): sub_parser.add_argument('--no_hashtree', action='store_true', help='Do not append hashtree') + sub_parser.add_argument('--check_at_most_once', + action='store_true', + help='Set to verify data block only once') self._add_common_args(sub_parser) self._add_common_footer_args(sub_parser) sub_parser.set_defaults(func=self.add_hashtree_footer) @@ -4729,7 +4776,7 @@ class AvbTool(object): """Implements the 'add_hash_footer' sub-command.""" args = self._fixup_common_args(args) self.avb.add_hash_footer(args.image.name if args.image else None, - args.partition_size, + args.partition_size, args.dynamic_partition_size, args.partition_name, args.hash_algorithm, args.salt, args.chain_partition, args.algorithm, args.key, @@ -4784,7 +4831,8 @@ class AvbTool(object): args.print_required_libavb_version, args.use_persistent_digest, args.do_not_use_ab, - args.no_hashtree) + args.no_hashtree, + args.check_at_most_once) def erase_footer(self, args): """Implements the 'erase_footer' sub-command.""" diff --git a/bbootimg/build.gradle.kts b/bbootimg/build.gradle.kts index 19833eb..499fb5b 100644 --- a/bbootimg/build.gradle.kts +++ b/bbootimg/build.gradle.kts @@ -15,7 +15,7 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { - kotlin("jvm") version "1.7.0" + kotlin("jvm") version "1.7.10" application } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 41dfb87..8049c68 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/helper/build.gradle.kts b/helper/build.gradle.kts index 34b9439..8618fd9 100644 --- a/helper/build.gradle.kts +++ b/helper/build.gradle.kts @@ -15,7 +15,7 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { - id("org.jetbrains.kotlin.jvm") version "1.7.0" + kotlin("jvm") version "1.7.0" `java-library` application } diff --git a/helper/src/main/kotlin/cfig/helper/Launcher.kt b/helper/src/main/kotlin/cfig/helper/Launcher.kt index a2644fc..dab8203 100644 --- a/helper/src/main/kotlin/cfig/helper/Launcher.kt +++ b/helper/src/main/kotlin/cfig/helper/Launcher.kt @@ -90,7 +90,7 @@ class Launcher { } FileOutputStream(kFile, false).use { it.write(keyPair.private.encoded) - log.info("RSA priv key(len=$kLen) written to $kFile") + log.info("RSA priv key(len=$kLen) written to $kFile.pk8") } dumpPem(PemObject("RSA PRIVATE KEY", pk8toPk1(keyPair.private).encoded), "$kFile.pem.pk1") @@ -99,6 +99,7 @@ class Launcher { it.write(keyPair.public.encoded) log.info("RSA pub key(len=$kLen) written to $kFile.pub") } + dumpPem(PemObject("RSA PUBLIC KEY", keyPair.public.encoded), "$kFile.pem.pub") } "toPub" -> { val k = CryptoHelper.KeyBox.parse4(File(kFile).readBytes()) @@ -201,4 +202,4 @@ fun main(args: Array) { log.info("Writing to " + info2.getProperty("propFile")) } } -} \ No newline at end of file +} diff --git a/tools/work_from_China.diff b/tools/work_from_China.diff new file mode 100644 index 0000000..6ede066 --- /dev/null +++ b/tools/work_from_China.diff @@ -0,0 +1,63 @@ +diff --git a/aosp/boot_signer/build.gradle.kts b/aosp/boot_signer/build.gradle.kts +index be7419c..a8826c9 100644 +--- a/aosp/boot_signer/build.gradle.kts ++++ b/aosp/boot_signer/build.gradle.kts +@@ -5,7 +5,7 @@ plugins { + } + + repositories { +- mavenCentral() ++ maven { setUrl("https://maven.aliyun.com/repository/public/") } + } + + dependencies { +diff --git a/bbootimg/build.gradle.kts b/bbootimg/build.gradle.kts +index 19833eb..f3b73a6 100644 +--- a/bbootimg/build.gradle.kts ++++ b/bbootimg/build.gradle.kts +@@ -20,7 +20,7 @@ plugins { + } + + repositories { +- mavenCentral() ++ maven { setUrl("https://maven.aliyun.com/repository/public/") } + } + + dependencies { +diff --git a/build.gradle.kts b/build.gradle.kts +index e133f36..7537313 100644 +--- a/build.gradle.kts ++++ b/build.gradle.kts +@@ -15,7 +15,7 @@ if (parseGradleVersion(gradle.gradleVersion) < 6) { + + buildscript { + repositories { +- mavenCentral() ++ maven { setUrl("https://maven.aliyun.com/repository/public/") } + } + dependencies { + classpath("org.apache.commons:commons-exec:1.3") +diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties +index 41dfb87..931fed7 100644 +--- a/gradle/wrapper/gradle-wrapper.properties ++++ b/gradle/wrapper/gradle-wrapper.properties +@@ -1,5 +1,5 @@ + distributionBase=GRADLE_USER_HOME + distributionPath=wrapper/dists +-distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip ++distributionUrl=https\://mirrors.cloud.tencent.com/gradle/gradle-7.4-bin.zip + zipStoreBase=GRADLE_USER_HOME + zipStorePath=wrapper/dists +diff --git a/helper/build.gradle.kts b/helper/build.gradle.kts +index 34b9439..2fd6e84 100644 +--- a/helper/build.gradle.kts ++++ b/helper/build.gradle.kts +@@ -21,7 +21,7 @@ plugins { + } + + repositories { +- mavenCentral() ++ maven { setUrl("https://maven.aliyun.com/repository/public/") } + } + + dependencies {