diff --git a/.travis.yml b/.travis.yml index 6b9a084..662e954 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: trusty +dist: bionic language: java before_install: - sudo apt -qq update diff --git a/README.md b/README.md index 123df56..0c5a4fe 100644 --- a/README.md +++ b/README.md @@ -111,3 +111,6 @@ https://android.googlesource.com/platform/build/+/refs/heads/master/tools/extrac mkdtboimg https://android.googlesource.com/platform/system/libufdt/ + +libsparse +https://android.googlesource.com/platform/system/core/+/refs/heads/master/libsparse/ diff --git a/aosp/boot_signer/build.gradle.kts b/aosp/boot_signer/build.gradle.kts new file mode 100644 index 0000000..0472c6e --- /dev/null +++ b/aosp/boot_signer/build.gradle.kts @@ -0,0 +1,30 @@ +import org.gradle.jvm.tasks.Jar + +plugins { + java +} + +repositories { + jcenter() +} + +dependencies { + implementation("org.bouncycastle:bcprov-jdk15on:1.57") + implementation("org.bouncycastle:bcpkix-jdk15on:1.57") +} + +val fatJar = task("fatJar", type = Jar::class) { + manifest { + attributes["Implementation-Title"] = "AOSP boot signer" + attributes["Main-Class"] = "com.android.verity.BootSignature" + } + from(configurations.runtimeClasspath.get().map({ if (it.isDirectory) it else zipTree(it) })) + excludes.addAll(mutableSetOf("META-INF/*.RSA", "META-INF/*.SF", "META-INF/*.DSA")) + with(tasks.jar.get() as CopySpec) +} + +tasks { + "build" { + dependsOn(fatJar) + } +} diff --git a/boot_signer/src/main/java/BootSignature.java b/aosp/boot_signer/src/main/java/BootSignature.java similarity index 100% rename from boot_signer/src/main/java/BootSignature.java rename to aosp/boot_signer/src/main/java/BootSignature.java diff --git a/boot_signer/src/main/java/Utils.java b/aosp/boot_signer/src/main/java/Utils.java similarity index 100% rename from boot_signer/src/main/java/Utils.java rename to aosp/boot_signer/src/main/java/Utils.java diff --git a/boot_signer/src/main/java/VeritySigner.java b/aosp/boot_signer/src/main/java/VeritySigner.java similarity index 100% rename from boot_signer/src/main/java/VeritySigner.java rename to aosp/boot_signer/src/main/java/VeritySigner.java diff --git a/aosp/libsparse/append2simg/build.gradle.kts b/aosp/libsparse/append2simg/build.gradle.kts new file mode 100644 index 0000000..8249e94 --- /dev/null +++ b/aosp/libsparse/append2simg/build.gradle.kts @@ -0,0 +1,14 @@ +plugins { + `cpp-application` +} + +application { + targetMachines.set(listOf(machines.linux.x86_64, machines.macOS.x86_64)) + dependencies { + implementation(project(":aosp:libsparse:sparse")) + } +} + +tasks.withType(LinkExecutable::class.java).configureEach { + linkerArgs.add("-lz") +} diff --git a/aosp/libsparse/append2simg/src/main/cpp/append2simg.cpp b/aosp/libsparse/append2simg/src/main/cpp/append2simg.cpp new file mode 100644 index 0000000..99f4339 --- /dev/null +++ b/aosp/libsparse/append2simg/src/main/cpp/append2simg.cpp @@ -0,0 +1,137 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _FILE_OFFSET_BITS 64 +#define _LARGEFILE64_SOURCE 1 + +#include +#include +#include +#include +#include +#include + +#include +#include "backed_block.h" +#include "sparse_file.h" + +#ifndef O_BINARY +#define O_BINARY 0 +#endif + +#if defined(__APPLE__) && defined(__MACH__) +#define lseek64 lseek +#endif +#if defined(__APPLE__) && defined(__MACH__) +#define lseek64 lseek +#define off64_t off_t +#endif + +void usage() { + fprintf(stderr, "Usage: append2simg \n"); +} + +int main(int argc, char* argv[]) { + int output; + int output_block; + char* output_path; + struct sparse_file* sparse_output; + + int input; + char* input_path; + off64_t input_len; + + int tmp_fd; + char* tmp_path; + + int ret; + + if (argc == 3) { + output_path = argv[1]; + input_path = argv[2]; + } else { + usage(); + exit(-1); + } + + ret = asprintf(&tmp_path, "%s.append2simg", output_path); + if (ret < 0) { + fprintf(stderr, "Couldn't allocate filename\n"); + exit(-1); + } + + output = open(output_path, O_RDWR | O_BINARY); + if (output < 0) { + fprintf(stderr, "Couldn't open output file (%s)\n", strerror(errno)); + exit(-1); + } + + sparse_output = sparse_file_import_auto(output, false, true); + if (!sparse_output) { + fprintf(stderr, "Couldn't import output file\n"); + exit(-1); + } + + input = open(input_path, O_RDONLY | O_BINARY); + if (input < 0) { + fprintf(stderr, "Couldn't open input file (%s)\n", strerror(errno)); + exit(-1); + } + + input_len = lseek64(input, 0, SEEK_END); + if (input_len < 0) { + fprintf(stderr, "Couldn't get input file length (%s)\n", strerror(errno)); + exit(-1); + } else if (input_len % sparse_output->block_size) { + fprintf(stderr, "Input file is not a multiple of the output file's block size"); + exit(-1); + } + lseek64(input, 0, SEEK_SET); + + output_block = sparse_output->len / sparse_output->block_size; + if (sparse_file_add_fd(sparse_output, input, 0, input_len, output_block) < 0) { + fprintf(stderr, "Couldn't add input file\n"); + exit(-1); + } + sparse_output->len += input_len; + + tmp_fd = open(tmp_path, O_WRONLY | O_CREAT | O_BINARY, 0664); + if (tmp_fd < 0) { + fprintf(stderr, "Couldn't open temporary file (%s)\n", strerror(errno)); + exit(-1); + } + + lseek64(output, 0, SEEK_SET); + if (sparse_file_write(sparse_output, tmp_fd, false, true, false) < 0) { + fprintf(stderr, "Failed to write sparse file\n"); + exit(-1); + } + + sparse_file_destroy(sparse_output); + close(tmp_fd); + close(output); + close(input); + + ret = rename(tmp_path, output_path); + if (ret < 0) { + fprintf(stderr, "Failed to rename temporary file (%s)\n", strerror(errno)); + exit(-1); + } + + free(tmp_path); + + exit(0); +} diff --git a/aosp/libsparse/base/build.gradle.kts b/aosp/libsparse/base/build.gradle.kts new file mode 100644 index 0000000..f3e15d6 --- /dev/null +++ b/aosp/libsparse/base/build.gradle.kts @@ -0,0 +1,14 @@ +plugins { + `cpp-library` +} + +library { + targetMachines.set(listOf(machines.linux.x86_64, machines.macOS.x86_64)) + linkage.set(listOf(Linkage.STATIC)) +} + +extensions.configure { + source.from(file("src/main/cpp")) + privateHeaders.from(file("src/main/headers")) + publicHeaders.from(file("src/main/public")) +} diff --git a/aosp/libsparse/base/src/main/cpp/stringprintf.cpp b/aosp/libsparse/base/src/main/cpp/stringprintf.cpp new file mode 100644 index 0000000..d12c9a2 --- /dev/null +++ b/aosp/libsparse/base/src/main/cpp/stringprintf.cpp @@ -0,0 +1,85 @@ +/* + * ===================================================================================== + * + * Filename: stringprintf.cpp + * + * Description: + * + * Version: 1.0 + * Created: 08/14/2019 11:09:23 AM + * Revision: none + * Compiler: gcc + * + * Author: YOUR NAME (), + * Organization: + * + * ===================================================================================== + */ + +#include +#include +#include + +namespace android { +namespace base { + +void StringAppendV(std::string* dst, const char* format, va_list ap) { + // First try with a small fixed size buffer + char space[1024]; + + // It's possible for methods that use a va_list to invalidate + // the data in it upon use. The fix is to make a copy + // of the structure before using it and use that copy instead. + va_list backup_ap; + va_copy(backup_ap, ap); + int result = vsnprintf(space, sizeof(space), format, backup_ap); + va_end(backup_ap); + + if (result < static_cast(sizeof(space))) { + if (result >= 0) { + // Normal case -- everything fit. + dst->append(space, result); + return; + } + + if (result < 0) { + // Just an error. + return; + } + } + + // Increase the buffer size to the size requested by vsnprintf, + // plus one for the closing \0. + int length = result + 1; + char* buf = new char[length]; + + // Restore the va_list before we use it again + va_copy(backup_ap, ap); + result = vsnprintf(buf, length, format, backup_ap); + va_end(backup_ap); + + if (result >= 0 && result < length) { + // It fit + dst->append(buf, result); + } + delete[] buf; +} + +std::string StringPrintf(const char* fmt, ...) { + va_list ap; + va_start(ap, fmt); + std::string result; + StringAppendV(&result, fmt, ap); + va_end(ap); + return result; +} + +void StringAppendF(std::string* dst, const char* format, ...) { + va_list ap; + va_start(ap, format); + StringAppendV(dst, format, ap); + va_end(ap); +} + +} // namespace base +} // namespace android diff --git a/aosp/libsparse/base/src/main/public/android-base/stringprintf.h b/aosp/libsparse/base/src/main/public/android-base/stringprintf.h new file mode 100644 index 0000000..93c56af --- /dev/null +++ b/aosp/libsparse/base/src/main/public/android-base/stringprintf.h @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +namespace android { +namespace base { + +// These printf-like functions are implemented in terms of vsnprintf, so they +// use the same attribute for compile-time format string checking. + +// Returns a string corresponding to printf-like formatting of the arguments. +std::string StringPrintf(const char* fmt, ...) __attribute__((__format__(__printf__, 1, 2))); + +// Appends a printf-like formatting of the arguments to 'dst'. +void StringAppendF(std::string* dst, const char* fmt, ...) + __attribute__((__format__(__printf__, 2, 3))); + +// Appends a printf-like formatting of the arguments to 'dst'. +void StringAppendV(std::string* dst, const char* format, va_list ap) + __attribute__((__format__(__printf__, 2, 0))); + +} // namespace base +} // namespace android diff --git a/aosp/libsparse/base/src/test/cpp/hello_test.cpp b/aosp/libsparse/base/src/test/cpp/hello_test.cpp new file mode 100644 index 0000000..971ca23 --- /dev/null +++ b/aosp/libsparse/base/src/test/cpp/hello_test.cpp @@ -0,0 +1,9 @@ +/* + * This C++ source file was generated by the Gradle 'init' task. + */ + +#include + +int main() { + return 0; +} diff --git a/aosp/libsparse/img2simg/build.gradle.kts b/aosp/libsparse/img2simg/build.gradle.kts new file mode 100644 index 0000000..8249e94 --- /dev/null +++ b/aosp/libsparse/img2simg/build.gradle.kts @@ -0,0 +1,14 @@ +plugins { + `cpp-application` +} + +application { + targetMachines.set(listOf(machines.linux.x86_64, machines.macOS.x86_64)) + dependencies { + implementation(project(":aosp:libsparse:sparse")) + } +} + +tasks.withType(LinkExecutable::class.java).configureEach { + linkerArgs.add("-lz") +} diff --git a/aosp/libsparse/img2simg/src/main/cpp/img2simg.cpp b/aosp/libsparse/img2simg/src/main/cpp/img2simg.cpp new file mode 100644 index 0000000..4c2c6ca --- /dev/null +++ b/aosp/libsparse/img2simg/src/main/cpp/img2simg.cpp @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _FILE_OFFSET_BITS 64 +#define _LARGEFILE64_SOURCE 1 + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifndef O_BINARY +#define O_BINARY 0 +#endif + +#if defined(__APPLE__) && defined(__MACH__) +#define lseek64 lseek +#define off64_t off_t +#endif + +void usage() { + fprintf(stderr, "Usage: img2simg []\n"); +} + +int main(int argc, char* argv[]) { + int in; + int out; + int ret; + struct sparse_file* s; + unsigned int block_size = 4096; + off64_t len; + + if (argc < 3 || argc > 4) { + usage(); + exit(-1); + } + + if (argc == 4) { + block_size = atoi(argv[3]); + } + + if (block_size < 1024 || block_size % 4 != 0) { + usage(); + exit(-1); + } + + if (strcmp(argv[1], "-") == 0) { + in = STDIN_FILENO; + } else { + in = open(argv[1], O_RDONLY | O_BINARY); + if (in < 0) { + fprintf(stderr, "Cannot open input file %s\n", argv[1]); + exit(-1); + } + } + + if (strcmp(argv[2], "-") == 0) { + out = STDOUT_FILENO; + } else { + out = open(argv[2], O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0664); + if (out < 0) { + fprintf(stderr, "Cannot open output file %s\n", argv[2]); + exit(-1); + } + } + + len = lseek64(in, 0, SEEK_END); + lseek64(in, 0, SEEK_SET); + + s = sparse_file_new(block_size, len); + if (!s) { + fprintf(stderr, "Failed to create sparse file\n"); + exit(-1); + } + + sparse_file_verbose(s); + ret = sparse_file_read(s, in, false, false); + if (ret) { + fprintf(stderr, "Failed to read file\n"); + exit(-1); + } + + ret = sparse_file_write(s, out, false, true, false); + if (ret) { + fprintf(stderr, "Failed to write sparse file\n"); + exit(-1); + } + + close(in); + close(out); + + exit(0); +} diff --git a/aosp/libsparse/simg2img/build.gradle.kts b/aosp/libsparse/simg2img/build.gradle.kts new file mode 100644 index 0000000..d8e587c --- /dev/null +++ b/aosp/libsparse/simg2img/build.gradle.kts @@ -0,0 +1,17 @@ +plugins { + `cpp-application` +} + +application { + targetMachines.set(listOf(machines.linux.x86_64, machines.macOS.x86_64)) + dependencies { + implementation(project(":aosp:libsparse:sparse")) + } +} + +tasks.withType(LinkExecutable::class.java).configureEach { + linkerArgs.add("-lz") +} + +tasks.register("hello") { +} diff --git a/aosp/libsparse/simg2img/src/main/cpp/simg2img.cpp b/aosp/libsparse/simg2img/src/main/cpp/simg2img.cpp new file mode 100644 index 0000000..8ba5f69 --- /dev/null +++ b/aosp/libsparse/simg2img/src/main/cpp/simg2img.cpp @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef O_BINARY +#define O_BINARY 0 +#endif + +void usage() { + fprintf(stderr, "Usage: simg2img \n"); +} + +int main(int argc, char* argv[]) { + int in; + int out; + int i; + struct sparse_file* s; + + if (argc < 3) { + usage(); + exit(-1); + } + + out = open(argv[argc - 1], O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0664); + if (out < 0) { + fprintf(stderr, "Cannot open output file %s\n", argv[argc - 1]); + exit(-1); + } + + for (i = 1; i < argc - 1; i++) { + if (strcmp(argv[i], "-") == 0) { + in = STDIN_FILENO; + } else { + in = open(argv[i], O_RDONLY | O_BINARY); + if (in < 0) { + fprintf(stderr, "Cannot open input file %s\n", argv[i]); + exit(-1); + } + } + + s = sparse_file_import(in, true, false); + if (!s) { + fprintf(stderr, "Failed to read sparse file\n"); + exit(-1); + } + + if (lseek(out, 0, SEEK_SET) == -1) { + perror("lseek failed"); + exit(EXIT_FAILURE); + } + + if (sparse_file_write(s, out, false, false, false) < 0) { + fprintf(stderr, "Cannot write output file\n"); + exit(-1); + } + sparse_file_destroy(s); + close(in); + } + + close(out); + + exit(0); +} diff --git a/aosp/libsparse/simg2simg/build.gradle.kts b/aosp/libsparse/simg2simg/build.gradle.kts new file mode 100644 index 0000000..8249e94 --- /dev/null +++ b/aosp/libsparse/simg2simg/build.gradle.kts @@ -0,0 +1,14 @@ +plugins { + `cpp-application` +} + +application { + targetMachines.set(listOf(machines.linux.x86_64, machines.macOS.x86_64)) + dependencies { + implementation(project(":aosp:libsparse:sparse")) + } +} + +tasks.withType(LinkExecutable::class.java).configureEach { + linkerArgs.add("-lz") +} diff --git a/aosp/libsparse/simg2simg/src/main/cpp/simg2simg.cpp b/aosp/libsparse/simg2simg/src/main/cpp/simg2simg.cpp new file mode 100644 index 0000000..41f9ebf --- /dev/null +++ b/aosp/libsparse/simg2simg/src/main/cpp/simg2simg.cpp @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _FILE_OFFSET_BITS 64 +#define _LARGEFILE64_SOURCE 1 + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifndef O_BINARY +#define O_BINARY 0 +#endif + +void usage() { + fprintf(stderr, "Usage: simg2simg \n"); +} + +int main(int argc, char* argv[]) { + int in; + int out; + int i; + int ret; + struct sparse_file* s; + int64_t max_size; + struct sparse_file** out_s; + int files; + char filename[4096]; + + if (argc != 4) { + usage(); + exit(-1); + } + + max_size = atoll(argv[3]); + + in = open(argv[1], O_RDONLY | O_BINARY); + if (in < 0) { + fprintf(stderr, "Cannot open input file %s\n", argv[1]); + exit(-1); + } + + s = sparse_file_import(in, true, false); + if (!s) { + fprintf(stderr, "Failed to import sparse file\n"); + exit(-1); + } + + files = sparse_file_resparse(s, max_size, nullptr, 0); + if (files < 0) { + fprintf(stderr, "Failed to resparse\n"); + exit(-1); + } + + out_s = (sparse_file**) calloc(sizeof(struct sparse_file*), files); + if (!out_s) { + fprintf(stderr, "Failed to allocate sparse file array\n"); + exit(-1); + } + + files = sparse_file_resparse(s, max_size, out_s, files); + if (files < 0) { + fprintf(stderr, "Failed to resparse\n"); + exit(-1); + } + + for (i = 0; i < files; i++) { + ret = snprintf(filename, sizeof(filename), "%s.%d", argv[2], i); + if (ret >= (int)sizeof(filename)) { + fprintf(stderr, "Filename too long\n"); + exit(-1); + } + + out = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0664); + if (out < 0) { + fprintf(stderr, "Cannot open output file %s\n", argv[2]); + exit(-1); + } + + ret = sparse_file_write(out_s[i], out, false, true, false); + if (ret) { + fprintf(stderr, "Failed to write sparse file\n"); + exit(-1); + } + close(out); + } + + close(in); + + exit(0); +} diff --git a/aosp/libsparse/sparse/build.gradle.kts b/aosp/libsparse/sparse/build.gradle.kts new file mode 100644 index 0000000..ae504c2 --- /dev/null +++ b/aosp/libsparse/sparse/build.gradle.kts @@ -0,0 +1,24 @@ +plugins { + `cpp-library` +} + +library { + targetMachines.set(listOf(machines.linux.x86_64, machines.macOS.x86_64)) + linkage.set(listOf(Linkage.STATIC)) + dependencies { + implementation(project(":aosp:libsparse:base")) + } +} + +tasks.withType(CppCompile::class.java).configureEach { + macros.put("NDEBUG", null) + compilerArgs.add("-Wall") + compilerArgs.add("-std=c++17") +} + +tasks.withType(LinkSharedLibrary::class.java).configureEach { + linkerArgs.add("-lz") +} + +tasks.withType(CreateStaticLibrary::class.java).configureEach { +} diff --git a/aosp/libsparse/sparse/src/main/cpp/Android.bp b/aosp/libsparse/sparse/src/main/cpp/Android.bp new file mode 100644 index 0000000..2ec4754 --- /dev/null +++ b/aosp/libsparse/sparse/src/main/cpp/Android.bp @@ -0,0 +1,84 @@ +// Copyright 2010 The Android Open Source Project + +cc_library { + name: "libsparse", + host_supported: true, + recovery_available: true, + unique_host_soname: true, + srcs: [ + "backed_block.cpp", + "output_file.cpp", + "sparse.cpp", + "sparse_crc32.cpp", + "sparse_err.cpp", + "sparse_read.cpp", + ], + cflags: ["-Werror"], + local_include_dirs: ["include"], + export_include_dirs: ["include"], + shared_libs: [ + "libz", + "libbase", + ], + target: { + windows: { + enabled: true, + }, + }, +} + +cc_binary { + name: "simg2img", + host_supported: true, + srcs: [ + "simg2img.cpp", + "sparse_crc32.cpp", + ], + static_libs: [ + "libsparse", + "libz", + "libbase", + ], + + cflags: ["-Werror"], +} + +cc_binary { + name: "img2simg", + host_supported: true, + srcs: ["img2simg.cpp"], + static_libs: [ + "libsparse", + "libz", + "libbase", + ], + + cflags: ["-Werror"], +} + +cc_binary_host { + name: "append2simg", + srcs: ["append2simg.cpp"], + static_libs: [ + "libsparse", + "libz", + "libbase", + ], + + cflags: ["-Werror"], +} + +python_binary_host { + name: "simg_dump.py", + main: "simg_dump.py", + srcs: ["simg_dump.py"], + version: { + py2: { + embedded_launcher: true, + enabled: true, + }, + py3: { + enabled: false, + }, + }, +} diff --git a/aosp/libsparse/sparse/src/main/cpp/backed_block.cpp b/aosp/libsparse/sparse/src/main/cpp/backed_block.cpp new file mode 100644 index 0000000..f3d8022 --- /dev/null +++ b/aosp/libsparse/sparse/src/main/cpp/backed_block.cpp @@ -0,0 +1,380 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "backed_block.h" +#include "sparse_defs.h" + +struct backed_block { + unsigned int block; + unsigned int len; + enum backed_block_type type; + union { + struct { + void* data; + } data; + struct { + char* filename; + int64_t offset; + } file; + struct { + int fd; + int64_t offset; + } fd; + struct { + uint32_t val; + } fill; + }; + struct backed_block* next; +}; + +struct backed_block_list { + struct backed_block* data_blocks; + struct backed_block* last_used; + unsigned int block_size; +}; + +struct backed_block* backed_block_iter_new(struct backed_block_list* bbl) { + return bbl->data_blocks; +} + +struct backed_block* backed_block_iter_next(struct backed_block* bb) { + return bb->next; +} + +unsigned int backed_block_len(struct backed_block* bb) { + return bb->len; +} + +unsigned int backed_block_block(struct backed_block* bb) { + return bb->block; +} + +void* backed_block_data(struct backed_block* bb) { + assert(bb->type == BACKED_BLOCK_DATA); + return bb->data.data; +} + +const char* backed_block_filename(struct backed_block* bb) { + assert(bb->type == BACKED_BLOCK_FILE); + return bb->file.filename; +} + +int backed_block_fd(struct backed_block* bb) { + assert(bb->type == BACKED_BLOCK_FD); + return bb->fd.fd; +} + +int64_t backed_block_file_offset(struct backed_block* bb) { + assert(bb->type == BACKED_BLOCK_FILE || bb->type == BACKED_BLOCK_FD); + if (bb->type == BACKED_BLOCK_FILE) { + return bb->file.offset; + } else { /* bb->type == BACKED_BLOCK_FD */ + return bb->fd.offset; + } +} + +uint32_t backed_block_fill_val(struct backed_block* bb) { + assert(bb->type == BACKED_BLOCK_FILL); + return bb->fill.val; +} + +enum backed_block_type backed_block_type(struct backed_block* bb) { + return bb->type; +} + +void backed_block_destroy(struct backed_block* bb) { + if (bb->type == BACKED_BLOCK_FILE) { + free(bb->file.filename); + } + + free(bb); +} + +struct backed_block_list* backed_block_list_new(unsigned int block_size) { + struct backed_block_list* b = + reinterpret_cast(calloc(sizeof(struct backed_block_list), 1)); + b->block_size = block_size; + return b; +} + +void backed_block_list_destroy(struct backed_block_list* bbl) { + if (bbl->data_blocks) { + struct backed_block* bb = bbl->data_blocks; + while (bb) { + struct backed_block* next = bb->next; + backed_block_destroy(bb); + bb = next; + } + } + + free(bbl); +} + +void backed_block_list_move(struct backed_block_list* from, struct backed_block_list* to, + struct backed_block* start, struct backed_block* end) { + struct backed_block* bb; + + if (start == nullptr) { + start = from->data_blocks; + } + + if (!end) { + for (end = start; end && end->next; end = end->next) + ; + } + + if (start == nullptr || end == nullptr) { + return; + } + + from->last_used = nullptr; + to->last_used = nullptr; + if (from->data_blocks == start) { + from->data_blocks = end->next; + } else { + for (bb = from->data_blocks; bb; bb = bb->next) { + if (bb->next == start) { + bb->next = end->next; + break; + } + } + } + + if (!to->data_blocks) { + to->data_blocks = start; + end->next = nullptr; + } else { + for (bb = to->data_blocks; bb; bb = bb->next) { + if (!bb->next || bb->next->block > start->block) { + end->next = bb->next; + bb->next = start; + break; + } + } + } +} + +/* may free b */ +static int merge_bb(struct backed_block_list* bbl, struct backed_block* a, struct backed_block* b) { + unsigned int block_len; + + /* Block doesn't exist (possible if one block is the last block) */ + if (!a || !b) { + return -EINVAL; + } + + assert(a->block < b->block); + + /* Blocks are of different types */ + if (a->type != b->type) { + return -EINVAL; + } + + /* Blocks are not adjacent */ + block_len = a->len / bbl->block_size; /* rounds down */ + if (a->block + block_len != b->block) { + return -EINVAL; + } + + switch (a->type) { + case BACKED_BLOCK_DATA: + /* Don't support merging data for now */ + return -EINVAL; + case BACKED_BLOCK_FILL: + if (a->fill.val != b->fill.val) { + return -EINVAL; + } + break; + case BACKED_BLOCK_FILE: + /* Already make sure b->type is BACKED_BLOCK_FILE */ + if (strcmp(a->file.filename, b->file.filename) || a->file.offset + a->len != b->file.offset) { + return -EINVAL; + } + break; + case BACKED_BLOCK_FD: + if (a->fd.fd != b->fd.fd || a->fd.offset + a->len != b->fd.offset) { + return -EINVAL; + } + break; + } + + /* Blocks are compatible and adjacent, with a before b. Merge b into a, + * and free b */ + a->len += b->len; + a->next = b->next; + + backed_block_destroy(b); + + return 0; +} + +static int queue_bb(struct backed_block_list* bbl, struct backed_block* new_bb) { + struct backed_block* bb; + + if (bbl->data_blocks == nullptr) { + bbl->data_blocks = new_bb; + return 0; + } + + if (bbl->data_blocks->block > new_bb->block) { + new_bb->next = bbl->data_blocks; + bbl->data_blocks = new_bb; + return 0; + } + + /* Optimization: blocks are mostly queued in sequence, so save the + pointer to the last bb that was added, and start searching from + there if the next block number is higher */ + if (bbl->last_used && new_bb->block > bbl->last_used->block) + bb = bbl->last_used; + else + bb = bbl->data_blocks; + bbl->last_used = new_bb; + + for (; bb->next && bb->next->block < new_bb->block; bb = bb->next) + ; + + if (bb->next == nullptr) { + bb->next = new_bb; + } else { + new_bb->next = bb->next; + bb->next = new_bb; + } + + merge_bb(bbl, new_bb, new_bb->next); + if (!merge_bb(bbl, bb, new_bb)) { + /* new_bb destroyed, point to retained as last_used */ + bbl->last_used = bb; + } + + return 0; +} + +/* Queues a fill block of memory to be written to the specified data blocks */ +int backed_block_add_fill(struct backed_block_list* bbl, unsigned int fill_val, unsigned int len, + unsigned int block) { + struct backed_block* bb = reinterpret_cast(calloc(1, sizeof(struct backed_block))); + if (bb == nullptr) { + return -ENOMEM; + } + + bb->block = block; + bb->len = len; + bb->type = BACKED_BLOCK_FILL; + bb->fill.val = fill_val; + bb->next = nullptr; + + return queue_bb(bbl, bb); +} + +/* Queues a block of memory to be written to the specified data blocks */ +int backed_block_add_data(struct backed_block_list* bbl, void* data, unsigned int len, + unsigned int block) { + struct backed_block* bb = reinterpret_cast(calloc(1, sizeof(struct backed_block))); + if (bb == nullptr) { + return -ENOMEM; + } + + bb->block = block; + bb->len = len; + bb->type = BACKED_BLOCK_DATA; + bb->data.data = data; + bb->next = nullptr; + + return queue_bb(bbl, bb); +} + +/* Queues a chunk of a file on disk to be written to the specified data blocks */ +int backed_block_add_file(struct backed_block_list* bbl, const char* filename, int64_t offset, + unsigned int len, unsigned int block) { + struct backed_block* bb = reinterpret_cast(calloc(1, sizeof(struct backed_block))); + if (bb == nullptr) { + return -ENOMEM; + } + + bb->block = block; + bb->len = len; + bb->type = BACKED_BLOCK_FILE; + bb->file.filename = strdup(filename); + bb->file.offset = offset; + bb->next = nullptr; + + return queue_bb(bbl, bb); +} + +/* Queues a chunk of a fd to be written to the specified data blocks */ +int backed_block_add_fd(struct backed_block_list* bbl, int fd, int64_t offset, unsigned int len, + unsigned int block) { + struct backed_block* bb = reinterpret_cast(calloc(1, sizeof(struct backed_block))); + if (bb == nullptr) { + return -ENOMEM; + } + + bb->block = block; + bb->len = len; + bb->type = BACKED_BLOCK_FD; + bb->fd.fd = fd; + bb->fd.offset = offset; + bb->next = nullptr; + + return queue_bb(bbl, bb); +} + +int backed_block_split(struct backed_block_list* bbl, struct backed_block* bb, + unsigned int max_len) { + struct backed_block* new_bb; + + max_len = ALIGN_DOWN(max_len, bbl->block_size); + + if (bb->len <= max_len) { + return 0; + } + + new_bb = reinterpret_cast(malloc(sizeof(struct backed_block))); + if (new_bb == nullptr) { + return -ENOMEM; + } + + *new_bb = *bb; + + new_bb->len = bb->len - max_len; + new_bb->block = bb->block + max_len / bbl->block_size; + new_bb->next = bb->next; + bb->next = new_bb; + bb->len = max_len; + + switch (bb->type) { + case BACKED_BLOCK_DATA: + new_bb->data.data = (char*)bb->data.data + max_len; + break; + case BACKED_BLOCK_FILE: + new_bb->file.offset += max_len; + break; + case BACKED_BLOCK_FD: + new_bb->fd.offset += max_len; + break; + case BACKED_BLOCK_FILL: + break; + } + + return 0; +} diff --git a/aosp/libsparse/sparse/src/main/cpp/defs.h b/aosp/libsparse/sparse/src/main/cpp/defs.h new file mode 100644 index 0000000..28e5cab --- /dev/null +++ b/aosp/libsparse/sparse/src/main/cpp/defs.h @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _LIBSPARSE_DEFS_H_ + +#ifndef __unused +#define __unused __attribute__((__unused__)) +#endif + +#endif /* _LIBSPARSE_DEFS_H_ */ diff --git a/aosp/libsparse/sparse/src/main/cpp/output_file.cpp b/aosp/libsparse/sparse/src/main/cpp/output_file.cpp new file mode 100644 index 0000000..c5c4960 --- /dev/null +++ b/aosp/libsparse/sparse/src/main/cpp/output_file.cpp @@ -0,0 +1,714 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _FILE_OFFSET_BITS 64 +#define _LARGEFILE64_SOURCE 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "defs.h" +#include "output_file.h" +#include "sparse_crc32.h" +#include "sparse_format.h" + +#ifndef _WIN32 +#include +#define O_BINARY 0 +#else +#define ftruncate64 ftruncate +#endif + +#if defined(__APPLE__) && defined(__MACH__) +#define lseek64 lseek +#define ftruncate64 ftruncate +#define mmap64 mmap +#define off64_t off_t +#endif + +#define SPARSE_HEADER_MAJOR_VER 1 +#define SPARSE_HEADER_MINOR_VER 0 +#define SPARSE_HEADER_LEN (sizeof(sparse_header_t)) +#define CHUNK_HEADER_LEN (sizeof(chunk_header_t)) + +#define container_of(inner, outer_t, elem) ((outer_t*)((char*)(inner)-offsetof(outer_t, elem))) + +struct output_file_ops { + int (*open)(struct output_file*, int fd); + int (*skip)(struct output_file*, int64_t); + int (*pad)(struct output_file*, int64_t); + int (*write)(struct output_file*, void*, size_t); + void (*close)(struct output_file*); +}; + +struct sparse_file_ops { + int (*write_data_chunk)(struct output_file* out, unsigned int len, void* data); + int (*write_fill_chunk)(struct output_file* out, unsigned int len, uint32_t fill_val); + int (*write_skip_chunk)(struct output_file* out, int64_t len); + int (*write_end_chunk)(struct output_file* out); +}; + +struct output_file { + int64_t cur_out_ptr; + unsigned int chunk_cnt; + uint32_t crc32; + struct output_file_ops* ops; + struct sparse_file_ops* sparse_ops; + int use_crc; + unsigned int block_size; + int64_t len; + char* zero_buf; + uint32_t* fill_buf; + char* buf; +}; + +struct output_file_gz { + struct output_file out; + gzFile gz_fd; +}; + +#define to_output_file_gz(_o) container_of((_o), struct output_file_gz, out) + +struct output_file_normal { + struct output_file out; + int fd; +}; + +#define to_output_file_normal(_o) container_of((_o), struct output_file_normal, out) + +struct output_file_callback { + struct output_file out; + void* priv; + int (*write)(void* priv, const void* buf, size_t len); +}; + +#define to_output_file_callback(_o) container_of((_o), struct output_file_callback, out) + +static int file_open(struct output_file* out, int fd) { + struct output_file_normal* outn = to_output_file_normal(out); + + outn->fd = fd; + return 0; +} + +static int file_skip(struct output_file* out, int64_t cnt) { + off64_t ret; + struct output_file_normal* outn = to_output_file_normal(out); + + ret = lseek64(outn->fd, cnt, SEEK_CUR); + if (ret < 0) { + error_errno("lseek64"); + return -1; + } + return 0; +} + +static int file_pad(struct output_file* out, int64_t len) { + int ret; + struct output_file_normal* outn = to_output_file_normal(out); + + ret = ftruncate64(outn->fd, len); + if (ret < 0) { + return -errno; + } + + return 0; +} + +static int file_write(struct output_file* out, void* data, size_t len) { + ssize_t ret; + struct output_file_normal* outn = to_output_file_normal(out); + + while (len > 0) { + ret = write(outn->fd, data, len); + if (ret < 0) { + if (errno == EINTR) { + continue; + } + error_errno("write"); + return -1; + } + + data = (char*)data + ret; + len -= ret; + } + + return 0; +} + +static void file_close(struct output_file* out) { + struct output_file_normal* outn = to_output_file_normal(out); + + free(outn); +} + +static struct output_file_ops file_ops = { + .open = file_open, + .skip = file_skip, + .pad = file_pad, + .write = file_write, + .close = file_close, +}; + +static int gz_file_open(struct output_file* out, int fd) { + struct output_file_gz* outgz = to_output_file_gz(out); + + outgz->gz_fd = gzdopen(fd, "wb9"); + if (!outgz->gz_fd) { + error_errno("gzopen"); + return -errno; + } + + return 0; +} + +static int gz_file_skip(struct output_file* out, int64_t cnt) { + off64_t ret; + struct output_file_gz* outgz = to_output_file_gz(out); + + ret = gzseek(outgz->gz_fd, cnt, SEEK_CUR); + if (ret < 0) { + error_errno("gzseek"); + return -1; + } + return 0; +} + +static int gz_file_pad(struct output_file* out, int64_t len) { + off64_t ret; + struct output_file_gz* outgz = to_output_file_gz(out); + + ret = gztell(outgz->gz_fd); + if (ret < 0) { + return -1; + } + + if (ret >= len) { + return 0; + } + + ret = gzseek(outgz->gz_fd, len - 1, SEEK_SET); + if (ret < 0) { + return -1; + } + + gzwrite(outgz->gz_fd, "", 1); + + return 0; +} + +static int gz_file_write(struct output_file* out, void* data, size_t len) { + int ret; + struct output_file_gz* outgz = to_output_file_gz(out); + + while (len > 0) { + ret = gzwrite(outgz->gz_fd, data, std::min(len, (unsigned int)INT_MAX)); + if (ret == 0) { + error("gzwrite %s", gzerror(outgz->gz_fd, nullptr)); + return -1; + } + len -= ret; + data = (char*)data + ret; + } + + return 0; +} + +static void gz_file_close(struct output_file* out) { + struct output_file_gz* outgz = to_output_file_gz(out); + + gzclose(outgz->gz_fd); + free(outgz); +} + +static struct output_file_ops gz_file_ops = { + .open = gz_file_open, + .skip = gz_file_skip, + .pad = gz_file_pad, + .write = gz_file_write, + .close = gz_file_close, +}; + +static int callback_file_open(struct output_file* out __unused, int fd __unused) { + return 0; +} + +static int callback_file_skip(struct output_file* out, int64_t off) { + struct output_file_callback* outc = to_output_file_callback(out); + int to_write; + int ret; + + while (off > 0) { + to_write = std::min(off, (int64_t)INT_MAX); + ret = outc->write(outc->priv, nullptr, to_write); + if (ret < 0) { + return ret; + } + off -= to_write; + } + + return 0; +} + +static int callback_file_pad(struct output_file* out __unused, int64_t len __unused) { + return -1; +} + +static int callback_file_write(struct output_file* out, void* data, size_t len) { + struct output_file_callback* outc = to_output_file_callback(out); + + return outc->write(outc->priv, data, len); +} + +static void callback_file_close(struct output_file* out) { + struct output_file_callback* outc = to_output_file_callback(out); + + free(outc); +} + +static struct output_file_ops callback_file_ops = { + .open = callback_file_open, + .skip = callback_file_skip, + .pad = callback_file_pad, + .write = callback_file_write, + .close = callback_file_close, +}; + +int read_all(int fd, void* buf, size_t len) { + size_t total = 0; + int ret; + char* ptr = reinterpret_cast(buf); + + while (total < len) { + ret = read(fd, ptr, len - total); + + if (ret < 0) return -errno; + + if (ret == 0) return -EINVAL; + + ptr += ret; + total += ret; + } + + return 0; +} + +static int write_sparse_skip_chunk(struct output_file* out, int64_t skip_len) { + chunk_header_t chunk_header; + int ret; + + if (skip_len % out->block_size) { + error("don't care size %" PRIi64 " is not a multiple of the block size %u", skip_len, + out->block_size); + return -1; + } + + /* We are skipping data, so emit a don't care chunk. */ + chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE; + chunk_header.reserved1 = 0; + chunk_header.chunk_sz = skip_len / out->block_size; + chunk_header.total_sz = CHUNK_HEADER_LEN; + ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); + if (ret < 0) return -1; + + out->cur_out_ptr += skip_len; + out->chunk_cnt++; + + return 0; +} + +static int write_sparse_fill_chunk(struct output_file* out, unsigned int len, uint32_t fill_val) { + chunk_header_t chunk_header; + int rnd_up_len, count; + int ret; + + /* Round up the fill length to a multiple of the block size */ + rnd_up_len = ALIGN(len, out->block_size); + + /* Finally we can safely emit a chunk of data */ + chunk_header.chunk_type = CHUNK_TYPE_FILL; + chunk_header.reserved1 = 0; + chunk_header.chunk_sz = rnd_up_len / out->block_size; + chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val); + ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); + + if (ret < 0) return -1; + ret = out->ops->write(out, &fill_val, sizeof(fill_val)); + if (ret < 0) return -1; + + if (out->use_crc) { + count = out->block_size / sizeof(uint32_t); + while (count--) out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t)); + } + + out->cur_out_ptr += rnd_up_len; + out->chunk_cnt++; + + return 0; +} + +static int write_sparse_data_chunk(struct output_file* out, unsigned int len, void* data) { + chunk_header_t chunk_header; + int rnd_up_len, zero_len; + int ret; + + /* Round up the data length to a multiple of the block size */ + rnd_up_len = ALIGN(len, out->block_size); + zero_len = rnd_up_len - len; + + /* Finally we can safely emit a chunk of data */ + chunk_header.chunk_type = CHUNK_TYPE_RAW; + chunk_header.reserved1 = 0; + chunk_header.chunk_sz = rnd_up_len / out->block_size; + chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len; + ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); + + if (ret < 0) return -1; + ret = out->ops->write(out, data, len); + if (ret < 0) return -1; + if (zero_len) { + ret = out->ops->write(out, out->zero_buf, zero_len); + if (ret < 0) return -1; + } + + if (out->use_crc) { + out->crc32 = sparse_crc32(out->crc32, data, len); + if (zero_len) out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len); + } + + out->cur_out_ptr += rnd_up_len; + out->chunk_cnt++; + + return 0; +} + +int write_sparse_end_chunk(struct output_file* out) { + chunk_header_t chunk_header; + int ret; + + if (out->use_crc) { + chunk_header.chunk_type = CHUNK_TYPE_CRC32; + chunk_header.reserved1 = 0; + chunk_header.chunk_sz = 0; + chunk_header.total_sz = CHUNK_HEADER_LEN + 4; + + ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); + if (ret < 0) { + return ret; + } + out->ops->write(out, &out->crc32, 4); + if (ret < 0) { + return ret; + } + + out->chunk_cnt++; + } + + return 0; +} + +static struct sparse_file_ops sparse_file_ops = { + .write_data_chunk = write_sparse_data_chunk, + .write_fill_chunk = write_sparse_fill_chunk, + .write_skip_chunk = write_sparse_skip_chunk, + .write_end_chunk = write_sparse_end_chunk, +}; + +static int write_normal_data_chunk(struct output_file* out, unsigned int len, void* data) { + int ret; + unsigned int rnd_up_len = ALIGN(len, out->block_size); + + ret = out->ops->write(out, data, len); + if (ret < 0) { + return ret; + } + + if (rnd_up_len > len) { + ret = out->ops->skip(out, rnd_up_len - len); + } + + return ret; +} + +static int write_normal_fill_chunk(struct output_file* out, unsigned int len, uint32_t fill_val) { + int ret; + unsigned int i; + unsigned int write_len; + + /* Initialize fill_buf with the fill_val */ + for (i = 0; i < out->block_size / sizeof(uint32_t); i++) { + out->fill_buf[i] = fill_val; + } + + while (len) { + write_len = std::min(len, out->block_size); + ret = out->ops->write(out, out->fill_buf, write_len); + if (ret < 0) { + return ret; + } + + len -= write_len; + } + + return 0; +} + +static int write_normal_skip_chunk(struct output_file* out, int64_t len) { + return out->ops->skip(out, len); +} + +int write_normal_end_chunk(struct output_file* out) { + return out->ops->pad(out, out->len); +} + +static struct sparse_file_ops normal_file_ops = { + .write_data_chunk = write_normal_data_chunk, + .write_fill_chunk = write_normal_fill_chunk, + .write_skip_chunk = write_normal_skip_chunk, + .write_end_chunk = write_normal_end_chunk, +}; + +void output_file_close(struct output_file* out) { + out->sparse_ops->write_end_chunk(out); + out->ops->close(out); +} + +static int output_file_init(struct output_file* out, int block_size, int64_t len, bool sparse, + int chunks, bool crc) { + int ret; + + out->len = len; + out->block_size = block_size; + out->cur_out_ptr = 0LL; + out->chunk_cnt = 0; + out->crc32 = 0; + out->use_crc = crc; + + out->zero_buf = reinterpret_cast(calloc(block_size, 1)); + if (!out->zero_buf) { + error_errno("malloc zero_buf"); + return -ENOMEM; + } + + out->fill_buf = reinterpret_cast(calloc(block_size, 1)); + if (!out->fill_buf) { + error_errno("malloc fill_buf"); + ret = -ENOMEM; + goto err_fill_buf; + } + + if (sparse) { + out->sparse_ops = &sparse_file_ops; + } else { + out->sparse_ops = &normal_file_ops; + } + + if (sparse) { + sparse_header_t sparse_header = { + .magic = SPARSE_HEADER_MAGIC, + .major_version = SPARSE_HEADER_MAJOR_VER, + .minor_version = SPARSE_HEADER_MINOR_VER, + .file_hdr_sz = SPARSE_HEADER_LEN, + .chunk_hdr_sz = CHUNK_HEADER_LEN, + .blk_sz = out->block_size, + .total_blks = static_cast(DIV_ROUND_UP(out->len, out->block_size)), + .total_chunks = static_cast(chunks), + .image_checksum = 0}; + + if (out->use_crc) { + sparse_header.total_chunks++; + } + + ret = out->ops->write(out, &sparse_header, sizeof(sparse_header)); + if (ret < 0) { + goto err_write; + } + } + + return 0; + +err_write: + free(out->fill_buf); +err_fill_buf: + free(out->zero_buf); + return ret; +} + +static struct output_file* output_file_new_gz(void) { + struct output_file_gz* outgz = + reinterpret_cast(calloc(1, sizeof(struct output_file_gz))); + if (!outgz) { + error_errno("malloc struct outgz"); + return nullptr; + } + + outgz->out.ops = &gz_file_ops; + + return &outgz->out; +} + +static struct output_file* output_file_new_normal(void) { + struct output_file_normal* outn = + reinterpret_cast(calloc(1, sizeof(struct output_file_normal))); + if (!outn) { + error_errno("malloc struct outn"); + return nullptr; + } + + outn->out.ops = &file_ops; + + return &outn->out; +} + +struct output_file* output_file_open_callback(int (*write)(void*, const void*, size_t), void* priv, + unsigned int block_size, int64_t len, int gz __unused, + int sparse, int chunks, int crc) { + int ret; + struct output_file_callback* outc; + + outc = + reinterpret_cast(calloc(1, sizeof(struct output_file_callback))); + if (!outc) { + error_errno("malloc struct outc"); + return nullptr; + } + + outc->out.ops = &callback_file_ops; + outc->priv = priv; + outc->write = write; + + ret = output_file_init(&outc->out, block_size, len, sparse, chunks, crc); + if (ret < 0) { + free(outc); + return nullptr; + } + + return &outc->out; +} + +struct output_file* output_file_open_fd(int fd, unsigned int block_size, int64_t len, int gz, + int sparse, int chunks, int crc) { + int ret; + struct output_file* out; + + if (gz) { + out = output_file_new_gz(); + } else { + out = output_file_new_normal(); + } + if (!out) { + return nullptr; + } + + out->ops->open(out, fd); + + ret = output_file_init(out, block_size, len, sparse, chunks, crc); + if (ret < 0) { + free(out); + return nullptr; + } + + return out; +} + +/* Write a contiguous region of data blocks from a memory buffer */ +int write_data_chunk(struct output_file* out, unsigned int len, void* data) { + return out->sparse_ops->write_data_chunk(out, len, data); +} + +/* Write a contiguous region of data blocks with a fill value */ +int write_fill_chunk(struct output_file* out, unsigned int len, uint32_t fill_val) { + return out->sparse_ops->write_fill_chunk(out, len, fill_val); +} + +int write_fd_chunk(struct output_file* out, unsigned int len, int fd, int64_t offset) { + int ret; + int64_t aligned_offset; + int aligned_diff; + uint64_t buffer_size; + char* ptr; + + aligned_offset = offset & ~(4096 - 1); + aligned_diff = offset - aligned_offset; + buffer_size = (uint64_t)len + (uint64_t)aligned_diff; + +#ifndef _WIN32 + if (buffer_size > SIZE_MAX) return -E2BIG; + char* data = + reinterpret_cast(mmap64(nullptr, buffer_size, PROT_READ, MAP_SHARED, fd, aligned_offset)); + if (data == MAP_FAILED) { + return -errno; + } + ptr = data + aligned_diff; +#else + off64_t pos; + char* data = reinterpret_cast(malloc(len)); + if (!data) { + return -errno; + } + pos = lseek64(fd, offset, SEEK_SET); + if (pos < 0) { + free(data); + return -errno; + } + ret = read_all(fd, data, len); + if (ret < 0) { + free(data); + return ret; + } + ptr = data; +#endif + + ret = out->sparse_ops->write_data_chunk(out, len, ptr); + +#ifndef _WIN32 + munmap(data, buffer_size); +#else + free(data); +#endif + + return ret; +} + +/* Write a contiguous region of data blocks from a file */ +int write_file_chunk(struct output_file* out, unsigned int len, const char* file, int64_t offset) { + int ret; + + int file_fd = open(file, O_RDONLY | O_BINARY); + if (file_fd < 0) { + return -errno; + } + + ret = write_fd_chunk(out, len, file_fd, offset); + + close(file_fd); + + return ret; +} + +int write_skip_chunk(struct output_file* out, int64_t len) { + return out->sparse_ops->write_skip_chunk(out, len); +} diff --git a/aosp/libsparse/sparse/src/main/cpp/output_file.h b/aosp/libsparse/sparse/src/main/cpp/output_file.h new file mode 100644 index 0000000..278430b --- /dev/null +++ b/aosp/libsparse/sparse/src/main/cpp/output_file.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _OUTPUT_FILE_H_ +#define _OUTPUT_FILE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +struct output_file; + +struct output_file* output_file_open_fd(int fd, unsigned int block_size, int64_t len, int gz, + int sparse, int chunks, int crc); +struct output_file* output_file_open_callback(int (*write)(void*, const void*, size_t), void* priv, + unsigned int block_size, int64_t len, int gz, + int sparse, int chunks, int crc); +int write_data_chunk(struct output_file* out, unsigned int len, void* data); +int write_fill_chunk(struct output_file* out, unsigned int len, uint32_t fill_val); +int write_file_chunk(struct output_file* out, unsigned int len, const char* file, int64_t offset); +int write_fd_chunk(struct output_file* out, unsigned int len, int fd, int64_t offset); +int write_skip_chunk(struct output_file* out, int64_t len); +void output_file_close(struct output_file* out); + +int read_all(int fd, void* buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/aosp/libsparse/sparse/src/main/cpp/simg_dump.py b/aosp/libsparse/sparse/src/main/cpp/simg_dump.py new file mode 100755 index 0000000..82a03ad --- /dev/null +++ b/aosp/libsparse/sparse/src/main/cpp/simg_dump.py @@ -0,0 +1,221 @@ +#! /usr/bin/env python + +# Copyright (C) 2012 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +import csv +import getopt +import hashlib +import posixpath +import signal +import struct +import sys + + +def usage(argv0): + print(""" +Usage: %s [-v] [-s] [-c ] sparse_image_file ... + -v verbose output + -s show sha1sum of data blocks + -c save .csv file of blocks +""" % (argv0)) + sys.exit(2) + + +def main(): + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + me = posixpath.basename(sys.argv[0]) + + # Parse the command line + verbose = 0 # -v + showhash = 0 # -s + csvfilename = None # -c + try: + opts, args = getopt.getopt(sys.argv[1:], + "vsc:", + ["verbose", "showhash", "csvfile"]) + except getopt.GetoptError, e: + print(e) + usage(me) + for o, a in opts: + if o in ("-v", "--verbose"): + verbose += 1 + elif o in ("-s", "--showhash"): + showhash = True + elif o in ("-c", "--csvfile"): + csvfilename = a + else: + print("Unrecognized option \"%s\"" % (o)) + usage(me) + + if not args: + print("No sparse_image_file specified") + usage(me) + + if csvfilename: + csvfile = open(csvfilename, "wb") + csvwriter = csv.writer(csvfile) + + output = verbose or csvfilename or showhash + + for path in args: + FH = open(path, "rb") + header_bin = FH.read(28) + header = struct.unpack(" 0: + print(" input_bytes output_blocks") + print("chunk offset number offset number") + + if csvfilename: + csvwriter.writerow(["chunk", "input offset", "input bytes", + "output offset", "output blocks", "type", "hash"]) + + offset = 0 + for i in xrange(1, total_chunks + 1): + header_bin = FH.read(12) + header = struct.unpack("<2H2I", header_bin) + chunk_type = header[0] + chunk_sz = header[2] + total_sz = header[3] + data_sz = total_sz - 12 + curhash = "" + curtype = "" + curpos = FH.tell() + + if verbose > 0: + print("%4u %10u %10u %7u %7u" % (i, curpos, data_sz, offset, chunk_sz), + end=" ") + + if chunk_type == 0xCAC1: + if data_sz != (chunk_sz * blk_sz): + print("Raw chunk input size (%u) does not match output size (%u)" + % (data_sz, chunk_sz * blk_sz)) + break + else: + curtype = "Raw data" + data = FH.read(data_sz) + if showhash: + h = hashlib.sha1() + h.update(data) + curhash = h.hexdigest() + elif chunk_type == 0xCAC2: + if data_sz != 4: + print("Fill chunk should have 4 bytes of fill, but this has %u" + % (data_sz)) + break + else: + fill_bin = FH.read(4) + fill = struct.unpack(" 0: + print("%-18s" % (curtype), end=" ") + + if verbose > 1: + header = struct.unpack("<12B", header_bin) + print(" (%02X%02X %02X%02X %02X%02X%02X%02X %02X%02X%02X%02X)" + % (header[0], header[1], header[2], header[3], + header[4], header[5], header[6], header[7], + header[8], header[9], header[10], header[11]), end=" ") + + print(curhash) + + if csvfilename: + csvwriter.writerow([i, curpos, data_sz, offset, chunk_sz, curtype, + curhash]) + + offset += chunk_sz + + if verbose > 0: + print(" %10u %7u End" % (FH.tell(), offset)) + + if total_blks != offset: + print("The header said we should have %u output blocks, but we saw %u" + % (total_blks, offset)) + + junk_len = len(FH.read()) + if junk_len: + print("There were %u bytes of extra data at the end of the file." + % (junk_len)) + + if csvfilename: + csvfile.close() + + sys.exit(0) + +if __name__ == "__main__": + main() diff --git a/aosp/libsparse/sparse/src/main/cpp/sparse.cpp b/aosp/libsparse/sparse/src/main/cpp/sparse.cpp new file mode 100644 index 0000000..cb288c5 --- /dev/null +++ b/aosp/libsparse/sparse/src/main/cpp/sparse.cpp @@ -0,0 +1,348 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include + +#include "defs.h" +#include "sparse_file.h" + +#include "backed_block.h" +#include "output_file.h" +#include "sparse_defs.h" +#include "sparse_format.h" + +struct sparse_file* sparse_file_new(unsigned int block_size, int64_t len) { + struct sparse_file* s = reinterpret_cast(calloc(sizeof(struct sparse_file), 1)); + if (!s) { + return nullptr; + } + + s->backed_block_list = backed_block_list_new(block_size); + if (!s->backed_block_list) { + free(s); + return nullptr; + } + + s->block_size = block_size; + s->len = len; + + return s; +} + +void sparse_file_destroy(struct sparse_file* s) { + backed_block_list_destroy(s->backed_block_list); + free(s); +} + +int sparse_file_add_data(struct sparse_file* s, void* data, unsigned int len, unsigned int block) { + return backed_block_add_data(s->backed_block_list, data, len, block); +} + +int sparse_file_add_fill(struct sparse_file* s, uint32_t fill_val, unsigned int len, + unsigned int block) { + return backed_block_add_fill(s->backed_block_list, fill_val, len, block); +} + +int sparse_file_add_file(struct sparse_file* s, const char* filename, int64_t file_offset, + unsigned int len, unsigned int block) { + return backed_block_add_file(s->backed_block_list, filename, file_offset, len, block); +} + +int sparse_file_add_fd(struct sparse_file* s, int fd, int64_t file_offset, unsigned int len, + unsigned int block) { + return backed_block_add_fd(s->backed_block_list, fd, file_offset, len, block); +} +unsigned int sparse_count_chunks(struct sparse_file* s) { + struct backed_block* bb; + unsigned int last_block = 0; + unsigned int chunks = 0; + + for (bb = backed_block_iter_new(s->backed_block_list); bb; bb = backed_block_iter_next(bb)) { + if (backed_block_block(bb) > last_block) { + /* If there is a gap between chunks, add a skip chunk */ + chunks++; + } + chunks++; + last_block = backed_block_block(bb) + DIV_ROUND_UP(backed_block_len(bb), s->block_size); + } + if (last_block < DIV_ROUND_UP(s->len, s->block_size)) { + chunks++; + } + + return chunks; +} + +static int sparse_file_write_block(struct output_file* out, struct backed_block* bb) { + int ret = -EINVAL; + + switch (backed_block_type(bb)) { + case BACKED_BLOCK_DATA: + ret = write_data_chunk(out, backed_block_len(bb), backed_block_data(bb)); + break; + case BACKED_BLOCK_FILE: + ret = write_file_chunk(out, backed_block_len(bb), backed_block_filename(bb), + backed_block_file_offset(bb)); + break; + case BACKED_BLOCK_FD: + ret = write_fd_chunk(out, backed_block_len(bb), backed_block_fd(bb), + backed_block_file_offset(bb)); + break; + case BACKED_BLOCK_FILL: + ret = write_fill_chunk(out, backed_block_len(bb), backed_block_fill_val(bb)); + break; + } + + return ret; +} + +static int write_all_blocks(struct sparse_file* s, struct output_file* out) { + struct backed_block* bb; + unsigned int last_block = 0; + int64_t pad; + int ret = 0; + + for (bb = backed_block_iter_new(s->backed_block_list); bb; bb = backed_block_iter_next(bb)) { + if (backed_block_block(bb) > last_block) { + unsigned int blocks = backed_block_block(bb) - last_block; + write_skip_chunk(out, (int64_t)blocks * s->block_size); + } + ret = sparse_file_write_block(out, bb); + if (ret) return ret; + last_block = backed_block_block(bb) + DIV_ROUND_UP(backed_block_len(bb), s->block_size); + } + + pad = s->len - (int64_t)last_block * s->block_size; + assert(pad >= 0); + if (pad > 0) { + write_skip_chunk(out, pad); + } + + return 0; +} + +int sparse_file_write(struct sparse_file* s, int fd, bool gz, bool sparse, bool crc) { + int ret; + int chunks; + struct output_file* out; + + chunks = sparse_count_chunks(s); + out = output_file_open_fd(fd, s->block_size, s->len, gz, sparse, chunks, crc); + + if (!out) return -ENOMEM; + + ret = write_all_blocks(s, out); + + output_file_close(out); + + return ret; +} + +int sparse_file_callback(struct sparse_file* s, bool sparse, bool crc, + int (*write)(void* priv, const void* data, size_t len), void* priv) { + int ret; + int chunks; + struct output_file* out; + + chunks = sparse_count_chunks(s); + out = output_file_open_callback(write, priv, s->block_size, s->len, false, sparse, chunks, crc); + + if (!out) return -ENOMEM; + + ret = write_all_blocks(s, out); + + output_file_close(out); + + return ret; +} + +struct chunk_data { + void* priv; + unsigned int block; + unsigned int nr_blocks; + int (*write)(void* priv, const void* data, size_t len, unsigned int block, unsigned int nr_blocks); +}; + +static int foreach_chunk_write(void* priv, const void* data, size_t len) { + struct chunk_data* chk = reinterpret_cast(priv); + + return chk->write(chk->priv, data, len, chk->block, chk->nr_blocks); +} + +int sparse_file_foreach_chunk(struct sparse_file* s, bool sparse, bool crc, + int (*write)(void* priv, const void* data, size_t len, + unsigned int block, unsigned int nr_blocks), + void* priv) { + int ret; + int chunks; + struct chunk_data chk; + struct output_file* out; + struct backed_block* bb; + + chk.priv = priv; + chk.write = write; + chk.block = chk.nr_blocks = 0; + chunks = sparse_count_chunks(s); + out = output_file_open_callback(foreach_chunk_write, &chk, s->block_size, s->len, false, sparse, + chunks, crc); + + if (!out) return -ENOMEM; + + for (bb = backed_block_iter_new(s->backed_block_list); bb; bb = backed_block_iter_next(bb)) { + chk.block = backed_block_block(bb); + chk.nr_blocks = (backed_block_len(bb) - 1) / s->block_size + 1; + ret = sparse_file_write_block(out, bb); + if (ret) return ret; + } + + output_file_close(out); + + return ret; +} + +static int out_counter_write(void* priv, const void* data __unused, size_t len) { + int64_t* count = reinterpret_cast(priv); + *count += len; + return 0; +} + +int64_t sparse_file_len(struct sparse_file* s, bool sparse, bool crc) { + int ret; + int chunks = sparse_count_chunks(s); + int64_t count = 0; + struct output_file* out; + + out = output_file_open_callback(out_counter_write, &count, s->block_size, s->len, false, sparse, + chunks, crc); + if (!out) { + return -1; + } + + ret = write_all_blocks(s, out); + + output_file_close(out); + + if (ret < 0) { + return -1; + } + + return count; +} + +unsigned int sparse_file_block_size(struct sparse_file* s) { + return s->block_size; +} + +static struct backed_block* move_chunks_up_to_len(struct sparse_file* from, struct sparse_file* to, + unsigned int len) { + int64_t count = 0; + struct output_file* out_counter; + struct backed_block* last_bb = nullptr; + struct backed_block* bb; + struct backed_block* start; + unsigned int last_block = 0; + int64_t file_len = 0; + int ret; + + /* + * overhead is sparse file header, the potential end skip + * chunk and crc chunk. + */ + int overhead = sizeof(sparse_header_t) + 2 * sizeof(chunk_header_t) + sizeof(uint32_t); + len -= overhead; + + start = backed_block_iter_new(from->backed_block_list); + out_counter = output_file_open_callback(out_counter_write, &count, to->block_size, to->len, false, + true, 0, false); + if (!out_counter) { + return nullptr; + } + + for (bb = start; bb; bb = backed_block_iter_next(bb)) { + count = 0; + if (backed_block_block(bb) > last_block) count += sizeof(chunk_header_t); + last_block = backed_block_block(bb) + DIV_ROUND_UP(backed_block_len(bb), to->block_size); + + /* will call out_counter_write to update count */ + ret = sparse_file_write_block(out_counter, bb); + if (ret) { + bb = nullptr; + goto out; + } + if (file_len + count > len) { + /* + * If the remaining available size is more than 1/8th of the + * requested size, split the chunk. Results in sparse files that + * are at least 7/8ths of the requested size + */ + file_len += sizeof(chunk_header_t); + if (!last_bb || (len - file_len > (len / 8))) { + backed_block_split(from->backed_block_list, bb, len - file_len); + last_bb = bb; + } + goto move; + } + file_len += count; + last_bb = bb; + } + +move: + backed_block_list_move(from->backed_block_list, to->backed_block_list, start, last_bb); + +out: + output_file_close(out_counter); + + return bb; +} + +int sparse_file_resparse(struct sparse_file* in_s, unsigned int max_len, struct sparse_file** out_s, + int out_s_count) { + struct backed_block* bb; + struct sparse_file* s; + struct sparse_file* tmp; + int c = 0; + + tmp = sparse_file_new(in_s->block_size, in_s->len); + if (!tmp) { + return -ENOMEM; + } + + do { + s = sparse_file_new(in_s->block_size, in_s->len); + + bb = move_chunks_up_to_len(in_s, s, max_len); + + if (c < out_s_count) { + out_s[c] = s; + } else { + backed_block_list_move(s->backed_block_list, tmp->backed_block_list, nullptr, nullptr); + sparse_file_destroy(s); + } + c++; + } while (bb); + + backed_block_list_move(tmp->backed_block_list, in_s->backed_block_list, nullptr, nullptr); + + sparse_file_destroy(tmp); + + return c; +} + +void sparse_file_verbose(struct sparse_file* s) { + s->verbose = true; +} diff --git a/aosp/libsparse/sparse/src/main/cpp/sparse_crc32.cpp b/aosp/libsparse/sparse/src/main/cpp/sparse_crc32.cpp new file mode 100644 index 0000000..267322c --- /dev/null +++ b/aosp/libsparse/sparse/src/main/cpp/sparse_crc32.cpp @@ -0,0 +1,97 @@ +/*- + * COPYRIGHT (C) 1986 Gary S. Brown. You may use this program, or + * code or tables extracted from it, as desired without restriction. + */ + +/* + * First, the polynomial itself and its table of feedback terms. The + * polynomial is + * X^32+X^26+X^23+X^22+X^16+X^12+X^11+X^10+X^8+X^7+X^5+X^4+X^2+X^1+X^0 + * + * Note that we take it "backwards" and put the highest-order term in + * the lowest-order bit. The X^32 term is "implied"; the LSB is the + * X^31 term, etc. The X^0 term (usually shown as "+1") results in + * the MSB being 1 + * + * Note that the usual hardware shift register implementation, which + * is what we're using (we're merely optimizing it by doing eight-bit + * chunks at a time) shifts bits into the lowest-order term. In our + * implementation, that means shifting towards the right. Why do we + * do it this way? Because the calculated CRC must be transmitted in + * order from highest-order term to lowest-order term. UARTs transmit + * characters in order from LSB to MSB. By storing the CRC this way + * we hand it to the UART in the order low-byte to high-byte; the UART + * sends each low-bit to hight-bit; and the result is transmission bit + * by bit from highest- to lowest-order term without requiring any bit + * shuffling on our part. Reception works similarly + * + * The feedback terms table consists of 256, 32-bit entries. Notes + * + * The table can be generated at runtime if desired; code to do so + * is shown later. It might not be obvious, but the feedback + * terms simply represent the results of eight shift/xor opera + * tions for all combinations of data and CRC register values + * + * The values must be right-shifted by eight bits by the "updcrc + * logic; the shift must be unsigned (bring in zeroes). On some + * hardware you could probably optimize the shift in assembler by + * using byte-swap instructions + * polynomial $edb88320 + * + * + * CRC32 code derived from work by Gary S. Brown. + */ + +/* Code taken from FreeBSD 8 */ +#include +#include + +static uint32_t crc32_tab[] = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, + 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, + 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, + 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, + 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, + 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, + 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, + 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, + 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, + 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, + 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, + 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, + 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, + 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, + 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, + 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, + 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, + 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d}; + +/* + * A function that calculates the CRC-32 based on the table above is + * given below for documentation purposes. An equivalent implementation + * of this function that's actually used in the kernel can be found + * in sys/libkern.h, where it can be inlined. + */ + +uint32_t sparse_crc32(uint32_t crc_in, const void* buf, size_t size) { + const uint8_t* p = reinterpret_cast(buf); + uint32_t crc; + + crc = crc_in ^ ~0U; + while (size--) crc = crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8); + return crc ^ ~0U; +} diff --git a/aosp/libsparse/sparse/src/main/cpp/sparse_crc32.h b/aosp/libsparse/sparse/src/main/cpp/sparse_crc32.h new file mode 100644 index 0000000..2702c4f --- /dev/null +++ b/aosp/libsparse/sparse/src/main/cpp/sparse_crc32.h @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _LIBSPARSE_SPARSE_CRC32_H_ +#define _LIBSPARSE_SPARSE_CRC32_H_ + +#include + +uint32_t sparse_crc32(uint32_t crc, const void* buf, size_t size); + +#endif diff --git a/aosp/libsparse/sparse/src/main/cpp/sparse_defs.h b/aosp/libsparse/sparse/src/main/cpp/sparse_defs.h new file mode 100644 index 0000000..9137805 --- /dev/null +++ b/aosp/libsparse/sparse/src/main/cpp/sparse_defs.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _LIBSPARSE_SPARSE_DEFS_ +#define _LIBSPARSE_SPARSE_DEFS_ + +#include +#include + +#define __le64 u64 +#define __le32 u32 +#define __le16 u16 + +#define __be64 u64 +#define __be32 u32 +#define __be16 u16 + +#define __u64 u64 +#define __u32 u32 +#define __u16 u16 +#define __u8 u8 + +typedef unsigned long long u64; +typedef signed long long s64; +typedef unsigned int u32; +typedef unsigned short int u16; +typedef unsigned char u8; + +#define DIV_ROUND_UP(x, y) (((x) + (y)-1) / (y)) +#define ALIGN(x, y) ((y)*DIV_ROUND_UP((x), (y))) +#define ALIGN_DOWN(x, y) ((y) * ((x) / (y))) + +#define error(fmt, args...) \ + do { \ + fprintf(stderr, "error: %s: " fmt "\n", __func__, ##args); \ + } while (0) +#define error_errno(s, args...) error(s ": %s", ##args, strerror(errno)) + +#endif diff --git a/aosp/libsparse/sparse/src/main/cpp/sparse_err.cpp b/aosp/libsparse/sparse/src/main/cpp/sparse_err.cpp new file mode 100644 index 0000000..6886d31 --- /dev/null +++ b/aosp/libsparse/sparse/src/main/cpp/sparse_err.cpp @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include + +void sparse_default_print(const char* fmt, ...) { + va_list argp; + + va_start(argp, fmt); + vfprintf(stderr, fmt, argp); + va_end(argp); +} + +void (*sparse_print_error)(const char* fmt, ...) = sparse_default_print; +void (*sparse_print_verbose)(const char* fmt, ...) = sparse_default_print; diff --git a/aosp/libsparse/sparse/src/main/cpp/sparse_format.h b/aosp/libsparse/sparse/src/main/cpp/sparse_format.h new file mode 100644 index 0000000..a8a721e --- /dev/null +++ b/aosp/libsparse/sparse/src/main/cpp/sparse_format.h @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _LIBSPARSE_SPARSE_FORMAT_H_ +#define _LIBSPARSE_SPARSE_FORMAT_H_ +#include "sparse_defs.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct sparse_header { + __le32 magic; /* 0xed26ff3a */ + __le16 major_version; /* (0x1) - reject images with higher major versions */ + __le16 minor_version; /* (0x0) - allow images with higer minor versions */ + __le16 file_hdr_sz; /* 28 bytes for first revision of the file format */ + __le16 chunk_hdr_sz; /* 12 bytes for first revision of the file format */ + __le32 blk_sz; /* block size in bytes, must be a multiple of 4 (4096) */ + __le32 total_blks; /* total blocks in the non-sparse output image */ + __le32 total_chunks; /* total chunks in the sparse input image */ + __le32 image_checksum; /* CRC32 checksum of the original data, counting "don't care" */ + /* as 0. Standard 802.3 polynomial, use a Public Domain */ + /* table implementation */ +} sparse_header_t; + +#define SPARSE_HEADER_MAGIC 0xed26ff3a + +#define CHUNK_TYPE_RAW 0xCAC1 +#define CHUNK_TYPE_FILL 0xCAC2 +#define CHUNK_TYPE_DONT_CARE 0xCAC3 +#define CHUNK_TYPE_CRC32 0xCAC4 + +typedef struct chunk_header { + __le16 chunk_type; /* 0xCAC1 -> raw; 0xCAC2 -> fill; 0xCAC3 -> don't care */ + __le16 reserved1; + __le32 chunk_sz; /* in blocks in output image */ + __le32 total_sz; /* in bytes of chunk input file including chunk header and data */ +} chunk_header_t; + +/* Following a Raw or Fill or CRC32 chunk is data. + * For a Raw chunk, it's the data in chunk_sz * blk_sz. + * For a Fill chunk, it's 4 bytes of the fill data. + * For a CRC32 chunk, it's 4 bytes of CRC32 + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/aosp/libsparse/sparse/src/main/cpp/sparse_read.cpp b/aosp/libsparse/sparse/src/main/cpp/sparse_read.cpp new file mode 100644 index 0000000..c4c1823 --- /dev/null +++ b/aosp/libsparse/sparse/src/main/cpp/sparse_read.cpp @@ -0,0 +1,577 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _FILE_OFFSET_BITS 64 +#define _LARGEFILE64_SOURCE 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "android-base/stringprintf.h" +#include "defs.h" +#include "output_file.h" +#include "sparse_crc32.h" +#include "sparse_file.h" +#include "sparse_format.h" + +#if defined(__APPLE__) && defined(__MACH__) +#define lseek64 lseek +#define off64_t off_t +#endif + +#define SPARSE_HEADER_MAJOR_VER 1 +#define SPARSE_HEADER_LEN (sizeof(sparse_header_t)) +#define CHUNK_HEADER_LEN (sizeof(chunk_header_t)) + +static constexpr int64_t COPY_BUF_SIZE = 1024 * 1024; +static char* copybuf; + +static std::string ErrorString(int err) { + if (err == -EOVERFLOW) return "EOF while reading file"; + if (err == -EINVAL) return "Invalid sparse file format"; + if (err == -ENOMEM) return "Failed allocation while reading file"; + return android::base::StringPrintf("Unknown error %d", err); +} + +class SparseFileSource { + public: + /* Seeks the source ahead by the given offset. */ + virtual void Seek(int64_t offset) = 0; + + /* Return the current offset. */ + virtual int64_t GetOffset() = 0; + + /* Set the current offset. Return 0 if successful. */ + virtual int SetOffset(int64_t offset) = 0; + + /* Adds the given length from the current offset of the source to the file at the given block. + * Return 0 if successful. */ + virtual int AddToSparseFile(struct sparse_file* s, int64_t len, unsigned int block) = 0; + + /* Get data of fixed size from the current offset and seek len bytes. Return 0 if successful. */ + virtual int ReadValue(void* ptr, int len) = 0; + + /* Find the crc32 of the next len bytes and seek ahead len bytes. Return 0 if successful. */ + virtual int GetCrc32(uint32_t* crc32, int64_t len) = 0; + + virtual ~SparseFileSource(){}; +}; + +class SparseFileFdSource : public SparseFileSource { + private: + int fd; + + public: + SparseFileFdSource(int fd) : fd(fd) {} + ~SparseFileFdSource() override {} + + void Seek(int64_t off) override { lseek64(fd, off, SEEK_CUR); } + + int64_t GetOffset() override { return lseek64(fd, 0, SEEK_CUR); } + + int SetOffset(int64_t offset) override { + return lseek64(fd, offset, SEEK_SET) == offset ? 0 : -errno; + } + + int AddToSparseFile(struct sparse_file* s, int64_t len, unsigned int block) override { + return sparse_file_add_fd(s, fd, GetOffset(), len, block); + } + + int ReadValue(void* ptr, int len) override { return read_all(fd, ptr, len); } + + int GetCrc32(uint32_t* crc32, int64_t len) override { + int chunk; + int ret; + while (len) { + chunk = std::min(len, COPY_BUF_SIZE); + ret = read_all(fd, copybuf, chunk); + if (ret < 0) { + return ret; + } + *crc32 = sparse_crc32(*crc32, copybuf, chunk); + len -= chunk; + } + return 0; + } +}; + +class SparseFileBufSource : public SparseFileSource { + private: + char* buf; + int64_t offset; + + public: + SparseFileBufSource(char* buf) : buf(buf), offset(0) {} + ~SparseFileBufSource() override {} + + void Seek(int64_t off) override { + buf += off; + offset += off; + } + + int64_t GetOffset() override { return offset; } + + int SetOffset(int64_t off) override { + buf += off - offset; + offset = off; + return 0; + } + + int AddToSparseFile(struct sparse_file* s, int64_t len, unsigned int block) override { + return sparse_file_add_data(s, buf, len, block); + } + + int ReadValue(void* ptr, int len) override { + memcpy(ptr, buf, len); + Seek(len); + return 0; + } + + int GetCrc32(uint32_t* crc32, int64_t len) override { + *crc32 = sparse_crc32(*crc32, buf, len); + Seek(len); + return 0; + } +}; + +static void verbose_error(bool verbose, int err, const char* fmt, ...) { + if (!verbose) return; + + std::string msg = ErrorString(err); + if (fmt) { + msg += " at "; + va_list argp; + va_start(argp, fmt); + android::base::StringAppendV(&msg, fmt, argp); + va_end(argp); + } + sparse_print_verbose("%s\n", msg.c_str()); +} + +static int process_raw_chunk(struct sparse_file* s, unsigned int chunk_size, + SparseFileSource* source, unsigned int blocks, unsigned int block, + uint32_t* crc32) { + int ret; + int64_t len = blocks * s->block_size; + + if (chunk_size % s->block_size != 0) { + return -EINVAL; + } + + if (chunk_size / s->block_size != blocks) { + return -EINVAL; + } + + ret = source->AddToSparseFile(s, len, block); + if (ret < 0) { + return ret; + } + + if (crc32) { + ret = source->GetCrc32(crc32, len); + if (ret < 0) { + return ret; + } + } else { + source->Seek(len); + } + + return 0; +} + +static int process_fill_chunk(struct sparse_file* s, unsigned int chunk_size, + SparseFileSource* source, unsigned int blocks, unsigned int block, + uint32_t* crc32) { + int ret; + int chunk; + int64_t len = (int64_t)blocks * s->block_size; + uint32_t fill_val; + uint32_t* fillbuf; + unsigned int i; + + if (chunk_size != sizeof(fill_val)) { + return -EINVAL; + } + + ret = source->ReadValue(&fill_val, sizeof(fill_val)); + if (ret < 0) { + return ret; + } + + ret = sparse_file_add_fill(s, fill_val, len, block); + if (ret < 0) { + return ret; + } + + if (crc32) { + /* Fill copy_buf with the fill value */ + fillbuf = (uint32_t*)copybuf; + for (i = 0; i < (COPY_BUF_SIZE / sizeof(fill_val)); i++) { + fillbuf[i] = fill_val; + } + + while (len) { + chunk = std::min(len, COPY_BUF_SIZE); + *crc32 = sparse_crc32(*crc32, copybuf, chunk); + len -= chunk; + } + } + + return 0; +} + +static int process_skip_chunk(struct sparse_file* s, unsigned int chunk_size, + SparseFileSource* source __unused, unsigned int blocks, + unsigned int block __unused, uint32_t* crc32) { + if (chunk_size != 0) { + return -EINVAL; + } + + if (crc32) { + int64_t len = (int64_t)blocks * s->block_size; + memset(copybuf, 0, COPY_BUF_SIZE); + + while (len) { + int chunk = std::min(len, COPY_BUF_SIZE); + *crc32 = sparse_crc32(*crc32, copybuf, chunk); + len -= chunk; + } + } + + return 0; +} + +static int process_crc32_chunk(SparseFileSource* source, unsigned int chunk_size, uint32_t* crc32) { + uint32_t file_crc32; + + if (chunk_size != sizeof(file_crc32)) { + return -EINVAL; + } + + int ret = source->ReadValue(&file_crc32, sizeof(file_crc32)); + if (ret < 0) { + return ret; + } + + if (crc32 != nullptr && file_crc32 != *crc32) { + return -EINVAL; + } + + return 0; +} + +static int process_chunk(struct sparse_file* s, SparseFileSource* source, unsigned int chunk_hdr_sz, + chunk_header_t* chunk_header, unsigned int cur_block, uint32_t* crc_ptr) { + int ret; + unsigned int chunk_data_size; + int64_t offset = source->GetOffset(); + + chunk_data_size = chunk_header->total_sz - chunk_hdr_sz; + + switch (chunk_header->chunk_type) { + case CHUNK_TYPE_RAW: + ret = + process_raw_chunk(s, chunk_data_size, source, chunk_header->chunk_sz, cur_block, crc_ptr); + if (ret < 0) { + verbose_error(s->verbose, ret, "data block at %" PRId64, offset); + return ret; + } + return chunk_header->chunk_sz; + case CHUNK_TYPE_FILL: + ret = process_fill_chunk(s, chunk_data_size, source, chunk_header->chunk_sz, cur_block, + crc_ptr); + if (ret < 0) { + verbose_error(s->verbose, ret, "fill block at %" PRId64, offset); + return ret; + } + return chunk_header->chunk_sz; + case CHUNK_TYPE_DONT_CARE: + ret = process_skip_chunk(s, chunk_data_size, source, chunk_header->chunk_sz, cur_block, + crc_ptr); + if (chunk_data_size != 0) { + if (ret < 0) { + verbose_error(s->verbose, ret, "skip block at %" PRId64, offset); + return ret; + } + } + return chunk_header->chunk_sz; + case CHUNK_TYPE_CRC32: + ret = process_crc32_chunk(source, chunk_data_size, crc_ptr); + if (ret < 0) { + verbose_error(s->verbose, -EINVAL, "crc block at %" PRId64, offset); + return ret; + } + return 0; + default: + verbose_error(s->verbose, -EINVAL, "unknown block %04X at %" PRId64, chunk_header->chunk_type, + offset); + } + + return 0; +} + +static int sparse_file_read_sparse(struct sparse_file* s, SparseFileSource* source, bool crc) { + int ret; + unsigned int i; + sparse_header_t sparse_header; + chunk_header_t chunk_header; + uint32_t crc32 = 0; + uint32_t* crc_ptr = nullptr; + unsigned int cur_block = 0; + + if (!copybuf) { + copybuf = (char*)malloc(COPY_BUF_SIZE); + } + + if (!copybuf) { + return -ENOMEM; + } + + if (crc) { + crc_ptr = &crc32; + } + + ret = source->ReadValue(&sparse_header, sizeof(sparse_header)); + if (ret < 0) { + return ret; + } + + if (sparse_header.magic != SPARSE_HEADER_MAGIC) { + return -EINVAL; + } + + if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) { + return -EINVAL; + } + + if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) { + return -EINVAL; + } + + if (sparse_header.chunk_hdr_sz < sizeof(chunk_header)) { + return -EINVAL; + } + + if (sparse_header.file_hdr_sz > SPARSE_HEADER_LEN) { + /* Skip the remaining bytes in a header that is longer than + * we expected. + */ + source->Seek(sparse_header.file_hdr_sz - SPARSE_HEADER_LEN); + } + + for (i = 0; i < sparse_header.total_chunks; i++) { + ret = source->ReadValue(&chunk_header, sizeof(chunk_header)); + if (ret < 0) { + return ret; + } + + if (sparse_header.chunk_hdr_sz > CHUNK_HEADER_LEN) { + /* Skip the remaining bytes in a header that is longer than + * we expected. + */ + source->Seek(sparse_header.chunk_hdr_sz - CHUNK_HEADER_LEN); + } + + ret = process_chunk(s, source, sparse_header.chunk_hdr_sz, &chunk_header, cur_block, crc_ptr); + if (ret < 0) { + return ret; + } + + cur_block += ret; + } + + if (sparse_header.total_blks != cur_block) { + return -EINVAL; + } + + return 0; +} + +static int sparse_file_read_normal(struct sparse_file* s, int fd) { + int ret; + uint32_t* buf = (uint32_t*)malloc(s->block_size); + unsigned int block = 0; + int64_t remain = s->len; + int64_t offset = 0; + unsigned int to_read; + unsigned int i; + bool sparse_block; + + if (!buf) { + return -ENOMEM; + } + + while (remain > 0) { + to_read = std::min(remain, (int64_t)(s->block_size)); + ret = read_all(fd, buf, to_read); + if (ret < 0) { + error("failed to read sparse file"); + free(buf); + return ret; + } + + if (to_read == s->block_size) { + sparse_block = true; + for (i = 1; i < s->block_size / sizeof(uint32_t); i++) { + if (buf[0] != buf[i]) { + sparse_block = false; + break; + } + } + } else { + sparse_block = false; + } + + if (sparse_block) { + /* TODO: add flag to use skip instead of fill for buf[0] == 0 */ + sparse_file_add_fill(s, buf[0], to_read, block); + } else { + sparse_file_add_fd(s, fd, offset, to_read, block); + } + + remain -= to_read; + offset += to_read; + block++; + } + + free(buf); + return 0; +} + +int sparse_file_read(struct sparse_file* s, int fd, bool sparse, bool crc) { + if (crc && !sparse) { + return -EINVAL; + } + + if (sparse) { + SparseFileFdSource source(fd); + return sparse_file_read_sparse(s, &source, crc); + } else { + return sparse_file_read_normal(s, fd); + } +} + +int sparse_file_read_buf(struct sparse_file* s, char* buf, bool crc) { + SparseFileBufSource source(buf); + return sparse_file_read_sparse(s, &source, crc); +} + +static struct sparse_file* sparse_file_import_source(SparseFileSource* source, bool verbose, + bool crc) { + int ret; + sparse_header_t sparse_header; + int64_t len; + struct sparse_file* s; + + ret = source->ReadValue(&sparse_header, sizeof(sparse_header)); + if (ret < 0) { + verbose_error(verbose, ret, "header"); + return nullptr; + } + + if (sparse_header.magic != SPARSE_HEADER_MAGIC) { + verbose_error(verbose, -EINVAL, "header magic"); + return nullptr; + } + + if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) { + verbose_error(verbose, -EINVAL, "header major version"); + return nullptr; + } + + if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) { + return nullptr; + } + + if (sparse_header.chunk_hdr_sz < sizeof(chunk_header_t)) { + return nullptr; + } + + len = (int64_t)sparse_header.total_blks * sparse_header.blk_sz; + s = sparse_file_new(sparse_header.blk_sz, len); + if (!s) { + verbose_error(verbose, -EINVAL, nullptr); + return nullptr; + } + + ret = source->SetOffset(0); + if (ret < 0) { + verbose_error(verbose, ret, "seeking"); + sparse_file_destroy(s); + return nullptr; + } + + s->verbose = verbose; + + ret = sparse_file_read_sparse(s, source, crc); + if (ret < 0) { + sparse_file_destroy(s); + return nullptr; + } + + return s; +} + +struct sparse_file* sparse_file_import(int fd, bool verbose, bool crc) { + SparseFileFdSource source(fd); + return sparse_file_import_source(&source, verbose, crc); +} + +struct sparse_file* sparse_file_import_buf(char* buf, bool verbose, bool crc) { + SparseFileBufSource source(buf); + return sparse_file_import_source(&source, verbose, crc); +} + +struct sparse_file* sparse_file_import_auto(int fd, bool crc, bool verbose) { + struct sparse_file* s; + int64_t len; + int ret; + + s = sparse_file_import(fd, verbose, crc); + if (s) { + return s; + } + + len = lseek64(fd, 0, SEEK_END); + if (len < 0) { + return nullptr; + } + + lseek64(fd, 0, SEEK_SET); + + s = sparse_file_new(4096, len); + if (!s) { + return nullptr; + } + + ret = sparse_file_read_normal(s, fd); + if (ret < 0) { + sparse_file_destroy(s); + return nullptr; + } + + return s; +} diff --git a/aosp/libsparse/sparse/src/main/public/backed_block.h b/aosp/libsparse/sparse/src/main/public/backed_block.h new file mode 100644 index 0000000..3a75460 --- /dev/null +++ b/aosp/libsparse/sparse/src/main/public/backed_block.h @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _BACKED_BLOCK_H_ +#define _BACKED_BLOCK_H_ + +#include + +struct backed_block_list; +struct backed_block; + +enum backed_block_type { + BACKED_BLOCK_DATA, + BACKED_BLOCK_FILE, + BACKED_BLOCK_FD, + BACKED_BLOCK_FILL, +}; + +int backed_block_add_data(struct backed_block_list* bbl, void* data, unsigned int len, + unsigned int block); +int backed_block_add_fill(struct backed_block_list* bbl, unsigned int fill_val, unsigned int len, + unsigned int block); +int backed_block_add_file(struct backed_block_list* bbl, const char* filename, int64_t offset, + unsigned int len, unsigned int block); +int backed_block_add_fd(struct backed_block_list* bbl, int fd, int64_t offset, unsigned int len, + unsigned int block); + +struct backed_block* backed_block_iter_new(struct backed_block_list* bbl); +struct backed_block* backed_block_iter_next(struct backed_block* bb); +unsigned int backed_block_len(struct backed_block* bb); +unsigned int backed_block_block(struct backed_block* bb); +void* backed_block_data(struct backed_block* bb); +const char* backed_block_filename(struct backed_block* bb); +int backed_block_fd(struct backed_block* bb); +int64_t backed_block_file_offset(struct backed_block* bb); +uint32_t backed_block_fill_val(struct backed_block* bb); +enum backed_block_type backed_block_type(struct backed_block* bb); +int backed_block_split(struct backed_block_list* bbl, struct backed_block* bb, unsigned int max_len); + +struct backed_block* backed_block_iter_new(struct backed_block_list* bbl); +struct backed_block* backed_block_iter_next(struct backed_block* bb); + +struct backed_block_list* backed_block_list_new(unsigned int block_size); +void backed_block_list_destroy(struct backed_block_list* bbl); + +void backed_block_list_move(struct backed_block_list* from, struct backed_block_list* to, + struct backed_block* start, struct backed_block* end); + +#endif diff --git a/aosp/libsparse/sparse/src/main/public/sparse/sparse.h b/aosp/libsparse/sparse/src/main/public/sparse/sparse.h new file mode 100644 index 0000000..3d5fb0c --- /dev/null +++ b/aosp/libsparse/sparse/src/main/public/sparse/sparse.h @@ -0,0 +1,346 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _LIBSPARSE_SPARSE_H_ +#define _LIBSPARSE_SPARSE_H_ + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct sparse_file; + +// The callbacks in sparse_file_callback() and sparse_file_foreach_chunk() take +// size_t as the length type (was `int` in past). This allows clients to keep +// their codes compatibile with both versions as needed. +#define SPARSE_CALLBACK_USES_SIZE_T + +/** + * sparse_file_new - create a new sparse file cookie + * + * @block_size - minimum size of a chunk + * @len - size of the expanded sparse file. + * + * Creates a new sparse_file cookie that can be used to associate data + * blocks. Can later be written to a file with a variety of options. + * block_size specifies the minimum size of a chunk in the file. The maximum + * size of the file is 2**32 * block_size (16TB for 4k block size). + * + * Returns the sparse file cookie, or NULL on error. + */ +struct sparse_file *sparse_file_new(unsigned int block_size, int64_t len); + +/** + * sparse_file_destroy - destroy a sparse file cookie + * + * @s - sparse file cookie + * + * Destroys a sparse file cookie. After destroy, all memory passed in to + * sparse_file_add_data can be freed by the caller + */ +void sparse_file_destroy(struct sparse_file *s); + +/** + * sparse_file_add_data - associate a data chunk with a sparse file + * + * @s - sparse file cookie + * @data - pointer to data block + * @len - length of the data block + * @block - offset in blocks into the sparse file to place the data chunk + * + * Associates a data chunk with a sparse file cookie. The region + * [block * block_size : block * block_size + len) must not already be used in + * the sparse file. If len is not a multiple of the block size the data + * will be padded with zeros. + * + * The data pointer must remain valid until the sparse file is closed or the + * data block is removed from the sparse file. + * + * Returns 0 on success, negative errno on error. + */ +int sparse_file_add_data(struct sparse_file *s, + void *data, unsigned int len, unsigned int block); + +/** + * sparse_file_add_fill - associate a fill chunk with a sparse file + * + * @s - sparse file cookie + * @fill_val - 32 bit fill data + * @len - length of the fill block + * @block - offset in blocks into the sparse file to place the fill chunk + * + * Associates a chunk filled with fill_val with a sparse file cookie. + * The region [block * block_size : block * block_size + len) must not already + * be used in the sparse file. If len is not a multiple of the block size the + * data will be padded with zeros. + * + * Returns 0 on success, negative errno on error. + */ +int sparse_file_add_fill(struct sparse_file *s, + uint32_t fill_val, unsigned int len, unsigned int block); + +/** + * sparse_file_add_file - associate a chunk of a file with a sparse file + * + * @s - sparse file cookie + * @filename - filename of the file to be copied + * @file_offset - offset into the copied file + * @len - length of the copied block + * @block - offset in blocks into the sparse file to place the file chunk + * + * Associates a chunk of an existing file with a sparse file cookie. + * The region [block * block_size : block * block_size + len) must not already + * be used in the sparse file. If len is not a multiple of the block size the + * data will be padded with zeros. + * + * Allows adding large amounts of data to a sparse file without needing to keep + * it all mapped. File size is limited by available virtual address space, + * exceptionally large files may need to be added in multiple chunks. + * + * Returns 0 on success, negative errno on error. + */ +int sparse_file_add_file(struct sparse_file *s, + const char *filename, int64_t file_offset, unsigned int len, + unsigned int block); + +/** + * sparse_file_add_file - associate a chunk of a file with a sparse file + * + * @s - sparse file cookie + * @filename - filename of the file to be copied + * @file_offset - offset into the copied file + * @len - length of the copied block + * @block - offset in blocks into the sparse file to place the file chunk + * + * Associates a chunk of an existing fd with a sparse file cookie. + * The region [block * block_size : block * block_size + len) must not already + * be used in the sparse file. If len is not a multiple of the block size the + * data will be padded with zeros. + * + * Allows adding large amounts of data to a sparse file without needing to keep + * it all mapped. File size is limited by available virtual address space, + * exceptionally large files may need to be added in multiple chunks. + * + * The fd must remain open until the sparse file is closed or the fd block is + * removed from the sparse file. + * + * Returns 0 on success, negative errno on error. + */ +int sparse_file_add_fd(struct sparse_file *s, + int fd, int64_t file_offset, unsigned int len, unsigned int block); + +/** + * sparse_file_write - write a sparse file to a file + * + * @s - sparse file cookie + * @fd - file descriptor to write to + * @gz - write a gzipped file + * @sparse - write in the Android sparse file format + * @crc - append a crc chunk + * + * Writes a sparse file to a file. If gz is true, the data will be passed + * through zlib. If sparse is true, the file will be written in the Android + * sparse file format. If sparse is false, the file will be written by seeking + * over unused chunks, producing a smaller file if the filesystem supports + * sparse files. If crc is true, the crc of the expanded data will be + * calculated and appended in a crc chunk. + * + * Returns 0 on success, negative errno on error. + */ +int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse, + bool crc); + +/** + * sparse_file_len - return the length of a sparse file if written to disk + * + * @s - sparse file cookie + * @sparse - write in the Android sparse file format + * @crc - append a crc chunk + * + * Returns the size a sparse file would be on disk if it were written in the + * specified format. If sparse is true, this is the size of the data in the + * sparse format. If sparse is false, this is the size of the normal + * non-sparse file. + */ +int64_t sparse_file_len(struct sparse_file *s, bool sparse, bool crc); + +/** + * sparse_file_block_size + * + * @s - sparse file cookie + */ +unsigned int sparse_file_block_size(struct sparse_file *s); + +/** + * sparse_file_callback - call a callback for blocks in sparse file + * + * @s - sparse file cookie + * @sparse - write in the Android sparse file format + * @crc - append a crc chunk + * @write - function to call for each block + * @priv - value that will be passed as the first argument to write + * + * Writes a sparse file by calling a callback function. If sparse is true, the + * file will be written in the Android sparse file format. If crc is true, the + * crc of the expanded data will be calculated and appended in a crc chunk. + * The callback 'write' will be called with data and length for each data, + * and with data==NULL to skip over a region (only used for non-sparse format). + * The callback should return negative on error, 0 on success. + * + * Returns 0 on success, negative errno on error. + */ +int sparse_file_callback(struct sparse_file *s, bool sparse, bool crc, + int (*write)(void *priv, const void *data, size_t len), void *priv); + +/** + * sparse_file_foreach_chunk - call a callback for data blocks in sparse file + * + * @s - sparse file cookie + * @sparse - write in the Android sparse file format + * @crc - append a crc chunk + * @write - function to call for each block + * @priv - value that will be passed as the first argument to write + * + * The function has the same behavior as 'sparse_file_callback', except it only + * iterates on blocks that contain data. + * + * Returns 0 on success, negative errno on error. + */ +int sparse_file_foreach_chunk(struct sparse_file *s, bool sparse, bool crc, + int (*write)(void *priv, const void *data, size_t len, unsigned int block, + unsigned int nr_blocks), + void *priv); +/** + * sparse_file_read - read a file into a sparse file cookie + * + * @s - sparse file cookie + * @fd - file descriptor to read from + * @sparse - read a file in the Android sparse file format + * @crc - verify the crc of a file in the Android sparse file format + * + * Reads a file into a sparse file cookie. If sparse is true, the file is + * assumed to be in the Android sparse file format. If sparse is false, the + * file will be sparsed by looking for block aligned chunks of all zeros or + * another 32 bit value. If crc is true, the crc of the sparse file will be + * verified. + * + * Returns 0 on success, negative errno on error. + */ +int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc); + +/** + * sparse_file_read_buf - read a buffer into a sparse file cookie + * + * @s - sparse file cookie + * @buf - buffer to read from + * @crc - verify the crc of a file in the Android sparse file format + * + * Reads a buffer into a sparse file cookie. The buffer must remain + * valid until the sparse file cookie is freed. If crc is true, the + * crc of the sparse file will be verified. + * + * Returns 0 on success, negative errno on error. + */ +int sparse_file_read_buf(struct sparse_file *s, char *buf, bool crc); + +/** + * sparse_file_import - import an existing sparse file + * + * @fd - file descriptor to read from + * @verbose - print verbose errors while reading the sparse file + * @crc - verify the crc of a file in the Android sparse file format + * + * Reads an existing sparse file into a sparse file cookie, recreating the same + * sparse cookie that was used to write it. If verbose is true, prints verbose + * errors when the sparse file is formatted incorrectly. + * + * Returns a new sparse file cookie on success, NULL on error. + */ +struct sparse_file *sparse_file_import(int fd, bool verbose, bool crc); + +/** + * sparse_file_import_buf - import an existing sparse file from a buffer + * + * @buf - buffer to read from + * @verbose - print verbose errors while reading the sparse file + * @crc - verify the crc of a file in the Android sparse file format + * + * Reads existing sparse file data into a sparse file cookie, recreating the same + * sparse cookie that was used to write it. If verbose is true, prints verbose + * errors when the sparse file is formatted incorrectly. + * + * Returns a new sparse file cookie on success, NULL on error. + */ +struct sparse_file *sparse_file_import_buf(char* buf, bool verbose, bool crc); + +/** + * sparse_file_import_auto - import an existing sparse or normal file + * + * @fd - file descriptor to read from + * @crc - verify the crc of a file in the Android sparse file format + * @verbose - whether to use verbose logging + * + * Reads an existing sparse or normal file into a sparse file cookie. + * Attempts to determine if the file is sparse or not by looking for the sparse + * file magic number in the first 4 bytes. If the file is not sparse, the file + * will be sparsed by looking for block aligned chunks of all zeros or another + * 32 bit value. If crc is true, the crc of the sparse file will be verified. + * + * Returns a new sparse file cookie on success, NULL on error. + */ +struct sparse_file *sparse_file_import_auto(int fd, bool crc, bool verbose); + +/** sparse_file_resparse - rechunk an existing sparse file into smaller files + * + * @in_s - sparse file cookie of the existing sparse file + * @max_len - maximum file size + * @out_s - array of sparse file cookies + * @out_s_count - size of out_s array + * + * Splits chunks of an existing sparse file into smaller sparse files such that + * each sparse file is less than max_len. Returns the number of sparse_files + * that would have been written to out_s if out_s were big enough. + */ +int sparse_file_resparse(struct sparse_file *in_s, unsigned int max_len, + struct sparse_file **out_s, int out_s_count); + +/** + * sparse_file_verbose - set a sparse file cookie to print verbose errors + * + * @s - sparse file cookie + * + * Print verbose sparse file errors whenever using the sparse file cookie. + */ +void sparse_file_verbose(struct sparse_file *s); + +/** + * sparse_print_verbose - function called to print verbose errors + * + * By default, verbose errors will print to standard error. + * sparse_print_verbose may be overridden to log verbose errors somewhere else. + * + */ +extern void (*sparse_print_verbose)(const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/aosp/libsparse/sparse/src/main/public/sparse_file.h b/aosp/libsparse/sparse/src/main/public/sparse_file.h new file mode 100644 index 0000000..e565f63 --- /dev/null +++ b/aosp/libsparse/sparse/src/main/public/sparse_file.h @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _LIBSPARSE_SPARSE_FILE_H_ +#define _LIBSPARSE_SPARSE_FILE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +struct sparse_file { + unsigned int block_size; + int64_t len; + bool verbose; + + struct backed_block_list* backed_block_list; + struct output_file* out; +}; + +#ifdef __cplusplus +} +#endif + +#endif /* _LIBSPARSE_SPARSE_FILE_H_ */ diff --git a/aosp/mkbootfs/build.gradle.kts b/aosp/mkbootfs/build.gradle.kts new file mode 100644 index 0000000..c7966c1 --- /dev/null +++ b/aosp/mkbootfs/build.gradle.kts @@ -0,0 +1,14 @@ +plugins { + `cpp-application` +} + +application { + targetMachines.set(listOf(machines.linux.x86_64, machines.macOS.x86_64)) +} + +tasks.withType(CppCompile::class.java).configureEach { + macros.put("__ANDROID_VNDK__", null) + //macros.put("CFIG_NO_FIX_STAT", 1) + compilerArgs.add("-std=c++17") + compilerArgs.add("-Wno-write-strings") +} diff --git a/mkbootfs/src/mkbootfs/cpp/fs_config.cpp b/aosp/mkbootfs/src/main/cpp/fs_config.cpp similarity index 100% rename from mkbootfs/src/mkbootfs/cpp/fs_config.cpp rename to aosp/mkbootfs/src/main/cpp/fs_config.cpp diff --git a/mkbootfs/src/mkbootfs/cpp/mkbootfs.c b/aosp/mkbootfs/src/main/cpp/mkbootfs.cpp similarity index 99% rename from mkbootfs/src/mkbootfs/cpp/mkbootfs.c rename to aosp/mkbootfs/src/main/cpp/mkbootfs.cpp index eba3657..df4b0a2 100644 --- a/mkbootfs/src/mkbootfs/cpp/mkbootfs.c +++ b/aosp/mkbootfs/src/main/cpp/mkbootfs.cpp @@ -88,7 +88,7 @@ static void fix_stat(const char *path, struct stat *s) unsigned st_mode = s->st_mode; int is_dir = S_ISDIR(s->st_mode) || strcmp(path, TRAILER) == 0; fs_config(path, is_dir, target_out_path, &s->st_uid, &s->st_gid, &st_mode, &capabilities); - s->st_mode = (typeof(s->st_mode)) st_mode; + s->st_mode = (decltype(s->st_mode)) st_mode; } } diff --git a/mkbootfs/src/mkbootfs/headers/log/log.h b/aosp/mkbootfs/src/main/headers/log/log.h similarity index 100% rename from mkbootfs/src/mkbootfs/headers/log/log.h rename to aosp/mkbootfs/src/main/headers/log/log.h diff --git a/mkbootfs/src/mkbootfs/headers/private/android_filesystem_capability.h b/aosp/mkbootfs/src/main/headers/private/android_filesystem_capability.h similarity index 100% rename from mkbootfs/src/mkbootfs/headers/private/android_filesystem_capability.h rename to aosp/mkbootfs/src/main/headers/private/android_filesystem_capability.h diff --git a/mkbootfs/src/mkbootfs/headers/private/android_filesystem_config.h b/aosp/mkbootfs/src/main/headers/private/android_filesystem_config.h similarity index 100% rename from mkbootfs/src/mkbootfs/headers/private/android_filesystem_config.h rename to aosp/mkbootfs/src/main/headers/private/android_filesystem_config.h diff --git a/mkbootfs/src/mkbootfs/headers/private/fs_config.h b/aosp/mkbootfs/src/main/headers/private/fs_config.h similarity index 100% rename from mkbootfs/src/mkbootfs/headers/private/fs_config.h rename to aosp/mkbootfs/src/main/headers/private/fs_config.h diff --git a/mkbootfs/src/mkbootfs/headers/utils/Compat.h b/aosp/mkbootfs/src/main/headers/utils/Compat.h similarity index 100% rename from mkbootfs/src/mkbootfs/headers/utils/Compat.h rename to aosp/mkbootfs/src/main/headers/utils/Compat.h diff --git a/bbootimg/build.gradle b/bbootimg/build.gradle deleted file mode 100644 index 0653606..0000000 --- a/bbootimg/build.gradle +++ /dev/null @@ -1,72 +0,0 @@ -buildscript { - ext { - kotlinVersion = "1.3.41" - } - repositories { - mavenCentral() - } - dependencies { - classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:${kotlinVersion}") - classpath("org.jetbrains.kotlin:kotlin-allopen:${kotlinVersion}") - } -} - -apply plugin: "kotlin" -apply plugin: "application" - -sourceCompatibility = 1.8 -compileKotlin { - kotlinOptions { - freeCompilerArgs = ["-Xjsr305=strict"] - jvmTarget = "1.8" - } -} -compileTestKotlin { - kotlinOptions { - freeCompilerArgs = ["-Xjsr305=strict"] - jvmTarget = "1.8" - } -} - -repositories { - jcenter() -} - -dependencies { - compile("org.jetbrains.kotlin:kotlin-stdlib-jdk8") - compile("org.jetbrains.kotlin:kotlin-reflect") - compile("org.slf4j:slf4j-simple:1.7.25") - compile("org.slf4j:slf4j-api:1.7.25") - compile("com.fasterxml.jackson.core:jackson-annotations:2.9.4") - compile("com.fasterxml.jackson.core:jackson-databind:2.9.4") - compile("org.apache.commons:commons-exec:1.3") - compile("org.apache.commons:commons-compress:1.16.1") - compile("commons-codec:commons-codec:1.11") - compile("junit:junit:4.12") - //compile("org.nd4j:nd4j-api:0.9.1") - compile("com.google.guava:guava:18.0") - compile("org.bouncycastle:bcprov-jdk15on:1.57") - compile("org.apache.commons:commons-exec:1.3") - compile("de.vandermeer:asciitable:0.3.2") -} - -mainClassName = "cfig.RKt" - -jar { - //from files("build/classes/kotlin/main") - doFirst { - from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } } - } - - manifest { - attributes "Main-Class": "cfig.RKt" - } - - exclude 'META-INF/*.RSA', 'META-INF/*.SF','META-INF/*.DSA' -} - -test { - testLogging { - showStandardStreams = true - } -} diff --git a/bbootimg/build.gradle.kts b/bbootimg/build.gradle.kts new file mode 100644 index 0000000..fc1fa34 --- /dev/null +++ b/bbootimg/build.gradle.kts @@ -0,0 +1,49 @@ +plugins { + id("org.jetbrains.kotlin.jvm").version("1.3.41") + application +} + +repositories { + jcenter() +} + +dependencies { + implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") + implementation("org.jetbrains.kotlin:kotlin-reflect") + + implementation("org.slf4j:slf4j-simple:1.7.25") + implementation("org.slf4j:slf4j-api:1.7.25") + implementation("com.fasterxml.jackson.core:jackson-annotations:2.9.4") + implementation("com.fasterxml.jackson.core:jackson-databind:2.9.4") + implementation("com.google.guava:guava:18.0") + implementation("org.apache.commons:commons-exec:1.3") + implementation("org.apache.commons:commons-compress:1.16.1") + implementation("commons-codec:commons-codec:1.11") + implementation("junit:junit:4.12") + implementation("org.bouncycastle:bcprov-jdk15on:1.57") + implementation("de.vandermeer:asciitable:0.3.2") + + testImplementation("org.jetbrains.kotlin:kotlin-test") + testImplementation("org.jetbrains.kotlin:kotlin-test-junit") +} + +application { + mainClassName = "cfig.packable.PackableLauncherKt" +} + +tasks { + jar { + manifest { + attributes["Implementation-Title"] = "Android image reverse engineering toolkit" + attributes["Main-Class"] = "cfig.packable.PackableLauncherKt" + } + from(configurations.runtimeClasspath.get().map({ if (it.isDirectory) it else zipTree(it) })) + excludes.addAll(mutableSetOf("META-INF/*.RSA", "META-INF/*.SF", "META-INF/*.DSA")) + } + test { + testLogging { + showExceptions = true + showStackTraces = true + } + } +} diff --git a/bbootimg/src/main/kotlin/Helper.kt b/bbootimg/src/main/kotlin/Helper.kt index 7aac2cc..ffe5844 100644 --- a/bbootimg/src/main/kotlin/Helper.kt +++ b/bbootimg/src/main/kotlin/Helper.kt @@ -23,6 +23,7 @@ import java.util.zip.GZIPOutputStream import javax.crypto.Cipher class Helper { + @ExperimentalStdlibApi @ExperimentalUnsignedTypes companion object { fun joinWithNulls(vararg source: ByteArray?): ByteArray { @@ -84,6 +85,7 @@ class Helper { } //similar to this.toString(StandardCharsets.UTF_8).replace("${Character.MIN_VALUE}", "") + @Deprecated("by 1.3.41 experimental api: String.decodeToString()") fun toCString(ba: ByteArray): String { val str = ba.toString(StandardCharsets.UTF_8) val nullPos = str.indexOf(Character.MIN_VALUE) @@ -300,6 +302,23 @@ class Helper { log.info("Dumping data to $dumpFile done") } + fun String.check_call(): Boolean { + var ret = false + try { + val cmd = CommandLine.parse(this) + log.info(cmd.toString()) + DefaultExecutor().execute(cmd) + ret = true + } catch (e: java.lang.IllegalArgumentException) { + log.error("$e: can not parse command: [$this]") + } catch (e: ExecuteException) { + log.error("$e: can not exec command") + } catch (e: IOException) { + log.error("$e: can not exec command") + } + return ret + } + private val log = LoggerFactory.getLogger("Helper") } } diff --git a/bbootimg/src/main/kotlin/ParamConfig.kt b/bbootimg/src/main/kotlin/ParamConfig.kt index 73c88c0..dc39581 100644 --- a/bbootimg/src/main/kotlin/ParamConfig.kt +++ b/bbootimg/src/main/kotlin/ParamConfig.kt @@ -9,4 +9,4 @@ data class ParamConfig( var dtbo: String? = UnifiedConfig.workDir + "recoveryDtbo", var dtb: String? = UnifiedConfig.workDir + "dtb", var cfg: String = UnifiedConfig.workDir + "bootimg.json", - val mkbootimg: String = "./tools/mkbootimg") + val mkbootimg: String = "./external/mkbootimg") diff --git a/bbootimg/src/main/kotlin/avb/AVBInfo.kt b/bbootimg/src/main/kotlin/avb/AVBInfo.kt old mode 100755 new mode 100644 diff --git a/bbootimg/src/main/kotlin/avb/Avb.kt b/bbootimg/src/main/kotlin/avb/Avb.kt old mode 100755 new mode 100644 diff --git a/bbootimg/src/main/kotlin/avb/desc/Descriptor.kt b/bbootimg/src/main/kotlin/avb/desc/Descriptor.kt old mode 100755 new mode 100644 diff --git a/bbootimg/src/main/kotlin/avb/desc/HashDescriptor.kt b/bbootimg/src/main/kotlin/avb/desc/HashDescriptor.kt old mode 100755 new mode 100644 diff --git a/bbootimg/src/main/kotlin/avb/desc/HashTreeDescriptor.kt b/bbootimg/src/main/kotlin/avb/desc/HashTreeDescriptor.kt old mode 100755 new mode 100644 diff --git a/bbootimg/src/main/kotlin/avb/desc/KernelCmdlineDescriptor.kt b/bbootimg/src/main/kotlin/avb/desc/KernelCmdlineDescriptor.kt old mode 100755 new mode 100644 diff --git a/bbootimg/src/main/kotlin/avb/desc/PropertyDescriptor.kt b/bbootimg/src/main/kotlin/avb/desc/PropertyDescriptor.kt old mode 100755 new mode 100644 diff --git a/bbootimg/src/main/kotlin/avb/desc/UnknownDescriptor.kt b/bbootimg/src/main/kotlin/avb/desc/UnknownDescriptor.kt old mode 100755 new mode 100644 diff --git a/bbootimg/src/main/kotlin/bootimg/ImgInfo.kt b/bbootimg/src/main/kotlin/bootimg/ImgInfo.kt index 3128b4b..5ca39f6 100644 --- a/bbootimg/src/main/kotlin/bootimg/ImgInfo.kt +++ b/bbootimg/src/main/kotlin/bootimg/ImgInfo.kt @@ -34,5 +34,5 @@ data class ImgInfo( var path: String = "/boot", var verity_pk8: String = "security/verity.pk8", var verity_pem: String = "security/verity.x509.pem", - var jarPath: String = "boot_signer/build/libs/boot_signer.jar") + var jarPath: String = "aosp/boot_signer/build/libs/boot_signer.jar") } diff --git a/bbootimg/src/main/kotlin/cfig/io/Struct3.kt b/bbootimg/src/main/kotlin/cfig/io/Struct3.kt index 71dd6da..509ef00 100644 --- a/bbootimg/src/main/kotlin/cfig/io/Struct3.kt +++ b/bbootimg/src/main/kotlin/cfig/io/Struct3.kt @@ -10,6 +10,7 @@ import java.nio.ByteOrder import java.util.* import java.util.regex.Pattern +@ExperimentalStdlibApi @ExperimentalUnsignedTypes class Struct3 { private val log = LoggerFactory.getLogger(Struct3::class.java) @@ -410,7 +411,7 @@ class Struct3 { if (format[0] === String) { val data = ByteArray(format[1] as Int) Assert.assertEquals(format[1] as Int, iS.read(data)) - ret.add(Helper.toCString(data)) + ret.add(data.decodeToString()) continue } diff --git a/bbootimg/src/main/kotlin/packable/BootImgParser.kt b/bbootimg/src/main/kotlin/packable/BootImgParser.kt index 33dc6c6..3373099 100644 --- a/bbootimg/src/main/kotlin/packable/BootImgParser.kt +++ b/bbootimg/src/main/kotlin/packable/BootImgParser.kt @@ -49,7 +49,8 @@ class BootImgParser : IPackable { } override fun pack(fileName: String) { - Packer().pack(mkbootfsBin = "mkbootfs/build/exe/mkbootfs/mkbootfs") - Signer.sign(avbtool = "avb/avbtool", bootSigner = "boot_signer/build/libs/boot_signer.jar") + val osSuffix = if (EnvironmentVerifier().isMacOS) "macos" else "linux" + Packer().pack(mkbootfsBin = "./aosp/mkbootfs/build/install/main/release/$osSuffix/mkbootfs") + Signer.sign(avbtool = "avb/avbtool", bootSigner = "aosp/boot_signer/build/libs/boot_signer.jar") } -} \ No newline at end of file +} diff --git a/bbootimg/src/main/kotlin/packable/PackableLauncher.kt b/bbootimg/src/main/kotlin/packable/PackableLauncher.kt index ce30bf0..83982ad 100644 --- a/bbootimg/src/main/kotlin/packable/PackableLauncher.kt +++ b/bbootimg/src/main/kotlin/packable/PackableLauncher.kt @@ -1,6 +1,7 @@ package cfig.packable import cfig.UnifiedConfig +import cfig.sparse_util.SparseImg import org.slf4j.LoggerFactory import java.io.File import java.util.regex.Pattern @@ -13,7 +14,7 @@ class PackableLauncher fun main(args: Array) { val log = LoggerFactory.getLogger(PackableLauncher::class.java) val packablePool = mutableMapOf, KClass>() - listOf(DtboParser(), VBMetaParser(), BootImgParser()).forEach { + listOf(DtboParser(), VBMetaParser(), BootImgParser(), SparseImg()).forEach { @Suppress("UNCHECKED_CAST") packablePool.put(it.capabilities(), it::class as KClass) } diff --git a/bbootimg/src/main/kotlin/sparse_util/SparseImg.kt b/bbootimg/src/main/kotlin/sparse_util/SparseImg.kt new file mode 100644 index 0000000..f443645 --- /dev/null +++ b/bbootimg/src/main/kotlin/sparse_util/SparseImg.kt @@ -0,0 +1,47 @@ +package cfig.sparse_util + +import cfig.EnvironmentVerifier +import cfig.packable.IPackable +import org.slf4j.LoggerFactory +import cfig.Helper.Companion.check_call + +@ExperimentalUnsignedTypes +class SparseImg : IPackable { + private val log = LoggerFactory.getLogger(SparseImg::class.java) + private val simg2imgBin: String + private val img2simgBin: String + + init { + val osSuffix = if (EnvironmentVerifier().isMacOS) "macos" else "linux" + simg2imgBin = "./aosp/libsparse/simg2img/build/install/main/release/$osSuffix/simg2img" + img2simgBin = "./aosp/libsparse/img2simg/build/install/main/release/$osSuffix/img2simg" + } + + override fun capabilities(): List { + return listOf("^(system|vendor|product|cache|userdata)\\.img$") + } + + override fun unpack(fileName: String) { + simg2img(fileName, "$fileName.unsparse") + } + + override fun pack(fileName: String) { + img2simg("$fileName.unsparse", "$fileName.new") + } + + private fun simg2img(sparseIn: String, flatOut: String) { + log.info("parsing Android sparse image $sparseIn ...") + "$simg2imgBin $sparseIn $flatOut".check_call() + "file $sparseIn".check_call() + "file $flatOut".check_call() + log.info("parsed Android sparse image $sparseIn -> $flatOut") + } + + private fun img2simg(flatIn: String, sparseOut: String) { + log.info("transforming image to Android sparse format: $flatIn ...") + "$img2simgBin $flatIn $sparseOut".check_call() + "file $flatIn".check_call() + "file $sparseOut".check_call() + log.info("transformed Android sparse image: $flatIn -> $sparseOut") + } +} diff --git a/boot_signer/build.gradle b/boot_signer/build.gradle deleted file mode 100644 index cc2c301..0000000 --- a/boot_signer/build.gradle +++ /dev/null @@ -1,21 +0,0 @@ -repositories { - mavenCentral() -} - -apply plugin: 'java' - -jar() { - manifest { - attributes "Implementation-Title": "AOSP boot signer", - "Main-Class": "com.android.verity.BootSignature" - } - doFirst { - from {configurations.compile.collect { it.isDirectory() ? it : zipTree(it) }} //pack dependent jar - } - exclude 'META-INF/*.RSA', 'META-INF/*.SF','META-INF/*.DSA' -} - -dependencies { - compile("org.bouncycastle:bcprov-jdk15on:1.57") - compile("org.bouncycastle:bcpkix-jdk15on:1.57") -} diff --git a/build.gradle b/build.gradle index 340aef5..f40114e 100644 --- a/build.gradle +++ b/build.gradle @@ -116,7 +116,7 @@ task unpack(type: JavaExec, dependsOn: ["bbootimg:jar"]) { args "unpack" } -task pack(type: JavaExec, dependsOn: ["bbootimg:jar", "mkbootfs:mkbootfsExecutable"]) { +task pack(type: JavaExec, dependsOn: ["bbootimg:jar", "aosp:boot_signer:build"]) { group GROUP_ANDROID main = "cfig.packable.PackableLauncherKt" classpath = files("bbootimg/build/libs/bbootimg.jar") @@ -132,6 +132,17 @@ task flash(type: JavaExec, dependsOn: ["bbootimg:jar"]) { args "flash" } +//sparse image dependencies +if (System.getProperty("os.name").contains("Mac")) { + unpack.dependsOn("aosp:libsparse:simg2img:installReleaseMacos") + pack.dependsOn("aosp:libsparse:img2simg:installReleaseMacos") + pack.dependsOn("aosp:mkbootfs:installReleaseMacos") +} else { + unpack.dependsOn("aosp:libsparse:simg2img:installReleaseLinux") + pack.dependsOn("aosp:libsparse:img2simg:installReleaseLinux") + pack.dependsOn("aosp:mkbootfs:installReleaseLinux") +} + int parseGradleVersion(String version) { Pattern VERSION_PATTERN = Pattern.compile("((\\d+)(\\.\\d+)+)(-(\\p{Alpha}+)-(\\w+))?(-(SNAPSHOT|\\d{14}([-+]\\d{4})?))?") Matcher matcher = VERSION_PATTERN.matcher(version) diff --git a/doc/additional_tricks.md b/doc/additional_tricks.md index 2df6eaa..1233957 100644 --- a/doc/additional_tricks.md +++ b/doc/additional_tricks.md @@ -46,3 +46,9 @@ Fickle Google removed "BOARD_BUILD_SYSTEM_ROOT_IMAGE" and added "ro.boot.dynamic Thanks to the work by [CallMESuper], ZTE NX606J boot.img is also verified to be compatible with this toolkit. ROM download page: [http://ui.nubia.cn/rom/detail/56](http://ui.nubia.cn/rom/detail/56) + +## about porting + +#### libsparse: output\_file.cpp + +*typeof* is missing in macos clang++, need to change it to *decltype* instead. diff --git a/tools/mkbootimg b/external/mkbootimg similarity index 99% rename from tools/mkbootimg rename to external/mkbootimg index c62481f..e38c106 100755 --- a/tools/mkbootimg +++ b/external/mkbootimg @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python2.7 # Copyright 2015, The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/external/mkdtboimg.diff b/external/mkdtboimg.diff new file mode 100644 index 0000000..a326ed2 --- /dev/null +++ b/external/mkdtboimg.diff @@ -0,0 +1,10 @@ +diff --git a/external/mkdtboimg.py b/external/mkdtboimg.py +index 03f0fd1..4b2de3d 100755 +--- a/external/mkdtboimg.py ++++ b/external/mkdtboimg.py +@@ -1,4 +1,4 @@ +-#! /usr/bin/env python ++#! /usr/bin/env python2 + # Copyright 2017, The Android Open Source Project + # + # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/external/mkdtboimg.py b/external/mkdtboimg.py index 03f0fd1..777f5cb 100755 --- a/external/mkdtboimg.py +++ b/external/mkdtboimg.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python2.7 # Copyright 2017, The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 75b8c7c..ef9a9e0 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.0-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-5.6-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/mkbootfs/build.gradle b/mkbootfs/build.gradle deleted file mode 100644 index 8d7f6cd..0000000 --- a/mkbootfs/build.gradle +++ /dev/null @@ -1,18 +0,0 @@ -apply plugin: 'cpp' -apply plugin: 'c' - -model { - buildTypes { - release - } - - components { - mkbootfs(NativeExecutableSpec) { - binaries.all { - cppCompiler.define "__ANDROID_VNDK__" - //cppCompiler.define 'CFIG_NO_FIX_STAT' - cppCompiler.args << '-std=gnu++11' << "-Wno-write-strings" - } - } - } -} diff --git a/settings.gradle b/settings.gradle index 28dee5a..a1fdb7d 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,3 +1,9 @@ -include 'boot_signer' -include 'mkbootfs' -include 'bbootimg' +include "bbootimg" +include "aosp:boot_signer" +include "aosp:mkbootfs" +include "aosp:libsparse:base" +include "aosp:libsparse:sparse" +include "aosp:libsparse:img2simg" +include "aosp:libsparse:simg2img" +include "aosp:libsparse:simg2simg" +include "aosp:libsparse:append2simg"