From f18f99a00917898d998ee32707906cd9bfda1551 Mon Sep 17 00:00:00 2001 From: cfig Date: Fri, 4 Jul 2025 15:34:05 +0800 Subject: [PATCH] lazybox: suppport repo worker repo_lfs and repo_unshallow --- lazybox/src/main/kotlin/cfig/lazybox/App.kt | 15 +- .../kotlin/cfig/lazybox/staging/DiffCI.kt | 83 +++++++- .../kotlin/cfig/lazybox/staging/RepoWorker.kt | 181 ++++++++++++++++++ 3 files changed, 270 insertions(+), 9 deletions(-) create mode 100644 lazybox/src/main/kotlin/cfig/lazybox/staging/RepoWorker.kt diff --git a/lazybox/src/main/kotlin/cfig/lazybox/App.kt b/lazybox/src/main/kotlin/cfig/lazybox/App.kt index 45853df..1ab99cd 100644 --- a/lazybox/src/main/kotlin/cfig/lazybox/App.kt +++ b/lazybox/src/main/kotlin/cfig/lazybox/App.kt @@ -1,6 +1,7 @@ package cfig.lazybox import cfig.lazybox.staging.DiffCI +import cfig.lazybox.staging.RepoWorker import cfig.lazybox.sysinfo.BootChart import cfig.lazybox.sysinfo.CpuInfo import cfig.lazybox.sysinfo.Pidstat @@ -26,9 +27,11 @@ fun main(args: Array) { println("cpuinfo : get cpu info from /sys/devices/system/cpu/") println("sysinfo : get overall system info from Android") println("\nIncubating usage:") - println("apps : get apk file list from Android") - println("dmainfo : parse /d/dma_buf/bufinfo") - println("diffci : find changelist files from CI server based on date and time ranges") + println("apps : get apk file list from Android") + println("dmainfo : parse /d/dma_buf/bufinfo") + println("diffci : find changelist files from CI server based on date and time ranges") + println("repo_lfs : pull LFS files from Git repositories managed by 'repo'") + println("repo_unshallow: unshallow Git repositories managed by 'repo'") exitProcess(0) } if (args[0] == "cpuinfo") { @@ -102,4 +105,10 @@ fun main(args: Array) { if (args[0] == "diffci") { DiffCI().run(args.drop(1).toTypedArray()) } + if (args[0] == "repo_lfs") { + RepoWorker().lfsPullRepo(args.drop(1).toTypedArray()) + } + if (args[0] == "repo_unshallow") { + RepoWorker().unshallowRepo(args.drop(1).toTypedArray()) + } } diff --git a/lazybox/src/main/kotlin/cfig/lazybox/staging/DiffCI.kt b/lazybox/src/main/kotlin/cfig/lazybox/staging/DiffCI.kt index 4bfef42..bdcca8f 100644 --- a/lazybox/src/main/kotlin/cfig/lazybox/staging/DiffCI.kt +++ b/lazybox/src/main/kotlin/cfig/lazybox/staging/DiffCI.kt @@ -1,8 +1,12 @@ package cfig.lazybox.staging import org.slf4j.LoggerFactory +import java.io.File import java.io.IOException +import java.net.HttpURLConnection +import java.net.MalformedURLException import java.net.URI +import java.net.URL import java.net.http.HttpClient import java.net.http.HttpRequest import java.net.http.HttpResponse @@ -20,7 +24,7 @@ class DiffCI { private val dateFormat = DateTimeFormatter.ofPattern("yyyyMMdd") private val dateTimeFormat = DateTimeFormatter.ofPattern("yyyyMMddHHmm") private val buildIdRegex = ".*/(\\d{12})/changelist".toRegex() - private val log = LoggerFactory.getLogger(DiffCI::class.java) + private val log = LoggerFactory.getLogger(DiffCI::class.simpleName) fun run(args: Array) { if (baseUrl == FAKE_URL) { @@ -85,7 +89,7 @@ class DiffCI { val allUrls = mutableListOf() var currentDate = startDate while (currentDate <= endDate) { - log.info("\n----- Processing Date: $currentDate -----") + log.info("----- Processing Date: $currentDate -----") val dailyUrls = finder.findChangelistUrlsForDate(currentDate) allUrls.addAll(dailyUrls) currentDate = currentDate.plusDays(1) @@ -109,7 +113,7 @@ class DiffCI { val allUrls = mutableListOf() var currentDate = startDateTime.toLocalDate() while (currentDate <= endDateTime.toLocalDate()) { - log.info("\n----- Processing Date: $currentDate -----") + log.info("----- Processing Date: $currentDate -----") val dailyUrls = finder.findChangelistUrlsForDate(currentDate) val filteredUrls = dailyUrls.filter { url -> @@ -130,19 +134,85 @@ class DiffCI { private fun printUsageAndExit() { log.info("Error: Invalid number of arguments.") - log.info("\nUsage:") + log.info("Usage:") log.info(" Single Date: --args='diffci '") log.info(" Date Range: --args='diffci '") log.info(" Time Range: --args='diffci '") - log.info("\nExamples:") + log.info("Examples:") log.info(" --args='diffci 20250628'") log.info(" --args='diffci 20250627 20250628'") log.info(" --args='diffci 202506281000 202506281430'") exitProcess(1) } + fun fetchChangeList(urlsToFetch: List) { + val OUTPUT_FILENAME = "merged_content.txt" + if (urlsToFetch.isEmpty()) { + return + } + val outputFile = File(OUTPUT_FILENAME) + log.info("Starting to fetch content from ${urlsToFetch.size} URLs ...") + + outputFile.bufferedWriter().use { writer -> + // Iterate over each URL string in the list. + urlsToFetch.forEachIndexed { index, urlString -> + log.info("(${index + 1}/${urlsToFetch.size}) Fetching: $urlString") + try { + // Create a URL object from the string. + val url = URL(urlString) + val connection = (url.openConnection() as HttpURLConnection).also { + it.requestMethod = "GET" + it.connectTimeout = 5000 // 5 seconds + it.readTimeout = 5000 // 5 seconds + } + val responseCode = connection.responseCode + // Write a header for this URL's content in the output file. + writer.write("--- START: Content from $urlString ---") + writer.newLine() + // Check for a successful HTTP response (2xx status codes). + if (responseCode in 200..299) { + connection.inputStream.bufferedReader().use { reader -> + // Read the content line by line and write to the file. + reader.forEachLine { line -> + writer.write(line) + writer.newLine() + } + } + } else { + val errorMessage = "Received non-OK response code: $responseCode" + log.info("Warning: $errorMessage for $urlString") + writer.write("! FAILED: $errorMessage") + writer.newLine() + } + + } catch (e: MalformedURLException) { + val errorMessage = "Invalid URL format" + log.error("Error for '$urlString': $errorMessage. Skipping.") + writer.write("! FAILED: $errorMessage") + writer.newLine() + } catch (e: IOException) { + val errorMessage = "Error reading data (I/O problem): ${e.message}" + log.info("Error for '$urlString': $errorMessage. Skipping.") + writer.write("! FAILED: $errorMessage") + writer.newLine() + } catch (e: Exception) { + val errorMessage = "An unexpected error occurred: ${e.message}" + log.error("Error for '$urlString': $errorMessage. Skipping.") + writer.write("! FAILED: $errorMessage") + writer.newLine() + } finally { + // Write a footer and add extra newlines for separation. + writer.write("--- END: Content from $urlString ---") + writer.newLine() + writer.newLine() + } + } + } + log.info("Process complete. '${outputFile.absolutePath}'.") + } + private fun printResults(urls: List, isTimeRange: Boolean = false) { - log.info("\n--- Results ---") + log.info("--- Results ---") if (urls.isNotEmpty()) { log.info("Successfully found ${urls.size} changelist files:") urls.forEach { log.info(it) } @@ -151,6 +221,7 @@ class DiffCI { log.info("No changelist files were found for the specified $rangeType.") } log.info("---------------") + fetchChangeList(urls) } private fun parseDate(dateStr: String): LocalDate? { diff --git a/lazybox/src/main/kotlin/cfig/lazybox/staging/RepoWorker.kt b/lazybox/src/main/kotlin/cfig/lazybox/staging/RepoWorker.kt new file mode 100644 index 0000000..daf3400 --- /dev/null +++ b/lazybox/src/main/kotlin/cfig/lazybox/staging/RepoWorker.kt @@ -0,0 +1,181 @@ +package cfig.lazybox.staging + +import java.io.File +import java.io.ByteArrayOutputStream +import org.apache.commons.exec.CommandLine +import org.apache.commons.exec.DefaultExecutor +import org.apache.commons.exec.ExecuteWatchdog +import org.apache.commons.exec.PumpStreamHandler +import org.slf4j.LoggerFactory + +class RepoWorker { + private val log = LoggerFactory.getLogger(RepoWorker::class.simpleName) + fun lfsPullRepo(args: Array) { + val startPath = args.firstOrNull() ?: "." + val dirList = getMatchRepositories(startPath, ::isLfsEnabled, "LFS Enabled") + log.warn("Found ${dirList.size} repositories with LFS enabled.") + dirList.forEach { repoDir -> + val relativePath = repoDir.toRelativeString(File(startPath).canonicalFile).ifEmpty { "." } + log.info("Pulling [$relativePath]...") + pullLfsContent(repoDir) + } + log.info("✨ Scan and pull complete.") + } + + fun unshallowRepo(args: Array) { + val startPath = args.firstOrNull() ?: "." + val dirList = getMatchRepositories(startPath, ::isShallowClone, "Shallow Clone") + log.warn("Found ${dirList.size} shallow repositories.") + dirList.forEach { repoDir -> + val relativePath = repoDir.toRelativeString(File(startPath).canonicalFile).ifEmpty { "." } + log.info("Unshallowing [$relativePath]...") + unshallowGit(repoDir) + } + log.info("✨ Scan and unshallow complete.") + } + + private fun getMatchRepositories( + startPath: String, + checker: (File) -> Boolean, + checkerName: String = "LFS Enabled" + ): List { + val ret = mutableListOf() + val startDir = File(startPath).canonicalFile + log.info("🔍 Finding Git repositories using 'repo forall' in: ${startDir.absolutePath}") + val gitRepositories = findGitRepositoriesWithRepo(startDir) + if (gitRepositories.isEmpty()) { + log.info("No Git repositories found. Make sure you are running this in a directory managed by 'repo'.") + return listOf() + } + log.info("✅ Found ${gitRepositories.size} Git repositories. Now checking for $checkerName status ...") + gitRepositories.forEach { repoDir -> + val relativePath = repoDir.toRelativeString(startDir).ifEmpty { "." } + if (checker(repoDir)) { + log.info("Checking [$relativePath]...") + log.info(" -> ✅ $checkerName") + ret.add(repoDir) + } else { + //log.info("Checking [$relativePath]...") + //log.info(" -> 🔴 LFS Not Enabled.") + } + } + return ret + } + + private fun pullLfsContent(repoDir: File) { + try { + val commandLine = CommandLine("git").also { + it.addArgument("lfs") + it.addArgument("pull") + } + DefaultExecutor().also { + //it.watchdog = ExecuteWatchdog(600000) + it.streamHandler = PumpStreamHandler(System.out, System.err) + it.workingDirectory = repoDir + it.setExitValue(0) + it.execute(commandLine) + } + log.info(" -> ✅ 'git lfs pull' completed successfully.") + } catch (e: Exception) { + log.error(" -> ❌ 'git lfs pull' failed for ${repoDir.name}: ${e.message}") + } + } + + private fun unshallowGit(repoDir: File) { + try { + val commandLine = CommandLine("git").also { + it.addArgument("fetch") + it.addArgument("--progress") + it.addArgument("--unshallow") + } + DefaultExecutor().also { + //it.watchdog = ExecuteWatchdog(180000) + it.streamHandler = PumpStreamHandler(System.out, System.err) + it.workingDirectory = repoDir + it.setExitValue(0) + it.execute(commandLine) + } + log.info(" -> ✅ 'git fetch --unshallow' completed successfully.") + } catch (e: Exception) { + log.error(" -> ❌ 'git fetch --unshallow' failed for ${repoDir.name}: ${e.message}") + } + } + + private fun findGitRepositoriesWithRepo(workingDir: File): List { + return try { + val commandLine = CommandLine("repo").also { + it.addArgument("forall") + it.addArgument("-c") + it.addArgument("pwd") + } + val outputStream = ByteArrayOutputStream() + DefaultExecutor().also { + it.watchdog = ExecuteWatchdog(60000) // 60 seconds + it.streamHandler = PumpStreamHandler(outputStream) + it.workingDirectory = workingDir + it.setExitValue(0) + it.execute(commandLine) + } + val output = outputStream.toString().trim() + if (output.isEmpty()) { + emptyList() + } else { + output.split(System.lineSeparator()) + .map { it.trim() } + .filter { it.isNotEmpty() } + .map { File(it) } + } + } catch (e: Exception) { + log.error("Error executing 'repo forall': ${e.message}") + log.error("Please ensure the 'repo' tool is installed and you are in a valid repo workspace.") + emptyList() + } + } + + private fun isLfsEnabled(repoDir: File): Boolean { + return try { + val commandLine = CommandLine("git").also { + it.addArgument("lfs") + it.addArgument("track") + } + val outputStream = ByteArrayOutputStream() + DefaultExecutor().also { + it.watchdog = ExecuteWatchdog(5000) + it.streamHandler = PumpStreamHandler(outputStream) + it.workingDirectory = repoDir + it.setExitValue(0) + it.execute(commandLine) + } + val output = outputStream.toString() + val lines = output.lines().filter { it.isNotBlank() } + return lines.size > 1 + } catch (e: Exception) { + false + } + } + + fun isShallowClone(repoDir: File): Boolean { + if (!repoDir.isDirectory) { + return false + } + return try { + val commandLine = CommandLine("git").also { + it.addArgument("rev-parse") + it.addArgument("--is-shallow-repository") + } + val outputStream = ByteArrayOutputStream() + val executor = DefaultExecutor().also { + it.watchdog = ExecuteWatchdog(5000) + it.streamHandler = PumpStreamHandler(outputStream) + it.workingDirectory = repoDir + it.setExitValue(0) + } + executor.execute(commandLine) + val output = outputStream.toString().trim() + return output.equals("true", ignoreCase = true) + + } catch (e: Exception) { + false + } + } +}