ninjalog: Store last uploaded .ninja_log mtime

The current logic checks if the mtime of .ninja_log is 2s older or not
to avoid duplicated uploads. This is problematic if post process takes
longer than 2s. e.g. Siso and/or Reclient's post process.

This CL changes the logic to store the last uploaded mtime as `.ninja_log.last_upload` file.

Bug: 353644846
Fixed: 353644846
Change-Id: I7fb82e43808476174644c50d97e984358a43409e
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5724768
Commit-Queue: Junji Watanabe <jwata@google.com>
Reviewed-by: Takuto Ikuta <tikuta@chromium.org>

ninjalog_uploader.py

@@ -18,11 +18,13 @@ See also the privacy review. http://eldar/assessments/656778450
 
 import argparse
 import gzip
+import http
 import io
 import json
 import logging
 import multiprocessing
 import os
+import pathlib
 import platform
 import subprocess
 import sys
@@ -182,6 +184,43 @@ def GetNinjalog(cmdline):
     return os.path.join(ninjalog_dir, ".ninja_log")
 
 
+def UploadNinjaLog(ninjalog, server, cmdline):
+    output = io.BytesIO()
+
+    with open(ninjalog) as f:
+        with gzip.GzipFile(fileobj=output, mode="wb") as g:
+            g.write(f.read().encode())
+            g.write(b"# end of ninja log\n")
+
+            metadata = GetMetadata(cmdline, ninjalog)
+            logging.info("send metadata: %s", json.dumps(metadata))
+            g.write(json.dumps(metadata).encode())
+
+    status = None
+    err_msg = ""
+    try:
+        resp = urllib.request.urlopen(
+            urllib.request.Request(
+                "https://" + server + "/upload_ninja_log/",
+                data=output.getvalue(),
+                headers={"Content-Encoding": "gzip"},
+            ))
+        status = resp.status
+        logging.info("response header: %s", resp.headers)
+        logging.info("response content: %s", resp.read())
+    except urllib.error.HTTPError as e:
+        status = e.status
+        err_msg = e.msg
+
+    if status != http.HTTPStatus.OK:
+        logging.warning(
+            "unexpected status code for response: status: %s, msg: %s", status,
+            err_msg)
+        return 1
+
+    return 0
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -218,46 +257,22 @@ def main():
         logging.warning("ninjalog is not found in %s", ninjalog)
         return 1
 
-    # We assume that each ninja invocation interval takes at least 2 seconds.
+    # To avoid uploading duplicated ninjalog entries,
-    # This is not to have duplicate entry in server when current build is no-op.
+    # record the mtime of ninjalog that is uploaded.
-    if os.stat(ninjalog).st_mtime < time.time() - 2:
+    # If the recorded timestamp is older than the mtime of ninjalog,
-        logging.info("ninjalog is not updated recently %s", ninjalog)
+    # itt needs to be uploaded.
+    ninjalog_mtime = os.stat(ninjalog).st_mtime
+    last_upload_file = pathlib.Path(ninjalog + '.last_upload')
+    if last_upload_file.exists() and ninjalog_mtime <= last_upload_file.stat(
+    ).st_mtime:
+        logging.info("ninjalog is already uploaded.")
         return 0
 
-    output = io.BytesIO()
+    exit_code = UploadNinjaLog(ninjalog, args.server, args.cmdline)
-
+    if exit_code == 0:
-    with open(ninjalog) as f:
+        last_upload_file.touch()
-        with gzip.GzipFile(fileobj=output, mode="wb") as g:
+    return exit_code
-            g.write(f.read().encode())
-            g.write(b"# end of ninja log\n")
-
-            metadata = GetMetadata(args.cmdline, ninjalog)
-            logging.info("send metadata: %s", json.dumps(metadata))
-            g.write(json.dumps(metadata).encode())
 
-    status = None
-    err_msg = ""
-    try:
-        resp = urllib.request.urlopen(
-            urllib.request.Request(
-                "https://" + args.server + "/upload_ninja_log/",
-                data=output.getvalue(),
-                headers={"Content-Encoding": "gzip"},
-            ))
-        status = resp.status
-        logging.info("response header: %s", resp.headers)
-        logging.info("response content: %s", resp.read())
-    except urllib.error.HTTPError as e:
-        status = e.status
-        err_msg = e.msg
-
-    if status != 200:
-        logging.warning(
-            "unexpected status code for response: status: %s, msg: %s", status,
-            err_msg)
-        return 1
-
-    return 0
 
 
 if __name__ == "__main__":