diff --git a/ninjalog_uploader.py b/ninjalog_uploader.py index 18d20b2105..41ad11b04c 100755 --- a/ninjalog_uploader.py +++ b/ninjalog_uploader.py @@ -18,11 +18,13 @@ See also the privacy review. http://eldar/assessments/656778450 import argparse import gzip +import http import io import json import logging import multiprocessing import os +import pathlib import platform import subprocess import sys @@ -182,6 +184,43 @@ def GetNinjalog(cmdline): return os.path.join(ninjalog_dir, ".ninja_log") +def UploadNinjaLog(ninjalog, server, cmdline): + output = io.BytesIO() + + with open(ninjalog) as f: + with gzip.GzipFile(fileobj=output, mode="wb") as g: + g.write(f.read().encode()) + g.write(b"# end of ninja log\n") + + metadata = GetMetadata(cmdline, ninjalog) + logging.info("send metadata: %s", json.dumps(metadata)) + g.write(json.dumps(metadata).encode()) + + status = None + err_msg = "" + try: + resp = urllib.request.urlopen( + urllib.request.Request( + "https://" + server + "/upload_ninja_log/", + data=output.getvalue(), + headers={"Content-Encoding": "gzip"}, + )) + status = resp.status + logging.info("response header: %s", resp.headers) + logging.info("response content: %s", resp.read()) + except urllib.error.HTTPError as e: + status = e.status + err_msg = e.msg + + if status != http.HTTPStatus.OK: + logging.warning( + "unexpected status code for response: status: %s, msg: %s", status, + err_msg) + return 1 + + return 0 + + def main(): parser = argparse.ArgumentParser() parser.add_argument( @@ -218,46 +257,22 @@ def main(): logging.warning("ninjalog is not found in %s", ninjalog) return 1 - # We assume that each ninja invocation interval takes at least 2 seconds. - # This is not to have duplicate entry in server when current build is no-op. - if os.stat(ninjalog).st_mtime < time.time() - 2: - logging.info("ninjalog is not updated recently %s", ninjalog) + # To avoid uploading duplicated ninjalog entries, + # record the mtime of ninjalog that is uploaded. + # If the recorded timestamp is older than the mtime of ninjalog, + # itt needs to be uploaded. + ninjalog_mtime = os.stat(ninjalog).st_mtime + last_upload_file = pathlib.Path(ninjalog + '.last_upload') + if last_upload_file.exists() and ninjalog_mtime <= last_upload_file.stat( + ).st_mtime: + logging.info("ninjalog is already uploaded.") return 0 - output = io.BytesIO() - - with open(ninjalog) as f: - with gzip.GzipFile(fileobj=output, mode="wb") as g: - g.write(f.read().encode()) - g.write(b"# end of ninja log\n") - - metadata = GetMetadata(args.cmdline, ninjalog) - logging.info("send metadata: %s", json.dumps(metadata)) - g.write(json.dumps(metadata).encode()) + exit_code = UploadNinjaLog(ninjalog, args.server, args.cmdline) + if exit_code == 0: + last_upload_file.touch() + return exit_code - status = None - err_msg = "" - try: - resp = urllib.request.urlopen( - urllib.request.Request( - "https://" + args.server + "/upload_ninja_log/", - data=output.getvalue(), - headers={"Content-Encoding": "gzip"}, - )) - status = resp.status - logging.info("response header: %s", resp.headers) - logging.info("response content: %s", resp.read()) - except urllib.error.HTTPError as e: - status = e.status - err_msg = e.msg - - if status != 200: - logging.warning( - "unexpected status code for response: status: %s, msg: %s", status, - err_msg) - return 1 - - return 0 if __name__ == "__main__":