You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
depot_tools/tests/git_number_test.py

87 lines
2.5 KiB
Python

#!/usr/bin/env vpython3
Add git-number script to calculate generation numbers for commits. Compatible with any git topology (multiple roots, weird branching/merging, etc.) I can't get it to be any faster (in python). Suggestions welcome :). On z600/linux, this takes 5.1s to calculate the initial count for 2e3de954ef0a (HEAD on src.git at the time of writing). Subsequent lookups take ~0.06s. For reference, this machine takes 3s to just list the revisions in sorted order without any additional processing (using rev-list). All calculations are stored in a git-notes-style ref with the exception that the leaf 'tree' object which would normally be stored in a git-notes world is replaced with a packed binary file which consists of records [hash int]. Each run of this script will create only 1 commit object on this internal ref which will have as its parents: * The previous git number commit * All of the target commits we calculated numbers for. This ref is then excluded on subsequent invocations of rev-list, which means that git-number will only ever process commit objects which it hasn't already calculated a value for. It also prevents you from attempting to number this special ref :). This implementation only has a 1-byte fanout which seems to be the best performance for the repos we're dealing with (i.e. on the order of 500k commit objects). Bumping this up to a 2-byte fanout became extremely slow (I suspect the internal caching structures I'm using are not efficient in this mode and could be improved). Using no fanout is slower than the 1 byte fanout for lookups by about 30%. R=agable@chromium.org, stip@chromium.org, szager@chromium.org BUG=280154,309692,skia:1639 Review URL: https://codereview.chromium.org/26109002 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@236035 0039d316-1c4b-4281-b951-d872f2087c98
11 years ago
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Unit tests for git_number.py"""
import binascii
import os
import sys
DEPOT_TOOLS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, DEPOT_TOOLS_ROOT)
from testing_support import git_test_utils
from testing_support import coverage_utils
class Basic(git_test_utils.GitRepoReadWriteTestBase):
REPO_SCHEMA = """
Add git-number script to calculate generation numbers for commits. Compatible with any git topology (multiple roots, weird branching/merging, etc.) I can't get it to be any faster (in python). Suggestions welcome :). On z600/linux, this takes 5.1s to calculate the initial count for 2e3de954ef0a (HEAD on src.git at the time of writing). Subsequent lookups take ~0.06s. For reference, this machine takes 3s to just list the revisions in sorted order without any additional processing (using rev-list). All calculations are stored in a git-notes-style ref with the exception that the leaf 'tree' object which would normally be stored in a git-notes world is replaced with a packed binary file which consists of records [hash int]. Each run of this script will create only 1 commit object on this internal ref which will have as its parents: * The previous git number commit * All of the target commits we calculated numbers for. This ref is then excluded on subsequent invocations of rev-list, which means that git-number will only ever process commit objects which it hasn't already calculated a value for. It also prevents you from attempting to number this special ref :). This implementation only has a 1-byte fanout which seems to be the best performance for the repos we're dealing with (i.e. on the order of 500k commit objects). Bumping this up to a 2-byte fanout became extremely slow (I suspect the internal caching structures I'm using are not efficient in this mode and could be improved). Using no fanout is slower than the 1 byte fanout for lookups by about 30%. R=agable@chromium.org, stip@chromium.org, szager@chromium.org BUG=280154,309692,skia:1639 Review URL: https://codereview.chromium.org/26109002 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@236035 0039d316-1c4b-4281-b951-d872f2087c98
11 years ago
A B C D E
B F E
X Y E
"""
@classmethod
def setUpClass(cls):
super(Basic, cls).setUpClass()
import git_number
cls.gn = git_number
cls.old_POOL_KIND = cls.gn.POOL_KIND
cls.gn.POOL_KIND = 'threads'
@classmethod
def tearDownClass(cls):
cls.gn.POOL_KIND = cls.old_POOL_KIND
super(Basic, cls).tearDownClass()
def tearDown(self):
self.gn.clear_caches()
super(Basic, self).tearDown()
def _git_number(self, refs, cache=False):
refs = [binascii.unhexlify(ref) for ref in refs]
Add git-number script to calculate generation numbers for commits. Compatible with any git topology (multiple roots, weird branching/merging, etc.) I can't get it to be any faster (in python). Suggestions welcome :). On z600/linux, this takes 5.1s to calculate the initial count for 2e3de954ef0a (HEAD on src.git at the time of writing). Subsequent lookups take ~0.06s. For reference, this machine takes 3s to just list the revisions in sorted order without any additional processing (using rev-list). All calculations are stored in a git-notes-style ref with the exception that the leaf 'tree' object which would normally be stored in a git-notes world is replaced with a packed binary file which consists of records [hash int]. Each run of this script will create only 1 commit object on this internal ref which will have as its parents: * The previous git number commit * All of the target commits we calculated numbers for. This ref is then excluded on subsequent invocations of rev-list, which means that git-number will only ever process commit objects which it hasn't already calculated a value for. It also prevents you from attempting to number this special ref :). This implementation only has a 1-byte fanout which seems to be the best performance for the repos we're dealing with (i.e. on the order of 500k commit objects). Bumping this up to a 2-byte fanout became extremely slow (I suspect the internal caching structures I'm using are not efficient in this mode and could be improved). Using no fanout is slower than the 1 byte fanout for lookups by about 30%. R=agable@chromium.org, stip@chromium.org, szager@chromium.org BUG=280154,309692,skia:1639 Review URL: https://codereview.chromium.org/26109002 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@236035 0039d316-1c4b-4281-b951-d872f2087c98
11 years ago
self.repo.run(self.gn.load_generation_numbers, refs)
if cache:
self.repo.run(self.gn.finalize, refs)
return [self.gn.get_num(ref) for ref in refs]
Add git-number script to calculate generation numbers for commits. Compatible with any git topology (multiple roots, weird branching/merging, etc.) I can't get it to be any faster (in python). Suggestions welcome :). On z600/linux, this takes 5.1s to calculate the initial count for 2e3de954ef0a (HEAD on src.git at the time of writing). Subsequent lookups take ~0.06s. For reference, this machine takes 3s to just list the revisions in sorted order without any additional processing (using rev-list). All calculations are stored in a git-notes-style ref with the exception that the leaf 'tree' object which would normally be stored in a git-notes world is replaced with a packed binary file which consists of records [hash int]. Each run of this script will create only 1 commit object on this internal ref which will have as its parents: * The previous git number commit * All of the target commits we calculated numbers for. This ref is then excluded on subsequent invocations of rev-list, which means that git-number will only ever process commit objects which it hasn't already calculated a value for. It also prevents you from attempting to number this special ref :). This implementation only has a 1-byte fanout which seems to be the best performance for the repos we're dealing with (i.e. on the order of 500k commit objects). Bumping this up to a 2-byte fanout became extremely slow (I suspect the internal caching structures I'm using are not efficient in this mode and could be improved). Using no fanout is slower than the 1 byte fanout for lookups by about 30%. R=agable@chromium.org, stip@chromium.org, szager@chromium.org BUG=280154,309692,skia:1639 Review URL: https://codereview.chromium.org/26109002 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@236035 0039d316-1c4b-4281-b951-d872f2087c98
11 years ago
def testBasic(self):
self.assertEqual([0], self._git_number([self.repo['A']]))
self.assertEqual([2], self._git_number([self.repo['F']]))
self.assertEqual([0], self._git_number([self.repo['X']]))
self.assertEqual([4], self._git_number([self.repo['E']]))
def testInProcessCache(self):
self.assertEqual(
None,
self.repo.run(self.gn.get_num, binascii.unhexlify(self.repo['A'])))
self.assertEqual([4], self._git_number([self.repo['E']]))
self.assertEqual(
0,
self.repo.run(self.gn.get_num, binascii.unhexlify(self.repo['A'])))
def testOnDiskCache(self):
self.assertEqual(
None,
self.repo.run(self.gn.get_num, binascii.unhexlify(self.repo['A'])))
self.assertEqual([4], self._git_number([self.repo['E']], cache=True))
self.assertEqual([4], self._git_number([self.repo['E']], cache=True))
self.gn.clear_caches()
self.assertEqual(
0,
self.repo.run(self.gn.get_num, binascii.unhexlify(self.repo['A'])))
self.gn.clear_caches()
self.repo.run(self.gn.clear_caches, True)
self.assertEqual(
None,
self.repo.run(self.gn.get_num, binascii.unhexlify(self.repo['A'])))
if __name__ == '__main__':
sys.exit(coverage_utils.covered_main(
os.path.join(DEPOT_TOOLS_ROOT, 'git_number.py'),
'3.7'
))