diff --git a/gclient.py b/gclient.py index f6db225c1..22a4eb96a 100755 --- a/gclient.py +++ b/gclient.py @@ -1542,23 +1542,18 @@ been automagically updated. The previous version is available at %s.old. # Fix path separator on Windows. entry_fixed = entry.replace('/', os.path.sep) e_dir = os.path.join(self.root_dir, entry_fixed) - - def _IsParentOfAny(parent, path_list): - parent_plus_slash = parent + '/' - return any( - path[:len(parent_plus_slash)] == parent_plus_slash - for path in path_list) - # Use entry and not entry_fixed there. if (entry not in entries and (not any(path.startswith(entry + '/') for path in entries)) and os.path.exists(e_dir)): + # The entry has been removed from DEPS. scm = gclient_scm.CreateSCM( prev_url, self.root_dir, entry_fixed, self.outbuf) # Check to see if this directory is now part of a higher-up checkout. # The directory might be part of a git OR svn checkout. scm_root = None + scm_class = None for scm_class in (gclient_scm.scm.GIT, gclient_scm.scm.SVN): try: scm_root = scm_class.GetCheckoutRoot(scm.checkout_path) @@ -1571,9 +1566,45 @@ been automagically updated. The previous version is available at %s.old. 'determine whether it is part of a higher-level ' 'checkout, so not removing.' % entry) continue + + # This is to handle the case of third_party/WebKit migrating from + # being a DEPS entry to being part of the main project. + # If the subproject is a Git project, we need to remove its .git + # folder. Otherwise git operations on that folder will have different + # effects depending on the current working directory. + if scm_class == gclient_scm.scm.GIT and ( + os.path.abspath(scm_root) == os.path.abspath(e_dir)): + e_par_dir = os.path.join(e_dir, os.pardir) + if scm_class.IsInsideWorkTree(e_par_dir): + par_scm_root = scm_class.GetCheckoutRoot(e_par_dir) + # rel_e_dir : relative path of entry w.r.t. its parent repo. + rel_e_dir = os.path.relpath(e_dir, par_scm_root) + if scm_class.IsDirectoryVersioned(par_scm_root, rel_e_dir): + save_dir = scm.GetGitBackupDirPath() + # Remove any eventual stale backup dir for the same project. + if os.path.exists(save_dir): + gclient_utils.rmtree(save_dir) + os.rename(os.path.join(e_dir, '.git'), save_dir) + # When switching between the two states (entry/ is a subproject + # -> entry/ is part of the outer project), it is very likely + # that some files are changed in the checkout, unless we are + # jumping *exactly* across the commit which changed just DEPS. + # In such case we want to cleanup any eventual stale files + # (coming from the old subproject) in order to end up with a + # clean checkout. + scm_class.CleanupDir(par_scm_root, rel_e_dir) + assert not os.path.exists(os.path.join(e_dir, '.git')) + print(('\nWARNING: \'%s\' has been moved from DEPS to a higher ' + 'level checkout. The git folder containing all the local' + ' branches has been saved to %s.\n' + 'If you don\'t care about its state you can safely ' + 'remove that folder to free up space.') % + (entry, save_dir)) + continue + if scm_root in full_entries: - logging.info('%s is part of a higher level checkout, not ' - 'removing.', scm.GetCheckoutRoot()) + logging.info('%s is part of a higher level checkout, not removing', + scm.GetCheckoutRoot()) continue file_list = [] diff --git a/gclient_scm.py b/gclient_scm.py index 69dec5003..243387b6a 100644 --- a/gclient_scm.py +++ b/gclient_scm.py @@ -389,6 +389,20 @@ class GitWrapper(SCMWrapper): if mirror: url = mirror.mirror_path + # If we are going to introduce a new project, there is a possibility that + # we are syncing back to a state where the project was originally a + # sub-project rolled by DEPS (realistic case: crossing the Blink merge point + # syncing backwards, when Blink was a DEPS entry and not part of src.git). + # In such case, we might have a backup of the former .git folder, which can + # be used to avoid re-fetching the entire repo again (useful for bisects). + backup_dir = self.GetGitBackupDirPath() + target_dir = os.path.join(self.checkout_path, '.git') + if os.path.exists(backup_dir) and not os.path.exists(target_dir): + gclient_utils.safe_makedirs(self.checkout_path) + os.rename(backup_dir, target_dir) + # Reset to a clean state + self._Run(['reset', '--hard', 'HEAD'], options) + if (not os.path.exists(self.checkout_path) or (os.path.isdir(self.checkout_path) and not os.path.exists(os.path.join(self.checkout_path, '.git')))): @@ -799,6 +813,12 @@ class GitWrapper(SCMWrapper): base_url = self.url return base_url[:base_url.rfind('/')] + url + def GetGitBackupDirPath(self): + """Returns the path where the .git folder for the current project can be + staged/restored. Use case: subproject moved from DEPS <-> outer project.""" + return os.path.join(self._root_dir, + 'old_' + self.relpath.replace(os.sep, '_')) + '.git' + def _GetMirror(self, url, options): """Get a git_cache.Mirror object for the argument url.""" if not git_cache.Mirror.GetCachePath(): diff --git a/scm.py b/scm.py index 9bc96bc80..94b925e87 100644 --- a/scm.py +++ b/scm.py @@ -442,6 +442,16 @@ class GIT(object): except (OSError, subprocess2.CalledProcessError): return False + @staticmethod + def IsDirectoryVersioned(cwd, relative_dir): + """Checks whether the given |relative_dir| is part of cwd's repo.""" + return bool(GIT.Capture(['ls-tree', 'HEAD', relative_dir], cwd=cwd)) + + @staticmethod + def CleanupDir(cwd, relative_dir): + """Cleans up untracked file inside |relative_dir|.""" + return bool(GIT.Capture(['clean', '-df', relative_dir], cwd=cwd)) + @staticmethod def GetGitSvnHeadRev(cwd): """Gets the most recently pulled git-svn revision.""" diff --git a/testing_support/fake_repos.py b/testing_support/fake_repos.py index 1b77bc866..3f908441d 100755 --- a/testing_support/fake_repos.py +++ b/testing_support/fake_repos.py @@ -828,6 +828,40 @@ class FakeRepoSkiaDEPS(FakeReposBase): }) +class FakeRepoBlinkDEPS(FakeReposBase): + """Simulates the Blink DEPS transition in Chrome.""" + + NB_GIT_REPOS = 2 + DEPS_pre = 'deps = {"src/third_party/WebKit": "%(git_base)srepo_2",}' + DEPS_post = 'deps = {}' + + def populateGit(self): + # Blink repo. + self._commit_git('repo_2', { + 'OWNERS': 'OWNERS-pre', + 'Source/exists_always': '_ignored_', + 'Source/exists_before_but_not_after': '_ignored_', + }) + + # Chrome repo. + self._commit_git('repo_1', { + 'DEPS': self.DEPS_pre % {'git_base': self.git_base}, + 'myfile': 'myfile@1', + '.gitignore': '/third_party/WebKit', + }) + self._commit_git('repo_1', { + 'DEPS': self.DEPS_post % {'git_base': self.git_base}, + 'myfile': 'myfile@2', + '.gitignore': '', + 'third_party/WebKit/OWNERS': 'OWNERS-post', + 'third_party/WebKit/Source/exists_always': '_ignored_', + 'third_party/WebKit/Source/exists_after_but_not_before': '_ignored', + }) + + def populateSvn(self): + raise NotImplementedError() + + class FakeReposTestBase(trial_dir.TestCase): """This is vaguely inspired by twisted.""" # Static FakeRepos instances. Lazy loaded. diff --git a/tests/gclient_scm_test.py b/tests/gclient_scm_test.py index 974aff001..967d3606e 100755 --- a/tests/gclient_scm_test.py +++ b/tests/gclient_scm_test.py @@ -1245,6 +1245,8 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase): self.root_dir = '/tmp' if sys.platform != 'win32' else 't:\\tmp' self.relpath = 'fake' self.base_path = os.path.join(self.root_dir, self.relpath) + self.backup_base_path = os.path.join(self.root_dir, + 'old_%s.git' % self.relpath) def tearDown(self): BaseTestCase.tearDown(self) @@ -1354,6 +1356,7 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase): gclient_scm.os.path.isdir( os.path.join(self.base_path, '.git', 'hooks')).AndReturn(False) + gclient_scm.os.path.exists(self.backup_base_path).AndReturn(False) gclient_scm.os.path.exists(self.base_path).AndReturn(True) gclient_scm.os.path.isdir(self.base_path).AndReturn(True) gclient_scm.os.path.exists(os.path.join(self.base_path, '.git') @@ -1384,6 +1387,7 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase): gclient_scm.os.path.isdir( os.path.join(self.base_path, '.git', 'hooks')).AndReturn(False) + gclient_scm.os.path.exists(self.backup_base_path).AndReturn(False) gclient_scm.os.path.exists(self.base_path).AndReturn(True) gclient_scm.os.path.isdir(self.base_path).AndReturn(True) gclient_scm.os.path.exists(os.path.join(self.base_path, '.git') diff --git a/tests/gclient_smoketest.py b/tests/gclient_smoketest.py index 9c84fad9f..388d2a83a 100755 --- a/tests/gclient_smoketest.py +++ b/tests/gclient_smoketest.py @@ -22,7 +22,7 @@ sys.path.insert(0, ROOT_DIR) from testing_support.fake_repos import join, write from testing_support.fake_repos import FakeReposTestBase, FakeRepoTransitive, \ - FakeRepoSkiaDEPS + FakeRepoSkiaDEPS, FakeRepoBlinkDEPS import gclient_utils import scm as gclient_scm @@ -1538,6 +1538,138 @@ class SkiaDEPSTransitionSmokeTest(GClientSmokeBase): skia_src), src_git_url) +class BlinkDEPSTransitionSmokeTest(GClientSmokeBase): + """Simulate the behavior of bisect bots as they transition across the Blink + DEPS change.""" + + FAKE_REPOS_CLASS = FakeRepoBlinkDEPS + + def setUp(self): + super(BlinkDEPSTransitionSmokeTest, self).setUp() + self.enabled = self.FAKE_REPOS.set_up_git() + self.checkout_path = os.path.join(self.root_dir, 'src') + self.blink = os.path.join(self.checkout_path, 'third_party', 'WebKit') + self.blink_git_url = self.FAKE_REPOS.git_base + 'repo_2' + self.pre_merge_sha = self.githash('repo_1', 1) + self.post_merge_sha = self.githash('repo_1', 2) + + def CheckStatusPreMergePoint(self): + self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'], + self.blink), self.blink_git_url) + self.assertTrue(os.path.exists(join(self.blink, '.git'))) + self.assertTrue(os.path.exists(join(self.blink, 'OWNERS'))) + with open(join(self.blink, 'OWNERS')) as f: + owners_content = f.read() + self.assertEqual('OWNERS-pre', owners_content, 'OWNERS not updated') + self.assertTrue(os.path.exists(join(self.blink, 'Source', 'exists_always'))) + self.assertTrue(os.path.exists( + join(self.blink, 'Source', 'exists_before_but_not_after'))) + self.assertFalse(os.path.exists( + join(self.blink, 'Source', 'exists_after_but_not_before'))) + + def CheckStatusPostMergePoint(self): + # Check that the contents still exists + self.assertTrue(os.path.exists(join(self.blink, 'OWNERS'))) + with open(join(self.blink, 'OWNERS')) as f: + owners_content = f.read() + self.assertEqual('OWNERS-post', owners_content, 'OWNERS not updated') + self.assertTrue(os.path.exists(join(self.blink, 'Source', 'exists_always'))) + # Check that file removed between the branch point are actually deleted. + self.assertTrue(os.path.exists( + join(self.blink, 'Source', 'exists_after_but_not_before'))) + self.assertFalse(os.path.exists( + join(self.blink, 'Source', 'exists_before_but_not_after'))) + # But not the .git folder + self.assertFalse(os.path.exists(join(self.blink, '.git'))) + + def testBlinkDEPSChangeUsingGclient(self): + """Checks that {src,blink} repos are consistent when syncing going back and + forth using gclient sync src@revision.""" + if not self.enabled: + return + + self.gclient(['config', '--spec', + 'solutions=[' + '{"name": "src",' + ' "url": "' + self.git_base + 'repo_1",' + '}]']) + + # Go back and forth two times. + for _ in xrange(2): + res = self.gclient(['sync', '--jobs', '1', + '--revision', 'src@%s' % self.pre_merge_sha]) + self.assertEqual(res[2], 0, 'DEPS change sync failed.') + self.CheckStatusPreMergePoint() + + res = self.gclient(['sync', '--jobs', '1', + '--revision', 'src@%s' % self.post_merge_sha]) + self.assertEqual(res[2], 0, 'DEPS change sync failed.') + self.CheckStatusPostMergePoint() + + + def testBlinkDEPSChangeUsingGit(self): + """Like testBlinkDEPSChangeUsingGclient, but move the main project using + directly git and not gclient sync.""" + if not self.enabled: + return + + self.gclient(['config', '--spec', + 'solutions=[' + '{"name": "src",' + ' "url": "' + self.git_base + 'repo_1",' + ' "managed": False,' + '}]']) + + # Perform an initial sync to bootstrap the repo. + res = self.gclient(['sync', '--jobs', '1']) + self.assertEqual(res[2], 0, 'Initial gclient sync failed.') + + # Go back and forth two times. + for _ in xrange(2): + subprocess2.check_call(['git', 'checkout', '-q', self.pre_merge_sha], + cwd=self.checkout_path) + res = self.gclient(['sync', '--jobs', '1']) + self.assertEqual(res[2], 0, 'gclient sync failed.') + self.CheckStatusPreMergePoint() + + subprocess2.check_call(['git', 'checkout', '-q', self.post_merge_sha], + cwd=self.checkout_path) + res = self.gclient(['sync', '--jobs', '1']) + self.assertEqual(res[2], 0, 'DEPS change sync failed.') + self.CheckStatusPostMergePoint() + + + def testBlinkLocalBranchesArePreserved(self): + """Checks that the state of local git branches are effectively preserved + when going back and forth.""" + if not self.enabled: + return + + self.gclient(['config', '--spec', + 'solutions=[' + '{"name": "src",' + ' "url": "' + self.git_base + 'repo_1",' + '}]']) + + # Initialize to pre-merge point. + self.gclient(['sync', '--revision', 'src@%s' % self.pre_merge_sha]) + self.CheckStatusPreMergePoint() + + # Create a branch named "foo". + subprocess2.check_call(['git', 'checkout', '-qB', 'foo'], + cwd=self.blink) + + # Cross the pre-merge point. + self.gclient(['sync', '--revision', 'src@%s' % self.post_merge_sha]) + self.CheckStatusPostMergePoint() + + # Go backwards and check that we still have the foo branch. + self.gclient(['sync', '--revision', 'src@%s' % self.pre_merge_sha]) + self.CheckStatusPreMergePoint() + subprocess2.check_call( + ['git', 'show-ref', '-q', '--verify', 'refs/heads/foo'], cwd=self.blink) + + class GClientSmokeFromCheckout(GClientSmokeBase): # WebKit abuses this. It has a .gclient and a DEPS from a checkout. def setUp(self): diff --git a/tests/scm_unittest.py b/tests/scm_unittest.py index 982a918d0..15676aa21 100755 --- a/tests/scm_unittest.py +++ b/tests/scm_unittest.py @@ -77,6 +77,7 @@ class GitWrapperTestCase(BaseSCMTestCase): 'AssertVersion', 'Capture', 'CaptureStatus', + 'CleanupDir', 'current_version', 'FetchUpstreamTuple', 'GenerateDiff', @@ -92,6 +93,7 @@ class GitWrapperTestCase(BaseSCMTestCase): 'GetSha1ForSvnRev', 'GetSVNBranch', 'GetUpstreamBranch', + 'IsDirectoryVersioned', 'IsGitSvn', 'IsInsideWorkTree', 'IsValidRevision',