git-repo/git_superproject.py

# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Provide functionality to get projects and their commit ids from Superproject.

For more information on superproject, check out:
https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects

Examples:
  superproject = Superproject(manifest, name, remote, revision)
  UpdateProjectsResult = superproject.UpdateProjectsRevisionId(projects)
"""

import functools
import glob
import hashlib
import os
import sys
import tempfile
import time
from typing import NamedTuple
import urllib.parse

from git_command import git_require
from git_command import GitCommand
from git_config import IsId
from git_config import RepoConfig
from git_refs import GitRefs
import platform_utils


_SUPERPROJECT_GIT_NAME = "superproject.git"
_SUPERPROJECT_MANIFEST_NAME = "superproject_override.xml"


class SyncResult(NamedTuple):
    """Return the status of sync and whether caller should exit."""

    # Whether the superproject sync was successful.
    success: bool
    # Whether the caller should exit.
    fatal: bool


class CommitIdsResult(NamedTuple):
    """Return the commit ids and whether caller should exit."""

    # A dictionary with the projects/commit ids on success, otherwise None.
    commit_ids: dict
    # Whether the caller should exit.
    fatal: bool


class UpdateProjectsResult(NamedTuple):
    """Return the overriding manifest file and whether caller should exit."""

    # Path name of the overriding manifest file if successful, otherwise None.
    manifest_path: str
    # Whether the caller should exit.
    fatal: bool


class Superproject:
    """Get commit ids from superproject.

    Initializes a bare local copy of a superproject for the manifest. This
    allows lookup of commit ids for all projects. It contains
    _project_commit_ids which is a dictionary with project/commit id entries.
    """

    def __init__(
        self,
        manifest,
        name,
        remote,
        revision,
        superproject_dir="exp-superproject",
    ):
        """Initializes superproject.

        Args:
            manifest: A Manifest object that is to be written to a file.
            name: The unique name of the superproject
            remote: The RemoteSpec for the remote.
            revision: The name of the git branch to track.
            superproject_dir: Relative path under |manifest.subdir| to checkout
                superproject.
        """
        self._project_commit_ids = None
        self._manifest = manifest
        self.name = name
        self.remote = remote
        self.revision = revision
        self._repodir = manifest.repodir
        self._superproject_dir = superproject_dir
        self._superproject_path = manifest.SubmanifestInfoDir(
            manifest.path_prefix, superproject_dir
        )
        self._manifest_path = os.path.join(
            self._superproject_path, _SUPERPROJECT_MANIFEST_NAME
        )
        git_name = hashlib.md5(remote.name.encode("utf8")).hexdigest() + "-"
        self._remote_url = remote.url
        self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME
        self._work_git = os.path.join(
            self._superproject_path, self._work_git_name
        )

        # The following are command arguemnts, rather than superproject
        # attributes, and were included here originally.  They should eventually
        # become arguments that are passed down from the public methods, instead
        # of being treated as attributes.
        self._git_event_log = None
        self._quiet = False
        self._print_messages = False

    def SetQuiet(self, value):
        """Set the _quiet attribute."""
        self._quiet = value

    def SetPrintMessages(self, value):
        """Set the _print_messages attribute."""
        self._print_messages = value

    def SetRevisionId(self, revision_id: str) -> None:
        """Set the revisionId of the superproject to sync to."""
        self.revision = revision_id

    @property
    def commit_id(self):
        """Returns the commit ID of the superproject checkout."""
        cmd = ["rev-parse", self.revision]
        p = GitCommand(
            None,  # project
            cmd,
            gitdir=self._work_git,
            bare=True,
            capture_stdout=True,
            capture_stderr=True,
        )
        retval = p.Wait()
        if retval != 0:
            self._LogWarning(
                "git rev-parse call failed, command: git {}, "
                "return code: {}, stderr: {}",
                cmd,
                retval,
                p.stderr,
            )
            return None
        return p.stdout

    @property
    def project_commit_ids(self):
        """Returns a dictionary of projects and their commit ids."""
        return self._project_commit_ids

    @property
    def manifest_path(self):
        """Returns the manifest path if the path exists or None."""
        return (
            self._manifest_path if os.path.exists(self._manifest_path) else None
        )

    @property
    def repo_id(self):
        """Returns the repo ID for the superproject.

        For example, if the superproject points to:
            https://android-review.googlesource.com/platform/superproject/
        Then the repo_id would be:
            android/platform/superproject
        """
        review_url = self.remote.review
        if review_url:
            parsed_url = urllib.parse.urlparse(review_url)
            netloc = parsed_url.netloc
            if netloc:
                parts = netloc.split("-review", 1)
                host = parts[0]
                rev = GitRefs(self._work_git).get("HEAD")
                return f"{host}/{self.name}@{rev}"
        return None

    def _LogMessage(self, fmt, *inputs):
        """Logs message to stderr and _git_event_log."""
        message = f"{self._LogMessagePrefix()} {fmt.format(*inputs)}"
        if self._print_messages:
            print(message, file=sys.stderr)
        if self._git_event_log:
            self._git_event_log.ErrorEvent(message, fmt)

    def _LogMessagePrefix(self):
        """Returns the prefix string to be logged in each log message"""
        return (
            f"repo superproject revision: {self.revision} "
            f"url: {self._remote_url}"
        )

    def _LogError(self, fmt, *inputs):
        """Logs error message to stderr and _git_event_log."""
        self._LogMessage(f"error: {fmt}", *inputs)

    def _LogWarning(self, fmt, *inputs):
        """Logs warning message to stderr and _git_event_log."""
        self._LogMessage(f"warning: {fmt}", *inputs)

    def _Init(self):
        """Sets up a local Git repository to get a copy of a superproject.

        Returns:
            True if initialization is successful, or False.
        """
        if not os.path.exists(self._superproject_path):
            os.mkdir(self._superproject_path)

        if os.path.exists(self._work_git):
            return True

        if not self._quiet:
            print(
                "%s: Performing initial setup for superproject; this might "
                "take several minutes." % self._work_git
            )

        tmp_gitdir_prefix = ".tmp-superproject-initgitdir-"
        tmp_gitdir = tempfile.mkdtemp(
            prefix=tmp_gitdir_prefix,
            dir=self._superproject_path,
        )
        tmp_git_name = os.path.basename(tmp_gitdir)

        try:
            cmd = ["init", "--bare", tmp_git_name]
            p = GitCommand(
                None,
                cmd,
                cwd=self._superproject_path,
                capture_stdout=True,
                capture_stderr=True,
            )
            retval = p.Wait()
            if retval:
                self._LogWarning(
                    "git init call failed, command: git {}, "
                    "return code: {}, stderr: {}",
                    cmd,
                    retval,
                    p.stderr,
                )
                return False

            platform_utils.rename(tmp_gitdir, self._work_git)
            tmp_gitdir = None
            return True
        finally:
            # Clean up the temporary directory created during the process,
            # as well as any stale ones left over from previous attempts.
            if tmp_gitdir and os.path.exists(tmp_gitdir):
                platform_utils.rmtree(tmp_gitdir)

            age_threshold = 60 * 60 * 24  # 1 day in seconds
            now = time.time()
            for tmp_dir in glob.glob(
                os.path.join(self._superproject_path, f"{tmp_gitdir_prefix}*")
            ):
                try:
                    mtime = os.path.getmtime(tmp_dir)
                    if now - mtime > age_threshold:
                        platform_utils.rmtree(tmp_dir)
                except OSError:
                    pass

    def _Fetch(self):
        """Fetches a superproject for the manifest based on |_remote_url|.

        This runs git fetch which stores a local copy the superproject.

        Returns:
            True if fetch is successful, or False.
        """
        if not os.path.exists(self._work_git):
            self._LogWarning("git fetch missing directory: {}", self._work_git)
            return False
        if not git_require((2, 28, 0)):
            self._LogWarning(
                "superproject requires a git version 2.28 or later"
            )
            return False
        cmd = [
            "fetch",
            self._remote_url,
            "--depth",
            "1",
            "--force",
            "--no-tags",
            "--filter",
            "blob:none",
        ]

        # Check if there is a local ref that we can pass to --negotiation-tip.
        # If this is the first fetch, it does not exist yet.
        # We use --negotiation-tip to speed up the fetch. Superproject branches
        # do not share commits. So this lets git know it only needs to send
        # commits reachable from the specified local refs.
        rev_commit = GitRefs(self._work_git).get(f"refs/heads/{self.revision}")
        if rev_commit:
            cmd.extend(["--negotiation-tip", rev_commit])

        if self.revision:
            # If revision is a commit hash, fetch it directly to avoid
            # creating a local branch of the same name.
            refspec = (
                self.revision
                if IsId(self.revision)
                else f"{self.revision}:{self.revision}"
            )
            cmd.append(refspec)
        p = GitCommand(
            None,
            cmd,
            gitdir=self._work_git,
            bare=True,
            capture_stdout=True,
            capture_stderr=True,
        )
        retval = p.Wait()
        if retval:
            self._LogWarning(
                "git fetch call failed, command: git {}, "
                "return code: {}, stderr: {}",
                cmd,
                retval,
                p.stderr,
            )
            return False
        return True

    def _LsTree(self):
        """Gets the commit ids for all projects.

        Works only in git repositories.

        Returns:
            data: data returned from 'git ls-tree ...'. None on error.
        """
        if not os.path.exists(self._work_git):
            self._LogWarning(
                "git ls-tree missing directory: {}", self._work_git
            )
            return None
        data = None
        branch = "HEAD" if not self.revision else self.revision
        cmd = ["ls-tree", "-z", "-r", branch]

        p = GitCommand(
            None,
            cmd,
            gitdir=self._work_git,
            bare=True,
            capture_stdout=True,
            capture_stderr=True,
        )
        retval = p.Wait()
        if retval == 0:
            data = p.stdout
        else:
            self._LogWarning(
                "git ls-tree call failed, command: git {}, "
                "return code: {}, stderr: {}",
                cmd,
                retval,
                p.stderr,
            )
            return None
        return data

    def Sync(self, git_event_log):
        """Gets a local copy of a superproject for the manifest.

        Args:
            git_event_log: an EventLog, for git tracing.

        Returns:
            SyncResult
        """
        self._git_event_log = git_event_log
        if not self._manifest.superproject:
            self._LogWarning(
                "superproject tag is not defined in manifest: {}",
                self._manifest.manifestFile,
            )
            return SyncResult(False, False)

        should_exit = True
        if not self._remote_url:
            self._LogWarning(
                "superproject URL is not defined in manifest: {}",
                self._manifest.manifestFile,
            )
            return SyncResult(False, should_exit)

        if not self._Init():
            return SyncResult(False, should_exit)
        if IsId(self.revision) and self.commit_id:
            return SyncResult(True, False)
        if not self._Fetch():
            return SyncResult(False, should_exit)
        if not self._quiet:
            print(
                "%s: Initial setup for superproject completed." % self._work_git
            )
        return SyncResult(True, False)

    def _GetAllProjectsCommitIds(self):
        """Get commit ids for all projects from superproject and save them.

        Commit ids are saved in _project_commit_ids.

        Returns:
            CommitIdsResult
        """
        sync_result = self.Sync(self._git_event_log)
        if not sync_result.success:
            return CommitIdsResult(None, sync_result.fatal)

        data = self._LsTree()
        if not data:
            self._LogWarning(
                "git ls-tree failed to return data for manifest: {}",
                self._manifest.manifestFile,
            )
            return CommitIdsResult(None, True)

        # Parse lines like the following to select lines starting with '160000'
        # and build a dictionary with project path (last element) and its commit
        # id (3rd element).
        #
        # 160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00
        # 120000 blob acc2cbdf438f9d2141f0ae424cec1d8fc4b5d97f\tbootstrap.bash\x00  # noqa: E501
        commit_ids = {}
        for line in data.split("\x00"):
            ls_data = line.split(None, 3)
            if not ls_data:
                break
            if ls_data[0] == "160000":
                commit_ids[ls_data[3]] = ls_data[2]

        self._project_commit_ids = commit_ids
        return CommitIdsResult(commit_ids, False)

    def _WriteManifestFile(self):
        """Writes manifest to a file.

        Returns:
            manifest_path: Path name of the file into which manifest is written
                instead of None.
        """
        if not os.path.exists(self._superproject_path):
            self._LogWarning(
                "missing superproject directory: {}", self._superproject_path
            )
            return None
        manifest_str = self._manifest.ToXml(
            filter_groups=self._manifest.GetManifestGroupsStr(),
            omit_local=True,
        ).toxml()
        manifest_path = self._manifest_path
        try:
            with open(manifest_path, "w", encoding="utf-8") as fp:
                fp.write(manifest_str)
        except OSError as e:
            self._LogError("cannot write manifest to : {} {}", manifest_path, e)
            return None
        return manifest_path

    def _SkipUpdatingProjectRevisionId(self, project):
        """Checks if a project's revision id needs to be updated or not.

        Revision id for projects from local manifest will not be updated.

        Args:
            project: project whose revision id is being updated.

        Returns:
            True if a project's revision id should not be updated, or False,
        """
        path = project.relpath
        if not path:
            return True
        # Skip the project with revisionId.
        if project.revisionId:
            return True
        # Skip the project if it comes from the local manifest.
        return project.manifest.IsFromLocalManifest(project)

    def UpdateProjectsRevisionId(self, projects, git_event_log):
        """Update revisionId of every project in projects with the commit id.

        Args:
            projects: a list of projects whose revisionId needs to be updated.
            git_event_log: an EventLog, for git tracing.

        Returns:
            UpdateProjectsResult
        """
        self._git_event_log = git_event_log
        commit_ids_result = self._GetAllProjectsCommitIds()
        commit_ids = commit_ids_result.commit_ids
        if not commit_ids:
            return UpdateProjectsResult(None, commit_ids_result.fatal)

        projects_missing_commit_ids = []
        for project in projects:
            if self._SkipUpdatingProjectRevisionId(project):
                continue
            path = project.relpath
            commit_id = commit_ids.get(path)
            if not commit_id:
                projects_missing_commit_ids.append(path)

        # If superproject doesn't have a commit id for a project, then report an
        # error event and continue as if do not use superproject is specified.
        if projects_missing_commit_ids:
            self._LogWarning(
                "please file a bug using {} to report missing "
                "commit_ids for: {}",
                self._manifest.contactinfo.bugurl,
                projects_missing_commit_ids,
            )
            return UpdateProjectsResult(None, False)

        for project in projects:
            if not self._SkipUpdatingProjectRevisionId(project):
                project.SetRevisionId(commit_ids.get(project.relpath))

        manifest_path = self._WriteManifestFile()
        return UpdateProjectsResult(manifest_path, False)


@functools.lru_cache(maxsize=None)
def _UseSuperprojectFromConfiguration():
    """Returns the user choice of whether to use superproject."""
    user_cfg = RepoConfig.ForUser()
    time_now = int(time.time())

    user_value = user_cfg.GetBoolean("repo.superprojectChoice")
    if user_value is not None:
        user_expiration = user_cfg.GetInt("repo.superprojectChoiceExpire")
        if (
            user_expiration is None
            or user_expiration <= 0
            or user_expiration >= time_now
        ):
            # TODO(b/190688390) - Remove prompt when we are comfortable with the
            # new default value.
            if user_value:
                print(
                    (
                        "You are currently enrolled in Git submodules "
                        "experiment (go/android-submodules-quickstart).  Use "
                        "--no-use-superproject to override.\n"
                    ),
                    file=sys.stderr,
                )
            else:
                print(
                    (
                        "You are not currently enrolled in Git submodules "
                        "experiment (go/android-submodules-quickstart).  Use "
                        "--use-superproject to override.\n"
                    ),
                    file=sys.stderr,
                )
            return user_value

    # We don't have an unexpired choice, ask for one.
    system_cfg = RepoConfig.ForSystem()
    system_value = system_cfg.GetBoolean("repo.superprojectChoice")
    if system_value:
        # The system configuration is proposing that we should enable the
        # use of superproject. Treat the user as enrolled for two weeks.
        #
        # TODO(b/190688390) - Remove prompt when we are comfortable with the new
        # default value.
        userchoice = True
        time_choiceexpire = time_now + (86400 * 14)
        user_cfg.SetString(
            "repo.superprojectChoiceExpire", str(time_choiceexpire)
        )
        user_cfg.SetBoolean("repo.superprojectChoice", userchoice)
        print(
            "You are automatically enrolled in Git submodules experiment "
            "(go/android-submodules-quickstart) for another two weeks.\n",
            file=sys.stderr,
        )
        return True

    # For all other cases, we would not use superproject by default.
    return False


def PrintMessages(use_superproject, manifest):
    """Returns a boolean if error/warning messages are to be printed.

    Args:
        use_superproject: option value from optparse.
        manifest: manifest to use.
    """
    return use_superproject is not None or bool(manifest.superproject)


def UseSuperproject(use_superproject, manifest):
    """Returns a boolean if use-superproject option is enabled.

    Args:
        use_superproject: option value from optparse.
        manifest: manifest to use.

    Returns:
        Whether the superproject should be used.
    """

    if not manifest.superproject:
        # This (sub) manifest does not have a superproject definition.
        return False
    elif use_superproject is not None:
        return use_superproject
    else:
        client_value = manifest.manifestProject.use_superproject
        if client_value is not None:
            return client_value
        elif manifest.superproject:
            return _UseSuperprojectFromConfiguration()
        else:
            return False