svn - 快速获取远程 svn 存储库的所有 svn:externals 列表

Question

我们有一个包含大量目录和文件的 svn 存储库，我们的构建系统需要能够找到所有 svn:externals 属性，递归地查找存储库中的一个分支，然后再检查它。目前我们使用：

svn propget svn:externals -R http://url.of.repo/Branch

事实证明，这非常耗时，并且是真正的带宽消耗。似乎客户端正在接收 repo 中所有内容的所有道具并在本地进行过滤（尽管我尚未通过wireshark 确认这一点）。有没有更快的方法来做到这一点？最好以某种方式让服务器只返回所需的数据。

score 30 · Accepted Answer

正如您所提到的，它确实会消耗网络带宽。但是，如果您有权访问托管这些存储库的服务器，则可以通过file://协议运行它。事实证明，它更快且不消耗网络。

svn propget svn:externals -R file:///path/to/repo/Branch

此外，如果你有整个工作副本，你也可以在你的 WC 中运行它。

svn propget svn:externals -R /path/to/WC

希望它可以帮助您更快地获得结果！

score 3 · Accepted Answer

我终于想出了一个解决方案。我决定将请求分解为多个小的 svn 请求，然后将每个请求都设为由线程池运行的任务。这种抨击 svn 服务器，但在我们的例子中，svn 服务器在 LAN 上，并且这个查询只在完整构建期间进行，所以这似乎不是问题。

import os
import sys
import threading
import ThreadPool

thread_pool = ThreadPool.ThreadPool(8)
externs_dict = {}
externs_lock = threading.Lock()

def getExternRev( path, url ):
    cmd = 'svn info "%s"' % url
    pipe = os.popen(cmd, 'r')
    data = pipe.read().splitlines()

    #Harvest last changed rev
    for line in data:
        if "Last Changed Rev" in line:
            revision = line.split(":")[1].strip()
            externs_lock.acquire()
            externs_dict[path] = (url, revision)
            externs_lock.release()

def getExterns(url, base_dir):
    cmd = 'svn propget svn:externals "%s"' % url
    pipe = os.popen(cmd, 'r')
    data = pipe.read().splitlines()
    pipe.close()

    for line in data:
        if line:
            line = line.split()
            path = base_dir + line[0]
            url = line[1]
            thread_pool.add_task( getExternRev, path, url )

def processDir(url, base_dir):
    thread_pool.add_task( getExterns, url, base_dir )

    cmd = 'svn list "%s"' % url
    pipe = os.popen(cmd, 'r')
    listing = pipe.read().splitlines()
    pipe.close()

    dir_list = []
    for node in listing:
        if node.endswith('/'):
            dir_list.append(node)

    for node in dir_list:
        #externs_data.extend( analyzePath( url + node, base_dir + node ) )
        thread_pool.add_task( processDir, url+node, base_dir+node )

def analyzePath(url, base_dir = ''):
    thread_pool.add_task( processDir, url, base_dir )
    thread_pool.wait_completion()


analyzePath( "http://url/to/repository" )
print externs_dict

score 3 · Accepted Answer

不知道我在哪里找到了这个宝石，但是用自己的外部组件查看外部组件非常有用：

Windows:
    svn status . | findstr /R "^X"

Linux/Unix:
    svn status . | grep -E "^X"

score 0 · Accepted Answer

由于 -R 开关，它很慢；递归搜索存储库路径中的所有目录以查找该属性，这需要大量工作。

score 0 · Accepted Answer

不是理想的解决方案（可能有副作用）并且不能回答您的问题，但是

您可以重写所有外部定义并在一个常见的已知位置添加（重写） - 这样您将在更改后消除 pg 中的递归

score 0 · Accepted Answer

如果您不介意使用 Python 和 pysvn 库，这里是我用于 SVN 外部的完整命令行程序：

"""
@file
@brief SVN externals utilities.
@author Lukasz Matecki
"""
import sys
import os
import pysvn
import argparse

class External(object):

    def __init__(self, parent, remote_loc, local_loc, revision):
        self.parent = parent
        self.remote_loc = remote_loc
        self.local_loc = local_loc
        self.revision = revision

    def __str__(self):
        if self.revision.kind == pysvn.opt_revision_kind.number:
            return """\
Parent:     {0}
Source:     {1}@{2}
Local name: {3}""".format(self.parent, self.remote_loc, self.revision.number, self.local_loc)
        else:
            return """\
Parent:     {0}
Source:     {1} 
Local name: {2}""".format(self.parent, self.remote_loc, self.local_loc)


def find_externals(client, repo_path, external_path=None):
    """
    @brief Find SVN externals.
    @param client (pysvn.Client) The client to use.
    @param repo_path (str) The repository path to analyze.
    @param external_path (str) The URL of the external to find; if omitted, all externals will be searched.
    @returns [External] The list of externals descriptors or empty list if none found.
    """
    repo_root = client.root_url_from_path(repo_path)

    def parse(ext_prop):
        for parent in ext_prop:
            external = ext_prop[parent]
            for line in external.splitlines():
                path, name = line.split()
                path = path.replace("^", repo_root)
                parts = path.split("@")
                if len(parts) > 1:
                    url = parts[0]
                    rev = pysvn.Revision(pysvn.opt_revision_kind.number, int(parts[1]))
                else:
                    url = parts[0]
                    rev = pysvn.Revision(pysvn.opt_revision_kind.head)
                retval = External(parent, url, name, rev)
                if external_path and not external_path == url:
                    continue
                else:
                    yield retval

    for entry in client.ls(repo_path, recurse=True):
        if entry["kind"] == pysvn.node_kind.dir and entry["has_props"] == True:
            externals = client.propget("svn:externals", entry["name"])
            if externals:
                for e in parse(externals):
                    yield e


def check_externals(client, externals_list):
    for i, e in enumerate(externals_list):
        url = e.remote_loc
        rev = e.revision
        try:
            info = client.info2(url, revision=rev, recurse=False)
            props = info[0][1]
            url = props.URL
            print("[{0}] Existing:\n{1}".format(i + 1, "\n".join(["   {0}".format(line) for line in str(e).splitlines()])))
        except:
            print("[{0}] Not found:\n{1}".format(i + 1, "\n".join(["   {0}".format(line) for line in str(e).splitlines()])))

def main(cmdargs):
    parser = argparse.ArgumentParser(description="SVN externals processing.",
                                     formatter_class=argparse.RawDescriptionHelpFormatter,
                                     prefix_chars='-+')

    SUPPORTED_COMMANDS = ("check", "references")

    parser.add_argument(
        "action",
        type=str,
        default="check",
        choices=SUPPORTED_COMMANDS,
        help="""\
the operation to execute:
   'check' to validate all externals in a given location;
   'references' to print all references to a given location""")

    parser.add_argument(
        "url",
        type=str,
        help="the URL to operate on")

    parser.add_argument(
        "--repo", "-r",
        dest="repo",
        type=str,
        default=None,
        help="the repository (or path within) to perform the operation on, if omitted is inferred from url parameter")

    args = parser.parse_args()

    client = pysvn.Client()

    if args.action == "check":
        externals = find_externals(client, args.url)
        check_externals(client, externals)
    elif args.action == "references":
        if args.repo:
            repo_root = args.repo
        else:
            repo_root = client.root_url_from_path(args.url)
        for i, e in enumerate(find_externals(client, repo_root, args.url)):
            print("[{0}] Reference:\n{1}".format(i + 1, "\n".join(["   {0}".format(line) for line in str(e).splitlines()])))

if __name__ == "__main__":
    sys.exit(main(sys.argv))

这应该适用于 Python 2 和 Python 3。您可以像这样使用它（实际地址已删除）：

python svn_externals.py references https://~~~~~~~~~~~~~~/cmd_utils.py
[1] Reference:
   Parent:     https://~~~~~~~~~~~~~~/BEFORE_MK2/scripts/utils
   Source:     https://~~~~~~~~~~~~~~/tools/python/cmd_utils.py
   Local name: cmd_utils.py
[2] Reference:
   Parent:     https://~~~~~~~~~~~~~~/VTB-1425_PCU/scripts/utils
   Source:     https://~~~~~~~~~~~~~~/tools/python/cmd_utils.py
   Local name: cmd_utils.py
[3] Reference:
   Parent:     https://~~~~~~~~~~~~~~/scripts/utils
   Source:     https://~~~~~~~~~~~~~~/tools/python/cmd_utils.py
   Local name: cmd_utils.py

至于性能，这工作得非常快（尽管我的存储库很小）。你必须自己检查一下。

svn - 快速获取远程 svn 存储库的所有 svn:externals 列表

6 回答 6

Related

Reference