packaging - zygaena/hammerhead

root / base / usr / src / contrib / rsync / packaging
packaging Plain Text 1985 lines 66.9 KB
TARGETS := all install install-ssl-daemon install-all install-strip conf gen reconfigure restatus \
	proto man clean cleantests distclean test check check29 check30 installcheck splint \
	doxygen doxygen-upload finddead rrsync

.PHONY: $(TARGETS) auto-prep

$(TARGETS): auto-prep
	make -C build $@

auto-prep:
	@if test x`packaging/prep-auto-dir` = x; then echo "auto-build-save is not setup"; exit 1; fi
	@echo 'Build branch: '`readlink build/.branch | tr % /`
#!/usr/bin/env -S python3 -B

# This script turns one or more diff files in the patches dir (which is
# expected to be a checkout of the rsync-patches git repo) into a branch
# in the main rsync git checkout. This allows the applied patch to be
# merged with the latest rsync changes and tested.  To update the diff
# with the resulting changes, see the patch-update script.

import os, sys, re, argparse, glob

sys.path = ['packaging'] + sys.path

from pkglib import *

def main():
    global created, info, local_branch

    cur_branch, args.base_branch = check_git_state(args.base_branch, not args.skip_check, args.patches_dir)

    local_branch = get_patch_branches(args.base_branch)

    if args.delete_local_branches:
        for name in sorted(local_branch):
            branch = f"patch/{args.base_branch}/{name}"
            cmd_chk(['git', 'branch', '-D', branch])
        local_branch = set()

    if args.add_missing:
        for fn in sorted(glob.glob(f"{args.patches_dir}/*.diff")):
            name = re.sub(r'\.diff$', '', re.sub(r'.+/', '', fn))
            if name not in local_branch and fn not in args.patch_files:
                args.patch_files.append(fn)

    if not args.patch_files:
        return

    for fn in args.patch_files:
        if not fn.endswith('.diff'):
            die(f"Filename is not a .diff file: {fn}")
        if not os.path.isfile(fn):
            die(f"File not found: {fn}")

    scanned = set()
    info = { }

    patch_list = [ ]
    for fn in args.patch_files:
        m = re.match(r'^(?P<dir>.*?)(?P<name>[^/]+)\.diff$', fn)
        patch = argparse.Namespace(**m.groupdict())
        if patch.name in scanned:
            continue
        patch.fn = fn

        lines = [ ]
        commit_hash = None
        with open(patch.fn, 'r', encoding='utf-8') as fh:
            for line in fh:
                m = re.match(r'^based-on: (\S+)', line)
                if m:
                    commit_hash = m[1]
                    break
                if (re.match(r'^index .*\.\..* \d', line)
                 or re.match(r'^diff --git ', line)
                 or re.match(r'^--- (old|a)/', line)):
                    break
                lines.append(re.sub(r'\s*\Z', "\n", line, 1))
        info_txt = ''.join(lines).strip() + "\n"
        lines = None

        parent = args.base_branch
        patches = re.findall(r'patch -p1 <%s/(\S+)\.diff' % args.patches_dir, info_txt)
        if patches:
            last = patches.pop()
            if last != patch.name:
                warn(f"No identity patch line in {patch.fn}")
                patches.append(last)
            if patches:
                parent = patches.pop()
                if parent not in scanned:
                    diff_fn = patch.dir + parent + '.diff'
                    if not os.path.isfile(diff_fn):
                        die(f"Failed to find parent of {patch.fn}: {parent}")
                    # Add parent to args.patch_files so that we will look for the
                    # parent's parent.  Any duplicates will be ignored.
                    args.patch_files.append(diff_fn)
        else:
            warn(f"No patch lines found in {patch.fn}")

        info[patch.name] = [ parent, info_txt, commit_hash ]

        patch_list.append(patch)

    created = set()
    for patch in patch_list:
        create_branch(patch)

    cmd_chk(['git', 'checkout', args.base_branch])


def create_branch(patch):
    if patch.name in created:
        return
    created.add(patch.name)

    parent, info_txt, commit_hash = info[patch.name]
    parent = argparse.Namespace(dir=patch.dir, name=parent, fn=patch.dir + parent + '.diff')

    if parent.name == args.base_branch:
        parent_branch = commit_hash if commit_hash else args.base_branch
    else:
        create_branch(parent)
        parent_branch = '/'.join(['patch', args.base_branch, parent.name])

    branch = '/'.join(['patch', args.base_branch, patch.name])
    print("\n" + '=' * 64)
    print(f"Processing {branch} ({parent_branch})")

    if patch.name in local_branch:
        cmd_chk(['git', 'branch', '-D', branch])

    cmd_chk(['git', 'checkout', '-b', branch, parent_branch])

    info_fn = 'PATCH.' + patch.name
    with open(info_fn, 'w', encoding='utf-8') as fh:
        fh.write(info_txt)
    cmd_chk(['git', 'add', info_fn])

    with open(patch.fn, 'r', encoding='utf-8') as fh:
        patch_txt = fh.read()

    cmd_run('patch -p1'.split(), input=patch_txt)

    for fn in glob.glob('*.orig') + glob.glob('*/*.orig'):
        os.unlink(fn)

    pos = 0
    new_file_re = re.compile(r'\nnew file mode (?P<mode>\d+)\s+--- /dev/null\s+\+\+\+ b/(?P<fn>.+)')
    while True:
        m = new_file_re.search(patch_txt, pos)
        if not m:
            break
        os.chmod(m['fn'], int(m['mode'], 8))
        cmd_chk(['git', 'add', m['fn']])
        pos = m.end()

    while True:
        cmd_chk('git status'.split())
        ans = input('Press Enter to commit, Ctrl-C to abort, or type a wild-name to add a new file: ')
        if ans == '':
            break
        cmd_chk("git add " + ans, shell=True)

    while True:
        s = cmd_run(['git', 'commit', '-a', '-m', f"Creating branch from {patch.name}.diff."])
        if not s.returncode:
            break
        s = cmd_run([os.environ.get('SHELL', '/bin/sh')])
        if s.returncode:
            die('Aborting due to shell error code')


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Create a git patch branch from an rsync patch file.", add_help=False)
    parser.add_argument('--branch', '-b', dest='base_branch', metavar='BASE_BRANCH', default='master', help="The branch the patch is based on. Default: master.")
    parser.add_argument('--add-missing', '-a', action='store_true', help="Add a branch for every patches/*.diff that doesn't have a branch.")
    parser.add_argument('--skip-check', action='store_true', help="Skip the check that ensures starting with a clean branch.")
    parser.add_argument('--delete', dest='delete_local_branches', action='store_true', help="Delete all the local patch/BASE/* branches, not just the ones that are being recreated.")
    parser.add_argument('--patches-dir', '-p', metavar='DIR', default='patches', help="Override the location of the rsync-patches dir. Default: patches.")
    parser.add_argument('patch_files', metavar='patches/DIFF_FILE', nargs='*', help="Specify what patch diff files to process. Default: all of them.")
    parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
    args = parser.parse_args()
    main()

# vim: sw=4 et ft=python
#!/usr/bin/env python3
# This script outputs either perl or python code that parses all possible options
# that the code in options.c might send to the server.  The resulting code is then
# included in the rrsync script.

import re, argparse

short_no_arg = { }
short_with_num = { '@': 1 }
long_opts = { # These include some extra long-args that BackupPC uses:
        'block-size': 1,
        'daemon': -1,
        'debug': 1,
        'fake-super': 0,
        'fuzzy': 0,
        'group': 0,
        'hard-links': 0,
        'ignore-times': 0,
        'info': 1,
        'links': 0,
        'log-file': 3,
        'munge-links': 0,
        'no-munge-links': -1,
        'one-file-system': 0,
        'owner': 0,
        'perms': 0,
        'recursive': 0,
        'stderr': 1,
        'times': 0,
        'copy-devices': -1,
        'write-devices': -1,
        }

def main():
    last_long_opt = None

    with open('../options.c') as fh:
        for line in fh:
            m = re.search(r"argstr\[x\+\+\] = '([^.ie])'", line)
            if m:
                short_no_arg[m.group(1)] = 1
                last_long_opt = None
                continue

            m = re.search(r'asprintf\([^,]+, "-([a-zA-Z0-9])\%l?[ud]"', line)
            if m:
                short_with_num[m.group(1)] = 1
                last_long_opt = None
                continue

            m = re.search(r'args\[ac\+\+\] = "--([^"=]+)"', line)
            if m:
                last_long_opt = m.group(1)
                if last_long_opt not in long_opts:
                    long_opts[last_long_opt] = 0
                else:
                    last_long_opt = None
                continue

            if last_long_opt:
                m = re.search(r'args\[ac\+\+\] = safe_arg\("", ([^[("\s]+)\);', line)
                if m:
                    long_opts[last_long_opt] = 2
                    last_long_opt = None
                    continue
                if 'args[ac++] = ' in line:
                    last_long_opt = None

            m = re.search(r'return "--([^"]+-dest)";', line)
            if m:
                long_opts[m.group(1)] = 2
                last_long_opt = None
                continue

            m = re.search(r'asprintf\([^,]+, "--([^"=]+)=', line)
            if not m:
                m = re.search(r'args\[ac\+\+\] = "--([^"=]+)=', line)
                if not m:
                    m = re.search(r'args\[ac\+\+\] = safe_arg\("--([^"=]+)"', line)
                    if not m:
                        m = re.search(r'fmt = .*: "--([^"=]+)=', line)
            if m:
                long_opts[m.group(1)] = 1
                last_long_opt = None

    long_opts['files-from'] = 3

    txt = """\
### START of options data produced by the cull-options script. ###

# To disable a short-named option, add its letter to this string:
"""

    txt += str_assign('short_disabled', 's') + "\n"
    txt += '# These are also disabled when the restricted dir is not "/":\n'
    txt += str_assign('short_disabled_subdir', 'KLk') + "\n"
    txt += '# These are all possible short options that we will accept (when not disabled above):\n'
    txt += str_assign('short_no_arg', ''.join(sorted(short_no_arg)), 'DO NOT REMOVE ANY')
    txt += str_assign('short_with_num', ''.join(sorted(short_with_num)), 'DO NOT REMOVE ANY')
   
    txt += """
# To disable a long-named option, change its value to a -1.  The values mean:
# 0 = the option has no arg; 1 = the arg doesn't need any checking; 2 = only
# check the arg when receiving; and 3 = always check the arg.
"""

    print(txt, end='')

    if args.python:
        print("long_opts = {")
        sep = ':'
    else:
        print("our %long_opt = (")
        sep = ' =>'

    for opt in sorted(long_opts):
        if opt.startswith(('min-', 'max-')):
            val = 1
        else:
            val = long_opts[opt]
        print(' ', repr(opt) + sep, str(val) + ',')

    if args.python:
        print("}")
    else:
        print(");")
    print("\n### END of options data produced by the cull-options script. ###")


def str_assign(name, val, comment=None):
    comment = ' # ' + comment if comment else ''
    if args.python:
        return name + ' = ' + repr(val) + comment + "\n"
    return 'our $' + name + ' = ' + repr(val) + ';' + comment + "\n"


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Output culled rsync options for rrsync.", add_help=False)
    out_group = parser.add_mutually_exclusive_group()
    out_group.add_argument('--perl', action='store_true', help="Output perl code.")
    out_group.add_argument('--python', action='store_true', help="Output python code (the default).")
    parser.add_argument('--help', '-h', action='help', help="Output this help message and exit.")
    args = parser.parse_args()
    if not args.perl:
        args.python = True
    main()

# vim: sw=4 et
Summary: A fast, versatile, remote (and local) file-copying tool
Name: rsync
Version: 3.4.1
%define fullversion %{version}
Release: 1
%define srcdir src
Group: Applications/Internet
License: GPL
Source0: https://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-%{fullversion}.tar.gz
#Source1: https://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-patches-%{fullversion}.tar.gz
URL: https://rsync.samba.org/

Prefix: %{_prefix}
BuildRoot: /var/tmp/%{name}-root

%package ssl-daemon
Summary: An stunnel config file to support ssl rsync daemon connections.
Group: Applications/Internet
Requires: rsync, stunnel >= 4

%description
Rsync is a fast and extraordinarily versatile file copying tool.  It can
copy locally, to/from another host over any remote shell, or to/from a
remote rsync daemon.  It offers a large number of options that control
every aspect of its behavior and permit very flexible specification of the
set of files to be copied.  It is famous for its delta-transfer algorithm,
which reduces the amount of data sent over the network by sending only the
differences between the source files and the existing files in the
destination.  Rsync is widely used for backups and mirroring and as an
improved copy command for everyday use.

%description ssl-daemon
Provides a config file for stunnel that will (if you start your stunnel
service) cause stunnel to listen for ssl rsync-daemon connections and run
"rsync --daemon" to handle them.

%prep
# Choose one -- setup source only, or setup source + rsync-patches:
%setup -q -n rsync-%{fullversion}
#%setup -q -b1 -n rsync-%{fullversion}

# If you you used "%setup -q -b1 ...", choose the patches you wish to apply:
#patch -p1 <patches/acls.diff
#patch -p1 <patches/xattrs.diff
#patch -p1 <patches/remote-option.diff
#patch -p1 <patches/db.diff

# Avoid extra perl dependencies for scripts going into doc dir.
chmod -x support/*

%build
#./prepare-source
%configure

make

%install
rm -rf $RPM_BUILD_ROOT
make install install-ssl-daemon DESTDIR=$RPM_BUILD_ROOT

mkdir -p $RPM_BUILD_ROOT/etc/xinetd.d $RPM_BUILD_ROOT/etc/rsync-ssl/certs
install -m 644 packaging/lsb/rsync.xinetd $RPM_BUILD_ROOT/etc/xinetd.d/rsync

%clean
rm -rf $RPM_BUILD_ROOT

%files
%defattr(-,root,root)
%doc COPYING NEWS.md README.md support/ tech_report.tex
%config(noreplace) /etc/xinetd.d/rsync
%{_prefix}/bin/rsync
%{_prefix}/bin/rsync-ssl
%{_mandir}/man1/rsync.1*
%{_mandir}/man1/rsync-ssl.1*
%{_mandir}/man5/rsyncd.conf.5*

%files ssl-daemon
%config(noreplace) /etc/stunnel/rsyncd.conf
%dir /etc/rsync-ssl/certs

%changelog
* Thu Jan 16 2025 Rsync Project <rsync.project@gmail.com>
Released 3.4.1.
# default: off
# description: The rsync server is a good addition to an ftp server, as it
#	allows crc checksumming etc.
service rsync
{
	disable	= yes
	socket_type     = stream
	wait            = no
	user            = root
	server          = /usr/bin/rsync
	server_args     = --daemon
	log_on_failure  += USERID
}
# This config file can be used with rsync to enable legacy digests
# (such as MD4) by using the OPENSSL_CONF environment variable.
# See rsync's configure --with-openssl-conf=/path/name option.

openssl_conf = openssl_init

[openssl_init]
providers = provider_sect

[provider_sect]
default = default_sect
legacy = legacy_sect

[default_sect]
activate = 1

[legacy_sect]
activate = 1
#!/usr/bin/env -S python3 -B

# This script is used to turn one or more of the "patch/BASE/*" branches
# into one or more diffs in the "patches" directory.  Pass the option
# --gen if you want generated files in the diffs.  Pass the name of
# one or more diffs if you want to just update a subset of all the
# diffs.

import os, sys, re, argparse, time, shutil

sys.path = ['packaging'] + sys.path

from pkglib import *

MAKE_GEN_CMDS = [
        './prepare-source'.split(),
        'cd build && if test -f config.status ; then ./config.status ; else ../configure ; fi',
        'make -C build gen'.split(),
        ]
TMP_DIR = "patches.gen"

os.environ['GIT_MERGE_AUTOEDIT'] = 'no'

def main():
    global master_commit, parent_patch, description, completed, last_touch

    if not os.path.isdir(args.patches_dir):
        die(f'No "{args.patches_dir}" directory was found.')
    if not os.path.isdir('.git'):
        die('No ".git" directory present in the current dir.')

    starting_branch, args.base_branch = check_git_state(args.base_branch, not args.skip_check, args.patches_dir)

    master_commit = latest_git_hash(args.base_branch)

    if cmd_txt_chk(['packaging/prep-auto-dir']).out == '':
        die('You must setup an auto-build-save dir to use this script.')

    if args.gen:
        if os.path.lexists(TMP_DIR):
            die(f'"{TMP_DIR}" must not exist in the current directory.')
        gen_files = get_gen_files()
        os.mkdir(TMP_DIR, 0o700)
        for cmd in MAKE_GEN_CMDS:
            cmd_chk(cmd)
        cmd_chk(['rsync', '-a', *gen_files, f'{TMP_DIR}/master/'])

    last_touch = int(time.time())

    # Start by finding all patches so that we can load all possible parents.
    patches = sorted(list(get_patch_branches(args.base_branch)))

    parent_patch = { }
    description = { }

    for patch in patches:
        branch = f"patch/{args.base_branch}/{patch}"
        desc = ''
        proc = cmd_pipe(['git', 'diff', '-U1000', f"{args.base_branch}...{branch}", '--', f"PATCH.{patch}"])
        in_diff = False
        for line in proc.stdout:
            if in_diff:
                if not re.match(r'^[ +]', line):
                    continue
                line = line[1:]
                m = re.search(r'patch -p1 <patches/(\S+)\.diff', line)
                if m and m[1] != patch:
                    parpat = parent_patch[patch] = m[1]
                    if not parpat in patches:
                        die(f"Parent of {patch} is not a local branch: {parpat}")
                desc += line
            elif re.match(r'^@@ ', line):
                in_diff = True
        description[patch] = desc
        proc.communicate()

    if args.patch_files: # Limit the list of patches to actually process
        valid_patches = patches
        patches = [ ]
        for fn in args.patch_files:
            name = re.sub(r'\.diff$', '', re.sub(r'.+/', '', fn))
            if name not in valid_patches:
                die(f"Local branch not available for patch: {name}")
            patches.append(name)

    completed = set()

    for patch in patches:
        if patch in completed:
            continue
        if not update_patch(patch):
            break

    if args.gen:
        shutil.rmtree(TMP_DIR)

    while last_touch >= int(time.time()):
        time.sleep(1)
    cmd_chk(['git', 'checkout', starting_branch])
    cmd_chk(['packaging/prep-auto-dir'], discard='output')


def update_patch(patch):
    global last_touch

    completed.add(patch) # Mark it as completed early to short-circuit any (bogus) dependency loops.

    parent = parent_patch.get(patch, None)
    if parent:
        if parent not in completed:
            if not update_patch(parent):
                return 0
        based_on = parent = f"patch/{args.base_branch}/{parent}"
    else:
        parent = args.base_branch
        based_on = master_commit

    print(f"======== {patch} ========")

    while args.gen and last_touch >= int(time.time()):
        time.sleep(1)

    branch = f"patch/{args.base_branch}/{patch}"
    s = cmd_run(['git', 'checkout', branch])
    if s.returncode != 0:
        return 0

    s = cmd_run(['git', 'merge', based_on])
    ok = s.returncode == 0
    skip_shell = False
    if not ok or args.cmd or args.make or args.shell:
        cmd_chk(['packaging/prep-auto-dir'], discard='output')
    if not ok:
        print(f'"git merge {based_on}" incomplete -- please fix.')
        if not run_a_shell(parent, patch):
            return 0
        if not args.make and not args.cmd:
            skip_shell = True
    if args.make:
        if cmd_run(['packaging/smart-make']).returncode != 0:
            if not run_a_shell(parent, patch):
                return 0
            if not args.cmd:
                skip_shell = True
    if args.cmd:
        if cmd_run(args.cmd).returncode != 0:
            if not run_a_shell(parent, patch):
                return 0
            skip_shell = True
    if args.shell and not skip_shell:
        if not run_a_shell(parent, patch):
            return 0

    with open(f"{args.patches_dir}/{patch}.diff", 'w', encoding='utf-8') as fh:
        fh.write(description[patch])
        fh.write(f"\nbased-on: {based_on}\n")

        if args.gen:
            gen_files = get_gen_files()
            for cmd in MAKE_GEN_CMDS:
                cmd_chk(cmd)
            cmd_chk(['rsync', '-a', *gen_files, f"{TMP_DIR}/{patch}/"])
        else:
            gen_files = [ ]
        last_touch = int(time.time())

        proc = cmd_pipe(['git', 'diff', based_on])
        skipping = False
        for line in proc.stdout:
            if skipping:
                if not re.match(r'^diff --git a/', line):
                    continue
                skipping = False
            elif re.match(r'^diff --git a/PATCH', line):
                skipping = True
                continue
            if not re.match(r'^index ', line):
                fh.write(line)
        proc.communicate()

        if args.gen:
            e_tmp_dir = re.escape(TMP_DIR)
            diff_re  = re.compile(r'^(diff -Nurp) %s/[^/]+/(.*?) %s/[^/]+/(.*)' % (e_tmp_dir, e_tmp_dir))
            minus_re = re.compile(r'^\-\-\- %s/[^/]+/([^\t]+)\t.*' % e_tmp_dir)
            plus_re  = re.compile(r'^\+\+\+ %s/[^/]+/([^\t]+)\t.*' % e_tmp_dir)

            if parent == args.base_branch:
                parent_dir = 'master'
            else:
                m = re.search(r'([^/]+)$', parent)
                parent_dir = m[1]

            proc = cmd_pipe(['diff', '-Nurp', f"{TMP_DIR}/{parent_dir}", f"{TMP_DIR}/{patch}"])
            for line in proc.stdout:
                line = diff_re.sub(r'\1 a/\2 b/\3', line)
                line = minus_re.sub(r'--- a/\1', line)
                line =  plus_re.sub(r'+++ b/\1', line)
                fh.write(line)
            proc.communicate()

    return 1


def run_a_shell(parent, patch):
    m = re.search(r'([^/]+)$', parent)
    parent_dir = m[1]
    os.environ['PS1'] = f"[{parent_dir}] {patch}: "

    while True:
        s = cmd_run([os.environ.get('SHELL', '/bin/sh')])
        if s.returncode != 0:
            ans = input("Abort? [n/y] ")
            if re.match(r'^y', ans, flags=re.I):
                return False
            continue
        cur_branch, is_clean, status_txt = check_git_status(0)
        if is_clean:
            break
        print(status_txt, end='')

    cmd_run('rm -f build/*.o build/*/*.o')

    return True


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Turn a git branch back into a diff files in the patches dir.", add_help=False)
    parser.add_argument('--branch', '-b', dest='base_branch', metavar='BASE_BRANCH', default='master', help="The branch the patch is based on. Default: master.")
    parser.add_argument('--skip-check', action='store_true', help="Skip the check that ensures starting with a clean branch.")
    parser.add_argument('--make', '-m', action='store_true', help="Run the smart-make script in every patch branch.")
    parser.add_argument('--cmd', '-c', help="Run a command in every patch branch.")
    parser.add_argument('--shell', '-s', action='store_true', help="Launch a shell for every patch/BASE/* branch updated, not just when a conflict occurs.")
    parser.add_argument('--gen', metavar='DIR', nargs='?', const='', help='Include generated files. Optional DIR value overrides the default of using the "patches" dir.')
    parser.add_argument('--patches-dir', '-p', metavar='DIR', default='patches', help="Override the location of the rsync-patches dir. Default: patches.")
    parser.add_argument('patch_files', metavar='patches/DIFF_FILE', nargs='*', help="Specify what patch diff files to process. Default: all of them.")
    parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
    args = parser.parse_args()
    if args.gen == '':
        args.gen = args.patches_dir
    elif args.gen is not None:
        args.patches_dir = args.gen
    main()

# vim: sw=4 et ft=python
import os, sys, re, subprocess, argparse

# This python3 library provides a few helpful routines that are
# used by the latest packaging scripts.

default_encoding = 'utf-8'

# Output the msg args to stderr.  Accepts all the args that print() accepts.
def warn(*msg):
    print(*msg, file=sys.stderr)


# Output the msg args to stderr and die with a non-zero return-code.
# Accepts all the args that print() accepts.
def die(*msg):
    warn(*msg)
    sys.exit(1)


# Set this to an encoding name or set it to None to avoid the default encoding idiom.
def set_default_encoding(enc):
    default_encoding = enc


# Set shell=True if the cmd is a string; sets a default encoding unless raw=True was specified.
def _tweak_opts(cmd, opts, **maybe_set_args):
    def _maybe_set(o, **msa): # Only set a value if the user didn't already set it.
        for var, val in msa.items():
            if var not in o:
                o[var] = val

    opts = opts.copy()
    _maybe_set(opts, **maybe_set_args)

    if isinstance(cmd, str):
        _maybe_set(opts, shell=True)

    want_raw = opts.pop('raw', False)
    if default_encoding and not want_raw:
        _maybe_set(opts, encoding=default_encoding)

    capture = opts.pop('capture', None)
    if capture:
        if capture == 'stdout':
            _maybe_set(opts, stdout=subprocess.PIPE)
        elif capture == 'stderr':
            _maybe_set(opts, stderr=subprocess.PIPE)
        elif capture == 'output':
            _maybe_set(opts, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        elif capture == 'combined':
            _maybe_set(opts, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

    discard = opts.pop('discard', None)
    if discard:
        # We DO want to override any already set stdout|stderr values (unlike above).
        if discard == 'stdout' or discard == 'output':
            opts['stdout'] = subprocess.DEVNULL
        if discard == 'stderr' or discard == 'output':
            opts['stderr'] = subprocess.DEVNULL

    return opts


# This does a normal subprocess.run() with some auto-args added to make life easier.
def cmd_run(cmd, **opts):
    return subprocess.run(cmd, **_tweak_opts(cmd, opts))


# Like cmd_run() with a default check=True specified.
def cmd_chk(cmd, **opts):
    return subprocess.run(cmd, **_tweak_opts(cmd, opts, check=True))


# Capture stdout in a string and return an object with out, err, and rc (return code).
# It defaults to capture='stdout' (so err is empty) but can be overridden using
# capture='combined' or capture='output' (the latter populates the err value).
def cmd_txt(cmd, **opts):
    input = opts.pop('input', None)
    if input is not None:
        opts['stdin'] = subprocess.PIPE
    proc = subprocess.Popen(cmd, **_tweak_opts(cmd, opts, capture='stdout'))
    out, err = proc.communicate(input=input)
    return argparse.Namespace(out=out, err=err, rc=proc.returncode)


# Just like calling cmd_txt() except that it raises an error if the command has a non-0 return code.
# The raised error includes the cmd, the return code, and the captured output.
def cmd_txt_chk(cmd, **opts):
    ct = cmd_txt(cmd, **opts)
    if ct.rc != 0:
        cmd_err = f'Command "{cmd}" returned non-0 exit status "{ct.rc}" and output:\n{ct.out}{ct.err}'
        raise Exception(cmd_err)
    return ct


# Starts a piped-output command of stdout (by default) and leaves it up to you to read
# the output and call communicate() on the returned object.
def cmd_pipe(cmd, **opts):
    return subprocess.Popen(cmd, **_tweak_opts(cmd, opts, capture='stdout'))


# Runs a "git status" command and dies if the checkout is not clean (the
# arg fatal_unless_clean can be used to make that non-fatal.  Returns a
# tuple of the current branch, the is_clean flag, and the status text.
def check_git_status(fatal_unless_clean=True, subdir='.'):
    status_txt = cmd_txt_chk(f"cd '{subdir}' && git status").out
    is_clean = re.search(r'\nnothing to commit.+working (directory|tree) clean', status_txt) != None

    if not is_clean and fatal_unless_clean:
        if subdir == '.':
            subdir = ''
        else:
            subdir = f" *{subdir}*"
        die(f"The{subdir} checkout is not clean:\n" + status_txt)

    m = re.match(r'^(?:# )?On branch (.+)\n', status_txt)
    cur_branch = m[1] if m else None

    return (cur_branch, is_clean, status_txt)


# Calls check_git_status() on the current git checkout and (optionally) a subdir path's
# checkout. Use fatal_unless_clean to indicate if an unclean checkout is fatal or not.
# The master_branch arg indicates what branch we want both checkouts to be using, and
# if the branch is wrong the user is given the option of either switching to the right
# branch or aborting.
def check_git_state(master_branch, fatal_unless_clean=True, check_extra_dir=None):
    cur_branch = check_git_status(fatal_unless_clean)[0]
    branch = re.sub(r'^patch/([^/]+)/[^/]+$', r'\1', cur_branch) # change patch/BRANCH/PATCH_NAME into BRANCH
    if branch != master_branch:
        print(f"The checkout is not on the {master_branch} branch.")
        if master_branch != 'master':
            sys.exit(1)
        ans = input(f"Do you want me to continue with --branch={branch}? [n] ")
        if not ans or not re.match(r'^y', ans, flags=re.I):
            sys.exit(1)
        master_branch = branch

    if check_extra_dir and os.path.isdir(os.path.join(check_extra_dir, '.git')):
        branch = check_git_status(fatal_unless_clean, check_extra_dir)[0]
        if branch != master_branch:
            print(f"The *{check_extra_dir}* checkout is on branch {branch}, not branch {master_branch}.")
            ans = input(f"Do you want to change it to branch {master_branch}? [n] ")
            if not ans or not re.match(r'^y', ans, flags=re.I):
                sys.exit(1)
            subdir.check_call(f"cd {check_extra_dir} && git checkout '{master_branch}'", shell=True)

    return (cur_branch, master_branch)


# Return the git hash of the most recent commit.
def latest_git_hash(branch):
    out = cmd_txt_chk(['git', 'log', '-1', '--no-color', branch]).out
    m = re.search(r'^commit (\S+)', out, flags=re.M)
    if not m:
        die(f"Unable to determine commit hash for master branch: {branch}")
    return m[1]


# Return a set of all branch names that have the format "patch/BASE_BRANCH/NAME"
# for the given base_branch string.  Just the NAME portion is put into the set.
def get_patch_branches(base_branch):
    branches = set()
    proc = cmd_pipe('git branch -l'.split())
    for line in proc.stdout:
        m = re.search(r' patch/([^/]+)/(.+)', line)
        if m and m[1] == base_branch:
            branches.add(m[2])
    proc.communicate()
    return branches


# Snag the GENFILES values out of the Makefile file and return them as a list.
def get_gen_files(want_dir_plus_list=False):
    cont_re = re.compile(r'\\\n')

    gen_files = [ ]

    auto_dir = os.path.join('auto-build-save', cmd_txt('git rev-parse --abbrev-ref HEAD').out.strip().replace('/', '%'))

    with open(auto_dir + '/Makefile', 'r', encoding='utf-8') as fh:
        for line in fh:
            if not gen_files:
                chk = re.sub(r'^GENFILES=', '', line)
                if line == chk:
                    continue
                line = chk
            m = re.search(r'\\$', line)
            line = re.sub(r'^\s+|\s*\\\n?$|\s+$', '', line)
            gen_files += line.split()
            if not m:
                break

    if want_dir_plus_list:
        return (auto_dir, gen_files)

    return [ os.path.join(auto_dir, fn) for fn in gen_files ]


def get_rsync_version():
    with open('version.h', 'r', encoding='utf-8') as fh:
        txt = fh.read()
    m = re.match(r'^#define\s+RSYNC_VERSION\s+"(\d.+?)"', txt)
    if m:
        return m[1]
    die("Unable to find RSYNC_VERSION define in version.h")


def get_NEWS_version_info():
    rel_re = re.compile(r'^\| \S{2} \w{3} \d{4}\s+\|\s+(?P<ver>\d+\.\d+\.\d+)\s+\|\s+(?P<pdate>\d{2} \w{3} \d{4})?\s+\|\s+(?P<pver>\d+)\s+\|')
    last_version = last_protocol_version = None
    pdate = { }

    with open('NEWS.md', 'r', encoding='utf-8') as fh:
        for line in fh:
            if not last_version: # Find the first non-dev|pre version with a release date.
                m = re.search(r'rsync (\d+\.\d+\.\d+) .*\d\d\d\d', line)
                if m:
                    last_version = m[1]
            m = rel_re.match(line)
            if m:
                if m['pdate']:
                    pdate[m['ver']] = m['pdate']
                if m['ver'] == last_version:
                    last_protocol_version = m['pver']

    if not last_protocol_version:
        die(f"Unable to determine protocol_version for {last_version}.")

    return last_version, last_protocol_version, pdate


def get_protocol_versions():
    protocol_version = subprotocol_version = None

    with open('rsync.h', 'r', encoding='utf-8') as fh:
        for line in fh:
            m = re.match(r'^#define\s+PROTOCOL_VERSION\s+(\d+)', line)
            if m:
                protocol_version = m[1]
                continue
            m = re.match(r'^#define\s+SUBPROTOCOL_VERSION\s+(\d+)', line)
            if m:
                subprotocol_version = m[1]
                break

    if not protocol_version:
        die("Unable to determine the current PROTOCOL_VERSION.")

    if not subprotocol_version:
        die("Unable to determine the current SUBPROTOCOL_VERSION.")

    return protocol_version, subprotocol_version

# vim: sw=4 et
#!/bin/sh -e

# This script will setup the build dir based on the current git branch and the
# directory auto-build-save/$BRANCH.  We don't use a symlink for the build dir
# because we want to maximize the ccache reuse, so all builds must happen in
# the same real dir.  When a dir is moved out of auto-build-save/$BRANCH to the
# build dir, it is replaced with a symlink so that it can still be found under
# that dir.  The build dir also gets a .branch -> $BRANCH symlink so that we
# can figure out the current build dir's branch.

# To get started, just clone the rsync git repo and create the auto-build-save
# dir.  If you have an existing git checkout and it is not in a pristine state,
# run "make distclean" before creating the auto-build-save dir.

auto_top='auto-build-save'
if test -d $auto_top && test -d .git; then
    desired_branch=`git rev-parse --abbrev-ref HEAD | tr / %`
    if test "$desired_branch" = HEAD; then
	echo "ERROR: switch to the right build dir manually when in detached HEAD mode." 1>&2
	exit 1
    fi
    auto_dir="$auto_top/$desired_branch"
    if test -d build; then
	cur_branch=`readlink build/.branch`
    else
	cur_branch='/'
    fi
    if test "$desired_branch" != "$cur_branch"; then
	if test "$cur_branch" != /; then
	    rm -f "$auto_top/$cur_branch"
	    mv build "$auto_top/$cur_branch"
	fi
	test -d "$auto_dir" || mkdir "$auto_dir"
	test -h "$auto_dir/.branch" || ln -s "$desired_branch" "$auto_dir/.branch"
	mv "$auto_dir" build
	ln -s ../build "$auto_dir"
    fi
    if test ! -h Makefile; then
	rm -f Makefile
	ln -s packaging/auto-Makefile Makefile
    fi
    echo $desired_branch
fi
#!/usr/bin/env -S python3 -B

# This script expects the directory ~/samba-rsync-ftp to exist and to be a
# copy of the /home/ftp/pub/rsync dir on samba.org.  When the script is done,
# the git repository in the current directory will be updated, and the local
# ~/samba-rsync-ftp dir will be ready to be rsynced to samba.org. See the
# script samba-rsync for an easy way to initialize the local ftp copy and to
# thereafter update the remote files from your local copy.

# This script also expects to be able to gpg sign the resulting tar files
# using your default gpg key. Make sure that the html download.html file
# has a link to the relevant keys that are authorized to sign the tar files
# and also make sure that the following commands work as expected:
#
#   touch TeMp
#   gpg --sign TeMp
#   gpg --verify TeMp.gpg
#   gpg --sign TeMp
#   rm TeMp*
#
# The second time you sign the file it should NOT prompt you for your password
# (unless the timeout period has passed).  It will prompt about overriding the
# existing TeMp.gpg file, though.

import os, sys, re, argparse, glob, shutil, signal
from datetime import datetime
from getpass import getpass

sys.path = ['packaging'] + sys.path

from pkglib import *

os.environ['LESS'] = 'mqeiXR'; # Make sure that -F is turned off and -R is turned on.
dest = os.environ['HOME'] + '/samba-rsync-ftp'
ORIGINAL_PATH = os.environ['PATH']

def main():
    if not os.path.isfile('packaging/release-rsync'):
        die('You must run this script from the top of your rsync checkout.')

    now = datetime.now()
    cl_today = now.strftime('* %a %b %d %Y')
    year = now.strftime('%Y')
    ztoday = now.strftime('%d %b %Y')
    today = ztoday.lstrip('0')

    curdir = os.getcwd()

    signal.signal(signal.SIGINT, signal_handler)

    if cmd_txt_chk(['packaging/prep-auto-dir']).out == '':
        die('You must setup an auto-build-save dir to use this script.');

    auto_dir, gen_files = get_gen_files(True)
    gen_pathnames = [ os.path.join(auto_dir, fn) for fn in gen_files ]

    dash_line = '=' * 74

    print(f"""\
{dash_line}
== This will release a new version of rsync onto an unsuspecting world. ==
{dash_line}
""")

    with open('build/rsync.1') as fh:
        for line in fh:
            if line.startswith(r'.\" prefix='):
                doc_prefix = line.split('=')[1].strip()
                if doc_prefix != '/usr':
                    warn(f"*** The documentation was built with prefix {doc_prefix} instead of /usr ***")
                    die("*** Read the md2man script for a way to override this. ***")
                break
            if line.startswith('.P'):
                die("Failed to find the prefix comment at the start of the rsync.1 manpage.")

    if not os.path.isdir(dest):
        die(dest, "dest does not exist")
    if not os.path.isdir('.git'):
        die("There is no .git dir in the current directory.")
    if os.path.lexists('a'):
        die('"a" must not exist in the current directory.')
    if os.path.lexists('b'):
        die('"b" must not exist in the current directory.')
    if os.path.lexists('patches.gen'):
        die('"patches.gen" must not exist in the current directory.')

    check_git_state(args.master_branch, True, 'patches')

    curversion = get_rsync_version()

    # All version values are strings!
    lastversion, last_protocol_version, pdate = get_NEWS_version_info()
    protocol_version, subprotocol_version = get_protocol_versions()

    version = curversion
    m = re.search(r'pre(\d+)', version)
    if m:
        version = re.sub(r'pre\d+', 'pre' + str(int(m[1]) + 1), version)
    else:
        version = version.replace('dev', 'pre1')

    ans = input(f"Please enter the version number of this release: [{version}] ")
    if ans == '.':
        version = re.sub(r'pre\d+', '', version)
    elif ans != '':
        version = ans
    if not re.match(r'^[\d.]+(pre\d+)?$', version):
        die(f'Invalid version: "{version}"')

    v_ver = 'v' + version
    rsync_ver = 'rsync-' + version

    if os.path.lexists(rsync_ver):
        die(f'"{rsync_ver}" must not exist in the current directory.')

    out = cmd_txt_chk(['git', 'tag', '-l', v_ver]).out
    if out != '':
        print(f"Tag {v_ver} already exists.")
        ans = input("\nDelete tag or quit? [Q/del] ")
        if not re.match(r'^del', ans, flags=re.I):
            die("Aborted")
        cmd_chk(['git', 'tag', '-d', v_ver])
        if os.path.isdir('patches/.git'):
            cmd_chk(f"cd patches && git tag -d '{v_ver}'")

    version = re.sub(r'[-.]*pre[-.]*', 'pre', version)
    if 'pre' in version and not curversion.endswith('dev'):
        lastversion = curversion

    ans = input(f"Enter the previous version to produce a patch against: [{lastversion}] ")
    if ans != '':
        lastversion = ans
    lastversion = re.sub(r'[-.]*pre[-.]*', 'pre', lastversion)

    rsync_lastver = 'rsync-' + lastversion
    if os.path.lexists(rsync_lastver):
        die(f'"{rsync_lastver}" must not exist in the current directory.')

    m = re.search(r'(pre\d+)', version)
    pre = m[1] if m else ''

    release = '0.1' if pre else '1'
    ans = input(f"Please enter the RPM release number of this release: [{release}] ")
    if ans != '':
        release = ans
    if pre:
        release += '.' + pre

    finalversion = re.sub(r'pre\d+', '', version)
    proto_changed = protocol_version != last_protocol_version
    if proto_changed:
        if finalversion in pdate:
            proto_change_date = pdate[finalversion]
        else:
            while True:
                ans = input("On what date did the protocol change to {protocol_version} get checked in? (dd Mmm yyyy) ")
                if re.match(r'^\d\d \w\w\w \d\d\d\d$', ans):
                    break
            proto_change_date = ans
    else:
        proto_change_date = ' ' * 11

    if 'pre' in lastversion:
        if not pre:
            die("You should not diff a release version against a pre-release version.")
        srcdir = srcdiffdir = lastsrcdir = 'src-previews'
        skipping = ' ** SKIPPING **'
    elif pre:
        srcdir = srcdiffdir = 'src-previews'
        lastsrcdir = 'src'
        skipping = ' ** SKIPPING **'
    else:
        srcdir = lastsrcdir = 'src'
        srcdiffdir = 'src-diffs'
        skipping = ''

    print(f"""
{dash_line}
version is "{version}"
lastversion is "{lastversion}"
dest is "{dest}"
curdir is "{curdir}"
srcdir is "{srcdir}"
srcdiffdir is "{srcdiffdir}"
lastsrcdir is "{lastsrcdir}"
release is "{release}"

About to:
    - tweak SUBPROTOCOL_VERSION in rsync.h, if needed
    - tweak the version in version.h and the spec files
    - tweak NEWS.md to ensure header values are correct
    - generate configure.sh, config.h.in, and proto.h
    - page through the differences
""")
    ans = input("<Press Enter to continue> ")

    specvars = {
        'Version:': finalversion,
        'Release:': release,
        '%define fullversion': f'%{{version}}{pre}',
        'Released': version + '.',
        '%define srcdir': srcdir,
        }

    tweak_files = 'version.h rsync.h'.split()
    tweak_files += glob.glob('packaging/*.spec')
    tweak_files += glob.glob('packaging/*/*.spec')

    for fn in tweak_files:
        with open(fn, 'r', encoding='utf-8') as fh:
            old_txt = txt = fh.read()
        if fn == 'version.h':
            x_re = re.compile(r'^(#define RSYNC_VERSION).*', re.M)
            msg = f"Unable to update RSYNC_VERSION in {fn}"
            txt = replace_or_die(x_re, r'\1 "%s"' % version, txt, msg)
        elif '.spec' in fn:
            for var, val in specvars.items():
                x_re = re.compile(r'^%s .*' % re.escape(var), re.M)
                txt = replace_or_die(x_re, var + ' ' + val, txt, f"Unable to update {var} in {fn}")
            x_re = re.compile(r'^\* \w\w\w \w\w\w \d\d \d\d\d\d (.*)', re.M)
            txt = replace_or_die(x_re, r'%s \1' % cl_today, txt, f"Unable to update ChangeLog header in {fn}")
        elif fn == 'rsync.h':
            x_re = re.compile('(#define\s+SUBPROTOCOL_VERSION)\s+(\d+)')
            repl = lambda m: m[1] + ' ' + ('0' if not pre or not proto_changed else '1' if m[2] == '0' else m[2])
            txt = replace_or_die(x_re, repl, txt, f"Unable to find SUBPROTOCOL_VERSION define in {fn}")
        elif fn == 'NEWS.md':
            efv = re.escape(finalversion)
            x_re = re.compile(r'^# NEWS for rsync %s \(UNRELEASED\)\s+## Changes in this version:\n' % efv
                    + r'(\n### PROTOCOL NUMBER:\s+- The protocol number was changed to \d+\.\n)?')
            rel_day = 'UNRELEASED' if pre else today
            repl = (f'# NEWS for rsync {finalversion} ({rel_day})\n\n'
                + '## Changes in this version:\n')
            if proto_changed:
                repl += f'\n### PROTOCOL NUMBER:\n\n - The protocol number was changed to {protocol_version}.\n'
            good_top = re.sub(r'\(.*?\)', '(UNRELEASED)', repl, 1)
            msg = f"The top lines of {fn} are not in the right format.  It should be:\n" + good_top
            txt = replace_or_die(x_re, repl, txt, msg)
            x_re = re.compile(r'^(\| )(\S{2} \S{3} \d{4})(\s+\|\s+%s\s+\| ).{11}(\s+\| )\S{2}(\s+\|+)$' % efv, re.M)
            repl = lambda m: m[1] + (m[2] if pre else ztoday) + m[3] + proto_change_date + m[4] + protocol_version + m[5]
            txt = replace_or_die(x_re, repl, txt, f'Unable to find "| ?? ??? {year} | {finalversion} | ... |" line in {fn}')
        else:
            die(f"Unrecognized file in tweak_files: {fn}")

        if txt != old_txt:
            print(f"Updating {fn}")
            with open(fn, 'w', encoding='utf-8') as fh:
                fh.write(txt)

    cmd_chk(['packaging/year-tweak'])

    print(dash_line)
    cmd_run("git diff".split())

    srctar_name = f"{rsync_ver}.tar.gz"
    pattar_name = f"rsync-patches-{version}.tar.gz"
    diff_name = f"{rsync_lastver}-{version}.diffs.gz"
    srctar_file = os.path.join(dest, srcdir, srctar_name)
    pattar_file = os.path.join(dest, srcdir, pattar_name)
    diff_file = os.path.join(dest, srcdiffdir, diff_name)
    lasttar_file = os.path.join(dest, lastsrcdir, rsync_lastver + '.tar.gz')

    print(f"""\
{dash_line}

About to:
    - git commit all changes
    - run a full build, ensuring that the manpages & configure.sh are up-to-date
    - merge the {args.master_branch} branch into the patch/{args.master_branch}/* branches
    - update the files in the "patches" dir and OPTIONALLY (if you type 'y') to
      run patch-update with the --make option (which opens a shell on error)
""")
    ans = input("<Press Enter OR 'y' to continue> ")

    s = cmd_run(['git', 'commit', '-a', '-m', f'Preparing for release of {version} [buildall]'])
    if s.returncode:
        die('Aborting')

    cmd_chk('touch configure.ac && packaging/smart-make && make gen')

    print('Creating any missing patch branches.')
    s = cmd_run(f'packaging/branch-from-patch --branch={args.master_branch} --add-missing')
    if s.returncode:
        die('Aborting')

    print('Updating files in "patches" dir ...')
    s = cmd_run(f'packaging/patch-update --branch={args.master_branch}')
    if s.returncode:
        die('Aborting')

    if re.match(r'^y', ans, re.I):
        print(f'\nRunning smart-make on all "patch/{args.master_branch}/*" branches ...')
        cmd_run(f"packaging/patch-update --branch={args.master_branch} --skip-check --make")

    if os.path.isdir('patches/.git'):
        s = cmd_run(f"cd patches && git commit -a -m 'The patches for {version}.'")
        if s.returncode:
            die('Aborting')

    print(f"""\
{dash_line}

About to:
    - create signed tag for this release: {v_ver}
    - create release diffs, "{diff_name}"
    - create release tar, "{srctar_name}"
    - generate {rsync_ver}/patches/* files
    - create patches tar, "{pattar_name}"
    - update top-level README.md, NEWS.md, TODO, and ChangeLog
    - update top-level rsync*.html manpages
    - gpg-sign the release files
    - update hard-linked top-level release files{skipping}
""")
    ans = input("<Press Enter to continue> ")

    # TODO: is there a better way to ensure that our passphrase is in the agent?
    cmd_run("touch TeMp; gpg --sign TeMp; rm TeMp*")

    out = cmd_txt(f"git tag -s -m 'Version {version}.' {v_ver}", capture='combined').out
    print(out, end='')
    if 'bad passphrase' in out or 'failed' in out:
        die('Aborting')

    if os.path.isdir('patches/.git'):
        out = cmd_txt(f"cd patches && git tag -s -m 'Version {version}.' {v_ver}", capture='combined').out
        print(out, end='')
        if 'bad passphrase' in out or 'failed' in out:
            die('Aborting')

    os.environ['PATH'] = ORIGINAL_PATH

    # Extract the generated files from the old tar.
    tweaked_gen_files = [ os.path.join(rsync_lastver, fn) for fn in gen_files ]
    cmd_run(['tar', 'xzf', lasttar_file, *tweaked_gen_files])
    os.rename(rsync_lastver, 'a')

    print(f"Creating {diff_file} ...")
    cmd_chk(['rsync', '-a', *gen_pathnames, 'b/'])

    sed_script = r's:^((---|\+\+\+) [ab]/[^\t]+)\t.*:\1:' # CAUTION: must not contain any single quotes!
    cmd_chk(f"(git diff v{lastversion} {v_ver} -- ':!.github'; diff -upN a b | sed -r '{sed_script}') | gzip -9 >{diff_file}")
    shutil.rmtree('a')
    os.rename('b', rsync_ver)

    print(f"Creating {srctar_file} ...")
    cmd_chk(f"git archive --format=tar --prefix={rsync_ver}/ {v_ver} | tar xf -")
    cmd_chk(f"support/git-set-file-times --quiet --prefix={rsync_ver}/")
    cmd_chk(['fakeroot', 'tar', 'czf', srctar_file, '--exclude=.github', rsync_ver])
    shutil.rmtree(rsync_ver)

    print(f'Updating files in "{rsync_ver}/patches" dir ...')
    os.mkdir(rsync_ver, 0o755)
    os.mkdir(f"{rsync_ver}/patches", 0o755)
    cmd_chk(f"packaging/patch-update --skip-check --branch={args.master_branch} --gen={rsync_ver}/patches".split())

    print(f"Creating {pattar_file} ...")
    cmd_chk(['fakeroot', 'tar', 'chzf', pattar_file, rsync_ver + '/patches'])
    shutil.rmtree(rsync_ver)

    print(f"Updating the other files in {dest} ...")
    md_files = 'README.md NEWS.md INSTALL.md'.split()
    html_files = [ fn for fn in gen_pathnames if fn.endswith('.html') ]
    cmd_chk(['rsync', '-a', *md_files, *html_files, dest])
    cmd_chk(["./md-convert", "--dest", dest, *md_files])

    cmd_chk(f"git log --name-status | gzip -9 >{dest}/ChangeLog.gz")

    for fn in (srctar_file, pattar_file, diff_file):
        asc_fn = fn + '.asc'
        if os.path.lexists(asc_fn):
            os.unlink(asc_fn)
        res = cmd_run(['gpg', '--batch', '-ba', fn])
        if res.returncode != 0 and res.returncode != 2:
            die("gpg signing failed")

    if not pre:
        for find in f'{dest}/rsync-*.gz {dest}/rsync-*.asc {dest}/src-previews/rsync-*diffs.gz*'.split():
            for fn in glob.glob(find):
                os.unlink(fn)
        top_link = [
                srctar_file, f"{srctar_file}.asc",
                pattar_file, f"{pattar_file}.asc",
                diff_file, f"{diff_file}.asc",
                ]
        for fn in top_link:
            os.link(fn, re.sub(r'/src(-\w+)?/', '/', fn))

    print(f"""\
{dash_line}

Local changes are done.  When you're satisfied, push the git repository
and rsync the release files.  Remember to announce the release on *BOTH*
rsync-announce@lists.samba.org and rsync@lists.samba.org (and the web)!
""")


def replace_or_die(regex, repl, txt, die_msg):
    m = regex.search(txt)
    if not m:
        die(die_msg)
    return regex.sub(repl, txt, 1)


def signal_handler(sig, frame):
    die("\nAborting due to SIGINT.")


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Prepare a new release of rsync in the git repo & ftp dir.", add_help=False)
    parser.add_argument('--branch', '-b', dest='master_branch', default='master', help="The branch to release. Default: master.")
    parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
    args = parser.parse_args()
    main()

# vim: sw=4 et ft=python
#!/bin/bash
# This script makes it easy to update the ftp & html directories on the samba.org server.
# It expects the 2 *_DEST directories to contain updated files that need to be sent to
# the remote server. If these directories don't exist yet, they will be copied from the
# remote server (while also making the html dir a git checkout).

FTP_SRC="$HOME/samba-rsync-ftp"
HTML_SRC="$HOME/samba-rsync-html"

FTP_DEST="/home/ftp/pub/rsync"
HTML_DEST="/home/httpd/html/rsync"

HTML_GIT='git.samba.org:/data/git/rsync-web.git'

export RSYNC_PARTIAL_DIR=''

case "$RSYNC_SAMBA_HOST" in
    *.samba.org) ;;
    *)
	echo "You must set RSYNC_SAMBA_HOST in your environment to the samba hostname to use." >&2
	exit 1
	;;
esac

MODE=''
REVERSE=''
while (( $# )); do
    case "$1" in
	-R|--reverse) REVERSE=yes ;;
	f|ftp) MODE=ftp ;;
	h|html) MODE=html ;;
	-h|--help)
	    echo "Usage: [-R] [f|ftp|h|html]"
	    echo "-R --reverse  Copy the files from the server to the local host."
	    echo "              The default is to update the remote files."
	    echo "-h --help     Output this help message."
	    echo " "
	    echo "The script will prompt if ftp or html is not specified on the command line."
	    echo "Only one category can be copied at a time. When pulling html files, a git"
	    echo "checkout will be either created or updated prior to the rsync copy."
	    exit
	    ;;
	*)
	    echo "Invalid option: $1" >&2
	    exit 1
	    ;;
    esac
    shift
done

while [ ! "$MODE" ]; do
    if [ "$REVERSE" = yes ]; then
	DIRECTION=FROM
    else
	DIRECTION=TO
    fi
    echo -n "Copy which files $DIRECTION the server? ftp or html? "
    read ans
    case "$ans" in
	f*) MODE=ftp ;;
	h*) MODE=html ;;
	'') exit 1 ;;
	*) echo "You must answer f or h to copy the ftp or html data." ;;
    esac
done

if [ "$MODE" = ftp ]; then
    SRC_DIR="$FTP_SRC"
    DEST_DIR="$FTP_DEST"
    FILT=".filt"
else
    SRC_DIR="$HTML_SRC"
    DEST_DIR="$HTML_DEST"
    FILT="filt"
fi

function do_rsync {
    rsync --dry-run "${@}" | grep -v 'is uptodate$'
    echo ''
    echo -n "Run without --dry-run? [n] "
    read ans
    case "$ans" in
	y*) rsync "${@}" | grep -v 'is uptodate$' ;;
    esac
}

if [ -d "$SRC_DIR" ]; then
    REVERSE_RSYNC=do_rsync
else
    echo "The directory $SRC_DIR does not exist yet."
    echo -n "Do you want to create it? [n] "
    read ans
    case "$ans" in
	y*) ;;
	*) exit 1 ;;
    esac
    REVERSE=yes
    REVERSE_RSYNC=rsync
fi

if [ "$REVERSE" = yes ]; then
    OPTS='-aivOHP'
    TMP_FILT="$SRC_DIR/tmp-filt"
    echo "Copying files from $RSYNC_SAMBA_HOST to $SRC_DIR ..."
    if [ "$MODE" = html ]; then
	if [ $REVERSE_RSYNC = rsync ]; then
	    git clone "$HTML_GIT" "$SRC_DIR" || exit 1
	else
	    cd "$SRC_DIR" || exit 1
	    git pull || exit 1
	fi
	sed -n -e 's/[-P]/H/p' "$SRC_DIR/$FILT" >"$TMP_FILT"
	OPTS="${OPTS}f._$TMP_FILT"
    else
	OPTS="${OPTS}f:_$FILT"
    fi
    $REVERSE_RSYNC "$OPTS" "$RSYNC_SAMBA_HOST:$DEST_DIR/" "$SRC_DIR/"
    rm -f "$TMP_FILT"
    exit
fi

cd "$SRC_DIR" || exit 1
echo "Copying files from $SRC_DIR to $RSYNC_SAMBA_HOST ..."
do_rsync -aivOHP --chown=:rsync --del -f._$FILT . "$RSYNC_SAMBA_HOST:$DEST_DIR/"
#!/bin/bash -e

# This script expects the ~/src/rsync directory to contain the rsync
# source that has been updated. It also expects the auto-build-save
# directory to have been created prior to the running of configure so
# that each branch has its own build directory underneath. This supports
# the maintainer workflow for the rsync-patches files maintenace.

FTP_SRC="$HOME/samba-rsync-ftp"
FTP_DEST="/home/ftp/pub/rsync"
MD_FILES="README.md INSTALL.md NEWS.md"

case "$RSYNC_SAMBA_HOST" in
    *.samba.org) ;;
    *)
	echo "You must set RSYNC_SAMBA_HOST in your environment to the samba hostname to use." >&2
	exit 1
	;;
esac

if [ ! -d "$FTP_SRC" ]; then
    packaging/samba-rsync ftp # Ask to initialize the local ftp dir
fi

cd ~/src/rsync

make man
./md-convert --dest="$FTP_SRC" $MD_FILES
rsync -aiic $MD_FILES auto-build-save/master/*.?.html "$FTP_SRC"

cd "$FTP_SRC"

rsync -aiic README.* INSTALL.* NEWS.* *.?.html "$RSYNC_SAMBA_HOST:$FTP_DEST/"
#!/bin/sh
 
set -e

export LANG=C

branch=`packaging/prep-auto-dir`
if test x"$branch" = x; then
    srcdir=.
else
    cd build
    srcdir=..
fi

if test -f configure.sh; then
    cp -p configure.sh configure.sh.old
else
    touch configure.sh.old
fi

if test -f .fetch; then
    $srcdir/prepare-source fetch
else
    $srcdir/prepare-source
fi

if diff configure.sh configure.sh.old >/dev/null 2>&1; then
    echo "configure.sh is unchanged."
    rm configure.sh.old
else
    echo "configure.sh has CHANGED."
    if test -f config.status; then
	./config.status --recheck
    else
	$srcdir/configure
    fi
fi

./config.status

make all

if test x"$1" = x"check"; then
    make check
fi
#!/bin/sh
# Shell script for building Solaris package of rsync
# Author: Jens Apel <jens.apel@web.de>
# License: GPL
#
# BASEDIR is /usr/local and should be the same as the
# --prefix parameter of configure
#
# this script should be copied under
# packaging/solaris/5.8/build_pkg.sh

# Definitions start here
# you can edit this, if you like

# The Package name under which rsync will b installed
PKGNAME=SMBrsync

# Extract common info requires for the 'info' part of the package.
# This should be made generic and generated by the configure script
# but for now it is hard coded
BASEDIR=/usr/local
VERSION="2.5.5"
ARCH=`uname -p`
NAME=rsync

# Definitions end here
# Please do not edit below this line or you know what you do.

## Start by faking root install
echo "Creating install directory (fake $BASEDIR)..."
START=`pwd`
FAKE_ROOT=$START/${PKGNAME}
mkdir $FAKE_ROOT

# copy the binary and the man page to their places
mkdir $FAKE_ROOT/bin
mkdir -p $FAKE_ROOT/doc/rsync
mkdir -p $FAKE_ROOT/man/man1
mkdir -p $FAKE_ROOT/man/man5

cp ../../../rsync $FAKE_ROOT/bin/rsync
cp ../../../rsync.1 $FAKE_ROOT/man/man1/rsync.1
cp ../../../rsyncd.conf.5 $FAKE_ROOT/man/man5/rsyncd.conf.5
cp ../../../README.md $FAKE_ROOT/doc/rsync/README.md
cp ../../../COPYING $FAKE_ROOT/doc/rsync/COPYING
cp ../../../tech_report.pdf $FAKE_ROOT/doc/rsync/tech_report.pdf
cp ../../../COPYING $FAKE_ROOT/COPYING

## Build info file
echo "Building pkginfo file..."
cat > $FAKE_ROOT/pkginfo << EOF_INFO
PKG=$PKGNAME
NAME=$NAME
DESC="Program for efficient remote updates of files."
VENDOR="Samba Team URL: http://samba.anu.edu.au/rsync/"
BASEDIR=$BASEDIR
ARCH=$ARCH
VERSION=$VERSION
CATEGORY=application
CLASSES=none
EOF_INFO

## Build prototype file
cat > $FAKE_ROOT/prototype << EOFPROTO
i copyright=COPYING
i pkginfo=pkginfo
d none bin 0755 bin bin
f none bin/rsync 0755 bin bin
d none doc 0755 bin bin
d none doc/$NAME 0755 bin bin
f none doc/$NAME/README.md 0644 bin bin
f none doc/$NAME/COPYING 0644 bin bin
f none doc/$NAME/tech_report.pdf 0644 bin bin
d none man 0755 bin bin
d none man/man1 0755 bin bin
f none man/man1/rsync.1 0644 bin bin
d none man/man5 0755 bin bin
f none man/man5/rsyncd.conf.5 0644 bin bin
EOFPROTO

## And now build the package.
OUTPUTFILE=$PKGNAME-$VERSION-sol8-$ARCH-local.pkg
echo "Building package.."
echo FAKE_ROOT = $FAKE_ROOT
cd $FAKE_ROOT
pkgmk -d . -r . -f ./prototype -o
pkgtrans -os . $OUTPUTFILE $PKGNAME

mv $OUTPUTFILE ..
cd ..

# Comment this out if you want to see, which file structure has been created
rm -rf $FAKE_ROOT

[Unit]
Description=fast remote file copy program daemon
ConditionPathExists=/etc/rsyncd.conf
After=network.target
Documentation=man:rsync(1) man:rsyncd.conf(5)

[Service]
ExecStart=/usr/bin/rsync --daemon --no-detach
RestartSec=1
Restart=on-failure

# Citing README.md:
#
#   [...] Using ssh is recommended for its security features.
#
#   Alternatively, rsync can run in `daemon' mode, listening on a socket.
#   This is generally used for public file distribution, [...]
#
# So let's assume some extra security is more than welcome here. We do full
# system protection (which makes /usr, /boot, & /etc read-only) and hide
# devices. To override these defaults, it's best to do so in the drop-in
# directory, often done via `systemctl edit rsync.service`. The file needs
# just the bare minimum of the right [heading] and override values.
# See systemd.unit(5) and search for "drop-in" for full details.

ProtectSystem=full
#ProtectHome=on|off|read-only
PrivateDevices=on
NoNewPrivileges=on

[Install]
WantedBy=multi-user.target
[Unit]
Description=socket for fast remote file copy program daemon
Conflicts=rsync.service

[Socket]
ListenStream=873
Accept=true

[Install]
WantedBy=sockets.target
[Unit]
Description=fast remote file copy program daemon
After=network.target

[Service]
ExecStart=-/usr/bin/rsync --daemon
StandardInput=socket
StandardOutput=inherit
StandardError=journal

# Citing README.md:
#
#   [...] Using ssh is recommended for its security features.
#
#   Alternatively, rsync can run in `daemon' mode, listening on a socket.
#   This is generally used for public file distribution, [...]
#
# So let's assume some extra security is more than welcome here. We do full
# system protection (which makes /usr, /boot, & /etc read-only) and hide
# devices. To override these defaults, it's best to do so in the drop-in
# directory, often done via `systemctl edit rsync@.service`. The file needs
# just the bare minimum of the right [heading] and override values.
# See systemd.unit(5) and search for "drop-in" for full details.

ProtectSystem=full
#ProtectHome=on|off|read-only
PrivateDevices=on
NoNewPrivileges=on
#!/usr/bin/env -S python3 -B

# This script checks the *.c files for extraneous "extern" variables,
# for vars that are defined but not used, and for inconsistent array
# sizes.  Run it from inside the main rsync directory.

import os, sys, re, argparse, glob

VARS_RE = re.compile(r'^(?!(?:extern|enum)\s)([a-zA-Z][^ \n\t:]*\s+.*);', re.M)
EXTERNS_RE = re.compile(r'^extern\s+(.*);', re.M)

types = { }
sizes = { }

def main():
    add_syscall_c = set('t_stub.c t_unsafe.c tls.c trimslash.c'.split())
    add_util_c = set('t_stub.c t_unsafe.c'.split())

    if not os.path.exists('syscall.c'):
        if os.path.exists('var-checker'):
            os.chdir('..')
        else:
            print("Couldn't find the source dir.")
            sys.exit(1)

    syscall_c = slurp_file('syscall.c', True)
    util_c = slurp_file('util1.c', True)

    for fn in sorted(glob.glob('*.c')):
        txt = slurp_file(fn)

        var_list = parse_vars(fn, VARS_RE.findall(txt))
        extern_list = parse_vars(fn, EXTERNS_RE.findall(txt))
        if not var_list and not extern_list:
            continue

        if fn in add_syscall_c:
            txt += syscall_c
        if fn in add_util_c:
            txt += util_c

        txt = re.sub(r'INFO_GTE', 'info_levels ', txt)
        txt = re.sub(r'DEBUG_GTE', 'debug_levels ', txt)
        txt = re.sub(r'SIGACTION\(', 'sigact (', txt)

        find = '|'.join([ re.escape(x) for x in var_list + extern_list ])
        var_re = re.compile(r'(?<!\sstruct )\b(%s)(?!\w)' % find)

        found = { x: 0 for x in var_list + extern_list }
        for var in var_re.findall(txt):
            found[var] += 1

        for var in sorted(var_list + extern_list):
            if found[var] == 1:
                vtype = 'var' if var in var_list else 'extern'
                print(fn, f'has extraneous {vtype}: "{var}"')


def slurp_file(fn, drop_externs=False):
    with open(fn, 'r', encoding='utf-8') as fh:
        txt = fh.read()
    if drop_externs:
        txt = EXTERNS_RE.sub('', txt)
    return txt


def parse_vars(fn, lines):
    ret = [ ]
    for line in lines:
        line = re.sub(r'\s*\{.*\}', '', line)
        line = re.sub(r'\s*\(.*\)', '', line)
        line = re.sub(r'\s*=\s*[^,]*', '', line)
        m = re.search(r'^(?:(?:static|extern)\s+)?(?P<type>[^\[,]+?)(?P<vars>\w+([\[,].+)?)$', line)
        if not m:
            print(f"Bogus match? ({line})")
            continue
        items = m['vars']
        main_type = m['type'].strip()
        mt_len = len(main_type)
        main_type = main_type.rstrip('*')
        first_stars = '*' * (mt_len - len(main_type))
        if first_stars:
            main_type = main_type.rstrip()
            items = first_stars + items
        for item in re.split(r'\s*,\s*', items):
            m = re.search(r'(?P<stars>\*+\s*)?(?P<var>\w+)(?P<sz>\[.*?\])?$', item)
            if not m:
                print(f"Bogus match? ({item})")
                continue
            typ = main_type
            if m['stars']:
                typ = typ + m['stars'].strip()
            chk = [
                    'type', typ, types,
                    'size', m['sz'], sizes,
                    ]
            while chk:
                label = chk.pop(0)
                new = chk.pop(0)
                lst = chk.pop(0)
                if label == 'type':
                    new = ' '.join(new.split()).replace(' *', '*')
                if m['var'] in lst:
                    old = lst[m['var']]
                    if new != old:
                        var = m['var']
                        print(fn, f'has inconsistent {label} for "{var}":', new, 'vs', old)
                else:
                    lst[m['var']] = new
            ret.append(m['var'])
    return ret


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Check the *.c files for extraneous extern vars.', add_help=False)
    parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
    args = parser.parse_args()
    main()

# vim: sw=4 et ft=python
#!/usr/bin/env python3

# This uses the output from "support/git-set-file-times --list" to discern
# the last-modified year of each *.c & *.h file and updates the copyright
# year if it isn't set right.

import sys, os, re, argparse, subprocess
from datetime import datetime

MAINTAINER_NAME = 'Wayne Davison'
MAINTAINER_SUF = ' ' + MAINTAINER_NAME + "\n"

def main():
    latest_year = '2000'

    proc = subprocess.Popen('support/git-set-file-times --list'.split(), stdout=subprocess.PIPE, encoding='utf-8')
    for line in proc.stdout:
        m = re.match(r'^\S\s+(?P<year>\d\d\d\d)\S+\s+\S+\s+(?P<fn>.+)', line)
        if not m:
            print("Failed to parse line from git-set-file-times:", line)
            sys.exit(1)
        m = argparse.Namespace(**m.groupdict())
        if m.year > latest_year:
            latest_year = m.year
        if m.fn.startswith('zlib/') or m.fn.startswith('popt/'):
            continue
        if re.search(r'\.(c|h|sh|test)$', m.fn):
            maybe_edit_copyright_year(m.fn, m.year)
    proc.communicate()

    fn = 'latest-year.h'
    with open(fn, 'r', encoding='utf-8') as fh:
        old_txt = fh.read()

    txt = f'#define LATEST_YEAR "{latest_year}"\n'
    if txt != old_txt:
        print(f"Updating {fn} with year {latest_year}")
        with open(fn, 'w', encoding='utf-8') as fh:
            fh.write(txt)


def maybe_edit_copyright_year(fn, year):
    opening_lines = [ ]
    copyright_line = None

    with open(fn, 'r', encoding='utf-8') as fh:
        for lineno, line in enumerate(fh):
            opening_lines.append(line)
            if lineno > 3 and not re.search(r'\S', line):
                break
            m = re.match(r'^(?P<pre>.*Copyright\s+\S+\s+)(?P<year>\d\d\d\d(?:-\d\d\d\d)?(,\s+\d\d\d\d)*)(?P<suf>.+)', line)
            if not m:
                continue
            copyright_line = argparse.Namespace(**m.groupdict())
            copyright_line.lineno = len(opening_lines)
            copyright_line.is_maintainer_line = MAINTAINER_NAME in copyright_line.suf
            copyright_line.txt = line
            if copyright_line.is_maintainer_line:
                break

        if not copyright_line:
            return

        if copyright_line.is_maintainer_line:
            cyears = copyright_line.year.split('-')
            if year == cyears[0]:
                cyears = [ year ]
            else:
                cyears = [ cyears[0], year ]
            txt = copyright_line.pre + '-'.join(cyears) + MAINTAINER_SUF
            if txt == copyright_line.txt:
                return
            opening_lines[copyright_line.lineno - 1] = txt
        else:
            if fn.startswith('lib/') or fn.startswith('testsuite/'):
                return
            txt = copyright_line.pre + year + MAINTAINER_SUF
            opening_lines[copyright_line.lineno - 1] += txt

        remaining_txt = fh.read()

    print(f"Updating {fn} with year {year}")

    with open(fn, 'w', encoding='utf-8') as fh:
        fh.write(''.join(opening_lines))
        fh.write(remaining_txt)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Grab the year of last mod for our c & h files and make sure the Copyright comment is up-to-date.")
    args = parser.parse_args()
    main()

# vim: sw=4 et