test-runner - zygaena/hammerhead

root / base / usr / src / test / test-runner
test-runner Plain Text 3170 lines 85.4 KB
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source.  A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#

#
# Copyright (c) 2012 by Delphix. All rights reserved.
#

.PARALLEL: $(SUBDIRS)

SUBDIRS = cmd stf man

include $(SRC)/test/Makefile.com
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source.  A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#

#
# Copyright (c) 2012 by Delphix. All rights reserved.
#

include $(SRC)/Makefile.master
include $(SRC)/test/Makefile.com

ROOTOPTPKG = $(ROOT)/opt/test-runner
ROOTBIN = $(ROOTOPTPKG)/bin

PROGS = run

CMDS = $(PROGS:%=$(ROOTBIN)/%)
$(CMDS) : FILEMODE = 0555

all clean clobber:

install: $(CMDS)

$(CMDS): $(ROOTBIN)

$(ROOTBIN):
	$(INS.dir)

$(ROOTBIN)/%: %
	$(INS.pyfile)
#!@PYTHON@

#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source.  A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#

#
# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
# Copyright (c) 2017, Chris Fraire <cfraire@me.com>.
# Copyright 2019 Joyent, Inc.
# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
#

from __future__ import print_function
import sys
PY3 = sys.version_info[0] == 3

if PY3:
    import configparser
else:
    import ConfigParser as configparser

import io
import os
import logging
import platform
import re
from logging.handlers import WatchedFileHandler
from datetime import datetime
from optparse import OptionParser
from pwd import getpwnam
from pwd import getpwuid
from select import select
from subprocess import PIPE
from subprocess import Popen
from sys import argv
from sys import exit
from sys import maxsize
from threading import Timer
from time import time

BASEDIR = '/var/tmp/test_results'
TESTDIR = '/opt/zfs-tests/'
KILL = '/usr/bin/kill'
TRUE = '/usr/bin/true'
SUDO = '/usr/bin/sudo'

retcode = 0

# Custom class to reopen the log file in case it is forcibly closed by a test.
class WatchedFileHandlerClosed(WatchedFileHandler):
    """Watch files, including closed files.
    Similar to (and inherits from) logging.handler.WatchedFileHandler,
    except that IOErrors are handled by reopening the stream and retrying.
    This will be retried up to a configurable number of times before
    giving up, default 5.
    """

    def __init__(self, filename, mode='a', encoding='utf-8', delay=0, max_tries=5):
        self.max_tries = max_tries
        self.tries = 0
        WatchedFileHandler.__init__(self, filename, mode, encoding, delay)

    def emit(self, record):
        while True:
            try:
                WatchedFileHandler.emit(self, record)
                self.tries = 0
                return
            except IOError as err:
                if self.tries == self.max_tries:
                    raise
                self.stream.close()
                self.stream = self._open()
                self.tries += 1

class Result(object):
    total = 0
    runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0}

    def __init__(self):
        self.starttime = None
        self.returncode = None
        self.runtime = ''
        self.stdout = []
        self.stderr = []
        self.result = ''

    def done(self, proc, killed):
        """
        Finalize the results of this Cmd.
        Report SKIP for return codes 3,4 (NOTINUSE, UNSUPPORTED)
        as defined in ../stf/include/stf.shlib
        """
        global retcode

        Result.total += 1
        m, s = divmod(time() - self.starttime, 60)
        self.runtime = '%02d:%02d' % (m, s)
        self.returncode = proc.returncode
        if killed:
            self.result = 'KILLED'
            Result.runresults['KILLED'] += 1
            retcode = 2;
        elif self.returncode == 0:
            self.result = 'PASS'
            Result.runresults['PASS'] += 1
        elif self.returncode == 3 or self.returncode == 4:
            self.result = 'SKIP'
            Result.runresults['SKIP'] += 1
        elif self.returncode != 0:
            self.result = 'FAIL'
            Result.runresults['FAIL'] += 1
            retcode = 1;


class Output(object):
    """
    This class is a slightly modified version of the 'Stream' class found
    here: http://goo.gl/aSGfv
    """
    def __init__(self, stream):
        self.stream = stream
        self._buf = ''
        self.lines = []

    def fileno(self):
        return self.stream.fileno()

    def read(self, drain=0):
        """
        Read from the file descriptor. If 'drain' set, read until EOF.
        """
        while self._read() is not None:
            if not drain:
                break

    def _read(self):
        """
        Read up to 4k of data from this output stream. Collect the output
        up to the last newline, and append it to any leftover data from a
        previous call. The lines are stored as a (timestamp, data) tuple
        for easy sorting/merging later.
        """
        fd = self.fileno()
        buf = os.read(fd, 4096).decode('utf-8', errors='ignore')
        if not buf:
            return None
        if '\n' not in buf:
            self._buf += buf
            return []

        buf = self._buf + buf
        tmp, rest = buf.rsplit('\n', 1)
        self._buf = rest
        now = datetime.now()
        rows = tmp.split('\n')
        self.lines += [(now, r) for r in rows]


class Cmd(object):
    verified_users = []

    def __init__(self, pathname, identifier=None, outputdir=None,
                 timeout=None, user=None, tags=None):
        self.pathname = pathname
        self.identifier = identifier
        self.outputdir = outputdir or 'BASEDIR'
        self.timeout = timeout
        self.user = user or ''
        self.killed = False
        self.result = Result()

        if self.timeout is None:
            self.timeout = 60

    def __str__(self):
        return '''\
Pathname: %s
Identifier: %s
Outputdir: %s
Timeout: %d
User: %s
''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user)

    def kill_cmd(self, proc):
        """
        Kill a running command due to timeout, or ^C from the keyboard. If
        sudo is required, this user was verified previously.
        """
        self.killed = True
        do_sudo = len(self.user) != 0
        signal = '-TERM'

        cmd = [SUDO, KILL, signal, str(proc.pid)]
        if not do_sudo:
            del cmd[0]

        try:
            kp = Popen(cmd)
            kp.wait()
        except:
            pass

    def update_cmd_privs(self, cmd, user):
        """
        If a user has been specified to run this Cmd and we're not already
        running as that user, prepend the appropriate sudo command to run
        as that user.
        """
        me = getpwuid(os.getuid())

        if not user or user == me.pw_name:
            return cmd

        ret = '%s -E -u %s %s' % (SUDO, user, cmd)
        return ret.split(' ')

    def collect_output(self, proc):
        """
        Read from stdout/stderr as data becomes available, until the
        process is no longer running. Return the lines from the stdout and
        stderr Output objects.
        """
        out = Output(proc.stdout)
        err = Output(proc.stderr)
        res = []
        while proc.returncode is None:
            proc.poll()
            res = select([out, err], [], [], .1)
            for fd in res[0]:
                fd.read()
        for fd in res[0]:
            fd.read(drain=1)

        return out.lines, err.lines

    def run(self, options):
        """
        This is the main function that runs each individual test.
        Determine whether or not the command requires sudo, and modify it
        if needed. Run the command, and update the result object.
        """
        if options.dryrun is True:
            print(self)
            return

        privcmd = self.update_cmd_privs(self.pathname, self.user)
        try:
            old = os.umask(0)
            if not os.path.isdir(self.outputdir):
                os.makedirs(self.outputdir, mode=0o777)
            os.umask(old)
        except OSError as e:
            fail('%s' % e)

        try:
            self.result.starttime = time()
            proc = Popen(privcmd, stdout=PIPE, stderr=PIPE, stdin=PIPE,
                         universal_newlines=True)
            proc.stdin.close()

            # Allow a special timeout value of 0 to mean infinity
            if int(self.timeout) == 0:
                self.timeout = maxsize
            t = Timer(int(self.timeout), self.kill_cmd, [proc])
            t.start()
            self.result.stdout, self.result.stderr = self.collect_output(proc)
        except KeyboardInterrupt:
            self.kill_cmd(proc)
            fail('\nRun terminated at user request.')
        finally:
            t.cancel()

        self.result.done(proc, self.killed)

    def skip(self):
        """
        Initialize enough of the test result that we can log a skipped
        command.
        """
        Result.total += 1
        Result.runresults['SKIP'] += 1
        self.result.stdout = self.result.stderr = []
        self.result.starttime = time()
        m, s = divmod(time() - self.result.starttime, 60)
        self.result.runtime = '%02d:%02d' % (m, s)
        self.result.result = 'SKIP'

    def log(self, logger, options):
        """
        This function is responsible for writing all output. This includes
        the console output, the logfile of all results (with timestamped
        merged stdout and stderr), and for each test, the unmodified
        stdout/stderr/merged in it's own file.
        """
        if logger is None:
            return

        logname = getpwuid(os.getuid()).pw_name
        user = ' (run as %s)' % (self.user if len(self.user) else logname)
        if self.identifier:
            msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user)
        else:
            msga = 'Test: %s%s ' % (self.pathname, user)
        msgb = '[%s] [%s]' % (self.result.runtime, self.result.result)
        pad = ' ' * (80 - (len(msga) + len(msgb)))

        # If -q is specified, only print a line for tests that didn't pass.
        # This means passing tests need to be logged as DEBUG, or the one
        # line summary will only be printed in the logfile for failures.
        if not options.quiet:
            logger.info('%s%s%s' % (msga, pad, msgb))
        elif self.result.result != 'PASS':
            logger.info('%s%s%s' % (msga, pad, msgb))
        else:
            logger.debug('%s%s%s' % (msga, pad, msgb))

        lines = sorted(self.result.stdout + self.result.stderr,
                       key=lambda x: x[0])

        for dt, line in lines:
            logger.debug('%s %s' % (dt.strftime("%H:%M:%S.%f ")[:11], line))

        if len(self.result.stdout):
            with io.open(os.path.join(self.outputdir, 'stdout'),
                                   encoding='utf-8',
                                   errors='surrogateescape',
                                   mode='w') as out:
                for _, line in self.result.stdout:
                    out.write('%s\n' % line)
        if len(self.result.stderr):
            with io.open(os.path.join(self.outputdir, 'stderr'),
                                   encoding='utf-8',
                                   errors='surrogateescape',
                                   mode='w') as err:
                for _, line in self.result.stderr:
                    err.write('%s\n' % line)
        if len(self.result.stdout) and len(self.result.stderr):
            with io.open(os.path.join(self.outputdir, 'merged'),
                                   encoding='utf-8',
                                   errors='surrogateescape',
                                   mode='w') as merged:
                for _, line in lines:
                    merged.write('%s\n' % line)


class Test(Cmd):
    props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post',
             'post_user', 'tags']

    def __init__(self, pathname,
                 pre=None, pre_user=None, post=None, post_user=None,
                 tags=None, **kwargs):
        super(Test, self).__init__(pathname, **kwargs)
        self.pre = pre or ''
        self.pre_user = pre_user or ''
        self.post = post or ''
        self.post_user = post_user or ''
        self.tags = tags or []

    def __str__(self):
        post_user = pre_user = ''
        if len(self.pre_user):
            pre_user = ' (as %s)' % (self.pre_user)
        if len(self.post_user):
            post_user = ' (as %s)' % (self.post_user)
        return '''\
Pathname: %s
Identifier: %s
Outputdir: %s
Timeout: %d
User: %s
Pre: %s%s
Post: %s%s
Tags: %s
''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user,
            self.pre, pre_user, self.post, post_user, self.tags)

    def verify(self, logger):
        """
        Check the pre/post scripts, user and Test. Omit the Test from this
        run if there are any problems.
        """
        files = [self.pre, self.pathname, self.post]
        users = [self.pre_user, self.user, self.post_user]

        for f in [f for f in files if len(f)]:
            if not verify_file(f):
                logger.info("Warning: Test '%s' not added to this run because"
                            " it failed verification." % f)
                return False

        for user in [user for user in users if len(user)]:
            if not verify_user(user, logger):
                logger.info("Not adding Test '%s' to this run." %
                            self.pathname)
                return False

        return True

    def run(self, logger, options):
        """
        Create Cmd instances for the pre/post scripts. If the pre script
        doesn't pass, skip this Test. Run the post script regardless.
        """
        odir = os.path.join(self.outputdir, os.path.basename(self.pre))
        pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir,
                      timeout=self.timeout, user=self.pre_user)
        test = Cmd(self.pathname, identifier=self.identifier,
                   outputdir=self.outputdir, timeout=self.timeout,
                   user=self.user)
        odir = os.path.join(self.outputdir, os.path.basename(self.post))
        posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir,
                       timeout=self.timeout, user=self.post_user)

        cont = True
        if len(pretest.pathname):
            pretest.run(options)
            cont = pretest.result.result == 'PASS'
            pretest.log(logger, options)

        if cont:
            test.run(options)
        else:
            test.skip()

        test.log(logger, options)

        if len(posttest.pathname):
            posttest.run(options)
            posttest.log(logger, options)


class TestGroup(Test):
    props = Test.props + ['tests']

    def __init__(self, pathname, tests=None, **kwargs):
        super(TestGroup, self).__init__(pathname, **kwargs)
        self.tests = tests or []

    def __str__(self):
        post_user = pre_user = ''
        if len(self.pre_user):
            pre_user = ' (as %s)' % (self.pre_user)
        if len(self.post_user):
            post_user = ' (as %s)' % (self.post_user)
        return '''\
Pathname: %s
Identifier: %s
Outputdir: %s
Tests: %s
Timeout: %d
User: %s
Pre: %s%s
Post: %s%s
Tags: %s
''' % (self.pathname, self.identifier, self.outputdir, self.tests,
            self.timeout, self.user, self.pre, pre_user, self.post, post_user,
            self.tags)

    def filter(self, keeplist):
        self.tests = [ x for x in self.tests if x in keeplist ]

    def verify(self, logger):
        """
        Check the pre/post scripts, user and tests in this TestGroup. Omit
        the TestGroup entirely, or simply delete the relevant tests in the
        group, if that's all that's required.
        """
        # If the pre or post scripts are relative pathnames, convert to
        # absolute, so they stand a chance of passing verification.
        if len(self.pre) and not os.path.isabs(self.pre):
            self.pre = os.path.join(self.pathname, self.pre)
        if len(self.post) and not os.path.isabs(self.post):
            self.post = os.path.join(self.pathname, self.post)

        auxfiles = [self.pre, self.post]
        users = [self.pre_user, self.user, self.post_user]

        for f in [f for f in auxfiles if len(f)]:
            if self.pathname != os.path.dirname(f):
                logger.info("Warning: TestGroup '%s' not added to this run. "
                            "Auxiliary script '%s' exists in a different "
                            "directory." % (self.pathname, f))
                return False

            if not verify_file(f):
                logger.info("Warning: TestGroup '%s' not added to this run. "
                            "Auxiliary script '%s' failed verification." %
                            (self.pathname, f))
                return False

        for user in [user for user in users if len(user)]:
            if not verify_user(user, logger):
                logger.info("Not adding TestGroup '%s' to this run." %
                            self.pathname)
                return False

        # If one of the tests is invalid, delete it, log it, and drive on.
        self.tests[:] = [f for f in self.tests if
          verify_file(os.path.join(self.pathname, f))]

        return len(self.tests) != 0

    def run(self, logger, options):
        """
        Create Cmd instances for the pre/post scripts. If the pre script
        doesn't pass, skip all the tests in this TestGroup. Run the post
        script regardless.
        """
        # tags assigned to this test group also include the test names
        if options.tags and not set(self.tags).intersection(set(options.tags)):
            return

        odir = os.path.join(self.outputdir, os.path.basename(self.pre))
        pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
                      user=self.pre_user, identifier=self.identifier)
        odir = os.path.join(self.outputdir, os.path.basename(self.post))
        posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
                       user=self.post_user, identifier=self.identifier)

        cont = True
        if len(pretest.pathname):
            pretest.run(options)
            cont = pretest.result.result == 'PASS'
            pretest.log(logger, options)

        for fname in self.tests:
            test = Cmd(os.path.join(self.pathname, fname),
                       outputdir=os.path.join(self.outputdir, fname),
                       timeout=self.timeout, user=self.user,
                       identifier=self.identifier)
            if cont:
                test.run(options)
            else:
                test.skip()

            test.log(logger, options)

        if len(posttest.pathname):
            posttest.run(options)
            posttest.log(logger, options)


class TestRun(object):
    props = ['quiet', 'outputdir']

    def __init__(self, options):
        self.tests = {}
        self.testgroups = {}
        self.starttime = time()
        self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S')
        self.outputdir = os.path.join(options.outputdir, self.timestamp)
        self.logger = self.setup_logging(options)
        self.defaults = [
            ('outputdir', BASEDIR),
            ('quiet', False),
            ('timeout', 60),
            ('user', ''),
            ('pre', ''),
            ('pre_user', ''),
            ('post', ''),
            ('post_user', ''),
            ('tags', [])
        ]

    def __str__(self):
        s = 'TestRun:\n    outputdir: %s\n' % self.outputdir
        s += 'TESTS:\n'
        for key in sorted(self.tests.keys()):
            s += '%s%s' % (self.tests[key].__str__(), '\n')
        s += 'TESTGROUPS:\n'
        for key in sorted(self.testgroups.keys()):
            s += '%s%s' % (self.testgroups[key].__str__(), '\n')
        return s

    def addtest(self, pathname, options):
        """
        Create a new Test, and apply any properties that were passed in
        from the command line. If it passes verification, add it to the
        TestRun.
        """
        test = Test(pathname)
        for prop in Test.props:
            setattr(test, prop, getattr(options, prop))

        if test.verify(self.logger):
            self.tests[pathname] = test

    def addtestgroup(self, dirname, filenames, options):
        """
        Create a new TestGroup, and apply any properties that were passed
        in from the command line. If it passes verification, add it to the
        TestRun.
        """
        if dirname not in self.testgroups:
            testgroup = TestGroup(dirname)
            for prop in Test.props:
                setattr(testgroup, prop, getattr(options, prop))

            # Prevent pre/post scripts from running as regular tests
            for f in [testgroup.pre, testgroup.post]:
                if f in filenames:
                    del filenames[filenames.index(f)]

            self.testgroups[dirname] = testgroup
            self.testgroups[dirname].tests = sorted(filenames)

            testgroup.verify(self.logger)

    def filter(self, keeplist):
        for group in list(self.testgroups.keys()):
            if group not in keeplist:
                del self.testgroups[group]
                continue

            g = self.testgroups[group]

            if g.pre and os.path.basename(g.pre) in keeplist[group]:
                continue

            g.filter(keeplist[group])

        for test in list(self.tests.keys()):
            directory, base = os.path.split(test)
            if directory not in keeplist or base not in keeplist[directory]:
                del self.tests[test]

    def read(self, logger, options):
        """
        Read in the specified runfile, and apply the TestRun properties
        listed in the 'DEFAULT' section to our TestRun. Then read each
        section, and apply the appropriate properties to the Test or
        TestGroup. Properties from individual sections override those set
        in the 'DEFAULT' section. If the Test or TestGroup passes
        verification, add it to the TestRun.
        """
        config = configparser.RawConfigParser()
        parsed = config.read(options.runfiles)
        failed = options.runfiles - set(parsed)
        if len(failed):
            files = ' '.join(sorted(failed))
            fail("Couldn't read config files: %s" % files)

        for opt in TestRun.props:
            if config.has_option('DEFAULT', opt):
                setattr(self, opt, config.get('DEFAULT', opt))
        self.outputdir = os.path.join(self.outputdir, self.timestamp)

        testdir = options.testdir

        for section in config.sections():
            if ('arch' in config.options(section) and
                platform.machine() != config.get(section, 'arch')):
                continue

            parts = section.split(':', 1)
            sectiondir = parts[0]
            identifier = parts[1] if len(parts) == 2 else None
            if os.path.isdir(sectiondir):
                pathname = sectiondir
            elif os.path.isdir(os.path.join(testdir, sectiondir)):
                pathname = os.path.join(testdir, sectiondir)
            else:
                pathname = sectiondir

            testgroup = TestGroup(os.path.abspath(pathname),
                                  identifier=identifier)
            if 'tests' in config.options(section):
                for prop in TestGroup.props:
                    for sect in ['DEFAULT', section]:
                        if config.has_option(sect, prop):
                            if prop == 'tags':
                                setattr(testgroup, prop,
                                        eval(config.get(sect, prop)))
                            else:
                                setattr(testgroup, prop,
                                        config.get(sect, prop))

                # Repopulate tests using eval to convert the string to a list
                testgroup.tests = eval(config.get(section, 'tests'))

                if testgroup.verify(logger):
                    self.testgroups[section] = testgroup

            elif 'autotests' in config.options(section):
                for prop in TestGroup.props:
                    for sect in ['DEFAULT', section]:
                        if config.has_option(sect, prop):
                            setattr(testgroup, prop, config.get(sect, prop))

                filenames = os.listdir(pathname)
                # only files starting with "tst." are considered tests
                filenames = [f for f in filenames if f.startswith("tst.")]
                testgroup.tests = sorted(filenames)

                if testgroup.verify(logger):
                    self.testgroups[section] = testgroup
            else:
                test = Test(section)
                for prop in Test.props:
                    for sect in ['DEFAULT', section]:
                        if config.has_option(sect, prop):
                            setattr(test, prop, config.get(sect, prop))

                if test.verify(logger):
                    self.tests[section] = test

    def write(self, options):
        """
        Create a configuration file for editing and later use. The
        'DEFAULT' section of the config file is created from the
        properties that were specified on the command line. Tests are
        simply added as sections that inherit everything from the
        'DEFAULT' section. TestGroups are the same, except they get an
        option including all the tests to run in that directory.
        """

        defaults = dict([(prop, getattr(options, prop)) for prop, _ in
                         self.defaults])
        config = configparser.RawConfigParser(defaults)

        for test in sorted(self.tests.keys()):
            config.add_section(test)
            for prop in Test.props:
                if prop not in self.props:
                    config.set(testgroup, prop,
                        getattr(self.testgroups[testgroup], prop))

        for testgroup in sorted(self.testgroups.keys()):
            config.add_section(testgroup)
            config.set(testgroup, 'tests', self.testgroups[testgroup].tests)
            for prop in TestGroup.props:
                if prop not in self.props:
                    config.set(testgroup, prop,
                        getattr(self.testgroups[testgroup], prop))

        try:
            with open(options.template, 'w') as f:
                return config.write(f)
        except IOError:
            fail('Could not open \'%s\' for writing.' % options.template)

    def complete_outputdirs(self):
        """
        Collect all the pathnames for Tests, and TestGroups. Work
        backwards one pathname component at a time, to create a unique
        directory name in which to deposit test output. Tests will be able
        to write output files directly in the newly modified outputdir.
        TestGroups will be able to create one subdirectory per test in the
        outputdir, and are guaranteed uniqueness because a group can only
        contain files in one directory. Pre and post tests will create a
        directory rooted at the outputdir of the Test or TestGroup in
        question for their output.
        """
        done = False
        components = 0
        tmp_dict = dict(list(self.tests.items()) + list(self.testgroups.items()))
        total = len(tmp_dict)
        base = self.outputdir

        while not done:
            l = []
            components -= 1
            for testfile in list(tmp_dict.keys()):
                uniq = '/'.join(testfile.split('/')[components:]).lstrip('/')
                if uniq not in l:
                    l.append(uniq)
                    tmp_dict[testfile].outputdir = os.path.join(base, uniq)
                else:
                    break
            done = total == len(l)

    def setup_logging(self, options):
        """
        Two loggers are set up here. The first is for the logfile which
        will contain one line summarizing the test, including the test
        name, result, and running time. This logger will also capture the
        timestamped combined stdout and stderr of each run. The second
        logger is optional console output, which will contain only the one
        line summary. The loggers are initialized at two different levels
        to facilitate segregating the output.
        """
        if options.dryrun is True:
            return

        testlogger = logging.getLogger(__name__)
        testlogger.setLevel(logging.DEBUG)

        if not options.template:
            try:
                old = os.umask(0)
                os.makedirs(self.outputdir, mode=0o777)
                os.umask(old)
            except OSError as e:
                fail('%s' % e)
            filename = os.path.join(self.outputdir, 'log')

            logfile = WatchedFileHandlerClosed(filename)
            logfile.setLevel(logging.DEBUG)
            logfilefmt = logging.Formatter('%(message)s')
            logfile.setFormatter(logfilefmt)
            testlogger.addHandler(logfile)

        cons = logging.StreamHandler()
        cons.setLevel(logging.INFO)
        consfmt = logging.Formatter('%(message)s')
        cons.setFormatter(consfmt)
        testlogger.addHandler(cons)

        return testlogger

    def run(self, options):
        """
        Walk through all the Tests and TestGroups, calling run().
        """
        if not options.dryrun:
            try:
                os.chdir(self.outputdir)
            except OSError:
                fail('Could not change to directory %s' % self.outputdir)
        for test in sorted(self.tests.keys()):
            self.tests[test].run(self.logger, options)
        for testgroup in sorted(self.testgroups.keys()):
            self.testgroups[testgroup].run(self.logger, options)

    def summary(self):
        if Result.total == 0:
            print('No tests to run')
            return

        print('\nResults Summary')
        for key in list(Result.runresults.keys()):
            if Result.runresults[key] != 0:
                print('%s\t% 4d' % (key, Result.runresults[key]))

        m, s = divmod(time() - self.starttime, 60)
        h, m = divmod(m, 60)
        print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s))
        print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) /
                                            float(Result.total)) * 100))
        print('Log directory:\t%s' % self.outputdir)


def verify_file(pathname):
    """
    Verify that the supplied pathname is an executable regular file.
    """
    if os.path.isdir(pathname) or os.path.islink(pathname):
        return False

    if os.path.isfile(pathname) and os.access(pathname, os.X_OK):
        return True

    return False


def verify_user(user, logger):
    """
    Verify that the specified user exists on this system, and can execute
    sudo without being prompted for a password.
    """
    testcmd = [SUDO, '-n', '-u', user, TRUE]

    if user in Cmd.verified_users:
        return True

    try:
        _ = getpwnam(user)
    except KeyError:
        logger.info("Warning: user '%s' does not exist.", user)
        return False

    p = Popen(testcmd)
    p.wait()
    if p.returncode != 0:
        logger.info("Warning: user '%s' cannot use passwordless sudo.", user)
        return False
    else:
        Cmd.verified_users.append(user)

    return True


def find_tests(testrun, options):
    """
    For the given list of pathnames, add files as Tests. For directories,
    if do_groups is True, add the directory as a TestGroup. If False,
    recursively search for executable files.
    """

    for p in sorted(options.pathnames):
        if os.path.isdir(p):
            for dirname, _, filenames in os.walk(p):
                if options.do_groups:
                    testrun.addtestgroup(dirname, filenames, options)
                else:
                    for f in sorted(filenames):
                        testrun.addtest(os.path.join(dirname, f), options)
        else:
            testrun.addtest(p, options)


def filter_tests(testrun, options):
    try:
        fh = open(options.logfile, "r", errors='replace')
    except Exception as e:
        fail('%s' % e)

    failed = {}
    while True:
            line = fh.readline()
            if not line:
                 break
            m = re.match(r'Test: (.*)/(\S+).*\[FAIL\]', line)
            if not m:
                continue
            group, test = m.group(1, 2)
            m = re.match(re.escape(options.testdir) + r'(.*)', group)
            if m:
                group = m.group(1)
            try:
                failed[group].append(test)
            except KeyError:
                failed[group] = [ test ]
    fh.close()

    testrun.filter(failed)


def fail(retstr, ret=1):
    print('%s: %s' % (argv[0], retstr))
    exit(ret)


def options_cb(option, opt_str, value, parser):
    path_options = ['outputdir', 'template', 'testdir', 'logfile']

    if opt_str in parser.rargs:
        fail('%s may only be specified once.' % opt_str)

    if option.dest == 'runfiles':
        parser.values.cmd = 'rdconfig'
        value = set(os.path.abspath(p) for p in value.split(','))
    if option.dest == 'tags':
        value = [x.strip() for x in value.split(',')]

    if option.dest in path_options:
        setattr(parser.values, option.dest, os.path.abspath(value))
    else:
        setattr(parser.values, option.dest, value)


def parse_args():
    parser = OptionParser()
    parser.add_option('-c', action='callback', callback=options_cb,
                      type='string', dest='runfiles', metavar='runfiles',
                      help='Specify tests to run via config files.')
    parser.add_option('-d', action='store_true', default=False, dest='dryrun',
                      help='Dry run. Print tests, but take no other action.')
    parser.add_option('-l', action='callback', callback=options_cb,
                      default=None, dest='logfile', metavar='logfile',
                      type='string',
                      help='Read logfile and re-run tests which failed.')
    parser.add_option('-g', action='store_true', default=False,
                      dest='do_groups', help='Make directories TestGroups.')
    parser.add_option('-o', action='callback', callback=options_cb,
                      default=BASEDIR, dest='outputdir', type='string',
                      metavar='outputdir', help='Specify an output directory.')
    parser.add_option('-i', action='callback', callback=options_cb,
                      default=TESTDIR, dest='testdir', type='string',
                      metavar='testdir', help='Specify a test directory.')
    parser.add_option('-p', action='callback', callback=options_cb,
                      default='', dest='pre', metavar='script',
                      type='string', help='Specify a pre script.')
    parser.add_option('-P', action='callback', callback=options_cb,
                      default='', dest='post', metavar='script',
                      type='string', help='Specify a post script.')
    parser.add_option('-q', action='store_true', default=False, dest='quiet',
                      help='Silence on the console during a test run.')
    parser.add_option('-t', action='callback', callback=options_cb, default=60,
                      dest='timeout', metavar='seconds', type='int',
                      help='Timeout (in seconds) for an individual test.')
    parser.add_option('-u', action='callback', callback=options_cb,
                      default='', dest='user', metavar='user', type='string',
                      help='Specify a different user name to run as.')
    parser.add_option('-w', action='callback', callback=options_cb,
                      default=None, dest='template', metavar='template',
                      type='string', help='Create a new config file.')
    parser.add_option('-x', action='callback', callback=options_cb, default='',
                      dest='pre_user', metavar='pre_user', type='string',
                      help='Specify a user to execute the pre script.')
    parser.add_option('-X', action='callback', callback=options_cb, default='',
                      dest='post_user', metavar='post_user', type='string',
                      help='Specify a user to execute the post script.')
    parser.add_option('-T', action='callback', callback=options_cb, default='',
                      dest='tags', metavar='tags', type='string',
                      help='Specify tags to execute specific test groups.')
    (options, pathnames) = parser.parse_args()

    if options.runfiles and len(pathnames):
        fail('Extraneous arguments.')

    options.pathnames = [os.path.abspath(path) for path in pathnames]

    return options


def main():
    options = parse_args()

    testrun = TestRun(options)

    if options.runfiles:
        testrun.read(testrun.logger, options)
    else:
        find_tests(testrun, options)

    if options.logfile:
        filter_tests(testrun, options)

    if options.template:
        testrun.write(options)
        exit(0)

    testrun.complete_outputdirs()
    testrun.run(options)
    testrun.summary()
    exit(retcode)


if __name__ == '__main__':
    main()
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source.  A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#

#
# Copyright (c) 2012 by Delphix. All rights reserved.
#

include $(SRC)/Makefile.master

MANSRCS = run.1

ROOTOPTPKG = $(ROOT)/opt/test-runner
MANDIR = $(ROOTOPTPKG)/man/man1

FILES = $(MANSRCS:%=$(MANDIR)/%)
$(FILES) : FILEMODE = 0444

all: $(MANSRCS)

install: $(FILES)

clean lint clobber:

$(FILES): $(MANDIR) $(MANSRCS)

$(MANDIR):
	$(INS.dir)

$(MANDIR)/%: %
	$(INS.file)
.\"
.\" This file and its contents are supplied under the terms of the
.\" Common Development and Distribution License ("CDDL"), version 1.0.
.\" You may only use this file in accordance with the terms of version
.\" 1.0 of the CDDL.
.\"
.\" A full copy of the text of the CDDL should have accompanied this
.\" source.  A copy of the CDDL is also available via the Internet at
.\" http://www.illumos.org/license/CDDL.
.\"
.\"
.\" Copyright (c) 2012 by Delphix. All rights reserved.
.\"
.TH run 1 "23 Sep 2012"
.SH NAME
run \- find, execute, and log the results of tests
.SH SYNOPSIS
.LP
.nf
\fBrun\fR [\fB-dgq] [\fB-o\fR \fIoutputdir\fR] [\fB-pP\fR \fIscript\fR] [\fB-t\fR \fIseconds\fR] [\fB-uxX\fR \fIusername\fR]
    \fIpathname\fR ...
.fi

.LP
.nf
\fBrun\fR \fB-w\fR \fIrunfile\fR [\fB-gq\fR] [\fB-o\fR \fIoutputdir\fR] [\fB-pP\fR \fIscript\fR] [\fB-t\fR \fIseconds\fR]
    [\fB-uxX\fR \fIusername\fR] \fIpathname\fR ...
.fi

.LP
.nf
\fBrun\fR \fB-c\fR \fIrunfile\fR [\fB-dq\fR]
.fi

.LP
.nf
\fBrun\fR [\fB-h\fR]
.fi

.SH DESCRIPTION
.sp
.LP
The \fBrun\fR command has three basic modes of operation. With neither the
\fB-c\fR nor the \fB-w\fR option, \fBrun\fR processes the arguments provided on
the command line, adding them to the list for this run. If a specified
\fIpathname\fR is an executable file, it is added as a test. If a specified
\fIpathname\fR is a directory, the behavior depends upon the \fB-g\fR option.
If \fB-g\fR is specified, the directory is treated as a test group. See the
section on "Test Groups" below. Without the \fB-g\fR option, \fBrun\fR simply
descends into the directory looking for executable files. The tests are then
executed, and the results are logged.

With the \fB-w\fR option, \fBrun\fR finds tests in the manner described above.
Rather than executing the tests and logging the results, the test configuration
is stored in a \fIrunfile\fR which can be used in future invocations, or edited
to modify which tests are executed and which options are applied. Options
included on the command line with \fB-w\fR become defaults in the
\fIrunfile\fR.

With the \fB-c\fR option, \fBrun\fR parses a \fIrunfile\fR, which can specify a
series of tests and test groups to be executed. The tests are then executed,
and the results are logged.
.sp
.SS "Test Groups"
.sp
.LP
A test group is comprised of a set of executable files, all of which exist in
one directory. The options specified on the command line or in a \fIrunfile\fR
apply to individual tests in the group. The exception is options pertaining to
pre and post scripts, which act on all tests as a group. Rather than running
before and after each test, these scripts are run only once each at the start
and end of the test group.
.SS "Test Execution"
.sp
.LP
The specified tests run serially, and are typically assigned results according
to exit values. Tests that exit zero and non-zero are marked "PASS" and "FAIL"
respectively. When a pre script fails for a test group, only the post script is
executed, and the remaining tests are marked "SKIPPED." Any test that exceeds
its \fItimeout\fR is terminated, and marked "KILLED."

By default, tests are executed with the credentials of the \fBrun\fR script.
Executing tests with other credentials is done via \fBsudo\fR(8), which must
be configured to allow execution without prompting for a password. Environment
variables from the calling shell are available to individual tests. During test
execution, the working directory is changed to \fIoutputdir\fR.
.SS "Output Logging"
.sp
.LP
By default, \fBrun\fR will print one line on standard output at the conclusion
of each test indicating the test name, result and elapsed time. Additionally,
for each invocation of \fBrun\fR, a directory is created using the ISO 8601
date format. Within this directory is a file named \fIlog\fR containing all the
test output with timestamps, and a directory for each test. Within the test
directories, there is one file each for standard output, standard error and
merged output. The default location for the \fIoutputdir\fR is
\fI/var/tmp/test_results\fR.
.SS "Runfiles"
.sp
.LP
The \fIrunfile\fR is an ini style configuration file that describes a test run.
The file has one section named "DEFAULT," which contains configuration option
names and their values in "name = value" format. The values in this section
apply to all the subsequent sections, unless they are also specified there, in
which case the default is overridden. The remaining section names are the
absolute pathnames of files and direcotries, describing tests and test groups
respectively. The legal option names are:
.sp
.ne 2
.na
\fBoutputdir\fR = \fIpathname\fR
.ad
.sp .6
.RS 4n
The name of the directory that holds test logs.
.RE
.sp
.ne 2
.na
\fBpre\fR = \fIscript\fR
.ad
.sp .6
.RS 4n
Run \fIscript\fR prior to the test or test group.
.RE
.sp
.ne 2
.na
\fBpre_user\fR = \fIusername\fR
.ad
.sp .6
.RS 4n
Execute the pre script as \fIusername\fR.
.RE
.sp
.ne 2
.na
\fBpost\fR = \fIscript\fR
.ad
.sp .6
.RS 4n
Run \fIscript\fR after the test or test group.
.RE
.sp
.ne 2
.na
\fBpost_user\fR = \fIusername\fR
.ad
.sp .6
.RS 4n
Execute the post script as \fIusername\fR.
.RE
.sp
.ne 2
.na
\fBquiet\fR = [\fITrue\fR|\fIFalse\fR]
.ad
.sp .6
.RS 4n
If set to True, only the results summary is printed to standard out.
.RE
.sp
.ne 2
.na
\fBtests\fR = [\fI'filename'\fR [,...]]
.ad
.sp .6
.RS 4n
Specify a list of \fIfilenames\fR for this test group. Only the basename of the
absolute path is required. This option is only valid for test groups, and each
\fIfilename\fR must be single quoted.
.RE
.sp
.ne 2
.na
\fBtimeout\fR = \fIn\fR
.ad
.sp .6
.RS 4n
A timeout value of \fIn\fR seconds.
.RE
.sp
.ne 2
.na
\fBuser\fR = \fIusername\fR
.ad
.sp .6
.RS 4n
Execute the test or test group as \fIusername\fR.
.RE

.SH OPTIONS
.sp
.LP
The following options are available for the \fBrun\fR command.
.sp
.ne 2
.na
\fB-c\fR \fIrunfile\fR
.ad
.RS 6n
Specify a \fIrunfile\fR to be consumed by the run command.
.RE

.ne 2
.na
\fB-d\fR
.ad
.RS 6n
Dry run mode. Execute no tests, but print a description of each test that would
have been run.
.RE

.ne 2
.na
\fB-g\fR
.ad
.RS 6n
Create test groups from any directories found while searching for tests.
.RE

.ne 2
.na
\fB-o\fR \fIoutputdir\fR
.ad
.RS 6n
Specify the directory in which to write test results.
.RE

.ne 2
.na
\fB-p\fR \fIscript\fR
.ad
.RS 6n
Run \fIscript\fR prior to any test or test group.
.RE

.ne 2
.na
\fB-P\fR \fIscript\fR
.ad
.RS 6n
Run \fIscript\fR after any test or test group.
.RE

.ne 2
.na
\fB-q\fR
.ad
.RS 6n
Print only the results sumary to the standard output.
.RE

.ne 2
.na
\fB-t\fR \fIn\fR
.ad
.RS 6n
Specify a timeout value of \fIn\fR seconds per test.
.RE

.ne 2
.na
\fB-u\fR \fIusername\fR
.ad
.RS 6n
Execute tests or test groups as \fIusername\fR.
.RE

.ne 2
.na
\fB-w\fR \fIrunfile\fR
.ad
.RS 6n
Specify the name of the \fIrunfile\fR to create.
.RE

.ne 2
.na
\fB-x\fR \fIusername\fR
.ad
.RS 6n
Execute the pre script as \fIusername\fR.
.RE

.ne 2
.na
\fB-X\fR \fIusername\fR
.ad
.RS 6n
Execute the post script as \fIusername\fR.
.RE

.SH EXAMPLES
.LP
\fBExample 1\fR Running ad-hoc tests.
.sp
.LP
This example demonstrates the simplest invocation of \fBrun\fR.

.sp
.in +2
.nf
% \fBrun my-tests\fR
Test: /home/jkennedy/my-tests/test-01                    [00:02] [PASS]
Test: /home/jkennedy/my-tests/test-02                    [00:04] [PASS]
Test: /home/jkennedy/my-tests/test-03                    [00:01] [PASS]

Results Summary
PASS       3

Running Time:   00:00:07
Percent passed: 100.0%
Log directory:  /var/tmp/test_results/20120923T180654
.fi
.in -2

.LP
\fBExample 2\fR Creating a \fIrunfile\fR for future use.
.sp
.LP
This example demonstrates creating a \fIrunfile\fR with non default options.

.sp
.in +2
.nf
% \fBrun -p setup -x root -g -w new-tests.run new-tests\fR
% \fBcat new-tests.run\fR
[DEFAULT]
pre = setup
post_user =
quiet = False
user =
timeout = 60
post =
pre_user = root
outputdir = /var/tmp/test_results

[/home/jkennedy/new-tests]
tests = ['test-01', 'test-02', 'test-03']
.fi
.in -2

.SH EXIT STATUS
.sp
.LP
The following exit values are returned:
.sp
.ne 2
.na
\fB\fB0\fR\fR
.ad
.sp .6
.RS 4n
Successful completion.
.RE
.sp
.ne 2
.na
\fB\fB1\fR\fR
.ad
.sp .6
.RS 4n
An error occurred.
.RE

.SH SEE ALSO
.sp
.LP
\fBsudo\fR(8)
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source.  A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#

#
# Copyright (c) 2012 by Delphix. All rights reserved.
#

.PARALLEL: $(SUBDIRS)

SUBDIRS = contrib/include include

include $(SRC)/test/Makefile.com
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source.  A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#

#
# Copyright (c) 2012 by Delphix. All rights reserved.
#

This directory contains shell libraries used by the Solaris Test
Framework (STF). They are duplicated here to allow test suites (like
the ZFS Test Suite) that were written for STF to be easily ported
to this framework.
                              The "Artistic License"

                                     Preamble

       The intent of this document is to state the conditions under which a
       Package may be copied, such that the Copyright Holder maintains some
       semblance of artistic control over the development of the package,
       while giving the users of the package the right to use and distribute
       the Package in a more-or-less customary fashion, plus the right to
       make reasonable modifications.

       Definitions:

          * "Package" refers to the collection of files distributed by the
            Copyright Holder, and derivatives of that collection of files
            created through textual modification.

          * "Standard Version" refers to such a Package if it has not been
            modified, or has been modified in accordance with the wishes of
            the Copyright Holder.

          * "Copyright Holder" is whoever is named in the copyright or
            copyrights for the package.

          * "You" is you, if you're thinking about copying or distributing
            this Package.

          * "Reasonable copying fee" is whatever you can justify on the
            basis of media cost, duplication charges, time of people
            involved, and so on. (You will not be required to justify it to
            the Copyright Holder, but only to the computing community at
            large as a market that must bear the fee.)

          * "Freely Available" means that no fee is charged for the item
            itself, though there may be fees involved in handling the item.
            It also means that recipients of the item may redistribute it
            under the same conditions they received it.

       1. You may make and give away verbatim copies of the source form of
       the Standard Version of this Package without restriction, provided
       that you duplicate all of the original copyright notices and
       associated disclaimers.

       2. You may apply bug fixes, portability fixes and other modifications
       derived from the Public Domain or from the Copyright Holder. A
       Package modified in such a way shall still be considered the Standard
       Version.

       3. You may otherwise modify your copy of this Package in any way,
       provided that you insert a prominent notice in each changed file
       stating how and when you changed that file, and provided that you do
       at least ONE of the following:

            a) place your modifications in the Public Domain or
            otherwise make them Freely Available, such as by posting
            said modifications to Usenet or an equivalent medium, or
            placing the modifications on a major archive site such as
            ftp.uu.net, or by allowing the Copyright Holder to include
            your modifications in the Standard Version of the Package.

            b) use the modified Package only within your corporation or
            organization.

            c) rename any non-standard executables so the names do not
            conflict with standard executables, which must also be
            provided, and provide a separate manual page for each
            non-standard executable that clearly documents how it
            differs from the Standard Version.

            d) make other distribution arrangements with the Copyright
            Holder.

       4. You may distribute the programs of this Package in object code or
       executable form, provided that you do at least ONE of the following:

            a) distribute a Standard Version of the executables and
            library files, together with instructions (in the manual
            page or equivalent) on where to get the Standard Version.

            b) accompany the distribution with the machine-readable
            source of the Package with your modifications.

            c) accompany any non-standard executables with their
            corresponding Standard Version executables, giving the
            non-standard executables non-standard names, and clearly
            documenting the differences in manual pages (or
            equivalent), together with instructions on where to get the
            Standard Version.

            d) make other distribution arrangements with the Copyright
            Holder.

       5. You may charge a reasonable copying fee for any distribution of
       this Package. You may charge any fee you choose for support of this
       Package. You may not charge a fee for this Package itself. However,
       you may distribute this Package in aggregate with other (possibly
       commercial) programs as part of a larger (possibly commercial)
       software distribution provided that you do not advertise this Package
       as a product of your own.

       6. The scripts and library files supplied as input to or produced as
       output from the programs of this Package do not automatically fall
       under the copyright of this Package, but belong to whomever generated
       them, and may be sold commercially, and may be aggregated with this
       Package.

       7. C or perl subroutines supplied by you and linked into this Package
       shall not be considered part of this Package.

       8. The name of the Copyright Holder may not be used to endorse or
       promote products derived from this software without specific prior
       written permission.

       9. THIS PACKAGE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
       IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
       WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.

                                      The End

CTIUTILS.SHLIB IN TEST-RUNNER
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source.  A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#

#
# Copyright (c) 2012 by Delphix. All rights reserved.
#

include $(SRC)/Makefile.master
include $(SRC)/test/Makefile.com

ROOTOPTPKG = $(ROOT)/opt/test-runner
INCLUDEDIR = $(ROOTOPTPKG)/stf/contrib/include

PROGS = logapi.shlib ctiutils.shlib

CMDS = $(PROGS:%=$(INCLUDEDIR)/%)
$(CMDS) : FILEMODE = 0555

all lint clean clobber:

install: $(CMDS)

$(CMDS): $(INCLUDEDIR)

$(INCLUDEDIR):
	$(INS.dir)

$(INCLUDEDIR)/%: %
	$(INS.file)
#
# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#

#
# This is free software; you can redistribute it and/or modify it
# under the terms of the "Artistic License" which is located in the
# file named "LICENSE.Artistic" which is included with this software.
#

#
# This file was originally part of the "contrib" portion of the
# TET3 test harness from The OpenGroup, originally found here:
# http://tetworks.opengroup.org/tet/ or ftp://ftp.xopen.org/pub/TET3/
# and later published in hg.opensolaris.org/hg/test-dev/stcnv-gate
#
# A build of that repository delivers this file as part of the
# SUNWstc-tetlite package, which carries the "Artistic" license
# referred to above.
#

#
# The "Artistic License" places some restrictions on the kinds of
# changes that may be made to this file.  The changes made here
# relative to the original are "portability fixes" to allow these
# functions to operate in the "test-runner" environment.
#
# The original from which this was derived can be found here:
#   https://bitbucket.org/illumos/illumos-stc/
# under the path:
#   usr/src/tools/tet/contrib/ctitools/src/lib/ksh/ctiutils.ksh

########################################################################
#
# NAME:		ctiutils.shlib
#
# SYNOPSIS:
#	cti_assert assertion_number assertion_msg
#	cti_report arg ...
#	cti_pass
#	cti_fail [msg ...]
#	cti_unresolved [msg ...]
#	cti_untested [msg ...]
#	cti_unsupported [msg ...]
#	cti_notinuse [msg ...]
#	cti_pathtrace
#	cti_checkpath expected-path-count
#	cti_deleteall reason
#	cti_reportfile path [title]
#	cti_rmf [files...]
#	cti_writefile path mode lines...
#	cti_appendfile path mode lines...
#	cti_execute [-c out|err] [-i input] result cmd [args...]
#	cti_runexpect failtext command pattern action [pattern action ...]
#	cti_expecttest failtext command pattern action [pattern action ...]
#	cti_cancel test_num [msg ...] [test result]
#	cti_cancelall [msg ...] [test result]
#
# DESCRIPTION:
#	Common korn shell functions for tests.
#
#	cti_report() writes an informational line to the journal.
#
#	The functions cti_pass(), cti_fail(), cti_unresolved(),
#	cti_untested(), cti_unsupported() and cti_notinuse() each
#	registers the corresponding result code against the current test,
#	and write any arguments to the execution results file, as a
#	single line.
#
#	The cti_pathtrace() and cti_checkpath() are used in path tracing.
#	cti_pathtrace() increments the variable pathok. cti_checkpath()
#	checks the path tracing and registers a PASS result if
#	appropriate.
#
#	cti_deleteall() cancels all tests in the current test case.
#
#	cti_reportfile() writes the contents of a file to the journal.
#
#	cti_rmf() removes files.
#
#	cti_writefile() writes to the contents of a file;
#	cti_appendfile() appends to contents of a file.
#
#	cti_execute() executes a command.
#
#	cti_runexpect() runs the expect utility. cti_expecttest() is
#	like cti_runexpect() except that it is written with path tracing
#	and is designed to do the complete work of a test purpose.
#       cti_runexpect() runs the expect utility. cti_expecttest() is
#       like cti_runexpect() except that it is written with path tracing
#       and is designed to do the complete work of a test purpose.
#
#       cti_cancel() cancels the dedicated test purpose in the current test
#       case from execution with the test result that user gives. It will work
#       in startup function.
#
#       cti_cancelall() cancels all tests in the current test case. It could
#       be used in startup function to cancel the execution of test cases
#       with the test result that user gives.
#
########################################################################

#
# cti_lf_checkargs() - check number of arguments passed to a shell function.
#
# Usage: cti_lf_checkargs argc expected-argc operator funcname
#
# operator can be EQ or GE.
#
# Returns 0 if the expected number of arguments were passed, non-zero
# otherwise.
#
function cti_lf_checkargs
{
	typeset -i cti_lv_argc=$1
	typeset -i cti_lv_expargc=$2
	typeset cti_lv_op="$3"
	typeset cti_lv_funcname="$4"

	case "$cti_lv_op" in
	EQ)
		if test $cti_lv_argc -ne $cti_lv_expargc
		then
			cti_unresolved "Test coding error:" \
				"$cti_lv_funcname() called with $cti_lv_argc" \
				"args, need $cti_lv_expargc"
			return 1
		fi
		;;
	GE)
		if test $cti_lv_argc -lt $cti_lv_expargc
		then
			cti_unresolved "Test coding error:" \
				"$cti_lv_funcname() called with $cti_lv_argc" \
				"args, need >= $cti_lv_expargc"
			return 1
		fi
		;;
	*)
		cti_unresolved "Internal error: cti_lf_checkargs()" \
			"called for $funcname() with operator $cti_lv_op"
		return 1
		;;
	esac

	return 0
}

#
# cti_result() - register a result and write informational line to journal.
#
# Usage: cti_result result [msg ...]
#
# If the current test function is not a startup or cleanup, this routine
# registers the specified result code for the current test. The remaining
# arguments, if any, are written to the execution results file as a single
# line.
#
# Modifications for test-runner:
# Print the result and test name (in place of tet_result)
# On failure, return non-zero to tell test-runner our status.
#
function cti_result
{
	typeset res
	typeset -i rv=0

	test $# -eq 0 && return
	res=$1
	shift

	my_host=`hostname`
	my_timestamp=`date | awk '{print $4}'`

	test $# -gt 0 && print "$my_host $my_timestamp $@"

	# Print the result and test name (as tet_result would)
	print "$res: ${tc_id:-$(basename $0)}"

	# Return non-zero for failures.  See codes in:
	# test-runner/stf/include/stf.shlib
	case "$res" in
	PASS)
		;;
	FAIL)
		rv=1
		;;
	UNRESOLVED)
		rv=2
		;;
	NOTINUSE)
		rv=3
		;;
	UNSUPPORTED)
		rv=4
		;;
	UNTESTED)
		rv=5
		;;
	UNINITIATED)
		rv=6
		;;
	NORESULT)
		rv=7
		;;
	WARNING)
		rv=8
		;;
	TIMED_OUT)
		rv=9
		;;
	*)
		echo "cti_result: $res: unknown result code"
		rv=10
		;;
	esac
	return $rv
}

#
# cti_report() - write an informational line to the journal
#
# Usage: cti_report arg ...
#
# Writes the arguments to the execution results file, as a single line.
#
function cti_report
{
	my_host=`hostname`
	my_timestamp=`date | awk '{print $4}'`

	print "$my_host $my_timestamp $@"
}

#
# cti_assert() - write an Assert line to the journal
#
# Usage: cti_assert assertion_number assertion_msg
#
# Writes the arguments to the execution results file, as a single line.
#
function cti_assert
{
	cti_lf_checkargs $# 2 GE cti_assert || return 1

	cti_report "ASSERT $1: $2"
}

#
# cti_pass() - register a PASS result.
#
# Usage: cti_pass [msg ...]
#
function cti_pass
{
	cti_result PASS "$@"
}

#
# cti_fail() - register a FAIL result.
#
# Usage: cti_fail [msg ...]
#
# Registers a FAIL result for the current test, and writes any arguments to
# the execution results file, as a single line.
#
function cti_fail
{
	cti_result FAIL "$@"
}

#
# cti_unresolved() - register an UNRESOLVED result.
#
# Usage: cti_unresolved [msg ...]
#
# Registers an UNRESOLVED result for the current test, and writes any arguments
# to the execution results file, as a single line.
#
function cti_unresolved
{
	cti_result UNRESOLVED "$@"
}


#
# cti_uninitiated() - register an UNINITIATED result.
#
# Usage: cti_uninitiated [msg ...]
#
# Registers an UNINITIATED result for the current test, and writes any arguments
# to the execution results file, as a single line.
#
function cti_uninitiated
{
	cti_result UNINITIATED "$@"
}

#
# cti_untested() - register an UNTESTED result.
#
# Usage: cti_untested [msg ...]
#
# Registers an UNTESTED result for the current test, and writes any arguments
# to the execution results file, as a single line.
#
function cti_untested
{
	cti_result UNTESTED "$@"
}

#
# cti_unsupported() - register an UNSUPPORTED result.
#
# Usage: cti_unsupported [msg ...]
#
# Registers an UNSUPPORTED result for the current test, and writes any
# arguments to the execution results file, as a single line.
#
function cti_unsupported
{
	cti_result UNSUPPORTED "$@"
}

#
# cti_notinuse() - register a NOTINUSE result.
#
# Usage: cti_notinuse [msg ...]
#
# Registers a NOTINUSE result for the current test, and writes any arguments
# to the execution results file, as a single line.
#
function cti_notinuse
{
	cti_result NOTINUSE "$@"
}

#
# cti_pathtrace() - increment path counter.
#
# Usage: cti_pathtrace
#
# Increments variable pathok. Like C macro PATH_TRACE.
#
function cti_pathtrace
{
	: $((pathok += 1))
}

#
# cti_checkpath() - check path tracing and register a PASS result.
#
# Usage: cti_checkpath expected-path-count
#
# Like C macro PATH_XS_RPT().
#
function cti_checkpath
{
	cti_lf_checkargs $# 1 EQ cti_checkpath || return

	if test $pathok -eq $1
	then
		cti_pass
	else
		cti_unresolved "Path tracing error: path counter $pathok," \
			"expecting $1"
	fi
}

#
# cti_deleteall() - cancel all tests.
#
# Usage: cti_deleteall reason
#
# Cancels all tests
#
function cti_deleteall
{
	typeset cti_lv_ic
	typeset cti_lv_tp

	test $# -eq 0 && return

	for cti_lv_ic in $iclist
	do
		for cti_lv_tp in `eval echo \\$$cti_lv_ic`
		do
			if test -n "$cti_lv_tp"
			then
				echo "Deleted test: $cti_lv_tp" "$@"
			fi
		done
	done
}

#
# cti_reportfile() - write the contents of a file to the journal.
#
# Usage: cti_reportfile path [title]
#
# Writes the contents of the file specified by path to the execution results
# file, line by line.
#
function cti_reportfile
{
	typeset cti_lv_path=$1
	typeset cti_lv_title="${2:-$cti_lv_path}"
	typeset cti_lv_line

	cti_lf_checkargs $# 1 GE cti_reportfile || return

	cti_report "+++ $cti_lv_title +++"

	/usr/bin/cat $cti_lv_path

	cti_report "+++ end +++"
	cti_report " "
}

#
# cti_rmf() - remove files.
#
# Usage: cti_rmf [files...]
#
# Calls "rm -f" to remove the files, and verifies that they have been removed.
#
# Returns 0 on success, non-zero if any of the files could not be removed.
#
function cti_rmf
{
	typeset cti_lv_file

	for cti_lv_file in "$@"
	do
		rm -f "$cti_lv_file"

		if test -f "$cti_lv_file"
		then
			cti_unresolved "Error removing file \"$cti_lv_file\""
			return 1
		fi
	done

	return 0
}

#
# cti_writefile() - write contents of a file.
#
# Usage: cti_writefile path mode lines...
#
# Truncates the file specified by path and then writes the third and
# subsequent arguments to the specified file as separate lines.
#
# Returns 0 on success, non-zero if any of the files could not be removed.
#
function cti_writefile
{
	cti_lf_checkargs $# 3 GE cti_writefile || return 1

	cti_rmf "$1" || return 1
	cti_appendfile "$@"
}

#
# cti_appendfile() - append to contents of a file.
#
# Usage: cti_appendfile path mode lines...
#
# Appends the third and subsequent arguments to the specified file as separate
# lines.
#
# Returns 0 on success, non-zero if any of the files could not be removed.
#
function cti_appendfile
{
	typeset cti_lv_path="$1"
	typeset cti_lv_mode="$2"
	typeset cti_lv_line

	cti_lf_checkargs $# 3 GE cti_appendfile || return 1
	shift 2

	for cti_lv_line in "$@"
	do
		echo "$cti_lv_line" >> "$cti_lv_path"

		if [[ $? -ne 0 ]]
		then
			cti_unresolved \
				"Error writing to file \"$cti_lv_path\""
			return 1
		fi
	done

	cti_execute UNRESOLVED chmod "$cti_lv_mode" "$cti_lv_path"
	return $?
}

#
# cti_execute() - execute a command
#
# Usage: cti_execute [-c out|err] [-i input] result cmd [args...]
#
# Executes a command. The standard output is redirected to the file cti_stdout
# and the standard error is redirected to the file cti_stderr.
#
# If the command has a non-zero exit status, cti_execute() registers a result
# code of `result'.
#
# Options:
#	-c out|err	Check standard output/error. If anything is written to
#			the specified output channel, a result code of `result'
#			is registered and the output written to the journal.
#			May have multiple -c options.
#	-i input	Use -i as an input line to the command.
#			May have multiple -i options.
#
# Returns 0 on success, non-zero on failure (returning the
#  exit status from the command when possible).
#
function cti_execute
{
	typeset cti_lv_opt
	typeset -i cti_lv_checkstdout=0
	typeset -i cti_lv_checkstderr=0
	typeset cti_lv_result
	typeset -i cti_lv_status
	typeset -i cti_lv_rv=0

	# Remove files used for redirecting standard I/O.
	cti_rmf cti_stdin cti_stdout cti_stderr || return 1

	# Create (empty) files to take standard output and error so there are
	# no problems later when we come to run the command.
	touch cti_stdout cti_stderr

	if [[ $? -ne 0 ]]
	then
		cti_unresolved "Error creating files cti_stdout and cti_stderr"
		return 1
	fi

	# Parse command line options
	while getopts "c:i:l:s:" cti_lv_opt
	do
		case $cti_lv_opt in
		c)
			case "$OPTARG" in
			out|err)
				eval cti_lv_checkstd$OPTARG=1
				;;
			*)
				cti_unresolved "cti_execute() called with" \
					"bad option argument -c $OPTARG"
				return 1
				;;
			esac
			;;
		i)
			echo "$OPTARG" >> cti_stdin
			if [[ $? -ne 0 ]]
			then
				cti_unresolved "Error writing to cti_stdin"
				return 1
			fi
			;;
		*)
			cti_unresolved "cti_execute() called with illegal" \
				"option $cti_lv_opt"
			return 1
			;;
		esac
	done

	shift $((OPTIND-1))

	# Verify the correct number of arguments were passed.
	cti_lf_checkargs $# 2 GE cti_execute || return 1

	# First (non-option) argument is the result code to use if the command
	# fails.
	cti_lv_result="${1:-UNRESOLVED}"
	shift

	# Execute the command, redirecting standard input if required.
	if test -f cti_stdin
	then
		eval "$@" < cti_stdin > cti_stdout 2> cti_stderr
	else
		eval "$@" > cti_stdout 2> cti_stderr
	fi

	cti_lv_status=$?

	# Check the exit status of the command
	if test $cti_lv_status -ne 0
	then
		if [[ "$cti_lv_result" = "CTI" ]]
		then
			cti_report CTI "Command \"$*\" failed "\
				"with status $cti_lv_status"
		else
			cti_result "$cti_lv_result"\
				"Command \"$*\" failed "\
				"with status $cti_lv_status"
			cti_lv_rv=$cti_lv_status
		fi
	fi

	# Always log output of cti_execute_cmd
	if [[ "$cti_lv_result" = "CTI" ]]
	then

		if test -s cti_stdout
		then
			cti_reportfile cti_stdout "Standard output from command \"$*\""
		fi
		if test -s cti_stderr
		then
			cti_reportfile cti_stderr "Standard error from command \"$*\""
		fi
		return $cti_lv_status
	fi

	# If cmd failed, or if "-c err", check standard error.
	if test \( $cti_lv_rv -ne 0 -o $cti_lv_checkstderr -eq 1 \) \
		-a -s cti_stderr
	then
		cti_result "$cti_lv_result" \
			"Command \"$*\" produced standard error"
		cti_reportfile cti_stderr "Standard error from command \"$*\""
		[[ $cti_lv_rv = 0 ]] && cti_lv_rv=1
	fi

	#  If cmd failed, or if "-c out", check standard output.
	if test \( $cti_lv_rv -ne 0 -o $cti_lv_checkstdout -eq 1 \) \
		-a -s cti_stdout
	then
		cti_result "$cti_lv_result" \
			"Command \"$*\" produced standard output"
		cti_reportfile cti_stdout "Standard output from command \"$*\""
		[[ $cti_lv_rv = 0 ]] && cti_lv_rv=1
	fi

	return $cti_lv_rv
}

#
# Exit values for expect scripts.
# N.B. Do not use 0, as expect uses this for e.g. syntax errors.
#
typeset -ri CTI_EXP_RV_TIMEDOUT=1
typeset -ri CTI_EXP_RV_TESTFAIL=2
typeset -ri CTI_EXP_RV_OK=3

#
# cti_runexpect() - run the expect utility.
#
# Usage: cti_runexpect failtext command pattern action [pattern action ...]
#
# Executes the expect utility using the command line specified in the second
# argument. Generates a temporary expect script which is removed at the end of
# the call. The arguments following the second are pattern-action pairs. The
# pattern can be a regular expression or "CALL", which indicates the action is
# simply a function call which is unconditionally executed at that point.
#
# The following expect functions are available:
#
#	startcritical	Indicates that failures from this point onwards
#			constitute a test failure. In that case the
#			``failtext'' argument is used to report the error
#			message.
#	endcritical	Indicates the end of the test failure section begun by
#			startcritical.
#	finish		Exit the expect script here - the test has passed.
#
# Returns 0 on success, non-zero on failure.
#
function cti_runexpect
{
	typeset -ri CTI_EXP_TIMEOUT=5
	typeset -r cti_lv_expfile="./cti_$tc_id-$$.exp"
	typeset cti_lv_failtext="$1"
	typeset cti_lv_command="$2"
	typeset -i cti_lv_rv=0
	typeset cti_lv_dopt

	# Verify the correct number of arguments were passed.
	cti_lf_checkargs $# 4 GE cti_runexpect || return 1

	shift 2

	# Generate expect script.
	{
	echo "set STATUS_OK       $CTI_EXP_RV_OK"
	echo "set STATUS_FAIL     $CTI_EXP_RV_TESTFAIL"
	echo "set STATUS_TIMEDOUT $CTI_EXP_RV_TIMEDOUT"
	echo ''
        echo "set timeout $CTI_EXP_TIMEOUT"
        echo 'set retval $STATUS_TIMEDOUT'
	echo ''
	echo 'eval spawn [lrange $argv 0 end]'
	echo ''
	echo 'proc startcritical {} {'
	echo '	global retval'
	echo '	global STATUS_FAIL'
	echo '	set retval $STATUS_FAIL'
	echo '}'
	echo ''
	echo 'proc endcritical {} {'
	echo '	global retval'
	echo '	global STATUS_TIMEDOUT'
	echo '	set retval $STATUS_TIMEDOUT'
	echo '}'
	echo ''
	echo 'proc finish {} {'
	echo '	global STATUS_OK'
	echo '	exit $STATUS_OK'
	echo '}'

	while test $# -gt 1
	do
		echo ''

		if test "$1" = "CALL"
		then
			echo "$2"
		else
			echo 'expect {'
			echo "	-re \"$1\" {"
			echo "		$2 "
			echo '	}'
			echo '	timeout {'
			echo '		exit $retval'
			echo '	}'
			echo '}'
		fi

		shift 2
	done
	} > $cti_lv_expfile

	# Check that there were no errors writing to the script file.
	if test $? -ne 0
	then
		cti_unresolved "Error writing expect script to file" \
			"\"$cti_lv_expfile\""
		return 1
	fi

	# If debug is on, turn on expect debug flag.
	case "$CTI_SHELL_DEBUG" in
	y*|Y*)
		cti_lv_dopt='-d'
		;;
	esac

	# Execute expect using generated script.
	expect $cti_lv_dopt -f $cti_lv_expfile $cti_lv_command > cti_expout 2>&1
	cti_lv_rv=$?

	# If debug is on, save expect script and output, otherwise remove
	# script.
	case "$CTI_SHELL_DEBUG" in
	y*|Y*)
		cp cti_expout ${cti_lv_expfile%.exp}.out
		cti_report "DEBUG: expect script is $PWD/$cti_lv_expfile," \
			"expect output is in $PWD/${cti_lv_expfile%.exp}.out"
		;;
	*)
		rm -f $cti_lv_expfile
	esac

	# Deal with return value from expect.
	case $cti_lv_rv in
	$CTI_EXP_RV_OK)
		return 0
		;;
	$CTI_EXP_RV_TIMEDOUT)
		cti_unresolved "Expect script timed-out during test setup"
		;;
	$CTI_EXP_RV_TESTFAIL)
		cti_fail "$cti_lv_failtext"
		;;
	*)
		cti_unresolved "Test coding error or expect bug:" \
			"unexpected exit status $cti_lv_rv from expect script"
		cti_reportfile cti_expout "Output from expect"
		;;
	esac

	return 1
}

#
# cti_expecttest() - run the expect utility.
#
# Usage: cti_expecttest failtext command pattern action [pattern action ...]
#
# Common test function for test purposes which use expect. Like cti_runexpect()
# except that this is written with path tracing and is designed to do the
# complete work of a test purpose.
#
function cti_expecttest
{
	# Verify the correct number of arguments were passed.
	cti_lf_checkargs $# 4 GE cti_expecttest || return

	# Use cti_runexpect() to execute expect utililty.
	if cti_runexpect "$@"
	then
		cti_pathtrace
	else
		return
	fi

	cti_checkpath 1
}

function cti_execute_cmd
{
	cti_execute CTI "$@"
}

#
# "private" functions for internal use by cti_cancel function
# used to replace test purpose functions which will be canceled.
#
function cancel_ic {
	cti_result $cticancel_result "$cticancel_msg"
}

#
# cti_cancel() - cancel an individual test purpose.
#
# Usage: cti_cancel tp reason [test result]
#
# Cancels an individual test by replace the tp function with
# cancel_ic function
#
function cti_cancel {
	test $# -gt 3 && return
	test_num=$1
	cticancel_msg="$2"
	cticancel_result=${3:-"UNSUPPORTED"}
	typeset cti_lv_ic cti_lv_ic_mod
	typeset cti_lv_tp

	cti_report "Canceling test $test_num: $cticancel_msg"
	#
	# translate the ic and point the test purpose ic to
	# cancel_ic function
	#
	for cti_lv_ic in $iclist
	do
		cti_lv_ic_mod=""
		for cti_lv_tp in `eval echo \\$$cti_lv_ic`
		do
			if [ X"$cti_lv_tp" == X"$test_num" ]; then
				cti_lv_ic_mod=$cti_lv_ic_mod" cancel_ic"
			else
				cti_lv_ic_mod=$cti_lv_ic_mod" $cti_lv_tp"
			fi
		done
		eval "$cti_lv_ic=\"$cti_lv_ic_mod\""
	done
}

#
# cti_cancelall() - cancel all tests.
#
# Usage: cti_cancelall reason [test result]
#
# Cancels all tests by replace the tests functions with cancel_ic function
#
function cti_cancelall {
	typeset cti_lv_ic
	typeset cti_lv_tp
	cticancel_msg=$1
	cticancel_result=${2:-"UNSUPPORTED"}

	test $# -eq 0 && return

	for cti_lv_ic in $iclist
	do
		for cti_lv_tp in `eval echo \\$$cti_lv_ic`
		do
			cti_cancel $cti_lv_tp "$cticancel_msg" \
			    $cticancel_result
		done
	done
}
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#

#
# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
#

. ${STF_TOOLS}/include/stf.shlib

# Output an assertion
#
# $@ - assertion text

function log_assert
{
	_printline ASSERTION: "$@"
}

# Output a comment
#
# $@ - comment text

function log_note
{
	_printline NOTE: "$@"
}

# Execute and print command with status where success equals non-zero result
#
# $@ - command to execute
#
# return 0 if command fails, otherwise return 1

function log_neg
{
	log_neg_expect ""  "$@"
	return $?
}

# Execute a positive test and exit $STF_FAIL is test fails
#
# $@ - command to execute

function log_must
{
	log_pos "$@"
	(( $? != 0 )) && log_fail
}

# Execute a negative test and exit $STF_FAIL if test passes
#
# $@ - command to execute

function log_mustnot
{
	log_neg "$@"
	(( $? != 0 )) && log_fail
}

# Execute a positive test but retry the command on failure if the output
# matches an expected pattern.  Otherwise behave like log_must and exit
# $STF_FAIL is test fails.
#
# $1 - retry keyword
# $2 - retry attempts
# $3-$@ - command to execute
#
function log_must_retry
{
	typeset out=""
	typeset logfile="/tmp/log.$$"
	typeset status=1
	typeset expect=$1
	typeset retry=$2
	typeset delay=1
	shift 2

	while [[ -e $logfile ]]; do
		logfile="$logfile.$$"
	done

	while (( $retry > 0 )); do
		"$@" 2>$logfile
		status=$?
		out="cat $logfile"

		if (( $status == 0 )); then
			$out | egrep -i "internal error|assertion failed" \
				> /dev/null 2>&1
			# internal error or assertion failed
			if [[ $? -eq 0 ]]; then
				print -u2 $($out)
				_printerror "$@" "internal error or" \
					" assertion failure exited $status"
				status=1
			else
				[[ -n $LOGAPI_DEBUG ]] && print $($out)
				_printsuccess "$@"
			fi
			break
		else
			$out | grep -i "$expect" > /dev/null 2>&1
			if (( $? == 0 )); then
				print -u2 $($out)
				_printerror "$@" "Retry in $delay seconds"
				sleep $delay

				(( retry=retry - 1 ))
				(( delay=delay * 2 ))
			else
				break;
			fi
		fi
	done

	if (( $status != 0 )) ; then
		print -u2 $($out)
		_printerror "$@" "exited $status"
	fi

	_recursive_output $logfile "false"
	return $status
}

# Execute a positive test and exit $STF_FAIL is test fails after being
# retried up to 5 times when the command returns the keyword "busy".
#
# $@ - command to execute
function log_must_busy
{
	log_must_retry "busy" 5 "$@"
	(( $? != 0 )) && log_fail
}

# Execute a negative test with keyword expected, and exit
# $STF_FAIL if test passes
#
# $1 - keyword expected
# $2-$@ - command to execute

function log_mustnot_expect
{
	log_neg_expect "$@"
	(( $? != 0 )) && log_fail
}

# Execute and print command with status where success equals non-zero result
# or output includes expected keyword
#
# $1 - keyword expected
# $2-$@ - command to execute
#
# return 0 if command fails, or the output contains the keyword expected,
# return 1 otherwise

function log_neg_expect
{
	typeset out=""
	typeset logfile="/tmp/log.$$"
	typeset ret=1
	typeset expect=$1
	shift

	while [[ -e $logfile ]]; do
		logfile="$logfile.$$"
	done

	"$@" 2>$logfile
	typeset status=$?
	out="/usr/bin/cat $logfile"

	# unexpected status
	if (( $status == 0 )); then
		 print -u2 $($out)
		_printerror "$@" "unexpectedly exited $status"
	# missing binary
	elif (( $status == 127 )); then
		print -u2 $($out)
		_printerror "$@" "unexpectedly exited $status (File not found)"
	# bus error - core dump
	elif (( $status == 138 )); then
		print -u2 $($out)
		_printerror "$@" "unexpectedly exited $status (Bus Error)"
	# segmentation violation - core dump
	elif (( $status == 139 )); then
		print -u2 $($out)
		_printerror "$@" "unexpectedly exited $status (SEGV)"
	else
		$out | /usr/bin/egrep -i "internal error|assertion failed" \
			> /dev/null 2>&1
		# internal error or assertion failed
		if (( $? == 0 )); then
			print -u2 $($out)
			_printerror "$@" "internal error or assertion failure" \
				" exited $status"
		elif [[ -n $expect ]] ; then
			$out | /usr/bin/grep -i "$expect" > /dev/null 2>&1
			if (( $? == 0 )); then
				ret=0
			else
				print -u2 $($out)
				_printerror "$@" "unexpectedly exited $status"
			fi
		else
			ret=0
		fi

		if (( $ret == 0 )); then
			[[ -n $LOGAPI_DEBUG ]] && print $($out)
			_printsuccess "$@" "exited $status"
		fi
	fi
	_recursive_output $logfile "false"
	return $ret
}

# Execute and print command with status where success equals zero result
#
# $@ command to execute
#
# return command exit status

function log_pos
{
	typeset out=""
	typeset logfile="/tmp/log.$$"

	while [[ -e $logfile ]]; do
		logfile="$logfile.$$"
	done

	"$@" 2>$logfile
	typeset status=$?
	out="/usr/bin/cat $logfile"

	if (( $status != 0 )) ; then
		print -u2 $($out)
		_printerror "$@" "exited $status"
	else
		$out | /usr/bin/egrep -i "internal error|assertion failed" \
			> /dev/null 2>&1
		# internal error or assertion failed
		if [[ $? -eq 0 ]]; then
			print -u2 $($out)
			_printerror "$@" "internal error or assertion failure" \
				" exited $status"
			status=1
		else
			[[ -n $LOGAPI_DEBUG ]] && print $($out)
			_printsuccess "$@"
		fi
	fi
	_recursive_output $logfile "false"
	return $status
}

# Set an exit handler
#
# $@ - function(s) to perform on exit

function log_onexit
{
	_CLEANUP="$@"
}

#
# Exit functions
#

# Perform cleanup and exit $STF_PASS
#
# $@ - message text

function log_pass
{
	_endlog $STF_PASS "$@"
}

# Perform cleanup and exit $STF_FAIL
#
# $@ - message text

function log_fail
{
	_endlog $STF_FAIL "$@"
}

# Perform cleanup and exit $STF_UNRESOLVED
#
# $@ - message text

function log_unresolved
{
	_endlog $STF_UNRESOLVED "$@"
}

# Perform cleanup and exit $STF_NOTINUSE
#
# $@ - message text

function log_notinuse
{
	_endlog $STF_NOTINUSE "$@"
}

# Perform cleanup and exit $STF_UNSUPPORTED
#
# $@ - message text

function log_unsupported
{
	_endlog $STF_UNSUPPORTED "$@"
}

# Perform cleanup and exit $STF_UNTESTED
#
# $@ - message text

function log_untested
{
	_endlog $STF_UNTESTED "$@"
}

# Perform cleanup and exit $STF_UNINITIATED
#
# $@ - message text

function log_uninitiated
{
	_endlog $STF_UNINITIATED "$@"
}

# Perform cleanup and exit $STF_NORESULT
#
# $@ - message text

function log_noresult
{
	_endlog $STF_NORESULT "$@"
}

# Perform cleanup and exit $STF_WARNING
#
# $@ - message text

function log_warning
{
	_endlog $STF_WARNING "$@"
}

# Perform cleanup and exit $STF_TIMED_OUT
#
# $@ - message text

function log_timed_out
{
	_endlog $STF_TIMED_OUT "$@"
}

# Perform cleanup and exit $STF_OTHER
#
# $@ - message text

function log_other
{
	_endlog $STF_OTHER "$@"
}

#
# Internal functions
#

# Execute custom callback scripts on test failure
#
# callback script paths are stored in TESTFAIL_CALLBACKS, delimited by ':'.

function _execute_testfail_callbacks
{
	typeset callback

	print "$TESTFAIL_CALLBACKS:" | while read -d ":" callback; do
		if [[ -n "$callback" ]] ; then
			log_note "Performing test-fail callback ($callback)"
			$callback
		fi
	done
}

# Perform cleanup and exit
#
# $1 - stf exit code
# $2-$n - message text

function _endlog
{
	typeset logfile="/tmp/log.$$"
	_recursive_output $logfile

	if [[ $1 == $STF_FAIL ]] ; then
		_execute_testfail_callbacks
	fi

	if [[ -n $_CLEANUP ]] ; then
		typeset cleanup=$_CLEANUP
		log_onexit ""
		log_note "Performing local cleanup via log_onexit ($cleanup)"
		$cleanup
	fi
	typeset exitcode=$1
	shift
	(( ${#@} > 0 )) && _printline "$@"
	exit $exitcode
}

# Output a formatted line
#
# $@ - message text

function _printline
{
	print "$@"
}

# Output an error message
#
# $@ - message text

function _printerror
{
	_printline ERROR: "$@"
}

# Output a success message
#
# $@ - message text

function _printsuccess
{
	_printline SUCCESS: "$@"
}

# Output logfiles recursively
#
# $1 - start file
# $2 - indicate whether output the start file itself, default as yes.

function _recursive_output #logfile
{
	typeset logfile=$1

	while [[ -e $logfile ]]; do
		if [[ -z $2 || $logfile != $1 ]]; then
			/usr/bin/cat $logfile
		fi
		/usr/bin/rm -f $logfile
		logfile="$logfile.$$"
        done
}
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source.  A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#

#
# Copyright (c) 2012 by Delphix. All rights reserved.
#

include $(SRC)/Makefile.master
include $(SRC)/test/Makefile.com

ROOTOPTPKG = $(ROOT)/opt/test-runner
INCLUDEDIR = $(ROOTOPTPKG)/stf/include

PROGS = stf.shlib

CMDS = $(PROGS:%=$(INCLUDEDIR)/%)
$(CMDS) : FILEMODE = 0555

all lint clean clobber:

install: $(CMDS)

$(CMDS): $(INCLUDEDIR)

$(INCLUDEDIR):
	$(INS.dir)

$(INCLUDEDIR)/%: %
	$(INS.file)
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#

#
# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
# Copyright (c) 2012 by Delphix. All rights reserved.
#


STF_PASS=0
STF_FAIL=1
STF_UNRESOLVED=2
STF_NOTINUSE=3
STF_UNSUPPORTED=4
STF_UNTESTED=5
STF_UNINITIATED=6
STF_NORESULT=7
STF_WARNING=8
STF_TIMED_OUT=9
STF_OTHER=10

# do this to use the names: eval echo \$STF_RESULT_NAME_${result}
STF_RESULT_NAME_0="PASS"
STF_RESULT_NAME_1="FAIL"
STF_RESULT_NAME_2="UNRESOLVED"
STF_RESULT_NAME_3="NOTINUSE"
STF_RESULT_NAME_4="UNSUPPORTED"
STF_RESULT_NAME_5="UNTESTED"
STF_RESULT_NAME_6="UNINITIATED"
STF_RESULT_NAME_7="NORESULT"
STF_RESULT_NAME_8="WARNING"
STF_RESULT_NAME_9="TIMED_OUT"
STF_RESULT_NAME_10="OTHER"

# do this to use the array: ${STF_RESULT_NAMES[$result]}
STF_RESULT_NAMES=( "PASS" "FAIL" "UNRESOLVED" "NOTINUSE" "UNSUPPORTED" \
    "UNTESTED" "UNINITIATED" "NORESULT" "WARNING" "TIMED_OUT" "OTHER" )