# # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright (c) 2012 by Delphix. All rights reserved. # .PARALLEL: $(SUBDIRS) SUBDIRS = cmd stf man include $(SRC)/test/Makefile.com # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright (c) 2012 by Delphix. All rights reserved. # include $(SRC)/Makefile.master include $(SRC)/test/Makefile.com ROOTOPTPKG = $(ROOT)/opt/test-runner ROOTBIN = $(ROOTOPTPKG)/bin PROGS = run CMDS = $(PROGS:%=$(ROOTBIN)/%) $(CMDS) : FILEMODE = 0555 all clean clobber: install: $(CMDS) $(CMDS): $(ROOTBIN) $(ROOTBIN): $(INS.dir) $(ROOTBIN)/%: % $(INS.pyfile) #!@PYTHON@ # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright (c) 2012, 2016 by Delphix. All rights reserved. # Copyright (c) 2017, Chris Fraire . # Copyright 2019 Joyent, Inc. # Copyright 2020 OmniOS Community Edition (OmniOSce) Association. # from __future__ import print_function import sys PY3 = sys.version_info[0] == 3 if PY3: import configparser else: import ConfigParser as configparser import io import os import logging import platform import re from logging.handlers import WatchedFileHandler from datetime import datetime from optparse import OptionParser from pwd import getpwnam from pwd import getpwuid from select import select from subprocess import PIPE from subprocess import Popen from sys import argv from sys import exit from sys import maxsize from threading import Timer from time import time BASEDIR = '/var/tmp/test_results' TESTDIR = '/opt/zfs-tests/' KILL = '/usr/bin/kill' TRUE = '/usr/bin/true' SUDO = '/usr/bin/sudo' retcode = 0 # Custom class to reopen the log file in case it is forcibly closed by a test. class WatchedFileHandlerClosed(WatchedFileHandler): """Watch files, including closed files. Similar to (and inherits from) logging.handler.WatchedFileHandler, except that IOErrors are handled by reopening the stream and retrying. This will be retried up to a configurable number of times before giving up, default 5. """ def __init__(self, filename, mode='a', encoding='utf-8', delay=0, max_tries=5): self.max_tries = max_tries self.tries = 0 WatchedFileHandler.__init__(self, filename, mode, encoding, delay) def emit(self, record): while True: try: WatchedFileHandler.emit(self, record) self.tries = 0 return except IOError as err: if self.tries == self.max_tries: raise self.stream.close() self.stream = self._open() self.tries += 1 class Result(object): total = 0 runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0} def __init__(self): self.starttime = None self.returncode = None self.runtime = '' self.stdout = [] self.stderr = [] self.result = '' def done(self, proc, killed): """ Finalize the results of this Cmd. Report SKIP for return codes 3,4 (NOTINUSE, UNSUPPORTED) as defined in ../stf/include/stf.shlib """ global retcode Result.total += 1 m, s = divmod(time() - self.starttime, 60) self.runtime = '%02d:%02d' % (m, s) self.returncode = proc.returncode if killed: self.result = 'KILLED' Result.runresults['KILLED'] += 1 retcode = 2; elif self.returncode == 0: self.result = 'PASS' Result.runresults['PASS'] += 1 elif self.returncode == 3 or self.returncode == 4: self.result = 'SKIP' Result.runresults['SKIP'] += 1 elif self.returncode != 0: self.result = 'FAIL' Result.runresults['FAIL'] += 1 retcode = 1; class Output(object): """ This class is a slightly modified version of the 'Stream' class found here: http://goo.gl/aSGfv """ def __init__(self, stream): self.stream = stream self._buf = '' self.lines = [] def fileno(self): return self.stream.fileno() def read(self, drain=0): """ Read from the file descriptor. If 'drain' set, read until EOF. """ while self._read() is not None: if not drain: break def _read(self): """ Read up to 4k of data from this output stream. Collect the output up to the last newline, and append it to any leftover data from a previous call. The lines are stored as a (timestamp, data) tuple for easy sorting/merging later. """ fd = self.fileno() buf = os.read(fd, 4096).decode('utf-8', errors='ignore') if not buf: return None if '\n' not in buf: self._buf += buf return [] buf = self._buf + buf tmp, rest = buf.rsplit('\n', 1) self._buf = rest now = datetime.now() rows = tmp.split('\n') self.lines += [(now, r) for r in rows] class Cmd(object): verified_users = [] def __init__(self, pathname, identifier=None, outputdir=None, timeout=None, user=None, tags=None): self.pathname = pathname self.identifier = identifier self.outputdir = outputdir or 'BASEDIR' self.timeout = timeout self.user = user or '' self.killed = False self.result = Result() if self.timeout is None: self.timeout = 60 def __str__(self): return '''\ Pathname: %s Identifier: %s Outputdir: %s Timeout: %d User: %s ''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user) def kill_cmd(self, proc): """ Kill a running command due to timeout, or ^C from the keyboard. If sudo is required, this user was verified previously. """ self.killed = True do_sudo = len(self.user) != 0 signal = '-TERM' cmd = [SUDO, KILL, signal, str(proc.pid)] if not do_sudo: del cmd[0] try: kp = Popen(cmd) kp.wait() except: pass def update_cmd_privs(self, cmd, user): """ If a user has been specified to run this Cmd and we're not already running as that user, prepend the appropriate sudo command to run as that user. """ me = getpwuid(os.getuid()) if not user or user == me.pw_name: return cmd ret = '%s -E -u %s %s' % (SUDO, user, cmd) return ret.split(' ') def collect_output(self, proc): """ Read from stdout/stderr as data becomes available, until the process is no longer running. Return the lines from the stdout and stderr Output objects. """ out = Output(proc.stdout) err = Output(proc.stderr) res = [] while proc.returncode is None: proc.poll() res = select([out, err], [], [], .1) for fd in res[0]: fd.read() for fd in res[0]: fd.read(drain=1) return out.lines, err.lines def run(self, options): """ This is the main function that runs each individual test. Determine whether or not the command requires sudo, and modify it if needed. Run the command, and update the result object. """ if options.dryrun is True: print(self) return privcmd = self.update_cmd_privs(self.pathname, self.user) try: old = os.umask(0) if not os.path.isdir(self.outputdir): os.makedirs(self.outputdir, mode=0o777) os.umask(old) except OSError as e: fail('%s' % e) try: self.result.starttime = time() proc = Popen(privcmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, universal_newlines=True) proc.stdin.close() # Allow a special timeout value of 0 to mean infinity if int(self.timeout) == 0: self.timeout = maxsize t = Timer(int(self.timeout), self.kill_cmd, [proc]) t.start() self.result.stdout, self.result.stderr = self.collect_output(proc) except KeyboardInterrupt: self.kill_cmd(proc) fail('\nRun terminated at user request.') finally: t.cancel() self.result.done(proc, self.killed) def skip(self): """ Initialize enough of the test result that we can log a skipped command. """ Result.total += 1 Result.runresults['SKIP'] += 1 self.result.stdout = self.result.stderr = [] self.result.starttime = time() m, s = divmod(time() - self.result.starttime, 60) self.result.runtime = '%02d:%02d' % (m, s) self.result.result = 'SKIP' def log(self, logger, options): """ This function is responsible for writing all output. This includes the console output, the logfile of all results (with timestamped merged stdout and stderr), and for each test, the unmodified stdout/stderr/merged in it's own file. """ if logger is None: return logname = getpwuid(os.getuid()).pw_name user = ' (run as %s)' % (self.user if len(self.user) else logname) if self.identifier: msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user) else: msga = 'Test: %s%s ' % (self.pathname, user) msgb = '[%s] [%s]' % (self.result.runtime, self.result.result) pad = ' ' * (80 - (len(msga) + len(msgb))) # If -q is specified, only print a line for tests that didn't pass. # This means passing tests need to be logged as DEBUG, or the one # line summary will only be printed in the logfile for failures. if not options.quiet: logger.info('%s%s%s' % (msga, pad, msgb)) elif self.result.result != 'PASS': logger.info('%s%s%s' % (msga, pad, msgb)) else: logger.debug('%s%s%s' % (msga, pad, msgb)) lines = sorted(self.result.stdout + self.result.stderr, key=lambda x: x[0]) for dt, line in lines: logger.debug('%s %s' % (dt.strftime("%H:%M:%S.%f ")[:11], line)) if len(self.result.stdout): with io.open(os.path.join(self.outputdir, 'stdout'), encoding='utf-8', errors='surrogateescape', mode='w') as out: for _, line in self.result.stdout: out.write('%s\n' % line) if len(self.result.stderr): with io.open(os.path.join(self.outputdir, 'stderr'), encoding='utf-8', errors='surrogateescape', mode='w') as err: for _, line in self.result.stderr: err.write('%s\n' % line) if len(self.result.stdout) and len(self.result.stderr): with io.open(os.path.join(self.outputdir, 'merged'), encoding='utf-8', errors='surrogateescape', mode='w') as merged: for _, line in lines: merged.write('%s\n' % line) class Test(Cmd): props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post', 'post_user', 'tags'] def __init__(self, pathname, pre=None, pre_user=None, post=None, post_user=None, tags=None, **kwargs): super(Test, self).__init__(pathname, **kwargs) self.pre = pre or '' self.pre_user = pre_user or '' self.post = post or '' self.post_user = post_user or '' self.tags = tags or [] def __str__(self): post_user = pre_user = '' if len(self.pre_user): pre_user = ' (as %s)' % (self.pre_user) if len(self.post_user): post_user = ' (as %s)' % (self.post_user) return '''\ Pathname: %s Identifier: %s Outputdir: %s Timeout: %d User: %s Pre: %s%s Post: %s%s Tags: %s ''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user, self.pre, pre_user, self.post, post_user, self.tags) def verify(self, logger): """ Check the pre/post scripts, user and Test. Omit the Test from this run if there are any problems. """ files = [self.pre, self.pathname, self.post] users = [self.pre_user, self.user, self.post_user] for f in [f for f in files if len(f)]: if not verify_file(f): logger.info("Warning: Test '%s' not added to this run because" " it failed verification." % f) return False for user in [user for user in users if len(user)]: if not verify_user(user, logger): logger.info("Not adding Test '%s' to this run." % self.pathname) return False return True def run(self, logger, options): """ Create Cmd instances for the pre/post scripts. If the pre script doesn't pass, skip this Test. Run the post script regardless. """ odir = os.path.join(self.outputdir, os.path.basename(self.pre)) pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir, timeout=self.timeout, user=self.pre_user) test = Cmd(self.pathname, identifier=self.identifier, outputdir=self.outputdir, timeout=self.timeout, user=self.user) odir = os.path.join(self.outputdir, os.path.basename(self.post)) posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir, timeout=self.timeout, user=self.post_user) cont = True if len(pretest.pathname): pretest.run(options) cont = pretest.result.result == 'PASS' pretest.log(logger, options) if cont: test.run(options) else: test.skip() test.log(logger, options) if len(posttest.pathname): posttest.run(options) posttest.log(logger, options) class TestGroup(Test): props = Test.props + ['tests'] def __init__(self, pathname, tests=None, **kwargs): super(TestGroup, self).__init__(pathname, **kwargs) self.tests = tests or [] def __str__(self): post_user = pre_user = '' if len(self.pre_user): pre_user = ' (as %s)' % (self.pre_user) if len(self.post_user): post_user = ' (as %s)' % (self.post_user) return '''\ Pathname: %s Identifier: %s Outputdir: %s Tests: %s Timeout: %d User: %s Pre: %s%s Post: %s%s Tags: %s ''' % (self.pathname, self.identifier, self.outputdir, self.tests, self.timeout, self.user, self.pre, pre_user, self.post, post_user, self.tags) def filter(self, keeplist): self.tests = [ x for x in self.tests if x in keeplist ] def verify(self, logger): """ Check the pre/post scripts, user and tests in this TestGroup. Omit the TestGroup entirely, or simply delete the relevant tests in the group, if that's all that's required. """ # If the pre or post scripts are relative pathnames, convert to # absolute, so they stand a chance of passing verification. if len(self.pre) and not os.path.isabs(self.pre): self.pre = os.path.join(self.pathname, self.pre) if len(self.post) and not os.path.isabs(self.post): self.post = os.path.join(self.pathname, self.post) auxfiles = [self.pre, self.post] users = [self.pre_user, self.user, self.post_user] for f in [f for f in auxfiles if len(f)]: if self.pathname != os.path.dirname(f): logger.info("Warning: TestGroup '%s' not added to this run. " "Auxiliary script '%s' exists in a different " "directory." % (self.pathname, f)) return False if not verify_file(f): logger.info("Warning: TestGroup '%s' not added to this run. " "Auxiliary script '%s' failed verification." % (self.pathname, f)) return False for user in [user for user in users if len(user)]: if not verify_user(user, logger): logger.info("Not adding TestGroup '%s' to this run." % self.pathname) return False # If one of the tests is invalid, delete it, log it, and drive on. self.tests[:] = [f for f in self.tests if verify_file(os.path.join(self.pathname, f))] return len(self.tests) != 0 def run(self, logger, options): """ Create Cmd instances for the pre/post scripts. If the pre script doesn't pass, skip all the tests in this TestGroup. Run the post script regardless. """ # tags assigned to this test group also include the test names if options.tags and not set(self.tags).intersection(set(options.tags)): return odir = os.path.join(self.outputdir, os.path.basename(self.pre)) pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, user=self.pre_user, identifier=self.identifier) odir = os.path.join(self.outputdir, os.path.basename(self.post)) posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, user=self.post_user, identifier=self.identifier) cont = True if len(pretest.pathname): pretest.run(options) cont = pretest.result.result == 'PASS' pretest.log(logger, options) for fname in self.tests: test = Cmd(os.path.join(self.pathname, fname), outputdir=os.path.join(self.outputdir, fname), timeout=self.timeout, user=self.user, identifier=self.identifier) if cont: test.run(options) else: test.skip() test.log(logger, options) if len(posttest.pathname): posttest.run(options) posttest.log(logger, options) class TestRun(object): props = ['quiet', 'outputdir'] def __init__(self, options): self.tests = {} self.testgroups = {} self.starttime = time() self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') self.outputdir = os.path.join(options.outputdir, self.timestamp) self.logger = self.setup_logging(options) self.defaults = [ ('outputdir', BASEDIR), ('quiet', False), ('timeout', 60), ('user', ''), ('pre', ''), ('pre_user', ''), ('post', ''), ('post_user', ''), ('tags', []) ] def __str__(self): s = 'TestRun:\n outputdir: %s\n' % self.outputdir s += 'TESTS:\n' for key in sorted(self.tests.keys()): s += '%s%s' % (self.tests[key].__str__(), '\n') s += 'TESTGROUPS:\n' for key in sorted(self.testgroups.keys()): s += '%s%s' % (self.testgroups[key].__str__(), '\n') return s def addtest(self, pathname, options): """ Create a new Test, and apply any properties that were passed in from the command line. If it passes verification, add it to the TestRun. """ test = Test(pathname) for prop in Test.props: setattr(test, prop, getattr(options, prop)) if test.verify(self.logger): self.tests[pathname] = test def addtestgroup(self, dirname, filenames, options): """ Create a new TestGroup, and apply any properties that were passed in from the command line. If it passes verification, add it to the TestRun. """ if dirname not in self.testgroups: testgroup = TestGroup(dirname) for prop in Test.props: setattr(testgroup, prop, getattr(options, prop)) # Prevent pre/post scripts from running as regular tests for f in [testgroup.pre, testgroup.post]: if f in filenames: del filenames[filenames.index(f)] self.testgroups[dirname] = testgroup self.testgroups[dirname].tests = sorted(filenames) testgroup.verify(self.logger) def filter(self, keeplist): for group in list(self.testgroups.keys()): if group not in keeplist: del self.testgroups[group] continue g = self.testgroups[group] if g.pre and os.path.basename(g.pre) in keeplist[group]: continue g.filter(keeplist[group]) for test in list(self.tests.keys()): directory, base = os.path.split(test) if directory not in keeplist or base not in keeplist[directory]: del self.tests[test] def read(self, logger, options): """ Read in the specified runfile, and apply the TestRun properties listed in the 'DEFAULT' section to our TestRun. Then read each section, and apply the appropriate properties to the Test or TestGroup. Properties from individual sections override those set in the 'DEFAULT' section. If the Test or TestGroup passes verification, add it to the TestRun. """ config = configparser.RawConfigParser() parsed = config.read(options.runfiles) failed = options.runfiles - set(parsed) if len(failed): files = ' '.join(sorted(failed)) fail("Couldn't read config files: %s" % files) for opt in TestRun.props: if config.has_option('DEFAULT', opt): setattr(self, opt, config.get('DEFAULT', opt)) self.outputdir = os.path.join(self.outputdir, self.timestamp) testdir = options.testdir for section in config.sections(): if ('arch' in config.options(section) and platform.machine() != config.get(section, 'arch')): continue parts = section.split(':', 1) sectiondir = parts[0] identifier = parts[1] if len(parts) == 2 else None if os.path.isdir(sectiondir): pathname = sectiondir elif os.path.isdir(os.path.join(testdir, sectiondir)): pathname = os.path.join(testdir, sectiondir) else: pathname = sectiondir testgroup = TestGroup(os.path.abspath(pathname), identifier=identifier) if 'tests' in config.options(section): for prop in TestGroup.props: for sect in ['DEFAULT', section]: if config.has_option(sect, prop): if prop == 'tags': setattr(testgroup, prop, eval(config.get(sect, prop))) else: setattr(testgroup, prop, config.get(sect, prop)) # Repopulate tests using eval to convert the string to a list testgroup.tests = eval(config.get(section, 'tests')) if testgroup.verify(logger): self.testgroups[section] = testgroup elif 'autotests' in config.options(section): for prop in TestGroup.props: for sect in ['DEFAULT', section]: if config.has_option(sect, prop): setattr(testgroup, prop, config.get(sect, prop)) filenames = os.listdir(pathname) # only files starting with "tst." are considered tests filenames = [f for f in filenames if f.startswith("tst.")] testgroup.tests = sorted(filenames) if testgroup.verify(logger): self.testgroups[section] = testgroup else: test = Test(section) for prop in Test.props: for sect in ['DEFAULT', section]: if config.has_option(sect, prop): setattr(test, prop, config.get(sect, prop)) if test.verify(logger): self.tests[section] = test def write(self, options): """ Create a configuration file for editing and later use. The 'DEFAULT' section of the config file is created from the properties that were specified on the command line. Tests are simply added as sections that inherit everything from the 'DEFAULT' section. TestGroups are the same, except they get an option including all the tests to run in that directory. """ defaults = dict([(prop, getattr(options, prop)) for prop, _ in self.defaults]) config = configparser.RawConfigParser(defaults) for test in sorted(self.tests.keys()): config.add_section(test) for prop in Test.props: if prop not in self.props: config.set(testgroup, prop, getattr(self.testgroups[testgroup], prop)) for testgroup in sorted(self.testgroups.keys()): config.add_section(testgroup) config.set(testgroup, 'tests', self.testgroups[testgroup].tests) for prop in TestGroup.props: if prop not in self.props: config.set(testgroup, prop, getattr(self.testgroups[testgroup], prop)) try: with open(options.template, 'w') as f: return config.write(f) except IOError: fail('Could not open \'%s\' for writing.' % options.template) def complete_outputdirs(self): """ Collect all the pathnames for Tests, and TestGroups. Work backwards one pathname component at a time, to create a unique directory name in which to deposit test output. Tests will be able to write output files directly in the newly modified outputdir. TestGroups will be able to create one subdirectory per test in the outputdir, and are guaranteed uniqueness because a group can only contain files in one directory. Pre and post tests will create a directory rooted at the outputdir of the Test or TestGroup in question for their output. """ done = False components = 0 tmp_dict = dict(list(self.tests.items()) + list(self.testgroups.items())) total = len(tmp_dict) base = self.outputdir while not done: l = [] components -= 1 for testfile in list(tmp_dict.keys()): uniq = '/'.join(testfile.split('/')[components:]).lstrip('/') if uniq not in l: l.append(uniq) tmp_dict[testfile].outputdir = os.path.join(base, uniq) else: break done = total == len(l) def setup_logging(self, options): """ Two loggers are set up here. The first is for the logfile which will contain one line summarizing the test, including the test name, result, and running time. This logger will also capture the timestamped combined stdout and stderr of each run. The second logger is optional console output, which will contain only the one line summary. The loggers are initialized at two different levels to facilitate segregating the output. """ if options.dryrun is True: return testlogger = logging.getLogger(__name__) testlogger.setLevel(logging.DEBUG) if not options.template: try: old = os.umask(0) os.makedirs(self.outputdir, mode=0o777) os.umask(old) except OSError as e: fail('%s' % e) filename = os.path.join(self.outputdir, 'log') logfile = WatchedFileHandlerClosed(filename) logfile.setLevel(logging.DEBUG) logfilefmt = logging.Formatter('%(message)s') logfile.setFormatter(logfilefmt) testlogger.addHandler(logfile) cons = logging.StreamHandler() cons.setLevel(logging.INFO) consfmt = logging.Formatter('%(message)s') cons.setFormatter(consfmt) testlogger.addHandler(cons) return testlogger def run(self, options): """ Walk through all the Tests and TestGroups, calling run(). """ if not options.dryrun: try: os.chdir(self.outputdir) except OSError: fail('Could not change to directory %s' % self.outputdir) for test in sorted(self.tests.keys()): self.tests[test].run(self.logger, options) for testgroup in sorted(self.testgroups.keys()): self.testgroups[testgroup].run(self.logger, options) def summary(self): if Result.total == 0: print('No tests to run') return print('\nResults Summary') for key in list(Result.runresults.keys()): if Result.runresults[key] != 0: print('%s\t% 4d' % (key, Result.runresults[key])) m, s = divmod(time() - self.starttime, 60) h, m = divmod(m, 60) print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)) print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / float(Result.total)) * 100)) print('Log directory:\t%s' % self.outputdir) def verify_file(pathname): """ Verify that the supplied pathname is an executable regular file. """ if os.path.isdir(pathname) or os.path.islink(pathname): return False if os.path.isfile(pathname) and os.access(pathname, os.X_OK): return True return False def verify_user(user, logger): """ Verify that the specified user exists on this system, and can execute sudo without being prompted for a password. """ testcmd = [SUDO, '-n', '-u', user, TRUE] if user in Cmd.verified_users: return True try: _ = getpwnam(user) except KeyError: logger.info("Warning: user '%s' does not exist.", user) return False p = Popen(testcmd) p.wait() if p.returncode != 0: logger.info("Warning: user '%s' cannot use passwordless sudo.", user) return False else: Cmd.verified_users.append(user) return True def find_tests(testrun, options): """ For the given list of pathnames, add files as Tests. For directories, if do_groups is True, add the directory as a TestGroup. If False, recursively search for executable files. """ for p in sorted(options.pathnames): if os.path.isdir(p): for dirname, _, filenames in os.walk(p): if options.do_groups: testrun.addtestgroup(dirname, filenames, options) else: for f in sorted(filenames): testrun.addtest(os.path.join(dirname, f), options) else: testrun.addtest(p, options) def filter_tests(testrun, options): try: fh = open(options.logfile, "r", errors='replace') except Exception as e: fail('%s' % e) failed = {} while True: line = fh.readline() if not line: break m = re.match(r'Test: (.*)/(\S+).*\[FAIL\]', line) if not m: continue group, test = m.group(1, 2) m = re.match(re.escape(options.testdir) + r'(.*)', group) if m: group = m.group(1) try: failed[group].append(test) except KeyError: failed[group] = [ test ] fh.close() testrun.filter(failed) def fail(retstr, ret=1): print('%s: %s' % (argv[0], retstr)) exit(ret) def options_cb(option, opt_str, value, parser): path_options = ['outputdir', 'template', 'testdir', 'logfile'] if opt_str in parser.rargs: fail('%s may only be specified once.' % opt_str) if option.dest == 'runfiles': parser.values.cmd = 'rdconfig' value = set(os.path.abspath(p) for p in value.split(',')) if option.dest == 'tags': value = [x.strip() for x in value.split(',')] if option.dest in path_options: setattr(parser.values, option.dest, os.path.abspath(value)) else: setattr(parser.values, option.dest, value) def parse_args(): parser = OptionParser() parser.add_option('-c', action='callback', callback=options_cb, type='string', dest='runfiles', metavar='runfiles', help='Specify tests to run via config files.') parser.add_option('-d', action='store_true', default=False, dest='dryrun', help='Dry run. Print tests, but take no other action.') parser.add_option('-l', action='callback', callback=options_cb, default=None, dest='logfile', metavar='logfile', type='string', help='Read logfile and re-run tests which failed.') parser.add_option('-g', action='store_true', default=False, dest='do_groups', help='Make directories TestGroups.') parser.add_option('-o', action='callback', callback=options_cb, default=BASEDIR, dest='outputdir', type='string', metavar='outputdir', help='Specify an output directory.') parser.add_option('-i', action='callback', callback=options_cb, default=TESTDIR, dest='testdir', type='string', metavar='testdir', help='Specify a test directory.') parser.add_option('-p', action='callback', callback=options_cb, default='', dest='pre', metavar='script', type='string', help='Specify a pre script.') parser.add_option('-P', action='callback', callback=options_cb, default='', dest='post', metavar='script', type='string', help='Specify a post script.') parser.add_option('-q', action='store_true', default=False, dest='quiet', help='Silence on the console during a test run.') parser.add_option('-t', action='callback', callback=options_cb, default=60, dest='timeout', metavar='seconds', type='int', help='Timeout (in seconds) for an individual test.') parser.add_option('-u', action='callback', callback=options_cb, default='', dest='user', metavar='user', type='string', help='Specify a different user name to run as.') parser.add_option('-w', action='callback', callback=options_cb, default=None, dest='template', metavar='template', type='string', help='Create a new config file.') parser.add_option('-x', action='callback', callback=options_cb, default='', dest='pre_user', metavar='pre_user', type='string', help='Specify a user to execute the pre script.') parser.add_option('-X', action='callback', callback=options_cb, default='', dest='post_user', metavar='post_user', type='string', help='Specify a user to execute the post script.') parser.add_option('-T', action='callback', callback=options_cb, default='', dest='tags', metavar='tags', type='string', help='Specify tags to execute specific test groups.') (options, pathnames) = parser.parse_args() if options.runfiles and len(pathnames): fail('Extraneous arguments.') options.pathnames = [os.path.abspath(path) for path in pathnames] return options def main(): options = parse_args() testrun = TestRun(options) if options.runfiles: testrun.read(testrun.logger, options) else: find_tests(testrun, options) if options.logfile: filter_tests(testrun, options) if options.template: testrun.write(options) exit(0) testrun.complete_outputdirs() testrun.run(options) testrun.summary() exit(retcode) if __name__ == '__main__': main() # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright (c) 2012 by Delphix. All rights reserved. # include $(SRC)/Makefile.master MANSRCS = run.1 ROOTOPTPKG = $(ROOT)/opt/test-runner MANDIR = $(ROOTOPTPKG)/man/man1 FILES = $(MANSRCS:%=$(MANDIR)/%) $(FILES) : FILEMODE = 0444 all: $(MANSRCS) install: $(FILES) clean lint clobber: $(FILES): $(MANDIR) $(MANSRCS) $(MANDIR): $(INS.dir) $(MANDIR)/%: % $(INS.file) .\" .\" This file and its contents are supplied under the terms of the .\" Common Development and Distribution License ("CDDL"), version 1.0. .\" You may only use this file in accordance with the terms of version .\" 1.0 of the CDDL. .\" .\" A full copy of the text of the CDDL should have accompanied this .\" source. A copy of the CDDL is also available via the Internet at .\" http://www.illumos.org/license/CDDL. .\" .\" .\" Copyright (c) 2012 by Delphix. All rights reserved. .\" .TH run 1 "23 Sep 2012" .SH NAME run \- find, execute, and log the results of tests .SH SYNOPSIS .LP .nf \fBrun\fR [\fB-dgq] [\fB-o\fR \fIoutputdir\fR] [\fB-pP\fR \fIscript\fR] [\fB-t\fR \fIseconds\fR] [\fB-uxX\fR \fIusername\fR] \fIpathname\fR ... .fi .LP .nf \fBrun\fR \fB-w\fR \fIrunfile\fR [\fB-gq\fR] [\fB-o\fR \fIoutputdir\fR] [\fB-pP\fR \fIscript\fR] [\fB-t\fR \fIseconds\fR] [\fB-uxX\fR \fIusername\fR] \fIpathname\fR ... .fi .LP .nf \fBrun\fR \fB-c\fR \fIrunfile\fR [\fB-dq\fR] .fi .LP .nf \fBrun\fR [\fB-h\fR] .fi .SH DESCRIPTION .sp .LP The \fBrun\fR command has three basic modes of operation. With neither the \fB-c\fR nor the \fB-w\fR option, \fBrun\fR processes the arguments provided on the command line, adding them to the list for this run. If a specified \fIpathname\fR is an executable file, it is added as a test. If a specified \fIpathname\fR is a directory, the behavior depends upon the \fB-g\fR option. If \fB-g\fR is specified, the directory is treated as a test group. See the section on "Test Groups" below. Without the \fB-g\fR option, \fBrun\fR simply descends into the directory looking for executable files. The tests are then executed, and the results are logged. With the \fB-w\fR option, \fBrun\fR finds tests in the manner described above. Rather than executing the tests and logging the results, the test configuration is stored in a \fIrunfile\fR which can be used in future invocations, or edited to modify which tests are executed and which options are applied. Options included on the command line with \fB-w\fR become defaults in the \fIrunfile\fR. With the \fB-c\fR option, \fBrun\fR parses a \fIrunfile\fR, which can specify a series of tests and test groups to be executed. The tests are then executed, and the results are logged. .sp .SS "Test Groups" .sp .LP A test group is comprised of a set of executable files, all of which exist in one directory. The options specified on the command line or in a \fIrunfile\fR apply to individual tests in the group. The exception is options pertaining to pre and post scripts, which act on all tests as a group. Rather than running before and after each test, these scripts are run only once each at the start and end of the test group. .SS "Test Execution" .sp .LP The specified tests run serially, and are typically assigned results according to exit values. Tests that exit zero and non-zero are marked "PASS" and "FAIL" respectively. When a pre script fails for a test group, only the post script is executed, and the remaining tests are marked "SKIPPED." Any test that exceeds its \fItimeout\fR is terminated, and marked "KILLED." By default, tests are executed with the credentials of the \fBrun\fR script. Executing tests with other credentials is done via \fBsudo\fR(8), which must be configured to allow execution without prompting for a password. Environment variables from the calling shell are available to individual tests. During test execution, the working directory is changed to \fIoutputdir\fR. .SS "Output Logging" .sp .LP By default, \fBrun\fR will print one line on standard output at the conclusion of each test indicating the test name, result and elapsed time. Additionally, for each invocation of \fBrun\fR, a directory is created using the ISO 8601 date format. Within this directory is a file named \fIlog\fR containing all the test output with timestamps, and a directory for each test. Within the test directories, there is one file each for standard output, standard error and merged output. The default location for the \fIoutputdir\fR is \fI/var/tmp/test_results\fR. .SS "Runfiles" .sp .LP The \fIrunfile\fR is an ini style configuration file that describes a test run. The file has one section named "DEFAULT," which contains configuration option names and their values in "name = value" format. The values in this section apply to all the subsequent sections, unless they are also specified there, in which case the default is overridden. The remaining section names are the absolute pathnames of files and direcotries, describing tests and test groups respectively. The legal option names are: .sp .ne 2 .na \fBoutputdir\fR = \fIpathname\fR .ad .sp .6 .RS 4n The name of the directory that holds test logs. .RE .sp .ne 2 .na \fBpre\fR = \fIscript\fR .ad .sp .6 .RS 4n Run \fIscript\fR prior to the test or test group. .RE .sp .ne 2 .na \fBpre_user\fR = \fIusername\fR .ad .sp .6 .RS 4n Execute the pre script as \fIusername\fR. .RE .sp .ne 2 .na \fBpost\fR = \fIscript\fR .ad .sp .6 .RS 4n Run \fIscript\fR after the test or test group. .RE .sp .ne 2 .na \fBpost_user\fR = \fIusername\fR .ad .sp .6 .RS 4n Execute the post script as \fIusername\fR. .RE .sp .ne 2 .na \fBquiet\fR = [\fITrue\fR|\fIFalse\fR] .ad .sp .6 .RS 4n If set to True, only the results summary is printed to standard out. .RE .sp .ne 2 .na \fBtests\fR = [\fI'filename'\fR [,...]] .ad .sp .6 .RS 4n Specify a list of \fIfilenames\fR for this test group. Only the basename of the absolute path is required. This option is only valid for test groups, and each \fIfilename\fR must be single quoted. .RE .sp .ne 2 .na \fBtimeout\fR = \fIn\fR .ad .sp .6 .RS 4n A timeout value of \fIn\fR seconds. .RE .sp .ne 2 .na \fBuser\fR = \fIusername\fR .ad .sp .6 .RS 4n Execute the test or test group as \fIusername\fR. .RE .SH OPTIONS .sp .LP The following options are available for the \fBrun\fR command. .sp .ne 2 .na \fB-c\fR \fIrunfile\fR .ad .RS 6n Specify a \fIrunfile\fR to be consumed by the run command. .RE .ne 2 .na \fB-d\fR .ad .RS 6n Dry run mode. Execute no tests, but print a description of each test that would have been run. .RE .ne 2 .na \fB-g\fR .ad .RS 6n Create test groups from any directories found while searching for tests. .RE .ne 2 .na \fB-o\fR \fIoutputdir\fR .ad .RS 6n Specify the directory in which to write test results. .RE .ne 2 .na \fB-p\fR \fIscript\fR .ad .RS 6n Run \fIscript\fR prior to any test or test group. .RE .ne 2 .na \fB-P\fR \fIscript\fR .ad .RS 6n Run \fIscript\fR after any test or test group. .RE .ne 2 .na \fB-q\fR .ad .RS 6n Print only the results sumary to the standard output. .RE .ne 2 .na \fB-t\fR \fIn\fR .ad .RS 6n Specify a timeout value of \fIn\fR seconds per test. .RE .ne 2 .na \fB-u\fR \fIusername\fR .ad .RS 6n Execute tests or test groups as \fIusername\fR. .RE .ne 2 .na \fB-w\fR \fIrunfile\fR .ad .RS 6n Specify the name of the \fIrunfile\fR to create. .RE .ne 2 .na \fB-x\fR \fIusername\fR .ad .RS 6n Execute the pre script as \fIusername\fR. .RE .ne 2 .na \fB-X\fR \fIusername\fR .ad .RS 6n Execute the post script as \fIusername\fR. .RE .SH EXAMPLES .LP \fBExample 1\fR Running ad-hoc tests. .sp .LP This example demonstrates the simplest invocation of \fBrun\fR. .sp .in +2 .nf % \fBrun my-tests\fR Test: /home/jkennedy/my-tests/test-01 [00:02] [PASS] Test: /home/jkennedy/my-tests/test-02 [00:04] [PASS] Test: /home/jkennedy/my-tests/test-03 [00:01] [PASS] Results Summary PASS 3 Running Time: 00:00:07 Percent passed: 100.0% Log directory: /var/tmp/test_results/20120923T180654 .fi .in -2 .LP \fBExample 2\fR Creating a \fIrunfile\fR for future use. .sp .LP This example demonstrates creating a \fIrunfile\fR with non default options. .sp .in +2 .nf % \fBrun -p setup -x root -g -w new-tests.run new-tests\fR % \fBcat new-tests.run\fR [DEFAULT] pre = setup post_user = quiet = False user = timeout = 60 post = pre_user = root outputdir = /var/tmp/test_results [/home/jkennedy/new-tests] tests = ['test-01', 'test-02', 'test-03'] .fi .in -2 .SH EXIT STATUS .sp .LP The following exit values are returned: .sp .ne 2 .na \fB\fB0\fR\fR .ad .sp .6 .RS 4n Successful completion. .RE .sp .ne 2 .na \fB\fB1\fR\fR .ad .sp .6 .RS 4n An error occurred. .RE .SH SEE ALSO .sp .LP \fBsudo\fR(8) # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright (c) 2012 by Delphix. All rights reserved. # .PARALLEL: $(SUBDIRS) SUBDIRS = contrib/include include include $(SRC)/test/Makefile.com # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright (c) 2012 by Delphix. All rights reserved. # This directory contains shell libraries used by the Solaris Test Framework (STF). They are duplicated here to allow test suites (like the ZFS Test Suite) that were written for STF to be easily ported to this framework. The "Artistic License" Preamble The intent of this document is to state the conditions under which a Package may be copied, such that the Copyright Holder maintains some semblance of artistic control over the development of the package, while giving the users of the package the right to use and distribute the Package in a more-or-less customary fashion, plus the right to make reasonable modifications. Definitions: * "Package" refers to the collection of files distributed by the Copyright Holder, and derivatives of that collection of files created through textual modification. * "Standard Version" refers to such a Package if it has not been modified, or has been modified in accordance with the wishes of the Copyright Holder. * "Copyright Holder" is whoever is named in the copyright or copyrights for the package. * "You" is you, if you're thinking about copying or distributing this Package. * "Reasonable copying fee" is whatever you can justify on the basis of media cost, duplication charges, time of people involved, and so on. (You will not be required to justify it to the Copyright Holder, but only to the computing community at large as a market that must bear the fee.) * "Freely Available" means that no fee is charged for the item itself, though there may be fees involved in handling the item. It also means that recipients of the item may redistribute it under the same conditions they received it. 1. You may make and give away verbatim copies of the source form of the Standard Version of this Package without restriction, provided that you duplicate all of the original copyright notices and associated disclaimers. 2. You may apply bug fixes, portability fixes and other modifications derived from the Public Domain or from the Copyright Holder. A Package modified in such a way shall still be considered the Standard Version. 3. You may otherwise modify your copy of this Package in any way, provided that you insert a prominent notice in each changed file stating how and when you changed that file, and provided that you do at least ONE of the following: a) place your modifications in the Public Domain or otherwise make them Freely Available, such as by posting said modifications to Usenet or an equivalent medium, or placing the modifications on a major archive site such as ftp.uu.net, or by allowing the Copyright Holder to include your modifications in the Standard Version of the Package. b) use the modified Package only within your corporation or organization. c) rename any non-standard executables so the names do not conflict with standard executables, which must also be provided, and provide a separate manual page for each non-standard executable that clearly documents how it differs from the Standard Version. d) make other distribution arrangements with the Copyright Holder. 4. You may distribute the programs of this Package in object code or executable form, provided that you do at least ONE of the following: a) distribute a Standard Version of the executables and library files, together with instructions (in the manual page or equivalent) on where to get the Standard Version. b) accompany the distribution with the machine-readable source of the Package with your modifications. c) accompany any non-standard executables with their corresponding Standard Version executables, giving the non-standard executables non-standard names, and clearly documenting the differences in manual pages (or equivalent), together with instructions on where to get the Standard Version. d) make other distribution arrangements with the Copyright Holder. 5. You may charge a reasonable copying fee for any distribution of this Package. You may charge any fee you choose for support of this Package. You may not charge a fee for this Package itself. However, you may distribute this Package in aggregate with other (possibly commercial) programs as part of a larger (possibly commercial) software distribution provided that you do not advertise this Package as a product of your own. 6. The scripts and library files supplied as input to or produced as output from the programs of this Package do not automatically fall under the copyright of this Package, but belong to whomever generated them, and may be sold commercially, and may be aggregated with this Package. 7. C or perl subroutines supplied by you and linked into this Package shall not be considered part of this Package. 8. The name of the Copyright Holder may not be used to endorse or promote products derived from this software without specific prior written permission. 9. THIS PACKAGE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. The End CTIUTILS.SHLIB IN TEST-RUNNER # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright (c) 2012 by Delphix. All rights reserved. # include $(SRC)/Makefile.master include $(SRC)/test/Makefile.com ROOTOPTPKG = $(ROOT)/opt/test-runner INCLUDEDIR = $(ROOTOPTPKG)/stf/contrib/include PROGS = logapi.shlib ctiutils.shlib CMDS = $(PROGS:%=$(INCLUDEDIR)/%) $(CMDS) : FILEMODE = 0555 all lint clean clobber: install: $(CMDS) $(CMDS): $(INCLUDEDIR) $(INCLUDEDIR): $(INS.dir) $(INCLUDEDIR)/%: % $(INS.file) # # Copyright 2008 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # # This is free software; you can redistribute it and/or modify it # under the terms of the "Artistic License" which is located in the # file named "LICENSE.Artistic" which is included with this software. # # # This file was originally part of the "contrib" portion of the # TET3 test harness from The OpenGroup, originally found here: # http://tetworks.opengroup.org/tet/ or ftp://ftp.xopen.org/pub/TET3/ # and later published in hg.opensolaris.org/hg/test-dev/stcnv-gate # # A build of that repository delivers this file as part of the # SUNWstc-tetlite package, which carries the "Artistic" license # referred to above. # # # The "Artistic License" places some restrictions on the kinds of # changes that may be made to this file. The changes made here # relative to the original are "portability fixes" to allow these # functions to operate in the "test-runner" environment. # # The original from which this was derived can be found here: # https://bitbucket.org/illumos/illumos-stc/ # under the path: # usr/src/tools/tet/contrib/ctitools/src/lib/ksh/ctiutils.ksh ######################################################################## # # NAME: ctiutils.shlib # # SYNOPSIS: # cti_assert assertion_number assertion_msg # cti_report arg ... # cti_pass # cti_fail [msg ...] # cti_unresolved [msg ...] # cti_untested [msg ...] # cti_unsupported [msg ...] # cti_notinuse [msg ...] # cti_pathtrace # cti_checkpath expected-path-count # cti_deleteall reason # cti_reportfile path [title] # cti_rmf [files...] # cti_writefile path mode lines... # cti_appendfile path mode lines... # cti_execute [-c out|err] [-i input] result cmd [args...] # cti_runexpect failtext command pattern action [pattern action ...] # cti_expecttest failtext command pattern action [pattern action ...] # cti_cancel test_num [msg ...] [test result] # cti_cancelall [msg ...] [test result] # # DESCRIPTION: # Common korn shell functions for tests. # # cti_report() writes an informational line to the journal. # # The functions cti_pass(), cti_fail(), cti_unresolved(), # cti_untested(), cti_unsupported() and cti_notinuse() each # registers the corresponding result code against the current test, # and write any arguments to the execution results file, as a # single line. # # The cti_pathtrace() and cti_checkpath() are used in path tracing. # cti_pathtrace() increments the variable pathok. cti_checkpath() # checks the path tracing and registers a PASS result if # appropriate. # # cti_deleteall() cancels all tests in the current test case. # # cti_reportfile() writes the contents of a file to the journal. # # cti_rmf() removes files. # # cti_writefile() writes to the contents of a file; # cti_appendfile() appends to contents of a file. # # cti_execute() executes a command. # # cti_runexpect() runs the expect utility. cti_expecttest() is # like cti_runexpect() except that it is written with path tracing # and is designed to do the complete work of a test purpose. # cti_runexpect() runs the expect utility. cti_expecttest() is # like cti_runexpect() except that it is written with path tracing # and is designed to do the complete work of a test purpose. # # cti_cancel() cancels the dedicated test purpose in the current test # case from execution with the test result that user gives. It will work # in startup function. # # cti_cancelall() cancels all tests in the current test case. It could # be used in startup function to cancel the execution of test cases # with the test result that user gives. # ######################################################################## # # cti_lf_checkargs() - check number of arguments passed to a shell function. # # Usage: cti_lf_checkargs argc expected-argc operator funcname # # operator can be EQ or GE. # # Returns 0 if the expected number of arguments were passed, non-zero # otherwise. # function cti_lf_checkargs { typeset -i cti_lv_argc=$1 typeset -i cti_lv_expargc=$2 typeset cti_lv_op="$3" typeset cti_lv_funcname="$4" case "$cti_lv_op" in EQ) if test $cti_lv_argc -ne $cti_lv_expargc then cti_unresolved "Test coding error:" \ "$cti_lv_funcname() called with $cti_lv_argc" \ "args, need $cti_lv_expargc" return 1 fi ;; GE) if test $cti_lv_argc -lt $cti_lv_expargc then cti_unresolved "Test coding error:" \ "$cti_lv_funcname() called with $cti_lv_argc" \ "args, need >= $cti_lv_expargc" return 1 fi ;; *) cti_unresolved "Internal error: cti_lf_checkargs()" \ "called for $funcname() with operator $cti_lv_op" return 1 ;; esac return 0 } # # cti_result() - register a result and write informational line to journal. # # Usage: cti_result result [msg ...] # # If the current test function is not a startup or cleanup, this routine # registers the specified result code for the current test. The remaining # arguments, if any, are written to the execution results file as a single # line. # # Modifications for test-runner: # Print the result and test name (in place of tet_result) # On failure, return non-zero to tell test-runner our status. # function cti_result { typeset res typeset -i rv=0 test $# -eq 0 && return res=$1 shift my_host=`hostname` my_timestamp=`date | awk '{print $4}'` test $# -gt 0 && print "$my_host $my_timestamp $@" # Print the result and test name (as tet_result would) print "$res: ${tc_id:-$(basename $0)}" # Return non-zero for failures. See codes in: # test-runner/stf/include/stf.shlib case "$res" in PASS) ;; FAIL) rv=1 ;; UNRESOLVED) rv=2 ;; NOTINUSE) rv=3 ;; UNSUPPORTED) rv=4 ;; UNTESTED) rv=5 ;; UNINITIATED) rv=6 ;; NORESULT) rv=7 ;; WARNING) rv=8 ;; TIMED_OUT) rv=9 ;; *) echo "cti_result: $res: unknown result code" rv=10 ;; esac return $rv } # # cti_report() - write an informational line to the journal # # Usage: cti_report arg ... # # Writes the arguments to the execution results file, as a single line. # function cti_report { my_host=`hostname` my_timestamp=`date | awk '{print $4}'` print "$my_host $my_timestamp $@" } # # cti_assert() - write an Assert line to the journal # # Usage: cti_assert assertion_number assertion_msg # # Writes the arguments to the execution results file, as a single line. # function cti_assert { cti_lf_checkargs $# 2 GE cti_assert || return 1 cti_report "ASSERT $1: $2" } # # cti_pass() - register a PASS result. # # Usage: cti_pass [msg ...] # function cti_pass { cti_result PASS "$@" } # # cti_fail() - register a FAIL result. # # Usage: cti_fail [msg ...] # # Registers a FAIL result for the current test, and writes any arguments to # the execution results file, as a single line. # function cti_fail { cti_result FAIL "$@" } # # cti_unresolved() - register an UNRESOLVED result. # # Usage: cti_unresolved [msg ...] # # Registers an UNRESOLVED result for the current test, and writes any arguments # to the execution results file, as a single line. # function cti_unresolved { cti_result UNRESOLVED "$@" } # # cti_uninitiated() - register an UNINITIATED result. # # Usage: cti_uninitiated [msg ...] # # Registers an UNINITIATED result for the current test, and writes any arguments # to the execution results file, as a single line. # function cti_uninitiated { cti_result UNINITIATED "$@" } # # cti_untested() - register an UNTESTED result. # # Usage: cti_untested [msg ...] # # Registers an UNTESTED result for the current test, and writes any arguments # to the execution results file, as a single line. # function cti_untested { cti_result UNTESTED "$@" } # # cti_unsupported() - register an UNSUPPORTED result. # # Usage: cti_unsupported [msg ...] # # Registers an UNSUPPORTED result for the current test, and writes any # arguments to the execution results file, as a single line. # function cti_unsupported { cti_result UNSUPPORTED "$@" } # # cti_notinuse() - register a NOTINUSE result. # # Usage: cti_notinuse [msg ...] # # Registers a NOTINUSE result for the current test, and writes any arguments # to the execution results file, as a single line. # function cti_notinuse { cti_result NOTINUSE "$@" } # # cti_pathtrace() - increment path counter. # # Usage: cti_pathtrace # # Increments variable pathok. Like C macro PATH_TRACE. # function cti_pathtrace { : $((pathok += 1)) } # # cti_checkpath() - check path tracing and register a PASS result. # # Usage: cti_checkpath expected-path-count # # Like C macro PATH_XS_RPT(). # function cti_checkpath { cti_lf_checkargs $# 1 EQ cti_checkpath || return if test $pathok -eq $1 then cti_pass else cti_unresolved "Path tracing error: path counter $pathok," \ "expecting $1" fi } # # cti_deleteall() - cancel all tests. # # Usage: cti_deleteall reason # # Cancels all tests # function cti_deleteall { typeset cti_lv_ic typeset cti_lv_tp test $# -eq 0 && return for cti_lv_ic in $iclist do for cti_lv_tp in `eval echo \\$$cti_lv_ic` do if test -n "$cti_lv_tp" then echo "Deleted test: $cti_lv_tp" "$@" fi done done } # # cti_reportfile() - write the contents of a file to the journal. # # Usage: cti_reportfile path [title] # # Writes the contents of the file specified by path to the execution results # file, line by line. # function cti_reportfile { typeset cti_lv_path=$1 typeset cti_lv_title="${2:-$cti_lv_path}" typeset cti_lv_line cti_lf_checkargs $# 1 GE cti_reportfile || return cti_report "+++ $cti_lv_title +++" /usr/bin/cat $cti_lv_path cti_report "+++ end +++" cti_report " " } # # cti_rmf() - remove files. # # Usage: cti_rmf [files...] # # Calls "rm -f" to remove the files, and verifies that they have been removed. # # Returns 0 on success, non-zero if any of the files could not be removed. # function cti_rmf { typeset cti_lv_file for cti_lv_file in "$@" do rm -f "$cti_lv_file" if test -f "$cti_lv_file" then cti_unresolved "Error removing file \"$cti_lv_file\"" return 1 fi done return 0 } # # cti_writefile() - write contents of a file. # # Usage: cti_writefile path mode lines... # # Truncates the file specified by path and then writes the third and # subsequent arguments to the specified file as separate lines. # # Returns 0 on success, non-zero if any of the files could not be removed. # function cti_writefile { cti_lf_checkargs $# 3 GE cti_writefile || return 1 cti_rmf "$1" || return 1 cti_appendfile "$@" } # # cti_appendfile() - append to contents of a file. # # Usage: cti_appendfile path mode lines... # # Appends the third and subsequent arguments to the specified file as separate # lines. # # Returns 0 on success, non-zero if any of the files could not be removed. # function cti_appendfile { typeset cti_lv_path="$1" typeset cti_lv_mode="$2" typeset cti_lv_line cti_lf_checkargs $# 3 GE cti_appendfile || return 1 shift 2 for cti_lv_line in "$@" do echo "$cti_lv_line" >> "$cti_lv_path" if [[ $? -ne 0 ]] then cti_unresolved \ "Error writing to file \"$cti_lv_path\"" return 1 fi done cti_execute UNRESOLVED chmod "$cti_lv_mode" "$cti_lv_path" return $? } # # cti_execute() - execute a command # # Usage: cti_execute [-c out|err] [-i input] result cmd [args...] # # Executes a command. The standard output is redirected to the file cti_stdout # and the standard error is redirected to the file cti_stderr. # # If the command has a non-zero exit status, cti_execute() registers a result # code of `result'. # # Options: # -c out|err Check standard output/error. If anything is written to # the specified output channel, a result code of `result' # is registered and the output written to the journal. # May have multiple -c options. # -i input Use -i as an input line to the command. # May have multiple -i options. # # Returns 0 on success, non-zero on failure (returning the # exit status from the command when possible). # function cti_execute { typeset cti_lv_opt typeset -i cti_lv_checkstdout=0 typeset -i cti_lv_checkstderr=0 typeset cti_lv_result typeset -i cti_lv_status typeset -i cti_lv_rv=0 # Remove files used for redirecting standard I/O. cti_rmf cti_stdin cti_stdout cti_stderr || return 1 # Create (empty) files to take standard output and error so there are # no problems later when we come to run the command. touch cti_stdout cti_stderr if [[ $? -ne 0 ]] then cti_unresolved "Error creating files cti_stdout and cti_stderr" return 1 fi # Parse command line options while getopts "c:i:l:s:" cti_lv_opt do case $cti_lv_opt in c) case "$OPTARG" in out|err) eval cti_lv_checkstd$OPTARG=1 ;; *) cti_unresolved "cti_execute() called with" \ "bad option argument -c $OPTARG" return 1 ;; esac ;; i) echo "$OPTARG" >> cti_stdin if [[ $? -ne 0 ]] then cti_unresolved "Error writing to cti_stdin" return 1 fi ;; *) cti_unresolved "cti_execute() called with illegal" \ "option $cti_lv_opt" return 1 ;; esac done shift $((OPTIND-1)) # Verify the correct number of arguments were passed. cti_lf_checkargs $# 2 GE cti_execute || return 1 # First (non-option) argument is the result code to use if the command # fails. cti_lv_result="${1:-UNRESOLVED}" shift # Execute the command, redirecting standard input if required. if test -f cti_stdin then eval "$@" < cti_stdin > cti_stdout 2> cti_stderr else eval "$@" > cti_stdout 2> cti_stderr fi cti_lv_status=$? # Check the exit status of the command if test $cti_lv_status -ne 0 then if [[ "$cti_lv_result" = "CTI" ]] then cti_report CTI "Command \"$*\" failed "\ "with status $cti_lv_status" else cti_result "$cti_lv_result"\ "Command \"$*\" failed "\ "with status $cti_lv_status" cti_lv_rv=$cti_lv_status fi fi # Always log output of cti_execute_cmd if [[ "$cti_lv_result" = "CTI" ]] then if test -s cti_stdout then cti_reportfile cti_stdout "Standard output from command \"$*\"" fi if test -s cti_stderr then cti_reportfile cti_stderr "Standard error from command \"$*\"" fi return $cti_lv_status fi # If cmd failed, or if "-c err", check standard error. if test \( $cti_lv_rv -ne 0 -o $cti_lv_checkstderr -eq 1 \) \ -a -s cti_stderr then cti_result "$cti_lv_result" \ "Command \"$*\" produced standard error" cti_reportfile cti_stderr "Standard error from command \"$*\"" [[ $cti_lv_rv = 0 ]] && cti_lv_rv=1 fi # If cmd failed, or if "-c out", check standard output. if test \( $cti_lv_rv -ne 0 -o $cti_lv_checkstdout -eq 1 \) \ -a -s cti_stdout then cti_result "$cti_lv_result" \ "Command \"$*\" produced standard output" cti_reportfile cti_stdout "Standard output from command \"$*\"" [[ $cti_lv_rv = 0 ]] && cti_lv_rv=1 fi return $cti_lv_rv } # # Exit values for expect scripts. # N.B. Do not use 0, as expect uses this for e.g. syntax errors. # typeset -ri CTI_EXP_RV_TIMEDOUT=1 typeset -ri CTI_EXP_RV_TESTFAIL=2 typeset -ri CTI_EXP_RV_OK=3 # # cti_runexpect() - run the expect utility. # # Usage: cti_runexpect failtext command pattern action [pattern action ...] # # Executes the expect utility using the command line specified in the second # argument. Generates a temporary expect script which is removed at the end of # the call. The arguments following the second are pattern-action pairs. The # pattern can be a regular expression or "CALL", which indicates the action is # simply a function call which is unconditionally executed at that point. # # The following expect functions are available: # # startcritical Indicates that failures from this point onwards # constitute a test failure. In that case the # ``failtext'' argument is used to report the error # message. # endcritical Indicates the end of the test failure section begun by # startcritical. # finish Exit the expect script here - the test has passed. # # Returns 0 on success, non-zero on failure. # function cti_runexpect { typeset -ri CTI_EXP_TIMEOUT=5 typeset -r cti_lv_expfile="./cti_$tc_id-$$.exp" typeset cti_lv_failtext="$1" typeset cti_lv_command="$2" typeset -i cti_lv_rv=0 typeset cti_lv_dopt # Verify the correct number of arguments were passed. cti_lf_checkargs $# 4 GE cti_runexpect || return 1 shift 2 # Generate expect script. { echo "set STATUS_OK $CTI_EXP_RV_OK" echo "set STATUS_FAIL $CTI_EXP_RV_TESTFAIL" echo "set STATUS_TIMEDOUT $CTI_EXP_RV_TIMEDOUT" echo '' echo "set timeout $CTI_EXP_TIMEOUT" echo 'set retval $STATUS_TIMEDOUT' echo '' echo 'eval spawn [lrange $argv 0 end]' echo '' echo 'proc startcritical {} {' echo ' global retval' echo ' global STATUS_FAIL' echo ' set retval $STATUS_FAIL' echo '}' echo '' echo 'proc endcritical {} {' echo ' global retval' echo ' global STATUS_TIMEDOUT' echo ' set retval $STATUS_TIMEDOUT' echo '}' echo '' echo 'proc finish {} {' echo ' global STATUS_OK' echo ' exit $STATUS_OK' echo '}' while test $# -gt 1 do echo '' if test "$1" = "CALL" then echo "$2" else echo 'expect {' echo " -re \"$1\" {" echo " $2 " echo ' }' echo ' timeout {' echo ' exit $retval' echo ' }' echo '}' fi shift 2 done } > $cti_lv_expfile # Check that there were no errors writing to the script file. if test $? -ne 0 then cti_unresolved "Error writing expect script to file" \ "\"$cti_lv_expfile\"" return 1 fi # If debug is on, turn on expect debug flag. case "$CTI_SHELL_DEBUG" in y*|Y*) cti_lv_dopt='-d' ;; esac # Execute expect using generated script. expect $cti_lv_dopt -f $cti_lv_expfile $cti_lv_command > cti_expout 2>&1 cti_lv_rv=$? # If debug is on, save expect script and output, otherwise remove # script. case "$CTI_SHELL_DEBUG" in y*|Y*) cp cti_expout ${cti_lv_expfile%.exp}.out cti_report "DEBUG: expect script is $PWD/$cti_lv_expfile," \ "expect output is in $PWD/${cti_lv_expfile%.exp}.out" ;; *) rm -f $cti_lv_expfile esac # Deal with return value from expect. case $cti_lv_rv in $CTI_EXP_RV_OK) return 0 ;; $CTI_EXP_RV_TIMEDOUT) cti_unresolved "Expect script timed-out during test setup" ;; $CTI_EXP_RV_TESTFAIL) cti_fail "$cti_lv_failtext" ;; *) cti_unresolved "Test coding error or expect bug:" \ "unexpected exit status $cti_lv_rv from expect script" cti_reportfile cti_expout "Output from expect" ;; esac return 1 } # # cti_expecttest() - run the expect utility. # # Usage: cti_expecttest failtext command pattern action [pattern action ...] # # Common test function for test purposes which use expect. Like cti_runexpect() # except that this is written with path tracing and is designed to do the # complete work of a test purpose. # function cti_expecttest { # Verify the correct number of arguments were passed. cti_lf_checkargs $# 4 GE cti_expecttest || return # Use cti_runexpect() to execute expect utililty. if cti_runexpect "$@" then cti_pathtrace else return fi cti_checkpath 1 } function cti_execute_cmd { cti_execute CTI "$@" } # # "private" functions for internal use by cti_cancel function # used to replace test purpose functions which will be canceled. # function cancel_ic { cti_result $cticancel_result "$cticancel_msg" } # # cti_cancel() - cancel an individual test purpose. # # Usage: cti_cancel tp reason [test result] # # Cancels an individual test by replace the tp function with # cancel_ic function # function cti_cancel { test $# -gt 3 && return test_num=$1 cticancel_msg="$2" cticancel_result=${3:-"UNSUPPORTED"} typeset cti_lv_ic cti_lv_ic_mod typeset cti_lv_tp cti_report "Canceling test $test_num: $cticancel_msg" # # translate the ic and point the test purpose ic to # cancel_ic function # for cti_lv_ic in $iclist do cti_lv_ic_mod="" for cti_lv_tp in `eval echo \\$$cti_lv_ic` do if [ X"$cti_lv_tp" == X"$test_num" ]; then cti_lv_ic_mod=$cti_lv_ic_mod" cancel_ic" else cti_lv_ic_mod=$cti_lv_ic_mod" $cti_lv_tp" fi done eval "$cti_lv_ic=\"$cti_lv_ic_mod\"" done } # # cti_cancelall() - cancel all tests. # # Usage: cti_cancelall reason [test result] # # Cancels all tests by replace the tests functions with cancel_ic function # function cti_cancelall { typeset cti_lv_ic typeset cti_lv_tp cticancel_msg=$1 cticancel_result=${2:-"UNSUPPORTED"} test $# -eq 0 && return for cti_lv_ic in $iclist do for cti_lv_tp in `eval echo \\$$cti_lv_ic` do cti_cancel $cti_lv_tp "$cticancel_msg" \ $cticancel_result done done } # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2007 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # Copyright (c) 2012, 2016 by Delphix. All rights reserved. # . ${STF_TOOLS}/include/stf.shlib # Output an assertion # # $@ - assertion text function log_assert { _printline ASSERTION: "$@" } # Output a comment # # $@ - comment text function log_note { _printline NOTE: "$@" } # Execute and print command with status where success equals non-zero result # # $@ - command to execute # # return 0 if command fails, otherwise return 1 function log_neg { log_neg_expect "" "$@" return $? } # Execute a positive test and exit $STF_FAIL is test fails # # $@ - command to execute function log_must { log_pos "$@" (( $? != 0 )) && log_fail } # Execute a negative test and exit $STF_FAIL if test passes # # $@ - command to execute function log_mustnot { log_neg "$@" (( $? != 0 )) && log_fail } # Execute a positive test but retry the command on failure if the output # matches an expected pattern. Otherwise behave like log_must and exit # $STF_FAIL is test fails. # # $1 - retry keyword # $2 - retry attempts # $3-$@ - command to execute # function log_must_retry { typeset out="" typeset logfile="/tmp/log.$$" typeset status=1 typeset expect=$1 typeset retry=$2 typeset delay=1 shift 2 while [[ -e $logfile ]]; do logfile="$logfile.$$" done while (( $retry > 0 )); do "$@" 2>$logfile status=$? out="cat $logfile" if (( $status == 0 )); then $out | egrep -i "internal error|assertion failed" \ > /dev/null 2>&1 # internal error or assertion failed if [[ $? -eq 0 ]]; then print -u2 $($out) _printerror "$@" "internal error or" \ " assertion failure exited $status" status=1 else [[ -n $LOGAPI_DEBUG ]] && print $($out) _printsuccess "$@" fi break else $out | grep -i "$expect" > /dev/null 2>&1 if (( $? == 0 )); then print -u2 $($out) _printerror "$@" "Retry in $delay seconds" sleep $delay (( retry=retry - 1 )) (( delay=delay * 2 )) else break; fi fi done if (( $status != 0 )) ; then print -u2 $($out) _printerror "$@" "exited $status" fi _recursive_output $logfile "false" return $status } # Execute a positive test and exit $STF_FAIL is test fails after being # retried up to 5 times when the command returns the keyword "busy". # # $@ - command to execute function log_must_busy { log_must_retry "busy" 5 "$@" (( $? != 0 )) && log_fail } # Execute a negative test with keyword expected, and exit # $STF_FAIL if test passes # # $1 - keyword expected # $2-$@ - command to execute function log_mustnot_expect { log_neg_expect "$@" (( $? != 0 )) && log_fail } # Execute and print command with status where success equals non-zero result # or output includes expected keyword # # $1 - keyword expected # $2-$@ - command to execute # # return 0 if command fails, or the output contains the keyword expected, # return 1 otherwise function log_neg_expect { typeset out="" typeset logfile="/tmp/log.$$" typeset ret=1 typeset expect=$1 shift while [[ -e $logfile ]]; do logfile="$logfile.$$" done "$@" 2>$logfile typeset status=$? out="/usr/bin/cat $logfile" # unexpected status if (( $status == 0 )); then print -u2 $($out) _printerror "$@" "unexpectedly exited $status" # missing binary elif (( $status == 127 )); then print -u2 $($out) _printerror "$@" "unexpectedly exited $status (File not found)" # bus error - core dump elif (( $status == 138 )); then print -u2 $($out) _printerror "$@" "unexpectedly exited $status (Bus Error)" # segmentation violation - core dump elif (( $status == 139 )); then print -u2 $($out) _printerror "$@" "unexpectedly exited $status (SEGV)" else $out | /usr/bin/egrep -i "internal error|assertion failed" \ > /dev/null 2>&1 # internal error or assertion failed if (( $? == 0 )); then print -u2 $($out) _printerror "$@" "internal error or assertion failure" \ " exited $status" elif [[ -n $expect ]] ; then $out | /usr/bin/grep -i "$expect" > /dev/null 2>&1 if (( $? == 0 )); then ret=0 else print -u2 $($out) _printerror "$@" "unexpectedly exited $status" fi else ret=0 fi if (( $ret == 0 )); then [[ -n $LOGAPI_DEBUG ]] && print $($out) _printsuccess "$@" "exited $status" fi fi _recursive_output $logfile "false" return $ret } # Execute and print command with status where success equals zero result # # $@ command to execute # # return command exit status function log_pos { typeset out="" typeset logfile="/tmp/log.$$" while [[ -e $logfile ]]; do logfile="$logfile.$$" done "$@" 2>$logfile typeset status=$? out="/usr/bin/cat $logfile" if (( $status != 0 )) ; then print -u2 $($out) _printerror "$@" "exited $status" else $out | /usr/bin/egrep -i "internal error|assertion failed" \ > /dev/null 2>&1 # internal error or assertion failed if [[ $? -eq 0 ]]; then print -u2 $($out) _printerror "$@" "internal error or assertion failure" \ " exited $status" status=1 else [[ -n $LOGAPI_DEBUG ]] && print $($out) _printsuccess "$@" fi fi _recursive_output $logfile "false" return $status } # Set an exit handler # # $@ - function(s) to perform on exit function log_onexit { _CLEANUP="$@" } # # Exit functions # # Perform cleanup and exit $STF_PASS # # $@ - message text function log_pass { _endlog $STF_PASS "$@" } # Perform cleanup and exit $STF_FAIL # # $@ - message text function log_fail { _endlog $STF_FAIL "$@" } # Perform cleanup and exit $STF_UNRESOLVED # # $@ - message text function log_unresolved { _endlog $STF_UNRESOLVED "$@" } # Perform cleanup and exit $STF_NOTINUSE # # $@ - message text function log_notinuse { _endlog $STF_NOTINUSE "$@" } # Perform cleanup and exit $STF_UNSUPPORTED # # $@ - message text function log_unsupported { _endlog $STF_UNSUPPORTED "$@" } # Perform cleanup and exit $STF_UNTESTED # # $@ - message text function log_untested { _endlog $STF_UNTESTED "$@" } # Perform cleanup and exit $STF_UNINITIATED # # $@ - message text function log_uninitiated { _endlog $STF_UNINITIATED "$@" } # Perform cleanup and exit $STF_NORESULT # # $@ - message text function log_noresult { _endlog $STF_NORESULT "$@" } # Perform cleanup and exit $STF_WARNING # # $@ - message text function log_warning { _endlog $STF_WARNING "$@" } # Perform cleanup and exit $STF_TIMED_OUT # # $@ - message text function log_timed_out { _endlog $STF_TIMED_OUT "$@" } # Perform cleanup and exit $STF_OTHER # # $@ - message text function log_other { _endlog $STF_OTHER "$@" } # # Internal functions # # Execute custom callback scripts on test failure # # callback script paths are stored in TESTFAIL_CALLBACKS, delimited by ':'. function _execute_testfail_callbacks { typeset callback print "$TESTFAIL_CALLBACKS:" | while read -d ":" callback; do if [[ -n "$callback" ]] ; then log_note "Performing test-fail callback ($callback)" $callback fi done } # Perform cleanup and exit # # $1 - stf exit code # $2-$n - message text function _endlog { typeset logfile="/tmp/log.$$" _recursive_output $logfile if [[ $1 == $STF_FAIL ]] ; then _execute_testfail_callbacks fi if [[ -n $_CLEANUP ]] ; then typeset cleanup=$_CLEANUP log_onexit "" log_note "Performing local cleanup via log_onexit ($cleanup)" $cleanup fi typeset exitcode=$1 shift (( ${#@} > 0 )) && _printline "$@" exit $exitcode } # Output a formatted line # # $@ - message text function _printline { print "$@" } # Output an error message # # $@ - message text function _printerror { _printline ERROR: "$@" } # Output a success message # # $@ - message text function _printsuccess { _printline SUCCESS: "$@" } # Output logfiles recursively # # $1 - start file # $2 - indicate whether output the start file itself, default as yes. function _recursive_output #logfile { typeset logfile=$1 while [[ -e $logfile ]]; do if [[ -z $2 || $logfile != $1 ]]; then /usr/bin/cat $logfile fi /usr/bin/rm -f $logfile logfile="$logfile.$$" done } # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright (c) 2012 by Delphix. All rights reserved. # include $(SRC)/Makefile.master include $(SRC)/test/Makefile.com ROOTOPTPKG = $(ROOT)/opt/test-runner INCLUDEDIR = $(ROOTOPTPKG)/stf/include PROGS = stf.shlib CMDS = $(PROGS:%=$(INCLUDEDIR)/%) $(CMDS) : FILEMODE = 0555 all lint clean clobber: install: $(CMDS) $(CMDS): $(INCLUDEDIR) $(INCLUDEDIR): $(INS.dir) $(INCLUDEDIR)/%: % $(INS.file) # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2007 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # Copyright (c) 2012 by Delphix. All rights reserved. # STF_PASS=0 STF_FAIL=1 STF_UNRESOLVED=2 STF_NOTINUSE=3 STF_UNSUPPORTED=4 STF_UNTESTED=5 STF_UNINITIATED=6 STF_NORESULT=7 STF_WARNING=8 STF_TIMED_OUT=9 STF_OTHER=10 # do this to use the names: eval echo \$STF_RESULT_NAME_${result} STF_RESULT_NAME_0="PASS" STF_RESULT_NAME_1="FAIL" STF_RESULT_NAME_2="UNRESOLVED" STF_RESULT_NAME_3="NOTINUSE" STF_RESULT_NAME_4="UNSUPPORTED" STF_RESULT_NAME_5="UNTESTED" STF_RESULT_NAME_6="UNINITIATED" STF_RESULT_NAME_7="NORESULT" STF_RESULT_NAME_8="WARNING" STF_RESULT_NAME_9="TIMED_OUT" STF_RESULT_NAME_10="OTHER" # do this to use the array: ${STF_RESULT_NAMES[$result]} STF_RESULT_NAMES=( "PASS" "FAIL" "UNRESOLVED" "NOTINUSE" "UNSUPPORTED" \ "UNTESTED" "UNINITIATED" "NORESULT" "WARNING" "TIMED_OUT" "OTHER" )