Skip to content

Commit

Permalink
Don't completely skip a case when excluded, we still want to let repo…
Browse files Browse the repository at this point in the history 
…rter know (closes #10)
  • Loading branch information
@kevinresol
kevinresol committed Feb 22, 2019
1 parent 6de2252 commit a347d1b
Show file tree
Hide file tree
Showing 3 changed files with 88 additions and 44 deletions.
22 changes: 12 additions & 10 deletions src/tink/testrunner/Reporter.hx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ interface Reporter {
enum ReportType {
BatchStart;
SuiteStart(info:SuiteInfo, hasCasesToRun:Bool);
CaseStart(info:CaseInfo);
CaseStart(info:CaseInfo, shouldRun:Bool);
Assertion(assertion:Assertion);
CaseFinish(result:CaseResult);
SuiteFinish(result:SuiteResult);
Expand Down Expand Up @@ -91,11 +91,13 @@ class BasicReporter implements Reporter {
println(formatter.info(info.name));
}

case CaseStart(info):
var m = formatter.info(indent(info.name, 2)) + ': ';
if(info.pos != null) m += formatter.extra('[${info.pos.fileName}:${info.pos.lineNumber}] ');
if(info.description != null) m += formatter.mute(info.description);
println(m);
case CaseStart(info, shouldRun):
if(shouldRun) {
var m = formatter.info(indent(info.name, 2)) + ': ';
if(info.pos != null) m += formatter.extra('[${info.pos.fileName}:${info.pos.lineNumber}] ');
if(info.description != null) m += formatter.mute(info.description);
println(m);
}

case Assertion(assertion):

Expand All @@ -111,11 +113,11 @@ class BasicReporter implements Reporter {
println(m);
if(failure != null) println(formatter.error(indent(failure, 8)));

case CaseFinish({results: results}):
switch results {
case Success(_):
case Failure(e):
case CaseFinish({result: result}):
switch result {
case Failed(e):
println(formatter.error(indent('- ${formatError(e)}', 4)));
case _:
}

case SuiteFinish(result):
Expand Down
80 changes: 48 additions & 32 deletions src/tink/testrunner/Runner.hx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,12 +70,12 @@ class Runner {
function setup() return hasCases ? suite.setup() : Promise.NOISE;
function teardown() return hasCases ? suite.teardown() : Promise.NOISE;

var iter = cases.iterator();
var iter = suite.cases.iterator();
var results = [];
function next() {
if(iter.hasNext()) {
var caze = iter.next();
runCase(caze, suite, reporter, timers).handle(function(r) {
runCase(caze, suite, reporter, timers, includeMode).handle(function(r) {
results.push(r);
next();
});
Expand All @@ -97,38 +97,46 @@ class Runner {
});
}

static function runCase(caze:Case, suite:Suite, reporter:Reporter, timers:TimerManager):Future<CaseResult> {
static function runCase(caze:Case, suite:Suite, reporter:Reporter, timers:TimerManager, includeMode:Bool):Future<CaseResult> {
return Future.async(function(cb) {
reporter.report(CaseStart(caze.info)).handle(function(_) {

suite.before().timeout(caze.timeout, timers, caze.pos)
.next(function(_) {
var assertions = [];
return caze.execute()
#if pure .forEach #else .forEachAsync #end(function(a) {
assertions.push(a);
return reporter.report(Assertion(a)).map(function(_) return #if pure Resume #else true #end);
})
#if pure
.next(function(o):Outcome<Array<Assertion>, Error> return switch o {
case Depleted: Success(assertions);
case Halted(_): throw 'unreachable';
case Failed(e): Failure(e);
})
#else
.next(function(_) return assertions)
#end
.timeout(caze.timeout, timers);
})
.next(function(result) return suite.after().timeout(caze.timeout, timers, caze.pos).next(function(_) return result))
.handle(function(result) {
if(caze.shouldRun(includeMode)) {
reporter.report(CaseStart(caze.info, true)).handle(function(_) {
suite.before().timeout(caze.timeout, timers, caze.pos)
.next(function(_) {
var assertions = [];
return caze.execute().forEach(function(a) {
assertions.push(a);
return reporter.report(Assertion(a)).map(function(_) return Resume);
})
.next(function(o):Outcome<Array<Assertion>, Error> return switch o {
case Depleted: Success(assertions);
case Halted(_): throw 'unreachable';
case Failed(e): Failure(e);
})
.timeout(caze.timeout, timers);
})
.next(function(result) return suite.after().timeout(caze.timeout, timers, caze.pos).next(function(_) return result))
.handle(function(result) {
var results = {
info: caze.info,
result: switch result {
case Success(v): Succeeded(v);
case Failure(e): Failed(e);
},
}
reporter.report(CaseFinish(results)).handle(function(_) cb(results));
});
});
} else {
reporter.report(CaseStart(caze.info, false))
.handle(function(_) {
var results = {
info: caze.info,
results: result,
result: Excluded,
}
reporter.report(CaseFinish(results)).handle(function(_) cb(results));
});
});
}
});
}

Expand Down Expand Up @@ -163,15 +171,17 @@ abstract BatchResult(Array<SuiteResult>) from Array<SuiteResult> to Array<SuiteR
};

function handleCases(cases:Array<CaseResult>)
for(c in cases) switch c.results {
case Success(assertions):
for(c in cases) switch c.result {
case Succeeded(assertions):
ret.assertions = ret.assertions.concat(assertions);
ret.failures = ret.failures.concat(
assertions.filter(function(a) return !a.holds)
.map(function(a) return AssertionFailed(a))
);
case Failure(e):
case Failed(e):
ret.failures.push(CaseFailed(e));
case Excluded:
// do nothing
}

for(s in this) switch s.result {
Expand All @@ -195,7 +205,7 @@ typedef SuiteResult = {

typedef CaseResult = {
info:CaseInfo,
results:Outcome<Array<Assertion>, Error>,
result:CaseResultType,
}

enum SuiteResultType {
Expand All @@ -204,6 +214,12 @@ enum SuiteResultType {
TeardownFailed(e:Error, cases:Array<CaseResult>);
}

enum CaseResultType {
Succeeded(assertions:Array<Assertion>);
Failed(e:Error);
Excluded;
}

enum FailureType {
AssertionFailed(assertion:Assertion);
CaseFailed(err:Error);
Expand Down
30 changes: 28 additions & 2 deletions tests/RunTests.hx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@ import tink.testrunner.*;
import tink.testrunner.Assertion.*;
import tink.testrunner.Case;
import tink.testrunner.Suite;
import tink.testrunner.Reporter;
import travix.Logger.*;

using tink.CoreApi;

using Lambda;

class RunTests {
static function main() {
Expand Down Expand Up @@ -49,6 +50,7 @@ class RunTests {
);

// Test: empty suite (reporter should not print the empty suite)
var reporter = new MemoryReporter();
futures.push(
function() return Runner.run([
new BasicSuite({name: 'SingleSuite'}, [
Expand All @@ -57,8 +59,20 @@ class RunTests {
new BasicSuite({name: 'EmptySuite'}, [
new ExcludedCase(),
]),
]).map(function(result) {
new BasicSuite({name: 'MixedSuite'}, [
single,
new ExcludedCase(),
]),
], reporter).map(function(result) {
assertEquals(0, result.summary().failures.length);
assertEquals(3, reporter.logs.filter(function(t) return t.match(SuiteStart(_))).length);
assertEquals(true, reporter.logs.exists(function(t) return t.match(SuiteStart({name: 'SingleSuite'}, true))));
assertEquals(true, reporter.logs.exists(function(t) return t.match(SuiteStart({name: 'EmptySuite'}, false))));
assertEquals(true, reporter.logs.exists(function(t) return t.match(SuiteStart({name: 'MixedSuite'}, true))));
assertEquals(2, reporter.logs.filter(function(t) return t.match(CaseStart({name: 'SingleCase'}, true))).length);
assertEquals(2, reporter.logs.filter(function(t) return t.match(CaseStart({name: 'ExcludedCase'}, false))).length);
assertEquals(2, reporter.logs.filter(function(t) return t.match(CaseFinish({info: {name: 'SingleCase'}, result: Succeeded(_)}))).length);
assertEquals(2, reporter.logs.filter(function(t) return t.match(CaseFinish({info: {name: 'ExcludedCase'}, result: Excluded}))).length);
return Noise;
})
);
Expand Down Expand Up @@ -118,4 +132,16 @@ class ExcludedCase extends BasicCase {
override function execute():Assertions {
return new Assertion(true, 'Dummy');
}
}

class MemoryReporter implements Reporter {

public var logs:Array<ReportType> = [];

public function new() {}

public function report(type:ReportType):Future<Noise> {
logs.push(type);
return Future.NOISE;
}
}

0 comments on commit a347d1b

Please sign in to comment.