Skip to content

New feature - Add basic support for JUnit reports #64

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
scyzoryck wants to merge 2 commits into
base: master
Choose a base branch
from
Open

New feature - Add basic support for JUnit reports #64

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b286038
Add basic support for JUnit Integration
Oct 10, 2021
ef7252e
Update help message for junit
scyzoryck Oct 21, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions Player/Console/PlayerCommand.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
use Blackfire\Player\Extension\BlackfireExtension;
use Blackfire\Player\Extension\CliFeedbackExtension;
use Blackfire\Player\Extension\DisableInternalNetworkExtension;
use Blackfire\Player\Extension\JUnitExtension;
use Blackfire\Player\Extension\SecurityExtension;
use Blackfire\Player\Extension\TracerExtension;
use Blackfire\Player\Guzzle\CurlFactory;
Expand Down Expand Up @@ -60,6 +61,7 @@ protected function configure()
new InputOption('sandbox', '', InputOption::VALUE_NONE, 'Enable the sandbox mode', null),
new InputOption('ssl-no-verify', '', InputOption::VALUE_NONE, 'Disable SSL certificate verification', null),
new InputOption('blackfire-env', '', InputOption::VALUE_REQUIRED, 'The blackfire environment to use'),
new InputOption('junit', '', InputOption::VALUE_REQUIRED, 'Save output execution report as JUnit into file - in path provided as value'),
Comment thread
scyzoryck marked this conversation as resolved.
Outdated
])
->setDescription('Runs scenario files')
->setHelp('Read https://blackfire.io/docs/builds-cookbooks/player to learn about all supported options.')
Expand All @@ -80,6 +82,12 @@ protected function execute(InputInterface $input, OutputInterface $output)
}

$json = $input->getOption('json');
$junit = $input->getOption('junit');
if ('' !== $junit && !extension_loaded('dom')) {
$output->writeln('<error>To create junit report you need to install DOM extension first.</error>');
$junit = '';
}

$sslNoVerify = $input->getOption('ssl-no-verify');

$clients = [$this->createClient($sslNoVerify)];
Expand Down Expand Up @@ -114,6 +122,10 @@ protected function execute(InputInterface $input, OutputInterface $output)
if ($input->getOption('disable-internal-network')) {
$player->addExtension(new DisableInternalNetworkExtension());
}
if ('' !== $junit) {
$junitExtension = new JUnitExtension();

@iamluc iamluc Oct 15, 2021

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As the XML extension is currently not a requirement of the player, we should add a runtime check and display a nice warning/error message if the extension is missing.

@scyzoryck scyzoryck Oct 20, 2021

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Check added in line 86.

$player->addExtension($junitExtension);
}

$scenarios = (new ScenarioHydrator())->hydrate($input);

Expand Down Expand Up @@ -160,6 +172,10 @@ protected function execute(InputInterface $input, OutputInterface $output)
]));
}

if ('' !== $junit) {
file_put_contents($junit, $junitExtension->getXml());
}

return $exitCode;
}

Expand Down
172 changes: 172 additions & 0 deletions Player/Extension/JUnitExtension.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
<?php

/*
* This file is part of the Blackfire Player package.
*
* (c) Fabien Potencier <fabien@blackfire.io>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/

namespace Blackfire\Player\Extension;

use Blackfire\Player\Context;
use Blackfire\Player\Exception\ExpectationFailureException;
use Blackfire\Player\Result;
use Blackfire\Player\Results;
use Blackfire\Player\Scenario;
use Blackfire\Player\ScenarioSet;
use Blackfire\Player\Step\AbstractStep;
use Blackfire\Player\Step\Step;
use Psr\Http\Message\RequestInterface;
use Psr\Http\Message\ResponseInterface;

/**
* @author Marcin Czarnecki <scyzoryck@gmail.com>
*/
class JUnitExtension extends AbstractExtension
{
/**
* @var \DOMDocument
*/
private $document;

/**
* @var \DOMElement
*/
private $currentScenarioSet;

private $scenarioSetErrorCount = 0;

private $scenarioSetFailureCount = 0;

private $scenarioSetTestsCount = 0;

/**
* @var \DOMElement
*/
private $currentScenario;

private $scenarioErrorCount = 0;

private $scenarioFailureCount = 0;

private $scenarioTestsCount = 0;

/**
* @var \DOMElement
*/
private $currentStep;

public function __construct()
{
$this->document = new \DOMDocument('1.0', 'UTF-8');
$this->document->formatOutput = true;
}

public function enterScenarioSet(ScenarioSet $scenarios, $concurrency)
{
$this->currentScenarioSet = $this->document->createElement('testsuites');
$this->currentScenarioSet->setAttribute('name', $scenarios->getName());
$this->document->appendChild($this->currentScenarioSet);
}

public function enterScenario(Scenario $scenario, Context $context)
{
$this->currentScenario = $this->document->createElement('testsuite');
$this->currentScenario->setAttribute('name', $scenario->getName());
$this->currentScenarioSet->appendChild($this->currentScenario);
}

public function enterStep(AbstractStep $step, RequestInterface $request, Context $context)
{
$this->currentStep = $this->document->createElement('testcase');
$this->currentStep->setAttribute('name', $step->getName());
$this->currentScenario->appendChild($this->currentStep);

if ($step instanceof Step) {
$assertionsCount = \count($step->getExpectations()) + \count($step->getAssertions());
} else {
$assertionsCount = 0;
}

$this->currentStep->setAttribute('assertions', $assertionsCount);

++$this->scenarioTestsCount;
++$this->scenarioSetTestsCount;

return $request;
}

public function leaveStep(
AbstractStep $step,
RequestInterface $request,
ResponseInterface $response,
Context $context
) {
foreach ($step->getErrors() as $failedAssertion) {
$failure = $this->document->createElement('failure');
$failure->setAttribute('message', $failedAssertion);
$failure->setAttribute('type', 'performance assertion');
$this->currentStep->appendChild($failure);

++$this->scenarioFailureCount;
++$this->scenarioSetFailureCount;
}

return $response;
}

public function abortStep(AbstractStep $step, RequestInterface $request, \Exception $exception, Context $context)
{
if ($exception instanceof ExpectationFailureException) {
$failure = $this->document->createElement('failure');
$failure->setAttribute('message', $exception->getMessage());
$failure->setAttribute('type', 'expectation');
$this->currentStep->appendChild($failure);

++$this->scenarioFailureCount;
++$this->scenarioSetFailureCount;

return;
}

++$this->scenarioErrorCount;
++$this->scenarioSetErrorCount;

$error = $this->document->createElement('error');
$error->setAttribute('message', $exception->getMessage());
$error->setAttribute('type', \get_class($exception));
$this->currentStep->appendChild($error);
}

public function leaveScenario(Scenario $scenario, Result $result, Context $context)
{
$this->currentScenario->setAttribute('errors', $this->scenarioErrorCount);
$this->scenarioErrorCount = 0;

$this->currentScenario->setAttribute('failures', $this->scenarioFailureCount);
$this->scenarioFailureCount = 0;

$this->currentScenario->setAttribute('tests', $this->scenarioTestsCount);
$this->scenarioTestsCount = 0;
}

public function leaveScenarioSet(ScenarioSet $scenarios, Results $results)
{
$this->currentScenarioSet->setAttribute('errors', $this->scenarioSetErrorCount);
$this->scenarioSetErrorCount = 0;

$this->currentScenarioSet->setAttribute('failures', $this->scenarioSetFailureCount);
$this->scenarioSetFailureCount = 0;

$this->currentScenarioSet->setAttribute('tests', $this->scenarioSetTestsCount);
$this->scenarioSetTestsCount = 0;
}

public function getXml()
{
return $this->document->saveXML();
}
}
93 changes: 93 additions & 0 deletions Player/Tests/Extension/JUnitExtensionTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
<?php

/*
* This file is part of the Blackfire Player package.
*
* (c) Fabien Potencier <fabien@blackfire.io>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/

namespace Blackfire\Player\Tests\Extension;

use Blackfire\Player\Context;
use Blackfire\Player\Exception\ExpectationFailureException;
use Blackfire\Player\Extension\JUnitExtension;
use Blackfire\Player\Result;
use Blackfire\Player\Results;
use Blackfire\Player\Scenario;
use Blackfire\Player\ScenarioSet;
use Blackfire\Player\Step\Step;
use PHPUnit\Framework\TestCase;
use Psr\Http\Message\RequestInterface;
use Psr\Http\Message\ResponseInterface;

class JUnitExtensionTest extends TestCase
{
public function testCreatingReport()
{
$extension = new JUnitExtension();
$set = new ScenarioSet();

$scenario = new Scenario('scenario 1');
$scenario->name('Example scenario');
$stepSucceed = new Step();
$stepSucceed->name('Successful step');
$stepSucceed->assert('main.wall_time < 50ms');
$stepSucceed->expect('status_code() == 200');

$stepFailedAssertion = new Step();
$stepFailedAssertion->name('Assertion failed');
$stepFailedAssertion->assert('main.wall_time < 5ms');
$stepFailedAssertion->addError('Example assertion error');

$stepFailedExpectation = new Step();
$stepFailedExpectation->name('Expectation failed');
$stepFailedExpectation->expect('status_code() == 400');
$expectationException = new ExpectationFailureException('Expectation failed');

$stepFailedException = new Step();
$stepFailedException->name('Other exception');
$exception = new \Exception('Some exception');

$request = $this->createMock(RequestInterface::class);
$response = $this->createMock(ResponseInterface::class);
$context = $this->createMock(Context::class);
$result = $this->createMock(Result::class);
$results = $this->createMock(Results::class);

$extension->enterScenarioSet($set, 1);
$extension->enterScenario($scenario, $context);
$extension->enterStep($stepSucceed, $request, $context);
$extension->leaveStep($stepSucceed, $request, $response, $context);
$extension->enterStep($stepFailedAssertion, $request, $context);
$extension->leaveStep($stepFailedAssertion, $request, $response, $context);
$extension->enterStep($stepFailedExpectation, $request, $context);
$extension->abortStep($stepFailedExpectation, $request, $expectationException, $context);
$extension->enterStep($stepFailedException, $request, $context);
$extension->abortStep($stepFailedException, $request, $exception, $context);
$extension->leaveScenario($scenario, $result, $context);
$extension->leaveScenarioSet($set, $results);

self::assertSame(<<<'XML'
<?xml version="1.0" encoding="UTF-8"?>
<testsuites name="" errors="1" failures="2" tests="4">
<testsuite name="Example scenario" errors="1" failures="2" tests="4">
<testcase name="Successful step" assertions="2"/>
<testcase name="Assertion failed" assertions="1">
<failure message="Example assertion error" type="performance assertion"/>
</testcase>
<testcase name="Expectation failed" assertions="1">
<failure message="Expectation failed" type="expectation"/>
</testcase>
<testcase name="Other exception" assertions="0">
<error message="Some exception" type="Exception"/>
</testcase>
</testsuite>
</testsuites>

XML
, $extension->getXml());
}
}