-
Notifications
You must be signed in to change notification settings - Fork 430
Enforce console output checks in core unit tests #4073
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Closed
Closed
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
f7fedd6
Fail tests when console output occurs
shai-almog 6bad8f7
Fix AnnotatedElement import
shai-almog 21ba216
Enable JUnit 5 extension autodetection for core tests
shai-almog fe9d239
Allow console output for failure listener test
shai-almog 3484807
Stop logging test failures twice
shai-almog 5f8e51f
Mock Log.e during TestRunnerComponent failure test
shai-almog 84100c4
Restore immediate logging for test runner failures
shai-almog File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
21 changes: 21 additions & 0 deletions
21
maven/core-unittests/src/test/java/com/codename1/testing/AllowConsoleOutput.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| package com.codename1.testing; | ||
|
|
||
| import java.lang.annotation.ElementType; | ||
| import java.lang.annotation.Retention; | ||
| import java.lang.annotation.RetentionPolicy; | ||
| import java.lang.annotation.Target; | ||
|
|
||
| /** | ||
| * Indicates that console output produced during a test should be allowed without causing the | ||
| * {@link UnexpectedLogExtension} to fail the test. This can be applied to individual test methods or | ||
| * entire test classes when console output is part of the expected behaviour under test. | ||
| */ | ||
| @Retention(RetentionPolicy.RUNTIME) | ||
| @Target({ElementType.TYPE, ElementType.METHOD}) | ||
| public @interface AllowConsoleOutput { | ||
| /** | ||
| * When set to {@code true}, console output is permitted. The attribute exists to allow future | ||
| * flexibility should tests wish to dynamically enable or disable console output. | ||
| */ | ||
| boolean value() default true; | ||
| } |
154 changes: 154 additions & 0 deletions
154
maven/core-unittests/src/test/java/com/codename1/testing/UnexpectedLogExtension.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,154 @@ | ||
| package com.codename1.testing; | ||
|
|
||
| import java.io.ByteArrayOutputStream; | ||
| import java.io.OutputStream; | ||
| import java.io.PrintStream; | ||
| import java.lang.annotation.AnnotatedElement; | ||
| import java.nio.charset.StandardCharsets; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
|
|
||
| import org.junit.jupiter.api.extension.AfterEachCallback; | ||
| import org.junit.jupiter.api.extension.BeforeEachCallback; | ||
| import org.junit.jupiter.api.extension.ExtensionContext; | ||
|
|
||
| /** | ||
| * Captures console output generated during each test execution and fails the test if any unexpected | ||
| * messages are produced. This helps surface silent failures that only manifest via stack traces or | ||
| * log statements printed to {@code System.out} or {@code System.err}. | ||
| */ | ||
| public class UnexpectedLogExtension implements BeforeEachCallback, AfterEachCallback { | ||
| private static final ExtensionContext.Namespace NAMESPACE = | ||
| ExtensionContext.Namespace.create(UnexpectedLogExtension.class); | ||
|
|
||
| @Override | ||
| public void beforeEach(ExtensionContext context) { | ||
| CapturedStreams captured = new CapturedStreams(System.out, System.err); | ||
| getStore(context).put(context.getUniqueId(), captured); | ||
|
|
||
| System.setOut(captured.createInterceptingStream(captured.originalOut, captured.capturedOut)); | ||
| System.setErr(captured.createInterceptingStream(captured.originalErr, captured.capturedErr)); | ||
| } | ||
|
|
||
| @Override | ||
| public void afterEach(ExtensionContext context) { | ||
| CapturedStreams captured = | ||
| getStore(context).remove(context.getUniqueId(), CapturedStreams.class); | ||
| if (captured == null) { | ||
| return; | ||
| } | ||
|
|
||
| System.setOut(captured.originalOut); | ||
| System.setErr(captured.originalErr); | ||
|
|
||
| if (isOutputAllowed(context)) { | ||
| return; | ||
| } | ||
|
|
||
| List<String> problems = new ArrayList<>(); | ||
| String stdout = new String(captured.capturedOut.toByteArray(), StandardCharsets.UTF_8); | ||
| String stderr = new String(captured.capturedErr.toByteArray(), StandardCharsets.UTF_8); | ||
|
|
||
| if (!stdout.trim().isEmpty()) { | ||
| problems.add("System.out:\n" + stdout.trim()); | ||
| } | ||
| if (!stderr.trim().isEmpty()) { | ||
| problems.add("System.err:\n" + stderr.trim()); | ||
| } | ||
|
|
||
| if (!problems.isEmpty()) { | ||
| String message = | ||
| "Unexpected console output detected during test '" | ||
| + context.getDisplayName() | ||
| + "'."; | ||
| throw new AssertionError(message + System.lineSeparator() + System.lineSeparator() | ||
| + String.join(System.lineSeparator() + System.lineSeparator(), problems)); | ||
| } | ||
| } | ||
|
|
||
| private ExtensionContext.Store getStore(ExtensionContext context) { | ||
| return context.getStore(NAMESPACE); | ||
| } | ||
|
|
||
| private boolean isOutputAllowed(ExtensionContext context) { | ||
| if (context.getElement().isPresent() | ||
| && isAnnotatedWithAllow(context.getElement().get())) { | ||
| return true; | ||
| } | ||
| Class<?> testClass = context.getTestClass().orElse(null); | ||
| return testClass != null && isAnnotatedWithAllow(testClass); | ||
| } | ||
|
|
||
| private boolean isAnnotatedWithAllow(AnnotatedElement element) { | ||
| AllowConsoleOutput annotation = element.getAnnotation(AllowConsoleOutput.class); | ||
| return annotation != null && annotation.value(); | ||
| } | ||
|
|
||
| private static class CapturedStreams { | ||
| final PrintStream originalOut; | ||
| final PrintStream originalErr; | ||
| final ByteArrayOutputStream capturedOut = new ByteArrayOutputStream(); | ||
| final ByteArrayOutputStream capturedErr = new ByteArrayOutputStream(); | ||
|
|
||
| CapturedStreams(PrintStream originalOut, PrintStream originalErr) { | ||
| this.originalOut = originalOut; | ||
| this.originalErr = originalErr; | ||
| } | ||
|
|
||
| PrintStream createInterceptingStream(PrintStream original, ByteArrayOutputStream capture) { | ||
| return new PrintStream(new TeeOutputStream(original, capture), true); | ||
| } | ||
| } | ||
|
|
||
| private static class TeeOutputStream extends OutputStream { | ||
| private final OutputStream first; | ||
| private final OutputStream second; | ||
|
|
||
| TeeOutputStream(OutputStream first, OutputStream second) { | ||
| this.first = first; | ||
| this.second = second; | ||
| } | ||
|
|
||
| @Override | ||
| public void write(int b) { | ||
| try { | ||
| first.write(b); | ||
| } catch (Exception ignore) { | ||
| // Swallow exceptions from the original stream to ensure we still capture the output. | ||
| } | ||
| try { | ||
| second.write(b); | ||
| } catch (Exception ignore) { | ||
| // ByteArrayOutputStream should not fail, but swallow just in case. | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void write(byte[] b, int off, int len) { | ||
| try { | ||
| first.write(b, off, len); | ||
| } catch (Exception ignore) { | ||
| // Swallow to ensure output continues to be captured even if the original stream fails. | ||
| } | ||
| try { | ||
| second.write(b, off, len); | ||
| } catch (Exception ignore) { | ||
| // Ignore failures while capturing. | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void flush() { | ||
| try { | ||
| first.flush(); | ||
| } catch (Exception ignore) { | ||
| // Ignore flush failures on the original stream. | ||
| } | ||
| try { | ||
| second.flush(); | ||
| } catch (Exception ignore) { | ||
| // Ignore flush failures on the captured stream. | ||
| } | ||
| } | ||
| } | ||
| } | ||
1 change: 1 addition & 0 deletions
1
...-unittests/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| com.codename1.testing.UnexpectedLogExtension |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The extension unconditionally replaces the JVM‑wide
System.outandSystem.errfor each test run and restores them afterwards. That works when tests execute sequentially, but JUnit 5 allows concurrent execution and surefire can enable it viajunit.jupiter.execution.parallel.enabled=true. With parallel tests, one invocation ofbeforeEachwill overwrite the streams that another test is already using, and whicheverafterEachruns last will restore the wrong streams. This leads to nondeterministic failures and, worse, some tests’ console output escaping the check entirely. Consider synchronizing access or opting out of parallel execution to ensure per‑test isolation.Useful? React with 👍 / 👎.