Skip to content

Get rid of OpenCV dependency #991

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 6, 2018
Merged

Get rid of OpenCV dependency #991

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6ae7a12
Get rid of OpenCV dependency
Aug 4, 2018
6318b4e
Fix constructor
Aug 4, 2018
9c47dfb
Fix linter
Aug 4, 2018
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ dependencies {
compile 'commons-io:commons-io:2.6'
compile 'org.springframework:spring-context:5.0.5.RELEASE'
compile 'org.aspectj:aspectjweaver:1.9.1'
compile 'org.openpnp:opencv:3.2.0-1'

testCompile 'junit:junit:4.12'
testCompile 'org.hamcrest:hamcrest-library:1.3'
Expand Down
177 changes: 36 additions & 141 deletions src/main/java/io/appium/java_client/ScreenshotState.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,29 +16,26 @@

package io.appium.java_client;

import static nu.pattern.OpenCV.loadShared;
import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Optional.ofNullable;

import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.Size;
import org.opencv.imgproc.Imgproc;

import java.awt.AlphaComposite;
import java.awt.Graphics2D;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferByte;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.Optional;
import java.util.Base64;
import java.util.function.Function;
import java.util.function.Supplier;

import javax.imageio.ImageIO;

public class ScreenshotState {
private static final Duration DEFAULT_INTERVAL_MS = Duration.ofMillis(500);

private Optional<BufferedImage> previousScreenshot = Optional.empty();
private Supplier<BufferedImage> stateProvider;
private BufferedImage previousScreenshot;
private final Supplier<BufferedImage> stateProvider;
private final ComparesImages comparator;

private Duration comparisonInterval = DEFAULT_INTERVAL_MS;

Expand Down Expand Up @@ -72,10 +69,15 @@ public class ScreenshotState {
*
* @param stateProvider lambda function, which returns a screenshot for further comparison
*/
public ScreenshotState(Supplier<BufferedImage> stateProvider) {
public ScreenshotState(ComparesImages comparator, Supplier<BufferedImage> stateProvider) {
this.comparator = checkNotNull(comparator);
this.stateProvider = stateProvider;
}

public ScreenshotState(ComparesImages comparator) {
this(comparator, null);
}

/**
* Gets the interval value in ms between similarity verification rounds in <em>verify*</em> methods.
*
Expand Down Expand Up @@ -103,7 +105,7 @@ public ScreenshotState setComparisonInterval(Duration comparisonInterval) {
* @return self instance for chaining
*/
public ScreenshotState remember() {
this.previousScreenshot = Optional.of(stateProvider.get());
this.previousScreenshot = stateProvider.get();
return this;
}

Expand All @@ -116,7 +118,7 @@ public ScreenshotState remember() {
* @return self instance for chaining
*/
public ScreenshotState remember(BufferedImage customInitialState) {
this.previousScreenshot = Optional.of(customInitialState);
this.previousScreenshot = checkNotNull(customInitialState);
return this;
}

Expand All @@ -134,7 +136,7 @@ public static class ScreenshotComparisonError extends RuntimeException {

public static class ScreenshotComparisonTimeout extends RuntimeException {
private static final long serialVersionUID = 6336247721154252476L;
private double currentScore = Double.NaN;
private final double currentScore;

ScreenshotComparisonTimeout(String message, double currentScore) {
super(message);
Expand All @@ -147,17 +149,13 @@ public double getCurrentScore() {
}

private ScreenshotState checkState(Function<Double, Boolean> checkerFunc, Duration timeout) {
return checkState(checkerFunc, timeout, ResizeMode.NO_RESIZE);
}

private ScreenshotState checkState(Function<Double, Boolean> checkerFunc, Duration timeout, ResizeMode resizeMode) {
final LocalDateTime started = LocalDateTime.now();
double score;
do {
final BufferedImage currentState = stateProvider.get();
score = getOverlapScore(this.previousScreenshot
score = getOverlapScore(ofNullable(this.previousScreenshot)
.orElseThrow(() -> new ScreenshotComparisonError("Initial screenshot state is not set. "
+ "Nothing to compare")), currentState, resizeMode);
+ "Nothing to compare")), currentState);
if (checkerFunc.apply(score)) {
return this;
}
Expand Down Expand Up @@ -188,25 +186,6 @@ public ScreenshotState verifyChanged(Duration timeout, double minScore) {
return checkState((x) -> x < minScore, timeout);
}

/**
* Verifies whether the state of the screenshot provided by stateProvider lambda function
* is changed within the given timeout.
*
* @param timeout timeout value
* @param minScore the value in range (0.0, 1.0)
* @param resizeMode one of <em>ResizeMode</em> enum values.
* Set it to a value different from <em>NO_RESIZE</em>
* if the actual screenshot is expected to have different
* dimensions in comparison to the previously remembered one
* @return self instance for chaining
* @throws ScreenshotComparisonTimeout if the calculated score is still
* greater or equal to the given score after timeout happens
* @throws ScreenshotComparisonError if {@link #remember()} method has not been invoked yet
*/
public ScreenshotState verifyChanged(Duration timeout, double minScore, ResizeMode resizeMode) {
return checkState((x) -> x < minScore, timeout, resizeMode);
}

/**
* Verifies whether the state of the screenshot provided by stateProvider lambda function
* is not changed within the given timeout.
Expand All @@ -222,112 +201,28 @@ public ScreenshotState verifyNotChanged(Duration timeout, double minScore) {
return checkState((x) -> x >= minScore, timeout);
}

/**
* Verifies whether the state of the screenshot provided by stateProvider lambda function
* is changed within the given timeout.
*
* @param timeout timeout value
* @param minScore the value in range (0.0, 1.0)
* @param resizeMode one of <em>ResizeMode</em> enum values.
* Set it to a value different from <em>NO_RESIZE</em>
* if the actual screenshot is expected to have different
* dimensions in comparison to the previously remembered one
* @return self instance for chaining
* @throws ScreenshotComparisonTimeout if the calculated score is still
* less than the given score after timeout happens
* @throws ScreenshotComparisonError if {@link #remember()} method has not been invoked yet
*/
public ScreenshotState verifyNotChanged(Duration timeout, double minScore, ResizeMode resizeMode) {
return checkState((x) -> x >= minScore, timeout, resizeMode);
}

private static Mat prepareImageForComparison(BufferedImage srcImage) {
final BufferedImage normalizedBitmap = new BufferedImage(srcImage.getWidth(), srcImage.getHeight(),
BufferedImage.TYPE_3BYTE_BGR);
final Graphics2D g = normalizedBitmap.createGraphics();
try {
g.setComposite(AlphaComposite.Src);
g.drawImage(srcImage, 0, 0, null);
} finally {
g.dispose();
}
final byte[] pixels = ((DataBufferByte) normalizedBitmap.getRaster().getDataBuffer()).getData();
final Mat result = new Mat(normalizedBitmap.getHeight(), normalizedBitmap.getWidth(), CvType.CV_8UC3);
result.put(0, 0, pixels);
return result;
}

private static Mat resizeFirstMatrixToSecondMatrixResolution(Mat first, Mat second) {
if (first.width() != second.width() || first.height() != second.height()) {
final Mat result = new Mat();
final Size sz = new Size(second.width(), second.height());
Imgproc.resize(first, result, sz);
return result;
}
return first;
}

/**
* A shortcut to {@link #getOverlapScore(BufferedImage, BufferedImage, ResizeMode)} method
* for the case if both reference and template images are expected to have the same dimensions.
*
* @param refImage reference image
* @param tplImage template
* @return similarity score value in range (-1.0, 1.0). 1.0 is returned if the images are equal
* @throws ScreenshotComparisonError if provided images are not valid or have different resolution
*/
public static double getOverlapScore(BufferedImage refImage, BufferedImage tplImage) {
return getOverlapScore(refImage, tplImage, ResizeMode.NO_RESIZE);
}

/**
* Compares two valid java bitmaps and calculates similarity score between them.
* Both images are expected to be of the same size/resolution. The method
* implicitly invokes {@link ComparesImages#getImagesSimilarity(byte[], byte[])}.
*
* @param refImage reference image
* @param tplImage template
* @param resizeMode one of possible enum values. Set it either to <em>TEMPLATE_TO_REFERENCE_RESOLUTION</em> or
* <em>REFERENCE_TO_TEMPLATE_RESOLUTION</em> if given bitmaps have different dimensions
* @return similarity score value in range (-1.0, 1.0). 1.0 is returned if the images are equal
* @return similarity score value in range (-1.0, 1.0]. 1.0 is returned if the images are equal
* @throws ScreenshotComparisonError if provided images are not valid or have
* different resolution, but resizeMode has been set to <em>NO_RESIZE</em>
* different resolution
*/
public static double getOverlapScore(BufferedImage refImage, BufferedImage tplImage, ResizeMode resizeMode) {
Mat ref = prepareImageForComparison(refImage);
if (ref.empty()) {
throw new ScreenshotComparisonError("Reference image cannot be converted for further comparison");
}
Mat tpl = prepareImageForComparison(tplImage);
if (tpl.empty()) {
throw new ScreenshotComparisonError("Template image cannot be converted for further comparison");
}
switch (resizeMode) {
case TEMPLATE_TO_REFERENCE_RESOLUTION:
tpl = resizeFirstMatrixToSecondMatrixResolution(tpl, ref);
break;
case REFERENCE_TO_TEMPLATE_RESOLUTION:
ref = resizeFirstMatrixToSecondMatrixResolution(ref, tpl);
break;
default:
// do nothing
}

if (ref.width() != tpl.width() || ref.height() != tpl.height()) {
throw new ScreenshotComparisonError(
"Resolutions of template and reference images are expected to be equal. "
+ "Try different resizeMode value."
);
public double getOverlapScore(BufferedImage refImage, BufferedImage tplImage) {
try (ByteArrayOutputStream img1 = new ByteArrayOutputStream();
ByteArrayOutputStream img2 = new ByteArrayOutputStream()) {
ImageIO.write(refImage, "png", img1);
ImageIO.write(tplImage, "png", img2);
return comparator
.getImagesSimilarity(Base64.getEncoder().encode(img1.toByteArray()),
Base64.getEncoder().encode(img2.toByteArray()))
.getScore();
} catch (IOException e) {
throw new ScreenshotComparisonError(e);
}

Mat res = new Mat(ref.rows() - tpl.rows() + 1, ref.cols() - tpl.cols() + 1, CvType.CV_32FC1);
Imgproc.matchTemplate(ref, tpl, res, Imgproc.TM_CCOEFF_NORMED);
return Core.minMaxLoc(res).maxVal;
}

public enum ResizeMode {
NO_RESIZE, TEMPLATE_TO_REFERENCE_RESOLUTION, REFERENCE_TO_TEMPLATE_RESOLUTION
}

static {
loadShared();
}
}
Loading