Add image comparison helpers (appium#870)

* Add image comparison helpers * Fix style * Add docstrings * Add some debug fixes

Author: Mykola Mokhnach

src/main/java/io/appium/java_client/AppiumDriver.java

@@ -64,7 +64,7 @@
 */
 @SuppressWarnings("unchecked")
 public class AppiumDriver<T extends WebElement>
- extends DefaultGenericMobileDriver<T> {
+ extends DefaultGenericMobileDriver<T> implements ComparesImages {
 
  private static final ErrorHandler errorHandler = new ErrorHandler(new ErrorCodesMobile(), true);
  // frequently used command parameters

src/main/java/io/appium/java_client/ComparesImages.java

@@ -0,0 +1,227 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * See the NOTICE file distributed with this work for additional
+ * information regarding copyright ownership.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.appium.java_client;
+
+import static io.appium.java_client.MobileCommand.compareImagesCommand;
+
+import io.appium.java_client.imagecomparison.ComparisonMode;
+import io.appium.java_client.imagecomparison.FeaturesMatchingOptions;
+import io.appium.java_client.imagecomparison.FeaturesMatchingResult;
+import io.appium.java_client.imagecomparison.OccurrenceMatchingOptions;
+import io.appium.java_client.imagecomparison.OccurrenceMatchingResult;
+import io.appium.java_client.imagecomparison.SimilarityMatchingOptions;
+import io.appium.java_client.imagecomparison.SimilarityMatchingResult;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.FileUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+public interface ComparesImages extends ExecutesMethod {
+
+ /**
+ * Performs images matching by features with default options. Read
+ * https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_feature2d/py_matcher/py_matcher.html
+ * for more details on this topic.
+ *
+ * @param base64image1 base64-encoded representation of the first image
+ * @param base64Image2 base64-encoded representation of the second image
+ * @return The matching result.
+ */
+ default FeaturesMatchingResult matchImagesFeatures(byte[] base64image1, byte[] base64Image2) {
+ return matchImagesFeatures(base64image1, base64Image2, null);
+ }
+
+ /**
+ * Performs images matching by features. Read
+ * https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_feature2d/py_matcher/py_matcher.html
+ * for more details on this topic.
+ *
+ * @param base64image1 base64-encoded representation of the first image
+ * @param base64Image2 base64-encoded representation of the second image
+ * @param options comparison options
+ * @return The matching result. The configuration of fields in the result depends on comparison options.
+ */
+ default FeaturesMatchingResult matchImagesFeatures(byte[] base64image1, byte[] base64Image2,
+ @Nullable FeaturesMatchingOptions options) {
+ Object response = CommandExecutionHelper.execute(this,
+ compareImagesCommand(ComparisonMode.MATCH_FEATURES, base64image1, base64Image2, options));
+ //noinspection unchecked
+ return new FeaturesMatchingResult((Map<String, Object>) response);
+ }
+
+ /**
+ * Performs images matching by features with default options. Read
+ * https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_feature2d/py_matcher/py_matcher.html
+ * for more details on this topic.
+ *
+ * @param image1 The location of the first image
+ * @param image2 The location of the second image
+ * @return The matching result.
+ */
+ default FeaturesMatchingResult matchImagesFeatures(File image1, File image2) throws IOException {
+ return matchImagesFeatures(image1, image2, null);
+ }
+
+ /**
+ * Performs images matching by features. Read
+ * https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_feature2d/py_matcher/py_matcher.html
+ * for more details on this topic.
+ *
+ * @param image1 The location of the first image
+ * @param image2 The location of the second image
+ * @param options comparison options
+ * @return The matching result. The configuration of fields in the result depends on comparison options.
+ */
+ default FeaturesMatchingResult matchImagesFeatures(File image1, File image2,
+ @Nullable FeaturesMatchingOptions options) throws IOException {
+ return matchImagesFeatures(Base64.encodeBase64(FileUtils.readFileToByteArray(image1)),
+ Base64.encodeBase64(FileUtils.readFileToByteArray(image2)), options);
+ }
+
+ /**
+ * Performs images matching by template to find possible occurrence of the partial image
+ * in the full image with default options. Read
+ * https://docs.opencv.org/2.4/doc/tutorials/imgproc/histograms/template_matching/template_matching.html
+ * for more details on this topic.
+ *
+ * @param fullImage base64-encoded representation of the full image
+ * @param partialImage base64-encoded representation of the partial image
+ * @return The matching result.
+ */
+ default OccurrenceMatchingResult findImageOccurrence(byte[] fullImage, byte[] partialImage) {
+ return findImageOccurrence(fullImage, partialImage, null);
+ }
+
+ /**
+ * Performs images matching by template to find possible occurrence of the partial image
+ * in the full image. Read
+ * https://docs.opencv.org/2.4/doc/tutorials/imgproc/histograms/template_matching/template_matching.html
+ * for more details on this topic.
+ *
+ * @param fullImage base64-encoded representation of the full image
+ * @param partialImage base64-encoded representation of the partial image
+ * @param options comparison options
+ * @return The matching result. The configuration of fields in the result depends on comparison options.
+ */
+ default OccurrenceMatchingResult findImageOccurrence(byte[] fullImage, byte[] partialImage,
+ @Nullable OccurrenceMatchingOptions options) {
+ Object response = CommandExecutionHelper.execute(this,
+ compareImagesCommand(ComparisonMode.MATCH_TEMPLATE, fullImage, partialImage, options));
+ //noinspection unchecked
+ return new OccurrenceMatchingResult((Map<String, Object>) response);
+ }
+
+ /**
+ * Performs images matching by template to find possible occurrence of the partial image
+ * in the full image with default options. Read
+ * https://docs.opencv.org/2.4/doc/tutorials/imgproc/histograms/template_matching/template_matching.html
+ * for more details on this topic.
+ *
+ * @param fullImage The location of the full image
+ * @param partialImage The location of the partial image
+ * @return The matching result. The configuration of fields in the result depends on comparison options.
+ */
+ default OccurrenceMatchingResult findImageOccurrence(File fullImage, File partialImage) throws IOException {
+ return findImageOccurrence(fullImage, partialImage, null);
+ }
+
+ /**
+ * Performs images matching by template to find possible occurrence of the partial image
+ * in the full image. Read
+ * https://docs.opencv.org/2.4/doc/tutorials/imgproc/histograms/template_matching/template_matching.html
+ * for more details on this topic.
+ *
+ * @param fullImage The location of the full image
+ * @param partialImage The location of the partial image
+ * @param options comparison options
+ * @return The matching result. The configuration of fields in the result depends on comparison options.
+ */
+ default OccurrenceMatchingResult findImageOccurrence(File fullImage, File partialImage,
+ @Nullable OccurrenceMatchingOptions options)
+ throws IOException {
+ return findImageOccurrence(Base64.encodeBase64(FileUtils.readFileToByteArray(fullImage)),
+ Base64.encodeBase64(FileUtils.readFileToByteArray(partialImage)), options);
+ }
+
+ /**
+ * Performs images matching to calculate the similarity score between them
+ * with default options. The flow there is similar to the one used in
+ * {@link #findImageOccurrence(byte[], byte[], OccurrenceMatchingOptions)},
+ * but it is mandatory that both images are of equal size.
+ *
+ * @param base64image1 base64-encoded representation of the first image
+ * @param base64Image2 base64-encoded representation of the second image
+ * @return Matching result. The configuration of fields in the result depends on comparison options.
+ */
+ default SimilarityMatchingResult getImagesSimilarity(byte[] base64image1, byte[] base64Image2) {
+ return getImagesSimilarity(base64image1, base64Image2, null);
+ }
+
+ /**
+ * Performs images matching to calculate the similarity score between them.
+ * The flow there is similar to the one used in
+ * {@link #findImageOccurrence(byte[], byte[], OccurrenceMatchingOptions)},
+ * but it is mandatory that both images are of equal size.
+ *
+ * @param base64image1 base64-encoded representation of the first image
+ * @param base64Image2 base64-encoded representation of the second image
+ * @param options comparison options
+ * @return Matching result. The configuration of fields in the result depends on comparison options.
+ */
+ default SimilarityMatchingResult getImagesSimilarity(byte[] base64image1, byte[] base64Image2,
+ @Nullable SimilarityMatchingOptions options) {
+ Object response = CommandExecutionHelper.execute(this,
+ compareImagesCommand(ComparisonMode.GET_SIMILARITY, base64image1, base64Image2, options));
+ //noinspection unchecked
+ return new SimilarityMatchingResult((Map<String, Object>) response);
+ }
+
+ /**
+ * Performs images matching to calculate the similarity score between them
+ * with default options. The flow there is similar to the one used in
+ * {@link #findImageOccurrence(byte[], byte[], OccurrenceMatchingOptions)},
+ * but it is mandatory that both images are of equal size.
+ *
+ * @param image1 The location of the full image
+ * @param image2 The location of the partial image
+ * @return Matching result. The configuration of fields in the result depends on comparison options.
+ */
+ default SimilarityMatchingResult getImagesSimilarity(File image1, File image2) throws IOException {
+ return getImagesSimilarity(image1, image2, null);
+ }
+
+ /**
+ * Performs images matching to calculate the similarity score between them.
+ * The flow there is similar to the one used in
+ * {@link #findImageOccurrence(byte[], byte[], OccurrenceMatchingOptions)},
+ * but it is mandatory that both images are of equal size.
+ *
+ * @param image1 The location of the full image
+ * @param image2 The location of the partial image
+ * @param options comparison options
+ * @return Matching result. The configuration of fields in the result depends on comparison options.
+ */
+ default SimilarityMatchingResult getImagesSimilarity(File image1, File image2,
+ @Nullable SimilarityMatchingOptions options)
+ throws IOException {
+ return getImagesSimilarity(Base64.encodeBase64(FileUtils.readFileToByteArray(image1)),
+ Base64.encodeBase64(FileUtils.readFileToByteArray(image2)), options);
+ }
+}

src/main/java/io/appium/java_client/MobileCommand.java

@@ -18,6 +18,8 @@
 
 import com.google.common.collect.ImmutableMap;
 
+import io.appium.java_client.imagecomparison.BaseComparisonOptions;
+import io.appium.java_client.imagecomparison.ComparisonMode;
 import io.appium.java_client.screenrecording.BaseStartScreenRecordingOptions;
 import io.appium.java_client.screenrecording.BaseStopScreenRecordingOptions;
 import org.apache.commons.lang3.StringUtils;
@@ -29,6 +31,7 @@
 import java.util.AbstractMap;
 import java.util.HashMap;
 import java.util.Map;
+import javax.annotation.Nullable;
 
 /**
  * The repository of mobile commands defined in the Mobile JSON
@@ -106,6 +109,7 @@ public class MobileCommand {
  protected static final String TOGGLE_WIFI;
  protected static final String TOGGLE_AIRPLANE_MODE;
  protected static final String TOGGLE_DATA;
+ protected static final String COMPARE_IMAGES;
 
  public static final Map<String, CommandInfo> commandRepository;
 
@@ -179,6 +183,7 @@ public class MobileCommand {
  TOGGLE_WIFI = "toggleWiFi";
  TOGGLE_AIRPLANE_MODE = "toggleFlightMode";
  TOGGLE_DATA = "toggleData";
+ COMPARE_IMAGES = "compareImages";
 
  commandRepository = new HashMap<>();
  commandRepository.put(RESET, postC("/session/:sessionId/appium/app/reset"));
@@ -262,6 +267,7 @@ public class MobileCommand {
  commandRepository.put(TOGGLE_WIFI, postC("/session/:sessionId/appium/device/toggle_wifi"));
  commandRepository.put(TOGGLE_AIRPLANE_MODE, postC("/session/:sessionId/appium/device/toggle_airplane_mode"));
  commandRepository.put(TOGGLE_DATA, postC("/session/:sessionId/appium/device/toggle_data"));
+ commandRepository.put(COMPARE_IMAGES, postC("/session/:sessionId/appium/compare_images"));
  }
 
  /**
@@ -435,7 +441,7 @@ public static ImmutableMap<String, Object> prepareArguments(String[] params,
  * This method forms a {@link java.util.Map} of parameters for the
  * device unlocking.
  *
- * @return  a key-value pair. The key is the command name. The value is a
+ * @return a key-value pair. The key is the command name. The value is a
  * {@link java.util.Map} command arguments.
  */
  public static Map.Entry<String, Map<String, ?>> unlockDeviceCommand() {
@@ -446,7 +452,7 @@ public static ImmutableMap<String, Object> prepareArguments(String[] params,
  * This method forms a {@link java.util.Map} of parameters for the
  * device locked query.
  *
- * @return  a key-value pair. The key is the command name. The value is a
+ * @return a key-value pair. The key is the command name. The value is a
  * {@link java.util.Map} command arguments.
  */
  public static Map.Entry<String, Map<String, ?>> getIsDeviceLockedCommand() {
@@ -472,7 +478,7 @@ public static ImmutableMap<String, Object> prepareArguments(String[] params,
  * {@link java.util.Map} command arguments.
  */
  public static Map.Entry<String, Map<String, ?>> pushFileCommand(String remotePath, byte[] base64Data) {
- String[] parameters = new String[] {"path", "data"};
+ String[] parameters = new String[]{"path", "data"};
  Object[] values = new Object[]{remotePath, new String(base64Data, StandardCharsets.UTF_8)};
  return new AbstractMap.SimpleEntry<>(PUSH_FILE, prepareArguments(parameters, values));
  }
@@ -486,4 +492,27 @@ public static ImmutableMap<String, Object> prepareArguments(String[] params,
  return new AbstractMap.SimpleEntry<>(STOP_RECORDING_SCREEN,
  prepareArguments("options", opts.build()));
  }
+
+ /**
+ * Forms a {@link java.util.Map} of parameters for images comparison.
+ *
+ * @param mode one of possible comparison modes
+ * @param img1Data base64-encoded data of the first image
+ * @param img2Data base64-encoded data of the second image
+ * @param options comparison options
+ * @return key-value pairs
+ */
+ public static Map.Entry<String, Map<String, ?>> compareImagesCommand(ComparisonMode mode,
+ byte[] img1Data, byte[] img2Data,
+ @Nullable BaseComparisonOptions options) {
+ String[] parameters = options == null
+ ? new String[]{"mode", "firstImage", "secondImage"}
+ : new String[]{"mode", "firstImage", "secondImage", "options"};
+ Object[] values = options == null
+ ? new Object[]{mode.toString(), new String(img1Data, StandardCharsets.UTF_8),
+ new String(img2Data, StandardCharsets.UTF_8)}
+ : new Object[]{mode.toString(), new String(img1Data, StandardCharsets.UTF_8),
+ new String(img2Data, StandardCharsets.UTF_8), options.build()};
+ return new AbstractMap.SimpleEntry<>(COMPARE_IMAGES, prepareArguments(parameters, values));
+ }
 }

src/main/java/io/appium/java_client/imagecomparison/BaseComparisonOptions.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * See the NOTICE file distributed with this work for additional
+ * information regarding copyright ownership.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.appium.java_client.imagecomparison;
+
+import static java.util.Optional.ofNullable;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+
+public abstract class BaseComparisonOptions<T extends BaseComparisonOptions<T>> {
+ private Boolean visualize;
+
+ /**
+ * Makes the endpoint to return an image,
+ * which contains the visualized result of the corresponding
+ * picture matching operation. This option is disabled by default.
+ *
+ * @return self instance for chaining
+ */
+ public T withEnabledVisualization() {
+ visualize = true;
+ //noinspection unchecked
+ return (T) this;
+ }
+
+ /**
+ * Builds a map, which is ready to be passed to the subordinated
+ * Appium API.
+ *
+ * @return comparison options mapping.
+ */
+ @SuppressWarnings("unchecked")
+ public Map<String, Object> build() {
+ final ImmutableMap.Builder builder = new ImmutableMap.Builder<String, Object>();
+ ofNullable(visualize).map(x -> builder.put("visualize", x));
+ return builder.build();
+ }
+}