Update arrow version to fix plasma bugs (ray-project#4127)

* Update arrow * Change to 2c511979b13b230e73a179dab1d55b03cd81ec02 which is rebased on Arrow 46f75d7 * Update to fix comment * disable tests which use python/ray/rllib/tests/data/cartpole_small * Fix get order of meta and data in MockObjectStore.java

Author: guoyuhong

ci/jenkins_tests/run_rllib_tests.sh

@@ -258,19 +258,21 @@ docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
  --stop '{"training_iteration": 1}' \
  --config '{"num_workers": 2, "optimizer": {"num_replay_buffer_shards": 1}, "learning_starts": 100, "min_iter_time_s": 1, "batch_mode": "complete_episodes", "parameter_noise": true}'
 
-docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
- /ray/python/ray/rllib/tests/run_silent.sh train.py \
- --env CartPole-v0 \
- --run MARWIL \
- --stop '{"training_iteration": 1}' \
- --config '{"input": "/ray/python/ray/rllib/tests/data/cartpole_small", "learning_starts": 0, "input_evaluation": ["wis", "is"], "shuffle_buffer_size": 10}'
-
-docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
- /ray/python/ray/rllib/tests/run_silent.sh train.py \
- --env CartPole-v0 \
- --run DQN \
- --stop '{"training_iteration": 1}' \
- --config '{"input": "/ray/python/ray/rllib/tests/data/cartpole_small", "learning_starts": 0, "input_evaluation": ["wis", "is"], "soft_q": true}'
+# TODO(ericl): reenable the test after fix the arrow serialization error.
+# https://github.com/ray-project/ray/pull/4127#issuecomment-468903577
+#docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
+# /ray/python/ray/rllib/tests/run_silent.sh train.py \
+# --env CartPole-v0 \
+# --run MARWIL \
+# --stop '{"training_iteration": 1}' \
+# --config '{"input": "/ray/python/ray/rllib/tests/data/cartpole_small", "learning_starts": 0, "input_evaluation": ["wis", "is"], "shuffle_buffer_size": 10}'
+
+#docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
+# /ray/python/ray/rllib/tests/run_silent.sh train.py \
+# --env CartPole-v0 \
+# --run DQN \
+# --stop '{"training_iteration": 1}' \
+# --config '{"input": "/ray/python/ray/rllib/tests/data/cartpole_small", "learning_starts": 0, "input_evaluation": ["wis", "is"], "soft_q": true}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
  /ray/python/ray/rllib/tests/run_silent.sh tests/test_local.py
@@ -360,8 +362,10 @@ docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
  /ray/python/ray/rllib/tests/run_silent.sh examples/cartpole_lstm.py --stop=200 --use-prev-action-reward
 
-docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
- /ray/python/ray/rllib/tests/run_silent.sh examples/custom_loss.py --iters=2
+# TODO(ericl): reenable the test after fix the arrow serialization error.
+# https://github.com/ray-project/ray/pull/4127#issuecomment-468903577
+#docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
+# /ray/python/ray/rllib/tests/run_silent.sh examples/custom_loss.py --iters=2
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
  /ray/python/ray/rllib/tests/run_silent.sh examples/custom_metrics_and_callbacks.py --num-iters=2

cmake/Modules/ArrowExternalProject.cmake

@@ -20,12 +20,12 @@ set(arrow_URL https://github.com/ray-project/arrow.git)
 # Arrow often rewrites git history and invalidates certain commits.
 # It has been patched to fix an upstream symbol clash with TensorFlow,
 # the patch is available at
-# https://github.com/ray-project/arrow/commit/007e1ca289e979bac80231fa9ee7510be744b60b
+# https://github.com/ray-project/arrow/commit/68299c5f48289c4f39a948cbd0426b9199a9df1e
 # See the discussion in https://github.com/apache/arrow/pull/3177
 # WARNING: If the arrow version is updated, you need to also update the
 # SETUPTOOLS_SCM_PRETEND_VERSION version string in the ThirdpartyToolchain.cmake
 # file
-set(arrow_TAG 007e1ca289e979bac80231fa9ee7510be744b60b)
+set(arrow_TAG 68299c5f48289c4f39a948cbd0426b9199a9df1e)
 
 set(ARROW_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/arrow-install)
 set(ARROW_HOME ${ARROW_INSTALL_PREFIX})

java/runtime/src/main/java/org/ray/runtime/objectstore/MockObjectStore.java

@@ -1,6 +1,5 @@
 package org.ray.runtime.objectstore;
 
-import java.lang.reflect.Constructor;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -65,7 +64,7 @@ public byte[] get(byte[] objectId, int timeoutMs, boolean isMetadata) {
  public List<byte[]> get(byte[][] objectIds, int timeoutMs, boolean isMetadata) {
  return get(objectIds, timeoutMs)
  .stream()
- .map(data -> isMetadata ? data.data : data.metadata)
+ .map(data -> isMetadata ? data.metadata : data.data)
  .collect(Collectors.toList());
  }
 
@@ -93,15 +92,9 @@ public List<ObjectStoreData> get(byte[][] objectIds, int timeoutMs) {
  firstCheck = false;
  }
  ArrayList<ObjectStoreData> rets = new ArrayList<>();
- for (byte[] id : objectIds) {
- try {
- Constructor<?> constructor = ObjectStoreData.class.getDeclaredConstructors()[0];
- constructor.setAccessible(true);
- rets.add((ObjectStoreData) constructor.newInstance(data.get(new UniqueId(id)),
- metadata.get(new UniqueId(id))));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
+ for (byte[] objId : objectIds) {
+ UniqueId uniqueId = new UniqueId(objId);
+ rets.add(new ObjectStoreData(metadata.get(uniqueId), data.get(uniqueId)));
  }
  return rets;
  }

java/runtime/src/main/java/org/ray/runtime/objectstore/ObjectStoreProxy.java

@@ -78,10 +78,8 @@ public <T> List<GetResult<T>> get(List<UniqueId> ids, int timeoutMs) {
 
  List<GetResult<T>> results = new ArrayList<>();
  for (int i = 0; i < dataAndMetaList.size(); i++) {
- // TODO(hchen): Plasma API returns data and metadata in wrong order, this should be fixed
- // from the arrow side first.
- byte[] meta = dataAndMetaList.get(i).data;
- byte[] data = dataAndMetaList.get(i).metadata;
+ byte[] meta = dataAndMetaList.get(i).metadata;
+ byte[] data = dataAndMetaList.get(i).data;
 
  GetResult<T> result;
  if (meta != null) {