update java version to 11

.github/workflows/ci.yml

@@ -22,7 +22,7 @@ jobs:
  uses: actions/setup-java@v2
  with:
  java-version: '8'
- distribution: 'adopt'
+ distribution: 'zulu'
 
  - name: Setup Python3
  uses: actions/setup-python@v2
@@ -57,9 +57,6 @@ jobs:
  with:
  fetch-depth: 2
 
- - name: Compile
- run: mvn clean compile -Dmaven.javadoc.skip=true | grep -v "Downloading\|Downloaded"
-
  - name: Prepare env and service
  env:
  TRAVIS_DIR: computer-dist/src/assembly/travis
@@ -69,6 +66,15 @@ jobs:
  $TRAVIS_DIR/install-hugegraph-from-source.sh $COMMIT_ID | grep -v "Downloading\|Downloaded"
  $TRAVIS_DIR/load-data-into-hugegraph.sh
 
+ - name: Install JDK 11
+ uses: actions/setup-java@v2
+ with:
+ java-version: '11'
+ distribution: 'zulu'
+
+ - name: Compile
+ run: mvn clean compile -Dmaven.javadoc.skip=true | grep -v "Downloading\|Downloaded"
+
  - name: Integrate test
  run: mvn test -P integrate-test
 

.github/workflows/publish_images.yml

@@ -36,11 +36,11 @@ jobs:
  username: ${{ secrets.DOCKERHUB_USERNAME }}
  password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
- - name: Install JDK 8
+ - name: Install JDK 11
  uses: actions/setup-java@v2
  with:
- java-version: '8'
- distribution: 'adopt'
+ java-version: '11'
+ distribution: 'zulu'
 
  - name: Cache Maven packages
  uses: actions/cache@v2

.github/workflows/release.yml

@@ -8,10 +8,10 @@ jobs:
  build:
  runs-on: ubuntu-20.04
  steps:
- - name: Install JDK 8
+ - name: Install JDK 11
  uses: actions/setup-java@v2
  with:
- java-version: '8'
+ java-version: '11'
  distribution: 'adopt'
 
  - name: Cache Maven packages

computer-core/src/main/java/com/baidu/hugegraph/computer/core/config/ComputerOptions.java

@@ -562,15 +562,15 @@ public static synchronized ComputerOptions instance() {
  "transport.server_threads",
  "The number of transport threads for server.",
  positiveInt(),
- TransportConf.NUMBER_CPU_CORES
+ TransportConf.DEFAULT_THREADS
  );
 
  public static final ConfigOption<Integer> TRANSPORT_CLIENT_THREADS =
  new ConfigOption<>(
  "transport.client_threads",
  "The number of transport threads for client.",
  positiveInt(),
- TransportConf.NUMBER_CPU_CORES
+ TransportConf.DEFAULT_THREADS
  );
 
  public static final ConfigOption<Class<?>> TRANSPORT_PROVIDER_CLASS =

computer-core/src/main/java/com/baidu/hugegraph/computer/core/network/TransportConf.java

@@ -35,10 +35,10 @@ public class TransportConf {
  "transport-netty-server";
  public static final String CLIENT_THREAD_GROUP_NAME =
  "transport-netty-client";
- public static final int NUMBER_CPU_CORES =
- Runtime.getRuntime().availableProcessors();
  private final Config config;
 
+ public static final int DEFAULT_THREADS = 4;
+
  public static TransportConf wrapConfig(Config config) {
  return new TransportConf(config);
  }
@@ -72,11 +72,7 @@ public int clientThreads() {
  }
 
  private int maxTransportThreads() {
- Integer workerCount = this.config
- .get(ComputerOptions.JOB_WORKERS_COUNT);
- Integer partitions = this.config
- .get(ComputerOptions.JOB_PARTITIONS_COUNT);
- return partitions / workerCount + 1;
+ return this.config.get(ComputerOptions.JOB_WORKERS_COUNT);
  }
 
  public TransportProvider transportProvider() {

computer-dist/Dockerfile

@@ -1,6 +1,7 @@
-FROM openjdk:8-jre
+FROM openjdk:11-jre
 LABEL maintainer="HugeGraph Docker Maintainers <hugegraph@googlegroups.com>"
-ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=2 -XshowSettings:vm"
+# use ParallelGC which is more friendly to olap system
+ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseParallelGC -XX:+UseContainerSupport -XX:MaxRAMPercentage=50 -XshowSettings:vm"
 COPY . /etc/local/hugegraph-computer
 WORKDIR /etc/local/hugegraph-computer
 RUN apt-get update && apt-get -y install gettext-base && apt-get -y install wget

computer-dist/src/assembly/static/bin/start-computer.sh

@@ -164,6 +164,17 @@ else
  JAVA="$JAVA_HOME/bin/java -server"
 fi
 
+# Set up count of cpu if it unspecified from k8s drive,
+# avoid `Runtime.getRuntime().availableProcessors()` always return 1
+if [[ "${DRIVE}" = "${K8S_DRIVE}" && -z "${CPU_LIMIT}" ]]; then
+ PROCESSOR_COUNT="$(cat /proc/cpuinfo | grep "processor" | wc -l)"
+ let MAX_PROCESSOR_COUNT=8
+ if [[ ${PROCESSOR_COUNT} -gt ${MAX_PROCESSOR_COUNT} ]]; then
+ PROCESSOR_COUNT="$MAX_PROCESSOR_COUNT"
+ fi
+ JAVA_OPTS="${JAVA_OPTS} -XX:ActiveProcessorCount=${PROCESSOR_COUNT}"
+fi
+
 if [ ! -a "${CONF_DIR}" ];then
  mkdir -p "${CONF_DIR}"
 fi
@@ -185,9 +196,9 @@ fi
 MAIN_CLASS=com.baidu.hugegraph.computer.dist.HugeGraphComputer
 
 if [ "${LOG4j_CONF}" != "" ]; then
- exec ${JAVA} -Dname="hugegraph-computer" "${LOG4j_CONF}" ${JVM_OPTIONS} \
+ exec ${JAVA} -Dname="hugegraph-computer" "${LOG4j_CONF}" ${JAVA_OPTS} ${JVM_OPTIONS} \
  -cp "${CP}" ${MAIN_CLASS} "${NEW_COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE}
 else
- exec ${JAVA} -Dname="hugegraph-computer" ${JVM_OPTIONS} -cp "${CP}" \
+ exec ${JAVA} -Dname="hugegraph-computer" ${JAVA_OPTS} ${JVM_OPTIONS} -cp "${CP}" \
  ${MAIN_CLASS} "${NEW_COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE}
 fi

computer-k8s-operator/Dockerfile

@@ -1,7 +1,5 @@
-FROM openjdk:8-jre-alpine
+FROM openjdk:11-jre
 LABEL maintainer="HugeGraph Docker Maintainers <hugegraph@googlegroups.com>"
-
-ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=2 -XshowSettings:vm"
 WORKDIR /opt/app
 COPY target/hugegraph-computer-operator-*.jar hugegraph-computer-operator.jar
 ENTRYPOINT ["java", "-jar", "hugegraph-computer-operator.jar"]

computer-k8s-operator/README.md

@@ -52,12 +52,14 @@ spec:
  image: hugegraph/hugegraph-builtin-algorithms:latest # algorithm image url
  jarFile: /opt/jars/hugegraph-builtin-algorithms.jar
  pullPolicy: Always
- workerInstances: 50
+ workerCpu: "4"
+ workerMemory: "4Gi"
+ workerInstances: 5
  computerConf:
+ job.partitions_count: "20"
  algorithm.params_class: com.baidu.hugegraph.computer.algorithm.centrality.pagerank.PageRankParams
  hugegraph.url: http://${hugegraph-server-host}:${hugegraph-server-port} # hugegraph server url
- job.partitions_count: "200"
- hugegraph.name: "hugegraph"
+ hugegraph.name: hugegraph
 EOF
 ```
 

computer-k8s-operator/src/main/java/com/baidu/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java

@@ -85,7 +85,6 @@ public class ComputerJobDeployer {
  private static final String RPC_PORT_NAME = "rpc-port";
  private static final int DEFAULT_TRANSPORT_PORT = 8099;
  private static final int DEFAULT_RPC_PORT = 8090;
- private static final int DEFAULT_TRANSPORT_THREADS = 8;
  private static final String COMPUTER_CONFIG_MAP_VOLUME =
  "computer-config-map-volume";
 
@@ -198,19 +197,6 @@ private Set<ContainerPort> handleConfig(ComputerJobSpec spec) {
  config.put(ComputerOptions.RPC_SERVER_PORT.name(), rpcPort);
  }
 
- /*
- Set a default number of transport threads,
- if the number of CPU quantity of the worker is not specified
- */
- if (spec.getWorkerCpu() == null) {
- String defaultThreads = String.valueOf(DEFAULT_TRANSPORT_THREADS);
-
- config.putIfAbsent(ComputerOptions.TRANSPORT_CLIENT_THREADS.name(),
- defaultThreads);
- config.putIfAbsent(ComputerOptions.TRANSPORT_SERVER_THREADS.name(),
- defaultThreads);
- }
-
  ContainerPort transportContainerPort = new ContainerPortBuilder()
  .withName(TRANSPORT_PORT_NAME)
  .withContainerPort(Integer.valueOf(transportPort))
@@ -471,6 +457,22 @@ private Container getContainer(String name, ComputerJobSpec spec,
  memory = spec.getWorkerMemory();
  }
 
+ if (cpu != null) {
+ EnvVar cpuLimit = new EnvVarBuilder()
+ .withName(Constants.ENV_CPU_LIMIT)
+ .withValue(cpu.toString())
+ .build();
+ envVars.add(cpuLimit);
+ }
+
+ if (memory != null) {
+ EnvVar memoryLimit = new EnvVarBuilder()
+ .withName(Constants.ENV_MEMORY_LIMIT)
+ .withValue(memory.toString())
+ .build();
+ envVars.add(memoryLimit);
+ }
+
  List<VolumeMount> volumeMounts = spec.getVolumeMounts();
  if (volumeMounts == null) {
  volumeMounts = new ArrayList<>();

computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/Constants.java

@@ -42,6 +42,8 @@ public class Constants {
  public static final String ENV_JAR_FILE_PATH = "JAR_FILE_PATH";
  public static final String ENV_JOB_JAR_URI = "JOB_JAR_URI";
  public static final String ENV_JVM_OPTIONS = "JVM_OPTIONS";
+ public static final String ENV_CPU_LIMIT = "CPU_LIMIT";
+ public static final String ENV_MEMORY_LIMIT = "MEMORY_LIMIT";
 
  public static final String CONFIG_DIR = "/opt/hugegraph-computer/conf";
  public static final String COMPUTER_CONF_FILE = "computer.properties";

computer-test/src/main/java/com/baidu/hugegraph/computer/core/graph/value/ListValueTest.java

@@ -130,14 +130,10 @@ public void testGet() {
 
  Assert.assertThrows(IndexOutOfBoundsException.class, () -> {
  value1.get(3);
- }, e -> {
- Assert.assertContains("Index: 3, Size: 3", e.getMessage());
  });
 
  Assert.assertThrows(IndexOutOfBoundsException.class, () -> {
  value2.get(3);
- }, e -> {
- Assert.assertContains("Index: 3, Size: 2", e.getMessage());
  });
  }
 

pom.xml

@@ -42,8 +42,8 @@
  <top.level.dir>${project.basedir}/..</top.level.dir>
  <release.name>hugegraph-computer</release.name>
  <final.name>${release.name}-${project.version}</final.name>
- <compiler.source>1.8</compiler.source>
- <compiler.target>1.8</compiler.target>
+ <compiler.source>11</compiler.source>
+ <compiler.target>11</compiler.target>
  <shell-executable>bash</shell-executable>
  <etcd.version>0.5.4</etcd.version>
  <hadoop-version>3.1.2</hadoop-version>