remove tabs

Author: rmuir

src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java

@@ -44,10 +44,10 @@
  * <p/>
  * Quick facts about S3:
  * <p/>
- * Maximum object size: 5 TB
- * Maximum number of parts per upload:10,000
+ * Maximum object size:  5 TB
+ * Maximum number of parts per upload: 10,000
  * Part numbers: 1 to 10,000 (inclusive)
- * Part size: 5 MB to 5 GB, last part can be < 5 MB
+ * Part size:  5 MB to 5 GB, last part can be < 5 MB
  * <p/>
  * See http://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html
  * See http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html
@@ -120,7 +120,7 @@ private void upload(byte[] bytes, int off, int len) throws IOException {
  }
 
  protected void doUpload(S3BlobStore blobStore, String bucketName, String blobName, InputStream is, int length,
-  boolean serverSideEncryption) throws AmazonS3Exception {
+ boolean serverSideEncryption) throws AmazonS3Exception {
  ObjectMetadata md = new ObjectMetadata();
  if (serverSideEncryption) {
  md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
@@ -202,15 +202,15 @@ private void uploadMultipart(byte[] bytes, int off, int len, boolean lastPart) t
  }
 
  protected PartETag doUploadMultipart(S3BlobStore blobStore, String bucketName, String blobName, String uploadId, InputStream is,
-  int length, boolean lastPart) throws AmazonS3Exception {
+ int length, boolean lastPart) throws AmazonS3Exception {
  UploadPartRequest request = new UploadPartRequest()
-  .withBucketName(bucketName)
-  .withKey(blobName)
-  .withUploadId(uploadId)
-  .withPartNumber(multipartChunks)
-  .withInputStream(is)
-  .withPartSize(length)
-  .withLastPart(lastPart);
+ .withBucketName(bucketName)
+ .withKey(blobName)
+ .withUploadId(uploadId)
+ .withPartNumber(multipartChunks)
+ .withInputStream(is)
+ .withPartSize(length)
+ .withLastPart(lastPart);
 
  UploadPartResult response = blobStore.client().uploadPart(request);
  return response.getPartETag();

src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java

@@ -91,7 +91,7 @@ public void testSimpleWorkflow() {
  .setType("s3").setSettings(ImmutableSettings.settingsBuilder()
  .put("base_path", basePath)
  .put("chunk_size", randomIntBetween(1000, 10000))
- ).get();
+  ).get();
  assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
 
  createIndex("test-idx-1", "test-idx-2", "test-idx-3");
@@ -154,94 +154,94 @@ public void testSimpleWorkflow() {
  assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true));
  assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false));
  }
- 
+
  @Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211")
  public void testEncryption() {
-Client client = client();
-logger.info("--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath);
-PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
-.setType("s3").setSettings(ImmutableSettings.settingsBuilder()
-.put("base_path", basePath)
-.put("chunk_size", randomIntBetween(1000, 10000))
-.put("server_side_encryption", true)
-).get();
-assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
-
-createIndex("test-idx-1", "test-idx-2", "test-idx-3");
-ensureGreen();
-
-logger.info("--> indexing some data");
-for (int i = 0; i < 100; i++) {
- index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
- index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
- index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
-}
-refresh();
-assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L));
-assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L));
-assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(100L));
-
-logger.info("--> snapshot");
-CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
-assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
-assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
-
-assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
-
-Settings settings = internalCluster().getInstance(Settings.class);
-Settings bucket = settings.getByPrefix("repositories.s3.");
-AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(
-null,
-null,
-bucket.get("region", settings.get("repositories.s3.region")),
-bucket.get("access_key", settings.get("cloud.aws.access_key")),
-bucket.get("secret_key", settings.get("cloud.aws.secret_key")));
-
- String bucketName = bucket.get("bucket");
-logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath);
-List<S3ObjectSummary> summaries = s3Client.listObjects(bucketName, basePath).getObjectSummaries();
-for (S3ObjectSummary summary : summaries) {
- assertThat(s3Client.getObjectMetadata(bucketName, summary.getKey()).getSSEAlgorithm(), equalTo("AES256"));
-}
-
-logger.info("--> delete some data");
-for (int i = 0; i < 50; i++) {
- client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get();
-}
-for (int i = 50; i < 100; i++) {
- client.prepareDelete("test-idx-2", "doc", Integer.toString(i)).get();
-}
-for (int i = 0; i < 100; i += 2) {
- client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get();
-}
-refresh();
-assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(50L));
-assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(50L));
-assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L));
-
-logger.info("--> close indices");
-client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get();
-
-logger.info("--> restore all indices from the snapshot");
-RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet();
-assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
-
-ensureGreen();
-assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L));
-assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L));
-assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L));
-
-// Test restore after index deletion
-logger.info("--> delete indices");
-cluster().wipeIndices("test-idx-1", "test-idx-2");
-logger.info("--> restore one index after deletion");
-restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet();
-assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
-ensureGreen();
-assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L));
-ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
-assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true));
-assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false));
+ Client client = client();
+ logger.info("--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath);
+ PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
+ .setType("s3").setSettings(ImmutableSettings.settingsBuilder()
+ .put("base_path", basePath)
+ .put("chunk_size", randomIntBetween(1000, 10000))
+ .put("server_side_encryption", true)
+ ).get();
+ assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
+
+ createIndex("test-idx-1", "test-idx-2", "test-idx-3");
+ ensureGreen();
+
+ logger.info("--> indexing some data");
+ for (int i = 0; i < 100; i++) {
+  index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
+  index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
+  index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
+ }
+ refresh();
+ assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L));
+ assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L));
+ assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(100L));
+
+ logger.info("--> snapshot");
+ CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
+ assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
+ assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
+
+ assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
+
+ Settings settings = internalCluster().getInstance(Settings.class);
+ Settings bucket = settings.getByPrefix("repositories.s3.");
+ AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(
+ null,
+ null,
+ bucket.get("region", settings.get("repositories.s3.region")),
+ bucket.get("access_key", settings.get("cloud.aws.access_key")),
+ bucket.get("secret_key", settings.get("cloud.aws.secret_key")));
+
+  String bucketName = bucket.get("bucket");
+ logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath);
+ List<S3ObjectSummary> summaries = s3Client.listObjects(bucketName, basePath).getObjectSummaries();
+ for (S3ObjectSummary summary : summaries) {
+  assertThat(s3Client.getObjectMetadata(bucketName, summary.getKey()).getSSEAlgorithm(), equalTo("AES256"));
+ }
+
+ logger.info("--> delete some data");
+ for (int i = 0; i < 50; i++) {
+  client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get();
+ }
+ for (int i = 50; i < 100; i++) {
+  client.prepareDelete("test-idx-2", "doc", Integer.toString(i)).get();
+ }
+ for (int i = 0; i < 100; i += 2) {
+  client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get();
+ }
+ refresh();
+ assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(50L));
+ assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(50L));
+ assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L));
+
+ logger.info("--> close indices");
+ client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get();
+
+ logger.info("--> restore all indices from the snapshot");
+ RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet();
+ assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
+
+ ensureGreen();
+ assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L));
+ assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L));
+ assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L));
+
+ // Test restore after index deletion
+ logger.info("--> delete indices");
+ cluster().wipeIndices("test-idx-1", "test-idx-2");
+ logger.info("--> restore one index after deletion");
+ restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet();
+ assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
+ ensureGreen();
+ assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L));
+ ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
+ assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true));
+ assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false));
  }
 
  /**
@@ -254,9 +254,9 @@ public void assertRepositoryWithCustomCredentialsIsNotAccessibleByDefaultCredent
  Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.private-bucket.");
  logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath);
  client.admin().cluster().preparePutRepository("test-repo")
-  .setType("s3").setSettings(ImmutableSettings.settingsBuilder()
-  .put("base_path", basePath)
-  .put("bucket", bucketSettings.get("bucket"))
+ .setType("s3").setSettings(ImmutableSettings.settingsBuilder()
+ .put("base_path", basePath)
+ .put("bucket", bucketSettings.get("bucket"))
  ).get();
  fail("repository verification should have raise an exception!");
  }
@@ -273,7 +273,7 @@ public void testRepositoryWithCustomCredentials() {
  .put("access_key", bucketSettings.get("access_key"))
  .put("secret_key", bucketSettings.get("secret_key"))
  .put("bucket", bucketSettings.get("bucket"))
- ).get();
+  ).get();
  assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
 
  assertRepositoryIsOperational(client, "test-repo");
@@ -291,7 +291,7 @@ public void testRepositoryWithCustomEndpointProtocol() {
  .put("access_key", bucketSettings.get("access_key"))
  .put("secret_key", bucketSettings.get("secret_key"))
  .put("base_path", basePath)
- ).get();
+  ).get();
  assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
  assertRepositoryIsOperational(client, "test-repo");
  }
@@ -306,11 +306,11 @@ public void assertRepositoryInRemoteRegionIsRemote() {
  Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.remote-bucket.");
  logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath);
  client.admin().cluster().preparePutRepository("test-repo")
-  .setType("s3").setSettings(ImmutableSettings.settingsBuilder()
-  .put("base_path", basePath)
-  .put("bucket", bucketSettings.get("bucket"))
-// Below setting intentionally omitted to assert bucket is not available in default region.
-// .put("region", privateBucketSettings.get("region"))
+ .setType("s3").setSettings(ImmutableSettings.settingsBuilder()
+ .put("base_path", basePath)
+ .put("bucket", bucketSettings.get("bucket"))
+ // Below setting intentionally omitted to assert bucket is not available in default region.
+ // .put("region", privateBucketSettings.get("region"))
  ).get();
 
  fail("repository verification should have raise an exception!");
@@ -327,7 +327,7 @@ public void testRepositoryInRemoteRegion() {
  .put("base_path", basePath)
  .put("bucket", bucketSettings.get("bucket"))
  .put("region", bucketSettings.get("region"))
- ).get();
+  ).get();
  assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
 
  assertRepositoryIsOperational(client, "test-repo");
@@ -342,8 +342,8 @@ public void testNonExistingRepo_86() {
  logger.info("--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath);
  PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
  .setType("s3").setSettings(ImmutableSettings.settingsBuilder()
-  .put("base_path", basePath)
- ).get();
+ .put("base_path", basePath)
+  ).get();
  assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
 
  logger.info("--> restore non existing snapshot");
@@ -364,8 +364,8 @@ public void testGetDeleteNonExistingSnapshot_86() {
  logger.info("--> creating s3 repository without any path");
  PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo")
  .setType("s3").setSettings(ImmutableSettings.settingsBuilder()
-  .put("base_path", basePath)
- ).get();
+ .put("base_path", basePath)
+  ).get();
  assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
 
  try {
@@ -383,7 +383,7 @@ public void testGetDeleteNonExistingSnapshot_86() {
  }
  }
 
-  private void assertRepositoryIsOperational(Client client, String repository) {
+ private void assertRepositoryIsOperational(Client client, String repository) {
  createIndex("test-idx-1");
  ensureGreen();
 
@@ -447,7 +447,7 @@ public void cleanRepositoryFiles(String basePath) {
  settings.getByPrefix("repositories.s3.private-bucket."),
  settings.getByPrefix("repositories.s3.remote-bucket."),
  settings.getByPrefix("repositories.s3.external-bucket.")
-  };
+ };
  for (Settings bucket : buckets) {
  String endpoint = bucket.get("endpoint", settings.get("repositories.s3.endpoint"));
  String protocol = bucket.get("protocol", settings.get("repositories.s3.protocol"));