Skip to content
Closed
Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
178e596
Added some logging messages. Made sure to shutdown executorService.
rkhaja Jul 26, 2019
ab3abfa
Measuring and reporting import performance.
rkhaja Jul 26, 2019
44aac4d
Automatically adjusting logging performance messages for second and m…
rkhaja Jul 26, 2019
34516a1
Adjusted logging messages.
rkhaja Jul 26, 2019
628c4ce
More logging.
rkhaja Jul 29, 2019
71b6d9c
Corrected counting of completed batches.
rkhaja Jul 29, 2019
4bdf76c
Attempt to combine individual DocumentImportEntity objects into singl…
rkhaja Jul 30, 2019
a651bd3
Consolidate logging messages.
rkhaja Jul 30, 2019
c8cd43c
Implemented 2 additional JUnit tessts for importDocumentsBatchSizeNum…
rkhaja Jul 30, 2019
83dddf1
Creating a combined DocumentImportEntity is not a good idea, because …
rkhaja Jul 30, 2019
2fed0d2
Put back combinedDocumentImportEntity. Adjusted measurement of perfor…
rkhaja Jul 30, 2019
1fd05a2
Fixed incorrect counts for combinedDocumentImportEntity.
rkhaja Jul 31, 2019
c1a5efc
Refactoring.
rkhaja Aug 1, 2019
d592ab0
Added Gson library to easily generate JSON string representation of o…
rkhaja Aug 2, 2019
6cd7ddc
Implemented toString() and toJSONString() methods.
rkhaja Aug 2, 2019
ec6891e
Implemented toString() and toJSONString() methods.
rkhaja Aug 2, 2019
4a7bb5a
Refactored loggin to use toJSONString() methods of DocumentImportOpti…
rkhaja Aug 2, 2019
54df698
Merge branch 'master' into feature/import-documents-logging
rkhaja Aug 6, 2019
0a3e40e
Made method parameters final.
rkhaja Aug 7, 2019
2f22b3f
Merge remote-tracking branch 'origin/feature/import-documents-logging…
rkhaja Aug 7, 2019
File filter

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Put back combinedDocumentImportEntity. Adjusted measurement of perfor…
…mance metrics.
  • Loading branch information
rkhaja committed Jul 30, 2019
commit 2fed0d2da3aacbbbd00fe899c51244479907ff12
51 changes: 44 additions & 7 deletions src/main/java/com/arangodb/internal/ArangoCollectionImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ public Collection<DocumentImportEntity> importDocuments(Collection<?> values, Do

int completedBatchNumber = 1;
List<DocumentImportEntity> documentImportEntityList = new ArrayList<>();
DocumentImportEntity combinedDocumentImportEntity = new DocumentImportEntity();
for (CompletableFuture<DocumentImportEntity> completableFuture : completableFutureList) {
DocumentImportEntity documentImportEntity = null;
try {
Expand All @@ -145,30 +146,66 @@ public Collection<DocumentImportEntity> importDocuments(Collection<?> values, Do
throw new ArangoDBException(e);
}
documentImportEntityList.add(documentImportEntity);
LOGGER.info("Completed import of batch: [{}/{}]. Created: [{}], Errors: [{}], Updated: [{}], Ignored: [{}],"
LOGGER.info("Completed import of batch: [{}/{}]. Created: [{}], Errors: [{}], Updated: [{}], Ignored: [{}],"
+ " Empty: [{}], Details: [{}]", completedBatchNumber, batches.size(),
documentImportEntity.getCreated(), documentImportEntity.getErrors(),
documentImportEntity.getUpdated(), documentImportEntity.getIgnored(),
documentImportEntity.getEmpty(), documentImportEntity.getDetails()
);
combinedDocumentImportEntity.setCreated(
combinedDocumentImportEntity.getCreated() == null ? 0 : combinedDocumentImportEntity.getCreated()
+ documentImportEntity.getCreated()
);
combinedDocumentImportEntity.setErrors(
combinedDocumentImportEntity.getErrors() == null ? 0 : combinedDocumentImportEntity.getErrors()
+ documentImportEntity.getErrors()
);
combinedDocumentImportEntity.setUpdated(
combinedDocumentImportEntity.getUpdated() == null ? 0 : combinedDocumentImportEntity.getUpdated()
+ documentImportEntity.getUpdated()
);
combinedDocumentImportEntity.setIgnored(
combinedDocumentImportEntity.getIgnored() == null ? 0 : combinedDocumentImportEntity.getIgnored()
+ documentImportEntity.getIgnored()
);
combinedDocumentImportEntity.setEmpty(
combinedDocumentImportEntity.getEmpty() == null ? 0 : combinedDocumentImportEntity.getEmpty()
+ documentImportEntity.getEmpty()
);
Collection<String> combinedDetails = new ArrayList<>();
combinedDetails.addAll(combinedDocumentImportEntity.getDetails());
combinedDetails.addAll(documentImportEntity.getDetails());
combinedDocumentImportEntity.setDetails(combinedDetails);
completedBatchNumber++;
}
LOGGER.info("Finished importing batches into collection: [{}].", this.name);
LOGGER.info("Totals. Created: [{}], Errors: [{}], Updated: [{}], Ignored: [{}], Empty: [{}]",
combinedDocumentImportEntity.getCreated(), combinedDocumentImportEntity.getErrors(),
combinedDocumentImportEntity.getUpdated(), combinedDocumentImportEntity.getIgnored(),
combinedDocumentImportEntity.getEmpty()
);

executorService.shutdown();
LOGGER.info("Shutdown fixed thread pool of [{}] threads.", numThreads);

long elapsedTimeMillis = System.currentTimeMillis() - startTimeMillis;
long elapsedTimeSeconds = TimeUnit.SECONDS.convert(elapsedTimeMillis, TimeUnit.MILLISECONDS);
int numDocumentsImported = combinedDocumentImportEntity.getCreated() + combinedDocumentImportEntity.getUpdated();
if( elapsedTimeSeconds > 0) {
String performance = String.format("%.2f", (float) values.size() / elapsedTimeSeconds);
LOGGER.info("Total number of documents imported: [{}] Time taken: [{}] seconds" +
" Performance: [{}] documents/second.", values.size(), elapsedTimeSeconds, performance);
String performance = String.format("%.2f", (float) numDocumentsImported / elapsedTimeSeconds);
LOGGER.info("Total number of documents imported (created + updated): [{}] Time taken: [{}] seconds" +
" Performance: [{}] documents/second.", numDocumentsImported, elapsedTimeSeconds, performance);
} else {
String performance = String.format("%.2f", (float) values.size() / elapsedTimeMillis);
LOGGER.info("Total number of documents imported: [{}] Time taken: [{}] milliseconds" +
" Performance: [{}] documents/millisecond.", values.size(), elapsedTimeMillis, performance);
String performance = String.format("%.2f", (float) numDocumentsImported / elapsedTimeMillis);
LOGGER.info("Total number of documents imported (created + updated): [{}] Time taken: [{}] milliseconds" +
" Performance: [{}] documents/millisecond.", numDocumentsImported, elapsedTimeMillis, performance);
}
// It would be nice to return a combinedDocumentImportEntity, but since details reflect the position of
// documents within each batch, such as when reporting errors as shown below it doesn't seem like a good idea to
// return a combinedDocumentImportEntity unless the positions can be adjusted to reflect the positions in the
// original Collection<?> values.
// [at position 9: creating document failed with error 'unique constraint violated', offending document: {"_key":"9"}]
// [at position 9: creating document failed with error 'unique constraint violated', offending document: {"_key":"19"}]
return documentImportEntityList;
}

Expand Down