Skip to content

Commit bf43f62

Browse files
pedroslopezpedroslopez
authored
feat: generate full connector catalog json (#18562)
* move combo catalog generator from cloud to oss, trigger on processResources * generate catalog * regenerate catalog * add test * add explicit gradle task for generating combo catalog * run format * ignore generated file from the formatter * update generated catalog * ignore oss catalog * fix ignore path
1 parent 74792c1 commit bf43f62

File tree

5 files changed

+250
-0
lines changed

5 files changed

+250
-0
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@ docs/SUMMARY.md
7171
# Files generated by unit tests
7272
**/specs_secrets_mask.yaml
7373

74+
# Files generated for uploading to GCS
75+
airbyte-config/**/resources/seed/oss_catalog.json
76+
7477
# Helm charts .tgz dependencies
7578
charts/**/charts
7679

‎airbyte-config/specs/build.gradle‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ dependencies {
77

88
implementation project(':airbyte-commons')
99
implementation project(':airbyte-commons-cli')
10+
implementation project(':airbyte-commons-docker')
1011
implementation project(':airbyte-config:config-models')
1112
implementation project(':airbyte-protocol:protocol-models')
1213
implementation project(':airbyte-json-validation')
@@ -32,4 +33,18 @@ task generateConnectorSpecsMask(type: JavaExec, dependsOn: generateSeedConnector
3233

3334
project(":airbyte-config:init").tasks.processResources.dependsOn(generateConnectorSpecsMask)
3435

36+
task generateCombinedConnectorCatalog(type: JavaExec, dependsOn: generateSeedConnectorSpecs) {
37+
classpath = sourceSets.main.runtimeClasspath
38+
39+
mainClass = 'io.airbyte.config.specs.CombinedConnectorCatalogGenerator'
40+
41+
args '--seed-root'
42+
args new File(project(":airbyte-config:init").projectDir, '/src/main/resources/seed')
43+
44+
args '--output-filename'
45+
args 'oss_catalog.json'
46+
}
47+
48+
project(":airbyte-config:init").tasks.processResources.dependsOn(generateCombinedConnectorCatalog)
49+
3550
Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)

‎airbyte-config/specs/src/main/java/io/airbyte/config/specs/CombinedConnectorCatalogGenerator.java‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
/*
2+
* Copyright (c) 2022 Airbyte, Inc., all rights reserved.
3+
*/
4+
5+
package io.airbyte.config.specs;
6+
7+
import com.fasterxml.jackson.databind.JsonNode;
8+
import com.fasterxml.jackson.databind.node.BooleanNode;
9+
import com.fasterxml.jackson.databind.node.ObjectNode;
10+
import com.google.common.annotations.VisibleForTesting;
11+
import io.airbyte.commons.cli.Clis;
12+
import io.airbyte.commons.docker.DockerUtils;
13+
import io.airbyte.commons.io.IOs;
14+
import io.airbyte.commons.json.Jsons;
15+
import io.airbyte.commons.util.MoreIterators;
16+
import io.airbyte.commons.yaml.Yamls;
17+
import io.airbyte.config.AirbyteConfigValidator;
18+
import io.airbyte.config.CombinedConnectorCatalog;
19+
import io.airbyte.config.ConfigSchema;
20+
import io.airbyte.config.DockerImageSpec;
21+
import io.airbyte.config.StandardDestinationDefinition;
22+
import io.airbyte.config.StandardSourceDefinition;
23+
import java.nio.file.Path;
24+
import java.util.List;
25+
import java.util.Map;
26+
import java.util.stream.Collectors;
27+
import org.apache.commons.cli.CommandLine;
28+
import org.apache.commons.cli.Option;
29+
import org.apache.commons.cli.Options;
30+
31+
/**
32+
* Generates a combined representation of the connector catalog that includes Sources, Destinations
33+
* and their specs all in one. This connector catalog can then be served and loaded from a
34+
* RemoteDefinitionsProvider.
35+
*/
36+
public class CombinedConnectorCatalogGenerator {
37+
38+
private static final Option SEED_ROOT_OPTION = Option.builder("s").longOpt("seed-root").hasArg(true).required(true)
39+
.desc("path to where seed resource files are stored").build();
40+
private static final Option OUTPUT_FILENAME_OPTION = Option.builder("o").longOpt("output-filename").hasArg(true).required(true)
41+
.desc("name for the generated catalog json file").build();
42+
private static final Options OPTIONS = new Options().addOption(SEED_ROOT_OPTION).addOption(OUTPUT_FILENAME_OPTION);
43+
44+
public static void main(final String[] args) throws Exception {
45+
final CommandLine parsed = Clis.parse(args, OPTIONS);
46+
final Path outputRoot = Path.of(parsed.getOptionValue(SEED_ROOT_OPTION.getOpt()));
47+
final String outputFileName = parsed.getOptionValue(OUTPUT_FILENAME_OPTION.getOpt());
48+
49+
final CombinedConnectorCatalogGenerator combinedConnectorCatalogGenerator = new CombinedConnectorCatalogGenerator();
50+
combinedConnectorCatalogGenerator.run(outputRoot, outputFileName);
51+
}
52+
53+
public void run(final Path outputRoot, final String outputFileName) {
54+
final List<JsonNode> destinationDefinitionsJson = getSeedJson(outputRoot, SeedConnectorType.DESTINATION.getDefinitionFileName());
55+
final List<JsonNode> destinationSpecsJson = getSeedJson(outputRoot, SeedConnectorType.DESTINATION.getSpecFileName());
56+
final List<JsonNode> sourceDefinitionsJson = getSeedJson(outputRoot, SeedConnectorType.SOURCE.getDefinitionFileName());
57+
final List<JsonNode> sourceSpecsJson = getSeedJson(outputRoot, SeedConnectorType.SOURCE.getSpecFileName());
58+
59+
mergeSpecsIntoDefinitions(destinationDefinitionsJson, destinationSpecsJson, ConfigSchema.STANDARD_DESTINATION_DEFINITION);
60+
mergeSpecsIntoDefinitions(sourceDefinitionsJson, sourceSpecsJson, ConfigSchema.STANDARD_SOURCE_DEFINITION);
61+
62+
final CombinedConnectorCatalog combinedCatalog = new CombinedConnectorCatalog()
63+
.withDestinations(destinationDefinitionsJson.stream().map(j -> Jsons.object(j, StandardDestinationDefinition.class)).toList())
64+
.withSources(sourceDefinitionsJson.stream().map(j -> Jsons.object(j, StandardSourceDefinition.class)).toList());
65+
66+
IOs.writeFile(outputRoot.resolve(outputFileName), Jsons.toPrettyString(Jsons.jsonNode(combinedCatalog)));
67+
}
68+
69+
private List<JsonNode> getSeedJson(final Path root, final String fileName) {
70+
final String jsonString = IOs.readFile(root, fileName);
71+
return MoreIterators.toList(Yamls.deserialize(jsonString).elements());
72+
}
73+
74+
/**
75+
* Updates all connector definitions with provided specs.
76+
*
77+
* @param definitions - List of Source or Destination Definitions as generated in the seed files
78+
* @param specs - List of connector specs as generated in the seed files (see
79+
* {@link DockerImageSpec})
80+
*/
81+
@VisibleForTesting
82+
void mergeSpecsIntoDefinitions(final List<JsonNode> definitions, final List<JsonNode> specs, final ConfigSchema configSchema) {
83+
final Map<String, JsonNode> specsByImage = specs.stream().collect(Collectors.toMap(
84+
json -> json.get("dockerImage").asText(),
85+
json -> json.get("spec")));
86+
87+
for (final JsonNode definition : definitions) {
88+
final String dockerImage = DockerUtils.getTaggedImageName(
89+
definition.get("dockerRepository").asText(),
90+
definition.get("dockerImageTag").asText());
91+
final JsonNode specConfigJson = specsByImage.get(dockerImage);
92+
93+
if (specConfigJson == null) {
94+
throw new UnsupportedOperationException(String.format("A spec for docker image %s was not found", dockerImage));
95+
}
96+
97+
((ObjectNode) definition).set("spec", specConfigJson);
98+
99+
if (!definition.hasNonNull("public")) {
100+
// All definitions in the catalog are public by default
101+
((ObjectNode) definition).set("public", BooleanNode.TRUE);
102+
}
103+
104+
AirbyteConfigValidator.AIRBYTE_CONFIG_VALIDATOR.ensureAsRuntime(configSchema, definition);
105+
}
106+
107+
}
108+
109+
}

‎airbyte-config/specs/src/test/java/io/airbyte/config/specs/CombinedConnectorCatalogGeneratorTest.java‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/*
2+
* Copyright (c) 2022 Airbyte, Inc., all rights reserved.
3+
*/
4+
5+
package io.airbyte.config.specs;
6+
7+
import static org.junit.jupiter.api.Assertions.assertEquals;
8+
import static org.junit.jupiter.api.Assertions.assertThrows;
9+
10+
import com.fasterxml.jackson.databind.JsonNode;
11+
import com.google.common.collect.ImmutableMap;
12+
import io.airbyte.commons.json.Jsons;
13+
import io.airbyte.config.ConfigSchema;
14+
import io.airbyte.config.DockerImageSpec;
15+
import io.airbyte.config.StandardDestinationDefinition;
16+
import io.airbyte.protocol.models.ConnectorSpecification;
17+
import java.util.List;
18+
import java.util.UUID;
19+
import org.junit.jupiter.api.BeforeEach;
20+
import org.junit.jupiter.api.Test;
21+
22+
class CombinedConnectorCatalogGeneratorTest {
23+
24+
private static final UUID DEF_ID1 = UUID.randomUUID();
25+
private static final UUID DEF_ID2 = UUID.randomUUID();
26+
private static final String CONNECTOR_NAME1 = "connector1";
27+
private static final String CONNECTOR_NAME2 = "connector2";
28+
private static final String DOCUMENTATION_URL = "https://www.example.com";
29+
private static final String DOCKER_REPOSITORY1 = "airbyte/connector1";
30+
private static final String DOCKER_REPOSITORY2 = "airbyte/connector2";
31+
private static final String DOCKER_TAG1 = "0.1.0";
32+
private static final String DOCKER_TAG2 = "0.2.0";
33+
34+
private CombinedConnectorCatalogGenerator catalogGenerator;
35+
36+
@BeforeEach
37+
void setup() {
38+
catalogGenerator = new CombinedConnectorCatalogGenerator();
39+
}
40+
41+
@Test
42+
void testMergeSpecsIntoDefinitions() {
43+
final StandardDestinationDefinition destinationDefinition1 = new StandardDestinationDefinition()
44+
.withDestinationDefinitionId(DEF_ID1)
45+
.withDockerRepository(DOCKER_REPOSITORY1)
46+
.withDockerImageTag(DOCKER_TAG1)
47+
.withName(CONNECTOR_NAME1)
48+
.withDocumentationUrl(DOCUMENTATION_URL)
49+
.withSpec(new ConnectorSpecification());
50+
final StandardDestinationDefinition destinationDefinition2 = new StandardDestinationDefinition()
51+
.withDestinationDefinitionId(DEF_ID2)
52+
.withDockerRepository(DOCKER_REPOSITORY2)
53+
.withDockerImageTag(DOCKER_TAG2)
54+
.withName(CONNECTOR_NAME2)
55+
.withDocumentationUrl(DOCUMENTATION_URL)
56+
.withSpec(new ConnectorSpecification());
57+
final DockerImageSpec destinationSpec1 = new DockerImageSpec().withDockerImage(DOCKER_REPOSITORY1 + ":" + DOCKER_TAG1)
58+
.withSpec(new ConnectorSpecification().withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of(
59+
"foo1",
60+
"bar1"))));
61+
final DockerImageSpec destinationSpec2 = new DockerImageSpec().withDockerImage(DOCKER_REPOSITORY2 + ":" + DOCKER_TAG2)
62+
.withSpec(new ConnectorSpecification().withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of(
63+
"foo2",
64+
"bar2"))));
65+
66+
final List<JsonNode> definitions = List.of(Jsons.jsonNode(destinationDefinition1), Jsons.jsonNode(destinationDefinition2));
67+
final List<JsonNode> specs = List.of(Jsons.jsonNode(destinationSpec1), Jsons.jsonNode(destinationSpec2));
68+
69+
catalogGenerator.mergeSpecsIntoDefinitions(definitions, specs, ConfigSchema.STANDARD_DESTINATION_DEFINITION);
70+
71+
final StandardDestinationDefinition expectedDefinition1 = new StandardDestinationDefinition()
72+
.withDestinationDefinitionId(DEF_ID1)
73+
.withDockerRepository(DOCKER_REPOSITORY1)
74+
.withDockerImageTag(DOCKER_TAG1)
75+
.withName(CONNECTOR_NAME1)
76+
.withDocumentationUrl(DOCUMENTATION_URL)
77+
.withSpec(destinationSpec1.getSpec());
78+
79+
final StandardDestinationDefinition expectedDefinition2 = new StandardDestinationDefinition()
80+
.withDestinationDefinitionId(DEF_ID2)
81+
.withDockerRepository(DOCKER_REPOSITORY2)
82+
.withDockerImageTag(DOCKER_TAG2)
83+
.withName(CONNECTOR_NAME2)
84+
.withDocumentationUrl(DOCUMENTATION_URL)
85+
.withSpec(destinationSpec2.getSpec());
86+
87+
assertEquals(Jsons.jsonNode(expectedDefinition1), definitions.get(0));
88+
assertEquals(Jsons.jsonNode(expectedDefinition2), definitions.get(1));
89+
}
90+
91+
@Test
92+
void testMergeSpecsIntoDefinitionsThrowsOnMissingSpec() {
93+
final StandardDestinationDefinition destinationDefinition1 = new StandardDestinationDefinition()
94+
.withDestinationDefinitionId(DEF_ID1)
95+
.withDockerRepository(DOCKER_REPOSITORY1)
96+
.withDockerImageTag(DOCKER_TAG1)
97+
.withName(CONNECTOR_NAME1)
98+
.withDocumentationUrl(DOCUMENTATION_URL)
99+
.withSpec(new ConnectorSpecification());
100+
final List<JsonNode> definitions = List.of(Jsons.jsonNode(destinationDefinition1));
101+
final List<JsonNode> specs = List.of();
102+
103+
assertThrows(UnsupportedOperationException.class,
104+
() -> catalogGenerator.mergeSpecsIntoDefinitions(definitions, specs, ConfigSchema.STANDARD_DESTINATION_DEFINITION));
105+
}
106+
107+
@Test
108+
void testMergeSpecsIntoDefinitionsThrowsOnInvalidFormat() {
109+
final JsonNode invalidDefinition = Jsons.jsonNode(ImmutableMap.of("dockerRepository", DOCKER_REPOSITORY1, "dockerImageTag", DOCKER_TAG1));
110+
final DockerImageSpec destinationSpec = new DockerImageSpec().withDockerImage(DOCKER_REPOSITORY1 + ":" + DOCKER_TAG1)
111+
.withSpec(new ConnectorSpecification().withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of(
112+
"foo1",
113+
"bar1"))));
114+
115+
final List<JsonNode> definitions = List.of(Jsons.jsonNode(invalidDefinition));
116+
final List<JsonNode> specs = List.of(Jsons.jsonNode(destinationSpec));
117+
118+
assertThrows(RuntimeException.class,
119+
() -> catalogGenerator.mergeSpecsIntoDefinitions(definitions, specs, ConfigSchema.STANDARD_DESTINATION_DEFINITION));
120+
}
121+
122+
}

‎build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ def createSpotlessTarget = { pattern ->
121121
'secrets',
122122
'charts', // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately.
123123
'resources/seed/*_specs.yaml', // Do not remove - this is necessary to prevent diffs in our github workflows, as the file diff check runs between the Format step and the Build step, the latter of which generates the file.
124+
'resources/seed/*_catalog.json', // Do not remove - this is also necessary to prevent diffs in our github workflows
124125
'airbyte-integrations/connectors/source-amplitude/unit_tests/api_data/zipped.json', // Zipped file presents as non-UTF-8 making spotless sad
125126
'airbyte-webapp', // The webapp module uses its own auto-formatter, so spotless is not necessary here
126127
'airbyte-webapp-e2e-tests', // This module also uses its own auto-formatter

0 commit comments

Comments
 (0)