[source-hubspot] - migrate deals_pipelinesto low code (#59195)

Author: pnilan

airbyte-integrations/connectors/source-hubspot/metadata.yaml

@@ -10,7 +10,7 @@ data:
  connectorSubtype: api
  connectorType: source
  definitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
- dockerImageTag: 4.9.0
+ dockerImageTag: 4.10.0
  dockerRepository: airbyte/source-hubspot
  documentationUrl: https://docs.airbyte.com/integrations/sources/hubspot
  erdUrl: https://dbdocs.io/airbyteio/source-hubspot?view=relationships

airbyte-integrations/connectors/source-hubspot/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "4.9.0"
+version = "4.10.0"
 name = "source-hubspot"
 description = "Source implementation for HubSpot."
 authors = [ "Airbyte <contact@airbyte.io>",]

airbyte-integrations/connectors/source-hubspot/source_hubspot/manifest.yaml

@@ -579,14 +579,56 @@ definitions:
  type: KeyTransformation
  prefix: counters_
 
+ deal_pipelines_stream:
+ $ref: "#/definitions/stream_base"
+ name: deal_pipelines
+ primary_key:
+ - pipelineId
+ retriever:
+ type: SimpleRetriever
+ requester:
+ $ref: "#/definitions/base_requester"
+ path: /crm-pipelines/v1/pipelines/deals
+ record_selector:
+ type: RecordSelector
+ transform_before_filtering: true
+ extractor:
+ type: DpathExtractor
+ field_path:
+ - results
+ decoder:
+ type: JsonDecoder
+ incremental_sync:
+ type: DatetimeBasedCursor
+ cursor_field: updatedAt
+ start_datetime:
+ type: MinMaxDatetime
+ datetime: "{{ format_datetime(config.get('start_date', '2006-06-01T00:00:00Z'), '%ms', '%Y-%m-%dT%H:%M:%SZ') }}"
+ datetime_format: "%ms"
+ datetime_format: "%ms"
+ lookback_window: P{{ config.get('lookback_window', 0) }}D
+ cursor_datetime_formats:
+ - "%ms"
+ is_client_side_incremental: true
+ schema_loader:
+ type: InlineSchemaLoader
+ schema:
+ $ref: "#/schemas/deal_pipelines"
+ transformations:
+ - type: AddFields
+ fields:
+ - path: ["updatedAt"]
+ value: "{{ record.get('updatedAt') or record['createdAt'] }}"
+
 streams:
+ - "#/definitions/campaigns_stream"
  - "#/definitions/companies_property_history_stream"
  - "#/definitions/contacts_property_history_stream"
+ - "#/definitions/deal_pipelines_stream"
  - "#/definitions/deals_property_history_stream"
  - "#/definitions/email_subscriptions_stream"
  - "#/definitions/marketing_emails_stream"
  - "#/definitions/ticket_pipelines_stream"
- - "#/definitions/campaigns_stream"
 
 # HubSpot account is limited to 110 requests every 10 seconds https://developers.hubspot.com/docs/guides/apps/api-usage/usage-details#rate-limits
 concurrency_level:
@@ -595,6 +637,111 @@ concurrency_level:
  max_concurrency: 40
 
 schemas:
+ deal_pipelines:
+ $schema: http://json-schema.org/draft-07/schema#
+ type:
+ - "null"
+ - object
+ additionalProperties: true
+ properties:
+ label:
+ description: The label or name of the deal pipeline.
+ type:
+ - "null"
+ - string
+ displayOrder:
+ description: The ordering of the deal pipeline for display.
+ type:
+ - "null"
+ - integer
+ active:
+ description: Indicates if the deal pipeline is currently active or not.
+ type:
+ - "null"
+ - boolean
+ stages:
+ description: List of deal stages within the pipeline.
+ type:
+ - "null"
+ - array
+ items:
+ type:
+ - "null"
+ - object
+ properties:
+ label:
+ description: The label or name of the deal stage.
+ type:
+ - "null"
+ - string
+ displayOrder:
+ description: The ordering of the deal stage for display within the pipeline.
+ type:
+ - "null"
+ - integer
+ metadata:
+ description: Additional information related to the deal stage.
+ type:
+ - "null"
+ - object
+ properties:
+ isClosed:
+ description: Indicates if the deal stage is considered closed or not.
+ type:
+ - "null"
+ - string
+ probability:
+ description: The probability of closing a deal at this stage.
+ type:
+ - "null"
+ - string
+ stageId:
+ description: The unique identifier of the deal stage.
+ type:
+ - "null"
+ - string
+ createdAt:
+ description: Timestamp for the creation date of the deal stage.
+ type:
+ - "null"
+ - integer
+ updatedAt:
+ description: Timestamp for the last update to the deal stage.
+ type:
+ - "null"
+ - integer
+ active:
+ description: Indicates if the deal stage is currently active or not.
+ type:
+ - "null"
+ - boolean
+ objectType:
+ description: The type of object this deal pipeline is associated with.
+ type:
+ - "null"
+ - string
+ objectTypeId:
+ description: The ID of the object type this deal pipeline is associated with.
+ type:
+ - "null"
+ - string
+ pipelineId:
+ description: The unique identifier of the deal pipeline.
+ type:
+ - "null"
+ - string
+ createdAt:
+ description: Timestamp for the creation date of the deal pipeline.
+ type:
+ - "null"
+ - integer
+ updatedAt:
+ description: Timestamp for the last update to the deal pipeline.
+ type:
+ - "null"
+ - integer
+ default:
+ description: Indicates if this pipeline is the default one in the system.
  companies_property_history:
  $schema: "http://json-schema.org/draft-07/schema"
  type:

airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deal_pipelines.json

@@ -29,7 +29,6 @@
  ContactsMergedAudit,
  ContactsWebAnalytics,
  CustomObject,
- DealPipelines,
  Deals,
  DealsArchived,
  DealSplits,
@@ -74,6 +73,7 @@
 scopes = {
  "companies_property_history": {"crm.objects.companies.read"},
  "contacts_property_history": {"crm.objects.contacts.read"},
+ "deal_pipelines": {"crm.objects.contacts.read"},
  "deals_property_history": {"crm.objects.deals.read"},
  "email_subscriptions": {"content"},
  "marketing_emails": {"content"},
@@ -199,7 +199,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
  ContactsFormSubmissions(**common_params),
  ContactsListMemberships(**common_params),
  ContactsMergedAudit(**common_params),
- DealPipelines(**common_params),
  DealSplits(**common_params),
  Deals(**common_params),
  DealsArchived(**common_params),

airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py

@@ -1578,20 +1578,6 @@ def request_params(
  return params
 
 
-class DealPipelines(ClientSideIncrementalStream):
- """Deal pipelines, API v1,
- This endpoint requires the contacts scope the tickets scope.
- Docs: https://legacydocs.hubspot.com/docs/methods/pipelines/get_pipelines_for_object_type
- """
-
- url = "/crm-pipelines/v1/pipelines/deals"
- updated_at_field = "updatedAt"
- created_at_field = "createdAt"
- cursor_field_datetime_format = "x"
- primary_key = "pipelineId"
- scopes = {"crm.objects.contacts.read"}
-
-
 class DealSplits(CRMSearchStream):
  """Deal splits, API v3"""