Skip to content

feat(file-api): new upload component for declarative cdk and update protocol support for file-based connectors #433

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Apr 22, 2025
Merged

feat(file-api): new upload component for declarative cdk and update protocol support for file-based connectors #433

Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
d1d5107
PoC for file upload
Mar 19, 2025
de09ae8
Auto-fix lint and format issues
Mar 19, 2025
b410187
fix mypy
Apr 2, 2025
b122c0a
PoC for emit file reference record (#443)
aldogonzalez8 Apr 2, 2025
4c29c91
merge from main
aldogonzalez8 Apr 15, 2025
b5b8197
feat(PoC): adjust file-based and file uploader component to latest pr…
aldogonzalez8 Apr 17, 2025
d060a8b
merge from main
aldogonzalez8 Apr 17, 2025
87c20b6
poetry lock
aldogonzalez8 Apr 17, 2025
ab5d4e7
file-api: add checks for missin values/paths
aldogonzalez8 Apr 17, 2025
5242876
file-api: fix unit tests
aldogonzalez8 Apr 17, 2025
8f14512
Auto-fix lint and format issues
Apr 17, 2025
0b449a6
file-mode-api: remove unnecessary check
aldogonzalez8 Apr 19, 2025
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
file-api: fix unit tests
  • Loading branch information
@aldogonzalez8
aldogonzalez8 committed Apr 17, 2025
commit 524287645b70e89f64a22b39550bc9c440a797d7
12 changes: 7 additions & 5 deletions unit_tests/sources/file_based/test_file_based_stream_reader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,11 @@
from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
from airbyte_cdk.sources.file_based.remote_file import RemoteFile
from airbyte_cdk.sources.utils.files_directory import get_files_directory
from unit_tests.sources.file_based.helpers import make_remote_files

reader = AbstractFileBasedStreamReader
files_directory = get_files_directory()

"""
The rules are:
Expand Down Expand Up @@ -402,7 +404,7 @@ def test_globs_and_prefixes_from_globs(
},
"mirror_paths_testing/not_duplicates/data/jan/monthly-kickoff-202402.mpeg",
"mirror_paths_testing/not_duplicates/data/jan/monthly-kickoff-202402.mpeg",
"/tmp/transfer-files/mirror_paths_testing/not_duplicates/data/jan/monthly-kickoff-202402.mpeg",
f"{files_directory}/mirror_paths_testing/not_duplicates/data/jan/monthly-kickoff-202402.mpeg",
id="preserve_directories_present_and_true",
),
pytest.param(
Expand All @@ -415,21 +417,21 @@ def test_globs_and_prefixes_from_globs(
},
"mirror_paths_testing/not_duplicates/data/jan/monthly-kickoff-202402.mpeg",
"monthly-kickoff-202402.mpeg",
"/tmp/transfer-files/monthly-kickoff-202402.mpeg",
f"{files_directory}/monthly-kickoff-202402.mpeg",
id="preserve_directories_present_and_false",
),
pytest.param(
{"streams": [], "delivery_method": {"delivery_type": "use_file_transfer"}},
"mirror_paths_testing/not_duplicates/data/jan/monthly-kickoff-202402.mpeg",
"mirror_paths_testing/not_duplicates/data/jan/monthly-kickoff-202402.mpeg",
"/tmp/transfer-files/mirror_paths_testing/not_duplicates/data/jan/monthly-kickoff-202402.mpeg",
f"{files_directory}/mirror_paths_testing/not_duplicates/data/jan/monthly-kickoff-202402.mpeg",
id="preserve_directories_not_present_defaults_true",
),
pytest.param(
{"streams": []},
"mirror_paths_testing/not_duplicates/data/jan/monthly-kickoff-202402.mpeg",
"mirror_paths_testing/not_duplicates/data/jan/monthly-kickoff-202402.mpeg",
"/tmp/transfer-files/mirror_paths_testing/not_duplicates/data/jan/monthly-kickoff-202402.mpeg",
f"{files_directory}/mirror_paths_testing/not_duplicates/data/jan/monthly-kickoff-202402.mpeg",
id="file_transfer_flag_not_present_defaults_true",
),
],
Expand All @@ -447,7 +449,7 @@ def test_preserve_sub_directories_scenarios(
reader = TestStreamReader()
reader.config = TestSpec(**config)
file_paths = reader._get_file_transfer_paths(
source_file_path, staging_directory="/tmp/transfer-files/"
source_file_path, staging_directory=f"{files_directory}/"
)

assert (
Expand Down
Loading