We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 762454c commit 71b05c3Copy full SHA for 71b05c3
utils.py
@@ -44,11 +44,12 @@ def check_for_json_files(file_path: str) -> bool:
44
def download_files_from_gcs(file_path: str) -> None:
45
"""Download from GCS to local directory"""
46
try:
47
+ source_prefix = f"{SCRAPE_LAKE_PREFIX}/{file_path}"
48
+ logger.info(f"Attempting to download from {source_prefix}")
49
+
50
cloud_storage_client = storage.Client(project=GCP_PROJECT)
51
bucket = cloud_storage_client.bucket(BUCKET_NAME)
- source_prefix = f"{SCRAPE_LAKE_PREFIX}/{file_path}"
52
- logger.info(f"Attempting to download from {source_prefix}")
53
blobs = bucket.list_blobs(prefix=source_prefix)
54
datadir = f"_data/{file_path}"
55
# Create local file directory
0 commit comments