""" Create an STS transfer job (Singapore -> Netherlands) and monitor its status. Based on official Google Cloud samples: https://github.com/GoogleCloudPlatform/python-docs-samples/tree/main/storagetransfer Usage: python3 sts_transfer.py """ import json import time from google.cloud import storage_transfer # ↓↓↓ 请根据自己的环境修改 ↓↓↓ PROJECT_ID = "gpu-launchpad-playground" SRC_BUCKET = "sts-demo-singapore-src" DST_BUCKET = "sts-demo-netherlands-dst" # ↑↑↑ def create_transfer_job(client): transfer_job_request = storage_transfer.CreateTransferJobRequest( transfer_job=storage_transfer.TransferJob( project_id=PROJECT_ID, description=f"STS: {SRC_BUCKET} -> {DST_BUCKET}", transfer_spec=storage_transfer.TransferSpec( gcs_data_source=storage_transfer.GcsData(bucket_name=SRC_BUCKET), gcs_data_sink=storage_transfer.GcsData(bucket_name=DST_BUCKET), ), status=storage_transfer.TransferJob.Status.ENABLED, ) ) result = client.create_transfer_job(request=transfer_job_request) print(f"Created transfer job: {result.name}") return result def run_transfer_job(client, job_name): request = storage_transfer.RunTransferJobRequest( job_name=job_name, project_id=PROJECT_ID, ) operation = client.run_transfer_job(request=request) print("Transfer operation started. Polling for completion...") return operation def poll_transfer_status(client, job_name): """Poll the transfer operation status until completion.""" poll_interval = 10 last_bytes = 0 ops_client = client.transport.operations_client job_filter = json.dumps({"projectId": PROJECT_ID, "jobNames": [job_name]}) while True: response = ops_client.list_operations("transferOperations", job_filter) operations = list(response) if not operations: print(" Waiting for operation to appear...") time.sleep(poll_interval) continue op = operations[-1] metadata = storage_transfer.TransferOperation.deserialize(op.metadata.value) status = storage_transfer.TransferOperation.Status(metadata.status) counters = metadata.counters bytes_found = counters.bytes_found_from_source bytes_copied = counters.bytes_copied_to_sink objects_copied = counters.objects_copied_to_sink objects_found = counters.objects_found_from_source speed_mbps = 0.0 if bytes_copied > last_bytes: speed_mbps = (bytes_copied - last_bytes) / (poll_interval * 1024 * 1024) last_bytes = bytes_copied progress = (bytes_copied / bytes_found * 100) if bytes_found > 0 else 0.0 print( f" Status: {status.name:12s} | " f"Progress: {progress:5.1f}% | " f"Copied: {bytes_copied / (1024**3):.2f}/{bytes_found / (1024**3):.2f} GB | " f"Objects: {objects_copied}/{objects_found} | " f"Speed: ~{speed_mbps:.1f} MB/s" ) if status == storage_transfer.TransferOperation.Status.SUCCESS: print("\nTransfer completed successfully!") print_final_stats(counters, metadata) return True elif status in ( storage_transfer.TransferOperation.Status.FAILED, storage_transfer.TransferOperation.Status.ABORTED, ): print(f"\nTransfer {status.name}!") if metadata.error_breakdowns: for err in metadata.error_breakdowns: print(f" Error: {err}") return False time.sleep(poll_interval) def print_final_stats(counters, metadata): """Print final transfer statistics.""" print("-" * 60) print("Final Transfer Statistics:") print(f" Bytes found from source: {counters.bytes_found_from_source / (1024**3):.2f} GB") print(f" Bytes copied to sink: {counters.bytes_copied_to_sink / (1024**3):.2f} GB") print(f" Objects found: {counters.objects_found_from_source}") print(f" Objects copied: {counters.objects_copied_to_sink}") print(f" Objects skipped (sync): {counters.objects_from_source_skipped_by_sync}") print(f" Objects failed: {counters.objects_from_source_failed}") if metadata.start_time and metadata.end_time: duration = metadata.end_time.timestamp() - metadata.start_time.timestamp() if duration > 0: avg_speed = counters.bytes_copied_to_sink / duration / (1024 * 1024) print(f" Duration: {duration:.1f}s") print(f" Average speed: {avg_speed:.1f} MB/s") print("-" * 60) def check_latest_operation(client, job_name): """ Check the latest transfer operation status. Based on: check_latest_transfer_operation.py from official samples. Returns (is_complete: bool, status_name: str, metadata: TransferOperation|None) """ transfer_job = client.get_transfer_job({ "project_id": PROJECT_ID, "job_name": job_name, }) if not transfer_job.latest_operation_name: return False, "NO_OPERATION", None response = client.transport.operations_client.get_operation( transfer_job.latest_operation_name ) metadata = storage_transfer.TransferOperation.deserialize(response.metadata.value) status = storage_transfer.TransferOperation.Status(metadata.status) is_complete = status in ( storage_transfer.TransferOperation.Status.SUCCESS, storage_transfer.TransferOperation.Status.FAILED, storage_transfer.TransferOperation.Status.ABORTED, ) return is_complete, status.name, metadata def main(): client = storage_transfer.StorageTransferServiceClient() print("=" * 60) print("STS Transfer Demo: Singapore -> Netherlands") print("=" * 60) # Step 1: Create transfer job job = create_transfer_job(client) # Step 2: Run the transfer job immediately run_transfer_job(client, job.name) # Step 3: Poll status until completion print("\nPolling transfer status (every 10s):") success = poll_transfer_status(client, job.name) # Step 4: Demo the completion-check utility print("\n--- Completion Check Demo ---") is_complete, status_name, metadata = check_latest_operation(client, job.name) print(f" is_complete = {is_complete}") print(f" status = {status_name}") if is_complete and status_name == "SUCCESS": print(" -> Ready for next step!") else: print(f" -> Transfer ended with status: {status_name}") if __name__ == "__main__": main()