""" Generate 100GB of mock data and upload to GCS via Private Google Access. Uses gcloud storage cp for fastest parallel upload. Usage: python3 upload_mock_data.py """ import os import subprocess import time # ↓↓↓ 请根据自己的环境修改 ↓↓↓ BUCKET_NAME = "sts-demo-singapore-src" # ↑↑↑ LOCAL_DIR = "/tmp/mock_data" CHUNK_SIZE_MB = 100 TOTAL_SIZE_GB = 100 NUM_FILES = TOTAL_SIZE_GB * 1024 // CHUNK_SIZE_MB # 1024 files def generate_local_files(): """Generate mock data files locally.""" os.makedirs(LOCAL_DIR, exist_ok=True) print(f"Generating {NUM_FILES} x {CHUNK_SIZE_MB}MB files in {LOCAL_DIR}/") for i in range(NUM_FILES): filepath = os.path.join(LOCAL_DIR, f"chunk_{i:04d}.bin") if os.path.exists(filepath) and os.path.getsize(filepath) == CHUNK_SIZE_MB * 1024 * 1024: print(f" Skipping {filepath} (already exists)") continue with open(filepath, "wb") as f: f.write(os.urandom(CHUNK_SIZE_MB * 1024 * 1024)) print(f" [{(i+1)/NUM_FILES*100:5.1f}%] Generated {filepath}") print(f"Done generating {TOTAL_SIZE_GB}GB of mock data.\n") def upload_via_gcloud(): """Upload using gcloud storage cp (recommended, fastest).""" print("Uploading via: gcloud storage cp -r (parallel, fastest)") print(f" Source: {LOCAL_DIR}/") print(f" Dest: gs://{BUCKET_NAME}/mock_data/") print("-" * 60) start = time.time() result = subprocess.run( [ "gcloud", "storage", "cp", "-r", f"{LOCAL_DIR}/*", f"gs://{BUCKET_NAME}/mock_data/", ], capture_output=True, text=True, ) elapsed = time.time() - start if result.returncode == 0: speed = (TOTAL_SIZE_GB * 1024) / elapsed print(f"Upload complete in {elapsed:.1f}s ({speed:.1f} MB/s)") else: print(f"Upload failed: {result.stderr}") return result.returncode == 0 if __name__ == "__main__": generate_local_files() upload_via_gcloud()