2019-07-01 13:21:06 +00:00
|
|
|
# This file contains google utils: https://cloud.google.com/storage/docs/reference/libraries
|
|
|
|
|
# pip install --upgrade google-cloud-storage
|
|
|
|
|
|
2019-08-04 15:34:58 +00:00
|
|
|
import os
|
2019-08-04 15:50:20 +00:00
|
|
|
import time
|
2019-08-04 15:34:58 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
# from google.cloud import storage
|
|
|
|
|
|
|
|
|
|
|
2019-08-04 15:50:20 +00:00
|
|
|
def gdrive_download(id='1HaXkef9z6y5l4vUnCYgdmEAj61c6bfWO', name='coco.zip'):
|
2019-08-04 15:34:58 +00:00
|
|
|
# https://gist.github.com/tanaikech/f0f2d122e05bf5f971611258c22c110f
|
|
|
|
|
# Downloads a file from Google Drive, accepting presented query
|
|
|
|
|
# from utils.google_utils import *; gdrive_download()
|
2019-08-04 15:50:20 +00:00
|
|
|
t = time.time()
|
|
|
|
|
|
2019-08-04 15:53:03 +00:00
|
|
|
print('Downloading https://drive.google.com/uc?export=download&id=%s as %s... ' % (id, name), end='')
|
2019-08-04 15:50:20 +00:00
|
|
|
if os.path.exists(name): # remove existing
|
|
|
|
|
os.remove(name)
|
|
|
|
|
|
|
|
|
|
# Attempt small file download
|
|
|
|
|
s = 'curl -f -L -o %s https://drive.google.com/uc?export=download&id=%s' % (name, id)
|
|
|
|
|
os.system(s)
|
|
|
|
|
|
|
|
|
|
# Attempt large file download
|
|
|
|
|
if not os.path.exists(name): # file size > 40MB
|
|
|
|
|
s = ["curl -c ./cookie -s -L \"https://drive.google.com/uc?export=download&id=%s\" > /dev/null" % id,
|
|
|
|
|
"curl -Lb ./cookie \"https://drive.google.com/uc?export=download&confirm=`awk '/download/ {print $NF}' ./cookie`&id=%s\" -o %s" % (
|
|
|
|
|
id, name),
|
|
|
|
|
'rm ./cookie']
|
|
|
|
|
[os.system(x) for x in s] # run commands
|
|
|
|
|
|
|
|
|
|
# Unzip if archive
|
|
|
|
|
if name.endswith('.zip'):
|
2019-08-04 15:54:03 +00:00
|
|
|
print('unzipping... ', end='')
|
2019-08-04 17:59:02 +00:00
|
|
|
os.system('unzip -q %s' % name) # unzip
|
|
|
|
|
os.remove(name) # remove zip to free space
|
2019-08-04 15:50:20 +00:00
|
|
|
|
2019-08-04 15:53:03 +00:00
|
|
|
print('Done (%.1fs)' % (time.time() - t))
|
2019-07-01 13:21:06 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def upload_blob(bucket_name, source_file_name, destination_blob_name):
|
|
|
|
|
# Uploads a file to a bucket
|
|
|
|
|
# https://cloud.google.com/storage/docs/uploading-objects#storage-upload-object-python
|
|
|
|
|
|
|
|
|
|
storage_client = storage.Client()
|
|
|
|
|
bucket = storage_client.get_bucket(bucket_name)
|
|
|
|
|
blob = bucket.blob(destination_blob_name)
|
|
|
|
|
|
|
|
|
|
blob.upload_from_filename(source_file_name)
|
|
|
|
|
|
|
|
|
|
print('File {} uploaded to {}.'.format(
|
|
|
|
|
source_file_name,
|
|
|
|
|
destination_blob_name))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def download_blob(bucket_name, source_blob_name, destination_file_name):
|
|
|
|
|
# Uploads a blob from a bucket
|
|
|
|
|
storage_client = storage.Client()
|
|
|
|
|
bucket = storage_client.get_bucket(bucket_name)
|
|
|
|
|
blob = bucket.blob(source_blob_name)
|
|
|
|
|
|
|
|
|
|
blob.download_to_filename(destination_file_name)
|
|
|
|
|
|
|
|
|
|
print('Blob {} downloaded to {}.'.format(
|
|
|
|
|
source_blob_name,
|
|
|
|
|
destination_file_name))
|
