大数据系列之：上传图片到cos、cos
安装python cos sdk 上传图片到cos 高级上传接口 分页列举桶内对象 cos桶之间复制 cos桶之间复制图片数据
pip install -U cos-python-sdk-v5
from qcloud_cos import CosConfig
from qcloud_cos import CosS3Client
import sys
import os
import logging
logging. basicConfig( level= logging. INFO, stream= sys. stdout)
secret_id = "*********************"
secret_key = "********************"
region = 'ap-shanghai'
token = None
scheme = 'https'
config = CosConfig( Region= region, SecretId= secret_id, SecretKey= secret_key, Token= token, Scheme= scheme)
client = CosS3Client( config)
folder_path = ""
for root, dirs, files in os. walk( folder_path) :
for file in files:
if file . lower( ) . endswith( ( '.png' , '.jpg' , '.jpeg' , '.gif' ) ) :
file_path = os. path. join( root, file )
print ( file_path)
key = os. path. relpath( file_path, folder_path)
key_path = "" + key
print ( key_path)
client. upload_file(
Bucket= 'optics-data-1253431691' ,
LocalFilePath= file_path,
Key= key_path,
PartSize= 1 ,
MAXThread= 10 ,
EnableMD5= False
)
print ( f'Uploaded { file_path} to COS' )
根据文件大小自动选择简单上传或分块上传，分块上传具备断点续传功能。
response = client. upload_file(
Bucket= 'optics-data-1253431691' ,
LocalFilePath= '' ,
Key= '/picture.jpg' ,
PartSize= 1 ,
MAXThread= 10 ,
EnableMD5= False
)
print ( response[ 'ETag' ] )
marker = ""
while True :
response = client. list_objects(
Bucket= 'optics-data-1253431691' , Prefix= '' , Marker= marker, MaxKeys= 10 )
if 'Contents' in response:
for content in response[ 'Contents' ] :
allPictures. append( content[ 'Key' ] )
if response[ 'IsTruncated' ] == 'false' :
break
marker = response[ "NextMarker" ]
for source_path in allPictures:
print ( source_path)
picName = source_path. split( "/" ) [ - 1 ]
response = client. copy(
Bucket= 'optics-ai-data-1253431691' ,
Key= f"*/ { picName} " ,
CopySource= {
'Bucket' : 'optics-data-1253431691' ,
'Key' : source_path,
'Region' : 'ap-shanghai'
}
)
from qcloud_cos import CosConfig
from qcloud_cos import CosS3Client
import sys
import logging
logging. basicConfig( level= logging. INFO, stream= sys. stdout)
secret_id = "************"
secret_key = "***********"
region = 'ap-shanghai'
token = None
scheme = 'https'
config = CosConfig( Region= region, SecretId= secret_id, SecretKey= secret_key, Token= token, Scheme= scheme)
client = CosS3Client( config)
allPictures = [ ]
marker = ""
while True :
response = client. list_objects(
Bucket= 'optics-data-1253431691' , Prefix= '*/' , Marker= marker, MaxKeys= 10 )
if 'Contents' in response:
for content in response[ 'Contents' ] :
allPictures. append( content[ 'Key' ] )
if response[ 'IsTruncated' ] == 'false' :
break
marker = response[ "NextMarker" ]
for source_path in allPictures:
print ( source_path)
picName = source_path. split( "/" ) [ - 1 ]
response = client. copy(
Bucket= 'optics-ai-data-1253431691' ,
Key= f"*/ { picName} " ,
CopySource= {
'Bucket' : 'optics-data-1253431691' ,
'Key' : source_path,
'Region' : 'ap-shanghai'
}
)
print ( "成功把业务cos桶中图片同步到大数据cos桶中" )
