parallelized multipart s3 uploads
root@ip-10-108-242-120:~# s3_multi_upload --access-key-id xxx \
--secret-access-key xxx \
--bucket s3_multi_upload \
--chunk-size 25:mb \
--threads 40 \
--key uploads/u.iso \
--file ubuntu-12.04-beta2-dvd-amd64.iso
uploading ubuntu-12.04-beta2-dvd-amd64.iso to s3://s3_multi_upload/uploads/u.iso using 40 threads in chunks of 25mb
upload: 20% |oooooooooooooooooooooo | ETA: 0:01:36installation
gem install s3_multi_upload
docker
docker pull dallasmarlow/s3_multi_upload
usage
Usage:
s3_multi_upload --access-key-id=ACCESS_KEY_ID \
--secret-access-key=SECRET_ACCESS_KEY \
--bucket=BUCKET \
--file=FILE
Options:
--access-key-id=ACCESS_KEY_ID
--secret-access-key=SECRET_ACCESS_KEY
--bucket=BUCKET
--file=FILE
--threads=N
# Default: 5
--chunk-size=amount:unit
# Default: {10=>:mb}
[--create-bucket]
# Default: false
[--progress-bar]
# Default: true
[--checksum]
# Default: true
[--key=KEY] # s3 object key (default: file name)
example
s3_multi_upload --access-key-id xxx \
--secret-access-key xxx \
--bucket xxx \
--file /path/to/file \
--no-progress-bar \ # disable the progress bar
--chunk-size 50:mb \ # increase the chunk size
--threads 40 # increase the number of threads
docker run -it --rm -v /tmp/upload:/root \
dallasmarlow/s3_multi_upload \
--access-key-id xxx \
--secret-access-key xxx \
--bucket xxx \
--file /root/test.iso