diff --git a/README.rst b/README.rst index 4603e3b..9a925ef 100644 --- a/README.rst +++ b/README.rst @@ -24,6 +24,12 @@ The configuration is stored in a YAML file like this: .. code-block:: yaml site: _site + + aws_id: XXXXXXXXXXX + aws_secret: XXXXXXXXXXX + region: eu-central-1 + aws_profile: deploy + s3_bucket: example.com cloudfront_distribution_id: XXXXXXXXXXX @@ -53,17 +59,23 @@ same directory: Credentials ----------- -AWS credentials can be provided through the environment variables -``AWS_ACCESS_KEY_ID`` and ``AWS_SECRET_ACCESS_KEY``. +AWS credentials can be provided in following ways (in order of priority): -.. code-block:: shell +1. Using the ``--profile PROFILE`` command line option selecting the AWS profile to use. - $ export AWS_ACCESS_KEY_ID=XXXXXX - $ export AWS_SECTER_ACCESS_KEY=XXXXXX - $ s3-deploy-website +2. Using the ``aws_id`` and ``aws_secret`` settings in configuration file. + +3. Using the ``aws_profile`` setting in configuration file to select the AWS profile to use. + +4. Environment variables ``AWS_ACCESS_KEY_ID`` and ``AWS_SECRET_ACCESS_KEY``. + + .. code-block:: shell + + $ export AWS_ACCESS_KEY_ID=XXXXXX + $ export AWS_SECTER_ACCESS_KEY=XXXXXX + $ s3-deploy-website -They can also be provided through the various configuration files that boto_ -reads. +5. They can also be provided through the various configuration files that boto_reads. .. _boto: https://boto.readthedocs.org/en/latest/boto_config_tut.html @@ -73,6 +85,26 @@ Configuration file **site** The directory of the static content to be uploaded (relative to the location of the configuration file (e.g. ``_site`` for Jekyll sites). + +**aws_id** + The AWS access key ID used to connect to the AWS. If ``aws_id`` is + specified, also ``aws_secret`` has to be specified. If either ``aws_id`` + or ``aws_secret`` and also ``aws_profile`` is specified, the specified + access key is used to connect. + +**aws_secret** + The AWS secret access key used to connect to the AWS. If ``aws_secret`` + is specified, also ``aws_id`` has to be specified. If either ``aws_id`` + or ``aws_secret`` and also ``aws_profile`` is specified, the specified + access key is used to connect. + +**region** + The region used to connect to the AWS. + +**aws_profile** + The AWS profile used to connect to the AWS. If ``aws_profile`` and also + either ``aws_id`` or ``aws_secret`` is specified, the specified access + key is used to connect. **s3_bucket** The name of the S3 bucket to upload the files to. You have to allow the diff --git a/s3_deploy/deploy.py b/s3_deploy/deploy.py index bff496d..d26dd82 100644 --- a/s3_deploy/deploy.py +++ b/s3_deploy/deploy.py @@ -7,12 +7,10 @@ import shutil import mimetypes from datetime import datetime +from six import BytesIO import boto3 -from six import BytesIO -from six.moves.urllib.parse import quote_plus - from . import config from .prefixcovertree import PrefixCoverTree @@ -47,6 +45,40 @@ def dst(self, dt): mimetypes.init() +def create_session(conf): + session = None + if conf.get('aws_id', False) or conf.get('aws_secret', False): + if conf.get('aws_id', False) and conf.get('aws_secret', False): + logger.info('Creating AWS session using AWS key id \"{}\" ' + 'and region \"{}\"...' + .format(conf['aws_id'], + conf.get('region', ''))) + + session = boto3.Session(conf['aws_id'], + conf['aws_secret'], + region_name=conf.get('region', None)) + else: + logger.error('Both AWS access key ID and AWS secret access key has' + 'to specified if either is specified.') + + elif conf.get('aws_profile', False): + logger.info('Creating AWS session using AWS profile \"{}\" ' + 'and region \"{}\"...' + .format(conf['aws_profile'], + conf.get('region', ''))) + + session = boto3.Session(profile_name=conf['aws_profile'], + region_name=conf.get('region', None)) + else: + logger.info('Creating AWS session using default profile ' + 'and region \"{}\"...' + .format(conf.get('region', ''))) + + session = boto3.Session(region_name=conf.get('region', None)) + + return session + + def key_name_from_path(path): """Convert a relative path into a key name.""" key_parts = [] @@ -112,26 +144,7 @@ def upload_key(obj, path, cache_rules, dry, storage_class=None): content_file.close() -def main(): - logging.basicConfig(level=logging.INFO) - logging.getLogger('boto3').setLevel(logging.WARNING) - - parser = argparse.ArgumentParser( - description='AWS S3 website deployment tool') - parser.add_argument( - '-f', '--force', action='store_true', dest='force', - help='force upload of all files') - parser.add_argument( - '-n', '--dry-run', action='store_true', dest='dry', - help='run without uploading any files') - parser.add_argument( - 'path', help='the .s3_website.yaml configuration file or directory', - default='.', nargs='?') - args = parser.parse_args() - - # Open configuration file - conf, base_path = config.load_config_file(args.path) - +def upload_to_bucket(session, site_dir, conf, dry): bucket_name = conf['s3_bucket'] cache_rules = conf.get('cache_rules', []) if conf.get('s3_reduced_redundancy', False): @@ -141,11 +154,10 @@ def main(): logger.info('Connecting to bucket {}...'.format(bucket_name)) - s3 = boto3.resource('s3') +# s3 = boto3.resource('s3') + s3 = session.resource('s3') bucket = s3.Bucket(bucket_name) - site_dir = os.path.join(base_path, conf['site']) - logger.info('Site: {}'.format(site_dir)) processed_keys = set() @@ -158,21 +170,21 @@ def main(): # Delete keys that have been deleted locally if not os.path.isfile(path): logger.info('Deleting {}...'.format(obj.key)) - if not args.dry: + if not dry: obj.delete() updated_keys.add(obj.key) continue # Skip keys that have not been updated mtime = datetime.fromtimestamp(os.path.getmtime(path), UTC) - if not args.force: + if not conf.get('force', False): if (mtime <= obj.last_modified and obj.storage_class == storage_class): logger.info('Not modified, skipping {}.'.format(obj.key)) continue upload_key( - obj, path, cache_rules, args.dry, storage_class=storage_class) + obj, path, cache_rules, dry, storage_class=storage_class) updated_keys.add(obj.key) for dirpath, dirnames, filenames in os.walk(site_dir): @@ -188,60 +200,112 @@ def main(): logger.info('Creating key {}...'.format(obj.key)) - upload_key( - obj, path, cache_rules, args.dry, storage_class=storage_class) + upload_key(obj, + path, + cache_rules, + dry, + storage_class=storage_class) updated_keys.add(key_name) logger.info('Bucket update done.') + return processed_keys, updated_keys + + +def invalidate_cloudfront(session, processed_keys, updated_keys, conf, dry): + logger.info('Connecting to Cloudfront distribution {}...' + .format(conf['cloudfront_distribution_id'])) + + index_pattern = None + if 'index_document' in conf: + index_doc = conf['index_document'] + index_pattern = r'(^(?:.*/)?)' + re.escape(index_doc) + '$' + + def path_from_key_name(key_name): + if index_pattern is not None: + m = re.match(index_pattern, key_name) + if m: + return m.group(1) + return key_name + + t = PrefixCoverTree() + for key_name in updated_keys: + t.include(path_from_key_name(key_name)) + for key_name in processed_keys - updated_keys: + t.exclude(path_from_key_name(key_name)) + + paths = [] + for prefix, exact in t.matches(): + path = '/' + prefix + ('' if exact else '*') + logger.info('Preparing to invalidate {}...'.format(path)) + paths.append(path) + + cloudfront = session.client('cloudfront') + + if len(paths) > 0: + dist_id = conf['cloudfront_distribution_id'] + if not dry: + logger.info('Creating invalidation request...') + response = cloudfront.create_invalidation( + DistributionId=dist_id, + InvalidationBatch=dict( + Paths=dict( + Quantity=len(paths), + Items=paths + ), + CallerReference='s3-deploy-website' + ) + ) + invalidation = response['Invalidation'] + logger.info('Invalidation request {} is {}'.format( + invalidation['Id'], invalidation['Status'])) + else: + logger.info('Nothing updated, invalidation skipped.') + + +def main(): + logging.basicConfig(level=logging.INFO) + logging.getLogger('boto3').setLevel(logging.WARNING) + + parser = argparse.ArgumentParser( + description='AWS S3 website deployment tool') + parser.add_argument( + '-f', '--force', action='store_true', dest='force', + help='force upload of all files') + parser.add_argument( + '-n', '--dry-run', action='store_true', dest='dry', + help='run without uploading any files') + parser.add_argument( + '--profile', dest='profile', + help='AWS configuration profile') + parser.add_argument( + 'path', help='the .s3_website.yaml configuration file or directory', + default='.', nargs='?') + args = parser.parse_args() + + # Open configuration file + conf, base_path = config.load_config_file(args.path) + + if args.force: + conf['force'] = True + if args.profile: + conf['aws_profile'] = args.profile + + session = create_session(conf) + if not session: + logger.error('Error creating AWS session!') + return + + processed_keys, updated_keys = upload_to_bucket( + session, + os.path.join(base_path, conf['site']), + conf, + args.dry) + # Invalidate files in cloudfront distribution if 'cloudfront_distribution_id' in conf: - logger.info('Connecting to Cloudfront distribution {}...'.format( - conf['cloudfront_distribution_id'])) - - index_pattern = None - if 'index_document' in conf: - index_doc = conf['index_document'] - index_pattern = r'(^(?:.*/)?)' + re.escape(index_doc) + '$' - - def path_from_key_name(key_name): - if index_pattern is not None: - m = re.match(index_pattern, key_name) - if m: - return m.group(1) - return key_name - - t = PrefixCoverTree() - for key_name in updated_keys: - t.include(path_from_key_name(key_name)) - for key_name in processed_keys - updated_keys: - t.exclude(path_from_key_name(key_name)) - - paths = [] - for prefix, exact in t.matches(): - path = '/' + prefix + ('' if exact else '*') - logger.info('Preparing to invalidate {}...'.format(path)) - paths.append(path) - - cloudfront = boto3.client('cloudfront') - - if len(paths) > 0: - dist_id = conf['cloudfront_distribution_id'] - if not args.dry: - logger.info('Creating invalidation request...') - response = cloudfront.create_invalidation( - DistributionId=dist_id, - InvalidationBatch=dict( - Paths=dict( - Quantity=len(paths), - Items=['' + quote_plus(p) + '' - for p in paths] - ), - CallerReference='s3-deploy-website' - ) - ) - invalidation = response['Invalidation'] - logger.info('Invalidation request {} is {}'.format( - invalidation['Id'], invalidation['Status'])) - else: - logger.info('Nothing updated, invalidation skipped.') + invalidate_cloudfront(session, + processed_keys, + updated_keys, + conf, + args.dry)