Root Zanli
Home
Console
Upload
information
Create File
Create Folder
About
Tools
:
/
opt
/
gsutil
/
gslib
/
utils
/
Filename :
boto_util.py
back
Copy
# -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared utility structures and methods that require importing boto. This module also imports httplib2 (as it is Boto's http transport and closely tied to some of Boto's core functionality) and oauth2client. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import pkgutil import tempfile import textwrap import six import boto from boto import config import boto.auth from boto.exception import NoAuthHandlerFound from boto.gs.connection import GSConnection from boto.provider import Provider from boto.pyami.config import BotoConfigLocations import gslib from gslib import context_config from gslib.exception import CommandException from gslib.utils import system_util from gslib.utils.constants import DEFAULT_GCS_JSON_API_VERSION from gslib.utils.constants import DEFAULT_GSUTIL_STATE_DIR from gslib.utils.constants import SSL_TIMEOUT_SEC from gslib.utils.constants import UTF8 from gslib.utils.unit_util import HumanReadableToBytes from gslib.utils.unit_util import ONE_MIB import httplib2 from oauth2client.client import HAS_CRYPTO if six.PY3: long = int # Globals in this module are set according to values in the boto config. BOTO_IS_SECURE = config.get('Boto', 'is_secure', True) CERTIFICATE_VALIDATION_ENABLED = config.get('Boto', 'https_validate_certificates', True) configured_certs_file = None # Single certs file for use across all processes. temp_certs_file = None # Temporary certs file for cleanup upon exit. def ConfigureCertsFile(): """Configures and returns the CA Certificates file. If one is already configured, use it. Otherwise, use the cert roots distributed with gsutil. Returns: string filename of the certs file to use. """ certs_file = boto.config.get('Boto', 'ca_certificates_file', None) # The 'system' keyword indicates to use the system installed certs. Some # Linux distributions patch the stack such that the Python SSL # infrastructure picks up the system installed certs by default, thus no # action necessary on our part if certs_file == 'system': return None if not certs_file: global configured_certs_file, temp_certs_file if not configured_certs_file: configured_certs_file = os.path.abspath( os.path.join(gslib.GSLIB_DIR, 'data', 'cacerts.txt')) if not os.path.exists(configured_certs_file): # If the file is not present on disk, this means the gslib module # doesn't actually exist on disk anywhere. This can happen if it's # being imported from a zip file. Unfortunately, we have to copy the # certs file to a local temp file on disk because the underlying SSL # socket requires it to be a filesystem path. certs_data = pkgutil.get_data('gslib', 'data/cacerts.txt') if not certs_data: raise CommandException('Certificates file not found. Please ' 'reinstall gsutil from scratch') certs_data = six.ensure_str(certs_data) fd, fname = tempfile.mkstemp(suffix='.txt', prefix='gsutil-cacerts') f = os.fdopen(fd, 'w') f.write(certs_data) f.close() temp_certs_file = fname configured_certs_file = temp_certs_file certs_file = configured_certs_file return certs_file def ConfigureNoOpAuthIfNeeded(): """Sets up no-op auth handler if no boto credentials are configured.""" if not HasConfiguredCredentials(): if (config.has_option('Credentials', 'gs_service_client_id') and not HAS_CRYPTO): if system_util.InvokedViaCloudSdk(): raise CommandException('\n'.join( textwrap.wrap( 'Your gsutil is configured with an OAuth2 service account, but ' 'you do not have PyOpenSSL or PyCrypto 2.6 or later installed. ' 'Service account authentication requires one of these libraries; ' 'please reactivate your service account via the gcloud auth ' 'command and ensure any gcloud packages necessary for ' 'service accounts are present.'))) else: raise CommandException('\n'.join( textwrap.wrap( 'Your gsutil is configured with an OAuth2 service account, but ' 'you do not have PyOpenSSL or PyCrypto 2.6 or later installed. ' 'Service account authentication requires one of these libraries; ' 'please install either of them to proceed, or configure a ' 'different type of credentials with "gsutil config".'))) else: # With no boto config file the user can still access publicly readable # buckets and objects. from gslib import no_op_auth_plugin # pylint: disable=unused-variable def GetCertsFile(): return configured_certs_file def GetCleanupFiles(): """Returns a list of temp files to delete (if possible) when program exits.""" return [temp_certs_file] if temp_certs_file else [] def GetConfigFilePaths(): """Returns a list of the path(s) to the boto config file(s) to be loaded.""" potential_config_paths = BotoConfigLocations # When Boto's pyami.config.Config class is initialized, it attempts to read # this file, transform it into valid Boto config syntax, and load credentials # from it, but it does not add this file to BotoConfigLocations. if 'AWS_CREDENTIAL_FILE' in os.environ: potential_config_paths.append(os.environ['AWS_CREDENTIAL_FILE']) # Provider-specific config files, like the one below, are checked for and # loaded when Boto's Provider class is initialized. These aren't listed in # BotoConfigLocations, but can still affect what credentials are loaded, so # we display them in our config list to make auth'n debugging easier. aws_cred_file = os.path.join(os.path.expanduser('~'), '.aws', 'credentials') if os.path.isfile(aws_cred_file): potential_config_paths.append(aws_cred_file) # Only return credential files which we have permission to read (and thus can # actually load credentials from). readable_config_paths = [] for path in potential_config_paths: try: with open(path, 'r'): readable_config_paths.append(path) except IOError: pass return readable_config_paths def GetFriendlyConfigFilePaths(): """Like GetConfigFilePaths but returns a not-found message if paths empty.""" readable_config_paths = GetConfigFilePaths() if len(readable_config_paths) == 0: readable_config_paths.append('No config found') return readable_config_paths def GetGsutilStateDir(): """Returns the location of the directory for gsutil state files. Certain operations, such as cross-process credential sharing and resumable transfer tracking, need a known location for state files which are created by gsutil as-needed. This location should only be used for storing data that is required to be in a static location. Returns: Path to directory for gsutil static state files. """ config_file_dir = config.get('GSUtil', 'state_dir', DEFAULT_GSUTIL_STATE_DIR) system_util.CreateDirIfNeeded(config_file_dir) return config_file_dir def GetCredentialStoreFilename(): # As of gsutil 4.29, this changed from 'credstore' to 'credstore2' because # of a change to the underlying credential storage format. return os.path.join(GetGsutilStateDir(), 'credstore2') def GetGceCredentialCacheFilename(): return os.path.join(GetGsutilStateDir(), 'gcecredcache') def GetGcsJsonApiVersion(): return config.get('GSUtil', 'json_api_version', DEFAULT_GCS_JSON_API_VERSION) # Resumable downloads and uploads make one HTTP call per chunk (and must be # in multiples of 256KiB). Overridable for testing. def GetJsonResumableChunkSize(): chunk_size = config.getint('GSUtil', 'json_resumable_chunk_size', long(1024 * 1024 * 100)) if chunk_size == 0: chunk_size = long(1024 * 256) elif chunk_size % long(1024 * 256) != 0: chunk_size += (long(1024 * 256) - (chunk_size % (long(1024 * 256)))) return chunk_size def GetLastCheckedForGsutilUpdateTimestampFile(): return os.path.join(GetGsutilStateDir(), '.last_software_update_check') def GetMaxConcurrentCompressedUploads(): """Gets the max concurrent transport compressed uploads allowed in parallel. Returns: The max number of concurrent transport compressed uploads allowed in parallel without exceeding the max_upload_compression_buffer_size. """ upload_chunk_size = GetJsonResumableChunkSize() # From apitools compression.py. compression_chunk_size = 16 * ONE_MIB total_upload_size = (upload_chunk_size + compression_chunk_size + 17 + 5 * (((compression_chunk_size - 1) / 16383) + 1)) max_concurrent_uploads = (GetMaxUploadCompressionBufferSize() / total_upload_size) if max_concurrent_uploads <= 0: max_concurrent_uploads = 1 return max_concurrent_uploads def GetMaxRetryDelay(): return config.getint('Boto', 'max_retry_delay', 32) def GetMaxUploadCompressionBufferSize(): """Get the max amount of memory compressed transport uploads may buffer.""" return HumanReadableToBytes( config.get('GSUtil', 'max_upload_compression_buffer_size', '2GiB')) def GetNewHttp(http_class=httplib2.Http, **kwargs): """Creates and returns a new httplib2.Http instance. Args: http_class: Optional custom Http class to use. **kwargs: Arguments to pass to http_class constructor. Returns: An initialized httplib2.Http instance. """ ##Get Proxy configuration from boto file, defaults are None, 0 and False proxy_host = config.get('Boto', 'proxy', None) #needed for if clause below boto_proxy_config = { 'proxy_host': proxy_host, 'proxy_type': config.get('Boto', 'proxy_type', 'http'), 'proxy_port': config.getint('Boto', 'proxy_port'), 'proxy_user': config.get('Boto', 'proxy_user', None), 'proxy_pass': config.get('Boto', 'proxy_pass', None), 'proxy_rdns': config.get('Boto', 'proxy_rdns', True if proxy_host else None) } #Use SetProxyInfo to convert boto config to httplib2.proxyinfo object proxy_info = SetProxyInfo(boto_proxy_config) # Some installers don't package a certs file with httplib2, so use the # one included with gsutil. kwargs['ca_certs'] = GetCertsFile() # Use a non-infinite SSL timeout to avoid hangs during network flakiness. kwargs['timeout'] = SSL_TIMEOUT_SEC http = http_class(proxy_info=proxy_info, **kwargs) http.disable_ssl_certificate_validation = (not config.getbool( 'Boto', 'https_validate_certificates')) global_context_config = context_config.get_context_config() if global_context_config and global_context_config.use_client_certificate: http.add_certificate(key=global_context_config.client_cert_path, cert=global_context_config.client_cert_path, domain='', password=global_context_config.client_cert_password) return http # Retry for 10 minutes with exponential backoff, which corresponds to # the maximum Downtime Period specified in the GCS SLA # (https://cloud.google.com/storage/sla) def GetNumRetries(): return config.getint('Boto', 'num_retries', 23) def GetTabCompletionLogFilename(): return os.path.join(GetGsutilStateDir(), 'tab-completion-logs') def GetTabCompletionCacheFilename(): tab_completion_dir = os.path.join(GetGsutilStateDir(), 'tab-completion') # Limit read permissions on the directory to owner for privacy. system_util.CreateDirIfNeeded(tab_completion_dir, mode=0o700) return os.path.join(tab_completion_dir, 'cache') def HasConfiguredCredentials(): """Determines if boto credential/config file exists.""" has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and config.has_option('Credentials', 'gs_secret_access_key')) has_amzn_creds = (config.has_option('Credentials', 'aws_access_key_id') and config.has_option('Credentials', 'aws_secret_access_key')) has_oauth_creds = (config.has_option('Credentials', 'gs_oauth2_refresh_token')) has_external_creds = (config.has_option('Credentials', 'gs_external_account_file')) has_external_account_authorized_user_creds = (config.has_option( 'Credentials', 'gs_external_account_authorized_user_file')) has_service_account_creds = ( HAS_CRYPTO and config.has_option('Credentials', 'gs_service_client_id') and config.has_option('Credentials', 'gs_service_key_file')) if (has_goog_creds or has_amzn_creds or has_oauth_creds or has_service_account_creds or has_external_creds or has_external_account_authorized_user_creds): return True valid_auth_handler = None try: valid_auth_handler = boto.auth.get_auth_handler(GSConnection.DefaultHost, config, Provider('google'), requested_capability=['s3']) # Exclude the no-op auth handler as indicating credentials are configured. # Note we can't use isinstance() here because the no-op module may not be # imported so we can't get a reference to the class type. if 'NoOpAuth' == getattr( getattr(valid_auth_handler, '__class__', None), '__name__', None): # yapf: disable valid_auth_handler = None except NoAuthHandlerFound: pass return valid_auth_handler def JsonResumableChunkSizeDefined(): chunk_size_defined = config.get('GSUtil', 'json_resumable_chunk_size', None) return chunk_size_defined is not None def MonkeyPatchBoto(): """Apply gsutil-specific patches to Boto. Here be dragons. Sorry. Note that this method should not be used as a replacement for contributing fixes to the upstream Boto library. However, the Boto library has historically not been consistent about release cadence, so upstream fixes may not be available immediately in a version which we can pin to. Also, some fixes may only be applicable to gsutil. In such cases, patches should be applied to the Boto library here (and removed if/when they are included in the upstream repository and included in an official new release that we pull in). This method should be invoked after all other Boto-related initialization has been completed. """ # We have to do all sorts of gross things here (dynamic imports, invalid names # to resolve symbols in copy/pasted methods, invalid spacing from copy/pasted # methods, etc.), so we just disable pylint warnings for this whole method. # yapf: disable # pylint: disable=all # This should have already been imported if this method was called in the # correct place, but we need to perform the import within this method for # this module to recognize it. We don't import this at module level because # its import timing is important and is managed elsewhere. import gcs_oauth2_boto_plugin # TODO(https://github.com/boto/boto/issues/3831): Remove this if the GitHub # issue is ever addressed. orig_get_plugin_method = boto.plugin.get_plugin def _PatchedGetPluginMethod(cls, requested_capability=None): handler_subclasses = orig_get_plugin_method( cls, requested_capability=requested_capability) # In Boto's logic, higher precedence handlers should be closer to the end # of the list. We always want to prefer OAuth2 credentials over HMAC # credentials if both are present, so we shuffle OAuth2 handlers to the end # of the handler list. xml_oauth2_handlers = ( gcs_oauth2_boto_plugin.oauth2_plugin.OAuth2ServiceAccountAuth, gcs_oauth2_boto_plugin.oauth2_plugin.OAuth2Auth) new_result = ( # We need to sort each of these to avoid inconsistent handler selection # when multiple credential types are configured in Python 3.5. See # https://issuetracker.google.com/issues/135709541 for specific logs. sorted( [r for r in handler_subclasses if r not in xml_oauth2_handlers], # Types aren't sortable, so we use their names: key=(lambda handler_t: handler_t.__name__), ) + # Now append XML handlers to the end (highest precedence) sorted( [r for r in handler_subclasses if r in xml_oauth2_handlers], key=(lambda handler_t: handler_t.__name__), )) return new_result boto.plugin.get_plugin = _PatchedGetPluginMethod ######################################################################### # TODO(boto>2.49.0): Remove this. # Fixes SSL issue where SNI was not being used for OpenSSL 1.1.1+ # https://github.com/boto/boto/pull/3843/files # Import modules and resolve symbols used by our patch method. import socket import ssl InvalidCertificateException = ( boto.https_connection.InvalidCertificateException) ValidateCertificateHostname = ( boto.https_connection.ValidateCertificateHostname) def _PatchedConnectMethod(self): # The lines below were copied directly from the Boto file, so we don't lint # or otherwise alter them. if hasattr(self, "timeout"): sock = socket.create_connection((self.host, self.port), self.timeout) else: sock = socket.create_connection((self.host, self.port)) msg = "wrapping ssl socket; " if self.ca_certs: msg += "CA certificate file=%s" % self.ca_certs else: msg += "using system provided SSL certs" boto.log.debug(msg) if hasattr(ssl, 'SSLContext') and getattr(ssl, 'HAS_SNI', False): # Use SSLContext so we can specify server_hostname for SNI # (Required for connections to storage.googleapis.com) context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.verify_mode = ssl.CERT_REQUIRED if self.ca_certs: context.load_verify_locations(self.ca_certs) if self.cert_file: context.load_cert_chain(self.cert_file, self.key_file) self.sock = context.wrap_socket(sock, server_hostname=self.host) # Add attributes only set in SSLSocket constructor without context: self.sock.keyfile = self.key_file self.sock.certfile = self.cert_file self.sock.cert_reqs = context.verify_mode self.sock.ssl_version = ssl.PROTOCOL_SSLv23 self.sock.ca_certs = self.ca_certs self.sock.ciphers = None else: self.sock = ssl.wrap_socket(sock, keyfile=self.key_file, certfile=self.cert_file, cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.ca_certs) cert = self.sock.getpeercert() hostname = self.host.split(':', 0)[0] if not ValidateCertificateHostname(cert, hostname): raise InvalidCertificateException( hostname, cert, 'remote hostname "%s" does not match ' 'certificate' % hostname) # End `_PatchedConnectMethod` declaration. boto.https_connection.CertValidatingHTTPSConnection.connect = ( _PatchedConnectMethod) def ProxyInfoFromEnvironmentVar(proxy_env_var): """Reads proxy info from the environment and converts to httplib2.ProxyInfo. Args: proxy_env_var: Environment variable string to read, such as http_proxy or https_proxy. Returns: httplib2.ProxyInfo constructed from the environment string. """ proxy_url = os.environ.get(proxy_env_var) if not proxy_url or not proxy_env_var.lower().startswith('http'): return httplib2.ProxyInfo(httplib2.socks.PROXY_TYPE_HTTP, None, 0) proxy_protocol = proxy_env_var.lower().split('_')[0] if not proxy_url.lower().startswith('http'): # proxy_info_from_url requires a protocol, which is always http or https. proxy_url = proxy_protocol + '://' + proxy_url return httplib2.proxy_info_from_url(proxy_url, method=proxy_protocol) def ResumableThreshold(): return config.getint('GSUtil', 'resumable_threshold', 8 * ONE_MIB) def SetProxyInfo(boto_proxy_config): """Sets proxy info from boto and environment and converts to httplib2.ProxyInfo. Args: dict: Values read from the .boto file Returns: httplib2.ProxyInfo constructed from boto or environment variable string. """ #Defining proxy_type based on httplib2 library, accounting for None entry too. proxy_type_spec = {'socks4': 1, 'socks5': 2, 'http': 3, 'https': 3} #proxy_type defaults to 'http (3)' for backwards compatibility proxy_type = proxy_type_spec.get( boto_proxy_config.get('proxy_type').lower(), proxy_type_spec['http']) proxy_host = boto_proxy_config.get('proxy_host') proxy_port = boto_proxy_config.get('proxy_port') proxy_user = boto_proxy_config.get('proxy_user') proxy_pass = boto_proxy_config.get('proxy_pass') proxy_rdns = bool(boto_proxy_config.get('proxy_rdns')) #For proxy_info below, proxy_rdns fails for socks4 and socks5 so restricting use #to http only proxy_info = httplib2.ProxyInfo(proxy_host=proxy_host, proxy_type=proxy_type, proxy_port=proxy_port, proxy_user=proxy_user, proxy_pass=proxy_pass, proxy_rdns=proxy_rdns) #Added to force socks proxies not to use rdns if not (proxy_info.proxy_type == proxy_type_spec['http']): proxy_info.proxy_rdns = False if not (proxy_info.proxy_host and proxy_info.proxy_port): # Fall back to using the environment variable. Use only http proxies. for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']: if proxy_env_var in os.environ and os.environ[proxy_env_var]: proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var) # Assume proxy_rnds is True if a proxy environment variable exists. if boto_proxy_config.get('proxy_rdns') == None: proxy_info.proxy_rdns = True break return proxy_info def UsingCrcmodExtension(): boto_opt = boto.config.get('GSUtil', 'test_assume_fast_crcmod', None) if boto_opt is not None: return boto_opt # Python 3 makes this attribute tough to access due to the way the top-level # crcmod package imports and (identically) names its crcmod module. The only # way to get it is "from crcmod.crcmod import _usingExtension". This is the # alternative form of that statement, but doesn't pollute this module's # namespace with a "_usingExtension" attribute. This also works in both Python # 2.7 and 3.5+. nested_crcmod = __import__( 'crcmod.crcmod', globals(), locals(), ['_usingExtension'], 0, ) return getattr(nested_crcmod, '_usingExtension', False) # TODO(boto-2.49.0): Remove when we pull in the next version of Boto. def _PatchedShouldRetryMethod(self, response, chunked_transfer=False): """Replaces boto.s3.key's should_retry() to handle KMS-encrypted objects.""" # We copy/pasted this from boto and slightly modified it; keep the old # formatting style. # yapf: disable provider = self.bucket.connection.provider if not chunked_transfer: if response.status in [500, 503]: # 500 & 503 can be plain retries. return True if response.getheader('location'): # If there's a redirect, plain retry. return True if 200 <= response.status <= 299: self.etag = response.getheader('etag') md5 = self.md5 if isinstance(md5, bytes): md5 = md5.decode(UTF8) # If you use customer-provided encryption keys, the ETag value that # Amazon S3 returns in the response will not be the MD5 of the # object. amz_server_side_encryption_customer_algorithm = response.getheader( 'x-amz-server-side-encryption-customer-algorithm', None) # The same is applicable for KMS-encrypted objects in gs buckets. goog_customer_managed_encryption = response.getheader( 'x-goog-encryption-kms-key-name', None) if (amz_server_side_encryption_customer_algorithm is None and goog_customer_managed_encryption is None): if self.etag != '"%s"' % md5: raise provider.storage_data_error( 'ETag from S3 did not match computed MD5. ' '%s vs. %s' % (self.etag, self.md5)) return True if response.status == 400: # The 400 must be trapped so the retry handler can check to # see if it was a timeout. # If ``RequestTimeout`` is present, we'll retry. Otherwise, bomb # out. body = response.read() err = provider.storage_response_error( response.status, response.reason, body ) if err.error_code in ['RequestTimeout']: raise boto.exception.PleaseRetryException( "Saw %s, retrying" % err.error_code, response=response ) return False def HasUserSpecifiedGsHost(): gs_host = boto.config.get('Credentials', 'gs_host', None) default_host = six.ensure_str(boto.gs.connection.GSConnection.DefaultHost) if gs_host is not None: return default_host == six.ensure_str(gs_host) return False def UsingGsHmac(): # NOTE: External credentials are omitted intentionally as HMAC takes priority # over the external credentials types. config = boto.config has_refresh_token = config.has_option('Credentials', 'gs_oauth2_refresh_token') has_service_account_credentials = ( config.has_option('Credentials', 'gs_service_client_id') and config.has_option('Credentials', 'gs_service_key_file')) has_hmac_credentials = ( config.has_option('Credentials', 'gs_access_key_id') and config.has_option('Credentials', 'gs_secret_access_key')) return (not has_refresh_token and not has_service_account_credentials and has_hmac_credentials)