# Copyright 2010 Google Inc. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Implementation of wildcarding over StorageUris. StorageUri is an abstraction that Google introduced in the boto library, for representing storage provider-independent bucket and object names with a shorthand URI-like syntax (see boto/boto/storage_uri.py) The current class provides wildcarding support for StorageUri objects (including both bucket and file system objects), allowing one to express collections of objects with syntax like the following: gs://mybucket/images/*.png file:///tmp/???abc??? We provide wildcarding support as part of gsutil rather than as part of boto because wildcarding is really part of shell command-like functionality. A comment about wildcard semantics: We support both single path component wildcards (e.g., using '*') and recursive wildcards (using '**'), for both file and cloud URIs. For example, gs://bucket/doc/*/*.html would enumerate HTML files one directory down from gs://bucket/doc, while gs://bucket/**/*.html would enumerate HTML files in all objects contained in the bucket. Note also that if you use file system wildcards it's likely your shell interprets the wildcarding before passing the command to gsutil. For example: % gsutil cp /opt/eclipse/*/*.html gs://bucket/eclipse would likely be expanded by the shell into the following before running gsutil: % gsutil cp /opt/eclipse/RUNNING.html gs://bucket/eclipse Note also that most shells don't support '**' wildcarding (I think only zsh does). If you want to use '**' wildcarding with such a shell you can single quote each wildcarded string, so it gets passed uninterpreted by the shell to gsutil (at which point gsutil will perform the wildcarding expansion): % gsutil cp '/opt/eclipse/**/*.html' gs://bucket/eclipse """ import boto import fnmatch import glob import os import re import sys import urllib from boto.s3.prefix import Prefix from boto.storage_uri import BucketStorageUri from bucket_listing_ref import BucketListingRef # Regex to determine if a string contains any wildcards. WILDCARD_REGEX = re.compile('[*?\[\]]') WILDCARD_OBJECT_ITERATOR = 'wildcard_object_iterator' WILDCARD_BUCKET_ITERATOR = 'wildcard_bucket_iterator' class WildcardIterator(object): """Base class for wildcarding over StorageUris. This class implements support for iterating over StorageUris that contain wildcards. The base class is abstract; you should instantiate using the wildcard_iterator() static factory method, which chooses the right implementation depending on the StorageUri. """ def __repr__(self): """Returns string representation of WildcardIterator.""" return 'WildcardIterator(%s)' % self.wildcard_uri class CloudWildcardIterator(WildcardIterator): """WildcardIterator subclass for buckets and objects. Iterates over BucketListingRef matching the StorageUri wildcard. It's much more efficient to request the Key from the BucketListingRef (via GetKey()) than to request the StorageUri and then call uri.get_key() to retrieve the key, for cases where you want to get metadata that's available in the Bucket (for example to get the name and size of each object), because that information is available in the bucket GET results. If you were to iterate over URIs for such cases and then get the name and size info from each resulting StorageUri, it would cause an additional object GET request for each of the result URIs. """ def __init__(self, wildcard_uri, proj_id_handler, bucket_storage_uri_class=BucketStorageUri, all_versions=False, headers=None, debug=0): """ Instantiates an iterator over BucketListingRef matching given wildcard URI. Args: wildcard_uri: StorageUri that contains the wildcard to iterate. proj_id_handler: ProjectIdHandler to use for current command. bucket_storage_uri_class: BucketStorageUri interface. Settable for testing/mocking. headers: Dictionary containing optional HTTP headers to pass to boto. debug: Debug level to pass in to boto connection (range 0..3). """ self.wildcard_uri = wildcard_uri # Make a copy of the headers so any updates we make during wildcard # expansion aren't left in the input params (specifically, so we don't # include the x-goog-project-id header needed by a subset of cases, in # the data returned to caller, which could then be used in other cases # where that header must not be passed). if headers is None: self.headers = {} else: self.headers = headers.copy() self.proj_id_handler = proj_id_handler self.bucket_storage_uri_class = bucket_storage_uri_class self.all_versions = all_versions self.debug = debug def __iter__(self): """Python iterator that gets called when iterating over cloud wildcard. Yields: BucketListingRef, or empty iterator if no matches. """ # First handle bucket wildcarding, if any. if ContainsWildcard(self.wildcard_uri.bucket_name): regex = fnmatch.translate(self.wildcard_uri.bucket_name) bucket_uris = [] prog = re.compile(regex) self.proj_id_handler.FillInProjectHeaderIfNeeded(WILDCARD_BUCKET_ITERATOR, self.wildcard_uri, self.headers) for b in self.wildcard_uri.get_all_buckets(headers=self.headers): if prog.match(b.name): # Use str(b.name) because get_all_buckets() returns Unicode # string, which when used to construct x-goog-copy-src metadata # requests for object-to-object copies causes pathname '/' chars # to be entity-encoded (bucket%2Fdir instead of bucket/dir), # which causes the request to fail. uri_str = '%s://%s' % (self.wildcard_uri.scheme, urllib.quote_plus(str(b.name))) bucket_uris.append( boto.storage_uri( uri_str, debug=self.debug, bucket_storage_uri_class=self.bucket_storage_uri_class, suppress_consec_slashes=False)) else: bucket_uris = [self.wildcard_uri.clone_replace_name('')] # Now iterate over bucket(s), and handle object wildcarding, if any. self.proj_id_handler.FillInProjectHeaderIfNeeded(WILDCARD_OBJECT_ITERATOR, self.wildcard_uri, self.headers) for bucket_uri in bucket_uris: if self.wildcard_uri.names_bucket(): # Bucket-only URI. yield BucketListingRef(bucket_uri, key=None, prefix=None, headers=self.headers) else: # URI contains an object name. If there's no wildcard just yield # the needed URI. if not ContainsWildcard(self.wildcard_uri.object_name): uri_to_yield = bucket_uri.clone_replace_name( self.wildcard_uri.object_name) yield BucketListingRef(uri_to_yield, key=None, prefix=None, headers=self.headers) else: # URI contains a wildcard. Expand iteratively by building # prefix/delimiter bucket listing request, filtering the results per # the current level's wildcard, and continuing with the next component # of the wildcard. See _BuildBucketFilterStrings() documentation # for details. # # Initialize the iteration with bucket name from bucket_uri but # object name from self.wildcard_uri. This is needed to handle cases # where both the bucket and object names contain wildcards. uris_needing_expansion = [ bucket_uri.clone_replace_name(self.wildcard_uri.object_name)] while len(uris_needing_expansion) > 0: uri = uris_needing_expansion.pop(0) (prefix, delimiter, prefix_wildcard, suffix_wildcard) = ( self._BuildBucketFilterStrings(uri.object_name)) prog = re.compile(fnmatch.translate(prefix_wildcard)) # List bucket for objects matching prefix up to delimiter. for key in bucket_uri.list_bucket(prefix=prefix, delimiter=delimiter, headers=self.headers, all_versions=self.all_versions): # Check that the prefix regex matches rstripped key.name (to # correspond with the rstripped prefix_wildcard from # _BuildBucketFilterStrings()). if prog.match(key.name.rstrip('/')): if suffix_wildcard and key.name.rstrip('/') != suffix_wildcard: if isinstance(key, Prefix): # There's more wildcard left to expand. uris_needing_expansion.append( uri.clone_replace_name(key.name.rstrip('/') + '/' + suffix_wildcard)) else: # Done expanding. expanded_uri = uri.clone_replace_key(key) if isinstance(key, Prefix): yield BucketListingRef(expanded_uri, key=None, prefix=key, headers=self.headers) else: if self.all_versions: yield BucketListingRef(expanded_uri, key=key, prefix=None, headers=self.headers) else: # Yield BLR wrapping version-less URI. yield BucketListingRef(expanded_uri.clone_replace_name( expanded_uri.object_name), key=key, prefix=None, headers=self.headers) def _BuildBucketFilterStrings(self, wildcard): """ Builds strings needed for querying a bucket and filtering results to implement wildcard object name matching. Args: wildcard: The wildcard string to match to objects. Returns: (prefix, delimiter, prefix_wildcard, suffix_wildcard) where: prefix is the prefix to be sent in bucket GET request. delimiter is the delimiter to be sent in bucket GET request. prefix_wildcard is the wildcard to be used to filter bucket GET results. suffix_wildcard is wildcard to be appended to filtered bucket GET results for next wildcard expansion iteration. For example, given the wildcard gs://bucket/abc/d*e/f*.txt we would build prefix= abc/d, delimiter=/, prefix_wildcard=d*e, and suffix_wildcard=f*.txt. Using this prefix and delimiter for a bucket listing request will then produce a listing result set that can be filtered using this prefix_wildcard; and we'd use this suffix_wildcard to feed into the next call(s) to _BuildBucketFilterStrings(), for the next iteration of listing/filtering. Raises: AssertionError if wildcard doesn't contain any wildcard chars. """ # Generate a request prefix if the object name part of the wildcard starts # with a non-wildcard string (e.g., that's true for 'gs://bucket/abc*xyz'). match = WILDCARD_REGEX.search(wildcard) if not match: # Input "wildcard" has no wildcard chars, so just return tuple that will # cause a bucket listing to match the given input wildcard. Example: if # previous iteration yielded gs://bucket/dir/ with suffix_wildcard abc, # the next iteration will call _BuildBucketFilterStrings() with # gs://bucket/dir/abc, and we will return prefix ='dir/abc', # delimiter='/', prefix_wildcard='dir/abc', and suffix_wildcard=''. prefix = wildcard delimiter = '/' prefix_wildcard = wildcard suffix_wildcard = '' else: if match.start() > 0: # Wildcard does not occur at beginning of object name, so construct a # prefix string to send to server. prefix = wildcard[:match.start()] wildcard_part = wildcard[match.start():] else: prefix = None wildcard_part = wildcard end = wildcard_part.find('/') if end != -1: wildcard_part = wildcard_part[:end+1] # Remove trailing '/' so we will match gs://bucket/abc* as well as # gs://bucket/abc*/ with the same wildcard regex. prefix_wildcard = ((prefix or '') + wildcard_part).rstrip('/') suffix_wildcard = wildcard[match.end():] end = suffix_wildcard.find('/') if end == -1: suffix_wildcard = '' else: suffix_wildcard = suffix_wildcard[end+1:] # To implement recursive (**) wildcarding, if prefix_wildcard # suffix_wildcard starts with '**' don't send a delimiter, and combine # suffix_wildcard at end of prefix_wildcard. if prefix_wildcard.find('**') != -1: delimiter = None prefix_wildcard = prefix_wildcard + suffix_wildcard suffix_wildcard = '' else: delimiter = '/' delim_pos = suffix_wildcard.find(delimiter) # The following debug output is useful for tracing how the algorithm # walks through a multi-part wildcard like gs://bucket/abc/d*e/f*.txt if self.debug > 1: sys.stderr.write( 'DEBUG: wildcard=%s, prefix=%s, delimiter=%s, ' 'prefix_wildcard=%s, suffix_wildcard=%s\n' % (wildcard, prefix, delimiter, prefix_wildcard, suffix_wildcard)) return (prefix, delimiter, prefix_wildcard, suffix_wildcard) def IterKeys(self): """ Convenience iterator that runs underlying iterator and returns Key for each iteration. Yields: Subclass of boto.s3.key.Key, or empty iterator if no matches. Raises: WildcardException: for bucket-only uri. """ for bucket_listing_ref in self. __iter__(): if bucket_listing_ref.HasKey(): yield bucket_listing_ref.GetKey() def IterUris(self): """ Convenience iterator that runs underlying iterator and returns StorageUri for each iteration. Yields: StorageUri, or empty iterator if no matches. """ for bucket_listing_ref in self. __iter__(): yield bucket_listing_ref.GetUri() def IterUrisForKeys(self): """ Convenience iterator that runs underlying iterator and returns the StorageUri for each iterated BucketListingRef that has a Key. Yields: StorageUri, or empty iterator if no matches. """ for bucket_listing_ref in self. __iter__(): if bucket_listing_ref.HasKey(): yield bucket_listing_ref.GetUri() class FileWildcardIterator(WildcardIterator): """WildcardIterator subclass for files and directories. If you use recursive wildcards ('**') only a single such wildcard is supported. For example you could use the wildcard '**/*.txt' to list all .txt files in any subdirectory of the current directory, but you couldn't use a wildcard like '**/abc/**/*.txt' (which would, if supported, let you find .txt files in any subdirectory named 'abc'). """ def __init__(self, wildcard_uri, headers=None, debug=0): """ Instantiate an iterator over BucketListingRefs matching given wildcard URI. Args: wildcard_uri: StorageUri that contains the wildcard to iterate. headers: Dictionary containing optional HTTP headers to pass to boto. debug: Debug level to pass in to boto connection (range 0..3). """ self.wildcard_uri = wildcard_uri self.headers = headers self.debug = debug def __iter__(self): wildcard = self.wildcard_uri.object_name match = re.search('\*\*', wildcard) if match: # Recursive wildcarding request ('.../**/...'). # Example input: wildcard = '/tmp/tmp2pQJAX/**/*' base_dir = wildcard[:match.start()-1] remaining_wildcard = wildcard[match.start()+2:] # At this point for the above example base_dir = '/tmp/tmp2pQJAX' and # remaining_wildcard = '/*' if remaining_wildcard.startswith('*'): raise WildcardException('Invalid wildcard with more than 2 consecutive ' '*s (%s)' % wildcard) # If there was no remaining wildcard past the recursive wildcard, # treat it as if it were a '*'. For example, file://tmp/** is equivalent # to file://tmp/**/* if not remaining_wildcard: remaining_wildcard = '*' # Skip slash(es). remaining_wildcard = remaining_wildcard.lstrip(os.sep) filepaths = [] for dirpath, unused_dirnames, filenames in os.walk(base_dir): filepaths.extend( os.path.join(dirpath, f) for f in fnmatch.filter(filenames, remaining_wildcard) ) else: # Not a recursive wildcarding request. filepaths = glob.glob(wildcard) for filepath in filepaths: expanded_uri = self.wildcard_uri.clone_replace_name(filepath) yield BucketListingRef(expanded_uri) def IterKeys(self): """ Placeholder to allow polymorphic use of WildcardIterator. Raises: WildcardException: in all cases. """ raise WildcardException( 'Iterating over Keys not possible for file wildcards') def IterUris(self): """ Convenience iterator that runs underlying iterator and returns StorageUri for each iteration. Yields: StorageUri, or empty iterator if no matches. """ for bucket_listing_ref in self. __iter__(): yield bucket_listing_ref.GetUri() class WildcardException(StandardError): """Exception thrown for invalid wildcard URIs.""" def __init__(self, reason): StandardError.__init__(self) self.reason = reason def __repr__(self): return 'WildcardException: %s' % self.reason def __str__(self): return 'WildcardException: %s' % self.reason def wildcard_iterator(uri_or_str, proj_id_handler, bucket_storage_uri_class=BucketStorageUri, all_versions=False, headers=None, debug=0): """Instantiate a WildCardIterator for the given StorageUri. Args: uri_or_str: StorageUri or URI string naming wildcard objects to iterate. proj_id_handler: ProjectIdHandler to use for current command. bucket_storage_uri_class: BucketStorageUri interface. Settable for testing/mocking. headers: Dictionary containing optional HTTP headers to pass to boto. debug: Debug level to pass in to boto connection (range 0..3). Returns: A WildcardIterator that handles the requested iteration. """ if isinstance(uri_or_str, basestring): # Disable enforce_bucket_naming, to allow bucket names containing wildcard # chars. uri = boto.storage_uri( uri_or_str, debug=debug, validate=False, bucket_storage_uri_class=bucket_storage_uri_class, suppress_consec_slashes=False) else: uri = uri_or_str if uri.is_cloud_uri(): return CloudWildcardIterator( uri, proj_id_handler, bucket_storage_uri_class=bucket_storage_uri_class, all_versions=all_versions, headers=headers, debug=debug) elif uri.is_file_uri(): return FileWildcardIterator(uri, headers=headers, debug=debug) else: raise WildcardException('Unexpected type of StorageUri (%s)' % uri) def ContainsWildcard(uri_or_str): """Checks whether uri_or_str contains a wildcard. Args: uri_or_str: StorageUri or URI string to check. Returns: bool indicator. """ if isinstance(uri_or_str, basestring): return bool(WILDCARD_REGEX.search(uri_or_str)) else: return bool(WILDCARD_REGEX.search(uri_or_str.uri))