diff --git a/landsat/downloader.py b/landsat/downloader.py index 101d73a..8a5080c 100644 --- a/landsat/downloader.py +++ b/landsat/downloader.py @@ -11,14 +11,17 @@ import settings class RemoteFileDoesntExist(Exception): + """ Exception to be used when the remote file does not exist """ pass class IncorrectSceneId(Exception): + """ Exception to be used when scene id is incorrect """ pass class Downloader(VerbosityMixin): + """ The downloader class """ def __init__(self, verbose=False, download_dir=None): self.download_dir = download_dir if download_dir else settings.DOWNLOAD_DIR @@ -32,9 +35,17 @@ class Downloader(VerbosityMixin): """ Download scenese from Google Storage or Amazon S3 if bands are provided - @params - scenes - A list of sceneIDs - bands - A list of bands + :param scenes: + A list of scene IDs + :type scenes: + List + :param bands: + A list of bands. Default value is None. + :type scenes: + List + + :returns: + Boolean """ if isinstance(scenes, list): @@ -63,7 +74,21 @@ class Downloader(VerbosityMixin): raise Exception('Expected sceneIDs list') def google_storage(self, scene, path): - """ Google Storage Downloader """ + """ + Google Storage Downloader. + + :param scene: + The scene id + :type scene: + String + :param path: + The directory path to where the image should be stored + :type path: + String + + :returns: + Boolean + """ sat = self.scene_interpreter(scene) filename = scene + '.tar.bz' @@ -76,7 +101,25 @@ class Downloader(VerbosityMixin): raise RemoteFileDoesntExist('%s is not available on Google Storage' % filename) def amazon_s3(self, scene, band, path): - """ Amazon S3 downloader """ + """ + Amazon S3 downloader + + :param scene: + The scene ID. + :type scene: + String + :param band: + The band number. + :type band: + String, Integer + :param path: + The directory path to where the image should be stored + :type path: + String + + :returns: + Boolean + """ sat = self.scene_interpreter(scene) if band != 'MTL': @@ -92,6 +135,24 @@ class Downloader(VerbosityMixin): raise RemoteFileDoesntExist('%s is not available on Amazon S3' % filename) def fetch(self, url, path, filename): + """ Downloads the given url. + + :param url: + The url to be downloaded. + :type url: + String + :param path: + The directory path to where the image should be stored + :type path: + String + :param filename: + The filename that has to be downloaded + :type filename: + String + + :returns: + Boolean + """ self.output('Downloading: %s' % filename, normal=True, arrow=True) @@ -108,22 +169,48 @@ class Downloader(VerbosityMixin): def google_storage_url(self, sat): """ - Return a google storage url the contains the scene provided - @params - sat - expects an object created by scene_interpreter method + Returns a google storage url the contains the scene provided. + + :param sat: + Expects an object created by scene_interpreter method + :type sat: + dict + + :returns: + (String) The URL to a google storage file """ filename = sat['scene'] + '.tar.bz' return join(self.google, sat['sat'], sat['path'], sat['row'], filename) def amazon_s3_url(self, sat, filename): """ - Return an amazon s3 url the contains the scene and band provided - @params - sat - expects an object created by scene_interpreter method + Return an amazon s3 url the contains the scene and band provided. + + :param sat: + Expects an object created by scene_interpreter method + :type sat: + dict + :param filename: + The filename that has to be downloaded from Amazon + :type filename: + String + + :returns: + (String) The URL to a S3 file """ return join(self.s3, sat['sat'], sat['path'], sat['row'], sat['scene'], filename) def remote_file_exists(self, url): + """ Checks whether the remote file exists. + + :param url: + The url that has to be checked. + :type url: + String + + :returns: + **True** if remote file exists and **False** if it doesn't exist. + """ status = requests.head(url).status_code if status == 200: @@ -132,12 +219,39 @@ class Downloader(VerbosityMixin): return False def get_remote_file_size(self, url): - """ Gets the filesize of a remote file """ + """ Gets the filesize of a remote file. + + :param url: + The url that has to be checked. + :type url: + String + + :returns: + int + """ headers = requests.head(url).headers return int(headers['content-length']) def scene_interpreter(self, scene): - """ Conver sceneID to rows, paths and dates """ + """ Conver sceneID to rows, paths and dates. + + :param scene: + The scene ID. + :type scene: + String + + :returns: + dict + + :Example output: + + >>> anatomy = { + 'path': None, + 'row': None, + 'sat': None, + 'scene': scene + } + """ anatomy = { 'path': None, 'row': None, diff --git a/landsat/image.py b/landsat/image.py index fd7013e..ccdb759 100644 --- a/landsat/image.py +++ b/landsat/image.py @@ -23,24 +23,36 @@ from utils import get_file, timer, check_create_folder, exit class FileDoesNotExist(Exception): + """ Exception to be used when the file does not exist. """ pass class Process(VerbosityMixin): """ Image procssing class + + To initiate the following parameters must be passed: + + :param path: + Path of the image. + :type path: + String + :param bands: + The band sequence for the final image. Must be a python list. (optional) + :type bands: + List + :param dst_path: + Path to the folder where the image should be stored. (optional) + :type dst_path: + String + :param verbose: + Whether the output should be verbose. Default is False. + :type verbose: + boolean + """ def __init__(self, path, bands=None, dst_path=None, verbose=False): - """ - @params - scene - the scene ID - bands - The band sequence for the final image. Must be a python list - src_path - The path to the source image bundle - dst_path - The destination path - zipped - Set to true if the scene is in zip format and requires unzipping - verbose - Whether to sh ow verbose output - """ self.projection = {'init': 'epsg:3857'} self.dst_crs = {'init': u'epsg:3857'} @@ -66,6 +78,16 @@ class Process(VerbosityMixin): self.bands_path.append(join(self.scene_path, self._get_full_filename(band))) def run(self, pansharpen=True): + """ Executes the image processing. + + :param pansharpen: + Whether the process should also run pansharpenning. Default is True + :type pansharpen: + boolean + + :returns: + (String) the path to the processed image + """ self.output("* Image processing started for bands %s" % "-".join(map(str, self.bands)), normal=True) @@ -120,10 +142,8 @@ class Process(VerbosityMixin): dst_shape = src_data['shape'] dst_corner_ys = [crn[k]['y'][1][0] for k in crn.keys()] dst_corner_xs = [crn[k]['x'][1][0] for k in crn.keys()] - y_pixel = abs(max(dst_corner_ys) - - min(dst_corner_ys)) / dst_shape[0] - x_pixel = abs(max(dst_corner_xs) - - min(dst_corner_xs)) / dst_shape[1] + y_pixel = abs(max(dst_corner_ys) - min(dst_corner_ys)) / dst_shape[0] + x_pixel = abs(max(dst_corner_xs) - min(dst_corner_xs)) / dst_shape[1] dst_transform = (min(dst_corner_xs), x_pixel, diff --git a/landsat/landsat.py b/landsat/landsat.py index c81f4d2..328f01a 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -119,6 +119,12 @@ search, download, and process Landsat imagery. def args_options(): + """ Generates an arugment parser. + + :returns: + Parser object + """ + parser = argparse.ArgumentParser(prog='landsat', formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent(DESCRIPTION)) @@ -198,7 +204,18 @@ def args_options(): def main(args): """ - Main function - launches the program + Main function - launches the program. + + :param args: + The Parser arguments + :type args: + Parser object + + :returns: + List + + :example: + >>> ["The latitude and longitude values must be valid numbers", 1] """ v = VerbosityMixin() @@ -282,6 +299,28 @@ def main(args): def process_image(path, bands=None, verbose=False, pansharpen=False): + """ Handles constructing and image process. + + :param path: + The path to the image that has to be processed + :type path: + String + :param bands: + List of bands that has to be processed. (optional) + :type bands: + List + :param verbose: + Sets the level of verbosity. Default is False. + :type verbose: + boolean + :param pansharpen: + Whether to pansharpen the image. Default is False. + :type pansharpen: + boolean + + :returns: + (String) path to the processed image + """ try: bands = convert_to_integer_list(bands) p = Process(path, bands=bands, verbose=verbose) diff --git a/landsat/mixins.py b/landsat/mixins.py index a50d8b4..0adbf66 100644 --- a/landsat/mixins.py +++ b/landsat/mixins.py @@ -10,9 +10,6 @@ from termcolor import colored class VerbosityMixin(object): """ Verbosity Mixin that generates beautiful stdout outputs. - - Main method: - output() """ verbose = False @@ -24,14 +21,33 @@ class VerbosityMixin(object): if class instance verbose is True, the value is printed - @param - - value: (string) the message to be printed - - nomral: (boolean) if set to true the message is always printed, - otherwise it is only shown if verbosity is set - - color: (string) The color of the message, choices: 'red', 'green', 'blue' - - error: (boolean) if set to true the message appears in red - - arrow: (boolean) if set to true an arrow appears before the message - - indent: (integer) indents the message based on the number provided + :param value: + a string representing the message to be printed + :type value: + String + :param normal: + if set to true the message is always printed, otherwise it is only shown if verbosity is set + :type normal: + boolean + :param color: + The color of the message, choices: 'red', 'green', 'blue' + :type normal: + String + :param error: + if set to true the message appears in red + :type error: + Boolean + :param arrow: + if set to true an arrow appears before the message + :type arrow: + Boolean + :param indent: + indents the message based on the number provided + :type indent: + Boolean + + :returns: + void """ if error and value and (normal or self.verbose): @@ -44,8 +60,16 @@ class VerbosityMixin(object): def subprocess(self, argv): """ - Execute subprocess commands with proper ouput + Execute subprocess commands with proper ouput. This is no longer used in landsat-util + + :param argv: + A list of subprocess arguments + :type argv: + List + + :returns: + void """ if self.verbose: @@ -59,13 +83,22 @@ class VerbosityMixin(object): return def exit(self, message): - """ Print an exist message and exit """ + """ outputs an exit message and exits + + :param message: + The message to be outputed + :type message: + String + + :returns: + void + """ self.output(message, normal=True, color="green") sys.exit() def _print(self, msg, color=None, arrow=False, indent=None): - """ Print the msg with the color provided """ + """ Print the msg with the color provided. """ if color: msg = colored(msg, color) diff --git a/landsat/search.py b/landsat/search.py index c3d6bfd..65ebc46 100644 --- a/landsat/search.py +++ b/landsat/search.py @@ -10,6 +10,7 @@ from utils import three_digit, create_paired_list class Search(object): + """ The search class """ def __init__(self): self.api_url = settings.API_URL @@ -17,43 +18,64 @@ class Search(object): def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=None, cloud_min=None, cloud_max=None, limit=1): """ - The main method of Search class. It searches the DevSeed Landsat API + The main method of Search class. It searches Development Seed's Landsat API. - Returns python dictionary + :param paths_rows: + A string in this format: "003,003,004,004". Must be in pairs and separated by comma. + :type paths_rows: + String + :param lat: + The latitude + :type lat: + String, float, integer + :param lon: + The The longitude + :type lon: + String, float, integer + :param start_date: + Date string. format: YYYY-MM-DD + :type start_date: + String + :param end_date: + date string. format: YYYY-MM-DD + :type end_date: + String + :param cloud_min: + float specifying the minimum percentage. e.g. 4.3 + :type cloud_min: + float + :param cloud_max: + float specifying the maximum percentage. e.g. 78.9 + :type cloud_max: + float + :param limit: + integer specigying the maximum results return. + :type limit: + integer - Arguments: - paths_rows -- A string in this format: "003,003,004,004". Must be in pairs - lat -- The latitude - lon -- The longitude - start_date -- date string. format: YYYY-MM-DD - end_date -- date string. format: YYYY-MM-DD - cloud_min -- float specifying the minimum percentage. e.g. 4.3 - cloud_max -- float specifying the maximum percentage. e.g. 78.9 - limit -- integer specigying the maximum results return. + :returns: + dict - Example: - - search('003,003', '2014-01-01', '2014-06-01') - - will return: - - { - 'status': u'SUCCESS', - 'total_returned': 1, - 'total': 1, - 'limit': 1 - 'results': [ - { - 'sat_type': u'L8', - 'sceneID': u'LC80030032014142LGN00', - 'date': u'2014-05-22', - 'path': u'003', - 'thumbnail': u'http://....../landsat_8/2014/003/003/LC80030032014142LGN00.jpg', - 'cloud': 33.36, - 'row': u'003 - } - ] - } + :example: + >>> search = Search() + >>> search('003,003', '2014-01-01', '2014-06-01') + >>> { + 'status': u'SUCCESS', + 'total_returned': 1, + 'total': 1, + 'limit': 1 + 'results': [ + { + 'sat_type': u'L8', + 'sceneID': u'LC80030032014142LGN00', + 'date': u'2014-05-22', + 'path': u'003', + 'thumbnail': u'http://....../landsat_8/2014/003/003/LC80030032014142LGN00.jpg', + 'cloud': 33.36, + 'row': u'003 + } + ] + } """ search_string = self.query_builder(paths_rows, lat, lon, start_date, end_date, cloud_min, cloud_max) @@ -89,7 +111,40 @@ class Search(object): def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=None, cloud_min=None, cloud_max=None): - """ Builds the proper search syntax (query) for Landsat API """ + """ Builds the proper search syntax (query) for Landsat API. + + :param paths_rows: + A string in this format: "003,003,004,004". Must be in pairs and separated by comma. + :type paths_rows: + String + :param lat: + The latitude + :type lat: + String, float, integer + :param lon: + The The longitude + :type lon: + String, float, integer + :param start_date: + Date string. format: YYYY-MM-DD + :type start_date: + String + :param end_date: + date string. format: YYYY-MM-DD + :type end_date: + String + :param cloud_min: + float specifying the minimum percentage. e.g. 4.3 + :type cloud_min: + float + :param cloud_max: + float specifying the maximum percentage. e.g. 78.9 + :type cloud_max: + float + + :returns: + String + """ query = [] or_string = '' @@ -131,15 +186,37 @@ class Search(object): def row_path_builder(self, path='', row=''): """ - Builds row and path query - Accepts row and path in XXX format, e.g. 003 + Builds row and path query. + + :param path: + Landsat path. Must be three digits + :type path: + String + :param row: + Landsat row. Must be three digits + :type row: + String + + :returns: + String """ return 'path:%s+AND+row:%s' % (path, row) def date_range_builder(self, start='2013-02-11', end=None): """ - Builds date range query - Accepts start and end date in this format YYYY-MM-DD + Builds date range query. + + :param start: + Date string. format: YYYY-MM-DD + :type start: + String + :param end: + date string. format: YYYY-MM-DD + :type end: + String + + :returns: + String """ if not end: end = time.strftime('%Y-%m-%d') @@ -148,13 +225,37 @@ class Search(object): def cloud_cover_prct_range_builder(self, min=0, max=100): """ - Builds cloud cover percentage range query - Accepts bottom and top range in float, e.g. 1.00 + Builds cloud cover percentage range query. + + :param min: + float specifying the minimum percentage. Default is 0 + :type min: + float + :param max: + float specifying the maximum percentage. Default is 100 + :type max: + float + + :returns: + String """ return 'cloudCoverFull:[%s+TO+%s]' % (min, max) def lat_lon_builder(self, lat=0, lon=0): - """ Builds lat and lon query """ + """ Builds lat and lon query. + + :param lat: + The latitude. Default is 0 + :type lat: + float + :param lon: + The The longitude. Default is 0 + :type lon: + float + + :returns: + String + """ return ('upperLeftCornerLatitude:[%s+TO+1000]+AND+lowerRightCornerLatitude:[-1000+TO+%s]' '+AND+lowerLeftCornerLongitude:[-1000+TO+%s]+AND+upperRightCornerLongitude:[%s+TO+1000]' % (lat, lat, lon, lon)) diff --git a/landsat/uploader.py b/landsat/uploader.py index bbbc151..c1e9e4f 100644 --- a/landsat/uploader.py +++ b/landsat/uploader.py @@ -27,6 +27,25 @@ STREAM = sys.stderr class Uploader(VerbosityMixin): + """ + The Uploader class. + + To initiate the following parameters must be passed: + + :param key: + AWS access key id (optional) + :type key: + String + :param secret: + AWS access secret key (optional) + :type secret: + String + :param host: + AWS host, e.g. s3.amazonaws.com (optional) + :type host: + String + """ + progress_template = \ 'File Size:%(size)4d MB | Uploaded:%(uploaded)4d MB' + ' ' * 8 @@ -37,6 +56,25 @@ class Uploader(VerbosityMixin): self.conn = S3Connection(key, secret, host=host) def run(self, bucket_name, filename, path): + """ + Initiate the upload. + + :param bucket_name: + Name of the S3 bucket + :type bucket_name: + String + :param filename: + The filname + :type filename: + String + :param path: + The path to the file that needs to be uploaded + :type path: + String + + :returns: + void + """ f = open(path, 'rb') self.source_size = os.stat(path).st_size @@ -66,15 +104,16 @@ class Uploader(VerbosityMixin): def data_collector(iterable, def_buf_size=5242880): - ''' Buffers n bytes of data + """ Buffers n bytes of data. - Args: - iterable: could be a list, generator or string - def_buf_size: number of bytes to buffer, default is 5mb + :param iterable: + Could be a list, generator or string + :type iterable: + List, generator, String - Returns: - A generator object - ''' + :returns: + A generator object + """ buf = '' for data in iterable: buf += data @@ -108,23 +147,54 @@ def upload(bucket, aws_access_key, aws_secret_key, iterable, key, progress_cb=None, threads=5, replace=False, secure=True, connection=None): - ''' Upload data to s3 using the s3 multipart upload API. + """ Upload data to s3 using the s3 multipart upload API. - Args: - bucket: name of s3 bucket - aws_access_key: aws access key - aws_secret_key: aws secret key - iterable: The data to upload. Each 'part' in the list - will be uploaded in parallel. Each part must be at - least 5242880 bytes (5mb). - key: the name of the key to create in the s3 bucket - progress_cb: will be called with (part_no, uploaded, total) - each time a progress update is available. - threads: the number of threads to use while uploading. (Default is 5) - replace: will replace the key in s3 if set to true. (Default is false) - secure: use ssl when talking to s3. (Default is true) - connection: used for testing - ''' + :param bucket: + Name of the S3 bucket + :type bucket: + String + :param aws_access_key: + AWS access key id (optional) + :type aws_access_key: + String + :param aws_secret_key: + AWS access secret key (optional) + :type aws_secret_key: + String + :param iterable: + The data to upload. Each 'part' in the list. will be uploaded in parallel. Each part must be at + least 5242880 bytes (5mb). + :type iterable: + An iterable object + :param key: + The name of the key (filename) to create in the s3 bucket + :type key: + String + :param progress_cb: + Progress callback, will be called with (part_no, uploaded, total) each time a progress update + is available. (optional) + :type progress_cb: + function + :param threads: + the number of threads to use while uploading. (Default is 5) + :type threads: + int + :param replace: + will replace the key (filename) on S3 if set to true. (Default is false) + :type replace: + boolean + :param secure: + Use ssl when talking to s3. (Default is true) + :type secure: + boolean + :param connection: + Used for testing (optional) + :type connection: + S3 connection class + + :returns: + void + """ if not connection: from boto.s3.connection import S3Connection as connection diff --git a/landsat/utils.py b/landsat/utils.py index 9591f0b..b5dc6e3 100644 --- a/landsat/utils.py +++ b/landsat/utils.py @@ -12,7 +12,13 @@ from mixins import VerbosityMixin class Capturing(list): - """ Captures a subprocess stdout """ + """ + Captures a subprocess stdout. + + :Usage: + >>> with Capturing(): + ... subprocess(args) + """ def __enter__(self): self._stdout = sys.stdout sys.stdout = self._stringio = StringIO() @@ -25,11 +31,11 @@ class Capturing(list): class timer(object): """ - A time class + A timer class. - Usage: - with timer(): - your code + :Usage: + >>> with timer(): + ... your code """ def __enter__(self): self.start = time.time() @@ -40,6 +46,21 @@ class timer(object): def exit(message, code=0): + """ output a message to stdout and terminates the process. + + :param message: + Message to be outputed. + :type message: + String + :param code: + The termination code. Default is 0 + :type code: + int + + :returns: + void + """ + v = VerbosityMixin() if code == 0: v.output(message, normal=True, arrow=True) @@ -50,13 +71,20 @@ def exit(message, code=0): def create_paired_list(value): - """ Create a list of paired items from a string + """ Create a list of paired items from a string. - Arguments: - i - the format must be 003,003,004,004 (commas with no space) + :param value: + the format must be 003,003,004,004 (commas with no space) + :type value: + String - Returns: + :returns: + List + + :example: + >>> create_paired_list('003,003,004,004') [['003','003'], ['004', '004']] + """ if isinstance(value, list): @@ -75,8 +103,15 @@ def create_paired_list(value): def check_create_folder(folder_path): - """ Check whether a folder exists, if not the folder is created - Always return folder_path + """ Check whether a folder exists, if not the folder is created. + + :param folder_path: + Path to the folder + :type folder_path: + String + + :returns: + (String) the path to the folder """ if not os.path.exists(folder_path): os.makedirs(folder_path) @@ -85,20 +120,55 @@ def check_create_folder(folder_path): def get_file(path): - """ Separate the name of the file or folder from the path and return it - Example: /path/to/file ---> file + """ Separate the name of the file or folder from the path and return it. + + :param path: + Path to the folder + :type path: + String + + :returns: + (String) the filename + + :example: + >>> get_file('/path/to/file.jpg') + 'file.jpg' """ return os.path.basename(path) def get_filename(path): - """ Return the filename without extension. e.g. index.html --> index """ + """ Return the filename without extension. + + :param path: + Path to the folder + :type path: + String + + :returns: + (String) the filename without extension + + :example: + >>> get_filename('/path/to/file.jpg') + 'file' + """ return os.path.splitext(get_file(path))[0] def three_digit(number): """ Add 0s to inputs that their length is less than 3. - For example: 1 --> 001 | 02 --> 020 | st --> 0st + + :param number: + The number to convert + :type number: + int + + :returns: + String + + :example: + >>> three_digit(1) + '001' """ number = str(number) if len(number) == 1: @@ -110,8 +180,19 @@ def three_digit(number): def georgian_day(date): - """ Returns the number of days passed since the start of the year - Accepted format: %m/%d/%Y + """ Returns the number of days passed since the start of the year. + + :param date: + The string date with this format %m/%d/%Y + :type date: + String + + :returns: + int + + :example: + >>> georgian_day('05/1/2015') + 121 """ try: fmt = '%m/%d/%Y' @@ -121,8 +202,19 @@ def georgian_day(date): def year(date): - """ Returns the year - Accepted format: %m/%d/%Y + """ Returns the year. + + :param date: + The string date with this format %m/%d/%Y + :type date: + String + + :returns: + int + + :example: + >>> year('05/1/2015') + 2015 """ try: fmt = '%m/%d/%Y' @@ -132,8 +224,23 @@ def year(date): def reformat_date(date, new_fmt='%Y-%m-%d'): - """ Return reformated date. Example: 01/28/2014 & %d/%m/%Y -> 28/01/2014 - Accepted date format: %m/%d/%Y + """ Returns reformated date. + + :param date: + The string date with this format %m/%d/%Y + :type date: + String + :param new_fmt: + date format string. Default is '%Y-%m-%d' + :type date: + String + + :returns: + int + + :example: + >>> reformat_date('05/1/2015', '%d/%m/%Y') + '1/05/2015' """ try: if isinstance(date, datetime): @@ -146,7 +253,21 @@ def reformat_date(date, new_fmt='%Y-%m-%d'): def convert_to_integer_list(value): - """ convert a comma separate string to a list where all values are integers """ + """ Converts a comma separate string to a list + + :param value: + the format must be 003,003,004,004 (commas with no space) + :type value: + String + + :returns: + List + + :example: + >>> convert_to_integer_list('003,003,004,004') + ['003', '003', '004', '004'] + + """ if value and isinstance(value, str): if ',' in value: