jlengrand/landsat-util
1
0
Fork 0
mirror of https://github.com/Skyai-io/landsat-util.git synced 2026-03-10 09:01:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

improved doc strings

Browse Source
This commit is contained in:
Scisco
2015-05-29 13:11:59 +01:00
parent 3531d17339
commit 9795ece4bf
7 changed files with 626 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
landsat/downloader.py
View File

@@ -11,14 +11,17 @@ import settings
class RemoteFileDoesntExist(Exception):
""" Exception to be used when the remote file does not exist """
pass
class IncorrectSceneId(Exception):
""" Exception to be used when scene id is incorrect """
pass
class Downloader(VerbosityMixin):
""" The downloader class """
def __init__(self, verbose=False, download_dir=None):
self.download_dir = download_dir if download_dir else settings.DOWNLOAD_DIR
@@ -32,9 +35,17 @@ class Downloader(VerbosityMixin):
"""
Download scenese from Google Storage or Amazon S3 if bands are provided
@params
scenes - A list of sceneIDs
bands - A list of bands
:param scenes:
A list of scene IDs
:type scenes:
List
:param bands:
A list of bands. Default value is None.
:type scenes:
List
:returns:
Boolean
"""
if isinstance(scenes, list):
@@ -63,7 +74,21 @@ class Downloader(VerbosityMixin):
raise Exception('Expected sceneIDs list')
def google_storage(self, scene, path):
""" Google Storage Downloader """
"""
Google Storage Downloader.
:param scene:
The scene id
:type scene:
String
:param path:
The directory path to where the image should be stored
:type path:
String
:returns:
Boolean
"""
sat = self.scene_interpreter(scene)
filename = scene + '.tar.bz'
@@ -76,7 +101,25 @@ class Downloader(VerbosityMixin):
raise RemoteFileDoesntExist('%s is not available on Google Storage' % filename)
def amazon_s3(self, scene, band, path):
""" Amazon S3 downloader """
"""
Amazon S3 downloader
:param scene:
The scene ID.
:type scene:
String
:param band:
The band number.
:type band:
String, Integer
:param path:
The directory path to where the image should be stored
:type path:
String
:returns:
Boolean
"""
sat = self.scene_interpreter(scene)
if band != 'MTL':
@@ -92,6 +135,24 @@ class Downloader(VerbosityMixin):
raise RemoteFileDoesntExist('%s is not available on Amazon S3' % filename)
def fetch(self, url, path, filename):
""" Downloads the given url.
:param url:
The url to be downloaded.
:type url:
String
:param path:
The directory path to where the image should be stored
:type path:
String
:param filename:
The filename that has to be downloaded
:type filename:
String
:returns:
Boolean
"""
self.output('Downloading: %s' % filename, normal=True, arrow=True)
@@ -108,22 +169,48 @@ class Downloader(VerbosityMixin):
def google_storage_url(self, sat):
"""
Return a google storage url the contains the scene provided
@params
sat - expects an object created by scene_interpreter method
Returns a google storage url the contains the scene provided.
:param sat:
Expects an object created by scene_interpreter method
:type sat:
dict
:returns:
(String) The URL to a google storage file
"""
filename = sat['scene'] + '.tar.bz'
return join(self.google, sat['sat'], sat['path'], sat['row'], filename)
def amazon_s3_url(self, sat, filename):
"""
Return an amazon s3 url the contains the scene and band provided
@params
sat - expects an object created by scene_interpreter method
Return an amazon s3 url the contains the scene and band provided.
:param sat:
Expects an object created by scene_interpreter method
:type sat:
dict
:param filename:
The filename that has to be downloaded from Amazon
:type filename:
String
:returns:
(String) The URL to a S3 file
"""
return join(self.s3, sat['sat'], sat['path'], sat['row'], sat['scene'], filename)
def remote_file_exists(self, url):
""" Checks whether the remote file exists.
:param url:
The url that has to be checked.
:type url:
String
:returns:
**True** if remote file exists and **False** if it doesn't exist.
"""
status = requests.head(url).status_code
if status == 200:
@@ -132,12 +219,39 @@ class Downloader(VerbosityMixin):
return False
def get_remote_file_size(self, url):
""" Gets the filesize of a remote file """
""" Gets the filesize of a remote file.
:param url:
The url that has to be checked.
:type url:
String
:returns:
int
"""
headers = requests.head(url).headers
return int(headers['content-length'])
def scene_interpreter(self, scene):
""" Conver sceneID to rows, paths and dates """
""" Conver sceneID to rows, paths and dates.
:param scene:
The scene ID.
:type scene:
String
:returns:
dict
:Example output:
>>> anatomy = {
'path': None,
'row': None,
'sat': None,
'scene': scene
}
"""
anatomy = {
'path': None,
'row': None,

46
landsat/image.py
View File

@@ -23,24 +23,36 @@ from utils import get_file, timer, check_create_folder, exit
class FileDoesNotExist(Exception):
""" Exception to be used when the file does not exist. """
pass
class Process(VerbosityMixin):
"""
Image procssing class
To initiate the following parameters must be passed:
:param path:
Path of the image.
:type path:
String
:param bands:
The band sequence for the final image. Must be a python list. (optional)
:type bands:
List
:param dst_path:
Path to the folder where the image should be stored. (optional)
:type dst_path:
String
:param verbose:
Whether the output should be verbose. Default is False.
:type verbose:
boolean
"""
def __init__(self, path, bands=None, dst_path=None, verbose=False):
"""
@params
scene - the scene ID
bands - The band sequence for the final image. Must be a python list
src_path - The path to the source image bundle
dst_path - The destination path
zipped - Set to true if the scene is in zip format and requires unzipping
verbose - Whether to sh ow verbose output
"""
self.projection = {'init': 'epsg:3857'}
self.dst_crs = {'init': u'epsg:3857'}
@@ -66,6 +78,16 @@ class Process(VerbosityMixin):
self.bands_path.append(join(self.scene_path, self._get_full_filename(band)))
def run(self, pansharpen=True):
""" Executes the image processing.
:param pansharpen:
Whether the process should also run pansharpenning. Default is True
:type pansharpen:
boolean
:returns:
(String) the path to the processed image
"""
self.output("* Image processing started for bands %s" % "-".join(map(str, self.bands)), normal=True)
@@ -120,10 +142,8 @@ class Process(VerbosityMixin):
dst_shape = src_data['shape']
dst_corner_ys = [crn[k]['y'][1][0] for k in crn.keys()]
dst_corner_xs = [crn[k]['x'][1][0] for k in crn.keys()]
y_pixel = abs(max(dst_corner_ys) -
min(dst_corner_ys)) / dst_shape[0]
x_pixel = abs(max(dst_corner_xs) -
min(dst_corner_xs)) / dst_shape[1]
y_pixel = abs(max(dst_corner_ys) - min(dst_corner_ys)) / dst_shape[0]
x_pixel = abs(max(dst_corner_xs) - min(dst_corner_xs)) / dst_shape[1]
dst_transform = (min(dst_corner_xs),
x_pixel,

41
landsat/landsat.py
View File

@@ -119,6 +119,12 @@ search, download, and process Landsat imagery.
def args_options():
""" Generates an arugment parser.
:returns:
Parser object
"""
parser = argparse.ArgumentParser(prog='landsat',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent(DESCRIPTION))
@@ -198,7 +204,18 @@ def args_options():
def main(args):
"""
Main function - launches the program
Main function - launches the program.
:param args:
The Parser arguments
:type args:
Parser object
:returns:
List
:example:
>>> ["The latitude and longitude values must be valid numbers", 1]
"""
v = VerbosityMixin()
@@ -282,6 +299,28 @@ def main(args):
def process_image(path, bands=None, verbose=False, pansharpen=False):
""" Handles constructing and image process.
:param path:
The path to the image that has to be processed
:type path:
String
:param bands:
List of bands that has to be processed. (optional)
:type bands:
List
:param verbose:
Sets the level of verbosity. Default is False.
:type verbose:
boolean
:param pansharpen:
Whether to pansharpen the image. Default is False.
:type pansharpen:
boolean
:returns:
(String) path to the processed image
"""
try:
bands = convert_to_integer_list(bands)
p = Process(path, bands=bands, verbose=verbose)

61
landsat/mixins.py
View File

@@ -10,9 +10,6 @@ from termcolor import colored
class VerbosityMixin(object):
"""
Verbosity Mixin that generates beautiful stdout outputs.
Main method:
output()
"""
verbose = False
@@ -24,14 +21,33 @@ class VerbosityMixin(object):
if class instance verbose is True, the value is printed
@param
- value: (string) the message to be printed
- nomral: (boolean) if set to true the message is always printed,
otherwise it is only shown if verbosity is set
- color: (string) The color of the message, choices: 'red', 'green', 'blue'
- error: (boolean) if set to true the message appears in red
- arrow: (boolean) if set to true an arrow appears before the message
- indent: (integer) indents the message based on the number provided
:param value:
a string representing the message to be printed
:type value:
String
:param normal:
if set to true the message is always printed, otherwise it is only shown if verbosity is set
:type normal:
boolean
:param color:
The color of the message, choices: 'red', 'green', 'blue'
:type normal:
String
:param error:
if set to true the message appears in red
:type error:
Boolean
:param arrow:
if set to true an arrow appears before the message
:type arrow:
Boolean
:param indent:
indents the message based on the number provided
:type indent:
Boolean
:returns:
void
"""
if error and value and (normal or self.verbose):
@@ -44,8 +60,16 @@ class VerbosityMixin(object):
def subprocess(self, argv):
"""
Execute subprocess commands with proper ouput
Execute subprocess commands with proper ouput.
This is no longer used in landsat-util
:param argv:
A list of subprocess arguments
:type argv:
List
:returns:
void
"""
if self.verbose:
@@ -59,13 +83,22 @@ class VerbosityMixin(object):
return
def exit(self, message):
""" Print an exist message and exit """
""" outputs an exit message and exits
:param message:
The message to be outputed
:type message:
String
:returns:
void
"""
self.output(message, normal=True, color="green")
sys.exit()
def _print(self, msg, color=None, arrow=False, indent=None):
""" Print the msg with the color provided """
""" Print the msg with the color provided. """
if color:
msg = colored(msg, color)

185
landsat/search.py
View File

@@ -10,6 +10,7 @@ from utils import three_digit, create_paired_list
class Search(object):
""" The search class """
def __init__(self):
self.api_url = settings.API_URL
@@ -17,43 +18,64 @@ class Search(object):
def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=None, cloud_min=None,
cloud_max=None, limit=1):
"""
The main method of Search class. It searches the DevSeed Landsat API
The main method of Search class. It searches Development Seed's Landsat API.
Returns python dictionary
:param paths_rows:
A string in this format: "003,003,004,004". Must be in pairs and separated by comma.
:type paths_rows:
String
:param lat:
The latitude
:type lat:
String, float, integer
:param lon:
The The longitude
:type lon:
String, float, integer
:param start_date:
Date string. format: YYYY-MM-DD
:type start_date:
String
:param end_date:
date string. format: YYYY-MM-DD
:type end_date:
String
:param cloud_min:
float specifying the minimum percentage. e.g. 4.3
:type cloud_min:
float
:param cloud_max:
float specifying the maximum percentage. e.g. 78.9
:type cloud_max:
float
:param limit:
integer specigying the maximum results return.
:type limit:
integer
Arguments:
paths_rows -- A string in this format: "003,003,004,004". Must be in pairs
lat -- The latitude
lon -- The longitude
start_date -- date string. format: YYYY-MM-DD
end_date -- date string. format: YYYY-MM-DD
cloud_min -- float specifying the minimum percentage. e.g. 4.3
cloud_max -- float specifying the maximum percentage. e.g. 78.9
limit -- integer specigying the maximum results return.
:returns:
dict
Example:
search('003,003', '2014-01-01', '2014-06-01')
will return:
{
'status': u'SUCCESS',
'total_returned': 1,
'total': 1,
'limit': 1
'results': [
{
'sat_type': u'L8',
'sceneID': u'LC80030032014142LGN00',
'date': u'2014-05-22',
'path': u'003',
'thumbnail': u'http://....../landsat_8/2014/003/003/LC80030032014142LGN00.jpg',
'cloud': 33.36,
'row': u'003
}
]
}
:example:
>>> search = Search()
>>> search('003,003', '2014-01-01', '2014-06-01')
>>> {
'status': u'SUCCESS',
'total_returned': 1,
'total': 1,
'limit': 1
'results': [
{
'sat_type': u'L8',
'sceneID': u'LC80030032014142LGN00',
'date': u'2014-05-22',
'path': u'003',
'thumbnail': u'http://....../landsat_8/2014/003/003/LC80030032014142LGN00.jpg',
'cloud': 33.36,
'row': u'003
}
]
}
"""
search_string = self.query_builder(paths_rows, lat, lon, start_date, end_date, cloud_min, cloud_max)
@@ -89,7 +111,40 @@ class Search(object):
def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=None,
cloud_min=None, cloud_max=None):
""" Builds the proper search syntax (query) for Landsat API """
""" Builds the proper search syntax (query) for Landsat API.
:param paths_rows:
A string in this format: "003,003,004,004". Must be in pairs and separated by comma.
:type paths_rows:
String
:param lat:
The latitude
:type lat:
String, float, integer
:param lon:
The The longitude
:type lon:
String, float, integer
:param start_date:
Date string. format: YYYY-MM-DD
:type start_date:
String
:param end_date:
date string. format: YYYY-MM-DD
:type end_date:
String
:param cloud_min:
float specifying the minimum percentage. e.g. 4.3
:type cloud_min:
float
:param cloud_max:
float specifying the maximum percentage. e.g. 78.9
:type cloud_max:
float
:returns:
String
"""
query = []
or_string = ''
@@ -131,15 +186,37 @@ class Search(object):
def row_path_builder(self, path='', row=''):
"""
Builds row and path query
Accepts row and path in XXX format, e.g. 003
Builds row and path query.
:param path:
Landsat path. Must be three digits
:type path:
String
:param row:
Landsat row. Must be three digits
:type row:
String
:returns:
String
"""
return 'path:%s+AND+row:%s' % (path, row)
def date_range_builder(self, start='2013-02-11', end=None):
"""
Builds date range query
Accepts start and end date in this format YYYY-MM-DD
Builds date range query.
:param start:
Date string. format: YYYY-MM-DD
:type start:
String
:param end:
date string. format: YYYY-MM-DD
:type end:
String
:returns:
String
"""
if not end:
end = time.strftime('%Y-%m-%d')
@@ -148,13 +225,37 @@ class Search(object):
def cloud_cover_prct_range_builder(self, min=0, max=100):
"""
Builds cloud cover percentage range query
Accepts bottom and top range in float, e.g. 1.00
Builds cloud cover percentage range query.
:param min:
float specifying the minimum percentage. Default is 0
:type min:
float
:param max:
float specifying the maximum percentage. Default is 100
:type max:
float
:returns:
String
"""
return 'cloudCoverFull:[%s+TO+%s]' % (min, max)
def lat_lon_builder(self, lat=0, lon=0):
""" Builds lat and lon query """
""" Builds lat and lon query.
:param lat:
The latitude. Default is 0
:type lat:
float
:param lon:
The The longitude. Default is 0
:type lon:
float
:returns:
String
"""
return ('upperLeftCornerLatitude:[%s+TO+1000]+AND+lowerRightCornerLatitude:[-1000+TO+%s]'
'+AND+lowerLeftCornerLongitude:[-1000+TO+%s]+AND+upperRightCornerLongitude:[%s+TO+1000]'
% (lat, lat, lon, lon))

116
landsat/uploader.py
View File

@@ -27,6 +27,25 @@ STREAM = sys.stderr
class Uploader(VerbosityMixin):
"""
The Uploader class.
To initiate the following parameters must be passed:
:param key:
AWS access key id (optional)
:type key:
String
:param secret:
AWS access secret key (optional)
:type secret:
String
:param host:
AWS host, e.g. s3.amazonaws.com (optional)
:type host:
String
"""
progress_template = \
'File Size:%(size)4d MB | Uploaded:%(uploaded)4d MB' + ' ' * 8
@@ -37,6 +56,25 @@ class Uploader(VerbosityMixin):
self.conn = S3Connection(key, secret, host=host)
def run(self, bucket_name, filename, path):
"""
Initiate the upload.
:param bucket_name:
Name of the S3 bucket
:type bucket_name:
String
:param filename:
The filname
:type filename:
String
:param path:
The path to the file that needs to be uploaded
:type path:
String
:returns:
void
"""
f = open(path, 'rb')
self.source_size = os.stat(path).st_size
@@ -66,15 +104,16 @@ class Uploader(VerbosityMixin):
def data_collector(iterable, def_buf_size=5242880):
''' Buffers n bytes of data
""" Buffers n bytes of data.
Args:
iterable: could be a list, generator or string
def_buf_size: number of bytes to buffer, default is 5mb
:param iterable:
Could be a list, generator or string
:type iterable:
List, generator, String
Returns:
A generator object
'''
:returns:
A generator object
"""
buf = ''
for data in iterable:
buf += data
@@ -108,23 +147,54 @@ def upload(bucket, aws_access_key, aws_secret_key,
iterable, key, progress_cb=None,
threads=5, replace=False, secure=True,
connection=None):
''' Upload data to s3 using the s3 multipart upload API.
""" Upload data to s3 using the s3 multipart upload API.
Args:
bucket: name of s3 bucket
aws_access_key: aws access key
aws_secret_key: aws secret key
iterable: The data to upload. Each 'part' in the list
will be uploaded in parallel. Each part must be at
least 5242880 bytes (5mb).
key: the name of the key to create in the s3 bucket
progress_cb: will be called with (part_no, uploaded, total)
each time a progress update is available.
threads: the number of threads to use while uploading. (Default is 5)
replace: will replace the key in s3 if set to true. (Default is false)
secure: use ssl when talking to s3. (Default is true)
connection: used for testing
'''
:param bucket:
Name of the S3 bucket
:type bucket:
String
:param aws_access_key:
AWS access key id (optional)
:type aws_access_key:
String
:param aws_secret_key:
AWS access secret key (optional)
:type aws_secret_key:
String
:param iterable:
The data to upload. Each 'part' in the list. will be uploaded in parallel. Each part must be at
least 5242880 bytes (5mb).
:type iterable:
An iterable object
:param key:
The name of the key (filename) to create in the s3 bucket
:type key:
String
:param progress_cb:
Progress callback, will be called with (part_no, uploaded, total) each time a progress update
is available. (optional)
:type progress_cb:
function
:param threads:
the number of threads to use while uploading. (Default is 5)
:type threads:
int
:param replace:
will replace the key (filename) on S3 if set to true. (Default is false)
:type replace:
boolean
:param secure:
Use ssl when talking to s3. (Default is true)
:type secure:
boolean
:param connection:
Used for testing (optional)
:type connection:
S3 connection class
:returns:
void
"""
if not connection:
from boto.s3.connection import S3Connection as connection

165
landsat/utils.py
View File

@@ -12,7 +12,13 @@ from mixins import VerbosityMixin
class Capturing(list):
""" Captures a subprocess stdout """
"""
Captures a subprocess stdout.
:Usage:
>>> with Capturing():
... subprocess(args)
"""
def __enter__(self):
self._stdout = sys.stdout
sys.stdout = self._stringio = StringIO()
@@ -25,11 +31,11 @@ class Capturing(list):
class timer(object):
"""
A time class
A timer class.
Usage:
with timer():
your code
:Usage:
>>> with timer():
... your code
"""
def __enter__(self):
self.start = time.time()
@@ -40,6 +46,21 @@ class timer(object):
def exit(message, code=0):
""" output a message to stdout and terminates the process.
:param message:
Message to be outputed.
:type message:
String
:param code:
The termination code. Default is 0
:type code:
int
:returns:
void
"""
v = VerbosityMixin()
if code == 0:
v.output(message, normal=True, arrow=True)
@@ -50,13 +71,20 @@ def exit(message, code=0):
def create_paired_list(value):
""" Create a list of paired items from a string
""" Create a list of paired items from a string.
Arguments:
i - the format must be 003,003,004,004 (commas with no space)
:param value:
the format must be 003,003,004,004 (commas with no space)
:type value:
String
Returns:
:returns:
List
:example:
>>> create_paired_list('003,003,004,004')
[['003','003'], ['004', '004']]
"""
if isinstance(value, list):
@@ -75,8 +103,15 @@ def create_paired_list(value):
def check_create_folder(folder_path):
""" Check whether a folder exists, if not the folder is created
Always return folder_path
""" Check whether a folder exists, if not the folder is created.
:param folder_path:
Path to the folder
:type folder_path:
String
:returns:
(String) the path to the folder
"""
if not os.path.exists(folder_path):
os.makedirs(folder_path)
@@ -85,20 +120,55 @@ def check_create_folder(folder_path):
def get_file(path):
""" Separate the name of the file or folder from the path and return it
Example: /path/to/file ---> file
""" Separate the name of the file or folder from the path and return it.
:param path:
Path to the folder
:type path:
String
:returns:
(String) the filename
:example:
>>> get_file('/path/to/file.jpg')
'file.jpg'
"""
return os.path.basename(path)
def get_filename(path):
""" Return the filename without extension. e.g. index.html --> index """
""" Return the filename without extension.
:param path:
Path to the folder
:type path:
String
:returns:
(String) the filename without extension
:example:
>>> get_filename('/path/to/file.jpg')
'file'
"""
return os.path.splitext(get_file(path))[0]
def three_digit(number):
""" Add 0s to inputs that their length is less than 3.
For example: 1 --> 001 | 02 --> 020 | st --> 0st
:param number:
The number to convert
:type number:
int
:returns:
String
:example:
>>> three_digit(1)
'001'
"""
number = str(number)
if len(number) == 1:
@@ -110,8 +180,19 @@ def three_digit(number):
def georgian_day(date):
""" Returns the number of days passed since the start of the year
Accepted format: %m/%d/%Y
""" Returns the number of days passed since the start of the year.
:param date:
The string date with this format %m/%d/%Y
:type date:
String
:returns:
int
:example:
>>> georgian_day('05/1/2015')
121
"""
try:
fmt = '%m/%d/%Y'
@@ -121,8 +202,19 @@ def georgian_day(date):
def year(date):
""" Returns the year
Accepted format: %m/%d/%Y
""" Returns the year.
:param date:
The string date with this format %m/%d/%Y
:type date:
String
:returns:
int
:example:
>>> year('05/1/2015')
2015
"""
try:
fmt = '%m/%d/%Y'
@@ -132,8 +224,23 @@ def year(date):
def reformat_date(date, new_fmt='%Y-%m-%d'):
""" Return reformated date. Example: 01/28/2014 & %d/%m/%Y -> 28/01/2014
Accepted date format: %m/%d/%Y
""" Returns reformated date.
:param date:
The string date with this format %m/%d/%Y
:type date:
String
:param new_fmt:
date format string. Default is '%Y-%m-%d'
:type date:
String
:returns:
int
:example:
>>> reformat_date('05/1/2015', '%d/%m/%Y')
'1/05/2015'
"""
try:
if isinstance(date, datetime):
@@ -146,7 +253,21 @@ def reformat_date(date, new_fmt='%Y-%m-%d'):
def convert_to_integer_list(value):
""" convert a comma separate string to a list where all values are integers """
""" Converts a comma separate string to a list
:param value:
the format must be 003,003,004,004 (commas with no space)
:type value:
String
:returns:
List
:example:
>>> convert_to_integer_list('003,003,004,004')
['003', '003', '004', '004']
"""
if value and isinstance(value, str):
if ',' in value: