jlengrand/landsat-util
1
0
Fork 0
mirror of https://github.com/Skyai-io/landsat-util.git synced 2026-03-10 09:01:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added setup.py and reorganized forlder structure

Browse Source
This commit is contained in:
Scisco
2014-08-07 12:28:04 -04:00
parent 994e1d2730
commit cb2b0a951a
12 changed files with 43 additions and 821 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
MANIFEST.in Normal file
View File

@@ -0,0 +1,4 @@
include README.md
include .gitignore
include LICENSE
recursive-include landsat/assests *

24
api/Gruntfile.js
View File

@@ -1,24 +0,0 @@
// USGS Landsat Imagery Metadata RES API Gruntfile
//
// Forked from https://github.com/FDA/openfda/tree/master/api
// Exposes /landsat/metadata.json and /healthcheck GET endpoints
//
// Author: developmentseed
// Contributer: scisco
//
// License: CC0 1.0 Universal
module.exports = function(grunt) {
var path = require('path');
grunt.loadNpmTasks('grunt-contrib-nodeunit');
grunt.initConfig({
nodeunit: {
all: ['*_test.js'],
options: {
reporter: 'verbose'
}
}
});
};

1
api/Procfile
View File

@@ -1 +0,0 @@
web: node api.js

65
api/README.md
View File

@@ -1,65 +0,0 @@
## Landsat-util metadata API
*This is a fork of the excellent OpenFDA API: https://github.com/FDA/openfda/tree/master/api*
## Getting started
### Installing Elastic Search on Ubuntu:
First you need Oracle Java
$: sudo add-apt-repository ppa:webupd8team/java
$: sudo apt-get update
$: sudo apt-get install oracle-java7-installer -y
Then you need to get the latest version of elastic search debian package from [ES website](http://www.elasticsearch.org/download/)
$: wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.3.1.deb
$: sudo dpkg -i elasticsearch-1.3.1.deb
$: sudo update-rc.d elasticsearch defaults 95 10
$: sudo /etc/init.d/elasticsearch start
ES will be accessible on `http://localhost:9200`
#### Installing Elastic Search on Mac
Donwload the lastest version of elastic search zip from [ES website](http://www.elasticsearch.org/download/)
Unzip and run `bin/elasticsearch`
ES will be accessible on `http://localhost:9200`
## Running the API
Install NodeJS, npm and forever (This is ONLY needed if you want to run the API engine)
$: sudo apt-get install nodejs ruby ruby1.9.1-dev npm -y
$: sudo ln -s /usr/bin/nodejs /usr/bin/node
$: npm install
If you want to update the metadata data on Elastic Search, make sure you have followed steps [described here](https://github.com/developmentseed/landsat-util) and then run the below command from the parent folder:
$: ./landsat_util.py --update-metadata
To run the api:
$: node api.js
To test the API run:
$: curl localhost:8000/landsat?search=LC81660362014196LGN00
To run the API in the background run:
$: forever start api.js
To list forever jobs:
$: forever list
info: Forever processes running
data: uid command script forever pid logfile uptime
data: [0] v0MM /usr/bin/nodejs api.js 19708 19710 /home/ubuntu/.forever/v0MM.log 0:0:5:46.387
To Kill the forever job:
$: forever stop 0

252
api/api.js
View File

@@ -1,252 +0,0 @@
// USGS Landsat Imagery Metadata RES API
//
// Forked from https://github.com/FDA/openfda/tree/master/api
// Exposes /landsat/metadata.json and /healthcheck GET endpoints
//
// Author: developmentseed
// Contributer: scisco
//
// License: CC0 1.0 Universal
var ejs = require('elastic.js');
var elasticsearch = require('elasticsearch');
var express = require('express');
var moment = require('moment');
var underscore = require('underscore');
var api_request = require('./api_request.js');
var elasticsearch_query = require('./elasticsearch_query.js');
var logging = require('./logging.js');
var META = {
'credit': 'This API is based on the openFDA\'s API ' +
'https://github.com/FDA/openfda/tree/master/api ',
'author': 'Development Seed',
'contributor': 'Scisco',
'license': 'http://creativecommons.org/publicdomain/zero/1.0/legalcode',
'last_updated': '2014-08-01'
};
var HTTP_CODE = {
OK: 200,
BAD_REQUEST: 400,
NOT_FOUND: 404,
SERVER_ERROR: 500
};
// Internal fields to remove from ES objects before serving
// via the API.
var FIELDS_TO_REMOVE = [
];
var MAIN_INDEX = 'landsat';
var app = express();
app.disable('x-powered-by');
// Set caching headers for Amazon Cloudfront
CacheMiddleware = function(seconds) {
return function(request, response, next) {
response.setHeader('Cache-Control', 'public, max-age=' + seconds);
return next();
};
};
app.use(CacheMiddleware(60));
// Use gzip compression
app.use(express.compress());
// Setup defaults for API JSON error responses
app.set('json spaces', 2);
app.set('json replacer', undefined);
var log = logging.GetLogger();
var client = new elasticsearch.Client({
host: process.env.ES_HOST || 'localhost:9200',
log: logging.ElasticsearchLogger,
// Note that this doesn't abort the query.
requestTimeout: 10000 // milliseconds
});
app.get('/healthcheck', function(request, response) {
client.cluster.health({
index: MAIN_INDEX,
timeout: 1000 * 60,
waitForStatus: 'yellow'
}, function(error, health_response, status) {
health_json = JSON.stringify(health_response, undefined, 2);
if (error != undefined) {
response.send(500, 'NAK.\n' + error + '\n');
} else if (health_response['status'] == 'red') {
response.send(500, 'NAK.\nStatus: ' + health_json + '\n');
} else {
response.send('OK\n\n' + health_json + '\n');
}
});
});
ApiError = function(response, code, message) {
error_response = {};
error_response.error = {};
error_response.error.code = code;
error_response.error.message = message;
response.json(HTTP_CODE[code], error_response);
};
LogRequest = function(request) {
log.info(request.headers, 'Request Headers');
log.info(request.query, 'Request Query');
};
SetHeaders = function(response) {
response.header('Server', 'api.developmentseed.org');
// http://john.sh/blog/2011/6/30/cross-domain-ajax-expressjs-
// and-access-control-allow-origin.html
response.header('Access-Control-Allow-Origin', '*');
response.header('Access-Control-Allow-Headers', 'X-Requested-With');
response.header('Content-Security-Policy', "default-src 'none'");
// https://www.owasp.org/index.php/REST_Security_Cheat_Sheet
// #Send_security_headers
response.header('X-Content-Type-Options', 'nosniff');
response.header('X-Frame-Options', 'deny');
response.header('X-XSS-Protection', '1; mode=block');
};
TryToCheckApiParams = function(request, response) {
try {
return api_request.CheckParams(request.query);
} catch (e) {
log.error(e);
if (e.name == api_request.API_REQUEST_ERROR) {
ApiError(response, 'BAD_REQUEST', e.message);
} else {
ApiError(response, 'BAD_REQUEST', '');
}
return null;
}
};
TryToBuildElasticsearchParams = function(params, elasticsearch_index, response) {
try {
var es_query = elasticsearch_query.BuildQuery(params);
log.info(es_query.toString(), 'Elasticsearch Query');
} catch (e) {
log.error(e);
if (e.name == elasticsearch_query.ELASTICSEARCH_QUERY_ERROR) {
ApiError(response, 'BAD_REQUEST', e.message);
} else {
ApiError(response, 'BAD_REQUEST', '');
}
return null;
}
var es_search_params = {
index: elasticsearch_index,
body: es_query.toString()
};
if (!params.count) {
es_search_params.from = params.skip;
es_search_params.size = params.limit;
}
return es_search_params;
};
TrySearch = function(index, params, es_search_params, response) {
client.search(es_search_params).then(function(body) {
if (body.hits.hits.length == 0) {
ApiError(response, 'NOT_FOUND', 'No matches found!');
}
var response_json = {};
response_json.meta = underscore.clone(META);
if (!params.count) {
response_json.meta.results = {
'skip': params.skip,
'limit': params.limit,
'total': body.hits.total
};
response_json.results = [];
for (i = 0; i < body.hits.hits.length; i++) {
var es_results = body.hits.hits[i]._source;
for (j = 0; j < FIELDS_TO_REMOVE.length; j++) {
delete es_results[FIELDS_TO_REMOVE[j]];
}
response_json.results.push(es_results);
}
response.json(HTTP_CODE.OK, response_json);
} else if (params.count) {
if (body.facets.count.terms) {
// Term facet count
if (body.facets.count.terms.length != 0) {
response_json.results = body.facets.count.terms;
response.json(HTTP_CODE.OK, response_json);
} else {
ApiError(response, 'NOT_FOUND', 'Nothing to count');
}
} else if (body.facets.count.entries) {
// Date facet count
if (body.facets.count.entries.length != 0) {
for (i = 0; i < body.facets.count.entries.length; i++) {
var day = moment(body.facets.count.entries[i].time);
body.facets.count.entries[i].time = day.format('YYYYMMDD');
}
response_json.results = body.facets.count.entries;
response.json(HTTP_CODE.OK, response_json);
} else {
ApiError(response, 'NOT_FOUND', 'Nothing to count');
}
} else {
ApiError(response, 'NOT_FOUND', 'Nothing to count');
}
} else {
ApiError(response, 'NOT_FOUND', 'No matches found!');
}
}, function(error) {
log.error(error);
ApiError(response, 'SERVER_ERROR', 'Check your request and try again');
});
};
Endpoint = function(noun) {
app.get('/' + noun, function(request, response) {
LogRequest(request);
SetHeaders(response);
var params = TryToCheckApiParams(request, response);
if (params == null) {
return;
}
var index = noun;
var es_search_params =
TryToBuildElasticsearchParams(params, index, response);
if (es_search_params == null) {
return;
}
TrySearch(index, params, es_search_params, response);
});
};
Endpoint('landsat');
// From http://strongloop.com/strongblog/
// robust-node-applications-error-handling/
if (process.env.NODE_ENV === 'production') {
process.on('uncaughtException', function(e) {
log.error(e);
process.exit(1);
});
}
var port = process.env.PORT || 8000;
app.listen(port, function() {
console.log('Listening on ' + port);
});

91
api/api_request.js
View File

@@ -1,91 +0,0 @@
// USGS Landsat Imagery Metadata RES API Request Helpers
//
// Forked from https://github.com/FDA/openfda/tree/master/api
// Exposes /landsat/metadata.json and /healthcheck GET endpoints
//
// Author: developmentseed
// Contributer: scisco
//
// License: CC0 1.0 Universal
var underscore = require('underscore');
var EXPECTED_PARAMS = ['search', 'count', 'limit', 'skip'];
exports.API_REQUEST_ERROR = 'ApiRequestError';
var API_REQUEST_ERROR = exports.API_REQUEST_ERROR;
exports.CheckParams = function(params) {
// Ensure we only have params that are expected.
underscore.each(underscore.keys(params), function(param) {
if (EXPECTED_PARAMS.indexOf(param) == -1) {
throw {
name: API_REQUEST_ERROR,
message: 'Invalid parameter: ' + param
};
}
});
if (params.limit) {
var limit = parseInt(params.limit);
if (isNaN(limit)) {
throw {
name: API_REQUEST_ERROR,
message: 'Invalid limit parameter value.'
};
}
params.limit = limit;
}
if (params.skip) {
var skip = parseInt(params.skip);
if (isNaN(skip)) {
throw {
name: API_REQUEST_ERROR,
message: 'Invalid skip parameter value.'
};
}
params.skip = skip;
}
// Limit to 100 results per search request.
if (!params.count && params.limit && params.limit > 100) {
throw {
name: API_REQUEST_ERROR,
message: 'Limit cannot exceed 100 results for search requests. Use ' +
'the skip param to get additional results.'
};
}
// Limit to 1000 results per count request.
if (params.count && params.limit && params.limit > 1000) {
throw {
name: API_REQUEST_ERROR,
message: 'Limit cannot exceed 1000 results for count requests.'
};
}
// Do not allow ski param with count requests.
if (params.count && params.skip) {
throw {
name: API_REQUEST_ERROR,
message: 'Should not use skip param when using count.'
};
}
// Set default values for missing params
params.skip = params.skip || 0;
if (!params.limit) {
if (params.count) {
params.limit = 100;
} else {
params.limit = 1;
}
}
var clean_params = {};
underscore.extend(clean_params,
underscore.pick(params, EXPECTED_PARAMS));
return clean_params;
};

90
api/api_request_test.js
View File

@@ -1,90 +0,0 @@
// USGS Landsat Imagery Metadata RES API Request Test
//
// Forked from https://github.com/FDA/openfda/tree/master/api
// Exposes /landsat/metadata.json and /healthcheck GET endpoints
//
// Author: developmentseed
// Contributer: scisco
//
// License: CC0 1.0 Universal
var querystring = require('querystring');
var api_request = require('./api_request.js');
apiRequestError = function(test, params) {
test.throws(function() { api_request.CheckParams(params) },
api_request.API_REQUEST_ERROR,
'Should be an API error: ' + JSON.stringify(params));
};
exports.testInvalidParam = function(test) {
var request = 'search=foo&notvalid=true&skip=10';
var params = querystring.parse(request);
apiRequestError(test, params);
test.done();
};
exports.testTooBigSearchLimit = function(test) {
var request = 'search=foo&limit=101';
var params = querystring.parse(request);
apiRequestError(test, params);
test.done();
};
exports.testTooBigCountLimit = function(test) {
var request = 'search=foo&count=foo&limit=1001';
var params = querystring.parse(request);
apiRequestError(test, params);
test.done();
};
exports.testCountRequestWithSkip = function(test) {
// with skip
var request = 'search=foo&count=bar&skip=10';
var params = querystring.parse(request);
apiRequestError(test, params);
test.done();
};
apiRequestValid = function(test, params) {
test.doesNotThrow(function() { api_request.CheckParams(params) },
api_request.API_REQUEST_ERROR,
'Should be valid: ' + JSON.stringify(params));
};
exports.testMaxLimit = function(test) {
var request = 'search=foo&limit=100';
var params = querystring.parse(request);
apiRequestValid(test, params);
test.done();
};
exports.testCountWithNoSearchParam = function(test) {
var request = 'count=bar';
var params = querystring.parse(request);
apiRequestValid(test, params);
test.done();
};
exports.testCountWithDot = function(test) {
var request = 'count=primarysource.qualification';
var params = querystring.parse(request);
apiRequestValid(test, params);
test.done();
};
exports.testCountMaxLimit = function(test) {
var request = 'search=foo&count=bar&limit=1000';
var params = querystring.parse(request);
apiRequestValid(test, params);
test.done();
};

76
api/elasticsearch_query.js
View File

@@ -1,76 +0,0 @@
// Elasticsearch Query Builder
var ejs = require('elastic.js');
var ELASTICSEARCH_QUERY_ERROR = 'ElasticsearchQueryError';
// Supported characters:
// all letters and numbers
// . for long.field.names
// _ for other_fields
// : for fields
// ( ) for grouping
// " for quoting
// [ ] and { } for ranges
// >, < and = for ranges
// - for dates and boolean
// + for boolean
// space for terms
var SUPPORTED_QUERY_RE = '^[0-9a-zA-Z\.\_\:\(\)\"\\[\\]\{\}\\-\\+\>\<\= ]+$';
var DATE_FIELDS = [
// FAERS
'drugstartdate',
'drugenddate',
'patient.patientdeath.patientdeathdate',
'receiptdate',
'receivedate',
'transmissiondate',
// RES
'report_date',
'recall_initiation_date'
];
exports.SupportedQueryString = function(query) {
var supported_query_re = new RegExp(SUPPORTED_QUERY_RE);
return supported_query_re.test(query);
};
// For the openfda section, we have field_exact rather than field.exact stored
// in elasticsearch.
exports.ReplaceExact = function(search_or_count) {
return search_or_count.replace(/openfda\.([\w\.]+).exact/g,
'openfda.$1_exact');
};
exports.BuildQuery = function(params) {
q = ejs.Request();
if (!params.search && !params.count) {
q.query(ejs.MatchAllQuery());
}
if (params.search) {
if (!exports.SupportedQueryString(params.search)) {
throw {
name: ELASTICSEARCH_QUERY_ERROR,
message: 'Search not supported: ' + params.search
};
}
q.query(ejs.QueryStringQuery(exports.ReplaceExact(params.search)));
}
if (params.count) {
if (DATE_FIELDS.indexOf(params.count) != -1) {
q.facet(ejs.DateHistogramFacet('count').
field(params.count).interval('day').order('time'));
} else {
var limit = parseInt(params.limit);
q.facet(ejs.TermsFacet('count').
fields([exports.ReplaceExact(params.count)]).size(limit));
}
}
return q;
};

146
api/elasticsearch_query_test.js
View File

@@ -1,146 +0,0 @@
// Elasticsearch Query Builder Test
var elasticsearch_query = require('./elasticsearch_query.js');
notSupported = function(test, query) {
test.ok(!elasticsearch_query.SupportedQueryString(query),
"Shouldn't be supported: " + query);
};
// All the different query string queries types from
// http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/
// query-dsl-query-string-query.html#query-string-syntax
// that we don't want to support at this time for performance reasons.
exports.testSupportedQueryString_NotSupported = function(test) {
// wildcard field
notSupported(test, 'city.\*:something');
notSupported(test, 'book.\*:(quick brown)');
// wildcard
notSupported(test, 'qu?ck bro*');
// leading wildcard
notSupported(test, '*ing');
// regular expression
notSupported(test, 'name:/joh?n(ath[oa]n)/');
// fuzziness
notSupported(test, 'quikc~ brwn~ foks~');
// proximity
notSupported(test, '"fox quick"~5');
// -------------------------------------------------------------
// We support most range queries except for those with wildcards.
// These can be rewritten using a supported alternative syntax.
// alternative: count:>=10
notSupported(test, 'count:[10 TO *]');
// alternative: date:<2012-01-01
notSupported(test, 'date:{* TO 2012/01/01}');
// alternative date:[2012-01-01 TO 2012-12-31]
notSupported(test, 'date:[2012/01/01 TO 2012/12/31]');
// -------------------------------------------------------------
// boosting
notSupported(test, 'quick^2 fox');
notSupported(test, '"john smith"^2 (foo bar)^4');
test.done();
};
supported = function(test, query) {
test.ok(elasticsearch_query.SupportedQueryString(query),
'Should be supported: ' + query);
};
// All the query string query types we want to support
exports.testSupportedQueryString_Supported = function(test) {
supported(test, 'active');
supported(test, 'status:active');
supported(test, 'msg.status:active');
supported(test, 'msg_status:active');
supported(test, 'title:(quick brown)');
supported(test, 'author:"John Smith"');
// ranges
supported(test, 'date:[2012-01-01 TO 2012-12-31]');
supported(test, 'count:[1 TO 5]');
supported(test, 'tag:{alpha TO omega}');
supported(test, 'count:>=10');
supported(test, 'date:<2012-01-01');
// boolean operators
supported(test, 'quick brown +fox -news');
// missing check
supported(test, '_missing_:title');
// exist check
supported(test, '_exists_:title');
test.done();
};
exports.testReplaceExact = function(test) {
// patient.drug.openfda section, exact but no value
test.ok(elasticsearch_query.ReplaceExact(
'patient.drug.openfda.product_ndc.exact') ==
'patient.drug.openfda.product_ndc_exact',
'patient.drug.openfda.product_ndc.exact');
// openfda section, exact with value
test.ok(elasticsearch_query.ReplaceExact(
'patient.drug.openfda.product_ndc.exact:10') ==
'patient.drug.openfda.product_ndc_exact:10',
'patient.drug.openfda.product_ndc.exact:10');
// multiple patient.drug.openfda exacts with values
test.ok(elasticsearch_query.ReplaceExact(
'patient.drug.openfda.product_ndc.exact:10 AND ' +
'patient.drug.openfda.spl_id.exact:a') ==
'patient.drug.openfda.product_ndc_exact:10 AND ' +
'patient.drug.openfda.spl_id_exact:a',
'patient.drug.openfda.product_ndc.exact:10 AND ' +
'patient.drug.openfda.spl_id.exact:a');
// patient.drug.openfda section, exact with space then value
test.ok(elasticsearch_query.ReplaceExact(
'patient.drug.openfda.product_ndc.exact: 10') ==
'patient.drug.openfda.product_ndc_exact: 10',
'patient.drug.openfda.product_ndc.exact: 10');
// No exact but in patient.drug.openfda section
test.ok(elasticsearch_query.ReplaceExact(
'patient.drug.openfda.unii:"nonsteroidal+anti-inflammatory+drug"') ==
'patient.drug.openfda.unii:"nonsteroidal+anti-inflammatory+drug"',
'patient.drug.openfda.unii:"nonsteroidal+anti-inflammatory+drug"');
// No section, no exact
test.ok(elasticsearch_query.ReplaceExact(
'receivedate:[2004-01-01+TO+2008-12-31]') ==
'receivedate:[2004-01-01+TO+2008-12-31]',
'receivedate:[2004-01-01+TO+2008-12-31]');
// Patient section, exact
test.ok(elasticsearch_query.ReplaceExact(
'patient.reaction.reactionmeddrapt.exact') ==
'patient.reaction.reactionmeddrapt.exact',
'patient.reaction.reactionmeddrapt.exact');
test.done();
};

43
api/logging.js
View File

@@ -1,43 +0,0 @@
// USGS Landsat Imagery Metadata RES API Logging
//
// Forked from https://github.com/FDA/openfda/tree/master/api
// Exposes /landsat/metadata.json and /healthcheck GET endpoints
//
// Author: developmentseed
// Contributer: scisco
//
// License: CC0 1.0 Universal
// Based on: http://www.elasticsearch.org/guide/
// en/elasticsearch/client/javascript-api/current/logging.html
var bunyan = require('bunyan');
exports.GetLogger = function() {
return bunyan.createLogger({
name: 'openfda-api-logger',
stream: process.stderr,
level: 'info'
});
};
exports.ElasticsearchLogger = function(config) {
// config is the object passed to the client constructor.
var logger = exports.GetLogger();
this.error = logger.error.bind(logger);
this.warning = logger.warn.bind(logger);
this.info = logger.info.bind(logger);
this.debug = logger.debug.bind(logger);
this.trace = function(method, requestUrl, body,
responseBody, responseStatus) {
logger.trace({
method: method,
requestUrl: requestUrl,
body: body,
responseBody: responseBody,
responseStatus: responseStatus
});
};
this.close = function() { /* bunyan's loggers do not need to be closed */ };
};

33
api/package.json
View File

@@ -1,33 +0,0 @@
{
"name": "landsat-api",
"version": "0.0.1",
"description": "An API for USGS Landsat8 imagery metadata.",
"author": "developmentseed",
"private": true,
"repository": {
"type": "git",
"url": "https://github.com/developmentseed/landsat-util"
},
"dependencies": {
"bunyan": "~0.22.3",
"elasticsearch": "~2.1.4",
"elastic.js": "~1.1.1",
"express": "~3.5.2",
"forever" : "~0.11.1",
"moment" : "~2.6.0",
"underscore": "~1.6.0"
},
"devDependencies": {
"grunt": "~0.4.4",
"grunt-contrib-nodeunit": "~0.3.3"
},
"scripts": {
"start": "node api.js",
"test": "grunt nodeunit:all"
},
"engines": {
"node": "0.10.x"
},
"license": "CC0",
"subdomain": "api"
}

39
setup.py Normal file
View File

@@ -0,0 +1,39 @@
#!/usr/bin/env python
# USGS Landsat Imagery Util
#
#
# Author: developmentseed
# Contributer: scisco
#
# License: CC0 1.0 Universal
try:
from setuptools import setup
except ImportError:
from distutils.core import setup
def readme():
with open("README.md") as f:
return f.read()
setup(name = "landsat",
version = '0.1.0',
description = "A utility to search, download and process Landsat 8" +
" satellite imagery",
long_description = readme(),
author = "Scisco",
author_email = "alireza@developmentseed.org",
scripts = ["bin/landsat"],
url = "https://github.com/developmentseed/landsat-util",
packages = ["landsat"],
license = "CCO",
platforms = "Posix; MacOS X; Windows",
install_requires=[
"GDAL==1.11.0",
"elasticsearch==1.1.1",
"gsutil==4.4",
],
)