Implement an abuse handler

If too many requests are made within X amount of time (not the traditional hourly rate limit), github may begin returning 403.  Then we should wait for a bit to attempt to access the API again.  In this case, we parse out the Retry-After field returned and sleep until that (it's usually 60 seconds)
This commit is contained in:
Matt Mitchell
2016-07-21 13:53:23 -07:00
parent a2f0837d14
commit 9f5a6ee549
4 changed files with 34 additions and 2 deletions

View File

@@ -83,6 +83,7 @@ public class GitHub {
private final String apiUrl;
/*package*/ final RateLimitHandler rateLimitHandler;
/*package*/ final RateLimitHandler abuseLimitHandler;
private HttpConnector connector = HttpConnector.DEFAULT;
@@ -122,7 +123,7 @@ public class GitHub {
* @param connector
* HttpConnector to use. Pass null to use default connector.
*/
/* package */ GitHub(String apiUrl, String login, String oauthAccessToken, String password, HttpConnector connector, RateLimitHandler rateLimitHandler) throws IOException {
/* package */ GitHub(String apiUrl, String login, String oauthAccessToken, String password, HttpConnector connector, RateLimitHandler rateLimitHandler, RateLimitHandler abuseLimitHandler) throws IOException {
if (apiUrl.endsWith("/")) apiUrl = apiUrl.substring(0, apiUrl.length()-1); // normalize
this.apiUrl = apiUrl;
if (null != connector) this.connector = connector;
@@ -140,6 +141,7 @@ public class GitHub {
}
this.rateLimitHandler = rateLimitHandler;
this.abuseLimitHandler = abuseLimitHandler;
if (login==null && encodedAuthorization!=null)
login = getMyself().getLogin();

View File

@@ -30,6 +30,7 @@ public class GitHubBuilder {
private HttpConnector connector;
private RateLimitHandler rateLimitHandler = RateLimitHandler.WAIT;
private RateLimitHandler abuseLimitHandler = RateLimitHandler.ABUSE;
public GitHubBuilder() {
}
@@ -193,6 +194,6 @@ public class GitHubBuilder {
}
public GitHub build() throws IOException {
return new GitHub(endpoint, user, oauthToken, password, connector, rateLimitHandler);
return new GitHub(endpoint, user, oauthToken, password, connector, rateLimitHandler, abuseLimitHandler);
}
}

View File

@@ -48,6 +48,27 @@ public abstract class RateLimitHandler {
return Math.max(10000, Long.parseLong(v)*1000 - System.currentTimeMillis());
}
};
/**
* Wait until the API abuse "wait time" is passed.
*/
public static final RateLimitHandler ABUSE = new RateLimitHandler() {
@Override
public void onError(IOException e, HttpURLConnection uc) throws IOException {
try {
Thread.sleep(parseWaitTime(uc));
} catch (InterruptedException _) {
throw (InterruptedIOException)new InterruptedIOException().initCause(e);
}
}
private long parseWaitTime(HttpURLConnection uc) {
String v = uc.getHeaderField("Retry-After");
if (v==null) return 60 * 1000; // can't tell, return 1 min
return Math.max(1000, Long.parseLong(v)*1000);
}
};
/**
* Fail immediately.

View File

@@ -572,6 +572,14 @@ class Requester {
root.rateLimitHandler.onError(e,uc);
return;
}
// Check to see whether we hit a 403, and the Retry-After field is
// available.
if (responseCode == HttpURLConnection.HTTP_FORBIDDEN &&
uc.getHeaderField("Retry-After") != null) {
this.root.abuseLimitHandler.onError(e,uc);
return;
}
InputStream es = wrapStream(uc.getErrorStream());
try {