[maven-release-plugin] prepare release github-api-1.22

pom.xml

@@ -7,7 +7,7 @@
   </parent>
 
   <artifactId>github-api</artifactId>
-  <version>1.20</version>
+  <version>1.22</version>
   <name>GitHub API for Java</name>
   <url>http://github-api.kohsuke.org/</url>
   <description>GitHub API for Java</description>

src/main/java/org/kohsuke/github/GHPerson.java

@@ -2,10 +2,15 @@ package org.kohsuke.github;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 
+import static org.kohsuke.github.ApiVersion.*;
+
 /**
  * Common part of {@link GHUser} and {@link GHOrganization}.
  * 
@@ -31,18 +36,48 @@ public abstract class GHPerson {
      */
     public synchronized Map<String,GHRepository> getRepositories() throws IOException {
         Map<String,GHRepository> repositories = new TreeMap<String, GHRepository>();
-        for (int i=1; ; i++) {
+        for (List<GHRepository> batch : iterateRepositories(100)) {
-            GHRepository[] array = root.retrieve3("/users/" + login + "/repos?per_page=100&page=" + i, GHRepository[].class);
+            for (GHRepository r : batch)
-            for (GHRepository r : array) {
-                r.root = root;
                 repositories.put(r.getName(),r);
-            }
-            if (array.length==0)  break;
         }
-
         return Collections.unmodifiableMap(repositories);
     }
 
+    /**
+     * Loads repository list in a pagenated fashion.
+     * 
+     * <p>
+     * For a person with a lot of repositories, GitHub returns the list of repositories in a pagenated fashion.
+     * Unlike {@link #getRepositories()}, this method allows the caller to start processing data as it arrives.
+     * 
+     * Every {@link Iterator#next()} call results in I/O. Exceptions that occur during the processing is wrapped
+     * into {@link Error}.
+     */
+    public synchronized Iterable<List<GHRepository>> iterateRepositories(final int pageSize) {
+        return new Iterable<List<GHRepository>>() {
+            public Iterator<List<GHRepository>> iterator() {
+                final Iterator<GHRepository[]> pager = root.retrievePaged("/users/" + login + "/repos?per_page="+pageSize,GHRepository[].class,false, V3);
+
+                return new Iterator<List<GHRepository>>() {
+                    public boolean hasNext() {
+                        return pager.hasNext();
+                    }
+
+                    public List<GHRepository> next() {
+                        GHRepository[] batch = pager.next();
+                        for (GHRepository r : batch)
+                            r.root = root;
+                        return Arrays.asList(batch);
+                    }
+
+                    public void remove() {
+                        throw new UnsupportedOperationException();
+                    }
+                };
+            }
+        };
+    }
+
     /**
      *
      * @return

src/main/java/org/kohsuke/github/GHRepository.java

@@ -156,6 +156,11 @@ public class GHRepository {
         return open_issues;
     }
 
+    /**
+     *
+     * @return
+     *      null if the repository was never pushed at.
+     */
     public Date getPushedAt() {
         return GitHub.parseDate(pushed_at);
     }

src/main/java/org/kohsuke/github/GitHub.java

@@ -44,13 +44,17 @@ import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.NoSuchElementException;
 import java.util.Properties;
 import java.util.TimeZone;
+import java.util.zip.GZIPInputStream;
 
 import com.infradna.tool.bridge_method_injector.WithBridgeMethods;
 import org.apache.commons.io.IOUtils;
+import org.codehaus.jackson.JsonParseException;
 import org.codehaus.jackson.map.DeserializationConfig.Feature;
 import org.codehaus.jackson.map.ObjectMapper;
 import org.codehaus.jackson.map.introspect.VisibilityChecker.Std;
@@ -170,7 +174,7 @@ public class GitHub {
     }
 
     /*package*/ <T> T retrieveWithAuth(String tailApiUrl, Class<T> type) throws IOException {
-        return retrieveWithAuth(tailApiUrl,type,"GET");
+        return retrieveWithAuth(tailApiUrl, type, "GET");
     }
 
     /*package*/ <T> T retrieveWithAuth3(String tailApiUrl, Class<T> type) throws IOException {
@@ -187,32 +191,141 @@ public class GitHub {
 
     private <T> T _retrieve(String tailApiUrl, Class<T> type, String method, boolean withAuth, ApiVersion v) throws IOException {
         while (true) {// loop while API rate limit is hit
-        	
+            HttpURLConnection uc = setupConnection(method, withAuth, getApiURL(v, tailApiUrl));
-            HttpURLConnection uc = (HttpURLConnection) getApiURL(v,tailApiUrl).openConnection();
-
-            if (withAuth && this.oauthAccessToken == null)
-                uc.setRequestProperty("Authorization", "Basic " + encodedAuthorization);
-            
-            uc.setRequestMethod(method);
-            if (method.equals("PUT")) {
-                uc.setDoOutput(true);
-                uc.setRequestProperty("Content-Length","0");
-                uc.getOutputStream().close();
-            }
-
             try {
-                InputStreamReader r = new InputStreamReader(uc.getInputStream(), "UTF-8");
+                return parse(uc,type);
-                if (type==null) {
-                    String data = IOUtils.toString(r);
-                    return null;
-                }
-                return MAPPER.readValue(r,type);
             } catch (IOException e) {
                 handleApiError(e,uc);
             }
         }
     }
 
+    /**
+     * Loads pagenated resources.
+     *
+     * Every iterator call reports a new batch.
+     */
+    /*package*/ <T> Iterator<T> retrievePaged(final String tailApiUrl, final Class<T> type, final boolean withAuth, final ApiVersion v) {
+        return new Iterator<T>() {
+            /**
+             * The next batch to be returned from {@link #next()}.
+             */
+            T next;
+            /**
+             * URL of the next resource to be retrieved, or null if no more data is available.
+             */
+            URL url;
+
+            {
+                try {
+                    url = getApiURL(v, tailApiUrl);
+                } catch (IOException e) {
+                    throw new Error(e);
+                }
+            }
+
+            public boolean hasNext() {
+                fetch();
+                return next!=null;
+            }
+
+            public T next() {
+                fetch();
+                T r = next;
+                if (r==null)    throw new NoSuchElementException();
+                next = null;
+                return r;
+            }
+
+            public void remove() {
+                throw new UnsupportedOperationException();
+            }
+
+            private void fetch() {
+                if (next!=null) return; // already fetched
+                if (url==null)  return; // no more data to fetch
+
+                try {
+                    while (true) {// loop while API rate limit is hit
+                        HttpURLConnection uc = setupConnection("GET", withAuth, url);
+                        try {
+                            next = parse(uc,type);
+                            assert next!=null;
+                            findNextURL(uc);
+                            return;
+                        } catch (IOException e) {
+                            handleApiError(e,uc);
+                        }
+                    }
+                } catch (IOException e) {
+                    throw new Error(e);
+                }
+            }
+
+            /**
+             * Locate the next page from the pagination "Link" tag.
+             */
+            private void findNextURL(HttpURLConnection uc) throws MalformedURLException {
+                url = null; // start defensively
+                String link = uc.getHeaderField("Link");
+                if (link==null) return;
+
+                for (String token : link.split(", ")) {
+                    if (token.endsWith("rel=\"next\"")) {
+                        // found the next page. This should look something like
+                        // <https://api.github.com/repos?page=3&per_page=100>; rel="next"
+                        int idx = token.indexOf('>');
+                        url = new URL(token.substring(1,idx));
+                        return;
+                    }
+                }
+
+                // no more "next" link. we are done.
+            }
+        };
+    }
+
+    private HttpURLConnection setupConnection(String method, boolean withAuth, URL url) throws IOException {
+        HttpURLConnection uc = (HttpURLConnection) url.openConnection();
+
+        if (withAuth && this.oauthAccessToken == null)
+            uc.setRequestProperty("Authorization", "Basic " + encodedAuthorization);
+
+        uc.setRequestMethod(method);
+        if (method.equals("PUT")) {
+            uc.setDoOutput(true);
+            uc.setRequestProperty("Content-Length","0");
+            uc.getOutputStream().close();
+        }
+        uc.setRequestProperty("Accept-Encoding", "gzip");
+        return uc;
+    }
+
+    private <T> T parse(HttpURLConnection uc, Class<T> type) throws IOException {
+        InputStreamReader r = null;
+        try {
+            r = new InputStreamReader(wrapStream(uc, uc.getInputStream()), "UTF-8");
+            if (type==null) {
+                String data = IOUtils.toString(r);
+                return null;
+            }
+            return MAPPER.readValue(r,type);
+        } finally {
+            IOUtils.closeQuietly(r);
+        }
+    }
+
+    /**
+     * Handles the "Content-Encoding" header.
+     */
+    private InputStream wrapStream(HttpURLConnection uc, InputStream in) throws IOException {
+        String encoding = uc.getContentEncoding();
+        if (encoding==null) return in;
+        if (encoding.equals("gzip"))    return new GZIPInputStream(in);
+
+        throw new UnsupportedOperationException("Unexpected Content-Encoding: "+encoding);
+    }
+
     /**
      * If the error is because of the API limit, wait 10 sec and return normally.
      * Otherwise throw an exception reporting an error.
@@ -231,11 +344,15 @@ public class GitHub {
         if (e instanceof FileNotFoundException)
             throw e;    // pass through 404 Not Found to allow the caller to handle it intelligently
 
-        InputStream es = uc.getErrorStream();
+        InputStream es = wrapStream(uc, uc.getErrorStream());
-        if (es!=null)
+        try {
-            throw (IOException)new IOException(IOUtils.toString(es,"UTF-8")).initCause(e);
+            if (es!=null)
-        else
+                throw (IOException)new IOException(IOUtils.toString(es,"UTF-8")).initCause(e);
-            throw e;
+            else
+                throw e;
+        } finally {
+            IOUtils.closeQuietly(es);
+        }
     }
 
     /**
@@ -372,6 +489,7 @@ public class GitHub {
     }
 
     /*package*/ static Date parseDate(String timestamp) {
+        if (timestamp==null)    return null;
         for (String f : TIME_FORMATS) {
             try {
                 SimpleDateFormat df = new SimpleDateFormat(f);

src/test/java/org/kohsuke/AppTest.java

@@ -44,6 +44,11 @@ public class AppTest extends TestCase {
         System.out.println(me);
         GHUser u = hub.getUser("kohsuke2");
         System.out.println(u);
+        for (List<GHRepository> lst : me.iterateRepositories(100)) {
+            for (GHRepository r : lst) {
+                System.out.println(r.getPushedAt());
+            }
+        }
     }
 
     public void tryOrgFork() throws Exception {
@@ -192,6 +197,15 @@ public class AppTest extends TestCase {
         System.out.println(hooks);
     }
 
+    public void testOrgRepositories() throws IOException {
+        GitHub gitHub = GitHub.connect();
+        GHOrganization j = gitHub.getOrganization("jenkinsci");
+        long start = System.currentTimeMillis();
+        Map<String, GHRepository> repos = j.getRepositories();
+        long end = System.currentTimeMillis();
+        System.out.printf("%d repositories in %dms\n",repos.size(),end-start);
+    }
+    
     public void testOrganization() throws IOException {
         GitHub gitHub = GitHub.connect();
         GHOrganization j = gitHub.getOrganization("jenkinsci");