diff --git a/README.md b/README.md index f90b632..0b85bf8 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ from https://docs.docker.com/get-docker/. Then run the following command to star have not tested it on Windows yet): ``` -touch ~/.remarkable-pocket ~/.rmapi && mkdir -p ~/.rmapi-cache && docker run -it --env TZ=Europe/Amsterdam -p 65112:65112 -v ~/.remarkable-pocket:/root/.remarkable-pocket -v ~/.rmapi:/root/.rmapi -v ~/.rmapi-cache:/root/.cache/rmapi ghcr.io/nov1n/remarkable-pocket:0.2.2 +touch ~/.remarkable-pocket ~/.rmapi && mkdir -p ~/.rmapi-cache && docker run -it --env TZ=Europe/Amsterdam -p 65112:65112 -v ~/.remarkable-pocket:/root/.remarkable-pocket -v ~/.rmapi:/root/.rmapi -v ~/.rmapi-cache:/root/.cache/rmapi ghcr.io/nov1n/remarkable-pocket:0.2.3 ``` The first time you run the application, you will be asked to authorize Pocket and Remarkable Cloud. Once you have done diff --git a/build.gradle b/build.gradle index e79d1c4..6b48071 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ plugins { } group = "nl.carosi" -version = "0.2.2" +version = "0.2.3" java { toolchain { diff --git a/docker-compose.yml b/docker-compose.yml index cb2deaf..a2261ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: remarkable-pocket: - image: ghcr.io/nov1n/remarkable-pocket:0.2.2 + image: ghcr.io/nov1n/remarkable-pocket:0.2.3 restart: always ports: - 65112:65112 diff --git a/nl.carosi.remarkable-pocket.plist b/nl.carosi.remarkable-pocket.plist index 68f8559..2083ea6 100644 --- a/nl.carosi.remarkable-pocket.plist +++ b/nl.carosi.remarkable-pocket.plist @@ -8,7 +8,7 @@ /bin/sh -c - while ! /usr/local/bin/docker version > /dev/null 2>&1; do sleep 5; done && /usr/local/bin/docker run --env TZ=Europe/Amsterdam -v ~/.remarkable-pocket:/root/.remarkable-pocket -v ~/.rmapi:/root/.rmapi -v ~/.rmapi-cache:/root/.cache/rmapi ghcr.io/nov1n/remarkable-pocket:0.2.2 1>>$HOME/.remarkable-pocket.log 2>&1 + while ! /usr/local/bin/docker version > /dev/null 2>&1; do sleep 5; done && /usr/local/bin/docker run --env TZ=Europe/Amsterdam -v ~/.remarkable-pocket:/root/.remarkable-pocket -v ~/.rmapi:/root/.rmapi -v ~/.rmapi-cache:/root/.cache/rmapi ghcr.io/nov1n/remarkable-pocket:0.2.3 1>>$HOME/.remarkable-pocket.log 2>&1 RunAtLoad diff --git a/src/main/java/nl/carosi/remarkablepocket/ArticleValidator.java b/src/main/java/nl/carosi/remarkablepocket/ArticleValidator.java new file mode 100644 index 0000000..71d10bb --- /dev/null +++ b/src/main/java/nl/carosi/remarkablepocket/ArticleValidator.java @@ -0,0 +1,19 @@ +package nl.carosi.remarkablepocket; + +import java.util.HashSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class ArticleValidator { + private static final Logger LOG = LoggerFactory.getLogger(ArticleValidator.class); + private final HashSet invalidArticles = new HashSet<>(); + + void invalidate(String articleName) { + invalidArticles.add(articleName); + LOG.debug("Invalid articles: {}", invalidArticles); + } + + boolean isValid(String articleName) { + return !invalidArticles.contains(articleName); + } +} diff --git a/src/main/java/nl/carosi/remarkablepocket/DownloadService.java b/src/main/java/nl/carosi/remarkablepocket/DownloadService.java index 75b71f2..8308bcd 100644 --- a/src/main/java/nl/carosi/remarkablepocket/DownloadService.java +++ b/src/main/java/nl/carosi/remarkablepocket/DownloadService.java @@ -1,14 +1,13 @@ package nl.carosi.remarkablepocket; -import com.google.common.collect.Streams; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; -import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.PostConstruct; import nl.carosi.remarkablepocket.model.Article; import org.slf4j.Logger; @@ -18,11 +17,12 @@ final class DownloadService { private static final Logger LOG = LoggerFactory.getLogger(DownloadService.class); private final ArticleDownloader downloader; - private final HashSet
invalidArticles = new HashSet<>(); + private final ArticleValidator validator; private Path storageDir; - DownloadService(ArticleDownloader downloader) { + DownloadService(ArticleDownloader downloader, ArticleValidator validator) { this.downloader = downloader; + this.validator = validator; } @PostConstruct @@ -35,32 +35,33 @@ final class DownloadService { List download(List
articles, int articleLimit, int nArticlesOnRm) { int limit = articleLimit - nArticlesOnRm; int pocketCount = articles.size(); - long total = Math.min(pocketCount, limit); + int total = Math.min(pocketCount, limit); LOG.info( "Found {} unread article(s) on Remarkable. Downloading {} more from Pocket.", nArticlesOnRm, total); - return Streams.mapWithIndex(articles.stream(), (e, i) -> logProgress(e, i, total)) - .filter(e -> !invalidArticles.contains(e)) - .map(this::tryDownload) + AtomicInteger count = new AtomicInteger(1); + return articles.stream() + .filter(article -> validator.isValid(article.title())) + .map(article -> tryDownload(article, count, total)) .flatMap(Optional::stream) .limit(limit) .toList(); } - private Optional tryDownload(Article e) { - Optional path = downloader.tryDownload(e, storageDir); + private Optional tryDownload(Article article, AtomicInteger count, int total) { + String title = article.title(); + LOG.info("({}/{}) Downloading: '{}'.", count.get(), total, title); + Optional path = downloader.tryDownload(article, storageDir); if (path.isEmpty()) { - invalidArticles.add(e); + validator.invalidate(title); + } else { + LOG.info("Download successful."); + count.incrementAndGet(); } return path; } - private Article logProgress(Article article, long index, long total) { - LOG.info("({}/{}) Downloading: '{}'.", index + 1, total, article.title()); - return article; - } - void clearDownloads() throws IOException { try (DirectoryStream paths = Files.newDirectoryStream(storageDir, "*." + downloader.getFileType())) { diff --git a/src/main/java/nl/carosi/remarkablepocket/MetadataProvider.java b/src/main/java/nl/carosi/remarkablepocket/MetadataProvider.java index 1350ca9..9a2c0d3 100644 --- a/src/main/java/nl/carosi/remarkablepocket/MetadataProvider.java +++ b/src/main/java/nl/carosi/remarkablepocket/MetadataProvider.java @@ -7,12 +7,9 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipInputStream; -import javax.annotation.PostConstruct; import javax.xml.parsers.DocumentBuilder; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathExpressionException; @@ -30,14 +27,18 @@ final class MetadataProvider { private final RemarkableApi rmapi; private final ObjectMapper objectMapper; private final DocumentBuilder documentBuilder; + private final ArticleValidator validator; private final XPath publisherXpath; - private Path workDir; public MetadataProvider( - RemarkableApi rmapi, ObjectMapper objectMapper, DocumentBuilder documentBuilder) { + RemarkableApi rmapi, + ObjectMapper objectMapper, + DocumentBuilder documentBuilder, + ArticleValidator validator) { this.rmapi = rmapi; this.objectMapper = objectMapper; this.documentBuilder = documentBuilder; + this.validator = validator; this.publisherXpath = constructXpath(); } @@ -50,21 +51,15 @@ final class MetadataProvider { return opfXPath; } - @PostConstruct - void createWorkDir() throws IOException { - workDir = Files.createTempDirectory(null); - LOG.debug("Created temporary working directory: {}.", workDir); - } - DocumentMetadata getMetadata(String name) { LOG.debug("Getting metadata for document: {}.", name); - try (ZipFile zip = new ZipFile(rmapi.download(name, workDir.toString()))) { + try (ZipFile zip = new ZipFile(rmapi.download(name).toFile())) { String fileHash = zip.entries().nextElement().getName().split("\\.")[0]; if (zip.getEntry(fileHash + ".pdf") != null) { // In this case the article was converted to pdf because Remarkable // couldn't read the epub file. This means we lost the metadata, // so we delete it. - return null; + throw new RuntimeException("Article was converted to PDF"); } try (InputStream linesStream = zip.getInputStream(zip.getEntry(fileHash + ".content")); InputStream epubStream = zip.getInputStream(zip.getEntry(fileHash + ".epub"))) { @@ -76,7 +71,9 @@ final class MetadataProvider { LOG.info( "Article '{}' is corrupted. Deleting file and retrieving new article in next sync.", name); + LOG.debug("Article invalid because", e); rmapi.delete(name); + validator.invalidate(name); return null; } } diff --git a/src/main/java/nl/carosi/remarkablepocket/RemarkableApi.java b/src/main/java/nl/carosi/remarkablepocket/RemarkableApi.java index fd5065e..c6fb944 100644 --- a/src/main/java/nl/carosi/remarkablepocket/RemarkableApi.java +++ b/src/main/java/nl/carosi/remarkablepocket/RemarkableApi.java @@ -9,6 +9,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.List; import java.util.Scanner; @@ -38,6 +40,7 @@ public class RemarkableApi { : ""); private final String rmStorageDir; private final ObjectMapper objectMapper; + private String workDir; public RemarkableApi( ObjectMapper objectMapper, @Value("${rm.storage-dir}") String rmStorageDir) { @@ -87,6 +90,12 @@ public class RemarkableApi { .start(); } + @PostConstruct + void createWorkDir() throws IOException { + workDir = Files.createTempDirectory(null).toAbsolutePath().toString(); + LOG.debug("Created temporary working directory: {}.", workDir); + } + @PostConstruct public void login() { try { @@ -104,10 +113,10 @@ public class RemarkableApi { } } - public String download(String name, String dest) { - exec(RMAPI_EXECUTABLE, "-ni", "get", rmStorageDir + name); - exec("mv", name + ".zip", dest); - return dest + File.separator + name + ".zip"; + public Path download(String articleName) { + exec(RMAPI_EXECUTABLE, "-ni", "get", rmStorageDir + articleName); + exec("mv", articleName + ".zip", workDir); + return Path.of(workDir, articleName + ".zip"); } public List list() { @@ -118,11 +127,13 @@ public class RemarkableApi { new String[] {"cut", "-b5-"})); } - public Document info(String name) { + public Document info(String articleName) { List info = exec( List.of( - new String[] {RMAPI_EXECUTABLE, "-ni", "stat", rmStorageDir + name}, + new String[] { + RMAPI_EXECUTABLE, "-ni", "stat", rmStorageDir + articleName + }, new String[] {"sed", "/{/,$!d"})); try { return objectMapper.readValue(Strings.join(info, '\n'), Document.class); @@ -131,12 +142,12 @@ public class RemarkableApi { } } - public void upload(String path) { - exec(RMAPI_EXECUTABLE, "-ni", "put", path, rmStorageDir); + public void upload(Path path) { + exec(RMAPI_EXECUTABLE, "-ni", "put", path.toString(), rmStorageDir); } - public void delete(String name) { - exec(RMAPI_EXECUTABLE, "-ni", "rm", rmStorageDir + name); + public void delete(String articleName) { + exec(RMAPI_EXECUTABLE, "-ni", "rm", rmStorageDir + articleName); } public void createDir(String path) { diff --git a/src/main/java/nl/carosi/remarkablepocket/RemarkableService.java b/src/main/java/nl/carosi/remarkablepocket/RemarkableService.java index 84c8216..f3af247 100644 --- a/src/main/java/nl/carosi/remarkablepocket/RemarkableService.java +++ b/src/main/java/nl/carosi/remarkablepocket/RemarkableService.java @@ -74,6 +74,6 @@ final class RemarkableService { } private void upload(Path path) { - rmapi.upload(path.toAbsolutePath().toString()); + rmapi.upload(path); } } diff --git a/src/main/java/nl/carosi/remarkablepocket/SyncApplication.java b/src/main/java/nl/carosi/remarkablepocket/SyncApplication.java index 865db07..73bd0d9 100644 --- a/src/main/java/nl/carosi/remarkablepocket/SyncApplication.java +++ b/src/main/java/nl/carosi/remarkablepocket/SyncApplication.java @@ -18,6 +18,7 @@ import pl.codeset.pocket.Pocket; @EnableRetry @Import({ ArticleDownloader.class, + ArticleValidator.class, DownloadService.class, EpubReader.class, EpubWriter.class, diff --git a/src/main/java/nl/carosi/remarkablepocket/SyncCommand.java b/src/main/java/nl/carosi/remarkablepocket/SyncCommand.java index 6be6733..e5bbc4d 100644 --- a/src/main/java/nl/carosi/remarkablepocket/SyncCommand.java +++ b/src/main/java/nl/carosi/remarkablepocket/SyncCommand.java @@ -19,7 +19,7 @@ import picocli.CommandLine.Option; sortOptions = false, usageHelpAutoWidth = true, // TODO: Read from gradle.properties - version = "0.2.2", + version = "0.2.3", mixinStandardHelpOptions = true) class SyncCommand implements Callable { @Option(