From 3e8c7b70ab76710706616c3632f9391c8a5efaf5 Mon Sep 17 00:00:00 2001 From: Robert Carosi Date: Fri, 23 Sep 2022 10:54:16 +0200 Subject: [PATCH] Gracefully handle converted articles Sometimes Remarkable will convert epubs to PDF files. It is not clear when this happens, but in the process it removes the epub file and thus failing the program. We now check if the file will be uploaded correctly as an epub, and skip it if it's not. --- README.md | 2 +- build.gradle | 2 +- docker-compose.yml | 2 +- nl.carosi.remarkable-pocket.plist | 2 +- .../remarkablepocket/ArticleValidator.java | 47 +++++++++++++++++++ .../remarkablepocket/DownloadService.java | 19 ++------ .../remarkablepocket/MetadataProvider.java | 39 +++++---------- .../remarkablepocket/RemarkableApi.java | 31 ++++++++---- .../remarkablepocket/RemarkableService.java | 2 +- .../remarkablepocket/SyncApplication.java | 1 + .../carosi/remarkablepocket/SyncCommand.java | 2 +- 11 files changed, 91 insertions(+), 58 deletions(-) create mode 100644 src/main/java/nl/carosi/remarkablepocket/ArticleValidator.java diff --git a/README.md b/README.md index 1ede6fd..f90b632 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ from https://docs.docker.com/get-docker/. Then run the following command to star have not tested it on Windows yet): ``` -touch ~/.remarkable-pocket ~/.rmapi && mkdir -p ~/.rmapi-cache && docker run -it --env TZ=Europe/Amsterdam -p 65112:65112 -v ~/.remarkable-pocket:/root/.remarkable-pocket -v ~/.rmapi:/root/.rmapi -v ~/.rmapi-cache:/root/.cache/rmapi ghcr.io/nov1n/remarkable-pocket:0.2.1 +touch ~/.remarkable-pocket ~/.rmapi && mkdir -p ~/.rmapi-cache && docker run -it --env TZ=Europe/Amsterdam -p 65112:65112 -v ~/.remarkable-pocket:/root/.remarkable-pocket -v ~/.rmapi:/root/.rmapi -v ~/.rmapi-cache:/root/.cache/rmapi ghcr.io/nov1n/remarkable-pocket:0.2.2 ``` The first time you run the application, you will be asked to authorize Pocket and Remarkable Cloud. Once you have done diff --git a/build.gradle b/build.gradle index 2d78fa6..29fbd76 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ plugins { } group = "nl.carosi" -version = "0.2.1" +version = "0.2.2" java { toolchain { diff --git a/docker-compose.yml b/docker-compose.yml index ae7b447..cb2deaf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: remarkable-pocket: - image: ghcr.io/nov1n/remarkable-pocket:0.2.1 + image: ghcr.io/nov1n/remarkable-pocket:0.2.2 restart: always ports: - 65112:65112 diff --git a/nl.carosi.remarkable-pocket.plist b/nl.carosi.remarkable-pocket.plist index 33d3ec3..68f8559 100644 --- a/nl.carosi.remarkable-pocket.plist +++ b/nl.carosi.remarkable-pocket.plist @@ -8,7 +8,7 @@ /bin/sh -c - while ! /usr/local/bin/docker version > /dev/null 2>&1; do sleep 5; done && /usr/local/bin/docker run --env TZ=Europe/Amsterdam -v ~/.remarkable-pocket:/root/.remarkable-pocket -v ~/.rmapi:/root/.rmapi -v ~/.rmapi-cache:/root/.cache/rmapi ghcr.io/nov1n/remarkable-pocket:0.2.1 1>>$HOME/.remarkable-pocket.log 2>&1 + while ! /usr/local/bin/docker version > /dev/null 2>&1; do sleep 5; done && /usr/local/bin/docker run --env TZ=Europe/Amsterdam -v ~/.remarkable-pocket:/root/.remarkable-pocket -v ~/.rmapi:/root/.rmapi -v ~/.rmapi-cache:/root/.cache/rmapi ghcr.io/nov1n/remarkable-pocket:0.2.2 1>>$HOME/.remarkable-pocket.log 2>&1 RunAtLoad diff --git a/src/main/java/nl/carosi/remarkablepocket/ArticleValidator.java b/src/main/java/nl/carosi/remarkablepocket/ArticleValidator.java new file mode 100644 index 0000000..e048a45 --- /dev/null +++ b/src/main/java/nl/carosi/remarkablepocket/ArticleValidator.java @@ -0,0 +1,47 @@ +package nl.carosi.remarkablepocket; + +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Optional; +import nl.carosi.remarkablepocket.model.Article; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class ArticleValidator { + private static final Logger LOG = LoggerFactory.getLogger(ArticleValidator.class); + private final HashSet
invalidArticles = new HashSet<>(); + private final MetadataProvider metadataProvider; + private final RemarkableApi remarkableApi; + + public ArticleValidator(MetadataProvider metadataProvider, RemarkableApi remarkableApi) { + this.metadataProvider = metadataProvider; + this.remarkableApi = remarkableApi; + } + + public Optional validate(Optional path, Article article) { + if (path.isEmpty()) { + invalidate(article); + return path; + } + + remarkableApi.upload(path.get()); + try { + metadataProvider.getMetadata(article.title()); + remarkableApi.delete(article.title()); + LOG.debug("Article is valid: {}", article); + return path; + } catch (RuntimeException e) { + invalidate(article); + return Optional.empty(); + } + } + + private void invalidate(Article article) { + LOG.debug("Article is invalid: {}", article); + invalidArticles.add(article); + } + + public boolean isValid(Article article) { + return !invalidArticles.contains(article); + } +} diff --git a/src/main/java/nl/carosi/remarkablepocket/DownloadService.java b/src/main/java/nl/carosi/remarkablepocket/DownloadService.java index 75b71f2..db1a0c4 100644 --- a/src/main/java/nl/carosi/remarkablepocket/DownloadService.java +++ b/src/main/java/nl/carosi/remarkablepocket/DownloadService.java @@ -6,7 +6,6 @@ import java.io.UncheckedIOException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; -import java.util.HashSet; import java.util.List; import java.util.Optional; import javax.annotation.PostConstruct; @@ -16,13 +15,13 @@ import org.slf4j.LoggerFactory; final class DownloadService { private static final Logger LOG = LoggerFactory.getLogger(DownloadService.class); - private final ArticleDownloader downloader; - private final HashSet
invalidArticles = new HashSet<>(); + private final ArticleValidator validator; private Path storageDir; - DownloadService(ArticleDownloader downloader) { + DownloadService(ArticleDownloader downloader, ArticleValidator validator) { this.downloader = downloader; + this.validator = validator; } @PostConstruct @@ -41,21 +40,13 @@ final class DownloadService { nArticlesOnRm, total); return Streams.mapWithIndex(articles.stream(), (e, i) -> logProgress(e, i, total)) - .filter(e -> !invalidArticles.contains(e)) - .map(this::tryDownload) + .filter(validator::isValid) + .map(a -> validator.validate(downloader.tryDownload(a, storageDir), a)) .flatMap(Optional::stream) .limit(limit) .toList(); } - private Optional tryDownload(Article e) { - Optional path = downloader.tryDownload(e, storageDir); - if (path.isEmpty()) { - invalidArticles.add(e); - } - return path; - } - private Article logProgress(Article article, long index, long total) { LOG.info("({}/{}) Downloading: '{}'.", index + 1, total, article.title()); return article; diff --git a/src/main/java/nl/carosi/remarkablepocket/MetadataProvider.java b/src/main/java/nl/carosi/remarkablepocket/MetadataProvider.java index 130df24..dd7e298 100644 --- a/src/main/java/nl/carosi/remarkablepocket/MetadataProvider.java +++ b/src/main/java/nl/carosi/remarkablepocket/MetadataProvider.java @@ -7,12 +7,9 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipInputStream; -import javax.annotation.PostConstruct; import javax.xml.parsers.DocumentBuilder; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathExpressionException; @@ -26,12 +23,10 @@ import org.xml.sax.SAXException; final class MetadataProvider { private static final Logger LOG = LoggerFactory.getLogger(MetadataProvider.class); - private final RemarkableApi rmapi; private final ObjectMapper objectMapper; private final DocumentBuilder documentBuilder; private final XPath publisherXpath; - private Path workDir; public MetadataProvider( RemarkableApi rmapi, ObjectMapper objectMapper, DocumentBuilder documentBuilder) { @@ -50,29 +45,17 @@ final class MetadataProvider { return opfXPath; } - @PostConstruct - void createWorkDir() throws IOException { - workDir = Files.createTempDirectory(null); - LOG.debug("Created temporary working directory: {}.", workDir); - } - - DocumentMetadata getMetadata(String name) { - LOG.debug("Getting metadata for document: {}.", name); - ZipFile zip; - try { - zip = new ZipFile(rmapi.download(name, workDir.toString())); - } catch (IOException e) { - throw new RuntimeException(e); - } - - String fileHash = zip.entries().nextElement().getName().split("\\.")[0]; - - try (InputStream lines = zip.getInputStream(zip.getEntry(fileHash + ".content")); - InputStream epub = zip.getInputStream(zip.getEntry(fileHash + ".epub"))) { - int pageCount = objectMapper.readValue(lines, Lines.class).pageCount(); - String pocketId = extractPocketId(epub); - return new DocumentMetadata(rmapi.info(name), pageCount, pocketId); - } catch (IOException | SAXException | XPathExpressionException e) { + DocumentMetadata getMetadata(String articleName) { + LOG.debug("Getting metadata for document: {}.", articleName); + try (ZipFile zip = new ZipFile(rmapi.download(articleName).toFile())) { + String fileHash = zip.entries().nextElement().getName().split("\\.")[0]; + try (InputStream lines = zip.getInputStream(zip.getEntry(fileHash + ".content")); + InputStream epub = zip.getInputStream(zip.getEntry(fileHash + ".epub"))) { + int pageCount = objectMapper.readValue(lines, Lines.class).pageCount(); + String pocketId = extractPocketId(epub); + return new DocumentMetadata(rmapi.info(articleName), pageCount, pocketId); + } + } catch (Exception e) { throw new RuntimeException(e); } } diff --git a/src/main/java/nl/carosi/remarkablepocket/RemarkableApi.java b/src/main/java/nl/carosi/remarkablepocket/RemarkableApi.java index fd5065e..c6fb944 100644 --- a/src/main/java/nl/carosi/remarkablepocket/RemarkableApi.java +++ b/src/main/java/nl/carosi/remarkablepocket/RemarkableApi.java @@ -9,6 +9,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.List; import java.util.Scanner; @@ -38,6 +40,7 @@ public class RemarkableApi { : ""); private final String rmStorageDir; private final ObjectMapper objectMapper; + private String workDir; public RemarkableApi( ObjectMapper objectMapper, @Value("${rm.storage-dir}") String rmStorageDir) { @@ -87,6 +90,12 @@ public class RemarkableApi { .start(); } + @PostConstruct + void createWorkDir() throws IOException { + workDir = Files.createTempDirectory(null).toAbsolutePath().toString(); + LOG.debug("Created temporary working directory: {}.", workDir); + } + @PostConstruct public void login() { try { @@ -104,10 +113,10 @@ public class RemarkableApi { } } - public String download(String name, String dest) { - exec(RMAPI_EXECUTABLE, "-ni", "get", rmStorageDir + name); - exec("mv", name + ".zip", dest); - return dest + File.separator + name + ".zip"; + public Path download(String articleName) { + exec(RMAPI_EXECUTABLE, "-ni", "get", rmStorageDir + articleName); + exec("mv", articleName + ".zip", workDir); + return Path.of(workDir, articleName + ".zip"); } public List list() { @@ -118,11 +127,13 @@ public class RemarkableApi { new String[] {"cut", "-b5-"})); } - public Document info(String name) { + public Document info(String articleName) { List info = exec( List.of( - new String[] {RMAPI_EXECUTABLE, "-ni", "stat", rmStorageDir + name}, + new String[] { + RMAPI_EXECUTABLE, "-ni", "stat", rmStorageDir + articleName + }, new String[] {"sed", "/{/,$!d"})); try { return objectMapper.readValue(Strings.join(info, '\n'), Document.class); @@ -131,12 +142,12 @@ public class RemarkableApi { } } - public void upload(String path) { - exec(RMAPI_EXECUTABLE, "-ni", "put", path, rmStorageDir); + public void upload(Path path) { + exec(RMAPI_EXECUTABLE, "-ni", "put", path.toString(), rmStorageDir); } - public void delete(String name) { - exec(RMAPI_EXECUTABLE, "-ni", "rm", rmStorageDir + name); + public void delete(String articleName) { + exec(RMAPI_EXECUTABLE, "-ni", "rm", rmStorageDir + articleName); } public void createDir(String path) { diff --git a/src/main/java/nl/carosi/remarkablepocket/RemarkableService.java b/src/main/java/nl/carosi/remarkablepocket/RemarkableService.java index 2a8ad17..b74ac46 100644 --- a/src/main/java/nl/carosi/remarkablepocket/RemarkableService.java +++ b/src/main/java/nl/carosi/remarkablepocket/RemarkableService.java @@ -72,6 +72,6 @@ final class RemarkableService { } private void upload(Path path) { - rmapi.upload(path.toAbsolutePath().toString()); + rmapi.upload(path); } } diff --git a/src/main/java/nl/carosi/remarkablepocket/SyncApplication.java b/src/main/java/nl/carosi/remarkablepocket/SyncApplication.java index 865db07..73bd0d9 100644 --- a/src/main/java/nl/carosi/remarkablepocket/SyncApplication.java +++ b/src/main/java/nl/carosi/remarkablepocket/SyncApplication.java @@ -18,6 +18,7 @@ import pl.codeset.pocket.Pocket; @EnableRetry @Import({ ArticleDownloader.class, + ArticleValidator.class, DownloadService.class, EpubReader.class, EpubWriter.class, diff --git a/src/main/java/nl/carosi/remarkablepocket/SyncCommand.java b/src/main/java/nl/carosi/remarkablepocket/SyncCommand.java index 7abba07..6be6733 100644 --- a/src/main/java/nl/carosi/remarkablepocket/SyncCommand.java +++ b/src/main/java/nl/carosi/remarkablepocket/SyncCommand.java @@ -19,7 +19,7 @@ import picocli.CommandLine.Option; sortOptions = false, usageHelpAutoWidth = true, // TODO: Read from gradle.properties - version = "0.2.1", + version = "0.2.2", mixinStandardHelpOptions = true) class SyncCommand implements Callable { @Option(