diff --git a/README.md b/README.md index 1ede6fd..f90b632 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ from https://docs.docker.com/get-docker/. Then run the following command to star have not tested it on Windows yet): ``` -touch ~/.remarkable-pocket ~/.rmapi && mkdir -p ~/.rmapi-cache && docker run -it --env TZ=Europe/Amsterdam -p 65112:65112 -v ~/.remarkable-pocket:/root/.remarkable-pocket -v ~/.rmapi:/root/.rmapi -v ~/.rmapi-cache:/root/.cache/rmapi ghcr.io/nov1n/remarkable-pocket:0.2.1 +touch ~/.remarkable-pocket ~/.rmapi && mkdir -p ~/.rmapi-cache && docker run -it --env TZ=Europe/Amsterdam -p 65112:65112 -v ~/.remarkable-pocket:/root/.remarkable-pocket -v ~/.rmapi:/root/.rmapi -v ~/.rmapi-cache:/root/.cache/rmapi ghcr.io/nov1n/remarkable-pocket:0.2.2 ``` The first time you run the application, you will be asked to authorize Pocket and Remarkable Cloud. Once you have done diff --git a/build.gradle b/build.gradle index 2d78fa6..29fbd76 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ plugins { } group = "nl.carosi" -version = "0.2.1" +version = "0.2.2" java { toolchain { diff --git a/docker-compose.yml b/docker-compose.yml index ae7b447..cb2deaf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: remarkable-pocket: - image: ghcr.io/nov1n/remarkable-pocket:0.2.1 + image: ghcr.io/nov1n/remarkable-pocket:0.2.2 restart: always ports: - 65112:65112 diff --git a/nl.carosi.remarkable-pocket.plist b/nl.carosi.remarkable-pocket.plist index 33d3ec3..68f8559 100644 --- a/nl.carosi.remarkable-pocket.plist +++ b/nl.carosi.remarkable-pocket.plist @@ -8,7 +8,7 @@ /bin/sh -c - while ! /usr/local/bin/docker version > /dev/null 2>&1; do sleep 5; done && /usr/local/bin/docker run --env TZ=Europe/Amsterdam -v ~/.remarkable-pocket:/root/.remarkable-pocket -v ~/.rmapi:/root/.rmapi -v ~/.rmapi-cache:/root/.cache/rmapi ghcr.io/nov1n/remarkable-pocket:0.2.1 1>>$HOME/.remarkable-pocket.log 2>&1 + while ! /usr/local/bin/docker version > /dev/null 2>&1; do sleep 5; done && /usr/local/bin/docker run --env TZ=Europe/Amsterdam -v ~/.remarkable-pocket:/root/.remarkable-pocket -v ~/.rmapi:/root/.rmapi -v ~/.rmapi-cache:/root/.cache/rmapi ghcr.io/nov1n/remarkable-pocket:0.2.2 1>>$HOME/.remarkable-pocket.log 2>&1 RunAtLoad diff --git a/src/main/java/nl/carosi/remarkablepocket/ArticleValidator.java b/src/main/java/nl/carosi/remarkablepocket/ArticleValidator.java new file mode 100644 index 0000000..e048a45 --- /dev/null +++ b/src/main/java/nl/carosi/remarkablepocket/ArticleValidator.java @@ -0,0 +1,47 @@ +package nl.carosi.remarkablepocket; + +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Optional; +import nl.carosi.remarkablepocket.model.Article; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class ArticleValidator { + private static final Logger LOG = LoggerFactory.getLogger(ArticleValidator.class); + private final HashSet
invalidArticles = new HashSet<>(); + private final MetadataProvider metadataProvider; + private final RemarkableApi remarkableApi; + + public ArticleValidator(MetadataProvider metadataProvider, RemarkableApi remarkableApi) { + this.metadataProvider = metadataProvider; + this.remarkableApi = remarkableApi; + } + + public Optional validate(Optional path, Article article) { + if (path.isEmpty()) { + invalidate(article); + return path; + } + + remarkableApi.upload(path.get()); + try { + metadataProvider.getMetadata(article.title()); + remarkableApi.delete(article.title()); + LOG.debug("Article is valid: {}", article); + return path; + } catch (RuntimeException e) { + invalidate(article); + return Optional.empty(); + } + } + + private void invalidate(Article article) { + LOG.debug("Article is invalid: {}", article); + invalidArticles.add(article); + } + + public boolean isValid(Article article) { + return !invalidArticles.contains(article); + } +} diff --git a/src/main/java/nl/carosi/remarkablepocket/DownloadService.java b/src/main/java/nl/carosi/remarkablepocket/DownloadService.java index 75b71f2..db1a0c4 100644 --- a/src/main/java/nl/carosi/remarkablepocket/DownloadService.java +++ b/src/main/java/nl/carosi/remarkablepocket/DownloadService.java @@ -6,7 +6,6 @@ import java.io.UncheckedIOException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; -import java.util.HashSet; import java.util.List; import java.util.Optional; import javax.annotation.PostConstruct; @@ -16,13 +15,13 @@ import org.slf4j.LoggerFactory; final class DownloadService { private static final Logger LOG = LoggerFactory.getLogger(DownloadService.class); - private final ArticleDownloader downloader; - private final HashSet
invalidArticles = new HashSet<>(); + private final ArticleValidator validator; private Path storageDir; - DownloadService(ArticleDownloader downloader) { + DownloadService(ArticleDownloader downloader, ArticleValidator validator) { this.downloader = downloader; + this.validator = validator; } @PostConstruct @@ -41,21 +40,13 @@ final class DownloadService { nArticlesOnRm, total); return Streams.mapWithIndex(articles.stream(), (e, i) -> logProgress(e, i, total)) - .filter(e -> !invalidArticles.contains(e)) - .map(this::tryDownload) + .filter(validator::isValid) + .map(a -> validator.validate(downloader.tryDownload(a, storageDir), a)) .flatMap(Optional::stream) .limit(limit) .toList(); } - private Optional tryDownload(Article e) { - Optional path = downloader.tryDownload(e, storageDir); - if (path.isEmpty()) { - invalidArticles.add(e); - } - return path; - } - private Article logProgress(Article article, long index, long total) { LOG.info("({}/{}) Downloading: '{}'.", index + 1, total, article.title()); return article; diff --git a/src/main/java/nl/carosi/remarkablepocket/MetadataProvider.java b/src/main/java/nl/carosi/remarkablepocket/MetadataProvider.java index 130df24..dd7e298 100644 --- a/src/main/java/nl/carosi/remarkablepocket/MetadataProvider.java +++ b/src/main/java/nl/carosi/remarkablepocket/MetadataProvider.java @@ -7,12 +7,9 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipInputStream; -import javax.annotation.PostConstruct; import javax.xml.parsers.DocumentBuilder; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathExpressionException; @@ -26,12 +23,10 @@ import org.xml.sax.SAXException; final class MetadataProvider { private static final Logger LOG = LoggerFactory.getLogger(MetadataProvider.class); - private final RemarkableApi rmapi; private final ObjectMapper objectMapper; private final DocumentBuilder documentBuilder; private final XPath publisherXpath; - private Path workDir; public MetadataProvider( RemarkableApi rmapi, ObjectMapper objectMapper, DocumentBuilder documentBuilder) { @@ -50,29 +45,17 @@ final class MetadataProvider { return opfXPath; } - @PostConstruct - void createWorkDir() throws IOException { - workDir = Files.createTempDirectory(null); - LOG.debug("Created temporary working directory: {}.", workDir); - } - - DocumentMetadata getMetadata(String name) { - LOG.debug("Getting metadata for document: {}.", name); - ZipFile zip; - try { - zip = new ZipFile(rmapi.download(name, workDir.toString())); - } catch (IOException e) { - throw new RuntimeException(e); - } - - String fileHash = zip.entries().nextElement().getName().split("\\.")[0]; - - try (InputStream lines = zip.getInputStream(zip.getEntry(fileHash + ".content")); - InputStream epub = zip.getInputStream(zip.getEntry(fileHash + ".epub"))) { - int pageCount = objectMapper.readValue(lines, Lines.class).pageCount(); - String pocketId = extractPocketId(epub); - return new DocumentMetadata(rmapi.info(name), pageCount, pocketId); - } catch (IOException | SAXException | XPathExpressionException e) { + DocumentMetadata getMetadata(String articleName) { + LOG.debug("Getting metadata for document: {}.", articleName); + try (ZipFile zip = new ZipFile(rmapi.download(articleName).toFile())) { + String fileHash = zip.entries().nextElement().getName().split("\\.")[0]; + try (InputStream lines = zip.getInputStream(zip.getEntry(fileHash + ".content")); + InputStream epub = zip.getInputStream(zip.getEntry(fileHash + ".epub"))) { + int pageCount = objectMapper.readValue(lines, Lines.class).pageCount(); + String pocketId = extractPocketId(epub); + return new DocumentMetadata(rmapi.info(articleName), pageCount, pocketId); + } + } catch (Exception e) { throw new RuntimeException(e); } } diff --git a/src/main/java/nl/carosi/remarkablepocket/RemarkableApi.java b/src/main/java/nl/carosi/remarkablepocket/RemarkableApi.java index fd5065e..c6fb944 100644 --- a/src/main/java/nl/carosi/remarkablepocket/RemarkableApi.java +++ b/src/main/java/nl/carosi/remarkablepocket/RemarkableApi.java @@ -9,6 +9,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.List; import java.util.Scanner; @@ -38,6 +40,7 @@ public class RemarkableApi { : ""); private final String rmStorageDir; private final ObjectMapper objectMapper; + private String workDir; public RemarkableApi( ObjectMapper objectMapper, @Value("${rm.storage-dir}") String rmStorageDir) { @@ -87,6 +90,12 @@ public class RemarkableApi { .start(); } + @PostConstruct + void createWorkDir() throws IOException { + workDir = Files.createTempDirectory(null).toAbsolutePath().toString(); + LOG.debug("Created temporary working directory: {}.", workDir); + } + @PostConstruct public void login() { try { @@ -104,10 +113,10 @@ public class RemarkableApi { } } - public String download(String name, String dest) { - exec(RMAPI_EXECUTABLE, "-ni", "get", rmStorageDir + name); - exec("mv", name + ".zip", dest); - return dest + File.separator + name + ".zip"; + public Path download(String articleName) { + exec(RMAPI_EXECUTABLE, "-ni", "get", rmStorageDir + articleName); + exec("mv", articleName + ".zip", workDir); + return Path.of(workDir, articleName + ".zip"); } public List list() { @@ -118,11 +127,13 @@ public class RemarkableApi { new String[] {"cut", "-b5-"})); } - public Document info(String name) { + public Document info(String articleName) { List info = exec( List.of( - new String[] {RMAPI_EXECUTABLE, "-ni", "stat", rmStorageDir + name}, + new String[] { + RMAPI_EXECUTABLE, "-ni", "stat", rmStorageDir + articleName + }, new String[] {"sed", "/{/,$!d"})); try { return objectMapper.readValue(Strings.join(info, '\n'), Document.class); @@ -131,12 +142,12 @@ public class RemarkableApi { } } - public void upload(String path) { - exec(RMAPI_EXECUTABLE, "-ni", "put", path, rmStorageDir); + public void upload(Path path) { + exec(RMAPI_EXECUTABLE, "-ni", "put", path.toString(), rmStorageDir); } - public void delete(String name) { - exec(RMAPI_EXECUTABLE, "-ni", "rm", rmStorageDir + name); + public void delete(String articleName) { + exec(RMAPI_EXECUTABLE, "-ni", "rm", rmStorageDir + articleName); } public void createDir(String path) { diff --git a/src/main/java/nl/carosi/remarkablepocket/RemarkableService.java b/src/main/java/nl/carosi/remarkablepocket/RemarkableService.java index 2a8ad17..b74ac46 100644 --- a/src/main/java/nl/carosi/remarkablepocket/RemarkableService.java +++ b/src/main/java/nl/carosi/remarkablepocket/RemarkableService.java @@ -72,6 +72,6 @@ final class RemarkableService { } private void upload(Path path) { - rmapi.upload(path.toAbsolutePath().toString()); + rmapi.upload(path); } } diff --git a/src/main/java/nl/carosi/remarkablepocket/SyncApplication.java b/src/main/java/nl/carosi/remarkablepocket/SyncApplication.java index 865db07..73bd0d9 100644 --- a/src/main/java/nl/carosi/remarkablepocket/SyncApplication.java +++ b/src/main/java/nl/carosi/remarkablepocket/SyncApplication.java @@ -18,6 +18,7 @@ import pl.codeset.pocket.Pocket; @EnableRetry @Import({ ArticleDownloader.class, + ArticleValidator.class, DownloadService.class, EpubReader.class, EpubWriter.class, diff --git a/src/main/java/nl/carosi/remarkablepocket/SyncCommand.java b/src/main/java/nl/carosi/remarkablepocket/SyncCommand.java index 7abba07..6be6733 100644 --- a/src/main/java/nl/carosi/remarkablepocket/SyncCommand.java +++ b/src/main/java/nl/carosi/remarkablepocket/SyncCommand.java @@ -19,7 +19,7 @@ import picocli.CommandLine.Option; sortOptions = false, usageHelpAutoWidth = true, // TODO: Read from gradle.properties - version = "0.2.1", + version = "0.2.2", mixinStandardHelpOptions = true) class SyncCommand implements Callable { @Option(