jlengrand/RemarkablePocket
1
0
Fork 0
mirror of https://github.com/jlengrand/RemarkablePocket.git synced 2026-03-10 08:41:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Gracefully handle converted articles

Browse Source 
Sometimes Remarkable will convert epubs to PDF files. It is not clear
when this happens, but in the process  it removes the epub file and
thus failing the program.

We now check if the file will be uploaded correctly as an epub, and skip
it if it's not.
This commit is contained in:
Robert Carosi
2022-09-23 10:54:16 +02:00
parent edcec60eae
commit 3e8c7b70ab
11 changed files with 91 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -52,7 +52,7 @@ from https://docs.docker.com/get-docker/. Then run the following command to star
have not tested it on Windows yet):
```
touch ~/.remarkable-pocket ~/.rmapi && mkdir -p ~/.rmapi-cache && docker run -it --env TZ=Europe/Amsterdam -p 65112:65112 -v ~/.remarkable-pocket:/root/.remarkable-pocket -v ~/.rmapi:/root/.rmapi -v ~/.rmapi-cache:/root/.cache/rmapi ghcr.io/nov1n/remarkable-pocket:0.2.1
touch ~/.remarkable-pocket ~/.rmapi && mkdir -p ~/.rmapi-cache && docker run -it --env TZ=Europe/Amsterdam -p 65112:65112 -v ~/.remarkable-pocket:/root/.remarkable-pocket -v ~/.rmapi:/root/.rmapi -v ~/.rmapi-cache:/root/.cache/rmapi ghcr.io/nov1n/remarkable-pocket:0.2.2
```
The first time you run the application, you will be asked to authorize Pocket and Remarkable Cloud. Once you have done

2
build.gradle
View File

@@ -7,7 +7,7 @@ plugins {
}
group = "nl.carosi"
version = "0.2.1"
version = "0.2.2"
java {
toolchain {

2
docker-compose.yml
View File

@@ -2,7 +2,7 @@ version: '3'
services:
remarkable-pocket:
image: ghcr.io/nov1n/remarkable-pocket:0.2.1
image: ghcr.io/nov1n/remarkable-pocket:0.2.2
restart: always
ports:
- 65112:65112

2
nl.carosi.remarkable-pocket.plist
View File

@@ -8,7 +8,7 @@
<array>
<string>/bin/sh</string>
<string>-c</string>
<string>while ! /usr/local/bin/docker version > /dev/null 2>&amp;1; do sleep 5; done &amp;&amp; /usr/local/bin/docker run --env TZ=Europe/Amsterdam -v ~/.remarkable-pocket:/root/.remarkable-pocket -v ~/.rmapi:/root/.rmapi -v ~/.rmapi-cache:/root/.cache/rmapi ghcr.io/nov1n/remarkable-pocket:0.2.1 1>>$HOME/.remarkable-pocket.log 2>&amp;1</string>
<string>while ! /usr/local/bin/docker version > /dev/null 2>&amp;1; do sleep 5; done &amp;&amp; /usr/local/bin/docker run --env TZ=Europe/Amsterdam -v ~/.remarkable-pocket:/root/.remarkable-pocket -v ~/.rmapi:/root/.rmapi -v ~/.rmapi-cache:/root/.cache/rmapi ghcr.io/nov1n/remarkable-pocket:0.2.2 1>>$HOME/.remarkable-pocket.log 2>&amp;1</string>
</array>
<key>RunAtLoad</key>
<true/>

47
src/main/java/nl/carosi/remarkablepocket/ArticleValidator.java Normal file
View File

@@ -0,0 +1,47 @@
package nl.carosi.remarkablepocket;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.Optional;
import nl.carosi.remarkablepocket.model.Article;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
class ArticleValidator {
private static final Logger LOG = LoggerFactory.getLogger(ArticleValidator.class);
private final HashSet<Article> invalidArticles = new HashSet<>();
private final MetadataProvider metadataProvider;
private final RemarkableApi remarkableApi;
public ArticleValidator(MetadataProvider metadataProvider, RemarkableApi remarkableApi) {
this.metadataProvider = metadataProvider;
this.remarkableApi = remarkableApi;
}
public Optional<Path> validate(Optional<Path> path, Article article) {
if (path.isEmpty()) {
invalidate(article);
return path;
}
remarkableApi.upload(path.get());
try {
metadataProvider.getMetadata(article.title());
remarkableApi.delete(article.title());
LOG.debug("Article is valid: {}", article);
return path;
} catch (RuntimeException e) {
invalidate(article);
return Optional.empty();
}
}
private void invalidate(Article article) {
LOG.debug("Article is invalid: {}", article);
invalidArticles.add(article);
}
public boolean isValid(Article article) {
return !invalidArticles.contains(article);
}
}

19
src/main/java/nl/carosi/remarkablepocket/DownloadService.java
View File

@@ -6,7 +6,6 @@ import java.io.UncheckedIOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import javax.annotation.PostConstruct;
@@ -16,13 +15,13 @@ import org.slf4j.LoggerFactory;
final class DownloadService {
private static final Logger LOG = LoggerFactory.getLogger(DownloadService.class);
private final ArticleDownloader downloader;
private final HashSet<Article> invalidArticles = new HashSet<>();
private final ArticleValidator validator;
private Path storageDir;
DownloadService(ArticleDownloader downloader) {
DownloadService(ArticleDownloader downloader, ArticleValidator validator) {
this.downloader = downloader;
this.validator = validator;
}
@PostConstruct
@@ -41,21 +40,13 @@ final class DownloadService {
nArticlesOnRm,
total);
return Streams.mapWithIndex(articles.stream(), (e, i) -> logProgress(e, i, total))
.filter(e -> !invalidArticles.contains(e))
.map(this::tryDownload)
.filter(validator::isValid)
.map(a -> validator.validate(downloader.tryDownload(a, storageDir), a))
.flatMap(Optional::stream)
.limit(limit)
.toList();
}
private Optional<Path> tryDownload(Article e) {
Optional<Path> path = downloader.tryDownload(e, storageDir);
if (path.isEmpty()) {
invalidArticles.add(e);
}
return path;
}
private Article logProgress(Article article, long index, long total) {
LOG.info("({}/{}) Downloading: '{}'.", index + 1, total, article.title());
return article;

39
src/main/java/nl/carosi/remarkablepocket/MetadataProvider.java
View File

@@ -7,12 +7,9 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;
import javax.annotation.PostConstruct;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathExpressionException;
@@ -26,12 +23,10 @@ import org.xml.sax.SAXException;
final class MetadataProvider {
private static final Logger LOG = LoggerFactory.getLogger(MetadataProvider.class);
private final RemarkableApi rmapi;
private final ObjectMapper objectMapper;
private final DocumentBuilder documentBuilder;
private final XPath publisherXpath;
private Path workDir;
public MetadataProvider(
RemarkableApi rmapi, ObjectMapper objectMapper, DocumentBuilder documentBuilder) {
@@ -50,29 +45,17 @@ final class MetadataProvider {
return opfXPath;
}
@PostConstruct
void createWorkDir() throws IOException {
workDir = Files.createTempDirectory(null);
LOG.debug("Created temporary working directory: {}.", workDir);
}
DocumentMetadata getMetadata(String name) {
LOG.debug("Getting metadata for document: {}.", name);
ZipFile zip;
try {
zip = new ZipFile(rmapi.download(name, workDir.toString()));
} catch (IOException e) {
throw new RuntimeException(e);
}
String fileHash = zip.entries().nextElement().getName().split("\\.")[0];
try (InputStream lines = zip.getInputStream(zip.getEntry(fileHash + ".content"));
InputStream epub = zip.getInputStream(zip.getEntry(fileHash + ".epub"))) {
int pageCount = objectMapper.readValue(lines, Lines.class).pageCount();
String pocketId = extractPocketId(epub);
return new DocumentMetadata(rmapi.info(name), pageCount, pocketId);
} catch (IOException | SAXException | XPathExpressionException e) {
DocumentMetadata getMetadata(String articleName) {
LOG.debug("Getting metadata for document: {}.", articleName);
try (ZipFile zip = new ZipFile(rmapi.download(articleName).toFile())) {
String fileHash = zip.entries().nextElement().getName().split("\\.")[0];
try (InputStream lines = zip.getInputStream(zip.getEntry(fileHash + ".content"));
InputStream epub = zip.getInputStream(zip.getEntry(fileHash + ".epub"))) {
int pageCount = objectMapper.readValue(lines, Lines.class).pageCount();
String pocketId = extractPocketId(epub);
return new DocumentMetadata(rmapi.info(articleName), pageCount, pocketId);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}

31
src/main/java/nl/carosi/remarkablepocket/RemarkableApi.java
View File

@@ -9,6 +9,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
import java.util.Scanner;
@@ -38,6 +40,7 @@ public class RemarkableApi {
: "");
private final String rmStorageDir;
private final ObjectMapper objectMapper;
private String workDir;
public RemarkableApi(
ObjectMapper objectMapper, @Value("${rm.storage-dir}") String rmStorageDir) {
@@ -87,6 +90,12 @@ public class RemarkableApi {
.start();
}
@PostConstruct
void createWorkDir() throws IOException {
workDir = Files.createTempDirectory(null).toAbsolutePath().toString();
LOG.debug("Created temporary working directory: {}.", workDir);
}
@PostConstruct
public void login() {
try {
@@ -104,10 +113,10 @@ public class RemarkableApi {
}
}
public String download(String name, String dest) {
exec(RMAPI_EXECUTABLE, "-ni", "get", rmStorageDir + name);
exec("mv", name + ".zip", dest);
return dest + File.separator + name + ".zip";
public Path download(String articleName) {
exec(RMAPI_EXECUTABLE, "-ni", "get", rmStorageDir + articleName);
exec("mv", articleName + ".zip", workDir);
return Path.of(workDir, articleName + ".zip");
}
public List<String> list() {
@@ -118,11 +127,13 @@ public class RemarkableApi {
new String[] {"cut", "-b5-"}));
}
public Document info(String name) {
public Document info(String articleName) {
List<String> info =
exec(
List.of(
new String[] {RMAPI_EXECUTABLE, "-ni", "stat", rmStorageDir + name},
new String[] {
RMAPI_EXECUTABLE, "-ni", "stat", rmStorageDir + articleName
},
new String[] {"sed", "/{/,$!d"}));
try {
return objectMapper.readValue(Strings.join(info, '\n'), Document.class);
@@ -131,12 +142,12 @@ public class RemarkableApi {
}
}
public void upload(String path) {
exec(RMAPI_EXECUTABLE, "-ni", "put", path, rmStorageDir);
public void upload(Path path) {
exec(RMAPI_EXECUTABLE, "-ni", "put", path.toString(), rmStorageDir);
}
public void delete(String name) {
exec(RMAPI_EXECUTABLE, "-ni", "rm", rmStorageDir + name);
public void delete(String articleName) {
exec(RMAPI_EXECUTABLE, "-ni", "rm", rmStorageDir + articleName);
}
public void createDir(String path) {

2
src/main/java/nl/carosi/remarkablepocket/RemarkableService.java
View File

@@ -72,6 +72,6 @@ final class RemarkableService {
}
private void upload(Path path) {
rmapi.upload(path.toAbsolutePath().toString());
rmapi.upload(path);
}
}

1
src/main/java/nl/carosi/remarkablepocket/SyncApplication.java
View File

@@ -18,6 +18,7 @@ import pl.codeset.pocket.Pocket;
@EnableRetry
@Import({
ArticleDownloader.class,
ArticleValidator.class,
DownloadService.class,
EpubReader.class,
EpubWriter.class,

2
src/main/java/nl/carosi/remarkablepocket/SyncCommand.java
View File

@@ -19,7 +19,7 @@ import picocli.CommandLine.Option;
sortOptions = false,
usageHelpAutoWidth = true,
// TODO: Read from gradle.properties
version = "0.2.1",
version = "0.2.2",
mixinStandardHelpOptions = true)
class SyncCommand implements Callable<Integer> {
@Option(