jlengrand/exposed-imdb
1
0
Fork 0
mirror of https://github.com/jlengrand/exposed-imdb.git synced 2026-03-10 00:11:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

last commit

Browse Source
This commit is contained in:
Julien Lengrand-Lambert
2020-05-29 15:43:18 +02:00
parent 3f41796ae8
commit cf7839c8a7
8 changed files with 70 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.idea/misc.xml generated
View File

@@ -8,6 +8,9 @@
<component name="FrameworkDetectionExcludesConfiguration">
<file type="web" url="file://$PROJECT_DIR$" />
</component>
<component name="JavaScriptSettings">
<option name="languageLevel" value="ES6" />
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_11" default="true" project-jdk-name="11" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>

59
README.md
View File

@@ -1,5 +1,7 @@
# Exposed Imdb
**NOTE : I am archiving this project because imdb [closed access to its datasets](https://developer.imdb.com/) and I could not find any other complete equivalents online. Sorry!**
A simple project to learn more about [exposed](https://github.com/JetBrains/Exposed), using the [imdb datasets](https://datasets.imdbws.com/).
Please note that if you use this project, you have to comply with the [Imbd license](https://www.imdb.com/interfaces/).
@@ -10,61 +12,16 @@ In short, [CC BY-NC-SA 4.0](https://tldrlegal.com/license/creative-commons-attri
See [LICENSE](/LICENSE)
## Running the app
```
$ docker-compose up to get the database running
```
## Learnings
* Use `;DB_CLOSE_DELAY=-1` if you want to persist the in-memory database information over more than a single transaction.
* `Database.connect("jdbc:h2:mem:test;DB_CLOSE_DELAY=-1", driver = "org.h2.Driver", user = "root", password = "")`
* Use `?useSSL=false` to avoid SSL exceptions (for dev only!) on MySQL.
* `Database.connect("jdbc:mysql://localhost:3308/imdb?useSSL=false", driver = "com.mysql.jdbc.Driver", user = "root", password = "aRootPassword")`
* Use `rewriteBatchedStatements=true` when inserting large volumes of data to have your driver rewrite your query
* `.map` keeps stack of memory while `for` loop doesn't? I get a OME when running with map
* reason to use partitions
* dsl loading copy paste prone
* Cross Join
* references
```
1
[dsl.Titles.primaryTitle=The Lego Batman Movie, dsl.Titles.titleType=movie, dsl.Ratings.averageRating=7.3, dsl.Ratings.numVotes=123790]
dsl.Titles.primaryTitle=The Lego Batman Movie, dsl.Titles.titleType=movie, dsl.Ratings.averageRating=7.3, dsl.Ratings.numVotes=123790
Ratings query took : 3170
1
[dsl.Titles.primaryTitle=The Lego Batman Movie, dsl.Titles.titleType=movie, dsl.Ratings.averageRating=7.3, dsl.Ratings.numVotes=123790]
dsl.Titles.primaryTitle=The Lego Batman Movie, dsl.Titles.titleType=movie, dsl.Ratings.averageRating=7.3, dsl.Ratings.numVotes=123790
Ratings query took : 2159
object Titles : IntIdTable() {
val tconst: Column<String> = (varchar("tconst", 10) references Ratings.tconst).uniqueIndex()
val titleType: Column<String> = varchar("titleType", 50) // TODO: Own Table?
val primaryTitle: Column<String> = varchar("primaryTitle", 500)
val originalTitle: Column<String> = varchar("originalTitle", 500)
val isAdult: Column<Boolean> = bool("isAdult")
val startYear: Column<Int?> = integer("startYear").nullable()
val endYear: Column<Int?> = integer("endYear").nullable()
val runtimeMinutes: Column<Long?> = long("runtimeMinutes").nullable()
val genres: Column<String> = varchar("genres", 50) // TODO: Own Table?
}
object Ratings : IntIdTable(){
val tconst : Column<String> = varchar("tconst", 10).uniqueIndex()
val averageRating : Column<Float?> = float("averageRating").nullable()
val numVotes : Column<Int?> = integer("numVotes").nullable()
}
println("Ratings query took : ${measureTimeMillis() {
transaction(db) {
var result = (Titles innerJoin Ratings).slice(Titles.primaryTitle, Titles.titleType, Ratings.averageRating, Ratings.numVotes).select {
((Titles.primaryTitle like "The Lego Batman Movie") and (Titles.titleType like "movie")
and Titles.tconst.eq(Ratings.tconst))
}.toList()
println(result.size)
println(result)
println(result.first())
}
}}");
```
## Author

84
src/main/kotlin/nl/lengrand/imdb/HeavyQueries.kt
View File

@@ -1,9 +1,14 @@
package nl.lengrand.imdb.queries
package nl.lengrand.imdb
import nl.lengrand.imdb.dsl.Names
//-XX:StartFlightRecording=duration=6000s,filename=myheavyrecording.jfr
import nl.lengrand.imdb.dsl.Ratings
import nl.lengrand.imdb.dsl.Titles
import nl.lengrand.imdb.dsl.Titles.primaryTitle
import org.jetbrains.exposed.sql.Database
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.select
import org.jetbrains.exposed.sql.transactions.transaction
import kotlin.system.measureTimeMillis
fun main(){
val db = Database.connect(
@@ -13,64 +18,21 @@ fun main(){
password = ""
)
// Find a actors with a given name
transaction(db) {
var result = Names.select { Names.primaryName like "%cotillard" }.toList()
println(result.size)
repeat(100){
println("Ratings query took : ${measureTimeMillis() {
transaction(db) {
var result = (Titles crossJoin Ratings).slice(Titles.primaryTitle, Titles.titleType, Ratings.averageRating, Ratings.numVotes).select {
((Titles.primaryTitle like "%batman%") and (Titles.titleType like "movie")
and Titles.tconst.eq(Ratings.tconst))
}
.orderBy(Ratings.averageRating)
.toList()
println(result.size)
println(result)
println(result.last()[primaryTitle])
}
}}");
}
// Trying with DAO
// Find all the movies someone played in
// Not possible currently, need join table
// Find all the batman movies
// transaction(db) {
// var result = Titles.select { Titles.primaryTitle like "%batman%"}.toList()
// println(result.size)
// println(result)
// println(result.first())
// }
// Find the rating for a specific movie
// println("Ratings query took : ${measureTimeMillis() {
// transaction(db) {
// var result = (Titles crossJoin Ratings).slice(Titles.primaryTitle, Titles.titleType, Ratings.averageRating, Ratings.numVotes).select {
// ((Titles.primaryTitle like "The Lego Batman Movie") and (Titles.titleType like "movie")
// and Titles.tconst.eq(Ratings.tconst))
// }.toList()
//
// println(result.size)
// println(result)
// println(result.first())
// }
// }}");
// println("Ratings query took : ${measureTimeMillis() {
// transaction(db) {
// var result = (Titles innerJoin Ratings).slice(Titles.primaryTitle, Titles.titleType, Ratings.averageRating, Ratings.numVotes).select {
// ((Titles.primaryTitle like "The Lego Batman Movie") and (Titles.titleType like "movie")
// and Titles.tconst.eq(Ratings.tconst))
// }.toList()
//
// println(result.size)
// println(result)
// println(result.first())
// }
// }}");
// println("Ratings query took : ${measureTimeMillis() {
// transaction(db) {
// var result = (Titles innerJoin Ratings).slice(Titles.primaryTitle, Titles.titleType, Ratings.averageRating, Ratings.numVotes).select {
// ((Titles.primaryTitle like "%batman%") and (Titles.titleType like "movie")
// and Titles.tconst.eq(Ratings.tconst))
// }.orderBy(Ratings.averageRating)
// .toList()
//
// println(result.size)
// println(result)
// println(result.last())
// }
// }}");
}

22
src/main/kotlin/nl/lengrand/imdb/LoadImdb.kt
View File

@@ -1,5 +1,8 @@
package nl.lengrand.imdb
import nl.lengrand.imdb.loader.NamesLoader
import nl.lengrand.imdb.loader.RatingsLoader
import nl.lengrand.imdb.loader.TitleLoader
import org.jetbrains.exposed.sql.Database
import org.jetbrains.exposed.sql.SchemaUtils
import org.jetbrains.exposed.sql.transactions.transaction
@@ -8,32 +11,30 @@ import kotlin.system.measureTimeMillis
fun main() {
// var db = Database.connect("jdbc:h2:mem:test;DB_CLOSE_DELAY=-1", driver = "org.h2.Driver", user = "root", password = "")
// var db = Database.connect("jdbc:mysql://localhost:3308?useSSL=false&allowPublicKeyRetrieval=true&rewriteBatchedStatements=true", driver = "com.mysql.jdbc.Driver", user = "root", password = "aRootPassword")
var db = Database.connect(
"jdbc:mysql://localhost:3306?useSSL=false&allowPublicKeyRetrieval=true&rewriteBatchedStatements=true",
"jdbc:mysql://localhost:3308?useSSL=false&allowPublicKeyRetrieval=true&rewriteBatchedStatements=true",
driver = "com.mysql.jdbc.Driver",
user = "root",
password = ""
password = "aRootPassword"
)
// transaction(db) { SchemaUtils.dropDatabase("imdb") }
transaction(db) { SchemaUtils.dropDatabase("imdb") }
transaction(db) { SchemaUtils.createDatabase("imdb") }
db = Database.connect(
"jdbc:mysql://localhost:3306/imdb?useSSL=false&allowPublicKeyRetrieval=true&rewriteBatchedStatements=true",
"jdbc:mysql://localhost:3308/imdb?useSSL=false&allowPublicKeyRetrieval=true&rewriteBatchedStatements=true",
driver = "com.mysql.jdbc.Driver",
user = "root",
password = ""
password = "aRootPassword"
)
// db = Database.connect("jdbc:mysql://localhost:3308/imdb?useSSL=false&allowPublicKeyRetrieval=true&rewriteBatchedStatements=true", driver = "com.mysql.jdbc.Driver", user = "root", password = "aRootPassword")
val time = measureTimeMillis() { // duplication
// TODO: Avoid overwriting
// RatingsLoader.load(db)
// TitleLoader.load(db)
// NamesLoader.load(db)
RatingsLoader.load(db)
TitleLoader.load(db)
NamesLoader.load(db)
// CrewsLoader.load(db)
// EpisodesLoader.load(db)
// PrincipalsLoader.load(db) // Time was : 21 minutes 23 seconds
@@ -41,5 +42,4 @@ fun main() {
}
println("---- Total time")
println("Total Time was : ${time / 1000 / 60 } minutes ${time / 1000 % 60 } seconds")
}

1
src/main/kotlin/nl/lengrand/imdb/api/ImdbServer.kt
View File

@@ -6,6 +6,5 @@ fun main(){
Vertx.vertx().createHttpServer()
.requestHandler { request ->
request.response().end("Imdb Server")
}.listen(8080)
}

8
src/main/kotlin/nl/lengrand/imdb/api/NamesApi.kt
View File

@@ -1,12 +1,16 @@
package nl.lengrand.imdb.api
import nl.lengrand.imdb.dsl.Names
import org.jetbrains.exposed.sql.Database
import org.jetbrains.exposed.sql.select
import org.jetbrains.exposed.sql.transactions.transaction
class NamesApi {
companion object{
fun get(searchTerm: String){
Names.select { Names.primaryName like "%cotillard" }.toList()
fun get(db: Database, searchTerm: String){
transaction(db) {
Names.select { Names.primaryName like searchTerm }.toList()
}
}
}
}

22
src/main/kotlin/nl/lengrand/imdb/dsl/Names.kt
View File

@@ -1,5 +1,8 @@
package nl.lengrand.imdb.dsl
import org.jetbrains.exposed.dao.IntEntity
import org.jetbrains.exposed.dao.IntEntityClass
import org.jetbrains.exposed.dao.id.EntityID
import org.jetbrains.exposed.dao.id.IntIdTable
import org.jetbrains.exposed.sql.Table
@@ -11,10 +14,21 @@ object Names : IntIdTable(){
val primaryProfession = varchar("primaryProfession", 500)
}
object KnownForTitles : Table(){
val id = integer("id").autoIncrement() // Column<Int>
class Name(id: EntityID<Int>) : IntEntity(id) {
companion object : IntEntityClass<Name>(Names)
var nconst by Names.nconst
var primaryName by Names.primaryName
var birthYear by Names.birthYear
var deathYear by Names.deathYear
var primaryProfession by Names.primaryProfession
}
object KnownForTitles : IntIdTable(){
// val id = integer("id").autoIncrement() // Column<Int>
val nconst = varchar("nconst", 10).index("knownfor_names")
val tconst = (varchar("tconst", 10) references Titles.tconst).index("knownfor_titles")
override val primaryKey = PrimaryKey(id)
}
// override val primaryKey = PrimaryKey(id)
}

6
src/main/kotlin/nl/lengrand/imdb/queries/RandomQueries.kt
View File

@@ -1,13 +1,9 @@
package nl.lengrand.imdb
package nl.lengrand.imdb.queries
import nl.lengrand.imdb.dsl.Names
import nl.lengrand.imdb.dsl.Ratings
import nl.lengrand.imdb.dsl.Titles
import org.jetbrains.exposed.sql.Database
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.select
import org.jetbrains.exposed.sql.transactions.transaction
import kotlin.system.measureTimeMillis
fun main(){
val db = Database.connect(