Using batch insert

README.md

@@ -15,7 +15,8 @@ See [LICENSE](/LICENSE)
 
 * Use `;DB_CLOSE_DELAY=-1` if you want to persist the in-memory database information over more than a single transaction.
   * `Database.connect("jdbc:h2:mem:test;DB_CLOSE_DELAY=-1", driver = "org.h2.Driver", user = "root", password = "")`
-
+* Use `?useSSL=false` to avoid SSL exceptions (for dev only!) on MySQL.
+  * `Database.connect("jdbc:mysql://localhost:3308/imdb?useSSL=false", driver = "com.mysql.jdbc.Driver", user = "root", password = "aRootPassword")`
 
 ## Author
 

docker-compose.yml

@@ -9,3 +9,7 @@ services:
       MYSQL_DATABASE: imdb
     ports:
       - "3308:3306"
+    volumes:
+      - my-datavolume:/var/lib/mysql
+volumes:
+  my-datavolume:

src/main/kotlin/dsl/TitleRatings.kt

@@ -1,19 +1,18 @@
 package dsl
 
-import org.jetbrains.exposed.dao.IntEntity
-import org.jetbrains.exposed.dao.IntEntityClass
-import org.jetbrains.exposed.dao.id.EntityID
-import org.jetbrains.exposed.dao.id.IntIdTable
 import org.jetbrains.exposed.sql.Column
 import org.jetbrains.exposed.sql.Table
 import org.jetbrains.exposed.sql.insert
+import org.jetbrains.exposed.sql.insertIgnore
 import tsv.Reader.NO_DATA
 
 object TitleRatings : Table(){
-    val tconst : Column<String> = varchar("tconst", 10).uniqueIndex()
+    val tconst : Column<String> = varchar("tconst", 10)//.uniqueIndex()
     val averageRating : Column<Float?> = float("averageRating").nullable()
     val numVotes : Column<Int?> = integer("numVotes").nullable()
 
+    override val primaryKey = PrimaryKey(tconst, name = "tconst")
+
     fun insertFromListString(values : List<String>){
         TitleRatings.insert {
                 it[tconst] = values[0]
@@ -22,11 +21,3 @@ object TitleRatings : Table(){
             }
     }
 }
-
-//class TitleRating(id: EntityID<Int>) : IntEntity(id) {
-//    companion object : IntEntityClass<TitleRating>(TitleRatings)
-//
-//    var tconst by TitleRatings.tconst
-//    var averageRating by TitleRatings.averageRating
-//    var numVotes by TitleRatings.numVotes
-//}

src/main/kotlin/loader/LoadImdb.kt

@@ -4,8 +4,8 @@ import org.jetbrains.exposed.sql.Database
 
 fun main(){
 
-    var db = Database.connect("jdbc:h2:mem:test;DB_CLOSE_DELAY=-1", driver = "org.h2.Driver", user = "root", password = "")
+//    var db = Database.connect("jdbc:h2:mem:test;DB_CLOSE_DELAY=-1", driver = "org.h2.Driver", user = "root", password = "")
-//    var db = Database.connect("jdbc:mysql://localhost:3308/imdb", driver = "com.mysql.jdbc.Driver", user = "root", password = "aRootPassword")
+    var db = Database.connect("jdbc:mysql://localhost:3308/imdb?useSSL=false&allowPublicKeyRetrieval=true", driver = "com.mysql.jdbc.Driver", user = "root", password = "aRootPassword")
     println("Running loader")
 
     val titleRatings = TitleRatingsLoader(db)

src/main/kotlin/loader/TitleRatingsLoader.kt

@@ -3,8 +3,10 @@ package loader
 import dsl.TitleRatings
 import org.jetbrains.exposed.sql.Database
 import org.jetbrains.exposed.sql.SchemaUtils
+import org.jetbrains.exposed.sql.batchInsert
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
+import tsv.Reader
 import java.io.File
 import java.util.concurrent.atomic.AtomicInteger
 
@@ -15,7 +17,7 @@ class TitleRatingsLoader(private val db: Database) {
         transaction(db) { SchemaUtils.create (TitleRatings) }
     }
 
-    fun loadData(){
+    fun loadDataMultiInsert(){
         val nameBasicsReader = File("./datasets/title.ratings.tsv").bufferedReader()
         nameBasicsReader.readLine()
 
@@ -33,6 +35,28 @@ class TitleRatingsLoader(private val db: Database) {
         println("Done loading title ratings!")
     }
 
+    fun loadData(){
+        val nameBasicsReader = File("./datasets/title.ratings.tsv").bufferedReader()
+        nameBasicsReader.readLine()
+
+        println("Title.Ratings loaded")
+
+        val loader = AtomicInteger()
+        transaction {
+            val lines = nameBasicsReader.readLines()
+            TitleRatings.batchInsert(lines){
+                val items = it.split("\t")
+                this[TitleRatings.tconst] = items[0]
+                this[TitleRatings.averageRating] = if (items[1] != Reader.NO_DATA) items[1].toFloat() else null
+                this[TitleRatings.numVotes] = if (items[2] != Reader.NO_DATA) items[2].toInt() else null
+                if (loader.incrementAndGet() % 10000 == 0) println(it)
+            }
+
+        }
+
+        println("Done loading title ratings!")
+    }
+
 
     fun showSome(){
         transaction(db) {