Also support importing likes

Author: SamSaffron

config.json.sample

@@ -3,5 +3,6 @@
   "api_key": "API_KEY",
   "topics_query_id": "QUERY_ID",
   "posts_query_id": "QUERY_ID",
+  "likes_query_id": "QUERY_ID",
   "domain": "YOUR_DISCOURSE.com"
 }

download_topics.rb

@@ -17,6 +17,7 @@
 API_USERNAME = config["api_username"]
 TOPIC_QUERY_ID = config["topics_query_id"]
 POST_QUERY_ID = config["posts_query_id"]
+LIKES_QUERY_ID = config["likes_query_id"]
 
 sqlite_conn = SQLite3::Database.new("dump.db")
 conn = MiniSql::Connection.get(sqlite_conn)
@@ -37,11 +38,17 @@ def run_report(query_id:, min_id: 0, limit:)
   request["Api-Username"] = API_USERNAME
 
   response = http.request(request)
+  if response.code != "200"
+    puts "Error: #{response.code} #{response.message}"
+    puts response.body
+    exit 1
+  end
+
   JSON.parse(response.body)
 end
 
 def create_schema(conn)
-  conn.exec <<-SQL
+  conn.exec <<~SQL
     CREATE TABLE IF NOT EXISTS topics (
       id INTEGER PRIMARY KEY,
       category,
@@ -52,15 +59,15 @@ def create_schema(conn)
     )
   SQL
 
-  conn.exec <<-SQL
+  conn.exec <<~SQL
     CREATE TABLE IF NOT EXISTS users(
       id INTEGER PRIMARY KEY,
       username,
       name
     )
   SQL
 
-  conn.exec <<-SQL
+  conn.exec <<~SQL
     CREATE TABLE IF NOT EXISTS posts(
       id INTEGER PRIMARY KEY,
       raw,
@@ -71,7 +78,21 @@ def create_schema(conn)
     )
   SQL
 
-  conn.exec("create index idxTopic on posts(topic_id,post_number)")
+  conn.exec <<~SQL
+    CREATE TABLE IF NOT EXISTS likes(
+      post_id,
+      user_id,
+      created_at
+    )
+  SQL
+
+  conn.exec(
+    "create unique index IF NOT EXISTS idxLikes on likes(post_id,user_id)"
+  )
+
+  conn.exec(
+    "create index IF NOT EXISTS idxTopic on posts(topic_id,post_number)"
+  )
 end
 
 def load_posts(conn, rows)
@@ -139,32 +160,74 @@ def load_users_from_json(conn, json)
   end
 end
 
-create_schema(conn)
+def load_likes(conn, json)
+  result = { highest_id: 0, likes_loaded: 0 }
+
+  conn.exec "BEGIN TRANSACTION"
+
+  json["rows"].each do |row|
+    conn.exec <<~SQL, *row
+      -- id: ?
+      INSERT OR IGNORE INTO likes(post_id, user_id, created_at)
+      VALUES (?, ?, ?)
+    SQL
+    result[:highest_id] = row[0] if row[0] > result[:highest_id]
+    result[:likes_loaded] += 1
+  end
 
-min_id = 0
-while true
-  response_data =
-    run_report(query_id: TOPIC_QUERY_ID, min_id: min_id, limit: 10_000)
+  conn.exec "COMMIT TRANSACTION"
+
+  result
+end
+
+def download_topics(conn)
+  min_id = 0
+  while true
+    response_data =
+      run_report(query_id: TOPIC_QUERY_ID, min_id: min_id, limit: 10_000)
+
+    load_users_from_json(conn, response_data)
+
+    result = load_topics(conn, response_data["rows"])
+    puts "Loaded #{result[:topics_loaded]} topics (highest id is #{result[:highest_id]})"
+
+    min_id = result[:highest_id]
+    break if result[:topics_loaded] == 0
+  end
+end
 
-  load_users_from_json(conn, response_data)
+def download_posts(conn)
+  min_id = 0
+  while true
+    response_data =
+      run_report(query_id: POST_QUERY_ID, min_id: min_id, limit: 10_000)
 
-  result = load_topics(conn, response_data["rows"])
-  puts "Loaded #{result[:topics_loaded]} topics (highest id is #{result[:highest_id]})"
+    load_users_from_json(conn, response_data)
 
-  min_id = result[:highest_id]
-  break if result[:topics_loaded] == 0
+    result = load_posts(conn, response_data["rows"])
+    puts "Loaded #{result[:posts_loaded]} posts (highest id is #{result[:highest_id]})"
+
+    min_id = result[:highest_id]
+    break if result[:posts_loaded] == 0
+  end
 end
 
-min_id = 0
-while true
-  response_data =
-    run_report(query_id: POST_QUERY_ID, min_id: min_id, limit: 10_000)
+def download_likes(conn)
+  min_id = 0
+  while true
+    response_data =
+      run_report(query_id: LIKES_QUERY_ID, min_id: min_id, limit: 10_000)
 
-  load_users_from_json(conn, response_data)
+    result = load_likes(conn, response_data)
 
-  result = load_posts(conn, response_data["rows"])
-  puts "Loaded #{result[:posts_loaded]} posts (highest id is #{result[:highest_id]})"
+    puts "Loaded #{result[:likes_loaded]} likes (highest id is #{result[:highest_id]})"
 
-  min_id = result[:highest_id]
-  break if result[:posts_loaded] == 0
+    min_id = result[:highest_id]
+    break if result[:likes_loaded] == 0
+  end
 end
+
+create_schema(conn)
+download_topics(conn)
+download_posts(conn)
+download_likes(conn)

import_db.rb

@@ -123,9 +123,45 @@ def import_posts(conn)
     end
 end
 
+def import_likes(conn)
+  bang
+  puts "creating likes..."
+
+  created = 0
+  conn
+    .query("SELECT post_id,user_id,created_at FROM likes order by id asc")
+    .each_slice(100) do |slice|
+      PostAction.transaction do
+        slice.each do |row|
+          if DB.query(
+               "SELECT 1 FROM post_actions where (post_id = ? and user_id = ? and post_action_type_id = ?)",
+               row.post_id,
+               row.user_id,
+               2
+             ).blank?
+            p =
+              PostAction.new(
+                post_id: row.post_id,
+                user_id: row.user_id,
+                post_action_type_id: 2,
+                created_at: row.created_at,
+                updated_at: row.created_at
+              )
+            p.save!(validate: false)
+            print "."
+          end
+
+          created += 1
+          puts "#{created} likes created" if created % 500 == 0
+        end
+      end
+    end
+end
+
 import_users(conn)
 import_topics(conn)
 import_posts(conn)
+import_likes(conn)
 
 Jobs::EnsureDbConsistency.new.execute(nil)
 Topic.reset_all_highest!

likes_query.sql

@@ -0,0 +1,22 @@
+-- [params]
+-- int :min_id = 0
+
+SELECT id, post_id pid, user_id uid, created_at
+FROM post_actions
+WHERE post_action_type_id = 2
+AND deleted_at is NULL
+AND id > :min_id
+AND post_id IN (
+  SELECT p.id
+   FROM topics t
+   JOIN posts p ON p.topic_id = t.id
+   JOIN categories c ON c.id = t.category_id
+   WHERE
+    NOT c.read_restricted
+     AND t.deleted_at IS NULL
+     AND p.deleted_at IS NULL
+     AND p.post_type = 1
+     AND NOT p.hidden
+)
+ORDER BY id ASC
+