Add importer model, validations to paper model

app/models/importer.rb

@@ -0,0 +1,3 @@
+class Importer < ActiveRecord::Base
+  validates :url, presence: true, uniqueness: true
+end

app/models/paper.rb

@@ -5,20 +5,28 @@ class Paper < ActiveRecord::Base
   include Elasticsearch::Model
   include Elasticsearch::Model::Callbacks
 
+  validates_presence_of :body, :content, :name, :originator, :paper_type, :published_at, :reference, :url
+  validates :url, uniqueness: true
+
   class << self
     def import_from_json(json_string)
+      old_count = count
       JSON.parse(json_string).each do |record|
         attributes = {
+          body: record['body'],
+          content: record['content'],
           name: record['name'],
-          url: record['url'],
-          reference: record['reference'],
-          paper_type: record['paper_type'],
+          resolution: record['resolution'],
           originator: record['originator'],
+          paper_type: record['paper_type'],
           published_at: record['published_at'],
+          reference: record['reference'],
+          url: record['url'],
         }
         record = find_or_initialize_by(url: attributes[:url])
-        record.update_attributes!(attributes)
-      end
+        record.update_attributes(attributes)
+      end
+      puts "Imported #{count - old_count} Papers!"
     end
   end
 end

db/migrate/20150413193656_create_papers.rb

@@ -5,6 +5,7 @@ class CreatePapers < ActiveRecord::Migration
       t.string :url
       t.string :reference
       t.string :name
+      t.string :body
       t.datetime :published_at
       t.datetime :scraped_at
       t.string :paper_type
@@ -14,5 +15,10 @@ class CreatePapers < ActiveRecord::Migration
 
       t.timestamps
     end
+
+    add_index(:papers, :reference)
+    add_index(:papers, :originator)
+    add_index(:papers, :body)
+    add_index(:papers, [:reference, :body], unique: true)
   end
 end

db/migrate/20150517152218_create_importers.rb

@@ -0,0 +1,8 @@
+class CreateImporters < ActiveRecord::Migration
+  def change
+    create_table :importers do |t|
+      t.string :url
+      t.timestamps null: false
+    end
+  end
+end

db/schema.rb

@@ -11,12 +11,19 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20150413193656) do
+ActiveRecord::Schema.define(version: 20150517152218) do
 
-  create_table "papers", force: true do |t|
+  create_table "importers", force: :cascade do |t|
+    t.string   "url"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  create_table "papers", force: :cascade do |t|
     t.string   "name"
     t.string   "url"
     t.string   "reference"
+    t.string   "body"
     t.datetime "published_at"
     t.datetime "scraped_at"
     t.string   "paper_type"
@@ -27,7 +34,12 @@ ActiveRecord::Schema.define(version: 20150413193656) do
     t.datetime "updated_at"
   end
 
-  create_table "users", force: true do |t|
+  add_index "papers", ["body"], name: "index_papers_on_body"
+  add_index "papers", ["originator"], name: "index_papers_on_originator"
+  add_index "papers", ["reference", "body"], name: "index_papers_on_reference_and_body", unique: true
+  add_index "papers", ["reference"], name: "index_papers_on_reference"
+
+  create_table "users", force: :cascade do |t|
     t.string   "email"
     t.datetime "created_at"
     t.datetime "updated_at"

lib/tasks/import_papers.rake

@@ -1,11 +1,10 @@
 namespace :import_papers do
-
   desc 'Import Paper records from CSV'
-  task :from_morph => :environment do |t, args|
+  task :from_morph => :environment do
     require 'open-uri'
-    api_key = ENV['MORPH_API_KEY']
-    uri = URI.parse "https://api.morph.io/ahx/city_council_leipzig_recent_papers/data.json?key=#{api_key}&query=select%20*%20from%20%27data%27"
-    puts "Download files from #{uri}…"
+    Importer.all.each do |importer|
+      uri = URI.parse(importer.url)
       Paper.import_from_json(uri.read)
     end
   end
+end

test/fixtures/importers.yml

@@ -0,0 +1,11 @@
+# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
+
+# This model initially had no columns defined.  If you add columns to the
+# model remove the '{}' from the fixture names and add the columns immediately
+# below each fixture, per the syntax in the comments below
+#
+one: {}
+# column: value
+#
+two: {}
+#  column: value

test/models/importer_test.rb

@@ -0,0 +1,7 @@
+require 'test_helper'
+
+class ImporterTest < ActiveSupport::TestCase
+  # test "the truth" do
+  #   assert true
+  # end
+end