mirror of
https://github.com/CodeforLeipzig/stadtratmonitor.git
synced 2024-12-22 23:53:15 +01:00
Add importer model, validations to paper model
This commit is contained in:
parent
568abb631f
commit
46de026234
8 changed files with 67 additions and 13 deletions
3
app/models/importer.rb
Normal file
3
app/models/importer.rb
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
class Importer < ActiveRecord::Base
|
||||||
|
validates :url, presence: true, uniqueness: true
|
||||||
|
end
|
|
@ -5,20 +5,28 @@ class Paper < ActiveRecord::Base
|
||||||
include Elasticsearch::Model
|
include Elasticsearch::Model
|
||||||
include Elasticsearch::Model::Callbacks
|
include Elasticsearch::Model::Callbacks
|
||||||
|
|
||||||
|
validates_presence_of :body, :content, :name, :originator, :paper_type, :published_at, :reference, :url
|
||||||
|
validates :url, uniqueness: true
|
||||||
|
|
||||||
class << self
|
class << self
|
||||||
def import_from_json(json_string)
|
def import_from_json(json_string)
|
||||||
|
old_count = count
|
||||||
JSON.parse(json_string).each do |record|
|
JSON.parse(json_string).each do |record|
|
||||||
attributes = {
|
attributes = {
|
||||||
|
body: record['body'],
|
||||||
|
content: record['content'],
|
||||||
name: record['name'],
|
name: record['name'],
|
||||||
url: record['url'],
|
resolution: record['resolution'],
|
||||||
reference: record['reference'],
|
|
||||||
paper_type: record['paper_type'],
|
|
||||||
originator: record['originator'],
|
originator: record['originator'],
|
||||||
|
paper_type: record['paper_type'],
|
||||||
published_at: record['published_at'],
|
published_at: record['published_at'],
|
||||||
|
reference: record['reference'],
|
||||||
|
url: record['url'],
|
||||||
}
|
}
|
||||||
record = find_or_initialize_by(url: attributes[:url])
|
record = find_or_initialize_by(url: attributes[:url])
|
||||||
record.update_attributes!(attributes)
|
record.update_attributes(attributes)
|
||||||
end
|
end
|
||||||
|
puts "Imported #{count - old_count} Papers!"
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -5,6 +5,7 @@ class CreatePapers < ActiveRecord::Migration
|
||||||
t.string :url
|
t.string :url
|
||||||
t.string :reference
|
t.string :reference
|
||||||
t.string :name
|
t.string :name
|
||||||
|
t.string :body
|
||||||
t.datetime :published_at
|
t.datetime :published_at
|
||||||
t.datetime :scraped_at
|
t.datetime :scraped_at
|
||||||
t.string :paper_type
|
t.string :paper_type
|
||||||
|
@ -14,5 +15,10 @@ class CreatePapers < ActiveRecord::Migration
|
||||||
|
|
||||||
t.timestamps
|
t.timestamps
|
||||||
end
|
end
|
||||||
|
|
||||||
|
add_index(:papers, :reference)
|
||||||
|
add_index(:papers, :originator)
|
||||||
|
add_index(:papers, :body)
|
||||||
|
add_index(:papers, [:reference, :body], unique: true)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
8
db/migrate/20150517152218_create_importers.rb
Normal file
8
db/migrate/20150517152218_create_importers.rb
Normal file
|
@ -0,0 +1,8 @@
|
||||||
|
class CreateImporters < ActiveRecord::Migration
|
||||||
|
def change
|
||||||
|
create_table :importers do |t|
|
||||||
|
t.string :url
|
||||||
|
t.timestamps null: false
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
18
db/schema.rb
18
db/schema.rb
|
@ -11,12 +11,19 @@
|
||||||
#
|
#
|
||||||
# It's strongly recommended that you check this file into your version control system.
|
# It's strongly recommended that you check this file into your version control system.
|
||||||
|
|
||||||
ActiveRecord::Schema.define(version: 20150413193656) do
|
ActiveRecord::Schema.define(version: 20150517152218) do
|
||||||
|
|
||||||
create_table "papers", force: true do |t|
|
create_table "importers", force: :cascade do |t|
|
||||||
|
t.string "url"
|
||||||
|
t.datetime "created_at", null: false
|
||||||
|
t.datetime "updated_at", null: false
|
||||||
|
end
|
||||||
|
|
||||||
|
create_table "papers", force: :cascade do |t|
|
||||||
t.string "name"
|
t.string "name"
|
||||||
t.string "url"
|
t.string "url"
|
||||||
t.string "reference"
|
t.string "reference"
|
||||||
|
t.string "body"
|
||||||
t.datetime "published_at"
|
t.datetime "published_at"
|
||||||
t.datetime "scraped_at"
|
t.datetime "scraped_at"
|
||||||
t.string "paper_type"
|
t.string "paper_type"
|
||||||
|
@ -27,7 +34,12 @@ ActiveRecord::Schema.define(version: 20150413193656) do
|
||||||
t.datetime "updated_at"
|
t.datetime "updated_at"
|
||||||
end
|
end
|
||||||
|
|
||||||
create_table "users", force: true do |t|
|
add_index "papers", ["body"], name: "index_papers_on_body"
|
||||||
|
add_index "papers", ["originator"], name: "index_papers_on_originator"
|
||||||
|
add_index "papers", ["reference", "body"], name: "index_papers_on_reference_and_body", unique: true
|
||||||
|
add_index "papers", ["reference"], name: "index_papers_on_reference"
|
||||||
|
|
||||||
|
create_table "users", force: :cascade do |t|
|
||||||
t.string "email"
|
t.string "email"
|
||||||
t.datetime "created_at"
|
t.datetime "created_at"
|
||||||
t.datetime "updated_at"
|
t.datetime "updated_at"
|
||||||
|
|
|
@ -1,11 +1,10 @@
|
||||||
namespace :import_papers do
|
namespace :import_papers do
|
||||||
|
|
||||||
desc 'Import Paper records from CSV'
|
desc 'Import Paper records from CSV'
|
||||||
task :from_morph => :environment do |t, args|
|
task :from_morph => :environment do
|
||||||
require 'open-uri'
|
require 'open-uri'
|
||||||
api_key = ENV['MORPH_API_KEY']
|
Importer.all.each do |importer|
|
||||||
uri = URI.parse "https://api.morph.io/ahx/city_council_leipzig_recent_papers/data.json?key=#{api_key}&query=select%20*%20from%20%27data%27"
|
uri = URI.parse(importer.url)
|
||||||
puts "Download files from #{uri}…"
|
|
||||||
Paper.import_from_json(uri.read)
|
Paper.import_from_json(uri.read)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
end
|
||||||
|
|
11
test/fixtures/importers.yml
vendored
Normal file
11
test/fixtures/importers.yml
vendored
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
|
||||||
|
|
||||||
|
# This model initially had no columns defined. If you add columns to the
|
||||||
|
# model remove the '{}' from the fixture names and add the columns immediately
|
||||||
|
# below each fixture, per the syntax in the comments below
|
||||||
|
#
|
||||||
|
one: {}
|
||||||
|
# column: value
|
||||||
|
#
|
||||||
|
two: {}
|
||||||
|
# column: value
|
7
test/models/importer_test.rb
Normal file
7
test/models/importer_test.rb
Normal file
|
@ -0,0 +1,7 @@
|
||||||
|
require 'test_helper'
|
||||||
|
|
||||||
|
class ImporterTest < ActiveSupport::TestCase
|
||||||
|
# test "the truth" do
|
||||||
|
# assert true
|
||||||
|
# end
|
||||||
|
end
|
Loading…
Reference in a new issue