Add method to import Oparl docs to Paper

This commit is contained in:
Lars Henrik Mai 2020-03-10 11:16:28 +01:00
parent 7a68ef0f26
commit 92f30ebaec
3 changed files with 76 additions and 0 deletions

View file

@ -97,6 +97,22 @@ class Paper < ActiveRecord::Base
puts "Imported #{count - old_count} Papers!" puts "Imported #{count - old_count} Papers!"
end end
def import_from_oparl(oparl_doc)
doc = JSON.parse(oparl_doc)
attributes = {
name: doc['name'],
body: doc['body'],
paper_type: doc['paperType'],
reference: doc['reference'],
url: doc['web'],
published_at: doc['modified'],
content: 'n.a.',
originator: doc['leipzig:originator']
}
record = find_or_initialize_by(url: attributes[:url])
record.update(attributes) ? record : nil
end
# use DSL to define search queries # use DSL to define search queries
# see https://github.com/elastic/elasticsearch-ruby/tree/master/elasticsearch-dsl # see https://github.com/elastic/elasticsearch-ruby/tree/master/elasticsearch-dsl
# and https://github.com/elastic/elasticsearch-rails/tree/master/elasticsearch-rails/lib/rails/templates # and https://github.com/elastic/elasticsearch-rails/tree/master/elasticsearch-rails/lib/rails/templates

34
spec/fixtures/oparl-paper.json vendored Normal file
View file

@ -0,0 +1,34 @@
{
"id": "https://ratsinfo.leipzig.de/bi/oparl/1.0/papers.asp?id=1014936",
"type": "https://schema.oparl.org/1.0/Paper",
"body": "https://ratsinfo.leipzig.de/bi/oparl/1.0/bodies.asp?id=2387",
"name": "Wiederherstellung des Haltepunktes an der Bahnstrecke und Zugänglichkeit zur Naherholung des Leipziger Ortsteils Knautnaundorf",
"reference": "VII-P-00651",
"paperType": "Petition",
"date": "2020-02-28",
"mainFile": {
"id": "https://ratsinfo.leipzig.de/bi/oparl/1.0/files.asp?dtyp=130&id=1612340",
"type": "https://schema.oparl.org/1.0/File",
"name": "Vorlage-Sammeldokument",
"fileName": "1612340.pdf",
"mimeType": "application/pdf",
"modified": "2020-03-01T22:05:35+01:00",
"size": 199412,
"accessUrl": "https://ratsinfo.leipzig.de/bi/oparl/1.0/download.asp?dtyp=130&id=1612340"
},
"consultation": [
{
"id": "https://ratsinfo.leipzig.de/bi/oparl/1.0/consultations.asp?typ=s&id=1051155",
"type": "https://schema.oparl.org/1.0/Consultation",
"organization": [
"https://ratsinfo.leipzig.de/bi/oparl/1.0/organizations.asp?typ=gr&id=2346"
],
"role": "Vorberatung",
"paper": "https://ratsinfo.leipzig.de/bi/oparl/1.0/papers.asp?id=1014936"
}
],
"web": "https://ratsinfo.leipzig.de/bi/vo020.asp?VOLFDNR=1014936",
"created": "2019-12-11T12:00:00+01:00",
"modified": "2020-02-28T12:49:29+01:00",
"leipzig:originator": "Petitionsausschuss"
}

View file

@ -51,4 +51,30 @@ RSpec.describe Paper do
it { should validate_length_of(:resolution).is_at_most(30_000) } it { should validate_length_of(:resolution).is_at_most(30_000) }
end end
context 'Paper.import_from_oparl' do
let(:oparl_paper_json) { File.read(Rails.root.join('spec/fixtures/oparl-paper.json')) }
it 'creates a database record for the document' do
expect { Paper.import_from_oparl(oparl_paper_json) }.to change { Paper.count }.by(1)
end
it 'populates the fields of the record correctly' do
source = JSON.parse(oparl_paper_json)
record = Paper.import_from_oparl(oparl_paper_json)
mapping = {
'name' => 'name',
'body' => 'body',
'paperType' => 'paper_type',
'reference' => 'reference',
'web' => 'url',
'modified' => 'published_at',
'leipzig:originator' => 'originator'
}
mapping.each do |source_field, target_field|
expect(source[source_field]).to be_present, "#{source_field} not present in source"
expect(record[target_field]).to eq(source[source_field])
end
end
end
end end