Merge pull request #43 from CodeforLeipzig/update-elasticsearch

Update elasticsearch

Dockerfile

@@ -1,8 +1,19 @@
 FROM ruby:2.5.7
-RUN curl -sL https://deb.nodesource.com/setup_9.x | bash - && apt-get update && \
+RUN gem install bundler
+
+# Add google package repository for google chrome
+RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
+    && echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list
+
+RUN apt-get update && \
   apt-get install -y build-essential zlib1g-dev libsqlite3-dev nodejs npm \
-  libxml2-dev libxslt1-dev pkg-config \
+  libxml2-dev libxslt1-dev pkg-config google-chrome-stable
-  libqtwebkit4 libqtwebkit-dev libqt4-dev xvfb
+
+
+ENV DOCKERIZE_VERSION v0.6.1
+RUN curl -sSLO https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
+    && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
+    && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
 
 RUN mkdir -p /app
 

Gemfile

@@ -31,8 +31,8 @@ gem 'omniauth'
 gem 'omniauth-browserid'
 
 gem 'elasticsearch', '~> 6'
-gem 'elasticsearch-model'
+gem 'elasticsearch-model', '~> 6'
-gem 'elasticsearch-rails'
+gem 'elasticsearch-rails', '~> 6'
 gem 'elasticsearch-dsl'
 
 gem 'leaflet-rails'
@@ -68,7 +68,7 @@ group :test do
   gem 'simplecov', require: false
   gem 'database_cleaner'
   gem 'coveralls', require: false
-  gem 'capybara-webkit'
+  gem 'apparition'
   gem 'rails-controller-testing'
 end
 

Gemfile.lock

@@ -38,21 +38,22 @@ GEM
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    addressable (2.5.2)
+    addressable (2.7.0)
-      public_suffix (>= 2.0.2, < 4.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    apparition (0.4.0)
+      capybara (~> 3.13, < 4)
+      websocket-driver (>= 0.6.5)
     arel (8.0.0)
     awesome_print (1.8.0)
     builder (3.2.3)
-    capybara (3.0.2)
+    capybara (3.30.0)
       addressable
       mini_mime (>= 0.1.3)
       nokogiri (~> 1.8)
       rack (>= 1.6.0)
       rack-test (>= 0.6.3)
-      xpath (~> 3.0)
+      regexp_parser (~> 1.5)
-    capybara-webkit (1.15.0)
+      xpath (~> 3.2)
-      capybara (>= 2.3, < 4.0)
-      json
     coderay (1.1.2)
     coffee-rails (4.2.2)
       coffee-script (>= 2.2.0)
@@ -78,11 +79,11 @@ GEM
     elasticsearch-api (6.0.2)
       multi_json
     elasticsearch-dsl (0.1.5)
-    elasticsearch-model (5.0.0)
+    elasticsearch-model (6.1.0)
       activesupport (> 3)
       elasticsearch (> 1)
       hashie
-    elasticsearch-rails (5.0.2)
+    elasticsearch-rails (6.1.0)
     elasticsearch-transport (6.0.2)
       faraday
       multi_json
@@ -138,14 +139,14 @@ GEM
     mime-types (3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
-    mini_mime (1.0.0)
+    mini_mime (1.0.2)
-    mini_portile2 (2.3.0)
+    mini_portile2 (2.4.0)
     minitest (5.11.3)
     multi_json (1.13.1)
     multipart-post (2.0.0)
     nio4r (2.3.0)
-    nokogiri (1.8.2)
+    nokogiri (1.10.7)
-      mini_portile2 (~> 2.3.0)
+      mini_portile2 (~> 2.4.0)
     omniauth (1.8.1)
       hashie (>= 3.4.6, < 3.6.0)
       rack (>= 1.6.2, < 3)
@@ -158,10 +159,10 @@ GEM
       method_source (~> 0.9.0)
     pry-rails (0.3.6)
       pry (>= 0.10.4)
-    public_suffix (3.0.2)
+    public_suffix (4.0.3)
     puma (3.11.4)
-    rack (2.0.4)
+    rack (2.1.1)
-    rack-test (1.0.0)
+    rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rails (5.1.6)
       actioncable (= 5.1.6)
@@ -195,6 +196,7 @@ GEM
       thor (>= 0.18.1, < 2.0)
     rake (12.3.1)
     rdoc (4.3.0)
+    regexp_parser (1.6.0)
     rest-client (1.6.7)
       mime-types (>= 1.16)
     rspec-core (3.7.1)
@@ -270,23 +272,23 @@ GEM
     websocket-driver (0.6.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.3)
-    xpath (3.0.0)
+    xpath (3.2.0)
       nokogiri (~> 1.8)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  apparition
   awesome_print
   capybara
-  capybara-webkit
   coffee-rails
   coveralls
   database_cleaner
   elasticsearch (~> 6)
   elasticsearch-dsl
-  elasticsearch-model
+  elasticsearch-model (~> 6)
-  elasticsearch-rails
+  elasticsearch-rails (~> 6)
   factory_bot_rails
   faker
   foundation-rails (~> 5.5)
@@ -316,3 +318,6 @@ DEPENDENCIES
   tzinfo-data
   uglifier
   validate_url
+
+BUNDLED WITH
+   2.1.4

app/models/paper.rb

@@ -33,9 +33,13 @@ class Paper < ActiveRecord::Base
           type: "stemmer",
           language: "light_german"
         },
-		  decomp: {
+        german_decompounder: {
-		    type: "decompound"
+          type: "hyphenation_decompounder",
-          }
+          word_list_path: "analysis/dictionary-de.txt",
+          hyphenation_patterns_path: "analysis/de_DR.xml",
+          only_longest_match: true,
+          min_subword_size: 4
+        },
       },
       analyzer: {
         german: {
@@ -43,21 +47,21 @@ class Paper < ActiveRecord::Base
           filter: [
             "lowercase",
             "german_stop",
+              "german_decompounder",
             "german_normalization",
-		      "german_stemmer",
+            "german_stemmer"
-		      "decomp"
           ]
         }
       }
     }
     } do mappings dynamic: false do
-      indexes :name, type: :string, analyzer: "german"
+      indexes :name, type: :text, analyzer: "german"
-      indexes :content, type: :string, analyzer: "german"
+      indexes :content, type: :text, analyzer: "german"
-      indexes :resolution, type: :string, analyzer: "german"
+      indexes :resolution, type: :text, analyzer: "german"
-      indexes :reference, type: :string, index: :not_analyzed
+      indexes :reference, type: :keyword, index: true
-      indexes :paper_type, type: :string, index: :not_analyzed
+      indexes :paper_type, type: :keyword, index: true
-      indexes :published_at, type: :date, index: :not_analyzed
+      indexes :published_at, type: :date, index: true
-      indexes :originator, type: :string, index: :not_analyzed
+      indexes :originator, type: :keyword, index: true
     end
   end
 

bin/run-tests

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-xvfb-run bundle exec rspec
+dockerize -wait $ELASTICSEARCH_URL -timeout 30s bundle exec rspec

docker-compose.yml

@@ -1,4 +1,8 @@
-web:
+version: "3.7"
+volumes:
+  elasticsearch:
+services:
+  web:
     build: .
     volumes:
       - .:/app
@@ -8,15 +12,17 @@ web:
       - elasticsearch
     environment:
       ELASTICSEARCH_URL: 'http://elasticsearch:9200'
-elasticsearch:
+  elasticsearch:
-  image: elasticsearch:5.4.3
+    build: ./docker/elasticsearch
-  command:
+    environment:
-    - sh
+      - discovery.type=single-node
-    - -c
-    - "./bin/elasticsearch-plugin install http://xbib.org/repository/org/xbib/elasticsearch/plugin/elasticsearch-analysis-decompound/5.4.3.0/elasticsearch-analysis-decompound-5.4.3.0-plugin.zip;
-      ./bin/elasticsearch-plugin install https://github.com/royrusso/elasticsearch-HQ/zipball/master;
-       /docker-entrypoint.sh elasticsearch"
     ports:
       - "9200:9200"
     volumes:
-    - .:/apps
+      - elasticsearch:/usr/share/elasticsearch/data
+  elastichq:
+    image: elastichq/elasticsearch-hq
+    ports:
+      - "5000:5000"
+    links:
+      - elasticsearch

docker/elasticsearch/Dockerfile

@@ -0,0 +1,8 @@
+FROM elasticsearch:6.8.6
+
+RUN \
+  mkdir -p /usr/share/elasticsearch/config/analysis && \
+  pushd /usr/share/elasticsearch/config/analysis && \
+  curl -sSLO https://github.com/uschindler/german-decompounder/raw/master/de_DR.xml && \
+  curl -sSLO https://github.com/uschindler/german-decompounder/raw/master/dictionary-de.txt && \
+  popd

spec/support/capybara.rb

@@ -1 +1,13 @@
-Capybara.javascript_driver = :webkit
+require 'capybara/apparition'
+Capybara.register_driver :apparition do |app|
+   Capybara::Apparition::Driver.new(
+     app,
+     headless: true,
+     browser_options: [ :no_sandbox, disable_features: 'VizDisplayCompositor']
+   )
+end
+
+Capybara.javascript_driver = :apparition
+
+# https://github.com/teamcapybara/capybara#setup
+Capybara.server = :puma, { Silent: true }