Merge pull request #43 from CodeforLeipzig/update-elasticsearch

Update elasticsearch
2024-10-18 15:54:36 +02:00 · 2020-02-29 16:23:01 +01:00 · 2020-02-29 16:23:01 +01:00 · 4f455c0584
commit 4f455c0584
parent 2050e1c46d 4d4ed38dfc
8 changed files with 130 additions and 84 deletions
--- a/17
+++ b/17
@ -1,8 +1,19 @@
 FROM ruby:2.5.7
-RUN curl -sL https://deb.nodesource.com/setup_9.x | bash - && apt-get update && \
+RUN gem install bundler
+
+# Add google package repository for google chrome
+RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
+    && echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list
+
+RUN apt-get update && \
  apt-get install -y build-essential zlib1g-dev libsqlite3-dev nodejs npm \
-  libxml2-dev libxslt1-dev pkg-config \
-  libqtwebkit4 libqtwebkit-dev libqt4-dev xvfb
+  libxml2-dev libxslt1-dev pkg-config google-chrome-stable
+
+
+ENV DOCKERIZE_VERSION v0.6.1
+RUN curl -sSLO https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
+    && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
+    && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz

 RUN mkdir -p /app

--- a/6
+++ b/6
@ -31,8 +31,8 @@ gem 'omniauth'
 gem 'omniauth-browserid'

 gem 'elasticsearch', '~> 6'
-gem 'elasticsearch-model'
-gem 'elasticsearch-rails'
+gem 'elasticsearch-model', '~> 6'
+gem 'elasticsearch-rails', '~> 6'
 gem 'elasticsearch-dsl'

 gem 'leaflet-rails'
@ -68,7 +68,7 @@ group :test do
  gem 'simplecov', require: false
  gem 'database_cleaner'
  gem 'coveralls', require: false
-  gem 'capybara-webkit'
+  gem 'apparition'
  gem 'rails-controller-testing'
 end

--- a/Gemfile.lock
+++ b/Gemfile.lock
@ -38,21 +38,22 @@ GEM
      i18n (>= 0.7, < 2)
      minitest (~> 5.1)
      tzinfo (~> 1.1)
-    addressable (2.5.2)
-      public_suffix (>= 2.0.2, < 4.0)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    apparition (0.4.0)
+      capybara (~> 3.13, < 4)
+      websocket-driver (>= 0.6.5)
    arel (8.0.0)
    awesome_print (1.8.0)
    builder (3.2.3)
-    capybara (3.0.2)
+    capybara (3.30.0)
      addressable
      mini_mime (>= 0.1.3)
      nokogiri (~> 1.8)
      rack (>= 1.6.0)
      rack-test (>= 0.6.3)
-      xpath (~> 3.0)
-    capybara-webkit (1.15.0)
-      capybara (>= 2.3, < 4.0)
-      json
+      regexp_parser (~> 1.5)
+      xpath (~> 3.2)
    coderay (1.1.2)
    coffee-rails (4.2.2)
      coffee-script (>= 2.2.0)
@ -78,11 +79,11 @@ GEM
    elasticsearch-api (6.0.2)
      multi_json
    elasticsearch-dsl (0.1.5)
-    elasticsearch-model (5.0.0)
+    elasticsearch-model (6.1.0)
      activesupport (> 3)
      elasticsearch (> 1)
      hashie
-    elasticsearch-rails (5.0.2)
+    elasticsearch-rails (6.1.0)
    elasticsearch-transport (6.0.2)
      faraday
      multi_json
@ -138,14 +139,14 @@ GEM
    mime-types (3.1)
      mime-types-data (~> 3.2015)
    mime-types-data (3.2016.0521)
-    mini_mime (1.0.0)
-    mini_portile2 (2.3.0)
+    mini_mime (1.0.2)
+    mini_portile2 (2.4.0)
    minitest (5.11.3)
    multi_json (1.13.1)
    multipart-post (2.0.0)
    nio4r (2.3.0)
-    nokogiri (1.8.2)
-      mini_portile2 (~> 2.3.0)
+    nokogiri (1.10.7)
+      mini_portile2 (~> 2.4.0)
    omniauth (1.8.1)
      hashie (>= 3.4.6, < 3.6.0)
      rack (>= 1.6.2, < 3)
@ -158,10 +159,10 @@ GEM
      method_source (~> 0.9.0)
    pry-rails (0.3.6)
      pry (>= 0.10.4)
-    public_suffix (3.0.2)
+    public_suffix (4.0.3)
    puma (3.11.4)
-    rack (2.0.4)
-    rack-test (1.0.0)
+    rack (2.1.1)
+    rack-test (1.1.0)
      rack (>= 1.0, < 3)
    rails (5.1.6)
      actioncable (= 5.1.6)
@ -195,6 +196,7 @@ GEM
      thor (>= 0.18.1, < 2.0)
    rake (12.3.1)
    rdoc (4.3.0)
+    regexp_parser (1.6.0)
    rest-client (1.6.7)
      mime-types (>= 1.16)
    rspec-core (3.7.1)
@ -270,23 +272,23 @@ GEM
    websocket-driver (0.6.5)
      websocket-extensions (>= 0.1.0)
    websocket-extensions (0.1.3)
-    xpath (3.0.0)
+    xpath (3.2.0)
      nokogiri (~> 1.8)

 PLATFORMS
  ruby

 DEPENDENCIES
+  apparition
  awesome_print
  capybara
-  capybara-webkit
  coffee-rails
  coveralls
  database_cleaner
  elasticsearch (~> 6)
  elasticsearch-dsl
-  elasticsearch-model
-  elasticsearch-rails
+  elasticsearch-model (~> 6)
+  elasticsearch-rails (~> 6)
  factory_bot_rails
  faker
  foundation-rails (~> 5.5)
@ -316,3 +318,6 @@ DEPENDENCIES
  tzinfo-data
  uglifier
  validate_url
+
+BUNDLED WITH
+   2.1.4
--- a/app/models/paper.rb
+++ b/app/models/paper.rb
@ -21,43 +21,47 @@ class Paper < ActiveRecord::Base

  index_name ['srm', Rails.env, self.base_class.to_s.pluralize.underscore].join('_')

-  settings index: { 
-	  number_of_shards: 1,
-	  analysis: {
-		filter: {
-		  german_stop: {
-		    type: "stop",
-		    stopwords: "_german_"
-          }, 
-		  german_stemmer: {
-		    type: "stemmer",
-		    language: "light_german"
-          },
-		  decomp: {
-		    type: "decompound"
-          }
-        }, 
-		analyzer: {
-		  german: {
-		    tokenizer: "standard",
-		    filter: [
-		      "lowercase",
-		      "german_stop",
-		      "german_normalization",
-		      "german_stemmer",
-		      "decomp"
-		    ]
-          }
+  settings index: {
+    number_of_shards: 1,
+    analysis: {
+      filter: {
+        german_stop: {
+          type: "stop",
+          stopwords: "_german_"
+        },
+        german_stemmer: {
+          type: "stemmer",
+          language: "light_german"
+        },
+        german_decompounder: {
+          type: "hyphenation_decompounder",
+          word_list_path: "analysis/dictionary-de.txt",
+          hyphenation_patterns_path: "analysis/de_DR.xml",
+          only_longest_match: true,
+          min_subword_size: 4
+        },
+      },
+      analyzer: {
+        german: {
+          tokenizer: "standard",
+          filter: [
+            "lowercase",
+            "german_stop",
+              "german_decompounder",
+            "german_normalization",
+            "german_stemmer"
+          ]
        }
      }
+    }
    } do mappings dynamic: false do
-      indexes :name, type: :string, analyzer: "german"
-      indexes :content, type: :string, analyzer: "german"
-      indexes :resolution, type: :string, analyzer: "german"
-      indexes :reference, type: :string, index: :not_analyzed
-      indexes :paper_type, type: :string, index: :not_analyzed
-      indexes :published_at, type: :date, index: :not_analyzed
-      indexes :originator, type: :string, index: :not_analyzed
+      indexes :name, type: :text, analyzer: "german"
+      indexes :content, type: :text, analyzer: "german"
+      indexes :resolution, type: :text, analyzer: "german"
+      indexes :reference, type: :keyword, index: true
+      indexes :paper_type, type: :keyword, index: true
+      indexes :published_at, type: :date, index: true
+      indexes :originator, type: :keyword, index: true
    end
  end

--- a/bin/run-tests
+++ b/bin/run-tests
@ -1,3 +1,3 @@
 #!/usr/bin/env bash

-xvfb-run bundle exec rspec
+dockerize -wait $ELASTICSEARCH_URL -timeout 30s bundle exec rspec
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,22 +1,28 @@
-web:
-  build: .
-  volumes:
-    - .:/app
-  ports:
-    - "3000:3000"
-  links:
-    - elasticsearch
-  environment:
-    ELASTICSEARCH_URL: 'http://elasticsearch:9200'
-elasticsearch:
-  image: elasticsearch:5.4.3
-  command:
-    - sh
-    - -c
-    - "./bin/elasticsearch-plugin install http://xbib.org/repository/org/xbib/elasticsearch/plugin/elasticsearch-analysis-decompound/5.4.3.0/elasticsearch-analysis-decompound-5.4.3.0-plugin.zip;
-      ./bin/elasticsearch-plugin install https://github.com/royrusso/elasticsearch-HQ/zipball/master;
-       /docker-entrypoint.sh elasticsearch"
-  ports:
-    - "9200:9200"
-  volumes:
-    - .:/apps
+version: "3.7"
+volumes:
+  elasticsearch:
+services:
+  web:
+    build: .
+    volumes:
+      - .:/app
+    ports:
+      - "3000:3000"
+    links:
+      - elasticsearch
+    environment:
+      ELASTICSEARCH_URL: 'http://elasticsearch:9200'
+  elasticsearch:
+    build: ./docker/elasticsearch
+    environment:
+      - discovery.type=single-node
+    ports:
+      - "9200:9200"
+    volumes:
+      - elasticsearch:/usr/share/elasticsearch/data
+  elastichq:
+    image: elastichq/elasticsearch-hq
+    ports:
+      - "5000:5000"
+    links:
+      - elasticsearch
--- a/docker/elasticsearch/Dockerfile
+++ b/docker/elasticsearch/Dockerfile
@ -0,0 +1,8 @@
+FROM elasticsearch:6.8.6
+
+RUN \
+  mkdir -p /usr/share/elasticsearch/config/analysis && \
+  pushd /usr/share/elasticsearch/config/analysis && \
+  curl -sSLO https://github.com/uschindler/german-decompounder/raw/master/de_DR.xml && \
+  curl -sSLO https://github.com/uschindler/german-decompounder/raw/master/dictionary-de.txt && \
+  popd
--- a/spec/support/capybara.rb
+++ b/spec/support/capybara.rb
@ -1 +1,13 @@
-Capybara.javascript_driver = :webkit
+require 'capybara/apparition'
+Capybara.register_driver :apparition do |app|
+   Capybara::Apparition::Driver.new(
+     app,
+     headless: true,
+     browser_options: [ :no_sandbox, disable_features: 'VizDisplayCompositor']
+   )
+end
+
+Capybara.javascript_driver = :apparition
+
+# https://github.com/teamcapybara/capybara#setup
+Capybara.server = :puma, { Silent: true }