diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 00000000..81e00690
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,7 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+ directory: "/"
+ schedule:
+ interval: weekly
+ open-pull-requests-limit: 10
diff --git a/.gitignore b/.gitignore
index ce1e7efa..3799e7bb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,6 @@
tmp/**
public/system/**
.idea/**
+.byebug_history
+pg_data/
+.vscode/
diff --git a/.ruby-version b/.ruby-version
index c1026d29..0b4a2023 100644
--- a/.ruby-version
+++ b/.ruby-version
@@ -1 +1 @@
-ruby-2.3.1
+ruby-2.4.6
diff --git a/.travis.yml b/.travis.yml
index 597b83e6..e46178f8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,20 +9,20 @@ addons:
services:
- postgresql
- redis
+ - elasticsearch
language: ruby
rvm:
- - 2.3.1
+ - 2.4.6
cache: bundler
env:
- RAILS_ENV=test
+ - SIDEKIQ_SOURCE=https://gems.contribsys.com
before_install:
- - curl -O https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.2.1.deb && sudo dpkg -i --force-confnew elasticsearch-1.2.1.deb && sudo service elasticsearch restart
- psql -c 'create database travis_ci_test;' -U postgres
- cp config/database.travis.yml config/database.yml
- git --git-dir=spec/fixtures/repository.git init --bare
+ - bundle config gems.contribsys.com $BUNDLE_GEMS__CONTRIBSYS__COM
script:
- bundle exec rake db:test:prepare
- - bundle exec rake environment elasticsearch:import:model FORCE=y CLASS=Commit
- - bundle exec rake environment elasticsearch:import:model FORCE=y CLASS=Key
- - bundle exec rake environment elasticsearch:import:model FORCE=y CLASS=Translation
+ - bundle exec rake chewy:reset
- bundle exec rspec
diff --git a/Brewfile b/Brewfile
index fb2f44b7..cb1d310f 100644
--- a/Brewfile
+++ b/Brewfile
@@ -1,6 +1,7 @@
brew 'cmake'
brew 'libarchive'
brew 'libgit2'
-brew 'postgresql'
+brew 'postgresql@9.4'
brew 'qt'
brew 'redis'
+brew 'elasticsearch@5.6'
diff --git a/Dockerfile b/Dockerfile
index 6dc217e2..e2318847 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,10 +1,15 @@
-FROM ruby:2.3.1
+FROM ruby:2.4.6
ARG BUNDLE_GEMS__CONTRIBSYS__COM
ENV APP_HOME /app
RUN mkdir $APP_HOME
WORKDIR $APP_HOME
+COPY square_primary_certificate_authority_g2.crt /usr/local/share/ca-certificates/
+COPY square_service_authority_g2.crt /usr/local/share/ca-certificates/
+RUN update-ca-certificates
+
+# RUN printf "deb http://archive.debian.org/debian/ jessie main\ndeb-src http://archive.debian.org/debian/ jessie main\ndeb http://security.debian.org jessie/updates main\ndeb-src http://security.debian.org jessie/updates main" > /etc/apt/sources.list
RUN apt-get update -qq \
&& apt-get install -y build-essential nodejs libarchive-dev libpq-dev \
postgresql-client cmake tidy git \
diff --git a/Gemfile b/Gemfile
index e9bdc31f..1a187d84 100644
--- a/Gemfile
+++ b/Gemfile
@@ -1,7 +1,8 @@
-source 'https://rubygems.org'
git_source(:github) { |repo| "https://github.com/#{repo}.git" }
-ruby '2.3.1'
+source 'https://rubygems.org'
+
+ruby '2.4.6'
# FRAMEWORK
gem 'rails', '4.2.10'
@@ -15,6 +16,7 @@ gem 'rubyzip'
# AUTHENTICATION
gem 'devise'
+gem 'devise_security_extension'
# MODELS
gem 'pg', '< 1.0'
@@ -29,11 +31,8 @@ gem 'after-commit-on-action'
gem 'postgres_ext'
gem 'active_record_union'
-# ElASTICSEARCH
-gem 'elasticsearch', '< 6.0'
-gem 'elasticsearch-rails'
-gem 'elasticsearch-model'
-gem 'elasticsearch-dsl'
+# ELASTICSEARCH
+gem 'chewy'
# VIEWS
gem 'jquery-rails'
@@ -46,8 +45,9 @@ gem 'slim-rails'
gem 'autoprefixer-rails'
# UTILITIES
+gem 'bundler'
gem 'json'
-gem 'rugged', github: 'brandonweeks/rugged', tag: 'v0.24.0-square0', submodules: true
+gem 'rugged', github: 'squarit/rugged', tag: 'v0.27.2-square0', submodules: true
gem 'coffee-script'
gem 'unicode_scanner'
gem 'httparty'
@@ -59,6 +59,7 @@ gem 'aws-sdk', '< 3'
gem 'execjs'
gem 'safemode'
gem 'pivot_table'
+gem 'sentry-raven'
# IMPORTING
gem 'nokogiri'
@@ -80,9 +81,10 @@ gem 'uea-stemmer'
gem 'faker'
# BACKGROUND JOBS
-source "https://gems.contribsys.com/" do
+source 'https://gems.contribsys.com' do
gem 'sidekiq-pro', '= 3.4.5'
end
+
gem 'sidekiq-failures', github: 'mhfs/sidekiq-failures'
gem 'sinatra', require: nil
gem 'whenever', require: nil
@@ -97,6 +99,14 @@ gem 'sass-rails'
gem 'coffee-rails'
gem 'uglifier'
gem 'hogan_assets', github: 'rubenrails/hogan_assets', branch: 'fix_for_sprockets' # sprockets 3 compatibility
+gem 'awesome_print'
+
+source 'https://rails-assets.org' do
+ gem 'rails-assets-raven-js'
+end
+
+# METRICS
+gem 'newrelic_rpm'
group :development do
gem 'capistrano'
@@ -113,6 +123,13 @@ group :development do
gem 'binding_of_caller'
end
+group :dummy do
+ # needed for deployment
+ gem 'rbnacl', '< 5.0'
+ gem 'rbnacl-libsodium'
+ gem 'bcrypt_pbkdf', '< 2.0'
+end
+
group :test do
gem 'rspec-rails'
gem 'factory_bot_rails'
@@ -121,3 +138,5 @@ group :test do
gem 'database_cleaner'
gem 'capybara'
end
+
+gem "message_format", "~> 0.0.5"
diff --git a/Gemfile.lock b/Gemfile.lock
index d04cbda9..77830f77 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,11 +1,3 @@
-GIT
- remote: https://github.com/brandonweeks/rugged.git
- revision: cdabe1e11ab85428c380df21cbd3117ca232a865
- tag: v0.24.0-square0
- submodules: true
- specs:
- rugged (0.24.0)
-
GIT
remote: https://github.com/mhfs/sidekiq-failures.git
revision: ade3c0035e8b7df0821a71328fe31dc706226aeb
@@ -23,6 +15,14 @@ GIT
sprockets (>= 2.0.3)
tilt (>= 1.3.3)
+GIT
+ remote: https://github.com/squarit/rugged.git
+ revision: e521d4903a624f119e77c7e66cf24a34687f9b33
+ tag: v0.27.2-square0
+ submodules: true
+ specs:
+ rugged (0.27.2)
+
GIT
remote: https://github.com/thoughtbot/paperclip.git
revision: 523bd46c768226893f23889079a7aa9c73b57d68
@@ -54,6 +54,7 @@ GIT
GEM
remote: https://rubygems.org/
remote: https://gems.contribsys.com/
+ remote: https://rails-assets.org/
specs:
CFPropertyList (2.3.6)
actionmailer (4.2.10)
@@ -101,6 +102,7 @@ GEM
arel (6.0.4)
autoprefixer-rails (7.2.4)
execjs
+ awesome_print (1.8.0)
aws-sdk (2.10.115)
aws-sdk-resources (= 2.10.115)
aws-sdk-core (2.10.115)
@@ -110,6 +112,7 @@ GEM
aws-sdk-core (= 2.10.115)
aws-sigv4 (1.0.2)
bcrypt (3.1.11)
+ bcrypt_pbkdf (1.0.1)
better_errors (2.4.0)
coderay (>= 1.0.0)
erubi (>= 1.0.0)
@@ -123,6 +126,7 @@ GEM
sass (>= 3.5.2)
builder (3.2.3)
byebug (10.0.1)
+ camertron-eprun (1.1.1)
capistrano (3.10.1)
airbrussh (>= 1.0.0)
i18n
@@ -147,7 +151,12 @@ GEM
rack (>= 1.0.0)
rack-test (>= 0.5.4)
xpath (>= 2.0, < 4.0)
+ chewy (5.0.0)
+ activesupport (>= 4.0)
+ elasticsearch (>= 2.0.0)
+ elasticsearch-dsl
chronic (0.10.2)
+ cldr-plurals-runtime-rb (1.0.1)
climate_control (0.2.0)
cocaine (0.5.8)
climate_control (>= 0.0.3, < 1.0)
@@ -172,21 +181,19 @@ GEM
railties (>= 4.1.0, < 5.2)
responders
warden (~> 1.2.3)
+ devise_security_extension (0.9.2)
+ devise (>= 2.0.0)
+ rails (>= 3.1.1)
diff-lcs (1.3)
dropzonejs-rails (0.8.2)
rails (> 3.1)
- elasticsearch (5.0.4)
- elasticsearch-api (= 5.0.4)
- elasticsearch-transport (= 5.0.4)
- elasticsearch-api (5.0.4)
+ elasticsearch (6.1.0)
+ elasticsearch-api (= 6.1.0)
+ elasticsearch-transport (= 6.1.0)
+ elasticsearch-api (6.1.0)
multi_json
- elasticsearch-dsl (0.1.5)
- elasticsearch-model (5.0.2)
- activesupport (> 3)
- elasticsearch (~> 5)
- hashie
- elasticsearch-rails (5.0.2)
- elasticsearch-transport (5.0.4)
+ elasticsearch-dsl (0.1.6)
+ elasticsearch-transport (6.1.0)
faraday
multi_json
erubi (1.7.0)
@@ -223,7 +230,6 @@ GEM
guard (~> 2.1)
guard-compat (~> 1.1)
rspec (>= 2.99.0, < 4.0)
- hashie (3.5.7)
html_validation (1.1.5)
httparty (0.15.6)
multi_xml (>= 0.5.2)
@@ -258,6 +264,8 @@ GEM
lumberjack (1.0.13)
mail (2.7.0)
mini_mime (>= 0.1.1)
+ message_format (0.0.5)
+ twitter_cldr (~> 4.0)
method_source (0.9.0)
mime-types (3.1)
mime-types-data (~> 3.2015)
@@ -274,6 +282,7 @@ GEM
net-scp (1.2.1)
net-ssh (>= 2.6.5)
net-ssh (4.2.0)
+ newrelic_rpm (6.3.0.355)
nokogiri (1.10.3)
mini_portile2 (~> 2.4.0)
notiffany (0.1.1)
@@ -316,6 +325,7 @@ GEM
bundler (>= 1.3.0, < 2.0)
railties (= 4.2.10)
sprockets-rails
+ rails-assets-raven-js (3.20.1)
rails-deprecated_sanitizer (1.0.3)
activesupport (>= 4.2.0.alpha)
rails-dom-testing (1.0.9)
@@ -331,10 +341,14 @@ GEM
activesupport (= 4.2.10)
rake (>= 0.8.7)
thor (>= 0.18.1, < 2.0)
- rake (12.3.0)
+ rake (12.3.1)
rb-fsevent (0.10.2)
rb-inotify (0.9.10)
ffi (>= 0.5.0, < 2)
+ rbnacl (4.0.2)
+ ffi
+ rbnacl-libsodium (1.0.16)
+ rbnacl (>= 3.0.1)
redcarpet (3.4.0)
redis (3.3.5)
redis-actionpack (5.0.2)
@@ -408,6 +422,8 @@ GEM
sprockets (>= 2.8, < 4.0)
sprockets-rails (>= 2.0, < 4.0)
tilt (>= 1.1, < 3)
+ sentry-raven (2.7.1)
+ faraday (>= 0.7.6, < 1.0)
sexp_processor (4.10.0)
shellany (0.0.1)
sidekiq (4.2.10)
@@ -457,6 +473,10 @@ GEM
actionpack (>= 3.1)
jquery-rails
railties (>= 3.1)
+ twitter_cldr (4.4.4)
+ camertron-eprun
+ cldr-plurals-runtime-rb (~> 1.0)
+ tzinfo
tzinfo (1.2.4)
thread_safe (~> 0.1)
uea-stemmer (0.10.3)
@@ -481,29 +501,30 @@ DEPENDENCIES
active_record_union
after-commit-on-action
autoprefixer-rails
+ awesome_print
aws-sdk (< 3)
+ bcrypt_pbkdf (< 2.0)
better_errors
binding_of_caller
boolean
bootstrap
+ bundler
capistrano
capistrano-bundler
capistrano-rails
capistrano-rvm
capistrano-sidekiq
capybara
+ chewy
coffee-rails
coffee-script
configoro
database_cleaner
devise
+ devise_security_extension
diff-lcs
docx (= 0.3.1)!
dropzonejs-rails
- elasticsearch (< 6.0)
- elasticsearch-dsl
- elasticsearch-model
- elasticsearch-rails
execjs
factory_bot_rails
faker
@@ -518,7 +539,9 @@ DEPENDENCIES
json
kaminari
libarchive
+ message_format (~> 0.0.5)
mustache
+ newrelic_rpm
nokogiri
paperclip!
parslet
@@ -529,7 +552,10 @@ DEPENDENCIES
rack-attack
rack-cache
rails (= 4.2.10)
+ rails-assets-raven-js!
rails-observers
+ rbnacl (< 5.0)
+ rbnacl-libsodium
redcarpet
redis-mutex
redis-namespace
@@ -541,6 +567,7 @@ DEPENDENCIES
rugged!
safemode
sass-rails
+ sentry-raven
sidekiq-failures!
sidekiq-pro (= 3.4.5)!
similar_text
@@ -561,7 +588,7 @@ DEPENDENCIES
yard
RUBY VERSION
- ruby 2.3.1p112
+ ruby 2.4.6p354
BUNDLED WITH
- 1.16.1
+ 1.17.3
diff --git a/README.md b/README.md
index e0d6880b..b42fe345 100644
--- a/README.md
+++ b/README.md
@@ -83,35 +83,23 @@ Create and import your first project!
### Setting up your development environment without Docker
-Developing for Shuttle requires Ruby 2.3.1, PostgreSQL, Redis, Tidy, Sidekiq Pro
-ElasticSearch 1.7, and a modern version of libarchive. To run Shuttle for the
-first time:
+Developing for Shuttle requires Ruby 2.4.6, PostgreSQL 9.4, Redis, Tidy, Sidekiq
+Pro ElasticSearch 5.6, and a modern version of libarchive. To run Shuttle for
+the first time:
1. Clone this project.
-2. Install Ruby 2.3.1. If you are using RVM, you can do so using the
+2. Install Ruby 2.4.6. If you are using RVM, you can do so using the
`.ruby-version` file.
3. Run `brew bundle` to install all dependencies available via Homebrew, which
are specified in the `Brewfile`, or install them manually referencing the
Brewfile.
-4. Since ElasticSearch 1.7 is required, you will have to download and install it
- manually at https://www.elastic.co/downloads/past-releases/elasticsearch-1-7-6.
-
- If you are already running a more modern version of ElasticSearch, you can
- run this older version simultaneously on a different port (e.g. 9201) by
- altering the `config/elasticsearch.yml` file in the ElasticSearch install
- directory. If you do this, make sure you override the default ElasticSearch
- URL when running Shuttle by either creating a
- `config/environments/development/elasticsearch.yml` file (to override the
- file under `config/environments/common`), or setting the `ELASTICSEARCH_URL`
- instance variable.
-
-5. Buy Sidekiq Pro and place your private source URL in Gemfile as specified by
+4. Buy Sidekiq Pro and place your private source URL in Gemfile as specified by
the Sidekiq Pro documentation.
-6. Create a PostgreSQL user called `shuttle`, and make it the owner of two
+5. Create a PostgreSQL user called `shuttle`, and make it the owner of two
PostgreSQL databases, `shuttle_development` and `shuttle_test`:
``` sh
@@ -120,51 +108,45 @@ first time:
createdb -O shuttle shuttle_test
```
-7. You will need to tell Bundler where the libarchive install directory is. If
+6. You will need to tell Bundler where the libarchive install directory is. If
you installed libarchive using Homebrew, you can do this by running
``` sh
bundle config build.libarchive --with-opt-dir=$(brew --prefix libarchive)
```
-8. Likewise, for Rugged, you will need to tell Bundler where the libgit2 install
+7. Likewise, for Rugged, you will need to tell Bundler where the libgit2 install
directory is. If you installed libgit2 using Homebrew:
``` sh
bundle config build.rugged --with-opt-dir=$(brew --prefix libgit2)
```
-9. Make sure that PostgreSQL, Redis, and ElasticSearch are running. If you
- installed them via Homebrew, running `brew info postgresql` and
- `brew info redis` will tell you how to run them. For ElasticSearch, read the
- README in your install directory.
+8. Make sure that PostgreSQL, Redis, and ElasticSearch are running. If you
+ installed them via Homebrew, running `brew info postgresql@9.6`,
+ `brew info redis`, and `brew info elasticsearch` will tell you how to run
+ them.
-10. Install the `mailcatcher` gem, which is used to receive emails sent in
+9. Install the `mailcatcher` gem, which is used to receive emails sent in
development. (This gem is not a part of the Gemfile because it's typically
installed as part of a global or system-wide gemset to be used with
all projects.)
-11. Optionally, install the `foreman` gem, which runs all the processes
+
+10. Optionally, install the `foreman` gem, which runs all the processes
necessary for development.
-12. Install all required gems by running `bundle install`.
-13. Run `rake db:migrate db:seed` to migrate and seed the database.
-14. Run `RAILS_ENV=test rake db:migrate` to setup the test database.
-15. Initialize the ElasticSearch index for development by running
- ``` sh
- rake environment elasticsearch:import:model FORCE=y CLASS=Commit
- rake environment elasticsearch:import:model FORCE=y CLASS=Key
- rake environment elasticsearch:import:model FORCE=y CLASS=Translation
- ```
+11. Install all required gems by running `bundle install`.
+
+12. Run `rake db:migrate db:seed` to migrate and seed the database.
+
+13. Run `RAILS_ENV=test rake db:migrate` to setup the test database.
-16. Do the same for the test indexes:
+14. Initialize the ElasticSearch index for development by running
+ `rake chewy:reset`.
- ``` sh
- RAILS_ENV=test rake environment elasticsearch:import:model FORCE=y CLASS=Commit
- RAILS_ENV=test rake environment elasticsearch:import:model FORCE=y CLASS=Key
- RAILS_ENV=test rake environment elasticsearch:import:model FORCE=y CLASS=Translation
- ```
+15. Do the same for the test indexes: `RAILS_ENV=test rake chewy:reset`
-17. Verify that all specs pass with `rspec spec`.
+16. Verify that all specs pass with `rspec spec`.
#### Starting the server
@@ -457,7 +439,7 @@ RSpec specs under the `spec` directory. Views and JavaScript files are not
specced. Almost all unit tests use factories rather than mocks, putting them
somewhat closer to integration tests.
-If you are using Docker, first run `docker-compose -f docker-compose.test.yml`
+If you are using Docker, first run `docker-compose -f docker-compose.test.yml build`
to build the images. This only has to be done once, and each time you make a
change to the source code. Run `docker-compose -f docker-compose.test.yml up` to
start the environment, and
diff --git a/app/assets/javascripts/application.js.coffee b/app/assets/javascripts/application.js.coffee
index b6155c2e..d4a9b38e 100644
--- a/app/assets/javascripts/application.js.coffee
+++ b/app/assets/javascripts/application.js.coffee
@@ -68,6 +68,8 @@
#= require popper
#= require bootstrap
#=
+#= require raven-js
+#
#= require_tree ../templates
#= require_tree ./application
diff --git a/app/assets/javascripts/application/translation_workbench.js.coffee.erb b/app/assets/javascripts/application/translation_workbench.js.coffee.erb
index c308a6b3..efe2f719 100644
--- a/app/assets/javascripts/application/translation_workbench.js.coffee.erb
+++ b/app/assets/javascripts/application/translation_workbench.js.coffee.erb
@@ -336,8 +336,10 @@ class TranslationItem
fenced_p.append htmlEscape(copy[copy_index...range[0]])
copy_index = range[0]
# consume the range and enclose it in a span
- $('').addClass('fenced').text(copy[range[0]..range[1]]).appendTo fenced_p
- copy_index = range[1] + 1
+ copy_trimmed = copy[range[0]..range[1]]
+ if copy_trimmed.length > 0
+ $('').addClass('fenced').text(copy_trimmed).appendTo fenced_p
+ copy_index = range[1] + 1
# consume from the end of the last range to the end of the string
if copy_index < copy.length
diff --git a/app/assets/stylesheets/base/_base.scss b/app/assets/stylesheets/base/_base.scss
index 1754855e..4fc20776 100644
--- a/app/assets/stylesheets/base/_base.scss
+++ b/app/assets/stylesheets/base/_base.scss
@@ -1,15 +1,15 @@
@import "base/vars";
@import "css3-mixins";
-// ----- RESPONSIVE
-
-// @include ipad-and-larger {
-// .compact-only { display: none; }
-// }
-
-// @include iphone-landscape-and-smaller {
-// .full-size-only { display: none; }
-// }
+@import url("https://rsms.me/inter/inter.css");
+*, html {
+ font-family: "Inter", sans-serif;
+}
+@supports (font-variation-settings: normal) {
+ html {
+ font-family: "Inter var", sans-serif;
+ }
+}
// ----- LAYOUT
@@ -21,9 +21,8 @@ html {
body {
height: 100%;
background-color: white;
- font-family: "Helvetica Neue", Helvetica, sans-serif;
font-size: $base-font-size;
- line-height: 21px;
+ line-height: $base-line-height;
}
.content-container {
@@ -35,7 +34,7 @@ body {
.header {
h1 {
display: inline-block;
- color: $gray3;
+ color: $gray2;
margin-top: 0px;
}
@@ -51,7 +50,6 @@ body {
h6 {
color: $gray4;
- text-transform: uppercase;
margin-left: 30px;
line-height: 10px;
margin-top: 0px;
@@ -60,15 +58,13 @@ body {
.border {
border: 1px solid $gray5;
- overflow-y: scroll;
}
hr.divider {
border: 0;
height: 1px;
background-color: $gray5;
- margin-top: 5px;
- margin-bottom: 40px;
+ margin: 5px 0 10px;
}
.hide {
diff --git a/app/assets/stylesheets/base/_controls.scss b/app/assets/stylesheets/base/_controls.scss
index 09942f8d..2a5afe9b 100644
--- a/app/assets/stylesheets/base/_controls.scss
+++ b/app/assets/stylesheets/base/_controls.scss
@@ -15,7 +15,6 @@
font-weight: normal;
letter-spacing: 1px;
height: $input-size;
- text-transform: uppercase;
text-decoration: none !important;
padding: 5px 20px;
@@ -25,7 +24,6 @@
&:hover {
background: darken($bg-color, 5%);
- color: $color !important;
cursor: pointer;
text-decoration: none !important;
}
@@ -53,6 +51,15 @@
padding: 5px;
text-align: center;
}
+
+ &.button--secondary {
+ background-color: white;
+ color: $dark-blue;
+ border-color: $dark-blue;
+ &:hover {
+ background: rgba(41, 150, 204, 0.1);
+ }
+ }
}
@mixin field-options {
@@ -128,7 +135,6 @@ textarea {
select {
@include field-options;
@include appearance(none);
- font-family: "Helvetica Neue", Helvetica, sans-serif;
text-indent: 0.01px;
text-overflow: '';
diff --git a/app/assets/stylesheets/base/_forms.scss b/app/assets/stylesheets/base/_forms.scss
index 68dbeac6..53804799 100644
--- a/app/assets/stylesheets/base/_forms.scss
+++ b/app/assets/stylesheets/base/_forms.scss
@@ -37,7 +37,6 @@ fieldset {
legend {
font-size: $legend-size;
line-height: $legend-line-height;
- text-transform: uppercase;
font-weight: 600;
letter-spacing: 1px;
display: block;
diff --git a/app/assets/stylesheets/base/_grid.scss b/app/assets/stylesheets/base/_grid.scss
index 6faa5f8b..562a74c3 100644
--- a/app/assets/stylesheets/base/_grid.scss
+++ b/app/assets/stylesheets/base/_grid.scss
@@ -43,7 +43,7 @@
}
@include widescreen {
- .container { width: $widescreen-size; }
+ .container { width: 95%; max-width: $widescreen-size; }
.row { @include columns($widescreen-column-width); }
}
diff --git a/app/assets/stylesheets/base/_tables.scss b/app/assets/stylesheets/base/_tables.scss
index 313e15d6..23ae7a99 100644
--- a/app/assets/stylesheets/base/_tables.scss
+++ b/app/assets/stylesheets/base/_tables.scss
@@ -12,10 +12,6 @@
.checkbox + .checkbox, input + .checkbox { margin-left: 15px; }
input + input, input + select, select + input, select + select { margin-left: 15px; }
- .main {
-
- }
-
.collapse {
margin-top: 30px;
margin-left: 40px;
@@ -60,9 +56,8 @@ table.table {
tr {
text-align: left;
font-weight: bold;
- text-transform: uppercase;
th {
- padding: 15px 10px;
+ padding: 14px 8px;
}
}
}
@@ -75,7 +70,15 @@ table.table {
td {
padding: 10px;
+ vertical-align: middle;
}
}
}
}
+
+.header {
+ margin-bottom: 6px;
+}
+.header-buttons {
+ padding-bottom: 10px;
+}
diff --git a/app/assets/stylesheets/base/_typography.scss b/app/assets/stylesheets/base/_typography.scss
index ccb78107..9f8c1496 100644
--- a/app/assets/stylesheets/base/_typography.scss
+++ b/app/assets/stylesheets/base/_typography.scss
@@ -13,14 +13,10 @@ h6 { font-size: $h6-size; line-height: $h6-line-height; }
h2 { font-size: $h4-size; }
h3 { font-size: $h5-size; }
h4 { font-size: $h6-size; }
- h5 { font-size: 11px; text-transform: uppercase; }
- h6 { font-size: 11px; text-transform: uppercase; }
}
h1, h2, h3, h4, h5, h6 {
margin-top: 20px;
- margin-bottom: 10px;
- font-weight: 100;
&:first-child { margin-top: 0; }
}
@@ -42,14 +38,14 @@ a {
strong { font-weight: 500; }
em { font-style: italic; }
-p.small, small { font-size: 80%; }
+p.small, small, .small { font-size: 80%; }
dl {
dt { font-weight: bold; }
dd { margin: 5px 0 10px 10px; }
}
-tt, kbd, code, samp, pre { font-family: Menlo, monospace; }
+tt, kbd, code, samp, pre { font-family: 'SF Mono', 'Roboto Mono', Monaco, Menlo, monospace; }
// tt is used for code atoms such as variable or class names
tt { font-weight: bold; }
// kbd is used for keyboard input strings
@@ -61,7 +57,7 @@ code { border-bottom: 1px solid $gray5; }
code.short { border-bottom: none; }
// pre is used for large code blocks
pre {
- background-color: #f9f9f9;
+ background-color: $gray6;
@include border-radius($notification-radius-size);
padding: 20px;
@@ -89,4 +85,14 @@ pre {
.lowercase {
text-transform: lowercase;
-}
\ No newline at end of file
+}
+
+.monospace,
+.revision-prefix,
+.revision {
+ font-family: $monospace-font-family;
+}
+
+.subtitle {
+ color: $gray3;
+}
diff --git a/app/assets/stylesheets/base/_vars.scss b/app/assets/stylesheets/base/_vars.scss
index e65ddcb6..fe192f1b 100644
--- a/app/assets/stylesheets/base/_vars.scss
+++ b/app/assets/stylesheets/base/_vars.scss
@@ -1,3 +1,6 @@
+$font-family: "Inter", -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif,
+ "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
+$monospace-font-family: 'SF Mono', 'Roboto Mono', Monaco, monospace;
$base-font-size: 14px;
$nav-font-size: 12px;
$control-font-size: 14px;
@@ -11,7 +14,6 @@ $h4-size: 18px;
$h5-size: 14px;
$h6-size: 12px;
-
$base-line-height: 21px;
$legend-line-height: 28px;
@@ -23,7 +25,7 @@ $h4-line-height: 28px;
$h5-line-height: 24px;
$h6-line-height: 20px;
-$button-radius-size: 1px;
+$button-radius-size: 4px;
$input-radius-size: 1px;
$notification-radius-size: 4px;
@@ -32,6 +34,8 @@ $subnav-height: 40px;
$input-size: 32px;
$button-size: 32px;
+$medium-letter-spacing: 0.0365em;
+
// ----- COLORS
$gray1: #212121;
@@ -41,8 +45,9 @@ $gray4: #b3b3b3;
$gray5: #dfdfdf;
$gray6: #f9f9f9;
+
$off-blue: #def1f7;
-$dark-blue: #2884b4;
+$dark-blue: #2996cc;
$navbar-background: #2b2b2b;
$subnav-background: #373737;
@@ -68,7 +73,7 @@ $input-white: #fbfcfc;
$input-white-text: #6e797f;
$input-white-border: #d3d9de;
-$input-blue: #4296c2;
+$input-blue: $dark-blue;
$input-blue-text: white;
$input-blue-border: #3688b3;
@@ -92,10 +97,10 @@ $button-flag: #f5d68d;
$columns: 16;
-$widescreen-size: 1200px;
+$widescreen-size: 1440px;
$widescreen-column-width: $widescreen-size/$columns;
-$desktop-size: 960px;
+$desktop-size: 1200px;
$desktop-column-width: $desktop-size/$columns;
$ipad-size: 768px;
diff --git a/app/assets/stylesheets/pages/glossary.css.scss b/app/assets/stylesheets/pages/glossary.css.scss
index 66733a20..6e03c397 100644
--- a/app/assets/stylesheets/pages/glossary.css.scss
+++ b/app/assets/stylesheets/pages/glossary.css.scss
@@ -118,7 +118,6 @@ body.glossary {
body.source_glossary_entries, body.locale_glossary_entries {
.eight.columns {
- margin-top: -30px;
padding: 30px;
@include box-sizing;
position: relative;
diff --git a/app/assets/stylesheets/pages/groups.css.scss b/app/assets/stylesheets/pages/groups.css.scss
new file mode 100644
index 00000000..20422ee8
--- /dev/null
+++ b/app/assets/stylesheets/pages/groups.css.scss
@@ -0,0 +1,35 @@
+@import "base/vars";
+@import "css3-mixins";
+
+body.groups {
+ .control-group label {
+ padding-top: 0;
+ width: 170px;
+ }
+
+ &#groups-show, &#groups-issues {
+ .download-button {
+ margin-right: 10px;
+ }
+ }
+
+ &#groups-manifest {
+ .locale-manifest {
+ .locale {
+ text-transform: uppercase;
+ font-weight: bold;
+ font-size: 1.5em;
+ margin-bottom: 20px;
+ }
+
+ hr {
+ margin: 20px 0 30px 0;
+ }
+ }
+ }
+
+ .hide-btn, .show-btn {
+ width: 100px;
+ margin-left: 10px;
+ }
+}
diff --git a/app/assets/stylesheets/pages/home.css.scss b/app/assets/stylesheets/pages/home.css.scss
index 598d0c13..3ce0662d 100644
--- a/app/assets/stylesheets/pages/home.css.scss
+++ b/app/assets/stylesheets/pages/home.css.scss
@@ -20,15 +20,33 @@ body.home {
}
.filter-bar {
- input[type="submit"] { width: 125px; margin-left: 15px; }
- select[name="filter__rfc5646_locales"] { width: 200px; }
- select[name="filter__status"] { width: 200px; }
- input[name="commits_filter__sha"], input[name="articles_filter__name"], input[name="groups_filter__name"] { width: 350px; }
- select[name="commits_filter__project_id"], select[name="articles_filter__project_id"], select[name="groups_filter__project_id"] { width: 160px; }
+ input[type="submit"] {
+ width: 125px;
+ margin-left: 15px;
+ }
+ select[name="filter__rfc5646_locales"] {
+ width: 200px;
+ }
+ select[name="filter__status"] {
+ min-width: 200px;
+ }
+ input[name="commits_filter__sha"],
+ input[name="articles_filter__name"],
+ input[name="groups_filter__name"] {
+ width: 230px;
+ }
+ select[name="commits_filter__project_id"],
+ select[name="articles_filter__project_id"],
+ select[name="groups_filter__project_id"] {
+ min-width: 160px;
+ max-width: 300px;
+ }
.twitter-typeahead {
margin-left: 15px;
- input + input { margin: 0px; }
+ input + input {
+ margin: 0px;
+ }
}
.advanced-filters-label {
@@ -45,7 +63,10 @@ body.home {
tbody {
tr {
border-top: 3px white solid;
- &:hover {cursor: pointer;}
+ transition: 0.3s background-color;
+ &:hover {
+ cursor: pointer;
+ }
td {
&.status-translating,
@@ -54,45 +75,95 @@ body.home {
padding: 0;
width: 5px;
}
- &.status-translating { background-color: $commit-red; }
- &.status-loading { background-color: $commit-blue; }
- &.status-ready { background-color: $commit-green; }
+ &.status-translating {
+ background-color: $commit-red;
+ }
+ &.status-loading {
+ background-color: $commit-blue;
+ }
+ &.status-ready {
+ background-color: $commit-green;
+ }
- &.due-date { width: 110px;}
- &.centered { text-align: center; }
+ &.due-date {
+ width: 110px;
+ }
+ &.centered {
+ text-align: center;
+ }
- select { width: 100%; }
- input[type="text"] { width: 110px; }
- a { text-decoration: underline; }
- .description { width: 280px; }
- .translate-link { padding: 5px 10px; }
+ select {
+ width: 100%;
+ }
+ input[type="text"] {
+ width: 110px;
+ }
+ a {
+ text-decoration: underline;
+ }
+ .description {
+ text-align: center;
+ font-size: 10px;
+ line-height: 12px;
+ vertical-align: top;
+ }
+ .translate-link {
+ padding: 5px 10px;
+ }
}
}
}
}
- .priority-0 { color: #E94239; font-weight: bold; }
- .priority-1 { color: #E98C39; font-weight: bold; }
- .priority-2 { color: #E9B039; font-weight: bold; }
- .priority-3 { color: $input-green; font-weight: bold; }
+ .priority-0 {
+ color: #e94239;
+ font-weight: bold;
+ }
+ .priority-1 {
+ color: #e98c39;
+ font-weight: bold;
+ }
+ .priority-2 {
+ color: #e9b039;
+ font-weight: bold;
+ }
+ .priority-3 {
+ color: $input-green;
+ font-weight: bold;
+ }
.pagination-links {
width: 100%;
font-size: 20px;
margin: 10px 0px;
.pagination {
- display:-moz-box; /* Firefox */
- display:-webkit-box; /* Safari and Chrome */
- display:-ms-flexbox; /* Internet Explorer 10 */
- display:box;
+ display: -moz-box; /* Firefox */
+ display: -webkit-box; /* Safari and Chrome */
+ display: -ms-flexbox; /* Internet Explorer 10 */
+ display: box;
+ }
+ .switch-page {
+ width: 198px;
}
- .switch-page { width: 198px; }
.pages {
@include flex;
text-align: center;
- .page + .page { margin-left: 8px; }
+ .page + .page {
+ margin-left: 8px;
+ }
}
}
- .pagination-info { width: 100%; text-align: center; }
+ .pagination-info {
+ width: 100%;
+ text-align: center;
+ }
+
+ .csv-button {
+ margin-right: 10px;
+ }
+}
+.description-container {
+ padding: 0 !important; // yes, I'm doing this so I don't have to chase this down
+ width: 120px;
}
diff --git a/app/assets/stylesheets/pages/projects.css.scss b/app/assets/stylesheets/pages/projects.css.scss
index 1f76248b..24859b46 100644
--- a/app/assets/stylesheets/pages/projects.css.scss
+++ b/app/assets/stylesheets/pages/projects.css.scss
@@ -35,10 +35,9 @@ body.projects {
tr {
text-align: left;
font-weight: bold;
- text-transform: uppercase;
-
th {
padding: 15px;
+ min-width: 100px;
}
}
}
@@ -87,7 +86,6 @@ body.projects {
&#projects-new,
&#projects-update {
.eight.columns {
- margin-top: -30px;
padding: 30px;
box-sizing: border-box;
}
diff --git a/app/assets/stylesheets/pages/reports.css.scss b/app/assets/stylesheets/pages/reports.css.scss
new file mode 100644
index 00000000..f3ed8da9
--- /dev/null
+++ b/app/assets/stylesheets/pages/reports.css.scss
@@ -0,0 +1,36 @@
+.day {
+ border: 1px solid rgba(128, 128, 128, 0.212);
+ border-radius: 4px;
+ padding: 10px;
+ margin: 10px 0;
+ h2 {
+ text-align: center;
+ margin-bottom: 20px;
+ }
+ .report-buttons {
+ display: flex;
+ display: flex;
+ justify-content: space-around;
+ a {
+ border: 1px solid;
+ border-radius: 3px;
+ padding: 5px 13px;
+ box-shadow: 3px 3px 0px rebeccapurple;
+ transition: 0.3s box-shadow;
+ &:hover {
+ box-shadow: 1px 1px 0px rebeccapurple;
+ text-decoration: none;
+ }
+ &:active {
+ box-shadow: inset 2px 2px 0 rebeccapurple;
+ }
+ }
+ }
+ .date-picker {
+ width: 380px;
+ margin: 10px auto 30px;
+ label {
+ margin-right: 10px;
+ }
+ }
+}
diff --git a/app/assets/stylesheets/pages/translations.scss b/app/assets/stylesheets/pages/translations.scss
index da70c4ba..9a0a0283 100644
--- a/app/assets/stylesheets/pages/translations.scss
+++ b/app/assets/stylesheets/pages/translations.scss
@@ -4,7 +4,6 @@
body.translations {
.eight, .sixteen {
&.columns {
- margin-top: -30px;
padding: 30px;
@include box-sizing;
position: relative;
@@ -164,4 +163,4 @@ body.translations {
margin-left: 10px;
}
}
-}
\ No newline at end of file
+}
diff --git a/app/assets/stylesheets/pages/unified_authentication.scss b/app/assets/stylesheets/pages/unified_authentication.scss
index 1b3ca7ce..679a4697 100644
--- a/app/assets/stylesheets/pages/unified_authentication.scss
+++ b/app/assets/stylesheets/pages/unified_authentication.scss
@@ -5,10 +5,14 @@
$form-radius-size: 4px;
body.unified-authentication, body.passwords {
- background-image: image-url('login-background.jpg');
- background-color: $gray3;
- background-repeat: repeat-y;
- background-size: cover;
+ // background-image: image-url('login-background.jpg');
+ // background-color: $gray3;
+ // background-repeat: repeat-y;
+ // background-size: cover;
+ background: #00F260; /* fallback for old browsers */
+ background: -webkit-linear-gradient(to left, #0575E6, #00F260); /* Chrome 10-25, Safari 5.1-6 */
+ background: linear-gradient(to left, #0575E6, #00F260); /* W3C, IE 10+/ Edge, Firefox 16+, Chrome 26+, Opera 12+, Safari 7+ */
+
#navbar-container {
background-color: transparent;
@@ -31,7 +35,6 @@ body.unified-authentication, body.passwords {
&>.ten.columns {
background-color: $gray1;
- @include opacity(0.9);
color: white;
font-weight: 300;
text-align: center;
@@ -40,7 +43,7 @@ body.unified-authentication, body.passwords {
h1 {
font-size: 50px;
- font-weight: 100;
+ font-weight: 200;
line-height: 58px;
}
diff --git a/app/assets/stylesheets/pages/users.css.scss b/app/assets/stylesheets/pages/users.css.scss
index 1750cf9d..ed7924b2 100644
--- a/app/assets/stylesheets/pages/users.css.scss
+++ b/app/assets/stylesheets/pages/users.css.scss
@@ -9,8 +9,6 @@ body.users {
tr {
text-align: left;
font-weight: bold;
- text-transform: uppercase;
-
th {
padding: 15px;
}
@@ -96,4 +94,4 @@ body.users {
}
}
}
-}
\ No newline at end of file
+}
diff --git a/app/assets/stylesheets/partials/_footer.scss b/app/assets/stylesheets/partials/_footer.scss
index e71ca197..9c8878bf 100644
--- a/app/assets/stylesheets/partials/_footer.scss
+++ b/app/assets/stylesheets/partials/_footer.scss
@@ -15,25 +15,32 @@ footer {
margin-top: -9em;
height: 9em;
overflow: hidden;
- background-color: $navbar-background;
+ background: #DA4453; /* fallback for old browsers */
+ background: -webkit-linear-gradient(to top, #89216B, #DA4453); /* Chrome 10-25, Safari 5.1-6 */
+ background: linear-gradient(to top, #89216B, #DA4453); /* W3C, IE 10+/ Edge, Firefox 16+, Chrome 26+, Opera 12+, Safari 7+ */
+
+
p {
font-size: 12px;
font-weight: 500;
letter-spacing: 1px;
text-align: center;
text-transform: uppercase;
- color: #555555;
+ color: #fff;
margin-top: 20px;
a {
- color: #555555;
+ color: #fff;
text-decoration: underline;
+ &:hover {
+ color: #eee;
+ }
}
i {
- color: darkred;
+ color: #fff;
}
}
img {
- opacity:0.2;
+ opacity: 0.2;
}
}
diff --git a/app/assets/stylesheets/partials/_navbar.scss b/app/assets/stylesheets/partials/_navbar.scss
index 3d131e4f..a6e81aa6 100644
--- a/app/assets/stylesheets/partials/_navbar.scss
+++ b/app/assets/stylesheets/partials/_navbar.scss
@@ -7,10 +7,8 @@ $navbar-min-width: 910px;
@include clearfix;
padding: 1.3em 1em;
- color: rgba(255,255,255,.5);
- text-transform: uppercase;
- font-weight: bold;
- font-size: 12px;
+ font-size: $base-font-size;
+ letter-spacing: $medium-letter-spacing;
&.navbar-dark {
background-color: $navbar-background;
@@ -18,7 +16,6 @@ $navbar-min-width: 910px;
.navbar-brand {
float: left;
- text-transform: uppercase;
margin-right: 1.5em;
img {
@@ -28,7 +25,6 @@ $navbar-min-width: 910px;
}
button.navbar-toggler {
- color: rgba(255,255,255,.5);
border: solid 1px rgba(255,255,255,.1);
padding: .25rem .75rem;
background: 0 0;
@@ -55,7 +51,7 @@ $navbar-min-width: 910px;
.nav-link {
display: block;
padding: .5em 1em;
- font-size: 12px;
+ font-size: $base-font-size;
&.active {
a {
@@ -64,7 +60,7 @@ $navbar-min-width: 910px;
}
a {
- color: $navbar-gray-text;
+ color: $gray4;
&:hover {
text-decoration: none;
diff --git a/app/assets/stylesheets/partials/_sidebar.scss b/app/assets/stylesheets/partials/_sidebar.scss
index ebc0d986..dbc43ce4 100644
--- a/app/assets/stylesheets/partials/_sidebar.scss
+++ b/app/assets/stylesheets/partials/_sidebar.scss
@@ -34,7 +34,7 @@
&.progress-bar {
dl {
@include box-sizing(border-box);
- margin: 30px;
+ margin: 20px 10px;
dt {
text-transform: uppercase;
strong {
@@ -62,6 +62,11 @@
margin: 0;
}
}
+ a {
+ margin-right: 0;
+ padding: 0;
+ text-transform: none;
+ }
}
}
}
diff --git a/app/assets/stylesheets/partials/_worker_status.scss b/app/assets/stylesheets/partials/_worker_status.scss
index 84b436f6..5735b945 100644
--- a/app/assets/stylesheets/partials/_worker_status.scss
+++ b/app/assets/stylesheets/partials/_worker_status.scss
@@ -6,19 +6,55 @@
box-shadow: inset 0 4px 5px darken($navbar-background, 10%);
background-color: darken($navbar-background, 5%);
- color: $gray3;
+ color: $gray4;
+ .fa {
+ display: none;
+ }
&.clickable:hover {
box-shadow: inset 0 4px 5px darken($navbar-background, 15%);
background-color: darken($navbar-background, 10%);
cursor: pointer;
- &.worker-status-idle { i { color: $input-green; } }
- &.worker-status-busy { i { color: darken($input-orange, 20%) !important; } }
- &.worker-status-swamped { i { color: $input-red !important; } }
+ &.worker-status-idle {
+ i {
+ color: $input-green;
+ }
+ }
+ &.worker-status-busy {
+ i {
+ color: darken($input-orange, 20%) !important;
+ }
+ }
+ &.worker-status-swamped {
+ i {
+ color: $input-red !important;
+ }
+ }
}
- &.worker-status-idle { i { color: darken($input-green, 10%); } }
- &.worker-status-busy { i { color: darken($input-orange, 30%) !important; } }
- &.worker-status-swamped { i { color: darken($input-red, 10%) !important; } }
+ &.worker-status-idle {
+ i {
+ color: darken($input-green, 10%);
+ &.fa-check {
+ display: inline-block;
+ }
+ }
+ }
+ &.worker-status-busy {
+ i {
+ color: darken($input-orange, 30%) !important;
+ &.fa-refresh {
+ display: inline-block;
+ }
+ }
+ }
+ &.worker-status-swamped {
+ i {
+ color: darken($input-red, 10%) !important;
+ &.fa-cloud {
+ display: inline-block;
+ }
+ }
+ }
}
diff --git a/app/chewy/commits_index.rb b/app/chewy/commits_index.rb
new file mode 100644
index 00000000..ec150921
--- /dev/null
+++ b/app/chewy/commits_index.rb
@@ -0,0 +1,25 @@
+class CommitsIndex < Chewy::Index
+ settings analysis: {
+ analyzer: {
+ sha: {
+ tokenizer: 'keyword'
+ }
+ }
+ }
+
+ define_type Commit.includes(:commits_keys) do
+ field :id, type: 'integer'
+ field :project_id, type: 'integer'
+ field :user_id, type: 'integer'
+ field :priority, type: 'integer'
+ field :due_date, type: 'date'
+ field :created_at, type: 'date'
+ field :revision, analyzer: 'sha'
+ field :loading, type: 'boolean'
+ field :ready, type: 'boolean'
+ field :exported, type: 'boolean'
+ field :fingerprint, analyzer: 'sha'
+ field :duplicate, type: 'boolean'
+ field :key_ids, value: ->(c) { c.commits_keys.pluck(:key_id) }
+ end
+end
diff --git a/app/chewy/keys_index.rb b/app/chewy/keys_index.rb
new file mode 100644
index 00000000..03175bc6
--- /dev/null
+++ b/app/chewy/keys_index.rb
@@ -0,0 +1,19 @@
+class KeysIndex < Chewy::Index
+ settings analysis: {
+ tokenizer: {
+ key_tokenizer: {type: 'pattern', pattern: '[^A-Za-z0-9]'}
+ },
+ analyzer: {
+ key_analyzer: {type: 'custom', tokenizer: 'key_tokenizer', filter: 'lowercase'}
+ }
+ }
+
+ define_type Key do
+ field :id, type: 'integer'
+ field :original_key, type: 'text', analyzer: 'key_analyzer'
+ field :original_key_exact, type: 'keyword', value: ->(t) { t.original_key }
+ field :project_id, type: 'integer'
+ field :ready, type: 'boolean'
+ field :hidden_in_search, type: 'boolean'
+ end
+end
diff --git a/app/chewy/translations_index.rb b/app/chewy/translations_index.rb
new file mode 100644
index 00000000..c4625fe9
--- /dev/null
+++ b/app/chewy/translations_index.rb
@@ -0,0 +1,40 @@
+class TranslationsIndex < Chewy::Index
+ TRANSLATION_STATE_NEW = 0
+ TRANSLATION_STATE_TRANSLATED = 1
+ TRANSLATION_STATE_APPROVED = 2
+ TRANSLATION_STATE_REJECTED = 3
+
+ define_type Translation.includes(key: :section) do
+ field :id, type: 'integer'
+ field :copy, analyzer: 'snowball', type: 'text', similarity: 'classic'
+ field :source_copy, analyzer: 'snowball', type: 'text', similarity: 'classic'
+ field :project_id, type: 'integer', value: ->(t) { t.key.project_id }
+ field :article_id, type: 'integer', value: ->(t) { t.key.section&.article_id }
+ field :section_id, type: 'integer', value: ->(t) { t.key.section_id }
+ field :is_block_tag, type: 'boolean', value: ->(t) { t.key.is_block_tag }
+ field :section_active, type: 'boolean', value: ->(t) { t.key.section&.active }
+ field :index_in_section, type: 'integer', value: ->(t) { t.key.index_in_section }
+ field :translator_id, type: 'integer'
+ field :reviewer_id, type: 'integer'
+ field :rfc5646_locale, type: 'keyword'
+ field :created_at, type: 'date'
+ field :updated_at, type: 'date'
+ field :translation_state, 'integer', value: -> (t) do
+ if t.approved
+ TRANSLATION_STATE_APPROVED
+ elsif t.translated
+ if t.approved.nil?
+ TRANSLATION_STATE_TRANSLATED
+ else
+ TRANSLATION_STATE_REJECTED
+ end
+
+ else
+ TRANSLATION_STATE_NEW
+ end
+ end
+ field :translated, type: 'boolean', value: -> (t) { !!t.translated } # converts nil to false
+ field :approved, type: 'boolean', value: ->(t) { !!t.approved } # converts nil to false
+ field :hidden_in_search, type: 'boolean', value: ->(t) { t.key.hidden_in_search }
+ end
+end
diff --git a/app/controllers/api/v1/groups_controller.rb b/app/controllers/api/v1/groups_controller.rb
index 58918a63..c3a04e10 100644
--- a/app/controllers/api/v1/groups_controller.rb
+++ b/app/controllers/api/v1/groups_controller.rb
@@ -23,6 +23,7 @@ module API
module V1
class GroupsController < ApplicationController
respond_to :json, only: [:index, :create, :update, :destroy]
+ respond_to :html, only: [:show]
skip_before_filter :authenticate_user!, if: :api_request?
skip_before_action :verify_authenticity_token, if: :api_request?
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 77772fe0..1eb74430 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -124,6 +124,7 @@ class ApplicationController < ActionController::Base
self.responder = JsonDetailResponder
before_filter :authenticate_user!
+ before_filter :set_raven_context
rescue_from(ActiveRecord::RecordNotFound) do
respond_to do |format|
@@ -252,4 +253,12 @@ def fix_empty_hashes(*fields)
end
end
end
+
+ private
+
+ def set_raven_context
+ return unless user_signed_in?
+ Raven.user_context user_id: current_user.id, email: current_user.email
+ end
+
end
diff --git a/app/controllers/commits_controller.rb b/app/controllers/commits_controller.rb
index 63d5ae2e..e6ffef18 100644
--- a/app/controllers/commits_controller.rb
+++ b/app/controllers/commits_controller.rb
@@ -126,14 +126,16 @@ def issues
# | `id` | The SHA of a Commit. |
def search
+ pending_locales = @commit.translations.where('approved IS NOT TRUE').group(:rfc5646_locale).count.keys
@locales = @project.locale_requirements.inject({}) do |hsh, (locale, required)|
hsh[locale.rfc5646] = {
required: required,
targeted: true,
- finished: @commit.translations.where('approved IS NOT TRUE AND rfc5646_locale = ?', locale.rfc5646).first.nil?
+ finished: !pending_locales.include?(locale.rfc5646)
}
hsh
end
+
@keys = CommitsSearchKeysFinder.new(params, @commit).find_keys
@keys_presenter = CommitsSearchPresenter.new(params[:locales], current_user.translator?, @project, @commit)
end
@@ -177,6 +179,7 @@ def create
flash[:alert] = t('controllers.commits.create.project_not_linked_error', revision: params[:commit][:revision].strip)
redirect_to root_url
rescue Timeout::Error
+ Raven.capture_exception err, extra: { project_id: project_id, sha: sha }
flash[:alert] = t('controllers.commits.create.timeout', revision: revision)
redirect_to root_url
end
@@ -300,7 +303,8 @@ def ping_stash
end
def reindex
- Translation.batch_refresh_elastic_search @commit
+ CommitsIndex.import! @commit
+ TranslationsIndex.import! @commit.translations
flash[:success] = t('controllers.commits.reindex.success')
respond_with @commit, location: project_commit_url(@project, @commit)
end
@@ -345,7 +349,8 @@ def manifest
compiler = Compiler.new(@commit)
file = compiler.manifest(request.format,
locale: params[:locale].presence,
- partial: params[:partial].parse_bool)
+ partial: params[:partial].parse_bool,
+ encoding: params[:encoding])
response.charset = file.encoding
response.headers['X-Git-Revision'] = @commit.revision
diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb
index 617e24c1..5d4dce03 100644
--- a/app/controllers/home_controller.rb
+++ b/app/controllers/home_controller.rb
@@ -14,8 +14,10 @@
# Contains landing pages appropriate for each of the User roles.
-class HomeController < ApplicationController
+require 'csv'
+require 'date'
+class HomeController < ApplicationController
# Typical number of commits to show per page.
PER_PAGE = 15
@@ -53,4 +55,100 @@ def index
@assets = items_finder.find_assets
@presenter = HomeIndexPresenter.new(@commits, @articles, @groups, @assets, @form[:filter__locales])
end
+
+ def csv
+ # manually setting how many rows to download
+ params[:limit] = 1000
+ @form = HomeIndexForm.new(params, cookies)
+ type = params[:type]
+ items_finder = HomeIndexItemsFinder.new(current_user, @form)
+ @commits = items_finder.find_commits
+ @articles = items_finder.find_articles
+ @groups = items_finder.find_groups
+ @assets = items_finder.find_assets
+ @items = items_finder.public_send("find_#{type}s")
+ @presenter = HomeIndexPresenter.new(@commits, @articles, @groups, @assets, @form[:filter__locales])
+ csv_file = CSV.generate do |csv|
+ identifier = type == 'commit' ? 'sha' : 'name'
+ if identifier == 'sha'
+ csv << ['Project',
+ 'SHA',
+ 'Created',
+ 'Due Date',
+ 'Priority',
+ 'New Strings',
+ 'New Words',
+ 'Review Strings',
+ 'Review Words',
+ 'Translate Link',
+ 'Requester Email']
+ @items.each do |item|
+ project = Project.find item.project_id
+ display_created_date = "#{item.created_at.month}/#{item.created_at.day}/#{item.created_at.year}"
+ unless item.due_date.nil?
+ display_due_date = "#{item.due_date.month}/#{item.due_date.day}/#{item.due_date.year}"
+ end
+ strings_to_translate = @presenter.item_stat(item, :translations, :new)
+ strings_to_review = @presenter.item_stat(item, :translations, :pending)
+ words_to_translate = @presenter.item_stat(item, :words, :new)
+ words_to_review = @presenter.item_stat(item, :words, :pending)
+ commit_url = project_commit_url(item.project, item)
+ requester_email = item.user.email if item.user
+ csv << [project.name,
+ commit_url,
+ display_created_date,
+ display_due_date,
+ item.priority,
+ strings_to_translate,
+ words_to_review,
+ strings_to_review,
+ words_to_translate,
+ project_commit_url(project, item),
+ requester_email]
+ end
+ else
+ csv << ['Project',
+ 'Name',
+ 'Created',
+ 'Due Date',
+ 'Priority',
+ 'Groups',
+ 'Review Strings',
+ 'Review Words',
+ 'New Strings',
+ 'New Words',
+ 'Translate Link',
+ 'Requester Email']
+ @items.each do |item|
+ project = Project.find item.project_id
+ display_created_date = "#{item.created_at.month}/#{item.created_at.day}/#{item.created_at.year}"
+ unless item.due_date.nil?
+ display_due_date = "#{item.due_date.month}/#{item.due_date.day}/#{item.due_date.year}"
+ end
+ groups = item.groups.count
+ strings_to_translate = @presenter.item_stat(item, :translations, :new)
+ strings_to_review = @presenter.item_stat(item, :translations, :pending)
+ words_to_translate = @presenter.item_stat(item, :words, :new)
+ words_to_review = @presenter.item_stat(item, :words, :pending)
+ article_link = api_v1_project_article_url(item.project.id, item.name)
+ requester_email = item.email
+ csv << [project.name,
+ article_link,
+ display_created_date,
+ display_due_date,
+ item.priority,
+ groups,
+ strings_to_translate,
+ words_to_translate,
+ strings_to_review,
+ words_to_review,
+ project_commit_url(project, item),
+ requester_email]
+ end
+ end
+ end
+ send_data csv_file, type: 'text/plain',
+ filename: "#{type}s.csv",
+ disposition: 'attachment'
+ end
end
diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb
index 70c83aec..aaab7edf 100644
--- a/app/controllers/projects_controller.rb
+++ b/app/controllers/projects_controller.rb
@@ -405,4 +405,26 @@ def project_params
project_params["targeted_rfc5646_locales"] = targeted_rfc5646_locales
project_params
end
+
+ def project_path_changed?(params)
+ # checks if only_paths have been modified
+ if @project.only_paths.sort != params['only_paths'].sort
+ return true
+ end
+
+ # checks if locales have been modified in key_locale_inclusions
+ if @project.key_locale_inclusions.keys.sort != params['key_locale_inclusions'].keys.sort
+ return true
+ end
+
+ # checks if filters in each locale have been modified
+ @project.key_locale_inclusions.each do |locale, filters|
+ if filters.sort != params['key_locale_inclusions'][locale].sort
+ return true
+ end
+ end
+
+ # neither only_paths, nor key_locale_inclusions has been modified.
+ return false;
+ end
end
diff --git a/app/controllers/reports_controller.rb b/app/controllers/reports_controller.rb
new file mode 100644
index 00000000..7db313ce
--- /dev/null
+++ b/app/controllers/reports_controller.rb
@@ -0,0 +1,44 @@
+require 'csv'
+
+class ReportsController < ApplicationController
+ def index
+ @today = Date.today
+ @yesterday = Date.yesterday
+ @range = Date.today...Date.today - 30
+ end
+
+ def incoming
+ parsed_date = Date.parse(params[:date])
+ range = parsed_date.beginning_of_day...parsed_date.end_of_day
+ @report = Report.where(date: range).where(report_type: 'incoming')
+ send_data convert_json_to_csv(@report), type: 'text/plain',
+ filename: "incoming-#{params[:date]}.csv",
+ disposition: :attachment
+ end
+
+ def pending
+ range = Date.parse(params[:date])...Date.tomorrow
+ @report = Report.where(date: range).where(report_type: 'pending')
+ send_data convert_json_to_csv(@report), type: 'text/plain',
+ filename: "pending-#{params[:date]}.csv",
+ disposition: :attachment
+ end
+
+ def completed
+ range = Date.parse(params[:date])...Date.tomorrow
+ @report = Report.where(date: range).where(report_type: 'completed')
+ send_data convert_json_to_csv(@report), type: 'text/plain',
+ filename: "completed-#{params[:date]}.csv",
+ disposition: :attachment
+ end
+
+ def convert_json_to_csv(jobs)
+ csv_file = CSV.generate do |csv|
+ csv << ['project_name', 'targeted_locale', 'strings', 'words', 'date']
+ jobs.each do |job|
+ csv << [job[:project], job[:locale], job[:strings], job[:words], job[:date]]
+ end
+ end
+ csv_file
+ end
+end
diff --git a/app/controllers/translations_controller.rb b/app/controllers/translations_controller.rb
index 81faeb01..54af4974 100644
--- a/app/controllers/translations_controller.rb
+++ b/app/controllers/translations_controller.rb
@@ -325,7 +325,6 @@ def translation_params
def change_hidden_in_search_to(state)
@key.update!(hidden_in_search: state)
- @translation.update_elasticsearch_index
end
def decorate_fuzzy_match(translations, source_copy)
@@ -337,7 +336,7 @@ def decorate_fuzzy_match(translations, source_copy)
rfc5646_locale: translation.rfc5646_locale,
project_name: translation.key.project.name.truncate(30)
}
- end.reject { |t| t[:match_percentage] < FuzzyMatchTranslationsFinder::MINIMUM_FUZZY_MATCH }
+ end.reject { |t| t[:match_percentage] < FuzzyMatchTranslationsFinder::FUZZY_MATCH_MIN_SCORE }
translations.sort! { |a, b| b[:match_percentage] <=> a[:match_percentage] }
end
end
diff --git a/app/helpers/custom_metric_helper.rb b/app/helpers/custom_metric_helper.rb
new file mode 100644
index 00000000..1f118fdd
--- /dev/null
+++ b/app/helpers/custom_metric_helper.rb
@@ -0,0 +1,73 @@
+module CustomMetricHelper
+ extend self
+
+ SIDEKIQ_WORKER_LONGEVITY = 'SidekiqWorker/longevity'
+ SIDEKIQ_WORKER_JOBS_BUSY = 'SidekiqWorker/jobs/busy'
+ SIDEKIQ_WORKER_JOBS_ENQUEUED = 'SidekiqWorker/jobs/enqueued'
+
+ PROJECT_PROCESSING_LOADING_TIME = 'project/processing/loading'
+ PROJECT_PROCESSING_TRANSLATING_TIME = 'project/processing/translating'
+ PROJECT_PROCESSING_REVIEWING_TIME = 'project/processing/translating'
+ PROJECT_PROCESSING_READY_TIME = 'project/processing/ready'
+ PROJECT_PROESSSING_STASH_TIME = 'project/processing/stash'
+
+ PROJECT_STATISTICS_FILES = 'project/statistics/files'
+ PROJECT_STATISTICS_STRINGS = 'project/statistics/strings'
+ PROJECT_STATISTICS_WORDS = 'project/statistics/words'
+
+ def record_sidekiq_longevity(host_to_longevities)
+ host_to_longevities.each do |hostname, longevity|
+ record_metric(longevity, SIDEKIQ_WORKER_LONGEVITY, hostname)
+ end
+ end
+
+ def record_sidekiq_jobs(busy_jobs, enqueued_jobs)
+ record_metric(busy_jobs, SIDEKIQ_WORKER_JOBS_BUSY)
+ record_metric(enqueued_jobs, SIDEKIQ_WORKER_JOBS_ENQUEUED)
+ end
+
+ # time from project created to loaded
+ def record_project_loading_time(project_name, loading_time)
+ record_metric(loading_time, PROJECT_PROCESSING_LOADING_TIME, project_name)
+ end
+
+ # time from project loaded to fully translated
+ def record_project_translating_time(project_name, locale_name, translating_time)
+ record_metric(translating_time, PROJECT_PROCESSING_TRANSLATING_TIME, project_name, locale_name)
+ end
+
+ # time from project fully translated to fully reviewed
+ def record_project_reviewing_time(project_name, locale_name, reviewing_time)
+ record_metric(reviewing_time, PROJECT_PROCESSING_REVIEWING_TIME, project_name, locale_name)
+ end
+
+ # time from project created to ready
+ def record_project_ready_time(project_name, ready_time)
+ record_metric(ready_time, PROJECT_PROCESSING_READY_TIME, project_name)
+ end
+
+ # time from project approved to ping done
+ def record_project_ping_stash_time(project_name, ping_stash_time)
+ record_metric(ping_stash_time, PROJECT_PROESSSING_STASH_TIME, project_name)
+ end
+
+ # counts for project files, keys per locale and words per locale
+ def record_project_statistics(project_name, blobs, locales_to_keys, locales_to_words)
+ record_metric(blobs, PROJECT_STATISTICS_FILES, project_name)
+
+ locales_to_keys.each do |locale_name, keys|
+ record_metric(keys, PROJECT_STATISTICS_STRINGS, project_name, locale_name)
+ end
+
+ locales_to_words.each do |locale_name, words|
+ record_metric(words, PROJECT_STATISTICS_WORDS, project_name, locale_name)
+ end
+ end
+
+ private
+
+ def record_metric(metric, metric_name, *sub_metric_names)
+ full_metric_name = (['Custom', metric_name] + sub_metric_names).join('/')
+ ::NewRelic::Agent.record_metric(full_metric_name, metric)
+ end
+end
diff --git a/app/mailers/fencer_validation_mailer.rb b/app/mailers/fencer_validation_mailer.rb
new file mode 100644
index 00000000..f060f939
--- /dev/null
+++ b/app/mailers/fencer_validation_mailer.rb
@@ -0,0 +1,91 @@
+# Copyright 2014 Square Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Sends emails related to Comments
+
+class FencerValidationMailer < ActionMailer::Base
+ include ActionView::Helpers::TextHelper
+
+ default from: Shuttle::Configuration.mailer.from
+
+ # Notifies shuttle team and localization team about suspicious source strings.
+ #
+ # @return [Mail::Message] The email to be delivered.
+
+ def suspicious_source_found(job, suspicious_keys_errors)
+ @job = job
+ @suspicious_keys_errors = suspicious_keys_errors
+
+ @author_name = author_name
+ @author_email = author_email
+ @job_url = job_url
+ @formatted_keys_errors = formatted_keys_errors
+
+ mail_addresses = [Shuttle::Configuration.mailer.from, Shuttle::Configuration.mailer.localization_list]
+ if Rails.env.production?
+ subject = t('mailer.fencer_validation.suspicious_translation_found.subject.production')
+ else
+ subject = t('mailer.fencer_validation.suspicious_translation_found.subject.staging')
+ end
+ mail to: mail_addresses, subject: subject
+ end
+
+ def author_name
+ case @job.project.job_type
+ when 'commit'
+ @job.author || author_email
+ when 'article'
+ author_email
+ when 'asset'
+ author_email
+ else
+ raise ArgumentError, "job type not supported: #{job.project.job_type}"
+ end
+ end
+
+ def author_email
+ case @job.project.job_type
+ when 'commit'
+ @job.author_email
+ when 'article'
+ @job.email
+ when 'asset'
+ @job.email
+ else
+ raise ArgumentError, "job type not supported: #{job.project.job_type}"
+ end
+ end
+
+ def job_url
+ case @job.project.job_type
+ when 'commit'
+ project_commit_url(@job.project, @job)
+ when 'article'
+ api_v1_project_article_url(@job.project, @job.name)
+ when 'asset'
+ project_asset_url(@job.project, @job)
+ else
+ raise ArgumentError, "job type not supported: #{job.project.job_type}"
+ end
+ end
+
+ def formatted_keys_errors
+ @suspicious_keys_errors.map do |key, reason|
+ [
+ project_key_translation_url(key.project, key, key.translations.first),
+ reason
+ ]
+ end
+ end
+end
diff --git a/app/mediators/translation_update_mediator.rb b/app/mediators/translation_update_mediator.rb
index 45069d6e..ec797b75 100644
--- a/app/mediators/translation_update_mediator.rb
+++ b/app/mediators/translation_update_mediator.rb
@@ -45,13 +45,11 @@ def update!
copy_to_translations = translations_that_should_be_multi_updated
return if failure?
- ActiveRecord::Base.observers.disable :translation_observer do
- Translation.transaction do
- copy_to_translations.each do |translation|
- update_single_translation!(translation)
- end
- update_single_translation!(@primary_translation)
+ Translation.transaction do
+ copy_to_translations.each do |translation|
+ update_single_translation!(translation)
end
+ update_single_translation!(@primary_translation)
end
check_and_invoke_article_pinger
@@ -141,7 +139,11 @@ def update_single_translation!(translation)
if (translation.copy || "").empty? && !@params[:blank_string].parse_bool
untranslate(translation)
else
- translation.translator = @user if translation.copy != translation.copy_was
+ if translation.copy != translation.copy_was
+ if translation.translator.nil? || @user.translator_only?
+ translation.translator = @user
+ end
+ end
if @user.reviewer?
translation.reviewer = @user
translation.review_date = Time.now
@@ -150,8 +152,8 @@ def update_single_translation!(translation)
end
end
+ translation.updating_params = @params.merge(is_edit: is_edit) # for creating TranslationChange
translation.save!
- TranslationChange.create_from_params!(translation, @params, is_edit)
end
# Untranslates a translation, but doesn't call `save`.
diff --git a/app/middleware/chewy_atomic.rb b/app/middleware/chewy_atomic.rb
new file mode 100644
index 00000000..0f91ac4d
--- /dev/null
+++ b/app/middleware/chewy_atomic.rb
@@ -0,0 +1,11 @@
+# Wraps Sidekiq jobs with a `Chewy.strategy(:atomic)` call.
+
+class ChewyAtomic
+ def initialize(options=nil)
+
+ end
+
+ def call(_worker, _msg, _queue)
+ Chewy.strategy(:atomic) { yield }
+ end
+end
diff --git a/app/middleware/health_check.rb b/app/middleware/health_check.rb
index d0ed559c..d18140a9 100644
--- a/app/middleware/health_check.rb
+++ b/app/middleware/health_check.rb
@@ -12,7 +12,7 @@ def call(env)
if env['ORIGINAL_FULLPATH'] == '/_status'
db = Project.connection.select_all('SELECT CURRENT_TIME') rescue nil
redis = (Shuttle::Redis.get('s') || true) rescue nil
- elasticsearch = Elasticsearch::Model.search({size: 1}, Translation).results.first rescue nil
+ elasticsearch = TranslationsIndex.limit(1).first rescue nil
status = (db && redis && elasticsearch) ? 'OK' : 'error'
json = {
diff --git a/app/models/commit.rb b/app/models/commit.rb
index 2db7f0dc..c5a0d064 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -92,33 +92,8 @@ class Commit < ActiveRecord::Base
alias_method :active_keys, :keys # called in ArticleOrCommitStats
alias_method :active_issues, :issues # called in ArticleOrCommitIssuesPresenter
- include Elasticsearch::Model
- include Elasticsearch::Model::Callbacks
- include IndexHelper
- Commit.index_name "shuttle_#{Rails.env}_commits"
-
- mapping do
- indexes :project_id, type: 'integer'
- indexes :user_id, type: 'integer'
- indexes :priority, type: 'integer'
- indexes :due_date, type: 'date'
- indexes :created_at, type: 'date'
- indexes :revision, index: :not_analyzed, type: 'string'
- indexes :loading, type: 'boolean'
- indexes :ready, type: 'boolean'
- indexes :exported, type: 'boolean'
- indexes :fingerprint, type: 'string'
- indexes :duplicate, type: 'boolean'
- end
-
- def regular_index_fields
- %w(project_id user_id priority due_date created_at revision ready exported loading fingerprint duplicate)
- end
-
- def special_index_fields
- {
- key_ids: commits_keys.pluck(:key_id)
- }
+ update_index('commits#commit') do
+ self unless previous_changes.blank? && persisted?
end
validates :project,
@@ -200,10 +175,16 @@ def revision_prefix
# approved_at to be the current time.
def recalculate_ready!
+ already_ready = self.ready
self.ready = successfully_loaded? && keys_are_ready? && !errored_during_import?
self.approved_at = Time.current if self.ready && self.approved_at.nil?
- index_elasticsearch_document
save!
+
+ # records metric only when becoming ready
+ if !already_ready and self.ready and self.approved_at and self.created_at
+ ready_time = self.approved_at - self.created_at
+ CustomMetricHelper.record_project_ready_time(self.project.slug, ready_time)
+ end
end
# Returns `true` if all Translations applying to this commit have been
diff --git a/app/models/concerns/index_helper.rb b/app/models/concerns/index_helper.rb
deleted file mode 100644
index 1722b3c6..00000000
--- a/app/models/concerns/index_helper.rb
+++ /dev/null
@@ -1,49 +0,0 @@
-# Copyright 2016 Square Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Include this module in model after including `Elasticsearch::Model`.
-# Define INDEX_FIELD array including regular fields that are able to reuse
-# rails active record methods to retrieve value from database. Create method
-# `special_index_fields` to return a hash including all special fileds
-
-module IndexHelper
-
- def self.included(base)
- base.extend ClassMethods
- end
-
- def regular_index_fields
- []
- end
-
- # override Elasticsearch::Model::Serializing.as_indexed_json method to only include mapped fields
- def as_indexed_json(_options={})
- Hash[regular_index_fields.map { |field| [field, self.send(field)] }].merge(special_index_fields.stringify_keys)
- end
-
- def update_elasticsearch_index
- __elasticsearch__.update_document
- end
-
- def index_elasticsearch_document
- __elasticsearch__.index_document
- end
-
- module ClassMethods
- def refresh_elasticsearch_index!
- __elasticsearch__.refresh_index!
- end
- end
-end
\ No newline at end of file
diff --git a/app/models/group.rb b/app/models/group.rb
index 34e121db..7b2bb6a6 100644
--- a/app/models/group.rb
+++ b/app/models/group.rb
@@ -78,4 +78,17 @@ def recalculate_ready!
self.save!
end
+ # Inherited from ArticleOrCommitStats
+ include ArticleOrCommitStats
+ def translations_not_done(*locales)
+ articles.map { |article| article.translations_not_done(*locales) }.sum
+ end
+
+ def translations_done(*locales)
+ articles.map { |article| article.translations_done(*locales) }.sum
+ end
+
+ def translations_total(*locales)
+ articles.map { |article| article.translations_total(*locales) }.sum
+ end
end
diff --git a/app/models/key.rb b/app/models/key.rb
index c7887c8a..a0640633 100644
--- a/app/models/key.rb
+++ b/app/models/key.rb
@@ -137,32 +137,11 @@ def apply_readiness_hooks?() !skip_readiness_hooks end
digest_field :key, scope: :for_key
digest_field :source_copy, scope: :source_copy_matches
- include Elasticsearch::Model
- include Elasticsearch::Model::Callbacks
- include IndexHelper
- Key.index_name "shuttle_#{Rails.env}_keys"
-
- settings analysis: {tokenizer: {key_tokenizer: {type: 'pattern', pattern: '[^A-Za-z0-9]'}},
- analyzer: {key_analyzer: {type: 'custom', tokenizer: 'key_tokenizer', filter: 'lowercase'}}}
-
- mapping do
- indexes :original_key, type: 'multi_field', fields: {
- original_key: {type: 'string', analyzer: 'key_analyzer'},
- original_key_exact: {type: 'string', index: :not_analyzed}
- }
- indexes :project_id, type: 'integer'
- indexes :ready, type: 'boolean'
- indexes :hidden_in_search, type: 'boolean'
+ update_index('keys#key') do
+ self unless previous_changes.blank? && persisted?
end
-
- def regular_index_fields
- %w(original_key project_id ready)
- end
-
- def special_index_fields
- {
- hidden_in_search: formatted_hidden_in_search
- }
+ update_index('translations#translation') do
+ translations.reload unless previous_changes.blank?
end
validates :project,
@@ -220,39 +199,47 @@ def importer_name() importer_class.try(:human_name) end
# to create pending Translation requests for each string in the new locale.
def add_pending_translations(model = nil)
- locales = model && model.targeted_locales ? model.targeted_locales : targeted_locales
- base = model && model.base_locale ? model.base_locale : base_locale
-
- translations.in_locale(base).find_or_create!(
- source_copy: source_copy,
- copy: source_copy,
- source_locale: base,
- locale: base,
- approved: true,
- preserve_reviewed_status: true,
- )
-
- locales.each do |locale|
- next if skip_key?(locale)
- t = translations.in_locale(locale).find_or_create!(
- source_copy: source_copy,
- source_locale: base_locale,
- locale: locale,
+ Chewy.strategy(:atomic) do
+ locales = model && model.targeted_locales ? model.targeted_locales : targeted_locales
+ base = model && model.base_locale ? model.base_locale : base_locale
+
+ translations.in_locale(base).find_or_create!(
+ source_copy: source_copy,
+ copy: source_copy,
+ source_locale: base,
+ locale: base,
+ approved: true,
+ preserve_reviewed_status: true,
)
- if is_auto_approved_string?(source_copy) || (article.present? && is_block_tag)
- t.update!(
- copy: source_copy,
- approved: true,
- translated: true,
- preserve_reviewed_status: true,
+ locales.each do |locale|
+ next if skip_key?(locale)
+
+ translation_updated = false
+ t = translations.in_locale(locale).find_or_create!(
+ source_copy: source_copy,
+ source_locale: base_locale,
+ locale: locale,
)
+ translation_updated ||= t.previous_changes.present?
+
+ if is_auto_approved_string?(source_copy) || (article.present? && is_block_tag)
+ t.update!(
+ copy: source_copy,
+ approved: true,
+ translated: true,
+ preserve_reviewed_status: true,
+ )
+ translation_updated ||= t.previous_changes.present?
+ end
+
+ if translation_updated && !t.approved?
+ finder = FuzzyMatchTranslationsFinder.new(source_copy, t)
+ t.update(
+ tm_match: finder.top_fuzzy_match_percentage
+ )
+ end
end
-
- finder = FuzzyMatchTranslationsFinder.new(source_copy, t)
- t.update(
- tm_match: finder.top_fuzzy_match_percentage
- )
end
end
@@ -263,9 +250,11 @@ def add_pending_translations(model = nil)
# only pending Translations.
def remove_excluded_pending_translations
- translations.not_base.not_translated.where(approved: nil).find_each do |translation|
- if skip_key?(translation.locale) || !targeted_locales.include?(translation.locale)
- translation.destroy
+ Chewy.strategy(:atomic) do
+ translations.not_base.not_translated.where(approved: nil).find_each do |translation|
+ if skip_key?(translation.locale) || !targeted_locales.include?(translation.locale)
+ translation.destroy
+ end
end
end
end
@@ -287,18 +276,12 @@ def recalculate_ready!
# @param [Commit, Project, Article] obj The object whose keys should be batch recalculated
def self.batch_recalculate_ready!(obj)
- not_ready_key_ids = obj.translations.in_locale(*obj.required_locales).not_approved.not_block_tag.select(:key_id).uniq.pluck(:key_id)
- ready_key_ids = obj.keys.pluck(:id) - not_ready_key_ids
- ready_key_ids.in_groups_of(500, false) { |group| Key.where(id: group).update_all(ready: true) }
- not_ready_key_ids.in_groups_of(500, false) { |group| Key.where(id: group).update_all(ready: false) }
-
- # Since update_all bypasses all callbacks, `ready` field for some Keys should be out of sync in ElasticSearch at this point.
- # We need to update ElasticSearch with the new ready fields.
- obj.keys.each { |key| key.update_elasticsearch_index }
- end
-
- def formatted_hidden_in_search
- !!hidden_in_search
+ Chewy.strategy(:atomic) do
+ not_ready_key_ids = obj.translations.in_locale(*obj.required_locales).not_approved.not_block_tag.select(:key_id).uniq.pluck(:key_id)
+ ready_key_ids = obj.keys.pluck(:id) - not_ready_key_ids
+ ready_key_ids.in_groups_of(500, false) { |group| Key.where(id: group).update_all(ready: true) }
+ not_ready_key_ids.in_groups_of(500, false) { |group| Key.where(id: group).update_all(ready: false) }
+ end
end
# checks if a string should be auto approved.
diff --git a/app/models/report.rb b/app/models/report.rb
new file mode 100644
index 00000000..22168966
--- /dev/null
+++ b/app/models/report.rb
@@ -0,0 +1,2 @@
+class Report < ActiveRecord::Base
+end
diff --git a/app/models/section.rb b/app/models/section.rb
index df88e119..90a8c4a5 100644
--- a/app/models/section.rb
+++ b/app/models/section.rb
@@ -67,6 +67,8 @@ class Section < ActiveRecord::Base
digest_field :name, scope: :for_name
digest_field :source_copy, scope: :source_copy_matches
+ update_index('translations#translation') { translations.reload }
+
validates :name, presence: true, uniqueness: { scope: :article_id }
validates :source_copy, presence: true
validates :article, presence: true, strict: true
diff --git a/app/models/translation.rb b/app/models/translation.rb
index d2edf55c..8b43c45b 100644
--- a/app/models/translation.rb
+++ b/app/models/translation.rb
@@ -49,7 +49,7 @@ class Translation < ActiveRecord::Base
belongs_to :key, inverse_of: :translations
belongs_to :translator, class_name: 'User', foreign_key: 'translator_id', inverse_of: :authored_translations
belongs_to :reviewer, class_name: 'User', foreign_key: 'reviewer_id', inverse_of: :reviewed_translations
- has_many :translation_changes, inverse_of: :translation, dependent: :delete_all
+ has_many :translation_changes, inverse_of: :translation, dependent: :destroy
has_many :issues, inverse_of: :translation, dependent: :destroy
has_many :commits_keys, primary_key: :key_id, foreign_key: :key_id
has_many :assets_keys, primary_key: :key_id, foreign_key: :key_id
@@ -61,44 +61,8 @@ class Translation < ActiveRecord::Base
locale_field :source_locale, from: :source_rfc5646_locale
locale_field :locale
- include Elasticsearch::Model
- include Elasticsearch::Model::Callbacks
- include IndexHelper
- Translation.index_name "shuttle_#{Rails.env}_translations"
-
- def regular_index_fields
- %w(copy source_copy id translator_id rfc5646_locale created_at updated_at translated)
- end
- mapping do
- indexes :copy, analyzer: 'snowball', type: 'string'
- indexes :source_copy, analyzer: 'snowball', type: 'string'
- indexes :id, type: 'integer', index: :not_analyzed
- indexes :project_id, type: 'integer'
- indexes :article_id, type: 'integer'
- indexes :section_id, type: 'integer'
- indexes :is_block_tag, type: 'boolean'
- indexes :section_active, type: 'boolean'
- indexes :index_in_section, type: 'integer'
- indexes :translator_id, type: 'integer'
- indexes :rfc5646_locale, type: 'string', index: :not_analyzed
- indexes :created_at, type: 'date'
- indexes :updated_at, type: 'date'
- indexes :translated, type: 'boolean'
- indexes :approved, type: 'integer'
- indexes :hidden_in_search, type: 'boolean'
- end
-
- def special_index_fields
- {
- project_id: self.key.project_id,
- article_id: self.key.section.try(:article_id),
- section_id: self.key.section_id,
- is_block_tag: self.key.is_block_tag,
- section_active: self.key.section.try(:active),
- index_in_section: self.key.index_in_section,
- approved: if approved==true then 1 elsif approved==false then 0 else nil end,
- hidden_in_search: self.key.formatted_hidden_in_search
- }
+ update_index('translations#translation') do
+ self unless previous_changes.blank? && persisted?
end
validates :key,
@@ -130,6 +94,8 @@ def special_index_fields
# @private
# @return [User] The person who changed this Translation
attr_accessor :modifier
+ # passing updating params to create TranslationChange
+ attr_accessor :updating_params
# TODO: Fold this into DailyMetric?
def self.total_words_per_project
@@ -271,19 +237,4 @@ def fences_must_match
# ` "べ"[1..27].hash == "".hash ` returns false
# ` "べ"[1..27] == "" ` returns true
end
-
- # TODO elasticsearch-rails doesn't have batch import, we should update this method once we find better solution
- # Batch updates `obj`s Translations in elastic search.
- # Expects `obj` to have a `keys` association.
- # This is currently only run in ArticleImporter::Finisher.
- #
- # @param [Commit, Project, Article, Asset] obj The object whose translations should be batch refreshed in ElasticSearch
-
- def self.batch_refresh_elastic_search(obj)
- obj.keys.includes(:translations, :section).find_in_batches do |keys|
- keys.map(&:translations).flatten.each do |translation|
- translation.update_elasticsearch_index
- end
- end
- end
end
diff --git a/app/models/translation_change.rb b/app/models/translation_change.rb
index ca2d9024..e3baf98c 100644
--- a/app/models/translation_change.rb
+++ b/app/models/translation_change.rb
@@ -35,7 +35,7 @@ class TranslationChange < ActiveRecord::Base
belongs_to :project
belongs_to :article
belongs_to :asset
- has_many :edit_reasons
+ has_many :edit_reasons, dependent: :delete_all
has_many :reasons, through: :edit_reasons
serialize :diff, Hash
@@ -57,11 +57,10 @@ class TranslationChange < ActiveRecord::Base
}
def self.create_from_translation!(translation)
- diff = translation.previous_changes.slice(*TRACKED_ATTRIBUTES)
- TranslationChange.create(translation: translation, user: translation.modifier, diff: diff) if diff.present?
+ create_from_params!(translation, translation.updating_params || {})
end
- def self.create_from_params!(translation, params, is_edit)
+ def self.create_from_params!(translation, params)
diff = translation.previous_changes.slice(*TRACKED_ATTRIBUTES)
if diff.present?
project_id = Project.joins(:slugs).where('slugs.slug = ?', params[:project_id]).pluck('projects.id').first
@@ -73,8 +72,8 @@ def self.create_from_params!(translation, params, is_edit)
translation: translation,
user: translation.modifier,
diff: diff,
- role: translation.modifier.role,
- is_edit: is_edit,
+ role: translation.modifier&.role,
+ is_edit: params[:is_edit],
tm_match: translation.tm_match,
sha: commit,
article_id: article_id,
diff --git a/app/models/user.rb b/app/models/user.rb
index 9bb8f7f4..c7f32410 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -60,7 +60,7 @@ class User < ActiveRecord::Base
ROLES = %w(monitor translator reviewer admin)
devise :database_authenticatable, :registerable, :confirmable,
- :recoverable, :rememberable, :trackable, :validatable, :lockable
+ :recoverable, :rememberable, :trackable, :validatable, :lockable, :expirable
has_many :authored_translations, class_name: 'Translation', foreign_key: 'translator_id', inverse_of: :translator, dependent: :nullify
has_many :reviewed_translations, class_name: 'Translation', foreign_key: 'reviewer_id', inverse_of: :reviewer, dependent: :nullify
@@ -107,6 +107,15 @@ def after_confirmation
# @private Used by Devise.
def active_for_authentication?() super && role? end
+ def self.reset_password_by_token(attributes={})
+ recoverable = super(attributes)
+
+ # re-enable expired account caused by inactivity
+ recoverable.update_last_activity! if recoverable.persisted?
+
+ recoverable
+ end
+
# @return [String] The User's full name.
def name() I18n.t 'models.user.name', first: first_name, last: last_name end
# @return [String] An abbreviated name for the user.
@@ -115,6 +124,7 @@ def abbreviated_name() I18n.t('models.user.name', first: first_name, last: last_
ROLES.each do |role|
define_method(:"#{role}?") { self.role == role || self.role == 'admin' }
+ define_method(:"#{role}_only?") { self.role == role }
end
# @private
diff --git a/app/presenters/locale_projects_show_presenter.rb b/app/presenters/locale_projects_show_presenter.rb
index 5a90f929..6ae50c16 100644
--- a/app/presenters/locale_projects_show_presenter.rb
+++ b/app/presenters/locale_projects_show_presenter.rb
@@ -53,6 +53,12 @@ def selected_asset
@_selected_asset ||= @project.assets.find_by_id(form[:asset_id])
end
+ # @return [Group, nil] selected Group if there is one
+
+ def selected_group
+ @_selected_group ||= @project.groups.where(display_name: form[:group]).first
+ end
+
# @return [Array>] an array of selectable options for Sections
def selectable_sections
diff --git a/app/services/commits_search_keys_finder.rb b/app/services/commits_search_keys_finder.rb
index 0391defa..28b45f71 100644
--- a/app/services/commits_search_keys_finder.rb
+++ b/app/services/commits_search_keys_finder.rb
@@ -16,7 +16,6 @@
class CommitsSearchKeysFinder
attr_reader :commit, :form
- include Elasticsearch::DSL
# The number of records to return by default.
PER_PAGE = 50
@@ -27,39 +26,18 @@ def initialize(form, commit)
end
def search_query
- query_filter = form[:filter]
- status = form[:status]
- key_ids = commit.keys.pluck(:id)
- current_page = page
+ query_params = []
+ query_params << { match: { original_key: { query: form[:filter], operator: 'and'} } } if form[:filter].present?
- search {
- query do
- filtered do
- if query_filter.present?
- query do
- match 'original_key' do
- query query_filter
- operator 'and'
- end
- end
- end
- filter do
- bool do
- must { ids values: key_ids }
- case status
- when 'approved'
- must { term ready: 1 }
- when 'pending'
- must { term ready: 0 }
- end
- end
- end
- end
- end
- sort { by :original_key_exact, order: 'asc', ignore_unmapped: true } unless query_filter
- from (current_page - 1) * PER_PAGE
- size PER_PAGE
- }.to_hash
+ filter_params = []
+ filter_params << { ids: { values: commit.keys.pluck(:id) } }
+ filter_params << { term: { ready: true } } if form[:status] == 'approved'
+ filter_params << { term: { ready: false } } if form[:status] == 'pending'
+
+ query = KeysIndex.query(query_params).filter(filter_params).offset((page - 1) * PER_PAGE).limit(PER_PAGE)
+ query = query.order(original_key_exact: { order: :asc }) unless form[:filter]
+
+ return query
end
def page
@@ -67,14 +45,7 @@ def page
end
def find_keys
- keys_in_es = Elasticsearch::Model.search(search_query, Key).results
-
- keys = Key.where(id: keys_in_es.map(&:id))
- .where('commits_keys.commit_id': commit.id)
- .includes(:translations, :commits_keys)
- .order('commits_keys.created_at asc, original_key asc')
-
- # Don't sort the keys since they are sorted in the line above
- PaginatableObjects.new(keys, keys_in_es, page, PER_PAGE, false)
+ keys = search_query.load(scope: -> { includes(:translations, :commits_keys) })
+ return PaginatableObjects.new(keys, page, PER_PAGE)
end
end
diff --git a/app/services/fuzzy_match_translations_finder.rb b/app/services/fuzzy_match_translations_finder.rb
index b61e7c33..132d1d39 100644
--- a/app/services/fuzzy_match_translations_finder.rb
+++ b/app/services/fuzzy_match_translations_finder.rb
@@ -14,9 +14,10 @@
class FuzzyMatchTranslationsFinder
attr_reader :translation
- include Elasticsearch::DSL
- MINIMUM_FUZZY_MATCH = 60
+ FUZZY_MATCH_MIN_SCORE = 60
+ FUZZY_MATCH_RESULT_SIZE = 5
+ ES_SEARCH_BATCH_SIZE = 5
def initialize(query_filter, translation)
@query_filter = query_filter
@@ -24,36 +25,25 @@ def initialize(query_filter, translation)
end
def search_query
- limit = 5
- query_filter = @query_filter
- target_locales = translation.locale.fallbacks.map(&:rfc5646)
- search {
- query do
- filtered do
- query do
- match 'source_copy' do
- query query_filter
- operator 'or'
- end
- end
- filter do
- bool do
- must { term approved: 1 }
- must { terms rfc5646_locale: target_locales }
- must { term hidden_in_search: false }
- end
- end
- end
- end
+ query_params = []
+ query_params << { match: { source_copy: { query: @query_filter, operator: 'or' } } }
- size limit
- }.to_hash
+ filter_params = []
+ filter_params << { term: { approved: true } }
+ filter_params << { terms: { rfc5646_locale: translation.locale.fallbacks.map(&:rfc5646) } }
+ filter_params << { term: { hidden_in_search: false } }
+
+ query = TranslationsIndex.query(query_params).filter(filter_params)
+ query = query.limit(ES_SEARCH_BATCH_SIZE)
+
+ return query
end
def find_fuzzy_match
- translations_in_es = Elasticsearch::Model.search(search_query, Translation).results
- translations = Translation.where(id: translations_in_es.map(&:id)).includes(key: :project)
- SortingHelper.order_by_elasticsearch_result_order(translations, translations_in_es)
+ translations = search_query.load(scope: -> { includes(key: :project) }).objects
+ translations = translations.reject { |t| @query_filter.similar(t.source_copy) < FUZZY_MATCH_MIN_SCORE }
+ translations = translations.sort { |a, b| @query_filter.similar(b.source_copy) <=> @query_filter.similar(a.source_copy) }
+ translations.first(FUZZY_MATCH_RESULT_SIZE).map { |t| Translation.find(t.id) }
end
def top_fuzzy_match_percentage
@@ -62,7 +52,7 @@ def top_fuzzy_match_percentage
{
match_percentage: @translation.source_copy.similar(tran.source_copy),
}
- end.reject { |t| t[:match_percentage] < MINIMUM_FUZZY_MATCH }
+ end.reject { |t| t[:match_percentage] < FUZZY_MATCH_MIN_SCORE }
translations.sort! { |a, b| b[:match_percentage] <=> a[:match_percentage] }
translations.any? ? translations.first[:match_percentage] : 0.0
end
diff --git a/app/services/home_index_items_finder.rb b/app/services/home_index_items_finder.rb
index 0d97ec33..026e5bd0 100644
--- a/app/services/home_index_items_finder.rb
+++ b/app/services/home_index_items_finder.rb
@@ -16,7 +16,6 @@
class HomeIndexItemsFinder
attr_reader :user, :form
- include Elasticsearch::DSL
def initialize(user, form)
@user = user
@@ -24,91 +23,58 @@ def initialize(user, form)
end
def search_query
- # FILTERS AND SORTING
- status = form[:filter__status]
- locales = form[:filter__locales]
- sort_field = form[:sort__field]
- sort_direction = form[:sort__direction]
- sha = form[:commits_filter__sha]
- project_id = form[:commits_filter__project_id]
- hide_exported = form[:commits_filter__hide_exported]
- hide_autoimported = form[:commits_filter__hide_autoimported]
- show_only_mine = form[:commits_filter__show_only_mine]
- hide_duplicates = form[:commits_filter__hide_duplicates]
-
- # PAGINATION
- offset = form[:offset]
- limit = form[:limit]
-
- # UNCOMPLETED IN SPECIFIC LOCALES
- if locales.present? && (status == 'uncompleted')
- uncompleted_key_ids_in_locales = uncompleted_key_ids_in_locales()
- end
-
- search {
- query do
- filtered do
- filter do
- bool do
- must { prefix revision: sha } if sha
- must { term project_id: project_id } unless project_id == 'all'
- must { term exported: false } if hide_exported
- must { exists field: :user_id } if hide_autoimported
- must { term user_id: 1 } if show_only_mine
- must { term loading: false }
- must { term duplicate: false } if hide_duplicates
-
- case status
- when 'uncompleted'
- locales.present? ? must { terms key_ids: uncompleted_key_ids_in_locales } : must { term ready: false }
- when 'completed'
- must { term ready: true }
- when 'hidden'
- # Do nothing as Commit has no such state. We have this to line up with Article since
- # Commit and Article share the same frontend template.
- must { match_all }
- when 'all'
- must { match_all }
+ commits = CommitsIndex.filter(bool: {must: [
+ form[:commits_filter__sha] ? {prefix: {revision: form[:commits_filter__sha]}} : nil,
+ form[:commits_filter__project_id] == 'all' ? nil : {term: {project_id: form[:commits_filter__project_id]}},
+ form[:commits_filter__hide_exported] ? {term: {exported: false}} : nil,
+ form[:commits_filter__hide_autoimported] ? {exists: {field: :user_id}} : nil,
+ form[:commits_filter__show_only_mine] ? {term: {user_id: 1}} : nil,
+ {term: {loading: false}},
+ form[:commits_filter__hide_duplicates] ? {term: {duplicate: :false}} : nil,
+ (case form[:filter__status]
+ when 'uncompleted'
+ if form[:filter__locales].present?
+ {terms: {key_ids: uncompleted_key_ids_in_locales}}
+ else
+ {term: {ready: false}}
+ end
+ when 'completed'
+ {term: {ready: true}}
+ when 'hidden'
+ # Do nothing as Commit has no such state. We have this to line up with Article since
+ # Commit and Article share the same frontend template.
+ nil
+ when 'all'
+ nil
+ end)
+ ].compact}).
+ offset(form[:offset]).limit(form[:limit])
+
+ commits = case form[:sort__field]
+ when 'due'
+ commits.order(due_date: (form[:sort__direction].nil? ? 'asc' : form[:sort__direction]),
+ priority: 'asc',
+ created_at: 'desc')
+ when 'create'
+ commits.order(created_at: (form[:sort__direction].nil? ? 'desc' : form[:sort__direction]),
+ priority: 'asc',
+ due_date: 'asc')
+ else
+ commits.order(priority: (form[:sort__direction].nil? ? 'asc' : form[:sort__direction]),
+ due_date: 'asc',
+ created_at: 'desc')
end
- end
- end
- end
- end
- from offset
- size limit
- sort do
- case sort_field
- when 'due'
- by :due_date, order: sort_direction.nil? ? 'asc' : sort_direction
- by :priority, order: 'asc'
- by :created_at, order: 'desc'
- when 'create'
- by :created_at, order: sort_direction.nil? ? 'desc' : sort_direction
- by :priority, order: 'asc'
- by :due_date, order: 'asc'
- else
- by :priority, order: sort_direction.nil? ? 'asc' : sort_direction
- by :due_date, order: 'asc'
- by :created_at, order: 'desc'
- end
- end
- }.to_hash
+ return commits
end
def find_commits
- #Search
- commits_in_es = Elasticsearch::Model.search(search_query, Commit).results
-
- # LOAD
- commits = Commit
- .where(id: commits_in_es.map(&:id))
- .includes(:user, project: :slugs)
- PaginatableObjects.new(commits, commits_in_es, form[:page], form[:limit])
+ commits = search_query.load(scope: -> { includes(:user, project: :slugs) })
+ PaginatableObjects.new(commits, form[:page], form[:limit])
end
def find_articles
- articles = Article.includes(:project).showing
+ articles = Article.includes(:project, :groups).showing
# filter by name
articles = articles.for_name(form[:articles_filter__name]) if form[:articles_filter__name]
@@ -136,7 +102,7 @@ def find_articles
when 'due'
"due_date #{direction || 'asc'}"
when 'create'
- "created_at #{direction || 'desc'}"
+ "last_import_requested_at #{direction || 'desc'}"
when 'priority'
"priority #{direction || 'asc'}"
end
@@ -188,7 +154,7 @@ def find_assets
end
def find_groups
- groups = Group.includes(:project).showing.joins(:articles)
+ groups = Group.includes(:project, :articles).showing.joins(:articles)
# filter by name
groups = groups.where("groups.display_name like '%#{form[:groups_filter__name]}%'") if form[:groups_filter__name]
@@ -214,11 +180,11 @@ def find_groups
direction = %w(asc desc).include?(form[:sort__direction]) ? form[:sort__direction] : nil
order_by = case form[:sort__field]
when 'due'
- "due_date #{direction || 'asc'}"
+ "groups.due_date #{direction || 'asc'}"
when 'create'
- "created_at #{direction || 'desc'}"
+ "groups.created_at #{direction || 'desc'}"
when 'priority'
- "priority #{direction || 'asc'}"
+ "groups.priority #{direction || 'asc'}"
end
groups = groups.order(order_by) if order_by
diff --git a/app/services/locale_projects_show_finder.rb b/app/services/locale_projects_show_finder.rb
index 2a3a5fd4..becd6eaf 100644
--- a/app/services/locale_projects_show_finder.rb
+++ b/app/services/locale_projects_show_finder.rb
@@ -15,7 +15,6 @@
class LocaleProjectsShowFinder
attr_reader :form
- include Elasticsearch::DSL
PER_PAGE = 50
@@ -24,92 +23,39 @@ def initialize(form)
end
def search_query
- include_translated = form[:include_translated]
- include_approved = form[:include_approved]
- include_new = form[:include_new]
- include_block_tags = form[:include_block_tags]
-
- current_page = page
- query_filter = form[:query_filter]
- translation_ids_in_commit = form[:translation_ids_in_commit]
- article_id = form[:article_id]
- section_id = form[:section_id]
- translation_ids_in_assest = form[:translation_ids_in_assest]
- locale = form[:locale]
- project_id = form[:project_id]
- project = form[:project]
- filter_source = form[:filter_source]
+ query_params = []
+ text_to_search = form[:query_filter]
+ if text_to_search.present?
+ if form[:filter_source] == 'source'
+ query_params << { match: { source_copy: {query: text_to_search, operator: 'and' } } }
+ elsif form[:filter_source] == 'translated'
+ query_params << { match: { copy: {query: text_to_search, operator: 'and'} } }
+ end
+ end
- search {
- query do
- filtered do
- if query_filter.present?
- if filter_source == 'source'
- query do
- match 'source_copy' do
- query query_filter
- operator 'and'
- end
- end
- elsif filter_source == 'translated'
- query do
- match 'copy' do
- query query_filter
- operator 'and'
- end
- end
- end
- end
+ filter_params = []
+ filter_params << { term: { project_id: form[:project_id] } }
+ filter_params << { term: { rfc5646_locale: form[:locale].rfc5646 } } if form[:locale]
+ filter_params << { ids: { values: form[:translation_ids_in_commit] } } if form[:translation_ids_in_commit]
+ filter_params << { term: { article_id: form[:article_id] } } if form[:article_id].present?
+ filter_params << { ids: { values: form[:translation_ids_in_assest] } } if form[:translation_ids_in_assest]
+ filter_params << { term: { section_id: form[:section_id] } } if form[:section_id].present?
+ filter_params << { term: { section_active: true } } if form[:project].article?
+ filter_params << { exists: { field: :index_in_section } } if form[:project].article?
+ filter_params << { bool: { must_not: { term: { is_block_tag: true } } } } if form[:project].article? && !form[:include_block_tags]
- filter do
- bool do
- must { term project_id: project_id }
- must { term rfc5646_locale: locale.rfc5646 } if locale
- must { ids values: translation_ids_in_commit } if translation_ids_in_commit
- must { term article_id: article_id } if article_id.present?
- must { ids values: translation_ids_in_assest } if translation_ids_in_assest
- must { term section_id: section_id } if section_id.present?
- must { term section_active: true } if project.article? # active sections
- must { exists field: :index_in_section } if project.article? # active keys in sections
- must_not { term is_block_tag: true } if project.article? && !include_block_tags
+ state_params = []
+ state_params << TranslationsIndex::TRANSLATION_STATE_APPROVED if form[:include_approved]
+ state_params << TranslationsIndex::TRANSLATION_STATE_TRANSLATED if form[:include_translated]
+ state_params << TranslationsIndex::TRANSLATION_STATE_NEW if form[:include_new]
+ state_params << TranslationsIndex::TRANSLATION_STATE_REJECTED if form[:include_new]
+ filter_params << { terms: { translation_state: state_params } } unless state_params.empty?
- if include_translated && include_approved && include_new
- #include everything
- elsif include_translated && include_approved
- must { term translated: 1 }
- elsif include_translated && include_new
- should { missing field: 'approved', existence: true, null_value: true }
- should { term approved: 0 }
- elsif include_approved && include_new
- should { term approved: 1 }
- should { term translated: 0 }
- elsif include_approved
- must { term approved: 1 }
- elsif include_new
- should { term translated: 0 }
- should { term approved: 0 }
- elsif include_translated
- must { missing field: 'approved', existence: true, null_value: true }
- must { term translated: 1 }
- else
- # include nothing
- throw :include_nothing
- end
- end
- end
- end
- end
+ query = TranslationsIndex.query(query_params).filter(filter_params)
+ query = query.order(section_id: :asc, index_in_section: :asc) if form[:project].article?
+ query = query.offset((page-1) * PER_PAGE).limit(PER_PAGE)
- if project.article?
- sort do
- by :section_id, order: 'asc'
- by :index_in_section, order: 'asc'
- end
- end
-
- from (current_page - 1) * PER_PAGE
- size PER_PAGE
- }.to_hash
+ return query
end
def page
@@ -117,23 +63,18 @@ def page
end
def find_translations
- translations_in_es = Elasticsearch::Model.search(search_query, Translation).results
- translations = Translation
- .where(id: translations_in_es.map(&:id))
- .where('commits.revision': form[:commit])
- .includes({key: [:project, :commits, :assets, :translations, :section, {article: :project}]}, :locale_associations, :translation_changes)
+ include_tables = [{ key: [:project, :assets, :translations, :section, { article: [:project, article_groups: :group] }] }, :locale_associations, :translation_changes]
+
+ scope = -> { includes(include_tables) }
if form[:article_id]
- translations = translations.order('keys.section_id, keys.index_in_section')
+ scope = -> { includes(include_tables).order('keys.section_id, keys.index_in_section') }
elsif form[:commit]
- translations = translations.order('commits_keys.created_at, keys.original_key')
- elsif form[:asset_id]
- translations = translations.order('assets_keys.created_at, keys.original_key')
+ scope = -> { includes(include_tables).order('translations.created_at') }
elsif form[:group]
- translations = translations.joins({article: {article_groups: :group}}).where("groups.display_name = ?", form[:group])
- translations = translations.order('article_groups.index_in_group, keys.section_id, keys.index_in_section')
+ scope = -> { includes(include_tables).order('article_groups.index_in_group, keys.section_id, keys.index_in_section') }
end
- # Don't sort the keys since they are sorted in the line above
- PaginatableObjects.new(translations, translations_in_es, page, PER_PAGE, false)
+ translations = search_query.load(scope: scope)
+ return PaginatableObjects.new(translations, page, PER_PAGE)
end
end
diff --git a/app/services/match_translations_finder.rb b/app/services/match_translations_finder.rb
index 753ecdfc..823b8819 100644
--- a/app/services/match_translations_finder.rb
+++ b/app/services/match_translations_finder.rb
@@ -14,37 +14,30 @@
class MatchTranslationsFinder
attr_reader :translation
- include Elasticsearch::DSL
def initialize(translation)
@translation = translation
end
def search_query(rfc5646, source_copy)
- search {
- query do
- filtered do
- filter do
- bool do
- must { term approved: 1 }
- must { term rfc5646_locale: rfc5646 }
- must { term source_copy: source_copy }
- end
- end
- end
- end
- sort { by :created_at, order: 'desc' }
- size 1
- }.to_hash
+ filter_params = []
+ filter_params << { term: { approved: true } }
+ filter_params << { term: { rfc5646_locale: rfc5646 } }
+ filter_params << { term: { source_copy: source_copy } }
+
+ query = TranslationsIndex.filter(filter_params)
+ query = query.order(created_at: :desc)
+ query = query.limit(1)
+
+ return query
end
def find_first_match_translation
source_copy = translation.source_copy
translation.locale.fallbacks.each do |fallback|
- query = search_query(fallback.rfc5646, source_copy)
- first_matched_translation = Elasticsearch::Model.search(query, Translation).results.first
- return first_matched_translation.to_hash["_source"] if first_matched_translation
+ first_matched_translation = search_query(fallback.rfc5646, source_copy).load.objects.first
+ return first_matched_translation if first_matched_translation
end
- nil
+ return nil
end
-end
\ No newline at end of file
+end
diff --git a/app/services/search_commits_finder.rb b/app/services/search_commits_finder.rb
index b59c4746..c5400624 100644
--- a/app/services/search_commits_finder.rb
+++ b/app/services/search_commits_finder.rb
@@ -14,38 +14,26 @@
class SearchCommitsFinder
attr_reader :params
- include Elasticsearch::DSL
def initialize(params)
@params = params
end
def search_query
- sha = params[:sha]
- project_id = params[:project_id].to_i
- limit = params.fetch(:limit, 50)
+ query_params = []
- search {
- query do
- filtered do
- filter do
- bool do
- must { prefix revision: sha } if sha
- must { term project_id: project_id } if project_id > 0
- must { match_all }
- end
- end
- end
- end
+ query_params << { prefix: { revision: params[:sha] } } if params[:sha]
+ query_params << { term: { project_id: params[:project_id].to_i } } if params[:project_id].present?
+ query_params << { bool: { must: { match_all: {} } } }
- size limit
- sort { by :created_at, order: 'desc' }
- }.to_hash
+ query = CommitsIndex.filter(query_params)
+ query = query.order(created_at: :desc)
+ query = query.limit(params.fetch(:limit, 50))
+
+ return query
end
def find_commits
- commits_in_es = Elasticsearch::Model.search(search_query, Commit).results
- commits = Commit.where(id: commits_in_es.map(&:id)).includes(:project)
- SortingHelper.order_by_elasticsearch_result_order(commits, commits_in_es)
+ search_query.load(scope: -> { includes(:project) }).objects
end
-end
\ No newline at end of file
+end
diff --git a/app/services/search_keys_finder.rb b/app/services/search_keys_finder.rb
index 89be55ee..c757cd46 100644
--- a/app/services/search_keys_finder.rb
+++ b/app/services/search_keys_finder.rb
@@ -23,44 +23,29 @@ def initialize(user, params)
end
def search_query
- query_filter = @params[:filter]
- status = @params[:status]
- offset = @params[:offset].to_i
- project_id = @params[:project_id]
- limit = @params.fetch(:limit, PER_PAGE)
- not_elastic = @params[:not_elastic_search]
- hidden_keys = @params[:hidden_in_search]
-
- search {
- query do
- filtered do
- if query_filter.present? && !not_elastic
- query do
- match 'original_key' do
- query query_filter
- operator 'and'
- end
- end
- end
- filter do
- bool do
- must { term original_key_exact: query_filter } if query_filter && not_elastic
- must { term project_id: project_id }
- must { term ready: status } unless status.blank?
- hidden_keys ? must { term hidden_in_search: true } : must { term hidden_in_search: false }
- end
- end
- end
+ query_params = []
+ text_to_search = @params[:filter]
+ if text_to_search.present?
+ if !@params[:not_elastic_search]
+ query_params << { match: { original_key: { query: text_to_search, operator: 'and' } } }
+ else
+ query_params << { term: { original_key_exact: text_to_search } }
end
- sort { by :original_key, order: 'asc' } unless query_filter.present?
- from offset
- size limit
- }.to_hash
+ end
+
+ filter_params = []
+ filter_params << { term: { project_id: @params[:project_id].to_i } } if @params[:project_id].present?
+ filter_params << { term: { ready: @params[:status].to_b } } if @params[:status].present?
+ filter_params << { term: { hidden_in_search: @params[:hidden_in_search].to_b } }
+
+ query = KeysIndex.query(query_params).filter(filter_params)
+ query = query.order(original_key_exact: :asc) if @params[:filter].blank?
+ query = query.offset(@params[:offset].to_i).limit(@params.fetch(:limit, PER_PAGE))
+
+ return query
end
def find_keys
- keys_in_es = Elasticsearch::Model.search(search_query, Key).results
- keys = Key.where(id: keys_in_es.map(&:id)).includes(:translations, :project)
- SortingHelper.order_by_elasticsearch_result_order(keys, keys_in_es)
+ search_query.load(scope: -> { includes(:translations, :project) }).objects
end
-end
\ No newline at end of file
+end
diff --git a/app/services/search_translations_finder.rb b/app/services/search_translations_finder.rb
index 4587b3e7..ea90821f 100644
--- a/app/services/search_translations_finder.rb
+++ b/app/services/search_translations_finder.rb
@@ -17,77 +17,35 @@ class SearchTranslationsFinder
PER_PAGE = 50
attr_reader :form
- include Elasticsearch::DSL
def initialize(form)
@form = form
end
def search_query
- project_id = form[:project_id]
- query_filter = form[:query]
- field = form[:field]
- translator_id = form[:translator_id]
-
- start_date = form[:start_date]
- end_date = form[:end_date]
- if form[:target_locales].present? && form[:target_locales].size > 0
- target_locales = form[:target_locales].first.rfc5646
- end
- hidden_keys = form[:hidden_keys] if form[:hidden_keys].present?
-
- offset = (page - 1) * PER_PAGE
- limit = PER_PAGE
-
- search {
- query do
- filtered do
- if query_filter.present?
- query do
- if field == 'searchable_source_copy'
- match 'source_copy' do
- query query_filter
- operator 'or'
- end
- else
- match 'copy' do
- query query_filter
- operator 'or'
- end
- end
- end
- end
-
- filter do
- bool do
- must { term rfc5646_locale: target_locales } if target_locales
- must { term project_id: project_id } if project_id && project_id > 0
- must { term translator_id: translator_id } if translator_id && translator_id > 0
- if start_date
- must {
- range 'updated_at' do
- gte start_date
- end
- }
- end
- if end_date
- must {
- range 'updated_at' do
- lte end_date
- end
- }
- end
-
- hidden_keys ? must { term hidden_in_search: true } : must { term hidden_in_search: false }
- end
- end
- end
+ query_params = []
+ text_to_search = form[:query]
+ if text_to_search.present?
+ if form[:field] == 'searchable_source_copy'
+ query_params << { match: { source_copy: { query: text_to_search, operator: 'or' } } }
+ else
+ query_params << { match: { copy: { query: text_to_search, operator: 'or' } } }
end
+ end
- sort { by :id, order: 'desc' } unless query_filter.present?
- size limit
- from offset
- }.to_hash
+ filter_params = []
+ filter_params << { term: { rfc5646_locale: form[:target_locales].first.rfc5646} } if form[:target_locales].present? && form[:target_locales].size > 0
+ filter_params << { term: { project_id: form[:project_id].to_i } } if form[:project_id].present?
+ filter_params << { term: { translator_id: form[:translator_id].to_i } } if form[:translator_id].present?
+ filter_params << { term: { reviewer_id: form[:reviewer_id].to_i } } if form[:reviewer_id].present?
+ filter_params << { range: { updated_at: { gte: form[:start_date] } } } if form[:start_date].present?
+ filter_params << { range: { updated_at: { lte: form[:end_date] } } } if form[:end_date].present?
+ filter_params << { term: { hidden_in_search: form[:hidden_keys] || false } }
+
+ query = TranslationsIndex.query(query_params).filter(filter_params)
+ query = query.order(id: :desc) if text_to_search.blank?
+ query = query.offset((page - 1) * PER_PAGE).limit(PER_PAGE)
+ return query
end
def page
@@ -95,9 +53,7 @@ def page
end
def find_translations
- limit = PER_PAGE
- translations_in_es = Elasticsearch::Model.search(search_query, Translation).results
- translations = Translation.where(id: translations_in_es.map(&:id)).includes(key: :project)
- PaginatableObjects.new(translations, translations_in_es, page, limit)
+ translations = search_query.load(scope: -> { includes(key: :project) })
+ return PaginatableObjects.new(translations, page, PER_PAGE)
end
end
diff --git a/app/support/article_and_commit_not_approved_translation_stats.rb b/app/support/article_and_commit_not_approved_translation_stats.rb
index a77e1c59..7dd38d59 100644
--- a/app/support/article_and_commit_not_approved_translation_stats.rb
+++ b/app/support/article_and_commit_not_approved_translation_stats.rb
@@ -63,7 +63,7 @@ def item_stat(item, type, state)
def commit_translation_groups_with_stats
query = Translation.not_base.not_approved.joins(:commits_keys)
- query = query.where(commits_keys: { commit_id: @commits.map(&:id) } )
+ query = query.where(commits_keys: { commit_id: @commits.map(&:id) })
query = query.where(translations: { rfc5646_locale: @locales.map(&:rfc5646) }) if @locales.present?
query = query.group("commit_id, translated, rfc5646_locale")
query.select("commit_id as item_id, translated, rfc5646_locale, COUNT(*) AS translations_count, SUM(words_count) AS words_count")
diff --git a/app/support/home_index_form.rb b/app/support/home_index_form.rb
index 3106900f..c290801d 100644
--- a/app/support/home_index_form.rb
+++ b/app/support/home_index_form.rb
@@ -70,7 +70,7 @@ def [](key)
def set_pagination_variables
vars[:page] = Integer(params[:page]) rescue 1
vars[:offset] = (vars[:page] - 1) * HomeController::PER_PAGE
- vars[:limit] = HomeController::PER_PAGE
+ vars[:limit] = @params[:limit] || HomeController::PER_PAGE
end
def set_filter__status
@@ -169,12 +169,12 @@ def set_articles_filter__project_id
def set_groups_filter__name
vars[:groups_filter__name] = params[:groups_filter__name].presence
end
-
+
def set_groups_filter__project_id
vars[:groups_filter__project_id] = cookies[:home_index__groups_filter__project_id] =
params[:groups_filter__project_id].to_s.presence || cookies[:home_index__groups_filter__project_id].to_s.presence || 'all'
end
-
+
# ASSET SPECIFIC
def set_assets_filter__name
diff --git a/app/support/search_translations_form.rb b/app/support/search_translations_form.rb
index 446c4209..223ff33c 100644
--- a/app/support/search_translations_form.rb
+++ b/app/support/search_translations_form.rb
@@ -10,8 +10,9 @@ def initialize(form)
# @param form hash of parameters to process
def extract_translations_search_params(form)
processed_params = form.deep_dup
- processed_params[:project_id] = processed_params[:project_id].to_i
- processed_params[:translator_id] = processed_params[:translator_id].to_i
+ processed_params[:project_id] = processed_params[:project_id].to_i if processed_params[:project_id].present?
+ processed_params[:translator_id] = processed_params[:translator_id].to_i if processed_params[:translator_id].present?
+ processed_params[:reviewer_id] = processed_params[:reviewer_id].to_i if processed_params[:reviewer_id].present?
start_date = Date.strptime(processed_params[:start_date], "%m/%d/%Y") rescue nil
end_date = Date.strptime(processed_params[:end_date], "%m/%d/%Y") rescue nil
@@ -37,4 +38,4 @@ def extract_translations_search_params(form)
def [](key)
form[key]
end
-end
\ No newline at end of file
+end
diff --git a/app/views/api/v1/articles/_form.html.slim b/app/views/api/v1/articles/_form.html.slim
index f530f8e7..aad48612 100644
--- a/app/views/api/v1/articles/_form.html.slim
+++ b/app/views/api/v1/articles/_form.html.slim
@@ -31,7 +31,13 @@
.control-group
= f.label :priority, class: 'control-label'
.controls
- = @article.priority || '-'
+ - if current_user.admin?
+ = f.select :priority, t("models.article.priority").to_a.map(&:reverse).unshift(['-', nil]), {}, class: 'styled'
+ - else
+ - if @article.priority
+ = t("models.article.priority")[@article.priority]
+ - else
+ | -
.control-group
= f.label :due_date, class: 'control-label'
@@ -74,5 +80,5 @@
= text_area_tag "article[sections_hash[#{section_name}]]", section_source_copy, rows: 6
.form-actions
- = f.submit class: 'primary', value: 'save', data: { confirm: 'Are you sure? Changes are irreversible.'}
+ = f.submit class: 'primary', value: 'Save', data: { confirm: 'Are you sure? Changes are irreversible.'}
button.default href=(@article.persisted? ? api_v1_project_article_path(@project.id, @article.name_was) : root_path) Cancel
diff --git a/app/views/api/v1/articles/_layout.slim b/app/views/api/v1/articles/_layout.slim
index 4c2967a2..812a1e8e 100644
--- a/app/views/api/v1/articles/_layout.slim
+++ b/app/views/api/v1/articles/_layout.slim
@@ -23,7 +23,10 @@
h1
| #{@article.project.name}
strong >
- | Article #{@article.id}
+ | Article
+ strong >
+ | #{@article.id}
+
hr.divider
.row
diff --git a/app/views/api/v1/groups/_layout.slim b/app/views/api/v1/groups/_layout.slim
new file mode 100644
index 00000000..b779aa90
--- /dev/null
+++ b/app/views/api/v1/groups/_layout.slim
@@ -0,0 +1,37 @@
+/ Copyright 2014 Square Inc.
+/
+/ Licensed under the Apache License, Version 2.0 (the "License");
+/ you may not use this file except in compliance with the License.
+/ You may obtain a copy of the License at
+/
+/ http://www.apache.org/licenses/LICENSE-2.0
+/
+/ Unless required by applicable law or agreed to in writing, software
+/ distributed under the License is distributed on an "AS IS" BASIS,
+/ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/ See the License for the specific language governing permissions and
+/ limitations under the License.
+
+.header
+ h1
+ | #{@group.project.name}
+ strong >
+ | Group
+ strong >
+ | #{@group.id}
+
+hr.divider
+
+.row
+ .three.columns.sidebar
+ ul
+ li class=('active' if action_name == 'show')
+ a General
+ / li class=('active' if action_name == 'issues')
+
+ li.divider
+
+ = render partial: 'common/progress_tracker', locals: { item: @group }
+
+ .thirteen.columns.sidebar-main
+ = yield
diff --git a/app/views/api/v1/groups/show.html.slim b/app/views/api/v1/groups/show.html.slim
new file mode 100644
index 00000000..eafa6872
--- /dev/null
+++ b/app/views/api/v1/groups/show.html.slim
@@ -0,0 +1,99 @@
+/ Copyright 2014 Square Inc.
+/
+/ Licensed under the Apache License, Version 2.0 (the "License");
+/ you may not use this file except in compliance with the License.
+/ You may obtain a copy of the License at
+/
+/ http://www.apache.org/licenses/LICENSE-2.0
+/
+/ Unless required by applicable law or agreed to in writing, software
+/ distributed under the License is distributed on an "AS IS" BASIS,
+/ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/ See the License for the specific language governing permissions and
+/ limitations under the License.
+
+- content_for :shuttle_title do
+ = "group Group #{@group.id}"
+- content_for :file_name do
+ = 'views/api/v1/groups/show'
+
+= render layout: 'api/v1/groups/layout' do
+ .row
+ .seven.columns
+ fieldset
+ .control-group
+ = label_tag :name, nil, class: 'control-label'
+ .controls
+ = @group.name
+
+ .control-group
+ = label_tag :display_name, nil, class: 'control-label'
+ .controls
+ = @group.display_name
+
+ .control-group
+ = label_tag :priority, nil, class: 'control-label'
+ .controls
+ = @group.priority || '-'
+
+ .control-group
+ = label_tag :due_date, nil, class: 'control-label'
+ .controls
+ = @group.due_date || '-'
+
+ .control-group
+ = label_tag :description, nil, class: 'control-label'
+ .controls
+ - if @group.description.present?
+ = sanitize @group.description, tags: %w(strong em a br), attributes: %w(href)
+ - else
+ = '-'
+
+ .six.columns
+ fieldset
+ .control-group
+ = label_tag :readiness_status, nil, class: 'control-label'
+ .controls
+ = @group.ready? ? 'Ready' : 'Not Ready'
+
+ .control-group
+ = label_tag :loading_status, nil, class: 'control-label'
+ .controls
+ = @group.loading? ? 'Loading' : 'Loaded'
+
+ .control-group
+ = label_tag :created_at, nil, class: 'control-label'
+ .controls
+ = @group.created_at.try(:to_s, :long) || '-'
+
+ .control-group
+ = label_tag :updated_at, nil, class: 'control-label'
+ .controls
+ = @group.updated_at.try(:to_s, :long) || '-'
+
+ .control-group
+ = label_tag :creator, nil, class: 'control-label'
+ .controls
+ = @group.creator.try(:name) || '-'
+
+ .control-group
+ = label_tag :updater, nil, class: 'control-label'
+ .controls
+ = @group.updater.try(:name) || '-'
+
+ .control-group
+ = label_tag 'Creation method', nil, class: 'control-label'
+ .controls
+ = @group.created_via_api ? 'Via API' : 'Via Website'
+
+ .row
+ .thirteen.columns
+ fieldset
+ legend Linked Articles
+ - articles = @group.article_groups.order(article_id: :asc).map(&:article)
+ - articles.reject { |article| article.ready }.each do |article|
+ .control-group
+ = link_to '⏳ ' + truncate(article.name), api_v1_project_article_url(article.project.id, article.name)
+ - articles.select {|article| article.ready}.each do |article|
+ .control-group
+ = link_to '✅ ' + truncate(article.name), api_v1_project_article_url(article.project.id, article.name)
diff --git a/app/views/assets/_form.html.slim b/app/views/assets/_form.html.slim
index 155ef38f..daf39f28 100644
--- a/app/views/assets/_form.html.slim
+++ b/app/views/assets/_form.html.slim
@@ -44,7 +44,13 @@
.control-group
= f.label :priority, class: 'control-label'
.controls
- = @asset.priority || '-'
+ - if current_user.admin?
+ = f.select :priority, t("models.asset.priority").to_a.map(&:reverse).unshift(['-', nil]), {}, class: 'styled'
+ - else
+ - if @asset.priority
+ = t("models.asset.priority")[@asset.priority]
+ - else
+ | -
.control-group
= f.label :due_date, class: 'control-label'
@@ -94,7 +100,7 @@
.qtip-tooltip#asset-description So that translators understand the context!
.form-actions
- = f.submit class: 'primary', value: 'save', data: { confirm: 'Are you sure? Changes are irreversible.'}
+ = f.submit class: 'primary', value: 'Save', data: { confirm: 'Are you sure? Changes are irreversible.'}
button.default href=(@asset.persisted? ? project_asset_path(@project, @asset) : root_path) Cancel
diff --git a/app/views/assets/_layout.slim b/app/views/assets/_layout.slim
index 291f0874..5afc7df2 100644
--- a/app/views/assets/_layout.slim
+++ b/app/views/assets/_layout.slim
@@ -23,7 +23,10 @@
h1
| #{@asset.project.name}
strong >
- | Asset #{@asset.id}
+ | Asset
+ strong >
+ | #{@asset.id}
+
hr.divider
.row
diff --git a/app/views/commits/_layout.slim b/app/views/commits/_layout.slim
index 315f7fa8..884460c3 100644
--- a/app/views/commits/_layout.slim
+++ b/app/views/commits/_layout.slim
@@ -33,9 +33,11 @@
h1
| #{@commit.project.name}
strong >
- | Commit #{@commit.revision_prefix}
+ | Commit
+ strong >
+ | #{@commit.id}
- h6
+ .subtitle
strong
- if @commit.loading?
- if @commit.import_batch_status
@@ -43,11 +45,11 @@
- else
- current_status = "Hung (Loading, No Import Batch)"
- elsif @commit.ready?
- - current_status = 'Ready'
+ - current_status = 'Ready ☑️'
- elsif @commit.import_errors.present?
- current_status = 'Errored - Import Aborted'
-else
- - current_status = 'Translating'
+ - current_status = 'Translating ⏳'
= "Currently #{current_status}"
span.separator /
= "Found #{@commit.keys.count} Keys"
diff --git a/app/views/commits/show.slim b/app/views/commits/show.slim
index c8bffec4..1f426508 100644
--- a/app/views/commits/show.slim
+++ b/app/views/commits/show.slim
@@ -24,7 +24,7 @@
.nine.columns
.control-group
- = f.label 'SHA', class: 'control-label info'
+ = f.label 'SHA', class: 'control-label info revision'
.controls
= link_to @commit.revision, @commit.git_url
.control-group
@@ -43,6 +43,16 @@
br
+ .control-group
+ = f.label :priority, class: 'control-label'
+ .controls
+ - if current_user.admin?
+ = f.select :priority, t("models.commit.priority").to_a.map(&:reverse).unshift(['-', nil]), {}, class: 'styled'
+ - else
+ - if @commit.priority
+ = t("models.commit.priority")[@commit.priority]
+ - else
+ | -
.control-group
= f.label :due_date, class: 'control-label'
.controls
diff --git a/app/views/common/_progress_tracker.slim b/app/views/common/_progress_tracker.slim
index ca0436ca..95c5b22e 100644
--- a/app/views/common/_progress_tracker.slim
+++ b/app/views/common/_progress_tracker.slim
@@ -22,11 +22,22 @@ li.progress-bar
- finished = (item.translations_not_done(locale) == 0)
li.progress-bar
dl
- dt class=(required ? (finished ? 'text-success' : 'text-error') : nil)
- = locale.rfc5646
- small.lowercase = " (#{required ? 'required' : 'optional'})"
- strong = (finished ? '100%' : "#{(item.fraction_done(locale) * 100).round(2)}%")
- dd.bar-status
- span class=(finished ? 'finished' : 'inprogress') style="width: #{(item.fraction_done(locale) * 100).round(2)}%;"
- dd.text-status class=(required ? (finished ? 'text-success' : 'text-error') : nil)
- = "#{item.translations_not_done(locale)} left (#{item.translations_total(locale)} total)"
+ - if current_user.translator?
+ a href=HomeIndexPresenter.new([],[],[],[],[locale]).translate_link_path(current_user, item)
+ dt class=(required ? (finished ? 'text-success' : 'text-error') : nil)
+ = locale.rfc5646
+ small.lowercase = " (#{required ? 'required' : 'optional'})"
+ strong = (finished ? '100%' : "#{(item.fraction_done(locale) * 100).round(2)}%")
+ dd.bar-status
+ span class=(finished ? 'finished' : 'inprogress') style="width: #{(item.fraction_done(locale) * 100).round(2)}%;"
+ dd.text-status class=(required ? (finished ? 'text-success' : 'text-error') : nil)
+ = "#{item.translations_not_done(locale)} left (#{item.translations_total(locale)} total)"
+ - else
+ dt class=(required ? (finished ? 'text-success' : 'text-error') : nil)
+ = locale.rfc5646
+ small.lowercase = " (#{required ? 'required' : 'optional'})"
+ strong = (finished ? '100%' : "#{(item.fraction_done(locale) * 100).round(2)}%")
+ dd.bar-status
+ span class=(finished ? 'finished' : 'inprogress') style="width: #{(item.fraction_done(locale) * 100).round(2)}%;"
+ dd.text-status class=(required ? (finished ? 'text-success' : 'text-error') : nil)
+ = "#{item.translations_not_done(locale)} left (#{item.translations_total(locale)} total)"
diff --git a/app/views/fencer_validation_mailer/suspicious_source_found.text.erb b/app/views/fencer_validation_mailer/suspicious_source_found.text.erb
new file mode 100644
index 00000000..e5a6a11e
--- /dev/null
+++ b/app/views/fencer_validation_mailer/suspicious_source_found.text.erb
@@ -0,0 +1,12 @@
+Dear localization team,
+
+Shuttle found the following source strings are suspicious in translation request <%= @job_url %>.
+
+<% for @item in @formatted_keys_errors -%>
+ <%= @item.first %> Reason: <%= @item.second %>
+<% end -%>
+
+Please double check these source strings are correct. Contact requester <%= @author_name %> at <%= @author_email %> for correcting source strings if needed.
+
+Sincerely,
+The Shuttle Automated Mailer
diff --git a/app/views/home/_headers.slim b/app/views/home/_headers.slim
index e718b552..146d995b 100644
--- a/app/views/home/_headers.slim
+++ b/app/views/home/_headers.slim
@@ -63,6 +63,14 @@ tr
th Description
+ / group count in articles & article count in groups
+ - if item_type == 'article'
+ th
+ = t("controllers.home.articles.groups")
+ - elsif item_type == 'group'
+ th
+ = t("controllers.home.groups.articles")
+
- if current_user.translator?
th Translate
th Review
diff --git a/app/views/home/_item.slim b/app/views/home/_item.slim
index 472a0032..6abce3b0 100644
--- a/app/views/home/_item.slim
+++ b/app/views/home/_item.slim
@@ -26,7 +26,7 @@ tr.row_class
td = item.project.name
/ Commit SHA or Article Name
- td
+ td class=('revision-prefix' if item.is_a?(Commit))
- if item.is_a?(Commit)
= link_to item.revision_prefix, project_commit_url(item.project, item)
- elsif item.is_a?(Article)
@@ -34,9 +34,9 @@ tr.row_class
- elsif item.is_a?(Asset)
= link_to truncate(item.name), project_asset_url(item.project, item)
- elsif item.is_a?(Group)
- = truncate(item.display_name)
+ = link_to truncate(item.display_name), api_v1_project_group_url(item.project.id, item.name)
/ Create Date
- td.centered
+ td
- if item.is_a?(Commit)
= item.created_at.strftime('%m/%d/%Y')
- elsif item.is_a?(Article)
@@ -47,7 +47,7 @@ tr.row_class
= item.created_at.strftime('%m/%d/%Y')
/ Due Date
- td.due-date.centered
+ td.due-date
- if current_user.admin? or current_user.monitor?
= form_for item, url: @presenter.update_item_path(item) do |f|
= f.text_field :due_date, value: (f.object.due_date.strftime('%m/%d/%Y') if !f.object.due_date.nil?), class: 'datepicker'
@@ -55,7 +55,7 @@ tr.row_class
span = l(item.due_date, format: :mon_day_year)
/ Priority
- td.centered
+ td
- if current_user.admin?
= form_for item, url: @presenter.update_item_path(item) do |f|
= f.select :priority, t("models.#{item_type}.priority").to_a.map(&:reverse).unshift(['-', nil]), {}, class: 'styled'
@@ -67,12 +67,20 @@ tr.row_class
| -
/ Description
- td.centered
+ td.description-container
div.description[data-full-description=@presenter.full_description(item)
data-short-description=@presenter.short_description(item)
data-sub-description=@presenter.sub_description(item)]
= @presenter.short_description(item)
+ / group count in articles & article count in groups
+ - if item.is_a?(Article)
+ td
+ = t("controllers.home.articles.groups_cell", pending_count: item&.groups&.reject(&:ready)&.count, linked_count: item&.groups&.count)
+ - elsif item.is_a?(Group)
+ td
+ = t("controllers.home.groups.articles_cell", pending_count: item&.articles&.reject(&:ready)&.count, linked_count: item&.articles&.count)
+
/ Stats
- if current_user.translator?
/ Word Translation Count
diff --git a/app/views/home/_items.slim b/app/views/home/_items.slim
index d9ddb9f4..c6552979 100644
--- a/app/views/home/_items.slim
+++ b/app/views/home/_items.slim
@@ -34,13 +34,14 @@ ruby:
div id="#{item_type}s"
.header
- - if showRequestTranslationButton
+ .header-buttons
+ - if showRequestTranslationButton
+ .pull-right
+ button.primary href="#add-#{item_type}-translation" rel='modal' disabled=(item_type == 'commit' ? Project.git : Project.not_git).count.zero? Request Translation
.pull-right
- button.primary href="#add-#{item_type}-translation" rel='modal' disabled=(item_type == 'commit' ? Project.git : Project.not_git).count.zero? Request Translation
+ button.button--secondary.csv-button href="/csv?type=#{item_type}" ⇩ Download CSV
h1 = itemsHeader
- hr.divider
-
.border
= render partial: "home/#{item_type}s/filter_bar"
@@ -57,6 +58,9 @@ div id="#{item_type}s"
.row
.pagination-info
= page_entries_info(items, entry_name: item_type)
+ - if current_user.admin?
+ br
+ = "Items shown on page: #{items.size}"
- if item_type != 'group'
= render partial: "home/#{item_type}s/add_translation_modal"
diff --git a/app/views/layouts/_navbar.slim b/app/views/layouts/_navbar.slim
index df2c49c6..2099dbc4 100644
--- a/app/views/layouts/_navbar.slim
+++ b/app/views/layouts/_navbar.slim
@@ -28,6 +28,8 @@ nav.navbar.navbar-dark
- if current_user.translator?
= nav_link "Locale Associations", "locale_associations", locale_associations_url
= nav_link "Glossary", "glossary", glossary_url
+
+ = nav_link "Reports", "reports", reports_url
li.nav-link.dropdown
a.dropdown-toggle href="#" data-toggle="dropdown"
@@ -38,7 +40,7 @@ nav.navbar.navbar-dark
= nav_dropdown_link "Project Translation Report", "stats", "project_translation_report", stats_project_translation_report_url
= nav_dropdown_link "Incoming New Words Report", "stats", "incoming_new_words_report", stats_incoming_new_words_report_url
= nav_dropdown_link "Translator Report", "stats", "translator_report", stats_translator_report_url
- = nav_dropdown_link "Backlog Report", "stats", "backlog_report", stats_backlog_report_url
+ /= nav_dropdown_link "Backlog Report", "stats", "backlog_report", stats_backlog_report_url
= nav_dropdown_link "Quality Report", "stats", "quality_report", stats_quality_report_url
- if current_user.admin?
@@ -57,5 +59,7 @@ nav.navbar.navbar-dark
- if current_user.monitor?
li.nav-link.worker-status class=("clickable" if current_user.admin?)
- = fa_icon "circle"
+ = fa_icon "check"
+ = fa_icon "refresh"
+ = fa_icon "cloud"
| Workers
diff --git a/app/views/layouts/application.slim b/app/views/layouts/application.slim
index 19c1c8c1..49331b30 100644
--- a/app/views/layouts/application.slim
+++ b/app/views/layouts/application.slim
@@ -49,4 +49,7 @@ html lang="en"
= raw(ERB.new(File.read(js_erb_file)).result(binding)) if File.exist?(js_erb_file)
= raw(CoffeeScript.compile(ERB.new(File.read(js_coffee_erb_file)).result(binding))) if File.exist?(js_coffee_erb_file)
+ - if ENV['SENTRY_PUBLIC_DSN']
+ script type='text/javascript' Raven.config("#{ENV['SENTRY_PUBLIC_DSN']}").install()
+
= render partial: 'layouts/flashes'
diff --git a/app/views/locale/projects/show.slim b/app/views/locale/projects/show.slim
index 7b3333cb..de19367a 100644
--- a/app/views/locale/projects/show.slim
+++ b/app/views/locale/projects/show.slim
@@ -29,21 +29,36 @@
= button_tag 'Translate', id: 'translate-link', class: 'submit', type: 'button'
h1
=> @project.name
- - if @presenter.form[:article_id]
+ - if @presenter.form[:commit]&.present?
+ strong >
+ | Commit
+ strong >
+ small.monospace
+ = link_to @presenter.selected_commit.revision_prefix, project_commit_url(@project, @presenter.selected_commit)
+ - elsif @presenter.form[:article_id]&.present?
+ strong >
+ | Article
+ strong >
small
- = link_to @presenter.selected_article.name, api_v1_project_article_path(@project.id, @presenter.selected_article.name)
- - elsif @presenter.form[:group]
+ = link_to truncate(@presenter.selected_article.name), api_v1_project_article_path(@project.id, @presenter.selected_article.name)
+ - elsif @presenter.form[:asset_id]&.present?
+ strong >
+ | Asset
+ strong >
small
- = @presenter.form[:group]
+ = link_to truncate(@presenter.selected_asset.name), project_asset_url(@project, @presenter.selected_asset)
+ - elsif @presenter.form[:group]&.present?
+ strong >
+ | Group
+ strong >
+ small
+ = link_to truncate(@presenter.selected_group.display_name), api_v1_project_group_url(@project.id, @presenter.selected_group.name)
- h6
+ .subtitle
strong
- - last_commit = @project.commits.order('committed_at DESC').first
- - if last_commit
+ - if @presenter.form[:commit]&.present?
| Last imported
- = "#{time_ago_in_words(last_commit.committed_at)} ago"
- | /
- = "#{last_commit.revision_prefix}"
+ = "#{time_ago_in_words(@presenter.selected_commit.committed_at)} ago"
- else
| Never imported before
@@ -61,10 +76,10 @@ div
- if @project.commit?
= select_tag 'commit', options_for_select(@presenter.selectable_commits, @presenter.selected_commit.try!(:revision))
- - elsif @presenter.form[:article_id]
+ - elsif @presenter.form[:article_id]&.present?
= hidden_field_tag 'article_id', @presenter.selected_article.id
= select_tag 'section_id', options_for_select(@presenter.selectable_sections, @presenter.selected_section.try!(:id))
- - elsif @presenter.form[:group]
+ - elsif @presenter.form[:group]&.present?
= hidden_field_tag 'group', @presenter.form[:group]
- elsif @project.asset?
= hidden_field_tag 'asset_id', @presenter.selected_asset.id
diff --git a/app/views/locale_associations/_form.html.slim b/app/views/locale_associations/_form.html.slim
index 0c4029d1..4752f357 100644
--- a/app/views/locale_associations/_form.html.slim
+++ b/app/views/locale_associations/_form.html.slim
@@ -44,5 +44,5 @@ hr.divider
= f.check_box :uncheck_disabled
span.help-block Do you want to disable unchecking the checkbox if it's checked?
.form-actions
- = f.submit class: 'primary', value: 'save', data: { confirm: 'Are you sure? Changes are irreversible.'}
+ = f.submit class: 'primary', value: 'Save', data: { confirm: 'Are you sure? Changes are irreversible.'}
button.default href=locale_associations_url Cancel
diff --git a/app/views/projects/_form.slim b/app/views/projects/_form.slim
index 753d95e1..f3192fec 100644
--- a/app/views/projects/_form.slim
+++ b/app/views/projects/_form.slim
@@ -299,7 +299,7 @@
= @project.api_token
.form-actions
- = f.submit class: 'primary', value: 'save', data: { confirm: 'Are you sure? Changes are irreversible.'}
+ = f.submit class: 'primary', value: 'Save', data: { confirm: 'Are you sure? Changes are irreversible.'}
button.default href=projects_url Cancel
- content_for :javascript do
diff --git a/app/views/projects/index.slim b/app/views/projects/index.slim
index f4c39eb7..2c971d69 100644
--- a/app/views/projects/index.slim
+++ b/app/views/projects/index.slim
@@ -37,6 +37,6 @@ hr.divider
tr onclick="document.location = '#{edit_project_url(project)}'"
td = project.name
td = project.job_type.titlecase
- td = project.repository_url
+ td.small.monospace = project.repository_url
td = project.required_rfc5646_locales.join(" / ").upcase
td = project.other_rfc5646_locales.join(" / ").upcase
diff --git a/app/views/reports/index.slim b/app/views/reports/index.slim
new file mode 100644
index 00000000..12c264e6
--- /dev/null
+++ b/app/views/reports/index.slim
@@ -0,0 +1,57 @@
+/ Copyright 2014 Square Inc.
+/
+/ Licensed under the Apache License, Version 2.0 (the "License");
+/ you may not use this file except in compliance with the License.
+/ You may obtain a copy of the License at
+/
+/ http://www.apache.org/licenses/LICENSE-2.0
+/
+/ Unless required by applicable law or agreed to in writing, software
+/ distributed under the License is distributed on an "AS IS" BASIS,
+/ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/ See the License for the specific language governing permissions and
+/ limitations under the License.
+
+- content_for :shuttle_title do
+ = "Reports - Shuttle"
+- content_for :file_name do
+ = 'views/reports/index'
+
+
+.header
+ h1 Reports based on job status
+
+
+.day
+ h2 =@today
+ div.report-buttons
+ a href="/reports/download/incoming/#{@today}" ↙️ Incoming jobs report
+ a href="/reports/download/pending/#{@today}" ⏳ Pending jobs report
+ a href="/reports/download/completed/#{@today}" ✓ Completed jobs report
+
+.day
+ h2 =@yesterday
+ div.report-buttons
+ a href="/reports/download/incoming/#{@yesterday}" ↙️ Incoming jobs report
+ a href="/reports/download/pending/#{@yesterday}" ⏳ Pending jobs report
+ a href="/reports/download/completed/#{@yesterday}" ✓ Completed jobs report
+
+.day
+ h2 Custom date in the past 30 days
+ .date-picker.center
+ label for="pickedDate" Pick a date and then a report type
+ input type="date" id="pickedDate" name="pickedDate" value="#{@today}" min="#{@today - 30}" max="#{@today}"
+ div.report-buttons.custom-dates
+ a href="#" data-type="incoming" ↙️ Incoming jobs report
+ a href="#" data-type="pending" ⏳ Pending jobs report
+ a href="#" data-type="completed" ✓ Completed jobs report
+
+javascript:
+ var buttons = document.querySelectorAll('.custom-dates a')
+ for (var i = 0; i < buttons.length; i++) {
+ buttons[i].addEventListener('click', function(event) {
+ var date = document.getElementById('pickedDate').value;
+ var reportType = event.target.dataset.type;
+ window.location = '/reports/download/' + reportType + '/' + date;
+ });
+ }
diff --git a/app/views/search/translations.slim b/app/views/search/translations.slim
index 43d923a2..798bcf38 100644
--- a/app/views/search/translations.slim
+++ b/app/views/search/translations.slim
@@ -51,6 +51,11 @@ div
.controls
= select_tag 'translator_id', options_for_select(User.order('email ASC').map { |u| [u.name, u.id] }.unshift(['Anyone', nil]))
+ .control-group
+ = label_tag 'reviewer_id', 'Reviewer', class: 'control-label'
+ .controls
+ = select_tag 'reviewer_id', options_for_select(User.order('email ASC').map {|u| [u.name, u.id]}.unshift(['Anyone', nil]))
+
.control-group
= label_tag 'start_date', 'Start Date', class: 'control-label'
.controls
@@ -60,6 +65,7 @@ div
= label_tag 'end_date', 'End Date', class: 'control-label'
.controls
= text_field_tag 'end_date', '', class: 'datepicker'
+
.control-group
= label_tag 'hidden_keys', 'Hidden Translations', class: 'control-label'
.controls
diff --git a/app/views/users/index.slim b/app/views/users/index.slim
index 29a4f9d7..f6f659d7 100644
--- a/app/views/users/index.slim
+++ b/app/views/users/index.slim
@@ -20,8 +20,6 @@
.header
h1 Users
-hr.divider
-
.border
table.table.hover-rows
thead
diff --git a/app/views/users/show.slim b/app/views/users/show.slim
index a8fbd791..e0847aa5 100644
--- a/app/views/users/show.slim
+++ b/app/views/users/show.slim
@@ -106,5 +106,5 @@ hr.divider
.form-actions
- = f.submit class: 'primary', value: 'save'
+ = f.submit class: 'primary', value: 'Save'
button.danger href=user_url(@user) data-method='DELETE' data-confirm='Are you sure you want to delete this account?' Delete
diff --git a/app/workers/article_importer.rb b/app/workers/article_importer.rb
index 58d0bae4..3998fca1 100644
--- a/app/workers/article_importer.rb
+++ b/app/workers/article_importer.rb
@@ -77,7 +77,9 @@ def on_success(_status, options)
# Keys are refreshed as part of `Key.batch_recalculate_ready!`.
# Translations need to be refreshed in case section data (such as `activeness`) changed in
# the last re-import. CommitKeyCreator takes care of refreshing Translations in a Commit during a Commit import.
- Translation.batch_refresh_elastic_search(article)
+ TranslationsIndex.import! article.reload.translations
+
+ PostLoadingChecker.launch(article)
end
end
end
diff --git a/app/workers/asset_importer.rb b/app/workers/asset_importer.rb
index 7252971d..82479e1b 100644
--- a/app/workers/asset_importer.rb
+++ b/app/workers/asset_importer.rb
@@ -59,7 +59,9 @@ def on_success(_status, options)
Key.batch_recalculate_ready!(asset)
AssetRecalculator.new.perform(asset.id)
- Translation.batch_refresh_elastic_search(asset)
+ TranslationsIndex.import! asset.reload.translations
+
+ PostLoadingChecker.launch(asset)
end
end
end
diff --git a/app/workers/auto_importer.rb b/app/workers/auto_importer.rb
index 3b9dd07b..1f991401 100644
--- a/app/workers/auto_importer.rb
+++ b/app/workers/auto_importer.rb
@@ -54,13 +54,16 @@ def perform(project_id)
begin
project.commit! branch, other_fields: {description: "Automatically imported from the #{branch} branch"}
rescue Git::CommitNotFoundError => err
- branches_to_delete << branch # branch doesn't actually exist; remove from watched branches and ignore
+ # TODO: Disable removing watched branch. SHUTTLE-913
+ Rails.logger.warn("[AutoImporter] Watched branch #{branch} not found in project #{project.name}")
+ # branches_to_delete << branch # branch doesn't actually exist; remove from watched branches and ignore
end
end
project.watched_branches = project.watched_branches - branches_to_delete
project.save!
rescue Timeout::Error => err
+ Raven.capture_exception err, extra: { project_id: project_id }
self.class.perform_in 2.minutes, project_id
end
diff --git a/app/workers/commit_creator.rb b/app/workers/commit_creator.rb
index 56b1bb70..b00c30c4 100644
--- a/app/workers/commit_creator.rb
+++ b/app/workers/commit_creator.rb
@@ -34,6 +34,7 @@ def perform(project_id, sha, options={})
rescue Git::CommitNotFoundError, Project::NotLinkedToAGitRepositoryError => err
CommitMailer.notify_import_errors_in_commit_creator(options[:other_fields].try!(:symbolize_keys).try!(:[], :user_id), project_id, sha, err).deliver_now
rescue Timeout::Error => err
+ Raven.capture_exception err, extra: { project_id: project_id, sha: sha }
self.class.perform_in 2.minutes, project_id, sha
end
diff --git a/app/workers/commit_importer.rb b/app/workers/commit_importer.rb
index ca529c5d..ddf14c71 100644
--- a/app/workers/commit_importer.rb
+++ b/app/workers/commit_importer.rb
@@ -41,6 +41,7 @@ class Finisher
def on_success(_status, options)
commit = Commit.find(options['commit_id'])
+ already_loaded = commit.loaded_at.present?
# mark related blobs as parsed so that we don't parse them again
mark_not_errored_blobs_as_parsed(commit)
@@ -59,8 +60,21 @@ def on_success(_status, options)
# finish loading
commit.update!(loading: false, import_batch_id: nil)
+ # records metric only when never loaded before
+ if !already_loaded and commit.loaded_at and commit.created_at
+ loading_time = commit.loaded_at - commit.created_at
+ CustomMetricHelper.record_project_loading_time(commit.project.slug, loading_time)
+
+ active_translations = commit.active_translations.group_by { |t| t.rfc5646_locale }
+ locale_to_keys = active_translations.map { |locale, ts| [locale, ts.count] }.to_h
+ locale_to_words = active_translations.map { |locale, ts| [locale, ts.map(&:words_count).sum] }.to_h
+ CustomMetricHelper.record_project_statistics(commit.project.slug, commit.blobs.count, locale_to_keys, locale_to_words)
+ end
+
# the readiness hooks were all disabled, so now we need to go through and calculate commit readiness and stats.
CommitRecalculator.new.perform commit.id
+
+ PostLoadingChecker.launch(commit)
end
private
diff --git a/app/workers/commit_key_creator.rb b/app/workers/commit_key_creator.rb
index ef0ece2f..8d57a779 100644
--- a/app/workers/commit_key_creator.rb
+++ b/app/workers/commit_key_creator.rb
@@ -55,6 +55,7 @@ def perform(blob_id, commit_id, importer, keys)
def self.update_key_associations(keys, commit)
keys.reject! { |key| skip_key?(key, commit) }
keys.map(&:id).uniq.each { |k| commit.commits_keys.where(key_id: k).find_or_create! }
+ CommitsIndex.import!(commit.id)
end
include SidekiqLocking
diff --git a/app/workers/commits_cleaner.rb b/app/workers/commits_cleaner.rb
index eda28628..438e29d1 100644
--- a/app/workers/commits_cleaner.rb
+++ b/app/workers/commits_cleaner.rb
@@ -14,7 +14,7 @@
class CommitsCleaner
include Sidekiq::Worker
- sidekiq_options queue: :low, retry: 5
+ sidekiq_options queue: :low, retry: false
def perform
log("Cleaning old commits for #{Date.today}")
diff --git a/app/workers/inactive_user_decommissioner.rb b/app/workers/inactive_user_decommissioner.rb
new file mode 100644
index 00000000..380e4485
--- /dev/null
+++ b/app/workers/inactive_user_decommissioner.rb
@@ -0,0 +1,68 @@
+# Copyright 2016 Square Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This workder puts ex-Squarer into expired state to prevent them from accessing Shuttle.
+# The expired users can use Forgot Password to re-activate their accounts.
+class InactiveUserDecommissioner
+ include Sidekiq::Worker
+ sidekiq_options queue: :high, retry: false
+
+ def perform
+ report_message('started')
+
+ # Retrieves domain users
+ decomission_inactive_user_url = Shuttle::Configuration.features[:decomission_inactive_user_url]
+ decomission_inactive_user_domain = Shuttle::Configuration.features[:decomission_inactive_user_domain]
+
+ ldap_user_response = HTTParty.get(Shuttle::Configuration.features[:decomission_inactive_user_url])
+ unless ldap_user_response.code == 200
+ report_message("Failed to retrieve LDAP users. error code: #{ldap_user_response.code}")
+ return
+ end
+
+ ldap_user_details = ldap_user_response.parsed_response
+ active_user_details = ldap_user_details.reject { |u| u['state'] == 'disabled' }
+ if active_user_details.count < 3000
+ # Stop processing in case LDAP returns empty or partial accounts back.
+ # This will avoid putting all Square users into expired state.
+ report_message("Found #{active_user_details.count} active LDAP users. Skip processing because of too few.")
+ return
+ end
+ active_user_names = active_user_details.map { |detail| detail['username'] }
+
+ # Finds non-expired non-domain accounts
+ shuttle_users = User.where("email like '%#{decomission_inactive_user_domain}'")
+ inactive_shuttle_users = shuttle_users.reject do |user|
+ email = user.email.downcase
+
+ email_domains = email.split('@')
+ raise "Not expected domain user #{user.id}" unless email_domains.count == 2 and email_domains[1] == decomission_inactive_user_domain
+
+ email_users = email_domains[0].split('+')
+ user.expired? || active_user_names.include?(email_users[0])
+ end
+
+ # Puts the inactive accounts as expired.
+ inactive_shuttle_users.each do |inactive_user|
+ report_message("Inactivate user #{inactive_user.email}")
+ inactive_user.update(last_activity_at: User.expire_after.ago)
+ end
+ end
+
+ def report_message(message)
+ Rails.logger.info("InactiveUserDecommissioner - #{message}")
+ end
+
+ include SidekiqLocking
+end
diff --git a/app/workers/post_loading_checker.rb b/app/workers/post_loading_checker.rb
new file mode 100644
index 00000000..8ee177b6
--- /dev/null
+++ b/app/workers/post_loading_checker.rb
@@ -0,0 +1,58 @@
+# Copyright 2019 Square Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Processing the Commit, Article or Asset after loading them into Shuttle.
+
+class PostLoadingChecker
+ include Sidekiq::Worker
+ sidekiq_options queue: :high
+
+ VALIDATORS = [
+ TranslationValidator::SourceFencerValidator,
+ TranslationValidator::TranslationAutoMigration,
+ ]
+
+ def perform(job_type, job_id)
+ job = find_job(job_type, job_id)
+ return if job.nil?
+
+ VALIDATORS.each do |validator|
+ begin
+ validator.new(job).run
+ rescue => e
+ Rails.logger.error("#{PostLoadingChecker} - Failed to run #{validator} on (#{job_type}, #{job_id})")
+ Rails.logger.info("#{PostLoadingChecker} - due to exception: #{e.inspect}")
+ end
+ end
+ end
+
+ def find_job(job_type, job_id)
+ case job_type
+ when 'commit'
+ Commit.where(id: job_id).first
+ when 'article'
+ Article.where(id: job_id).first
+ when 'asset'
+ Asset.where(id: job_id).first
+ else
+ raise ArgumentError, "invalid model type: #{job_type}, #{job_id}"
+ end
+ end
+
+ def self.launch(job)
+ PostLoadingChecker.perform_once(job.project.job_type, job.id)
+ end
+
+ include SidekiqLocking
+end
diff --git a/app/workers/sidekiq_worker_restarter.rb b/app/workers/sidekiq_worker_restarter.rb
new file mode 100644
index 00000000..f221b091
--- /dev/null
+++ b/app/workers/sidekiq_worker_restarter.rb
@@ -0,0 +1,76 @@
+# Copyright 2016 Square Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This worker stops sidekiq worker gracefully after draining existing running jobs.
+class SidekiqWorkerRestarter
+ include Sidekiq::Worker
+ sidekiq_options queue: :high, retry: false
+
+ MIN_RUNNING_PROCESSES = 4
+ MAX_RUNNING_DURATION = 3.hours
+
+ def perform
+ Rails.logger.info("Sidekiq Worker Restarter - started")
+ processes = Sidekiq::ProcessSet.new.sort { |p1, p2| p1['started_at'] - p2['started_at'] }
+ record_sidekiq_metrics(processes)
+ restart_oldest_process(processes)
+ end
+
+ def restart_oldest_process(processes)
+ # finds process in quiet state
+ process = processes.select { |p| p['quiet'] == 'true' }.first
+ if process.present?
+ if process['busy'] == 0
+ Rails.logger.info("Sidekiq Worker Restarter - signal quiet worker #{process['pid']} to stop")
+ process.stop!
+ else
+ Rails.logger.info("Sidekiq Worker Restarter - quiet worker #{process['pid']} is busy (#{process['busy']} jobs)")
+ end
+ return
+ end
+
+ # Sidekiq::ProcessSet misses some workers sometimes for unknown reason.
+ # checks if there are enough available running processes
+ if processes.count <= MIN_RUNNING_PROCESSES
+ Rails.logger.info("Sidekiq Worker Restarter - no enough running workers (#{processes.count} workers)")
+ return
+ end
+
+ # finds process running longer enough to restart
+ expiration_time = MAX_RUNNING_DURATION.ago
+ process = processes.select { |p| p['started_at'] < expiration_time.to_i }.first
+ if process.present?
+ Rails.logger.info("Sidekiq Worker Restarter - signal worker #{process['pid']} to quiet")
+ process.quiet!
+ else
+ Rails.logger.info("Sidekiq Worker Restarter - no expired worker found")
+ end
+ end
+
+ def record_sidekiq_metrics(processes)
+ host_processes = processes.group_by { |p| p['hostname'] }
+ host_to_longevities = {}
+ host_processes.each do |hostname, ps|
+ min_started_at = ps.map { |p| p['started_at'] }.min
+ host_to_longevities[hostname] = Time.now.to_i - min_started_at
+ end
+ CustomMetricHelper.record_sidekiq_longevity(host_to_longevities)
+
+ busy_jobs = processes.map { |x| x['busy'] }.sum
+ enqueued_jobs = Sidekiq::Stats.new.enqueued
+ CustomMetricHelper.record_sidekiq_jobs(busy_jobs, enqueued_jobs)
+ end
+
+ include SidekiqLocking
+end
diff --git a/app/workers/stash_webhook_pinger.rb b/app/workers/stash_webhook_pinger.rb
index 6db2589f..1897743a 100644
--- a/app/workers/stash_webhook_pinger.rb
+++ b/app/workers/stash_webhook_pinger.rb
@@ -20,7 +20,7 @@ class StashWebhookPinger
include Sidekiq::Worker
include Rails.application.routes.url_helpers
- sidekiq_options queue: :high, retry: 5
+ sidekiq_options queue: :high, retry: 10
# Executes this worker.
#
diff --git a/bin/docker-setup b/bin/docker-setup
index 74d6ed80..cbd2a45c 100755
--- a/bin/docker-setup
+++ b/bin/docker-setup
@@ -5,9 +5,5 @@ require 'pathname'
APP_ROOT = Pathname.new File.expand_path('../../', __FILE__)
Dir.chdir APP_ROOT do
- system 'bin/rake', 'db:migrate', 'db:seed'
-
- system({'FORCE' => 'y', 'CLASS' => 'Commit'}, 'bin/rake', 'environment', 'elasticsearch:import:model')
- system({'FORCE' => 'y', 'CLASS' => 'Key'}, 'bin/rake', 'environment', 'elasticsearch:import:model')
- system({'FORCE' => 'y', 'CLASS' => 'Translation'}, 'bin/rake', 'environment', 'elasticsearch:import:model')
+ system 'bin/rake', 'db:migrate', 'db:seed', 'chewy:reset'
end
diff --git a/bin/docker-tests b/bin/docker-tests
index 4121c5c9..e42221cc 100755
--- a/bin/docker-tests
+++ b/bin/docker-tests
@@ -7,9 +7,9 @@ APP_ROOT = Pathname.new File.expand_path('../../', __FILE__)
Dir.chdir APP_ROOT do
system 'bin/rake', 'db:migrate'
- system({'FORCE' => 'y', 'CLASS' => 'Commit'}, 'bin/rake', 'environment', 'elasticsearch:import:model')
- system({'FORCE' => 'y', 'CLASS' => 'Key'}, 'bin/rake', 'environment', 'elasticsearch:import:model')
- system({'FORCE' => 'y', 'CLASS' => 'Translation'}, 'bin/rake', 'environment', 'elasticsearch:import:model')
+ # TODO: update with better way of waiting for ElasticSearch ready
+ system 'echo', 'wait 10 seconds for elasticsearch readiness......'
+ system 'sleep', '10'
exec 'bundle', 'exec', 'rspec', 'spec'
end
diff --git a/bin/setup b/bin/setup
index 53f5757f..baa71c2b 100755
--- a/bin/setup
+++ b/bin/setup
@@ -21,9 +21,7 @@ Dir.chdir APP_ROOT do
system "bin/rake db:setup"
puts "\n== Preparing ElasticSearch =="
- system 'bin/rake', 'elasticsearch:import:model', 'FORCE=y', 'CLASS=Commit'
- system 'bin/rake', 'elasticsearch:import:model', 'FORCE=y', 'CLASS=Key'
- system 'bin/rake', 'elasticsearch:import:model', 'FORCE=y', 'CLASS=Translation'
+ system 'bin/rake', 'chewy:reset'
puts "\n== Removing old logs and tempfiles =="
system "rm -f log/*"
diff --git a/config/application.rb b/config/application.rb
index c0c96aa2..f665ad27 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -22,6 +22,9 @@
module Shuttle
class Application < Rails::Application
+ # Do not swallow errors in after_commit/after_rollback callbacks.
+ config.active_record.raise_in_transactional_callbacks = true
+
# Load configoro settings here so that the settings can be used in application.rb, development.rb, production.rb, etc...
config.before_configuration do
Configoro.initialize
@@ -51,9 +54,6 @@ class Application < Rails::Application
# config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
# config.i18n.default_locale = :de
- # Do not swallow errors in after_commit/after_rollback callbacks.
- config.active_record.raise_in_transactional_callbacks = true
-
# Use SQL instead of Active Record's schema dumper when creating the database.
# This is necessary if your schema can't be completely dumped by the schema dumper,
# like if you have constraints or database-specific column types
diff --git a/config/chewy.yml b/config/chewy.yml
new file mode 100644
index 00000000..3ee20d76
--- /dev/null
+++ b/config/chewy.yml
@@ -0,0 +1,7 @@
+---
+development:
+ host: <%= ENV.fetch('SHUTTLE_ES_URL', 'localhost:9200') %>
+ prefix: shuttle_development
+test:
+ host: <%= ENV.fetch('SHUTTLE_ES_URL', 'localhost:9200') %>
+ prefix: shuttle_test
diff --git a/config/deploy.rb b/config/deploy.rb
index 80458bb0..3718661c 100644
--- a/config/deploy.rb
+++ b/config/deploy.rb
@@ -14,12 +14,13 @@
set :application, 'shuttle'
-set :repo_url, 'https://github.com/Square/shuttle.git'
+set :repo_url, 'https://git.sqcorp.co/scm/intl/shuttle.git'
ask(:branch, proc { `git rev-parse --abbrev-ref HEAD`.chomp })
-set :deploy_to, "/var/www/#{fetch :application}"
+set :deploy_to, "/app/#{fetch :application}"
append :linked_files,
+ 'config/chewy.yml',
'config/database.yml',
'config/secrets.yml'
append :linked_dirs,
@@ -29,15 +30,49 @@
'vendor/bundle'
set :rvm_type, :system
-set :rvm_ruby_version, "2.3.1@#{fetch :application}"
+set :rvm_ruby_version, "2.4.6@#{fetch :application}"
+set :runit_timeout, 60
set :whenever_roles, [:app, :primary_cron]
namespace :deploy do
desc 'Restart application'
task :restart do
- on roles(:app), in: :sequence, wait: 5 do
+ on roles(:web), in: :sequence, wait: 5 do
execute :touch, release_path.join('tmp/restart.txt')
end
end
end
+
+namespace :sidekiq do
+ Rake::Task["sidekiq:quiet"].clear_actions
+ task :quiet do
+ on roles fetch(:sidekiq_roles) do
+ within release_path do
+ execute :sidekiqctl, 'quiet', release_path.join('tmp/pids/sidekiq-0.pid')
+ execute :sidekiqctl, 'quiet', release_path.join('tmp/pids/sidekiq-1.pid')
+ execute :sidekiqctl, 'quiet', release_path.join('tmp/pids/sidekiq-2.pid')
+ end
+ end
+ end
+
+ Rake::Task["sidekiq:stop"].clear_actions
+ task :stop do
+ on roles fetch(:sidekiq_roles) do
+ sudo "sv -w #{fetch(:runit_timeout)} stop sidekiq0"
+ sudo "sv -w #{fetch(:runit_timeout)} stop sidekiq1"
+ sudo "sv -w #{fetch(:runit_timeout)} stop sidekiq2"
+ end
+ end
+
+ Rake::Task["sidekiq:start"].clear_actions
+ task :start do
+ on roles fetch(:sidekiq_roles) do
+ sudo "sv start sidekiq0"
+ sudo "sv start sidekiq1"
+ sudo "sv start sidekiq2"
+ end
+ end
+end
+
+after 'deploy:publishing', 'deploy:restart'
diff --git a/config/deploy/production.rb b/config/deploy/production.rb
index 65254332..320f176e 100644
--- a/config/deploy/production.rb
+++ b/config/deploy/production.rb
@@ -1,12 +1,14 @@
set :stage, :production
-WEB_BOXES = %w[user@shuttle.example.com]
-WORKER_BOXES = %w[user@shuttle.example.com]
+
+WEB_BOXES = (1..2).map { |i| "square@shuttle-web-b-#{i.to_s.rjust(2, '0')}.sqcorp.co" }
+WORKER_BOXES = (1..2).map { |i| "square@shuttle-worker-b-#{i.to_s.rjust(2, '0')}.sqcorp.co" }
role :app, WEB_BOXES + WORKER_BOXES
role :web, WEB_BOXES
role :db, WEB_BOXES.first
role :sidekiq, WORKER_BOXES
+set :sidekiq_roles, [:sidekiq]
role :primary_cron, WORKER_BOXES.first
set :branch, 'deployable'
diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb
index fcb60b4c..2fc28ced 100644
--- a/config/deploy/staging.rb
+++ b/config/deploy/staging.rb
@@ -1,13 +1,15 @@
set :stage, :staging
set :rails_env, :staging
-STAGING_BOXES = %w[user@shuttle-staging.example.com]
+DEV_WEB_BOXES = (1..1).map { |i| "square@shuttle-web-dev-a-#{i.to_s.rjust(2, '0')}.sqcorp.co" }
+DEV_WORKER_BOXES = (1..1).map { |i| "square@shuttle-worker-dev-a-#{i.to_s.rjust(2, '0')}.sqcorp.co" }
-role :app, STAGING_BOXES
-role :web, STAGING_BOXES
-role :db, STAGING_BOXES.first
-role :sidekiq, STAGING_BOXES
-role :primary_cron, STAGING_BOXES.first
+role :app, DEV_WEB_BOXES + DEV_WORKER_BOXES
+role :web, DEV_WEB_BOXES
+role :db, DEV_WEB_BOXES.first
+role :sidekiq, DEV_WORKER_BOXES
+set :sidekiq_roles, [:sidekiq]
+role :primary_cron, DEV_WORKER_BOXES.first
append :linked_files,
'config/environments/staging/credentials.yml',
diff --git a/config/environments/common/automatic_user_privileges.yml b/config/environments/common/automatic_user_privileges.yml
index 9569e0ce..a00cc273 100644
--- a/config/environments/common/automatic_user_privileges.yml
+++ b/config/environments/common/automatic_user_privileges.yml
@@ -1,7 +1,7 @@
# After a user signs up and confirms their email address, he will automatically get 'monitor' access
# if his email is from one of the domains listed here. We recommend putting in your company email
# domain here if you trust them and don't want to have to manually activate users from within the company.
-domains_to_get_monitor_role_after_email_confirmation: ['example.com']
+domains_to_get_monitor_role_after_email_confirmation: ['squareup.com']
# Will be used to determine if the current user can use the autofill feature
-domains_who_can_search_users: ['example.com']
+domains_who_can_search_users: ['squareup.com']
diff --git a/config/environments/common/features.yml b/config/environments/common/features.yml
index a681585e..0b47af85 100644
--- a/config/environments/common/features.yml
+++ b/config/environments/common/features.yml
@@ -1 +1,6 @@
enable_blank_string_auto_approval: true
+
+decomission_inactive_user_url: https://registry-office-accessible.global.square/api/v1/users/basic_info
+decomission_inactive_user_domain: squareup.com
+
+skip_levenshtein_distance_diff: true
diff --git a/config/environments/common/git.yml b/config/environments/common/git.yml
index 8d1e9036..1b4e12cb 100644
--- a/config/environments/common/git.yml
+++ b/config/environments/common/git.yml
@@ -1,4 +1,4 @@
# The author info that will be used when Shuttle is updating the touchdown branch, i.e. when pushing manifest file
author:
name: Shuttle
- email: shuttle-noreply@example.com
+ email: shuttle-noreply@squareup.com
diff --git a/config/environments/common/globalsight.yml b/config/environments/common/globalsight.yml
new file mode 100644
index 00000000..2fbf0ffd
--- /dev/null
+++ b/config/environments/common/globalsight.yml
@@ -0,0 +1 @@
+--- {}
diff --git a/config/environments/common/mailer.yml b/config/environments/common/mailer.yml
index d561edb5..f52f6146 100644
--- a/config/environments/common/mailer.yml
+++ b/config/environments/common/mailer.yml
@@ -1,2 +1,3 @@
-from: shuttle-engineers@example.com
-translators_list: translators@example.com
+from: shuttle-engineering@squareup.com
+localization_list: g11n@squareup.com
+translators_list: l10n@squareup.com
diff --git a/config/environments/common/reports.yml b/config/environments/common/reports.yml
index 5da714b6..63d0fc95 100644
--- a/config/environments/common/reports.yml
+++ b/config/environments/common/reports.yml
@@ -1 +1 @@
-internal_domains: ['@example.com']
+internal_domains: ['@squareup.com']
diff --git a/config/environments/common/services.yml b/config/environments/common/services.yml
index 86c77bb0..f104a6f1 100644
--- a/config/environments/common/services.yml
+++ b/config/environments/common/services.yml
@@ -1,8 +1,8 @@
# Will be used to determine the url of a commit
-github_enterprise_domain: "git.example.com"
+github_enterprise_domain: "git.squareup.com"
# Will be used to determine the url of a commit
-stash_domain: "stash.example.com"
+stash_domain: "git.sqcorp.co"
# Will be linked to in the footer to give users a way to provide feedback to the engineers running Shuttle
-feedback_link: "https://jira.example.com"
+feedback_link: "https://jira.corp.squareup.com/secure/CreateIssueDetails!init.jspa?pid=12827&issuetype=6"
diff --git a/config/environments/common/webhook_pinger.yml b/config/environments/common/webhook_pinger.yml
index eddc8a2d..71f0408d 100644
--- a/config/environments/common/webhook_pinger.yml
+++ b/config/environments/common/webhook_pinger.yml
@@ -1,2 +1,2 @@
---
-host: bitbucket.example.com
+host: git.sqcorp.co
diff --git a/config/environments/development/elasticsearch.yml b/config/environments/development/elasticsearch.yml
new file mode 100644
index 00000000..46387bb5
--- /dev/null
+++ b/config/environments/development/elasticsearch.yml
@@ -0,0 +1,2 @@
+---
+xpack.security.enabled: false
diff --git a/config/environments/development/sentry.yml b/config/environments/development/sentry.yml
new file mode 100644
index 00000000..8f5f0277
--- /dev/null
+++ b/config/environments/development/sentry.yml
@@ -0,0 +1,4 @@
+---
+# disabling Sentry Raven logging by not setting DSN and secret key
+dsn:
+secret_key:
diff --git a/config/environments/production.rb b/config/environments/production.rb
index a9a7035a..e0af4ccb 100644
--- a/config/environments/production.rb
+++ b/config/environments/production.rb
@@ -60,7 +60,7 @@
# Use the lowest log level to ensure availability of diagnostic information
# when problems arise.
- config.log_level = :debug
+ config.log_level = :info
# Prepend all log lines with the following tags.
# config.log_tags = [ :subdomain, :uuid ]
@@ -97,7 +97,7 @@
config.action_mailer.delivery_method = :smtp
config.action_mailer.smtp_settings = {
- address: 'localhost',
+ address: '127.0.0.1',
port: 25
}
end
diff --git a/config/environments/production/default_url_options.yml b/config/environments/production/default_url_options.yml
index 72714587..69a6f1f1 100644
--- a/config/environments/production/default_url_options.yml
+++ b/config/environments/production/default_url_options.yml
@@ -1,2 +1,2 @@
-host: shuttle.example.com
+host: shuttle.squareup.com
protocol: https
diff --git a/config/environments/production/elasticsearch.yml b/config/environments/production/elasticsearch.yml
index 3645f7c7..3c672dc7 100644
--- a/config/environments/production/elasticsearch.yml
+++ b/config/environments/production/elasticsearch.yml
@@ -1,2 +1,2 @@
---
-url: "http://shuttle-es-production.example.com:9200"
+url: "http://es-a.corp.squareup.com:9206"
diff --git a/config/environments/production/redis.yml b/config/environments/production/redis.yml
deleted file mode 100644
index 0c2a6300..00000000
--- a/config/environments/production/redis.yml
+++ /dev/null
@@ -1,2 +0,0 @@
----
-host: shuttle-redis.example.com
diff --git a/config/environments/production/workbench.yml b/config/environments/production/workbench.yml
index 9cc3a987..cfe42df9 100644
--- a/config/environments/production/workbench.yml
+++ b/config/environments/production/workbench.yml
@@ -1,2 +1,2 @@
enable_reasons: true
-show_article_groups: false
+show_article_groups: true
diff --git a/config/environments/staging.rb b/config/environments/staging.rb
index e1a5b23c..26cf7e86 100644
--- a/config/environments/staging.rb
+++ b/config/environments/staging.rb
@@ -90,4 +90,10 @@
config.log_formatter = ::Logger::Formatter.new
config.use_ssl = true
+
+ config.action_mailer.delivery_method = :smtp
+ config.action_mailer.smtp_settings = {
+ address: '127.0.0.1',
+ port: 25
+ }
end
diff --git a/config/environments/staging/default_url_options.yml b/config/environments/staging/default_url_options.yml
index 1c7fcd28..bfe3b369 100644
--- a/config/environments/staging/default_url_options.yml
+++ b/config/environments/staging/default_url_options.yml
@@ -1,2 +1,2 @@
-host: shuttle-stage.example.com
+host: shuttle-dev.sqcorp.co
protocol: https
diff --git a/config/environments/staging/elasticsearch.yml b/config/environments/staging/elasticsearch.yml
index 8592736a..8fe66f47 100644
--- a/config/environments/staging/elasticsearch.yml
+++ b/config/environments/staging/elasticsearch.yml
@@ -1,2 +1,2 @@
---
-url: "http://shuttle-es-staging.example.com:9200"
+url: "https://shuttle-es-dev.sqcorp.co"
diff --git a/config/environments/staging/services.yml b/config/environments/staging/services.yml
new file mode 100644
index 00000000..603fe4b8
--- /dev/null
+++ b/config/environments/staging/services.yml
@@ -0,0 +1,5 @@
+# Will be used to determine the url of a commit
+github_enterprise_domain: "bitbucket-staging.sqcorp.co"
+
+# Will be used to determine the url of a commit
+stash_domain: "bitbucket-staging.sqcorp.co"
diff --git a/config/environments/staging/webhook_pinger.yml b/config/environments/staging/webhook_pinger.yml
new file mode 100644
index 00000000..9935b08e
--- /dev/null
+++ b/config/environments/staging/webhook_pinger.yml
@@ -0,0 +1,2 @@
+---
+host: bitbucket-staging.sqcorp.co
diff --git a/config/environments/staging/workbench.yml b/config/environments/staging/workbench.yml
index af96205e..cfe42df9 100644
--- a/config/environments/staging/workbench.yml
+++ b/config/environments/staging/workbench.yml
@@ -1,2 +1,2 @@
-enable_reasons: false
+enable_reasons: true
show_article_groups: true
diff --git a/config/environments/test/sentry.yml b/config/environments/test/sentry.yml
new file mode 100644
index 00000000..3eeec656
--- /dev/null
+++ b/config/environments/test/sentry.yml
@@ -0,0 +1,3 @@
+---
+# disabling Sentry Raven logging by not setting a DSN
+dsn:
diff --git a/config/environments/test/sidekiq.yml b/config/environments/test/sidekiq.yml
new file mode 100644
index 00000000..e20e3426
--- /dev/null
+++ b/config/environments/test/sidekiq.yml
@@ -0,0 +1,3 @@
+---
+server_pool_size: 1
+client_pool_size: 1
diff --git a/config/initializers/chewy.rb b/config/initializers/chewy.rb
new file mode 100644
index 00000000..27a4b4b6
--- /dev/null
+++ b/config/initializers/chewy.rb
@@ -0,0 +1,23 @@
+# Instrument ElasticSearch code in NewRelic
+
+ActiveSupport::Notifications.subscribe('import_objects.chewy') do |_name, start, finish|
+ metric_name = 'Database/ElasticSearch/import'
+ duration = (finish - start).to_f
+
+ self.class.trace_execution_scoped([metric_name]) do
+ # NewRelic::Agent.instance.transaction_sampler.notice_sql(logged, nil, duration)
+ # NewRelic::Agent.instance.sql_sampler.notice_sql(logged, metric_name, nil, duration)
+ NewRelic::Agent.record_metric(metric_name, duration)
+ end
+end
+
+ActiveSupport::Notifications.subscribe('search_query.chewy') do |_name, start, finish|
+ metric_name = 'Database/ElasticSearch/search'
+ duration = (finish - start).to_f
+
+ self.class.trace_execution_scoped([metric_name]) do
+ # NewRelic::Agent.instance.transaction_sampler.notice_sql(logged, nil, duration)
+ # NewRelic::Agent.instance.sql_sampler.notice_sql(logged, metric_name, nil, duration)
+ NewRelic::Agent.record_metric(metric_name, duration)
+ end
+end
diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb
index 37b7fe88..42b7bcee 100644
--- a/config/initializers/devise.rb
+++ b/config/initializers/devise.rb
@@ -137,7 +137,7 @@
# their account can't be confirmed with the token any more.
# Default is nil, meaning there is no restriction on how long a user can take
# before confirming their account.
- # config.confirm_within = 3.days
+ config.confirm_within = 3.days
# If true, requires any email changes to be confirmed (exactly the same way as
# initial account confirmation) to be applied. Requires additional unconfirmed_email
@@ -281,4 +281,42 @@
# When using OmniAuth, Devise cannot automatically set OmniAuth path,
# so you need to do it manually. For the users scope, it would be:
# config.omniauth_path_prefix = '/my_engine/users/auth'
+
+ # ==> Security Extension
+ # Configure security extension for devise
+
+ # Should the password expire (e.g 3.months)
+ # config.expire_password_after = false
+
+ # Need 1 char of A-Z, a-z and 0-9
+ # config.password_regex = /(?=.*\d)(?=.*[a-z])(?=.*[A-Z])/
+
+ # How many passwords to keep in archive
+ # config.password_archiving_count = 5
+
+ # Deny old password (true, false, count)
+ # config.deny_old_passwords = true
+
+ # enable email validation for :secure_validatable. (true, false, validation_options)
+ # dependency: need an email validator like rails_email_validator
+ # config.email_validation = true
+
+ # captcha integration for recover form
+ # config.captcha_for_recover = true
+
+ # captcha integration for sign up form
+ # config.captcha_for_sign_up = true
+
+ # captcha integration for sign in form
+ # config.captcha_for_sign_in = true
+
+ # captcha integration for unlock form
+ # config.captcha_for_unlock = true
+
+ # captcha integration for confirmation form
+ # config.captcha_for_confirmation = true
+
+ # Time period for account expiry from last_activity_at
+ config.expire_after = 90.days
+
end
diff --git a/config/initializers/no_proxy.rb b/config/initializers/no_proxy.rb
new file mode 100644
index 00000000..ca0fc9fb
--- /dev/null
+++ b/config/initializers/no_proxy.rb
@@ -0,0 +1 @@
+ENV['no_proxy']='square,sqcorp.co,corp.squareup.com'
diff --git a/config/initializers/sentry.rb b/config/initializers/sentry.rb
new file mode 100644
index 00000000..e37d57d9
--- /dev/null
+++ b/config/initializers/sentry.rb
@@ -0,0 +1,10 @@
+require 'raven'
+
+if !Rails.env.test?
+ Raven.configure do |config|
+ config.dsn = Shuttle::Configuration.sentry.dsn if Shuttle::Configuration.sentry.dsn
+ config.secret_key = Shuttle::Configuration.sentry.secret_key if Shuttle::Configuration.sentry.secret_key
+ config.tags = { environment: Rails.env }
+ config.excluded_exceptions = ['ActionController::RoutingError', 'Sidekiq::Shutdown']
+ end
+end
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
index 74869991..75586f93 100644
--- a/config/initializers/sidekiq.rb
+++ b/config/initializers/sidekiq.rb
@@ -25,6 +25,7 @@ def process_id() Process.pid end
Sidekiq.configure_client do |config|
config.redis = redis
end
+
Sidekiq.configure_server do |config|
begin
require 'sidekiq/pro/reliable_fetch'
@@ -32,6 +33,11 @@ def process_id() Process.pid end
# no sidekiq pro
end
config.redis = redis
+
+ require 'chewy_atomic'
+ config.server_middleware do |chain|
+ chain.add ChewyAtomic
+ end
end
end
diff --git a/config/locales/devise.security_extension.en.yml b/config/locales/devise.security_extension.en.yml
new file mode 100644
index 00000000..e15bf5cb
--- /dev/null
+++ b/config/locales/devise.security_extension.en.yml
@@ -0,0 +1,14 @@
+en:
+ errors:
+ messages:
+ taken_in_past: "was already taken in the past!"
+ equal_to_current_password: "must be different to the current password!"
+ password_format: "must contain big, small letters and digits"
+ devise:
+ invalid_captcha: "The captcha input is not valid!"
+ password_expired:
+ updated: "Your new password is saved."
+ change_required: "Your password is expired. Please renew your password!"
+ failure:
+ session_limited: 'Your login credentials were used in another browser. Please sign in again to continue in this browser.'
+ expired: 'Your account has expired due to inactivity. Please use Forgot Password to re-activate your account.'
diff --git a/config/locales/en.yml b/config/locales/en.yml
index b211ceff..21c8fcbf 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -166,10 +166,14 @@ en:
header: Commits
articles:
header: Articles
+ groups: Groups
+ groups_cell: "%{pending_count} (%{linked_count})"
assets:
header: Assets
groups:
header: Article Groups
+ articles: Articles
+ articles_cell: "%{pending_count} (%{linked_count})"
assets:
project_not_found: Invalid project
asset_not_found: Asset doesn't exist
@@ -302,6 +306,7 @@ en:
Android: "{android}"
Erb: "<%= ERb %>"
Html: ""
+ IntlMessageFormat: "{ICU Message}"
MessageFormat: "Java MessageFormat [{0}]"
Mustache: "{{Mustache}}"
Printf: "printf() [%s]"
@@ -408,3 +413,8 @@ en:
request_screenshot:
subject: "[Shuttle] Requesting screenshots for commit %{sha}"
success: "Successfully requested screenshots for commit %{sha}"
+ fencer_validation:
+ suspicious_translation_found:
+ subject:
+ production: "[ACTION REQUIRED] [Shuttle] Found suspicious source strings"
+ staging: "[NO ACTION REQUIRED] [Shuttle Staging] Found suspicious source strings"
diff --git a/config/locales/zzz_manifest.yml b/config/locales/zzz_manifest.yml
new file mode 100644
index 00000000..28583753
--- /dev/null
+++ b/config/locales/zzz_manifest.yml
@@ -0,0 +1,49 @@
+---
+ja:
+ workers:
+ translations_mass_copier:
+ from:
+ not_a_targeted_or_base_locale: ソースロケールは基本ロケールでも、プロジェクトのターゲットロケールのいずれもありません
+ from_and_to_cannot_be_equal: ソースおよびターゲットロケールが互いに等しくすることができない
+ invalid_rfc5646_locale: 無効な %{kind} ロケール
+ iso639s_doesnt_match: ソースとターゲットのロケールが同じ言語ファミリに含まれていない（彼らのISO639sが一致しない）
+ project_translations_adder_and_remover_batch_still_running: プロジェクト翻訳加算バッチがまだ実行されている。それが終了した後にしてみてください。
+ to:
+ cannot_copy_to_projects_base_locale: 基本ロケールを投影するコピーすることはできません
+ not_a_targeted_locale: ターゲットロケールは、プロジェクトのターゲットロケールのものではありません
+en-AU:
+ workers:
+ translations_mass_copier:
+ from:
+ not_a_targeted_or_base_locale: Source locale is neither the base locale nor one of the project target locales
+ from_and_to_cannot_be_equal: Source and target locales cannot be equal to each other
+ invalid_rfc5646_locale: Invalid %{kind} locale
+ iso639s_doesnt_match: Source and target locales are not in the same language family (their ISO639s do not match)
+ project_translations_adder_and_remover_batch_still_running: Project Translations Adder and Remover batch is still running. Try after it finishes.
+ to:
+ cannot_copy_to_projects_base_locale: Cannot copy to project base locale
+ not_a_targeted_locale: Target locale is not one of the project target locales
+en-GB:
+ workers:
+ translations_mass_copier:
+ from:
+ not_a_targeted_or_base_locale: Source locale is neither the base locale nor one of the project target locales
+ from_and_to_cannot_be_equal: Source and target locales cannot be equal to each other
+ invalid_rfc5646_locale: Invalid %{kind} locale
+ iso639s_doesnt_match: Source and target locales are not in the same language family (their ISO639s do not match)
+ project_translations_adder_and_remover_batch_still_running: Project Translations Adder And Remover batch is still running. Try after it finishes.
+ to:
+ cannot_copy_to_projects_base_locale: Cannot copy to project base locale
+ not_a_targeted_locale: Target locale is not one of the project target locales
+en-IN:
+ workers:
+ translations_mass_copier:
+ from:
+ not_a_targeted_or_base_locale: Source locale is neither the base locale nor one of the project target locales
+ from_and_to_cannot_be_equal: Source and target locales cannot be equal to each other
+ invalid_rfc5646_locale: Invalid %{kind} locale
+ iso639s_doesnt_match: Source and target locales are not in the same language family (their ISO639s do not match)
+ project_translations_adder_and_remover_batch_still_running: Project Translations Adder And Remover batch is still running. Try after it finishes.
+ to:
+ cannot_copy_to_projects_base_locale: Cannot copy to project base locale
+ not_a_targeted_locale: Target locale is not one of the project target locales
diff --git a/config/routes.rb b/config/routes.rb
index 907b7352..7fa6095a 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -153,12 +153,19 @@
end
end
+ get 'reports' => 'reports#index', as: :reports
+ get 'reports/download/incoming/:date' => 'reports#incoming'
+ get 'reports/download/pending/:date' => 'reports#pending'
+ get 'reports/download/completed/:date' => 'reports#completed'
+
# HOME PAGES
get 'administrators' => 'home#administrators', as: :administrators
get 'translators' => 'home#translators', as: :translators
get 'reviewers' => 'home#reviewers', as: :reviewers
root to: 'home#index'
+ get 'csv' => 'home#csv'
+
require 'sidekiq/web'
begin
require 'sidekiq/pro/web'
diff --git a/config/schedule.rb b/config/schedule.rb
index 79a92332..c0bb73d0 100644
--- a/config/schedule.rb
+++ b/config/schedule.rb
@@ -18,11 +18,19 @@
runner 'AutoImporter.perform_once'
end
+every 5.minutes, roles: [:primary_cron] do
+ runner 'SidekiqWorkerRestarter.perform_once'
+end
+
+every :day, at: '12:00 am', roles: [:primary_cron] do
+ runner 'InactiveUserDecommissioner.perform_once'
+end
+
every :day, at: '12:00 am', roles: [:primary_cron] do
rake 'metrics:update'
end
-every 1.minute, roles: [:primary_cron] do
+every 5.minute, roles: [:primary_cron] do
rake 'touchdown:update'
end
@@ -30,10 +38,22 @@
rake 'maintenance:cleanup_commits'
end
-every :saturday, at: '1am', roles: [:app] do
+every :day, at: '1am', roles: [:app] do
rake 'maintenance:cleanup_repos'
end
every :day, roles: [:primary_cron] do
rake 'maintenance:reap_deleted_commits'
end
+
+every :day, at: '2am', roles: [:app] do
+ rake 'reports:generate:incoming'
+end
+
+every :day, at: '3am', roles: [:app] do
+ rake 'reports:generate:pending'
+end
+
+every :day, at: '4am', roles: [:app] do
+ rake 'reports:generate:completed'
+end
diff --git a/db/migrate/20190517232627_add_devise_extenstion.rb b/db/migrate/20190517232627_add_devise_extenstion.rb
new file mode 100644
index 00000000..77904e30
--- /dev/null
+++ b/db/migrate/20190517232627_add_devise_extenstion.rb
@@ -0,0 +1,11 @@
+class AddDeviseExtenstion < ActiveRecord::Migration
+ def change
+ # https://github.com/phatworx/devise_security_extension
+
+ # Expirable on inactivity
+ add_column :users, :last_activity_at, :datetime
+ add_column :users, :expired_at, :datetime
+ add_index :users, :last_activity_at
+ add_index :users, :expired_at
+ end
+end
diff --git a/db/migrate/20190620214800_create_reports.rb b/db/migrate/20190620214800_create_reports.rb
new file mode 100644
index 00000000..8ec34187
--- /dev/null
+++ b/db/migrate/20190620214800_create_reports.rb
@@ -0,0 +1,15 @@
+class CreateReports < ActiveRecord::Migration
+ def change
+ create_table :reports do |t|
+ t.datetime :date
+ t.string :project
+ t.string :locale
+ t.integer :strings
+ t.integer :words
+ t.string :report_type
+
+ t.timestamps null: false
+ end
+ add_index :reports, :date
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index e732d683..fe617511 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -2,8 +2,8 @@
-- PostgreSQL database dump
--
--- Dumped from database version 9.6.10
--- Dumped by pg_dump version 9.6.10
+-- Dumped from database version 9.4.21
+-- Dumped by pg_dump version 9.6.13
SET statement_timeout = 0;
SET lock_timeout = 0;
@@ -12,6 +12,7 @@ SET client_encoding = 'UTF8';
SET standard_conforming_strings = on;
SELECT pg_catalog.set_config('search_path', '', false);
SET check_function_bodies = false;
+SET xmloption = content;
SET client_min_messages = warning;
SET row_security = off;
@@ -354,10 +355,10 @@ CREATE TABLE public.commits (
exported boolean DEFAULT false NOT NULL,
loaded_at timestamp without time zone,
description text,
- author character varying,
- author_email character varying,
+ author character varying(255),
+ author_email character varying(255),
pull_request_url text,
- import_batch_id character varying,
+ import_batch_id character varying(255),
import_errors text,
revision character varying(40) NOT NULL,
fingerprint character varying,
@@ -468,6 +469,39 @@ CREATE SEQUENCE public.edit_reasons_id_seq
ALTER SEQUENCE public.edit_reasons_id_seq OWNED BY public.edit_reasons.id;
+--
+-- Name: globalsight_api_records; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.globalsight_api_records (
+ id integer NOT NULL,
+ job_id integer NOT NULL,
+ status character varying(255) NOT NULL,
+ article_id integer,
+ created_at timestamp without time zone,
+ updated_at timestamp without time zone
+);
+
+
+--
+-- Name: globalsight_api_records_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.globalsight_api_records_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+
+--
+-- Name: globalsight_api_records_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.globalsight_api_records_id_seq OWNED BY public.globalsight_api_records.id;
+
+
--
-- Name: groups; Type: TABLE; Schema: public; Owner: -
--
@@ -565,7 +599,7 @@ CREATE TABLE public.keys (
original_key text NOT NULL,
source_copy text,
context text,
- importer character varying,
+ importer character varying(255),
source text,
fencers text,
other_data text,
@@ -604,8 +638,8 @@ ALTER SEQUENCE public.keys_id_seq OWNED BY public.keys.id;
CREATE TABLE public.locale_associations (
id integer NOT NULL,
- source_rfc5646_locale character varying NOT NULL,
- target_rfc5646_locale character varying NOT NULL,
+ source_rfc5646_locale character varying(255) NOT NULL,
+ target_rfc5646_locale character varying(255) NOT NULL,
checked boolean DEFAULT false NOT NULL,
uncheck_disabled boolean DEFAULT false NOT NULL,
created_at timestamp without time zone,
@@ -679,9 +713,9 @@ CREATE TABLE public.projects (
name character varying(256) NOT NULL,
repository_url character varying(256),
created_at timestamp without time zone,
- translations_adder_and_remover_batch_id character varying,
+ translations_adder_and_remover_batch_id character varying(255),
disable_locale_association_checkbox_settings boolean DEFAULT false NOT NULL,
- base_rfc5646_locale character varying DEFAULT 'en'::character varying NOT NULL,
+ base_rfc5646_locale character varying(255) DEFAULT 'en'::character varying NOT NULL,
targeted_rfc5646_locales text,
skip_imports text,
key_exclusions text,
@@ -692,11 +726,11 @@ CREATE TABLE public.projects (
only_paths text,
skip_importer_paths text,
only_importer_paths text,
- default_manifest_format character varying,
+ default_manifest_format character varying(255),
watched_branches text,
- touchdown_branch character varying,
+ touchdown_branch character varying(255),
manifest_directory text,
- manifest_filename character varying,
+ manifest_filename character varying(255),
github_webhook_url text,
stash_webhook_url text,
api_token character(240) NOT NULL,
@@ -758,12 +792,48 @@ CREATE SEQUENCE public.reasons_id_seq
ALTER SEQUENCE public.reasons_id_seq OWNED BY public.reasons.id;
+--
+-- Name: reports; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.reports (
+ id integer NOT NULL,
+ date timestamp without time zone,
+ project character varying,
+ locale character varying,
+ strings integer,
+ words integer,
+ report_type character varying,
+ created_at timestamp without time zone NOT NULL,
+ updated_at timestamp without time zone NOT NULL
+);
+
+
+--
+-- Name: reports_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.reports_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+
+--
+-- Name: reports_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.reports_id_seq OWNED BY public.reports.id;
+
+
--
-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.schema_migrations (
- version character varying NOT NULL
+ version character varying(255) NOT NULL
);
@@ -775,8 +845,8 @@ CREATE TABLE public.screenshots (
commit_id integer NOT NULL,
created_at timestamp without time zone,
updated_at timestamp without time zone,
- image_file_name character varying,
- image_content_type character varying,
+ image_file_name character varying(255),
+ image_content_type character varying(255),
image_file_size integer,
image_updated_at timestamp without time zone,
id integer NOT NULL
@@ -1007,20 +1077,22 @@ CREATE TABLE public.users (
role character varying(50) DEFAULT NULL::character varying,
created_at timestamp without time zone,
updated_at timestamp without time zone,
- confirmation_token character varying,
- first_name character varying NOT NULL,
- last_name character varying NOT NULL,
- encrypted_password character varying NOT NULL,
+ confirmation_token character varying(255),
+ first_name character varying(255) NOT NULL,
+ last_name character varying(255) NOT NULL,
+ encrypted_password character varying(255) NOT NULL,
remember_created_at timestamp without time zone,
current_sign_in_at timestamp without time zone,
last_sign_in_at timestamp without time zone,
- current_sign_in_ip character varying,
- last_sign_in_ip character varying,
+ current_sign_in_ip character varying(255),
+ last_sign_in_ip character varying(255),
confirmed_at timestamp without time zone,
confirmation_sent_at timestamp without time zone,
locked_at timestamp without time zone,
reset_password_sent_at timestamp without time zone,
approved_rfc5646_locales text,
+ last_activity_at timestamp without time zone,
+ expired_at timestamp without time zone,
CONSTRAINT encrypted_password_exists CHECK ((char_length((encrypted_password)::text) > 20)),
CONSTRAINT users_email_check CHECK ((char_length((email)::text) > 0)),
CONSTRAINT users_failed_attempts_check CHECK ((failed_attempts >= 0)),
@@ -1124,6 +1196,13 @@ ALTER TABLE ONLY public.daily_metrics ALTER COLUMN id SET DEFAULT nextval('publi
ALTER TABLE ONLY public.edit_reasons ALTER COLUMN id SET DEFAULT nextval('public.edit_reasons_id_seq'::regclass);
+--
+-- Name: globalsight_api_records id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.globalsight_api_records ALTER COLUMN id SET DEFAULT nextval('public.globalsight_api_records_id_seq'::regclass);
+
+
--
-- Name: groups id; Type: DEFAULT; Schema: public; Owner: -
--
@@ -1173,6 +1252,13 @@ ALTER TABLE ONLY public.projects ALTER COLUMN id SET DEFAULT nextval('public.pro
ALTER TABLE ONLY public.reasons ALTER COLUMN id SET DEFAULT nextval('public.reasons_id_seq'::regclass);
+--
+-- Name: reports id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.reports ALTER COLUMN id SET DEFAULT nextval('public.reports_id_seq'::regclass);
+
+
--
-- Name: screenshots id; Type: DEFAULT; Schema: public; Owner: -
--
@@ -1287,11 +1373,11 @@ ALTER TABLE ONLY public.comments
--
--- Name: commits commits_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+-- Name: commits commits_new_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.commits
- ADD CONSTRAINT commits_pkey PRIMARY KEY (id);
+ ADD CONSTRAINT commits_new_pkey PRIMARY KEY (id);
--
@@ -1310,6 +1396,14 @@ ALTER TABLE ONLY public.edit_reasons
ADD CONSTRAINT edit_reasons_pkey PRIMARY KEY (id);
+--
+-- Name: globalsight_api_records globalsight_api_records_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.globalsight_api_records
+ ADD CONSTRAINT globalsight_api_records_pkey PRIMARY KEY (id);
+
+
--
-- Name: groups groups_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -1366,6 +1460,14 @@ ALTER TABLE ONLY public.reasons
ADD CONSTRAINT reasons_pkey PRIMARY KEY (id);
+--
+-- Name: reports reports_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.reports
+ ADD CONSTRAINT reports_pkey PRIMARY KEY (id);
+
+
--
-- Name: screenshots screenshots_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -1437,10 +1539,10 @@ CREATE INDEX comments_user ON public.comments USING btree (user_id);
--
--- Name: commits_date; Type: INDEX; Schema: public; Owner: -
+-- Name: commits_date_new; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX commits_date ON public.commits USING btree (project_id, committed_at);
+CREATE INDEX commits_date_new ON public.commits USING btree (project_id, committed_at);
--
@@ -1458,10 +1560,10 @@ CREATE INDEX commits_priority ON public.commits USING btree (priority, due_date)
--
--- Name: commits_ready_date; Type: INDEX; Schema: public; Owner: -
+-- Name: commits_ready_date_new; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX commits_ready_date ON public.commits USING btree (project_id, ready, committed_at);
+CREATE INDEX commits_ready_date_new ON public.commits USING btree (project_id, ready, committed_at);
--
@@ -1667,6 +1769,13 @@ CREATE UNIQUE INDEX index_locale_associations_on_source_and_target_rfc5646_local
CREATE UNIQUE INDEX index_projects_on_api_token ON public.projects USING btree (api_token);
+--
+-- Name: index_reports_on_date; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_reports_on_date ON public.reports USING btree (date);
+
+
--
-- Name: index_sections_on_article_id; Type: INDEX; Schema: public; Owner: -
--
@@ -1730,6 +1839,20 @@ CREATE INDEX index_translations_on_rfc5646_locale ON public.translations USING b
CREATE UNIQUE INDEX index_users_on_confirmation_token ON public.users USING btree (confirmation_token);
+--
+-- Name: index_users_on_expired_at; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_users_on_expired_at ON public.users USING btree (expired_at);
+
+
+--
+-- Name: index_users_on_last_activity_at; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_users_on_last_activity_at ON public.users USING btree (last_activity_at);
+
+
--
-- Name: issues_translation; Type: INDEX; Schema: public; Owner: -
--
@@ -1923,6 +2046,14 @@ ALTER TABLE ONLY public.commits_keys
ADD CONSTRAINT commits_keys_commit_id_fkey FOREIGN KEY (commit_id) REFERENCES public.commits(id) ON DELETE CASCADE;
+--
+-- Name: commits_keys commits_keys_commit_id_fkey1; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.commits_keys
+ ADD CONSTRAINT commits_keys_commit_id_fkey1 FOREIGN KEY (commit_id) REFERENCES public.commits(id) ON DELETE CASCADE;
+
+
--
-- Name: commits_keys commits_keys_key_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -1932,11 +2063,11 @@ ALTER TABLE ONLY public.commits_keys
--
--- Name: commits commits_project_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+-- Name: commits commits_new_project_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.commits
- ADD CONSTRAINT commits_project_id_fkey FOREIGN KEY (project_id) REFERENCES public.projects(id) ON DELETE CASCADE;
+ ADD CONSTRAINT commits_new_project_id_fkey FOREIGN KEY (project_id) REFERENCES public.projects(id) ON DELETE CASCADE;
--
@@ -2127,7 +2258,7 @@ ALTER TABLE ONLY public.translations
-- PostgreSQL database dump complete
--
-SET search_path TO "$user", public;
+SET search_path TO "$user",public;
INSERT INTO schema_migrations (version) VALUES ('20130605211557');
@@ -2161,6 +2292,8 @@ INSERT INTO schema_migrations (version) VALUES ('20130821011614');
INSERT INTO schema_migrations (version) VALUES ('20131008220117');
+INSERT INTO schema_migrations (version) VALUES ('20131031034100');
+
INSERT INTO schema_migrations (version) VALUES ('20131111213136');
INSERT INTO schema_migrations (version) VALUES ('20131116042827');
@@ -2301,6 +2434,8 @@ INSERT INTO schema_migrations (version) VALUES ('20160404235737');
INSERT INTO schema_migrations (version) VALUES ('20160516051607');
+INSERT INTO schema_migrations (version) VALUES ('20170126001545');
+
INSERT INTO schema_migrations (version) VALUES ('20170508202319');
INSERT INTO schema_migrations (version) VALUES ('20171024225818');
@@ -2339,3 +2474,7 @@ INSERT INTO schema_migrations (version) VALUES ('20181028020548');
INSERT INTO schema_migrations (version) VALUES ('20181029221959');
+INSERT INTO schema_migrations (version) VALUES ('20190517232627');
+
+INSERT INTO schema_migrations (version) VALUES ('20190620214800');
+
diff --git a/doc/DEVELOPER_GUIDE.md b/doc/DEVELOPER_GUIDE.md
index 39786747..57a328ce 100644
--- a/doc/DEVELOPER_GUIDE.md
+++ b/doc/DEVELOPER_GUIDE.md
@@ -140,6 +140,9 @@ use one or both of them.
### Downloading manifest files
+If you're using the Shuttle Gem, it adds the `rake shuttle:complete[git_sha]`
+and `rake shuttle:preview[git_sha]` commands.
+
For localization libraries that store their translated strings in one file (this
is most libraries, including Rails i18n, Ember.js, and others), you can download
a manifest file. This file will contain solely translated strings for one or
@@ -171,7 +174,6 @@ The value of `:format` will depend on the localization library you are using:
| Library | `:format` | Notes |
|:------------------------------------|:-------------|:--------------------------------------------------------------------------------------------------------|
-| Android | `android` | Will be a gzipped tarball that can be extracted into your project root. |
| Rails i18n (YAML) | `yaml` | All locales will be merged into one file unless the `locale` query parameter is specified. |
| Ember.js | `js` | All locales will be merged into one file unless the `locale` query parameter is specified. |
| Ember.js (dependency-injected) | `jsm` | Similar to Ember.js, but places translations under a `module.exports` object. Locale must be specified. |
@@ -193,6 +195,9 @@ A normal response status is 200 OK. If the commit is not fully localized and
`:format` is not recognized, the response status will be 406 Not Acceptable. If
`locale` is required and not provided, the response status is 400 Bad Request.
+For Android developers, follow the **Downloading localized files**
+section and copy the localized `res` folders into your repo's `res` folder.
+
### Downloading localized files
For localization libraries that expect translated strings to be reintegrated
diff --git a/doc/README_FOR_SQUARES.md b/doc/README_FOR_SQUARES.md
new file mode 100644
index 00000000..1bffcf7a
--- /dev/null
+++ b/doc/README_FOR_SQUARES.md
@@ -0,0 +1,54 @@
+# Workflow for Square Engineers contributing to Shuttle
+
+Shuttle's codebase is opensouced on Github but also present on Bitbucket.
+Nearly all contributions to Shuttle should be made against the Github
+repository. There is a small amount of Square specific code that only exists in
+the Bitbucket repository. The list of Square specific code includes:
+
+* Sentry bug reporting
+* Square specific deploy configuration
+* Square specific Redis and ElasticSearch IPs
+* Square SSL certificates in Docker
+* Square gem mirror
+* Set `no_proxy` to avoid proxying corp requests
+* Kochiku CI script
+
+### Example steps to setup Shuttle for development with Github and Bitbucket as remotes
+
+Setup Repository:
+
+ mkdir ~/Development/shuttle
+ cd ~/Development/shuttle
+
+ git init
+ git remote add bitbucket ssh://git@git.corp.squareup.com/intl/kochiku.git
+ git remote add github git@github.com:square/shuttle.git
+ git fetch bitbucket
+ git fetch github
+
+ git checkout -b github-master github/master
+ git checkout -b bitbucket-master bitbucket/master
+
+Create and push a new Github branch:
+
+ git checkout github-master
+ git pull
+ git checkout -b new-github-branch-name
+ # ... make changes and commit them
+ git push -u github HEAD
+
+Create and push a new Bitbucket branch:
+
+ git checkout bitbucket-master
+ git pull
+ git checkout -b new-bitbucket-branch-name
+ # ... make changes and commit them
+ git push -u bitbucket HEAD
+
+Merge changes on Github into Bitbucket:
+
+ git checkout bitbucket-master
+ git pull
+ git fetch github
+ git merge --log --no-ff github/master
+ git push bitbucket master
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 3d400738..c41e6b25 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -12,7 +12,11 @@ services:
image: redis
elasticsearch:
- image: quay.io/trackmaven/elasticsearch:1.7
+ image: docker.elastic.co/elasticsearch/elasticsearch:6.5.1
+ ports:
+ - 9200:9200
+ environment:
+ - xpack.security.enabled=false
tests:
build:
@@ -21,18 +25,14 @@ services:
- BUNDLE_GEMS__CONTRIBSYS__COM
links: &links
- postgresql
- - postgresql:postgresql.shuttle.local
- redis
- - redis:redis.shuttle.local
- elasticsearch
- - elasticsearch:elasticsearch.shuttle.local
environment: &environment
- RAILS_ENV=test
- RACK_ENV=test
- - SHUTTLE_DB_HOST=postgresql.shuttle.local
- - SHUTTLE_REDIS_HOST=redis.shuttle.local
- - SHUTTLE_ES_URL=elasticsearch.shuttle.local:9200
- - SHUTTLE_MAIL_HOST=mail.shuttle.local
+ - SHUTTLE_DB_HOST=postgresql
+ - SHUTTLE_REDIS_HOST=redis
+ - SHUTTLE_ES_URL=elasticsearch:9200
# worker:
# build: .
diff --git a/docker-compose.yml b/docker-compose.yml
index c945de6f..af828c02 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -4,19 +4,25 @@ services:
postgresql:
image: postgres:9.4
volumes:
- - postgres_data:/var/lib/postgresql/data
+ - ./../pg_data:/var/lib/postgresql/data
environment:
- POSTGRES_USER=shuttle
- POSTGRES_PASSWORD=
- POSTGRES_DB=shuttle_development
+ ports:
+ - '5432:5432'
redis:
image: redis
elasticsearch:
- image: quay.io/trackmaven/elasticsearch:1.7
+ image: docker.elastic.co/elasticsearch/elasticsearch:6.5.1
volumes:
- es_data:/usr/share/elasticsearch/data
+ ports:
+ - '9200:9200'
+ environment:
+ - xpack.security.enabled=false
web:
build: .
@@ -24,11 +30,8 @@ services:
ports:
- '3000:3000'
links: &links
- - postgresql
- postgresql:postgresql.shuttle.local
- - redis
- redis:redis.shuttle.local
- - elasticsearch
- elasticsearch:elasticsearch.shuttle.local
- mailcatcher:mail.shuttle.local
environment: &environment
@@ -38,6 +41,8 @@ services:
- SHUTTLE_REDIS_HOST=redis.shuttle.local
- 'SHUTTLE_ES_URL=elasticsearch.shuttle.local:9200'
- SHUTTLE_MAIL_HOST=mail.shuttle.local
+ volumes:
+ - .:/app
worker:
build: .
diff --git a/kochiku.yml b/kochiku.yml
new file mode 100644
index 00000000..7757f86d
--- /dev/null
+++ b/kochiku.yml
@@ -0,0 +1,8 @@
+language: ruby
+ruby:
+ - ruby-2.4.6
+test_command: 'script/ci'
+targets:
+- type: specs
+ glob: spec/**/*_spec.rb
+ workers: 10
diff --git a/lib/assets/javascripts/fencer.js.coffee b/lib/assets/javascripts/fencer.js.coffee
index 90eb7546..46e9574e 100644
--- a/lib/assets/javascripts/fencer.js.coffee
+++ b/lib/assets/javascripts/fencer.js.coffee
@@ -34,7 +34,7 @@ class root.Fencer
when 'MessageFormat'
fence_index = fence.match(/^\{(\d+)[,}]/)[1]
new RegExp("\\{#{fence_index}(,[^}]+)?\\}")
- when 'Erb', 'Html', 'Strftime'
+ when 'Erb', 'Html', 'Strftime', 'IntlMessageFormat'
# these are all optional fences
null
else
diff --git a/lib/compiler.rb b/lib/compiler.rb
index 1703fb35..73f005d9 100644
--- a/lib/compiler.rb
+++ b/lib/compiler.rb
@@ -60,15 +60,16 @@ def manifest(format, options={})
io = StringIO.new
exporter = exporter.new(@commit)
+ exporter.override_encoding = options[:encoding]
exporter.export io, *locales_for_export(*locales, options[:partial])
io.rewind
filename = locales.size == 1 ? locales.first.rfc5646 : 'manifest'
return File.new(
io,
- exporter.class.character_encoding,
+ exporter.active_encoding,
"#{filename}.#{exporter.class.file_extension}",
- exporter.class.mime_type
+ exporter.mime_type
)
end
diff --git a/lib/exporter/base.rb b/lib/exporter/base.rb
index 08cd9ca8..aa41ae63 100644
--- a/lib/exporter/base.rb
+++ b/lib/exporter/base.rb
@@ -30,6 +30,8 @@ module Exporter
class Base
extend AbstractClass
+ attr_accessor :override_encoding
+
# Prepares an exporter for use with a Commit.
#
# @param [Commit] commit A Commit whose Translations will be exported.
@@ -75,9 +77,9 @@ def self.request_mime
# @return [String] The MIME type of this exporter's output format.
- def self.mime_type
- mime = request_mime.to_s
- mime << "; charset=#{character_encoding.downcase}" unless character_encoding == 'UTF-8'
+ def mime_type
+ mime = self.class.request_mime.to_s
+ mime += "; charset=#{active_encoding.downcase}" unless active_encoding == 'UTF-8'
mime
end
@@ -89,12 +91,20 @@ def self.human_name() I18n.t "exporter.#{ident}.name" end
def self.ident() to_s.demodulize.underscore end
# @return [String] The character encoding the output is in.
- def self.character_encoding() 'UTF-8' end
+ def self.default_encoding() 'UTF-8' end
# @return [true, false] Whether this exporter is capable of exporting
# multiple locales in a single file (default true).
def self.multilingual?() true end
+ def active_encoding
+ if override_encoding&.upcase == 'UTF-8'
+ override_encoding
+ else
+ self.class.default_encoding
+ end
+ end
+
# Locates an exporter subclass from its unique identifier.
#
# @param [String] ident An identifier.
@@ -215,6 +225,7 @@ def translation_hash(locale, deduplicate=[])
# or an array index (previous if did run)
this_object[last] = translation.copy
rescue => err
+ Raven.capture_exception err, extra: { translation_id: translation.id }
raise if Rails.env.test?
end
end
diff --git a/lib/exporter/ios.rb b/lib/exporter/ios.rb
index 04787c7c..43f662cc 100644
--- a/lib/exporter/ios.rb
+++ b/lib/exporter/ios.rb
@@ -23,8 +23,11 @@ module Exporter
class Ios < Base
include Multifile
+ def self.default_encoding() 'UTF-16LE' end
+
def export_files(receiver, *locales)
exporter = Exporter::Strings.new(@commit)
+ exporter.override_encoding = override_encoding
locales.each do |locale|
Translation.in_commit(@commit).includes(:key).
@@ -39,9 +42,11 @@ def export_files(receiver, *locales)
next unless source.end_with?('.strings')
stream = StringIO.new
- # write the BOM
- stream.putc 0xFF
- stream.putc 0xFE
+ if active_encoding == 'UTF-16LE'
+ # write the BOM
+ stream.putc 0xFF
+ stream.putc 0xFE
+ end
translations.sort_by { |t| t.key.key }.each do |translation|
exporter.export_translation stream, translation
end
diff --git a/lib/exporter/strings.rb b/lib/exporter/strings.rb
index 541d4241..16d29f3f 100644
--- a/lib/exporter/strings.rb
+++ b/lib/exporter/strings.rb
@@ -36,9 +36,11 @@ def export(io, *locales)
raise NoLocaleProvidedError, ".strings files can only be for a single locale" unless locales.size == 1
locale = locales.first
- # write the BOM
- io.putc 0xFF
- io.putc 0xFE
+ if active_encoding == 'UTF-16LE'
+ # write the BOM
+ io.putc 0xFF
+ io.putc 0xFE
+ end
translations = Translation.in_commit(@commit).where(rfc5646_locale: locale.rfc5646).
sort_by { |t| t.key.key }
@@ -56,11 +58,15 @@ def export_translation(io, translation)
part << %("#{escape translation.key.original_key}" = "#{escape translation.copy}";\n)
part << "\n"
- io.write part.encode('UTF-16LE').force_encoding('BINARY')
+ if active_encoding == 'UTF-16LE'
+ io.write part.encode(active_encoding).force_encoding('BINARY')
+ else
+ io.write part.encode(active_encoding)
+ end
end
def self.file_extension() 'strings' end
- def self.character_encoding() 'UTF-16LE' end
+ def self.default_encoding() 'UTF-16LE' end
def self.request_format() :strings end
def self.multilingual?() false end
diff --git a/lib/fencer/intl_message_format.rb b/lib/fencer/intl_message_format.rb
index bd1bfb00..4c27f0f6 100644
--- a/lib/fencer/intl_message_format.rb
+++ b/lib/fencer/intl_message_format.rb
@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+require 'message_format'
+
module Fencer
# Fences out Message Format tags using the intl-messageformat syntax.
@@ -21,52 +23,66 @@ module Fencer
module IntlMessageFormat
extend self
- UNESCAPED_LEFT_BRACE = /(^.?|[^\\][^\\]){/
- UNESCAPED_RIGHT_BRACE = /(^.?|[^\\][^\\])}/
+ ESCAPED_BRACE = /\\([{}])/
def fence(string)
- scanner = UnicodeScanner.new(string)
-
- tokens = Hash.new { |hsh, k| hsh[k] = [] }
- until scanner.eos?
- match = scanner.scan_until(UNESCAPED_LEFT_BRACE)
- break unless match
-
- start = scanner.pos - 1 # less the brace
- token = scanner.scan_until(UNESCAPED_RIGHT_BRACE)
- next unless token
-
- stop = scanner.pos - 1 # ranges are inclusive
- tokens['{' + token] << (start..stop)
+ begin
+ tokenize(::MessageFormat::Parser.parse(sanitize(string)))
+ rescue
+ {}
end
-
- return tokens
end
- # Scan string to ensure that left braces are paired with right braces
- # and that left braces are closed before encountering another left brace.
def valid?(string)
- scanner = UnicodeScanner.new(string)
-
- first_left_brace_match = scanner.scan_until(UNESCAPED_LEFT_BRACE)
- return true unless first_left_brace_match
+ begin
+ ::MessageFormat::Parser.parse(sanitize(string))
+ true
+ rescue
+ false
+ end
+ end
- until scanner.eos?
- # Make sure there's a right brace to match with the left one.
- right_brace_match = scanner.scan_until(UNESCAPED_RIGHT_BRACE)
- return false unless right_brace_match
+ private
- right_brace_index = scanner.pos
- scanner.unscan # Reset to last time we saw a left brace.
+ def tokenize(segments, prefix: nil)
+ # takes in the parsed structure from message_format and tokenize recursively all parameters in the string
+ # {name} => {":name" => [-1..0]}
+ # {num, plural , =0 {nothing!} one {one!}} => "{":number|plural|=0" => [-1..0], ":number|plural|one" => [-1..0]}"
+ tokens = {}
- # If there are no more left braces, we're good.
- left_brace_match = scanner.scan_until(UNESCAPED_LEFT_BRACE)
- return true unless left_brace_match
+ segments.each do |segment|
+ next unless segment.instance_of?(Array)
+ if segment.last.instance_of?(Hash)
+ # select, plural, selectordinal
+ *data, format = segment
+ format.each do |k, v|
+ key = generate_key(prefix, [*data, k])
- # Make sure the next right brace happens before the next left brace.
- left_brace_index = scanner.pos
- return false if left_brace_index < right_brace_index
+ if v.length == 1 && v[0].instance_of?(String)
+ # option is plain string
+ tokens[key] = (tokens[key] || []).push(-1..0)
+ else
+ tokens.merge!(tokenize(v, prefix: key))
+ end
+ end
+ else
+ # simple argument, number, date, time type
+ key = generate_key(prefix, segment)
+ tokens[key] = (tokens[key] || []).push(-1..0)
+ end
end
+
+ tokens
+ end
+
+ def generate_key(prefix, args)
+ "#{prefix}:#{args.join('|')}"
+ end
+
+ def sanitize(string)
+ # intl-message-format uses blackslash for escaping which is not consitent with ICU standard
+ # replace blackslash with single quote so that message_format lib can parse it properly
+ string.gsub(ESCAPED_BRACE, '\'\1\'')
end
end
end
diff --git a/lib/importer/android.rb b/lib/importer/android.rb
index 954745e0..c4d4ae4a 100644
--- a/lib/importer/android.rb
+++ b/lib/importer/android.rb
@@ -53,8 +53,9 @@ def import_strings
end
context = find_comment(tag).try!(:content)
+ content = has_cdata?(tag) ? tag.children[0].to_s : tag.content
add_string "#{file.path}:#{tag['name']}",
- unescape(strip(tag.content)),
+ unescape(strip(content)),
context: clean_comment(context),
original_key: tag['name']
end
@@ -143,6 +144,14 @@ def unescape(string)
return result
end
+ def has_cdata?(tag)
+ tag.children.each do |child|
+ if child.to_s.include?('CDATA')
+ return true
+ end
+ end
+ end
+
def find_comment(tag)
tag = tag.previous
tag = tag.previous while tag.try!(:text?)
diff --git a/lib/importer/strings.rb b/lib/importer/strings.rb
index f9f206a8..614d748d 100644
--- a/lib/importer/strings.rb
+++ b/lib/importer/strings.rb
@@ -68,6 +68,8 @@ def unescape(str)
result << "\r"
elsif scanner.scan(/t/)
result << "\t"
+ elsif scanner.scan(/\n/)
+ # join lines
elsif (match = scanner.scan(/[0-9a-f]{4}/))
result << Integer("0x#{match}").chr('utf-16')
elsif (match = scanner.scan(/[0-7]{3}/))
diff --git a/lib/paginatable_objects.rb b/lib/paginatable_objects.rb
index 9596eb8a..18accfe5 100644
--- a/lib/paginatable_objects.rb
+++ b/lib/paginatable_objects.rb
@@ -17,11 +17,9 @@ class PaginatableObjects
attr_reader :objects, :total_count, :current_page, :limit_value
delegate :map, :each, :first, :length, :size, :sort_by, to: :objects
- def initialize(objects, objects_in_es, current_page, limit_value, sort = true)
- @objects = sort ?
- SortingHelper.order_by_elasticsearch_result_order(objects, objects_in_es)
- : objects
- @total_count = objects_in_es.total
+ def initialize(objects, current_page, limit_value)
+ @objects = objects.objects
+ @total_count = objects.total
@current_page = current_page
@limit_value = limit_value
end
diff --git a/lib/sorting_helper.rb b/lib/sorting_helper.rb
deleted file mode 100644
index bcca8b61..00000000
--- a/lib/sorting_helper.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# Copyright 2016 Square Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-module SortingHelper
- def self.order_by_elasticsearch_result_order(objects, objects_in_es)
- ordered_ids = objects_in_es.map(&:id).map(&:to_i)
- hash = objects.index_by(&:id)
- ordered_objects = []
- ordered_ids.each { |id| ordered_objects << hash[id] if hash[id] }
- ordered_objects
- end
-end
diff --git a/lib/stash_webhook_helper.rb b/lib/stash_webhook_helper.rb
index aa00c674..2323b6d3 100644
--- a/lib/stash_webhook_helper.rb
+++ b/lib/stash_webhook_helper.rb
@@ -23,15 +23,31 @@ def ping(commit, opts = {})
num_pings = opts[:num_times] || DEFAULT_NUM_TIMES
if ping_stash_webhook?(commit)
- # Pretty awful but there's no way we can verify that Stash decided to ignore us
- # Other projects do it this way as well
num_pings.times do
- HTTParty.post(webhook_url(commit), {timeout: 5,
- body: webhook_post_parameters(commit, opts),
- headers: webhook_header_parameters,
- basic_auth: webhook_auth_parameters})
+ params = {
+ timeout: 5,
+ body: webhook_post_parameters(commit, opts),
+ headers: webhook_header_parameters,
+ basic_auth: webhook_auth_parameters
+ }
+ response = HTTParty.post(webhook_url(commit), params)
+
+ # fails with retry on failure.
+ unless response.code.between? 200, 299
+ message = "[StashWebhookHelper] Failed to ping stash for commit #{commit.id}, revision: #{commit.revision}, code: #{response.code}"
+ details = "#{current_commit_state(commit)} - #{response.inspect}"
+ Rails.logger.warn("#{message} - #{details}")
+ raise message
+ end
+
Kernel.sleep(DEFAULT_WAIT_TIME)
end
+
+ # record metric only when the commit is ready
+ if commit.ready and commit.approved_at
+ ping_stash_time = Time.current - commit.approved_at
+ CustomMetricHelper.record_project_ping_stash_time(commit.project.slug, ping_stash_time)
+ end
end
end
diff --git a/lib/tasks/elasticsearch.rake b/lib/tasks/elasticsearch.rake
deleted file mode 100644
index 001eeaea..00000000
--- a/lib/tasks/elasticsearch.rake
+++ /dev/null
@@ -1,54 +0,0 @@
-# Copyright 2016 Square Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# A collection of Rake tasks to facilitate importing data from your models into Elasticsearch.
-#
-# To import all elasticsearch indexed models, run:
-#
-# $ bundle exec rake RAILS_ENV=environment elasticsearch:import:model FORCE=y
-#
-# To import the records from your `Article` model, run:
-#
-# $ bundle exec rake RAILS_ENV=environment elasticsearch:import:model CLASS='MyModel'
-#
-# Run this command to display usage instructions:
-#
-# $ bundle exec rake -D elasticsearch
-#
-
-require 'elasticsearch/rails/tasks/import'
-
-Rake::Task['elasticsearch:import:all'].clear
-namespace :elasticsearch do
- namespace :import do
- task :all do
- dir = ENV['DIR'].presence || Rails.root.join('app', 'models')
- puts "[IMPORT] Loading models from: #{dir}"
- Pathname.glob(dir.join('**', '*.rb')).each do |path|
- next if path.each_filename.include?('concerns')
- next if path.each_filename.include?('observers')
- require path.relative_path_from(dir)
- end
-
- ActiveRecord::Base.subclasses.each do |klass|
- next unless klass.respond_to?(:__elasticsearch__)
- puts "[IMPORT] Processing model: #{klass}..."
-
- ENV['CLASS'] = klass.to_s
- Rake::Task['elasticsearch:import:model'].invoke
- Rake::Task['elasticsearch:import:model'].reenable
- end
- end
- end
-end
diff --git a/lib/tasks/maintenance/cleanup_commits.rake b/lib/tasks/maintenance/cleanup_commits.rake
index 1fa6c81f..87b3c065 100644
--- a/lib/tasks/maintenance/cleanup_commits.rake
+++ b/lib/tasks/maintenance/cleanup_commits.rake
@@ -20,13 +20,13 @@ namespace :maintenance do
desc "Cleans up commits that have been deleted from the repository"
task reap_deleted_commits: :environment do
- Commit.includes(:project).find_each do |c|
- begin
- c.commit!
- rescue Git::CommitNotFoundError
- c.destroy
- rescue Elasticsearch::Transport::Transport::Errors::NotFound
- next
+ Chewy.strategy(:atomic) do
+ Commit.includes(:project).find_each do |c|
+ begin
+ c.commit!
+ rescue Git::CommitNotFoundError
+ c.destroy
+ end
end
end
end
diff --git a/lib/tasks/reports.rake b/lib/tasks/reports.rake
new file mode 100644
index 00000000..16844e00
--- /dev/null
+++ b/lib/tasks/reports.rake
@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+# This namespace is used to generate reports for incoming, pending, and
+# completed jobs over a given time
+
+namespace :reports do
+ desc 'Get reports for projects in Shuttle'
+ namespace :generate do
+ task incoming: :environment do
+ system('clear') || system('cls')
+ puts "[reports:generate:incoming] Creating a new report for #{Date.today}"
+ generate_incoming_report
+ end
+
+ task pending: :environment do
+ system('clear') || system('cls')
+ puts "[reports:generate:pending] Creating a new report for #{Date.today}"
+ generate_pending_report
+ end
+
+ task completed: :environment do
+ system('clear') || system('cls')
+ puts "[reports:generate:completed] Creating a new report for #{Date.today}"
+ generate_completed_report
+ end
+ end
+
+ def generate_incoming_report
+ filename = "incoming-#{get_display_date}.csv"
+ date_start = Date.yesterday.beginning_of_day
+ date_range = date_start...Date.yesterday.end_of_day
+
+ commits_loaded = get_commits(date_range)
+ articles_loaded = get_articles(date_range)
+ untranslated_commit_keys = get_untranslated_keys(commits_loaded)
+ untranslated_article_keys = get_untranslated_keys(articles_loaded)
+
+ puts "Found #{untranslated_commit_keys.count} commits and #{untranslated_article_keys.count} articles from #{date_start} to #{Date.today}"
+
+ combined_jobs = commits_loaded.concat(articles_loaded)
+ jobs = create_hash_for_export(combined_jobs)
+ save_incoming_translations jobs
+ end
+
+ def generate_pending_report
+ filename = "pending-#{get_display_date}.csv"
+ yesterday = Date.yesterday
+ date_range = yesterday...Date.today
+
+ commits_loaded = get_commits(date_range)
+ articles_loaded = get_articles(date_range)
+ untranslated_commit_keys = get_untranslated_keys(commits_loaded)
+ untranslated_article_keys = get_untranslated_keys(articles_loaded)
+
+ puts "Found #{untranslated_commit_keys.count} commits and #{untranslated_article_keys.count} articles from #{yesterday} to #{Date.today}"
+ combined_jobs = commits_loaded.concat(articles_loaded)
+ jobs = create_hash_for_export(combined_jobs)
+ save_pending_translations jobs
+ end
+
+ def generate_completed_report
+ filename = "completed-#{get_display_date}.csv"
+ yesterday = Date.yesterday
+ date_range = yesterday...Date.today
+
+ completed_translations = get_completed_translations(date_range)
+ parsed_translations = get_report_from completed_translations
+ save_completed_translations parsed_translations
+ end
+
+ private
+
+ def get_commits(range)
+ Commit.where(loaded_at: range)
+ end
+
+ def get_articles(range)
+ Article.where(created_at: range)
+ end
+
+ def get_untranslated_keys(list_of_jobs)
+ list_of_untranslated_keys = []
+ list_of_jobs.where(ready: false).each do |job|
+ list_of_untranslated_keys << job.keys.where(ready: false)
+ end
+ end
+
+ def get_completed_translations(range)
+ commits = Commit.where(approved_at: range)
+ articles = Article.where(last_completed_at: range)
+ jobs = commits.concat(articles)
+ completed_translations = []
+ puts "jobs in tranlsated: #{jobs.count}"
+ jobs.each do |job|
+ translations = job.translations.where updated_at: range
+ completed_translations << translations.flatten
+ end
+ completed_translations.flatten
+ end
+
+ def get_report_from(translations)
+ export_hash = {}
+ translations.each do |translation|
+ project_name = translation.key.project.name
+ locale = translation.rfc5646_locale
+ identifier = :"#{project_name}_#{locale}"
+ unless export_hash[identifier]
+ export_hash[identifier] = {
+ project: project_name,
+ locale: locale,
+ strings: 0,
+ words: 0
+ }
+ end
+ export_hash[identifier][:strings] = export_hash[identifier][:strings] + 1
+ export_hash[identifier][:words] = export_hash[identifier][:words] + translation.words_count
+ end
+ export_hash
+ end
+
+ def create_hash_for_export(list_of_jobs)
+ jobs = {}
+ list_of_jobs.each do |job|
+ job.keys.where(ready: false).each do |key|
+ project = Project.find(key.project_id)
+ project_name = project.name
+ job.targeted_rfc5646_locales.each do |locale, _exists|
+ identifier = :"#{project_name}_#{locale}"
+ unless jobs[identifier]
+ jobs[identifier] = {
+ locale: locale,
+ project: project,
+ strings: 0,
+ words: 0
+ }
+ end
+ jobs[identifier][:strings] = jobs[identifier][:strings] + 1
+ key.translations.where(approved: [false, nil]).each do |translation|
+ jobs[identifier][:words] = jobs[identifier][:words] + translation.words_count
+ end
+ end
+ end
+ end
+ jobs
+ end
+
+ def save_completed_translations(translations)
+ translations.each do |_identifier, stats|
+ r = Report.new
+ r.project = stats[:project]
+ r.locale = stats[:locale]
+ r.strings = stats[:strings]
+ r.words = stats[:words]
+ r.date = Date.today
+ r.report_type = :completed
+ r.save
+ end
+ end
+
+ def save_incoming_translations(translations)
+ translations.each do |_identifier, stats|
+ r = Report.new
+ r.project = stats[:project].name
+ r.locale = stats[:locale]
+ r.strings = stats[:strings]
+ r.words = stats[:words]
+ r.date = Date.today
+ r.report_type = :incoming
+ r.save
+ end
+ end
+
+ def save_pending_translations(translations)
+ translations.each do |_identifier, stats|
+ r = Report.new
+ r.project = stats[:project].name
+ r.locale = stats[:locale]
+ r.strings = stats[:strings]
+ r.words = stats[:words]
+ r.date = Date.today
+ r.report_type = :pending
+ r.save
+ end
+ end
+
+ def get_display_date
+ Date.today.strftime("%Y-%m-%d")
+ end
+end
diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb
index d93eb93c..78a27bff 100644
--- a/lib/translation_diff.rb
+++ b/lib/translation_diff.rb
@@ -54,6 +54,14 @@ def chunks
# @return [Array] The two strings being compared, formatted to
# highlight the differences.
def diff(joiner="...")
+ if Shuttle::Configuration.features[:skip_levenshtein_distance_diff]
+ # see https://jira.sqcorp.co/browse/SHUTTLE-1138
+ return [
+ (@str1 && @str1.length > 100) ? (@str1[0...100] + '...') : @str1,
+ (@str2 && @str2.length > 100) ? (@str2[0...100] + '...') : @str2,
+ ]
+ end
+
# Base cases
return [@str1, @str2] unless @str1.present? || @str2.present?
return @diff if @diff
@@ -83,6 +91,11 @@ def diff(joiner="...")
# visually align words that are matched according to minimal Levenshtein
# distance
def aligned
+ if Shuttle::Configuration.features[:skip_levenshtein_distance_diff]
+ # see https://jira.sqcorp.co/browse/SHUTTLE-1138
+ return [@str1, @str2]
+ end
+
@aligned ||= [chunks.map { |m| m.one }.join(" "), chunks.map { |m| m.two }.join(" ")]
end
diff --git a/config/initializers/elasticsearch_rails.rb b/lib/translation_validator/base.rb
similarity index 65%
rename from config/initializers/elasticsearch_rails.rb
rename to lib/translation_validator/base.rb
index c22a0ddc..b54e4a14 100644
--- a/config/initializers/elasticsearch_rails.rb
+++ b/lib/translation_validator/base.rb
@@ -1,4 +1,4 @@
-# Copyright 2016 Square Inc.
+# Copyright 2019 Square Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,4 +12,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-Elasticsearch::Model.client = Elasticsearch::Client.new host: Shuttle::Configuration.elasticsearch.url
\ No newline at end of file
+require 'abstract_class'
+
+# Container module for {Validator::Base} and its subclasses.
+
+module TranslationValidator
+ class Base
+ extended AbstractClass
+
+ def initialize(job)
+ @job = job
+ end
+
+ def run
+ raise NotImplementedError
+ end
+ end
+end
diff --git a/lib/translation_validator/source_fencer_validator.rb b/lib/translation_validator/source_fencer_validator.rb
new file mode 100644
index 00000000..600398b0
--- /dev/null
+++ b/lib/translation_validator/source_fencer_validator.rb
@@ -0,0 +1,58 @@
+# Copyright 2019 Square Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module TranslationValidator
+
+ # Checks if the source strings violates their own fencers.
+ class SourceFencerValidator < Base
+ IGNORE_FENCERS = %w(Html)
+
+ # implementations for Validator::Base
+ def run
+ Rails.logger.info("#{SourceFencerValidator} - starting source string validation: #{@job.project.job_type} with id #{@job.id}")
+
+ suspicious_keys_errors = find_suspicious_keys_errors(find_pending_keys)
+ Rails.logger.info("#{SourceFencerValidator} - found #{suspicious_keys_errors.count} suspicious source strings")
+ return if suspicious_keys_errors.blank?
+
+ FencerValidationMailer.suspicious_source_found(@job, suspicious_keys_errors).deliver_now
+ Rails.logger.info("#{SourceFencerValidator} - email sent for these suspicious source strings")
+ end
+
+ # find new translations, which are created after the job.
+ def find_pending_keys
+ @job.translations.includes(key: :project).where(translated: [nil, false]).where("translations.created_at >= ?", @job.created_at).map(&:key).uniq
+ end
+
+ # returns keys failed to pass their fencers
+ def find_suspicious_keys_errors(keys)
+ suspicious_keys_errors = []
+ keys.each do |key|
+ if key.fencers.present?
+ offensive_fencers = key.fencers.reject do |fencer|
+ begin
+ IGNORE_FENCERS.include?(fencer) || Fencer.const_get(fencer).valid?(key.source_copy)
+ rescue
+ false
+ end
+ end
+ if offensive_fencers.present?
+ suspicious_keys_errors << [key, "Violate fencers: #{offensive_fencers.join(', ')}"]
+ end
+ end
+ end
+ suspicious_keys_errors
+ end
+ end
+end
diff --git a/lib/translation_validator/translation_auto_migration.rb b/lib/translation_validator/translation_auto_migration.rb
new file mode 100644
index 00000000..e9528381
--- /dev/null
+++ b/lib/translation_validator/translation_auto_migration.rb
@@ -0,0 +1,89 @@
+# Copyright 2019 Square Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module TranslationValidator
+
+ # This translation_validator walks through the pending translations and automatically migrates the translation from existing TMs.
+ # The order to match the translation:
+ # - from same project
+ # - from same job type
+ # - from other job type
+ class TranslationAutoMigration < Base
+
+ EN_BASE_LOCALES = %w(en en-US)
+ AUTO_TRANSLATION_MIGRATION_NOTE = 'AutoTM'
+
+ # implementations for Validator::Base
+ def run
+ Rails.logger.info("#{TranslationAutoMigration} - starting auto translation migration: #{@job.project.job_type} with id #{@job.id}")
+
+ pending_translations = find_pending_translations
+ Rails.logger.info("#{TranslationAutoMigration} - found #{pending_translations.count} pending translations")
+ return if pending_translations.blank?
+
+ migrated_translations = []
+ pending_translations.each do |translation|
+ approved_translations = find_approved_translations(translation)
+ matched_transaltion = find_approved_translation_from_project(translation, approved_translations) ||
+ find_approved_translation_from_job_type(translation, approved_translations) ||
+ find_approved_translation_from_other(translation, approved_translations)
+ if matched_transaltion
+ migrate_translation(translation, matched_transaltion)
+ migrated_translations << translation
+ end
+ end
+ Rails.logger.info("#{TranslationAutoMigration} - migrated #{migrated_translations.count} translations")
+ return if migrated_translations.blank?
+
+ keys = migrated_translations.map(&:key).uniq
+ keys.map(&:recalculate_ready!)
+ Rails.logger.info("#{TranslationAutoMigration} - recalculated #{keys.count} keys for translations")
+ end
+
+ def find_pending_translations
+ @job.translations.includes(key: :project).where(translated: [nil, false]).where("translations.created_at >= ?", @job.created_at)
+ end
+
+ # find existing approved translations, ordered by :updated_at descending
+ def find_approved_translations(translation)
+ base_locales = EN_BASE_LOCALES.include?(translation.source_rfc5646_locale) ? EN_BASE_LOCALES : translation.source_rfc5646_locale
+ Translation.includes(key: :project).where(source_copy: translation.source_copy, source_rfc5646_locale: base_locales, rfc5646_locale: translation.rfc5646_locale, approved: true).order(updated_at: :desc)
+ end
+
+ # find existing translation from the same project
+ def find_approved_translation_from_project(translation, approved_translations)
+ project_id = translation.key.project.id
+ approved_translations.detect { |t| t.key.project.id == project_id }
+ end
+
+ # find existing translation from same job_type
+ def find_approved_translation_from_job_type(translation, approved_translations)
+ job_type = translation.key.project.job_type
+ approved_translations.detect { |t| t.key.project.job_type == job_type }
+ end
+
+ # find existing translation from other job_types
+ def find_approved_translation_from_other(_translation, approved_translations)
+ approved_translations.first
+ end
+
+ def migrate_translation(translation, approved_translation)
+ translation.copy = approved_translation.copy
+ translation.translated = true
+ translation.notes = "#{AUTO_TRANSLATION_MIGRATION_NOTE}:#{approved_translation.id} #{translation.notes}"
+ translation.translation_date = Time.now
+ translation.save!
+ end
+ end
+end
diff --git a/script/ci b/script/ci
new file mode 100755
index 00000000..b37144c5
--- /dev/null
+++ b/script/ci
@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+
+export LC_CTYPE=en_US.UTF-8
+export LANG=en_US.UTF-8
+RUN_LIST=$(echo ${RUN_LIST} | tr "," " ")
+PIDFILE="tmp/elasticsearch.pid"
+
+function install_cmake() {
+ sudo yum install -y cmake
+}
+
+function install_tidy() {
+ sudo yum install -y tidy
+}
+
+function install_libarchive() {
+ sudo yum install -y libarchive-devel
+}
+
+function install_elasticsearch() {
+ ./script/install.d/elasticsearch
+}
+
+function start_elasticsearch() {
+ ./script/elasticsearch &
+
+ curl 'localhost:9200'
+ err=$?
+ while [ $err -ne 0 ]
+ do
+ echo 'Waiting for ElasticSearch to start'
+ sleep 1
+ curl 'localhost:9200'
+ err=$?
+ done
+}
+
+function stop_elasticsearch() {
+ if [ -f $PIDFILE ]; then
+ kill `cat $PIDFILE`
+ sleep 5
+ fi
+
+ ./script/elasticsearch-cleanup
+}
+
+function install_bundler_if_needed() {
+ echo "Checking for Bundler ..."
+ gem install bundler --conservative
+}
+
+function update_gems_if_needed() {
+ set -e # exit if bundle install fails
+
+ echo "Installing gems..."
+ if [[ -n $KOCHIKU_ENV ]]; then
+ bundle_install_from_shared_cache shuttle
+ else
+ bundle check || bundle install
+ fi
+
+ set +e
+}
+
+function prepare_database() {
+ dropdb -h 127.0.0.1 shuttle_test || true
+ dropuser -h 127.0.0.1 shuttle || true
+ createuser -D -R -S -h 127.0.0.1 shuttle || true
+ createdb -h 127.0.0.1 -O shuttle shuttle_test
+# psql -h 127.0.0.1 -U shuttle -f db/structure.sql shuttle_test
+ RAILS_ENV=test bundle exec rake db:migrate
+}
+
+function run_specs() {
+ bundle exec rspec ${RUN_LIST}
+}
+
+function run_jasmine() {
+ bundle exec rake guard:jasmine -t
+}
+
+function prepare() {
+ install_cmake
+ install_tidy
+ install_elasticsearch
+ install_libarchive
+ install_bundler_if_needed &&
+ update_gems_if_needed &&
+ prepare_database
+}
+
+prepare
+start_elasticsearch
+
+set -x
+case "${TEST_RUNNER}" in
+ jasmine)
+ run_jasmine
+ ;;
+
+ specs)
+ run_specs
+ ;;
+ *)
+ echo "unknown test runner: ${TEST_RUNNER}"
+ exit 127
+ ;;
+esac
+
+exit_status=$?
+stop_elasticsearch
+exit $exit_status
diff --git a/script/elasticsearch b/script/elasticsearch
index 912ccd61..5b34bb21 100755
--- a/script/elasticsearch
+++ b/script/elasticsearch
@@ -7,4 +7,4 @@ PIDFILE="tmp/elasticsearch.pid"
./script/elasticsearch-cleanup
export HOSTNAME=localhost
-./vendor/shuttle-es/elasticsearch-1.2.1/bin/elasticsearch -p $PIDFILE
\ No newline at end of file
+./vendor/shuttle-es/elasticsearch-5.6.16/bin/elasticsearch -p $PIDFILE
\ No newline at end of file
diff --git a/script/find_strings_to_replace.rb b/script/find_strings_to_replace.rb
new file mode 100644
index 00000000..f39a45d9
--- /dev/null
+++ b/script/find_strings_to_replace.rb
@@ -0,0 +1,207 @@
+#!/usr/local/bin/env ruby
+
+require "csv"
+require "byebug"
+
+# ABOUT
+#---------
+# this script can be used to replace words in en-US with variant words
+# in other locales, e.g. a word like color -> colour
+
+# clear the output buffer first
+system "clear" or system "cls"
+
+CSV_FILE_PATH = 'replacement_results.csv'
+# get variant words from Google Drive: https://docs.google.com/spreadsheets/d/1NVaPqEz-R5UiAYHfNPYOlOyIVCwA7dAz5xtuvjXbVAg/edit?usp=sharing
+variant_words=
+[
+ # [source, en-GB source, en-GB replacement, en-CA source, en-CA replacement, en-AU source, en-AU replacement]
+ ["Authorization", "Authorization", "Authorisation", "Authorisation", "Authorization", "Authorization", "Authorisation"],
+ ["Authorize", "Authorize", "Authorise", "Authorise", "Authorize", "Authorize", "Authorise"],
+ ["Authorized", "authorised", "authorised", "authorized", "authorized", "authorised", "authorised"],
+ ["authorizing", "authorising", "authorising", "authorizing", "authorizing", "authorising", "authorising"],
+ ["Canceled", "Cancelled", "Cancelled", "Cancelled", "Cancelled", "Cancelled", "Cancelled"],
+ ["canceling", "cancelling", "cancelling", "cancelling", "cancelling", "cancelling", "cancelling"],
+ ["cancellation", "cancellation", "cancellation", "cancellation", "cancellation", "cancellation", "cancellation"],
+ ["Customize", "Customise", "Customise", "Customize", "Customize", "Customise", "Customise"],
+ ["customized", "customised", "customised", "customized", "customized", "customised", "customised"],
+ ["Enroll", "Enrol", "Enrol", "Enrol", "Enrol", "Enrol", "Enrol"],
+ ["enrolled", "enrolled", "enrolled", "enrolled", "enrolled", "enrolled", "enrolled"],
+ ["enrollment", "enrolment", "enrolment", "enrolment", "enrolment", "enrolment", "enrolment"],
+ ["enrolling", "enrolling", "enrolling", "enrolling", "enrolling", "enrolling", "enrolling"],
+ ["initialize", "initialise", "initialise", "initialize", "initialize", "initialise", "initialise"],
+ ["initialized", "--", "initialised", "--", "initialized", "--", "initialised"],
+ ["initializing", "--", "initialising", "--", "initializing", "--", "initialising"],
+ ["initialization", "--", "initialisation", "--", "initialization", "--", "initialisation"],
+ ["Personalize", "Personalise", "Personalise", "Personalize", "Personalize", "Personalise", "Personalise"],
+ ["Personalization", "Personalisation", "Personalisation", "Personalization", "Personalization", "Personalisation", "Personalisation"],
+ ["Uncategorized", "Uncategorised", "Uncategorised", "Uncategorized", "Uncategorized", "Uncategorised", "Uncategorised"],
+ ["Wifi", "Wi-fi", "Wi-fi", "Wi-fi", "Wi-fi", "Wi-fi", "Wi-fi"],
+ ["fulfillment", "fulfillment", "fulfilment", "fulfillment", "fulfillment", "fulfillment", "fulfilment"],
+]
+
+# helper method to print things while debugging
+def pp(input, linesAfter=true, linesBefore=true)
+ if linesBefore == true
+ puts "-" * [(input.size + 2), 15].min
+ end
+ puts input
+ if linesAfter == true
+ puts "-" * [(input.size + 2), 15].min
+ end
+end
+
+# finds replaceable word pairs in a given locale
+# locale: string
+# variant_words: hash object containing all variants for given locale
+# returns Hash (of arrays for each locale)
+def find_replacement_translations(locale, variant_words, translations)
+ pp "Processing #{locale} strings"
+ unchanged = []
+ to_be_replaced = []
+ variant_words.each do |dict|
+ current = dict[:source]
+ origin = dict[:origin]
+ replacement = dict[:target]
+ # keeping a tally of how many will not change due to both current
+ # and replacement being the same
+ if current == replacement
+ unchanged << { current: current, replacement: replacement }
+ next
+ end
+ if current == '--'
+ t = translations.where('copy LIKE ?', "%#{origin}%")
+ puts "#{t.count} strings found in #{locale} for #{origin}"
+ else
+ t = translations.where('copy LIKE ?', "%#{current}%")
+ puts "#{t.count} strings found in #{locale} for #{current}"
+ end
+ # t = translations.where(source_copy: source)
+ # count = t.count
+ # t = t.concat(fuzzy_match)
+ unless (t.nil? or t.empty?) && current[0] != replacement[0]
+ # pp "#{current[0]} matched #{replacement[0]}"
+ t.each do |row|
+ # exact match with word boundaries around the word
+ # this will prevent words being part of ids/classes
+ # and it will also prevent words like "Unenroll"
+ # it's looking for "enroll"
+ unless row.copy.match(/#{current}\b/)
+ next
+ end
+ if current[0] == replacement[0]
+ pp "#{current} will be replaced with #{replacement}"
+ end
+ rep = {
+ locale: locale,
+ source: row.source_copy,
+ current: row.copy,
+ replacement: row.copy && row.copy.gsub(current, replacement),
+ id: row.id,
+ word: replacement,
+ }
+ if rep[:current] != rep[:replacement]
+ puts "Current and replacmeent match: #{rep[:current]} == #{rep[:replacement]}"
+ begin
+ if rep[:replacement].strip_html_tags == rep[:replacement]
+ to_be_replaced << rep
+ else
+ pp "Stripped #{rep[:replacement]} and didn't add to list"
+ end
+ end
+ end
+ end
+ end
+ end
+ puts "Ignoring: #{unchanged.size} strings"
+ puts "Changing: #{to_be_replaced.size} strings"
+ to_be_replaced
+end
+
+# this method builds a locale specific hash of all words
+# returns Hash (locale based arrays of words)
+def build_variant_replacements(variant_words)
+ # first check if the number of words in a given set is not 7
+ # (meaning doesn't include all source/target for each locale + source)
+ invalid_variant_words = variant_words.select { |words| words.count != 7 }
+ unless invalid_variant_words.empty?
+ pp "Found Invalid Variants: #{invalid_variant_words}"
+ raise Exception.new("Found Invalid Variants: #{invalid_variant_words.count}")
+ end
+ locale_words = { 'en-GB' => [], 'en-CA' => [], 'en-AU' => [], }
+ variant_words.each do |source, gb_source, gb_target, ca_source, ca_target, au_source, au_target|
+ puts "A single row below:"
+ puts "#{source}, #{gb_source}, #{gb_target}, #{ca_source}, #{ca_target}, #{au_source}, #{au_target}"
+ locale_words['en-GB'] << { origin: source, source: gb_source, target: gb_target }
+ # puts locale_words['en-GB']
+ locale_words['en-CA'] << { origin: source, source: ca_source, target: ca_target }
+ pp locale_words
+ # puts locale_words['en-CA']
+ locale_words['en-AU'] << { origin: source, source: au_source, target: au_target }
+ # puts locale_words['en-AU']
+ end
+ locale_words
+end
+
+# this method will case a given array into uppercase, lowercase, and sentence case
+# returns Array (all case converted words and original words)
+def add_casing_types(original_array)
+ lowercase_array = []
+ uppercase_array = []
+ capitalized_array = []
+ # go over original_array once to to create temporary arrays
+ original_array.each do |array|
+ temp_lowercase_array = []
+ temp_uppercase_array = []
+ temp_capitalized_array = []
+ array.each do |word|
+ temp_lowercase_array << word.downcase
+ temp_uppercase_array << word.upcase
+ # split the first letter, capitalize it
+ temp_capitalized_array << word.split.map(&:capitalize)[0]
+ end
+ lowercase_array << temp_lowercase_array
+ uppercase_array << temp_uppercase_array
+ capitalized_array << temp_capitalized_array
+ end
+ # appending because at this point, we'll want to append a cased array
+ # just like any other list of words
+ final_array = [].concat(lowercase_array)
+ final_array = final_array.concat(uppercase_array)
+ final_array = final_array.concat(capitalized_array)
+end
+
+# 1st:
+# we add all the different cases to each row
+# an example this will look like this:
+# original row: ['Enroll', 'Enrol', 'Enrol'...etc]
+# uppercase row: ['ENROLL', 'ENROL', 'ENROL'...etc]
+# lowercase row: ['enroll', 'enrol', 'enrol'...etc]
+# capitalized case: ['Enroll', 'Enrol', 'Enrol'...etc]
+cased_variant_words = add_casing_types(variant_words)
+
+# 2nd:
+# we build individual hashes of each locale
+# e.g.
+# { 'en-GB: {origin: 'Enroll', source: 'Enrol', replacement: 'Enrol'} }
+# note: origin is the en-US source origin
+variant_words_dict = build_variant_replacements(cased_variant_words)
+
+# 3rd:
+# Write headers to CSV file
+CSV.open(CSV_FILE_PATH, 'w') do |csv|
+ csv << ['locale', 'word', 'current', 'replacement', 'translation_id']
+end
+
+# 4th:
+# find replacements for each locale and add to CSV file
+variant_words_dict.each do |locale, words_for_locale|
+ translations = Translation.where(rfc5646_locale: locale)
+ locale_replacements = find_replacement_translations(locale, words_for_locale, translations)
+ locale_replacements.each do |rep|
+ CSV.open(CSV_FILE_PATH, 'a+') do |csv|
+ csv << [rep[:locale], rep[:word], rep[:current], rep[:replacement], rep[:id]]
+ end
+ end
+end
+
diff --git a/script/install.d/elasticsearch b/script/install.d/elasticsearch
index f3647ede..aa306c25 100755
--- a/script/install.d/elasticsearch
+++ b/script/install.d/elasticsearch
@@ -6,3 +6,6 @@ mkdir -p vendor
cd vendor
git clone --quiet --shared /mnt/nfs/git/sq/shuttle-es.git
+
+# ES 5.6.16 expects there is folder plugins.
+mkdir -p ./shuttle-es/elasticsearch-5.6.16/plugins
diff --git a/script/replace_strings.rb b/script/replace_strings.rb
new file mode 100644
index 00000000..433cfa8d
--- /dev/null
+++ b/script/replace_strings.rb
@@ -0,0 +1,10 @@
+# Using an imported list of strings from a CSV, this script can replace strings.
+
+arguments = ARGV
+
+file_name = arguments[0]
+dry_run = arguments[1] || true
+
+puts "Filename: #{file_name}"
+puts "Dry run: #{dry_run}"
+
diff --git a/script/reset-deployable.sh b/script/reset-deployable.sh
new file mode 100755
index 00000000..0f35dcc5
--- /dev/null
+++ b/script/reset-deployable.sh
@@ -0,0 +1,38 @@
+echo "Checking out master branch"
+# first we'll checkout master branch
+git checkout master
+
+# if master branch checkout fails due to reasons like
+# existing changes locally that haven't been merged
+# ongoing merge/rebase/bisect
+# we'll fail and exit
+# otherwise, we'll delete deployable branch after checking out master
+if [ $? -eq 0 ]; then
+ echo "\n\nDeleting deployable branch"
+ git branch -D deployable
+else
+ echo "\n\nLooks like something went wrong while checking out master"
+ exit;
+fi
+
+# now let's create a new branch off of master called deployable
+if [ $? -eq 0 ]; then
+ echo "\n\nCreating new version of deployable from master"
+ git checkout -b deployable
+else
+ echo "\n\nCouldn't delete deployable, might not exist"
+ echo "\nDo you still want to create a 'deployable' branch?"
+ select yn in "Yes" "No"; do
+ case $yn in
+ Yes ) git checkout -b deployable; break;;
+ No ) exit;
+ esac
+ done
+fi
+
+# if creating a branch succeeded, we can push it up to origin
+if [ $? -eq 0 ]; then
+ echo "\n\nPush latest version of deployable branch to bitbucket"
+ git push -f origin deployable
+fi
+
diff --git a/spec/controllers/commits_controller_spec.rb b/spec/controllers/commits_controller_spec.rb
index aec63545..d8bf95d3 100644
--- a/spec/controllers/commits_controller_spec.rb
+++ b/spec/controllers/commits_controller_spec.rb
@@ -202,6 +202,24 @@
C
end
+ it "should export a UTF-8 Strings file in one locale" do
+ get :manifest, project_id: @project.to_param, id: @commit.to_param, locale: 'fr', format: 'strings', encoding: 'utf-8'
+ expect(response.status).to eql(200)
+ expect(response.headers['Content-Disposition']).to eql('attachment; filename="fr.strings"')
+ expect(response.headers['Content-Type']).to eql('text/plain; charset=utf-8')
+ expect(response.body.encoding.to_s).to eql('UTF-8')
+
+ body = response.body
+ expect(body).to include(<<-C)
+/* Universal Greeting */
+"key1" = "Bonjour {name}! Avec anninas fromage {count} la bouches.";
+ C
+ expect(body).to include(<<-C)
+/* Shopping cart contents */
+"key2" = "Tu avec carté {count} itém has";
+ C
+ end
+
it "should export a Java properties file in one locale" do
get :manifest, project_id: @project.to_param, id: @commit.to_param, locale: 'fr', format: 'properties'
expect(response.status).to eql(200)
@@ -215,11 +233,65 @@
C
end
- it "should export an iOS tarball manifest" do
+ it "should export an iOS tarball manifest in UTF-16LE" do
+ Commit.find(@commit.id).keys.update_all(source: 'foo/bar.strings')
+
get :manifest, project_id: @project.to_param, id: @commit.to_param, format: 'ios'
expect(response.status).to eql(200)
expect(response.headers['Content-Disposition']).to eql('attachment; filename="manifest.tar.gz"')
- # check body?
+ expect(response.headers['Content-Type']).to eql('application/x-gzip; charset=utf-16le')
+ expect(response.body.encoding.to_s).to eql('UTF-16LE')
+
+ entries = Hash.new
+ Archive.read_open_memory(response.body, Archive::COMPRESSION_GZIP, Archive::FORMAT_TAR_GNUTAR) do |archive|
+ while (entry = archive.next_header)
+ contents = archive.read_data
+ expect(contents.bytes.to_a[0]).to eq(0xFF)
+ expect(contents.bytes.to_a[1]).to eq(0xFE)
+ entries[entry.pathname] = contents.force_encoding('UTF-16LE')
+ end
+ end
+ expect(entries.count).to eq(1)
+
+ body = entries['foo/bar.strings'].encode('UTF-8')
+ expect(body).to include(<<-C)
+/* Universal Greeting */
+"key1" = "Bonjour {name}! Avec anninas fromage {count} la bouches.";
+ C
+ expect(body).to include(<<-C)
+/* Shopping cart contents */
+"key2" = "Tu avec carté {count} itém has";
+ C
+ end
+
+ it "should export an iOS tarball manifest in UTF-8" do
+ Commit.find(@commit.id).keys.update_all(source: 'foo/bar.strings')
+
+ get :manifest, project_id: @project.to_param, id: @commit.to_param, format: 'ios', encoding: 'utf-8'
+ expect(response.status).to eql(200)
+ expect(response.headers['Content-Disposition']).to eql('attachment; filename="manifest.tar.gz"')
+ expect(response.headers['Content-Type']).to eql('application/x-gzip; charset=utf-8')
+ expect(response.body.encoding.to_s).to eql('UTF-8')
+
+ entries = Hash.new
+ Archive.read_open_memory(response.body, Archive::COMPRESSION_GZIP, Archive::FORMAT_TAR_GNUTAR) do |archive|
+ while (entry = archive.next_header)
+ contents = archive.read_data
+ expect(contents.force_encoding('UTF-8')).to eq(contents)
+ entries[entry.pathname] = contents
+ end
+ end
+ expect(entries.count).to eq(1)
+
+ body = entries['foo/bar.strings']
+ expect(body).to include(<<-C)
+/* Universal Greeting */
+"key1" = "Bonjour {name}! Avec anninas fromage {count} la bouches.";
+ C
+ expect(body).to include(<<-C)
+/* Shopping cart contents */
+"key2" = "Tu avec carté {count} itém has";
+ C
end
it "should export a Ruby file in one locale" do
@@ -529,8 +601,6 @@
describe '#search' do
before :each do
- reset_elastic_search
-
@project = FactoryBot.create(:project,
base_rfc5646_locale: 'en',
targeted_rfc5646_locales: { 'en' => true, 'fr' => true, 'es' => false },
@@ -538,17 +608,15 @@
@commit = @project.commit!('HEAD', skip_import: true)
other_commit = FactoryBot.create(:commit, project: @project)
- other_commit.keys = [FactoryBot.create(:key, project: @project).tap(&:add_pending_translations)]
+ other_commit.update(keys: [FactoryBot.create(:key, project: @project).tap(&:add_pending_translations)])
@keys = FactoryBot.create_list(:key, 51, project: @project, ready: false).sort_by(&:key)
@keys.each &:add_pending_translations
- @commit.keys = @keys
+ @commit.update(keys: @keys)
@user = FactoryBot.create(:user, :activated)
@request.env['devise.mapping'] = Devise.mappings[:user]
sign_in @user
-
- regenerate_elastic_search_indexes
end
it 'should return the first page of keys if page not specified' do
@@ -579,7 +647,9 @@
it 'filters by requested status' do
approved_key = FactoryBot.create(:key, project: @project, key: 'approved_key', ready: true)
@commit.keys = @keys << approved_key
- regenerate_elastic_search_indexes
+ CommitsIndex.reset!
+ KeysIndex.reset!
+ TranslationsIndex.reset!
get :search, project_id: @project.to_param, id: @commit.to_param, status: 'approved'
expect(response.status).to eql 200
@@ -666,7 +736,7 @@
@request.env['devise.mapping'] = Devise.mappings[:user]
@user = FactoryBot.create(:user, :confirmed, role: 'admin')
sign_in @user
- regenerate_elastic_search_indexes
+ CommitsIndex.reset!
end
it "should require a monitor" do
@@ -677,7 +747,6 @@
end
it "should delete a commit" do
- skip 'ElasticSearch flaky - deletion throws NotFound exception'
delete :destroy, project_id: @commit.project.to_param, id: @commit.to_param, format: 'json'
expect(response.status).to eql(204)
expect { @commit.reload }.to raise_error(ActiveRecord::RecordNotFound)
diff --git a/spec/controllers/glossary_controller_spec.rb b/spec/controllers/glossary_controller_spec.rb
index 2634f130..3828108f 100644
--- a/spec/controllers/glossary_controller_spec.rb
+++ b/spec/controllers/glossary_controller_spec.rb
@@ -17,8 +17,6 @@
RSpec.describe GlossaryController do
describe '#index' do
before :each do
- reset_elastic_search
-
update_date = DateTime.new(2014, 1, 1)
@user = FactoryBot.create(:user, :confirmed, role: 'translator')
@start_date = (update_date - 1.day).strftime('%m/%d/%Y')
@@ -49,7 +47,7 @@
end
end
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
@request.env['devise.mapping'] = Devise.mappings[:user]
sign_in @user
diff --git a/spec/controllers/home_controller_spec.rb b/spec/controllers/home_controller_spec.rb
index b881cc55..9a0649cd 100644
--- a/spec/controllers/home_controller_spec.rb
+++ b/spec/controllers/home_controller_spec.rb
@@ -18,7 +18,6 @@
before :each do
allow_any_instance_of(Article).to receive(:import!) # prevent auto import
- reset_elastic_search
@request.env['devise.mapping'] = Devise.mappings[:user]
@user = FactoryBot.create(:user, :confirmed, role: 'monitor')
sign_in @user
@@ -38,7 +37,8 @@
# red herring: loading commit
FactoryBot.create(:commit, project: @project, loading: true)
- regenerate_elastic_search_indexes
+ CommitsIndex.reset!
+ KeysIndex.reset!
end
context "[when 'remove duplicates' filter is selected]" do
@@ -56,7 +56,8 @@
commit1.keys << key2
commit2.keys << key2
- regenerate_elastic_search_indexes
+ CommitsIndex.reset!
+ KeysIndex.reset!
get :index, { commits_filter__hide_duplicates: 'true' }
expect(assigns(:commits).map(&:id)).to eq [commit3.id, commit2.id, @commit.id]
@@ -98,7 +99,7 @@
hidden_article = FactoryBot.create(:article, project: @project, hidden: true)
hidden_group = FactoryBot.create(:group, name: 'hidden-group', project: @project, hidden: true)
- get :index, { filter__status: 'hidden' }
+ get :index, { filter__status: 'hidden', sort__field: 'create' }
expect(assigns(:articles).map(&:id)).to eq([hidden_article.id])
expect(assigns(:groups).map(&:id)).to eq([hidden_group.id])
end
@@ -108,7 +109,7 @@
it 'should return all translations in all projects' do
commit1 = FactoryBot.create(:commit, project: @project, ready: false)
commit2 = FactoryBot.create(:commit, project: @project)
- regenerate_elastic_search_indexes
+ CommitsIndex.reset!
get :index, { filter__status: 'all', commits_filter__project_id: 'all' }
expect(assigns(:commits).map(&:id)).to eq([commit2.id, commit1.id, @commit.id])
diff --git a/spec/controllers/locales/projects_controller_spec.rb b/spec/controllers/locales/projects_controller_spec.rb
index d852ea2a..6320e0be 100644
--- a/spec/controllers/locales/projects_controller_spec.rb
+++ b/spec/controllers/locales/projects_controller_spec.rb
@@ -18,7 +18,6 @@
describe "#show" do
context "[status filtering]" do
before :each do
- reset_elastic_search
@user = FactoryBot.create(:user, :confirmed, role: 'translator', approved_rfc5646_locales: ['fr-CA'])
@project = FactoryBot.create(:project, base_rfc5646_locale: 'en-US', targeted_rfc5646_locales: {'fr-CA' => true})
@@ -77,7 +76,6 @@
copy: nil,
translated: false,
approved: nil)
- regenerate_elastic_search_indexes
@request.env["devise.mapping"] = Devise.mappings[:user]
sign_in @user
@@ -127,7 +125,7 @@
expect(response.status).to eql(200)
translations = assigns(:translations)
expect(translations.map { |t| t.key.key }).
- to match_array([@approved.key.key, @new.key.key])
+ to match_array([@approved.key.key, @new.key.key, @rejected.key.key])
end
it "should filter with include_translated = true, include_approved = true" do
@@ -135,7 +133,7 @@
expect(response.status).to eql(200)
translations = assigns(:translations)
expect(translations.map { |t| t.key.key }).
- to match_array([@translated.key.key, @approved.key.key, @rejected.key.key])
+ to match_array([@translated.key.key, @approved.key.key])
end
it "should filter with include_translated = true, include_approved = true, include_new = true" do
@@ -163,7 +161,6 @@
sign_in user
allow_any_instance_of(Article).to receive(:import!) # prevent auto import
- reset_elastic_search
@project = FactoryBot.create(:project, repository_url: nil, job_type: :article)
@article = FactoryBot.create(:article, project: @project)
@@ -179,14 +176,11 @@
@translation3 = FactoryBot.create(:translation, key: @key3, copy: nil, rfc5646_locale: 'fr')
@translation4 = FactoryBot.create(:translation, key: @key4, copy: nil, rfc5646_locale: 'fr')
@translation5 = FactoryBot.create(:translation, key: @key5, copy: nil, rfc5646_locale: 'fr')
-
- regenerate_elastic_search_indexes
end
it "returns active keys in an article in the right order" do
@section2.update! active: false # inactive section
@key3.update! index_in_section: nil # inactive key
- regenerate_elastic_search_indexes
get :show, id: @project.to_param, article_id: @article.id, locale_id: 'fr', include_new: 'true'
expect(response.status).to eql(200)
@@ -199,11 +193,17 @@
expect(assigns(:translations).map(&:id)).to eql([@translation4.id])
end
- it "filters with include_block_tags" do
+ it "filters with include_block_tags = true" do
get :show, id: @project.to_param, article_id: @article.id, section_id: @section2.id, locale_id: 'fr', include_new: 'true', include_block_tags: 'true'
expect(response.status).to eql(200)
expect(assigns(:translations).map(&:id)).to eql([@translation4.id, @translation5.id])
end
+
+ it "filters with include_block_tags = false" do
+ get :show, id: @project.to_param, article_id: @article.id, section_id: @section2.id, locale_id: 'fr', include_new: 'false', include_block_tags: 'false'
+ expect(response.status).to eql(200)
+ expect(assigns(:translations).map(&:id)).to eql([@translation4.id])
+ end
end
end
end
diff --git a/spec/controllers/search_controller_spec.rb b/spec/controllers/search_controller_spec.rb
index c72a6fd5..430c78b1 100644
--- a/spec/controllers/search_controller_spec.rb
+++ b/spec/controllers/search_controller_spec.rb
@@ -17,8 +17,6 @@
RSpec.describe SearchController do
describe "#translations" do
before :each do
- reset_elastic_search
-
update_date = DateTime.new(2014, 1, 1)
@user = FactoryBot.create(:user, :confirmed, role: 'translator')
@start_date = (update_date - 1.day).strftime('%m/%d/%Y')
@@ -42,7 +40,7 @@
end
end
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
@request.env['devise.mapping'] = Devise.mappings[:user]
sign_in @user
@@ -50,7 +48,7 @@
it "should filter by page" do
FactoryBot.create_list :translation, 50
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
get :translations, page: '2'
results = assigns(:results)
@@ -119,13 +117,12 @@
describe '#keys' do
before :each do
- reset_elastic_search
@user = FactoryBot.create(:user, :confirmed, role: 'translator')
@project = FactoryBot.create(:project)
5.times { |i| FactoryBot.create :key, project: @project, key: "t1_n#{i}" }
5.times { |i| FactoryBot.create :key, project: @project, key: "t2_n#{i}" }
- regenerate_elastic_search_indexes
+ KeysIndex.reset!
@request.env['devise.mapping'] = Devise.mappings[:user]
sign_in @user
@@ -141,7 +138,7 @@
it "should exlcude hidden key" do
FactoryBot.create(:key, project: @project, key: 'hide me', hidden_in_search: true)
- regenerate_elastic_search_indexes
+ KeysIndex.reset!
get :keys, project_id: @project.id, format: 'json'
expect(response.status).to eql(200)
@@ -151,9 +148,9 @@
it "should search for hidden key only" do
FactoryBot.create(:key, project: @project, key: 'hide me', hidden_in_search: true)
- regenerate_elastic_search_indexes
+ KeysIndex.reset!
- get :keys, project_id: @project.id, hidden_in_search: '', format: 'json'
+ get :keys, project_id: @project.id, hidden_in_search: '1', format: 'json'
expect(response.status).to eql(200)
results = JSON.parse(response.body)
expect(results.size).to eq(1)
@@ -210,8 +207,6 @@ def finish_sha(prefix="")
let(:prefix3) { "abc111" }
before :each do
- reset_elastic_search
-
@user = FactoryBot.create(:user, :confirmed, role: "translator")
@project1 = FactoryBot.create(:project)
@project2 = FactoryBot.create(:project)
@@ -221,7 +216,7 @@ def finish_sha(prefix="")
FactoryBot.create :commit, project: @project1, revision: finish_sha('abc111')
FactoryBot.create :commit, project: @project2, revision: finish_sha('abc111')
- regenerate_elastic_search_indexes
+ CommitsIndex.reset!
end
before :each do
diff --git a/spec/controllers/translations_controller_spec.rb b/spec/controllers/translations_controller_spec.rb
index 3b50fa25..befa82fe 100644
--- a/spec/controllers/translations_controller_spec.rb
+++ b/spec/controllers/translations_controller_spec.rb
@@ -152,7 +152,7 @@
expect(response.status).to eql(200)
expect(@translation.reload.copy).to eql('bye!')
expect(@translation).to be_approved
- expect(@translation.translator).to eql(@user)
+ expect(@translation.translator).to eql(translator)
expect(@translation.reviewer).to eql(@user)
expect(@translation.translation_changes.count).to eq(1)
@@ -163,6 +163,8 @@
end
it "should automatically approve reviewer changes to an untranslated string" do
+ translator = @translation.translator
+
patch :update,
project_id: @translation.key.project.to_param,
key_id: @translation.key.to_param,
@@ -173,7 +175,7 @@
expect(response.status).to eql(200)
expect(@translation.reload.copy).to eql('bye!')
expect(@translation).to be_approved
- expect(@translation.translator).to eql(@user)
+ expect(@translation.translator).to eql(translator)
expect(@translation.reviewer).to eql(@user)
expect(@translation.translation_changes.count).to eq(1)
@@ -366,7 +368,6 @@
end
before :each do
- reset_elastic_search
allow_any_instance_of(Locale).to receive(:fallbacks).and_return(
%w(fr-CA fr en).map { |l| Locale.from_rfc5646 l }
)
@@ -410,7 +411,7 @@
end
it "should 1. respond with a Translation with matching locale and source copy" do
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
get :match,
project_id: @project.to_param,
@@ -437,7 +438,7 @@
translation.update! modifier: @user
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
get :match,
project_id: @project.to_param,
@@ -452,7 +453,7 @@
it "should 3. respond with the Translation of the 1st fallback locale with matching project/key and source copy" do
@same_locale_sc.destroy
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
get :match,
project_id: @project.to_param,
@@ -465,7 +466,7 @@
it "should 4. respond with the Translation of the 1st fallback locale with source copy" do
[@same_locale_sc, @fallback1_sc].each(&:destroy)
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
get :match,
project_id: @project.to_param,
@@ -478,7 +479,7 @@
it "should 5. respond with a 204" do
[@same_locale_sc, @fallback1_sc, @fallback2_sc].each(&:destroy)
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
get :match,
project_id: @project.to_param,
@@ -592,7 +593,6 @@
before :each do
Translation.destroy_all
- reset_elastic_search
@translation = FactoryBot.create :translation,
source_copy: 'foo bar 1',
copy: 'something else',
@@ -611,7 +611,7 @@
copy: 'something else',
source_rfc5646_locale: 'en',
rfc5646_locale: 'fr'
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
get :fuzzy_match,
project_id: @translation.key.project.to_param,
@@ -631,7 +631,7 @@
approved: true,
source_rfc5646_locale: 'en',
rfc5646_locale: 'fr-CA'
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
# fr is a fallback of fr-CA
get :fuzzy_match,
@@ -661,7 +661,7 @@
copy: 'something else',
source_rfc5646_locale: 'en',
rfc5646_locale: 'fr'
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
get :fuzzy_match,
project_id: @translation.key.project.to_param,
@@ -683,7 +683,7 @@
source_rfc5646_locale: 'en',
rfc5646_locale: 'fr'
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
get :fuzzy_match,
project_id: @translation.key.project.to_param,
@@ -706,7 +706,7 @@
rfc5646_locale: 'fr'
end
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
get :fuzzy_match,
project_id: @translation.key.project.to_param,
@@ -729,7 +729,7 @@
rfc5646_locale: 'fr'
end
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
get :fuzzy_match,
project_id: @translation.key.project.to_param,
@@ -749,7 +749,7 @@
end
it "should truncate project name exceeds 30 chars" do
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
get :fuzzy_match,
project_id: @translation.key.project.to_param,
@@ -779,7 +779,7 @@
end
it "should not show in search result" do
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
get :fuzzy_match,
project_id: @translation.key.project.to_param,
@@ -812,6 +812,7 @@
%w(fr es).each do |locale|
FactoryBot.create(:translation, key: key, source_rfc5646_locale: 'en', rfc5646_locale: locale, source_copy: 'fake', copy: nil, approved: nil)
end
+ key.reload
end
@commit1 = FactoryBot.create(:commit, project: @project)
@@ -885,6 +886,7 @@
%w(fr es).each do |locale|
FactoryBot.create(:translation, key: key, source_rfc5646_locale: 'en', rfc5646_locale: locale, source_copy: 'fake', copy: nil, approved: nil)
end
+ key.reload
end
@project.keys.each { |k| k.recalculate_ready! }
diff --git a/spec/factories/edit_reason.rb b/spec/factories/edit_reason.rb
new file mode 100644
index 00000000..e11aef35
--- /dev/null
+++ b/spec/factories/edit_reason.rb
@@ -0,0 +1,7 @@
+# Read about factories at https://github.com/thoughtbot/factory_girl
+
+FactoryBot.define do
+ factory :edit_reason do
+ association :translation_change
+ end
+end
diff --git a/spec/factories/reports.rb b/spec/factories/reports.rb
new file mode 100644
index 00000000..51954ee0
--- /dev/null
+++ b/spec/factories/reports.rb
@@ -0,0 +1,10 @@
+FactoryBot.define do
+ factory :report do
+ date "2019-06-20 21:48:02"
+ project "MyString"
+ locale "MyString"
+ strings 1
+ words 1
+ report_type "MyString"
+ end
+end
diff --git a/spec/helpers/comments_helper_spec.rb b/spec/helpers/comments_helper_spec.rb
index 19ddcee8..2ed57675 100644
--- a/spec/helpers/comments_helper_spec.rb
+++ b/spec/helpers/comments_helper_spec.rb
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-require "spec_helper"
+require "rails_helper"
RSpec.describe CommentsHelper do
describe "#issue_url" do
diff --git a/spec/helpers/issues_helper_spec.rb b/spec/helpers/issues_helper_spec.rb
index 8143d0c3..a434ec1e 100644
--- a/spec/helpers/issues_helper_spec.rb
+++ b/spec/helpers/issues_helper_spec.rb
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-require "spec_helper"
+require "rails_helper"
RSpec.describe IssuesHelper do
describe "#issue_url" do
diff --git a/spec/lib/exporter/ios_spec.rb b/spec/lib/exporter/ios_spec.rb
index edec5f7f..e77fd82a 100644
--- a/spec/lib/exporter/ios_spec.rb
+++ b/spec/lib/exporter/ios_spec.rb
@@ -128,6 +128,48 @@
expect(entries).not_to include('/Resources/en-US.lproj/Some.xib')
end
+ it "should output in UTF-8 encoding when requested" do
+ io = StringIO.new
+ exporter = Exporter::Ios.new(@commit)
+ exporter.override_encoding = 'utf-8'
+ exporter.export(io, @de)
+ io.rewind
+
+ entries = Hash.new
+ Archive.read_open_memory(io.string, Archive::COMPRESSION_GZIP, Archive::FORMAT_TAR_GNUTAR) do |archive|
+ while (entry = archive.next_header)
+ expect(entry).to be_regular
+ contents = archive.read_data
+ expect(contents.force_encoding('UTF-8')).to eq(contents)
+ entries[entry.pathname] = contents
+ end
+ end
+
+ expect(entries.size).to eql(2)
+
+ body = entries['Resources/de-DE.lproj/Localizable-en-US.strings']
+ expect(body).to include(<<-C)
+/* This is a normal string. */
+"I'm a string!" = "Ich bin ein String!";
+ C
+ expect(body).to include(<<-C)
+/* This is also a normal string. */
+"I'm also a string!" = "Ich bin auch ein String!";
+ C
+
+ body = entries['Resources/de-DE.lproj/More.strings']
+ expect(body).to include(<<-C)
+/* Saying hello. */
+"Hello, world!" = "Hallo, Welt!";
+ C
+ expect(body).to include(<<-C)
+/* Saying goodbye. */
+"Goodbye, cruel world." = "Auf Wiedersehen, grausamer Welt.";
+ C
+
+ expect(entries).not_to include('/Resources/en-US.lproj/Some.xib')
+ end
+
describe ".valid?" do
it "should return true for a valid tar-gz file" do
smalltgz = "\x1F\x8B\b\b\xB1\x8E\xF8Q\x00\x03log.tar\x00\xED\x99\xCFO\xC20\x14\x80_'\xC6\x19/;\x19\x8F\xBDx\xF1\x80mi\xB7\xEBB\xF0hL\xDC\xC5\e\x121d\t?\x12\x1C\xF7\xFD\xE9\xB6\xF4I\x16\x10\x88\x89\e\"\xEFK\x9A\x0FX\xCB\xDE(\xAF\xEC\xD1\xF1lt\x0F5#\x84H\x8C\xE1\xD6J\xAB\xC4YH%\x96\xFE\x82K%\x93X\x8AD\x19\xCD\x85\x94\xC6$\xC0M\xDD\x819\x16\x1F\xC5`nC)\xF2\xC9\xCE~\x83\xE1$\x9F\xEE8\x8E\xD7\xB1\xF2\x910\xB6\xF3\xDF\xEE\xB7{Y?+f\xF3\xF7Z\xCEa?\x8FX\xEB\xED\xF3/\x95r\xF3o:\x89R\xB1\x8C\xED\xFCw\xB4Q\xC0E-\xD1\xACq\xE2\xF3\x0F\xE7\xD7\x17\x10\x00<\x0E\xDE\xF8S\xC6_8\xE2^\x83K\xDB\x94m\xDC6\xF7\xFC\xD9\rX\xF5\x88\x0E\x174\xF1[,\xF3\xBF\xD6\xEC\xDF\x97\xFFR\v\xA1\xD7\xF3_iI\xF9\xDF\x10\xAC\xBB\x18JX\xA6s\b\xDEp\xFB}\xD7\x10\xDB\x06A\xF5\xFD\x80\x96\x06\x82 \b\x828\x06\x98Wxu\xD80\b\x82\xF8\x83\xB8\xF5\x81\xA3St\xE9\xCD\xF0x\x80nU\xC6Dh\x8EN\xD1\xA57\xC3~\x01\xBA\x85\x0E\xD1\x11\x9A\xA3St\xE9\x8D\x8B\x16\xC3\xE2\x83\xE1\x99\x19V(\f\xAB\x10\xC6\xD1\xE9\x8F.\x99 N\x863\xAF\xC8\xFD\xFE?\xC0\xD6\xFA\x9F \x88\x7F\fk\xF5\xB2^\x17V\x05\xC1f\a\xDB^+\x8FK\xD8~\x13\x10\xF8?\vo*c9:E\x97\xDEt\#@\x10\x04\xD14\xCB\xFD\xBFQ^\xE4\xA3im\e\x80\xFB\xF6\xFF\x850\xEB\xFB\x7FF\xD3\xFE\x7F#\xDC\xB5\xED7\xE0\xD0A\x10\x04A\x10\x8D\xF3\t\n\xEE0\x8E\x00*\x00\x00"
diff --git a/spec/lib/fencer/android_spec.rb b/spec/lib/fencer/android_spec.rb
index 248b93ba..3a269672 100644
--- a/spec/lib/fencer/android_spec.rb
+++ b/spec/lib/fencer/android_spec.rb
@@ -35,5 +35,9 @@
it "should return false for a string that only contains any other character between { }" do
expect(Fencer::Android.valid?("String with {{two}} {tokens}.")).to be_falsey
end
+
+ it "should return true for a string that contains CDATA" do
+ expect(Fencer::Android.valid?("hello]]>")).to be_truthy
+ end
end
end
diff --git a/spec/lib/fencer/intl_message_format_spec.rb b/spec/lib/fencer/intl_message_format_spec.rb
index 30aa4412..edbe97b1 100644
--- a/spec/lib/fencer/intl_message_format_spec.rb
+++ b/spec/lib/fencer/intl_message_format_spec.rb
@@ -15,19 +15,61 @@
require 'rails_helper'
RSpec.describe Fencer::IntlMessageFormat do
describe ".fence" do
+ # test cases cover all sample from this link: https://formatjs.io/guides/message-syntax
+
it "should fence a message format token" do
expect(Fencer::IntlMessageFormat.fence("String with {two} {tokens}.")).
- to eql('{two}' => [12..16], '{tokens}' => [18..25])
+ to eql(':two' => [-1..0], ':tokens' => [-1..0])
end
it "should fence a message format token at the beginning of the string" do
expect(Fencer::IntlMessageFormat.fence("{one} token in this string.")).
- to eql('{one}' => [0..4])
+ to eql(':one' => [-1..0])
+ end
+
+ it "should fence a message number type format" do
+ expect(Fencer::IntlMessageFormat.fence("Almost {pctBlack, number, percent} of them are black.")).
+ to eql(':pctBlack|number|percent' => [-1..0])
+ end
+
+ it "should fence a message date type format" do
+ expect(Fencer::IntlMessageFormat.fence("Sale begins {start, date, medium}")).
+ to eql(':start|date|medium' => [-1..0])
+ end
+
+ it "should fence a message time type format" do
+ expect(Fencer::IntlMessageFormat.fence("Coupon expires at {expires, time, short}")).
+ to eql(':expires|time|short' => [-1..0])
+ end
+
+ it "should fence a message select type format" do
+ expect(Fencer::IntlMessageFormat.fence("{fruit, select, apple {Apple} banana {Banana} other {Unknown}}")).
+ to eql(':fruit|select|apple' => [-1..0], ':fruit|select|banana' => [-1..0], ':fruit|select|other' => [-1..0])
+ end
+
+ it "should fence a message nested select type format" do
+ expect(Fencer::IntlMessageFormat.fence("{taxableArea, select, yes {An additional {taxRate, number, percent} tax will be collected.} other {No taxes apply.}}")).
+ to eql(':taxableArea|select|yes:taxRate|number|percent' => [-1..0], ':taxableArea|select|other' => [-1..0])
+ end
+
+ it "should fence a message plural format" do
+ expect(Fencer::IntlMessageFormat.fence("Cart: {itemCount} {itemCount, plural, one {item} other {items}}")).
+ to eql(':itemCount' => [-1..0], ':itemCount|plural|0|one' => [-1..0], ':itemCount|plural|0|other' => [-1..0])
+ end
+
+ it "should fence a message selectordinal format" do
+ expect(Fencer::IntlMessageFormat.fence("It's my cat's {year, selectordinal, one {st} two {nd} few {rd} other {3th}} birthday!")).
+ to eql(':year|selectordinal|0|one' => [-1..0], ':year|selectordinal|0|two' => [-1..0], ':year|selectordinal|0|few' => [-1..0], ':year|selectordinal|0|other' => [-1..0])
end
it "should not fence a string with escaped braces" do
expect(Fencer::IntlMessageFormat.fence("String with \\{two\\} \\{tokens\\}.")).
- to eql({})
+ to eql({})
+ end
+
+ it "should return empty set if mal-formatted" do
+ expect(Fencer::IntlMessageFormat.fence("Sale begins {start, date, mediu")).
+ to eql({})
end
end
@@ -35,7 +77,7 @@
it "should return true for a string with valid interpolations" do
expect(Fencer::IntlMessageFormat.valid?("String with {valid} {interpolations}.")).to be_truthy
expect(Fencer::IntlMessageFormat.valid?("{String} that starts with an interpolation.")).to be_truthy
- expect(Fencer::IntlMessageFormat.valid?("String with an escaped brace '\\{'.")).to be_truthy
+ expect(Fencer::IntlMessageFormat.valid?("String with an escaped brace \\{.")).to be_truthy
expect(Fencer::IntlMessageFormat.valid?("String with no interpolations.")).to be_truthy
end
diff --git a/spec/lib/importer/strings_spec.rb b/spec/lib/importer/strings_spec.rb
index 5b60f668..775260c3 100644
--- a/spec/lib/importer/strings_spec.rb
+++ b/spec/lib/importer/strings_spec.rb
@@ -35,6 +35,11 @@
expect(@project.keys.for_key("/apple/en-US.lproj/example.strings:Something\nwith\tescapes\\").first.translations.base.first.copy).to eql("Something\nwith\tescapes\\")
end
+ it "should properly joins multiple lines" do
+ importer = Importer::Strings.new(@commit.blobs.first, @commit)
+ expect(importer.send(:unescape, "first line, \\\ncontinue with the line")).to eq("first line, continue with the line")
+ end
+
it "should still import strings that end with " do
expect(@project.keys.for_key('/apple/en-US.lproj/example.strings:quote.charlie.1').first.translations.find_by_rfc5646_locale('en-US').copy).to eql("I'm a patriot. You've gotta give me that.")
end
diff --git a/spec/lib/localizer/android_spec.rb b/spec/lib/localizer/android_spec.rb
index 0ecf19a5..d4e1084f 100644
--- a/spec/lib/localizer/android_spec.rb
+++ b/spec/lib/localizer/android_spec.rb
@@ -38,7 +38,8 @@
'/java/basic-hdpi/strings.xml:attributed_array[0]' => 'Hallo',
'/java/basic-hdpi/strings.xml:attributed_array[1]' => 'Welt',
'/java/basic-hdpi/strings.xml:plural[one]' => 'Welt',
- '/java/basic-hdpi/strings.xml:plural[other]' => 'Welten'
+ '/java/basic-hdpi/strings.xml:plural[other]' => 'Welten',
+ '/java/basic-hdpi/strings.xml:cdata' => 'Hallo]]>'
}.each do |key, value|
key_obj = FactoryBot.create(:key, key: key, project: @project, source: '/java/basic-hdpi/strings.xml')
FactoryBot.create :translation, key: key_obj, copy: value, source_locale: @en, locale: @de
@@ -75,6 +76,7 @@
- Hello
- World
+ Hello]]>
XML
output_file = Localizer::File.new
@@ -110,6 +112,7 @@
- Hallo
- Welt
+ Hallo]]>
XML
end
diff --git a/spec/lib/paginatable_objects_spec.rb b/spec/lib/paginatable_objects_spec.rb
index a896ad35..df7ed856 100644
--- a/spec/lib/paginatable_objects_spec.rb
+++ b/spec/lib/paginatable_objects_spec.rb
@@ -18,55 +18,35 @@
before do
project = FactoryBot.create(:project)
- @commit1 = FactoryBot.create(:commit, project: project)
- @commit2 = FactoryBot.create(:commit, project: project)
- @commit3 = FactoryBot.create(:commit, project: project)
- @commit4 = FactoryBot.create(:commit, project: project)
- @commit5 = FactoryBot.create(:commit, project: project)
-
- @commits = [@commit1, @commit2, @commit3, @commit4, @commit5]
- @es_objects = [double('es_result', id: @commit4.id),
- double('es_result', id: @commit1.id),
- double('es_result', id: @commit5.id),
- double('es_result', id: @commit2.id),
- double('es_result', id: @commit3.id)]
-
- class << @es_objects
- def total() 10 end
- end
- end
-
- describe '#initialize' do
- it 'keeps objects sorted' do
- ordered_commits = PaginatableObjects.new(@commits, @es_objects, 1, 5).objects
- expect(ordered_commits).to eql([@commit4, @commit1, @commit5, @commit2, @commit3])
- end
+ FactoryBot.create_list :commit, 10, project: project
+ CommitsIndex.reset!
+ @es_objects = CommitsIndex.filter(term: {project_id: project.id})
end
describe '#offset_value' do
it 'finds the correct offset_value' do
- expect(PaginatableObjects.new([], @es_objects, 1, 5).offset_value).to eql(0)
- expect(PaginatableObjects.new([], @es_objects, 2, 5).offset_value).to eql(5)
- expect(PaginatableObjects.new([], @es_objects, 3, 5).offset_value).to eql(10)
+ expect(PaginatableObjects.new(@es_objects, 1, 5).offset_value).to eql(0)
+ expect(PaginatableObjects.new(@es_objects, 2, 5).offset_value).to eql(5)
+ expect(PaginatableObjects.new(@es_objects, 3, 5).offset_value).to eql(10)
end
end
describe '#total_pages' do
it 'returns the total number of pages' do
- expect(PaginatableObjects.new([], @es_objects, 1, 4).total_pages).to eql(3)
- expect(PaginatableObjects.new([], @es_objects, 1, 5).total_pages).to eql(2)
- expect(PaginatableObjects.new([], @es_objects, 1, 6).total_pages).to eql(2)
+ expect(PaginatableObjects.new(@es_objects, 1, 4).total_pages).to eql(3)
+ expect(PaginatableObjects.new(@es_objects, 1, 5).total_pages).to eql(2)
+ expect(PaginatableObjects.new(@es_objects, 1, 6).total_pages).to eql(2)
end
end
describe '#last_page?' do
it 'returns true if last page' do
- expect(PaginatableObjects.new([], @es_objects, 3, 4).last_page?).to be_truthy
+ expect(PaginatableObjects.new(@es_objects, 3, 4).last_page?).to be_truthy
end
it 'returns false if not page' do
- expect(PaginatableObjects.new([], @es_objects, 1, 4).last_page?).to be_falsey
- expect(PaginatableObjects.new([], @es_objects, 2, 4).last_page?).to be_falsey
+ expect(PaginatableObjects.new(@es_objects, 1, 4).last_page?).to be_falsey
+ expect(PaginatableObjects.new(@es_objects, 2, 4).last_page?).to be_falsey
end
end
end
diff --git a/spec/lib/reports/quality_report_spec.rb b/spec/lib/reports/quality_report_spec.rb
index af100156..7c4a3c21 100644
--- a/spec/lib/reports/quality_report_spec.rb
+++ b/spec/lib/reports/quality_report_spec.rb
@@ -137,20 +137,20 @@
expect(result[7]).to eql expected_results
end
- it 'has the expected row 8' do
- expected_results = [@end_date.strftime("%Y-%m-%d"), project, sha, @key4.key, "Hello, world", "IT", @end_date.strftime("%Y-%m-%d 08:00:00 UTC"), "Rebecca (#{@translator.id})", "hello!", "bye!", @end_date.strftime("%Y-%m-%d 08:00:00 UTC"), "Mark (#{@reviewer.id})", "", nil]
- expect(result[8]).to eql expected_results
- end
-
- it 'has the expected row 9' do
- expected_results = [@end_date.strftime("%Y-%m-%d"), @asset_project.name, asset, @key6.key, "Hello, world", "IT", @end_date.strftime("%Y-%m-%d 08:00:00 UTC"), "Rebecca (#{@translator.id})", "hello!", "bye!", @end_date.strftime("%Y-%m-%d 08:00:00 UTC"), "Mark (#{@reviewer.id})", "", nil]
- expect(result[9]).to eql expected_results
- end
-
- it 'has the expected row 10' do
- expected_results = [@end_date.strftime("%Y-%m-%d"), @article_project.name, article, @key5.key, "Hello, world", "IT", @end_date.strftime("%Y-%m-%d 08:00:00 UTC"), "Rebecca (#{@translator.id})", "hello!", "bye!", @end_date.strftime("%Y-%m-%d 08:00:00 UTC"), "Mark (#{@reviewer.id})", "", nil]
- expect(result[10]).to eql expected_results
- end
+ # it 'has the expected row 8' do
+ # expected_results = [@end_date.strftime("%Y-%m-%d"), project, sha, @key4.key, "Hello, world", "IT", @end_date.strftime("%Y-%m-%d 08:00:00 UTC"), "Rebecca (#{@translator.id})", "hello!", "bye!", @end_date.strftime("%Y-%m-%d 08:00:00 UTC"), "Mark (#{@reviewer.id})", "", nil]
+ # expect(result[8]).to eql expected_results
+ # end
+
+ # it 'has the expected row 9' do
+ # expected_results = [@end_date.strftime("%Y-%m-%d"), @asset_project.name, asset, @key6.key, "Hello, world", "IT", @end_date.strftime("%Y-%m-%d 08:00:00 UTC"), "Rebecca (#{@translator.id})", "hello!", "bye!", @end_date.strftime("%Y-%m-%d 08:00:00 UTC"), "Mark (#{@reviewer.id})", "", nil]
+ # expect(result[9]).to eql expected_results
+ # end
+
+ # it 'has the expected row 10' do
+ # expected_results = [@end_date.strftime("%Y-%m-%d"), @article_project.name, article, @key5.key, "Hello, world", "IT", @end_date.strftime("%Y-%m-%d 08:00:00 UTC"), "Rebecca (#{@translator.id})", "hello!", "bye!", @end_date.strftime("%Y-%m-%d 08:00:00 UTC"), "Mark (#{@reviewer.id})", "", nil]
+ # expect(result[10]).to eql expected_results
+ # end
it 'has the expected row 11' do
expect(result[11]).to eql nil
diff --git a/spec/lib/sorting_helper_spec.rb b/spec/lib/sorting_helper_spec.rb
deleted file mode 100644
index 383ccd64..00000000
--- a/spec/lib/sorting_helper_spec.rb
+++ /dev/null
@@ -1,37 +0,0 @@
-# Copyright 2016 Square Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-require 'rails_helper'
-
-RSpec.describe SortingHelper do
- describe "#order_by_elasticsearch_result_order" do
- before do
- project = FactoryBot.create(:project)
- @commit1 = FactoryBot.create(:commit, project: project)
- @commit2 = FactoryBot.create(:commit, project: project)
- @commit3 = FactoryBot.create(:commit, project: project)
-
- @commits = [@commit1, @commit2, @commit3]
- @es_objects = [double('es_result', id: @commit2.id),
- double('es_result', id: @commit3.id),
- double('es_result', id: @commit1.id)]
- end
-
- it "orders items with the elasticsearch results order" do
- ordered_commits = SortingHelper.order_by_elasticsearch_result_order(@commits, @es_objects)
-
- expect(ordered_commits).to eql([@commit2, @commit3, @commit1])
- end
- end
-end
diff --git a/spec/lib/translation_validator/source_fencer_validator_spec.rb b/spec/lib/translation_validator/source_fencer_validator_spec.rb
new file mode 100644
index 00000000..c54e6ef5
--- /dev/null
+++ b/spec/lib/translation_validator/source_fencer_validator_spec.rb
@@ -0,0 +1,52 @@
+# Copyright 2014 Square Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rails_helper'
+
+RSpec.describe TranslationValidator::SourceFencerValidator do
+ let(:source_copy) { "this is a {{good-variable-name}}"}
+
+ before :each do
+ @project = FactoryBot.create(:project)
+ @commit = FactoryBot.create(:commit, project: @project, author: 'Foo Bar', author_email: "foo@example.com")
+
+ @key = FactoryBot.create(:key, project: @project, fencers: ['Mustache'], source_copy: source_copy)
+ @commit.keys << @key
+
+ @translation = FactoryBot.create(:translation, key: @key, translated: false, copy: nil)
+ @key.translations << @translation
+ end
+
+ describe "#run" do
+ subject { TranslationValidator::SourceFencerValidator.new(@commit).run }
+
+ context "without suspicious source strings" do
+ it "should not send email" do
+ expect_any_instance_of(FencerValidationMailer).to_not receive(:suspicious_source_found)
+
+ subject
+ end
+ end
+
+ context "with suspicious source strings" do
+ let(:source_copy) { "this is a {{bad variable name}}" }
+
+ it "should send email" do
+ expect_any_instance_of(FencerValidationMailer).to receive(:suspicious_source_found)
+
+ subject
+ end
+ end
+ end
+end
diff --git a/spec/lib/translation_validator/translation_auto_migration_spec.rb b/spec/lib/translation_validator/translation_auto_migration_spec.rb
new file mode 100644
index 00000000..3b663446
--- /dev/null
+++ b/spec/lib/translation_validator/translation_auto_migration_spec.rb
@@ -0,0 +1,76 @@
+# Copyright 2014 Square Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rails_helper'
+
+RSpec.describe TranslationValidator::TranslationAutoMigration do
+ let(:source_copy) { "this is a testing string"}
+
+ before :each do
+ @project = FactoryBot.create(:project)
+ @commit = FactoryBot.create(:commit, project: @project, author: 'Foo Bar', author_email: "foo@example.com")
+
+ @key = FactoryBot.create(:key, project: @project, source_copy: source_copy)
+ @commit.keys << @key
+
+ @translation = FactoryBot.create(:translation, key: @key, source_copy: source_copy, copy: nil, translated: false, source_rfc5646_locale: 'en', rfc5646_locale: 'en-CA')
+ @key.translations << @translation
+ end
+
+ describe "#run" do
+ subject { TranslationValidator::TranslationAutoMigration.new(@commit).run }
+
+ context "without any matched translation" do
+ it "should not migrate TM" do
+ subject
+
+ @translation.reload
+ expect(@translation.translated).to be_falsey
+ expect(@translation.approved).to be_falsey
+ expect(@translation.copy).to be_nil
+ expect(@translation.notes).to be_nil
+ end
+ end
+
+ context "with non-approved translation" do
+ it "should not migrate TM" do
+ key = FactoryBot.create(:key, project: @project, original_key: '1', source_copy: source_copy)
+ FactoryBot.create(:translation, key: key, source_copy: source_copy, copy: 'hello', translated: true, approved: nil, source_rfc5646_locale: 'en', rfc5646_locale: 'en-CA')
+
+ subject
+
+ @translation.reload
+ expect(@translation.translated).to be_falsey
+ expect(@translation.approved).to be_falsey
+ expect(@translation.copy).to be_nil
+ expect(@translation.notes).to be_nil
+ end
+ end
+
+ context "with approved translation" do
+ it "should migrate TM" do
+ key = FactoryBot.create(:key, project: @project, original_key: '1', source_copy: source_copy)
+ translation = FactoryBot.create(:translation, key: key, source_copy: source_copy, copy: 'hello', translated: true, approved: true, source_rfc5646_locale: 'en', rfc5646_locale: 'en-CA')
+
+ subject
+
+ @translation.reload
+ expect(@translation.translated).to be_truthy
+ expect(@translation.approved).to be_falsey
+ expect(@translation.copy).to eq('hello')
+ expect(@translation.notes).to eq("AutoTM:#{translation.id} ")
+ end
+ end
+ end
+end
diff --git a/spec/mailers/comment_mailer_spec.rb b/spec/mailers/comment_mailer_spec.rb
index 9943aa2d..83bb5b6c 100644
--- a/spec/mailers/comment_mailer_spec.rb
+++ b/spec/mailers/comment_mailer_spec.rb
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-require "spec_helper"
+require "rails_helper"
RSpec.describe CommentMailer do
context "[comment_created]" do
diff --git a/spec/mailers/commit_mailer_spec.rb b/spec/mailers/commit_mailer_spec.rb
index 2b25fe11..c11d6136 100644
--- a/spec/mailers/commit_mailer_spec.rb
+++ b/spec/mailers/commit_mailer_spec.rb
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-require "spec_helper"
+require "rails_helper"
RSpec.describe CommitMailer do
describe "#notify_submitter_of_import_errors" do
diff --git a/spec/mailers/fencer_validation_mailer_spec.rb b/spec/mailers/fencer_validation_mailer_spec.rb
new file mode 100644
index 00000000..344ce82e
--- /dev/null
+++ b/spec/mailers/fencer_validation_mailer_spec.rb
@@ -0,0 +1,39 @@
+# Copyright 2014 Square Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "rails_helper"
+
+RSpec.describe FencerValidationMailer do
+ describe "#suspicious_source_found" do
+ before :each do
+ @project = FactoryBot.create(:project)
+ @commit = FactoryBot.create(:commit, project: @project, author: 'Foo Bar', author_email: "foo@example.com")
+ @key = FactoryBot.create(:key, project: @project)
+
+ @translation = FactoryBot.create(:translation, translated: false, key: @key)
+ @key.reload
+
+ ActionMailer::Base.deliveries.clear
+ end
+
+ it 'sends an email to shuttle and localization teams' do
+ fake_suspicious_keys_errors = [[@key, 'Violate fencers: FakeFencer']]
+
+ mail = FencerValidationMailer.suspicious_source_found(@commit, fake_suspicious_keys_errors).deliver_now
+
+ expect(mail.subject).to eq('[NO ACTION REQUIRED] [Shuttle Staging] Found suspicious source strings')
+ expect(mail.body).to include('Violate fencers: FakeFencer')
+ end
+ end
+end
diff --git a/spec/mailers/issue_mailer_spec.rb b/spec/mailers/issue_mailer_spec.rb
index a27c7bc0..0f236b3d 100644
--- a/spec/mailers/issue_mailer_spec.rb
+++ b/spec/mailers/issue_mailer_spec.rb
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-require "spec_helper"
+require "rails_helper"
RSpec.describe IssueMailer do
context "issue_created" do
diff --git a/spec/mailers/screenshot_mailer_spec.rb b/spec/mailers/screenshot_mailer_spec.rb
index 0034f490..eb6aff96 100644
--- a/spec/mailers/screenshot_mailer_spec.rb
+++ b/spec/mailers/screenshot_mailer_spec.rb
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-require "spec_helper"
+require "rails_helper"
RSpec.describe ScreenshotMailer do
describe 'request_screenshot' do
diff --git a/spec/mediators/translation_update_mediator_spec.rb b/spec/mediators/translation_update_mediator_spec.rb
index 8ef1a0aa..5fba216e 100644
--- a/spec/mediators/translation_update_mediator_spec.rb
+++ b/spec/mediators/translation_update_mediator_spec.rb
@@ -43,7 +43,7 @@
expect(@key.reload).to_not be_ready
end
- it "updates a single translation; approve translation if translator is a reviewer; key becomes ready" do
+ it "updates a single translation; approve not-translated string" do
TranslationUpdateMediator.new(@fr_translation, reviewer, @params).update!
expect(@fr_translation.copy).to eql("test copy")
expect(@fr_translation.translator).to eql(reviewer)
@@ -52,6 +52,17 @@
expect(@key.reload).to be_ready
end
+ it "updates a single translation; approve translated string" do
+ @fr_translation.update(copy: 'this is old translation', translator: translator)
+
+ TranslationUpdateMediator.new(@fr_translation, reviewer, @params).update!
+ expect(@fr_translation.copy).to eql("test copy")
+ expect(@fr_translation.translator).to eql(translator)
+ expect(@fr_translation.approved).to be_truthy
+ expect(@fr_translation.reviewer).to eql(reviewer)
+ expect(@key.reload).to be_ready
+ end
+
it "updates a single translation; review_date is set if the translator is a reviewer" do
TranslationUpdateMediator.new(@fr_translation, reviewer, @params).update!
expect(@fr_translation.review_date).to_not be_nil
@@ -141,13 +152,11 @@
end
it "sets the tm_match" do
- reset_elastic_search
-
# create a translation that will be used for lookup for tm_match
FactoryBot.create(:translation, copy: "test", source_copy: 'test', approved: true, translated: true, rfc5646_locale: 'fr')
# finding the fuzzy match for a translation requires elasticsearch, update the index since we just created a translation
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
TranslationUpdateMediator.new(@fr_translation, reviewer, @params).update!
@@ -224,6 +233,7 @@
translation1 = FactoryBot.create(:translation, key: key, rfc5646_locale: 'fr')
translation2 = FactoryBot.create(:translation, key: key, rfc5646_locale: 'fr-CA')
translation3 = FactoryBot.create(:translation, key: key, rfc5646_locale: 'fr-FR')
+ key.reload
expect(TranslationUpdateMediator.multi_updateable_translations_to_locale_associations_hash(translation1)).to eql(translation2 => la1, translation3 => la2)
end
@@ -236,6 +246,7 @@
translation1 = FactoryBot.create(:translation, key: key, source_rfc5646_locale: 'en', rfc5646_locale: 'en-XX')
translation2 = FactoryBot.create(:translation, key: key, source_rfc5646_locale: 'en', rfc5646_locale: 'en')
translation3 = FactoryBot.create(:translation, key: key, source_rfc5646_locale: 'en', rfc5646_locale: 'en-YY')
+ key.reload
expect(TranslationUpdateMediator.multi_updateable_translations_to_locale_associations_hash(translation1)).to eql(translation3 => la2)
end
end
@@ -251,6 +262,7 @@
it "returns the Translation objects corresponding to the user provided copyToLocales param; doesn't add an error if all are valid" do
fr_CA_translation = FactoryBot.create(:translation, key: @key, copy: nil, translator: nil, rfc5646_locale: 'fr-CA')
fr_FR_translation = FactoryBot.create(:translation, key: @key, copy: nil, translator: nil, rfc5646_locale: 'fr-FR')
+ @key.reload
mediator = TranslationUpdateMediator.new(@fr_translation, translator, ActionController::Parameters.new( copyToLocales: %w(fr-CA fr-FR) ))
expect(mediator.send(:translations_that_should_be_multi_updated)).to eql([fr_CA_translation, fr_FR_translation])
expect(mediator.success?).to be_truthy
@@ -266,6 +278,7 @@
it "adds an error to the mediator if one of the locales user wanted to copy to is not valid because there is no Translation in one of those locales" do
fr_CA_translation = FactoryBot.create(:translation, key: @key, copy: nil, translator: nil, rfc5646_locale: 'fr-CA')
+ @key.reload
mediator = TranslationUpdateMediator.new(@fr_translation, translator, ActionController::Parameters.new( copyToLocales: %w(fr-CA fr-FR)))
mediator.send(:translations_that_should_be_multi_updated)
expect(mediator.success?).to be_falsey
diff --git a/spec/models/article_spec.rb b/spec/models/article_spec.rb
index bc804459..d94f07e0 100644
--- a/spec/models/article_spec.rb
+++ b/spec/models/article_spec.rb
@@ -377,7 +377,11 @@
allow(article).to receive(:import!) # prevent the import because we want to create the related keys manually
article.save!
- article.import_batch.jobs { regenerate_elastic_search_indexes }
+ article.import_batch.jobs do
+ CommitsIndex.reset!
+ KeysIndex.reset!
+ TranslationsIndex.reset!
+ end
bid = article.import_batch_id
article.import_batch # this should re-use the existing batch
expect(article.import_batch_id).to eql(bid)
diff --git a/spec/models/commit_spec.rb b/spec/models/commit_spec.rb
index cca1c2ab..8b0220ee 100644
--- a/spec/models/commit_spec.rb
+++ b/spec/models/commit_spec.rb
@@ -487,4 +487,21 @@
end
end
end
+
+ describe "#elastic_search" do
+ let!(:project) { FactoryBot.create(:project, repository_url: "https://github.com/example/my-project.git") }
+ let!(:commit) { FactoryBot.create(:commit, revision: 'abc123', project: project) }
+
+ it "should appear in both ES and DB after creation" do
+ expect(Commit.where(id: commit.id).count).to eq(1)
+ expect(CommitsIndex.query(term: { id: commit.id }).count).to eq(1)
+ end
+
+ it "should disappear in both ES and DB after destroy" do
+ commit.destroy
+
+ expect(Commit.where(id: commit.id).count).to eq(0)
+ expect(CommitsIndex.query(term: { id: commit.id }).count).to eq(0)
+ end
+ end
end
diff --git a/spec/models/key_spec.rb b/spec/models/key_spec.rb
index 9764b7c9..47ce89f4 100644
--- a/spec/models/key_spec.rb
+++ b/spec/models/key_spec.rb
@@ -208,7 +208,7 @@
FactoryBot.create :translation, key: key1, rfc5646_locale: 'fr', approved: true, source_copy: 'yes', copy: 'oui'
# finding the fuzzy match for a translation requires elasticsearch, update the index since we just created a translation
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
# this is a new key
key2 = FactoryBot.create(:key, project: project, source_copy: 'yes')
@@ -409,6 +409,8 @@
FactoryBot.create :translation, source_rfc5646_locale: 'en-US', rfc5646_locale: 'de', key: key
FactoryBot.create :translation, source_rfc5646_locale: 'en-US', rfc5646_locale: 'de', key: excluded, copy: nil
FactoryBot.create :translation, source_rfc5646_locale: 'en-US', rfc5646_locale: 'fr', key: excluded, copy: nil
+ key.reload
+ excluded.reload
key.remove_excluded_pending_translations
excluded.remove_excluded_pending_translations
@@ -428,6 +430,8 @@
FactoryBot.create :translation, source_rfc5646_locale: 'en-US', rfc5646_locale: 'de', key: included, copy: nil
FactoryBot.create :translation, source_rfc5646_locale: 'en-US', rfc5646_locale: 'de', key: excluded, copy: nil
FactoryBot.create :translation, source_rfc5646_locale: 'en-US', rfc5646_locale: 'fr', key: excluded, copy: "hello!"
+ included.reload
+ excluded.reload
included.remove_excluded_pending_translations
excluded.remove_excluded_pending_translations
@@ -447,6 +451,8 @@
FactoryBot.create :translation, source_rfc5646_locale: 'en-US', rfc5646_locale: 'de', key: key, copy: nil
FactoryBot.create :translation, source_rfc5646_locale: 'en-US', rfc5646_locale: 'de', key: excluded, copy: nil
FactoryBot.create :translation, source_rfc5646_locale: 'en-US', rfc5646_locale: 'fr', key: excluded, copy: "hello!"
+ key.reload
+ excluded.reload
key.remove_excluded_pending_translations
excluded.remove_excluded_pending_translations
@@ -608,4 +614,21 @@
expect(@key2.reload).to_not be_ready
end
end
+
+ describe "#elastic_search" do
+ let!(:project) { FactoryBot.create(:project, targeted_rfc5646_locales: {'fr' => true}, base_rfc5646_locale: 'en') }
+ let!(:key) { FactoryBot.create(:key, project: project) }
+
+ it "should appear in both ES and DB after creation" do
+ expect(Key.where(id: key.id).count).to eq(1)
+ expect(KeysIndex.query(term: { id: key.id }).count).to eq(1)
+ end
+
+ it "should disappear in both ES and DB after destroy" do
+ key.destroy
+
+ expect(Key.where(id: key.id).count).to eq(0)
+ expect(KeysIndex.query(term: { id: key.id }).count).to eq(0)
+ end
+ end
end
diff --git a/spec/models/report_spec.rb b/spec/models/report_spec.rb
new file mode 100644
index 00000000..92088eed
--- /dev/null
+++ b/spec/models/report_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe Report, type: :model do
+ pending "add some examples to (or delete) #{__FILE__}"
+end
diff --git a/spec/models/translation_spec.rb b/spec/models/translation_spec.rb
index eea51c58..313ff051 100644
--- a/spec/models/translation_spec.rb
+++ b/spec/models/translation_spec.rb
@@ -257,30 +257,6 @@
end
end
- describe "#batch_refresh_elastic_search" do
- it "refreshes the ElasticSearch index (section_active field in this test) of Article's Translations" do
- allow_any_instance_of(Article).to receive(:import!) # prevent auto import
- reset_elastic_search
-
- article = FactoryBot.create(:article)
- section = FactoryBot.create(:section, article: article, active: true)
- key = FactoryBot.create(:key, section: section, index_in_section: 0, project: article.project)
- translation = FactoryBot.create(:translation, key: key)
-
- regenerate_elastic_search_indexes
-
- expect(Elasticsearch::Model.search(section_active_query(true), Translation).results.first.id.to_i).to eql(translation.id)
- expect(Elasticsearch::Model.search(section_active_query(false), Translation).results.first).to be_nil
-
- section.update! active: false
- Translation.batch_refresh_elastic_search(article)
- regenerate_elastic_search_indexes
-
- expect(Elasticsearch::Model.search(section_active_query(true), Translation).results.first).to be_nil
- expect(Elasticsearch::Model.search(section_active_query(false), Translation).results.first.id.to_i).to eql(translation.id)
- end
- end
-
describe "#shared?" do
let(:article) { FactoryBot.create(:article) }
let(:section) { FactoryBot.create(:section, article: article, active: true) }
@@ -313,6 +289,40 @@
end
end
+ describe "#destroy" do
+ it "cleans all child objects properly" do
+ translation = FactoryBot.create(:translation)
+ translation_change = FactoryBot.create(:translation_change, translation: translation)
+ edit_reason = FactoryBot.create(:edit_reason, translation_change: translation_change)
+
+ expect(Translation.where(id: translation.id).count).to eq(1)
+ expect(TranslationChange.where(id: translation_change.id).count).to eq(1)
+ expect(EditReason.where(id: edit_reason.id).count).to eq(1)
+
+ translation.destroy
+
+ expect(Translation.where(id: translation.id).count).to eq(0)
+ expect(TranslationChange.where(id: translation_change.id).count).to eq(0)
+ expect(EditReason.where(id: edit_reason.id).count).to eq(0)
+ end
+ end
+
+ describe "#elastic_search" do
+ let!(:translation) { FactoryBot.create(:translation) }
+
+ it "should appear in both ES and DB after creation" do
+ expect(Translation.where(id: translation.id).count).to eq(1)
+ expect(TranslationsIndex.query(term: { id: translation.id }).count).to eq(1)
+ end
+
+ it "should disappear in both ES and DB after destroy" do
+ translation.destroy
+
+ expect(Translation.where(id: translation.id).count).to eq(0)
+ expect(TranslationsIndex.query(term: { id: translation.id }).count).to eq(0)
+ end
+ end
+
def section_active_query(active)
{
filter: {
diff --git a/spec/presenters/locale_projects_show_presenter_spec.rb b/spec/presenters/locale_projects_show_presenter_spec.rb
index e600e4de..3f5a9e66 100644
--- a/spec/presenters/locale_projects_show_presenter_spec.rb
+++ b/spec/presenters/locale_projects_show_presenter_spec.rb
@@ -48,6 +48,14 @@
end
end
+ describe "#selected_group" do
+ it "returns the selected Group if there is one" do
+ group = FactoryBot.create(:group, name: "hello", display_name: 'test display name')
+ presenter = LocaleProjectsShowPresenter.new(group.project, @user, { group: group.display_name })
+ expect(presenter.selected_group).to eql(group)
+ end
+ end
+
describe "#selectable_sections" do
before :each do
@project = FactoryBot.create(:project)
diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb
index a3c7b066..16a6fef2 100644
--- a/spec/rails_helper.rb
+++ b/spec/rails_helper.rb
@@ -8,6 +8,7 @@
# Add additional requires below this line. Rails is not loaded until this point!
require 'sidekiq/testing'
require 'paperclip/matchers'
+require 'chewy/rspec'
Dir[Rails.root.join('spec/spec_support/**/*.rb')].each { |f| require f }
@@ -88,8 +89,17 @@
DatabaseCleaner.strategy = :deletion
DatabaseCleaner.clean_with :truncation
end
+
config.around(:each) do |example|
- DatabaseCleaner.cleaning { example.run }
+ DatabaseCleaner.start
+ example.run
+
+ # Delete the table relationship individually.
+ models = [AssetsKey, ArticleGroup, BlobsCommit, TranslationChange]
+ models.each { |model| model.all.destroy_all }
+
+ DatabaseCleaner.clean
+ [CommitsIndex, KeysIndex, TranslationsIndex].each { |index| index.reset! }
end
# Paperclip
@@ -115,22 +125,7 @@
end
# ElasticSearch
- config.before(:suite) { reset_elastic_search }
-end
-
-def reset_elastic_search
- ActiveRecord::Base.subclasses.each do |model|
- next unless model.respond_to?(:__elasticsearch__)
- model.__elasticsearch__.create_index! force: true
- model.import(force: true)
- model.__elasticsearch__.client.indices.flush(index: model.__elasticsearch__.index_name, force: true)
- end
-end
-
-def regenerate_elastic_search_indexes
- ActiveRecord::Base.subclasses.each do |model|
- next unless model.respond_to?(:__elasticsearch__)
- model.import(refresh: true)
- model.__elasticsearch__.client.indices.flush(index: model.__elasticsearch__.index_name, force: true)
+ config.before(:suite) do
+ Chewy::strategy :urgent
end
end
diff --git a/spec/services/fuzzy_match_translations_finder_spec.rb b/spec/services/fuzzy_match_translations_finder_spec.rb
index 9757cebd..cf50d1d0 100644
--- a/spec/services/fuzzy_match_translations_finder_spec.rb
+++ b/spec/services/fuzzy_match_translations_finder_spec.rb
@@ -25,7 +25,7 @@
FactoryBot.create(:translation, copy: "oui", source_copy: source_copy, approved: true, translated: true, rfc5646_locale: 'fr')
# finding the fuzzy match for a translation requires elasticsearch, update the index since we just created a translation
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
translation = FactoryBot.build(:translation, source_copy: source_copy, rfc5646_locale: 'fr')
finder = FuzzyMatchTranslationsFinder.new(source_copy, translation)
@@ -44,7 +44,7 @@
FactoryBot.create(:translation, copy: "oui monsieur ", source_copy: 'yes sir', approved: true, translated: true, rfc5646_locale: 'fr')
# finding the fuzzy match for a translation requires elasticsearch, update the index since we just created a translation
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
# create a translation that will matches the above one with 57%
translation = FactoryBot.build(:translation, source_copy: 'yes madam', rfc5646_locale: 'fr')
diff --git a/spec/services/search_translations_finder_spec.rb b/spec/services/search_translations_finder_spec.rb
index bab966d4..839194bf 100644
--- a/spec/services/search_translations_finder_spec.rb
+++ b/spec/services/search_translations_finder_spec.rb
@@ -19,7 +19,6 @@
describe "#find_translations" do
before :each do
Translation.destroy_all
- reset_elastic_search
@project = FactoryBot.create(:project, repository_url: Rails.root.join('spec', 'fixtures', 'repository.git').to_s)
@key = create_key(@project)
@translation = create_translation(@key, copy: 'some copy here', rfc5646_locale: Locale.new('de-DE').rfc5646)
@@ -27,19 +26,19 @@
end
it "should include new translation" do
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
expect(@finder.find_translations.total_count).to eq(1)
new_key = create_key(@project)
create_translation(new_key)
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
expect(@finder.find_translations.total_count).to eq(2)
end
it "should filter target locales" do
create_translation(@key)
new_finder = create_finder(target_locales: [Locale.new('zh-CN'), Locale.new('ja-JP')])
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
expect(@finder.find_translations.total_count).to eq(2)
expect(new_finder.find_translations.total_count).to eq(1)
@@ -50,7 +49,7 @@
new_key = create_key(new_project)
create_translation(new_key)
new_finder = create_finder(project_id: new_project.id)
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
expect(@finder.find_translations.total_count).to eq(2)
expect(new_finder.find_translations.total_count).to eq(1)
@@ -58,7 +57,15 @@
it "should filter translation id" do
new_finder = create_finder(translator_id: 15875)
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
+
+ expect(@finder.find_translations.total_count).to eq(1)
+ expect(new_finder.find_translations.total_count).to eq(0)
+ end
+
+ it "should filter reviewer id" do
+ new_finder = create_finder(reviewer_id: 15875)
+ TranslationsIndex.reset!
expect(@finder.find_translations.total_count).to eq(1)
expect(new_finder.find_translations.total_count).to eq(0)
@@ -67,7 +74,7 @@
it "should filter start date" do
create_translation(@key, updated_at: Time.current - 7.days)
new_finder = create_finder(start_date: Time.current - 3.days)
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
expect(@finder.find_translations.total_count).to eq(2)
expect(new_finder.find_translations.total_count).to eq(1)
@@ -76,7 +83,7 @@
it "should filter end date" do
create_translation(@key, updated_at: Time.current + 7.days)
new_finder = create_finder(end_date: Time.current + 3.days)
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
expect(@finder.find_translations.total_count).to eq(2)
expect(new_finder.find_translations.total_count).to eq(1)
@@ -85,7 +92,7 @@
it "should filter hidden keys" do
hidden_key = create_key(@project, hidden_in_search: true)
create_translation(hidden_key)
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
expect(@finder.find_translations.total_count).to eq(1)
end
@@ -96,7 +103,7 @@
hidden_key = create_key(@project, hidden_in_search: true)
create_translation(hidden_key)
end
- regenerate_elastic_search_indexes
+ TranslationsIndex.reset!
expect(@finder.find_translations.total_count).to eq(1)
expect(new_finder.find_translations.total_count).to eq(3)
end
diff --git a/spec/workers/article_importer_spec.rb b/spec/workers/article_importer_spec.rb
index ea5896b1..b211e7b2 100644
--- a/spec/workers/article_importer_spec.rb
+++ b/spec/workers/article_importer_spec.rb
@@ -16,7 +16,7 @@
RSpec.describe ArticleImporter do
describe "#perform" do
- before :each do
+ before do
allow_any_instance_of(Article).to receive(:import!) # prevent auto imports
@article = FactoryBot.create(:article, sections_hash: { "title" => "a", "body" => "b
c
" })
ArticleImporter.new.perform(@article.id) # first import
@@ -117,7 +117,7 @@
RSpec.describe ArticleImporter::Finisher do
describe "#on_success" do
- before :each do
+ before do
# creation triggers the initial import
@article = FactoryBot.create(:article, sections_hash: { "main" => "hello
world
" }, base_rfc5646_locale: 'en', targeted_rfc5646_locales: { 'fr' => true, 'es' => true, 'ja' => false })
expect(@article.reload.keys.count).to eql(6)
diff --git a/spec/workers/auto_importer_spec.rb b/spec/workers/auto_importer_spec.rb
index 9478cb6a..7db57ad3 100644
--- a/spec/workers/auto_importer_spec.rb
+++ b/spec/workers/auto_importer_spec.rb
@@ -17,7 +17,7 @@
RSpec.describe AutoImporter do
describe "#perform" do
context "[watched branches]" do
- it "calls ProjectAutoImporter on the projects with watched_branches, removes the watched branch if it doesn't exist" do
+ it "calls ProjectAutoImporter on the projects with watched_branches, does not remove the watched branch if it doesn't exist" do
project1 = FactoryBot.create(:project, skip_imports: (Importer::Base.implementations.map(&:ident) - %w(yaml)), repository_url: Rails.root.join('spec', 'fixtures', 'repository.git').to_s, watched_branches: %w(master non_existent_branch))
project2 = FactoryBot.create(:project, watched_branches: [])
@@ -28,7 +28,7 @@
expect { AutoImporter.new.perform }.to_not raise_error
- expect(project1.reload.watched_branches).to eql(%w(master))
+ expect(project1.reload.watched_branches).to eql(%w(master non_existent_branch))
expect(project2.reload.watched_branches).to be_blank
end
end
@@ -47,12 +47,12 @@
describe "#perform" do
context "[watched branches]" do
context "[rescue Git::CommitNotFoundError]" do
- it "removes a watched branch if the branch doesn't exist anymore" do
+ it "does not remove a watched branch if the branch doesn't exist anymore" do
project = FactoryBot.create(:project, skip_imports: (Importer::Base.implementations.map(&:ident) - %w(yaml)), repository_url: Rails.root.join('spec', 'fixtures', 'repository.git').to_s, watched_branches: %w(master non_existent_branch))
expect(project.watched_branches).to eql(%w(master non_existent_branch))
expect { AutoImporter::ProjectAutoImporter.new.perform(project.id) }.to_not raise_error
- expect(project.reload.watched_branches).to eql(%w(master))
+ expect(project.reload.watched_branches).to eql(%w(master non_existent_branch))
end
end
end
diff --git a/spec/workers/commit_importer_spec.rb b/spec/workers/commit_importer_spec.rb
index ec5b8f55..ee3b65fd 100644
--- a/spec/workers/commit_importer_spec.rb
+++ b/spec/workers/commit_importer_spec.rb
@@ -19,11 +19,10 @@
describe "#perform" do
context "[rescue Git::CommitNotFoundError]" do
it "deletes the commit when commit importer fails due to a Git::CommitNotFoundError" do
- skip 'ElasticSearch flaky - deletion throws NotFound exception'
allow_any_instance_of(Project).to receive(:find_or_fetch_git_object).and_return(nil)
project = FactoryBot.create(:project)
commit = FactoryBot.create(:commit, revision: "abc123", project: project)
- regenerate_elastic_search_indexes
+ CommitsIndex.reset!
CommitImporter.new.perform commit.id
expect { commit.reload }.to raise_error(ActiveRecord::RecordNotFound)
diff --git a/spec/workers/commits_cleaner_spec.rb b/spec/workers/commits_cleaner_spec.rb
index f766dced..20b64900 100644
--- a/spec/workers/commits_cleaner_spec.rb
+++ b/spec/workers/commits_cleaner_spec.rb
@@ -29,9 +29,8 @@
end
it "should destory all commits" do
- skip 'ElasticSearch flaky - deletion throws NotFound exception'
FactoryBot.create_list :commit, 3, project: @project
- regenerate_elastic_search_indexes
+ CommitsIndex.reset!
expect(@project.commits.count).to eq(3)
@commits_cleaner.destroy_dangling_commits
@@ -56,10 +55,9 @@
describe "#destroy_old_commits_which_errored_during_import" do
it "should destroy all errored commits older than 2 days during import" do
- skip 'ElasticSearch flaky - deletion throws NotFound exception'
FactoryBot.create_list(:commit, 3, project: @project, created_at: 3.days.ago).
each { |c| c.add_import_error StandardError.new("This is a fake error") }
- regenerate_elastic_search_indexes
+ CommitsIndex.reset!
expect(@project.commits.count).to eq(3)
@commits_cleaner.destroy_old_commits_which_errored_during_import
diff --git a/spec/workers/post_loading_checker_spec.rb b/spec/workers/post_loading_checker_spec.rb
new file mode 100644
index 00000000..d560d604
--- /dev/null
+++ b/spec/workers/post_loading_checker_spec.rb
@@ -0,0 +1,31 @@
+# Copyright 2014 Square Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rails_helper'
+
+RSpec.describe PostLoadingChecker do
+ describe "#perform" do
+ before :each do
+ @project = FactoryBot.create(:project)
+ @commit = FactoryBot.create(:commit, project: @project, author: 'Foo Bar', author_email: "foo@example.com")
+ end
+
+ it "calls each validator" do
+ expect_any_instance_of(TranslationValidator::SourceFencerValidator).to receive(:run)
+ expect_any_instance_of(TranslationValidator::TranslationAutoMigration).to receive(:run)
+
+ PostLoadingChecker.new.perform('commit', @commit.id)
+ end
+ end
+end
diff --git a/spec/workers/stash_webhook_pinger_spec.rb b/spec/workers/stash_webhook_pinger_spec.rb
index f2014f25..bf047b21 100644
--- a/spec/workers/stash_webhook_pinger_spec.rb
+++ b/spec/workers/stash_webhook_pinger_spec.rb
@@ -16,6 +16,9 @@
RSpec.describe StashWebhookPinger do
include Rails.application.routes.url_helpers
+ let(:http_response_code) { 204 }
+ let(:http_response) { Net::HTTPResponse.new(1.0, http_response_code, "OK") }
+
before(:each) do
allow(Kernel).to receive(:sleep)
allow(HTTParty).to receive(:post)
@@ -36,7 +39,7 @@
expect(HTTParty).to receive(:post).with(
"#{url}/#{@commit.revision}",
anything()
- ).exactly(StashWebhookHelper::DEFAULT_NUM_TIMES).times
+ ).exactly(StashWebhookHelper::DEFAULT_NUM_TIMES).times.and_return(http_response)
subject.perform(@commit.id)
end
@@ -50,7 +53,7 @@
expected_state = @commit.ready? ? 'SUCCESSFUL' : 'INPROGRESS'
expect(commit_state).to eql(expected_state)
@commit.update_column :ready, !@commit.ready
- end.exactly(StashWebhookHelper::DEFAULT_NUM_TIMES).times
+ end.exactly(StashWebhookHelper::DEFAULT_NUM_TIMES).times.and_return(http_response)
subject.perform(@commit.id)
end
@@ -65,6 +68,20 @@
expect(HTTParty).not_to receive(:post)
expect { subject.perform(@commit.id) }.to raise_error(Project::NotLinkedToAGitRepositoryError)
end
+
+ context "with failure http response code" do
+ let(:http_response_code) { 400 }
+
+ it "when http returns failure" do
+ url = "http://www.example.com"
+ @commit.project.stash_webhook_url = url
+ @commit.project.save!
+
+ expect(HTTParty).to receive(:post).and_return(http_response)
+
+ expect { subject.perform(@commit.id) }.to raise_error(RuntimeError, "[StashWebhookHelper] Failed to ping stash for commit #{@commit.id}, revision: #{@commit.revision}, code: #{http_response_code}")
+ end
+ end
end
context "on_create" do
@@ -84,7 +101,7 @@
protocol: Shuttle::Configuration.default_url_options['protocol'] || 'http'),
state: 'INPROGRESS',
description: 'Currently loading',
- }.to_json))
+ }.to_json)).exactly(StashWebhookHelper::DEFAULT_NUM_TIMES).times.and_return(http_response)
@commit.save!
end
end
@@ -92,6 +109,8 @@
context "on_update" do
before(:each) do
+ expect(HTTParty).to receive(:post).exactly(StashWebhookHelper::DEFAULT_NUM_TIMES).times.and_return(http_response)
+
@commit = FactoryBot.build(:commit, ready: false, loading: true)
@url = "http://www.example.com"
@commit.project.stash_webhook_url = @url
@@ -110,7 +129,7 @@
protocol: Shuttle::Configuration.default_url_options['protocol'] || 'http'),
state: 'INPROGRESS',
description: 'Currently translating',
- }.to_json))
+ }.to_json)).exactly(StashWebhookHelper::DEFAULT_NUM_TIMES).times.and_return(http_response)
@commit.loading = false
# force commit not to be ready
@@ -130,7 +149,7 @@
protocol: Shuttle::Configuration.default_url_options['protocol'] || 'http'),
state: 'SUCCESSFUL',
description: 'Translations completed',
- }.to_json))
+ }.to_json)).exactly(StashWebhookHelper::DEFAULT_NUM_TIMES).times.and_return(http_response)
@commit.loading = false
@commit.ready = true # redundant since CSR will do this anyway
@commit.save!
diff --git a/square_primary_certificate_authority_g2.crt b/square_primary_certificate_authority_g2.crt
new file mode 100644
index 00000000..71fa6fe5
--- /dev/null
+++ b/square_primary_certificate_authority_g2.crt
@@ -0,0 +1,35 @@
+-----BEGIN CERTIFICATE-----
+MIIGEzCCA/ugAwIBAgIQbOEgSriEcg18Vw6n58DlXDANBgkqhkiG9w0BAQ0FADCB
+hDEyMDAGA1UEAwwpU3F1YXJlIFByaW1hcnkgQ2VydGlmaWNhdGUgQXV0aG9yaXR5
+IC0gRzIxFDASBgNVBAoMC1NxdWFyZSBJbmMuMRMwEQYDVQQIDApDYWxpZm9ybmlh
+MRYwFAYDVQQHDA1TYW4gRnJhbmNpc2NvMQswCQYDVQQGEwJVUzAeFw0xMjA1MDMx
+NzQ4NDVaFw0zNzA0MjcxNzQ4NDlaMIGEMTIwMAYDVQQDDClTcXVhcmUgUHJpbWFy
+eSBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjEUMBIGA1UECgwLU3F1YXJlIElu
+Yy4xEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28x
+CzAJBgNVBAYTAlVTMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1dK2
+hdAwnTyw+ja7Jm1r2ft3OOp4vX5ATIvt53OSUhxjWWZXmLYoIC2yXKg6ru0D6Pw+
+pc9pOHTXRNPRaCiMEZ+AzwdMHHgGpRZGTN+e5HUIVfYpJE9mcs1Q9KX/GWw05uAC
+xGyk7sbBqXSj/aJB6itydxhb2FXiRmn+QktYfjjjFgcPD2LK1YbuDRia7rkUoLLu
+EZQk0AkuNMNJ9TmdNFBsEb1wiZCYfYx1bF8d1adhgRLbeWLbV5aueVBeczkOcO3s
+XqvyoJSNq1AYx6T2xdDS2Pi2oN6//UnRk5h22M+ROfeVSGyIX7pAiJ8QojD0ajNn
+TfESy1FwQnHIadAxeqgkEZezr5NhzzgAczL0L0acXlve6sIPdaIREjn4XNxJS5W9
+WUdzqux64dVcGTLNshvArS/yLbCSC9M5t4M+OAI+FPt4wn8Uk3auEWvIi5jfWwwY
+nxitejb2kT2rKSb/ETji1lZorQEnlemqylEWfzG21P0weUa+ghKlYk+F2d5nJBly
+YakFgheSnzs9zoz1LISDBP1v8GY+xJJhZIKBA9np4HCGMUx5nXdj+ohXvN3oSzsU
+p6foRnaiRpY5vfIy3LITVdAZ3R/BJFUcjXXSH9ddI2tPyo470l+L5qTn2LAXGOzH
+wRaGDeFnJNU768PC7CckcqsiUkHouLnBNLz7g3UCAwEAAaN/MH0wDAYDVR0TBAUw
+AwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFH0MAs3/FcUHwggOwfwOFamH
+kHDbMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAfBgNVHSMEGDAWgBR9
+DALN/xXFB8IIDsH8DhWph5Bw2zANBgkqhkiG9w0BAQ0FAAOCAgEAKB8LQZme3mnV
+biFZknUt6htb/iSBYtT+45CDnM/q57R7ucme59CHiOAaAyVp8TlS5GfMLfdWaj2q
+wvdaEtLMunIco8qmJ52uQ7omi5SC0hqru8x1CcOdddiPeETeF5Ixg5Oa+uwsLyss
+ez+mkm0+jrUZaAQUd4Myhjb0qKZWv2gCr5pBoouwelThz4TTFW95Mm1zkqoxuiHi
+w/17w9ggiQA+cgb+/6cTv2HRdTTY36FLePATZa9hd6YDA8j7cDFdUmmMMeTTm7+1
+YKDgH2S44v2C28E4Kv9/ip5thRLirkiyIsXXF+And62Vm1O0q2CK7rsKyhJW4gSn
+8oTa7RAq2mABfFEIkVNjhNY3xE8Ij1D0FK7a5mo7G3xLJSRbcMvyqlQER4d32/uT
+/aN+xpyO8ug8QYLpoPaZDoa06L19IWkoz4CI152ZBOwPVjX38jenoNEpmvwroJ4Y
+Wr8VNy6xl95GjkZXp9lwYLByM5pt3S7+d2L9xUMZATC8tR4vgdK6GeUbj+O33IHT
+0Mx3+ZSh03C4/WdHKQPOWz0coBjMf9/1zrLrYSHmMXM/AishIzq8iPF6RtaFjDjQ
+fSJ/Zcz6z3KLFO4oHE6n5D284XuZoiCovHJT1NZGRPgBoQ5mlvTrLnX+DSufesLs
+8XVOleJ/4+k35H8d59s9vykPpVJxYa0=
+-----END CERTIFICATE-----
diff --git a/square_service_authority_g2.crt b/square_service_authority_g2.crt
new file mode 100644
index 00000000..aaf36811
--- /dev/null
+++ b/square_service_authority_g2.crt
@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIE6DCCAtCgAwIBAgIJAK17JkYW1k4SMA0GCSqGSIb3DQEBCwUAMIGEMTIwMAYD
+VQQDDClTcXVhcmUgUHJpbWFyeSBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjEU
+MBIGA1UECgwLU3F1YXJlIEluYy4xEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNV
+BAcMDVNhbiBGcmFuY2lzY28xCzAJBgNVBAYTAlVTMB4XDTE2MDQyMTIxMzczN1oX
+DTE5MTAyMzIxMzczN1owfTErMCkGA1UEAwwiU3F1YXJlIFNlcnZpY2UgQXV0aG9y
+aXR5IC0gRzIgMjAxNTEUMBIGA1UECgwLU3F1YXJlIEluYy4xEzARBgNVBAgMCkNh
+bGlmb3JuaWExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xCzAJBgNVBAYTAlVTMIIB
+IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1mZV8i8jLplpIqbIOT/CJ2Sw
+AvhoGx3OfrTGRMemGtKQ3Ne0qa7BeyLeIG5Hd2dv5IkNEdyNuP0zH73ji0SB+MdC
+RZPRa0h/1zRWdsD1FGMvhOP0yowMRKVMAHZTSMzCbJJW8qVFe61uNzWPXuxwSfy7
+gbjPcNRbR1o1AJPELuqNHfCEinL93mIvvLNM2u8w889IdQJVY9a0hQaKsIMNwh3G
+PcEoxsZbNPUjS6nseX1lAHFVd51qcU3zxpNXYxzH76Eqj/ltXgYmkggI7GZs6jDk
+9TvjwfU1hVoB+mCydXlhJAZkQIewnsDbHU+jH3Hx2vNvovTdGw6X+AhTI4NsAQID
+AQABo2MwYTAdBgNVHQ4EFgQUeNr6yOZXK2lPaoqKm8XiyNlZbAgwDwYDVR0TAQH/
+BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHwYDVR0jBBgwFoAUfQwCzf8VxQfCCA7B
+/A4VqYeQcNswDQYJKoZIhvcNAQELBQADggIBAIxUdCPzazg9rj9AxRy9kUdExc73
+/Wh5HecEECyQOWI/+wPZAyGFseMpGYT+4c7Y/hvBYYI2UHhD4+3knZmgh69wI9g2
+Drmzo5KSHK6aQVofPHdLVUPid+QfI2NvaJfv2KK2wJuBKR0JueTBBaml8L8jW/aA
+m2MhCW6X5PFS8rS8S5FtL1a2AHsHD31xgGM3/Ku7JJOuYcE0jpzlX3Tk/LzpchuV
+tSdJKBRbP14q/U/2IDzMLJ168plfbo7QFwkq7HOrgNMCJJJcRHoZZOjHv3NalPlw
+CM1QiM1uqVgCna33eA/W6PFNGn6iXS+W5Pdh3p7n1hMB6UTmnuB4l0n8eQuZU/SJ
+J14D6VNPVDfsa1y0rDbqx7WuJkVwDUK1JJhU7cdBHy4KjInMI7niaHdMXqt/Mn4A
+4Lmjg0W99zqda9gCVskKXlDne/9N2Z8LIhPePv9iRUCVV/riuy9Yhh3O7y+pYB+f
+A8GhlglC/B8Aps26YS8/Mlj/3l/uR1m/Hw73+h3IQCEWncwmX8TdCuHRpMbuZQLc
+OaYFMoikk+dBs32c4x4wD/FOoY/uBmFzTc6I96ZD58JKn3yG+0i0/fUMGsoBgRWt
+95o2Nq3peW86Htz8OjwHXEmwiU6LBKa2uBLUQ7d0Mwn+55Hku0EmkXiHMCR8F8cY
+mtqI2NT3nLrtwbSl
+-----END CERTIFICATE-----