Added ElasticSearch + Kibana, improved environment

Author: dmitriy

.circleci/config.yml

@@ -15,13 +15,16 @@ jobs:
             make start-test
 
       - run:
-          name: Wait for DB container is running and initialize DB & messenger component
+          name: Wait for DB container is running and initialize DB & messenger component / cron / elastic
           command: |
             make wait-for-db
             make drop-migrate
             make generate-jwt-keys
             make create-roles-groups
+            make migrate-cron-jobs
             make messenger-setup-transports
+            make wait-for-elastic
+            make elastic-create-or-update-template
 
       - run:
           name: Run unit/functional tests

.dockerignore

@@ -9,6 +9,7 @@
 /public/bundles/
 /var/mysql-data
 /var/rabbitmq
+/var/elasticsearch-data
 /vendor/
 /tools/*/vendor/
 

.env

@@ -13,6 +13,10 @@
 # Run "composer dump-env prod" to compile .env files for production use (requires symfony/flex >=1.2).
 # https://symfony.com/doc/current/best_practices.html#use-environment-variables-for-infrastructure-configuration
 
+###> docker-compose configuration ###
+COMPOSE_PROJECT_NAME=backend
+###< docker-compose configuration ###
+
 ###> symfony/framework-bundle ###
 APP_ENV=dev
 APP_DEBUG=1
@@ -30,13 +34,25 @@ DATABASE_URL=mysql://root:secret@mysql:3306/symfony
 ###< doctrine/doctrine-bundle ###
 
 ###> symfony/mailer ###
-# MAILER_DSN=smtp://localhost
+MAILER_DSN=smtp://user:pass@smtp.example.com
+APP_SENDER_EMAIL=admin@localhost
+APP_EMAIL_NOTIFICATION_ABOUT_ERROR=1
+APP_ERROR_RECEIVER_EMAIL=admin@localhost
 ###< symfony/mailer ###
 
 ###> symfony/messenger ###
 MESSENGER_TRANSPORT_DSN=amqp://guest:guest@rabbitmq:5672/%2f/messages
 ###< symfony/messenger ###
 
+###> Elasticsearch configuration ###
+ELK_VERSION=7.8.0
+ELASTICSEARCH_HOST=http://elasticsearch:9200
+ELASTICSEARCH_NUMBER_OF_SHARDS=1
+ELASTICSEARCH_NUMBER_OF_REPLICAS=0
+ELASTICSEARCH_USERNAME=elastic
+ELASTICSEARCH_PASSWORD=changeme
+###< Elasticsearch configuration ###
+
 ###> lexik/jwt-authentication-bundle ###
 JWT_SECRET_KEY=config/jwt/private.pem
 JWT_PUBLIC_KEY=config/jwt/public.pem
@@ -48,6 +64,9 @@ JWT_TOKEN_TTL=3600
 CORS_ALLOW_ORIGIN=^https?://localhost(:[0-9]+)?$
 ###< nelmio/cors-bundle ###
 
+# how many failure attempts for lock account, 0 - off
+LOCK_USER_ON_LOGIN_FAILURE_ATTEMPTS=0
+
 # how many days health history will be stored in the db
 DATABASE_HEALTH_HISTORY_DAYS=7
 

.env.prod

@@ -7,6 +7,21 @@ APP_DEBUG=0
 DATABASE_URL=mysql://root:secret@mysql:3306/symfony
 ###< doctrine/doctrine-bundle ###
 
+###> symfony/mailer ###
+MAILER_DSN=smtp://user:pass@smtp.example.com?encryption=tls
+APP_SENDER_EMAIL=admin@localhost
+APP_EMAIL_NOTIFICATION_ABOUT_ERROR=1
+APP_ERROR_RECEIVER_EMAIL=admin@localhost
+###< symfony/mailer ###
+
 ###> symfony/messenger ###
 MESSENGER_TRANSPORT_DSN=amqp://guest:guest@rabbitmq:5672/%2f/messages
 ###< symfony/messenger ###
+
+###> Elasticsearch configuration ###
+ELASTICSEARCH_HOST=http://elasticsearch:9200
+ELASTICSEARCH_NUMBER_OF_SHARDS=1
+ELASTICSEARCH_NUMBER_OF_REPLICAS=0
+ELASTICSEARCH_USERNAME=elastic
+ELASTICSEARCH_PASSWORD=changeme
+###< Elasticsearch configuration ###

.env.staging

@@ -2,3 +2,24 @@
 APP_ENV=staging
 APP_SECRET=42f011ec3a7bde0bec87364b1d967194
 APP_DEBUG=0
+
+###> doctrine/doctrine-bundle ###
+DATABASE_URL=mysql://root:secret@mysql:3306/symfony
+###< doctrine/doctrine-bundle ###
+
+###> symfony/mailer ###
+MAILER_DSN=smtp://user:pass@smtp.example.com?encryption=tls
+APP_SENDER_EMAIL=admin@localhost
+APP_EMAIL_NOTIFICATION_ABOUT_ERROR=1
+APP_ERROR_RECEIVER_EMAIL=admin@localhost
+###< symfony/mailer ###
+
+###> symfony/messenger ###
+MESSENGER_TRANSPORT_DSN=amqp://guest:guest@rabbitmq:5672/%2f/messages
+###< symfony/messenger ###
+
+###> Elasticsearch configuration ###
+ELASTICSEARCH_HOST=http://elasticsearch:9200
+ELASTICSEARCH_USERNAME=elastic
+ELASTICSEARCH_PASSWORD=changeme
+###< Elasticsearch configuration ###

.env.test

@@ -4,4 +4,7 @@ KERNEL_CLASS='App\Kernel'
 APP_SECRET='$ecretf0rt3st'
 APP_DEBUG=0
 SYMFONY_DEPRECATIONS_HELPER=999999
+
+###> doctrine/doctrine-bundle ###
 DATABASE_URL=mysql://root:secret@mysql:3306/symfony_testing
+###< doctrine/doctrine-bundle ###

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ jobs:
         steps:
             - uses: actions/checkout@v1
             - name: Build the docker images
-              run: docker-compose -f docker-compose-test-ci.yml build
+              run: make build-test
             - name: Start the docker images
               run: make start-test
             - name: Check running containers
@@ -34,8 +34,14 @@ jobs:
               run: make generate-jwt-keys
             - name: Create roles and groups
               run: make create-roles-groups
+            - name: Create cron jobs
+              run: make migrate-cron-jobs
             - name: Setup transports for Messenger component
               run: make messenger-setup-transports
+            - name: Wait for Elastic connection
+              run: make wait-for-elastic
+            - name: Create or update Elastic index template
+              run: make elastic-create-or-update-template
             - name: Run test suite
               run: make phpunit
             - name: Run coding standard

.gitignore

@@ -8,7 +8,8 @@ reports/*
 /.env.*.local
 /config/secrets/prod/prod.decrypt.private.php
 /public/bundles/
-/var/
+/var/*
+!/var/elasticsearch-data/nodes/0/.gitkeep
 /vendor/
 /tools/**/vendor
 ###< symfony/framework-bundle ###

.gitlab-ci.yml

@@ -23,14 +23,17 @@ stages:
 build:
   stage: build
   script:
-    - docker-compose -f docker-compose-test-ci.yml build
+    - make build-test
     - make start-test
     - docker ps -a
     - make wait-for-db
     - make drop-migrate
     - make generate-jwt-keys
     - make create-roles-groups
+    - make migrate-cron-jobs
     - make messenger-setup-transports
+    - make wait-for-elastic
+    - make elastic-create-or-update-template
     - *general_scripts
     - make phpunit
     - make ecs
@@ -48,18 +51,19 @@ build:
 push_staging_images:
   stage: deploy
   script:
-    - docker-compose -f docker-compose-staging.yml build
+    - make build-staging
     # TODO: set necessary image name in docker-compose-staging.yml according to your registry and edit lines bellow
     #- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
     #- docker-compose -f docker-compose-staging.yml push
   only:
     - master
     - develop
+    - /^release.*$/
 
 push_prod_images:
   stage: deploy
   script:
-    - docker-compose -f docker-compose-prod.yml build
+    - make build-prod
     # TODO: set necessary image name in docker-compose-prod.yml according to your registry and edit lines bellow
     #- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
     #- docker-compose -f docker-compose-prod.yml push

Makefile

@@ -1,6 +1,15 @@
 dir=${CURDIR}
-project=-p symfony
-service=symfony:latest
+
+ifndef APP_ENV
+	include .env
+	# Determine if .env.local file exist
+	ifneq ("$(wildcard .env.local)","")
+		include .env.local
+	endif
+endif
+
+project=-p ${COMPOSE_PROJECT_NAME}
+service=${COMPOSE_PROJECT_NAME}:latest
 openssl_bin:=$(shell which openssl)
 interactive:=$(shell [ -t 0 ] && echo 1)
 ifneq ($(interactive),1)
@@ -11,13 +20,17 @@ ifeq ($(GITLAB_CI),1)
 	phpunitOptions=--coverage-text --colors=never
 endif
 
-ifndef APP_ENV
-	include .env
-	# Determine if .env.local file exist
-	ifneq ("$(wildcard .env.local)","")
-		include .env.local
-	endif
-endif
+build:
+	@docker-compose -f docker-compose.yml build
+
+build-test:
+	@docker-compose -f docker-compose-test-ci.yml build
+
+build-staging:
+	@docker-compose -f docker-compose-staging.yml build
+
+build-prod:
+	@docker-compose -f docker-compose-prod.yml build
 
 start:
 	@docker-compose -f docker-compose.yml $(project) up -d
@@ -88,6 +101,12 @@ ssh-mysql:
 ssh-rabbitmq:
 	@docker-compose $(project) exec rabbitmq /bin/sh
 
+ssh-elasticsearch:
+	@docker-compose $(project) exec elasticsearch bash
+
+ssh-kibana:
+	@docker-compose $(project) exec kibana bash
+
 exec:
 	@docker-compose $(project) exec $(optionT) symfony $$cmd
 
@@ -103,6 +122,9 @@ report-clean:
 wait-for-db:
 	@make exec cmd="php bin/console db:wait"
 
+wait-for-elastic:
+	@make exec cmd="php bin/console elastic:wait"
+
 composer-install-no-dev:
 	@make exec-bash cmd="COMPOSER_MEMORY_LIMIT=-1 composer install --optimize-autoloader --no-dev"
 
@@ -117,19 +139,25 @@ info:
 	@make exec cmd="php --version"
 
 logs:
-	@docker logs -f symfony
+	@docker logs -f ${COMPOSE_PROJECT_NAME}_symfony
 
 logs-nginx:
-	@docker logs -f nginx
+	@docker logs -f ${COMPOSE_PROJECT_NAME}_nginx
 
 logs-supervisord:
-	@docker logs -f supervisord
+	@docker logs -f ${COMPOSE_PROJECT_NAME}_supervisord
 
 logs-mysql:
-	@docker logs -f mysql
+	@docker logs -f ${COMPOSE_PROJECT_NAME}_mysql
 
 logs-rabbitmq:
-	@docker logs -f rabbitmq
+	@docker logs -f ${COMPOSE_PROJECT_NAME}_rabbitmq
+
+logs-elasticsearch:
+	@docker logs -f ${COMPOSE_PROJECT_NAME}_elasticsearch
+
+logs-kibana:
+	@docker logs -f ${COMPOSE_PROJECT_NAME}_kibana
 
 drop-migrate:
 	@make exec cmd="php bin/console doctrine:schema:drop --full-database --force"
@@ -155,6 +183,9 @@ create-roles-groups:
 messenger-setup-transports:
 	@make exec cmd="php bin/console messenger:setup-transports"
 
+elastic-create-or-update-template:
+	@make exec cmd="php bin/console elastic:create-or-update-template"
+
 phpunit:
 	@make exec-bash cmd="rm -rf ./var/cache/test* && bin/console cache:warmup --env=test && ./vendor/bin/phpunit -c phpunit.xml.dist --coverage-html reports/coverage $(phpunitOptions) --coverage-clover reports/clover.xml --log-junit reports/junit.xml"