Added backend API

projects/web-scraper/.dockerignore

@@ -0,0 +1,6 @@
+node_modules
+coverage
+.nyc_output
+mongo
+.vscode
+logs

projects/web-scraper/.eslintignore

@@ -0,0 +1,3 @@
+# ignore these files
+test.js
+test/*

projects/web-scraper/.gitignore

@@ -1,2 +1,17 @@
 ./output/**
-./node_modules/**
+./node_modules/**
+node_modules
+coverage
+.nyc_output
+mongo
+.vscode
+logs
+variables.env
+prod.variables.env
+.env
+test.variables.env
+
+# semaphore secrets
+.semaphore/docker.secrets.yml
+.semaphore/kops.secrets.yml
+.semaphore/ssh.secrets.yml

projects/web-scraper/.semaphore/create-kubeconfig.secrets.bash

@@ -0,0 +1,6 @@
+KUBECONFIG=${HOME}/${KOPS_CLUSTER_NAME}-kubeconfig.yaml kops export kubecfg --name ${KOPS_CLUSTER_NAME} --state ${KOPS_STATE_STORE}
+
+sem create secret kubeconfig \
+  --file ${HOME}/${KOPS_CLUSTER_NAME}-kubeconfig.yaml:/home/semaphore/.kube/config
+
+sem get secrets kubeconfig

projects/web-scraper/.semaphore/deploy-prod.yml

@@ -0,0 +1,25 @@
+version: v1.0
+name: Deploy to Production K8S Cluster
+agent:
+  machine:
+    type: e1-standard-2
+    os_image: ubuntu1804
+
+blocks:
+  - name: "Deploy"
+    task:
+      jobs:
+        - name: kubectl apply  
+          commands:            
+            - checkout
+            - export GIT_HASH=$(git log --format=format:'%h' -1)
+            - export TAG=${SEMAPHORE_GIT_BRANCH}_${GIT_HASH}_${SEMAPHORE_WORKFLOW_ID}
+            - docker pull $CLIENT_IMAGE:$TAG            
+            - docker pull $API_IMAGE:$TAG
+            - envsubst '${TAG}' <api-deployment.yaml > api-deployment.prod.yaml
+            - envsubst '${TAG}' <client-deployment.yaml > client-deployment.prod.yaml
+            - kubectl apply -f api-deployment.prod.yaml
+            - kubectl apply -f client-deployment.prod.yaml
+      secrets:
+        - name: kubeconfig
+        - name: prod-variables

projects/web-scraper/.semaphore/native-integration.yml

@@ -0,0 +1,58 @@
+version: v1.0
+name: Native Integration
+agent:
+  machine:
+    type: e1-standard-2
+    os_image: ubuntu1804
+
+blocks:
+
+  # Native
+
+  - name: "Smoke Tests"
+    task:
+      jobs:
+        - name: curl /api
+          commands:
+            - checkout
+
+            - docker run -d -p 27017:27017 mongo:4.0
+            - mkdir docker_images
+            - docker save mongo:4.0 -o docker_images/mongo:4.0.tar
+            - cache restore node_modules-$SEMAPHORE_GIT_BRANCH-$(checksum package-lock.json),node_modules-$SEMAPHORE_GIT_BRANCH-,node_modules
+            - cache restore docker-images-$SEMAPHORE_WORKFLOW_ID
+
+            - npm i
+            - npm start &
+            - ./smoke_test.sh localhost:3000/api
+
+            - cache store node_modules-$SEMAPHORE_GIT_BRANCH-$(checksum package-lock.json) node_modules
+            - cache store docker-images-$SEMAPHORE_WORKFLOW_ID docker_images
+
+  - name: "Unit & Integration Tests"
+    task:
+      jobs:
+        - name: npm run lint
+          commands:
+            - checkout
+            - cache restore node_modules-$SEMAPHORE_GIT_BRANCH-$(checksum package-lock.json),node_modules-
+            - npm i
+            - npm run lint
+        - name: npm run test-unit
+          commands:
+            - checkout
+            - cache restore docker-images-$SEMAPHORE_WORKFLOW_ID
+            - docker load -i docker_images/mongo:4.0.tar
+            - docker run -d -p 27017:27017 mongo:4.0
+            - cache restore node_modules-$SEMAPHORE_GIT_BRANCH-$(checksum package-lock.json),node_modules-
+            - npm i
+            - npm run test-unit
+        - name: npm run test-integration
+          commands:
+            - checkout
+            - cache restore docker-images-$SEMAPHORE_WORKFLOW_ID
+            - docker load -i docker_images/mongo:4.0.tar
+            - docker run -d -p 27017:27017 mongo:4.0
+            - cache restore node_modules-$SEMAPHORE_GIT_BRANCH-$(checksum package-lock.json),node_modules-
+            - npm i
+            - npm run test-integration

projects/web-scraper/.semaphore/prod.variables.secrets.yml

@@ -0,0 +1,18 @@
+apiVersion: v1beta
+kind: Secret
+metadata:
+  name: prod-variables
+data:
+  env_vars:
+    - name: SECRET
+      value: secretkey
+    - name: NODE_ENV
+      value: prod
+    - name: DB
+      value: mongodb://mongo:27017/boilerplate_api
+    - name: PORT
+      value: 3000
+    - name: API_IMAGE
+      value: adnanrahic/boilerplate-api
+    - name: CLIENT_IMAGE
+      value: adnanrahic/smoketest

projects/web-scraper/.semaphore/push-images.yml

@@ -0,0 +1,36 @@
+version: v1.0
+name: Push images to Docker Hub
+agent:
+  machine:
+    type: e1-standard-2
+    os_image: ubuntu1804
+
+blocks:
+  - name: "Push Images"
+    task:
+      jobs:
+        - name: docker push images  
+          commands:
+            - checkout
+            - export GIT_HASH=$(git log --format=format:'%h' -1)
+            - export TAG=${SEMAPHORE_GIT_BRANCH}_${GIT_HASH}_${SEMAPHORE_WORKFLOW_ID}
+            - docker login -u $DOCKER_USER -p $DOCKER_PASS
+            - cache restore docker-images-$SEMAPHORE_WORKFLOW_ID
+            - ./build_image_if_not_exists.sh
+            - echo "Push API image"
+            - docker push $API_IMAGE:$TAG
+            - echo "Push CLIENT image"
+            - docker push $CLIENT_IMAGE:$TAG
+            - docker images
+            - cache delete docker-images-$SEMAPHORE_WORKFLOW_ID
+      secrets:
+        - name: docker-secrets
+        - name: prod-variables
+
+promotions:
+  - name: Deploy Production
+    pipeline_file: deploy-prod.yml
+    auto_promote_on:
+      - result: passed
+        branch:
+          - master

projects/web-scraper/.semaphore/semaphore.yml

@@ -0,0 +1,100 @@
+version: v1.0
+name: Build & Test
+agent:
+  machine:
+    type: e1-standard-2
+    os_image: ubuntu1804
+
+blocks:
+
+  # Docker only
+  - name: "Build and cache images"
+    task:
+      jobs:
+        - name: docker-compose build && cache store
+          commands:
+            - checkout
+            - export GIT_HASH=$(git log --format=format:'%h' -1)
+            - export TAG=${SEMAPHORE_GIT_BRANCH}_${GIT_HASH}_${SEMAPHORE_WORKFLOW_ID}
+            - docker pull mongo:4.0
+            - ./generate-node-env.sh
+            - docker-compose build
+
+            - mkdir docker_images
+            - docker save $API_IMAGE:$TAG -o docker_images/api_image_${TAG}.tar
+            - docker save $CLIENT_IMAGE:$TAG -o docker_images/client_image_${TAG}.tar
+            - docker save mongo:4.0 -o docker_images/mongo_4.0.tar
+
+            - cache store docker-images-$SEMAPHORE_WORKFLOW_ID docker_images
+      secrets:
+        - name: prod-variables
+
+  - name: "Smoke tests"
+    task:
+      jobs:
+        - name: CURL /api
+          commands:
+            - checkout
+            - export GIT_HASH=$(git log --format=format:'%h' -1)
+            - export TAG=${SEMAPHORE_GIT_BRANCH}_${GIT_HASH}_${SEMAPHORE_WORKFLOW_ID}
+            - cache restore docker-images-$SEMAPHORE_WORKFLOW_ID
+            - ls -l docker_images
+            - docker load -i docker_images/api_image_${TAG}.tar
+            - docker load -i docker_images/client_image_${TAG}.tar
+            - docker load -i docker_images/mongo_4.0.tar
+            - docker images
+            - ./generate-node-env.sh
+            - docker-compose up -d --build
+            - sleep 1
+            - ./smoke_test.sh localhost:3000/api
+            - ./smoke_test.sh localhost
+            - ./smoke_test.sh localhost/api
+      secrets:
+        - name: prod-variables
+
+  - name: "Unit & Integration tests"
+    task:
+      jobs:
+        - name: npm run lint
+          commands:
+            - checkout
+            - cache restore node_modules-$SEMAPHORE_GIT_BRANCH-$(checksum package-lock.json),node_modules-$SEMAPHORE_GIT_BRANCH-,node_modules
+            - npm i
+            - npm run lint
+            - cache store node_modules-$SEMAPHORE_GIT_BRANCH-$(checksum package-lock.json) node_modules
+        - name: npm run test-unit
+          commands:
+            - checkout
+            - export GIT_HASH=$(git log --format=format:'%h' -1)
+            - export TAG=${SEMAPHORE_GIT_BRANCH}_${GIT_HASH}_${SEMAPHORE_WORKFLOW_ID}
+            - cache restore docker-images-$SEMAPHORE_WORKFLOW_ID
+            - ls -l docker_images
+            - docker load -i docker_images/api_image_${TAG}.tar
+            - docker load -i docker_images/client_image_${TAG}.tar
+            - docker load -i docker_images/mongo_4.0.tar
+            - docker images
+            - ./generate-node-env.sh
+            - docker-compose build
+            - docker-compose run api node test-unit.js bash
+        - name: npm run test-integration
+          commands:
+            - checkout
+            - export GIT_HASH=$(git log --format=format:'%h' -1)
+            - export TAG=${SEMAPHORE_GIT_BRANCH}_${GIT_HASH}_${SEMAPHORE_WORKFLOW_ID}
+            - cache restore docker-images-$SEMAPHORE_WORKFLOW_ID
+            - ls -l docker_images
+            - docker load -i docker_images/api_image_${TAG}.tar
+            - docker load -i docker_images/client_image_${TAG}.tar
+            - docker load -i docker_images/mongo_4.0.tar
+            - docker images
+            - ./generate-node-env.sh
+            - docker-compose build
+            - docker-compose run api node test-integration.js bash
+      secrets:
+        - name: test-variables
+
+promotions:
+  - name: Push Images
+    pipeline_file: push-images.yml
+    auto_promote_on:
+      - result: passed

projects/web-scraper/.semaphore/test.variables.secrets.yml

@@ -0,0 +1,18 @@
+apiVersion: v1beta
+kind: Secret
+metadata:
+  name: test-variables
+data:
+  env_vars:
+    - name: SECRET
+      value: secretkey
+    - name: NODE_ENV
+      value: test
+    - name: DB
+      value: mongodb://mongo:27017/boilerplate_api_test
+    - name: PORT
+      value: 3000
+    - name: API_IMAGE
+      value: adnanrahic/boilerplate-api
+    - name: CLIENT_IMAGE
+      value: adnanrahic/smoketest

projects/web-scraper/CONTRIBUTING.md

@@ -0,0 +1,90 @@
+# Contributing
+
+When contributing to this repository, please first discuss the change you wish to make via issue,
+email, or any other method with the owners of this repository before making a change. 
+
+Please note we have a code of conduct, please follow it in all your interactions with the project.
+
+## Pull Request Process
+
+1. Ensure any install or build dependencies are removed before the end of the layer when doing a 
+   build.
+2. Update the README.md with details of changes to the interface, this includes new environment 
+   variables, exposed ports, useful file locations and container parameters.
+3. You may merge the Pull Request in once you have the sign-off of one other developer, or if you 
+   do not have permission to do that, you may request the second reviewer to merge it for you.
+
+## Code of Conduct
+
+### Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+### Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+### Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+### Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+### Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at [INSERT EMAIL ADDRESS]. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+### Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/