Feature/j browse test

.gitignore

@@ -33,6 +33,6 @@ build/
 ### VS Code ###
 .vscode/
 
-
 ## others
-run.sh
+run.sh
+.DS_Store

Jenkinsfile.groovy

@@ -84,146 +84,69 @@ spec:
                 }
             }
         }
+
         stage('Test') {
             steps {
                 container('jdk') {
                     sh "./mvnw test package"
                 }
             }
         }
-        stage('Build & Publish Develop') {
-            when {
-                branch "develop"
-            }
-            steps {
-                container('docker') {
-                    withCredentials([usernamePassword(credentialsId:'OvertureDockerHub', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
-                        sh 'docker login -u $USERNAME -p $PASSWORD'
-                    }
-                    sh "docker build --network=host --target client -f  Dockerfile . -t overture/dms:edge -t overture/dms:${version}-${commit}"
-                    sh "docker build --network=host --target latest-version-helper -f Dockerfile . -t overture/dms-version-helper:edge"
-                    sh "docker push overture/dms:${version}-${commit}"
-                    sh "docker push overture/dms:edge"
-                    sh "docker push overture/dms-version-helper:edge"
-                }
-                container('docker') {
-                    withCredentials([usernamePassword(credentialsId:'OvertureBioGithub', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
-                        sh 'docker login ghcr.io -u $USERNAME -p $PASSWORD'
-                    }
-                    sh "docker build --network=host --target client -f  Dockerfile . -t ${dockerOrg}/${dmsRepo}:edge -t ${dockerOrg}/${dmsRepo}:${version}-${commit}"
-                    sh "docker build --network=host --target latest-version-helper -f Dockerfile . -t ${dockerOrg}/${dmsVersionHelperRepo}:edge"
-                    sh "docker build --target insecure --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}:edge"
-                    sh "docker build --target secure --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}-secure:edge"
-                    sh "docker push ${dockerOrg}/${dmsRepo}:${version}-${commit}"
-                    sh "docker push ${dockerOrg}/${dmsRepo}:edge"
-                    sh "docker push ${dockerOrg}/${dmsGatewayRepo}:edge"
-                    sh "docker push ${dockerOrg}/${dmsVersionHelperRepo}:edge"
-                }
-            }
-        }
-        stage('Release & tag') {
-          when {
-            branch "master"
-          }
-          steps {
-                container('docker') {
-                    withCredentials([usernamePassword(credentialsId: 'OvertureBioGithub', passwordVariable: 'GIT_PASSWORD', usernameVariable: 'GIT_USERNAME')]) {
-                        sh "git tag ${version}"
-                        sh "git push https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/overture-stack/dms --tags"
-                    }
-                    withCredentials([usernamePassword(credentialsId:'OvertureBioGithub', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
-                        sh 'docker login ghcr.io -u $USERNAME -p $PASSWORD'
-                    }
-                    sh "docker build --network=host --target client -f Dockerfile . -t ${dockerOrg}/${dmsRepo}:latest -t ${dockerOrg}/${dmsRepo}:${version}"
-                    sh "docker build --network=host --target latest-version-helper  -f Dockerfile . -t ${dockerOrg}/${dmsVersionHelperRepo}:latest"
-                    sh "docker build --target insecure --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}:latest -t ${dockerOrg}/${dmsGatewayRepo}:${version}"
-                    sh "docker build --target secure --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}-secure:latest -t ${dockerOrg}/${dmsGatewayRepo}-secure:${version}"
-                    sh "docker push ${dockerOrg}/${dmsRepo}:${version}"
-                    sh "docker push ${dockerOrg}/${dmsGatewayRepo}:${version}"
-                    sh "docker push ${dockerOrg}/${dmsGatewayRepo}-secure:${version}"
-                    sh "docker push ${dockerOrg}/${dmsGatewayRepo}:latest"
-                    sh "docker push ${dockerOrg}/${dmsGatewayRepo}-secure:latest"
-                    sh "docker push ${dockerOrg}/${dmsRepo}:latest"
-                    sh "docker push ${dockerOrg}/${dmsVersionHelperRepo}:latest"
-                }
-                container('docker') {
-                  withCredentials([usernamePassword(credentialsId:'OvertureDockerHub', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
-                      sh 'docker login -u $USERNAME -p $PASSWORD'
-                  }
-                  sh "docker build --network=host --target client -f Dockerfile . -t overture/dms:latest -t overture/dms:${version}"
-                  sh "docker build --network=host --target latest-version-helper  -f Dockerfile . -t overture/dms-version-helper:latest"
-                  sh "docker push overture/dms:${version}"
-                  sh "docker push overture/dms:latest"
-                  sh "docker push overture/dms-version-helper:latest"
-                }
-            }
-        }
 
-        stage('Destination SNAPSHOT') {
+        stage('Build images') {
             when {
                 anyOf {
                     branch 'develop'
-                    branch 'test-develop'
+                    branch 'main'
+                    branch 'feature/jbrowseIntegeration'
                 }
             }
             steps {
-                script {
-                    repo = "dcc-snapshot/bio/overture"
-                }
-            }
-        }
+                container('docker') {
+                    sh 'docker build \
+                        --target client \
+                        --network=host \
+                        -f Dockerfile \
+                        -t dms .'
 
-        stage('Destination release') {
-            when {
-                anyOf {
-                    branch 'master'
-                    branch 'test-master'
-                }
-            }
-            steps {
-                script {
-                    repo = "dcc-release/bio/overture"
+                    sh 'docker build \
+                        --target latest-version-helper \
+                        --network=host \
+                        -f Dockerfile \
+                        -t dms-version-helper .'
                 }
             }
         }
 
-        stage('Upload Artifacts') {
+        stage('Publish Images') {
             when {
-                anyOf {
-                    branch 'master'
-                    branch 'test-master'
-                    branch 'develop'
-                    branch 'test-develop'
-                }
+                branch 'feature/jbrowseIntegeration'
             }
             steps {
-                script {
-
-                    project = "dms"
-                    versionName = "$version"
-                    subProjects = ['cli']
-
-                    files = []
-                    files.add([pattern: "pom.xml", target: "$repo/$project/$versionName/$project-${versionName}.pom"])
+                container('docker') {
+                    withCredentials([usernamePassword(credentialsId:'OvertureBioDockerHub', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
+                        sh 'docker login -u $USERNAME -p $PASSWORD'
+                    }
+                    // still necessary because the helper script relies on the dockerhub api to get versions
+                    sh "docker tag dms overture/dms:jbrowse-${commit}"
+                    sh "docker push overture/dms:jbrowse-${commit}"
+                }
+                container('docker') {
+                    withCredentials([usernamePassword(credentialsId:'OvertureBioGithub', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
+                        sh 'docker login ghcr.io -u $USERNAME -p $PASSWORD'
+                    }
+                    sh "docker tag dms ${dockerOrg}/${dmsRepo}:jbrowse-${commit}"
+                    sh "docker push ${dockerOrg}/${dmsRepo}:jbrowse-${commit}"
 
-                    for (s in subProjects) {
-                        name = "${project}-$s"
-                        target = "$repo/$name/$versionName/$name-$versionName"
-                        files.add(pom(name, target))
-                        files.add(jar(name, target))
+                    sh "docker tag dms-version-helper ${dockerOrg}/${dmsVersionHelperRepo}:jbrowse-${commit}"
+                    sh "docker push ${dockerOrg}/${dmsVersionHelperRepo}:jbrowse-${commit}"
 
-                        if (s in ['cli']) {
-                            files.add(runjar(name, target))
-                            files.add(tar(name, target))
-                        }
-                    }
+                    sh "docker build --target insecure --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}:jbrowse-${commit}"
+                    sh "docker push ${dockerOrg}/${dmsGatewayRepo}:jbrowse-${commit}"
 
-                    fileSet = JsonOutput.toJson([files: files])
-                    pretty = JsonOutput.prettyPrint(fileSet)
-                    print("Uploading files=${pretty}")
+                    sh "docker build --target secure --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}-secure:jbrowse-${commit}"
+                    sh "docker push ${dockerOrg}/${dmsGatewayRepo}-secure:jbrowse-${commit}"
                 }
-
-                rtUpload(serverId: 'artifactory', spec: fileSet)
             }
         }
     }

src/main/bin/dms-docker

@@ -6,6 +6,7 @@ export DOCKER_EXE=$DEFAULT_DOCKER_EXE
 export CONFIG_FILENAME=config.yaml
 export TEMP_CONFIG_FILE=/tmp/$CONFIG_FILENAME
 
+export HELPER_VERSION=${HELPER_VERSION:-latest}
 export DMS_HOME=$HOME/.dms
 export DMS_CONFIG_FILE=$DMS_HOME/$CONFIG_FILENAME
 
@@ -25,6 +26,7 @@ function isVersionGTE {
 	min_patch_version=$(parsePatchVersion $min_version)
 
 	result=true
+
 	if [ $input_major_version -lt $min_major_version ];then
 		result=false
 	elif [ $input_major_version -eq $min_major_version ];then
@@ -36,6 +38,7 @@ function isVersionGTE {
 			fi
 		fi
 	fi
+
 	echo $result
 }
 
@@ -52,14 +55,16 @@ function parsePatchVersion {
 }
 
 function checkDocker {
-	# check docker exists: show instructions 
+	# check docker exists: show instructions
 	result_output=$($DOCKER_EXE version -f '{{ .Server.Version }}' 2>&1)
 	result_code=$?
+
 	if [ $result_code -ne 0  ]; then
 		echo "Could not execute the docker command. Message: $result_output"
 		exit 1
 	else
 		version_result=$(isVersionGTE $result_output $MIN_VERSION 2>&1)
+
 		if [ "$version_result" == "false" ]; then
 			echo "[ERROR]: The installed docker version $result_output is not >= $MIN_VERSION"
 			exit 1
@@ -71,9 +76,9 @@ function checkDocker {
 function setupDockerExecutable {
 	if [ ! -f $DEFAULT_DOCKER_EXE ]; then
 		result=$(which docker)
+
 		if [ $? -ne 0 ]; then
-			#TODO: rtisma -- add instructions
-			echo "Docker is not installed. Please refer to ...."
+			echo "Docker is not installed. Please refer to https://www.docker.com/get-started/"
 			exit 1
 		else
 			export DOCKER_EXE=$result
@@ -85,23 +90,29 @@ function setupDockerExecutable {
 
 function getLatestVersion {
 	docker_engine_arch=$($DOCKER_EXE version -f '{{ .Server.Arch }}' 2>&1)
+
 	if [ $? -ne 0 ]; then
 		echo "[ERROR]: Could not find the docker engine server architecture. Message: $docker_engine_arch"
 		exit 1
 	fi
-	$DOCKER_EXE run --rm -i overture/dms-version-helper:latest /tmp/run.sh $docker_engine_arch
+
+	# silicon architecture is filtered out otherwise
+	[ $docker_engine_arch == 'arm64' ] && docker_engine_arch=amd64;
+
+	$DOCKER_EXE run --rm -i -e ARCH=$docker_engine_arch -e DMS_TAG=$HELPER_VERSION ghcr.io/overture-stack/dms-version-helper:$HELPER_VERSION /tmp/run.sh
 }
 
 function genLatestConfig {
 	output_path=$1
 
-	$DOCKER_EXE pull overture/dms-version-helper:latest > /dev/null
-	result=$(getLatestVersion 2>&1)
+	result=$(getLatestVersion)
+
 	if [ $? -ne 0 ]; then
 		echo "[ERROR]: Could not find the latest dms version. Try again later."
 		echo "$result"
 		exit 1
 	fi
+
 	echo -n "version: $result" > $output_path
 }
 
@@ -124,6 +135,7 @@ function provisionConfig {
 		echo "[ERROR]:  The config file \"$DMS_CONFIG_FILE\" was not created"
 		exit 1
 	fi
+
 	dms_version=$(cat $DMS_CONFIG_FILE | grep "^version" | sed 's/.*://g' |  tr -d '[:space:]' | sed  's/"//g' 2>&1)
 
 	# Return the version
@@ -137,7 +149,7 @@ function main {
 	all_dms_params=$@
 
 	# Constants
-	repo_name="overture/dms"
+	repo_name="ghcr.io/overture-stack/dms"
 	volume_name="dms-config"
 	config_filename=$CONFIG_FILENAME
 
@@ -146,6 +158,7 @@ function main {
 
 	# Get dms version from config
 	dms_version=$(provisionConfig $config_filename 2>&1)
+
 	if [ $? -ne 0 ]; then
 		echo "[ERROR]: There was an error obtaining the dms version"
 		echo "$dms_version"
@@ -155,17 +168,21 @@ function main {
 	#################################
 	# Define the docker options
 	#################################
-	# This is done so that unix line endings are output. Otherwise, with the -t option, 
+	# This is done so that unix line endings are output. Otherwise, with the -t option,
 	# dos line endings are present and the script cannot be sourced
 	docker_opts=""
+
 	if [ "${DMS_DEBUG:-}" != "true" ]; then
 		docker_opts='--rm'
 	fi
+
 	docker_opts="$docker_opts -i --network=$NETWORK_NAME -e \"DOCKER_MODE=true\""
+
 	if [ "$command_name" != "bash-completion" ];then
 		# Output colour to the terminal
 		docker_opts="$docker_opts -t"
 	fi
+
 	docker_opts="$docker_opts -v /var/run/docker.sock:/var/run/docker.sock:ro"
 	docker_opts="$docker_opts -v /usr/bin/docker:/usr/bin/docker:ro"
 	docker_opts="$docker_opts -v $DMS_HOME:/root/.dms"
@@ -175,15 +192,19 @@ function main {
 		docker_opts="$docker_opts -p 5005:5005"
 		set -x
 	fi
+
 	docker_opts="$docker_opts -e DOCKER_RUNAS=true -e DOCKER_DMSHOMEHOSTPATH=$DMS_HOME -e DOCKER_TAG=$dms_version"
+
 	# create dms-swarm-network if it doesn't exist, this is important for first run
 	# since the dms uses that in its run options
 	found_network=$(docker network ls --filter name=${NETWORK_NAME} --format="{{ .Name }}")
+
 	if [ -z $found_network ] ; then
 		echo "creating the dms network"
 		$DOCKER_EXE network create -d overlay --attachable ${NETWORK_NAME} ;
 	fi
 	$DOCKER_EXE run $docker_opts $repo_name:$dms_version dms $all_dms_params
+
 	set +x
 }
 

src/main/bin/getLatestDockerTag.sh

@@ -1,18 +1,15 @@
 #!/bin/bash
-arch=$1
+arch=${1:-${ARCH:-amd64}}
+tag=${DMS_TAG:-latest}
 
 set -euo pipefail
 function getLatestDigest {
-	arch=$1
-	curl -sL https://hub.docker.com/v2/repositories/overture/dms/tags/latest | jq -r '.images[] | select(.architecture=="amd64") | .digest '
+	curl -sL https://hub.docker.com/v2/repositories/overture/dms/tags/$tag | jq -r ".images[] | select(.architecture==\"$arch\") | .digest"
 }
 
-
 digest=$(getLatestDigest $arch)
 curl -sL https://hub.docker.com/v2/repositories/overture/dms/tags \
 	| jq -r -e ".results[] | select(.name!=\"latest\") | select( any(.images[]; .architecture==\"$arch\" and .digest==\"$digest\")) | .name" \
 	| grep "^[0-9]\+\.[0-9]\+\.[0-9]\+$" \
 	| sort -V \
 	| tail -1
-
-

src/main/java/bio/overture/dms/cli/model/Constants.java

@@ -48,7 +48,7 @@ public static final class DockerImagesConstants {
     public static final String ES_TAG = "7.6.0";
     public static final String DMS_GATEWAY_TAG = "1.0.0";
     public static final String MAESTRO_TAG = "3.8.0";
-    public static final String DMS_UI_TAG = "1.0.3";
+    public static final String DMS_UI_TAG = "fa1420b";
   }
 
   public static final class MESSAGES {
@@ -165,6 +165,8 @@ public class DmsUiQuestions {
         "What is the Project ID you will configure in Arranger (to be referenced by DMS UI)?";
     public static final String PROJ_NAME =
         "What is the Project Name you will configure in Arranger (to be referenced by DMS UI)?";
+    public static final String DMS_VIZ_TOOL =
+            "Which optional visualization tools would you like to install?";
     public static final String ALIAS =
         "What is the Elasticsearch alias name you will configure in Arranger (to be referenced by DMS UI and ALSO must match the alias name previously supplied for Maestro) be? ";
     public static final String ARRANGER_QUESTIONS_NOTE =