diff --git a/.gitignore b/.gitignore index 0abed45..510ed8c 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,6 @@ build/ ### VS Code ### .vscode/ - ## others -run.sh \ No newline at end of file +run.sh +.DS_Store diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 680adec..c33cd98 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -84,6 +84,7 @@ spec: } } } + stage('Test') { steps { container('jdk') { @@ -91,139 +92,61 @@ spec: } } } - stage('Build & Publish Develop') { - when { - branch "develop" - } - steps { - container('docker') { - withCredentials([usernamePassword(credentialsId:'OvertureDockerHub', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh 'docker login -u $USERNAME -p $PASSWORD' - } - sh "docker build --network=host --target client -f Dockerfile . -t overture/dms:edge -t overture/dms:${version}-${commit}" - sh "docker build --network=host --target latest-version-helper -f Dockerfile . -t overture/dms-version-helper:edge" - sh "docker push overture/dms:${version}-${commit}" - sh "docker push overture/dms:edge" - sh "docker push overture/dms-version-helper:edge" - } - container('docker') { - withCredentials([usernamePassword(credentialsId:'OvertureBioGithub', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh 'docker login ghcr.io -u $USERNAME -p $PASSWORD' - } - sh "docker build --network=host --target client -f Dockerfile . -t ${dockerOrg}/${dmsRepo}:edge -t ${dockerOrg}/${dmsRepo}:${version}-${commit}" - sh "docker build --network=host --target latest-version-helper -f Dockerfile . -t ${dockerOrg}/${dmsVersionHelperRepo}:edge" - sh "docker build --target insecure --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}:edge" - sh "docker build --target secure --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}-secure:edge" - sh "docker push ${dockerOrg}/${dmsRepo}:${version}-${commit}" - sh "docker push ${dockerOrg}/${dmsRepo}:edge" - sh "docker push ${dockerOrg}/${dmsGatewayRepo}:edge" - sh "docker push ${dockerOrg}/${dmsVersionHelperRepo}:edge" - } - } - } - stage('Release & tag') { - when { - branch "master" - } - steps { - container('docker') { - withCredentials([usernamePassword(credentialsId: 'OvertureBioGithub', passwordVariable: 'GIT_PASSWORD', usernameVariable: 'GIT_USERNAME')]) { - sh "git tag ${version}" - sh "git push https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/overture-stack/dms --tags" - } - withCredentials([usernamePassword(credentialsId:'OvertureBioGithub', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh 'docker login ghcr.io -u $USERNAME -p $PASSWORD' - } - sh "docker build --network=host --target client -f Dockerfile . -t ${dockerOrg}/${dmsRepo}:latest -t ${dockerOrg}/${dmsRepo}:${version}" - sh "docker build --network=host --target latest-version-helper -f Dockerfile . -t ${dockerOrg}/${dmsVersionHelperRepo}:latest" - sh "docker build --target insecure --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}:latest -t ${dockerOrg}/${dmsGatewayRepo}:${version}" - sh "docker build --target secure --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}-secure:latest -t ${dockerOrg}/${dmsGatewayRepo}-secure:${version}" - sh "docker push ${dockerOrg}/${dmsRepo}:${version}" - sh "docker push ${dockerOrg}/${dmsGatewayRepo}:${version}" - sh "docker push ${dockerOrg}/${dmsGatewayRepo}-secure:${version}" - sh "docker push ${dockerOrg}/${dmsGatewayRepo}:latest" - sh "docker push ${dockerOrg}/${dmsGatewayRepo}-secure:latest" - sh "docker push ${dockerOrg}/${dmsRepo}:latest" - sh "docker push ${dockerOrg}/${dmsVersionHelperRepo}:latest" - } - container('docker') { - withCredentials([usernamePassword(credentialsId:'OvertureDockerHub', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh 'docker login -u $USERNAME -p $PASSWORD' - } - sh "docker build --network=host --target client -f Dockerfile . -t overture/dms:latest -t overture/dms:${version}" - sh "docker build --network=host --target latest-version-helper -f Dockerfile . -t overture/dms-version-helper:latest" - sh "docker push overture/dms:${version}" - sh "docker push overture/dms:latest" - sh "docker push overture/dms-version-helper:latest" - } - } - } - stage('Destination SNAPSHOT') { + stage('Build images') { when { anyOf { branch 'develop' - branch 'test-develop' + branch 'main' + branch 'feature/jbrowseIntegeration' } } steps { - script { - repo = "dcc-snapshot/bio/overture" - } - } - } + container('docker') { + sh 'docker build \ + --target client \ + --network=host \ + -f Dockerfile \ + -t dms .' - stage('Destination release') { - when { - anyOf { - branch 'master' - branch 'test-master' - } - } - steps { - script { - repo = "dcc-release/bio/overture" + sh 'docker build \ + --target latest-version-helper \ + --network=host \ + -f Dockerfile \ + -t dms-version-helper .' } } } - stage('Upload Artifacts') { + stage('Publish Images') { when { - anyOf { - branch 'master' - branch 'test-master' - branch 'develop' - branch 'test-develop' - } + branch 'feature/jbrowseIntegeration' } steps { - script { - - project = "dms" - versionName = "$version" - subProjects = ['cli'] - - files = [] - files.add([pattern: "pom.xml", target: "$repo/$project/$versionName/$project-${versionName}.pom"]) + container('docker') { + withCredentials([usernamePassword(credentialsId:'OvertureBioDockerHub', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + sh 'docker login -u $USERNAME -p $PASSWORD' + } + // still necessary because the helper script relies on the dockerhub api to get versions + sh "docker tag dms overture/dms:jbrowse-${commit}" + sh "docker push overture/dms:jbrowse-${commit}" + } + container('docker') { + withCredentials([usernamePassword(credentialsId:'OvertureBioGithub', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + sh 'docker login ghcr.io -u $USERNAME -p $PASSWORD' + } + sh "docker tag dms ${dockerOrg}/${dmsRepo}:jbrowse-${commit}" + sh "docker push ${dockerOrg}/${dmsRepo}:jbrowse-${commit}" - for (s in subProjects) { - name = "${project}-$s" - target = "$repo/$name/$versionName/$name-$versionName" - files.add(pom(name, target)) - files.add(jar(name, target)) + sh "docker tag dms-version-helper ${dockerOrg}/${dmsVersionHelperRepo}:jbrowse-${commit}" + sh "docker push ${dockerOrg}/${dmsVersionHelperRepo}:jbrowse-${commit}" - if (s in ['cli']) { - files.add(runjar(name, target)) - files.add(tar(name, target)) - } - } + sh "docker build --target insecure --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}:jbrowse-${commit}" + sh "docker push ${dockerOrg}/${dmsGatewayRepo}:jbrowse-${commit}" - fileSet = JsonOutput.toJson([files: files]) - pretty = JsonOutput.prettyPrint(fileSet) - print("Uploading files=${pretty}") + sh "docker build --target secure --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}-secure:jbrowse-${commit}" + sh "docker push ${dockerOrg}/${dmsGatewayRepo}-secure:jbrowse-${commit}" } - - rtUpload(serverId: 'artifactory', spec: fileSet) } } } diff --git a/src/main/bin/dms-docker b/src/main/bin/dms-docker index 198a22e..01ac6ba 100755 --- a/src/main/bin/dms-docker +++ b/src/main/bin/dms-docker @@ -6,6 +6,7 @@ export DOCKER_EXE=$DEFAULT_DOCKER_EXE export CONFIG_FILENAME=config.yaml export TEMP_CONFIG_FILE=/tmp/$CONFIG_FILENAME +export HELPER_VERSION=${HELPER_VERSION:-latest} export DMS_HOME=$HOME/.dms export DMS_CONFIG_FILE=$DMS_HOME/$CONFIG_FILENAME @@ -25,6 +26,7 @@ function isVersionGTE { min_patch_version=$(parsePatchVersion $min_version) result=true + if [ $input_major_version -lt $min_major_version ];then result=false elif [ $input_major_version -eq $min_major_version ];then @@ -36,6 +38,7 @@ function isVersionGTE { fi fi fi + echo $result } @@ -52,14 +55,16 @@ function parsePatchVersion { } function checkDocker { - # check docker exists: show instructions + # check docker exists: show instructions result_output=$($DOCKER_EXE version -f '{{ .Server.Version }}' 2>&1) result_code=$? + if [ $result_code -ne 0 ]; then echo "Could not execute the docker command. Message: $result_output" exit 1 else version_result=$(isVersionGTE $result_output $MIN_VERSION 2>&1) + if [ "$version_result" == "false" ]; then echo "[ERROR]: The installed docker version $result_output is not >= $MIN_VERSION" exit 1 @@ -71,9 +76,9 @@ function checkDocker { function setupDockerExecutable { if [ ! -f $DEFAULT_DOCKER_EXE ]; then result=$(which docker) + if [ $? -ne 0 ]; then - #TODO: rtisma -- add instructions - echo "Docker is not installed. Please refer to ...." + echo "Docker is not installed. Please refer to https://www.docker.com/get-started/" exit 1 else export DOCKER_EXE=$result @@ -85,23 +90,29 @@ function setupDockerExecutable { function getLatestVersion { docker_engine_arch=$($DOCKER_EXE version -f '{{ .Server.Arch }}' 2>&1) + if [ $? -ne 0 ]; then echo "[ERROR]: Could not find the docker engine server architecture. Message: $docker_engine_arch" exit 1 fi - $DOCKER_EXE run --rm -i overture/dms-version-helper:latest /tmp/run.sh $docker_engine_arch + + # silicon architecture is filtered out otherwise + [ $docker_engine_arch == 'arm64' ] && docker_engine_arch=amd64; + + $DOCKER_EXE run --rm -i -e ARCH=$docker_engine_arch -e DMS_TAG=$HELPER_VERSION ghcr.io/overture-stack/dms-version-helper:$HELPER_VERSION /tmp/run.sh } function genLatestConfig { output_path=$1 - $DOCKER_EXE pull overture/dms-version-helper:latest > /dev/null - result=$(getLatestVersion 2>&1) + result=$(getLatestVersion) + if [ $? -ne 0 ]; then echo "[ERROR]: Could not find the latest dms version. Try again later." echo "$result" exit 1 fi + echo -n "version: $result" > $output_path } @@ -124,6 +135,7 @@ function provisionConfig { echo "[ERROR]: The config file \"$DMS_CONFIG_FILE\" was not created" exit 1 fi + dms_version=$(cat $DMS_CONFIG_FILE | grep "^version" | sed 's/.*://g' | tr -d '[:space:]' | sed 's/"//g' 2>&1) # Return the version @@ -137,7 +149,7 @@ function main { all_dms_params=$@ # Constants - repo_name="overture/dms" + repo_name="ghcr.io/overture-stack/dms" volume_name="dms-config" config_filename=$CONFIG_FILENAME @@ -146,6 +158,7 @@ function main { # Get dms version from config dms_version=$(provisionConfig $config_filename 2>&1) + if [ $? -ne 0 ]; then echo "[ERROR]: There was an error obtaining the dms version" echo "$dms_version" @@ -155,17 +168,21 @@ function main { ################################# # Define the docker options ################################# - # This is done so that unix line endings are output. Otherwise, with the -t option, + # This is done so that unix line endings are output. Otherwise, with the -t option, # dos line endings are present and the script cannot be sourced docker_opts="" + if [ "${DMS_DEBUG:-}" != "true" ]; then docker_opts='--rm' fi + docker_opts="$docker_opts -i --network=$NETWORK_NAME -e \"DOCKER_MODE=true\"" + if [ "$command_name" != "bash-completion" ];then # Output colour to the terminal docker_opts="$docker_opts -t" fi + docker_opts="$docker_opts -v /var/run/docker.sock:/var/run/docker.sock:ro" docker_opts="$docker_opts -v /usr/bin/docker:/usr/bin/docker:ro" docker_opts="$docker_opts -v $DMS_HOME:/root/.dms" @@ -175,15 +192,19 @@ function main { docker_opts="$docker_opts -p 5005:5005" set -x fi + docker_opts="$docker_opts -e DOCKER_RUNAS=true -e DOCKER_DMSHOMEHOSTPATH=$DMS_HOME -e DOCKER_TAG=$dms_version" + # create dms-swarm-network if it doesn't exist, this is important for first run # since the dms uses that in its run options found_network=$(docker network ls --filter name=${NETWORK_NAME} --format="{{ .Name }}") + if [ -z $found_network ] ; then echo "creating the dms network" $DOCKER_EXE network create -d overlay --attachable ${NETWORK_NAME} ; fi $DOCKER_EXE run $docker_opts $repo_name:$dms_version dms $all_dms_params + set +x } diff --git a/src/main/bin/getLatestDockerTag.sh b/src/main/bin/getLatestDockerTag.sh index 56331b2..a258aa0 100755 --- a/src/main/bin/getLatestDockerTag.sh +++ b/src/main/bin/getLatestDockerTag.sh @@ -1,18 +1,15 @@ #!/bin/bash -arch=$1 +arch=${1:-${ARCH:-amd64}} +tag=${DMS_TAG:-latest} set -euo pipefail function getLatestDigest { - arch=$1 - curl -sL https://hub.docker.com/v2/repositories/overture/dms/tags/latest | jq -r '.images[] | select(.architecture=="amd64") | .digest ' + curl -sL https://hub.docker.com/v2/repositories/overture/dms/tags/$tag | jq -r ".images[] | select(.architecture==\"$arch\") | .digest" } - digest=$(getLatestDigest $arch) curl -sL https://hub.docker.com/v2/repositories/overture/dms/tags \ | jq -r -e ".results[] | select(.name!=\"latest\") | select( any(.images[]; .architecture==\"$arch\" and .digest==\"$digest\")) | .name" \ | grep "^[0-9]\+\.[0-9]\+\.[0-9]\+$" \ | sort -V \ | tail -1 - - diff --git a/src/main/java/bio/overture/dms/cli/model/Constants.java b/src/main/java/bio/overture/dms/cli/model/Constants.java index 856e62e..d267436 100644 --- a/src/main/java/bio/overture/dms/cli/model/Constants.java +++ b/src/main/java/bio/overture/dms/cli/model/Constants.java @@ -48,7 +48,7 @@ public static final class DockerImagesConstants { public static final String ES_TAG = "7.6.0"; public static final String DMS_GATEWAY_TAG = "1.0.0"; public static final String MAESTRO_TAG = "3.8.0"; - public static final String DMS_UI_TAG = "1.0.3"; + public static final String DMS_UI_TAG = "fa1420b"; } public static final class MESSAGES { @@ -165,6 +165,8 @@ public class DmsUiQuestions { "What is the Project ID you will configure in Arranger (to be referenced by DMS UI)?"; public static final String PROJ_NAME = "What is the Project Name you will configure in Arranger (to be referenced by DMS UI)?"; + public static final String DMS_VIZ_TOOL = + "Which optional visualization tools would you like to install?"; public static final String ALIAS = "What is the Elasticsearch alias name you will configure in Arranger (to be referenced by DMS UI and ALSO must match the alias name previously supplied for Maestro) be? "; public static final String ARRANGER_QUESTIONS_NOTE = diff --git a/src/main/java/bio/overture/dms/cli/questionnaire/DmsUIQuestionnaire.java b/src/main/java/bio/overture/dms/cli/questionnaire/DmsUIQuestionnaire.java index a66f0fb..eabd97e 100644 --- a/src/main/java/bio/overture/dms/cli/questionnaire/DmsUIQuestionnaire.java +++ b/src/main/java/bio/overture/dms/cli/questionnaire/DmsUIQuestionnaire.java @@ -106,16 +106,24 @@ String.class, PROJ_ID, true, getDefaultValue(() -> existingConfig.getProjectConf DmsUIConfig.ArrangerProjectConfig.DEFAULT_PROJECT_ID, isNull(existingConfig))) .getAnswer(); - String projectName = - questionFactory - .newDefaultSingleQuestion( - String.class, - PROJ_NAME, - true, - getDefaultValue(() -> existingConfig.getProjectConfig().getName() , - DmsUIConfig.ArrangerProjectConfig.DEFAULT_PROJECT_NAME, isNull(existingConfig)) - ) - .getAnswer(); +// String projectName = +// questionFactory +// .newDefaultSingleQuestion( +// String.class, +// PROJ_NAME, +// true, +// getDefaultValue(() -> existingConfig.getDmsVizTool() , +// DmsUIConfig.ArrangerProjectConfig.DEFAULT_PROJECT_NAME, isNull(existingConfig)) +// ) +// .getAnswer(); + String visualizationTool = + questionFactory + .newDefaultSingleQuestion( + String.class, + DMS_VIZ_TOOL, + false, + existingConfig.getDmsVizTool()) + .getAnswer(); String elasticSearchIndexOrAlias = questionFactory @@ -127,13 +135,14 @@ DmsUIConfig.ArrangerProjectConfig.DEFAULT_PROJECT_NAME, isNull(existingConfig)) .projectConfig( DmsUIConfig.ArrangerProjectConfig.builder() .id(projectId) - .name(projectName) +// .name(projectName) .indexAlias(elasticSearchIndexOrAlias) .build()) .url(info.serverUrl) .labName(labName) .logoFileName(logoFileName) .assetsDir(assetsDirPath) + .dmsVizTool(visualizationTool) .ssoProviders( egoConfig.getApi().getSso().getConfiguredProviders().stream() .map(p -> p.name().toUpperCase()) diff --git a/src/main/java/bio/overture/dms/core/model/dmsconfig/DmsUIConfig.java b/src/main/java/bio/overture/dms/core/model/dmsconfig/DmsUIConfig.java index 8f571bc..1c558b1 100644 --- a/src/main/java/bio/overture/dms/core/model/dmsconfig/DmsUIConfig.java +++ b/src/main/java/bio/overture/dms/core/model/dmsconfig/DmsUIConfig.java @@ -33,6 +33,8 @@ public class DmsUIConfig { private String assetsDir; + private String dmsVizTool; + private ArrangerProjectConfig projectConfig; public static final int DEFAULT_PORT = 8000;