From 67d49ac15e7dc970d0d55a50c2e5f0ba3a5b3dd9 Mon Sep 17 00:00:00 2001 From: Azher2Ali <121898125+Azher2Ali@users.noreply.github.com> Date: Thu, 13 Jul 2023 13:00:18 -0400 Subject: [PATCH 1/3] Committing changes related to download failure debugging modifications --- score-client/pom.xml | 1 + score-client/src/main/conf/logback.xml | 4 +-- .../score/client/command/DownloadCommand.java | 25 ++++++++++++++++--- .../score/client/command/UploadCommand.java | 5 ++++ .../score/client/command/ViewCommand.java | 3 ++- .../client/download/DownloadService.java | 4 +++ .../client/download/DownloadStateStore.java | 3 ++- .../src/main/resources/application.yml | 8 +++--- 8 files changed, 42 insertions(+), 11 deletions(-) diff --git a/score-client/pom.xml b/score-client/pom.xml index 6ebba856..bb29a497 100644 --- a/score-client/pom.xml +++ b/score-client/pom.xml @@ -196,6 +196,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S org.apache.maven.plugins maven-jar-plugin + 3.2.2 diff --git a/score-client/src/main/conf/logback.xml b/score-client/src/main/conf/logback.xml index f0352b69..72fed3de 100644 --- a/score-client/src/main/conf/logback.xml +++ b/score-client/src/main/conf/logback.xml @@ -16,7 +16,7 @@ OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> - + @@ -36,7 +36,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S - + diff --git a/score-client/src/main/java/bio/overture/score/client/command/DownloadCommand.java b/score-client/src/main/java/bio/overture/score/client/command/DownloadCommand.java index e1f89107..7ce2cc1d 100644 --- a/score-client/src/main/java/bio/overture/score/client/command/DownloadCommand.java +++ b/score-client/src/main/java/bio/overture/score/client/command/DownloadCommand.java @@ -34,6 +34,8 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -60,6 +62,8 @@ public enum OutputLayout { BUNDLE, FILENAME, ID } + private Logger session = LoggerFactory.getLogger("session"); + /** * Options */ @@ -148,7 +152,8 @@ private List getIdsFromManifest(ManifestResource manifestResource) throw val manifest = manifestService.getDownloadManifest(manifestResource); validateManifest(manifest); - + session.info("Download Command Class Parameters : manifest.getEntries() ::::"+manifest.getEntries()); + terminal.println("Download Command Class Parameters : manifest.getEntries() ::::"+manifest.getEntries()); val entries = manifest.getEntries(); if (entries.isEmpty()) { throw new BadManifestException(format("Manifest '%s' is empty", manifestResource)); @@ -165,7 +170,10 @@ private List getIdsFromAnalysis(String programId, String analysisId) { private int downloadObjects(List objectIds) throws IOException { // Entities are defined in Meta service val entities = resolveEntities(objectIds); - + terminal + .printLine() + .println("Downloading objects size ::::"+objectIds.size()) + .printLine(); if (!verifyLocalAvailableSpace(entities)) { return FAILURE_STATUS; } @@ -178,6 +186,14 @@ private int downloadObjects(List objectIds) throws IOException { if (layout != OutputLayout.ID) { entitySet = filterEntities(entities); } + terminal + .printLine() + .println("Filtered Entites ::::"+entitySet.size()) + .printLine(); + terminal + .printLine() + .println("Force value ::::"+force) + .printLine(); // If --force is specified, delete all the old files first, so that we can resume later if the new downloads fail. if (force) { @@ -200,7 +216,10 @@ private int downloadObjects(List objectIds) throws IOException { val builder = DownloadRequest.builder(); val request = builder.outputDir(outputDir).entity(entity).objectId(entity.getId()).offset(offset).length(length) .validate(validate).build(); - + terminal + .printLine() + .println("Request payload ::::"+request.toString()) + .printLine(); // only try to re-download a file that exists if --force was specified on the command line val target = getLayoutTarget(entity); if (target.exists()) { diff --git a/score-client/src/main/java/bio/overture/score/client/command/UploadCommand.java b/score-client/src/main/java/bio/overture/score/client/command/UploadCommand.java index 926c2421..c0b02470 100644 --- a/score-client/src/main/java/bio/overture/score/client/command/UploadCommand.java +++ b/score-client/src/main/java/bio/overture/score/client/command/UploadCommand.java @@ -27,6 +27,8 @@ import com.beust.jcommander.Parameters; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; @@ -43,6 +45,7 @@ @Profile("!kf") public class UploadCommand extends RepositoryAccessCommand { + private Logger session = LoggerFactory.getLogger("session"); /** * Options. */ @@ -90,6 +93,8 @@ public int execute() throws Exception { uploadFile(objectId, file, checksum); } } else { + session.info("Upload Command Class Parameters : file ::::"+file+" objectId ::::::"+objectId+" manifestResource ::::::"+manifestResource+" md5 ::::"+md5); + terminal.println("Upload Command Class Parameters : file ::::"+file+" objectId ::::::"+objectId+" manifestResource ::::::"+manifestResource+" md5 ::::"+md5); checkParameter(file != null, "--file must be specified if --object-id is specified"); checkParameter(md5 != null, "--md5 must be specified if --object-id is specified"); uploadFile(objectId, file, md5); diff --git a/score-client/src/main/java/bio/overture/score/client/command/ViewCommand.java b/score-client/src/main/java/bio/overture/score/client/command/ViewCommand.java index 4e275d6c..b276005d 100644 --- a/score-client/src/main/java/bio/overture/score/client/command/ViewCommand.java +++ b/score-client/src/main/java/bio/overture/score/client/command/ViewCommand.java @@ -305,7 +305,8 @@ private void validateParms() { terminal.println("When --stdout specified, output format forced to SAM."); } } - + session.info("View Command Class parameters : objectId ::::"+objectId+" sequenceFile ::::"+sequenceFile+" manifestResource ::::"+manifestResource); + terminal.println("View Command Class parameters : objectId ::::"+objectId+" sequenceFile ::::"+sequenceFile+" manifestResource ::::"+manifestResource); checkParameter(objectId != null || sequenceFile != null || manifestResource != null, "One of --object-id, --input-file or --manifest must be specified. " + "For CRAM files, please additionally include --reference-file."); diff --git a/score-client/src/main/java/bio/overture/score/client/download/DownloadService.java b/score-client/src/main/java/bio/overture/score/client/download/DownloadService.java index c03af1e6..6345e720 100644 --- a/score-client/src/main/java/bio/overture/score/client/download/DownloadService.java +++ b/score-client/src/main/java/bio/overture/score/client/download/DownloadService.java @@ -155,6 +155,10 @@ private void resumeIfPossible(DownloadRequest request, boolean checksum) log.debug("Attempting to resume download for {}", request.toString()); ObjectSpecification spec = null; try { + terminal + .printLine() + .println("Request payload in Download Service ::::"+request.toString()) + .printLine(); spec = downloadStateStore.getProgress(request.getOutputDir(), request.getObjectId()); } catch (NotRetryableException e) { log.info("New download: {} because {}", request.getObjectId(), e.getMessage()); diff --git a/score-client/src/main/java/bio/overture/score/client/download/DownloadStateStore.java b/score-client/src/main/java/bio/overture/score/client/download/DownloadStateStore.java index 4842224f..6f36cb98 100644 --- a/score-client/src/main/java/bio/overture/score/client/download/DownloadStateStore.java +++ b/score-client/src/main/java/bio/overture/score/client/download/DownloadStateStore.java @@ -76,7 +76,8 @@ public boolean hasStarted(File stateDir, String objectId) { return Files.exists(new File(getObjectStateDir(stateDir, objectId), getSpecificationName()).toPath()); } - public ObjectSpecification getProgress(File stateDir, String objectId) throws IOException { + public ObjectSpecification + getProgress(File stateDir, String objectId) throws IOException { log.debug("Loading local progress for {} from {}", objectId, stateDir.toString()); val spec = loadSpecification(stateDir, objectId); log.debug("Completed loading local object specification (meta file)"); diff --git a/score-client/src/main/resources/application.yml b/score-client/src/main/resources/application.yml index cc9b8d28..8db784c8 100644 --- a/score-client/src/main/resources/application.yml +++ b/score-client/src/main/resources/application.yml @@ -23,10 +23,10 @@ ############################################################################### client: - ansi: ${ansi:true} - silent: ${silent:false} - quiet: ${quiet:false} - accessToken: "${accessToken:}" # Alias for backwards compatibility + ansi: true + silent: false + quiet: false + accessToken: dd9b5c3e-d07e-4b5e-b34e-2d471ee9c78c connectTimeoutSeconds: 60 readTimeoutSeconds: 60 # path of dir with WRITE access for score client upload state files From ef609cf442e9b0e2d6ddf692647ebec84a46c9da Mon Sep 17 00:00:00 2001 From: Azher2Ali <121898125+Azher2Ali@users.noreply.github.com> Date: Thu, 13 Jul 2023 13:41:09 -0400 Subject: [PATCH 2/3] Committing changes related to download failure debugging modifications --- pom.xml | 6 +++--- .../overture/score/client/download/DownloadStateStore.java | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index d66f44bb..96b3f4b4 100644 --- a/pom.xml +++ b/pom.xml @@ -258,9 +258,9 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S - 11 + 17 - 1.18.4 + 1.18.22 3.0.1u2 3.1 @@ -295,7 +295,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S 1.3.1 [1.0.0,) [3.0.0,) - [11,) + [17,) ${basedir}/src/main/pmd diff --git a/score-client/src/main/java/bio/overture/score/client/download/DownloadStateStore.java b/score-client/src/main/java/bio/overture/score/client/download/DownloadStateStore.java index 6f36cb98..4842224f 100644 --- a/score-client/src/main/java/bio/overture/score/client/download/DownloadStateStore.java +++ b/score-client/src/main/java/bio/overture/score/client/download/DownloadStateStore.java @@ -76,8 +76,7 @@ public boolean hasStarted(File stateDir, String objectId) { return Files.exists(new File(getObjectStateDir(stateDir, objectId), getSpecificationName()).toPath()); } - public ObjectSpecification - getProgress(File stateDir, String objectId) throws IOException { + public ObjectSpecification getProgress(File stateDir, String objectId) throws IOException { log.debug("Loading local progress for {} from {}", objectId, stateDir.toString()); val spec = loadSpecification(stateDir, objectId); log.debug("Completed loading local object specification (meta file)"); From 1701344a8f25ebc213b1ef51749a5d0532d326c7 Mon Sep 17 00:00:00 2001 From: Azher2Ali <121898125+Azher2Ali@users.noreply.github.com> Date: Thu, 13 Jul 2023 13:43:47 -0400 Subject: [PATCH 3/3] Committing changes related to download failure debugging modifications --- pom.xml | 6 +++--- score-client/pom.xml | 1 - score-client/src/main/resources/application.yml | 8 ++++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index 96b3f4b4..d66f44bb 100644 --- a/pom.xml +++ b/pom.xml @@ -258,9 +258,9 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S - 17 + 11 - 1.18.22 + 1.18.4 3.0.1u2 3.1 @@ -295,7 +295,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S 1.3.1 [1.0.0,) [3.0.0,) - [17,) + [11,) ${basedir}/src/main/pmd diff --git a/score-client/pom.xml b/score-client/pom.xml index bb29a497..6ebba856 100644 --- a/score-client/pom.xml +++ b/score-client/pom.xml @@ -196,7 +196,6 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S org.apache.maven.plugins maven-jar-plugin - 3.2.2 diff --git a/score-client/src/main/resources/application.yml b/score-client/src/main/resources/application.yml index 8db784c8..cc9b8d28 100644 --- a/score-client/src/main/resources/application.yml +++ b/score-client/src/main/resources/application.yml @@ -23,10 +23,10 @@ ############################################################################### client: - ansi: true - silent: false - quiet: false - accessToken: dd9b5c3e-d07e-4b5e-b34e-2d471ee9c78c + ansi: ${ansi:true} + silent: ${silent:false} + quiet: ${quiet:false} + accessToken: "${accessToken:}" # Alias for backwards compatibility connectTimeoutSeconds: 60 readTimeoutSeconds: 60 # path of dir with WRITE access for score client upload state files