diff --git a/Dockerfile b/Dockerfile index 22ff253..0e7e75d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,8 +32,10 @@ FROM eclipse-temurin:11-jre-focal ARG REPO_DIR -ARG JAR_FILE=target/data-export-exec.jar +ARG EXEC_JAR_FILE=target/data-export-exec.jar +ARG VERIFIER_JAR_FILE=target/data-export-verifier.jar WORKDIR ${REPO_DIR} -COPY --from=build-jar ${REPO_DIR}/${JAR_FILE} ./target/ +COPY --from=build-jar ${REPO_DIR}/${EXEC_JAR_FILE} ./target/ +COPY --from=build-jar ${REPO_DIR}/${VERIFIER_JAR_FILE} ./target/ diff --git a/Jenkinsfile b/Jenkinsfile index 824f148..d42d17d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,25 +8,25 @@ def utils = new Utilities() pipeline{ agent any + // Set output folder that will contain the files generated by this step. - environment { - OUTPUT_FOLDER = "export" + environment { + OUTPUT_FOLDER = "export" ECR_URL = 'public.ecr.aws/reactome/data-export' CONT_NAME = 'data_export_container' CONT_ROOT = '/opt/data-export' - } - + } + stages{ // This stage checks that upstream project 'DiagramConverter' was run successfully. stage('Check DiagramConverter build succeeded'){ steps{ script{ - utils.checkUpstreamBuildsSucceeded("File-Generation/job/DiagramConverter/") + utils.checkUpstreamBuildsSucceeded("File-Generation/job/DiagramConverter/") } } } - stage('Setup: Pull and clean docker environment'){ steps{ sh "docker pull ${ECR_URL}:latest" @@ -37,12 +37,13 @@ pipeline{ """ } } - + // Execute the jar file, producing data-export files. stage('Main: Run Data-Export'){ steps{ script{ def releaseVersion = utils.getReleaseVersion() + sh "mkdir -p ${env.OUTPUT_FOLDER}" sh "rm -rf ${env.OUTPUT_FOLDER}/*" withCredentials([usernamePassword(credentialsId: 'neo4jUsernamePassword', passwordVariable: 'pass', usernameVariable: 'user')]){ @@ -52,9 +53,15 @@ pipeline{ sh "sudo service tomcat9 stop" sh """\ - docker run -v \$(pwd)/${env.OUTPUT_FOLDER}:${CONT_ROOT}/${env.OUTPUT_FOLDER} --net=host --name ${CONT_NAME} ${ECR_URL}:latest /bin/bash -c 'java -Xmx${env.JAVA_MEM_MAX}m -jar target/data-export-exec.jar --user $user --password $pass --output ./${env.OUTPUT_FOLDER} --verbose' - """ - sh "sudo chown jenkins:jenkins ${env.OUTPUT_FOLDER}" + docker run \\ + -v \$(pwd)/${env.OUTPUT_FOLDER}:${CONT_ROOT}/${env.OUTPUT_FOLDER} \\ + --net=host \\ + --name ${CONT_NAME} \\ + ${ECR_URL}:latest \\ + /bin/bash -c 'java -Xmx${env.JAVA_MEM_MAX}m -jar target/data-export-exec.jar --user $user --password $pass --output ./${env.OUTPUT_FOLDER} --verbose' + """ + sh "sudo chown jenkins:jenkins ${env.OUTPUT_FOLDER}" + sh "sudo service mysql start" sh "sudo service tomcat9 start" // Archive the files produced by the step for S3. @@ -63,46 +70,86 @@ pipeline{ } } } + + // Execute the verifier jar file checking for the existence and proper file sizes of the data-export output + stage('Post: Verify DataExport ran correctly') { + steps { + script { + def releaseVersion = utils.getReleaseVersion() + + sh """ + docker run \\ + --rm \\ + -v ${pwd()}/${env.OUTPUT_FOLDER}:${CONT_ROOT}/${env.OUTPUT_FOLDER}/ \\ + -v \$HOME/.aws:/root/.aws:ro \\ + -e AWS_REGION=us-east-1 \\ + --net=host \\ + --name ${CONT_NAME}_verifier \\ + ${ECR_URL}:latest \\ + /bin/bash -c "java -jar target/data-export-verifier.jar --releaseNumber ${releaseVersion} --output ${CONT_ROOT}/${env.OUTPUT_FOLDER}" + """ + } + } + } + + // Creates a list of files and their sizes to use for comparison baseline during next release + stage('Post: Create files and sizes list to upload for next release\'s verifier') { + steps { + script { + def fileSizeList = "files_and_sizes.txt" + def releaseVersion = utils.getReleaseVersion() + + sh "find ${env.OUTPUT_FOLDER} -type f -printf \"%s\t%P\n\" > ${fileSizeList}" + sh "aws s3 --no-progress cp ${fileSizeList} s3://reactome/private/releases/${releaseVersion}/data_export/data/" + sh "rm ${fileSizeList}" + } + } + } + // This stage outputs the difference in line counts for data-export files between releases. stage('Post: Compare Data-Export file line counts between releases') { - steps{ - script{ - def releaseVersion = utils.getReleaseVersion() - def previousReleaseVersion = utils.getPreviousReleaseVersion() - def previousExportsArchive = "export-v${previousReleaseVersion}.tgz" - def currentDir = pwd() - - sh "mkdir -p ${previousReleaseVersion}" - - // Download data-export files archive from previous release from S3. - sh "aws s3 --no-progress cp s3://reactome/private/releases/${previousReleaseVersion}/data_export/data/${previousExportsArchive} ${previousReleaseVersion}/" - dir("${previousReleaseVersion}"){ - sh "tar -xf ${previousExportsArchive}" + steps{ + script{ + def releaseVersion = utils.getReleaseVersion() + def previousReleaseVersion = utils.getPreviousReleaseVersion() + def previousExportsArchive = "export-v${previousReleaseVersion}.tgz" + def currentDir = pwd() + + sh "mkdir -p ${previousReleaseVersion}" + + // Download data-export files archive from previous release from S3. + sh "aws s3 --no-progress cp s3://reactome/private/releases/${previousReleaseVersion}/data_export/data/${previousExportsArchive} ${previousReleaseVersion}/" + dir("${previousReleaseVersion}"){ + sh "tar -xf ${previousExportsArchive}" + } + // Output line counts between files. + utils.outputLineCountsOfFilesBetweenFolders("${env.OUTPUT_FOLDER}", "${previousReleaseVersion}/${env.OUTPUT_FOLDER}", "$currentDir") + sh "rm -r ${previousReleaseVersion}*" + } } - // Output line counts between files. - utils.outputLineCountsOfFilesBetweenFolders("${env.OUTPUT_FOLDER}", "${previousReleaseVersion}/${env.OUTPUT_FOLDER}", "$currentDir") - sh "rm -r ${previousReleaseVersion}*" - } - } } + // Move all data-export files to the downloads folder. At time of writing, these files aren't gzipped. stage('Post: Move export files to download folder') { - steps{ - script{ - def releaseVersion = utils.getReleaseVersion() - def downloadPath = "${env.ABS_DOWNLOAD_PATH}/${releaseVersion}" - sh "mv ${env.OUTPUT_FOLDER}/* ${downloadPath}/ " - } - } + steps{ + script{ + def releaseVersion = utils.getReleaseVersion() + def downloadPath = "${env.ABS_DOWNLOAD_PATH}/${releaseVersion}" + + sh "mv ${env.OUTPUT_FOLDER}/* ${downloadPath}/ " + } + } } + // Archive everything on S3, and move the 'diagram' folder to the download/vXX folder. stage('Post: Archive Outputs'){ steps{ script{ def releaseVersion = utils.getReleaseVersion() - def dataFiles = ["export-v${releaseVersion}.tgz"] + def dataFiles = ["export-v${releaseVersion}.tgz"] def logFiles = [] def foldersToDelete = [] + utils.cleanUpAndArchiveBuildFiles("data_export", dataFiles, logFiles, foldersToDelete) } } diff --git a/pom.xml b/pom.xml index 903a6ac..2e24f29 100644 --- a/pom.xml +++ b/pom.xml @@ -1,136 +1,172 @@ - - 4.0.0 - org.reactome.server.qa - data-export - jar - 2.0.7 - Data Export - - org.reactome.maven - reactome-parent - 1.0.5-SNAPSHOT - - Exports specific content in several files - - The European Bioinformatics Institute - https://www.ebi.ac.uk/ - - - - gviteri - Guilherme Viteri - - - fkorn - Florian Korninger - - - amundo - - - - org.reactome.server.export.Main - 11 - - - - - org.reactome.server.graph - graph-core - - - - com.martiansoftware - jsap - - - - com.fasterxml.jackson.core - jackson-databind - - - - - - nexus-ebi-repo - The EBI internal repository - https://www.ebi.ac.uk/Tools/maven/repos/content/groups/ebi-repo/ - - true - - - false - - - - - nexus-ebi-snapshot-repo - The EBI internal snapshot repository - https://www.ebi.ac.uk/Tools/maven/repos/content/groups/ebi-snapshots/ - - false - - - true - - - - - ${project.artifactId} - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven.compiler.version} - - ${java.version} - ${java.version} - - - - org.springframework.boot - spring-boot-maven-plugin - ${spring-boot.version} - - ${main.class} - true - exec - - - - - repackage - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - 3.1.1 - - - - com.puppycrawl.tools - checkstyle - 8.44 - - - - - checkstyle-check - - check - - - - - checkstyle.xml - - UTF-8 - - - - + + 4.0.0 + org.reactome.server.qa + data-export + jar + 2.0.7 + Data Export + + org.reactome.maven + reactome-parent + 1.0.5 + + Exports specific content in several files + + The European Bioinformatics Institute + https://www.ebi.ac.uk/ + + + + gviteri + Guilherme Viteri + + + fkorn + Florian Korninger + + + amundo + + + + org.reactome.server.export.Main + org.reactome.server.export.verifier.DataExportVerifier + 11 + + + + + org.reactome.server.graph + graph-core + + + + com.martiansoftware + jsap + + + + com.fasterxml.jackson.core + jackson-databind + + + org.reactome.release.verifier + verifier-lib + 1.0-SNAPSHOT + + + + + + nexus-ebi-repo + The EBI internal repository + https://www.ebi.ac.uk/Tools/maven/repos/content/groups/ebi-repo/ + + true + + + false + + + + + nexus-ebi-snapshot-repo + The EBI internal snapshot repository + https://www.ebi.ac.uk/Tools/maven/repos/content/groups/ebi-snapshots/ + + false + + + true + + + + + Central Portal Snapshots + central-portal-snapshots + https://central.sonatype.com/repository/maven-snapshots/ + + false + + + true + + + + + ${project.artifactId} + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven.compiler.version} + + ${java.version} + ${java.version} + + + + org.springframework.boot + spring-boot-maven-plugin + ${spring-boot.version} + + true + NONE + + + + primary-exec + + repackage + + + ${main.class} + exec + true + + + + verifier-exec + + repackage + + + ${verifier.main.class} + verifier + true + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 3.1.1 + + + + com.puppycrawl.tools + checkstyle + 8.44 + + + + + checkstyle-check + + check + + + + + checkstyle.xml + + UTF-8 + + + + \ No newline at end of file diff --git a/src/main/java/org/reactome/server/export/tasks/SequenceExporter.java b/src/main/java/org/reactome/server/export/tasks/SequenceExporter.java index 6a74ba7..fa9d92a 100644 --- a/src/main/java/org/reactome/server/export/tasks/SequenceExporter.java +++ b/src/main/java/org/reactome/server/export/tasks/SequenceExporter.java @@ -60,11 +60,6 @@ public String getName() { return "reactome_reaction_exporter"; } - @Override - public boolean printDbVersion() { - return true; - } - @Override public boolean addNullValues() { return false; diff --git a/src/main/java/org/reactome/server/export/verifier/DataExportVerifier.java b/src/main/java/org/reactome/server/export/verifier/DataExportVerifier.java new file mode 100644 index 0000000..babc244 --- /dev/null +++ b/src/main/java/org/reactome/server/export/verifier/DataExportVerifier.java @@ -0,0 +1,19 @@ +package org.reactome.server.export.verifier; + +import org.reactome.release.verifier.DefaultVerifier; +import org.reactome.release.verifier.Verifier; + +import java.io.IOException; + +/** + * @author Joel Weiser (joel.weiser@oicr.on.ca) + * Created 1/3/2025 + */ +public class DataExportVerifier { + + public static void main(String[] args) throws IOException { + Verifier verifier = new DefaultVerifier("data_export"); + verifier.parseCommandLineArgs(args); + verifier.run(); + } +}