Last active
October 21, 2022 13:55
-
-
Save marcelbirkner/9bc906b24348f31e03b2 to your computer and use it in GitHub Desktop.
Revisions
-
marcelbirkner revised this gist
Sep 24, 2015 . 1 changed file with 0 additions and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1,4 +1,3 @@ import groovy.sql.Sql import java.util.Date import java.text.SimpleDateFormat -
marcelbirkner revised this gist
Sep 24, 2015 . 1 changed file with 1 addition and 0 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1,3 +1,4 @@ // Import required dependencies import groovy.sql.Sql import java.util.Date import java.text.SimpleDateFormat -
marcelbirkner revised this gist
Sep 24, 2015 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -14,7 +14,7 @@ import java.text.SimpleDateFormat * 2. Call Configuration Management Database via JDBC and read additional information * 3. Iterate over all projects and get further project details via GitLab REST API * 4. Create all necessary jobs via Jenkins Job DSL. Typical jobs are: Build, Deploy, Acceptance Test-Jobs * 5. Create custom views */ // GitLab settings -
marcelbirkner revised this gist
Sep 24, 2015 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -30,7 +30,7 @@ javacMap = ['6': '1.6', '7': '1.7', '8': '1.8', '9': '1.9'] // Default build tool versions def antVersion = "Ant 1.9.6" def mavenVersion = "Maven 3.3.3" // valid company department id's departments = ["BIZ1","BIZ2","BIZ3","DEV1","DEV2","DEV3","OPS1","OPS2"] -
marcelbirkner created this gist
Sep 24, 2015 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,685 @@ import groovy.sql.Sql import java.util.Date import java.text.SimpleDateFormat /* * THIS IS AN EXAMPLE SNIPPET. FOR MORE DETAILS SEE THE FOLLOWING BLOG ARTICLE: * https://blog.codecentric.de/en/?p=30502 * * This Jenkins Job DSL Groovy Script creates Continuous Integration (CI) Jobs * for all Maven & Ant projects that exist on a GitLab Server. * * The script does the following steps: * 1. Call GitLab REST API and get all projects * 2. Call Configuration Management Database via JDBC and read additional information * 3. Iterate over all projects and get further project details via GitLab REST API * 4. Create all necessary jobs via Jenkins Job DSL. Typical jobs are: Build, Deploy, Acceptance Test-Jobs * 5. Create customer views */ // GitLab settings gitUrl = 'http://git/api/v3' // you find the private token in your GitLab profile gitPrivateToken = 'xYxYxYxYxYxYxYxYxYxY' // id of Jenkins GitLab credentials gitCredentials = "xYxYxYxY-xYxY-xYxY-xYxY-xYxYxYxYxYxY" // mapping of Java Version to Jenkins JDK Version jdkMap = ['6': 'JDK 6', '7': 'JDK 7', '8': 'JDK 8', '9': 'JDK 9'] javacMap = ['6': '1.6', '7': '1.7', '8': '1.8', '9': '1.9'] // Default build tool versions def antVersion = "Ant 1.9.6" def mavenVersion = "Maven 3.3.0" // valid company department id's departments = ["BIZ1","BIZ2","BIZ3","DEV1","DEV2","DEV3","OPS1","OPS2"] // GitLab REST API settings. REST API returns max 100 per page. Thats why we need pagination. def currentPage = 1 def projectsPerPage = 100 def currentProjectsSize = 100 // we will use this array to gather all CI job names def ciJobList = [] // read projects from GitLab REST API until finished while (currentProjectsSize == projectsPerPage) { def projectsApi = new URL("${gitUrl}/projects/all?page=${currentPage}&per_page=${projectsPerPage}&private_token=${gitPrivateToken}") println "############################################################################################################" println "Read GitLab REST API: ${projectsApi}" println "############################################################################################################" // convert returned JSON to object def projects = new groovy.json.JsonSlurper().parse(projectsApi.newReader()) // required for pagination currentProjectsSize = projects.size() currentPage++ // iterate over all projects projects.each { println "------------------------------------------------------------------------------------------------------------" println "Working on project: ${it.name}" def gitProjectName = it.name def projectId = it.id def projectGitSshUrlToRepo = it.ssh_url_to_repo // read project details from configuration management databse def projectDetails = getProjectDetailsFromConfigMgtDb(gitProjectName) def isJavaProject=projectDetails.isJavaProject if ( ! isJavaProject ) { println "-> skipping project, since no Config Mgt details are available." return } def webContextRoot=projectDetails.webContextRoot def mailRecipients=projectDetails.mailRecipients def isAnt=projectDetails.isAnt def isMaven=projectDetails.isMaven def isLibrary=projectDetails.isLibrary def isBatch=projectDetails.isBatch def isWebApp=projectDetails.isWebApp def isArchetype=projectDetails.isArchetype def javaVersion = projectDetails.javaVersion // get JDK Version and javac Version from javaVersion in Config Mgt DB def jenkinsJdkVersion = jdkMap.get(javaVersion) def projectJavacVersion = javacMap.get(javaVersion) println "-> Properties:" println "-> javaVersion=${javaVersion}" println "-> projectJavacVersion=${projectJavacVersion}" println "-> jenkinsJdkVersion=${jenkinsJdkVersion}" println "-> isMaven=${isMaven}" println "-> isAnt=${isAnt}" println "-> webContextRoot=${webContextRoot}" println "-> mailRecipients=${mailRecipients}" if ( javaVersion == null || projectJavacVersion == null || jenkinsJdkVersion == null ) { println "-> ERROR: Should never get here." return } // get additional information from GitLab repository def gitProjectDetails = getGitProjectDetails(projectId) def validProjectGruppe = gitProjectDetails.validProjectGruppe def isGitProjectActive = gitProjectDetails.isActive def defaultBranch = gitProjectDetails.defaultBranch println "-> isGitProjectActive=${isGitProjectActive}" println "-> validProjectGruppe=${validProjectGruppe}" println "-> defaultBranch=${defaultBranch}" if( ! isGitProjectActive || defaultBranch == null ) { println "Skipping project ${gitProjectName} since it is not active or does not have a default branch" return } // each project is assigned a department, therefore all jobs are prefixed with the department id println "-> Create Job names:" def ciJobName = "${validProjectGruppe}-${gitProjectName}-1-ci" def deployJobName = "${validProjectGruppe}-${gitProjectName}-2-deployment" def robotJobName = "${validProjectGruppe}-${gitProjectName}-3-robot" // create CI jobs if(isLibrary && isMaven) { println "-> create Maven Library Job: ${ciJobName}" createCIJobOnly(ciJobName, projectGitSshUrlToRepo, defaultBranch, mailRecipients, jenkinsJdkVersion) } else if(isBatch && isMaven) { println "-> create Maven Batch Job: ${ciJobName}" createCIJobOnly(ciJobName, projectGitSshUrlToRepo, defaultBranch, mailRecipients, jenkinsJdkVersion) } else if(isBatch && isAnt) { println "-> create Ant Batch Job: ${ciJobName}" createAntCIJobOnly(ciJobName, gitProjectName, defaultBranch, projectGitSshUrlToRepo, webContextRoot, mailRecipients, jenkinsJdkVersion, projectJavacVersion) } else if(isMaven && isWebApp ) { println "-> create Maven WebApp Job: ${ciJobName}" createMavenCIJob(ciJobName, projectGitSshUrlToRepo, defaultBranch, deployJobName, mailRecipients, jenkinsJdkVersion) } else if (isAnt && isWebApp) { println "-> create Ant WebApp Job: ${ciJobName}" createAntCIJob(ciJobName, gitProjectName, defaultBranch, projectGitSshUrlToRepo, webContextRoot, deployJobName, mailRecipients, jenkinsJdkVersion, projectJavacVersion) } else if ( isArchetype ) { createCIJobOnly(ciJobName, projectGitSshUrlToRepo, defaultBranch, mailRecipients, jenkinsJdkVersion) } else { println "No CI Jobs will be generated for ${gitProjectName}" } // create deployment and acceptance test jobs if( isWebApp ) { println "-> create Deploy Job: ${deployJobName}" createDeployJob(deployJobName, projectGitSshUrlToRepo, defaultBranch, robotJobName, gitProjectName, mailRecipients) println "-> create Robot Job: ${robotJobName}" createRobotJob(projectGitSshUrlToRepo, robotJobName, gitProjectName, mailRecipients) // add CI Job to list ciJobList += ciJobName } } } /* * CREATE VIEWS TO ORGANIZE JOBS */ // regular expression prefix for all company departments def groupPrefix="(" departments.each { groupPrefix += it+"|" } groupPrefix+="NOGROUP)" createListView('1-Build', 'All Build Jobs', "${groupPrefix}.*-1-ci") createListView('2-Deploy', 'All Deploy Jobs', "${groupPrefix}.*-2-deployment") createListView('3-Robot', 'All Robot Jobs', "${groupPrefix}.*-3-robot") createListView('WebApp', 'All project Jobs', "${groupPrefix}-.*-webapp") createListView('Batch', 'All Subproject Jobs', "${groupPrefix}-.*-batch-.*") createListView('Library', 'All Library Jobs', "${groupPrefix}-library-.*") createListView('Admin', 'All Admin Jobs', 'Administration.*') createNestedDepartmentView('Department') nestedView('Build Pipelines') { description('Automatically generated Build Pipelines for all CI Jobs') columns { weather() } views { ciJobList.each { def job = it println "Create Build Pipeline View for ${job}" view(job, type: BuildPipelineView) { selectedJob(job) triggerOnlyLatestJob(true) alwaysAllowManualTrigger(true) showPipelineParameters(true) showPipelineParametersInHeaders(true) showPipelineDefinitionHeader(true) startsWithParameters(true) displayedBuilds(5) } } } } /* * JOB DSL UTILITY METHODS */ // generate Ant -1-ci Job def createAntCIJobOnly(def ciJobName, def gitProjectName, def defaultBranch, def projectGitSshUrlToRepo, def webContextRoot, def mailRecipients, def projectJenkinsJdkVersion, def projectJavacVersion) { job(ciJobName) { logRotator { daysToKeep(-1) numToKeep(10) } parameters { stringParam("project.name", gitProjectName) stringParam("project.version", defaultBranch) stringParam("web.context.root", webContextRoot) booleanParam("junit.skip.tests", false) stringParam("javac.version", projectJavacVersion) } label("ci-slave") jdk(projectjenkinsJdkVersion) scm { git { remote { url(projectGitSshUrlToRepo) credentials(gitCredentials) } wipeOutWorkspace(true) createTag(false) branch('\${project.version}') } } triggers { cron("H * * * 1-5") } wrappers { preBuildCleanup() } steps { ant("build") { buildFile "build.xml" antInstallation antVersion javaOpt("-Xmx1G -XX:MaxPermSize=512M") } } publishers { chucknorris() archiveJunit("results/junit/**/*.xml") mailer(mailRecipients, true, true) } } } // generate Ant -1-ci Job for WebApps def createAntCIJob(def ciJobName, def gitProjectName, def defaultBranch, def projectGitSshUrlToRepo, def webContextRoot, def deployJobName, def mailRecipients, def projectjenkinsJdkVersion, def projectJavacVersion) { job(ciJobName) { logRotator { daysToKeep(-1) numToKeep(10) } parameters { stringParam("project.name", gitProjectName) stringParam("project.version", defaultBranch) stringParam("web.context.root", webContextRoot) booleanParam("junit.skip.tests", false) stringParam("javac.version", projectJavacVersion) } label("ci-slave") jdk(projectjenkinsJdkVersion) scm { git { remote { url(projectGitSshUrlToRepo) credentials(gitCredentials) } wipeOutWorkspace(true) createTag(false) branch('\${project.version}') } } triggers { cron("H * * * 1-5") } wrappers { preBuildCleanup() } steps { ant("build") { buildFile "build.xml" antInstallation antVersion javaOpt("-Xmx1G -XX:MaxPermSize=512M") } } publishers { chucknorris() archiveJunit("results/junit/**/*.xml") downstreamParameterized { trigger(deployJobName, 'UNSTABLE_OR_BETTER') { currentBuild() } } mailer(mailRecipients, true, true) } } } // generate Maven -1-ci Job for Library or Batch projects def createCIJobOnly(def ciJobName, def projectGitSshUrlToRepo, def defaultBranch, def mailRecipients, def projectjenkinsJdkVersion) { job(ciJobName) { logRotator { daysToKeep(-1) numToKeep(10) } parameters { stringParam("project.version", defaultBranch, "Select branch to build: master, branch, tag") } label("ci-slave") jdk(projectjenkinsJdkVersion) scm { git { remote { url(projectGitSshUrlToRepo) credentials(gitCredentials) } wipeOutWorkspace(true) createTag(false) branch('\${project.version}') } } triggers { scm('H/5 * * * *') } wrappers { preBuildCleanup() } steps { maven { goals('clean versions:set -DnewVersion=DEV-\${project.version}-\${BUILD_NUMBER} -P jenkins-build -U') mavenOpts('-Xms256m') mavenOpts('-Xmx512m') mavenInstallation(mavenVersion) } maven { goals('org.jacoco:jacoco-maven-plugin:prepare-agent deploy -P jenkins-build,sonar -Dmaven.test.failure.ignore=true') mavenOpts('-XX:PermSize=256m') mavenOpts('-XX:MaxPermSize=1024m') mavenInstallation(mavenVersion) } maven { goals('sonar:sonar -P sonar,jenkins-build') mavenOpts('-Xmx2G') mavenOpts('-XX:MaxPermSize=1G') mavenInstallation(mavenVersion) } } publishers { chucknorris() archiveJunit('**/target/surefire-reports/*.xml') mailer(mailRecipients, true, true) } } } // generate Maven -1-ci Job for WebApps def createMavenCIJob(def ciJobName, def projectGitSshUrlToRepo, def defaultBranch, def deployJobName, def mailRecipients, def projectjenkinsJdkVersion) { job(ciJobName) { logRotator { daysToKeep(-1) numToKeep(10) } parameters { stringParam("project.version", defaultBranch, "Select branch to build: master, branch, tag") } label("ci-slave") jdk(projectjenkinsJdkVersion) scm { git { remote { url(projectGitSshUrlToRepo) credentials(gitCredentials) } wipeOutWorkspace(true) createTag(false) branch('\${project.version}') } } triggers { scm('H/5 * * * *') } wrappers { preBuildCleanup() } steps { maven { goals('clean versions:set -DnewVersion=DEV-\${project.version}-\${BUILD_NUMBER} -P jenkins-build -U') mavenOpts('-Xms256m') mavenOpts('-Xmx512m') mavenInstallation(mavenVersion) } maven { goals('org.jacoco:jacoco-maven-plugin:prepare-agent deploy -P jenkins-build,sonar -Dmaven.test.failure.ignore=true') mavenOpts('-XX:PermSize=256m') mavenOpts('-XX:MaxPermSize=1024m') mavenInstallation(mavenVersion) } maven { goals('sonar:sonar -P sonar,jenkins-build') mavenOpts('-Xmx2G') mavenOpts('-XX:MaxPermSize=1G') mavenInstallation(mavenVersion) } } publishers { chucknorris() archiveJunit('**/target/surefire-reports/*.xml') downstreamParameterized { trigger(deployJobName, 'UNSTABLE_OR_BETTER') { currentBuild() } } mailer(mailRecipients, true, true) } } } // generate Deployment -2-deploy Job def createDeployJob(def deployJobName, def projectGitSshUrlToRepo, def defaultBranch, def robotJobName, def gitProjectName, def mailRecipients ) { job(deployJobName) { logRotator { daysToKeep(-1) numToKeep(10) } label("ci-slave") multiscm { git { remote { url('git@git:infrastructure/deployment-scripts.git') credentials(gitCredentials) } createTag(false) branch("master") } git { remote { url(projectGitSshUrlToRepo) credentials(gitCredentials) } createTag(false) branch(defaultBranch) relativeTargetDir(gitProjectName) } } wrappers { preBuildCleanup() } steps { shell("set -x && sh startDeployment.sh ${gitProjectName}") } publishers { chucknorris() downstreamParameterized { trigger(robotJobName, 'UNSTABLE_OR_BETTER') { currentBuild() } } mailer(mailRecipients, true, true) } } } // generate Robot -3-robot Job def createRobotJob(def projectGitSshUrlToRepo, def robotJobName, def gitProjectName, def mailRecipients) { job(robotJobName) { logRotator { daysToKeep(-1) numToKeep(10) } label("ci-robot") multiscm { git { remote { url(projectGitSshUrlToRepo) credentials(gitCredentials) } createTag(false) branch("master") relativeTargetDir(gitProjectName) } git { remote { url('git@git:test/robot-test-framework.git') credentials(gitCredentials) } createTag(false) branch("master") relativeTargetDir("robot-test-framework") } } wrappers { preBuildCleanup() } steps { ant() { buildFile "/src/robot/robot.xml" antInstallation antVersion } } publishers { chucknorris() publishRobotFrameworkReports { passThreshold(100.0) unstableThreshold(75.0) onlyCritical(false) outputPath("/src/robot/log") reportFileName('report.html') logFileName('log.html') outputFileName('output.xml') disableArchiveOutput(false) otherFiles('*.jpg', '*.png') } mailer(mailRecipients, true, true) } } } // create list view def createListView(def jobViewName, def jobDescription, def regularExpression) { println "createListView ${jobViewName} with ${jobDescription} and ${regularExpression}" listView(jobViewName) { description(jobDescription) filterBuildQueue() filterExecutors() jobs { regex(regularExpression) } columns { status() buildButton() weather() name() lastSuccess() lastFailure() lastDuration() } } } // create nested view def createNestedDepartmentView(def viewName) { println "createNestedDepartmentView for ${viewName}" nestedView(viewName) { description('Automatically generated department groups') columns { weather() } views { departments.each { def department = it println "Create build pipeline subview for ${department}" view("${department}", type: ListView) { description("All Jobs for department ${department}") filterBuildQueue() filterExecutors() jobs { regex("${department}-.*") } columns { status() buildButton() weather() name() lastSuccess() lastFailure() lastDuration() } } } view("NOGROUP", type: ListView) { description("All Jobs that are not assigned to a department") filterBuildQueue() filterExecutors() jobs { regex("NOGROUP-.*") } columns { status() buildButton() weather() name() lastSuccess() lastFailure() lastDuration() } } } } } ///////////////////////////////////////////////////////// // UTILITY METHODS ///////////////////////////////////////////////////////// // Get project details from Configuration Management Database // using JDBC and the Git Project Name. def getProjectDetailsFromConfigMgtDb(def gitProjectName) { def gitProjectNameQuery = gitProjectName.toUpperCase() def dbSchema = "configdb" def dbServer = "dbserver" def dbUser = 'dbuser' def dbPassword = 'dbpassword' // Important: Oracle Driver needs to be available in the classpath def dbDriver = 'oracle.jdbc.driver.OracleDriver' def dbUrl = 'jdbc:oracle:thin:@' + dbServer + ':1521:' + dbSchema sql = Sql.newInstance( dbUrl, dbUser, dbPassword, dbDriver ) // search project in database def result = sql.firstRow("SELECT * FROM projects WHERE UPPER(project_id) LIKE ${gitProjectNameQuery}") def isJavaProject=false def projectDetails = new LinkedHashMap(); if ( result != null ) { projectDetails.isJavaProject=true projectDetails.webContextRoot=result.webContextRoot projectDetails.mailRecipients=result.mailRecipients projectDetails.isAnt=result.isAnt projectDetails.isMaven=result.isMaven projectDetails.isLibrary=result.isLibrary projectDetails.isBatch=result.isBatch projectDetails.isWebApp=result.isWebApp projectDetails.isArchetype=result.isArchetype projectDetails.javaVersion = result.javaVersion } projectDetails } // Reads Git Project details via GitLab REST API using Git projectId def getGitProjectDetails(def projectId) { def projectDetailsApi = new URL("${gitUrl}/projects/${projectId}?private_token=${gitPrivateToken}") def projectDetails = new groovy.json.JsonSlurper().parse(projectDetailsApi.newReader()) def projectTags = projectDetails.tag_list def defaultBranch = projectDetails.default_branch // determines the Git projects assigned department // department name is stored in Git Repository Tag. // defaults to 'NOGROUP' def validProjectGroup = "NOGROUP" departments.each { if ( projectTags != null && projectTags.contains(it) ) { validProjectGroup = it } } // checks whether the git project is still active or already archived def isActive = true if( projectDetails.archived ) { isActive = false } println "-> Get project details from ${gitUrl}/projects/${projectId}?private_token=${gitPrivateToken}" println "-> validProjectGroup=${validProjectGroup}" println "-> defaultBranch=${defaultBranch}" println "-> isActive=${isActive}" def details = new LinkedHashMap(); details.validProjectGruppe=validProjectGroup details.defaultBranch=defaultBranch details.isActive=isActive details }