/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import java.util.regex.Matcher import java.util.regex.Pattern import org.apache.commons.codec.digest.DigestUtils description = 'Solr Docker image' apply plugin: 'base' // Solr Docker inputs def dockerImageSolrDist = "${ -> propertyOrEnvOrDefault("solr.docker.dist", "SOLR_DOCKER_DIST", 'full') }" def isDistSlim = {String dist -> dist.toLowerCase(Locale.ROOT) == "slim"} def isImageSlim = { -> isDistSlim(dockerImageSolrDist) } def distToSuffix = {String dist -> isDistSlim(dist) ? "-slim" : ""} def dockerImageDistSuffix = "${ -> distToSuffix(dockerImageSolrDist)}" def dockerImageTagSuffix = "${ -> propertyOrEnvOrDefault("solr.docker.imageTagSuffix", "SOLR_DOCKER_IMAGE_TAG_SUFFIX", '')}" def dockerImageRepo = "${ -> propertyOrEnvOrDefault("solr.docker.imageRepo", "SOLR_DOCKER_IMAGE_REPO", "apache/solr") }" def dockerImageTag = "${ -> propertyOrEnvOrDefault("solr.docker.imageTag", "SOLR_DOCKER_IMAGE_TAG", project.version + dockerImageDistSuffix) }" def dockerImageName = "${ -> propertyOrEnvOrDefault("solr.docker.imageName", "SOLR_DOCKER_IMAGE_NAME", "${dockerImageRepo}:${dockerImageTag}${dockerImageTagSuffix}") }" def baseDockerImage = "${ -> propertyOrEnvOrDefault("solr.docker.baseImage", "SOLR_DOCKER_BASE_IMAGE", 'eclipse-temurin:17-jre-jammy') }" def officialDockerImageName = {String dist -> "${ -> propertyOrEnvOrDefault("solr.docker.imageName", "SOLR_DOCKER_IMAGE_NAME", "${dockerImageRepo}-official:${dockerImageTag}${distToSuffix(dist)}${dockerImageTagSuffix}") }" } def releaseGpgFingerprint = "${ -> propertyOrDefault('signing.gnupg.keyName',propertyOrDefault('signing.keyId','')) }" // Build directory locations def imageIdFile = "$buildDir/image-id" def smokeTestOfficial = "$buildDir/smoke-check-official" def imageIdFileOfficial = { String dist -> "$smokeTestOfficial/$dist/image-id" } configurations { packaging { canBeResolved = true } packagingLocal { canBeResolved = true } packagingOfficial { canBeResolved = true } solrFullTgz { canBeConsumed = false canBeResolved = true } solrSlimTgz { canBeConsumed = false canBeResolved = true } solrFullTgzSignature { canBeConsumed = false canBeResolved = true } solrSlimTgzSignature { canBeConsumed = false canBeResolved = true } dockerImage { canBeResolved = true } dockerOfficialFullSmokeCheckImage { canBeConsumed = false canBeResolved = true } dockerOfficialSlimSmokeCheckImage { canBeConsumed = false canBeResolved = true } } ext { dockerfilesDirPath = "${buildDir}/dockerfiles" packagingDir = file("${buildDir}/packaging") } dependencies { packaging files(packagingDir) { builtBy 'assemblePackaging' } packagingLocal files("${dockerfilesDirPath}/Dockerfile.local") { builtBy 'createDockerfileLocal' } packagingOfficial files("${dockerfilesDirPath}/Dockerfile.official-full") { builtBy 'createDockerfileOfficialFull' } packagingOfficial files("${dockerfilesDirPath}/Dockerfile.official-slim") { builtBy 'createDockerfileOfficialSlim' } solrFullTgz project(path: ":solr:packaging", configuration: "solrFullTgz") solrSlimTgz project(path: ":solr:packaging", configuration: "solrSlimTgz") solrFullTgzSignature project(path: ":solr:packaging", configuration: 'solrFullTgzSignature') solrSlimTgzSignature project(path: ":solr:packaging", configuration: 'solrSlimTgzSignature') dockerImage files(imageIdFile) { builtBy 'dockerBuild' } dockerOfficialFullSmokeCheckImage files(imageIdFileOfficial("Full")) { builtBy 'testBuildDockerfileOfficialFull' } dockerOfficialSlimSmokeCheckImage files(imageIdFileOfficial("Slim")) { builtBy 'testBuildDockerfileOfficialSlim' } } // We're using commons-codec for computing checksums. buildscript { repositories { mavenCentral() } dependencies { classpath "commons-codec:commons-codec:${scriptDepVersions['commons-codec']}" } } def checksum = { file -> return new DigestUtils(DigestUtils.sha512Digest).digestAsHex(file).trim() } task assemblePackaging(type: Sync) { description = 'Assemble docker scripts and Dockerfile for Solr Packaging' dependsOn configurations.packagingLocal from(projectDir, { include "scripts/**" include "README.md" }) from(configurations.packagingLocal, { include 'Dockerfile.local' rename { 'Dockerfile' } }) into packagingDir } task dockerBuild() { group = 'Docker' description = 'Build Solr docker image' // Ensure that the docker image is rebuilt on build-arg changes or changes in the docker context inputs.properties([ baseDockerImage: baseDockerImage ]) var solrTgzConfiguration = isImageSlim() ? configurations.solrSlimTgz : configurations.solrFullTgz inputs.files(solrTgzConfiguration) inputs.property("isSlimImage", isImageSlim()) dependsOn(solrTgzConfiguration) doLast { exec { standardInput = solrTgzConfiguration.singleFile.newDataInputStream() commandLine "docker", "build", "-f", "solr-${ -> project.version }${dockerImageDistSuffix}/docker/Dockerfile", "--iidfile", imageIdFile, "--build-arg", "BASE_IMAGE=${ -> inputs.properties.baseDockerImage}", "-" } } // Print information on the image after it has been created doLast { def dockerImageId = file(imageIdFile).text project.logger.lifecycle("Solr Docker Image Created") project.logger.lifecycle("\tID: \t${ -> dockerImageId }") project.logger.lifecycle("\tBase Image: \t${ -> baseDockerImage }") project.logger.lifecycle("\tSolr Version: \t${ -> project.version }") project.logger.lifecycle("\tSolr Distribution: \t${isImageSlim() ? "Slim" : "Full"}") } outputs.files(imageIdFile) } task dockerTag(dependsOn: tasks.dockerBuild) { group = 'Docker' description = 'Tag Solr docker image' def dockerImageIdFile = file(imageIdFile) // Ensure that the docker image is re-tagged if the image ID or desired tag changes inputs.properties([ dockerImageName: dockerImageName, ]) inputs.file(dockerImageIdFile) doLast { def dockerImageId = dockerImageIdFile.text exec { commandLine "docker", "tag", dockerImageId, inputs.properties.dockerImageName } // Print information on the image after it has been created project.logger.lifecycle("Solr Docker Image Tagged") project.logger.lifecycle("\tID: \t$dockerImageId") project.logger.lifecycle("\tTag: \t$dockerImageName") } } task testDocker(type: TestDockerImageTask, dependsOn: tasks.dockerBuild) { group = 'Docker' description = 'Test Solr docker image built from the local Dockerfile' idFile = tasks.dockerBuild.outputs.files.singleFile outputDir = file("$buildDir/test-results") // include/exclude options are designed for people who know their customizations will break some tests testCasesInclude.value(Arrays.asList(propertyOrEnvOrDefault("solr.docker.tests.include", "SOLR_DOCKER_TESTS_INCLUDE", ",").split(","))) var excludeCases = new HashSet(Arrays.asList(propertyOrEnvOrDefault("solr.docker.tests.exclude", "SOLR_DOCKER_TESTS_EXCLUDE", ",").split(","))) if (isImageSlim()) { // The slim image does not contain the prometheus-exporter. excludeCases.add("prometheus-exporter-standalone") excludeCases.add("prometheus-exporter-cloud") } testCasesExclude.value(excludeCases) } task dockerPush(dependsOn: tasks.dockerTag) { group = 'Docker' description = 'Push Solr docker image' // Ensure that the docker image is re-pushed if the image ID or tag changes inputs.properties([ dockerImageName: dockerImageName, ]) inputs.file(imageIdFile) // We don't want to push a docker image unless the tests have passed mustRunAfter tasks.testDocker doLast { exec { commandLine "docker", "push", dockerImageName } // Print information on the image after it has been created project.logger.lifecycle("Solr Docker Image Pushed: \t$dockerImageName") } } // One task to build and tag a Solr docker image task docker { dependsOn tasks.dockerBuild, tasks.dockerTag } ext { // Filters/Patterns for re-use in multiple "template" related tasks commentFilter = { line -> if (line.startsWith('#-#')) { return null; } return line; } propReplacePattern = Pattern.compile('_REPLACE_((_|\\p{Upper})+)_') } task createBodySnippetDockerfile(type: Copy) { from 'templates/Dockerfile.body.template' into "$buildDir/snippets/" outputs.file("$buildDir/snippets/Dockerfile.body.snippet") rename { name -> name.replace("template","snippet") } filteringCharset 'UTF-8' // NOTE: The only "templating" the Dockerfile.body supports is removing comments. // // Any situation where it feel appropriate to add variable substitution should be reconsidered, and probably // implemented as either a build-arg or as a variable expanded in the header snippets (or both) filter( commentFilter ) } ext { // NOTE: 'props' are variables that will be replaced in the respective templates, // and they must consist solely of characters matching the regex '(_|\\p{Upper})+'. // They may only be used in ARG and FROM lines, via the syntax: '_REPLACE_FOO_' // where 'FOO' is the key used in 'props' (NOTE the leading and trailing literal '_' characters) dfLocalDetails = [ name: 'Local', template: 'local', desc: 'Dockerfile used to create local Solr docker images directly from Solr release tgz file', // NOTE: There should be no reason for Dockerfile.local to include unique values // // Values identical in both Dockerfiles should use consistent names in both templates and // be defined in the task creation. props: [:] ] dfOfficialFullDetails = [ name: 'OfficialFull', template: 'official', desc: 'Dockerfile used to create official Full Solr docker images published to hub.docker.io', props: [ // NOTE: Only include values here that are distinct and unique to the Official Dockerfiles // // Values identical in both Dockerfiles should use consistent names in both templates and // be defined in the task creation // NOTE: SHA is lazy computed... 'SOLR_TGZ_SHA': "${ -> checksum(configurations.solrFullTgz.singleFile) }", 'RELEASE_MANAGER_GPG_FINGERPRINT': "${ -> releaseGpgFingerprint}", 'SOLR_DIST': "" ] ] dfOfficialSlimDetails = [ name: 'OfficialSlim', template: 'official', desc: 'Dockerfile used to create official Slim Solr docker images published to hub.docker.io', props: [ // NOTE: Only include values here that are distinct and unique to the Official Dockerfiles // // Values identical in both Dockerfiles should use consistent names in both templates and // be defined in the task creation // NOTE: SHA is lazy computed... 'SOLR_TGZ_SHA': "${ -> checksum(configurations.solrSlimTgz.singleFile) }", 'RELEASE_MANAGER_GPG_FINGERPRINT': "${ -> releaseGpgFingerprint}", 'SOLR_DIST': "-slim" ] ] } /* This section creates the following gradle tasks: - createDockerfileOfficialFull - createDockerfileOfficialSlim - createDockerfileLocal Both will create a self-standing Dockerfile that can be used to build a Solr image. These are templated using a unique header file for each Dockerfile and the same body template for the logic on installing Solr. These templates can be found in the templates/ directory. The snippets of each section (header and body) are saved to build/snippets after they are templated and before they are combined. The final Dockerfiles are merely the snippet headers combined with the snippet body. */ [ dfLocalDetails, dfOfficialFullDetails, dfOfficialSlimDetails ].each{ details -> def fileName = "Dockerfile.${ -> details.name.replaceAll("(.)([A-Z])", "\$1-\$2").toLowerCase(Locale.ROOT) }" def outFile = file("$dockerfilesDirPath/${fileName}") tasks.create("createDockerfile${details.name}", Copy) { description "Creates ${details.desc}" def props = [ // Values defined here should be common (and consistent) across both Dockerfiles 'BASE_IMAGE': baseDockerImage, 'SOLR_VERSION': "${ -> project.version}", * : details.props ] dependsOn tasks.createBodySnippetDockerfile inputs.properties(props) inputs.file("$buildDir/snippets/Dockerfile.body.snippet") outputs.file(outFile) from "templates/Dockerfile.${details.template}.header.template" into "$buildDir/snippets/" rename { fileName + ".header.snippet" } filteringCharset 'UTF-8' filter( commentFilter ) filter( { line -> if ( line.startsWith("FROM ") || line.startsWith("ARG ") ) { Matcher matcher = project.ext.propReplacePattern.matcher(line); StringBuilder sb = new StringBuilder(); if (matcher.find()) { String key = matcher.group(1); if (null == key || key.isEmpty() || ( ! props.containsKey(key) ) ) { throw new GradleException("Line contains invalid REPLACE variable (" + key + "): " + line); } matcher.appendReplacement(sb, props.get(key) ); } matcher.appendTail(sb); return sb.toString(); } return line; }) doLast { outFile.withWriter('UTF-8') { writer -> files("$buildDir/snippets/${fileName}.header.snippet", "$buildDir/snippets/Dockerfile.body.snippet").each { snippet -> snippet.withReader('UTF-8') { reader -> writer << reader } } } } } } assemblePackaging.dependsOn tasks.createDockerfileLocal tasks.createDockerfileOfficialFull.dependsOn configurations.solrFullTgz // to lazy compute SHA tasks.createDockerfileOfficialSlim.dependsOn configurations.solrSlimTgz // to lazy compute SHA tasks.createDockerfileOfficialSlim.mustRunAfter tasks.createDockerfileOfficialFull // Eliminate a weird warning task createDockerfileOfficial { dependsOn tasks.createDockerfileOfficialFull dependsOn tasks.createDockerfileOfficialSlim } // sanity check... if (''.equals(releaseGpgFingerprint)) { gradle.taskGraph.whenReady { graph -> if ( graph.hasTask(tasks.createDockerfileOfficialFull) || graph.hasTask(tasks.createDockerfileOfficialSlim) ) { throw new GradleException("No GPG keyName found, please see help/publishing.txt (GPG key is neccessary to create Dockerfile.official)") } } } [ 'Full', 'Slim' ].each { variant -> def lowerVariant = variant.toLowerCase(Locale.ROOT) tasks.create("testBuildDockerfileOfficial${variant}", Copy) { description = "Test 'docker build' works with our generated Dockerfile.official-${lowerVariant} using Mocked URLs" dependsOn tasks.named("createDockerfileOfficial${variant}") def mockHttpdHome = file("$smokeTestOfficial/$lowerVariant/mock-httpd-home"); inputs.file("$dockerfilesDirPath/Dockerfile.official-$lowerVariant") outputs.dir(mockHttpdHome) outputs.file(imageIdFileOfficial(variant)) dependsOn configurations.named("solr${variant}Tgz") dependsOn configurations.named("solr${variant}TgzSignature") from configurations.named("solr${variant}Tgz") from configurations.named("solr${variant}TgzSignature") into mockHttpdHome doLast { // A file to record the container ID of our mock httpd def mockServerIdFile = file("$smokeTestOfficial/$lowerVariant/dockerfile-mock-artifact-server-cid.txt") // if we encounter any problems running our test, we'll fill this in and use it to suppress any // other exceptions we encounter on cleanup... def mainException = null; // TODO: setup a little 'suppressOrThrow(Exception)' closure for reuse below.... try { // run an httpd server to host our artifacts logger.lifecycle('Running mock HTTPD server our testing...'); exec { commandLine 'docker', 'run', '--cidfile', mockServerIdFile, '--rm', '-d', '-v', "${mockHttpdHome.absoluteFile}:/data/${-> project.version}", '-w', '/data', '-p', '9876:9876', 'python:3-alpine', 'python', '-m', 'http.server', '9876' } try { def mockServerId = mockServerIdFile.text def mockServerIpStdOut = new ByteArrayOutputStream() exec { commandLine 'docker', 'inspect', "--format={{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", mockServerId standardOutput = mockServerIpStdOut } def mockServerIp = mockServerIpStdOut.toString().trim() // *NOW* we can actually run our docker build command... logger.lifecycle('Running docker build on Dockerfile.official...'); def downloadHost = "mock-downloads.apache.org" // We are not signing, so do not use an apache.org host, as that will check the // GPG signature, which will not be created. if (!project(":solr:distribution").ext.withSignedArtifacts) { downloadHost = "mock-downloads.test.org" } exec { standardInput = file("${dockerfilesDirPath}/Dockerfile.official-${lowerVariant}").newDataInputStream() commandLine 'docker', 'build', '--add-host', "${downloadHost}:${mockServerIp}", '--no-cache', // force fresh downloads from our current network "--iidfile", imageIdFileOfficial(variant), '--build-arg', "SOLR_DOWNLOAD_SERVER=http://${downloadHost}:9876", '--tag', officialDockerImageName(variant), '-' } def officialDockerImageId = file(imageIdFileOfficial(variant)).text // Print information on the image after it has been created project.logger.lifecycle("\"Official $variant\" Solr Docker Image Tagged") project.logger.lifecycle("\tID: \t$officialDockerImageId") project.logger.lifecycle("\tTag: \t${officialDockerImageName(variant)}") } finally { // Try to shut down our mock httpd server.... if (mockServerIdFile.exists()) { def mockServerId = mockServerIdFile.text try { exec { commandLine 'docker', 'stop', mockServerId } } catch (Exception e) { logger.error("Unable to stop docker container ${mockServerId}", e) if (null != mainException) { mainException.addSuppressed(e); } else { mainException = e; throw e; } } finally { project.delete(mockServerIdFile) } } } } catch (Exception e) { mainException = e throw e; } } } } task testBuildDockerfileOfficial { dependsOn tasks.testBuildDockerfileOfficialFull dependsOn tasks.testBuildDockerfileOfficialSlim } task testDockerfileOfficialFull(type: TestDockerImageTask, dependsOn: configurations.dockerOfficialFullSmokeCheckImage) { description = 'Smoke Test Full Solr docker image built from the official Dockerfile' idFile = file(imageIdFileOfficial("Full")) outputDir = file("$smokeTestOfficial/full/test-results") // This test does not respect the include/exclude properties that `testDocker` does. // All docker tests will be run, no matter the properties specified. testCasesInclude.empty() testCasesExclude.empty() } task testDockerfileOfficialSlim(type: TestDockerImageTask, dependsOn: configurations.dockerOfficialSlimSmokeCheckImage) { description = 'Smoke Test Slim Solr docker image built from the official Slim Dockerfile' idFile = file(imageIdFileOfficial("Slim")) outputDir = file("$smokeTestOfficial/slim/test-results") // This test does not respect the include/exclude properties that `testDocker` does. // All docker tests will be run, no matter the properties specified. testCasesInclude.empty() testCasesExclude = ['prometheus-exporter-cloud', 'prometheus-exporter-standalone'] } task testDockerfileOfficial { dependsOn tasks.testDockerfileOfficialFull dependsOn tasks.testDockerfileOfficialSlim } // Re-usable class for running tests... abstract class TestDockerImageTask extends DefaultTask { // Ensure that the docker image is re-tested if the image ID changes or the test files change @InputFile abstract public RegularFileProperty getIdFile() // Ensure that the docker image is re-tested if the shared test library changes @InputFile File sharedTestLibraryFile = project.file("tests/shared.sh") @InputDirectory File sourceDir = project.file("tests/cases") @Input SetProperty testCasesInclude = project.objects.setProperty(String) @Input SetProperty testCasesExclude = project.objects.setProperty(String) @OutputDirectory abstract public DirectoryProperty getOutputDir() @Inject abstract public WorkerExecutor getWorkerExecutor(); public static interface SingleTestParameters extends WorkParameters { // NOTE: we explicitly don't use DirectoryProperty here because the way WorkerExecutor serializes the params // causes weird "wrapped" objects to come back that don't work when you try to call `.getAsFile()` or `.getAsFile().getPath()` Property getTestName(); Property getTestDir(); Property getWorkDir(); Property getImageId(); } public abstract static class SingleTestAction implements WorkAction { @Inject abstract public ExecOperations getExec(); @Override public void execute() { def testCaseName = getParameters().getTestName().get() def testCaseDir = getParameters().getTestDir().get() def testCaseWorkDir = getParameters().getWorkDir().get() def testCaseBuildDir = "${testCaseWorkDir}/build_dir" System.out.println("Starting Solr Docker test: ${testCaseName}") def res = getExec().exec { // we'll handle it ourselves so we can report the details of which test failed ignoreExitValue true environment "DEBUG", "true" environment "TEST_DIR", testCaseDir environment "BUILD_DIR", testCaseBuildDir standardOutput new FileOutputStream("${testCaseWorkDir}/test.std.out") errorOutput new FileOutputStream("${testCaseWorkDir}/test.err.out") commandLine "bash", "${testCaseDir}/test.sh", getParameters().getImageId().get() } def ev = res.getExitValue() if (0 != ev) { throw new GradleException("Docker test failure=${ev}: Test: ${testCaseName} Output: ${testCaseWorkDir}"); } else { System.out.println("Completed Solr Docker test: ${testCaseName}") } } } @TaskAction public void runTests() { // no easy way to control the amount of parallelization of the work queue (independent of `org.gradle.workers.max`) // so the best we can do (simply) is a boolean setting that controls if we `await()` after each `submit()` def workQueue = getWorkerExecutor().noIsolation(); def runConcurrentTests = project.propertyOrEnvOrDefault("solr.docker.tests.concurrent", "SOLR_DOCKER_TESTS_CONCURRENT", "false").toBoolean() def imageId = idFile.getAsFile().get().text // Print information on the image before it is tested logger.lifecycle("Testing Solr Image ID: $imageId ${ -> runConcurrentTests ? '(concurrently)' : '(sequentially)'}") def includes = testCasesInclude.get() def excludes = testCasesExclude.get() // "Run" each of the test cases sourceDir.eachFile { file -> def testName = file.getName() def outDir = outputDir.get() def testCaseWorkDir = outDir.dir(testName) // If specific tests are specified, only run those. Otherwise run all that are not ignored. def runTest = !includes.isEmpty() ? includes.contains(testName) : !excludes.contains(testName) if (runTest) { project.mkdir testCaseWorkDir def paramSetup = { params -> params.getTestName().set(testName) params.getTestDir().set(file.getPath()) params.getWorkDir().set(testCaseWorkDir.getAsFile().getPath()) params.getImageId().set(imageId) } workQueue.submit(SingleTestAction.class, paramSetup); if (! runConcurrentTests) { workQueue.await(); } } } } }