/* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-test-artifact' apply plugin: 'elasticsearch.dra-artifacts' esplugin { name = 'x-pack-ml' description = 'Elasticsearch Expanded Pack Plugin - Machine Learning' classname ='org.elasticsearch.xpack.ml.MachineLearning' hasNativeController =true nativeControllerEnabledSettings = ['xpack.ml.enabled', 'xpack.ml.autodetect_process'] extendedPlugins = ['x-pack-autoscaling', 'lang-painless'] } if (useDra == false) { repositories { exclusiveContent { forRepository { ivy { name = "ml-cpp" url = providers.systemProperty('build.ml_cpp.repo').orElse('https://prelert-artifacts.s3.amazonaws.com').get() metadataSources { // no repository metadata, look directly for the artifact artifact() } patternLayout { artifact "maven/org/elasticsearch/ml/ml-cpp/[revision]/[module]-[revision](-[classifier]).[ext]" } } } filter { includeGroup 'org.elasticsearch.ml' } } } } configurations { nativeBundle { resolutionStrategy.cacheChangingModulesFor 2, 'hours' } } esplugin.bundleSpec.from { configurations.nativeBundle.files.collect { zipTree(it) } } // this is required due to https://github.com/nebula-plugins/gradle-ospackage-plugin/issues/472 // those lib files should not be executable in the first place esplugin.bundleSpec.filesMatching("platform/**/lib/*") { details -> details.permissions { unix(0644) } } // We don't ship the individual nativeBundle licenses - instead // they get combined into the top level NOTICES file we ship esplugin.bundleSpec.exclude 'platform/licenses/**' // The controller-protocol.version marker (see verifyControllerProtocolVersion below) is // deliberately published in BOTH the deps and nodeps nativeBundle artifacts so that either one // alone is sufficient for the build-time compatibility check. Once both zips resolve // successfully, esplugin.bundleSpec merges their trees via a single Copy task, and a file // present at the same path in both zips is a duplicate Gradle refuses to copy without an // explicit strategy. The marker is build-time-only metadata for verifyControllerProtocolVersion // (which reads it directly from configurations.nativeBundle, not from this bundleSpec) and has // no runtime purpose in the shipped plugin, so exclude it here rather than pick a duplicates // strategy that would silently prefer one artifact's copy over the other. esplugin.bundleSpec.exclude 'controller-protocol.version' ["bundlePlugin", "explodedBundlePlugin"].each { bundleTaskName -> tasks.named(bundleTaskName).configure { dependsOn configurations.nativeBundle dependsOn "verifyControllerProtocolVersion" } } dependencies { testImplementation project(path: ':x-pack:plugin:inference') compileOnly project(':modules:lang-painless:spi') compileOnly project(path: xpackModule('core')) compileOnly project(path: xpackModule('autoscaling')) compileOnly project(path: xpackModule('ml-package-loader')) testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(path: xpackModule('ilm')) testImplementation project(path: xpackModule('shutdown')) testImplementation project(':modules:data-streams') testImplementation project(path: xpackModule('monitoring')) testImplementation project(':modules:ingest-common') testImplementation project(':modules:reindex') testImplementation project(':modules:analysis-common') testImplementation project(':modules:mapper-extras') testImplementation project(':modules:lang-mustache') // This should not be here testImplementation(testArtifact(project(xpackModule('security')))) testImplementation project(path: xpackModule('wildcard')) // ml deps api project(':libs:grok') api project(':modules:lang-mustache') api "org.apache.commons:commons-math3:${versions.commons_math3}" api "com.ibm.icu:icu4j:${versions.icu4j}" api "org.apache.lucene:lucene-analysis-icu:${versions.lucene}" api "org.apache.lucene:lucene-analysis-kuromoji:${versions.lucene}" implementation 'org.ojalgo:ojalgo:51.2.0' nativeBundle("org.elasticsearch.ml:ml-cpp:${mlCppVersion()}:deps@zip") { changing = true } nativeBundle("org.elasticsearch.ml:ml-cpp:${mlCppVersion()}:nodeps@zip") { changing = true } testImplementation 'org.ini4j:ini4j:0.5.2' testImplementation "com.google.jimfs:jimfs:${versions.jimfs}" } def mlCppVersion(){ return (project.gradle.parent != null && buildParams.snapshotBuild == false) ? (project.version + "-SNAPSHOT") : project.version; } // Minimum controller-protocol.version that ml-cpp's native controller must publish (see ml-cpp's own // build.gradle, next to its controller-protocol.version publication). Bumping this constant is how a // future breaking change to the controller protocol gets enforced at Elasticsearch build time - // instead of relying on convention - so we can never ship an ES build that assumes new controller // capabilities against an old, incompatible native controller. def REQUIRED_CONTROLLER_PROTOCOL_VERSION = 2 tasks.register("verifyControllerProtocolVersion") { def nativeBundleFiles = objects.fileCollection().from(configurations.nativeBundle) def archiveOps = project.services.get(org.gradle.api.file.ArchiveOperations) dependsOn nativeBundleFiles inputs.files(nativeBundleFiles).withPropertyName("nativeBundle") doLast { // This is a hard build-time failure, not an advisory-only check. PyTorchBuilder emits, on Linux, // exactly one of '--disableSandbox' (xpack.ml.trained_models.sandbox_enabled=false, the default) // or '--requireSandbox' (sandbox_enabled=true) on every deployment start - never neither. Only a // native controller built against this same controller-protocol version recognizes either token; // an older controller forwards an unrecognized token verbatim to the child, which aborts at // startup. Both settings values depend on this compatibility check equally - there is no // sandbox-setting value for which an incompatible pairing is safe. This Elasticsearch build must // not be assembled/shipped until the paired ml-cpp artifact (publishing 'controller-protocol.version' // at >= REQUIRED_CONTROLLER_PROTOCOL_VERSION) is bundled - see // https://github.com/elastic/ml-cpp/pull/3188. // // ml-cpp (elastic/ml-cpp#3188) deliberately publishes the marker in BOTH nativeBundle artifacts // (deps@zip and nodeps@zip), so that either one alone satisfies this check. This gate is therefore // an OR across the resolved artifacts: it accepts the marker in EITHER artifact, at ANY depth // rather than only the archive root, since the exact publication path is not part of the contract. // A single resolved artifact carrying a valid, sufficiently new marker is enough to pass, whatever // any other resolved artifact contains (missing, malformed, or stale marker). Requiring the marker // in every artifact would make the gate needlessly fragile against future repackaging on either side. // (The marker's runtime duplicate - both zips carrying it at the same path - is handled separately by // excluding it from esplugin.bundleSpec above; here it is read directly from configurations.nativeBundle.) def matchesByZip = nativeBundleFiles.files.collectEntries { zipFile -> [zipFile, archiveOps.zipTree(zipFile).matching { include '**/controller-protocol.version' }.files] } if (matchesByZip.values().every { it.isEmpty() }) { def checkedNames = nativeBundleFiles.files.collect { it.name }.join(', ') throw new GradleException( "None of the resolved native controller bundles (checked: ${checkedNames}) contain a " + "'controller-protocol.version' file, at any depth (required: controller-protocol-version >= " + "${REQUIRED_CONTROLLER_PROTOCOL_VERSION}). The resolved ml-cpp artifact predates the " + "controller-protocol.version publication and is incompatible with this Elasticsearch build." ) } // Genuine OR-across-artifacts: a single artifact with a valid, sufficiently new marker is // enough to pass, regardless of what any other resolved artifact contains (malformed marker, // stale version, or - per the nodeps case above - no marker at all). Collect failure reasons // from every checked artifact instead of throwing on the first one, so the eventual failure // message (only emitted if NO artifact passes) is complete rather than a single hit. def failureReasons = [] def foundPassingVersion = false matchesByZip.each { zipFile, matches -> matches.each { versionFile -> // Match line-by-line (each trimmed individually) rather than the whole trimmed file content, // so trailing blank lines, CRLF line endings, or an extra comment line the ml-cpp side might // legitimately add don't trip the "malformed" check below. def versionLinePattern = ~/^controller-protocol-version=(\d+)$/ def matcher = null for (fileLine in versionFile.text.readLines()) { def candidate = versionLinePattern.matcher(fileLine.trim()) if (candidate.matches()) { matcher = candidate break } } if (matcher == null) { failureReasons.add( "Native controller bundle '${zipFile.name}' has a malformed 'controller-protocol.version' file " + "(got '${versionFile.text.trim()}', expected a line matching 'controller-protocol-version=')." ) return } def foundVersion = matcher.group(1).toInteger() if (foundVersion < REQUIRED_CONTROLLER_PROTOCOL_VERSION) { failureReasons.add( "Native controller bundle '${zipFile.name}' publishes controller-protocol version ${foundVersion}, " + "but this Elasticsearch build requires controller-protocol version >= " + "${REQUIRED_CONTROLLER_PROTOCOL_VERSION}. Upgrade the resolved ml-cpp artifact." ) return } foundPassingVersion = true } } if (!foundPassingVersion) { throw new GradleException( "None of the resolved native controller bundles carry a valid, sufficiently new " + "'controller-protocol.version' marker (required: controller-protocol-version >= " + "${REQUIRED_CONTROLLER_PROTOCOL_VERSION}). Reasons:\n" + failureReasons.collect { " - ${it}" }.join('\n') ) } } } tasks.named('assemble').configure { dependsOn tasks.named('jar') } tasks.register("extractNativeLicenses", Copy) { dependsOn configurations.nativeBundle into "${buildDir}/extractedNativeLicenses" from { configurations.nativeBundle.files.collect { zipTree(it) } } include 'platform/licenses/**' } // Add an extra licenses directory to the combined notices tasks.named('generateNotice').configure { dependsOn "extractNativeLicenses" inputs.dir("${project.buildDir}/extractedNativeLicenses/platform/licenses") .withPropertyName('licensingDir') .withPathSensitivity(PathSensitivity.RELATIVE) licenseDirs.add(tasks.named("extractNativeLicenses").map {new File(it.destinationDir, "platform/licenses") }) } tasks.named("dependencyLicenses").configure { mapping from: /lucene-.*/, to: 'lucene' } addQaCheckDependencies(project)