diff --git a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java index 84faa2f2d2d..7bb8af95392 100644 --- a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java @@ -27,7 +27,6 @@ import org.apache.commons.cli.Option; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.core.ConfigSetService; -import org.apache.solr.util.FileTypeMagicUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -130,7 +129,6 @@ public class ConfigSetUploadTool extends ToolBase { + cli.getOptionValue("conf-name") + " to ZooKeeper at " + zkHost); - FileTypeMagicUtil.assertConfigSetFolderLegal(confPath); ZkMaintenanceUtils.uploadToZK( zkClient, confPath, diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.java index e678e44d725..8bbdc8ac098 100644 --- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java +++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java @@ -41,7 +41,7 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.CoreAdminRequest; import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.response.CoreAdminResponse; -import org.apache.solr.common.cloud.ZkMaintenanceUtils; +import org.apache.solr.cloud.ZkConfigSetService; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CommonParams; @@ -311,7 +311,10 @@ public class CreateTool extends ToolBase { confName = collectionName; } + // TODO: This should be done using the configSet API final Path configsetsDirPath = SolrCLI.getConfigSetsDir(solrInstallDirPath); + ConfigSetService configSetService = + new ZkConfigSetService(ZkStateReader.from(cloudSolrClient).getZkClient()); Path confPath = ConfigSetService.getConfigsetPath(confDir, configsetsDirPath.toString()); echoIfVerbose( @@ -322,11 +325,8 @@ public class CreateTool extends ToolBase { + " to ZooKeeper at " + cloudSolrClient.getClusterStateProvider().getQuorumHosts(), cli); - ZkMaintenanceUtils.uploadToZK( - ZkStateReader.from(cloudSolrClient).getZkClient(), - confPath, - ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + confName, - ZkMaintenanceUtils.UPLOAD_FILENAME_EXCLUDE_PATTERN); + // We will trust the config since we have the Zookeeper Address + configSetService.uploadConfig(confName, confPath, true); } // since creating a collection is a heavy-weight operation, check for existence first diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java index 5057e62613f..2b3d1d9d72a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java @@ -29,6 +29,7 @@ import org.apache.solr.client.solrj.cloud.SolrCloudManager; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkMaintenanceUtils; +import org.apache.solr.common.util.FileTypeMagicUtil; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Utils; import org.apache.solr.core.ConfigSetProperties; @@ -37,7 +38,6 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.util.FileTypeMagicUtil; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.data.Stat; @@ -167,7 +167,7 @@ public class ZkConfigSetService extends ConfigSetService { } @Override - public void uploadConfig(String configName, Path dir) throws IOException { + protected void uploadConfig(String configName, Path dir) throws IOException { zkClient.uploadToZK( dir, CONFIGS_ZKNODE + "/" + configName, ConfigSetService.UPLOAD_FILENAME_EXCLUDE_PATTERN); } @@ -230,6 +230,8 @@ public class ZkConfigSetService extends ConfigSetService { (Map) Utils.fromJSON(zkClient.getData(CONFIGS_ZKNODE + "/" + configName, null, null, true)); return data; + } catch (KeeperException.NoNodeException e) { + return Collections.emptyMap(); } catch (KeeperException | InterruptedException e) { throw new IOException("Error getting config metadata", SolrZkClient.checkInterrupted(e)); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java index 55d1725e6ec..9a4b99514d0 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java @@ -150,6 +150,7 @@ public class RestoreCmd implements CollApiCmds.CollectionApiCommand { final URI location; final URI backupPath; final List nodeList; + final boolean requestIsTrusted; final CoreContainer container; final BackupRepository repository; @@ -165,6 +166,7 @@ public class RestoreCmd implements CollApiCmds.CollectionApiCommand { this.asyncId = message.getStr(ASYNC); this.repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY); this.backupId = message.getInt(CoreAdminParams.BACKUP_ID, -1); + this.requestIsTrusted = message.getBool(CoreAdminParams.TRUSTED, false); this.container = ccc.getCoreContainer(); this.repository = this.container.newBackupRepository(repo); @@ -228,7 +230,8 @@ public class RestoreCmd implements CollApiCmds.CollectionApiCommand { rc.backupProperties.getConfigName(), rc.restoreConfigName, rc.backupManager, - rc.container.getConfigSetService()); + rc.container.getConfigSetService(), + rc.requestIsTrusted); log.info( "Starting restore into collection={} with backup_name={} at location={}", @@ -293,7 +296,8 @@ public class RestoreCmd implements CollApiCmds.CollectionApiCommand { String configName, String restoreConfigName, BackupManager backupMgr, - ConfigSetService configSetService) + ConfigSetService configSetService, + boolean requestIsTrusted) throws IOException { if (configSetService.checkConfigExists(restoreConfigName)) { log.info( @@ -305,7 +309,8 @@ public class RestoreCmd implements CollApiCmds.CollectionApiCommand { "Config with name {} does not already exist in ZooKeeper. Will restore from Backup.", restoreConfigName); - backupMgr.uploadConfigDir(configName, restoreConfigName, configSetService); + backupMgr.uploadConfigDir( + configName, restoreConfigName, configSetService, requestIsTrusted); } } diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java index 32432d9f18b..53160cd75ee 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java @@ -23,6 +23,7 @@ import java.lang.invoke.MethodHandles; import java.lang.reflect.Constructor; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -116,7 +117,7 @@ public abstract class ConfigSetService { "intended to be the default. Current 'solr.default.confdir' value:", System.getProperty(SolrDispatchFilter.SOLR_DEFAULT_CONFDIR_ATTRIBUTE)); } else { - this.uploadConfig(ConfigSetsHandler.DEFAULT_CONFIGSET_NAME, configDirPath); + this.uploadConfig(ConfigSetsHandler.DEFAULT_CONFIGSET_NAME, configDirPath, true); } } } @@ -131,7 +132,7 @@ public abstract class ConfigSetService { String confName = System.getProperty( ZkController.COLLECTION_PARAM_PREFIX + ZkController.CONFIGNAME_PROP, "configuration1"); - this.uploadConfig(confName, configPath); + this.uploadConfig(confName, configPath, true); } /** @@ -215,7 +216,7 @@ public abstract class ConfigSetService { if (StrUtils.isNullOrEmpty(confName)) confName = coreName; Path udir = cd.getInstanceDir().resolve("conf"); log.info("Uploading directory {} with name {} for solrCore {}", udir, confName, coreName); - cc.getConfigSetService().uploadConfig(confName, udir); + cc.getConfigSetService().uploadConfig(confName, udir, true); } } @@ -244,6 +245,18 @@ public abstract class ConfigSetService { return (flags == null || flags.get("trusted") == null || flags.getBooleanArg("trusted")); } + /** + * Change the trust of the given configSet. + * + * @param name name of the configSet + * @param isTrusted whether the given configSet should be trusted or not + */ + public void setConfigSetTrust(String name, boolean isTrusted) throws IOException { + Map contentMap = new HashMap<>(getConfigMetadata(name)); + contentMap.put("trusted", isTrusted); + setConfigMetadata(name, contentMap); + } + /** * Load the ConfigSet for a core * @@ -406,7 +419,20 @@ public abstract class ConfigSetService { * @param dir {@link Path} to the files * @throws IOException if an I/O error occurs or the path does not exist */ - public abstract void uploadConfig(String configName, Path dir) throws IOException; + protected abstract void uploadConfig(String configName, Path dir) throws IOException; + + /** + * Upload files from a given path to config, which will explicitly be trusted or not. + * + * @param configName the config name + * @param dir {@link Path} to the files + * @param isTrusted whether the config being uploaded is trusted + * @throws IOException if an I/O error occurs or the path does not exist + */ + public void uploadConfig(String configName, Path dir, boolean isTrusted) throws IOException { + setConfigSetTrust(configName, isTrusted); + uploadConfig(configName, dir); + } /** * Upload a file to config If file does not exist, it will be uploaded If overwriteOnExists is set @@ -479,9 +505,25 @@ public abstract class ConfigSetService { * @param configName the config name * @param data the metadata to be set on config */ - public abstract void setConfigMetadata(String configName, Map data) + protected abstract void setConfigMetadata(String configName, Map data) throws IOException; + /** + * Set the config metadata If config does not exist, it will be created and set metadata on it + * Else metadata will be replaced with the provided metadata. This will preserve the trust that + * the configSet already has if trust is not included in the metadata. + * + * @param configName the config name + * @param data the metadata to be set on config + */ + public void setConfigMetadataWithTrust(String configName, Map data) + throws IOException { + if (!data.containsKey("trusted")) { + data.put("trusted", isConfigSetTrusted(configName)); + } + setConfigMetadata(configName, data); + } + /** * Get the config metadata (mutable, non-null) * diff --git a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java index c4686195d64..f31101bb4d4 100644 --- a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java @@ -36,8 +36,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkMaintenanceUtils; +import org.apache.solr.common.util.FileTypeMagicUtil; import org.apache.solr.common.util.Utils; -import org.apache.solr.util.FileTypeMagicUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -139,7 +139,7 @@ public class FileSystemConfigSetService extends ConfigSetService { } @Override - public void uploadConfig(String configName, Path source) throws IOException { + protected void uploadConfig(String configName, Path source) throws IOException { Path dest = getConfigDir(configName); copyRecursively(source, dest); } @@ -169,7 +169,11 @@ public class FileSystemConfigSetService extends ConfigSetService { @Override public void setConfigMetadata(String configName, Map data) throws IOException { // store metadata in .metadata.json file - Path metadataPath = getConfigDir(configName).resolve(METADATA_FILE); + Path configDir = getConfigDir(configName); + if (!Files.exists(configDir)) { + Files.createDirectory(configDir); + } + Path metadataPath = configDir.resolve(METADATA_FILE); Files.write(metadataPath, Utils.toJSON(data)); } diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java index e846da51659..2eb928ec663 100644 --- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java +++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java @@ -38,10 +38,10 @@ import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.FileTypeMagicUtil; import org.apache.solr.common.util.Utils; import org.apache.solr.core.ConfigSetService; import org.apache.solr.core.backup.repository.BackupRepository; -import org.apache.solr.util.FileTypeMagicUtil; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; @@ -255,12 +255,16 @@ public class BackupManager { * @throws IOException in case of I/O errors. */ public void uploadConfigDir( - String sourceConfigName, String targetConfigName, ConfigSetService configSetService) + String sourceConfigName, + String targetConfigName, + ConfigSetService configSetService, + boolean requestIsTrusted) throws IOException { URI source = repository.resolveDirectory(getZkStateDir(), CONFIG_STATE_DIR, sourceConfigName); if (!repository.exists(source)) { throw new IllegalArgumentException("Configset expected at " + source + " does not exist"); } + configSetService.setConfigSetTrust(targetConfigName, requestIsTrusted); uploadConfigToSolrCloud(configSetService, source, targetConfigName, ""); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java index e1e071e6418..ff8d9ac9690 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java @@ -31,6 +31,7 @@ import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_ID; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY; +import static org.apache.solr.common.params.CoreAdminParams.TRUSTED; import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; @@ -57,6 +58,7 @@ import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.core.CoreContainer; import org.apache.solr.handler.admin.CollectionsHandler; +import org.apache.solr.handler.configsets.ConfigSetAPIBase; import org.apache.solr.jersey.JacksonReflectMapWriter; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; @@ -155,7 +157,7 @@ public class RestoreCollectionAPI extends BackupAPIBase { return response; } - public static ZkNodeProps createRemoteMessage( + public ZkNodeProps createRemoteMessage( String backupName, RestoreCollectionRequestBody requestBody) { final Map remoteMessage = requestBody.toMap(new HashMap<>()); @@ -181,6 +183,10 @@ public class RestoreCollectionAPI extends BackupAPIBase { if (requestBody.backupId != null) remoteMessage.put(BACKUP_ID, requestBody.backupId); if (requestBody.repository != null) remoteMessage.put(BACKUP_REPOSITORY, requestBody.repository); + remoteMessage.put( + TRUSTED, + ConfigSetAPIBase.isTrusted( + solrQueryRequest.getUserPrincipal(), coreContainer.getAuthenticationPlugin())); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/ConfigSetAPIBase.java b/solr/core/src/java/org/apache/solr/handler/configsets/ConfigSetAPIBase.java index 87c60fda94a..3f401e31bd8 100644 --- a/solr/core/src/java/org/apache/solr/handler/configsets/ConfigSetAPIBase.java +++ b/solr/core/src/java/org/apache/solr/handler/configsets/ConfigSetAPIBase.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.io.InputStream; import java.lang.invoke.MethodHandles; import java.security.Principal; -import java.util.Collections; import java.util.Iterator; import java.util.Map; import java.util.Optional; @@ -124,8 +123,6 @@ public class ConfigSetAPIBase { boolean requestIsTrusted, String configName) throws IOException { - Map metadata = Collections.singletonMap("trusted", requestIsTrusted); - if (overwritesExisting) { if (!requestIsTrusted) { ensureOverwritingUntrustedConfigSet(configName); @@ -133,7 +130,7 @@ public class ConfigSetAPIBase { // If the request is trusted and cleanup=true, then the configSet will be set to trusted after // the overwriting has been done. } else { - configSetService.setConfigMetadata(configName, metadata); + configSetService.setConfigSetTrust(configName, requestIsTrusted); } } diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetAPI.java b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetAPI.java index f9bde932cb0..79d1b34d5ca 100644 --- a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetAPI.java @@ -25,7 +25,6 @@ import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import org.apache.solr.api.EndPoint; @@ -110,8 +109,7 @@ public class UploadConfigSetAPI extends ConfigSetAPIBase { && requestIsTrusted && overwritesExisting && !configSetService.isConfigSetTrusted(configSetName)) { - Map metadata = Collections.singletonMap("trusted", true); - configSetService.setConfigMetadata(configSetName, metadata); + configSetService.setConfigSetTrust(configSetName, true); } } diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java index 2380a79a92b..1c4754458e3 100644 --- a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java @@ -24,10 +24,10 @@ import org.apache.solr.api.EndPoint; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.common.params.ConfigSetParams; +import org.apache.solr.common.util.FileTypeMagicUtil; import org.apache.solr.core.CoreContainer; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.util.FileTypeMagicUtil; /** * V2 API for adding or updating a single file within a configset. diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java index db35a805c51..0125e7988d8 100644 --- a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java +++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java @@ -1231,8 +1231,7 @@ class SchemaDesignerConfigSetHelper implements SchemaDesignerConstants { public void removeConfigSetTrust(String configSetName) { try { - Map metadata = Collections.singletonMap("trusted", false); - cc.getConfigSetService().setConfigMetadata(configSetName, metadata); + cc.getConfigSetService().setConfigSetTrust(configSetName, false); } catch (IOException e) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java index a45232a3af9..68142392df1 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java @@ -147,7 +147,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { public void testCreateErrors() throws Exception { final String baseUrl = cluster.getJettySolrRunners().get(0).getBaseUrl().toString(); try (final SolrClient solrClient = getHttpSolrClient(baseUrl)) { - getConfigSetService().uploadConfig("configSet", configset("configset-2")); + getConfigSetService().uploadConfig("configSet", configset("configset-2"), false); // no action CreateNoErrorChecking createNoAction = new CreateNoErrorChecking(); @@ -298,7 +298,9 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { getConfigSetProps(oldProps), UTF_8); } - getConfigSetService().uploadConfig(baseConfigSetName, tmpConfigDir); + getConfigSetService() + .uploadConfig( + baseConfigSetName, tmpConfigDir, isTrusted(zkClient(), baseConfigSetName, "")); } private void verifyCreate( @@ -1350,12 +1352,15 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { throws KeeperException, InterruptedException { String configSetZkPath = String.format(Locale.ROOT, "/configs/%s%s", configsetName, configsetSuffix); - byte[] configSetNodeContent = zkClient.getData(configSetZkPath, null, null, true); - ; + try { + byte[] configSetNodeContent = zkClient.getData(configSetZkPath, null, null, true); - @SuppressWarnings("unchecked") - Map contentMap = (Map) Utils.fromJSON(configSetNodeContent); - return (boolean) contentMap.getOrDefault("trusted", true); + @SuppressWarnings("unchecked") + Map contentMap = (Map) Utils.fromJSON(configSetNodeContent); + return (boolean) contentMap.getOrDefault("trusted", true); + } catch (KeeperException.NoNodeException e) { + return true; + } } private int getConfigZNodeVersion( @@ -1928,7 +1933,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { tmpConfigDir.resolve("configsetprops.json"), getConfigSetProps(Map.of("immutable", "true")), UTF_8); - getConfigSetService().uploadConfig("configSet", tmpConfigDir); + getConfigSetService().uploadConfig("configSet", tmpConfigDir, false); // no ConfigSet name DeleteNoErrorChecking delete = new DeleteNoErrorChecking(); @@ -1956,7 +1961,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { final String baseUrl = cluster.getJettySolrRunners().get(0).getBaseUrl().toString(); final SolrClient solrClient = getHttpSolrClient(baseUrl); final String configSet = "testDelete"; - getConfigSetService().uploadConfig(configSet, configset("configset-2")); + getConfigSetService().uploadConfig(configSet, configset("configset-2"), false); assertDelete(solrClient, configSet, true); assertDelete(solrClient, "configSetBogus", false); solrClient.close(); @@ -2005,7 +2010,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { Set configSets = new HashSet(); for (int i = 0; i < 5; ++i) { String configSet = "configSet" + i; - getConfigSetService().uploadConfig(configSet, configset("configset-2")); + getConfigSetService().uploadConfig(configSet, configset("configset-2"), false); configSets.add(configSet); } response = list.process(solrClient); diff --git a/solr/core/src/test/org/apache/solr/core/TestConfigSetService.java b/solr/core/src/test/org/apache/solr/core/TestConfigSetService.java index e1f3787fc88..726ad98e8db 100644 --- a/solr/core/src/test/org/apache/solr/core/TestConfigSetService.java +++ b/solr/core/src/test/org/apache/solr/core/TestConfigSetService.java @@ -24,6 +24,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -78,7 +79,7 @@ public class TestConfigSetService extends SolrTestCaseJ4 { Files.createDirectory(configDir.resolve("subdir")); Files.createFile(configDir.resolve("subdir").resolve("file3")); - configSetService.uploadConfig(configName, configDir); + configSetService.uploadConfig(configName, configDir, false); assertTrue(configSetService.checkConfigExists(configName)); assertFalse(configSetService.checkConfigExists("dummyConfig")); @@ -92,14 +93,15 @@ public class TestConfigSetService extends SolrTestCaseJ4 { assertArrayEquals(configSetService.downloadFileFromConfig(configName, "subdir/file4"), data); Map metadata = configSetService.getConfigMetadata(configName); - assertTrue(metadata.isEmpty()); + assertFalse(metadata.isEmpty()); + assertFalse(configSetService.isConfigSetTrusted(configName)); - configSetService.setConfigMetadata(configName, Collections.singletonMap("trusted", true)); - metadata = configSetService.getConfigMetadata(configName); - assertTrue(metadata.containsKey("trusted")); + configSetService.setConfigSetTrust(configName, true); + assertTrue(configSetService.isConfigSetTrusted(configName)); - configSetService.setConfigMetadata(configName, Collections.singletonMap("foo", true)); - assertFalse(configSetService.getConfigMetadata(configName).containsKey("trusted")); + configSetService.setConfigMetadataWithTrust( + configName, new HashMap<>(Collections.singletonMap("foo", true))); + assertTrue(configSetService.isConfigSetTrusted(configName)); assertTrue(configSetService.getConfigMetadata(configName).containsKey("foo")); List configFiles = configSetService.getAllConfigFiles(configName); diff --git a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java index b8405fba6b0..4aaea27c857 100644 --- a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java +++ b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java @@ -568,14 +568,14 @@ public class TestCoreContainer extends SolrTestCaseJ4 { public void copyConfig(String fromConfig, String toConfig) {} @Override - public void uploadConfig(String configName, Path dir) {} + protected void uploadConfig(String configName, Path dir) {} @Override public void uploadFileToConfig( String configName, String fileName, byte[] data, boolean overwriteOnExists) {} @Override - public void setConfigMetadata(String configName, Map data) {} + protected void setConfigMetadata(String configName, Map data) {} @Override public Map getConfigMetadata(String configName) { diff --git a/solr/core/src/test/org/apache/solr/core/TestFileSystemConfigSetService.java b/solr/core/src/test/org/apache/solr/core/TestFileSystemConfigSetService.java index 7b0e26d47cc..f9695a990e5 100644 --- a/solr/core/src/test/org/apache/solr/core/TestFileSystemConfigSetService.java +++ b/solr/core/src/test/org/apache/solr/core/TestFileSystemConfigSetService.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -61,12 +62,13 @@ public class TestFileSystemConfigSetService extends SolrTestCaseJ4 { fileSystemConfigSetService.uploadFileToConfig(configName, "testfile", testdata, true); // metadata is stored in .metadata.json - fileSystemConfigSetService.setConfigMetadata(configName, Map.of("key1", "val1")); + fileSystemConfigSetService.setConfigMetadataWithTrust( + configName, new HashMap<>(Map.of("key1", "val1"))); Map metadata = fileSystemConfigSetService.getConfigMetadata(configName); - assertEquals(metadata.toString(), "{key1=val1}"); + assertEquals("{key1=val1, trusted=true}", metadata.toString()); List allConfigFiles = fileSystemConfigSetService.getAllConfigFiles(configName); - assertEquals(allConfigFiles.toString(), "[schema.xml, solrconfig.xml, testfile]"); + assertEquals("[schema.xml, solrconfig.xml, testfile]", allConfigFiles.toString()); fileSystemConfigSetService.deleteFilesFromConfig( configName, List.of(METADATA_FILE, "testfile")); @@ -74,19 +76,19 @@ public class TestFileSystemConfigSetService extends SolrTestCaseJ4 { assertTrue(metadata.isEmpty()); allConfigFiles = fileSystemConfigSetService.getAllConfigFiles(configName); - assertEquals(allConfigFiles.toString(), "[schema.xml, solrconfig.xml]"); + assertEquals("[schema.xml, solrconfig.xml]", allConfigFiles.toString()); fileSystemConfigSetService.copyConfig(configName, "copytestconfig"); assertEquals(fileSystemConfigSetService.listConfigs().size(), 2); allConfigFiles = fileSystemConfigSetService.getAllConfigFiles("copytestconfig"); - assertEquals(allConfigFiles.toString(), "[schema.xml, solrconfig.xml]"); + assertEquals("[schema.xml, solrconfig.xml]", allConfigFiles.toString()); Path downloadConfig = createTempDir("downloadConfig"); fileSystemConfigSetService.downloadConfig(configName, downloadConfig); List configs = getFileList(downloadConfig); - assertEquals(configs.toString(), "[schema.xml, solrconfig.xml]"); + assertEquals("[schema.xml, solrconfig.xml]", configs.toString()); Exception ex = assertThrows( diff --git a/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperReadAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperReadAPITest.java index 39d494e177c..1a580953c5e 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperReadAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperReadAPITest.java @@ -72,10 +72,10 @@ public class ZookeeperReadAPITest extends SolrCloudTestCase { assertNotNull(o); o = Utils.executeGET(client.getHttpClient(), basezkls + "/configs", Utils.JSONCONSUMER); assertEquals( - "0", + "16", String.valueOf(getObjectByPath(o, true, split(":/configs:_default:dataLength", ':')))); assertEquals( - "0", String.valueOf(getObjectByPath(o, true, split(":/configs:conf:dataLength", ':')))); + "16", String.valueOf(getObjectByPath(o, true, split(":/configs:conf:dataLength", ':')))); assertEquals("0", String.valueOf(getObjectByPath(o, true, split("/stat/version", '/')))); o = Utils.executeGET(client.getHttpClient(), basezk + "/configs", Utils.JSONCONSUMER); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java index 17416f95786..922cc989726 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java @@ -30,6 +30,7 @@ import static org.apache.solr.common.params.CoreAdminParams.BACKUP_ID; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY; import static org.apache.solr.common.params.CoreAdminParams.NAME; +import static org.apache.solr.common.params.CoreAdminParams.TRUSTED; import static org.hamcrest.Matchers.containsString; import java.util.List; @@ -38,11 +39,28 @@ import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.CreateCollectionRequestBody; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.NodeConfig; +import org.apache.solr.request.LocalSolrQueryRequest; +import org.junit.BeforeClass; import org.junit.Test; /** Unit tests for {@link RestoreCollectionAPI} */ public class RestoreCollectionAPITest extends SolrTestCaseJ4 { + private static RestoreCollectionAPI restoreCollectionAPI; + + @BeforeClass + public static void setUpApi() { + restoreCollectionAPI = + new RestoreCollectionAPI( + new CoreContainer( + new NodeConfig.NodeConfigBuilder("testnode", createTempDir()).build()), + new LocalSolrQueryRequest(null, new NamedList<>()), + null); + } + @Test public void testReportsErrorIfBackupNameMissing() { final var requestBody = new RestoreCollectionAPI.RestoreCollectionRequestBody(); @@ -51,8 +69,7 @@ public class RestoreCollectionAPITest extends SolrTestCaseJ4 { expectThrows( SolrException.class, () -> { - final var api = new RestoreCollectionAPI(null, null, null); - api.restoreCollection(null, requestBody); + restoreCollectionAPI.restoreCollection(null, requestBody); }); assertEquals(400, thrown.code()); @@ -65,8 +82,7 @@ public class RestoreCollectionAPITest extends SolrTestCaseJ4 { expectThrows( SolrException.class, () -> { - final var api = new RestoreCollectionAPI(null, null, null); - api.restoreCollection("someBackupName", null); + restoreCollectionAPI.restoreCollection("someBackupName", null); }); assertEquals(400, thrown.code()); @@ -81,8 +97,7 @@ public class RestoreCollectionAPITest extends SolrTestCaseJ4 { expectThrows( SolrException.class, () -> { - final var api = new RestoreCollectionAPI(null, null, null); - api.restoreCollection("someBackupName", requestBody); + restoreCollectionAPI.restoreCollection("someBackupName", requestBody); }); assertEquals(400, thrown.code()); @@ -97,8 +112,7 @@ public class RestoreCollectionAPITest extends SolrTestCaseJ4 { expectThrows( SolrException.class, () -> { - final var api = new RestoreCollectionAPI(null, null, null); - api.restoreCollection("someBackupName", requestBody); + restoreCollectionAPI.restoreCollection("someBackupName", requestBody); }); assertEquals(400, thrown.code()); @@ -116,9 +130,9 @@ public class RestoreCollectionAPITest extends SolrTestCaseJ4 { requestBody.async = "someAsyncId"; final var remoteMessage = - RestoreCollectionAPI.createRemoteMessage("someBackupName", requestBody).getProperties(); + restoreCollectionAPI.createRemoteMessage("someBackupName", requestBody).getProperties(); - assertEquals(7, remoteMessage.size()); + assertEquals(8, remoteMessage.size()); assertEquals("restore", remoteMessage.get(QUEUE_OPERATION)); assertEquals("someCollectionName", remoteMessage.get(COLLECTION)); assertEquals("/some/location/path", remoteMessage.get(BACKUP_LOCATION)); @@ -126,6 +140,7 @@ public class RestoreCollectionAPITest extends SolrTestCaseJ4 { assertEquals("someRepositoryName", remoteMessage.get(BACKUP_REPOSITORY)); assertEquals("someAsyncId", remoteMessage.get(ASYNC)); assertEquals("someBackupName", remoteMessage.get(NAME)); + assertFalse((Boolean) remoteMessage.get(TRUSTED)); } @Test @@ -146,9 +161,9 @@ public class RestoreCollectionAPITest extends SolrTestCaseJ4 { createParams.properties = Map.of("foo", "bar"); final var remoteMessage = - RestoreCollectionAPI.createRemoteMessage("someBackupName", requestBody).getProperties(); + restoreCollectionAPI.createRemoteMessage("someBackupName", requestBody).getProperties(); - assertEquals(14, remoteMessage.size()); + assertEquals(15, remoteMessage.size()); assertEquals("restore", remoteMessage.get(QUEUE_OPERATION)); assertEquals("someCollectionName", remoteMessage.get(COLLECTION)); assertEquals("/some/location/path", remoteMessage.get(BACKUP_LOCATION)); @@ -162,6 +177,7 @@ public class RestoreCollectionAPITest extends SolrTestCaseJ4 { assertEquals(Integer.valueOf(456), remoteMessage.get(TLOG_REPLICAS)); assertEquals(Integer.valueOf(789), remoteMessage.get(PULL_REPLICAS)); assertEquals("node1,node2", remoteMessage.get(CREATE_NODE_SET_PARAM)); + assertFalse((Boolean) remoteMessage.get(TRUSTED)); assertEquals("bar", remoteMessage.get("property.foo")); } diff --git a/solr/solr-ref-guide/modules/deployment-guide/examples/ZkConfigFilesTest.java b/solr/solr-ref-guide/modules/deployment-guide/examples/ZkConfigFilesTest.java index f3d9c41fb1e..f2fb303a7dd 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/examples/ZkConfigFilesTest.java +++ b/solr/solr-ref-guide/modules/deployment-guide/examples/ZkConfigFilesTest.java @@ -70,7 +70,8 @@ public class ZkConfigFilesTest extends SolrCloudTestCase { assertConfigsContainOnly(); // tag::zk-configset-upload[] - getConfigSetService().uploadConfig("nameForConfigset", Paths.get(localConfigSetDirectory)); + getConfigSetService() + .uploadConfig("nameForConfigset", Paths.get(localConfigSetDirectory), false); // end::zk-configset-upload[] assertConfigsContainOnly("nameForConfigset"); diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java index a3fe06f4a11..c0ca106a012 100644 --- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java @@ -34,6 +34,7 @@ import java.util.Set; import java.util.function.Predicate; import java.util.regex.Pattern; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.common.util.FileTypeMagicUtil; import org.apache.solr.common.util.StrUtils; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -352,7 +353,14 @@ public class ZkMaintenanceUtils { USE_FORBIDDEN_FILE_TYPES); return FileVisitResult.CONTINUE; } - // TODO: Cannot check MAGIC header for file since FileTypeGuesser is in core + if (FileTypeMagicUtil.isFileForbiddenInConfigset(file)) { + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(file); + log.warn( + "uploadToZK skipping '{}', as it matched the MAGIC signature of a forbidden mime type {}", + file, + mimeType); + return FileVisitResult.CONTINUE; + } String zkNode = createZkNodeName(zkPath, rootPath, file); try { // if the path exists (and presumably we're uploading data to it) just set its data @@ -426,12 +434,26 @@ public class ZkMaintenanceUtils { private static int copyDataDown(SolrZkClient zkClient, String zkPath, Path file) throws IOException, KeeperException, InterruptedException { - byte[] data = zkClient.getData(zkPath, null, null, true); - if (data != null && data.length > 0) { // There are apparently basically empty ZNodes. - log.info("Writing file {}", file); - Files.write(file, data); - return data.length; + if (isFileForbiddenInConfigSets(zkPath)) { + log.warn("Skipping download of file from ZK, as it is a forbidden type: {}", zkPath); + } else { + byte[] data = zkClient.getData(zkPath, null, null, true); + + if (data != null && data.length > 0) { // There are apparently basically empty ZNodes. + if (FileTypeMagicUtil.isFileForbiddenInConfigset(data)) { + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data); + log.warn( + "Skipping download of file from ZK, as it matched the MAGIC signature of a forbidden mime type {}: {}", + mimeType, + zkPath); + } else { + log.info("Writing file {}", file); + Files.write(file, data); + } + return data.length; + } } + return 0; } @@ -444,13 +466,8 @@ public class ZkMaintenanceUtils { if (children.size() == 0) { // If we didn't copy data down, then we also didn't create the file. But we still need a // marker on the local disk so create an empty file. - if (isFileForbiddenInConfigSets(zkPath)) { - log.warn("Skipping download of file from ZK, as it is a forbidden type: {}", zkPath); - } else { - // TODO: Cannot check MAGIC header for file since FileTypeGuesser is in core - if (copyDataDown(zkClient, zkPath, file) == 0) { - Files.createFile(file); - } + if (copyDataDown(zkClient, zkPath, file) == 0) { + Files.createFile(file); } } else { Files.createDirectories(file); // Make parent dir. diff --git a/solr/solrj-zookeeper/src/test/org/apache/solr/common/cloud/TestZkConfigSetService.java b/solr/solrj-zookeeper/src/test/org/apache/solr/common/cloud/TestZkConfigSetService.java index 5375f62cdeb..8a2ff9f84cf 100644 --- a/solr/solrj-zookeeper/src/test/org/apache/solr/common/cloud/TestZkConfigSetService.java +++ b/solr/solrj-zookeeper/src/test/org/apache/solr/common/cloud/TestZkConfigSetService.java @@ -94,13 +94,14 @@ public class TestZkConfigSetService extends SolrTestCaseJ4 { Files.createDirectory(tempConfig.resolve(".ignoreddir")); Files.createFile(tempConfig.resolve(".ignoreddir").resolve("ignored")); - configSetService.uploadConfig("testconfig", tempConfig); + configSetService.uploadConfig("testconfig", tempConfig, true); // uploading a directory creates a new config List configs = configSetService.listConfigs(); assertEquals(1, configs.size()); assertEquals("testconfig", configs.get(0)); assertTrue(configSetService.checkConfigExists("testconfig")); + assertTrue(configSetService.isConfigSetTrusted("testconfig")); // check downloading Path downloadPath = createTempDir("download"); @@ -118,16 +119,17 @@ public class TestZkConfigSetService extends SolrTestCaseJ4 { // uploading to the same config overwrites byte[] overwritten = "new test data".getBytes(StandardCharsets.UTF_8); Files.write(tempConfig.resolve("file1"), overwritten); - configSetService.uploadConfig("testconfig", tempConfig); + configSetService.uploadConfig("testconfig", tempConfig, false); assertEquals(1, configSetService.listConfigs().size()); Path download2 = createTempDir("download2"); configSetService.downloadConfig("testconfig", download2); byte[] checkdata2 = Files.readAllBytes(download2.resolve("file1")); assertArrayEquals(overwritten, checkdata2); + assertFalse(configSetService.isConfigSetTrusted("testconfig")); // uploading same files to a new name creates a new config - configSetService.uploadConfig("config2", tempConfig); + configSetService.uploadConfig("config2", tempConfig, true); assertEquals(2, configSetService.listConfigs().size()); // Test copying a config works in both flavors @@ -136,6 +138,7 @@ public class TestZkConfigSetService extends SolrTestCaseJ4 { configs = configSetService.listConfigs(); assertTrue("config2copy should exist", configs.contains("config2copy")); assertTrue("config2copy2 should exist", configs.contains("config2copy2")); + assertTrue(configSetService.isConfigSetTrusted("config2")); } } @@ -210,7 +213,7 @@ public class TestZkConfigSetService extends SolrTestCaseJ4 { try (SolrZkClient client = buildZkClient(zkServer.getZkAddress("/acl"), aclProvider, writeable)) { ConfigSetService configSetService = new ZkConfigSetService(client); - configSetService.uploadConfig("acltest", configPath); + configSetService.uploadConfig("acltest", configPath, false); assertEquals(1, configSetService.listConfigs().size()); } @@ -221,7 +224,8 @@ public class TestZkConfigSetService extends SolrTestCaseJ4 { assertEquals(1, configSetService.listConfigs().size()); IOException ioException = assertThrows( - IOException.class, () -> configSetService.uploadConfig("acltest2", configPath)); + IOException.class, + () -> configSetService.uploadConfig("acltest2", configPath, false)); assertEquals(KeeperException.NoAuthException.class, ioException.getCause().getClass()); } diff --git a/solr/solrj/build.gradle b/solr/solrj/build.gradle index dc1656a4624..cffe8ecf5eb 100644 --- a/solr/solrj/build.gradle +++ b/solr/solrj/build.gradle @@ -50,6 +50,8 @@ dependencies { implementation 'org.apache.httpcomponents:httpclient' implementation 'org.apache.httpcomponents:httpcore' + implementation 'com.j256.simplemagic:simplemagic' + compileOnly 'com.github.stephenc.jcip:jcip-annotations' testImplementation project(':solr:test-framework') diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java index a85b3fe9ead..15d4397bd39 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java @@ -159,6 +159,9 @@ public abstract class CoreAdminParams { */ public static final String REPLICA_TYPE = "replicaType"; + /** Whether the request that generated the admin command is trusted */ + public static final String TRUSTED = "trusted"; + public enum CoreAdminAction { STATUS(true), UNLOAD, diff --git a/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java b/solr/solrj/src/java/org/apache/solr/common/util/FileTypeMagicUtil.java similarity index 94% rename from solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java rename to solr/solrj/src/java/org/apache/solr/common/util/FileTypeMagicUtil.java index 692cda83bd3..61a71780cbc 100644 --- a/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/FileTypeMagicUtil.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.solr.util; +package org.apache.solr.common.util; import com.j256.simplemagic.ContentInfo; import com.j256.simplemagic.ContentInfoUtil; @@ -34,7 +34,7 @@ import java.util.Set; import org.apache.solr.common.SolrException; /** Utility class to guess the mime type of file based on its magic number. */ -public class FileTypeMagicUtil implements ContentInfoUtil.ErrorCallBack { +public class FileTypeMagicUtil { private final ContentInfoUtil util; private static final Set SKIP_FOLDERS = new HashSet<>(Arrays.asList(".", "..")); @@ -42,7 +42,15 @@ public class FileTypeMagicUtil implements ContentInfoUtil.ErrorCallBack { FileTypeMagicUtil() { try { - util = new ContentInfoUtil("/magic/executables", this); + util = + new ContentInfoUtil( + "/magic/executables", + (line, details, e) -> { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + String.format(Locale.ROOT, "%s: %s", line, details), + e); + }); } catch (IOException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error parsing magic file", e); } @@ -120,14 +128,6 @@ public class FileTypeMagicUtil implements ContentInfoUtil.ErrorCallBack { return guessTypeFallbackToOctetStream(util.findMatch(bytes)); } - @Override - public void error(String line, String details, Exception e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - String.format(Locale.ROOT, "%s: %s", line, details), - e); - } - /** * Determine forbidden file type based on magic bytes matching of the file itself. Forbidden types * are: diff --git a/solr/solrj/src/resources/magic/executables b/solr/solrj/src/resources/magic/executables new file mode 100644 index 00000000000..04094eaf797 --- /dev/null +++ b/solr/solrj/src/resources/magic/executables @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# POSIX tar archives +# URL: https://en.wikipedia.org/wiki/Tar_(computing) +# Reference: https://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5&manpath=FreeBSD+8-current +# header mainly padded with nul bytes +500 quad 0 +!:strength /2 +# filename or extended attribute printable strings in range space null til umlaut ue +>0 ubeshort >0x1F00 +>>0 ubeshort <0xFCFD +# last 4 header bytes often null but tar\0 in gtarfail2.tar gtarfail.tar-bad +# at https://sourceforge.net/projects/s-tar/files/testscripts/ +>>>508 ubelong&0x8B9E8DFF 0 +# nul, space or ascii digit 0-7 at start of mode +>>>>100 ubyte&0xC8 =0 +>>>>>101 ubyte&0xC8 =0 +# nul, space at end of check sum +>>>>>>155 ubyte&0xDF =0 +# space or ascii digit 0 at start of check sum +>>>>>>>148 ubyte&0xEF =0x20 +# check for specific 1st member name that indicates other mime type and file name suffix +>>>>>>>>0 string TpmEmuTpms/permall +!:mime application/x-tar +!:ext tar +# other stuff in padding +# some implementations add new fields to the blank area at the end of the header record +# created for example by DOS TAR 3.20g 1994 Tim V.Shapore with -j option +>>257 ulong !0 tar archive (old) +!:mime application/x-tar +!:ext tar +# magic in newer, GNU, posix variants +>257 string =ustar +# 2 last char of magic and UStar version because string expression does not work +# 2 space characters followed by a null for GNU variant +>>261 ubelong =0x72202000 POSIX tar archive (GNU) +!:mime application/x-gtar +!:ext tar/gtar + + +# Zip archives (Greg Roelofs, c/o zip-bugs@wkuvx1.wku.edu) +0 string PK\005\006 Zip archive data (empty) +0 string PK\003\004 Zip archive data +!:strength +1 +!:mime application/zip +!:ext zip/cbz + + +# JAVA +0 belong 0xcafebabe +>4 ubelong >30 compiled Java class data, +!:mime application/x-java-applet +#!:mime application/java-byte-code +!:ext class + + +# SHELL scripts +#0 string/w : shell archive or script for antique kernel text +0 regex \^#!\\s?(/bin/|/usr/) POSIX shell script text executable +!:mime text/x-shellscript +!:ext sh/bash \ No newline at end of file diff --git a/solr/solrj/src/test-files/magic/HelloWorld.java.txt b/solr/solrj/src/test-files/magic/HelloWorld.java.txt new file mode 100644 index 00000000000..ca9518d2afa --- /dev/null +++ b/solr/solrj/src/test-files/magic/HelloWorld.java.txt @@ -0,0 +1,5 @@ +class HelloWorld { + public static void main(String[] args) { + System.out.println("Hellow world"); + } +} \ No newline at end of file diff --git a/solr/solrj/src/test-files/magic/HelloWorldJavaClass.class.bin b/solr/solrj/src/test-files/magic/HelloWorldJavaClass.class.bin new file mode 100644 index 00000000000..e15d0a6c5b9 Binary files /dev/null and b/solr/solrj/src/test-files/magic/HelloWorldJavaClass.class.bin differ diff --git a/solr/solrj/src/test-files/magic/README.md b/solr/solrj/src/test-files/magic/README.md new file mode 100644 index 00000000000..6e499a2f711 --- /dev/null +++ b/solr/solrj/src/test-files/magic/README.md @@ -0,0 +1,29 @@ + + +The two binary files were created by the following commands: + +```bash +echo "Hello" > hello.txt && \ + tar -cvf hello.tar.bin hello.txt && \ + rm hello.txt + +cp HelloWorld.java.txt HelloWorld.java && \ + javac HelloWorld.java && \ + mv HelloWorld.class HelloWorldJavaClass.class.bin && \ + rm HelloWorld.java +``` \ No newline at end of file diff --git a/solr/solrj/src/test-files/magic/hello.tar.bin b/solr/solrj/src/test-files/magic/hello.tar.bin new file mode 100644 index 00000000000..68ca23c362a Binary files /dev/null and b/solr/solrj/src/test-files/magic/hello.tar.bin differ diff --git a/solr/solrj/src/test-files/magic/plain.txt b/solr/solrj/src/test-files/magic/plain.txt new file mode 100644 index 00000000000..70c379b63ff --- /dev/null +++ b/solr/solrj/src/test-files/magic/plain.txt @@ -0,0 +1 @@ +Hello world \ No newline at end of file diff --git a/solr/solrj/src/test-files/magic/shell.sh.txt b/solr/solrj/src/test-files/magic/shell.sh.txt new file mode 100644 index 00000000000..9ea411e111d --- /dev/null +++ b/solr/solrj/src/test-files/magic/shell.sh.txt @@ -0,0 +1,2 @@ +#! /usr/bin/env bash +echo Hello \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java b/solr/solrj/src/test/org/apache/solr/common/util/FileTypeMagicUtilTest.java similarity index 98% rename from solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java rename to solr/solrj/src/test/org/apache/solr/common/util/FileTypeMagicUtilTest.java index 5c02528a7f4..1647ae9ced0 100644 --- a/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java +++ b/solr/solrj/src/test/org/apache/solr/common/util/FileTypeMagicUtilTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.solr.util; +package org.apache.solr.common.util; import java.io.IOException; import java.io.InputStream; diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java index 5e0461f5240..26f33c7c60d 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java @@ -70,7 +70,6 @@ import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.util.ExecutorUtil; @@ -568,11 +567,7 @@ public class MiniSolrCloudCluster { .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) .withConnTimeOut(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) .build()) { - ZkMaintenanceUtils.uploadToZK( - zkClient, - configDir, - ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + configName, - ZkMaintenanceUtils.UPLOAD_FILENAME_EXCLUDE_PATTERN); + new ZkConfigSetService(zkClient).uploadConfig(configName, configDir, true); } } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java index 79a5f59d12f..cd1414d6d5d 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java @@ -234,7 +234,6 @@ public abstract class AbstractIncrementalBackupTest extends SolrCloudTestCase { @SuppressWarnings("unchecked") @Test - @Nightly public void testBackupIncremental() throws Exception { setTestSuffix("testbackupinc"); CloudSolrClient solrClient = cluster.getSolrClient();