message:
Mark rule disabling
This PR adds the possibility to exclude mark rules from being loaded. The disabled mark rules are specified as fully qualified name (`package.markRuleName`) either in the configuration file or as a CLI option. The asterisk (`*`) is used as a wildcard to disable an entire package.

diff:
@@ -3,6 +3,7 @@ package de.fraunhofer.aisec.codyze.analysis;
import de.fraunhofer.aisec.codyze.JythonInterpreter;
import de.fraunhofer.aisec.codyze.analysis.markevaluation.Evaluator;
+import de.fraunhofer.aisec.codyze.config.DisabledMarkRulesValue;
import de.fraunhofer.aisec.codyze.crymlin.builtin.Builtin;
import de.fraunhofer.aisec.codyze.crymlin.builtin.BuiltinRegistry;
import de.fraunhofer.aisec.codyze.crymlin.connectors.lsp.CpgLanguageServer;
@@ -232,7 +233,7 @@ public class AnalysisServer {
* Load MARK model as given in configuration, if it has not been set manually before.
*/
File[] markModelLocations = Arrays.stream(config.markModelFiles).map(File::new).toArray(File[]::new);
- loadMarkRules(markModelLocations);
+ loadMarkRules(config.packageToDisabledMarkRules, markModelLocations);
}
/**
@@ -265,6 +266,16 @@ public class AnalysisServer {
* @param markFiles load all mark entities/rules from these files
*/
public void loadMarkRules(@NonNull File... markFiles) {
+ loadMarkRules(new HashMap<>(), markFiles);
+ }
+
+ /**
+ * Loads all MARK rules from a file or a directory but skips files specified in packageToDisabledMarkRules map
+ *
+ * @param markFiles load all mark entities/rules from these files
+ * @param packageToDisabledMarkRules skip specified files during loading
+ */
+ public void loadMarkRules(Map<String, DisabledMarkRulesValue> packageToDisabledMarkRules, @NonNull File... markFiles) {
File markDescriptionFile = null;
XtextParser parser = new XtextParser();
@@ -312,7 +323,7 @@ public class AnalysisServer {
start = Instant.now();
log.info("Transforming MARK Xtext to internal format");
- this.markModel = new MarkModelLoader().load(markModels);
+ this.markModel = new MarkModelLoader().load(markModels, packageToDisabledMarkRules);
log.info(
"Done Transforming MARK Xtext to internal format in {} ms",
Duration.between(start, Instant.now()).toMillis());

diff:
package de.fraunhofer.aisec.codyze.analysis;
+import de.fraunhofer.aisec.codyze.config.DisabledMarkRulesValue;
import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend;
import kotlin.Pair;
import org.checkerframework.checker.nullness.qual.NonNull;
import java.io.File;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
/** The configuration for the {@link AnalysisServer} holds all values used by the server. */
public class ServerConfiguration {
@@ -56,6 +59,9 @@ public class ServerConfiguration {
/** Additional registered languages */
public final List<Pair<Class<? extends LanguageFrontend>, List<String>>> additionalLanguages;
+ /** Disabled Mark rules. key=package, value=(disableEntirePackage, markRulesToDisable) */
+ public final Map<String, DisabledMarkRulesValue> packageToDisabledMarkRules;
+
private ServerConfiguration(
boolean launchConsole,
boolean launchLsp,
@@ -65,6 +71,7 @@ public class ServerConfiguration {
boolean useUnityBuild,
@NonNull File[] includePath,
boolean disableGoodFindings,
+ Map<String, DisabledMarkRulesValue> packageToDisabledMarkRules,
boolean pedantic,
List<Pair<Class<? extends LanguageFrontend>, List<String>>> additionalLanguages) {
this.launchConsole = launchConsole;
@@ -77,6 +84,7 @@ public class ServerConfiguration {
this.disableGoodFindings = disableGoodFindings;
this.pedantic = pedantic;
this.additionalLanguages = additionalLanguages;
+ this.packageToDisabledMarkRules = packageToDisabledMarkRules;
}
public static Builder builder() {
@@ -96,6 +104,7 @@ public class ServerConfiguration {
private boolean disableGoodFindings;
private boolean pedantic;
public final List<Pair<Class<? extends LanguageFrontend>, List<String>>> additionalLanguages = new ArrayList<>();
+ private Map<String, DisabledMarkRulesValue> packageToDisabledMarkRules = Collections.emptyMap();
public Builder launchConsole(boolean launchConsole) {
this.launchConsole = launchConsole;
@@ -156,6 +165,11 @@ public class ServerConfiguration {
return this;
}
+ public Builder disableMark(Map<String, DisabledMarkRulesValue> disabledMarkRules) {
+ this.packageToDisabledMarkRules = disabledMarkRules;
+ return this;
+ }
+
public ServerConfiguration build() {
return new ServerConfiguration(
launchConsole,
@@ -166,6 +180,7 @@ public class ServerConfiguration {
useUnityBuild,
includePath,
disableGoodFindings,
+ packageToDisabledMarkRules,
pedantic,
additionalLanguages);
}

diff:
@@ -69,7 +69,7 @@ class CodyzeConfiguration {
paramLabel = "<package.rule>",
description =
[
- "The specified mark rules will be excluded from being parsed and processed. The rule has to be specified by the package and the rule name."],
+ "The specified mark rules will be excluded from being parsed and processed. The rule has to be specified by its fully qualified name (package.rule). If there is no package name, specify rule as \".rule\". Use \'*\' to disable an entire package."],
split = ","
)
var disabledMarkRules: List<String> = emptyList()

diff:
@@ -105,6 +105,26 @@ class Configuration {
config.registerLanguage(golangFrontendClazz, extensions)
}
}
+
+ if (!codyze.pedantic) {
+ val disabledRulesMap = mutableMapOf<String, DisabledMarkRulesValue>()
+ for (mName in codyze.disabledMarkRules) {
+ val index = mName.lastIndexOf('.')
+ val packageName = mName.subSequence(0, index).toString()
+ val markName = mName.subSequence(index + 1, mName.length).toString()
+ if (markName.isNotEmpty()) {
+ disabledRulesMap.putIfAbsent(packageName, DisabledMarkRulesValue())
+ if (markName == "*")
+ disabledRulesMap.getValue(packageName).isDisablePackage = true
+ else disabledRulesMap[packageName]?.disabledMarkRuleNames?.add(markName)
+ } else
+ log.warn(
+ "Error while parsing disabled-mark-rules: \'$mName\' is not a valid name for a mark rule. Continue parsing disabled-mark-rules"
+ )
+ }
+ config.disableMark(disabledRulesMap)
+ }
+
return config.build()
}

diff:
+package de.fraunhofer.aisec.codyze.config
+
+/**
+ * Stores whether the associated package should be disabled and the name of the rules to be disabled
+ */
+data class DisabledMarkRulesValue(
+ var isDisablePackage: Boolean = false,
+ val disabledMarkRuleNames: MutableSet<String> = mutableSetOf()
+)

diff:
package de.fraunhofer.aisec.codyze.markmodel;
import de.fraunhofer.aisec.codyze.analysis.markevaluation.ExpressionHelper;
+import de.fraunhofer.aisec.codyze.config.DisabledMarkRulesValue;
import de.fraunhofer.aisec.mark.markDsl.EntityDeclaration;
import de.fraunhofer.aisec.mark.markDsl.EntityStatement;
import de.fraunhofer.aisec.mark.markDsl.MarkModel;
@@ -15,11 +16,7 @@ import org.python.jline.internal.Log;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
/**
* Parses a MarkModel provided by XText in the form of an ECore hierarchy into a simple (ECore-free) {@code Mark} model that the analysis server can work with.
@@ -31,7 +28,7 @@ public class MarkModelLoader {
private static final Logger log = LoggerFactory.getLogger(MarkModelLoader.class);
@NonNull
- public Mark load(Map<String, MarkModel> markModels, @Nullable String onlyfromthisfile) {
+ public Mark load(Map<String, MarkModel> markModels, Map<String, DisabledMarkRulesValue> packageToDisabledMarkRules, @Nullable String onlyfromthisfile) {
Mark m = new Mark();
for (Map.Entry<String, MarkModel> entry : markModels.entrySet()) {
@@ -59,14 +56,33 @@ public class MarkModelLoader {
continue;
}
MarkModel markModel = entry.getValue();
+
+ // For mark files without package the key is an empty string in the map
+ String key = markModel.getPackage() == null ? "" : markModel.getPackage().getName();
+
+ DisabledMarkRulesValue disabledRules = packageToDisabledMarkRules.getOrDefault(key,
+ new DisabledMarkRulesValue(false, Collections.emptySet()));
+
+ // check if entire package should be disabled
+ if (disabledRules.isDisablePackage()) {
+ log.info("Disabled all mark rules in package {}", markModel.getPackage().getName());
+ } else {
// Parse rules
for (RuleDeclaration r : markModel.getRule()) {
// todo @FW: should rules also have package names? what is the exact reasoning behind
// packages?
+
+ // check if rule should be disabled
+ if (disabledRules.getDisabledMarkRuleNames().contains(r.getName())) {
+ log.info("Disabled mark rule {} in package {}", r.getName(), markModel.getPackage() == null ? "" : markModel.getPackage().getName());
+ continue;
+ }
+
MRule rule = parseRule(r, m, entry.getKey());
m.getRules().add(rule);
}
}
+ }
/*
* VALIDATE todo validate MARK files. check e.g.: - Entities - one function call is not part of two ops (of one or two entities) if this would be the case, many
@@ -95,9 +111,19 @@ public class MarkModelLoader {
}
}
+ @NonNull
+ public Mark load(Map<String, MarkModel> markModels, Map<String, DisabledMarkRulesValue> packageToDisabledMarkRules) {
+ return load(markModels, packageToDisabledMarkRules, null);
+ }
+
+ @NonNull
+ public Mark load(Map<String, MarkModel> markModels, @Nullable String onlyfromthisfile) {
+ return load(markModels, Collections.emptyMap(), onlyfromthisfile);
+ }
+
@NonNull
public Mark load(Map<String, MarkModel> markModels) {
- return load(markModels, null);
+ return load(markModels, Collections.emptyMap(), null);
}
private MRule parseRule(RuleDeclaration rule, Mark mark, String containedInThisFile) {

diff:
+package de.fraunhofer.aisec.codyze.crymlin
+
+import de.fraunhofer.aisec.codyze.analysis.AnalysisServer
+import de.fraunhofer.aisec.codyze.config.DisabledMarkRulesValue
+import de.fraunhofer.aisec.codyze.markmodel.MRule
+import java.io.File
+import java.lang.Exception
+import kotlin.Throws
+import kotlin.test.*
+import org.junit.jupiter.api.*
+import org.junit.jupiter.api.Test
+
+internal class MarkLoadTest {
+
+ @Test
+ @Throws(Exception::class)
+ fun disabledRulesTest() {
+ val disabledMarkRules = mutableMapOf<String, DisabledMarkRulesValue>()
+ disabledMarkRules["java"] =
+ DisabledMarkRulesValue(
+ false,
+ mutableSetOf("UseValidAlgorithm", "RandomInitializationVector", "MockWhen2")
+ )
+ disabledMarkRules[""] = DisabledMarkRulesValue(false, mutableSetOf("SignatureOrder"))
+
+ analysisServer.loadMarkRules(
+ disabledMarkRules,
+ botanLocation,
+ javaLocation,
+ noPackageLocation
+ )
+ val actualMarkRules = analysisServer.markModel.rules
+ assertNotNull(actualMarkRules)
+ val actualMarkRuleNames = actualMarkRules.map { r -> r.name }
+
+ assertFalse(actualMarkRuleNames.isEmpty(), "Expected to contain mark rules but was empty")
+
+ val expectedSize =
+ allMarkRules.size -
+ disabledMarkRules.getOrDefault("java", DisabledMarkRulesValue())
+ .disabledMarkRuleNames
+ .size -
+ disabledMarkRules.getOrDefault("", DisabledMarkRulesValue())
+ .disabledMarkRuleNames
+ .size
+ assertEquals(
+ expectedSize,
+ actualMarkRuleNames.size,
+ "Expected size to be $expectedSize, but was ${actualMarkRuleNames.size}"
+ )
+
+ for (s in
+ disabledMarkRules.getOrDefault("java", DisabledMarkRulesValue())
+ .disabledMarkRuleNames) assertFalse(
+ actualMarkRuleNames.contains(s),
+ "Expected to have filtered out $s but was included in mark rules"
+ )
+
+ for (s in
+ disabledMarkRules.getOrDefault("", DisabledMarkRulesValue())
+ .disabledMarkRuleNames) assertFalse(
+ actualMarkRuleNames.contains(s),
+ "Expected to have filtered out $s but was included in mark rules"
+ )
+
+ for (s in botanMarkRuleNames) assertTrue(
+ actualMarkRuleNames.contains(s),
+ "Expected to have loaded $s but was not in mark rules"
+ )
+ }
+
+ @Test
+ @Throws(Exception::class)
+ fun disabledPackageTest() {
+ val disabledMarkRules = mutableMapOf<String, DisabledMarkRulesValue>()
+ disabledMarkRules["botan"] = DisabledMarkRulesValue(true, mutableSetOf())
+
+ analysisServer.loadMarkRules(
+ disabledMarkRules,
+ botanLocation,
+ javaLocation,
+ noPackageLocation
+ )
+ val actualMarkRules = analysisServer.markModel.rules
+ assertNotNull(actualMarkRules)
+ val actualMarkRuleNames = actualMarkRules.map { r -> r.name }
+
+ assertFalse(actualMarkRuleNames.isEmpty(), "Expected to contain mark rules but was empty")
+
+ val expectedSize = allMarkRules.size - botanMarkRuleNames.size
+ assertEquals(
+ expectedSize,
+ actualMarkRuleNames.size,
+ "Expected size to be $expectedSize, but was ${actualMarkRuleNames.size}"
+ )
+
+ for (s in botanMarkRuleNames) assertFalse(
+ actualMarkRuleNames.contains(s),
+ "Expected to have filtered out $s but was included in mark rules"
+ )
+
+ for (s in javaMarkRuleNames) assertTrue(
+ actualMarkRuleNames.contains(s),
+ "Expected to have loaded $s but was not in mark rules"
+ )
+
+ for (s in noPackageRuleNames) assertTrue(
+ actualMarkRuleNames.contains(s),
+ "Expected to have loaded $s but was not in mark rules"
+ )
+ }
+
+ @Test
+ @Throws(Exception::class)
+ fun disabledRulesAndPackageTest() {
+ val disabledMarkRules = mutableMapOf<String, DisabledMarkRulesValue>()
+ disabledMarkRules["java"] =
+ DisabledMarkRulesValue(
+ false,
+ mutableSetOf("UseValidAlgorithm", "RandomInitializationVector", "MockWhen2")
+ )
+ disabledMarkRules["botan"] = DisabledMarkRulesValue(true, mutableSetOf())
+
+ analysisServer.loadMarkRules(
+ disabledMarkRules,
+ botanLocation,
+ javaLocation,
+ noPackageLocation
+ )
+ val actualMarkRules = analysisServer.markModel.rules
+ assertNotNull(actualMarkRules)
+ val actualMarkRuleNames = actualMarkRules.map { r -> r.name }
+
+ assertFalse(actualMarkRuleNames.isEmpty(), "Expected to contain mark rules but was empty")
+
+ val expectedSize =
+ allMarkRules.size -
+ disabledMarkRules.getOrDefault("java", DisabledMarkRulesValue())
+ .disabledMarkRuleNames
+ .size -
+ botanMarkRuleNames.size
+ assertEquals(
+ expectedSize,
+ actualMarkRuleNames.size,
+ "Expected size to be $expectedSize, but was ${actualMarkRuleNames.size}"
+ )
+
+ for (s in
+ disabledMarkRules.getOrDefault("java", DisabledMarkRulesValue())
+ .disabledMarkRuleNames) assertFalse(
+ actualMarkRuleNames.contains(s),
+ "Expected to have filtered out $s but was included in mark rules"
+ )
+
+ for (s in botanMarkRuleNames) assertFalse(
+ actualMarkRuleNames.contains(s),
+ "Expected to have filtered out $s but was included in mark rules"
+ )
+
+ for (s in noPackageRuleNames) assertTrue(
+ actualMarkRuleNames.contains(s),
+ "Expected to have loaded $s but was not in mark rules"
+ )
+ }
+
+ companion object {
+ private lateinit var analysisServer: AnalysisServer
+ private lateinit var javaLocation: File
+ private lateinit var botanLocation: File
+ private lateinit var noPackageLocation: File
+ private lateinit var allMarkRules: MutableList<MRule>
+ private lateinit var botanMarkRuleNames: List<String>
+ private lateinit var javaMarkRuleNames: List<String>
+ private lateinit var noPackageRuleNames: List<String>
+
+ @BeforeAll
+ @JvmStatic
+ fun startup() {
+ analysisServer = AnalysisServer.builder().build()
+ assertNotNull(analysisServer)
+
+ val botanMarkResource =
+ MarkLoadTest::class.java.classLoader.getResource("real-examples/botan/MARK")
+ assertNotNull(botanMarkResource)
+ botanLocation = File(botanMarkResource.file)
+ assertNotNull(botanLocation)
+
+ val javaMarkResource =
+ MarkLoadTest::class.java.classLoader.getResource(
+ "real-examples/bc/rwedoff.Password-Manager"
+ )
+ assertNotNull(javaMarkResource)
+ javaLocation = File(javaMarkResource.file)
+ assertNotNull(javaLocation)
+
+ val noPackageResource =
+ MarkLoadTest::class.java.classLoader.getResource("unittests/nfa-test.mark")
+ assertNotNull(botanMarkResource)
+ noPackageLocation = File(noPackageResource.file)
+ assertNotNull(noPackageLocation)
+
+ analysisServer.loadMarkRules(botanLocation, javaLocation, noPackageLocation)
+ allMarkRules = analysisServer.markModel.rules
+ assertNotNull(allMarkRules)
+
+ analysisServer.loadMarkRules(botanLocation)
+ val botanMarkRules = analysisServer.markModel.rules
+ assertNotNull(botanMarkRules)
+ botanMarkRuleNames = botanMarkRules.map { r -> r.name }
+
+ analysisServer.loadMarkRules(javaLocation)
+ val javaMarkRules = analysisServer.markModel.rules
+ assertNotNull(javaMarkRules)
+ javaMarkRuleNames = javaMarkRules.map { r -> r.name }
+
+ analysisServer.loadMarkRules(noPackageLocation)
+ val noPackageMarkRules = analysisServer.markModel.rules
+ assertNotNull(noPackageMarkRules)
+ noPackageRuleNames = noPackageMarkRules.map { r -> r.name }
+ }
+ }
+}

diff:
@@ -24,7 +24,7 @@ class FSMBuilderTest {
val parser = XtextParser()
parser.addMarkFile(markPoC1)
val markModels = parser.parse()
- val mark = MarkModelLoader().load(markModels, null)
+ val mark = MarkModelLoader().load(markModels)
assertNotNull(mark)
var dfa = DFA()
for (rule in mark.rules) {
@@ -55,7 +55,7 @@ class FSMBuilderTest {
val parser = XtextParser()
parser.addMarkFile(markPoC1)
val markModels = parser.parse()
- val mark = MarkModelLoader().load(markModels, null)
+ val mark = MarkModelLoader().load(markModels)
assertNotNull(mark)
var dfa = DFA()
for (rule in mark.rules) {
@@ -97,7 +97,7 @@ class FSMBuilderTest {
val parser = XtextParser()
parser.addMarkFile(markPoC1)
val markModels = parser.parse()
- val mark = MarkModelLoader().load(markModels, null)
+ val mark = MarkModelLoader().load(markModels)
assertNotNull(mark)
var dfa = DFA()
for (rule in mark.rules) {
@@ -143,7 +143,7 @@ class FSMBuilderTest {
val parser = XtextParser()
parser.addMarkFile(markPoC1)
val markModels = parser.parse()
- val mark = MarkModelLoader().load(markModels, null)
+ val mark = MarkModelLoader().load(markModels)
assertNotNull(mark)
for (rule in mark.rules) {
if (rule.statement.ensure.exp is OrderExpression) {

original_message:
Mark rule disabling (#426)

This PR adds the possibility to exclude mark rules from being loaded. The disabled mark rules are specified as fully qualified name (`package.markRuleName`) either in the configuration file or as a CLI option. The asterisk (`*`) is used as a wildcard to disable an entire package.


message:
Add config normalizing
This PR adds a normalize function to the Configuration class. The function handles all interactions between configurations that cannot be specified through PicoCLI and the configuration file.

change_type: MODIFY

diff:
+
package de.fraunhofer.aisec.codyze;
import picocli.CommandLine;

diff:
@@ -12,8 +12,8 @@ import de.fraunhofer.aisec.cpg.graph.Node
import java.io.File
/**
- * Codyze-specific implementation of the [DFAOrderEvaluator].
- * Its main purpose is to collect the findings in case of violations to the order.
+ * Codyze-specific implementation of the [DFAOrderEvaluator]. Its main purpose is to collect the
+ * findings in case of violations to the order.
*/
class CodyzeDFAOrderEvaluator(
referencedVertices: Set<Long>,
@@ -25,8 +25,8 @@ class CodyzeDFAOrderEvaluator(
) : DFAOrderEvaluator(referencedVertices, nodesToOp, thisPositionOfNode) {
/**
- * Collects the finding in the AnalysisContext as the [node] makes an
- * operation which violates the desired order.
+ * Collects the finding in the AnalysisContext as the [node] makes an operation which violates
+ * the desired order.
*/
override fun actionMissingTransitionForNode(node: Node, fsm: DFA) {
val region = Utils.getRegionByNode(node)
@@ -64,9 +64,8 @@ class CodyzeDFAOrderEvaluator(
}
/**
- * Collects the finding in the AnalysisContext because the DFA finished
- * analyzing the function but the [base] did not terminate in an accepting
- * state (i.e., some operations are missing).
+ * Collects the finding in the AnalysisContext because the DFA finished analyzing the function
+ * but the [base] did not terminate in an accepting state (i.e., some operations are missing).
*/
override fun actionNonAcceptingTermination(base: String, fsm: DFA) {
if (fsm.executionTrace.size == 1)

diff:
@@ -16,9 +16,8 @@ class OrderEvaluation {
private val log = LoggerFactory.getLogger(OrderEvaluation::class.java)
/**
- * Collects the mark instances of the [orderExpression], maps CPG nodes
- * to the string representation of the MARK-Operators and starts the order
- * evaluation.
+ * Collects the mark instances of the [orderExpression], maps CPG nodes to the string
+ * representation of the MARK-Operators and starts the order evaluation.
*/
fun evaluateDFA(
rule: MRule,

diff:
@@ -69,46 +69,22 @@ class Configuration {
this.cpg = cpgConfiguration
}
- // Parse CLI arguments into config class
- private fun parseCLI(vararg args: String?) {
-
- CommandLine(this)
- // Added as Mixin so the already initialized objects are used instead of new ones
- // created
- .addMixin("codyze", codyze)
- .addMixin("cpg", cpg)
- .addMixin("analysis", codyze.analysis)
- .addMixin("translation", cpg.translation)
- .registerConverter(Pass::class.java, PassTypeConverter())
- .setCaseInsensitiveEnumValuesAllowed(true)
- // setUnmatchedArgumentsAllowed is true because both classes don't have the config path
- // option which would result in exceptions, side effect is that all unknown options are
- // ignored
- .setUnmatchedArgumentsAllowed(true)
- .parseArgs(*args)
- }
-
/**
* Builds ServerConfiguration object with available configurations
*
* @return ServerConfiguration
*/
fun buildServerConfiguration(): ServerConfiguration {
+ this.normalize()
val config =
ServerConfiguration.builder()
.launchLsp(executionMode.isLsp)
.launchConsole(executionMode.isTui)
.typestateAnalysis(codyze.analysis.tsMode)
- .disableGoodFindings(
- if (codyze.pedantic) {
- false
- } else {
- codyze.noGoodFindings
- }
- )
+ .disableGoodFindings(codyze.noGoodFindings)
.markFiles(*codyze.mark.map { m -> m.absolutePath }.toTypedArray())
+ .pedantic(codyze.pedantic)
- if (!codyze.pedantic) {
val disabledRulesMap = mutableMapOf<String, DisabledMarkRulesValue>()
for (mName in codyze.disabledMarkRules) {
val index = mName.lastIndexOf('.')
@@ -116,14 +92,12 @@ class Configuration {
val markName = mName.subSequence(index + 1, mName.length).toString()
if (markName.isNotEmpty()) {
disabledRulesMap.putIfAbsent(packageName, DisabledMarkRulesValue())
- if (markName == "*")
- disabledRulesMap.getValue(packageName).isDisablePackage = true
+ if (markName == "*") disabledRulesMap.getValue(packageName).isDisablePackage = true
else disabledRulesMap[packageName]?.disabledMarkRuleNames?.add(markName)
} else
log.warn(
"Error while parsing disabled-mark-rules: \'$mName\' is not a valid name for a mark rule. Continue parsing disabled-mark-rules"
)
- }
config.disableMark(disabledRulesMap)
}
@@ -136,13 +110,13 @@ class Configuration {
* @return TranslationConfiguration
*/
fun buildTranslationConfiguration(vararg sources: File): TranslationConfiguration {
+ this.normalize()
val translationConfig =
TranslationConfiguration.builder()
.debugParser(if (executionMode.isLsp) false else cpg.debugParser)
.failOnError(cpg.failOnError)
.codeInNodes(cpg.codeInNodes)
- // we need to force load includes for unity builds, otherwise nothing will be parsed
- .loadIncludes(cpg.translation.analyzeIncludes || cpg.useUnityBuild)
+ .loadIncludes(cpg.translation.analyzeIncludes)
.useUnityBuild(cpg.useUnityBuild)
.processAnnotations(cpg.processAnnotations)
.symbols(cpg.symbols)
@@ -205,6 +179,41 @@ class Configuration {
return translationConfig.build()
}
+ private fun normalize() {
+ // In pedantic analysis mode all MARK rules are analyzed and all findings reported
+ if (codyze.pedantic) {
+ codyze.noGoodFindings = false
+ codyze.disabledMarkRules = emptyList()
+ }
+
+ // we need to force load includes for unity builds, otherwise nothing will be parsed
+ if (cpg.useUnityBuild) cpg.translation.analyzeIncludes = true
+
+ if (executionMode.isLsp) {
+ // we don't want the parser to print to the terminal when in LSP mode
+ cpg.debugParser = false
+ }
+ }
+
+ // Parse CLI arguments into config class
+ private fun parseCLI(vararg args: String?) {
+
+ CommandLine(this)
+ // Added as Mixin so the already initialized objects are used instead of new ones
+ // created
+ .addMixin("codyze", codyze)
+ .addMixin("cpg", cpg)
+ .addMixin("analysis", codyze.analysis)
+ .addMixin("translation", cpg.translation)
+ .registerConverter(Pass::class.java, PassTypeConverter())
+ .setCaseInsensitiveEnumValuesAllowed(true)
+ // setUnmatchedArgumentsAllowed is true because both classes don't have the config path
+ // option which would result in exceptions, side effect is that all unknown options are
+ // ignored
+ .setUnmatchedArgumentsAllowed(true)
+ .parseArgs(*args)
+ }
+
companion object {
private val log = LoggerFactory.getLogger(Configuration::class.java)

diff:
@@ -9,7 +9,9 @@ import java.io.IOException
import java.util.*
import org.slf4j.LoggerFactory
-/** Custom deserializer to populate a list of languages derived from strings from the Jackson parser */
+/**
+ * Custom deserializer to populate a list of languages derived from strings from the Jackson parser
+ */
class LanguageDeserializer : StdDeserializer<EnumSet<Language>?>(null as JavaType?) {
@Throws(IOException::class)

change_type: ADD

diff:
+package de.fraunhofer.aisec.codyze.crymlin
+
+import de.fraunhofer.aisec.codyze.config.Configuration
+import kotlin.test.*
+import org.junit.jupiter.api.Test
+
+internal class BuildServerConfigTest {
+ @Test
+ fun pedanticNoGoodFindingsTest() {
+ val cliParameters = arrayOf("-c", "--pedantic", "--noGoodFindings")
+ val config = Configuration.initConfig(null, *cliParameters)
+ val serverConfig = config.buildServerConfiguration()
+
+ assertTrue(serverConfig.pedantic, "pedantic should be true but was false")
+ assertFalse(
+ serverConfig.disableGoodFindings,
+ "disableGoodFindings should be false because in pedantic mode all findings should be reported"
+ )
+ }
+
+ @Test
+ fun pedanticDisabledMarkRulesTest() {
+ val cliParameters =
+ arrayOf("-c", "--pedantic", "--disabled-mark-rules=java.UseValidAlgorithm")
+ val config = Configuration.initConfig(null, *cliParameters)
+ val serverConfig = config.buildServerConfiguration()
+
+ assertTrue(serverConfig.pedantic, "pedantic should be true but was false")
+ assertEquals(
+ 0,
+ serverConfig.packageToDisabledMarkRules.size,
+ "disableGoodFindings should be false because in pedantic mode all findings should be reported"
+ )
+ }
+}

diff:
@@ -120,6 +120,18 @@ internal class BuildTranslationConfigTest {
)
}
+ @Test
+ fun lspTest() {
+ val cliParameters = arrayOf("-l", "--debug-parser", "--source=test.java")
+ val config = Configuration.initConfig(null, *cliParameters)
+ val translationConfig = config.buildTranslationConfiguration()
+
+ assertFalse(
+ translationConfig.debugParser,
+ "DebugParser has to be set to false if in lsp mode"
+ )
+ }
+
companion object {
private lateinit var additionalOptionFile: File

license: Apache License 2.0

repo: fraunhofer-aisec/codyze

original_message:
Add config normalizing (#442)

This PR adds a normalize function to the Configuration class. The function handles all interactions between configurations that cannot be specified through PicoCLI and the configuration file.

Co-authored-by: Florian Wendland <florian.wendland@aisec.fraunhofer.de>


message:
Add schema for config file
This PR adds a JSON schema which users can use to generate and validate their YAML configuration files. The README is also updated to include a short exemplary codyze call and link to the configuration wiki page instead of listing them all in the README.

diff:
@@ -12,7 +12,7 @@ Codyze is a static code analyzer that focuses on verifying security compliance i
Documentation: https://www.codyze.io
-## Build & run Codyze
+## Build & Run Codyze
Java 11 (OpenJDK) is a prerequisite.
@@ -22,50 +22,28 @@ To build an executable version of Codyze, use the `installDist` task:
$ ./gradlew installDist
```
-This will provide you with an executable Codyze installation under `build/install/codyze`. Change to that directory and run Codyze:
+This will provide you with an executable Codyze installation under `build/install/codyze`.
+To start Codyze, change to the directory and run Codyze.
+
+Codyze has three execution modes:
+* commando line interface mode (`-c`)
+* language server protocol mode (`-l`)
+* interactive console mode (`-t`).
+
+One of these modes has to be specified as command line option when running Codyze.
+
+An exemplary call to start the commando line interface mode would be
```shell
$ cd build/install/codyze
-$ ./bin/codyze
+$ ./bin/codyze -c -m ./mark -s <sourcepath>
```
+where `<sourcepath>` denotes the path to the source directory or file which should be analyzed.
-Without further command line arguments, Codyze will print its command line help:
+Codyze can be further configured with more command line arguments or a YAML configuration file.
+For more information about the usage and configurations, please refer to https://www.codyze.io and the corresponding [wiki page](https://github.com/Fraunhofer-AISEC/codyze/wiki/Usage).
-```
-Usage: codyze (-c | -l | -t) [[--typestate=<NFA|WPDS>]] [[--analyze-includes]
- [--includes=<includesPath>[:|;<includesPath>...]] [--includes=<includesPath>[:|;
- <includesPath>...]]...] [-hV] [--no-good-findings] [-m=<path>] [-o=<file>]
- [-s=<path>] [--timeout=<minutes>]
-Codyze finds security flaws in source code
- -s, --source=<path> Source file or folder to analyze.
- -m, --mark=<path> Load MARK policy files from folder
- -o, --output=<file> Write results to file. Use -- for stdout.
- --timeout=<minutes> Terminate analysis after timeout
- Default: 120
- --no-good-findings Disable output of "positive" findings which indicate correct
- implementations
- -h, --help Show this help message and exit.
- -V, --version Print version information and exit.
-Execution mode
- -c Start in command line mode.
- -l Start in language server protocol (LSP) mode.
- -t Start interactive console (Text-based User Interface).
-Analysis settings
- --typestate=<NFA|WPDS>
- Typestate analysis mode
- NFA: Non-deterministic finite automaton (faster, intraprocedural)
- WPDS: Weighted pushdown system (slower, interprocedural)
-Translation settings
- --analyze-includes Enables parsing of include files. By default, if --includes are given,
- the parser will resolve symbols/templates from these include, but not
- load their parse tree.
- --includes=<includesPath>[:|;<includesPath>...]
- Path(s) containing include files. Path must be separated by :
- (Mac/Linux) or ; (Windows)
-```
-Please refer to https://www.codyze.io for further usage instructions.
-
## Research & Student Work
If you are looking for an exciting thesis project or student job in the field of static analysis, we are happy to discuss possible topics. Please contact us at _codyze [at] aisec.fraunhofer.de_.

diff:
@@ -91,6 +91,7 @@ class CpgConfiguration {
)
var codeInNodes = true
+ @JsonProperty("annotations")
@Option(
names = ["--annotations"],
description = ["Enables processing annotations or annotation-like elements"],
@@ -115,6 +116,7 @@ class CpgConfiguration {
)
var symbols: Map<String, String> = HashMap()
+ @JsonProperty("parallel-frontends")
@Option(
names = ["--parallel-frontends"],
description =

original_message:
Add schema for config file (#447)

This PR adds a JSON schema which users can use to generate and validate their YAML configuration files. The README is also updated to include a short exemplary codyze call and link to the configuration wiki page instead of listing them all in the README.


message:
Add new config defaults
This PR adds config defaults for source, sarif and config. The command line mode is also started if no mode was specified.

diff:
@@ -26,21 +26,19 @@ This will provide you with an executable Codyze installation under `build/instal
To start Codyze, change to the directory and run Codyze.
Codyze has three execution modes:
-* commando line interface mode (`-c`)
+* commando line interface mode (`-c`, default)
* language server protocol mode (`-l`)
* interactive console mode (`-t`).
-One of these modes has to be specified as command line option when running Codyze.
-
An exemplary call to start the commando line interface mode would be
```shell
$ cd build/install/codyze
-$ ./bin/codyze -c -m ./mark -s <sourcepath>
+$ ./bin/codyze -m ./mark -s <sourcepath>
```
where `<sourcepath>` denotes the path to the source directory or file which should be analyzed.
-Codyze can be further configured with more command line arguments or a YAML configuration file.
+Codyze can be further configured with additional command line arguments or a YAML configuration file.
For more information about the usage and configurations, please refer to https://www.codyze.io and the corresponding [wiki page](https://github.com/Fraunhofer-AISEC/codyze/wiki/Usage).

diff:
@@ -49,6 +49,7 @@ public class Main {
*/
public static void main(String... args) throws Exception {
ConfigFilePath firstPass = new ConfigFilePath();
+ firstPass.configFile = null;
CommandLine cmd = new CommandLine(firstPass);
cmd.parseArgs(args); // first pass to get potential config file path
if (cmd.isUsageHelpRequested()) {
@@ -93,7 +94,7 @@ public class Main {
server.start();
log.info("Analysis server started in {} in ms.", Duration.between(start, Instant.now()).toMillis());
- if (!configuration.getExecutionMode().isLsp() && configuration.getSource() != null) {
+ if (!configuration.getExecutionMode().isLsp()) {
log.info("Analyzing {}", configuration.getSource());
AnalysisContext ctx = server
.analyze(configuration.getSource().getAbsolutePath())
@@ -133,9 +134,9 @@ public class Main {
// Stores path to config file given as cli option
@Command(mixinStandardHelpOptions = true)
- static class ConfigFilePath {
- @Option(names = { "--config" }, paramLabel = "<path>", description = "Parse configuration settings from this file.")
- File configFile;
+ public static class ConfigFilePath {
+ @Option(names = { "--config" }, paramLabel = "<path>", description = "Parse configuration settings from this file.\n\t(Default: ${DEFAULT-VALUE})")
+ public File configFile = new File("codyze.yaml");
@Unmatched
List<String> remainder;

diff:
@@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature
import com.fasterxml.jackson.databind.MapperFeature
import com.fasterxml.jackson.databind.PropertyNamingStrategies
import com.fasterxml.jackson.dataformat.yaml.YAMLMapper
+import de.fraunhofer.aisec.codyze.Main.ConfigFilePath
import de.fraunhofer.aisec.codyze.analysis.ServerConfiguration
import de.fraunhofer.aisec.cpg.TranslationConfiguration
import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend
@@ -22,15 +23,15 @@ import picocli.CommandLine
class Configuration {
@JsonIgnore
- @CommandLine.ArgGroup(exclusive = true, multiplicity = "1", heading = "Execution Mode\n")
+ @CommandLine.ArgGroup(exclusive = true, heading = "Execution Mode\n")
val executionMode: ExecutionMode = ExecutionMode()
@CommandLine.Option(
names = ["-s", "--source"],
paramLabel = "<path>",
- description = ["Source file or folder to analyze."]
+ description = ["Source file or folder to analyze.\n\t(Default: \${DEFAULT-VALUE})"]
)
- var source: File? = null
+ var source = File("./")
private set
// TODO output standard stdout?
@@ -53,10 +54,13 @@ class Configuration {
@JsonProperty("sarif")
@CommandLine.Option(
names = ["--sarif"],
- description = ["Enables the SARIF output."],
+ negatable = true,
+ description =
+ [
+ "Controls whether the output is written in the SARIF format.\n\t(Default: \${DEFAULT-VALUE})"],
fallbackValue = "true"
)
- var sarifOutput: Boolean = false
+ var sarifOutput = true
private set
private var codyze = CodyzeConfiguration()
@@ -193,6 +197,10 @@ class Configuration {
// we don't want the parser to print to the terminal when in LSP mode
cpg.debugParser = false
}
+
+ if (!executionMode.isCli && !executionMode.isLsp && !executionMode.isTui) {
+ executionMode.isCli = true
+ }
}
// Parse CLI arguments into config class
@@ -228,7 +236,16 @@ class Configuration {
@JvmStatic
fun initConfig(configFile: File?, vararg args: String?): Configuration {
val config: Configuration =
- if (configFile != null) parseFile(configFile) else Configuration()
+ if (configFile != null) {
+ parseFile(configFile)
+ } else {
+ val defaultConfigFile = ConfigFilePath().configFile
+ if (defaultConfigFile.isFile) {
+ parseFile(defaultConfigFile)
+ } else {
+ Configuration()
+ }
+ }
config.parseCLI(*args)
return config
}
@@ -283,7 +300,7 @@ class ExecutionMode {
@CommandLine.Option(
names = ["-c"],
required = true,
- description = ["Start in command line mode."]
+ description = ["Start in command line mode (default)."]
)
var isCli = false

diff:
@@ -183,8 +183,7 @@ class ExpressionEvaluatorTest : AbstractTest() {
var myEntity: EntityDeclaration? = null
var myRule: RuleDeclaration? = null
- model =
- mark {
+ model = mark {
myEntity =
entity("MyClass") {
variable("field")

diff:
@@ -12,14 +12,6 @@ import picocli.CommandLine
internal class CLILoadTest {
- @Test
- @Throws(Exception::class)
- fun noArgsTest() {
- Assertions.assertThrows(CommandLine.MissingParameterException::class.java) {
- Configuration.initConfig(null, *emptyArray())
- }
- }
-
@Test
@Throws(Exception::class)
fun exclusiveOptionTest() {
@@ -39,10 +31,10 @@ internal class CLILoadTest {
assertTrue(config.executionMode.isCli)
assertFalse(config.executionMode.isLsp)
assertFalse(config.executionMode.isTui)
- assertNull(config.source)
+ assertEquals(File(".").absolutePath, config.source.absolutePath)
assertEquals(120L, config.timeout)
assertEquals("findings.sarif", config.output)
- assertFalse(config.sarifOutput)
+ assertTrue(config.sarifOutput)
assertContentEquals(
arrayOf("./").map { s -> File(s).absolutePath }.toTypedArray(),

diff:
@@ -70,7 +70,7 @@ internal class ConfigLoadTest {
assertEquals("source.java", config.source.toString())
assertEquals(120L, config.timeout)
assertEquals("result.out", config.output)
- assertFalse(config.sarifOutput)
+ assertTrue(config.sarifOutput)
assertContentEquals(
arrayOf("./").map { s -> File(s).absolutePath }.toTypedArray(),

original_message:
Add new config defaults (#448)

This PR adds config defaults for source, sarif and config. The command line mode is also started if no mode was specified.

Co-authored-by: Florian Wendland <florian.wendland@aisec.fraunhofer.de>


message:
Change config path default
This PR changes the config path default behavior so that the default value "codyze.yaml" is only used when --config is specified. When the option is not present, no config file is parsed.

diff:
@@ -49,7 +49,6 @@ public class Main {
*/
public static void main(String... args) throws Exception {
ConfigFilePath firstPass = new ConfigFilePath();
- firstPass.configFile = null;
CommandLine cmd = new CommandLine(firstPass);
cmd.parseArgs(args); // first pass to get potential config file path
if (cmd.isUsageHelpRequested()) {
@@ -135,8 +134,9 @@ public class Main {
// Stores path to config file given as cli option
@Command(mixinStandardHelpOptions = true)
public static class ConfigFilePath {
- @Option(names = { "--config" }, paramLabel = "<path>", description = "Parse configuration settings from this file.\n\t(Default: ${DEFAULT-VALUE})")
- public File configFile = new File("codyze.yaml");
+ @Option(names = {
+ "--config" }, paramLabel = "<path>", fallbackValue = "codyze.yaml", arity = "0..1", description = "Parse configuration settings from this file. If no file path is specified, codyze will try to load the configuration file from ./codyze.yaml")
+ public File configFile;
@Unmatched
List<String> remainder;

diff:
@@ -10,7 +10,6 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize
import com.fasterxml.jackson.databind.deser.BeanDeserializerModifier
import com.fasterxml.jackson.databind.module.SimpleModule
import com.fasterxml.jackson.dataformat.yaml.YAMLMapper
-import de.fraunhofer.aisec.codyze.Main.ConfigFilePath
import de.fraunhofer.aisec.codyze.analysis.ServerConfiguration
import de.fraunhofer.aisec.codyze.config.converters.FileDeserializer
import de.fraunhofer.aisec.codyze.config.converters.OutputDeserializer
@@ -243,14 +242,9 @@ class Configuration {
val config: Configuration =
if (configFile != null) {
parseFile(configFile)
- } else {
- val defaultConfigFile = ConfigFilePath().configFile
- if (defaultConfigFile.isFile) {
- parseFile(defaultConfigFile)
} else {
Configuration()
}
- }
config.parseCLI(*args)
return config
}

original_message:
Change config path default (#459)

This PR changes the config path default behavior so that the default value "codyze.yaml" is only used when --config is specified. When the option is not present, no config file is parsed.

Co-authored-by: Florian Wendland <florian.wendland@aisec.fraunhofer.de>


message:
Fix typos and inconsistencies in help message
This PR fixes typos and missing punctuation in the help message. All split characters for list and map type options are changed to the path separation character for each OS (: for Linux/Mac, ; for Windows) for consistency.

diff:
@@ -14,7 +14,7 @@ class CodyzeConfiguration {
names = ["-m", "--mark"],
paramLabel = "<path>",
description = ["Loads MARK policy files.\n\t(Default: \${DEFAULT-VALUE})"],
- split = ","
+ split = "\${sys:path.separator}"
)
var mark: Array<File> = arrayOf(File("./"))
@@ -23,7 +23,7 @@ class CodyzeConfiguration {
names = ["--no-good-findings"],
description =
[
- "Disable output of \"positive\" findings which indicate correct implementations\n" +
+ "Disable output of \"positive\" findings which indicate correct implementations.\n" +
"\t(Default: \${DEFAULT-VALUE})"],
fallbackValue = "true"
)
@@ -31,11 +31,11 @@ class CodyzeConfiguration {
@Option(
names = ["--disabled-mark-rules"],
- paramLabel = "<package.rule>",
+ paramLabel = "<package>.<rule>",
description =
[
- "The specified mark rules will be excluded from being parsed and processed. The rule has to be specified by its fully qualified name (package.rule). If there is no package name, specify rule as \".rule\". Use \'*\' to disable an entire package."],
- split = ","
+ "The specified mark rules will be excluded from being parsed and processed. The rule has to be specified by its fully qualified name. If there is no package name, specify rule as \".<rule>\". Use \'<package>.*\' to disable an entire package."],
+ split = "\${sys:path.separator}"
)
var disabledMarkRules: List<String> = emptyList()

diff:
@@ -16,8 +16,8 @@ class CpgConfiguration {
@Option(
names = ["--additional-languages"],
- split = ",",
- paramLabel = "language",
+ split = "\${sys:path.separator}",
+ paramLabel = "<language>",
description =
[
"Enables the experimental support for additional languages (currently \${COMPLETION-CANDIDATES}). Additional files need to be placed in certain locations. Please follow the CPG README."]
@@ -39,7 +39,7 @@ class CpgConfiguration {
negatable = true,
description =
[
- "If false, type listener system is only activated after the frontends are done building the initial AST structure.(Default: \${DEFAULT-VALUE})"],
+ "If false, type listener system is only activated after the frontends are done building the initial AST structure.\n\t(Default: \${DEFAULT-VALUE})"],
fallbackValue = "true"
)
var typeSystemInFrontend = true
@@ -55,11 +55,11 @@ class CpgConfiguration {
@Option(
names = ["--passes"],
- paramLabel = "pass",
+ paramLabel = "<pass>",
description =
[
"CPG passes in the order in which they should be executed, fully qualified name of the classes only. If default-passes is specified, the default passes are executed first."],
- split = ","
+ split = "\${sys:path.separator}"
)
@JvmName("setPassesNull")
fun setPasses(passes: List<Pass?>) {
@@ -69,7 +69,7 @@ class CpgConfiguration {
@Option(
names = ["--debug-parser"],
- description = ["Controls debug output generation for the cpg parser"],
+ description = ["Controls debug output generation for the cpg parser."],
fallbackValue = "true"
)
var debugParser = false
@@ -78,7 +78,7 @@ class CpgConfiguration {
names = ["--disable-cleanup"],
description =
[
- "Switch off cleaning up TypeManager memory after the analysis. Set to true only for testing"],
+ "Switch off cleaning up TypeManager memory after the analysis. Set to true only for testing."],
fallbackValue = "true"
)
var disableCleanup = false
@@ -88,7 +88,7 @@ class CpgConfiguration {
negatable = true,
description =
[
- "Controls showing the code of a node as parameter in the node\n\t(Default: \${DEFAULT-VALUE})"],
+ "Controls showing the code of a node as parameter in the node.\n\t(Default: \${DEFAULT-VALUE})"],
fallbackValue = "true"
)
var codeInNodes = true
@@ -96,7 +96,7 @@ class CpgConfiguration {
@JsonProperty("annotations")
@Option(
names = ["--annotations"],
- description = ["Enables processing annotations or annotation-like elements"],
+ description = ["Enables processing annotations or annotation-like elements."],
fallbackValue = "true"
)
var processAnnotations = false
@@ -105,16 +105,16 @@ class CpgConfiguration {
names = ["--fail-on-error"],
description =
[
- "Should the parser/translation fail on errors (true) or try to continue in a best-effort manner (false)\n\t(Default: \${DEFAULT-VALUE})"],
+ "Should the parser/translation fail on errors (true) or try to continue in a best-effort manner (false).\n\t(Default: \${DEFAULT-VALUE})"],
fallbackValue = "true"
)
var failOnError = false
@Option(
names = ["--symbols"],
- paramLabel = "<symbol=definition>",
- description = ["Definition of additional symbols"],
- split = ","
+ paramLabel = "<symbol>=<definition>",
+ description = ["Definition of additional symbols."],
+ split = "\${sys:path.separator}"
)
var symbols: Map<String, String> = HashMap()
@@ -123,7 +123,7 @@ class CpgConfiguration {
names = ["--parallel-frontends"],
description =
[
- "Enables parsing the ASTs for the source files in parallel, but the passes afterwards will still run in a single thread"],
+ "Enables parsing the ASTs for the source files in parallel, but the passes afterwards will still run in a single thread."],
fallbackValue = "true"
)
var useParallelFrontends = false
@@ -142,10 +142,9 @@ class TranslationSettings {
@Option(
names = ["--includes"],
- description =
- [
- "Path(s) containing include files. Path must be separated by \':\' (Mac/Linux) or \';\' (Windows)."],
- split = ":|;"
+ paramLabel = "<path>",
+ description = ["Path(s) containing include files."],
+ split = "\${sys:path.separator}"
)
var includes: Array<File> = emptyArray()
@@ -154,8 +153,8 @@ class TranslationSettings {
paramLabel = "<path>",
description =
[
- "If includes is not empty, only the specified files will be parsed and processed in the cpg, unless it is a part of the disabled list, in which it will be ignored. Path must be separated by \':\' (Mac/Linux) or \';\' (Windows)"],
- split = ":|;"
+ "If includes is not empty, only the specified files will be parsed and processed in the cpg, unless it is a part of the disabled list, in which it will be ignored."],
+ split = "\${sys:path.separator}"
)
var enabledIncludes: Array<File> = emptyArray()
@@ -164,8 +163,8 @@ class TranslationSettings {
paramLabel = "<path>",
description =
[
- "If includes is not empty, the specified files will be excluded from being parsed and processed in the cpg. The disabled list entries always take priority over the enabled list entries. Path must be separated by \':\' (Mac/Linux) or \';\' (Windows)"],
- split = ":|;"
+ "If includes is not empty, the specified files will be excluded from being parsed and processed in the cpg. The disabled list entries always take priority over the enabled list entries."],
+ split = "\${sys:path.separator}"
)
var disabledIncludes: Array<File> = emptyArray()
}

diff:
@@ -76,8 +76,8 @@ internal class CLILoadTest {
Configuration.initConfig(
null,
"-c",
- "--passes=de.fraunhofer.aisec.cpg.passes.EdgeCachePass," +
- "de.fraunhofer.aisec.cpg.passes.FilenameMapper," +
+ "--passes=de.fraunhofer.aisec.cpg.passes.EdgeCachePass${File.pathSeparator}" +
+ "de.fraunhofer.aisec.cpg.passes.FilenameMapper${File.pathSeparator}" +
"de.fraunhofer.aisec.cpg.passes.CallResolver"
)
val translationConfiguration = config.buildTranslationConfiguration(File("test.java"))
@@ -104,10 +104,10 @@ internal class CLILoadTest {
Configuration.initConfig(
null,
"-c",
- "--passes=de.fraunhofer.aisec.cpg.passes.MyPass," +
- "de.fraunhofer.aisec.cpg.passes.Pass," +
- "de.fraunhofer.aisec.cpg.passes.scopes.BlockScope," +
- "de.fraunhofer.aisec.cpg.passes.EdgeCachePass," +
+ "--passes=de.fraunhofer.aisec.cpg.passes.MyPass${File.pathSeparator}" +
+ "de.fraunhofer.aisec.cpg.passes.Pass${File.pathSeparator}" +
+ "de.fraunhofer.aisec.cpg.passes.scopes.BlockScope${File.pathSeparator}" +
+ "de.fraunhofer.aisec.cpg.passes.EdgeCachePass${File.pathSeparator}" +
"MyPass2"
)
val translationConfiguration = config.buildTranslationConfiguration(File("test.java"))
@@ -125,7 +125,8 @@ internal class CLILoadTest {
@Test
@Throws(Exception::class)
fun symbolsOptionTest() {
- val config = Configuration.initConfig(null, "-c", "--symbols=#=hash,*=star")
+ val config =
+ Configuration.initConfig(null, "-c", "--symbols=#=hash${File.pathSeparator}*=star")
val translationConfiguration = config.buildTranslationConfiguration(File("test.java"))
assertEquals(
2,

diff:
@@ -25,7 +25,7 @@ class ConfigCLILoadTest {
"--sarif", // test if true stays true
"--unity", // test if false is set to true
"--analyze-includes=false", // test if false is set to false
- "-m=mark5,mark7,mark6"
+ "-m=mark5${File.pathSeparator}mark7${File.pathSeparator}mark6"
)
val config = Configuration.initConfig(correctFile, *options)
val serverConfig = config.buildServerConfiguration()
@@ -82,9 +82,9 @@ class ConfigCLILoadTest {
Configuration.initConfig(
additionalOptionFile,
"-c",
- "--passes=de.fraunhofer.aisec.cpg.passes.FilenameMapper," +
+ "--passes=de.fraunhofer.aisec.cpg.passes.FilenameMapper${File.pathSeparator}" +
"de.fraunhofer.aisec.cpg.passes.CallResolver",
- "--symbols=&=and,+=plus",
+ "--symbols=&=and${File.pathSeparator}+=plus",
"--no-type-system-in-frontend",
"--default-passes"
)

original_message:
Fix typos and inconsistencies in help message (#464)

This PR fixes typos and missing punctuation in the help message. All split characters for list and map type options are changed to the path separation character for each OS (: for Linux/Mac, ; for Windows) for consistency.

Co-authored-by: Florian Wendland <florian.wendland@aisec.fraunhofer.de>


message:
Log unmatched arguments and unknown properties
This PR adds logging of all unmatched arguments and unknown properties which are currently just ignored. A custom Jackson Module and ProblemHandler are used to collect and log the unknown properties.

diff:
@@ -67,7 +67,7 @@ public class Main {
int returnCode = 0;
try {
- Configuration config = Configuration.initConfig(firstPass.configFile, args);
+ Configuration config = Configuration.initConfig(firstPass.configFile, firstPass.remainder.toArray(new String[0]));
// Start analysis setup
returnCode = start(config);
}

diff:
@@ -8,6 +8,7 @@ import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.*
import com.fasterxml.jackson.databind.annotation.JsonDeserialize
import com.fasterxml.jackson.databind.deser.BeanDeserializerModifier
+import com.fasterxml.jackson.databind.deser.DeserializationProblemHandler
import com.fasterxml.jackson.databind.module.SimpleModule
import com.fasterxml.jackson.dataformat.yaml.YAMLMapper
import de.fraunhofer.aisec.codyze.analysis.ServerConfiguration
@@ -19,6 +20,7 @@ import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend
import de.fraunhofer.aisec.cpg.passes.Pass
import java.io.File
import java.io.IOException
+import org.slf4j.Logger
import org.slf4j.LoggerFactory
import picocli.CommandLine
@@ -293,6 +295,7 @@ class Configuration {
// Parse CLI arguments into config class
private fun parseCLI(vararg args: String?) {
+ val cl =
CommandLine(this)
// Added as Mixin so the already initialized objects are used instead of new ones
// created
@@ -302,11 +305,20 @@ class Configuration {
.addMixin("translation", cpg.translation)
.registerConverter(Pass::class.java, PassTypeConverter())
.setCaseInsensitiveEnumValuesAllowed(true)
- // setUnmatchedArgumentsAllowed is true because both classes don't have the config path
- // option which would result in exceptions, side effect is that all unknown options are
- // ignored
+ // setUnmatchedArgumentsAllowed is true because both classes don't have the config
+ // path option which would result in exceptions, side effect is that all unknown
+ // options are ignored
.setUnmatchedArgumentsAllowed(true)
.parseArgs(*args)
+
+ // log any unmatched arguments
+ if (cl.unmatched() != null && cl.unmatched().isNotEmpty()) {
+ log.warn(
+ "{} argument(s) could not be matched: {}",
+ cl.unmatched().size,
+ cl.unmatched().toString()
+ )
+ }
}
companion object {
@@ -350,6 +362,8 @@ class Configuration {
}
}
)
+ val logProblemHandler = LogDeserializationProblemHandler(log)
+
val mapper =
YAMLMapper.builder().enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS).build()
mapper
@@ -357,16 +371,21 @@ class Configuration {
.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL)
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
.registerModule(module)
+ .registerModule(LogModule(logProblemHandler))
mapper.injectableValues =
InjectableValues.Std()
.addValue("configFileBasePath", configFile.absoluteFile.parentFile.absolutePath)
mapper.propertyNamingStrategy = PropertyNamingStrategies.KebabCaseStrategy()
+
var config: Configuration? = null
try {
config = mapper.readValue(configFile, Configuration::class.java)
} catch (e: IOException) {
printErrorMessage(configFile.absolutePath, e.toString())
}
+
+ logProblemHandler.printToLog()
+
return config ?: Configuration()
}
@@ -420,3 +439,35 @@ class ExecutionMode {
)
var isTui = false
}
+
+// Taken from:
+// https://stackoverflow.com/questions/46644099/cant-set-problemhandler-to-objectmapper-in-spring-boot
+class LogModule(private val problemHandler: LogDeserializationProblemHandler) : SimpleModule() {
+ override fun setupModule(context: SetupContext) {
+ super.setupModule(context)
+ context.addDeserializationProblemHandler(problemHandler)
+ }
+}
+
+class LogDeserializationProblemHandler(val log: Logger) : DeserializationProblemHandler() {
+ private val unknownPropNames = mutableListOf<String>()
+
+ fun printToLog() {
+ if (unknownPropNames.isNotEmpty()) {
+ log.warn(
+ "${if(unknownPropNames.size == 1) "1 property is" else "${unknownPropNames.size} properties are"} unknown: $unknownPropNames"
+ )
+ }
+ }
+
+ override fun handleUnknownProperty(
+ ctxt: DeserializationContext,
+ p: JsonParser,
+ deserializer: JsonDeserializer<*>,
+ beanOrClass: Any,
+ propertyName: String
+ ): Boolean {
+ unknownPropNames.add(propertyName)
+ return false
+ }
+}

original_message:
Log unmatched arguments and unknown properties (#465)

This PR adds logging of all unmatched arguments and unknown properties which are currently just ignored. A custom Jackson Module and ProblemHandler are used to collect and log the unknown properties.

Co-authored-by: Florian Wendland <florian.wendland@aisec.fraunhofer.de>


message:
Remove unnecessary "if" in buildTranslationConfig
Remove unnecessary if for debugParser in buildTranslationConfig() because it is already covered in normalize()

diff:
@@ -135,7 +135,7 @@ class Configuration {
this.normalize()
val translationConfig =
TranslationConfiguration.builder()
- .debugParser(if (executionMode.isLsp) false else cpg.debugParser)
+ .debugParser(cpg.debugParser)
.failOnError(cpg.failOnError)
.codeInNodes(cpg.codeInNodes)
.loadIncludes(cpg.translation.analyzeIncludes)

original_message:
Remove unnecessary "if" in buildTranslationConfig (#468)

Remove unnecessary if for debugParser in buildTranslationConfig() because it is already covered in normalize()

Co-authored-by: Florian Wendland <florian.wendland@aisec.fraunhofer.de>


message:
Make collection type options appendable
This PR add a new option name (e.g. --option-name+) for all list and map type options so that the data from the command line can be appended to the config file data instead of overwriting it.

diff:
@@ -54,7 +54,11 @@ public class Main {
cmd.parseArgs(args); // first pass to get potential config file path
if (cmd.isUsageHelpRequested()) {
// print help message
- CommandLine c = new CommandLine(new Help());
+ Help help = new Help();
+ System.setProperty("source", Arrays.toString(help.configuration.getSource()));
+ System.setProperty("mark", Arrays.toString(help.codyzeConfig.getMark()));
+
+ CommandLine c = new CommandLine(help);
c.getHelpSectionMap().put(SECTION_KEY_OPTION_LIST, new HelpRenderer());
c.usage(System.out);
System.exit(c.getCommandSpec().exitCodeOnUsageHelp());
@@ -136,7 +140,7 @@ public class Main {
@Command(mixinStandardHelpOptions = true)
public static class ConfigFilePath {
@Option(names = {
- "--config" }, paramLabel = "<path>", fallbackValue = "codyze.yaml", arity = "0..1", description = "Parse configuration settings from this file. If no file path is specified, codyze will try to load the configuration file from ./codyze.yaml")
+ "--config" }, paramLabel = "<path>", fallbackValue = "codyze.yaml", arity = "0..1", description = "Parse configuration settings from this file. If no file path is specified, codyze will try to load the configuration file from ${FALLBACK-VALUE}")
public File configFile;
@Unmatched
@@ -151,10 +155,10 @@ public class Main {
private ConfigFilePath configFilePath;
// ArgGroups only for display purposes
- @ArgGroup(heading = "@|bold,underline Codyze Options|@\n", exclusive = false)
+ @ArgGroup(heading = "@|bold,underline Codyze Options|@\n", exclusive = false, validate = true)
private CodyzeConfiguration codyzeConfig = new CodyzeConfiguration();
- @ArgGroup(heading = "", exclusive = false)
+ @ArgGroup(exclusive = false)
private Configuration configuration = new Configuration();
@ArgGroup(exclusive = false, heading = "Analysis Options\n")
@@ -182,7 +186,7 @@ public class Main {
for (CommandSpec c : mix.values()) {
sb.append(help.optionListExcludingGroups(c.options().stream().filter(optionSpec -> optionSpec.group() == null).collect(Collectors.toList())));
}
- sb.append("\n");
+
for (ArgGroupSpec group : spec.argGroups()) {
addHierachy(group, sb);
}
@@ -191,13 +195,15 @@ public class Main {
}
private void addHierachy(ArgGroupSpec argGroupSpec, StringBuilder sb) {
+ if (argGroupSpec.heading() != null) {
+ sb.append("\n");
sb.append(help.colorScheme().text(argGroupSpec.heading()).toString());
+ }
// render all options that are not in subgroups
sb.append(help.optionListExcludingGroups(
argGroupSpec.options().stream().filter(optionSpec -> optionSpec.group().equals(argGroupSpec)).collect(Collectors.toList())));
- sb.append("\n");
for (ArgGroupSpec group : argGroupSpec.subgroups()) {
addHierachy(group, sb);
}

diff:
package de.fraunhofer.aisec.codyze.config
+import com.fasterxml.jackson.annotation.JsonIgnore
import com.fasterxml.jackson.annotation.JsonProperty
import de.fraunhofer.aisec.codyze.analysis.TypestateMode
import java.io.File
+import picocli.CommandLine.ArgGroup
import picocli.CommandLine.Option
class CodyzeConfiguration {
@@ -10,12 +12,7 @@ class CodyzeConfiguration {
// TODO: names
val analysis = AnalysisMode()
- @Option(
- names = ["-m", "--mark"],
- paramLabel = "<path>",
- description = ["Loads MARK policy files.\n\t(Default: \${DEFAULT-VALUE})"],
- split = "\${sys:path.separator}"
- )
+ @JsonIgnore @ArgGroup(exclusive = true) val markCLI = MarkArgGroup()
var mark: Array<File> = arrayOf(File("./"))
// TODO: change name or make into warning levels
@@ -29,14 +26,7 @@ class CodyzeConfiguration {
)
var noGoodFindings = false
- @Option(
- names = ["--disabled-mark-rules"],
- paramLabel = "<package>.<rule>",
- description =
- [
- "The specified mark rules will be excluded from being parsed and processed. The rule has to be specified by its fully qualified name. If there is no package name, specify rule as \".<rule>\". Use \'<package>.*\' to disable an entire package."],
- split = "\${sys:path.separator}"
- )
+ @JsonIgnore @ArgGroup(exclusive = true) val disabledMarkRulesCLI = DisabledMarkArgGroup()
var disabledMarkRules: List<String> = emptyList()
@Option(
@@ -69,3 +59,65 @@ class AnalysisMode {
)
var tsMode = TypestateMode.DFA
}
+
+class MarkArgGroup {
+ var append = false
+ var matched = false
+
+ var mark: Array<File> = emptyArray()
+
+ @Option(
+ names = ["-m", "--mark"],
+ paramLabel = "<path>",
+ description = ["Loads MARK policy files.\n\t(Default: \${sys:mark})"],
+ split = "\${sys:path.separator}"
+ )
+ fun match(value: Array<File>) {
+ matched = true
+ this.mark = value
+ }
+
+ @Option(
+ names = ["--mark+"],
+ paramLabel = "<path>",
+ description =
+ ["See --mark, but appends the values to the ones specified in configuration file."],
+ split = "\${sys:path.separator}"
+ )
+ fun append(value: Array<File>) {
+ append = true
+ match(value)
+ }
+}
+
+class DisabledMarkArgGroup {
+ var append = false
+ var matched = false
+
+ var disabledMarkRules: List<String> = emptyList()
+ @Option(
+ names = ["--disabled-mark-rules"],
+ paramLabel = "<package>.<rule>",
+ description =
+ [
+ "The specified mark rules will be excluded from being parsed and processed. The rule has to be specified by its fully qualified name. If there is no package name, specify rule as \".<rule>\". Use \"<package>.*\" to disable an entire package."],
+ split = "\${sys:path.separator}"
+ )
+ fun match(value: List<String>) {
+ matched = true
+ this.disabledMarkRules = value
+ }
+
+ @Option(
+ names = ["--disabled-mark-rules+"],
+ paramLabel = "<package>.<rule>",
+ description =
+ [
+ "See --disabled-mark-rules, but appends the values to the ones specified in configuration file."],
+ split = "\${sys:path.separator}"
+ )
+ fun append(value: List<String>) {
+ append = true
+ match(value)
+ }
+}

diff:
@@ -27,29 +27,46 @@ import picocli.CommandLine
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY)
class Configuration {
- @JsonIgnore
- @CommandLine.ArgGroup(exclusive = true, heading = "Execution Mode\n")
- val executionMode: ExecutionMode = ExecutionMode()
-
- @CommandLine.Option(
- names = ["-s", "--source"],
- paramLabel = "<path>",
- split = "\${sys:path.separator}",
- description = ["Source files or folders to analyze.\n\t(Default: \${DEFAULT-VALUE})"]
- )
+ @JsonIgnore @CommandLine.ArgGroup(exclusive = true) private val sourceCLI = SourceArgGroup()
var source: Array<File> = arrayOf(File("./"))
private set
+ get() {
+ val result = mutableListOf<File>()
+ if (!sourceCLI.matched || sourceCLI.append) {
+ result.addAll(field)
+ }
+
+ if (sourceCLI.matched) {
+ result.addAll(sourceCLI.source)
+ }
+
+ return result.toTypedArray()
+ }
+
+ @JsonIgnore
+ @CommandLine.ArgGroup(exclusive = true)
+ private val disabledSourceCLI = DisabledSourceArgGroup()
@JsonProperty("disabled-sources")
- @CommandLine.Option(
- names = ["--disabled-sources"],
- paramLabel = "<path>",
- split = "\${sys:path.separator}",
- description =
- [
- "Files or folders specified here will not be analyzed. Symbolic links are not followed when filtering out these paths"]
- )
var disabledSource: Array<File> = emptyArray()
+ private set
+ get() {
+ val result = mutableListOf<File>()
+
+ if (!disabledSourceCLI.matched || disabledSourceCLI.append) {
+ result.addAll(field)
+ }
+
+ if (disabledSourceCLI.matched) {
+ result.addAll(disabledSourceCLI.disabledSource)
+ }
+
+ return result.toTypedArray()
+ }
+
+ @JsonIgnore
+ @CommandLine.ArgGroup(exclusive = true, heading = "Execution Mode\n")
+ val executionMode: ExecutionMode = ExecutionMode()
// TODO output standard stdout?
@CommandLine.Option(
@@ -104,11 +121,19 @@ class Configuration {
.launchConsole(executionMode.isTui)
.typestateAnalysis(codyze.analysis.tsMode)
.disableGoodFindings(codyze.noGoodFindings)
- .markFiles(*codyze.mark.map { m -> m.absolutePath }.toTypedArray())
.pedantic(codyze.pedantic)
+ val mark = mutableListOf<File>()
+ if (!codyze.markCLI.matched || codyze.markCLI.append) mark.addAll(codyze.mark)
+ if (codyze.markCLI.matched) mark.addAll(codyze.markCLI.mark)
+ config.markFiles(*mark.map { m -> m.absolutePath }.toTypedArray())
+
val disabledRulesMap = mutableMapOf<String, DisabledMarkRulesValue>()
- for (mName in codyze.disabledMarkRules) {
+ val disabledMarkRules = codyze.disabledMarkRulesCLI.disabledMarkRules.toMutableList()
+ if (!codyze.disabledMarkRulesCLI.matched || codyze.disabledMarkRulesCLI.append)
+ disabledMarkRules.addAll(codyze.disabledMarkRules)
+
+ for (mName in disabledMarkRules) {
val index = mName.lastIndexOf('.')
val packageName = mName.subSequence(0, index).toString()
val markName = mName.subSequence(index + 1, mName.length).toString()
@@ -120,9 +145,10 @@ class Configuration {
log.warn(
"Error while parsing disabled-mark-rules: \'$mName\' is not a valid name for a mark rule. Continue parsing disabled-mark-rules"
)
- config.disableMark(disabledRulesMap)
}
+ config.disableMark(disabledRulesMap)
+
return config.build()
}
@@ -141,42 +167,77 @@ class Configuration {
.loadIncludes(cpg.translation.analyzeIncludes)
.useUnityBuild(cpg.useUnityBuild)
.processAnnotations(cpg.processAnnotations)
- .symbols(cpg.symbols)
.useParallelFrontends(cpg.useParallelFrontends)
.typeSystemActiveInFrontend(cpg.typeSystemInFrontend)
.defaultLanguages()
.sourceLocations(*filterFiles(disabledSource, *sources))
+ val symbols = cpg.symbolsCLI.symbols.toMutableMap()
+ if (!cpg.symbolsCLI.matched || cpg.symbolsCLI.append) {
+ symbols.putAll(cpg.symbols)
+ }
+ translationConfig.symbols(symbols)
+
+ if (!cpg.translation.includesCLI.matched || cpg.translation.includesCLI.append) {
for (file in cpg.translation.includes) {
translationConfig.includePath(file.absolutePath)
}
+ }
+ if (cpg.translation.includesCLI.matched)
+ for (file in cpg.translation.includesCLI.includes) {
+ translationConfig.includePath(file.absolutePath)
+ }
+
+ if (!cpg.translation.enabledIncludesCLI.matched || cpg.translation.enabledIncludesCLI.append
+ ) {
for (s in cpg.translation.enabledIncludes) {
translationConfig.includeWhitelist(s.absolutePath)
}
+ }
+ if (cpg.translation.enabledIncludesCLI.matched)
+ for (file in cpg.translation.enabledIncludesCLI.enabledIncludes) {
+ translationConfig.includeWhitelist(file.absolutePath)
+ }
+
+ if (!cpg.translation.disabledIncludesCLI.matched ||
+ cpg.translation.disabledIncludesCLI.append
+ ) {
for (s in cpg.translation.disabledIncludes) {
translationConfig.includeBlacklist(s.absolutePath)
}
+ }
+ if (cpg.translation.disabledIncludesCLI.matched)
+ for (file in cpg.translation.disabledIncludesCLI.disabledIncludes) {
+ translationConfig.includeBlacklist(file.absolutePath)
+ }
if (cpg.disableCleanup) {
translationConfig.disableCleanup()
}
if (cpg.defaultPasses == null) {
- if (cpg.passes.isEmpty()) {
+ if (cpg.passesCLI.passes.isEmpty() && cpg.passes.isEmpty()) {
translationConfig.defaultPasses()
}
} else {
if (cpg.defaultPasses!!) {
translationConfig.defaultPasses()
} else {
- if (cpg.passes.isEmpty()) {
+ if (cpg.passesCLI.passes.isEmpty() && cpg.passes.isEmpty()) {
// TODO: error handling for no passes if needed
}
}
}
+
+ if (!cpg.passesCLI.matched || cpg.passesCLI.append) {
for (p in cpg.passes) {
translationConfig.registerPass(p)
}
+ }
+ if (cpg.passesCLI.matched)
+ for (p in cpg.passesCLI.passes) {
+ translationConfig.registerPass(p)
+ }
for (l in cpg.additionalLanguages) {
val frontendClazz =
@@ -277,6 +338,7 @@ class Configuration {
if (codyze.pedantic) {
codyze.noGoodFindings = false
codyze.disabledMarkRules = emptyList()
+ codyze.disabledMarkRulesCLI.disabledMarkRules = emptyList()
}
// we need to force load includes for unity builds, otherwise nothing will be parsed
@@ -440,6 +502,67 @@ class ExecutionMode {
var isTui = false
}
+class SourceArgGroup {
+ var append = false
+ var matched = false
+
+ var source: Array<File> = emptyArray()
+ @CommandLine.Option(
+ names = ["-s", "--source"],
+ paramLabel = "<path>",
+ split = "\${sys:path.separator}",
+ description = ["Source files or folders to analyze.\n\t(Default: \${sys:source})"]
+ )
+ fun match(value: Array<File>) {
+ matched = true
+ this.source = value
+ }
+
+ @CommandLine.Option(
+ names = ["--source+"],
+ paramLabel = "<path>",
+ split = "\${sys:path.separator}",
+ description =
+ ["See --source, but appends the values to the ones specified in configuration file."]
+ )
+ fun append(value: Array<File>) {
+ append = true
+ match(value)
+ }
+}
+
+class DisabledSourceArgGroup {
+ var append = false
+ var matched = false
+
+ var disabledSource: Array<File> = emptyArray()
+ @CommandLine.Option(
+ names = ["--disabled-sources"],
+ paramLabel = "<path>",
+ split = "\${sys:path.separator}",
+ description =
+ [
+ "Files or folders specified here will not be analyzed. Symbolic links are not followed when filtering out these paths"]
+ )
+ fun match(value: Array<File>) {
+ matched = true
+ this.disabledSource = value
+ }
+
+ @CommandLine.Option(
+ names = ["--disabled-sources+"],
+ paramLabel = "<path>",
+ split = "\${sys:path.separator}",
+ description =
+ [
+ "See --disabled-sources, but appends the values to the ones specified in configuration file."]
+ )
+ fun append(value: Array<File>) {
+ append = true
+ match(value)
+ }
+}
+
// Taken from:
// https://stackoverflow.com/questions/46644099/cant-set-problemhandler-to-objectmapper-in-spring-boot
class LogModule(private val problemHandler: LogDeserializationProblemHandler) : SimpleModule() {

diff:
package de.fraunhofer.aisec.codyze.config
+import com.fasterxml.jackson.annotation.JsonIgnore
import com.fasterxml.jackson.annotation.JsonProperty
import com.fasterxml.jackson.databind.annotation.JsonDeserialize
import de.fraunhofer.aisec.codyze.config.converters.LanguageDeserializer
@@ -7,6 +8,7 @@ import de.fraunhofer.aisec.codyze.config.converters.PassListDeserializer
import de.fraunhofer.aisec.cpg.passes.Pass
import java.io.File
import java.util.*
+import picocli.CommandLine.ArgGroup
import picocli.CommandLine.Option
/** This class consists of configuration settings for CPG. */
@@ -44,29 +46,6 @@ class CpgConfiguration {
)
var typeSystemInFrontend = true
- @Option(
- names = ["--default-passes"],
- negatable = true,
- description = ["Controls the usage of default passes for cpg.\n\t(Default: true)"],
- fallbackValue = "true"
- )
- // Set to null to differentiate if it was set or not
- var defaultPasses: Boolean? = null
-
- @Option(
- names = ["--passes"],
- paramLabel = "<pass>",
- description =
- [
- "CPG passes in the order in which they should be executed, fully qualified name of the classes only. If default-passes is specified, the default passes are executed first."],
- split = "\${sys:path.separator}"
- )
- @JvmName("setPassesNull")
- fun setPasses(passes: List<Pass?>) {
- this.passes = passes.filterNotNull()
- }
- @JsonDeserialize(using = PassListDeserializer::class) var passes: List<Pass> = ArrayList()
-
@Option(
names = ["--debug-parser"],
description = ["Controls debug output generation for the cpg parser."],
@@ -110,12 +89,7 @@ class CpgConfiguration {
)
var failOnError = false
- @Option(
- names = ["--symbols"],
- paramLabel = "<symbol>=<definition>",
- description = ["Definition of additional symbols."],
- split = "\${sys:path.separator}"
- )
+ @JsonIgnore @ArgGroup(exclusive = true) val symbolsCLI = SymbolsArgGroup()
var symbols: Map<String, String> = HashMap()
@JsonProperty("parallel-frontends")
@@ -127,6 +101,18 @@ class CpgConfiguration {
fallbackValue = "true"
)
var useParallelFrontends = false
+
+ @Option(
+ names = ["--default-passes"],
+ negatable = true,
+ description = ["Controls the usage of default passes for cpg.\n\t(Default: true)"],
+ fallbackValue = "true"
+ )
+ // Set to null to differentiate if it was set or not
+ var defaultPasses: Boolean? = null
+
+ @JsonIgnore @ArgGroup(exclusive = true) val passesCLI = PassesArgGroup()
+ @JsonDeserialize(using = PassListDeserializer::class) var passes: List<Pass> = ArrayList()
}
/** This class consists of settings that control how CPG will process includes. */
@@ -140,13 +126,113 @@ class TranslationSettings {
)
var analyzeIncludes = false
+ @JsonIgnore @ArgGroup(exclusive = true) val includesCLI = IncludesArgGroup()
+ var includes: Array<File> = emptyArray()
+
+ @JsonIgnore @ArgGroup(exclusive = true) val enabledIncludesCLI = EnabledIncludesArgGroup()
+ var enabledIncludes: Array<File> = emptyArray()
+
+ @JsonIgnore @ArgGroup(exclusive = true) val disabledIncludesCLI = DisabledIncludesArgGroup()
+ var disabledIncludes: Array<File> = emptyArray()
+}
+
+class PassesArgGroup {
+ var append = false
+ var matched = false
+
+ var passes: List<Pass> = emptyList()
+
+ @Option(
+ names = ["--passes"],
+ paramLabel = "<pass>",
+ description =
+ [
+ "CPG passes in the order in which they should be executed, fully qualified name of the classes only. If default-passes is specified, the default passes are executed first."],
+ split = "\${sys:path.separator}"
+ )
+ fun match(passes: List<Pass?>) {
+ matched = true
+ this.passes = passes.filterNotNull()
+ }
+
+ @Option(
+ names = ["--passes+"],
+ paramLabel = "<pass>",
+ description =
+ ["See passes, but appends the values to the ones specified in configuration file."],
+ split = "\${sys:path.separator}"
+ )
+ fun append(value: List<Pass?>) {
+ append = true
+ match(value)
+ }
+}
+
+class SymbolsArgGroup {
+ var append = false
+ var matched = false
+
+ var symbols: Map<String, String> = emptyMap()
+
+ @Option(
+ names = ["--symbols"],
+ paramLabel = "<symbol>=<definition>",
+ description = ["Definition of additional symbols."],
+ split = "\${sys:path.separator}"
+ )
+ fun match(value: Map<String, String>) {
+ matched = true
+ this.symbols = value
+ }
+
+ @Option(
+ names = ["--symbols+"],
+ paramLabel = "<symbol>=<definition>",
+ description =
+ ["See --symbols, but appends the values to the ones specified in configuration file."],
+ split = "\${sys:path.separator}"
+ )
+ fun append(value: Map<String, String>) {
+ append = true
+ match(value)
+ }
+}
+
+class IncludesArgGroup {
+ var append = false
+ var matched = false
+
+ var includes: Array<File> = emptyArray()
+
@Option(
names = ["--includes"],
paramLabel = "<path>",
description = ["Path(s) containing include files."],
split = "\${sys:path.separator}"
)
- var includes: Array<File> = emptyArray()
+ fun match(value: Array<File>) {
+ matched = true
+ this.includes = value
+ }
+
+ @Option(
+ names = ["--includes+"],
+ paramLabel = "<path>",
+ description =
+ ["See --includes, but appends the values to the ones specified in configuration file."],
+ split = "\${sys:path.separator}"
+ )
+ fun append(value: Array<File>) {
+ append = true
+ match(value)
+ }
+}
+
+class EnabledIncludesArgGroup {
+ var append = false
+ var matched = false
+
+ var enabledIncludes: Array<File> = emptyArray()
@Option(
names = ["--enabled-includes"],
@@ -156,7 +242,30 @@ class TranslationSettings {
"If includes is not empty, only the specified files will be parsed and processed in the cpg, unless it is a part of the disabled list, in which it will be ignored."],
split = "\${sys:path.separator}"
)
- var enabledIncludes: Array<File> = emptyArray()
+ fun match(value: Array<File>) {
+ matched = true
+ this.enabledIncludes = value
+ }
+
+ @Option(
+ names = ["--enabled-includes+"],
+ paramLabel = "<path>",
+ description =
+ [
+ "See --enabled-includes, but appends the values to the ones specified in configuration file."],
+ split = "\${sys:path.separator}"
+ )
+ fun append(value: Array<File>) {
+ append = true
+ match(value)
+ }
+}
+
+class DisabledIncludesArgGroup {
+ var append = false
+ var matched = false
+
+ var disabledIncludes: Array<File> = emptyArray()
@Option(
names = ["--disabled-includes"],
@@ -166,5 +275,21 @@ class TranslationSettings {
"If includes is not empty, the specified files will be excluded from being parsed and processed in the cpg. The disabled list entries always take priority over the enabled list entries."],
split = "\${sys:path.separator}"
)
- var disabledIncludes: Array<File> = emptyArray()
+ fun match(value: Array<File>) {
+ matched = true
+ this.disabledIncludes = value
+ }
+
+ @Option(
+ names = ["--disabled-includes+"],
+ paramLabel = "<path>",
+ description =
+ [
+ "See --disabled-includes, but appends the values to the ones specified in configuration file."],
+ split = "\${sys:path.separator}"
+ )
+ fun append(value: Array<File>) {
+ append = true
+ match(value)
+ }
}

diff:
@@ -2,9 +2,12 @@ package de.fraunhofer.aisec.codyze.crymlin
import de.fraunhofer.aisec.codyze.analysis.TypestateMode
import de.fraunhofer.aisec.codyze.config.Configuration
+import de.fraunhofer.aisec.codyze.config.DisabledMarkRulesValue
import de.fraunhofer.aisec.cpg.TranslationConfiguration
import de.fraunhofer.aisec.cpg.passes.CallResolver
+import de.fraunhofer.aisec.cpg.passes.EdgeCachePass
import de.fraunhofer.aisec.cpg.passes.FilenameMapper
+import de.fraunhofer.aisec.cpg.passes.UnreachableEOGPass
import java.io.File
import java.lang.Exception
import kotlin.Throws
@@ -144,10 +147,247 @@ class ConfigCLILoadTest {
assertContentEquals(expectedDisabledIncludes, translationConfiguration.includeBlacklist)
}
+ @Test
+ @Throws(Exception::class)
+ fun appendTest() {
+ val options =
+ arrayOf(
+ "--mark+=mark5${File.pathSeparator}mark7${File.pathSeparator}mark6",
+ "--includes+=include7${File.pathSeparator}include193${File.pathSeparator}include3",
+ "--source+=cliSource.java${File.pathSeparator}cliDir"
+ )
+ val config = Configuration.initConfig(correctFile, *options)
+ val serverConfig = config.buildServerConfiguration()
+ val translationConfig = config.buildTranslationConfiguration()
+ val configFileBasePath = correctFile.absoluteFile.parent
+
+ val expectedConfigMark =
+ listOf("mark1", "mark4", "mark3", "mark2")
+ .map { s -> File(configFileBasePath, s).absolutePath }
+ .toTypedArray()
+ assertContentEquals(
+ arrayOf(*expectedConfigMark, "mark5", "mark7", "mark6")
+ .map { s -> File(s).absolutePath }
+ .toTypedArray(),
+ serverConfig.markModelFiles,
+ "Option specified in config file should be appended to CLI option"
+ )
+
+ val expectedConfigIncludes =
+ listOf("include1", "include2")
+ .map { s -> File(configFileBasePath, s).absolutePath }
+ .toTypedArray()
+ assertContentEquals(
+ arrayOf(*expectedConfigIncludes, "include7", "include193", "include3")
+ .map { s -> File(s).absolutePath }
+ .toTypedArray(),
+ translationConfig.includePaths,
+ "Option specified in config file should be appended to CLI option"
+ )
+
+ assertContentEquals(
+ arrayOf(
+ File(configFileBasePath, "source.java"),
+ File("cliSource.java"),
+ File("cliDir")
+ ),
+ config.source,
+ "Option specified in config file should be appended to CLI option"
+ )
+ }
+
+ @Test
+ @Throws(java.lang.Exception::class)
+ fun additionalAppendTest() {
+ val config =
+ Configuration.initConfig(
+ additionalOptionFile,
+ "-c",
+ "--passes+=de.fraunhofer.aisec.cpg.passes.FilenameMapper${File.pathSeparator}" +
+ "de.fraunhofer.aisec.cpg.passes.CallResolver",
+ "--symbols+=&=and${File.pathSeparator}+=plus",
+ "--default-passes",
+ "--includes+=include9${File.pathSeparator}include193${File.pathSeparator}include13",
+ "--enabled-includes+=include9${File.pathSeparator}include52",
+ "--disabled-includes+=include13"
+ )
+ val translationConfiguration = config.buildTranslationConfiguration(File("test.java"))
+ val expectedTranslationConfig =
+ TranslationConfiguration.builder()
+ .defaultPasses()
+ .registerPass(EdgeCachePass())
+ .registerPass(UnreachableEOGPass())
+ .registerPass(FilenameMapper())
+ .registerPass(CallResolver())
+ .build()
+
+ val configFileBasePath = additionalOptionFile.absoluteFile.parent
+
+ assertEquals(
+ expectedTranslationConfig.registeredPasses.size,
+ translationConfiguration.registeredPasses.size,
+ "Expected size ${expectedTranslationConfig.registeredPasses.size} but was ${translationConfiguration.registeredPasses.size}"
+ )
+ val passesNames =
+ translationConfiguration.registeredPasses.map { s -> s.javaClass.name }.toTypedArray()
+ val expectedPassesNames =
+ expectedTranslationConfig.registeredPasses.map { s -> s.javaClass.name }.toTypedArray()
+ assertContentEquals(expectedPassesNames, passesNames)
+
+ assertEquals(
+ 5,
+ translationConfiguration.symbols.size,
+ "Expected size 5 but was ${translationConfiguration.symbols.size}"
+ )
+ assertTrue(
+ translationConfiguration.symbols.containsKey("#"),
+ "Did not contain \'#\' as a key"
+ )
+ assertEquals("hash", translationConfiguration.symbols["#"])
+ assertTrue(
+ translationConfiguration.symbols.containsKey("$"),
+ "Did not contain \'$\' as a key"
+ )
+ assertEquals("dollar", translationConfiguration.symbols["$"])
+ assertTrue(
+ translationConfiguration.symbols.containsKey("*"),
+ "Did not contain \'*\' as a key"
+ )
+ assertEquals("star", translationConfiguration.symbols["*"])
+ assertTrue(
+ translationConfiguration.symbols.containsKey("&"),
+ "Did not contain \'&\' as a key"
+ )
+ assertEquals("and", translationConfiguration.symbols["&"])
+ assertTrue(
+ translationConfiguration.symbols.containsKey("+"),
+ "Did not contain \'=\' as a key"
+ )
+ assertEquals("plus", translationConfiguration.symbols["+"])
+
+ val expectedIncludes =
+ arrayOf("include1", "include7", "include3", "include5")
+ .map { s -> File(configFileBasePath, s).absolutePath }
+ .toTypedArray()
+ assertContentEquals(
+ arrayOf(*expectedIncludes, "include9", "include193", "include13")
+ .map { s -> File(s).absolutePath }
+ .toTypedArray(),
+ translationConfiguration.includePaths
+ )
+
+ val expectedEnabledIncludes =
+ arrayOf("include3", "include5", "include1")
+ .map { s -> File(configFileBasePath, s).absolutePath }
+ .toTypedArray()
+ assertContentEquals(
+ arrayOf(*expectedEnabledIncludes, "include9", "include52").map { s ->
+ File(s).absolutePath
+ },
+ translationConfiguration.includeWhitelist
+ )
+
+ val expectedDisabledIncludes =
+ arrayOf("include7", "include3")
+ .map { s -> File(configFileBasePath, s).absolutePath }
+ .toTypedArray()
+ assertContentEquals(
+ arrayOf(*expectedDisabledIncludes, "include13").map { s -> File(s).absolutePath },
+ translationConfiguration.includeBlacklist
+ )
+ }
+
+ @Test
+ fun disabledMarkAppendTest() {
+ val config =
+ Configuration.initConfig(
+ disabledMarkFile,
+ "--disabled-mark-rules+=package.mark0${File.pathSeparator}package2.*"
+ )
+ val serverConfiguration = config.buildServerConfiguration()
+ val expectedMap =
+ mapOf(
+ Pair("package", DisabledMarkRulesValue(false, mutableSetOf("mark1", "mark0"))),
+ Pair("", DisabledMarkRulesValue(false, mutableSetOf("mark2"))),
+ Pair("package0", DisabledMarkRulesValue(false, mutableSetOf("mark123"))),
+ Pair("package2", DisabledMarkRulesValue(true))
+ )
+
+ for (expectedKey in expectedMap.keys) {
+ assertContains(
+ serverConfiguration.packageToDisabledMarkRules,
+ expectedKey,
+ "\"$expectedKey\" was not a key in map"
+ )
+
+ val expectedValue = expectedMap[expectedKey]
+ val actualValue = serverConfiguration.packageToDisabledMarkRules[expectedKey]
+ if (expectedValue != null && actualValue != null) {
+ assertEquals(
+ expectedValue.isDisablePackage,
+ actualValue.isDisablePackage,
+ "Boolean to disable entire package was not equal"
+ )
+ for (rule in expectedValue.disabledMarkRuleNames) {
+ assertTrue(
+ actualValue.disabledMarkRuleNames.contains(rule),
+ "Rule $rule was not in set"
+ )
+ }
+ }
+ }
+ }
+
+ @Test
+ fun disabledSourceAppendTest() {
+ val options =
+ arrayOf(
+ "--source+=cliDir1${File.pathSeparator}cliDir2/cliSource.java${File.pathSeparator}cliSource.java",
+ "--disabled-sources+=cliDir2${File.pathSeparator}cliSource.java"
+ )
+ val config = Configuration.initConfig(sourceDisablingFile, *options)
+ val configFileBasePath = sourceDisablingFile.absoluteFile.parent
+
+ val expectedConfigSource =
+ arrayOf(
+ "../real-examples/botan/blockciphers/Antidote1911.Arsenic",
+ "../real-examples/botan/blockciphers/obraunsdorf.playbook-creator",
+ "../real-examples/botan/MARK",
+ "../real-examples/botan/MARK",
+ "../real-examples/botan/streamciphers",
+ "../directory-structure"
+ )
+ .map { s -> File(configFileBasePath, s).absolutePath }
+ .toTypedArray()
+ assertContentEquals(
+ arrayOf(*expectedConfigSource, "cliDir1", "cliDir2/cliSource.java", "cliSource.java")
+ .map { s -> File(s).absolutePath },
+ config.source.map { f -> f.absolutePath }
+ )
+
+ val expectedConfigDisabledSource =
+ arrayOf(
+ "../real-examples/botan/blockciphers",
+ "../config-files/additional_options.yml",
+ "../real-examples/botan/MARK",
+ "../directory-structure/dir2/dir2dir1/dir2dir1file1.java"
+ )
+ .map { s -> File(configFileBasePath, s).absolutePath }
+ .toTypedArray()
+ assertContentEquals(
+ arrayOf(*expectedConfigDisabledSource, "cliDir2", "cliSource.java").map { s ->
+ File(s).absolutePath
+ },
+ config.disabledSource.map { f -> f.absolutePath }
+ )
+ }
+
companion object {
private lateinit var correctFile: File
private lateinit var incorrectFile: File
private lateinit var additionalOptionFile: File
+ private lateinit var disabledMarkFile: File
+ private lateinit var sourceDisablingFile: File
@BeforeAll
@JvmStatic
@@ -175,6 +415,20 @@ class ConfigCLILoadTest {
assertNotNull(additionalOptionResource)
additionalOptionFile = File(additionalOptionResource.file)
assertNotNull(additionalOptionFile)
+
+ val disabledMarkResource =
+ ConfigLoadTest::class.java.classLoader.getResource("config-files/disabled_mark.yml")
+ assertNotNull(disabledMarkResource)
+ disabledMarkFile = File(disabledMarkResource.file)
+ assertNotNull(disabledMarkFile)
+
+ val sourceDisablingResource =
+ ConfigLoadTest::class.java.classLoader.getResource(
+ "config-files/source_disabling.yml"
+ )
+ assertNotNull(sourceDisablingResource)
+ sourceDisablingFile = File(sourceDisablingResource.file)
+ assertNotNull(sourceDisablingFile)
}
}
}

diff:
+codyze:
+ disabled-mark-rules:
+ - package.mark1
+ - .mark2
+ - package0.mark123
\ No newline at end of file

original_message:
Make collection type options appendable (#463)

This PR add a new option name (e.g. --option-name+) for all list and map type options so that the data from the command line can be appended to the config file data instead of overwriting it.

Co-authored-by: Florian Wendland <florian.wendland@aisec.fraunhofer.de>