message: Adding new task repositories

diff:
+package org.apache.airavata.registry.core.repositories.expcatalog;
+
+import org.apache.airavata.model.commons.ErrorModel;
+import org.apache.airavata.model.task.TaskModel;
+import org.apache.airavata.registry.core.entities.expcatalog.TaskErrorEntity;
+import org.apache.airavata.registry.core.entities.expcatalog.TaskErrorPK;
+import org.apache.airavata.registry.core.utils.ExpCatalogUtils;
+import org.apache.airavata.registry.core.utils.ObjectMapperSingleton;
+import org.apache.airavata.registry.cpi.RegistryException;
+import org.dozer.Mapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+public class TaskErrorRepository extends ExpCatAbstractRepository<ErrorModel, TaskErrorEntity, TaskErrorPK> {
+ private final static Logger logger = LoggerFactory.getLogger(TaskErrorRepository.class);
+
+ public TaskErrorRepository() { super(ErrorModel.class, TaskErrorEntity.class); }
+
+ protected String saveTaskError(ErrorModel error, String taskId) throws RegistryException {
+ Mapper mapper = ObjectMapperSingleton.getInstance();
+ TaskErrorEntity taskErrorEntity = mapper.map(error, TaskErrorEntity.class);
+
+ if (taskErrorEntity.getTaskId() == null) {
+ logger.debug("Setting the TaskErrorEntity's TaskId");
+ taskErrorEntity.setTaskId(taskId);
+ }
+
+ execute(entityManager -> entityManager.merge(taskErrorEntity));
+ return taskErrorEntity.getErrorId();
+ }
+
+ public String addTaskError(ErrorModel taskError, String taskId) throws RegistryException {
+
+ if (taskError.getErrorId() == null) {
+ logger.debug("Setting the TaskError's ErrorId");
+ taskError.setErrorId(ExpCatalogUtils.getID("ERROR"));
+ }
+
+ return saveTaskError(taskError, taskId);
+ }
+
+ public String updateTaskError(ErrorModel updatedTaskError, String taskId) throws RegistryException {
+ return saveTaskError(updatedTaskError, taskId);
+ }
+
+ public List<ErrorModel> getTaskError(String taskId) throws RegistryException {
+ TaskRepository taskRepository = new TaskRepository();
+ TaskModel taskModel = taskRepository.getTask(taskId);
+ return taskModel.getTaskErrors();
+ }
+
+}

diff:
*/
package org.apache.airavata.registry.core.repositories.expcatalog;
-import org.apache.airavata.model.commons.ErrorModel;
import org.apache.airavata.model.commons.airavata_commonsConstants;
-import org.apache.airavata.model.status.TaskState;
-import org.apache.airavata.model.status.TaskStatus;
import org.apache.airavata.model.task.TaskModel;
import org.apache.airavata.registry.core.entities.expcatalog.TaskEntity;
import org.apache.airavata.registry.core.utils.DBConstants;
@@ -97,101 +94,6 @@ public class TaskRepository extends ExpCatAbstractRepository<TaskModel, TaskEnti
return taskRepository.get(taskId);
}
- public String addTaskStatus(TaskStatus taskStatus, String taskId) throws RegistryException {
- TaskModel taskModel = getTask(taskId);
- List<TaskStatus> taskStatusList = taskModel.getTaskStatuses();
-
- if (taskStatusList.size() == 0 || !taskStatusList.contains(taskStatus)) {
-
- if (taskStatus.getStatusId() == null) {
- logger.debug("Set TaskStatus's StatusId");
- taskStatus.setStatusId(ExpCatalogUtils.getID("STATUS"));
- }
-
- logger.debug("Adding the TaskStatus to the list");
- taskStatusList.add(taskStatus);
- }
-
- taskModel.setTaskStatuses(taskStatusList);
- updateTask(taskModel, taskId);
- return taskStatus.getStatusId();
- }
-
- public String updateTaskStatus(TaskStatus updatedTaskStatus, String taskId) throws RegistryException {
- TaskModel taskModel = getTask(taskId);
- List<TaskStatus> taskStatusList = taskModel.getTaskStatuses();
-
- for (TaskStatus retrievedTaskStatus : taskStatusList) {
-
- if (retrievedTaskStatus.getStatusId().equals(updatedTaskStatus.getStatusId())) {
- logger.debug("Updating the TaskStatus");
- taskStatusList.remove(retrievedTaskStatus);
- taskStatusList.add(updatedTaskStatus);
- }
-
- }
-
- taskModel.setTaskStatuses(taskStatusList);
- updateTask(taskModel, taskId);
- return updatedTaskStatus.getStatusId();
- }
-
- public TaskStatus getTaskStatus(String taskId) throws RegistryException {
- TaskModel taskModel = getTask(taskId);
- List<TaskStatus> taskStatusList = taskModel.getTaskStatuses();
-
- if(taskStatusList.size() == 0) {
- logger.debug("TaskStatus list is empty");
- return null;
- }
-
- else {
- TaskStatus latestTaskStatus = taskStatusList.get(0);
-
- for(int i = 1; i < taskStatusList.size(); i++) {
- Timestamp timeOfStateChange = new Timestamp(taskStatusList.get(i).getTimeOfStateChange());
-
- if (timeOfStateChange.after(new Timestamp(latestTaskStatus.getTimeOfStateChange()))
- || (timeOfStateChange.equals(latestTaskStatus.getTimeOfStateChange()) && taskStatusList.get(i).getState().equals(TaskState.COMPLETED.toString()))
- || (timeOfStateChange.equals(latestTaskStatus.getTimeOfStateChange()) && taskStatusList.get(i).getState().equals(TaskState.FAILED.toString()))
- || (timeOfStateChange.equals(latestTaskStatus.getTimeOfStateChange()) && taskStatusList.get(i).getState().equals(TaskState.CANCELED.toString()))) {
- latestTaskStatus = taskStatusList.get(i);
- }
-
- }
-
- return latestTaskStatus;
- }
- }
-
- public String addTaskError(ErrorModel taskError, String taskId) throws RegistryException {
- TaskModel taskModel = getTask(taskId);
- List<ErrorModel> errorModelList = taskModel.getTaskErrors();
-
- if (errorModelList == null) {
- logger.debug("Adding the first TaskError to the list");
- taskModel.setTaskErrors(Arrays.asList(taskError));
- }
-
- else if (!errorModelList.contains(taskError)) {
- logger.debug("Adding the TaskError to the list");
- errorModelList.add(taskError);
- taskModel.setTaskErrors(errorModelList);
- }
-
- updateTask(taskModel, taskId);
- return taskId;
- }
-
- public String updateTaskError(ErrorModel taskError, String taskId) throws RegistryException {
- return addTaskError(taskError, taskId);
- }
-
- public List<ErrorModel> getTaskError(String taskId) throws RegistryException {
- TaskModel taskModel = getTask(taskId);
- return taskModel.getTaskErrors();
- }
-
public List<TaskModel> getTaskList(String fieldName, Object value) throws RegistryException {
TaskRepository taskRepository = new TaskRepository();
List<TaskModel> taskModelList;

diff:
+package org.apache.airavata.registry.core.repositories.expcatalog;
+
+import org.apache.airavata.model.status.TaskState;
+import org.apache.airavata.model.status.TaskStatus;
+import org.apache.airavata.model.task.TaskModel;
+import org.apache.airavata.registry.core.entities.expcatalog.TaskStatusEntity;
+import org.apache.airavata.registry.core.entities.expcatalog.TaskStatusPK;
+import org.apache.airavata.registry.core.utils.ExpCatalogUtils;
+import org.apache.airavata.registry.core.utils.ObjectMapperSingleton;
+import org.apache.airavata.registry.cpi.RegistryException;
+import org.dozer.Mapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Timestamp;
+import java.util.List;
+
+public class TaskStatusRepository extends ExpCatAbstractRepository<TaskStatus, TaskStatusEntity, TaskStatusPK> {
+ private final static Logger logger = LoggerFactory.getLogger(TaskStatusRepository.class);
+
+ public TaskStatusRepository() { super(TaskStatus.class, TaskStatusEntity.class); }
+
+ protected String saveTaskStatus(TaskStatus taskStatus, String taskId) throws RegistryException {
+ Mapper mapper = ObjectMapperSingleton.getInstance();
+ TaskStatusEntity taskStatusEntity = mapper.map(taskStatus, TaskStatusEntity.class);
+
+ if (taskStatusEntity.getTaskId() == null) {
+ logger.debug("Setting the TaskStatusEntity's TaskId");
+ taskStatusEntity.setTaskId(taskId);
+ }
+
+ execute(entityManager -> entityManager.merge(taskStatusEntity));
+ return taskStatusEntity.getStatusId();
+ }
+
+ public String addTaskStatus(TaskStatus taskStatus, String taskId) throws RegistryException {
+
+ if (taskStatus.getStatusId() == null) {
+ logger.debug("Setting the TaskStatus's StatusId");
+ taskStatus.setStatusId(ExpCatalogUtils.getID("STATUS"));
+ }
+
+ return saveTaskStatus(taskStatus, taskId);
+ }
+
+ public String updateTaskStatus(TaskStatus updatedTaskStatus, String taskId) throws RegistryException {
+ return saveTaskStatus(updatedTaskStatus, taskId);
+ }
+
+ public TaskStatus getTaskStatus(String taskId) throws RegistryException {
+ TaskRepository taskRepository = new TaskRepository();
+ TaskModel taskModel = taskRepository.getTask(taskId);
+ List<TaskStatus> taskStatusList = taskModel.getTaskStatuses();
+
+ if(taskStatusList.size() == 0) {
+ logger.debug("TaskStatus list is empty");
+ return null;
+ }
+
+ else {
+ TaskStatus latestTaskStatus = taskStatusList.get(0);
+
+ for(int i = 1; i < taskStatusList.size(); i++) {
+ Timestamp timeOfStateChange = new Timestamp(taskStatusList.get(i).getTimeOfStateChange());
+
+ if (timeOfStateChange.after(new Timestamp(latestTaskStatus.getTimeOfStateChange()))
+ || (timeOfStateChange.equals(latestTaskStatus.getTimeOfStateChange()) && taskStatusList.get(i).getState().equals(TaskState.COMPLETED.toString()))
+ || (timeOfStateChange.equals(latestTaskStatus.getTimeOfStateChange()) && taskStatusList.get(i).getState().equals(TaskState.FAILED.toString()))
+ || (timeOfStateChange.equals(latestTaskStatus.getTimeOfStateChange()) && taskStatusList.get(i).getState().equals(TaskState.CANCELED.toString()))) {
+ latestTaskStatus = taskStatusList.get(i);
+ }
+
+ }
+
+ return latestTaskStatus;
+ }
+ }
+
+}

diff:
+package org.apache.airavata.registry.core.repositories.expcatalog;
+
+import org.apache.airavata.model.commons.ErrorModel;
+import org.apache.airavata.model.experiment.ExperimentModel;
+import org.apache.airavata.model.experiment.ExperimentType;
+import org.apache.airavata.model.process.ProcessModel;
+import org.apache.airavata.model.task.TaskModel;
+import org.apache.airavata.model.task.TaskTypes;
+import org.apache.airavata.model.workspace.Gateway;
+import org.apache.airavata.model.workspace.Project;
+import org.apache.airavata.registry.core.repositories.expcatalog.util.Initialize;
+import org.apache.airavata.registry.cpi.RegistryException;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TaskErrorRepositoryTest {
+
+ private static Initialize initialize;
+ GatewayRepository gatewayRepository;
+ ProjectRepository projectRepository;
+ ExperimentRepository experimentRepository;
+ ProcessRepository processRepository;
+ TaskRepository taskRepository;
+ TaskErrorRepository taskErrorRepository;
+ private static final Logger logger = LoggerFactory.getLogger(TaskErrorRepositoryTest.class);
+
+ @Before
+ public void setUp() {
+ try {
+ initialize = new Initialize("expcatalog-derby.sql");
+ initialize.initializeDB();
+ gatewayRepository = new GatewayRepository();
+ projectRepository = new ProjectRepository();
+ experimentRepository = new ExperimentRepository();
+ processRepository = new ProcessRepository();
+ taskRepository = new TaskRepository();
+ taskErrorRepository = new TaskErrorRepository();
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ System.out.println("********** TEAR DOWN ************");
+ initialize.stopDerbyServer();
+ }
+
+ @Test
+ public void TaskErrorRepositoryTest() throws RegistryException {
+ Gateway gateway = new Gateway();
+ gateway.setGatewayId("gateway");
+ gateway.setDomain("SEAGRID");
+ gateway.setEmailAddress("abc@d.com");
+ String gatewayId = gatewayRepository.addGateway(gateway);
+
+ Project project = new Project();
+ project.setName("projectName");
+ project.setOwner("user");
+ project.setGatewayId(gatewayId);
+
+ String projectId = projectRepository.addProject(project, gatewayId);
+
+ ExperimentModel experimentModel = new ExperimentModel();
+ experimentModel.setProjectId(projectId);
+ experimentModel.setGatewayId(gatewayId);
+ experimentModel.setExperimentType(ExperimentType.SINGLE_APPLICATION);
+ experimentModel.setUserName("user");
+ experimentModel.setExperimentName("name");
+
+ String experimentId = experimentRepository.addExperiment(experimentModel);
+
+ ProcessModel processModel = new ProcessModel(null, experimentId);
+ String processId = processRepository.addProcess(processModel, experimentId);
+
+ TaskModel taskModel = new TaskModel();
+ taskModel.setTaskType(TaskTypes.JOB_SUBMISSION);
+ taskModel.setParentProcessId(processId);
+
+ String taskId = taskRepository.addTask(taskModel, processId);
+ assertTrue(taskId != null);
+
+ ErrorModel errorModel = new ErrorModel();
+ errorModel.setErrorId("error");
+
+ String taskErrorId = taskErrorRepository.addTaskError(errorModel, taskId);
+ assertTrue(taskErrorId != null);
+ assertTrue(taskRepository.getTask(taskId).getTaskErrors().size() == 1);
+
+ errorModel.setActualErrorMessage("message");
+ taskErrorRepository.updateTaskError(errorModel, taskId);
+
+ List<ErrorModel> retrievedErrorList = taskErrorRepository.getTaskError(taskId);
+ assertTrue(retrievedErrorList.size() == 1);
+ assertEquals("message", retrievedErrorList.get(0).getActualErrorMessage());
+
+ experimentRepository.removeExperiment(experimentId);
+ processRepository.removeProcess(processId);
+ taskRepository.removeTask(taskId);
+ gatewayRepository.removeGateway(gatewayId);
+ projectRepository.removeProject(projectId);
+ }
+
+}

diff:
*/
package org.apache.airavata.registry.core.repositories.expcatalog;
-import org.apache.airavata.model.commons.ErrorModel;
import org.apache.airavata.model.experiment.ExperimentModel;
import org.apache.airavata.model.experiment.ExperimentType;
import org.apache.airavata.model.process.ProcessModel;
-import org.apache.airavata.model.status.TaskState;
-import org.apache.airavata.model.status.TaskStatus;
import org.apache.airavata.model.task.TaskModel;
import org.apache.airavata.model.task.TaskTypes;
import org.apache.airavata.model.workspace.Gateway;
@@ -114,28 +111,6 @@ public class TaskRepositoryTest {
taskRepository.updateTask(taskModel, taskId);
assertEquals(TaskTypes.MONITORING, taskRepository.getTask(taskId).getTaskType());
- TaskStatus taskStatus = new TaskStatus(TaskState.EXECUTING);
- String taskStatusId = taskRepository.addTaskStatus(taskStatus, taskId);
- assertTrue(taskStatusId != null);
-
- taskStatus.setState(TaskState.CREATED);
- taskRepository.updateTaskStatus(taskStatus, taskId);
-
- TaskStatus retrievedTaskStatus = taskRepository.getTaskStatus(taskId);
- assertEquals(TaskState.CREATED, retrievedTaskStatus.getState());
-
- ErrorModel errorModel = new ErrorModel();
- errorModel.setErrorId("error");
-
- assertEquals(taskId, taskRepository.addTaskError(errorModel, taskId));
-
- errorModel.setActualErrorMessage("message");
- taskRepository.updateTaskError(errorModel, taskId);
-
- List<ErrorModel> retrievedErrorList = taskRepository.getTaskError(taskId);
- assertTrue(retrievedErrorList.size() == 1);
- assertEquals("message", retrievedErrorList.get(0).getActualErrorMessage());
-
List<String> taskIdList = taskRepository.getTaskIds(DBConstants.Task.PARENT_PROCESS_ID, processId);
assertTrue(taskIdList.size() == 1);
assertTrue(taskIdList.get(0).equals(taskId));

diff:
+package org.apache.airavata.registry.core.repositories.expcatalog;
+
+import org.apache.airavata.model.experiment.ExperimentModel;
+import org.apache.airavata.model.experiment.ExperimentType;
+import org.apache.airavata.model.process.ProcessModel;
+import org.apache.airavata.model.status.TaskState;
+import org.apache.airavata.model.status.TaskStatus;
+import org.apache.airavata.model.task.TaskModel;
+import org.apache.airavata.model.task.TaskTypes;
+import org.apache.airavata.model.workspace.Gateway;
+import org.apache.airavata.model.workspace.Project;
+import org.apache.airavata.registry.core.repositories.expcatalog.util.Initialize;
+import org.apache.airavata.registry.cpi.RegistryException;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TaskStatusRepositoryTest {
+
+ private static Initialize initialize;
+ GatewayRepository gatewayRepository;
+ ProjectRepository projectRepository;
+ ExperimentRepository experimentRepository;
+ ProcessRepository processRepository;
+ TaskRepository taskRepository;
+ TaskStatusRepository taskStatusRepository;
+ private static final Logger logger = LoggerFactory.getLogger(TaskStatusRepositoryTest.class);
+
+ @Before
+ public void setUp() {
+ try {
+ initialize = new Initialize("expcatalog-derby.sql");
+ initialize.initializeDB();
+ gatewayRepository = new GatewayRepository();
+ projectRepository = new ProjectRepository();
+ experimentRepository = new ExperimentRepository();
+ processRepository = new ProcessRepository();
+ taskRepository = new TaskRepository();
+ taskStatusRepository = new TaskStatusRepository();
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ System.out.println("********** TEAR DOWN ************");
+ initialize.stopDerbyServer();
+ }
+
+ @Test
+ public void TaskStatusRepositoryTest() throws RegistryException {
+ Gateway gateway = new Gateway();
+ gateway.setGatewayId("gateway");
+ gateway.setDomain("SEAGRID");
+ gateway.setEmailAddress("abc@d.com");
+ String gatewayId = gatewayRepository.addGateway(gateway);
+
+ Project project = new Project();
+ project.setName("projectName");
+ project.setOwner("user");
+ project.setGatewayId(gatewayId);
+
+ String projectId = projectRepository.addProject(project, gatewayId);
+
+ ExperimentModel experimentModel = new ExperimentModel();
+ experimentModel.setProjectId(projectId);
+ experimentModel.setGatewayId(gatewayId);
+ experimentModel.setExperimentType(ExperimentType.SINGLE_APPLICATION);
+ experimentModel.setUserName("user");
+ experimentModel.setExperimentName("name");
+
+ String experimentId = experimentRepository.addExperiment(experimentModel);
+
+ ProcessModel processModel = new ProcessModel(null, experimentId);
+ String processId = processRepository.addProcess(processModel, experimentId);
+
+ TaskModel taskModel = new TaskModel();
+ taskModel.setTaskType(TaskTypes.JOB_SUBMISSION);
+ taskModel.setParentProcessId(processId);
+
+ String taskId = taskRepository.addTask(taskModel, processId);
+ assertTrue(taskId != null);
+
+ TaskStatus taskStatus = new TaskStatus(TaskState.EXECUTING);
+ String taskStatusId = taskStatusRepository.addTaskStatus(taskStatus, taskId);
+ assertTrue(taskStatusId != null);
+ assertTrue(taskRepository.getTask(taskId).getTaskStatuses().size() == 1);
+
+ taskStatus.setState(TaskState.CREATED);
+ taskStatusRepository.updateTaskStatus(taskStatus, taskId);
+
+ TaskStatus retrievedTaskStatus = taskStatusRepository.getTaskStatus(taskId);
+ assertEquals(TaskState.CREATED, retrievedTaskStatus.getState());
+
+ experimentRepository.removeExperiment(experimentId);
+ processRepository.removeProcess(processId);
+ taskRepository.removeTask(taskId);
+ gatewayRepository.removeGateway(gatewayId);
+ projectRepository.removeProject(projectId);
+ }
+
+}

diff:
@@ -152,6 +152,8 @@ public class RegistryServerHandler implements RegistryService.Iface {
private ProcessStatusRepository processStatusRepository = new ProcessStatusRepository();
private ProcessErrorRepository processErrorRepository = new ProcessErrorRepository();
private TaskRepository taskRepository = new TaskRepository();
+ private TaskStatusRepository taskStatusRepository = new TaskStatusRepository();
+ private TaskErrorRepository taskErrorRepository = new TaskErrorRepository();
private JobRepository jobRepository = new JobRepository();
private QueueStatusRepository queueStatusRepository = new QueueStatusRepository();
private DataProductRepository dataProductRepository = new DataProductRepository();
@@ -819,7 +821,7 @@ public class RegistryServerHandler implements RegistryService.Iface {
experimentErrorRepository.addExperimentError(errorModel, id);
}
else if (ExpCatChildDataType.TASK_ERROR.equals(ExpCatChildDataType.valueOf(errorType))) {
- taskRepository.addTaskError(errorModel, id);
+ taskErrorRepository.addTaskError(errorModel, id);
}
else if (ExpCatChildDataType.PROCESS_ERROR.equals(ExpCatChildDataType.valueOf(errorType))) {
processErrorRepository.addProcessError(errorModel, id);
@@ -836,7 +838,7 @@ public class RegistryServerHandler implements RegistryService.Iface {
@Override
public void addTaskStatus(TaskStatus taskStatus, String taskId) throws RegistryServiceException, TException {
try {
- taskRepository.addTaskStatus(taskStatus, taskId);
+ taskStatusRepository.addTaskStatus(taskStatus, taskId);
} catch (Exception e) {
logger.error(taskId, "Error while adding task status", e);
AiravataSystemException exception = new AiravataSystemException();

original_message: Adding new task repositories


message: Adding new job repository

change_type: MODIFY

diff:
@@ -22,11 +22,9 @@ package org.apache.airavata.registry.core.repositories.expcatalog;
import org.apache.airavata.model.commons.airavata_commonsConstants;
import org.apache.airavata.model.job.JobModel;
-import org.apache.airavata.model.status.JobStatus;
import org.apache.airavata.registry.core.entities.expcatalog.JobEntity;
import org.apache.airavata.registry.core.entities.expcatalog.JobPK;
import org.apache.airavata.registry.core.utils.DBConstants;
-import org.apache.airavata.registry.core.utils.ExpCatalogUtils;
import org.apache.airavata.registry.core.utils.ObjectMapperSingleton;
import org.apache.airavata.registry.core.utils.QueryConstants;
import org.apache.airavata.registry.cpi.RegistryException;
@@ -93,52 +91,6 @@ public class JobRepository extends ExpCatAbstractRepository<JobModel, JobEntity,
return get(jobPK);
}
- public String addJobStatus(JobStatus jobStatus, JobPK jobPK) throws RegistryException {
- JobModel jobModel = getJob(jobPK);
- List<JobStatus> jobStatusList = jobModel.getJobStatuses();
-
- if (jobStatusList.size() == 0 || !jobStatusList.contains(jobStatus)) {
-
- if (jobStatus.getStatusId() == null) {
- logger.debug("Set JobStatus's StatusId");
- jobStatus.setStatusId(ExpCatalogUtils.getID("STATUS"));
- }
-
- logger.debug("Adding the JobStatus to the list");
- jobStatusList.add(jobStatus);
-
- }
-
- jobModel.setJobStatuses(jobStatusList);
- updateJob(jobModel, jobPK);
- return jobStatus.getStatusId();
- }
-
- public String updateJobStatus(JobStatus updatedJobStatus, JobPK jobPK) throws RegistryException {
- JobModel jobModel = getJob(jobPK);
- List<JobStatus> jobStatusList = jobModel.getJobStatuses();
-
- for (JobStatus retrievedJobStatus : jobStatusList) {
-
- if (retrievedJobStatus.getStatusId().equals(updatedJobStatus.getStatusId())) {
- logger.debug("Updating the JobStatus");
- jobStatusList.remove(retrievedJobStatus);
- jobStatusList.add(updatedJobStatus);
- }
-
- }
-
- jobModel.setJobStatuses(jobStatusList);
- updateJob(jobModel, jobPK);
- return updatedJobStatus.getStatusId();
-
- }
-
- public List<JobStatus> getJobStatus(JobPK jobPK) throws RegistryException {
- JobModel jobModel = getJob(jobPK);
- return jobModel.getJobStatuses();
- }
-
public List<JobModel> getJobList(String fieldName, Object value) throws RegistryException {
JobRepository jobRepository = new JobRepository();
List<JobModel> jobModelList;

change_type: ADD

diff:
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.registry.core.repositories.expcatalog;
+
+import org.apache.airavata.model.job.JobModel;
+import org.apache.airavata.model.status.JobState;
+import org.apache.airavata.model.status.JobStatus;
+import org.apache.airavata.registry.core.entities.expcatalog.JobPK;
+import org.apache.airavata.registry.core.entities.expcatalog.JobStatusEntity;
+import org.apache.airavata.registry.core.entities.expcatalog.JobStatusPK;
+import org.apache.airavata.registry.core.utils.ExpCatalogUtils;
+import org.apache.airavata.registry.core.utils.ObjectMapperSingleton;
+import org.apache.airavata.registry.cpi.RegistryException;
+import org.dozer.Mapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Timestamp;
+import java.util.List;
+
+public class JobStatusRepository extends ExpCatAbstractRepository<JobStatus, JobStatusEntity, JobStatusPK> {
+ private final static Logger logger = LoggerFactory.getLogger(JobStatusRepository.class);
+
+ public JobStatusRepository() { super(JobStatus.class, JobStatusEntity.class); }
+
+ protected String saveJobStatus(JobStatus jobStatus, JobPK jobPK) throws RegistryException {
+ Mapper mapper = ObjectMapperSingleton.getInstance();
+ JobStatusEntity jobStatusEntity = mapper.map(jobStatus, JobStatusEntity.class);
+
+ if (jobStatusEntity.getJobId() == null) {
+ logger.debug("Setting the JobStatusEntity's JobId");
+ jobStatusEntity.setJobId(jobPK.getJobId());
+ }
+
+ if (jobStatusEntity.getTaskId() == null) {
+ logger.debug("Setting the JobStatusEntity's TaskId");
+ jobStatusEntity.setTaskId(jobPK.getTaskId());
+ }
+
+ execute(entityManager -> entityManager.merge(jobStatusEntity));
+ return jobStatusEntity.getStatusId();
+ }
+
+ public String addJobStatus(JobStatus jobStatus, JobPK jobPK) throws RegistryException {
+
+ if (jobStatus.getStatusId() == null) {
+ logger.debug("Setting the JobStatusEntity's StatusId");
+ jobStatus.setStatusId(ExpCatalogUtils.getID("STATUS"));
+ }
+
+ return saveJobStatus(jobStatus, jobPK);
+
+ }
+
+ public String updateJobStatus(JobStatus updatedJobStatus, JobPK jobPK) throws RegistryException {
+ return saveJobStatus(updatedJobStatus, jobPK);
+ }
+
+ public JobStatus getJobStatus(JobPK jobPK) throws RegistryException {
+ JobRepository jobRepository = new JobRepository();
+ JobModel jobModel = jobRepository.getJob(jobPK);
+ List<JobStatus> jobStatusList = jobModel.getJobStatuses();
+
+ if(jobStatusList.size() == 0) {
+ logger.debug("JobStatus list is empty");
+ return null;
+ }
+
+ else {
+ JobStatus latestJobStatus = jobStatusList.get(0);
+
+ for(int i = 1; i < jobStatusList.size(); i++) {
+ Timestamp timeOfStateChange = new Timestamp(jobStatusList.get(i).getTimeOfStateChange());
+
+ if (timeOfStateChange.after(new Timestamp(latestJobStatus.getTimeOfStateChange()))
+ || (timeOfStateChange.equals(latestJobStatus.getTimeOfStateChange()) && jobStatusList.get(i).getJobState().equals(JobState.COMPLETE.toString()))
+ || (timeOfStateChange.equals(latestJobStatus.getTimeOfStateChange()) && jobStatusList.get(i).getJobState().equals(JobState.FAILED.toString()))
+ || (timeOfStateChange.equals(latestJobStatus.getTimeOfStateChange()) && jobStatusList.get(i).getJobState().equals(JobState.CANCELED.toString()))) {
+ latestJobStatus = jobStatusList.get(i);
+ }
+
+ }
+
+ return latestJobStatus;
+ }
+ }
+
+
+}

diff:
@@ -24,8 +24,6 @@ import org.apache.airavata.model.experiment.ExperimentModel;
import org.apache.airavata.model.experiment.ExperimentType;
import org.apache.airavata.model.job.JobModel;
import org.apache.airavata.model.process.ProcessModel;
-import org.apache.airavata.model.status.JobState;
-import org.apache.airavata.model.status.JobStatus;
import org.apache.airavata.model.task.TaskModel;
import org.apache.airavata.model.task.TaskTypes;
import org.apache.airavata.model.workspace.Gateway;
@@ -132,17 +130,6 @@ public class JobRepositoryTest {
jobRepository.updateJob(jobModel, jobPK);
assertEquals("jobName", jobRepository.getJob(jobPK).getJobName());
- JobStatus jobStatus = new JobStatus(JobState.QUEUED);
- String jobStatusId = jobRepository.addJobStatus(jobStatus, jobPK);
- assertTrue(jobStatusId != null);
-
- jobStatus.setJobState(JobState.ACTIVE);
- jobRepository.updateJobStatus(jobStatus, jobPK);
-
- List<JobStatus> retrievedJobStatusList = jobRepository.getJobStatus(jobPK);
- assertTrue(retrievedJobStatusList.size() == 1);
- assertEquals(JobState.ACTIVE, retrievedJobStatusList.get(0).getJobState());
-
List<String> jobIdList = jobRepository.getJobIds(DBConstants.Job.TASK_ID, taskId);
assertTrue(jobIdList.size() == 1);
assertTrue(jobIdList.get(0).equals(jobId));

diff:
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.registry.core.repositories.expcatalog;
+
+import org.apache.airavata.model.experiment.ExperimentModel;
+import org.apache.airavata.model.experiment.ExperimentType;
+import org.apache.airavata.model.job.JobModel;
+import org.apache.airavata.model.process.ProcessModel;
+import org.apache.airavata.model.status.JobState;
+import org.apache.airavata.model.status.JobStatus;
+import org.apache.airavata.model.task.TaskModel;
+import org.apache.airavata.model.task.TaskTypes;
+import org.apache.airavata.model.workspace.Gateway;
+import org.apache.airavata.model.workspace.Project;
+import org.apache.airavata.registry.core.entities.expcatalog.JobPK;
+import org.apache.airavata.registry.core.repositories.expcatalog.util.Initialize;
+import org.apache.airavata.registry.cpi.RegistryException;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class JobStatusRepositoryTest {
+
+ private static Initialize initialize;
+ GatewayRepository gatewayRepository;
+ ProjectRepository projectRepository;
+ ExperimentRepository experimentRepository;
+ ProcessRepository processRepository;
+ TaskRepository taskRepository;
+ JobRepository jobRepository;
+ JobStatusRepository jobStatusRepository;
+ private static final Logger logger = LoggerFactory.getLogger(JobStatusRepositoryTest.class);
+
+ @Before
+ public void setUp() {
+ try {
+ initialize = new Initialize("expcatalog-derby.sql");
+ initialize.initializeDB();
+ gatewayRepository = new GatewayRepository();
+ projectRepository = new ProjectRepository();
+ experimentRepository = new ExperimentRepository();
+ processRepository = new ProcessRepository();
+ taskRepository = new TaskRepository();
+ jobRepository = new JobRepository();
+ jobStatusRepository = new JobStatusRepository();
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ System.out.println("********** TEAR DOWN ************");
+ initialize.stopDerbyServer();
+ }
+
+ @Test
+ public void JobStatusRepositoryTest() throws RegistryException {
+ Gateway gateway = new Gateway();
+ gateway.setGatewayId("gateway");
+ gateway.setDomain("SEAGRID");
+ gateway.setEmailAddress("abc@d.com");
+ String gatewayId = gatewayRepository.addGateway(gateway);
+
+ Project project = new Project();
+ project.setName("projectName");
+ project.setOwner("user");
+ project.setGatewayId(gatewayId);
+
+ String projectId = projectRepository.addProject(project, gatewayId);
+
+ ExperimentModel experimentModel = new ExperimentModel();
+ experimentModel.setProjectId(projectId);
+ experimentModel.setGatewayId(gatewayId);
+ experimentModel.setExperimentType(ExperimentType.SINGLE_APPLICATION);
+ experimentModel.setUserName("user");
+ experimentModel.setExperimentName("name");
+
+ String experimentId = experimentRepository.addExperiment(experimentModel);
+
+ ProcessModel processModel = new ProcessModel(null, experimentId);
+ String processId = processRepository.addProcess(processModel, experimentId);
+
+ TaskModel taskModel = new TaskModel();
+ taskModel.setTaskType(TaskTypes.JOB_SUBMISSION);
+ taskModel.setParentProcessId(processId);
+
+ String taskId = taskRepository.addTask(taskModel, processId);
+ assertTrue(taskId != null);
+
+ taskModel.setTaskType(TaskTypes.MONITORING);
+ taskRepository.updateTask(taskModel, taskId);
+
+ JobPK jobPK = new JobPK();
+ jobPK.setJobId("job");
+ jobPK.setTaskId(taskId);
+
+ JobModel jobModel = new JobModel();
+ jobModel.setJobId(jobPK.getJobId());
+ jobModel.setTaskId(jobPK.getTaskId());
+ jobModel.setJobDescription("jobDescription");
+
+ String jobId = jobRepository.addJob(jobModel, processId);
+ assertTrue(jobId != null);
+
+ JobStatus jobStatus = new JobStatus(JobState.QUEUED);
+ String jobStatusId = jobStatusRepository.addJobStatus(jobStatus, jobPK);
+ assertTrue(jobStatusId != null);
+ assertTrue(jobRepository.getJob(jobPK).getJobStatuses().size() == 1);
+
+ jobStatus.setJobState(JobState.ACTIVE);
+ jobStatusRepository.updateJobStatus(jobStatus, jobPK);
+
+ JobStatus retrievedJobStatus = jobStatusRepository.getJobStatus(jobPK);
+ assertEquals(JobState.ACTIVE, retrievedJobStatus.getJobState());
+
+ experimentRepository.removeExperiment(experimentId);
+ processRepository.removeProcess(processId);
+ taskRepository.removeTask(taskId);
+ jobRepository.removeJob(jobPK);
+ gatewayRepository.removeGateway(gatewayId);
+ projectRepository.removeProject(projectId);
+ }
+
+}

diff:
@@ -112,10 +112,8 @@ import org.apache.airavata.registry.core.repositories.replicacatalog.DataReplica
import org.apache.airavata.registry.core.repositories.workflowcatalog.WorkflowRepository;
import org.apache.airavata.registry.core.utils.DBConstants;
import org.apache.airavata.registry.cpi.AppCatalogException;
-import org.apache.airavata.registry.cpi.CompositeIdentifier;
import org.apache.airavata.registry.cpi.ComputeResource;
import org.apache.airavata.registry.cpi.ExpCatChildDataType;
-import org.apache.airavata.registry.cpi.ExperimentCatalog;
import org.apache.airavata.registry.cpi.ExperimentCatalogException;
import org.apache.airavata.registry.cpi.RegistryException;
import org.apache.airavata.registry.cpi.ResultOrderType;
@@ -155,6 +153,7 @@ public class RegistryServerHandler implements RegistryService.Iface {
private TaskStatusRepository taskStatusRepository = new TaskStatusRepository();
private TaskErrorRepository taskErrorRepository = new TaskErrorRepository();
private JobRepository jobRepository = new JobRepository();
+ private JobStatusRepository jobStatusRepository = new JobStatusRepository();
private QueueStatusRepository queueStatusRepository = new QueueStatusRepository();
private DataProductRepository dataProductRepository = new DataProductRepository();
private DataReplicaLocationRepository dataReplicaLocationRepository = new DataReplicaLocationRepository();
@@ -893,7 +892,7 @@ public class RegistryServerHandler implements RegistryService.Iface {
JobPK jobPK = new JobPK();
jobPK.setJobId(jobId);
jobPK.setTaskId(taskId);
- jobRepository.addJobStatus(jobStatus, jobPK);
+ jobStatusRepository.addJobStatus(jobStatus, jobPK);
} catch (Exception e) {
logger.error(jobId, "Error while adding job status", e);
AiravataSystemException exception = new AiravataSystemException();

license: Apache License 2.0

repo: apache/airavata

original_message: Adding new job repository


diff:
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
- <AdditionalDependencies>libhl64.lib::foreach libraries::;::name::64.lib::end::</AdditionalDependencies>
+ <AdditionalDependencies>libhl.lib::foreach libraries::;::name::.lib::end::</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation>
- <AdditionalDependencies>libhl64.lib::foreach libraries::;::name::64.lib::end::</AdditionalDependencies>
+ <AdditionalDependencies>libhl.lib::foreach libraries::;::name::.lib::end::</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemGroup>::foreach cfiles::

license: MIT License

repo: haxefoundation/hashlink

original_message: Match .lib names to defaults generated by solution (#449)


message:
[ci] Update linux environment
Azure pipelines removed 16.04

diff:
parameters:
name: 'BuildLinux'
- vmImage: 'ubuntu-16.04'
+ vmImage: 'ubuntu-18.04'
container: 'debian:stretch'
arch: '' # set it to i386 fof 32-bit build
buildSystem: 'cmake' # either 'cmake' or 'make'


message: Fix Visual Studio project templates

old_path: .gitignore

new_path: .gitignore

diff:
@@ -37,7 +37,7 @@ ReleaseStatic
/src/Makefile
/include/ffmpeg
shiroLibs
-ReleaseVS2013
+ReleaseVS*
/include/bullet
/libs/bullet/sample/sample.js
/libs/bullet/bullet_js.cpp

change_type: RENAME

original_message: Fix Visual Studio project templates (#505)


message: Publish linux 32 bit build

diff:
@@ -352,7 +352,7 @@ jobs:
--attach "artifact/hashlink-latest-linux-amd64.tar.gz" \
--attach "artifact/hashlink-latest-win32.zip" \
--attach "artifact/hashlink-latest-win64.zip"
- #--attach "artifact/hashlink-latest-linux-i386.tar.gz"
+ --attach "artifact/hashlink-latest-linux-i386.tar.gz"
- name: "Delete intermediate build artifacts"
uses: geekyeggo/delete-artifact@1-glob-support # https://github.com/GeekyEggo/delete-artifact/

original_message: Publish linux 32 bit build (#534)


message: [ci] Add missing slash

diff:
@@ -351,7 +351,7 @@ jobs:
--attach "artifact/hashlink-latest-darwin.tar.gz" \
--attach "artifact/hashlink-latest-linux-amd64.tar.gz" \
--attach "artifact/hashlink-latest-win32.zip" \
- --attach "artifact/hashlink-latest-win64.zip"
+ --attach "artifact/hashlink-latest-win64.zip" \
--attach "artifact/hashlink-latest-linux-i386.tar.gz"
- name: "Delete intermediate build artifacts"

original_message: [ci] Add missing slash (#535)


diff:
@@ -145,6 +145,10 @@ add_executable(hl
src/profile.c
)
+if (UNIX AND NOT APPLE)
+ set_target_properties(hl PROPERTIES INSTALL_RPATH "$ORIGIN;${CMAKE_INSTALL_PREFIX}/lib")
+endif()
+
target_link_libraries(hl libhl)
if(WIN32)

old_path: Makefile

new_path: Makefile

diff:
@@ -89,7 +89,7 @@ else
# Linux
CFLAGS += -m$(MARCH) -fPIC -pthread -fno-omit-frame-pointer
-LFLAGS += -lm -Wl,-rpath,. -Wl,--export-dynamic -Wl,--no-undefined
+LFLAGS += -lm -Wl,-rpath,.:'$$ORIGIN':$(INSTALL_LIB_DIR) -Wl,--export-dynamic -Wl,--no-undefined
ifeq ($(MARCH),32)
CFLAGS += -I /usr/include/i386-linux-gnu

original_message: Fix linux dynamic library issue when installing to `/usr/local/` (#483)


message: [make] Add sqlite.hdll target

diff:
@@ -96,7 +96,8 @@ jobs:
libuv1-dev:i386 \
libvorbis-dev:i386 \
libz-dev:i386 \
- zlib1g-dev:i386
+ zlib1g-dev:i386 \
+ libsqlite3-dev:i386
;;
linux-amd64)
@@ -109,7 +110,8 @@ jobs:
libsdl2-dev \
libturbojpeg-dev \
libuv1-dev \
- libvorbis-dev
+ libvorbis-dev \
+ libsqlite3-dev
;;
darwin*)

old_path: Brewfile

new_path: Brewfile

diff:
@@ -10,3 +10,4 @@ brew "openal-soft"
brew "mbedtls@2", link: true
brew "libuv"
brew "openssl"
+brew "sqlite"

diff:
@@ -7,9 +7,9 @@ INSTALL_BIN_DIR ?= $(PREFIX)/bin
INSTALL_LIB_DIR ?= $(PREFIX)/lib
INSTALL_INCLUDE_DIR ?= $(PREFIX)/include
-LIBS=fmt sdl ssl openal ui uv mysql
+LIBS=fmt sdl ssl openal ui uv mysql sqlite
-CFLAGS = -Wall -O3 -I src -msse2 -mfpmath=sse -std=c11 -I include -I include/pcre -I include/mikktspace -I include/minimp3 -D LIBHL_EXPORTS
+CFLAGS = -Wall -O3 -I src -msse2 -mfpmath=sse -std=c11 -D LIBHL_EXPORTS
LFLAGS = -L. -lhl
EXTRA_LFLAGS ?=
LIBFLAGS =
@@ -17,6 +17,8 @@ HLFLAGS = -ldl
LIBEXT = so
LIBTURBOJPEG = -lturbojpeg
+PCRE_INCLUDE = -I include/pcre
+
PCRE = include/pcre/pcre_chartables.o include/pcre/pcre_compile.o include/pcre/pcre_dfa_exec.o \
include/pcre/pcre_exec.o include/pcre/pcre_fullinfo.o include/pcre/pcre_globals.o \
include/pcre/pcre_newline.o include/pcre/pcre_string_utils.o include/pcre/pcre_tables.o include/pcre/pcre_xclass.o \
@@ -31,6 +33,8 @@ STD = src/std/array.o src/std/buffer.o src/std/bytes.o src/std/cast.o src/std/da
HL = src/code.o src/jit.o src/main.o src/module.o src/debugger.o src/profile.o
+FMT_INCLUDE = -I include/mikktspace -I include/minimp3
+
FMT = libs/fmt/fmt.o libs/fmt/sha1.o include/mikktspace/mikktspace.o libs/fmt/mikkt.o libs/fmt/dxt.o
SDL = libs/sdl/sdl.o libs/sdl/gl.o
@@ -45,6 +49,8 @@ UI = libs/ui/ui_stub.o
MYSQL = libs/mysql/socket.o libs/mysql/sha1.o libs/mysql/my_proto.o libs/mysql/my_api.o libs/mysql/mysql.o
+SQLITE = libs/sqlite/sqlite.o
+
LIB = ${PCRE} ${RUNTIME} ${STD}
BOOT = src/_main.o
@@ -66,7 +72,9 @@ else ifeq ($(UNAME),Darwin)
# Mac
LIBEXT=dylib
-CFLAGS += -m$(MARCH) -I /usr/local/include -I /usr/local/opt/libjpeg-turbo/include -I /usr/local/opt/jpeg-turbo/include -I /usr/local/opt/sdl2/include/SDL2 -I /usr/local/opt/libvorbis/include -I /usr/local/opt/openal-soft/include -Dopenal_soft -DGL_SILENCE_DEPRECATION
+CFLAGS += -m$(MARCH) -I include -I /usr/local/include -I /usr/local/opt/libjpeg-turbo/include \
+ -I /usr/local/opt/jpeg-turbo/include -I /usr/local/opt/sdl2/include/SDL2 -I /usr/local/opt/libvorbis/include \
+ -I /usr/local/opt/openal-soft/include -Dopenal_soft -DGL_SILENCE_DEPRECATION
LFLAGS += -Wl,-export_dynamic -L/usr/local/lib
ifdef OSX_SDK
@@ -130,6 +138,12 @@ uninstall:
libs: $(LIBS)
+./include/pcre/%.o: include/pcre/%.c
+ ${CC} ${CFLAGS} -o $@ -c $< ${PCRE_FLAGS}
+
+src/std/regexp.o: src/std/regexp.c
+ ${CC} ${CFLAGS} -o $@ -c $< ${PCRE_FLAGS}
+
libhl: ${LIB}
${CC} -o libhl.$(LIBEXT) -m${MARCH} ${LIBFLAGS} -shared ${LIB} -lpthread -lm
@@ -139,8 +153,11 @@ hlc: ${BOOT}
hl: ${HL} libhl
${CC} ${CFLAGS} -o hl ${HL} ${LFLAGS} ${EXTRA_LFLAGS} ${HLFLAGS}
+libs/fmt/%.o: libs/fmt/%.c
+ ${CC} ${CFLAGS} -o $@ -c $< ${FMT_INCLUDE}
+
fmt: ${FMT} libhl
- ${CC} ${CFLAGS} -I include/mikktspace -I include/minimp3 -shared -o fmt.hdll ${FMT} ${LIBFLAGS} -L. -lhl -lpng $(LIBTURBOJPEG) -lz -lvorbisfile
+ ${CC} ${CFLAGS} -shared -o fmt.hdll ${FMT} ${LIBFLAGS} -L. -lhl -lpng $(LIBTURBOJPEG) -lz -lvorbisfile
sdl: ${SDL} libhl
${CC} ${CFLAGS} -shared -o sdl.hdll ${SDL} ${LIBFLAGS} -L. -lhl -lSDL2 $(LIBOPENGL)
@@ -160,6 +177,9 @@ uv: ${UV} libhl
mysql: ${MYSQL} libhl
${CC} ${CFLAGS} -shared -o mysql.hdll ${MYSQL} ${LIBFLAGS} -L. -lhl
+sqlite: ${SQLITE} libhl
+ ${CC} ${CFLAGS} -shared -o sqlite.hdll ${SQLITE} ${LIBFLAGS} -L. -lhl -lsqlite3
+
mesa:
(cd libs/mesa && make)
@@ -224,7 +244,7 @@ codesign_osx:
${CC} ${CFLAGS} -o $@ -c $<
clean_o:
- rm -f ${STD} ${BOOT} ${RUNTIME} ${PCRE} ${HL} ${FMT} ${SDL} ${SSL} ${OPENAL} ${UI} ${UV} ${HL_DEBUG}
+ rm -f ${STD} ${BOOT} ${RUNTIME} ${PCRE} ${HL} ${FMT} ${SDL} ${SSL} ${OPENAL} ${UI} ${UV} ${MYSQL} ${SQLITE} ${HL_DEBUG}
clean: clean_o
rm -f hl hl.exe libhl.$(LIBEXT) *.hdll

diff:
@@ -17,16 +17,17 @@ Here's the dependencies that you install in order to compile all the libraries:
* sdl: libsdl2-dev
* ssl: libmbedtls-dev
* uv: libuv1-dev
+* sqlite: libsqlite3-dev
To install all dependencies on the latest **Ubuntu**, for example:
-`sudo apt-get install libpng-dev libturbojpeg-dev libvorbis-dev libopenal-dev libsdl2-dev libmbedtls-dev libuv1-dev`
+`sudo apt-get install libpng-dev libturbojpeg-dev libvorbis-dev libopenal-dev libsdl2-dev libmbedtls-dev libuv1-dev libsqlite3-dev`
For 16.04, see [this note](https://github.com/HaxeFoundation/hashlink/issues/147).
To install all dependencies on the latest **Fedora**, for example:
-`sudo dnf install libpng-devel turbojpeg-devel libvorbis-devel openal-soft-devel SDL2-devel mesa-libGLU-devel mbedtls-devel libuv-devel`
+`sudo dnf install libpng-devel turbojpeg-devel libvorbis-devel openal-soft-devel SDL2-devel mesa-libGLU-devel mbedtls-devel libuv-devel sqlite-devel`
**And on OSX:**

diff:
@@ -2,7 +2,7 @@ parameters:
name: 'BuildLinux'
vmImage: 'ubuntu-18.04'
container: 'debian:stretch'
- arch: '' # set it to i386 fof 32-bit build
+ arch: '' # set it to i386 for 32-bit build
buildSystem: 'cmake' # either 'cmake' or 'make'
cmakeConfig: 'RelWithDebInfo'
@@ -44,6 +44,7 @@ jobs:
libturbojpeg0-dev \
libuv1-dev \
libopenal-dev \
+ libsqlite3-dev \
neko \
curl \
ca-certificates
@@ -67,6 +68,7 @@ jobs:
libturbojpeg0-dev:i386 \
libuv1-dev:${{ parameters.arch }} \
libopenal-dev:${{ parameters.arch }} \
+ libsqlite3-dev:${{ parameters.arch }} \
neko \
curl \
ca-certificates

original_message: [make] Add sqlite.hdll target (#520)


message: Fix make error

diff:
@@ -17,7 +17,7 @@ HLFLAGS = -ldl
LIBEXT = so
LIBTURBOJPEG = -lturbojpeg
-PCRE_INCLUDE = -I include/pcre
+PCRE_FLAGS = -I include/pcre
PCRE = include/pcre/pcre_chartables.o include/pcre/pcre_compile.o include/pcre/pcre_dfa_exec.o \
include/pcre/pcre_exec.o include/pcre/pcre_fullinfo.o include/pcre/pcre_globals.o \

original_message: Fix make error (#545)


message: Fix building issues on some mac systems

diff:
@@ -82,7 +82,7 @@ else ifeq ($(UNAME),Darwin)
# Mac
LIBEXT=dylib
CFLAGS += -m$(MARCH) -I include -I /usr/local/include -I /usr/local/opt/libjpeg-turbo/include \
- -I /usr/local/opt/jpeg-turbo/include -I /usr/local/opt/sdl2/include/SDL2 -I /usr/local/opt/libvorbis/include \
+ -I /usr/local/opt/jpeg-turbo/include -I /usr/local/opt/sdl2/include -I /usr/local/opt/libvorbis/include \
-I /usr/local/opt/openal-soft/include -Dopenal_soft -DGL_SILENCE_DEPRECATION
LFLAGS += -Wl,-export_dynamic -L/usr/local/lib

diff:
# include <OpenGLES/ES3/gl.h>
# define HL_GLES
#elif defined(HL_MAC)
-# include <SDL.h>
+# include <SDL2/SDL.h>
# include <OpenGL/gl3.h>
# define glBindImageTexture(...) hl_error("Not supported on OSX")
# define glDispatchCompute(...) hl_error("Not supported on OSX")

diff:
#if defined(_WIN32) || defined(__ANDROID__) || defined(HL_IOS) || defined(HL_TVOS)
# include <SDL.h>
# include <SDL_syswm.h>
-#elif defined(HL_MAC)
-# include <SDL.h>
#else
# include <SDL2/SDL.h>
#endif

diff:
@@ -76,12 +76,21 @@ SET(SDL2_SEARCH_PATHS
${SDL2_PATH}
)
+if (APPLE)
+ FIND_PATH(SDL2_INCLUDE_DIR SDL2/SDL.h
+ HINTS
+ $ENV{SDL2DIR}
+ PATH_SUFFIXES include/SDL2 include
+ PATHS ${SDL2_SEARCH_PATHS}
+ )
+else()
FIND_PATH(SDL2_INCLUDE_DIR SDL.h
HINTS
$ENV{SDL2DIR}
PATH_SUFFIXES include/SDL2 include
PATHS ${SDL2_SEARCH_PATHS}
)
+endif()
if(CMAKE_SIZEOF_VOID_P EQUAL 8)
set(PATH_SUFFIXES lib64 lib/x64 lib)

original_message: Fix building issues on some mac systems (#570)


message: Add vs2019 hlc compilation template

diff:
*.n
Debug
Release
+ReleaseVS2013
x64
*.stackdump
*.hdll
@@ -38,7 +39,6 @@ ReleaseStatic
/src/Makefile
/include/ffmpeg
shiroLibs
-ReleaseVS*
/include/bullet
/libs/bullet/sample/sample.js
/libs/bullet/bullet_js.cpp

diff:
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
+ <VCProjectVersion>15.0</VCProjectVersion>
<ProjectGuid>{$$makeUID(::name::)}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>::name::</RootNamespace>
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
- <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;HL_MAKE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;HL_MAKE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <ConformanceMode>true</ConformanceMode>
<ObjectFileName>$(IntDir)\%(RelativeDir)\%(Filename).obj</ObjectFileName>
</ClCompile>
<Link>
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
- <PreprocessorDefinitions>_DEBUG;_CONSOLE;HL_MAKE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
+ <PreprocessorDefinitions>_DEBUG;_CONSOLE;HL_MAKE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <ConformanceMode>true</ConformanceMode>
<ObjectFileName>$(IntDir)\%(RelativeDir)\%(Filename).obj</ObjectFileName>
</ClCompile>
<Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
- <WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
+ <WarningLevel>Level3</WarningLevel>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
- <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;HL_MAKE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;HL_MAKE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <ConformanceMode>true</ConformanceMode>
<ObjectFileName>$(IntDir)\%(RelativeDir)\%(Filename).obj</ObjectFileName>
</ClCompile>
<Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
- <WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
+ <WarningLevel>Level3</WarningLevel>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
- <PreprocessorDefinitions>NDEBUG;_CONSOLE;HL_MAKE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
+ <PreprocessorDefinitions>NDEBUG;_CONSOLE;HL_MAKE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <ConformanceMode>true</ConformanceMode>
<ObjectFileName>$(IntDir)\%(RelativeDir)\%(Filename).obj</ObjectFileName>
</ClCompile>
<Link>

original_message: Add vs2019 hlc compilation template (#509)


diff:
@@ -29,6 +29,7 @@ entity Botan.PK_Verifier {
op create {
// C++
+ Botan::PK_Verifier(pub_key, emsa);
Botan::PK_Verifier(pub_key, emsa, ...); // additional params: Signature_Format format=IEEE_1363, const std::string &provider=""
// Fake Java

repo: fraunhofer-aisec/codyze

original_message: Fix errors in MARK to match example code (#26)


message:
Upgrade gradle build configuration marked for deprecation
*  Change application's `mainClassName` to `mainClass.set()` as it is
marked for deprecation

diff:
@@ -127,7 +127,7 @@ dependencies {
}
application {
- mainClassName = "de.fraunhofer.aisec.analysis.Main"
+ mainClass.set("de.fraunhofer.aisec.analysis.Main")
// Required to give Ehcache deep reflective access to fields to correctly esitmate the cache size.
applicationDefaultJvmArgs = listOf("--add-opens=java.base/java.lang=ALL-UNNAMED", "--add-opens=java.base/java.util=ALL-UNNAMED", "--add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED", "-Xss10M")
}

original_message:
Upgrade gradle build configuration marked for deprecation (#136)

*  Change application's `mainClassName` to `mainClass.set()` as it is
marked for deprecation


diff:
@@ -275,45 +275,6 @@ public class AnalysisServer {
}
}
- /**
- * extracts all mark-files and the findingDescription.js from a zip or jar file to a temp-folder (which is cleared by the JVM upon exiting)
- *
- * @param zipFilePath
- * @return
- * @throws IOException
- */
- private ArrayList<File> unzipMarkAndFindingDescription(String zipFilePath) throws IOException {
- ArrayList<File> ret = new ArrayList<>();
- Path tempDirWithPrefix = Files.createTempDirectory("mark_extracted_");
- try (ZipInputStream zipIn = new ZipInputStream(new FileInputStream(zipFilePath))) {
- ZipEntry entry = zipIn.getNextEntry();
- // iterates over entries in the zip file
- while (entry != null) {
- String filePath = tempDirWithPrefix.toString() + File.separator + entry.getName();
- if (!entry.isDirectory()
- && (entry.getName().endsWith(".mark") || entry.getName().equals(FINDING_DESCRIPTION_FILE))) {
- try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath))) {
- byte[] bytesIn = new byte[4096];
- int read;
- while ((read = zipIn.read(bytesIn)) != -1) {
- bos.write(bytesIn, 0, read);
- }
- }
- ret.add(new File(filePath));
- } else {
- // if the entry is a directory, make the directory
- File dir = new File(filePath);
- if (!dir.mkdir()) {
- throw new IOException("could not create folder " + dir.getAbsolutePath());
- }
- }
- zipIn.closeEntry();
- entry = zipIn.getNextEntry();
- }
- }
- return ret;
- }
-
/**
* Loads all MARK rules from a file or a directory.
*
@@ -345,21 +306,6 @@ public class AnalysisServer {
log.error("Failed to load MARK file", e);
}
markDescriptionFile = new File(markFile.getAbsolutePath() + File.separator + FINDING_DESCRIPTION_FILE);
- } else if (markFile.getName().endsWith(".jar") || markFile.getName().endsWith(".zip")) {
- try {
- ArrayList<File> allMarkFiles = unzipMarkAndFindingDescription(markFile.getAbsolutePath());
- for (File f : allMarkFiles) {
- if (f.getName().endsWith(".mark")) {
- log.info(" Loading MARK file {}", f.getAbsolutePath());
- parser.addMarkFile(f);
- } else if (f.getName().equals(FINDING_DESCRIPTION_FILE)) {
- markDescriptionFile = f;
- }
- }
- }
- catch (Exception e) {
- log.error("Failed to load MARK file", e);
- }
} else {
log.info("Loading MARK from file {}", markFile.getAbsolutePath());
parser.addMarkFile(markFile);

original_message: Remove ability to load MARK files from archives (#167)


message: Change e-mail on website

diff:
@@ -133,7 +133,7 @@ no = 'Sorry to hear that. Please <a href="https://github.com/USERNAME/REPOSITORY
# End user relevant links. These will show up on left side of footer and in the community page if you have one.
[[params.links.user]]
name = "Email"
- url = "mailto:info@aisec.fraunhofer.de"
+ url = "mailto:codyze@aisec.fraunhofer.de"
icon = "fa fa-envelope"
desc = "Send us an email"
[[params.links.user]]

original_message: Change e-mail on website (#202)


message:
Pull MARK dependency from master through jitpack.io
* Change MARK dependency to master via jitpack.io
We're using jitpack.io to build MARK automatically on request. This
moves the MARK dependency to its master branch because we're changing
MARK and want to pull in the latest changes.
* Reduce MARK library visiblility

diff:
@@ -58,6 +58,10 @@ repositories {
artifact("/[organisation].[module]_[revision].[ext]")
}
}
+
+ maven {
+ setUrl("https://jitpack.io")
+ }
}
java {
@@ -82,7 +86,9 @@ dependencies {
api("de.fraunhofer.aisec:cpg:3.5.1") // ok
// MARK DSL (use fat jar). changing=true circumvents gradle cache
- api("de.fraunhofer.aisec.mark:de.fraunhofer.aisec.mark:1.4.0-SNAPSHOT:repackaged") { isChanging = true } // ok
+ //api("de.fraunhofer.aisec.mark:de.fraunhofer.aisec.mark:1.4.0-SNAPSHOT:repackaged") { isChanging = true } // ok
+ api("com.github.Fraunhofer-AISEC.codyze-mark-eclipse-plugin:de.fraunhofer.aisec.mark:master-SNAPSHOT:repackaged")
+
// Pushdown Systems
api("de.breakpointsec:pushdown:1.1") // ok

original_message:
Pull MARK dependency from master through jitpack.io (#217)

* Change MARK dependency to master via jitpack.io

We're using jitpack.io to build MARK automatically on request. This
moves the MARK dependency to its master branch because we're changing
MARK and want to pull in the latest changes.

* Reduce MARK library visiblility


message:
Remove whitespaces on spitted strings
* Remove whitespaces on spitted strings
Java's `String.split()` removes only trailing whitespaces. With the various `_split_*`
it's easier to remove trailing and leading whitespaces. This simplifies the required
regex for the splitting.
* Remove unused imports

diff:
@@ -8,10 +8,9 @@ import org.checkerframework.checker.nullness.qual.NonNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
-import java.util.Objects;
-import java.util.Set;
import java.util.regex.PatternSyntaxException;
import java.util.stream.Collectors;
@@ -54,15 +53,15 @@ public class SplitDisjoint implements Builtin {
var providedSet = listSet.stream().map(mir -> ((ConstantValue) mir).getValue()).collect(Collectors.toSet());
if (s == null || regex == null || providedSet == null) {
- log.warn("One of the arguments for _split_match_unordered was not the expected type, or not initialized/resolved");
- return ErrorValue.newErrorValue("One of the arguments for _split_match_unordered was not the expected type, or not initialized/resolved",
+ log.warn("One of the arguments for _split_disjoint was not the expected type, or not initialized/resolved");
+ return ErrorValue.newErrorValue("One of the arguments for _split_disjoint was not the expected type, or not initialized/resolved",
argResultList.getAll());
}
log.info("args are: {}; {}; {}", s, regex, providedSet);
String[] splitted = s.split(regex);
- var values = Set.of(splitted);
+ var values = Arrays.stream(splitted).map(String::strip).collect(Collectors.toSet());
boolean isDisjoint = Collections.disjoint(values, providedSet);
ConstantValue cv = ConstantValue.of(isDisjoint);

diff:
@@ -8,9 +8,9 @@ import org.checkerframework.checker.nullness.qual.NonNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.Objects;
-import java.util.Set;
import java.util.regex.PatternSyntaxException;
import java.util.stream.Collectors;
@@ -69,7 +69,7 @@ public class SplitMatchUnordered implements Builtin {
log.info("args are: {}; {}; {}; {}", s, regex, providedSet, strict);
String[] splitted = s.split(regex);
- var values = Set.of(splitted);
+ var values = Arrays.stream(splitted).map(String::strip).collect(Collectors.toSet());
boolean isMatch = strict ? Objects.equals(values, providedSet) : providedSet.containsAll(values);
ConstantValue cv = ConstantValue.of(isMatch);

original_message:
Remove whitespaces on spitted strings (#224)

* Remove whitespaces on spitted strings

Java's `String.split()` removes only trailing whitespaces. With the various `_split_*`
it's easier to remove trailing and leading whitespaces. This simplifies the required
regex for the splitting.

* Remove unused imports


message:
Add explicit Kotlin dependency to prevent version mix
Kotlin compiler warns about mixing library versions. This pulls in
a matching version.

diff:
@@ -89,6 +89,9 @@ dependencies {
// will be a transitive dependency after CPG beta.3
implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.12.5")
+ // pull in explicitly to prevent mixing versions
+ implementation("org.jetbrains.kotlin:kotlin-reflect:1.5.30")
+
// Code Property Graph
api("de.fraunhofer.aisec:cpg:4.0.4") // ok

original_message:
Add explicit Kotlin dependency to prevent version mix (#269)

Kotlin compiler warns about mixing library versions. This pulls in
a matching version.


diff:
@@ -5,7 +5,6 @@ import de.fraunhofer.aisec.codyze.analysis.AnalysisServer
import de.fraunhofer.aisec.codyze.analysis.Finding
import de.fraunhofer.aisec.codyze.analysis.ServerConfiguration
import de.fraunhofer.aisec.codyze.analysis.TypestateMode
-import de.fraunhofer.aisec.codyze.analysis.wpds.NFA
import de.fraunhofer.aisec.cpg.TranslationManager
import java.io.*
import java.lang.Exception
@@ -44,7 +43,7 @@ abstract class AbstractMarkTest : AbstractTest() {
var resource = classLoader.getResource(sourceFileName)
assertNotNull(resource, "Resource $sourceFileName not found")
- var javaFile = File(resource!!.file)
+ var javaFile = File(resource.file)
assertNotNull(javaFile, "File $sourceFileName not found")
val toAnalyze = ArrayList<File>()
toAnalyze.add(javaFile)
@@ -166,7 +165,7 @@ abstract class AbstractMarkTest : AbstractTest() {
}
}
- if (!missingFindings.isEmpty()) {
+ if (missingFindings.isNotEmpty()) {
println("Missing findings:")
for (missing in missingFindings) {
println(missing)

diff:
@@ -5,7 +5,6 @@ import de.fraunhofer.aisec.codyze.JythonInterpreter
import de.fraunhofer.aisec.codyze.analysis.AnalysisServer
import de.fraunhofer.aisec.codyze.analysis.ServerConfiguration
import de.fraunhofer.aisec.codyze.analysis.TypestateMode
-import de.fraunhofer.aisec.codyze.analysis.wpds.NFA
import java.io.*
import kotlin.test.assertTrue
import org.junit.jupiter.api.*

diff:
@@ -21,7 +21,7 @@ internal class FindingDescriptionTest {
val onFailId = "WrongUseOfBotan_CipherMode"
val regions = listOf(Region(1, 2, 10, 12))
val isProblem = true
- val f = Finding(logMsg, artifactUri, onFailId, regions, isProblem)
+ Finding(logMsg, artifactUri, onFailId, regions, isProblem)
// Initialize database with explanations
val fd = instance

diff:
@@ -40,9 +40,9 @@ internal class GithubTest : AbstractTest() {
for (f in files) {
ret.add(f.absolutePath.substring(baseFolder!!.length + 1)) // only use the file name
}
- Collections.sort(ret)
+ ret.sort()
// random!
- Collections.shuffle(ret)
+ ret.shuffle()
if (MAX_FILES_TO_SCAN != -1) {
ret = ret.subList(FILES_OFFSET, FILES_OFFSET + MAX_FILES_TO_SCAN)
}
@@ -96,7 +96,7 @@ internal class GithubTest : AbstractTest() {
val tmpString = sourceFileName
val matchingFiles =
dir.listFiles { pathname: File -> pathname.name.endsWith("$tmpString.out") }
- if (!RESCAN && matchingFiles != null && matchingFiles.size > 0) {
+ if (!RESCAN && matchingFiles != null && matchingFiles.isNotEmpty()) {
println("File already scanned")
return
}

diff:
@@ -22,7 +22,7 @@ internal class MarkLoadOutputTest {
for ((key, markModel) in allModels) {
val reconstructed = StringBuilder()
val entities = ArrayList(markModel.entities)
- if (!entities.isEmpty()) {
+ if (entities.isNotEmpty()) {
// they all have the same packed name in our context
reconstructed.append("package ").append(entities[0].packageName).append("\n")
}
@@ -40,14 +40,14 @@ internal class MarkLoadOutputTest {
var full = String(Files.readAllBytes(Paths.get(key)))
full = full.replace("(?:/\\*(?:[^*]|(?:\\*+[^*/]))*\\*+/)|(?://.*)".toRegex(), "")
for (line in full.split("\n").toTypedArray()) {
- if (!line.strip().isEmpty()) {
- sanitizedOriginal.append(line.strip()).append("\n")
+ if (line.trim().isNotEmpty()) {
+ sanitizedOriginal.append(line.trim()).append("\n")
}
}
val sanitizedReconstructed = StringBuilder()
for (line in reconstructed.toString().split("\n").toTypedArray()) {
- if (!line.strip().isEmpty()) {
- sanitizedReconstructed.append(line.strip()).append("\n")
+ if (line.trim().isNotEmpty()) {
+ sanitizedReconstructed.append(line.trim()).append("\n")
}
}
assertEquals(sanitizedOriginal.toString(), sanitizedReconstructed.toString())
@@ -67,7 +67,7 @@ internal class MarkLoadOutputTest {
assertNotNull(markPoC1)
val markModelFiles = markPoC1.parent
val directories =
- File(markModelFiles).list { current: File?, name: String -> name.endsWith(".mark") }
+ File(markModelFiles).list { _: File?, name: String -> name.endsWith(".mark") }
assertNotNull(directories)
val parser = XtextParser()
for (markFile in directories) {

diff:
@@ -17,11 +17,11 @@ internal class RealBotanTest : AbstractMarkTest() {
@Throws(Exception::class)
fun testSimple() {
// Just a very simple test to explore the graph
- val findings =
performTest(
"real-examples/botan/streamciphers/bsex.cpp",
"real-examples/botan/streamciphers/bsex.mark"
)
+
val graph = ctx!!.graph
val variables =
graph

diff:
@@ -7,7 +7,6 @@ import de.fraunhofer.aisec.codyze.analysis.ServerConfiguration
import de.fraunhofer.aisec.codyze.analysis.TypestateMode
import de.fraunhofer.aisec.codyze.analysis.markevaluation.ExpressionEvaluator
import de.fraunhofer.aisec.codyze.analysis.resolution.ConstantValue
-import de.fraunhofer.aisec.codyze.analysis.wpds.NFA
import de.fraunhofer.aisec.codyze.markmodel.MarkModelLoader
import de.fraunhofer.aisec.cpg.ExperimentalGraph
import de.fraunhofer.aisec.cpg.TranslationManager
@@ -102,7 +101,7 @@ internal class RuleEnsureSemanticsTest {
assertNotNull(markFile)
var directoryContent =
- markFile.listFiles { current: File?, name: String -> name.endsWith(".mark") }
+ markFile.listFiles { _: File?, name: String -> name.endsWith(".mark") }
if (directoryContent == null) {
directoryContent = arrayOf(markFile)
}

diff:
@@ -5,23 +5,25 @@ import org.apache.logging.log4j.Level
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.core.*
import org.apache.logging.log4j.core.appender.AbstractAppender
+import org.apache.logging.log4j.core.config.Property
import org.apache.logging.log4j.core.config.plugins.Plugin
import org.apache.logging.log4j.core.config.plugins.PluginAttribute
import org.apache.logging.log4j.core.config.plugins.PluginElement
import org.apache.logging.log4j.core.config.plugins.PluginFactory
@Plugin(name = "TestAppender", category = Core.CATEGORY_NAME, elementType = Appender.ELEMENT_TYPE)
-class TestAppender(name: String?, filter: Filter?) : AbstractAppender(name, filter, null) {
+class TestAppender(name: String?, filter: Filter?) :
+ AbstractAppender(name, filter, null, true, Property.EMPTY_ARRAY) {
private val events = ArrayList<LogEvent>()
override fun append(event: LogEvent) {
synchronized(events) { events.add(event.toImmutable()) }
}
fun getLog(vararg levels: Level?): List<LogEvent> {
- val levelHashSet = HashSet(Arrays.asList(*levels))
+ val levelHashSet = HashSet(listOf(*levels))
val ret: MutableList<LogEvent> = ArrayList()
for (e in events) {
- if (levels.size == 0 || levelHashSet.contains(e.level)) {
+ if (levels.isEmpty() || levelHashSet.contains(e.level)) {
ret.add(e)
}
}

diff:
@@ -10,8 +10,6 @@ import de.fraunhofer.aisec.mark.markDsl.OrderExpression
import java.io.*
import java.lang.Exception
import java.util.HashSet
-import java.util.function.BinaryOperator
-import java.util.function.Function
import java.util.stream.Collectors
import kotlin.Throws
import kotlin.test.assertEquals
@@ -38,7 +36,7 @@ internal class WpdsTest : AbstractMarkTest() {
edgesFSM.add("START.START -> " + n.name) // artificial node existing in the NFA
}
val seen: MutableSet<StateNode> = HashSet()
- while (!worklist.isEmpty()) {
+ while (worklist.isNotEmpty()) {
val nextWorkList: MutableSet<StateNode> = HashSet()
for (n in worklist) {
for (succ in n.successors) {
@@ -110,14 +108,13 @@ internal class WpdsTest : AbstractMarkTest() {
val findings = performTest("unittests/wpds-vector-example.java", "unittests/vector.mark")
// Extract <line nr, isProblem> from findings
- val startLineNumbers =
findings
.stream()
.collect(
Collectors.toMap(
- Function { f: Finding? -> f!!.regions[0].startLine },
- Function { obj: Finding? -> obj!!.isProblem },
- BinaryOperator { isProblemA: Boolean, isProblemB: Boolean ->
+ { f: Finding? -> f!!.regions[0].startLine },
+ { obj: Finding? -> obj!!.isProblem },
+ { isProblemA: Boolean, isProblemB: Boolean ->
println("Several findings : $isProblemA/$isProblemB")
isProblemA && isProblemB
}
@@ -136,9 +133,9 @@ internal class WpdsTest : AbstractMarkTest() {
.stream()
.collect(
Collectors.toMap(
- Function { f: Finding? -> f!!.regions[0].startLine },
- Function { obj: Finding? -> obj!!.isProblem },
- BinaryOperator { isProblemA: Boolean, isProblemB: Boolean ->
+ { f: Finding? -> f!!.regions[0].startLine },
+ { obj: Finding? -> obj!!.isProblem },
+ { isProblemA: Boolean, isProblemB: Boolean ->
println("Several findings : $isProblemA/$isProblemB")
isProblemA && isProblemB
}
@@ -172,9 +169,9 @@ internal class WpdsTest : AbstractMarkTest() {
.stream()
.collect(
Collectors.toMap(
- Function { f: Finding? -> f!!.regions[0].startLine },
- Function { obj: Finding? -> obj!!.isProblem },
- BinaryOperator { isProblemA: Boolean, isProblemB: Boolean ->
+ { f: Finding? -> f!!.regions[0].startLine },
+ { obj: Finding? -> obj!!.isProblem },
+ { isProblemA: Boolean, isProblemB: Boolean ->
println("Several findings : $isProblemA/$isProblemB")
isProblemA && isProblemB
}
@@ -210,9 +207,9 @@ internal class WpdsTest : AbstractMarkTest() {
.stream()
.collect(
Collectors.toMap(
- Function { f: Finding? -> f!!.regions[0].startLine },
- Function { obj: Finding? -> obj!!.isProblem },
- BinaryOperator { isProblemA: Boolean, isProblemB: Boolean ->
+ { f: Finding? -> f!!.regions[0].startLine },
+ { obj: Finding? -> obj!!.isProblem },
+ { isProblemA: Boolean, isProblemB: Boolean ->
println("Several findings : $isProblemA/$isProblemB")
isProblemA && isProblemB
}
@@ -241,9 +238,9 @@ internal class WpdsTest : AbstractMarkTest() {
.stream()
.collect(
Collectors.toMap(
- Function { f: Finding? -> f!!.regions[0].startLine },
- Function { obj: Finding? -> obj!!.isProblem },
- BinaryOperator { isProblemA: Boolean, isProblemB: Boolean ->
+ { f: Finding? -> f!!.regions[0].startLine },
+ { obj: Finding? -> obj!!.isProblem },
+ { isProblemA: Boolean, isProblemB: Boolean ->
println("Several findings : $isProblemA/$isProblemB")
isProblemA && isProblemB
}
@@ -273,9 +270,9 @@ internal class WpdsTest : AbstractMarkTest() {
.stream()
.collect(
Collectors.toMap(
- Function { f: Finding? -> f!!.regions[0].startLine },
- Function { obj: Finding? -> obj!!.isProblem },
- BinaryOperator { isProblemA: Boolean, isProblemB: Boolean ->
+ { f: Finding? -> f!!.regions[0].startLine },
+ { obj: Finding? -> obj!!.isProblem },
+ { isProblemA: Boolean, isProblemB: Boolean ->
println("Several findings : $isProblemA/$isProblemB")
isProblemA && isProblemB
}
@@ -303,9 +300,9 @@ internal class WpdsTest : AbstractMarkTest() {
.stream()
.collect(
Collectors.toMap(
- Function { f: Finding? -> f!!.regions[0].startLine },
- Function { obj: Finding? -> obj!!.isProblem },
- BinaryOperator { isProblemA: Boolean, isProblemB: Boolean ->
+ { f: Finding? -> f!!.regions[0].startLine },
+ { obj: Finding? -> obj!!.isProblem },
+ { isProblemA: Boolean, isProblemB: Boolean ->
println("Several findings : $isProblemA/$isProblemB")
isProblemA && isProblemB
}
@@ -331,9 +328,9 @@ internal class WpdsTest : AbstractMarkTest() {
.stream()
.collect(
Collectors.toMap(
- Function { f: Finding? -> f!!.regions[0].startLine },
- Function { obj: Finding? -> obj!!.isProblem },
- BinaryOperator { isProblemA: Boolean, isProblemB: Boolean ->
+ { f: Finding? -> f!!.regions[0].startLine },
+ { obj: Finding? -> obj!!.isProblem },
+ { isProblemA: Boolean, isProblemB: Boolean ->
println("Several findings : $isProblemA/$isProblemB")
isProblemA && isProblemB
}
@@ -355,9 +352,9 @@ internal class WpdsTest : AbstractMarkTest() {
.stream()
.collect(
Collectors.toMap(
- Function { f: Finding? -> f!!.regions[0].startLine },
- Function { obj: Finding? -> obj!!.isProblem },
- BinaryOperator { isProblemA: Boolean, isProblemB: Boolean ->
+ { f: Finding? -> f!!.regions[0].startLine },
+ { obj: Finding? -> obj!!.isProblem },
+ { isProblemA: Boolean, isProblemB: Boolean ->
println("Several findings : $isProblemA/$isProblemB")
isProblemA || isProblemB
}
@@ -391,11 +388,11 @@ internal class WpdsTest : AbstractMarkTest() {
.stream()
.collect(
Collectors.toMap(
- Function { f: Finding? ->
+ { f: Finding? ->
if (f!!.regions.isEmpty()) -1 else f.regions[0].startLine
},
- Function { obj: Finding? -> obj!!.isProblem },
- BinaryOperator { isProblemA: Boolean, isProblemB: Boolean ->
+ { obj: Finding? -> obj!!.isProblem },
+ { isProblemA: Boolean, isProblemB: Boolean ->
println("Several findings : $isProblemA/$isProblemB")
isProblemA && isProblemB
}

original_message:
Remove kotlin compiler and intellij code warnings (#268)

Co-authored-by: Christian Banse <christian.banse@aisec.fraunhofer.de>


diff:
@@ -4,6 +4,7 @@ package de.fraunhofer.aisec.codyze.analysis;
import com.fasterxml.jackson.annotation.JsonIgnore;
import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation;
import de.fraunhofer.aisec.cpg.sarif.Region;
+import de.fraunhofer.aisec.mark.markDsl.Action;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
@@ -16,17 +17,35 @@ import java.util.stream.Collectors;
/**
* Representation of a vulnerability/non-vulnerability found in the source code.
- *
*/
public class Finding {
- private final String onFailIdentifier;
/**
- * True, if this Finding indicates a problem/vulnerability. False, if this Finding indicates a code snippet that has been checked and verified.
+ * Identifier for this finding.
+ */
+ private String id;
+
+ /**
+ *
+ */
+ private final Action action;
+
+ /**
+ * True, if this finding indicates a problem/vulnerability. False, if this
+ * finding indicates a code snippet that has been checked and verified.
*/
private boolean isProblem = true;
+ /**
+ * Simplified description of this finding. Usually state that a rule was
+ * validated or violated.
+ */
private final String logMsg;
+
+ /**
+ * Location (lines and columns) in source code files, where this finding was
+ * produced.
+ */
@NonNull
private final List<PhysicalLocation> locations = new ArrayList<>();
@@ -35,15 +54,16 @@ public class Finding {
*
* @param logMsg Log message for that specific finding. This message is created by the analysis module and may contain further descriptions and details of the
* finding.
- * @param onfailIdentifier Identifier of the generic finding, as given by the "onfail" construct of the MARK rule.
* @param startLine Line in code where the finding begins. Note that LSP starts counting at 1.
* @param endLine Line in code where the finding ends.
* @param startColumn Column in code where the finding begins. Note that LPS start counting at 1.
* @param endColumn Column in code where the finding ends.
*/
- public Finding(String logMsg, String onfailIdentifier, @Nullable URI artifactUri, int startLine, int endLine, int startColumn, int endColumn) {
+ public Finding(String id, Action action, String logMsg, @Nullable URI artifactUri, int startLine, int endLine, int startColumn,
+ int endColumn) {
+ this.id = id;
+ this.action = action;
this.logMsg = logMsg;
- this.onFailIdentifier = onfailIdentifier;
if (artifactUri != null) {
this.locations
.add(new PhysicalLocation(artifactUri, new Region(startLine, startColumn, endLine, endColumn)));
@@ -53,16 +73,18 @@ public class Finding {
/**
* Constructor.
*
+ * @param id
* @param logMsg Log message for that specific finding. This message is created by the analysis module and may contain further descriptions and details of the
* finding.
* @param artifactUri Absolute URI of the source file.
- * @param onfailIdentifier Identifier of the generic finding, as given by the "onfail" construct of the MARK rule.
* @param ranges List of LSP "ranges" determining the position(s) in code of this finding. Note that a LSP range starts counting at 1, while a CPG "region" starts
* @param isProblem true, if this Finding represents a vulnerability/weakness. False, if the Finding confirms that the code is actually correct.
*/
- public Finding(String logMsg, @Nullable URI artifactUri, String onfailIdentifier, List<Region> ranges, boolean isProblem) {
+ public Finding(String id, Action action, String logMsg, @Nullable URI artifactUri, List<Region> ranges, boolean isProblem) {
+ this.id = id;
+ this.action = action;
this.logMsg = logMsg;
- this.onFailIdentifier = onfailIdentifier;
+
for (Region r : ranges) {
if (artifactUri != null) {
this.locations.add(new PhysicalLocation(artifactUri, r));
@@ -71,6 +93,14 @@ public class Finding {
this.isProblem = isProblem;
}
+ public String getIdentifier() {
+ return id;
+ }
+
+ public Action getAction() {
+ return action;
+ }
+
public String getLogMsg() {
return logMsg;
}
@@ -92,7 +122,7 @@ public class Finding {
/**
* Returns a non-null, possibly empty list of physical locations of this finding.
- *
+ * <p>
* This method is required for a proper result JSON.
*
* @return
@@ -102,20 +132,14 @@ public class Finding {
return this.locations;
}
- public String getOnfailIdentifier() {
- return onFailIdentifier;
- }
-
public String toString() {
String addIfExists = "";
- if (onFailIdentifier != null) {
// simple for now
- String descriptionShort = FindingDescription.getInstance().getDescriptionShort(onFailIdentifier);
- if (descriptionShort != null && !descriptionShort.equals(onFailIdentifier)) {
+ String descriptionShort = FindingDescription.getInstance().getDescriptionShort(id);
+ if (descriptionShort != null && !descriptionShort.equals(id)) {
addIfExists = ": " + descriptionShort;
}
- }
String lines;
if (locations.size() == 1) {
@@ -130,7 +154,8 @@ public class Finding {
if (!(obj instanceof Finding)) {
return false;
}
- return Objects.equals(this.logMsg, ((Finding) obj).logMsg) && Objects.equals(this.onFailIdentifier, ((Finding) obj).onFailIdentifier);
+ return Objects.equals(this.logMsg, ((Finding) obj).logMsg)
+ && Objects.equals(this.id, ((Finding) obj).id);
}
public int hashCode() {
@@ -143,9 +168,9 @@ public class Finding {
* @param out
*/
public void prettyPrintShort(@NonNull PrintStream out) {
- String shortMsg = FindingDescription.getInstance().getDescriptionShort(onFailIdentifier);
+ String shortMsg = FindingDescription.getInstance().getDescriptionShort(id);
if (shortMsg == null) {
- shortMsg = onFailIdentifier;
+ shortMsg = id;
}
String lines;
@@ -157,4 +182,5 @@ public class Finding {
out.println(lines + ": " + (isProblem ? "(BAD) " : "(GOOD) ") + shortMsg + ": " + logMsg);
}
+
}

diff:
@@ -14,30 +14,35 @@ class FindingDescription private constructor() {
val helpUri: String?,
val fullDescription: Description?,
val shortDescription: Description?,
+ val passMessage: Description?,
val fixes: List<Fix>?
)
class Fix(val description: Description)
class Description(val text: String)
- fun get(onFailId: String): FindingDescriptionItem? {
- return items?.get(onFailId)
+ fun get(id: String): FindingDescriptionItem? {
+ return items?.get(id)
}
- fun getDescriptionFull(onFailId: String): String? {
- return items?.get(onFailId)?.fullDescription?.text
+ fun getDescriptionFull(id: String): String? {
+ return items?.get(id)?.fullDescription?.text
}
- fun getDescriptionShort(onFailId: String): String? {
- return items?.get(onFailId)?.shortDescription?.text
+ fun getDescriptionShort(id: String): String? {
+ return items?.get(id)?.shortDescription?.text
}
- fun getHelpUri(onFailId: String): String? {
- return items?.get(onFailId)?.helpUri
+ fun getDescriptionPass(id: String): String? {
+ return items?.get(id)?.passMessage?.text
}
- fun getFixes(onFailId: String): List<String>? {
- return items?.get(onFailId)?.fixes?.map { it.description.text }
+ fun getHelpUri(id: String): String? {
+ return items?.get(id)?.helpUri
+ }
+
+ fun getFixes(id: String): List<String>? {
+ return items?.get(id)?.fixes?.map { it.description.text }
}
fun init(descriptionFile: File) {

diff:
@@ -252,14 +252,17 @@ public class Evaluator {
}
boolean isRuleViolated = !(Boolean) evaluationResultUb;
+
findings.add(new Finding(
+ rule.getErrorMessage() != null ? rule.getErrorMessage() : rule.getName(),
+ rule.getStatement().getAction(),
"Rule "
+ rule.getName()
+ (isRuleViolated ? " violated" : " verified"),
currentFile,
- rule.getErrorMessage(),
ranges,
isRuleViolated));
+
}
} else if (evaluationResultUb == null) {
log.warn("Unable to evaluate rule {} in MARK context " + markCtx + "/" + markCtxHolder.getAllContexts().size()
@@ -353,7 +356,6 @@ public class Evaluator {
* In that case, the function will return [ [ (c1, v1) , (c1, v2), (c1, v3) ] [ (c2, c1), (c2, v2), (c2, v3) ] ]
*
* @param rule the MARK rule
- *
* @return
*/
protected List<List<Pair<String, Node>>> findInstancesForEntities(MRule rule) {

diff:
@@ -128,9 +128,10 @@ public class ExpressionEvaluator {
if (node != null) {
var ranges = List.of(Utils.getRegionByNode(node));
Finding f = new Finding(
+ (this.markRule.getErrorMessage() != null ? this.markRule.getErrorMessage() : this.markRule.getName()),
+ this.markRule.getStatement().getAction(),
"Verified Order: " + this.markRule.getName(),
new File(node.getFile()).toURI(),
- "",
ranges,
false);
this.resultCtx.getFindings().add(f);

diff:
@@ -5,6 +5,7 @@ import de.fraunhofer.aisec.codyze.analysis.Finding
import de.fraunhofer.aisec.codyze.analysis.utils.Utils
import de.fraunhofer.aisec.codyze.markmodel.MOp
import de.fraunhofer.aisec.codyze.markmodel.Mark
+import de.fraunhofer.aisec.mark.markDsl.Action
import de.fraunhofer.aisec.mark.markDsl.OpStatement
import java.io.File
import org.slf4j.LoggerFactory
@@ -65,8 +66,9 @@ class ForbiddenEvaluator(private val markModel: Mark) {
node.code)
val f =
Finding(
+ "FORBIDDEN_" + entity.name,
+ Action.FAIL,
message,
- "FORBIDDEN",
File(node.file).toURI(),
region.startLine,
region.endLine,

diff:
@@ -285,6 +285,8 @@ public class OrderNFAEvaluator {
var region = Utils.getRegionByNode(vertex);
Finding f = new Finding(
+ rule.getErrorMessage() != null ? rule.getErrorMessage() : rule.getName(),
+ rule.getStatement().getAction(),
"Violation against Order: "
+ vertex.getCode()
+ " ("
@@ -297,7 +299,6 @@ public class OrderNFAEvaluator {
+ " ("
+ rule.getErrorMessage()
+ ")",
- rule.getErrorMessage(),
new File(vertex.getFile()).toURI(),
region.getStartLine(),
region.getEndLine(),
@@ -442,6 +443,8 @@ public class OrderNFAEvaluator {
endCol = region.getEndColumn();
}
Finding f = new Finding(
+ rule.getErrorMessage() != null ? rule.getErrorMessage() : rule.getName(),
+ rule.getStatement().getAction(),
"Violation against Order: Base "
+ base
+ " is not correctly terminated. Expected one of ["
@@ -453,7 +456,6 @@ public class OrderNFAEvaluator {
+ " ("
+ rule.getErrorMessage()
+ ")",
- rule.getErrorMessage(),
file,
startLine,
endLine,

diff:
@@ -316,9 +316,10 @@ class TypestateAnalysis(private val markContextHolder: MarkContextHolder) {
}
return Finding(
+ rule.errorMessage ?: rule.name,
+ rule.statement.action,
name,
currentFile,
- rule.errorMessage,
listOf(Utils.getRegionByNode(stmt)),
true
)
@@ -336,9 +337,10 @@ class TypestateAnalysis(private val markContextHolder: MarkContextHolder) {
*/
private fun createGoodFinding(node: Node, `val`: Val?, currentFile: URI): Finding {
return Finding(
+ rule.errorMessage ?: rule.name,
+ rule.statement.action,
"Good: $`val` at $node",
currentFile,
- rule.errorMessage,
listOf(Utils.getRegionByNode(node)),
false
)

diff:
@@ -157,20 +157,41 @@ public class CpgDocumentService implements TextDocumentService {
for (Region reg : f.getRegions()) {
Diagnostic diagnostic = new Diagnostic();
// TODO Replace HINT for verified findings with Code Lens
- diagnostic.setSeverity(f.isProblem() ? DiagnosticSeverity.Error : DiagnosticSeverity.Information);
+
+ // Everything is informational until it becomes a problem
+ DiagnosticSeverity severity = DiagnosticSeverity.Information;
+ if (f.isProblem()) {
+ switch (f.getAction()) {
+ case INFO:
+ severity = DiagnosticSeverity.Information;
+ break;
+ case WARN:
+ severity = DiagnosticSeverity.Warning;
+ break;
+ case FAIL:
+ // Error
+ default:
+ severity = DiagnosticSeverity.Error;
+ break;
+ }
+ }
+ diagnostic.setSeverity(severity);
// Get human readable description, if available
- String msg = FindingDescription.getInstance().getDescriptionShort(f.getOnfailIdentifier());
+ String msg = (f.isProblem() ? FindingDescription.getInstance().getDescriptionShort(f.getIdentifier())
+ : FindingDescription.getInstance().getDescriptionPass(f.getIdentifier()));
if (msg == null) {
msg = f.getLogMsg();
}
- String longDesc = FindingDescription.getInstance().getDescriptionFull(f.getOnfailIdentifier());
+ if (f.isProblem()) {
+ String longDesc = FindingDescription.getInstance().getDescriptionFull(f.getIdentifier());
if (longDesc != null) {
msg += ": " + longDesc;
}
+ }
- diagnostic.setCode(Either.forLeft(f.getOnfailIdentifier()));
+ diagnostic.setCode(Either.forLeft(f.getIdentifier()));
diagnostic.setSource("Codyze");
diagnostic.setMessage(msg);
Range r = new Range(
@@ -181,7 +202,7 @@ public class CpgDocumentService implements TextDocumentService {
String line = ignoredLines.get(r.getStart().getLine());
if (line == null
|| (!line.contains(DISABLE_FINDING_ALL)
- && !line.contains(DISABLE_FINDING_PREFIX + f.getOnfailIdentifier()))) {
+ && !line.contains(DISABLE_FINDING_PREFIX + f.getIdentifier()))) {
allDiags.add(diagnostic);
} else {
log.warn("Skipping finding {}, disabled via comment", f);

diff:
@@ -15,6 +15,7 @@ public class MRule {
private String name;
private RuleStatement statement;
private String errorMessage;
+
/**
* stores Entity-alias to Pair(Name of Entity, EntityReference). The EntityReference can be NULL if the entity is not available/parsed. E.g. for the rule rule
* UseOfBotan_CipherMode { using Order as cm ensure order cm.start(), cm.finish() onfail WrongUseOfBotan_CipherMode } this would store: cm -> Pair(Order, Reference to

diff:
@@ -3,6 +3,7 @@ package de.fraunhofer.aisec.codyze.crymlin
import de.fraunhofer.aisec.codyze.analysis.Finding
import de.fraunhofer.aisec.codyze.analysis.FindingDescription.Companion.instance
import de.fraunhofer.aisec.cpg.sarif.Region
+import de.fraunhofer.aisec.mark.markDsl.Action
import java.io.File
import java.net.URI
import java.net.URISyntaxException
@@ -18,10 +19,10 @@ internal class FindingDescriptionTest {
// Create some Finding object
val logMsg = "Variable cm not initialized"
val artifactUri = URI.create("file:///tmp/test.cpp")
- val onFailId = "WrongUseOfBotan_CipherMode"
+ val id = "WrongUseOfBotan_CipherMode"
val regions = listOf(Region(1, 2, 10, 12))
val isProblem = true
- Finding(logMsg, artifactUri, onFailId, regions, isProblem)
+ Finding(id, Action.FAIL, logMsg, artifactUri, regions, isProblem)
// Initialize database with explanations
val fd = instance
@@ -31,15 +32,17 @@ internal class FindingDescriptionTest {
fd.init(File(url.toURI()))
- val item = fd.get(onFailId)
- val fullDescription = fd.getDescriptionFull(onFailId)
- val shortDescription = fd.getDescriptionShort(onFailId)
- val helpUri = fd.getHelpUri(onFailId)
- val fixes = fd.getFixes(onFailId)
+ val item = fd.get(id)
+ val fullDescription = fd.getDescriptionFull(id)
+ val shortDescription = fd.getDescriptionShort(id)
+ val passMessage = fd.getDescriptionPass(id)
+ val helpUri = fd.getHelpUri(id)
+ val fixes = fd.getFixes(id)
assertNotNull(item)
assertEquals("Full description", fullDescription)
assertEquals("The order of called Botan methods is wrong.", shortDescription)
+ assertEquals("Order of Botan methods validated.", passMessage)
assertEquals("https://www.codyze.io/explanations/10", helpUri)
assertEquals(1, fixes?.size)
assertEquals("Just fix it!", fixes?.get(0) ?: "")

diff:
@@ -15,6 +15,6 @@ internal class JacksonTest : AbstractMarkTest() {
val finding = findings.iterator().next()
assertEquals(9, finding.locations[0].region.startLine)
- assertEquals("FORBIDDEN", finding.onfailIdentifier)
+ assertEquals("FORBIDDEN_ObjectMapper", finding.identifier)
}
}

diff:
@@ -19,14 +19,14 @@ internal class RealBCTest : AbstractMarkTest() {
"real-examples/bc/rwedoff.Password-Manager/"
)
for (f in findings) {
- println(" ->" + f.onfailIdentifier + " " + f.regions[0] + " " + f.logMsg)
+ println(" ->" + f.identifier + " " + f.regions[0] + " " + f.logMsg)
}
// We expect three (positive) verification of the algorithm used
val wrongAlgorithmsFindings =
findings
.stream()
- .filter { f: Finding? -> f!!.onfailIdentifier == "Wrong_Algorithm" }
+ .filter { f: Finding? -> f!!.identifier == "Wrong_Algorithm" }
.collect(Collectors.toSet())
assertEquals(3, wrongAlgorithmsFindings.size) // 3 in total
assertEquals(

diff:
@@ -46,7 +46,7 @@ internal class RealBotanTest : AbstractMarkTest() {
val correctKeyLength =
findings
.stream()
- .filter { f: Finding? -> f!!.onfailIdentifier == "CorrectPrivateKeyLength" }
+ .filter { f: Finding? -> f!!.identifier == "CorrectPrivateKeyLength" }
.findFirst()
assertTrue(correctKeyLength.isPresent)
assertFalse(correctKeyLength.get().isProblem)
@@ -66,7 +66,7 @@ internal class RealBotanTest : AbstractMarkTest() {
val wrongKeyLength =
findings
.stream()
- .filter { f: Finding? -> f!!.onfailIdentifier == "WrongPrivateKeyLength" }
+ .filter { f: Finding? -> f!!.identifier == "WrongPrivateKeyLength" }
.findFirst()
assertTrue(wrongKeyLength.isPresent)
assertTrue(wrongKeyLength.get().isProblem)
@@ -118,7 +118,7 @@ internal class RealBotanTest : AbstractMarkTest() {
val blockCiphers =
findings
.stream()
- .filter { f: Finding? -> f!!.onfailIdentifier == "WrongBlockCipher" }
+ .filter { f: Finding? -> f!!.identifier == "WrongBlockCipher" }
.filter { f: Finding? -> !f!!.isProblem }
.collect(Collectors.toList())
assertTrue(blockCiphers.stream().anyMatch { f: Finding? -> f!!.regions[0].startLine == 15 })
@@ -129,7 +129,7 @@ internal class RealBotanTest : AbstractMarkTest() {
val keyLengths =
findings
.stream()
- .filter { f: Finding? -> f!!.onfailIdentifier == "BadKeyLength" }
+ .filter { f: Finding? -> f!!.identifier == "BadKeyLength" }
.filter { obj: Finding? -> obj!!.isProblem }
.collect(Collectors.toList())
assertEquals(2, keyLengths.size)

diff:
"text": "Just fix it!"
}
}
- ]
+ ],
+ "passMessage": {
+ "text": "Order of Botan methods validated."
+ }
}
}

original_message:
Add severity and pass message for findings (#270)

Co-authored-by: Christian Banse <christian.banse@aisec.fraunhofer.de>


message:
Bump VSCode engine and types
* Bump VSCode version
Make VSCode engine and the same version string. Hopefully,
this helps renovate to update both version concurrently.
* Fix relative path in script due to move in subdir

diff:
#/bin/bash
rm -rf codyze*
-cp ../build/distributions/codyze*.zip .
+cp ../../build/distributions/codyze*.zip .
unzip codyze*.zip
mv codyze*/ codyze || true

diff:
"devDependencies": {
"@types/mocha": "9.0.0",
"@types/node": "14.17.19",
- "@types/vscode": "1.57.0",
+ "@types/vscode": "^1.60.0",
"@typescript-eslint/parser": "4.32.0",
"eslint": "7.32.0",
"mocha": "9.1.2",
"webpack-cli": "4.8.0"
},
"engines": {
- "vscode": "^1.57.0"
+ "vscode": "^1.60.0"
}
},
"node_modules/@babel/code-frame": {
"dev": true
},
"node_modules/@types/vscode": {
- "version": "1.57.0",
- "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.57.0.tgz",
- "integrity": "sha512-FeznBFtIDCWRluojTsi9c3LLcCHOXP5etQfBK42+ixo1CoEAchkw39tuui9zomjZuKfUVL33KZUDIwHZ/xvOkQ==",
+ "version": "1.60.0",
+ "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.60.0.tgz",
+ "integrity": "sha512-wZt3VTmzYrgZ0l/3QmEbCq4KAJ71K3/hmMQ/nfpv84oH8e81KKwPEoQ5v8dNCxfHFVJ1JabHKmCvqdYOoVm1Ow==",
"dev": true
},
"node_modules/@typescript-eslint/parser": {
"dev": true
},
"@types/vscode": {
- "version": "1.57.0",
- "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.57.0.tgz",
- "integrity": "sha512-FeznBFtIDCWRluojTsi9c3LLcCHOXP5etQfBK42+ixo1CoEAchkw39tuui9zomjZuKfUVL33KZUDIwHZ/xvOkQ==",
+ "version": "1.60.0",
+ "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.60.0.tgz",
+ "integrity": "sha512-wZt3VTmzYrgZ0l/3QmEbCq4KAJ71K3/hmMQ/nfpv84oH8e81KKwPEoQ5v8dNCxfHFVJ1JabHKmCvqdYOoVm1Ow==",
"dev": true
},
"@typescript-eslint/parser": {

diff:
"publisher": "fraunhofer-aisec",
"categories": [],
"engines": {
- "vscode": "^1.57.0"
+ "vscode": "^1.60.0"
},
"dependencies": {
"vscode-languageclient": "^7.0.0"
"devDependencies": {
"@types/mocha": "9.0.0",
"@types/node": "14.17.19",
- "@types/vscode": "1.57.0",
+ "@types/vscode": "^1.60.0",
"@typescript-eslint/parser": "4.32.0",
"eslint": "7.32.0",
"mocha": "9.1.2",

original_message:
Bump VSCode engine and types (#291)

* Bump VSCode version

Make VSCode engine and @types/vscode the same version string. Hopefully,
this helps renovate to update both version concurrently.

* Fix relative path in script due to move in subdir


message:
Use GitHub Action to setup Node.js
* Use GitHub Action to setup Node.js
This should fix the EACCESS error in the current workflow.
* Split step with action and run
Didn't remember that uses and run cannot be mixed.

diff:
@@ -60,9 +60,13 @@ jobs:
with:
name: test
path: build/reports/tests/test
+ - name: Prepare Node.js
+ uses: actions/setup-node@v2
+ with:
+ node-version: "16"
- name: Build Visual Studio Code Plugin
run: |
- pushd plugins/vscode && npm version $VERSION || true && sudo npm install -g vsce && npm install && vsce package && popd
+ pushd plugins/vscode && npm version $VERSION || true && npm install -g vsce && npm install && vsce package && popd
env:
VERSION: ${{ steps.determine_version.outputs.version }}
- name: Archive Visual Studio Code Plugin

original_message:
Use GitHub Action to setup Node.js (#342)

* Use GitHub Action to setup Node.js

This should fix the EACCESS error in the current workflow.

* Split step with action and run

Didn't remember that uses and run cannot be mixed.


diff:
"matchPackagePatterns": ["^info.picocli:"]
},
{
- "matchPackagePatterns": ["@types/vscode"],
+ "groupName": "Codyze VSCode plugin",
+ "matchPaths": ["plugins/vscode/**"],
+ "extends": ["schedule:earlyMondays"]
+ },
+ {
+ "matchPackageNames": ["vscode"],
"rangeStrategy": "bump"
}
]

original_message: Adjust renovate config for vscode plugin (#357)


message: Update MARK to latest release

diff:
@@ -97,7 +97,9 @@ dependencies {
// MARK DSL (use fat jar). changing=true circumvents gradle cache
//api("de.fraunhofer.aisec.mark:de.fraunhofer.aisec.mark:1.4.0-SNAPSHOT:repackaged") { isChanging = true } // ok
- api("com.github.Fraunhofer-AISEC.codyze-mark-eclipse-plugin:de.fraunhofer.aisec.mark:bbd54a7b11:repackaged") // pin to specific commit before annotations
+ //api("com.github.Fraunhofer-AISEC.codyze-mark-eclipse-plugin:de.fraunhofer.aisec.mark:bbd54a7b11:repackaged") // pin to specific commit before annotations
+ api("com.github.Fraunhofer-AISEC.codyze-mark-eclipse-plugin:de.fraunhofer.aisec.mark:2.0.0:repackaged") // use GitHub release via JitPack
+
// Pushdown Systems
api("de.breakpointsec:pushdown:1.1") // ok

original_message: Update MARK to latest release (#373)


message: Bump Codyze version for CLI interface

diff:
@@ -26,7 +26,7 @@ import java.util.concurrent.TimeUnit;
* Start point of the standalone analysis server.
*/
@SuppressWarnings("java:S106")
-@Command(name = "codyze", mixinStandardHelpOptions = true, version = "1.5.0", description = "Codyze finds security flaws in source code", sortOptions = false, usageHelpWidth = 100)
+@Command(name = "codyze", mixinStandardHelpOptions = true, version = "2.0.0-beta1", description = "Codyze finds security flaws in source code", sortOptions = false, usageHelpWidth = 100)
public class Main implements Callable<Integer> {
private static final Logger log = LoggerFactory.getLogger(Main.class);

original_message: Bump Codyze version for CLI interface (#371)


diff:
],
"ignorePaths": ["scripts", "docs"],
"packageRules": [
+ {
+ "matchPackagePrefixes": ["de.fraunhofer.aisec:cpg"],
+ "groupName": "CPG packages"
+ },
{
"groupName": "picocli packages",
"matchPackagePatterns": ["^info.picocli:"]

original_message: Combine CPG packages into one group for simultaneous updates (#412)