# Objective Configuration

An objective configuration file is a TOML file used to create an objective in the Modality CLI. Objectives can be thought of as the encoding of an experimental design for Modality's experiment automation functionality. You can also apply objectives to previously recorded sessions to take your experimental measurements against historical data.

There are three main parts to an objective. The first is guidance for what mutations to perform. This allows Modality to rapidly explore the space of system states relevant to your experiment. Next are measurements, which allow you to complete the experimental circle, seeing how your system reacts to the different states it is put in. Measurements are recorded for each set of mutations so you can go back and gain insights into how your system behaves. Last are stopping conditions, which give you many different ways to specify when to stop generating new runs of your experiment.

A diagram of Modality's generative testing process, introducing mutations into your system, observing many conditions including possible bugs, and returning sophisticated analysis

# TOML File Fields

# Global Configuration

  • name - required - string - The human-oriented name of the objective.

# Mutation Configuration

  • mutation_constraints - optional - table - Global mutation constraints.
    • max_concurrent_mutations - optional - integer - Maximum number of mutations to perform at the same time.
    • mutation_precondition - optional - string - Trace expression to be used as a global precondition for the triggering of any generated mutation. Mutually exclusive with per-mutator precondition constraints.
    • mutator_tags - optional - array of strings - Only consider mutators which have at least one of the specified tags.
  • mutator - optional - array of tables - Defines per-mutator constraints.
    • name - required - string - The mutator name.
    • probe - optional - string - Restrict mutations to a specific probe (in cases where this mutator is used at multiple probes).
    • mutation_precondition - optional - string - Mutator specific trace expression to be used as a precondition for the triggering of any generated mutation. Mutually exclusive with global mutation_precondition.
    • param_constraints - optional - table - Constraints on potential mutation parameter values.
      • name - required - string - Name of the mutator parameter to constrain.
      • range - required, not permitted for boolean or enum types - array of length 2 - Range of values for the parameter. It is an error if range is not contained within the hard_range for this parameter.
      • enum_options - required for enum types, otherwise not permitted - array of strings - Set of values for the parameter.

# Measurements

  • measurement - required, at least 1 - array of tables - Measurements to take for each set of mutations.
    • name - required - string - The human-oriented name of this measurement.
    • should_pass - optional - boolean - Whether the check should pass or fail. Defaults to true.
    • check - required - string - The trace expression for this measurement, which will be given to check. This expression is parsed and sanity-checked when the objective is created in the CLI with modality objective create.
    • stopping_conditions - optional - table - Stopping conditions specific to this measurement. Note that passing and failing in this context is relative to the should_pass parameter above. Thus, if should_pass is false, a failing measurement is one where the check query returns true.
      • passing_measurements - optional - integer - Number of attempts where this measurement passed.
      • failing_measurements - optional - integer - Number of attempts where this measurement failed.

# Stopping Conditions

  • stopping_conditions - optional - table - General stopping conditions. Multiple conditions will be OR’d together so that any condition being met stops the mutation generator.
    • attempts - optional - integer - Number of sets of simultaneous mutations to attempt.
    • time - optional - string - Total time running this objective.
    • passing_measurements - optional - integer - Number of attempts where all the measurements passed.
    • failing_measurements - optional - integer - Number of attempts where not all the measurements passed.

# Example Objective Configuration

name = "sample objective"


# Global mutation constraints
[mutation_constraints]
# Only perform 1 mutation attempt per mutation-epoch
max_concurrent_mutations = 1

# Only consider mutators which have at least one of the specified tags
mutator_tags = ['drone', 'environmental-forces', 'simulator']

# A global precondition trace expression to be used as a
# causal prefix for the triggering of any generated mutation
# mutation_precondition = 'MATCH (name = "SYSTEM_STATE_READY") AS ReadyState'


# Per-mutator constraints
# For more information about this mutator, see its definition
# in modality-sut/sample_component/mutators/sim-env.toml
[[mutator]]
name = 'simulation-environment-mutator'

# Restrict mutations to the specific probe SIM_MUTATOR_PLUGIN
probe = 'SIM_MUTATOR_PLUGIN'

# Mutator specific precondition trace expression to be used as a
# causal prefix for the triggering of any generated mutation
#
# Wait until the drone altitude (in the simulator) has reached 20 meters before performing mutations
mutation_precondition = 'MATCH (SIM_ALTITUDE @ SIM_MUTATOR_PLUGIN AND payload > 20) AS HoverAltitudeReached'

    # Constrain the impact force to [0.1, 30.0] Newtons
    [[mutator.param_constraints]]
    name = 'impact-force-magnitude'
    range = [0.1, 30.0]

    # Constrain the impact force link to either ROTOR_0 or ROTOR_1
    [[mutator.param_constraints]]
    name = 'impact-force-location-link'
    range = ['ROTOR_0', 'ROTOR_1']

# Measurements to take
# This measurement simply fails if any failure-like events with severity greater than 8 occur
[[measurement]]
name = 'No critical events occur'

# The check is expected to fail
should_pass = false

# Measurement-specific stopping conditions, stop immediately if this check doesn't pass
failing_measurements = 0

# The trace query expression to check
check = 'MATCH (severity > 8) AS AnyCriticalFailures'

# This measurement checks that we only observe up to 5 low severity events
[[measurement]]
name = 'Tolerate up to 5 low severity events'

check = '''
MATCH
    (severity <= 3) AS LowSeverityEvents

AGGREGATE
    count() <= 5
'''

# This measurement checks that the IMU gyroscope instability metric is nominal
[[measurement]]
name = 'IMU gyroscope instability is acceptable'

check = '''
MATCH
    (name = "IMU_GYRO_INSTABILITY" AND probe = "DRONE_IMU") AS Metric

FILTER
    Metric.payload != 0.0

AGGREGATE
    max(Metric.payload) < 0.45,
    mean(Metric.payload) < 0.2,
    stddev(Metric.payload) < 0.1
'''


# This measurement checks that the tilt detection and reaction mechanism happens within
# the specified amount of time
[[measurement]]
name = 'Tilt detection works in a timely fashion'

# Additional measurement-specific stopping conditions may be specified
# passing_measurements = 1

check = '''
MATCH
    # Either roll or pitch is in bounds for the tilt detector
    ((name = "IMU_ROLL" OR name = "IMU_PITCH") AND (payload > 45 OR payload < -45)) AS ExcessiveTilt,

    # The tilt detection mechanism detected the unsafe conditions
    (name = "UNSAFE_TILT_DETECTED") AS UnsafeTiltDetected,

    # Commander module should react to the condition by terminating the flight
    (name = "FLIGHT_TERMINATED") AS FlightTerminated

WHEN
    # It is a requirement that unsafe tilt (either roll or pitch) is detected and reacted to
    ExcessiveTilt -> UnsafeTiltDetected AND UnsafeTiltDetected -> FlightTerminated
'''

# General stopping conditions, OR'd together
[stopping_conditions]
# Maximum number of related mutation-epoch attempts
attempts = 50

# Maximum wall-clock time of related mutation-epoch attempts
time = "2h 15min"

# Maximum number of times all the measurements passed
passing_measurements = 10

# Maximum number of times not all the measurements passed
failing_measurements = 5