Feature/cva6 (#21)

* fix cva6 build
* stats coverage saved on systemfile
* libNPI silent, simv silent by default
* track coverage on cpu core, disable fsm as broken, set proper size for cov map
* cov on ariane_tb.dut.i_ariane
* handle libNPI hangs (license issues or bug)
* always spawn run.sh to run the fuzzer
* plot.py
* remove old readme
* update doc

fuzzers/cva6-vcs-fuzzer/Cargo.toml

@@ -36,4 +36,4 @@ tempdir = "0.3.7"
 serde = { version = "1.0", default-features = false, features = ["alloc"] } # serialization lib
 clap = { version = "3.2", features = ["default"] }
 fs_extra = "1.2.0"
-
+wait-timeout = "0.1.5"

fuzzers/cva6-vcs-fuzzer/README.md

@@ -50,5 +50,33 @@ $ AFL_LAUNCHER_CLIENT=1 ./cva6_vcs_fuzzer
 
 To run multiple fuzzer instances:
 ```
-for i in {1..10}; do AFL_LAUNCHER_CLIENT=$i ./cva6_vcs_fuzzer ; done
+for i in {1..10}; do AFL_LAUNCHER_CLIENT=$i ./cva6_vcs_fuzzer & done
 ```
+
+# Customizing
+
+The fuzzer is bootstraped using the seed files into the `seeds` folder. Feel free to customize the content of this file with any interesting seed.
+When starting the fuzzer loads the initial inputs (i.e., the seeds), and only keep interesting ones in the corpus (i.e., coverage novelty).
+Coverage novelty consider any changes for all supported code coverage metrics on vcs, i.e., branch, conditional, line, toggle, and FSM.
+Then, starts the fuzzer loop that iteratively calls the different stages. 
+StdMutationalStage is responsible for generating new mutant by applying mutation to the existing testcase in the corpus. 
+The mutations work at the ISA level by first deserializing the binary testcase into stream of instruction, then different mutations might be applied (e.g., adding instruction, removing instruction, changing opcode, ..). 
+The mutation can easily be customized by changing `../../libpresifuzz_mutators/src/riscv_isa.rs`. 
+The generated testcase is then inserted into a template ELF file by simplify injecting the code after the `payload` label. 
+This template contains epilogue and prologue code. 
+The current version is very simple. We first init registers to some known values, and we change the `mtvec` to points to our own trap handler.
+The trap handler is there to stop earlier the testcase execution if we trop too often. Otherwise, it always try to return to the instruction after the failing one.
+This version is a naive implementation, better performance could be achieved with some changes on the testharness (e.g., early simulation stop, irq support).
+
+
+# Ploting data
+
+The fuzzer saves statistics into the `sync`directory.
+It is possible to plot coverage over time using the `plot.py`:
+
+```python
+python3 ./plot.py -m branch -d ./sync
+```
+
+The `-m` option is there to provide the coverage metric that is either tgl, cond, branch, line, fsm.
+The `-d` points to the directory where stats are saved.

fuzzers/cva6-vcs-fuzzer/build.rs

@@ -35,6 +35,15 @@ fn main() {
         let cva6_dir = cva6_dir.as_os_str().to_str().unwrap().to_string();
         std::env::set_current_dir(&cva6_dir).expect("Unable to change into cva6 directory");
 
+        println!("INFO: cheking out good commit");
+
+        assert!(Command::new("git")
+            .arg("checkout")
+            .arg("b401ab3868d053a00779add51ea37cf3b8c98b21")
+            .status()
+            .unwrap()
+            .success());
+
         println!("INFO: updating submodules");
 
         assert!(Command::new("git")
@@ -46,15 +55,6 @@ fn main() {
                 .unwrap()
                 .success());
 
-        println!("INFO: cheking out good commit");
-
-        assert!(Command::new("git")
-            .arg("checkout")
-            .arg("b401ab3868d053a00779add51ea37cf3b8c98b21")
-            .status()
-            .unwrap()
-            .success());
-
         assert!(Command::new("git")
             .arg("apply")
             .arg("../cva6.patch")

fuzzers/cva6-vcs-fuzzer/config.yml

@@ -6,7 +6,7 @@ simv:
   cov_enable: true
   coverage_metrics: "line+fsm+cond+tgl+branch"
   coverage_directory: "Coverage.vdb"
-  reset_coverage_before_use: true
+  reset_coverage_before_use: false
   system_timeout_s: 180
   vcs_timeout: "30us"
   multi_seed: 0

fuzzers/cva6-vcs-fuzzer/plot.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# SPDX-FileCopyrightText: 2022 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import re
+import time
+import copy
+from datetime import datetime
+import matplotlib.pyplot as plt
+import numpy as np
+from collections import OrderedDict
+import pandas
+import seaborn as sns
+import json
+import argparse
+
+print(sns.__version__)
+
+
+def is_valid_directory(arg):
+    if not os.path.isdir(arg):
+        raise argparse.ArgumentTypeError(f"'{arg}' is not a valid directory")
+    return arg
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Parse command line arguments")
+    parser.add_argument("-d", "--directory", type=is_valid_directory, required=True, help="Input directory")
+    parser.add_argument("-m", "--metric", choices=["line", "branch", "cond", "tgl", "fsm"], required=True, help="Metric to be provided")
+    return parser.parse_args()
+
+args = parse_args()
+print("Input directory:", args.directory)
+print("Metric:", args.metric)
+
+stats_directories = [args.directory]
+metric = args.metric
+
+# first, let's load all the data
+data = []
+for stats_directory in stats_directories:
+
+    i = 0
+    delta = 0
+    for stats_filename in os.scandir(stats_directory):
+
+        f = stats_filename
+
+        if "stats" not in f.name:
+            continue
+
+
+        print(f"Analyzing file {f.name}: ")
+
+        if os.path.isfile(f):
+            f = open(f)
+
+            last_runtime = 0
+            last_coverage = 0
+
+            lines = f.readlines()
+            for l in lines:
+
+                l = json.loads(l)
+
+                if  "coverage_verdi_"+metric in l["UpdateUserStats"]["name"]:
+                    a = l["UpdateUserStats"]["value"]["value"]["Ratio"][0]
+                    b = l["UpdateUserStats"]["value"]["value"]["Ratio"][1]
+                    last_coverage = a/b
+
+                if  "time_verdi_"+metric in l["UpdateUserStats"]["name"]:
+                    last_runtime = l["UpdateUserStats"]["value"]["value"]["Number"]
+                    data += [{"runtime": last_runtime, "score": last_coverage}]
+                i += 1
+
+# let's order the timepoint
+dataset = []
+runtime = []
+coverage = []
+max_cov = 0.0
+
+time_sorted = sorted(data, key=lambda x: x['runtime'])
+
+delta = time_sorted[0]["runtime"]
+
+for item in time_sorted:
+
+    runtime += [item["runtime"]]
+
+    if max_cov < item["score"]:
+        max_cov = item["score"]
+
+    coverage += [max_cov]
+
+dataset = {"Execution Time": runtime, "Score": coverage}
+print(dataset)
+ax = sns.lineplot(x=dataset["Execution Time"], y=dataset["Score"], legend="full")
+
+plt.title(f"{metric} coverage score over time.")
+plt.legend(loc='upper center')
+plt.savefig(f'{metric}.png')

fuzzers/cva6-vcs-fuzzer/run.sh

@@ -1 +1 @@
-./simv +vcs+finish+20us -cm line+fsm+cond+tgl+branch -cm_dir Coverage.vdb  +permissive +elf_file=./testcase.elf ++./testcase.elf +debug_disable=1 +ntb_random_seed=1 -sv_lib /home/sdp/riscv/lib/libfesvr  +ntb_random_seed=28072
+./simv +vcs+finish+20us -cm line+fsm+cond+tgl+branch -cm_dir Coverage.vdb  +permissive +elf_file=./testcase.elf ++./testcase.elf +debug_disable=1 +ntb_random_seed=1 -sv_lib $RISCV/lib/libfesvr  +ntb_random_seed=28072

fuzzers/cva6-vcs-fuzzer/seeds/seed.bin

@@ -0,0 +1 @@
+��l��(}���
�#�69����I	��&W[��NÔr0	$绿��9e��x��A�dqJ��y��w

fuzzers/cva6-vcs-fuzzer/src/differential_feedback.rs

@@ -29,7 +29,6 @@ extern crate fs_extra;
 #[cfg(feature = "debug")]
 use color_print::cprintln;
 
-use libpresifuzz_observers::xtrace_observer::XTraceObserver;
 use std::fmt::Write;
 use std::{
     fs,
@@ -63,44 +62,37 @@ where
         EM: EventFirer<State = S>,
         OT: ObserversTuple<S>,
     {
-        return Ok(false);
-        let mut interesting = false;
 
-        #[cfg(feature = "input_injection")]
-        {
-            #[cfg(feature = "debug")]
-            cprintln!("<red>[WARNING]</red> Skipping trace comparison feedback because it is not supported when input_injection is enabled ...");
-            return Ok(false);
-        }
+        #[cfg(feature = "debug")]
+        cprintln!("<red>[WARNING]</red> Skipping trace comparison feedback because it is not implemented for cva6 ...");
+        return Ok(false);
 
-        let observer = observers.match_name::<XTraceObserver>(self.first_name.as_str()).unwrap();
+        //TODO: implement differential testing for cva6 + Spike
+        //let observer = observers.match_name::<XTraceObserver>(self.first_name.as_str()).unwrap();
 
-        let mut output_arg = String::new();
-        write!(output_arg, "--ofile=backup_{}.log", self.counter).expect("Unable to backup compare.pl log");
+        //let mut output_arg = String::new();
+        //write!(output_arg, "--ofile=backup_{}.log", self.counter).expect("Unable to backup compare.pl log");
 
-        if observer.mismatch == true {
-
-            interesting = true;
-
-            let dst_file = format!("testcase_state_{}.log", self.counter); 
-            fs::copy(observer.logfile.as_str(), dst_file).expect("Unable to create copy of log file");
+        //if observer.mismatch == true {
+        //    
+        //    interesting = true;
+        //    
+        //    let dst_file = format!("testcase_state_{}.log", self.counter); 
+        //    fs::copy(observer.logfile.as_str(), dst_file).expect("Unable to create copy of log file");
 
-            self.counter += 1;
-
-            let dst_file = format!("testcase.elf_spike_{}.log", self.counter); 
-            fs::copy("testcase.elf_spike.log", dst_file).expect("Unable to create copy of log file");
-
-            let dst_file = format!("testcase_{}.elf", self.counter); 
-            fs::copy("testcase.elf", dst_file).expect("Unable to create copy of log file");
-
-            // let dst_file = format!("simv_{}.log", self.counter);
-            // fs::copy("simv_0.log", dst_file);
-        }
+        //    self.counter += 1;
+        //    
+        //    let dst_file = format!("testcase.elf_spike_{}.log", self.counter); 
+        //    fs::copy("testcase.elf_spike.log", dst_file).expect("Unable to create copy of log file");
+        //    
+        //    let dst_file = format!("testcase_{}.elf", self.counter); 
+        //    fs::copy("testcase.elf", dst_file).expect("Unable to create copy of log file");
+        //}
 
-        let _ = std::fs::remove_file("testcase_state.log");
-        let _ = std::fs::remove_file("testcase.elf_spike.log");
+        //let _ = std::fs::remove_file("testcase_state.log");
+        //let _ = std::fs::remove_file("testcase.elf_spike.log");
 
-        return Ok(interesting);
+        //return Ok(interesting);
     }
 
     #[inline]