Spike Tandem Implementation using VCS simulator (#1561)

.github/workflows/ci.yml

@@ -7,20 +7,50 @@ name: ci
 on: [push, pull_request]
 
 jobs:
-  riscv-tests:
-    name: riscv-tests
+  build-riscv-tests:
+    name: build-riscv-tests
+    runs-on: ubuntu-latest
+    env:
+      RISCV: /riscv
+      NUM_JOBS: 4
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+    - name: Prepare
+      run: |
+          ci/setup.sh
+          tar -cf tools.tar tools
+          tar -cf tmp.tar tmp
+    - name: Archive production artifacts
+      uses: actions/upload-artifact@v3
+      with:
+        name: compiled-tools
+        path: |
+          tools.tar
+          tmp.tar
+  execute-riscv-tests:
+    name: execute-riscv-tests
     runs-on: ubuntu-latest
     strategy:
       matrix:
         testcase: [asm-tests, mul, amo, fp, benchmarks]
         target: [cv64a6_imafdc_sv39, cv64a6_imafdc_sv39_wb]
     env:
       RISCV: /riscv
+    needs:
+      build-riscv-tests
     steps:
     - uses: actions/checkout@v2
       with:
         submodules: recursive
-    - name: Prepare
-      run: ci/setup.sh
-    - name: run tests
-      run: make run-${{ matrix.testcase}}-verilator target=${{ matrix.target }}
+    - name: Download a single artifact
+      uses: actions/download-artifact@v3
+      with:
+        name: compiled-tools
+    - name: Run Tests
+      run: |
+          tar xf tools.tar
+          tar xf tmp.tar
+          source verif/regress/install-cva6.sh
+          make run-${{ matrix.testcase}}-verilator target=${{ matrix.target }}

.gitlab-ci.yml

@@ -139,6 +139,24 @@ smoke:
     - source verif/regress/smoke-tests.sh
     - !reference [.simu_after_script]
 
+smoke-tandem:
+  extends:
+    - .fe_smoke_test
+  variables:
+    DASHBOARD_JOB_TITLE: "Smoke test $DV_SIMULATORS with tandem"
+    DASHBOARD_JOB_DESCRIPTION: "Short tests to challenge most architectures with most testbenchs configurations"
+    DASHBOARD_SORT_INDEX: 0
+    DASHBOARD_JOB_CATEGORY: "Basic"
+    SPIKE_TANDEM: 1
+  parallel:
+    matrix:
+      - DV_SIMULATORS:
+        - "vcs-testharness,spike"
+        - "vcs-uvm,spike"
+  script:
+    - source verif/regress/smoke-tests.sh
+    - !reference [.simu_after_script]
+
 gen_smoke:
   extends:
     - .fe_smoke_test

Makefile

@@ -25,17 +25,18 @@ VCOM ?= vcom$(questa_version)
 VLIB ?= vlib$(questa_version)
 VMAP ?= vmap$(questa_version)
 # verilator version
-verilator      ?= $(PWD)/tmp/verilator-v5.008/verilator/bin/verilator
+VERILATOR_INSTALL_DIR ?= $(PWD)/tmp/verilator-v5.008/verilator/
+verilator             ?= verilator
 # traget option
 target-options ?=
-# additional definess
+# additional defines
 defines        ?=
 # test name for torture runs (binary name)
 test-location  ?= output/test
 # set to either nothing or -log
 torture-logs   :=
 # custom elf bin to run with sim or sim-verilator
-elf-bin        ?= tmp/riscv-tests/build/benchmarks/dhrystone.riscv
+elf_file        ?= tmp/riscv-tests/build/benchmarks/dhrystone.riscv
 # board name for bitstream generation. Currently supported: kc705, genesys2
 BOARD          ?= genesys2
 # root path
@@ -58,7 +59,9 @@ ifndef RISCV
 $(error RISCV not set - please point your RISCV variable to your RISCV installation)
 endif
 
-# By default assume spike resides at $(root-dir)/tools/spike prefix.
+# Spike tandem mode: default to environment setting (DISABLED if envariable SPIKE_TANDEM is not set).
+spike-tandem ?= $(SPIKE_TANDEM)
+
 SPIKE_INSTALL_DIR     ?= $(root-dir)/tools/spike
 
 # setting additional xilinx board parameters for the selected board
@@ -79,11 +82,10 @@ $(error Unknown board - please specify a supported FPGA board)
 endif
 
 # spike tandem verification
-ifdef spike-tandem
+ifneq ($(spike-tandem),)
     compile_flag += -define SPIKE_TANDEM
-    ifndef preload
-        $(error Tandem verification requires preloading)
-    endif
+    CFLAGS += -I. -I$(SPIKE_INSTALL_DIR)/include/riscv
+    defines += +SPIKE_TANDEM=1
 endif
 
 # target takes one of the following cva6 hardware configuration:
@@ -123,14 +125,15 @@ test_pkg := $(wildcard tb/test/*/*sequence_pkg.sv*) \
 dpi := $(patsubst corev_apu/tb/dpi/%.cc, ${dpi-library}/%.o, $(wildcard corev_apu/tb/dpi/*.cc))
 
 # filter spike stuff if tandem is not activated
-ifndef spike-tandem
+ifeq ($(spike-tandem),)
     dpi := $(filter-out ${dpi-library}/spike.o ${dpi-library}/sim_spike.o, $(dpi))
 endif
 
 dpi_hdr := $(wildcard corev_apu/tb/dpi/*.h)
 dpi_hdr := $(addprefix $(root-dir), $(dpi_hdr))
 CFLAGS += -I$(QUESTASIM_HOME)/include         \
           -I$(VCS_HOME)/include               \
+          -I$(VL_INC_DIR)/vltstd              \
           -I$(RISCV)/include                  \
           -I$(SPIKE_INSTALL_DIR)/include      \
           -std=c++17 -I../corev_apu/tb/dpi -O3
@@ -141,13 +144,10 @@ else
 $(warning XCELIUM_HOME not set which is necessary for compiling DPIs when using XCELIUM)
 endif
 
-ifdef spike-tandem
-    CFLAGS += -Itb/riscv-isa-sim/install/include/spike
-endif
-
-
 # this list contains the standalone components
 src :=  core/include/$(target)_config_pkg.sv                                         \
+        $(if $(spike-tandem),verif/tb/core/rvfi_pkg.sv)                              \
+        $(if $(spike-tandem),corev_apu/tb/common/spike.sv)                           \
         corev_apu/src/ariane.sv                                                      \
         $(wildcard corev_apu/bootrom/*.sv)                                           \
         $(wildcard corev_apu/clint/*.sv)                                             \
@@ -215,6 +215,7 @@ fpga_src := $(addprefix $(root-dir), $(fpga_src))
 
 # look for testbenches
 tbs := core/include/$(target)_config_pkg.sv corev_apu/tb/ariane_tb.sv corev_apu/tb/ariane_testharness.sv
+
 tbs := $(addprefix $(root-dir), $(tbs))
 
 # RISCV asm tests and benchmark setup (used for CI)
@@ -233,7 +234,9 @@ riscv-fp-tests            := $(shell xargs printf '\n%s' < $(riscv-fp-tests-list
 riscv-benchmarks          := $(shell xargs printf '\n%s' < $(riscv-benchmarks-list) | cut -b 1-)
 
 # Search here for include files (e.g.: non-standalone components)
-incdir := vendor/pulp-platform/common_cells/include/ vendor/pulp-platform/axi/include/ corev_apu/register_interface/include/
+incdir := $(CVA6_REPO_DIR)/vendor/pulp-platform/common_cells/include/ $(CVA6_REPO_DIR)/vendor/pulp-platform/axi/include/ \
+          $(CVA6_REPO_DIR)/corev_apu/register_interface/include/ $(CVA6_REPO_DIR)/corev_apu/tb/common/ \
+          $(CVA6_REPO_DIR)/vendor/pulp-platform/axi/include/ $(CVA6_REPO_DIR)/verif/core-v-verif/lib/uvm_agents/uvma_rvfi/
 
 # Compile and sim flags
 compile_flag     += +cover=bcfst+/dut -incr -64 -nologo -quiet -suppress 13262 -permissive -svinputport=compat +define+$(defines)
@@ -263,11 +266,11 @@ endif
 # we want to preload the memories
 ifdef preload
 	questa-cmd += +PRELOAD=$(preload)
-	elf-bin = none
+	elf_file = none
 endif
 
 ifdef spike-tandem
-    questa-cmd += -gblso tb/riscv-isa-sim/install/lib/libriscv.so
+    questa-cmd += -gblso $(SPIKE_INSTALL_DIR)/lib/libriscv.so
 endif
 
 # remote bitbang is enabled
@@ -280,16 +283,19 @@ endif
 vcs_build: $(dpi-library)/ariane_dpi.so
 	mkdir -p $(vcs-library)
 	cd $(vcs-library) &&\
-	vlogan $(if $(VERDI), -kdb,) -full64 -nc -sverilog +define+$(defines) -assert svaext -f ../core/Flist.cva6 &&\
+	vlogan $(if $(VERDI), -kdb,) -full64 -nc -sverilog +define+$(defines) -assert svaext -f ../core/Flist.cva6 $(list_incdir) &&\
 	vlogan $(if $(VERDI), -kdb,) -full64 -nc -sverilog +define+$(defines) $(filter %.sv,$(ariane_pkg)) +incdir+core/include/+$(VCS_HOME)/etc/uvm-1.2/dpi &&\
 	vhdlan $(if $(VERDI), -kdb,) -full64 -nc $(filter %.vhd,$(uart_src)) &&\
-	vlogan $(if $(VERDI), -kdb,) -full64 -nc -sverilog -assert svaext +define+$(defines) $(filter %.sv,$(src)) +incdir+../vendor/pulp-platform/common_cells/include/+../vendor/pulp-platform/axi/include/+../corev_apu/register_interface/include/ &&\
+	vlogan $(if $(VERDI), -kdb,) -full64 -nc -sverilog -assert svaext +define+$(defines) +incdir+$(VCS_HOME)/etc/uvm/src $(VCS_HOME)/etc/uvm/src/uvm_pkg.sv  $(filter %.sv,$(src)) $(list_incdir) &&\
 	vlogan $(if $(VERDI), -kdb,) -full64 -nc -sverilog -ntb_opts uvm-1.2 &&\
-	vlogan $(if $(VERDI), -kdb,) -full64 -nc -sverilog -ntb_opts uvm-1.2 $(tbs) +define+$(defines) +incdir+../vendor/pulp-platform/axi/include/ &&\
+	vlogan $(if $(VERDI), -kdb,) -full64 -nc -sverilog -ntb_opts uvm-1.2 $(tbs) +define+$(defines) $(list_incdir) &&\
 	vcs $(if $(VERDI), -kdb -debug_access+all -lca,) -full64 -timescale=1ns/1ns -ntb_opts uvm-1.2 work.ariane_tb -error="IWNF"
 
 vcs: vcs_build
-	cd $(vcs-library) && ./simv  $(if $(VERDI), -verdi -do $(root-dir)/util/init_testharness.do,) +permissive -sv_lib ../work-dpi/ariane_dpi +PRELOAD=$(elf-bin) +permissive-off ++$(elf-bin)| tee vcs.log
+	cd $(vcs-library) && \
+	 ./simv +permissive $(if $(VERDI), -verdi -do $(root-dir)/init_testharness.do,) \
+		+elf_file=$(elf_file) ++$(elf_file) $(if $(spike-tandem),-sv_lib $(SPIKE_INSTALL_DIR)/libriscv) \
+		-sv_lib ../work-dpi/ariane_dpi | tee vcs.log
 
 # Build the TB and module using QuestaSim
 build: $(library) $(library)/.build-srcs $(library)/.build-tb $(dpi-library)/ariane_dpi.so
@@ -322,20 +328,20 @@ $(dpi-library)/%.o: corev_apu/tb/dpi/%.cc $(dpi_hdr)
 $(dpi-library)/ariane_dpi.so: $(dpi)
 	mkdir -p $(dpi-library)
 	# Compile C-code and generate .so file
-	$(CXX) -shared -m64 -o $(dpi-library)/ariane_dpi.so $? -L$(RISCV)/lib -L$(SPIKE_INSTALL_DIR)/lib -Wl,-rpath,$(RISCV)/lib -Wl,-rpath,$(SPIKE_INSTALL_DIR)/lib -lfesvr
+	$(CXX) -shared -m64 -o $(dpi-library)/ariane_dpi.so $? -L$(RISCV)/lib -L$(SPIKE_INSTALL_DIR)/lib -Wl,-rpath,$(RISCV)/lib -Wl,-rpath,$(SPIKE_INSTALL_DIR)/lib -lfesvr -lriscv
 
 # single test runs on Questa can be started by calling make <testname>, e.g. make towers.riscv
 # the test names are defined in ci/riscv-asm-tests.list, and in ci/riscv-benchmarks.list
 # if you want to run in batch mode, use make <testname> batch-mode=1
-# alternatively you can call make sim elf-bin=<path/to/elf-bin> in order to load an arbitrary binary
+# alternatively you can call make sim elf_file=<path/to/elf_file> in order to load an arbitrary binary
 generate-trace-vsim:
-	make sim preload=$(preload) elf-bin= batch-mode=1
+	make sim preload=$(preload) elf_file= batch-mode=1
 	make generate-trace
 
 sim: build
 	$(VSIM) +permissive $(questa-flags) $(questa-cmd) -lib $(library) +MAX_CYCLES=$(max_cycles) +UVM_TESTNAME=$(test_case) \
 	+BASEDIR=$(riscv-test-dir) $(uvm-flags) $(QUESTASIM_FLAGS) -gblso $(SPIKE_INSTALL_DIR)/lib/libfesvr.so -sv_lib $(dpi-library)/ariane_dpi  \
-	${top_level}_optimized +permissive-off ++$(elf-bin) ++$(target-options) | tee sim.log
+	${top_level}_optimized +permissive-off ++$(elf_file) ++$(target-options) | tee sim.log
 
 $(riscv-asm-tests): build
 	$(VSIM) +permissive $(questa-flags) $(questa-cmd) -lib $(library) +max-cycles=$(max_cycles) +UVM_TESTNAME=$(test_case) \
@@ -463,7 +469,7 @@ xrun_sim: xrun_comp
 		+UVM_TESTNAME=$(test_case)	\
 		-l $(XRUN_RUN_LOG)		\
 		+permissive-off			\
-		++$(elf-bin)
+		++$(elf_file)
 
 #-e "set_severity_pack_assert_off {warning}; set_pack_assert_off {numeric_std}" TODO: This will remove assertion warning at the beginning of the simulation.
 
@@ -561,24 +567,24 @@ verilate_command := $(verilator) --no-timing verilator_config.vlt
                     $(if $(DEBUG), --trace-structs,)                                                             \
                     $(if $(TRACE_COMPACT), --trace-fst $(VL_INC_DIR)/verilated_fst_c.cpp)                        \
                     $(if $(TRACE_FAST), --trace $(VL_INC_DIR)/verilated_vcd_c.cpp)                               \
-                    -LDFLAGS "-L$(RISCV)/lib -L$(SPIKE_INSTALL_DIR)/lib -Wl,-rpath,$(RISCV)/lib -Wl,-rpath,$(SPIKE_INSTALL_DIR)/lib -lfesvr$(if $(PROFILE), -g -pg,) -lpthread $(if $(TRACE_COMPACT), -lz,)" \
+                    -LDFLAGS "-L$(RISCV)/lib -L$(SPIKE_INSTALL_DIR)/lib -Wl,-rpath,$(RISCV)/lib -Wl,-rpath,$(SPIKE_INSTALL_DIR)/lib -lfesvr -lriscv $(if $(PROFILE), -g -pg,) -lpthread $(if $(TRACE_COMPACT), -lz,)" \
                     -CFLAGS "$(CFLAGS)$(if $(PROFILE), -g -pg,) -DVL_DEBUG"                                      \
+                    $(if $(SPIKE_TANDEM), +define+SPIKE_TANDEM, )                                                \
                     --cc  --vpi                                                                                  \
                     $(list_incdir) --top-module ariane_testharness                                               \
                     --threads-dpi none                                                                           \
                     --Mdir $(ver-library) -O3                                                                    \
                     --exe corev_apu/tb/ariane_tb.cpp corev_apu/tb/dpi/SimDTM.cc corev_apu/tb/dpi/SimJTAG.cc      \
                     corev_apu/tb/dpi/remote_bitbang.cc corev_apu/tb/dpi/msim_helper.cc
 
-
 # User Verilator, at some point in the future this will be auto-generated
 verilate:
 	@echo "[Verilator] Building Model$(if $(PROFILE), for Profiling,)"
 	$(verilate_command)
 	cd $(ver-library) && $(MAKE) -j${NUM_JOBS} -f Variane_testharness.mk
 
 sim-verilator: verilate
-	$(ver-library)/Variane_testharness $(elf-bin)
+	$(ver-library)/Variane_testharness $(elf_file)
 
 $(addsuffix -verilator,$(riscv-asm-tests)): verilate
 	$(ver-library)/Variane_testharness $(riscv-test-dir)/$(subst -verilator,,$@)

ci/setup.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 set -e
-ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+export ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+export ROOT_PROJECT=$ROOT
 
 export PATH=$RISCV/bin:/bin:$PATH
 export LIBRARY_PATH=$RISCV/lib
@@ -15,13 +16,14 @@ sudo apt install device-tree-compiler
 
 ci/make-tmp.sh
 
-ci/install-verilator.sh
-
 sudo mkdir -p $RISCV && sudo chmod 777 $RISCV
 RISCV64_UNKNOWN_ELF_GCC=riscv64-unknown-elf-gcc-8.3.0-2020.04.0-x86_64-linux-ubuntu14.tar.gz
 if [ ! -f "$RISCV64_UNKNOWN_ELF_GCC" ]; then
   wget https://static.dev.sifive.com/dev-tools/$RISCV64_UNKNOWN_ELF_GCC
 fi
 tar -x -f $RISCV64_UNKNOWN_ELF_GCC --strip-components=1 -C $RISCV
-ci/install-fesvr.sh
+
+sudo apt install libfl-dev help2man
+
+verif/regress/install-cva6.sh
 ci/build-riscv-tests.sh

core/csr_regfile.sv

@@ -562,7 +562,7 @@ module csr_regfile
       end
       instret_d = instret;
       // increment the cycle count
-      if (ENABLE_CYCLE_COUNT && !mcountinhibit_q[0]) cycle_d = cycle_q + 1'b1;
+      if (!mcountinhibit_q[0]) cycle_d = cycle_q + 1'b1;
       else cycle_d = instret;
     end
 

core/cva6.sv

@@ -1348,27 +1348,15 @@ module cva6
   if (IsRVFI) begin
     always_comb begin
       for (int i = 0; i < CVA6ExtendCfg.NrCommitPorts; i++) begin
-        logic exception, mem_exception;
+        logic exception;
         exception = commit_instr_id_commit[i].valid && ex_commit.valid;
-        mem_exception = exception &&
-          (ex_commit.cause == riscv::INSTR_ADDR_MISALIGNED ||
-           ex_commit.cause == riscv::INSTR_ACCESS_FAULT ||
-           ex_commit.cause == riscv::ILLEGAL_INSTR ||
-           ex_commit.cause == riscv::LD_ADDR_MISALIGNED ||
-           ex_commit.cause == riscv::LD_ACCESS_FAULT ||
-           ex_commit.cause == riscv::ST_ADDR_MISALIGNED ||
-           ex_commit.cause == riscv::ST_ACCESS_FAULT ||
-           ex_commit.cause == riscv::INSTR_PAGE_FAULT ||
-           ex_commit.cause == riscv::LOAD_PAGE_FAULT ||
-           ex_commit.cause == riscv::STORE_PAGE_FAULT);
-        // when rvfi_valid, the instruction is executed
         rvfi_o[i].valid    = (commit_ack[i] && !ex_commit.valid) ||
           (exception && (ex_commit.cause == riscv::ENV_CALL_MMODE ||
                     ex_commit.cause == riscv::ENV_CALL_SMODE ||
                     ex_commit.cause == riscv::ENV_CALL_UMODE));
-        rvfi_o[i].insn     = ex_commit.valid ? ex_commit.tval[31:0] : commit_instr_id_commit[i].ex.tval[31:0];
+        rvfi_o[i].insn = ex_commit.valid ? ex_commit.tval[31:0] : commit_instr_id_commit[i].ex.tval[31:0];
         // when trap, the instruction is not executed
-        rvfi_o[i].trap = mem_exception;
+        rvfi_o[i].trap = exception;
         rvfi_o[i].cause = ex_commit.cause;
         rvfi_o[i].mode = (CVA6Cfg.DebugEn && debug_mode) ? 2'b10 : priv_lvl;
         rvfi_o[i].ixl = riscv::XLEN == 64 ? 2 : 1;

core/decoder.sv

@@ -183,7 +183,7 @@ module decoder
                 end
                 // WFI
                 12'b1_0000_0101: begin
-                  if (ENABLE_WFI) instruction_o.op = ariane_pkg::WFI;
+                  instruction_o.op = ariane_pkg::WFI;
                   // if timeout wait is set, trap on an illegal instruction in S Mode
                   // (after 0 cycles timeout)
                   if (CVA6Cfg.RVS && priv_lvl_i == riscv::PRIV_LVL_S && tw_i) begin

core/include/ariane_pkg.sv

@@ -107,22 +107,17 @@ package ariane_pkg;
   // enables a commit log which matches spikes commit log format for easier trace comparison
   localparam bit ENABLE_SPIKE_COMMIT_LOG = 1'b1;
 
-  // ------------- Dangerouse -------------
+  // ------------- Dangerous -------------
   // if set to zero a flush will not invalidate the cache-lines, in a single core environment
   // where coherence is not necessary this can improve performance. This needs to be switched on
   // when more than one core is in a system
   localparam logic INVALIDATE_ON_FLUSH = 1'b1;
+
 `ifdef SPIKE_TANDEM
-  // enable performance cycle counter, if set to zero mcycle will be incremented
-  // with instret (non RISC-V conformal)
-  localparam bit ENABLE_CYCLE_COUNT = 1'b0;
-  // mark WIF as nop
-  localparam bit ENABLE_WFI = 1'b0;
-  // Spike zeros tval on all exception except memory faults
+  // Spike still places 0 in TVAL for ENV_CALL_* exceptions.
+  // This may eventually go away when Spike starts to handle TVAL for *all* exceptions.
   localparam bit ZERO_TVAL = 1'b1;
 `else
-  localparam bit ENABLE_CYCLE_COUNT = 1'b1;
-  localparam bit ENABLE_WFI = 1'b1;
   localparam bit ZERO_TVAL = 1'b0;
 `endif
   // read mask for SSTATUS over MMSTATUS