diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aba60ba391..c9705d5e87 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -149,16 +149,17 @@ jobs: --evm-backend ${{ matrix.evm-backend || 'revm' }} ${{ matrix.debug && '--enable-compiler-debug-mode' || '' }} ${{ matrix.experimental-codegen && '--experimental-codegen' || '' }} - --cov-branch - --cov-report xml:coverage.xml + --cov-config=setup.cfg --cov=vyper tests/ - - name: Upload Coverage - uses: codecov/codecov-action@v5 + - name: Upload coverage artifact + uses: actions/upload-artifact@v4 with: - token: ${{ secrets.CODECOV_TOKEN }} - file: ./coverage.xml + name: coverage-files-${{ github.job }}-${{ strategy.job-index }} + include-hidden-files: true + path: .coverage + if-no-files-found: error core-tests-success: if: always() @@ -209,16 +210,17 @@ jobs: --splits 120 \ --group ${{ matrix.group }} \ --splitting-algorithm least_duration \ - --cov-branch \ - --cov-report xml:coverage.xml \ + --cov-config=setup.cfg \ --cov=vyper \ tests/ - - name: Upload Coverage - uses: codecov/codecov-action@v5 + - name: Upload coverage artifact + uses: actions/upload-artifact@v4 with: - token: ${{ secrets.CODECOV_TOKEN }} - file: ./coverage.xml + name: coverage-files-${{ github.job }}-${{ strategy.job-index }} + include-hidden-files: true + path: .coverage + if-no-files-found: error slow-tests-success: if: always() @@ -231,3 +233,38 @@ jobs: - name: Check slow tests all succeeded if: ${{ needs.fuzzing.result != 'success' }} run: exit 1 + + consolidate-coverage: + # Consolidate code coverage using `coverage combine` and upload + # to the codecov app + runs-on: ubuntu-latest + needs: [tests, fuzzing] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: "pip" + + - name: Install coverage + run: pip install coverage + + - name: Download coverage artifacts + uses: actions/download-artifact@v4 + with: + pattern: coverage-files-* + path: coverage-files + + - name: Combine coverage + run: | + coverage combine coverage-files/**/.coverage + coverage xml + + - name: Upload Coverage + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: coverage.xml diff --git a/README.md b/README.md index 84c2948ceb..827d40d549 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ be a bit behind the latest version found in the master branch of this repository ```bash make dev-init -python setup.py test +./quicktest.sh -m "not fuzzing" ``` ## Developing (working on the compiler) diff --git a/quicktest.sh b/quicktest.sh index cd3aad1f15..af928f5d7c 100755 --- a/quicktest.sh +++ b/quicktest.sh @@ -2,8 +2,17 @@ # examples: # ./quicktest.sh +# ./quicktest.sh -m "not fuzzing" +# ./quicktest.sh -m "not fuzzing" -n (this is the most useful) +# ./quicktest.sh -m "not fuzzing" -n0 # ./quicktest.sh tests/.../mytest.py # run pytest but bail out on first error -# useful for dev workflow +# useful for dev workflow. + pytest -q -s --instafail -x --disable-warnings "$@" + +# useful options include: +# -n0 (uses only one core but faster startup) +# -nauto (uses only one core but faster startup) +# -m "not fuzzing" - skip slow/fuzzing tests diff --git a/setup.cfg b/setup.cfg index 5998961ee8..4cce85034d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,3 +33,15 @@ markers = fuzzing: Run Hypothesis fuzz test suite (deselect with '-m "not fuzzing"') requires_evm_version(version): Mark tests that require at least a specific EVM version and would throw `EvmVersionException` otherwise venom_xfail: mark a test case as a regression (expected to fail) under the venom pipeline + + +[coverage:run] +branch = True +source = vyper + +# this is not available on the CI step that performs `coverage combine` +omit = vyper/version.py + +# allow `coverage combine` to combine reports from heterogeneous OSes. +# (mainly important for consolidating coverage reports in the CI). +relative_files = True diff --git a/setup.py b/setup.py index 5b1ae1b81a..e6d4c5763d 100644 --- a/setup.py +++ b/setup.py @@ -98,8 +98,7 @@ def _global_version(version): "importlib-metadata", "wheel", ], - setup_requires=["pytest-runner", "setuptools_scm>=7.1.0,<8.0.0"], - tests_require=extras_require["test"], + setup_requires=["setuptools_scm>=7.1.0,<8.0.0"], extras_require=extras_require, entry_points={ "console_scripts": [ diff --git a/tests/functional/builtins/codegen/test_ecrecover.py b/tests/functional/builtins/codegen/test_ecrecover.py index 8db51fdd07..47a225068d 100644 --- a/tests/functional/builtins/codegen/test_ecrecover.py +++ b/tests/functional/builtins/codegen/test_ecrecover.py @@ -1,7 +1,10 @@ +import contextlib + from eth_account import Account from eth_account._utils.signing import to_bytes32 -from tests.utils import ZERO_ADDRESS +from tests.utils import ZERO_ADDRESS, check_precompile_asserts +from vyper.compiler.settings import OptimizationLevel def test_ecrecover_test(get_contract): @@ -86,3 +89,40 @@ def test_ecrecover() -> bool: """ c = get_contract(code) assert c.test_ecrecover() is True + + +def test_ecrecover_oog_handling(env, get_contract, tx_failed, optimize, experimental_codegen): + # GHSA-vgf2-gvx8-xwc3 + code = """ +@external +@view +def do_ecrecover(hash: bytes32, v: uint256, r:uint256, s:uint256) -> address: + return ecrecover(hash, v, r, s) + """ + check_precompile_asserts(code) + + c = get_contract(code) + + h = b"\x35" * 32 + local_account = Account.from_key(b"\x46" * 32) + sig = local_account.signHash(h) + v, r, s = sig.v, sig.r, sig.s + + assert c.do_ecrecover(h, v, r, s) == local_account.address + + gas_used = env.last_result.gas_used + + if optimize == OptimizationLevel.NONE and not experimental_codegen: + # if optimizations are off, enough gas is used by the contract + # that the gas provided to ecrecover (63/64ths rule) is enough + # for it to succeed + ctx = contextlib.nullcontext + else: + # in other cases, the gas forwarded is small enough for ecrecover + # to fail with oog, which we handle by reverting. + ctx = tx_failed + + with ctx(): + # provide enough spare gas for the top-level call to not oog but + # not enough for ecrecover to succeed + c.do_ecrecover(h, v, r, s, gas=gas_used) diff --git a/tests/functional/builtins/codegen/test_slice.py b/tests/functional/builtins/codegen/test_slice.py index d5d1efca0f..3f2ce44e1a 100644 --- a/tests/functional/builtins/codegen/test_slice.py +++ b/tests/functional/builtins/codegen/test_slice.py @@ -5,7 +5,12 @@ from vyper.compiler import compile_code from vyper.compiler.settings import OptimizationLevel, Settings from vyper.evm.opcodes import version_check -from vyper.exceptions import ArgumentException, CompilerPanic, TypeMismatch +from vyper.exceptions import ( + ArgumentException, + CompilerPanic, + StaticAssertionException, + TypeMismatch, +) _fun_bytes32_bounds = [(0, 32), (3, 29), (27, 5), (0, 5), (5, 3), (30, 2)] @@ -533,9 +538,15 @@ def do_slice(): @pytest.mark.parametrize("bad_code", oob_fail_list) def test_slice_buffer_oob_reverts(bad_code, get_contract, tx_failed): - c = get_contract(bad_code) - with tx_failed(): - c.do_slice() + try: + c = get_contract(bad_code) + with tx_failed(): + c.do_slice() + except StaticAssertionException: + # it should be ok if we + # catch the assert in compile time + # since it supposed to be revert + pass # tests all 3 adhoc locations: `msg.data`, `self.code`, `
.code` diff --git a/tests/functional/codegen/features/test_clampers.py b/tests/functional/codegen/features/test_clampers.py index b82a771962..2b015a1cce 100644 --- a/tests/functional/codegen/features/test_clampers.py +++ b/tests/functional/codegen/features/test_clampers.py @@ -5,7 +5,6 @@ from eth_utils import keccak from tests.utils import ZERO_ADDRESS, decimal_to_int -from vyper.exceptions import StackTooDeep from vyper.utils import int_bounds @@ -502,7 +501,6 @@ def foo(b: DynArray[int128, 10]) -> DynArray[int128, 10]: @pytest.mark.parametrize("value", [0, 1, -1, 2**127 - 1, -(2**127)]) -@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression") def test_multidimension_dynarray_clamper_passing(get_contract, value): code = """ @external diff --git a/tests/functional/codegen/types/test_dynamic_array.py b/tests/functional/codegen/types/test_dynamic_array.py index 2f647ac38c..e35bec9dbc 100644 --- a/tests/functional/codegen/types/test_dynamic_array.py +++ b/tests/functional/codegen/types/test_dynamic_array.py @@ -1,10 +1,12 @@ +import contextlib import itertools from typing import Any, Callable import pytest -from tests.utils import decimal_to_int +from tests.utils import check_precompile_asserts, decimal_to_int from vyper.compiler import compile_code +from vyper.evm.opcodes import version_check from vyper.exceptions import ( ArgumentException, ArrayIndexException, @@ -12,6 +14,7 @@ ImmutableViolation, OverflowException, StateAccessViolation, + StaticAssertionException, TypeMismatch, ) @@ -1861,9 +1864,16 @@ def should_revert() -> DynArray[String[65], 2]: @pytest.mark.parametrize("code", dynarray_length_no_clobber_cases) def test_dynarray_length_no_clobber(get_contract, tx_failed, code): # check that length is not clobbered before dynarray data copy happens - c = get_contract(code) - with tx_failed(): - c.should_revert() + try: + c = get_contract(code) + with tx_failed(): + c.should_revert() + except StaticAssertionException: + # this test should create + # assert error so if it is + # detected in compile time + # we can continue + pass def test_dynarray_make_setter_overlap(get_contract): @@ -1901,3 +1911,59 @@ def foo(): c = get_contract(code) with tx_failed(): c.foo() + + +def test_dynarray_copy_oog(env, get_contract, tx_failed): + # GHSA-vgf2-gvx8-xwc3 + code = """ + +@external +def foo(a: DynArray[uint256, 4000]) -> uint256: + b: DynArray[uint256, 4000] = a + return b[0] + """ + check_precompile_asserts(code) + + c = get_contract(code) + dynarray = [2] * 4000 + assert c.foo(dynarray) == 2 + + gas_used = env.last_result.gas_used + if version_check(begin="cancun"): + ctx = contextlib.nullcontext + else: + ctx = tx_failed + + with ctx(): + # depends on EVM version. pre-cancun, will revert due to checking + # success flag from identity precompile. + c.foo(dynarray, gas=gas_used) + + +def test_dynarray_copy_oog2(env, get_contract, tx_failed): + # GHSA-vgf2-gvx8-xwc3 + code = """ +@external +@view +def foo(x: String[1000000], y: String[1000000]) -> DynArray[String[1000000], 2]: + z: DynArray[String[1000000], 2] = [x, y] + # Some code + return z + """ + check_precompile_asserts(code) + + c = get_contract(code) + calldata0 = "a" * 10 + calldata1 = "b" * 1000000 + assert c.foo(calldata0, calldata1) == [calldata0, calldata1] + + gas_used = env.last_result.gas_used + if version_check(begin="cancun"): + ctx = contextlib.nullcontext + else: + ctx = tx_failed + + with ctx(): + # depends on EVM version. pre-cancun, will revert due to checking + # success flag from identity precompile. + c.foo(calldata0, calldata1, gas=gas_used) diff --git a/tests/functional/codegen/types/test_lists.py b/tests/functional/codegen/types/test_lists.py index 953a9a9f9f..26cd16ed32 100644 --- a/tests/functional/codegen/types/test_lists.py +++ b/tests/functional/codegen/types/test_lists.py @@ -1,8 +1,12 @@ +import contextlib import itertools import pytest -from tests.utils import decimal_to_int +from tests.evm_backends.base_env import EvmError +from tests.utils import check_precompile_asserts, decimal_to_int +from vyper.compiler.settings import OptimizationLevel +from vyper.evm.opcodes import version_check from vyper.exceptions import ArrayIndexException, OverflowException, TypeMismatch @@ -848,3 +852,73 @@ def foo() -> {return_type}: return MY_CONSTANT[0][0] """ assert_compile_failed(lambda: get_contract(code), TypeMismatch) + + +def test_array_copy_oog(env, get_contract, tx_failed, optimize, experimental_codegen, request): + # GHSA-vgf2-gvx8-xwc3 + code = """ +@internal +def bar(x: uint256[3000]) -> uint256[3000]: + a: uint256[3000] = x + return a + +@external +def foo(x: uint256[3000]) -> uint256: + s: uint256[3000] = self.bar(x) + return s[0] + """ + check_precompile_asserts(code) + + if optimize == OptimizationLevel.NONE and not experimental_codegen: + # fails in bytecode generation due to jumpdests too large + with pytest.raises(AssertionError): + get_contract(code) + return + + c = get_contract(code) + array = [2] * 3000 + assert c.foo(array) == array[0] + + # get the minimum gas for the contract complete execution + gas_used = env.last_result.gas_used + if version_check(begin="cancun"): + ctx = contextlib.nullcontext + else: + ctx = tx_failed + with ctx(): + # depends on EVM version. pre-cancun, will revert due to checking + # success flag from identity precompile. + c.foo(array, gas=gas_used) + + +def test_array_copy_oog2(env, get_contract, tx_failed, optimize, experimental_codegen, request): + # GHSA-vgf2-gvx8-xwc3 + code = """ +@external +def foo(x: uint256[2500]) -> uint256: + s: uint256[2500] = x + t: uint256[2500] = s + return t[0] + """ + check_precompile_asserts(code) + + if optimize == OptimizationLevel.NONE and not experimental_codegen: + # fails in creating contract due to code too large + with tx_failed(EvmError): + get_contract(code) + return + + c = get_contract(code) + array = [2] * 2500 + assert c.foo(array) == array[0] + + # get the minimum gas for the contract complete execution + gas_used = env.last_result.gas_used + if version_check(begin="cancun"): + ctx = contextlib.nullcontext + else: + ctx = tx_failed + with ctx(): + # depends on EVM version. pre-cancun, will revert due to checking + # success flag from identity precompile. + c.foo(array, gas=gas_used) diff --git a/tests/functional/codegen/types/test_string.py b/tests/functional/codegen/types/test_string.py index 1c186eeb6e..b4e6919ea7 100644 --- a/tests/functional/codegen/types/test_string.py +++ b/tests/functional/codegen/types/test_string.py @@ -1,5 +1,10 @@ +import contextlib + import pytest +from tests.utils import check_precompile_asserts +from vyper.evm.opcodes import version_check + def test_string_return(get_contract): code = """ @@ -359,3 +364,56 @@ def compare_var_storage_not_equal_false() -> bool: assert c.compare_var_storage_equal_false() is False assert c.compare_var_storage_not_equal_true() is True assert c.compare_var_storage_not_equal_false() is False + + +def test_string_copy_oog(env, get_contract, tx_failed): + # GHSA-vgf2-gvx8-xwc3 + code = """ +@external +@view +def foo(x: String[1000000]) -> String[1000000]: + return x + """ + check_precompile_asserts(code) + + c = get_contract(code) + calldata = "a" * 1000000 + assert c.foo(calldata) == calldata + + gas_used = env.last_result.gas_used + if version_check(begin="cancun"): + ctx = contextlib.nullcontext + else: + ctx = tx_failed + + with ctx(): + # depends on EVM version. pre-cancun, will revert due to checking + # success flag from identity precompile. + c.foo(calldata, gas=gas_used) + + +def test_string_copy_oog2(env, get_contract, tx_failed): + # GHSA-vgf2-gvx8-xwc3 + code = """ +@external +@view +def foo(x: String[1000000]) -> uint256: + y: String[1000000] = x + return len(y) + """ + check_precompile_asserts(code) + + c = get_contract(code) + calldata = "a" * 1000000 + assert c.foo(calldata) == len(calldata) + + gas_used = env.last_result.gas_used + if version_check(begin="cancun"): + ctx = contextlib.nullcontext + else: + ctx = tx_failed + + with ctx(): + # depends on EVM version. pre-cancun, will revert due to checking + # success flag from identity precompile. + c.foo(calldata, gas=gas_used) diff --git a/tests/functional/syntax/test_unbalanced_return.py b/tests/functional/syntax/test_unbalanced_return.py index 04835bb0f0..a1faa1c6a5 100644 --- a/tests/functional/syntax/test_unbalanced_return.py +++ b/tests/functional/syntax/test_unbalanced_return.py @@ -195,7 +195,7 @@ def test() -> int128: if 1 == 1 : return 1 else: - assert msg.sender != msg.sender + assert msg.sender != self return 0 """, """ diff --git a/tests/unit/compiler/venom/test_algebraic_binopt.py b/tests/unit/compiler/venom/test_algebraic_binopt.py new file mode 100644 index 0000000000..5486787225 --- /dev/null +++ b/tests/unit/compiler/venom/test_algebraic_binopt.py @@ -0,0 +1,584 @@ +import pytest + +from tests.venom_utils import assert_ctx_eq, parse_from_basic_block +from vyper.venom.analysis import IRAnalysesCache +from vyper.venom.passes import AlgebraicOptimizationPass, StoreElimination + +""" +Test abstract binop+unop optimizations in algebraic optimizations pass +""" + + +def _sccp_algebraic_runner(pre, post): + ctx = parse_from_basic_block(pre) + + for fn in ctx.functions.values(): + ac = IRAnalysesCache(fn) + StoreElimination(ac, fn).run_pass() + AlgebraicOptimizationPass(ac, fn).run_pass() + StoreElimination(ac, fn).run_pass() + + assert_ctx_eq(ctx, parse_from_basic_block(post)) + + +def test_sccp_algebraic_opt_sub_xor(): + # x - x -> 0 + # x ^ x -> 0 + pre = """ + _global: + %par = param + %1 = sub %par, %par + %2 = xor %par, %par + return %1, %2 + """ + post = """ + _global: + %par = param + return 0, 0 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_zero_sub_add_xor(): + # x + 0 == x - 0 == x ^ 0 -> x + # (this cannot be done for 0 - x) + pre = """ + _global: + %par = param + %1 = sub %par, 0 + %2 = xor %par, 0 + %3 = add %par, 0 + %4 = sub 0, %par + %5 = add 0, %par + %6 = xor 0, %par + return %1, %2, %3, %4, %5, %6 + """ + post = """ + _global: + %par = param + %4 = sub 0, %par + return %par, %par, %par, %4, %par, %par + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_sub_xor_max(): + # x ^ 0xFF..FF -> not x + # -1 - x -> ~x + pre = """ + _global: + %par = param + %tmp = -1 + %1 = xor -1, %par + %2 = xor %par, -1 + + %3 = sub -1, %par + + return %1, %2, %3 + """ + post = """ + _global: + %par = param + %1 = not %par + %2 = not %par + %3 = not %par + return %1, %2, %3 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_shift(): + # x << 0 == x >> 0 == x (sar) 0 -> x + # sar is right arithmetic shift + pre = """ + _global: + %par = param + %1 = shl 0, %par + %2 = shr 0, %1 + %3 = sar 0, %2 + return %1, %2, %3 + """ + post = """ + _global: + %par = param + return %par, %par, %par + """ + + _sccp_algebraic_runner(pre, post) + + +@pytest.mark.parametrize("opcode", ("mul", "and", "div", "sdiv", "mod", "smod")) +def test_mul_by_zero(opcode): + # x * 0 == 0 * x == x % 0 == 0 % x == x // 0 == 0 // x == x & 0 == 0 & x -> 0 + pre = f""" + _global: + %par = param + %1 = {opcode} 0, %par + %2 = {opcode} %par, 0 + return %1, %2 + """ + post = """ + _global: + %par = param + return 0, 0 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_multi_neutral_elem(): + # x * 1 == 1 * x == x / 1 -> x + # checks for non comutative ops + pre = """ + _global: + %par = param + %1_1 = mul 1, %par + %1_2 = mul %par, 1 + %2_1 = div 1, %par + %2_2 = div %par, 1 + %3_1 = sdiv 1, %par + %3_2 = sdiv %par, 1 + return %1_1, %1_2, %2_1, %2_2, %3_1, %3_2 + """ + post = """ + _global: + %par = param + %2_1 = div 1, %par + %3_1 = sdiv 1, %par + return %par, %par, %2_1, %par, %3_1, %par + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_mod_zero(): + # x % 1 -> 0 + pre = """ + _global: + %par = param + %1 = mod %par, 1 + %2 = smod %par, 1 + return %1, %2 + """ + post = """ + _global: + %par = param + return 0, 0 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_and_max(): + # x & 0xFF..FF == 0xFF..FF & x -> x + max_uint256 = 2**256 - 1 + pre = f""" + _global: + %par = param + %tmp = {max_uint256} + %1 = and %par, %tmp + %2 = and %tmp, %par + return %1, %2 + """ + post = """ + _global: + %par = param + return %par, %par + """ + + _sccp_algebraic_runner(pre, post) + + +# test powers of 2 from n==2 to n==255. +# (skip 1 since there are specialized rules for n==1) +@pytest.mark.parametrize("n", range(2, 256)) +def test_sccp_algebraic_opt_mul_div_to_shifts(n): + # x * 2**n -> x << n + # x / 2**n -> x >> n + y = 2**n + pre = f""" + _global: + %par = param + %1 = mul %par, {y} + %2 = mod %par, {y} + %3 = div %par, {y} + %4 = mul {y}, %par + %5 = mod {y}, %par ; note: this is blocked! + %6 = div {y}, %par ; blocked! + return %1, %2, %3, %4, %5, %6 + """ + post = f""" + _global: + %par = param + %1 = shl {n}, %par + %2 = and {y - 1}, %par + %3 = shr {n}, %par + %4 = shl {n}, %par + %5 = mod {y}, %par + %6 = div {y}, %par + return %1, %2, %3, %4, %5, %6 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_exp(): + # x ** 0 == 0 ** x -> 1 + # x ** 1 -> x + pre = """ + _global: + %par = param + %1 = exp %par, 0 + %2 = exp 1, %par + %3 = exp 0, %par + %4 = exp %par, 1 + return %1, %2, %3, %4 + """ + post = """ + _global: + %par = param + %3 = iszero %par + return 1, 1, %3, %par + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_compare_self(): + # x < x == x > x -> 0 + pre = """ + _global: + %par = param + %tmp = %par + %1 = gt %tmp, %par + %2 = sgt %tmp, %par + %3 = lt %tmp, %par + %4 = slt %tmp, %par + return %1, %2, %3, %4 + """ + post = """ + _global: + %par = param + return 0, 0, 0, 0 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_or(): + # x | 0 -> x + # x | 0xFF..FF -> 0xFF..FF + max_uint256 = 2**256 - 1 + pre = f""" + _global: + %par = param + %1 = or %par, 0 + %2 = or %par, {max_uint256} + %3 = or 0, %par + %4 = or {max_uint256}, %par + return %1, %2, %3, %4 + """ + post = f""" + _global: + %par = param + return %par, {max_uint256}, %par, {max_uint256} + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_eq(): + # (x == 0) == (0 == x) -> iszero x + # x == x -> 1 + # x == 0xFFFF..FF -> iszero(not x) + pre = """ + global: + %par = param + %1 = eq %par, 0 + %2 = eq 0, %par + + %3 = eq %par, -1 + %4 = eq -1, %par + + %5 = eq %par, %par + return %1, %2, %3, %4, %5 + """ + post = """ + global: + %par = param + %1 = iszero %par + %2 = iszero %par + %6 = not %par + %3 = iszero %6 + %7 = not %par + %4 = iszero %7 + return %1, %2, %3, %4, 1 + """ + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_boolean_or(): + # x | (non zero) -> 1 if it is only used as boolean + some_nonzero = 123 + pre = f""" + _global: + %par = param + %1 = or %par, {some_nonzero} + %2 = or %par, {some_nonzero} + assert %1 + %3 = or {some_nonzero}, %par + %4 = or {some_nonzero}, %par + assert %3 + return %2, %4 + """ + post = f""" + _global: + %par = param + %2 = or {some_nonzero}, %par + assert 1 + %4 = or {some_nonzero}, %par + assert 1 + return %2, %4 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_boolean_eq(): + # x == y -> iszero (x ^ y) if it is only used as boolean + pre = """ + _global: + %par = param + %par2 = param + %1 = eq %par, %par2 + %2 = eq %par, %par2 + assert %1 + return %2 + + """ + post = """ + _global: + %par = param + %par2 = param + %3 = xor %par, %par2 + %1 = iszero %3 + %2 = eq %par, %par2 + assert %1 + return %2 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_compare_never(): + # unsigned x > 0xFF..FF == x < 0 -> 0 + # signed: x > MAX_SIGNED (0x3F..FF) == x < MIN_SIGNED (0xF0..00) -> 0 + min_int256 = -(2**255) + max_int256 = 2**255 - 1 + min_uint256 = 0 + max_uint256 = 2**256 - 1 + pre = f""" + _global: + %par = param + + %1 = slt %par, {min_int256} + %2 = sgt %par, {max_int256} + %3 = lt %par, {min_uint256} + %4 = gt %par, {max_uint256} + + return %1, %2, %3, %4 + """ + post = """ + _global: + %par = param + return 0, 0, 0, 0 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_comparison_zero(): + # x > 0 => iszero(iszero x) + # 0 < x => iszero(iszero x) + pre = """ + _global: + %par = param + %1 = lt 0, %par + %2 = gt %par, 0 + return %1, %2 + """ + post = """ + _global: + %par = param + %3 = iszero %par + %1 = iszero %3 + %4 = iszero %par + %2 = iszero %4 + return %1, %2 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_comparison_almost_never(): + # unsigned: + # x < 1 => eq x 0 => iszero x + # MAX_UINT - 1 < x => eq x MAX_UINT => iszero(not x) + # signed + # x < MIN_INT + 1 => eq x MIN_INT + # MAX_INT - 1 < x => eq x MAX_INT + + max_uint256 = 2**256 - 1 + max_int256 = 2**255 - 1 + min_int256 = -(2**255) + pre1 = f""" + _global: + %par = param + %1 = lt %par, 1 + %2 = gt %par, {max_uint256 - 1} + %3 = sgt %par, {max_int256 - 1} + %4 = slt %par, {min_int256 + 1} + + return %1, %2, %3, %4 + """ + # commuted versions - produce same output + pre2 = f""" + _global: + %par = param + %1 = gt 1, %par + %2 = lt {max_uint256 - 1}, %par + %3 = slt {max_int256 - 1}, %par + %4 = sgt {min_int256 + 1}, %par + return %1, %2, %3, %4 + """ + post = f""" + _global: + %par = param + ; lt %par, 1 => eq 0, %par => iszero %par + %1 = iszero %par + ; x > MAX_UINT256 - 1 => eq MAX_UINT x => iszero(not x) + %5 = not %par + %2 = iszero %5 + %3 = eq {max_int256}, %par + %4 = eq {min_int256}, %par + return %1, %2, %3, %4 + """ + + _sccp_algebraic_runner(pre1, post) + _sccp_algebraic_runner(pre2, post) + + +def test_comparison_almost_always(): + # unsigned + # x > 0 => iszero(iszero x) + # 0 < x => iszero(iszero x) + # x < MAX_UINT => iszero(eq x MAX_UINT) => iszero(iszero(not x)) + # signed + # x < MAX_INT => iszero(eq MAX_INT) => iszero(iszero(xor MAX_INT x)) + + max_uint256 = 2**256 - 1 + max_int256 = 2**255 - 1 + min_int256 = -(2**255) + + pre1 = f""" + _global: + %par = param + %1 = gt %par, 0 + %2 = lt %par, {max_uint256} + assert %2 + %3 = slt %par, {max_int256} + assert %3 + %4 = sgt %par, {min_int256} + assert %4 + return %1 + """ + # commuted versions + pre2 = f""" + _global: + %par = param + %1 = lt 0, %par + %2 = gt {max_uint256}, %par + assert %2 + %3 = sgt {max_int256}, %par + assert %3 + %4 = slt {min_int256}, %par + assert %4 + return %1 + """ + post = f""" + _global: + %par = param + %5 = iszero %par + %1 = iszero %5 + %9 = not %par ; (eq -1 x) => (iszero (not x)) + %6 = iszero %9 + %2 = iszero %6 + assert %2 + %10 = xor %par, {max_int256} + %7 = iszero %10 + %3 = iszero %7 + assert %3 + %11 = xor %par, {min_int256} + %8 = iszero %11 + %4 = iszero %8 + assert %4 + return %1 + """ + + _sccp_algebraic_runner(pre1, post) + _sccp_algebraic_runner(pre2, post) + + +@pytest.mark.parametrize("val", (100, 2, 3, -100)) +def test_comparison_ge_le(val): + # iszero(x < 100) => 99 < x + # iszero(x > 100) => 101 > x + + up = val + 1 + down = val - 1 + + abs_val = abs(val) + abs_up = abs_val + 1 + abs_down = abs_val - 1 + + pre1 = f""" + _global: + %par = param + %1 = lt %par, {abs_val} + %3 = gt %par, {abs_val} + %2 = iszero %1 + %4 = iszero %3 + %5 = slt %par, {val} + %7 = sgt %par, {val} + %6 = iszero %5 + %8 = iszero %7 + return %2, %4, %6, %8 + """ + pre2 = f""" + _global: + %par = param + %1 = gt {abs_val}, %par + %3 = lt {abs_val}, %par + %2 = iszero %1 + %4 = iszero %3 + %5 = sgt {val}, %par + %7 = slt {val}, %par + %6 = iszero %5 + %8 = iszero %7 + return %2, %4, %6, %8 + """ + post = f""" + _global: + %par = param + %1 = lt {abs_down}, %par + %3 = gt {abs_up}, %par + %5 = slt {down}, %par + %7 = sgt {up}, %par + return %1, %3, %5, %7 + """ + + _sccp_algebraic_runner(pre1, post) + _sccp_algebraic_runner(pre2, post) diff --git a/tests/utils.py b/tests/utils.py index 8548c4f47a..b9dc443c0d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -3,6 +3,7 @@ import os from vyper import ast as vy_ast +from vyper.compiler.phases import CompilerData from vyper.semantics.analysis.constant_folding import constant_fold from vyper.utils import DECIMAL_EPSILON, round_towards_zero @@ -28,3 +29,24 @@ def parse_and_fold(source_code): def decimal_to_int(*args): s = decimal.Decimal(*args) return round_towards_zero(s / DECIMAL_EPSILON) + + +def check_precompile_asserts(source_code): + # common sanity check for some tests, that calls to precompiles + # are correctly wrapped in an assert. + + compiler_data = CompilerData(source_code) + deploy_ir = compiler_data.ir_nodes + runtime_ir = compiler_data.ir_runtime + + def _check(ir_node, parent=None): + if ir_node.value == "staticcall": + precompile_addr = ir_node.args[1] + if isinstance(precompile_addr.value, int) and precompile_addr.value < 10: + assert parent is not None and parent.value == "assert" + for arg in ir_node.args: + _check(arg, ir_node) + + _check(deploy_ir) + # technically runtime_ir is contained in deploy_ir, but check it anyways. + _check(runtime_ir) diff --git a/tests/venom_utils.py b/tests/venom_utils.py index 85298ccb87..6ddc61f615 100644 --- a/tests/venom_utils.py +++ b/tests/venom_utils.py @@ -18,9 +18,12 @@ def instructions_eq(i1: IRInstruction, i2: IRInstruction) -> bool: def assert_bb_eq(bb1: IRBasicBlock, bb2: IRBasicBlock): assert bb1.label.value == bb2.label.value - assert len(bb1.instructions) == len(bb2.instructions) for i1, i2 in zip(bb1.instructions, bb2.instructions): - assert instructions_eq(i1, i2), f"[{i1}] != [{i2}]" + assert instructions_eq(i1, i2), (bb1, f"[{i1}] != [{i2}]") + + # assert after individual instruction checks, makes it easier to debug + # if there is a difference. + assert len(bb1.instructions) == len(bb2.instructions) def assert_fn_eq(fn1: IRFunction, fn2: IRFunction): diff --git a/vyper/builtins/functions.py b/vyper/builtins/functions.py index 62539872bc..55d5443a8f 100644 --- a/vyper/builtins/functions.py +++ b/vyper/builtins/functions.py @@ -781,7 +781,7 @@ def build_IR(self, expr, args, kwargs, context): ["mstore", add_ofst(input_buf, 32), args[1]], ["mstore", add_ofst(input_buf, 64), args[2]], ["mstore", add_ofst(input_buf, 96), args[3]], - ["staticcall", "gas", 1, input_buf, 128, output_buf, 32], + ["assert", ["staticcall", "gas", 1, input_buf, 128, output_buf, 32]], ["mload", output_buf], ], typ=AddressT(), diff --git a/vyper/cli/vyper_compile.py b/vyper/cli/vyper_compile.py index 390416799a..09f8324dcf 100755 --- a/vyper/cli/vyper_compile.py +++ b/vyper/cli/vyper_compile.py @@ -5,7 +5,7 @@ import sys import warnings from pathlib import Path -from typing import Any, Iterable, Iterator, Optional, Set, TypeVar +from typing import Any, Optional import vyper import vyper.codegen.ir_node as ir_node @@ -15,8 +15,7 @@ from vyper.compiler.input_bundle import FileInput, FilesystemInputBundle from vyper.compiler.settings import VYPER_TRACEBACK_LIMIT, OptimizationLevel, Settings from vyper.typing import ContractPath, OutputFormats - -T = TypeVar("T") +from vyper.utils import uniq format_options_help = """Format to print, one or more of: bytecode (default) - Deployable bytecode @@ -263,20 +262,6 @@ def _parse_args(argv): _cli_helper(f, output_formats, compiled) -def uniq(seq: Iterable[T]) -> Iterator[T]: - """ - Yield unique items in ``seq`` in order. - """ - seen: Set[T] = set() - - for x in seq: - if x in seen: - continue - - seen.add(x) - yield x - - def exc_handler(contract_path: ContractPath, exception: Exception) -> None: print(f"Error compiling: {contract_path}") raise exception diff --git a/vyper/codegen/core.py b/vyper/codegen/core.py index 0ad7fa79c6..aaf6f35047 100644 --- a/vyper/codegen/core.py +++ b/vyper/codegen/core.py @@ -326,7 +326,7 @@ def copy_bytes(dst, src, length, length_bound): copy_op = ["mcopy", dst, src, length] gas_bound = _mcopy_gas_bound(length_bound) else: - copy_op = ["staticcall", "gas", 4, src, length, dst, length] + copy_op = ["assert", ["staticcall", "gas", 4, src, length, dst, length]] gas_bound = _identity_gas_bound(length_bound) elif src.location == CALLDATA: copy_op = ["calldatacopy", dst, src, length] diff --git a/vyper/utils.py b/vyper/utils.py index 5bebca7776..39d3093478 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -9,7 +9,7 @@ import time import traceback import warnings -from typing import Generic, List, TypeVar, Union +from typing import Generic, Iterable, Iterator, List, Set, TypeVar, Union from vyper.exceptions import CompilerPanic, DecimalOverrideException, VyperException @@ -129,6 +129,20 @@ def intersection(cls, *sets): return cls(tmp) +def uniq(seq: Iterable[_T]) -> Iterator[_T]: + """ + Yield unique items in ``seq`` in original sequence order. + """ + seen: Set[_T] = set() + + for x in seq: + if x in seen: + continue + + seen.add(x) + yield x + + class StringEnum(enum.Enum): # Must be first, or else won't work, specifies what .value is @staticmethod @@ -234,6 +248,13 @@ def int_to_fourbytes(n: int) -> bytes: return n.to_bytes(4, byteorder="big") +def wrap256(val: int, signed=False) -> int: + ret = val % (2**256) + if signed: + ret = unsigned_to_signed(ret, 256, strict=True) + return ret + + def signed_to_unsigned(int_, bits, strict=False): """ Reinterpret a signed integer with n bits as an unsigned integer. @@ -243,7 +264,7 @@ def signed_to_unsigned(int_, bits, strict=False): """ if strict: lo, hi = int_bounds(signed=True, bits=bits) - assert lo <= int_ <= hi + assert lo <= int_ <= hi, int_ if int_ < 0: return int_ + 2**bits return int_ @@ -258,7 +279,7 @@ def unsigned_to_signed(int_, bits, strict=False): """ if strict: lo, hi = int_bounds(signed=False, bits=bits) - assert lo <= int_ <= hi + assert lo <= int_ <= hi, int_ if int_ > (2 ** (bits - 1)) - 1: return int_ - (2**bits) return int_ diff --git a/vyper/venom/__init__.py b/vyper/venom/__init__.py index ddd9065194..bb3fe58a8d 100644 --- a/vyper/venom/__init__.py +++ b/vyper/venom/__init__.py @@ -56,18 +56,24 @@ def _run_passes(fn: IRFunction, optimize: OptimizationLevel) -> None: SimplifyCFGPass(ac, fn).run_pass() MakeSSA(ac, fn).run_pass() + # run algebraic opts before mem2var to reduce some pointer arithmetic + AlgebraicOptimizationPass(ac, fn).run_pass() StoreElimination(ac, fn).run_pass() Mem2Var(ac, fn).run_pass() MakeSSA(ac, fn).run_pass() SCCP(ac, fn).run_pass() + SimplifyCFGPass(ac, fn).run_pass() + StoreElimination(ac, fn).run_pass() + AlgebraicOptimizationPass(ac, fn).run_pass() LoadElimination(ac, fn).run_pass() + SCCP(ac, fn).run_pass() StoreElimination(ac, fn).run_pass() - MemMergePass(ac, fn).run_pass() + SimplifyCFGPass(ac, fn).run_pass() + MemMergePass(ac, fn).run_pass() LowerDloadPass(ac, fn).run_pass() - AlgebraicOptimizationPass(ac, fn).run_pass() # NOTE: MakeSSA is after algebraic optimization it currently produces # smaller code by adding some redundant phi nodes. This is not a # problem for us, but we need to be aware of it, and should be @@ -76,6 +82,8 @@ def _run_passes(fn: IRFunction, optimize: OptimizationLevel) -> None: # MakeSSA again. MakeSSA(ac, fn).run_pass() BranchOptimizationPass(ac, fn).run_pass() + + AlgebraicOptimizationPass(ac, fn).run_pass() RemoveUnusedVariablesPass(ac, fn).run_pass() StoreExpansionPass(ac, fn).run_pass() diff --git a/vyper/venom/analysis/dfg.py b/vyper/venom/analysis/dfg.py index a2e050094d..e528284422 100644 --- a/vyper/venom/analysis/dfg.py +++ b/vyper/venom/analysis/dfg.py @@ -30,6 +30,9 @@ def get_uses_in_bb(self, op: IRVariable, bb: IRBasicBlock): def get_producing_instruction(self, op: IRVariable) -> Optional[IRInstruction]: return self._dfg_outputs.get(op) + def set_producing_instruction(self, op: IRVariable, inst: IRInstruction): + self._dfg_outputs[op] = inst + def add_use(self, op: IRVariable, inst: IRInstruction): uses = self._dfg_inputs.setdefault(op, OrderedSet()) uses.add(inst) diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index 4c75c67700..8d86da73e7 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -4,6 +4,7 @@ import vyper.venom.effects as effects from vyper.codegen.ir_node import IRnode +from vyper.exceptions import CompilerPanic from vyper.utils import OrderedSet # instructions which can terminate a basic block @@ -92,6 +93,15 @@ from vyper.venom.function import IRFunction +def flip_comparison_opcode(opcode): + if opcode in ("gt", "sgt"): + return opcode.replace("g", "l") + elif opcode in ("lt", "slt"): + return opcode.replace("l", "g") + + raise CompilerPanic(f"unreachable {opcode}") # pragma: nocover + + class IRDebugInfo: """ IRDebugInfo represents debug information in IR, used to annotate IR @@ -318,10 +328,8 @@ def flip(self): if self.is_commutative: return - if self.opcode in ("gt", "sgt"): - self.opcode = self.opcode.replace("g", "l") - else: - self.opcode = self.opcode.replace("l", "g") + assert self.opcode in COMPARATOR_INSTRUCTIONS # sanity + self.opcode = flip_comparison_opcode(self.opcode) def replace_operands(self, replacements: dict) -> None: """ diff --git a/vyper/venom/passes/algebraic_optimization.py b/vyper/venom/passes/algebraic_optimization.py index 5d4291667e..b4f4104d5f 100644 --- a/vyper/venom/passes/algebraic_optimization.py +++ b/vyper/venom/passes/algebraic_optimization.py @@ -1,16 +1,105 @@ -from vyper.venom.analysis import DFGAnalysis, LivenessAnalysis -from vyper.venom.basicblock import IRInstruction, IRLabel, IRLiteral, IROperand +from vyper.utils import SizeLimits, int_bounds, int_log2, is_power_of_two, wrap256 +from vyper.venom.analysis.dfg import DFGAnalysis +from vyper.venom.analysis.liveness import LivenessAnalysis +from vyper.venom.basicblock import ( + COMPARATOR_INSTRUCTIONS, + IRInstruction, + IRLabel, + IRLiteral, + IROperand, + IRVariable, + flip_comparison_opcode, +) from vyper.venom.passes.base_pass import IRPass +TRUTHY_INSTRUCTIONS = ("iszero", "jnz", "assert", "assert_unreachable") + + +def lit_eq(op: IROperand, val: int) -> bool: + return isinstance(op, IRLiteral) and wrap256(op.value) == wrap256(val) + + +class InstructionUpdater: + """ + A helper class for updating instructions which also updates the + basic block and dfg in place + """ + + def __init__(self, dfg: DFGAnalysis): + self.dfg = dfg + + def _update_operands(self, inst: IRInstruction, replace_dict: dict[IROperand, IROperand]): + old_operands = inst.operands + new_operands = [replace_dict[op] if op in replace_dict else op for op in old_operands] + self._update(inst, inst.opcode, new_operands) + + def _update(self, inst: IRInstruction, opcode: str, new_operands: list[IROperand]): + assert opcode != "phi" + # sanity + assert all(isinstance(op, IROperand) for op in new_operands) + + old_operands = inst.operands + + for op in old_operands: + if not isinstance(op, IRVariable): + continue + uses = self.dfg.get_uses(op) + if inst in uses: + uses.remove(inst) + + for op in new_operands: + if isinstance(op, IRVariable): + self.dfg.add_use(op, inst) + + inst.opcode = opcode + inst.operands = new_operands + + def _store(self, inst: IRInstruction, op: IROperand): + self._update(inst, "store", [op]) + + def _add_before(self, inst: IRInstruction, opcode: str, args: list[IROperand]) -> IRVariable: + """ + Insert another instruction before the given instruction + """ + assert opcode != "phi" + index = inst.parent.instructions.index(inst) + var = inst.parent.parent.get_next_variable() + operands = list(args) + new_inst = IRInstruction(opcode, operands, output=var) + inst.parent.insert_instruction(new_inst, index) + for op in new_inst.operands: + if isinstance(op, IRVariable): + self.dfg.add_use(op, new_inst) + self.dfg.add_use(var, inst) + self.dfg.set_producing_instruction(var, new_inst) + return var + class AlgebraicOptimizationPass(IRPass): """ This pass reduces algebraic evaluatable expressions. It currently optimizes: - * iszero chains + - iszero chains + - binops + - offset adds """ + dfg: DFGAnalysis + updater: InstructionUpdater + + def run_pass(self): + self.dfg = self.analyses_cache.request_analysis(DFGAnalysis) # type: ignore + self.updater = InstructionUpdater(self.dfg) + self._handle_offset() + + self._algebraic_opt() + self._optimize_iszero_chains() + self._algebraic_opt() + + self.analyses_cache.invalidate_analysis(DFGAnalysis) + self.analyses_cache.invalidate_analysis(LivenessAnalysis) + def _optimize_iszero_chains(self) -> None: fn = self.function for bb in fn.get_basic_blocks(): @@ -23,7 +112,8 @@ def _optimize_iszero_chains(self) -> None: if iszero_count == 0: continue - for use_inst in self.dfg.get_uses(inst.output): + assert isinstance(inst.output, IRVariable) + for use_inst in self.dfg.get_uses(inst.output).copy(): opcode = use_inst.opcode if opcode == "iszero": @@ -42,12 +132,14 @@ def _optimize_iszero_chains(self) -> None: continue out_var = iszero_chain[keep_count].operands[0] - use_inst.replace_operands({inst.output: out_var}) + self.updater._update_operands(use_inst, {inst.output: out_var}) def _get_iszero_chain(self, op: IROperand) -> list[IRInstruction]: chain: list[IRInstruction] = [] while True: + if not isinstance(op, IRVariable): + break inst = self.dfg.get_producing_instruction(op) if inst is None or inst.opcode != "iszero": break @@ -57,24 +149,302 @@ def _get_iszero_chain(self, op: IROperand) -> list[IRInstruction]: chain.reverse() return chain - def _handle_offsets(self): + def _handle_offset(self): for bb in self.function.get_basic_blocks(): for inst in bb.instructions: - # check if the instruction is of the form - # `add