From d8671d9c14467251f35afb8d82354b4bf0ddad4e Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Mon, 30 Mar 2020 20:13:00 -0500 Subject: [PATCH 001/126] Add script to determine percentage of tests compilable via Yul. --- scripts/yul_coverage.sh | 150 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100755 scripts/yul_coverage.sh diff --git a/scripts/yul_coverage.sh b/scripts/yul_coverage.sh new file mode 100755 index 000000000000..d49065f69d92 --- /dev/null +++ b/scripts/yul_coverage.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash + +#------------------------------------------------------------------------------ +# Bash script to determine the percantage of tests that are compilable via Yul. +# +# Usage: +# ./yul_coverage.sh [--no-stats] [--successful] [--internal-compiler-errors] +# [--unimplemented-feature-errors] [--other-errors] [--list-files] +# +# --no-stats will not print the stats to stdout +# --successful print output of successful test-case compilations to stdout +# --internal-compiler-errors print output of test-case compilations that resulted in +# internal compilation errors to stdout +# --unimplemented-feature-errors print output of test-case compilations that resulted in +# unimplemented feature errors to stdout +# --other-errors print output of test-case compilations that resulted in +# errors that where not internal compiler errors or unimplemented feature errors +# to stdout +# --list-files will not print the compiler output to stdout, it will just print the files +# e.g. ./yul_coverage.sh --successful --list-files will just return a list of +# files where it's compilation result was successful +# Environment Variables +# SOLC can be set to change the used compiler. +# +# ./yul_coverage.sh +# run the script without any parameters to execute the tests will return stats. +# +# SOLC= ./yul_coverage.sh +# To change the used compiler, just set the SOLC environment variable. +# +# The documentation for solidity is hosted at: +# +# https://solidity.readthedocs.org +# +# ------------------------------------------------------------------------------ +# This file is part of solidity. +# +# solidity is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# solidity is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with solidity. If not, see +# +# (c) 2020 solidity contributors. +#------------------------------------------------------------------------------ + +set -e + +ROOT_DIR="$(dirname "$0")"/.. + +for arg in "$@"; do + case "$arg" in + --no-stats) NO_STATS=1 ;; + --successful) SHOW_SUCCESSFUL=1 ;; + --internal-compiler-errors) SHOW_INTERNAL_COMPILER_ERRORS=1 ;; + --unimplemented-feature-errors) SHOW_UNIMPLEMENTED_FEATURE_ERRORS=1 ;; + --other-errors) SHOW_OTHER_ERRORS=1 ;; + --list-files) ONLY_LIST_FILES=1 ;; + *) + echo "Usage:" + echo " $(basename "${0}") [--no-stats] [--successful] [--internal-compiler-errors] [--unimplemented-feature-errors] [--other-errors] [--list-files]" + echo " --no-stats will not print the stats to stdout" + echo " --successful print output of successful test-case compilations to stdout" + echo " --internal-compiler-errors print output of test-case compilations that resulted in" + echo " internal compilation errors to stdout" + echo " --unimplemented-feature-errors print output of test-case compilations that resulted in" + echo " unimplemented feature errors to stdout" + echo " --other-errors print output of test-case compilations that resulted in" + echo " errors that where not internal compiler errors or unimplemented feature errors" + echo " to stdout" + echo " --list-files will not print the compiler output to stdout, it will just print the files" + echo " e.g. './yul_coverage.sh --successful --list-files' will just return a list of" + echo " files where it's compilation result was successful" + exit 0 + ;; + esac +done + +show_output_if() { + local VAR=${1} + if [ -n "${VAR}" ]; then + echo "${SOL_FILE}" + if [ -z "${ONLY_LIST_FILES}" ]; then + echo "${OUTPUT}" + echo "" + fi + fi +} + +FAILED=() +SUCCESS=() +SOLC=${SOLC:-"$(command -v -- solc)"} +if [ ! -f "${SOLC}" ]; then + echo "error: solc '${SOLC}' not found." + exit 1 +fi + +test_file() { + local SOL_FILE + local OUTPUT + SOL_FILE=${1} + + if OUTPUT=$("${SOLC}" --ir "${SOL_FILE}" 2>&1); then + SUCCESS+=("${SOL_FILE}") + show_output_if ${SHOW_SUCCESSFUL} + else + FAILED+=("${SOL_FILE}") + if [[ ${OUTPUT} == *"UnimplementedFeatureError"* ]]; then + UNIMPLEMENTED_FEATURE_ERRORS+=("${SOL_FILE}") + show_output_if ${SHOW_UNIMPLEMENTED_FEATURE_ERRORS} + elif [[ ${OUTPUT} == *"InternalCompilerError"* ]]; then + INTERNAL_COMPILER_ERRORS+=("${SOL_FILE}") + show_output_if ${SHOW_INTERNAL_COMPILER_ERRORS} + else + OTHER_ERRORS+=("${SOL_FILE}") + show_output_if ${SHOW_OTHER_ERRORS} + fi + fi +} + +# we only want to use files that do not contain errors or multi-source files. +SOL_FILES=() +while IFS='' read -r line; do + SOL_FILES+=("$line") +done < <( + grep -riL -E \ + "^\/\/ (DocstringParsing|Syntax|Type|Parser|Declaration)Error|^==== Source:" \ + "${ROOT_DIR}/test/libsolidity/syntaxTests" \ + "${ROOT_DIR}/test/libsolidity/semanticTests" +) + +for SOL_FILE in "${SOL_FILES[@]}"; do + test_file "${SOL_FILE}" +done + +if [ -z "${NO_STATS}" ]; then + SUM=$((${#SUCCESS[@]} + ${#FAILED[@]})) + PERCENTAGE=$(echo "scale=4; ${#SUCCESS[@]} / ${SUM}" | bc) + echo "${#SUCCESS[@]} / ${SUM} = ${PERCENTAGE}" + echo "UnimplementedFeatureError(s): ${#UNIMPLEMENTED_FEATURE_ERRORS[@]}" + echo "InternalCompilerError(s): ${#INTERNAL_COMPILER_ERRORS[@]}" + echo "OtherError(s): ${#OTHER_ERRORS[@]}" +fi From 2750bb9b908713cbf201782e9f6e9ab8e0167af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 19 Mar 2020 17:08:53 +0100 Subject: [PATCH 002/126] [yul-phaser] TestHelpers: Generic operator << for printing tuples in boost tests --- test/yulPhaser/TestHelpers.h | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/yulPhaser/TestHelpers.h b/test/yulPhaser/TestHelpers.h index 2bf755b18423..514d2eda76bd 100644 --- a/test/yulPhaser/TestHelpers.h +++ b/test/yulPhaser/TestHelpers.h @@ -33,12 +33,39 @@ #include #include +#include + #include #include #include #include +#include #include +// OPERATORS FOR BOOST::TEST + +/// Output operator for arbitrary two-element tuples. +/// Necessary to make BOOST_TEST() work with such tuples. +template +std::ostream& operator<<(std::ostream& _output, std::tuple const& _tuple) +{ + _output << "(" << std::get<0>(_tuple) << ", " << std::get<1>(_tuple) << ")"; + return _output; +} + +namespace boost::test_tools::tt_detail +{ + +// Boost won't find find the << operator unless we put it in the std namespace which is illegal. +// The recommended solution is to overload print_log_value<> struct and make it use our global operator. +template +struct print_log_value> +{ + void operator()(std::ostream& _output, std::tuple const& _tuple) { ::operator<<(_output, _tuple); } +}; + +} + namespace solidity::phaser::test { From 7381068dcc2932eed964d61302cf8cf07dc7ed55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 11 Mar 2020 03:01:09 +0100 Subject: [PATCH 003/126] [yul-phaser] Make the Population constructor that takes individuals public and use it to speed up some operations --- test/yulPhaser/Population.cpp | 17 +++++++++++++++++ tools/yulPhaser/AlgorithmRunner.cpp | 6 +++--- tools/yulPhaser/Population.h | 7 +++---- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index 7a9172749bb7..0cb0fe29fda7 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -104,6 +104,23 @@ BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_compute_fitness_and_ BOOST_TEST(individuals[2].chromosome == chromosomes[1]); } +BOOST_FIXTURE_TEST_CASE(constructor_should_accept_individuals_without_recalculating_fitness, PopulationFixture) +{ + vector customIndividuals = { + Individual(Chromosome("aaaccc"), 20), + Individual(Chromosome("aaa"), 10), + Individual(Chromosome("aaaf"), 30), + }; + assert(customIndividuals[0].fitness != m_fitnessMetric->evaluate(customIndividuals[0].chromosome)); + assert(customIndividuals[1].fitness != m_fitnessMetric->evaluate(customIndividuals[1].chromosome)); + assert(customIndividuals[2].fitness != m_fitnessMetric->evaluate(customIndividuals[2].chromosome)); + + Population population(m_fitnessMetric, customIndividuals); + + vector expectedIndividuals{customIndividuals[1], customIndividuals[0], customIndividuals[2]}; + BOOST_TEST(population.individuals() == expectedIndividuals); +} + BOOST_FIXTURE_TEST_CASE(makeRandom_should_get_chromosome_lengths_from_specified_generator, PopulationFixture) { size_t chromosomeCount = 30; diff --git a/tools/yulPhaser/AlgorithmRunner.cpp b/tools/yulPhaser/AlgorithmRunner.cpp index c402e5d844d3..402735e0d2c3 100644 --- a/tools/yulPhaser/AlgorithmRunner.cpp +++ b/tools/yulPhaser/AlgorithmRunner.cpp @@ -187,16 +187,16 @@ Population AlgorithmRunner::randomiseDuplicates( if (_population.individuals().size() == 0) return _population; - vector chromosomes{_population.individuals()[0].chromosome}; + vector individuals{_population.individuals()[0]}; size_t duplicateCount = 0; for (size_t i = 1; i < _population.individuals().size(); ++i) if (_population.individuals()[i].chromosome == _population.individuals()[i - 1].chromosome) ++duplicateCount; else - chromosomes.push_back(_population.individuals()[i].chromosome); + individuals.push_back(_population.individuals()[i]); return ( - Population(_population.fitnessMetric(), chromosomes) + + Population(_population.fitnessMetric(), individuals) + Population::makeRandom(_population.fitnessMetric(), duplicateCount, _minChromosomeLength, _maxChromosomeLength) ); } diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 40d51498b181..cf82a5aa8afc 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -81,6 +81,9 @@ class Population _fitnessMetric, chromosomesToIndividuals(*_fitnessMetric, std::move(_chromosomes)) ) {} + explicit Population(std::shared_ptr _fitnessMetric, std::vector _individuals): + m_fitnessMetric(std::move(_fitnessMetric)), + m_individuals{sortedIndividuals(std::move(_individuals))} {} static Population makeRandom( std::shared_ptr _fitnessMetric, @@ -112,10 +115,6 @@ class Population friend std::ostream& operator<<(std::ostream& _stream, Population const& _population); private: - explicit Population(std::shared_ptr _fitnessMetric, std::vector _individuals): - m_fitnessMetric(std::move(_fitnessMetric)), - m_individuals{sortedIndividuals(std::move(_individuals))} {} - static std::vector chromosomesToIndividuals( FitnessMetric& _fitnessMetric, std::vector _chromosomes From b6f8ecf755df9a112e4be6212ed8eff4e65a0aec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 11 Mar 2020 02:07:54 +0100 Subject: [PATCH 004/126] [yul-phaser] Selections+PairSelections: Add RandomSubset and PairsFromRandomSubset --- test/yulPhaser/PairSelections.cpp | 72 ++++++++++++++++++++++++++++++ test/yulPhaser/Selections.cpp | 56 +++++++++++++++++++++++ tools/yulPhaser/PairSelections.cpp | 38 ++++++++++++++++ tools/yulPhaser/PairSelections.h | 22 +++++++++ tools/yulPhaser/Selections.cpp | 10 +++++ tools/yulPhaser/Selections.h | 22 +++++++++ 6 files changed, 220 insertions(+) diff --git a/test/yulPhaser/PairSelections.cpp b/test/yulPhaser/PairSelections.cpp index 64109470f6ed..62a4dd4bcfaf 100644 --- a/test/yulPhaser/PairSelections.cpp +++ b/test/yulPhaser/PairSelections.cpp @@ -119,6 +119,78 @@ BOOST_AUTO_TEST_CASE(materialise_should_return_no_pairs_if_collection_has_one_el BOOST_TEST(RandomPairSelection(2.0).materialise(1).empty()); } +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE(PairsFromRandomSubsetTest) + +BOOST_AUTO_TEST_CASE(materialise_should_return_random_values_with_equal_probabilities) +{ + constexpr int collectionSize = 1000; + constexpr double selectionChance = 0.7; + constexpr double relativeTolerance = 0.001; + constexpr double expectedValue = selectionChance; + constexpr double variance = selectionChance * (1 - selectionChance); + + SimulationRNG::reset(1); + vector> pairs = PairsFromRandomSubset(selectionChance).materialise(collectionSize); + vector bernoulliTrials(collectionSize, 0); + for (auto& pair: pairs) + { + BOOST_REQUIRE(get<1>(pair) < collectionSize); + BOOST_REQUIRE(get<1>(pair) < collectionSize); + bernoulliTrials[get<0>(pair)] = 1.0; + bernoulliTrials[get<1>(pair)] = 1.0; + } + + BOOST_TEST(abs(mean(bernoulliTrials) - expectedValue) < expectedValue * relativeTolerance); + BOOST_TEST(abs(meanSquaredError(bernoulliTrials, expectedValue) - variance) < variance * relativeTolerance); +} + +BOOST_AUTO_TEST_CASE(materialise_should_return_only_values_that_can_be_used_as_collection_indices) +{ + const size_t collectionSize = 200; + constexpr double selectionChance = 0.5; + + vector> pairs = PairsFromRandomSubset(selectionChance).materialise(collectionSize); + + BOOST_TEST(all_of(pairs.begin(), pairs.end(), [&](auto const& pair){ return get<0>(pair) <= collectionSize; })); + BOOST_TEST(all_of(pairs.begin(), pairs.end(), [&](auto const& pair){ return get<1>(pair) <= collectionSize; })); +} + +BOOST_AUTO_TEST_CASE(materialise_should_use_unique_indices) +{ + constexpr size_t collectionSize = 200; + constexpr double selectionChance = 0.5; + + vector> pairs = PairsFromRandomSubset(selectionChance).materialise(collectionSize); + set indices; + for (auto& pair: pairs) + { + indices.insert(get<0>(pair)); + indices.insert(get<1>(pair)); + } + + BOOST_TEST(indices.size() == 2 * pairs.size()); +} + +BOOST_AUTO_TEST_CASE(materialise_should_return_no_indices_if_collection_is_empty) +{ + BOOST_TEST(PairsFromRandomSubset(0.0).materialise(0).empty()); + BOOST_TEST(PairsFromRandomSubset(0.5).materialise(0).empty()); + BOOST_TEST(PairsFromRandomSubset(1.0).materialise(0).empty()); +} + +BOOST_AUTO_TEST_CASE(materialise_should_return_no_pairs_if_selection_chance_is_zero) +{ + BOOST_TEST(PairsFromRandomSubset(0.0).materialise(0).empty()); + BOOST_TEST(PairsFromRandomSubset(0.0).materialise(100).empty()); +} + +BOOST_AUTO_TEST_CASE(materialise_should_return_all_pairs_if_selection_chance_is_one) +{ + BOOST_TEST(PairsFromRandomSubset(1.0).materialise(0).empty()); + BOOST_TEST(PairsFromRandomSubset(1.0).materialise(100).size() == 50); +} + BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE(PairMosaicSelectionTest) diff --git a/test/yulPhaser/Selections.cpp b/test/yulPhaser/Selections.cpp index 02a85f4f33b5..d25766ae6284 100644 --- a/test/yulPhaser/Selections.cpp +++ b/test/yulPhaser/Selections.cpp @@ -25,9 +25,11 @@ #include #include +#include #include using namespace std; +using namespace solidity::util; namespace solidity::phaser::test { @@ -199,6 +201,60 @@ BOOST_AUTO_TEST_CASE(materialise_should_return_no_indices_if_collection_is_empty BOOST_TEST(RandomSelection(2.0).materialise(0).empty()); } +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE(RandomSubsetTest) + +BOOST_AUTO_TEST_CASE(materialise_should_return_random_values_with_equal_probabilities) +{ + constexpr int collectionSize = 1000; + constexpr double selectionChance = 0.7; + constexpr double relativeTolerance = 0.001; + constexpr double expectedValue = selectionChance; + constexpr double variance = selectionChance * (1 - selectionChance); + + SimulationRNG::reset(1); + auto indices = convertContainer>(RandomSubset(selectionChance).materialise(collectionSize)); + + vector bernoulliTrials(collectionSize); + for (size_t i = 0; i < collectionSize; ++i) + bernoulliTrials[i] = indices.count(i); + + BOOST_TEST(abs(mean(bernoulliTrials) - expectedValue) < expectedValue * relativeTolerance); + BOOST_TEST(abs(meanSquaredError(bernoulliTrials, expectedValue) - variance) < variance * relativeTolerance); +} + +BOOST_AUTO_TEST_CASE(materialise_should_return_only_values_that_can_be_used_as_collection_indices) +{ + const size_t collectionSize = 200; + vector indices = RandomSubset(0.5).materialise(collectionSize); + + BOOST_TEST(all_of(indices.begin(), indices.end(), [&](auto const& index){ return index <= collectionSize; })); +} + +BOOST_AUTO_TEST_CASE(materialise_should_return_indices_in_the_same_order_they_are_in_the_container) +{ + const size_t collectionSize = 200; + vector indices = RandomSubset(0.5).materialise(collectionSize); + + for (size_t i = 1; i < indices.size(); ++i) + BOOST_TEST(indices[i - 1] < indices[i]); +} + +BOOST_AUTO_TEST_CASE(materialise_should_return_no_indices_if_collection_is_empty) +{ + BOOST_TEST(RandomSubset(0.5).materialise(0).empty()); +} + +BOOST_AUTO_TEST_CASE(materialise_should_return_no_indices_if_selection_chance_is_zero) +{ + BOOST_TEST(RandomSubset(0.0).materialise(10).empty()); +} + +BOOST_AUTO_TEST_CASE(materialise_should_return_all_indices_if_selection_chance_is_one) +{ + BOOST_TEST(RandomSubset(1.0).materialise(10).size() == 10); +} + BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() diff --git a/tools/yulPhaser/PairSelections.cpp b/tools/yulPhaser/PairSelections.cpp index 8f3fd0f7f3c1..fe85fef9ccc2 100644 --- a/tools/yulPhaser/PairSelections.cpp +++ b/tools/yulPhaser/PairSelections.cpp @@ -17,6 +17,7 @@ #include +#include #include #include @@ -47,6 +48,43 @@ vector> RandomPairSelection::materialise(size_t _poolSize) return selection; } +vector> PairsFromRandomSubset::materialise(size_t _poolSize) const +{ + vector selectedIndices = RandomSubset(m_selectionChance).materialise(_poolSize); + + if (selectedIndices.size() % 2 != 0) + { + if (selectedIndices.size() < _poolSize && SimulationRNG::bernoulliTrial(0.5)) + { + do + { + size_t extraIndex = SimulationRNG::uniformInt(0, selectedIndices.size() - 1); + if (find(selectedIndices.begin(), selectedIndices.end(), extraIndex) == selectedIndices.end()) + selectedIndices.push_back(extraIndex); + } while (selectedIndices.size() % 2 != 0); + } + else + selectedIndices.erase(selectedIndices.begin() + SimulationRNG::uniformInt(0, selectedIndices.size() - 1)); + } + assert(selectedIndices.size() % 2 == 0); + + vector> selectedPairs; + for (size_t i = selectedIndices.size() / 2; i > 0; --i) + { + size_t position1 = SimulationRNG::uniformInt(0, selectedIndices.size() - 1); + size_t value1 = selectedIndices[position1]; + selectedIndices.erase(selectedIndices.begin() + position1); + size_t position2 = SimulationRNG::uniformInt(0, selectedIndices.size() - 1); + size_t value2 = selectedIndices[position2]; + selectedIndices.erase(selectedIndices.begin() + position2); + + selectedPairs.push_back({value1, value2}); + } + assert(selectedIndices.size() == 0); + + return selectedPairs; +} + vector> PairMosaicSelection::materialise(size_t _poolSize) const { if (_poolSize < 2) diff --git a/tools/yulPhaser/PairSelections.h b/tools/yulPhaser/PairSelections.h index 7778d656783a..a80fff8972be 100644 --- a/tools/yulPhaser/PairSelections.h +++ b/tools/yulPhaser/PairSelections.h @@ -69,6 +69,28 @@ class RandomPairSelection: public PairSelection double m_selectionSize; }; + +/** + * A selection that goes over all elements in a container, for each one independently decides + * whether to select it or not and then randomly combines those elements into pairs. If the number + * of elements is odd, randomly decides whether to take one more or exclude one. + * + * Each element has the same chance of being selected and can be selected at most once. + * The number of selected elements is random and can be different with each call to + * @a materialise(). + */ +class PairsFromRandomSubset: public PairSelection +{ +public: + explicit PairsFromRandomSubset(double _selectionChance): + m_selectionChance(_selectionChance) {} + + std::vector> materialise(size_t _poolSize) const override; + +private: + double m_selectionChance; +}; + /** * A selection that selects pairs of elements at specific, fixed positions indicated by a repeating * "pattern". If the positions in the pattern exceed the size of the container, they are capped at diff --git a/tools/yulPhaser/Selections.cpp b/tools/yulPhaser/Selections.cpp index abc080fdee6a..920a7d7f1af2 100644 --- a/tools/yulPhaser/Selections.cpp +++ b/tools/yulPhaser/Selections.cpp @@ -20,6 +20,7 @@ #include #include +#include using namespace std; using namespace solidity::phaser; @@ -58,3 +59,12 @@ vector RandomSelection::materialise(size_t _poolSize) const return selection; } +vector RandomSubset::materialise(size_t _poolSize) const +{ + vector selection; + for (size_t index = 0; index < _poolSize; ++index) + if (SimulationRNG::bernoulliTrial(m_selectionChance)) + selection.push_back(index); + + return selection; +} diff --git a/tools/yulPhaser/Selections.h b/tools/yulPhaser/Selections.h index 46d975bbd5e8..a0ed2657f0c1 100644 --- a/tools/yulPhaser/Selections.h +++ b/tools/yulPhaser/Selections.h @@ -118,4 +118,26 @@ class RandomSelection: public Selection double m_selectionSize; }; +/** + * A selection that goes over all elements in a container, for each one independently deciding + * whether to select it or not. Each element has the same chance of being selected and can be + * selected at most once. The order of selected elements is the same as the order of elements in + * the container. The number of selected elements is random and can be different with each call + * to @a materialise(). + */ +class RandomSubset: public Selection +{ +public: + explicit RandomSubset(double _selectionChance): + m_selectionChance(_selectionChance) + { + assert(0.0 <= _selectionChance && _selectionChance <= 1.0); + } + + std::vector materialise(size_t _poolSize) const override; + +private: + double m_selectionChance; +}; + } From 0837a62d5c0d40abae7884a0bc111dd8a64590e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 11 Mar 2020 02:11:58 +0100 Subject: [PATCH 005/126] [yul-phaser] Mutations: Add symmetricRandomPointCrossover() --- test/yulPhaser/Mutations.cpp | 14 +++++++++++++ tools/yulPhaser/Mutations.cpp | 39 ++++++++++++++++++++++++++++------- tools/yulPhaser/Mutations.h | 7 +++++++ 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/test/yulPhaser/Mutations.cpp b/test/yulPhaser/Mutations.cpp index df58cec540d9..d98fa76043ad 100644 --- a/test/yulPhaser/Mutations.cpp +++ b/test/yulPhaser/Mutations.cpp @@ -225,6 +225,20 @@ BOOST_AUTO_TEST_CASE(randomPointCrossover_should_swap_chromosome_parts_at_random BOOST_TEST(result2 == Chromosome("cccaaaaaaa")); } +BOOST_AUTO_TEST_CASE(symmetricRandomPointCrossover_should_swap_chromosome_parts_at_random_point) +{ + function crossover = symmetricRandomPointCrossover(); + + SimulationRNG::reset(1); + tuple result1 = crossover(Chromosome("aaaaaaaaaa"), Chromosome("cccccc")); + tuple expectedPair1 = {Chromosome("aaaccc"), Chromosome("cccaaaaaaa")}; + BOOST_TEST(result1 == expectedPair1); + + tuple result2 = crossover(Chromosome("cccccc"), Chromosome("aaaaaaaaaa")); + tuple expectedPair2 = {Chromosome("ccccccaaaa"), Chromosome("aaaaaa")}; + BOOST_TEST(result2 == expectedPair2); +} + BOOST_AUTO_TEST_CASE(randomPointCrossover_should_only_consider_points_available_on_both_chromosomes) { SimulationRNG::reset(1); diff --git a/tools/yulPhaser/Mutations.cpp b/tools/yulPhaser/Mutations.cpp index 86f815198d07..9cf7417073d0 100644 --- a/tools/yulPhaser/Mutations.cpp +++ b/tools/yulPhaser/Mutations.cpp @@ -98,7 +98,7 @@ function phaser::alternativeMutations( namespace { -Chromosome buildChromosomesBySwappingParts( +ChromosomePair fixedPointSwap( Chromosome const& _chromosome1, Chromosome const& _chromosome2, size_t _crossoverPoint @@ -109,11 +109,19 @@ Chromosome buildChromosomesBySwappingParts( auto begin1 = _chromosome1.optimisationSteps().begin(); auto begin2 = _chromosome2.optimisationSteps().begin(); - - return Chromosome( - vector(begin1, begin1 + _crossoverPoint) + - vector(begin2 + _crossoverPoint, _chromosome2.optimisationSteps().end()) - ); + auto end1 = _chromosome1.optimisationSteps().end(); + auto end2 = _chromosome2.optimisationSteps().end(); + + return { + Chromosome( + vector(begin1, begin1 + _crossoverPoint) + + vector(begin2 + _crossoverPoint, end2) + ), + Chromosome( + vector(begin2, begin2 + _crossoverPoint) + + vector(begin1 + _crossoverPoint, end1) + ), + }; } } @@ -129,7 +137,22 @@ function phaser::randomPointCrossover() assert(minPoint <= minLength); size_t randomPoint = SimulationRNG::uniformInt(minPoint, minLength); - return buildChromosomesBySwappingParts(_chromosome1, _chromosome2, randomPoint); + return get<0>(fixedPointSwap(_chromosome1, _chromosome2, randomPoint)); + }; +} + +function phaser::symmetricRandomPointCrossover() +{ + return [=](Chromosome const& _chromosome1, Chromosome const& _chromosome2) + { + size_t minLength = min(_chromosome1.length(), _chromosome2.length()); + + // Don't use position 0 (because this just swaps the values) unless it's the only choice. + size_t minPoint = (minLength > 0? 1 : 0); + assert(minPoint <= minLength); + + size_t randomPoint = SimulationRNG::uniformInt(minPoint, minLength); + return fixedPointSwap(_chromosome1, _chromosome2, randomPoint); }; } @@ -142,6 +165,6 @@ function phaser::fixedPointCrossover(double _crossoverPoint) size_t minLength = min(_chromosome1.length(), _chromosome2.length()); size_t concretePoint = static_cast(round(minLength * _crossoverPoint)); - return buildChromosomesBySwappingParts(_chromosome1, _chromosome2, concretePoint); + return get<0>(fixedPointSwap(_chromosome1, _chromosome2, concretePoint)); }; } diff --git a/tools/yulPhaser/Mutations.h b/tools/yulPhaser/Mutations.h index bff48c52b204..5554a1d2dd2e 100644 --- a/tools/yulPhaser/Mutations.h +++ b/tools/yulPhaser/Mutations.h @@ -28,8 +28,11 @@ namespace solidity::phaser { +using ChromosomePair = std::tuple; + using Mutation = Chromosome(Chromosome const&); using Crossover = Chromosome(Chromosome const&, Chromosome const&); +using SymmetricCrossover = ChromosomePair(Chromosome const&, Chromosome const&); // MUTATIONS @@ -61,6 +64,10 @@ std::function alternativeMutations( /// position at which to perform perform @a fixedPointCrossover. std::function randomPointCrossover(); +/// Symmetric version of @a randomPointCrossover(). Creates an operator that returns a pair +/// containing both possible results for the same crossover point. +std::function symmetricRandomPointCrossover(); + /// Creates a crossover operator that always chooses a point that lies at @a _crossoverPoint /// percent of the length of the shorter chromosome. Then creates a new chromosome by /// splitting both inputs at the crossover point and stitching output from the first half or first From 59011fcde6586150148792e3309cd3e15fe9e74a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 11 Mar 2020 23:45:46 +0100 Subject: [PATCH 006/126] [yul-phaser] Mutations: Add mutationSequence() --- test/yulPhaser/Mutations.cpp | 33 +++++++++++++++++++++++++++++++++ tools/yulPhaser/Mutations.cpp | 12 ++++++++++++ tools/yulPhaser/Mutations.h | 3 +++ 3 files changed, 48 insertions(+) diff --git a/test/yulPhaser/Mutations.cpp b/test/yulPhaser/Mutations.cpp index d98fa76043ad..33c623f84a56 100644 --- a/test/yulPhaser/Mutations.cpp +++ b/test/yulPhaser/Mutations.cpp @@ -212,6 +212,39 @@ BOOST_AUTO_TEST_CASE(alternativeMutations_should_always_choose_second_mutation_i BOOST_TEST(mutation(chromosome) == Chromosome("f")); } +BOOST_AUTO_TEST_CASE(mutationSequence_should_apply_all_mutations) +{ + Chromosome chromosome("aaaaa"); + function mutation = mutationSequence({ + geneSubstitution(3, Chromosome("g").optimisationSteps()[0]), + geneSubstitution(2, Chromosome("f").optimisationSteps()[0]), + geneSubstitution(1, Chromosome("c").optimisationSteps()[0]), + }); + + BOOST_TEST(mutation(chromosome) == Chromosome("acfga")); +} + +BOOST_AUTO_TEST_CASE(mutationSequence_apply_mutations_in_the_order_they_are_given) +{ + Chromosome chromosome("aa"); + function mutation = mutationSequence({ + geneSubstitution(0, Chromosome("g").optimisationSteps()[0]), + geneSubstitution(1, Chromosome("c").optimisationSteps()[0]), + geneSubstitution(0, Chromosome("f").optimisationSteps()[0]), + geneSubstitution(1, Chromosome("o").optimisationSteps()[0]), + }); + + BOOST_TEST(mutation(chromosome) == Chromosome("fo")); +} + +BOOST_AUTO_TEST_CASE(mutationSequence_should_return_unmodified_chromosome_if_given_no_mutations) +{ + Chromosome chromosome("aa"); + function mutation = mutationSequence({}); + + BOOST_TEST(mutation(chromosome) == chromosome); +} + BOOST_AUTO_TEST_CASE(randomPointCrossover_should_swap_chromosome_parts_at_random_point) { function crossover = randomPointCrossover(); diff --git a/tools/yulPhaser/Mutations.cpp b/tools/yulPhaser/Mutations.cpp index 9cf7417073d0..98689a810216 100644 --- a/tools/yulPhaser/Mutations.cpp +++ b/tools/yulPhaser/Mutations.cpp @@ -95,6 +95,18 @@ function phaser::alternativeMutations( }; } +function phaser::mutationSequence(vector> _mutations) +{ + return [=](Chromosome const& _chromosome) + { + Chromosome mutatedChromosome = _chromosome; + for (size_t i = 0; i < _mutations.size(); ++i) + mutatedChromosome = _mutations[i](move(mutatedChromosome)); + + return mutatedChromosome; + }; +} + namespace { diff --git a/tools/yulPhaser/Mutations.h b/tools/yulPhaser/Mutations.h index 5554a1d2dd2e..d545aa0934fa 100644 --- a/tools/yulPhaser/Mutations.h +++ b/tools/yulPhaser/Mutations.h @@ -58,6 +58,9 @@ std::function alternativeMutations( std::function _mutation2 ); +/// Creates a mutation operator that sequentially applies all the operators given in @a _mutations. +std::function mutationSequence(std::vector> _mutations); + // CROSSOVER /// Creates a crossover operator that randomly selects a number between 0 and 1 and uses it as the From ef8d0888af7accf12207c8288a9d82e11f5121cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 11 Mar 2020 02:13:55 +0100 Subject: [PATCH 007/126] [yul-phaser] Population: Add symmetricCrossoverWithRemainder() --- test/yulPhaser/Population.cpp | 55 ++++++++++++++++++++++++++++++++++ tools/yulPhaser/Population.cpp | 31 +++++++++++++++++++ tools/yulPhaser/Population.h | 4 +++ 3 files changed, 90 insertions(+) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index 0cb0fe29fda7..12555845c2e2 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -48,6 +48,14 @@ namespace solidity::phaser::test class PopulationFixture { protected: + static ChromosomePair twoStepSwap(Chromosome const& _chromosome1, Chromosome const& _chromosome2) + { + return ChromosomePair{ + Chromosome(vector{_chromosome1.optimisationSteps()[0], _chromosome2.optimisationSteps()[1]}), + Chromosome(vector{_chromosome2.optimisationSteps()[0], _chromosome1.optimisationSteps()[1]}), + }; + } + shared_ptr m_fitnessMetric = make_shared(); }; @@ -309,6 +317,53 @@ BOOST_FIXTURE_TEST_CASE(crossover_should_return_empty_population_if_selection_is BOOST_TEST(population.crossover(selection, fixedPointCrossover(0.5)).individuals().empty()); } +BOOST_FIXTURE_TEST_CASE(symmetricCrossoverWithRemainder_should_return_crossed_population_and_remainder, PopulationFixture) +{ + Population population(m_fitnessMetric, {Chromosome("aa"), Chromosome("cc"), Chromosome("gg"), Chromosome("hh")}); + PairMosaicSelection selection({{2, 1}}, 0.25); + assert(selection.materialise(population.individuals().size()) == (vector>{{2, 1}})); + + Population expectedCrossedPopulation(m_fitnessMetric, {Chromosome("gc"), Chromosome("cg")}); + Population expectedRemainder(m_fitnessMetric, {Chromosome("aa"), Chromosome("hh")}); + + BOOST_TEST( + population.symmetricCrossoverWithRemainder(selection, twoStepSwap) == + (tuple{expectedCrossedPopulation, expectedRemainder}) + ); +} + +BOOST_FIXTURE_TEST_CASE(symmetricCrossoverWithRemainder_should_allow_crossing_the_same_individual_multiple_times, PopulationFixture) +{ + Population population(m_fitnessMetric, {Chromosome("aa"), Chromosome("cc"), Chromosome("gg"), Chromosome("hh")}); + PairMosaicSelection selection({{0, 0}, {2, 1}}, 1.0); + assert(selection.materialise(population.individuals().size()) == (vector>{{0, 0}, {2, 1}, {0, 0}, {2, 1}})); + + Population expectedCrossedPopulation(m_fitnessMetric, { + Chromosome("aa"), Chromosome("aa"), + Chromosome("aa"), Chromosome("aa"), + Chromosome("gc"), Chromosome("cg"), + Chromosome("gc"), Chromosome("cg"), + }); + Population expectedRemainder(m_fitnessMetric, {Chromosome("hh")}); + + BOOST_TEST( + population.symmetricCrossoverWithRemainder(selection, twoStepSwap) == + (tuple{expectedCrossedPopulation, expectedRemainder}) + ); +} + +BOOST_FIXTURE_TEST_CASE(symmetricCrossoverWithRemainder_should_return_empty_population_if_selection_is_empty, PopulationFixture) +{ + Population population(m_fitnessMetric, {Chromosome("aa"), Chromosome("cc")}); + PairMosaicSelection selection({}, 0.0); + assert(selection.materialise(population.individuals().size()).empty()); + + BOOST_TEST( + population.symmetricCrossoverWithRemainder(selection, twoStepSwap) == + (tuple{Population(m_fitnessMetric), population}) + ); +} + BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index b336daf73f39..acdae1eb780a 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -117,6 +117,37 @@ Population Population::crossover(PairSelection const& _selection, function Population::symmetricCrossoverWithRemainder( + PairSelection const& _selection, + function _symmetricCrossover +) const +{ + vector indexSelected(m_individuals.size(), false); + + vector crossedIndividuals; + for (auto const& [i, j]: _selection.materialise(m_individuals.size())) + { + auto children = _symmetricCrossover( + m_individuals[i].chromosome, + m_individuals[j].chromosome + ); + crossedIndividuals.emplace_back(move(get<0>(children)), *m_fitnessMetric); + crossedIndividuals.emplace_back(move(get<1>(children)), *m_fitnessMetric); + indexSelected[i] = true; + indexSelected[j] = true; + } + + vector remainder; + for (size_t i = 0; i < indexSelected.size(); ++i) + if (!indexSelected[i]) + remainder.emplace_back(m_individuals[i]); + + return { + Population(m_fitnessMetric, crossedIndividuals), + Population(m_fitnessMetric, remainder), + }; +} + namespace solidity::phaser { diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index cf82a5aa8afc..2d79dfef26c4 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -100,6 +100,10 @@ class Population Population select(Selection const& _selection) const; Population mutate(Selection const& _selection, std::function _mutation) const; Population crossover(PairSelection const& _selection, std::function _crossover) const; + std::tuple symmetricCrossoverWithRemainder( + PairSelection const& _selection, + std::function _symmetricCrossover + ) const; friend Population operator+(Population _a, Population _b); From 8c86a4983d70eda377db17d3f4833698052c342b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 11 Mar 2020 02:14:30 +0100 Subject: [PATCH 008/126] [yul-phaser] Population: Add combine() --- test/yulPhaser/Population.cpp | 8 ++++++++ tools/yulPhaser/Population.cpp | 5 +++++ tools/yulPhaser/Population.h | 1 + 3 files changed, 14 insertions(+) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index 12555845c2e2..1657e97c99c9 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -364,6 +364,14 @@ BOOST_FIXTURE_TEST_CASE(symmetricCrossoverWithRemainder_should_return_empty_popu ); } +BOOST_FIXTURE_TEST_CASE(combine_should_add_two_populations_from_a_pair, PopulationFixture) +{ + Population population1(m_fitnessMetric, {Chromosome("aa"), Chromosome("hh")}); + Population population2(m_fitnessMetric, {Chromosome("gg"), Chromosome("cc")}); + + BOOST_TEST(Population::combine({population1, population2}) == population1 + population2); +} + BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index acdae1eb780a..1b846122e34f 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -163,6 +163,11 @@ Population operator+(Population _a, Population _b) } +Population Population::combine(std::tuple _populationPair) +{ + return get<0>(_populationPair) + get<1>(_populationPair); +} + bool Population::operator==(Population const& _other) const { // We consider populations identical only if they share the same exact instance of the metric. diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 2d79dfef26c4..c405f702ce9a 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -106,6 +106,7 @@ class Population ) const; friend Population operator+(Population _a, Population _b); + static Population combine(std::tuple _populationPair); std::shared_ptr fitnessMetric() { return m_fitnessMetric; } std::vector const& individuals() const { return m_individuals; } From 879f6e17e97bd772377471752b28f6e93afb1aca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 11 Mar 2020 02:29:20 +0100 Subject: [PATCH 009/126] [yul-phaser] GenerationalElitistWithExclusivePools: Refactor runNextRound() for less nesting and more readability --- tools/yulPhaser/GeneticAlgorithms.cpp | 36 +++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tools/yulPhaser/GeneticAlgorithms.cpp b/tools/yulPhaser/GeneticAlgorithms.cpp index 432f3fe38aae..c38c3bc95492 100644 --- a/tools/yulPhaser/GeneticAlgorithms.cpp +++ b/tools/yulPhaser/GeneticAlgorithms.cpp @@ -43,24 +43,24 @@ Population RandomAlgorithm::runNextRound(Population _population) Population GenerationalElitistWithExclusivePools::runNextRound(Population _population) { double elitePoolSize = 1.0 - (m_options.mutationPoolSize + m_options.crossoverPoolSize); - RangeSelection elite(0.0, elitePoolSize); + + RangeSelection elitePool(0.0, elitePoolSize); + RandomSelection mutationPoolFromElite(m_options.mutationPoolSize / elitePoolSize); + RandomPairSelection crossoverPoolFromElite(m_options.crossoverPoolSize / elitePoolSize); + + std::function mutationOperator = alternativeMutations( + m_options.randomisationChance, + geneRandomisation(m_options.percentGenesToRandomise), + alternativeMutations( + m_options.deletionVsAdditionChance, + geneDeletion(m_options.percentGenesToAddOrDelete), + geneAddition(m_options.percentGenesToAddOrDelete) + ) + ); + std::function crossoverOperator = randomPointCrossover(); return - _population.select(elite) + - _population.select(elite).mutate( - RandomSelection(m_options.mutationPoolSize / elitePoolSize), - alternativeMutations( - m_options.randomisationChance, - geneRandomisation(m_options.percentGenesToRandomise), - alternativeMutations( - m_options.deletionVsAdditionChance, - geneDeletion(m_options.percentGenesToAddOrDelete), - geneAddition(m_options.percentGenesToAddOrDelete) - ) - ) - ) + - _population.select(elite).crossover( - RandomPairSelection(m_options.crossoverPoolSize / elitePoolSize), - randomPointCrossover() - ); + _population.select(elitePool) + + _population.select(elitePool).mutate(mutationPoolFromElite, mutationOperator) + + _population.select(elitePool).crossover(crossoverPoolFromElite, crossoverOperator); } From f6783c60b215f859133f0ab2c395c04bf183934f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 11 Mar 2020 02:15:25 +0100 Subject: [PATCH 010/126] [yul-phaser] GeneticAlgorithms: Add ClassicGeneticAlgorithm --- test/yulPhaser/GeneticAlgorithms.cpp | 204 ++++++++++++++++++++++++++ tools/yulPhaser/GeneticAlgorithms.cpp | 62 ++++++++ tools/yulPhaser/GeneticAlgorithms.h | 55 +++++++ 3 files changed, 321 insertions(+) diff --git a/test/yulPhaser/GeneticAlgorithms.cpp b/test/yulPhaser/GeneticAlgorithms.cpp index f7a5e7a92dc4..1b902d97878b 100644 --- a/test/yulPhaser/GeneticAlgorithms.cpp +++ b/test/yulPhaser/GeneticAlgorithms.cpp @@ -31,6 +31,7 @@ using namespace std; using namespace boost::unit_test::framework; using namespace boost::test_tools; +using namespace solidity::util; namespace solidity::phaser::test { @@ -41,6 +42,18 @@ class GeneticAlgorithmFixture shared_ptr m_fitnessMetric = make_shared(); }; +class ClassicGeneticAlgorithmFixture: public GeneticAlgorithmFixture +{ +protected: + ClassicGeneticAlgorithm::Options m_options = { + /* elitePoolSize = */ 0.0, + /* crossoverChance = */ 0.0, + /* mutationChance = */ 0.0, + /* deletionChance = */ 0.0, + /* additionChance = */ 0.0, + }; +}; + BOOST_AUTO_TEST_SUITE(Phaser) BOOST_AUTO_TEST_SUITE(GeneticAlgorithmsTest) BOOST_AUTO_TEST_SUITE(RandomAlgorithmTest) @@ -186,6 +199,197 @@ BOOST_FIXTURE_TEST_CASE(runNextRound_should_generate_individuals_in_the_crossove })); } +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE(ClassicGeneticAlgorithmTest) + +BOOST_FIXTURE_TEST_CASE(runNextRound_should_select_individuals_with_probability_proportional_to_fitness, ClassicGeneticAlgorithmFixture) +{ + constexpr double relativeTolerance = 0.1; + constexpr size_t populationSize = 1000; + assert(populationSize % 4 == 0 && "Choose a number divisible by 4 for this test"); + + auto population = + Population::makeRandom(m_fitnessMetric, populationSize / 4, 0, 0) + + Population::makeRandom(m_fitnessMetric, populationSize / 4, 1, 1) + + Population::makeRandom(m_fitnessMetric, populationSize / 4, 2, 2) + + Population::makeRandom(m_fitnessMetric, populationSize / 4, 3, 3); + + map expectedProbabilities = { + {0, 4.0 / (4 + 3 + 2 + 1)}, + {1, 3.0 / (4 + 3 + 2 + 1)}, + {2, 2.0 / (4 + 3 + 2 + 1)}, + {3, 1.0 / (4 + 3 + 2 + 1)}, + }; + double const expectedValue = ( + 0.0 * expectedProbabilities[0] + + 1.0 * expectedProbabilities[1] + + 2.0 * expectedProbabilities[2] + + 3.0 * expectedProbabilities[3] + ); + double const variance = ( + (0.0 - expectedValue) * (0.0 - expectedValue) * expectedProbabilities[0] + + (1.0 - expectedValue) * (1.0 - expectedValue) * expectedProbabilities[1] + + (2.0 - expectedValue) * (2.0 - expectedValue) * expectedProbabilities[2] + + (3.0 - expectedValue) * (3.0 - expectedValue) * expectedProbabilities[3] + ); + + ClassicGeneticAlgorithm algorithm(m_options); + Population newPopulation = algorithm.runNextRound(population); + + BOOST_TEST(newPopulation.individuals().size() == population.individuals().size()); + + vector newFitness = chromosomeLengths(newPopulation); + BOOST_TEST(abs(mean(newFitness) - expectedValue) < expectedValue * relativeTolerance); + BOOST_TEST(abs(meanSquaredError(newFitness, expectedValue) - variance) < variance * relativeTolerance); +} + +BOOST_FIXTURE_TEST_CASE(runNextRound_should_select_only_individuals_existing_in_the_original_population, ClassicGeneticAlgorithmFixture) +{ + constexpr size_t populationSize = 1000; + auto population = Population::makeRandom(m_fitnessMetric, populationSize, 1, 10); + + set originalSteps; + for (auto const& individual: population.individuals()) + originalSteps.insert(toString(individual.chromosome)); + + ClassicGeneticAlgorithm algorithm(m_options); + Population newPopulation = algorithm.runNextRound(population); + + for (auto const& individual: newPopulation.individuals()) + BOOST_TEST(originalSteps.count(toString(individual.chromosome)) == 1); +} + +BOOST_FIXTURE_TEST_CASE(runNextRound_should_do_crossover, ClassicGeneticAlgorithmFixture) +{ + auto population = Population(m_fitnessMetric, { + Chromosome("aa"), Chromosome("aa"), Chromosome("aa"), + Chromosome("ff"), Chromosome("ff"), Chromosome("ff"), + Chromosome("gg"), Chromosome("gg"), Chromosome("gg"), + }); + + set originalSteps{"aa", "ff", "gg"}; + set crossedSteps{"af", "fa", "fg", "gf", "ga", "ag"}; + + m_options.crossoverChance = 0.8; + ClassicGeneticAlgorithm algorithm(m_options); + + SimulationRNG::reset(1); + Population newPopulation = algorithm.runNextRound(population); + + size_t totalCrossed = 0; + size_t totalUnchanged = 0; + for (auto const& individual: newPopulation.individuals()) + { + totalCrossed += crossedSteps.count(toString(individual.chromosome)); + totalUnchanged += originalSteps.count(toString(individual.chromosome)); + } + BOOST_TEST(totalCrossed + totalUnchanged == newPopulation.individuals().size()); + BOOST_TEST(totalCrossed >= 2); +} + +BOOST_FIXTURE_TEST_CASE(runNextRound_should_do_mutation, ClassicGeneticAlgorithmFixture) +{ + m_options.mutationChance = 0.6; + ClassicGeneticAlgorithm algorithm(m_options); + + constexpr size_t populationSize = 1000; + constexpr double relativeTolerance = 0.05; + double const expectedValue = m_options.mutationChance; + double const variance = m_options.mutationChance * (1 - m_options.mutationChance); + + Chromosome chromosome("aaaaaaaaaa"); + vector chromosomes(populationSize, chromosome); + Population population(m_fitnessMetric, chromosomes); + + SimulationRNG::reset(1); + Population newPopulation = algorithm.runNextRound(population); + + vector bernoulliTrials; + for (auto const& individual: newPopulation.individuals()) + { + string steps = toString(individual.chromosome); + for (char step: steps) + bernoulliTrials.push_back(static_cast(step != 'a')); + } + + BOOST_TEST(abs(mean(bernoulliTrials) - expectedValue) < expectedValue * relativeTolerance); + BOOST_TEST(abs(meanSquaredError(bernoulliTrials, expectedValue) - variance) < variance * relativeTolerance); +} + +BOOST_FIXTURE_TEST_CASE(runNextRound_should_do_deletion, ClassicGeneticAlgorithmFixture) +{ + m_options.deletionChance = 0.6; + ClassicGeneticAlgorithm algorithm(m_options); + + constexpr size_t populationSize = 1000; + constexpr double relativeTolerance = 0.05; + double const expectedValue = m_options.deletionChance; + double const variance = m_options.deletionChance * (1 - m_options.deletionChance); + + Chromosome chromosome("aaaaaaaaaa"); + vector chromosomes(populationSize, chromosome); + Population population(m_fitnessMetric, chromosomes); + + SimulationRNG::reset(1); + Population newPopulation = algorithm.runNextRound(population); + + vector bernoulliTrials; + for (auto const& individual: newPopulation.individuals()) + { + string steps = toString(individual.chromosome); + for (size_t i = 0; i < chromosome.length(); ++i) + bernoulliTrials.push_back(static_cast(i >= steps.size())); + } + + BOOST_TEST(abs(mean(bernoulliTrials) - expectedValue) < expectedValue * relativeTolerance); + BOOST_TEST(abs(meanSquaredError(bernoulliTrials, expectedValue) - variance) < variance * relativeTolerance); +} + +BOOST_FIXTURE_TEST_CASE(runNextRound_should_do_addition, ClassicGeneticAlgorithmFixture) +{ + m_options.additionChance = 0.6; + ClassicGeneticAlgorithm algorithm(m_options); + + constexpr size_t populationSize = 1000; + constexpr double relativeTolerance = 0.05; + double const expectedValue = m_options.additionChance; + double const variance = m_options.additionChance * (1 - m_options.additionChance); + + Chromosome chromosome("aaaaaaaaaa"); + vector chromosomes(populationSize, chromosome); + Population population(m_fitnessMetric, chromosomes); + + SimulationRNG::reset(1); + Population newPopulation = algorithm.runNextRound(population); + + vector bernoulliTrials; + for (auto const& individual: newPopulation.individuals()) + { + string steps = toString(individual.chromosome); + for (size_t i = 0; i < chromosome.length() + 1; ++i) + { + BOOST_REQUIRE(chromosome.length() <= steps.size() && steps.size() <= 2 * chromosome.length() + 1); + bernoulliTrials.push_back(static_cast(i < steps.size() - chromosome.length())); + } + } + + BOOST_TEST(abs(mean(bernoulliTrials) - expectedValue) < expectedValue * relativeTolerance); + BOOST_TEST(abs(meanSquaredError(bernoulliTrials, expectedValue) - variance) < variance * relativeTolerance); +} + +BOOST_FIXTURE_TEST_CASE(runNextRound_should_preserve_elite, ClassicGeneticAlgorithmFixture) +{ + auto population = Population::makeRandom(m_fitnessMetric, 4, 3, 3) + Population::makeRandom(m_fitnessMetric, 6, 5, 5); + assert((chromosomeLengths(population) == vector{3, 3, 3, 3, 5, 5, 5, 5, 5, 5})); + + m_options.elitePoolSize = 0.5; + m_options.deletionChance = 1.0; + ClassicGeneticAlgorithm algorithm(m_options); + Population newPopulation = algorithm.runNextRound(population); + + BOOST_TEST((chromosomeLengths(newPopulation) == vector{0, 0, 0, 0, 0, 3, 3, 3, 3, 5})); +} + BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() diff --git a/tools/yulPhaser/GeneticAlgorithms.cpp b/tools/yulPhaser/GeneticAlgorithms.cpp index c38c3bc95492..701bdf28c137 100644 --- a/tools/yulPhaser/GeneticAlgorithms.cpp +++ b/tools/yulPhaser/GeneticAlgorithms.cpp @@ -64,3 +64,65 @@ Population GenerationalElitistWithExclusivePools::runNextRound(Population _popul _population.select(elitePool).mutate(mutationPoolFromElite, mutationOperator) + _population.select(elitePool).crossover(crossoverPoolFromElite, crossoverOperator); } + +Population ClassicGeneticAlgorithm::runNextRound(Population _population) +{ + Population elite = _population.select(RangeSelection(0.0, m_options.elitePoolSize)); + Population rest = _population.select(RangeSelection(m_options.elitePoolSize, 1.0)); + + Population selectedPopulation = select(_population, rest.individuals().size()); + + Population crossedPopulation = Population::combine( + selectedPopulation.symmetricCrossoverWithRemainder( + PairsFromRandomSubset(m_options.crossoverChance), + symmetricRandomPointCrossover() + ) + ); + + std::function mutationOperator = mutationSequence({ + geneRandomisation(m_options.mutationChance), + geneDeletion(m_options.deletionChance), + geneAddition(m_options.additionChance), + }); + + RangeSelection all(0.0, 1.0); + Population mutatedPopulation = crossedPopulation.mutate(all, mutationOperator); + + return elite + mutatedPopulation; +} + +Population ClassicGeneticAlgorithm::select(Population _population, size_t _selectionSize) +{ + if (_population.individuals().size() == 0) + return _population; + + size_t maxFitness = 0; + for (auto const& individual: _population.individuals()) + maxFitness = max(maxFitness, individual.fitness); + + size_t rouletteRange = 0; + for (auto const& individual: _population.individuals()) + // Add 1 to make sure that every chromosome has non-zero probability of being chosen + rouletteRange += maxFitness + 1 - individual.fitness; + + vector selectedIndividuals; + for (size_t i = 0; i < _selectionSize; ++i) + { + uint32_t ball = SimulationRNG::uniformInt(0, rouletteRange - 1); + + size_t cumulativeFitness = 0; + for (auto const& individual: _population.individuals()) + { + size_t pocketSize = maxFitness + 1 - individual.fitness; + if (ball < cumulativeFitness + pocketSize) + { + selectedIndividuals.push_back(individual); + break; + } + cumulativeFitness += pocketSize; + } + } + + assert(selectedIndividuals.size() == _selectionSize); + return Population(_population.fitnessMetric(), selectedIndividuals); +} diff --git a/tools/yulPhaser/GeneticAlgorithms.h b/tools/yulPhaser/GeneticAlgorithms.h index a2a7484d674b..0d1f0375f12a 100644 --- a/tools/yulPhaser/GeneticAlgorithms.h +++ b/tools/yulPhaser/GeneticAlgorithms.h @@ -139,4 +139,59 @@ class GenerationalElitistWithExclusivePools: public GeneticAlgorithm Options m_options; }; +/** + * A typical genetic algorithm that works in three distinct phases, each resulting in a new, + * modified population: + * - selection: chromosomes are selected from the population with probability proportional to their + * fitness. A chromosome can be selected more than once. The new population has the same size as + * the old one. + * - crossover: first, for each chromosome we decide whether it undergoes crossover or not + * (according to crossover chance parameter). Then each selected chromosome is randomly paired + * with one other selected chromosome. Each pair produces a pair of children and gets replaced by + * it in the population. + * - mutation: we go over each gene in the population and independently decide whether to mutate it + * or not (according to mutation chance parameters). This is repeated for every mutation type so + * one gene can undergo mutations of multiple types in a single round. + * + * This implementation also has the ability to preserve the top chromosomes in each round. + */ +class ClassicGeneticAlgorithm: public GeneticAlgorithm +{ +public: + struct Options + { + double elitePoolSize; ///< Percentage of the population treated as the elite. + double crossoverChance; ///< The chance of a particular chromosome being selected for crossover. + double mutationChance; ///< The chance of a particular gene being randomised in @a geneRandomisation mutation. + double deletionChance; ///< The chance of a particular gene being deleted in @a geneDeletion mutation. + double additionChance; ///< The chance of a particular gene being added in @a geneAddition mutation. + + bool isValid() const + { + return ( + 0 <= elitePoolSize && elitePoolSize <= 1.0 && + 0 <= crossoverChance && crossoverChance <= 1.0 && + 0 <= mutationChance && mutationChance <= 1.0 && + 0 <= deletionChance && deletionChance <= 1.0 && + 0 <= additionChance && additionChance <= 1.0 + ); + } + }; + + ClassicGeneticAlgorithm(Options const& _options): + m_options(_options) + { + assert(_options.isValid()); + } + + Options const& options() const { return m_options; } + + Population runNextRound(Population _population) override; + +private: + static Population select(Population _population, size_t _selectionSize); + + Options m_options; +}; + } From 0efea99fa5e173a56364ddfee64783523bf9a8f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 11 Mar 2020 02:16:00 +0100 Subject: [PATCH 011/126] [yul-phaser] Phaser: Allow selecting the classic algorithm on the command line --- test/yulPhaser/Phaser.cpp | 17 ++++++++++++++ tools/yulPhaser/Phaser.cpp | 46 ++++++++++++++++++++++++++++++++++++++ tools/yulPhaser/Phaser.h | 6 +++++ 3 files changed, 69 insertions(+) diff --git a/test/yulPhaser/Phaser.cpp b/test/yulPhaser/Phaser.cpp index 633462865a23..c365ce310501 100644 --- a/test/yulPhaser/Phaser.cpp +++ b/test/yulPhaser/Phaser.cpp @@ -52,6 +52,11 @@ class GeneticAlgorithmFactoryFixture /* gewepDeletionVsAdditionChance = */ 0.3, /* gewepGenesToRandomise = */ 0.4, /* gewepGenesToAddOrDelete = */ 0.2, + /* classicElitePoolSize = */ 0.0, + /* classicCrossoverChance = */ 0.75, + /* classicMutationChance = */ 0.2, + /* classicDeletionChance = */ 0.2, + /* classicAdditionChance = */ 0.2, }; }; @@ -122,6 +127,18 @@ BOOST_FIXTURE_TEST_CASE(build_should_select_the_right_algorithm_and_pass_the_opt BOOST_TEST(gewepAlgorithm->options().deletionVsAdditionChance == m_options.gewepDeletionVsAdditionChance); BOOST_TEST(gewepAlgorithm->options().percentGenesToRandomise == m_options.gewepGenesToRandomise.value()); BOOST_TEST(gewepAlgorithm->options().percentGenesToAddOrDelete == m_options.gewepGenesToAddOrDelete.value()); + + m_options.algorithm = Algorithm::Classic; + unique_ptr algorithm3 = GeneticAlgorithmFactory::build(m_options, 100); + BOOST_REQUIRE(algorithm3 != nullptr); + + auto classicAlgorithm = dynamic_cast(algorithm3.get()); + BOOST_REQUIRE(classicAlgorithm != nullptr); + BOOST_TEST(classicAlgorithm->options().elitePoolSize == m_options.classicElitePoolSize); + BOOST_TEST(classicAlgorithm->options().crossoverChance == m_options.classicCrossoverChance); + BOOST_TEST(classicAlgorithm->options().mutationChance == m_options.classicMutationChance); + BOOST_TEST(classicAlgorithm->options().deletionChance == m_options.classicDeletionChance); + BOOST_TEST(classicAlgorithm->options().additionChance == m_options.classicAdditionChance); } BOOST_FIXTURE_TEST_CASE(build_should_set_random_algorithm_elite_pool_size_based_on_population_size_if_not_specified, GeneticAlgorithmFactoryFixture) diff --git a/tools/yulPhaser/Phaser.cpp b/tools/yulPhaser/Phaser.cpp index 2e38edbf097f..6f7be3257d2a 100644 --- a/tools/yulPhaser/Phaser.cpp +++ b/tools/yulPhaser/Phaser.cpp @@ -58,6 +58,7 @@ map const AlgorithmToStringMap = { {Algorithm::Random, "random"}, {Algorithm::GEWEP, "GEWEP"}, + {Algorithm::Classic, "classic"}, }; map const StringToAlgorithmMap = invertMap(AlgorithmToStringMap); @@ -107,6 +108,11 @@ GeneticAlgorithmFactory::Options GeneticAlgorithmFactory::Options::fromCommandLi _arguments.count("gewep-genes-to-add-or-delete") > 0 ? _arguments["gewep-genes-to-add-or-delete"].as() : optional{}, + _arguments["classic-elite-pool-size"].as(), + _arguments["classic-crossover-chance"].as(), + _arguments["classic-mutation-chance"].as(), + _arguments["classic-deletion-chance"].as(), + _arguments["classic-addition-chance"].as(), }; } @@ -151,6 +157,16 @@ unique_ptr GeneticAlgorithmFactory::build( /* percentGenesToAddOrDelete = */ percentGenesToAddOrDelete, }); } + case Algorithm::Classic: + { + return make_unique(ClassicGeneticAlgorithm::Options{ + /* elitePoolSize = */ _options.classicElitePoolSize, + /* crossoverChance = */ _options.classicCrossoverChance, + /* mutationChance = */ _options.classicMutationChance, + /* deletionChance = */ _options.classicDeletionChance, + /* additionChance = */ _options.classicAdditionChance, + }); + } default: assertThrow(false, solidity::util::Exception, "Invalid Algorithm value."); } @@ -475,6 +491,36 @@ Phaser::CommandLineDescription Phaser::buildCommandLineDescription() ; keywordDescription.add(gewepAlgorithmDescription); + po::options_description classicGeneticAlgorithmDescription("CLASSIC GENETIC ALGORITHM", lineLength, minDescriptionLength); + classicGeneticAlgorithmDescription.add_options() + ( + "classic-elite-pool-size", + po::value()->value_name("")->default_value(0), + "Percentage of population to regenerate using mutations in each round." + ) + ( + "classic-crossover-chance", + po::value()->value_name("")->default_value(0.75), + "Chance of a chromosome being selected for crossover." + ) + ( + "classic-mutation-chance", + po::value()->value_name("")->default_value(0.01), + "Chance of a gene being mutated." + ) + ( + "classic-deletion-chance", + po::value()->value_name("")->default_value(0.01), + "Chance of a gene being deleted." + ) + ( + "classic-addition-chance", + po::value()->value_name("")->default_value(0.01), + "Chance of a random gene being added." + ) + ; + keywordDescription.add(classicGeneticAlgorithmDescription); + po::options_description randomAlgorithmDescription("RANDOM ALGORITHM", lineLength, minDescriptionLength); randomAlgorithmDescription.add_options() ( diff --git a/tools/yulPhaser/Phaser.h b/tools/yulPhaser/Phaser.h index 77e9e48c8eae..2896dd090879 100644 --- a/tools/yulPhaser/Phaser.h +++ b/tools/yulPhaser/Phaser.h @@ -58,6 +58,7 @@ enum class Algorithm { Random, GEWEP, + Classic, }; enum class MetricChoice @@ -101,6 +102,11 @@ class GeneticAlgorithmFactory double gewepDeletionVsAdditionChance; std::optional gewepGenesToRandomise; std::optional gewepGenesToAddOrDelete; + double classicElitePoolSize; + double classicCrossoverChance; + double classicMutationChance; + double classicDeletionChance; + double classicAdditionChance; static Options fromCommandLine(boost::program_options::variables_map const& _arguments); }; From 5203503583b637d9917f2ae6fb892cb11751864d Mon Sep 17 00:00:00 2001 From: Mathias Baumann Date: Mon, 6 Apr 2020 14:47:44 +0200 Subject: [PATCH 012/126] Allow for per-parameter literalValues builtin functions --- libyul/AsmAnalysis.cpp | 10 +++--- libyul/Dialect.h | 5 +-- libyul/backends/evm/EVMDialect.cpp | 21 ++++++----- libyul/backends/evm/EVMDialect.h | 2 +- libyul/backends/wasm/WasmCodeTransform.cpp | 10 ++++-- libyul/backends/wasm/WasmDialect.cpp | 13 ++++--- libyul/backends/wasm/WasmDialect.h | 2 +- libyul/backends/wasm/WordSizeTransform.cpp | 36 +++++++++---------- libyul/backends/wasm/WordSizeTransform.h | 1 - .../CommonSubexpressionEliminator.cpp | 15 ++++++-- libyul/optimiser/ExpressionSplitter.cpp | 10 +++--- test/libyul/Parser.cpp | 2 +- 12 files changed, 75 insertions(+), 52 deletions(-) diff --git a/libyul/AsmAnalysis.cpp b/libyul/AsmAnalysis.cpp index 91f8181003ec..ca3f88c753e7 100644 --- a/libyul/AsmAnalysis.cpp +++ b/libyul/AsmAnalysis.cpp @@ -255,14 +255,14 @@ vector AsmAnalyzer::operator()(FunctionCall const& _funCall) yulAssert(!_funCall.functionName.name.empty(), ""); vector const* parameterTypes = nullptr; vector const* returnTypes = nullptr; - bool needsLiteralArguments = false; + vector const* needsLiteralArguments = nullptr; if (BuiltinFunction const* f = m_dialect.builtin(_funCall.functionName.name)) { parameterTypes = &f->parameters; returnTypes = &f->returns; if (f->literalArguments) - needsLiteralArguments = true; + needsLiteralArguments = &f->literalArguments.value(); } else if (!m_currentScope->lookup(_funCall.functionName.name, GenericVisitor{ [&](Scope::Variable const&) @@ -293,11 +293,13 @@ vector AsmAnalyzer::operator()(FunctionCall const& _funCall) ); vector argTypes; - for (auto const& arg: _funCall.arguments | boost::adaptors::reversed) + for (size_t i = _funCall.arguments.size(); i > 0; i--) { + Expression const& arg = _funCall.arguments[i - 1]; + argTypes.emplace_back(expectExpression(arg)); - if (needsLiteralArguments) + if (needsLiteralArguments && (*needsLiteralArguments)[i - 1]) { if (!holds_alternative(arg)) typeError( diff --git a/libyul/Dialect.h b/libyul/Dialect.h index a2a5961775db..870205833659 100644 --- a/libyul/Dialect.h +++ b/libyul/Dialect.h @@ -28,6 +28,7 @@ #include #include +#include namespace solidity::yul { @@ -46,8 +47,8 @@ struct BuiltinFunction ControlFlowSideEffects controlFlowSideEffects; /// If true, this is the msize instruction. bool isMSize = false; - /// If true, can only accept literals as arguments and they cannot be moved to variables. - bool literalArguments = false; + /// If set, same length as the arguments, if true at index i, the i'th argument has to be a literal which means it can't be moved to variables. + std::optional> literalArguments; }; struct Dialect: boost::noncopyable diff --git a/libyul/backends/evm/EVMDialect.cpp b/libyul/backends/evm/EVMDialect.cpp index ea6586323ce6..52cd1d23f48a 100644 --- a/libyul/backends/evm/EVMDialect.cpp +++ b/libyul/backends/evm/EVMDialect.cpp @@ -55,7 +55,7 @@ pair createEVMFunction( f.controlFlowSideEffects.terminates = evmasm::SemanticInformation::terminatesControlFlow(_instruction); f.controlFlowSideEffects.reverts = evmasm::SemanticInformation::reverts(_instruction); f.isMSize = _instruction == evmasm::Instruction::MSIZE; - f.literalArguments = false; + f.literalArguments.reset(); f.instruction = _instruction; f.generateCode = [_instruction]( FunctionCall const&, @@ -75,17 +75,22 @@ pair createFunction( size_t _params, size_t _returns, SideEffects _sideEffects, - bool _literalArguments, + vector _literalArguments, std::function)> _generateCode ) { + solAssert(_literalArguments.size() == _params || _literalArguments.empty(), ""); + YulString name{std::move(_name)}; BuiltinFunctionForEVM f; f.name = name; f.parameters.resize(_params); f.returns.resize(_returns); f.sideEffects = std::move(_sideEffects); - f.literalArguments = _literalArguments; + if (!_literalArguments.empty()) + f.literalArguments = std::move(_literalArguments); + else + f.literalArguments.reset(); f.isMSize = false; f.instruction = {}; f.generateCode = std::move(_generateCode); @@ -107,7 +112,7 @@ map createBuiltins(langutil::EVMVersion _evmVe if (_objectAccess) { - builtins.emplace(createFunction("datasize", 1, 1, SideEffects{}, true, []( + builtins.emplace(createFunction("datasize", 1, 1, SideEffects{}, {true}, []( FunctionCall const& _call, AbstractAssembly& _assembly, BuiltinContext& _context, @@ -128,7 +133,7 @@ map createBuiltins(langutil::EVMVersion _evmVe _assembly.appendDataSize(_context.subIDs.at(dataName)); } })); - builtins.emplace(createFunction("dataoffset", 1, 1, SideEffects{}, true, []( + builtins.emplace(createFunction("dataoffset", 1, 1, SideEffects{}, {true}, []( FunctionCall const& _call, AbstractAssembly& _assembly, BuiltinContext& _context, @@ -154,7 +159,7 @@ map createBuiltins(langutil::EVMVersion _evmVe 3, 0, SideEffects{false, false, false, false, true}, - false, + {}, []( FunctionCall const&, AbstractAssembly& _assembly, @@ -262,7 +267,7 @@ EVMDialectTyped::EVMDialectTyped(langutil::EVMVersion _evmVersion, bool _objectA m_functions["popbool"_yulstring] = m_functions["pop"_yulstring]; m_functions["popbool"_yulstring].name = "popbool"_yulstring; m_functions["popbool"_yulstring].parameters = {"bool"_yulstring}; - m_functions.insert(createFunction("bool_to_u256", 1, 1, {}, false, []( + m_functions.insert(createFunction("bool_to_u256", 1, 1, {}, {}, []( FunctionCall const&, AbstractAssembly&, BuiltinContext&, @@ -272,7 +277,7 @@ EVMDialectTyped::EVMDialectTyped(langutil::EVMVersion _evmVersion, bool _objectA })); m_functions["bool_to_u256"_yulstring].parameters = {"bool"_yulstring}; m_functions["bool_to_u256"_yulstring].returns = {"u256"_yulstring}; - m_functions.insert(createFunction("u256_to_bool", 1, 1, {}, false, []( + m_functions.insert(createFunction("u256_to_bool", 1, 1, {}, {}, []( FunctionCall const&, AbstractAssembly& _assembly, BuiltinContext&, diff --git a/libyul/backends/evm/EVMDialect.h b/libyul/backends/evm/EVMDialect.h index 2141ab98f5da..00852fe34b15 100644 --- a/libyul/backends/evm/EVMDialect.h +++ b/libyul/backends/evm/EVMDialect.h @@ -45,7 +45,7 @@ struct BuiltinContext std::map subIDs; }; -struct BuiltinFunctionForEVM: BuiltinFunction +struct BuiltinFunctionForEVM: public BuiltinFunction { std::optional instruction; /// Function to generate code for the given function call and append it to the abstract diff --git a/libyul/backends/wasm/WasmCodeTransform.cpp b/libyul/backends/wasm/WasmCodeTransform.cpp index f3ff04cd9fe7..cb1294d4424e 100644 --- a/libyul/backends/wasm/WasmCodeTransform.cpp +++ b/libyul/backends/wasm/WasmCodeTransform.cpp @@ -136,11 +136,15 @@ wasm::Expression WasmCodeTransform::operator()(FunctionCall const& _call) } typeConversionNeeded = true; } - else if (builtin->literalArguments) + else if (builtin->literalArguments && contains(builtin->literalArguments.value(), true)) { vector literals; - for (auto const& arg: _call.arguments) - literals.emplace_back(wasm::StringLiteral{std::get(arg).value.str()}); + for (size_t i = 0; i < _call.arguments.size(); i++) + if (builtin->literalArguments.value()[i]) + literals.emplace_back(wasm::StringLiteral{std::get(_call.arguments[i]).value.str()}); + else + literals.emplace_back(visitReturnByValue(_call.arguments[i])); + return wasm::BuiltinCall{_call.functionName.name.str(), std::move(literals)}; } else diff --git a/libyul/backends/wasm/WasmDialect.cpp b/libyul/backends/wasm/WasmDialect.cpp index c9817af7668c..c9dfe05c78e0 100644 --- a/libyul/backends/wasm/WasmDialect.cpp +++ b/libyul/backends/wasm/WasmDialect.cpp @@ -102,8 +102,8 @@ WasmDialect::WasmDialect() m_functions["unreachable"_yulstring].controlFlowSideEffects.terminates = true; m_functions["unreachable"_yulstring].controlFlowSideEffects.reverts = true; - addFunction("datasize", {i64}, {i64}, true, true); - addFunction("dataoffset", {i64}, {i64}, true, true); + addFunction("datasize", {i64}, {i64}, true, {true}); + addFunction("dataoffset", {i64}, {i64}, true, {true}); addEthereumExternals(); } @@ -204,7 +204,7 @@ void WasmDialect::addEthereumExternals() f.controlFlowSideEffects = ext.controlFlowSideEffects; f.isMSize = false; f.sideEffects.invalidatesStorage = (ext.name == "storageStore"); - f.literalArguments = false; + f.literalArguments.reset(); } } @@ -213,7 +213,7 @@ void WasmDialect::addFunction( vector _params, vector _returns, bool _movable, - bool _literalArguments + std::vector _literalArguments ) { YulString name{move(_name)}; @@ -224,5 +224,8 @@ void WasmDialect::addFunction( f.returns = std::move(_returns); f.sideEffects = _movable ? SideEffects{} : SideEffects::worst(); f.isMSize = false; - f.literalArguments = _literalArguments; + if (!_literalArguments.empty()) + f.literalArguments = std::move(_literalArguments); + else + f.literalArguments.reset(); } diff --git a/libyul/backends/wasm/WasmDialect.h b/libyul/backends/wasm/WasmDialect.h index 1de65dfdc898..e5f3c9fc4315 100644 --- a/libyul/backends/wasm/WasmDialect.h +++ b/libyul/backends/wasm/WasmDialect.h @@ -61,7 +61,7 @@ struct WasmDialect: public Dialect std::vector _params, std::vector _returns, bool _movable = true, - bool _literalArguments = false + std::vector _literalArguments = std::vector{} ); std::map m_functions; diff --git a/libyul/backends/wasm/WordSizeTransform.cpp b/libyul/backends/wasm/WordSizeTransform.cpp index eec04fae62c4..bc450a8e4891 100644 --- a/libyul/backends/wasm/WordSizeTransform.cpp +++ b/libyul/backends/wasm/WordSizeTransform.cpp @@ -41,15 +41,24 @@ void WordSizeTransform::operator()(FunctionDefinition& _fd) void WordSizeTransform::operator()(FunctionCall& _fc) { + vector const* literalArguments = nullptr; + if (BuiltinFunction const* fun = m_inputDialect.builtin(_fc.functionName.name)) if (fun->literalArguments) + literalArguments = &fun->literalArguments.value(); + + vector newArgs; + + for (size_t i = 0; i < _fc.arguments.size(); i++) + if (!literalArguments || !(*literalArguments)[i]) + newArgs += expandValueToVector(_fc.arguments[i]); + else { - for (Expression& arg: _fc.arguments) - get(arg).type = m_targetDialect.defaultType; - return; + get(_fc.arguments[i]).type = m_targetDialect.defaultType; + newArgs.emplace_back(std::move(_fc.arguments[i])); } - rewriteFunctionCallArguments(_fc.arguments); + _fc.arguments = std::move(newArgs); } void WordSizeTransform::operator()(If& _if) @@ -97,9 +106,9 @@ void WordSizeTransform::operator()(Block& _block) // Special handling for datasize and dataoffset - they will only need one variable. if (BuiltinFunction const* f = m_inputDialect.builtin(std::get(*varDecl.value).functionName.name)) - if (f->literalArguments) + if (f->name == "datasize"_yulstring || f->name == "dataoffset"_yulstring) { - yulAssert(f->name == "datasize"_yulstring || f->name == "dataoffset"_yulstring, ""); + yulAssert(f->literalArguments && f->literalArguments.value()[0], ""); yulAssert(varDecl.variables.size() == 1, ""); auto newLhs = generateU64IdentifierNames(varDecl.variables[0].name); vector ret; @@ -158,9 +167,9 @@ void WordSizeTransform::operator()(Block& _block) // Special handling for datasize and dataoffset - they will only need one variable. if (BuiltinFunction const* f = m_inputDialect.builtin(std::get(*assignment.value).functionName.name)) - if (f->literalArguments) + if (f->name == "datasize"_yulstring || f->name == "dataoffset"_yulstring) { - yulAssert(f->name == "datasize"_yulstring || f->name == "dataoffset"_yulstring, ""); + yulAssert(f->literalArguments && f->literalArguments.value()[0], ""); yulAssert(assignment.variableNames.size() == 1, ""); auto newLhs = generateU64IdentifierNames(assignment.variableNames[0].name); vector ret; @@ -268,17 +277,6 @@ void WordSizeTransform::rewriteIdentifierList(vector& _ids) ); } -void WordSizeTransform::rewriteFunctionCallArguments(vector& _args) -{ - iterateReplacing( - _args, - [&](Expression& _e) -> std::optional> - { - return expandValueToVector(_e); - } - ); -} - vector WordSizeTransform::handleSwitchInternal( langutil::SourceLocation const& _location, vector const& _splitExpressions, diff --git a/libyul/backends/wasm/WordSizeTransform.h b/libyul/backends/wasm/WordSizeTransform.h index 67dfe886439f..5771afea82ef 100644 --- a/libyul/backends/wasm/WordSizeTransform.h +++ b/libyul/backends/wasm/WordSizeTransform.h @@ -83,7 +83,6 @@ class WordSizeTransform: public ASTModifier void rewriteVarDeclList(std::vector&); void rewriteIdentifierList(std::vector&); - void rewriteFunctionCallArguments(std::vector&); std::vector handleSwitch(Switch& _switch); std::vector handleSwitchInternal( diff --git a/libyul/optimiser/CommonSubexpressionEliminator.cpp b/libyul/optimiser/CommonSubexpressionEliminator.cpp index 84074ad2773b..3b0b6bec179e 100644 --- a/libyul/optimiser/CommonSubexpressionEliminator.cpp +++ b/libyul/optimiser/CommonSubexpressionEliminator.cpp @@ -58,12 +58,21 @@ void CommonSubexpressionEliminator::visit(Expression& _e) // If this is a function call to a function that requires literal arguments, // do not try to simplify there. if (holds_alternative(_e)) - if (BuiltinFunction const* builtin = m_dialect.builtin(std::get(_e).functionName.name)) - if (builtin->literalArguments) + { + FunctionCall& funCall = std::get(_e); + + if (BuiltinFunction const* builtin = m_dialect.builtin(funCall.functionName.name)) + { + for (size_t i = funCall.arguments.size(); i > 0; i--) // We should not modify function arguments that have to be literals // Note that replacing the function call entirely is fine, // if the function call is movable. - descend = false; + if (!builtin->literalArguments || !builtin->literalArguments.value()[i - 1]) + visit(funCall.arguments[i - 1]); + + descend = false; + } + } // We visit the inner expression first to first simplify inner expressions, // which hopefully allows more matches. diff --git a/libyul/optimiser/ExpressionSplitter.cpp b/libyul/optimiser/ExpressionSplitter.cpp index 7b2ac14f6235..8144de49a788 100644 --- a/libyul/optimiser/ExpressionSplitter.cpp +++ b/libyul/optimiser/ExpressionSplitter.cpp @@ -47,13 +47,15 @@ void ExpressionSplitter::run(OptimiserStepContext& _context, Block& _ast) void ExpressionSplitter::operator()(FunctionCall& _funCall) { + vector const* literalArgs = nullptr; + if (BuiltinFunction const* builtin = m_dialect.builtin(_funCall.functionName.name)) if (builtin->literalArguments) - // We cannot outline function arguments that have to be literals - return; + literalArgs = &builtin->literalArguments.value(); - for (auto& arg: _funCall.arguments | boost::adaptors::reversed) - outlineExpression(arg); + for (size_t i = _funCall.arguments.size(); i > 0; i--) + if (!literalArgs || !(*literalArgs)[i - 1]) + outlineExpression(_funCall.arguments[i - 1]); } void ExpressionSplitter::operator()(If& _if) diff --git a/test/libyul/Parser.cpp b/test/libyul/Parser.cpp index 93b5287218ad..7b8e56119d09 100644 --- a/test/libyul/Parser.cpp +++ b/test/libyul/Parser.cpp @@ -539,7 +539,7 @@ BOOST_AUTO_TEST_CASE(builtins_analysis) { return _name == "builtin"_yulstring ? &f : nullptr; } - BuiltinFunction f{"builtin"_yulstring, vector(2), vector(3), {}, {}}; + BuiltinFunction f{"builtin"_yulstring, vector(2), vector(3), {}, {}, false, {}}; }; SimpleDialect dialect; From 5d9dd654cf6152a63b19ec08a905a9cd5691a61d Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Fri, 3 Apr 2020 15:10:16 +0200 Subject: [PATCH 013/126] [SMTChecker] Add and use tuple sort --- libsolidity/formal/CVC4Interface.cpp | 15 ++++++ libsolidity/formal/SMTEncoder.cpp | 64 ++++++++++++------------ libsolidity/formal/SMTLib2Interface.cpp | 26 ++++++++++ libsolidity/formal/SMTLib2Interface.h | 1 + libsolidity/formal/SolverInterface.h | 16 +++++- libsolidity/formal/Sorts.h | 43 +++++++++++++++- libsolidity/formal/SymbolicTypes.cpp | 43 ++++++++++++++++ libsolidity/formal/SymbolicTypes.h | 2 + libsolidity/formal/SymbolicVariables.cpp | 55 ++++++++++++-------- libsolidity/formal/SymbolicVariables.h | 18 ++++--- libsolidity/formal/Z3Interface.cpp | 27 ++++++++++ 11 files changed, 248 insertions(+), 62 deletions(-) diff --git a/libsolidity/formal/CVC4Interface.cpp b/libsolidity/formal/CVC4Interface.cpp index 9aecc2586862..06778606189d 100644 --- a/libsolidity/formal/CVC4Interface.cpp +++ b/libsolidity/formal/CVC4Interface.cpp @@ -196,6 +196,16 @@ CVC4::Expr CVC4Interface::toCVC4Expr(Expression const& _expr) solAssert(sortSort, ""); return m_context.mkConst(CVC4::ArrayStoreAll(cvc4Sort(*sortSort->inner), arguments[1])); } + else if (n == "tuple_get") + { + shared_ptr tupleSort = std::dynamic_pointer_cast(_expr.arguments[0].sort); + solAssert(tupleSort, ""); + CVC4::DatatypeType tt = m_context.mkTupleType(cvc4Sort(tupleSort->components)); + CVC4::Datatype const& dt = tt.getDatatype(); + size_t index = std::stoi(_expr.arguments[1].name); + CVC4::Expr s = dt[0][index].getSelector(); + return m_context.mkExpr(CVC4::kind::APPLY_SELECTOR, s, arguments[0]); + } solAssert(false, ""); } @@ -229,6 +239,11 @@ CVC4::Type CVC4Interface::cvc4Sort(Sort const& _sort) auto const& arraySort = dynamic_cast(_sort); return m_context.mkArrayType(cvc4Sort(*arraySort.domain), cvc4Sort(*arraySort.range)); } + case Kind::Tuple: + { + auto const& tupleSort = dynamic_cast(_sort); + return m_context.mkTupleType(cvc4Sort(tupleSort.components)); + } default: break; } diff --git a/libsolidity/formal/SMTEncoder.cpp b/libsolidity/formal/SMTEncoder.cpp index f02d32aae229..67a4885072a6 100644 --- a/libsolidity/formal/SMTEncoder.cpp +++ b/libsolidity/formal/SMTEncoder.cpp @@ -296,16 +296,22 @@ void SMTEncoder::endVisit(VariableDeclarationStatement const& _varDecl) { auto symbTuple = dynamic_pointer_cast(m_context.expression(*init)); solAssert(symbTuple, ""); - auto const& components = symbTuple->components(); + auto const& symbComponents = symbTuple->components(); + + auto tupleType = dynamic_cast(init->annotation().type); + solAssert(tupleType, ""); + solAssert(tupleType->components().size() == symbTuple->components().size(), ""); + auto const& components = tupleType->components(); + auto const& declarations = _varDecl.declarations(); - solAssert(components.size() == declarations.size(), ""); + solAssert(symbComponents.size() == declarations.size(), ""); for (unsigned i = 0; i < declarations.size(); ++i) if ( components.at(i) && declarations.at(i) && m_context.knownVariable(*declarations.at(i)) ) - assignment(*declarations.at(i), components.at(i)->currentValue(declarations.at(i)->type())); + assignment(*declarations.at(i), symbTuple->component(i, components.at(i), declarations.at(i)->type())); } } else if (m_context.knownVariable(*_varDecl.declarations().front())) @@ -354,7 +360,7 @@ void SMTEncoder::endVisit(Assignment const& _assignment) { auto const& type = _assignment.annotation().type; vector rightArguments; - if (_assignment.rightHandSide().annotation().type->category() == Type::Category::Tuple) + if (auto const* tupleTypeRight = dynamic_cast(_assignment.rightHandSide().annotation().type)) { auto symbTupleLeft = dynamic_pointer_cast(m_context.expression(_assignment.leftHandSide())); solAssert(symbTupleLeft, ""); @@ -365,17 +371,16 @@ void SMTEncoder::endVisit(Assignment const& _assignment) auto const& rightComponents = symbTupleRight->components(); solAssert(leftComponents.size() == rightComponents.size(), ""); - for (unsigned i = 0; i < leftComponents.size(); ++i) - { - auto const& left = leftComponents.at(i); - auto const& right = rightComponents.at(i); - /// Right hand side tuple component cannot be empty. - solAssert(right, ""); - if (left) - rightArguments.push_back(right->currentValue(left->originalType())); - else - rightArguments.push_back(right->currentValue()); - } + auto tupleTypeLeft = dynamic_cast(_assignment.leftHandSide().annotation().type); + solAssert(tupleTypeLeft, ""); + solAssert(tupleTypeLeft->components().size() == leftComponents.size(), ""); + auto const& typesLeft = tupleTypeLeft->components(); + + solAssert(tupleTypeRight->components().size() == rightComponents.size(), ""); + auto const& typesRight = tupleTypeRight->components(); + + for (unsigned i = 0; i < rightComponents.size(); ++i) + rightArguments.push_back(symbTupleRight->component(i, typesRight.at(i), typesLeft.at(i))); } else { @@ -418,17 +423,16 @@ void SMTEncoder::endVisit(TupleExpression const& _tuple) solAssert(symbComponents.size() == tupleComponents->size(), ""); for (unsigned i = 0; i < symbComponents.size(); ++i) { - auto sComponent = symbComponents.at(i); auto tComponent = tupleComponents->at(i); - if (sComponent && tComponent) + if (tComponent) { if (auto varDecl = identifierToVariable(*tComponent)) - m_context.addAssertion(sComponent->currentValue() == currentValue(*varDecl)); + m_context.addAssertion(symbTuple->component(i) == currentValue(*varDecl)); else { if (!m_context.knownExpression(*tComponent)) createExpr(*tComponent); - m_context.addAssertion(sComponent->currentValue() == expr(*tComponent)); + m_context.addAssertion(symbTuple->component(i) == expr(*tComponent)); } } } @@ -807,13 +811,15 @@ void SMTEncoder::endVisit(Return const& _return) { auto const& symbTuple = dynamic_pointer_cast(m_context.expression(*_return.expression())); solAssert(symbTuple, ""); - auto const& components = symbTuple->components(); - solAssert(components.size() == returnParams.size(), ""); + solAssert(symbTuple->components().size() == returnParams.size(), ""); + + auto const* tupleType = dynamic_cast(_return.expression()->annotation().type); + solAssert(tupleType, ""); + auto const& types = tupleType->components(); + solAssert(types.size() == returnParams.size(), ""); + for (unsigned i = 0; i < returnParams.size(); ++i) - { - solAssert(components.at(i), ""); - m_context.addAssertion(components.at(i)->currentValue(returnParams.at(i)->type()) == m_context.newValue(*returnParams.at(i))); - } + m_context.addAssertion(symbTuple->component(i, types.at(i), returnParams.at(i)->type()) == m_context.newValue(*returnParams.at(i))); } else if (returnParams.size() == 1) m_context.addAssertion(expr(*_return.expression(), returnParams.front()->type()) == m_context.newValue(*returnParams.front())); @@ -1676,14 +1682,10 @@ void SMTEncoder::createReturnedExpressions(FunctionCall const& _funCall) solAssert(symbComponents.size() == returnParams.size(), ""); for (unsigned i = 0; i < symbComponents.size(); ++i) { - auto sComponent = symbComponents.at(i); auto param = returnParams.at(i); solAssert(param, ""); - if (sComponent) - { - solAssert(m_context.knownVariable(*param), ""); - m_context.addAssertion(sComponent->currentValue() == currentValue(*param)); - } + solAssert(m_context.knownVariable(*param), ""); + m_context.addAssertion(symbTuple->component(i) == currentValue(*param)); } } else if (returnParams.size() == 1) diff --git a/libsolidity/formal/SMTLib2Interface.cpp b/libsolidity/formal/SMTLib2Interface.cpp index 74529b4c5c6e..0c6b8715de40 100644 --- a/libsolidity/formal/SMTLib2Interface.cpp +++ b/libsolidity/formal/SMTLib2Interface.cpp @@ -28,6 +28,7 @@ #include #include #include +#include using namespace std; using namespace solidity; @@ -50,6 +51,7 @@ void SMTLib2Interface::reset() m_accumulatedOutput.clear(); m_accumulatedOutput.emplace_back(); m_variables.clear(); + m_userSorts.clear(); write("(set-option :produce-models true)"); write("(set-logic ALL)"); } @@ -145,6 +147,14 @@ string SMTLib2Interface::toSExpr(smt::Expression const& _expr) sexpr += "(as const " + toSmtLibSort(*arraySort) + ") "; sexpr += toSExpr(_expr.arguments.at(1)); } + else if (_expr.name == "tuple_get") + { + solAssert(_expr.arguments.size() == 2, ""); + auto tupleSort = dynamic_pointer_cast(_expr.arguments.at(0).sort); + unsigned index = std::stoi(_expr.arguments.at(1).name); + solAssert(index < tupleSort->members.size(), ""); + sexpr += tupleSort->members.at(index) + " " + toSExpr(_expr.arguments.at(0)); + } else { sexpr += _expr.name; @@ -169,6 +179,22 @@ string SMTLib2Interface::toSmtLibSort(Sort const& _sort) solAssert(arraySort.domain && arraySort.range, ""); return "(Array " + toSmtLibSort(*arraySort.domain) + ' ' + toSmtLibSort(*arraySort.range) + ')'; } + case Kind::Tuple: + { + auto const& tupleSort = dynamic_cast(_sort); + if (!m_userSorts.count(tupleSort.name)) + { + m_userSorts.insert(tupleSort.name); + string decl("(declare-datatypes ((" + tupleSort.name + " 0)) (((" + tupleSort.name); + solAssert(tupleSort.members.size() == tupleSort.components.size(), ""); + for (unsigned i = 0; i < tupleSort.members.size(); ++i) + decl += " (" + tupleSort.members.at(i) + " " + toSmtLibSort(*tupleSort.components.at(i)) + ")"; + decl += "))))"; + write(decl); + } + + return tupleSort.name; + } default: solAssert(false, "Invalid SMT sort"); } diff --git a/libsolidity/formal/SMTLib2Interface.h b/libsolidity/formal/SMTLib2Interface.h index 17d385cb2f9e..e3ebd8b5eca8 100644 --- a/libsolidity/formal/SMTLib2Interface.h +++ b/libsolidity/formal/SMTLib2Interface.h @@ -74,6 +74,7 @@ class SMTLib2Interface: public SolverInterface, public boost::noncopyable std::vector m_accumulatedOutput; std::map m_variables; + std::set m_userSorts; std::map const& m_queryResponses; std::vector m_unhandledQueries; diff --git a/libsolidity/formal/SolverInterface.h b/libsolidity/formal/SolverInterface.h index 106ada82092c..cac412c4bb12 100644 --- a/libsolidity/formal/SolverInterface.h +++ b/libsolidity/formal/SolverInterface.h @@ -94,7 +94,8 @@ class Expression {"mod", 2}, {"select", 2}, {"store", 3}, - {"const_array", 2} + {"const_array", 2}, + {"tuple_get", 2} }; return operatorsArity.count(name) && operatorsArity.at(name) == arguments.size(); } @@ -166,6 +167,19 @@ class Expression ); } + static Expression tuple_get(Expression _tuple, size_t _index) + { + solAssert(_tuple.sort->kind == Kind::Tuple, ""); + std::shared_ptr tupleSort = std::dynamic_pointer_cast(_tuple.sort); + solAssert(tupleSort, ""); + solAssert(_index < tupleSort->components.size(), ""); + return Expression( + "tuple_get", + std::vector{std::move(_tuple), Expression(_index)}, + tupleSort->components.at(_index) + ); + } + friend Expression operator!(Expression _a) { return Expression("not", std::move(_a), Kind::Bool); diff --git a/libsolidity/formal/Sorts.h b/libsolidity/formal/Sorts.h index d266444625ea..2acd4697523b 100644 --- a/libsolidity/formal/Sorts.h +++ b/libsolidity/formal/Sorts.h @@ -33,7 +33,8 @@ enum class Kind Bool, Function, Array, - Sort + Sort, + Tuple }; struct Sort @@ -115,6 +116,46 @@ struct SortSort: public Sort SortPointer inner; }; +struct TupleSort: public Sort +{ + TupleSort( + std::string _name, + std::vector _members, + std::vector _components + ): + Sort(Kind::Tuple), + name(std::move(_name)), + members(std::move(_members)), + components(std::move(_components)) + {} + + bool operator==(Sort const& _other) const override + { + if (!Sort::operator==(_other)) + return false; + auto _otherTuple = dynamic_cast(&_other); + solAssert(_otherTuple, ""); + if (name != _otherTuple->name) + return false; + if (members != _otherTuple->members) + return false; + if (components.size() != _otherTuple->components.size()) + return false; + if (!std::equal( + components.begin(), + components.end(), + _otherTuple->components.begin(), + [&](SortPointer _a, SortPointer _b) { return *_a == *_b; } + )) + return false; + return true; + } + + std::string const name; + std::vector const members; + std::vector const components; +}; + /** Frequently used sorts.*/ struct SortProvider { diff --git a/libsolidity/formal/SymbolicTypes.cpp b/libsolidity/formal/SymbolicTypes.cpp index 1d9655a9c909..b90500ce40f5 100644 --- a/libsolidity/formal/SymbolicTypes.cpp +++ b/libsolidity/formal/SymbolicTypes.cpp @@ -75,6 +75,23 @@ SortPointer smtSort(frontend::Type const& _type) return make_shared(SortProvider::intSort, smtSortAbstractFunction(*arrayType->baseType())); } } + case Kind::Tuple: + { + auto tupleType = dynamic_cast(&_type); + solAssert(tupleType, ""); + vector members; + static unsigned tupleTypeId = 0; + for (auto const& component: tupleType->components()) + if (component) + members.emplace_back(component->identifier() + "_" + to_string(tupleTypeId++)); + else + members.emplace_back("null_type_" + to_string(tupleTypeId++)); + return make_shared( + _type.identifier() + "_" + to_string(tupleTypeId++), + members, + smtSortAbstractFunction(tupleType->components()) + ); + } default: // Abstract case. return SortProvider::intSort; @@ -96,6 +113,17 @@ SortPointer smtSortAbstractFunction(frontend::Type const& _type) return smtSort(_type); } +vector smtSortAbstractFunction(vector const& _types) +{ + vector sorts; + for (auto const& type: _types) + if (type) + sorts.push_back(smtSortAbstractFunction(*type)); + else + sorts.push_back(SortProvider::intSort); + return sorts; +} + Kind smtKind(frontend::Type::Category _category) { if (isNumber(_category)) @@ -106,6 +134,8 @@ Kind smtKind(frontend::Type::Category _category) return Kind::Function; else if (isMapping(_category) || isArray(_category)) return Kind::Array; + else if (isTuple(_category)) + return Kind::Tuple; // Abstract case. return Kind::Int; } @@ -350,4 +380,17 @@ void setSymbolicUnknownValue(Expression _expr, frontend::TypePointer const& _typ } } +optional symbolicTypeConversion(TypePointer _from, TypePointer _to) +{ + if (_to && _from) + // StringLiterals are encoded as SMT arrays in the generic case, + // but they can also be compared/assigned to fixed bytes, in which + // case they'd need to be encoded as numbers. + if (auto strType = dynamic_cast(_from)) + if (_to->category() == frontend::Type::Category::FixedBytes) + return smt::Expression(u256(toHex(util::asBytes(strType->value()), util::HexPrefix::Add))); + + return std::nullopt; +} + } diff --git a/libsolidity/formal/SymbolicTypes.h b/libsolidity/formal/SymbolicTypes.h index 55c88811d84d..a05d606fe53a 100644 --- a/libsolidity/formal/SymbolicTypes.h +++ b/libsolidity/formal/SymbolicTypes.h @@ -31,6 +31,7 @@ std::vector smtSort(std::vector const& _type /// If _type has type Function, abstract it to Integer. /// Otherwise return smtSort(_type). SortPointer smtSortAbstractFunction(frontend::Type const& _type); +std::vector smtSortAbstractFunction(std::vector const& _types); /// Returns the SMT kind that models the Solidity type type category _category. Kind smtKind(frontend::Type::Category _category); @@ -69,4 +70,5 @@ void setSymbolicZeroValue(Expression _expr, frontend::TypePointer const& _type, void setSymbolicUnknownValue(SymbolicVariable const& _variable, EncodingContext& _context); void setSymbolicUnknownValue(Expression _expr, frontend::TypePointer const& _type, EncodingContext& _context); +std::optional symbolicTypeConversion(TypePointer _from, TypePointer _to); } diff --git a/libsolidity/formal/SymbolicVariables.cpp b/libsolidity/formal/SymbolicVariables.cpp index 549fc05e9652..35cd4f9c26c6 100644 --- a/libsolidity/formal/SymbolicVariables.cpp +++ b/libsolidity/formal/SymbolicVariables.cpp @@ -230,16 +230,9 @@ SymbolicArrayVariable::SymbolicArrayVariable( smt::Expression SymbolicArrayVariable::currentValue(frontend::TypePointer const& _targetType) const { - if (_targetType) - { - solAssert(m_originalType, ""); - // StringLiterals are encoded as SMT arrays in the generic case, - // but they can also be compared/assigned to fixed bytes, in which - // case they'd need to be encoded as numbers. - if (auto strType = dynamic_cast(m_originalType)) - if (_targetType->category() == frontend::Type::Category::FixedBytes) - return smt::Expression(u256(toHex(util::asBytes(strType->value()), util::HexPrefix::Add))); - } + optional conversion = symbolicTypeConversion(m_originalType, _targetType); + if (conversion) + return *conversion; return SymbolicVariable::currentValue(_targetType); } @@ -262,16 +255,34 @@ SymbolicTupleVariable::SymbolicTupleVariable( SymbolicVariable(_type, _type, move(_uniqueName), _context) { solAssert(isTuple(m_type->category()), ""); - auto const& tupleType = dynamic_cast(*m_type); - auto const& componentsTypes = tupleType.components(); - for (unsigned i = 0; i < componentsTypes.size(); ++i) - if (componentsTypes.at(i)) - { - string componentName = m_uniqueName + "_component_" + to_string(i); - auto result = smt::newSymbolicVariable(*componentsTypes.at(i), componentName, m_context); - solAssert(result.second, ""); - m_components.emplace_back(move(result.second)); - } - else - m_components.emplace_back(nullptr); +} + +SymbolicTupleVariable::SymbolicTupleVariable( + SortPointer _sort, + string _uniqueName, + EncodingContext& _context +): + SymbolicVariable(move(_sort), move(_uniqueName), _context) +{ + solAssert(m_sort->kind == Kind::Tuple, ""); +} + +vector const& SymbolicTupleVariable::components() +{ + auto tupleSort = dynamic_pointer_cast(m_sort); + solAssert(tupleSort, ""); + return tupleSort->components; +} + +smt::Expression SymbolicTupleVariable::component( + size_t _index, + TypePointer _fromType, + TypePointer _toType +) +{ + optional conversion = symbolicTypeConversion(_fromType, _toType); + if (conversion) + return *conversion; + + return smt::Expression::tuple_get(currentValue(), _index); } diff --git a/libsolidity/formal/SymbolicVariables.h b/libsolidity/formal/SymbolicVariables.h index e1c28a8b5c73..6f7ad6ec4127 100644 --- a/libsolidity/formal/SymbolicVariables.h +++ b/libsolidity/formal/SymbolicVariables.h @@ -249,14 +249,18 @@ class SymbolicTupleVariable: public SymbolicVariable std::string _uniqueName, EncodingContext& _context ); + SymbolicTupleVariable( + SortPointer _sort, + std::string _uniqueName, + EncodingContext& _context + ); - std::vector> const& components() - { - return m_components; - } - -private: - std::vector> m_components; + std::vector const& components(); + Expression component( + size_t _index, + TypePointer _fromType = nullptr, + TypePointer _toType = nullptr + ); }; } diff --git a/libsolidity/formal/Z3Interface.cpp b/libsolidity/formal/Z3Interface.cpp index 20150c337b1b..f175e45d25e2 100644 --- a/libsolidity/formal/Z3Interface.cpp +++ b/libsolidity/formal/Z3Interface.cpp @@ -193,6 +193,11 @@ z3::expr Z3Interface::toZ3Expr(Expression const& _expr) solAssert(arraySort && arraySort->domain, ""); return z3::const_array(z3Sort(*arraySort->domain), arguments[1]); } + else if (n == "tuple_get") + { + size_t index = std::stoi(_expr.arguments[1].name); + return z3::func_decl(m_context, Z3_get_tuple_sort_field_decl(m_context, z3Sort(*_expr.arguments[0].sort), index))(arguments[0]); + } solAssert(false, ""); } @@ -217,6 +222,28 @@ z3::sort Z3Interface::z3Sort(Sort const& _sort) auto const& arraySort = dynamic_cast(_sort); return m_context.array_sort(z3Sort(*arraySort.domain), z3Sort(*arraySort.range)); } + case Kind::Tuple: + { + auto const& tupleSort = dynamic_cast(_sort); + vector cMembers; + for (auto const& member: tupleSort.members) + cMembers.emplace_back(member.c_str()); + /// Using this instead of the function below because with that one + /// we can't use `&sorts[0]` here. + vector sorts; + for (auto const& sort: tupleSort.components) + sorts.push_back(z3Sort(*sort)); + z3::func_decl_vector projs(m_context); + z3::func_decl tupleConstructor = m_context.tuple_sort( + tupleSort.name.c_str(), + tupleSort.members.size(), + &cMembers[0], + &sorts[0], + projs + ); + return tupleConstructor.range(); + } + default: break; } From 4fc9920112869304a1fa8c39a929717ab0ff83ca Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Thu, 9 Apr 2020 12:57:54 +0200 Subject: [PATCH 014/126] Use tuple sort name plus index for field name --- libsolidity/formal/SymbolicTypes.cpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/libsolidity/formal/SymbolicTypes.cpp b/libsolidity/formal/SymbolicTypes.cpp index b90500ce40f5..0364a04580c3 100644 --- a/libsolidity/formal/SymbolicTypes.cpp +++ b/libsolidity/formal/SymbolicTypes.cpp @@ -80,14 +80,12 @@ SortPointer smtSort(frontend::Type const& _type) auto tupleType = dynamic_cast(&_type); solAssert(tupleType, ""); vector members; - static unsigned tupleTypeId = 0; - for (auto const& component: tupleType->components()) - if (component) - members.emplace_back(component->identifier() + "_" + to_string(tupleTypeId++)); - else - members.emplace_back("null_type_" + to_string(tupleTypeId++)); + auto const& tupleName = _type.identifier(); + auto const& components = tupleType->components(); + for (unsigned i = 0; i < components.size(); ++i) + members.emplace_back(tupleName + "_accessor_" + to_string(i)); return make_shared( - _type.identifier() + "_" + to_string(tupleTypeId++), + tupleName, members, smtSortAbstractFunction(tupleType->components()) ); From eaff5c58a9cc2ebe18e7d58ffc895e618e27de25 Mon Sep 17 00:00:00 2001 From: Chris Chinchilla Date: Tue, 17 Dec 2019 14:41:51 +0100 Subject: [PATCH 015/126] Update contributing doc to clarify tests Fix line endings Update docs/contributing.rst Co-Authored-By: Bhargava Shastry Remove duplication Add back URL to correct evmone version Add Istanbul --- docs/contributing.rst | 74 ++++++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index dbfcebe892ca..571a298f23da 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -81,26 +81,55 @@ Thank you for your help! Running the compiler tests ========================== -The ``./scripts/tests.sh`` script executes most Solidity tests automatically, -but for quicker feedback, you might want to run specific tests. +Prerequisites +------------- + +Some tests require the `evmone `_ +library, others require `libz3 `_. The test script +tries to discover the location of the ``evmone`` library, which can be located +in the current directory, installed on the system level, or the ``deps`` folder +in the project top level. The required file is called ``libevmone.so`` on Linux +systems, ``evmone.dll`` on Windows systems and ``libevmone.dylib`` on macOS. + +Running the tests +----------------- Solidity includes different types of tests, most of them bundled into the `Boost C++ Test Framework `_ application ``soltest``. Running ``build/test/soltest`` or its wrapper ``scripts/soltest.sh`` is sufficient for most changes. -Some tests require the ``evmone`` library, others require ``libz3``. +The ``./scripts/tests.sh`` script executes most Solidity tests automatically, +including those bundled into the `Boost C++ Test Framework `_ application ``soltest`` (or its wrapper ``scripts/soltest.sh``), +as well as command line tests and compilation tests. -The test system will automatically try to discover the location of the ``evmone`` library +The test system automatically tries try to discover the location of the ``evmone`` library starting from the current directory. The required file is called ``libevmone.so`` on Linux systems, -``evmone.dll`` on Windows systems and ``libevmone.dylib`` on MacOS. If it is not found, the relevant tests -are skipped. To run all tests, download the library from -`Github `_ -and either place it in the project root path or inside the ``deps`` folder. +``evmone.dll`` on Windows systems and ``libevmone.dylib`` on macOS. If it is not found, tests that +use it are skipped. These tests are ``libsolididty/semanticTests``, ``libsolidity/GasCosts``, +``libsolidity/SolidityEndToEndTest``, part of the soltest suite. To run all tests, download the library from +`GitHub `_ +and place it in the project root path or inside the ``deps`` folder. + +If the ``libz3`` library is not installed on your system, you should disable the +SMT tests by exporting ``SMT_FLAGS=--no-smt`` before running ``./scripts/tests.sh`` or +running ``./scripts/soltest.sh --no-smt``. +These tests are ``libsolidity/smtCheckerTests`` and ``libsolidity/smtCheckerTestsJSON``. + +.. note :: + + To get a list of all unit tests run by Soltest, run ``./build/test/soltest --list_content=HRF``. + +For quicker results you can run a subset of, or specific tests. + +To run a subset of tests, you can use filters: +``./scripts/soltest.sh -t TestSuite/TestName``, +where ``TestName`` can be a wildcard ``*``. -If you do not have libz3 installed on your system, you should disable the SMT tests: -``./scripts/soltest.sh --no-smt``. +Or, for example, to run all the tests for the yul disambiguator: +``./scripts/soltest.sh -t "yulOptimizerTests/disambiguator/*" --no-smt``. ``./build/test/soltest --help`` has extensive help on all of the options available. + See especially: - `show_progress (-p) `_ to show test completion, @@ -109,20 +138,10 @@ See especially: .. note :: - Those working in a Windows environment wanting to run the above basic sets without libz3 in Git Bash, you would have to do: ``./build/test/Release/soltest.exe -- --no-smt``. + Those working in a Windows environment wanting to run the above basic sets + without libz3. Using Git Bash, you use: ``./build/test/Release/soltest.exe -- --no-smt``. If you are running this in plain Command Prompt, use ``.\build\test\Release\soltest.exe -- --no-smt``. -To run a subset of tests, you can use filters: -``./scripts/soltest.sh -t TestSuite/TestName``, -where ``TestName`` can be a wildcard ``*``. - -For example, here is an example test you might run; -``./scripts/soltest.sh -t "yulOptimizerTests/disambiguator/*" --no-smt``. -This will test all the tests for the disambiguator. - -To get a list of all tests, use -``./build/test/soltest --list_content=HRF``. - If you want to debug using GDB, make sure you build differently than the "usual". For example, you could run the following command in your ``build`` folder: :: @@ -130,14 +149,11 @@ For example, you could run the following command in your ``build`` folder: cmake -DCMAKE_BUILD_TYPE=Debug .. make -This will create symbols such that when you debug a test using the ``--debug`` flag, you will have access to functions and variables in which you can break or print with. - - -The script ``./scripts/tests.sh`` also runs commandline tests and compilation tests -in addition to those found in ``soltest``. - -The CI runs additional tests (including ``solc-js`` and testing third party Solidity frameworks) that require compiling the Emscripten target. +This creates symbols so that when you debug a test using the ``--debug`` flag, +you have access to functions and variables in which you can break or print with. +The CI runs additional tests (including ``solc-js`` and testing third party Solidity +frameworks) that require compiling the Emscripten target. Writing and running syntax tests -------------------------------- From 557c3ef7671edafaed273e80d6f1eb3de530e2e9 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 9 Apr 2020 16:50:31 +0200 Subject: [PATCH 016/126] Set version to 0.6.7. --- CMakeLists.txt | 2 +- Changelog.md | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 008f458aebc0..9a5026bad6d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ include(EthPolicy) eth_policy() # project name and version should be set after cmake_policy CMP0048 -set(PROJECT_VERSION "0.6.6") +set(PROJECT_VERSION "0.6.7") # OSX target needed in order to support std::visit set(CMAKE_OSX_DEPLOYMENT_TARGET "10.14") project(solidity VERSION ${PROJECT_VERSION} LANGUAGES C CXX) diff --git a/Changelog.md b/Changelog.md index 1c0952f19ad3..eb428c1edec0 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,15 @@ +### 0.6.7 (unreleased) + +Language Features: + + +Compiler Features: + + +Bugfixes: + + + ### 0.6.6 (2020-04-09) Important Bugfixes: From f682942f96c6b6a334a0167987abe3a891ca3814 Mon Sep 17 00:00:00 2001 From: David Cian Date: Mon, 16 Mar 2020 14:23:12 +0100 Subject: [PATCH 017/126] Make CircleCI post diff review for style violation The CircleCI robot now also posts a review comment directly on the diff (on GitHub, the changes tab), where the style violations take place. --- scripts/report_errors.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/scripts/report_errors.sh b/scripts/report_errors.sh index 55fc2e8c6c81..795cb006719b 100755 --- a/scripts/report_errors.sh +++ b/scripts/report_errors.sh @@ -44,6 +44,28 @@ function post_error_to_github --header 'content-type: application/json' \ -u stackenbotten:$GITHUB_ACCESS_TOKEN \ --data "{\"body\": \"There was an error when running \`$CIRCLE_JOB\` for commit \`$CIRCLE_SHA1\`:\n\`\`\`\n$FORMATTED_ERROR_MSG\n\`\`\`\nPlease check that your changes are working as intended.\"}" + + post_review_comment_to_github +} + +function post_review_comment_to_github +{ + GITHUB_API_URL="https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/pulls/$CIRCLE_PR_NUMBER/comments" + + sed -i 1d $ERROR_LOG + + while read line + do + ERROR_PATH=$(echo $line | grep -oE ".*\.cpp") + ERROR_LINE=$(echo $line | grep -oE "[0-9]*") + + curl --request POST \ + --url $GITHUB_API_URL \ + --header 'accept: application/vnd.github.v3+json, application/vnd.github.comfort-fade-preview+json' \ + --header 'content-type: application/json' \ + -u stackenbotten:$GITHUB_ACCESS_TOKEN \ + --data "{\"commit_id\": \"$CIRCLE_SHA1\", \"path\": \"$ERROR_PATH\", \"line\": $ERROR_LINE, \"side\": \"LEFT\", \"body\": \"Coding style error\"}" + done < $ERROR_LOG } trap report_error_to_github EXIT From ae336fe1c1e760bbe0901081a1c5a7454e75ff5b Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 9 Apr 2020 17:01:26 +0200 Subject: [PATCH 018/126] Fix 32 bit build failure. --- libsolutil/IpfsHash.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsolutil/IpfsHash.cpp b/libsolutil/IpfsHash.cpp index 95605cdc17d1..1f9435748848 100644 --- a/libsolutil/IpfsHash.cpp +++ b/libsolutil/IpfsHash.cpp @@ -160,7 +160,7 @@ bytes solidity::util::ipfsHash(string _data) Chunks allChunks; - for (unsigned long chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++) + for (size_t chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++) { bytes chunkBytes = asBytes( _data.substr(chunkIndex * maxChunkSize, min(maxChunkSize, _data.length() - chunkIndex * maxChunkSize)) From 8dd02f271230a402888ddfcc6a761118d19c336a Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 9 Apr 2020 17:25:03 +0200 Subject: [PATCH 019/126] Change error reporting from left to right. --- scripts/report_errors.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/report_errors.sh b/scripts/report_errors.sh index 795cb006719b..18a3c266d8ef 100755 --- a/scripts/report_errors.sh +++ b/scripts/report_errors.sh @@ -64,7 +64,7 @@ function post_review_comment_to_github --header 'accept: application/vnd.github.v3+json, application/vnd.github.comfort-fade-preview+json' \ --header 'content-type: application/json' \ -u stackenbotten:$GITHUB_ACCESS_TOKEN \ - --data "{\"commit_id\": \"$CIRCLE_SHA1\", \"path\": \"$ERROR_PATH\", \"line\": $ERROR_LINE, \"side\": \"LEFT\", \"body\": \"Coding style error\"}" + --data "{\"commit_id\": \"$CIRCLE_SHA1\", \"path\": \"$ERROR_PATH\", \"line\": $ERROR_LINE, \"side\": \"RIGHT\", \"body\": \"Coding style error\"}" done < $ERROR_LOG } From 1ef24562a97a7448b98149050a76098e45c616c2 Mon Sep 17 00:00:00 2001 From: Evan Saulpaugh Date: Fri, 10 Apr 2020 02:17:57 -0500 Subject: [PATCH 020/126] change positive to non-negative --- docs/abi-spec.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/abi-spec.rst b/docs/abi-spec.rst index d645a3d44d33..c0b4287d010d 100644 --- a/docs/abi-spec.rst +++ b/docs/abi-spec.rst @@ -202,7 +202,7 @@ on the type of ``X`` being - ``uint``: ``enc(X)`` is the big-endian encoding of ``X``, padded on the higher-order (left) side with zero-bytes such that the length is 32 bytes. - ``address``: as in the ``uint160`` case -- ``int``: ``enc(X)`` is the big-endian two's complement encoding of ``X``, padded on the higher-order (left) side with ``0xff`` for negative ``X`` and with zero bytes for positive ``X`` such that the length is 32 bytes. +- ``int``: ``enc(X)`` is the big-endian two's complement encoding of ``X``, padded on the higher-order (left) side with ``0xff`` for negative ``X`` and with zero bytes for non-negative ``X`` such that the length is 32 bytes. - ``bool``: as in the ``uint8`` case, where ``1`` is used for ``true`` and ``0`` for ``false`` - ``fixedx``: ``enc(X)`` is ``enc(X * 10**N)`` where ``X * 10**N`` is interpreted as a ``int256``. - ``fixed``: as in the ``fixed128x18`` case From 1a67b86f2414c0bba2ae3314fc45329d61cc6abd Mon Sep 17 00:00:00 2001 From: Evan Saulpaugh Date: Fri, 10 Apr 2020 02:23:13 -0500 Subject: [PATCH 021/126] clarify description of padding for int types --- docs/abi-spec.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/abi-spec.rst b/docs/abi-spec.rst index c0b4287d010d..59636acb6038 100644 --- a/docs/abi-spec.rst +++ b/docs/abi-spec.rst @@ -202,7 +202,7 @@ on the type of ``X`` being - ``uint``: ``enc(X)`` is the big-endian encoding of ``X``, padded on the higher-order (left) side with zero-bytes such that the length is 32 bytes. - ``address``: as in the ``uint160`` case -- ``int``: ``enc(X)`` is the big-endian two's complement encoding of ``X``, padded on the higher-order (left) side with ``0xff`` for negative ``X`` and with zero bytes for non-negative ``X`` such that the length is 32 bytes. +- ``int``: ``enc(X)`` is the big-endian two's complement encoding of ``X``, padded on the higher-order (left) side with ``0xff`` bytes for negative ``X`` and with zero-bytes for non-negative ``X`` such that the length is 32 bytes. - ``bool``: as in the ``uint8`` case, where ``1`` is used for ``true`` and ``0`` for ``false`` - ``fixedx``: ``enc(X)`` is ``enc(X * 10**N)`` where ``X * 10**N`` is interpreted as a ``int256``. - ``fixed``: as in the ``fixed128x18`` case From 3af43fd3508951a2583270697829146b6b700fbe Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Tue, 7 Apr 2020 19:31:48 +0200 Subject: [PATCH 022/126] Extract typing. --- libsolidity/CMakeLists.txt | 2 + .../analysis/DeclarationTypeChecker.cpp | 299 ++++++++++++++++++ libsolidity/analysis/DeclarationTypeChecker.h | 72 +++++ libsolidity/analysis/NameAndTypeResolver.cpp | 45 --- libsolidity/analysis/NameAndTypeResolver.h | 6 - libsolidity/analysis/ReferencesResolver.cpp | 263 +-------------- libsolidity/analysis/ReferencesResolver.h | 11 - libsolidity/analysis/TypeChecker.cpp | 46 +++ libsolidity/analysis/TypeChecker.h | 5 + libsolidity/ast/ASTAnnotations.h | 2 + libsolidity/interface/CompilerStack.cpp | 6 + test/libsolidity/Assembly.cpp | 8 + .../SolidityExpressionCompiler.cpp | 5 +- ..._cannot_be_constant_function_parameter.sol | 1 + .../function_type_struct_undefined_member.sol | 1 - .../150_array_with_nonconstant_length.sol | 1 + .../151_array_with_negative_length.sol | 1 + ...nvalid_array_declaration_with_rational.sol | 1 + ...ray_declaration_with_signed_fixed_type.sol | 1 + ...y_declaration_with_unsigned_fixed_type.sol | 1 + .../address_invalid_state_mutability.sol | 4 +- .../structs/member_type_eq_name.sol | 1 - .../syntaxTests/structs/member_type_func.sol | 1 - .../tupleAssignments/tuple_in_tuple_long.sol | 13 +- .../types/var_decl_val_mismatch.sol | 5 +- 25 files changed, 461 insertions(+), 340 deletions(-) create mode 100644 libsolidity/analysis/DeclarationTypeChecker.cpp create mode 100644 libsolidity/analysis/DeclarationTypeChecker.h diff --git a/libsolidity/CMakeLists.txt b/libsolidity/CMakeLists.txt index efb553e1a5d5..7e07b88079ec 100644 --- a/libsolidity/CMakeLists.txt +++ b/libsolidity/CMakeLists.txt @@ -12,6 +12,8 @@ set(sources analysis/ControlFlowGraph.h analysis/DeclarationContainer.cpp analysis/DeclarationContainer.h + analysis/DeclarationTypeChecker.cpp + analysis/DeclarationTypeChecker.h analysis/DocStringAnalyser.cpp analysis/DocStringAnalyser.h analysis/ImmutableValidator.cpp diff --git a/libsolidity/analysis/DeclarationTypeChecker.cpp b/libsolidity/analysis/DeclarationTypeChecker.cpp new file mode 100644 index 000000000000..12e84d53db0c --- /dev/null +++ b/libsolidity/analysis/DeclarationTypeChecker.cpp @@ -0,0 +1,299 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include + +#include + +#include + +#include + +using namespace std; +using namespace solidity::langutil; +using namespace solidity::frontend; + +bool DeclarationTypeChecker::visit(ElementaryTypeName const& _typeName) +{ + if (!_typeName.annotation().type) + { + _typeName.annotation().type = TypeProvider::fromElementaryTypeName(_typeName.typeName()); + if (_typeName.stateMutability().has_value()) + { + // for non-address types this was already caught by the parser + solAssert(_typeName.annotation().type->category() == Type::Category::Address, ""); + switch (*_typeName.stateMutability()) + { + case StateMutability::Payable: + _typeName.annotation().type = TypeProvider::payableAddress(); + break; + case StateMutability::NonPayable: + _typeName.annotation().type = TypeProvider::address(); + break; + default: + typeError( + _typeName.location(), + "Address types can only be payable or non-payable." + ); + break; + } + } + } + return true; +} + +void DeclarationTypeChecker::endVisit(UserDefinedTypeName const& _typeName) +{ + Declaration const* declaration = _typeName.annotation().referencedDeclaration; + solAssert(declaration, ""); + + if (StructDefinition const* structDef = dynamic_cast(declaration)) + _typeName.annotation().type = TypeProvider::structType(*structDef, DataLocation::Storage); + else if (EnumDefinition const* enumDef = dynamic_cast(declaration)) + _typeName.annotation().type = TypeProvider::enumType(*enumDef); + else if (ContractDefinition const* contract = dynamic_cast(declaration)) + _typeName.annotation().type = TypeProvider::contract(*contract); + else + { + _typeName.annotation().type = TypeProvider::emptyTuple(); + fatalTypeError(_typeName.location(), "Name has to refer to a struct, enum or contract."); + } +} +void DeclarationTypeChecker::endVisit(FunctionTypeName const& _typeName) +{ + switch (_typeName.visibility()) + { + case Visibility::Internal: + case Visibility::External: + break; + default: + fatalTypeError(_typeName.location(), "Invalid visibility, can only be \"external\" or \"internal\"."); + return; + } + + if (_typeName.isPayable() && _typeName.visibility() != Visibility::External) + { + fatalTypeError(_typeName.location(), "Only external function types can be payable."); + return; + } + + if (_typeName.visibility() == Visibility::External) + for (auto const& t: _typeName.parameterTypes() + _typeName.returnParameterTypes()) + { + solAssert(t->annotation().type, "Type not set for parameter."); + if (!t->annotation().type->interfaceType(false).get()) + { + fatalTypeError(t->location(), "Internal type cannot be used for external function type."); + return; + } + } + + _typeName.annotation().type = TypeProvider::function(_typeName); +} +void DeclarationTypeChecker::endVisit(Mapping const& _mapping) +{ + if (auto const* typeName = dynamic_cast(&_mapping.keyType())) + { + if (auto const* contractType = dynamic_cast(typeName->annotation().type)) + { + if (contractType->contractDefinition().isLibrary()) + m_errorReporter.fatalTypeError( + typeName->location(), + "Library types cannot be used as mapping keys." + ); + } + else if (typeName->annotation().type->category() != Type::Category::Enum) + m_errorReporter.fatalTypeError( + typeName->location(), + "Only elementary types, contract types or enums are allowed as mapping keys." + ); + } + else + solAssert(dynamic_cast(&_mapping.keyType()), ""); + + TypePointer keyType = _mapping.keyType().annotation().type; + TypePointer valueType = _mapping.valueType().annotation().type; + + // Convert key type to memory. + keyType = TypeProvider::withLocationIfReference(DataLocation::Memory, keyType); + + // Convert value type to storage reference. + valueType = TypeProvider::withLocationIfReference(DataLocation::Storage, valueType); + _mapping.annotation().type = TypeProvider::mapping(keyType, valueType); +} + +void DeclarationTypeChecker::endVisit(ArrayTypeName const& _typeName) +{ + TypePointer baseType = _typeName.baseType().annotation().type; + if (!baseType) + { + solAssert(!m_errorReporter.errors().empty(), ""); + return; + } + if (baseType->storageBytes() == 0) + fatalTypeError(_typeName.baseType().location(), "Illegal base type of storage size zero for array."); + if (Expression const* length = _typeName.length()) + { + TypePointer& lengthTypeGeneric = length->annotation().type; + if (!lengthTypeGeneric) + lengthTypeGeneric = ConstantEvaluator(m_errorReporter).evaluate(*length); + RationalNumberType const* lengthType = dynamic_cast(lengthTypeGeneric); + u256 lengthValue = 0; + if (!lengthType || !lengthType->mobileType()) + typeError(length->location(), "Invalid array length, expected integer literal or constant expression."); + else if (lengthType->isZero()) + typeError(length->location(), "Array with zero length specified."); + else if (lengthType->isFractional()) + typeError(length->location(), "Array with fractional length specified."); + else if (lengthType->isNegative()) + typeError(length->location(), "Array with negative length specified."); + else + lengthValue = lengthType->literalValue(nullptr); + _typeName.annotation().type = TypeProvider::array(DataLocation::Storage, baseType, lengthValue); + } + else + _typeName.annotation().type = TypeProvider::array(DataLocation::Storage, baseType); +} +void DeclarationTypeChecker::endVisit(VariableDeclaration const& _variable) +{ + if (_variable.annotation().type) + return; + + if (_variable.isConstant() && !_variable.isStateVariable()) + m_errorReporter.declarationError(_variable.location(), "The \"constant\" keyword can only be used for state variables."); + if (_variable.immutable() && !_variable.isStateVariable()) + m_errorReporter.declarationError(_variable.location(), "The \"immutable\" keyword can only be used for state variables."); + + if (!_variable.typeName()) + { + // This can still happen in very unusual cases where a developer uses constructs, such as + // `var a;`, however, such code will have generated errors already. + // However, we cannot blindingly solAssert() for that here, as the TypeChecker (which is + // invoking ReferencesResolver) is generating it, so the error is most likely(!) generated + // after this step. + return; + } + using Location = VariableDeclaration::Location; + Location varLoc = _variable.referenceLocation(); + DataLocation typeLoc = DataLocation::Memory; + + set allowedDataLocations = _variable.allowedDataLocations(); + if (!allowedDataLocations.count(varLoc)) + { + auto locationToString = [](VariableDeclaration::Location _location) -> string + { + switch (_location) + { + case Location::Memory: return "\"memory\""; + case Location::Storage: return "\"storage\""; + case Location::CallData: return "\"calldata\""; + case Location::Unspecified: return "none"; + } + return {}; + }; + + string errorString; + if (!_variable.hasReferenceOrMappingType()) + errorString = "Data location can only be specified for array, struct or mapping types"; + else + { + errorString = "Data location must be " + + util::joinHumanReadable( + allowedDataLocations | boost::adaptors::transformed(locationToString), + ", ", + " or " + ); + if (_variable.isCallableOrCatchParameter()) + errorString += + " for " + + string(_variable.isReturnParameter() ? "return " : "") + + "parameter in" + + string(_variable.isExternalCallableParameter() ? " external" : "") + + " function"; + else + errorString += " for variable"; + } + errorString += ", but " + locationToString(varLoc) + " was given."; + typeError(_variable.location(), errorString); + + solAssert(!allowedDataLocations.empty(), ""); + varLoc = *allowedDataLocations.begin(); + } + + // Find correct data location. + if (_variable.isEventParameter()) + { + solAssert(varLoc == Location::Unspecified, ""); + typeLoc = DataLocation::Memory; + } + else if (_variable.isStateVariable()) + { + solAssert(varLoc == Location::Unspecified, ""); + typeLoc = (_variable.isConstant() || _variable.immutable()) ? DataLocation::Memory : DataLocation::Storage; + } + else if ( + dynamic_cast(_variable.scope()) || + dynamic_cast(_variable.scope()) + ) + // The actual location will later be changed depending on how the type is used. + typeLoc = DataLocation::Storage; + else + switch (varLoc) + { + case Location::Memory: + typeLoc = DataLocation::Memory; + break; + case Location::Storage: + typeLoc = DataLocation::Storage; + break; + case Location::CallData: + typeLoc = DataLocation::CallData; + break; + case Location::Unspecified: + solAssert(!_variable.hasReferenceOrMappingType(), "Data location not properly set."); + } + + TypePointer type = _variable.typeName()->annotation().type; + if (auto ref = dynamic_cast(type)) + { + bool isPointer = !_variable.isStateVariable(); + type = TypeProvider::withLocation(ref, typeLoc, isPointer); + } + + _variable.annotation().type = type; + +} + +void DeclarationTypeChecker::typeError(SourceLocation const& _location, string const& _description) +{ + m_errorOccurred = true; + m_errorReporter.typeError(_location, _description); +} + +void DeclarationTypeChecker::fatalTypeError(SourceLocation const& _location, string const& _description) +{ + m_errorOccurred = true; + m_errorReporter.fatalTypeError(_location, _description); +} + +bool DeclarationTypeChecker::check(ASTNode const& _node) +{ + _node.accept(*this); + return !m_errorOccurred; +} diff --git a/libsolidity/analysis/DeclarationTypeChecker.h b/libsolidity/analysis/DeclarationTypeChecker.h new file mode 100644 index 000000000000..6878359ab3dc --- /dev/null +++ b/libsolidity/analysis/DeclarationTypeChecker.h @@ -0,0 +1,72 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include +#include +#include + +#include +#include +#include + +namespace solidity::langutil +{ +class ErrorReporter; +} + +namespace solidity::frontend +{ + +/** + * Assigns types to declarations. + */ +class DeclarationTypeChecker: private ASTConstVisitor +{ +public: + DeclarationTypeChecker( + langutil::ErrorReporter& _errorReporter, + langutil::EVMVersion _evmVersion + ): + m_errorReporter(_errorReporter), + m_evmVersion(_evmVersion) + {} + + bool check(ASTNode const& _contract); + +private: + + bool visit(ElementaryTypeName const& _typeName) override; + void endVisit(UserDefinedTypeName const& _typeName) override; + void endVisit(FunctionTypeName const& _typeName) override; + void endVisit(Mapping const& _mapping) override; + void endVisit(ArrayTypeName const& _typeName) override; + void endVisit(VariableDeclaration const& _variable) override; + + /// Adds a new error to the list of errors. + void typeError(langutil::SourceLocation const& _location, std::string const& _description); + + /// Adds a new error to the list of errors and throws to abort reference resolving. + void fatalTypeError(langutil::SourceLocation const& _location, std::string const& _description); + + langutil::ErrorReporter& m_errorReporter; + bool m_errorOccurred = false; + langutil::EVMVersion m_evmVersion; +}; + +} diff --git a/libsolidity/analysis/NameAndTypeResolver.cpp b/libsolidity/analysis/NameAndTypeResolver.cpp index 3643c64019b8..65831ec868e4 100644 --- a/libsolidity/analysis/NameAndTypeResolver.cpp +++ b/libsolidity/analysis/NameAndTypeResolver.cpp @@ -195,51 +195,6 @@ Declaration const* NameAndTypeResolver::pathFromCurrentScope(vector c return nullptr; } -vector NameAndTypeResolver::cleanedDeclarations( - Identifier const& _identifier, - vector const& _declarations -) -{ - solAssert(_declarations.size() > 1, ""); - vector uniqueFunctions; - - for (Declaration const* declaration: _declarations) - { - solAssert(declaration, ""); - // the declaration is functionDefinition, eventDefinition or a VariableDeclaration while declarations > 1 - solAssert( - dynamic_cast(declaration) || - dynamic_cast(declaration) || - dynamic_cast(declaration) || - dynamic_cast(declaration), - "Found overloading involving something not a function, event or a (magic) variable." - ); - - FunctionTypePointer functionType { declaration->functionType(false) }; - if (!functionType) - functionType = declaration->functionType(true); - solAssert(functionType, "Failed to determine the function type of the overloaded."); - - for (auto parameter: functionType->parameterTypes() + functionType->returnParameterTypes()) - if (!parameter) - m_errorReporter.fatalDeclarationError(_identifier.location(), "Function type can not be used in this context."); - - if (uniqueFunctions.end() == find_if( - uniqueFunctions.begin(), - uniqueFunctions.end(), - [&](Declaration const* d) - { - FunctionType const* newFunctionType = d->functionType(false); - if (!newFunctionType) - newFunctionType = d->functionType(true); - return newFunctionType && functionType->hasEqualParameterTypes(*newFunctionType); - } - )) - uniqueFunctions.push_back(declaration); - } - return uniqueFunctions; -} - void NameAndTypeResolver::warnVariablesNamedLikeInstructions() { for (auto const& instruction: evmasm::c_instructions) diff --git a/libsolidity/analysis/NameAndTypeResolver.h b/libsolidity/analysis/NameAndTypeResolver.h index dc4d32bb9dfc..845b2870f796 100644 --- a/libsolidity/analysis/NameAndTypeResolver.h +++ b/libsolidity/analysis/NameAndTypeResolver.h @@ -95,12 +95,6 @@ class NameAndTypeResolver: private boost::noncopyable /// @note Returns a null pointer if any component in the path was not unique or not found. Declaration const* pathFromCurrentScope(std::vector const& _path) const; - /// returns the vector of declarations without repetitions - std::vector cleanedDeclarations( - Identifier const& _identifier, - std::vector const& _declarations - ); - /// Generate and store warnings about variables that are named like instructions. void warnVariablesNamedLikeInstructions(); diff --git a/libsolidity/analysis/ReferencesResolver.cpp b/libsolidity/analysis/ReferencesResolver.cpp index 94a16d6f1552..f72253c1f622 100644 --- a/libsolidity/analysis/ReferencesResolver.cpp +++ b/libsolidity/analysis/ReferencesResolver.cpp @@ -22,9 +22,7 @@ #include #include -#include #include -#include #include #include @@ -37,7 +35,6 @@ #include #include -#include using namespace std; using namespace solidity::langutil; @@ -126,40 +123,10 @@ bool ReferencesResolver::visit(Identifier const& _identifier) else if (declarations.size() == 1) _identifier.annotation().referencedDeclaration = declarations.front(); else - _identifier.annotation().overloadedDeclarations = - m_resolver.cleanedDeclarations(_identifier, declarations); + _identifier.annotation().candidateDeclarations = declarations; return false; } -bool ReferencesResolver::visit(ElementaryTypeName const& _typeName) -{ - if (!_typeName.annotation().type) - { - _typeName.annotation().type = TypeProvider::fromElementaryTypeName(_typeName.typeName()); - if (_typeName.stateMutability().has_value()) - { - // for non-address types this was already caught by the parser - solAssert(_typeName.annotation().type->category() == Type::Category::Address, ""); - switch (*_typeName.stateMutability()) - { - case StateMutability::Payable: - _typeName.annotation().type = TypeProvider::payableAddress(); - break; - case StateMutability::NonPayable: - _typeName.annotation().type = TypeProvider::address(); - break; - default: - m_errorReporter.typeError( - _typeName.location(), - "Address types can only be payable or non-payable." - ); - break; - } - } - } - return true; -} - bool ReferencesResolver::visit(FunctionDefinition const& _functionDefinition) { m_returnParameters.push_back(_functionDefinition.returnParameterList().get()); @@ -194,113 +161,6 @@ void ReferencesResolver::endVisit(UserDefinedTypeName const& _typeName) } _typeName.annotation().referencedDeclaration = declaration; - - if (StructDefinition const* structDef = dynamic_cast(declaration)) - _typeName.annotation().type = TypeProvider::structType(*structDef, DataLocation::Storage); - else if (EnumDefinition const* enumDef = dynamic_cast(declaration)) - _typeName.annotation().type = TypeProvider::enumType(*enumDef); - else if (ContractDefinition const* contract = dynamic_cast(declaration)) - _typeName.annotation().type = TypeProvider::contract(*contract); - else - { - _typeName.annotation().type = TypeProvider::emptyTuple(); - fatalTypeError(_typeName.location(), "Name has to refer to a struct, enum or contract."); - } -} - -void ReferencesResolver::endVisit(FunctionTypeName const& _typeName) -{ - switch (_typeName.visibility()) - { - case Visibility::Internal: - case Visibility::External: - break; - default: - fatalTypeError(_typeName.location(), "Invalid visibility, can only be \"external\" or \"internal\"."); - return; - } - - if (_typeName.isPayable() && _typeName.visibility() != Visibility::External) - { - fatalTypeError(_typeName.location(), "Only external function types can be payable."); - return; - } - - if (_typeName.visibility() == Visibility::External) - for (auto const& t: _typeName.parameterTypes() + _typeName.returnParameterTypes()) - { - solAssert(t->annotation().type, "Type not set for parameter."); - if (!t->annotation().type->interfaceType(false).get()) - { - fatalTypeError(t->location(), "Internal type cannot be used for external function type."); - return; - } - } - - _typeName.annotation().type = TypeProvider::function(_typeName); -} - -void ReferencesResolver::endVisit(Mapping const& _mapping) -{ - if (auto const* typeName = dynamic_cast(&_mapping.keyType())) - { - if (auto const* contractType = dynamic_cast(typeName->annotation().type)) - { - if (contractType->contractDefinition().isLibrary()) - m_errorReporter.fatalTypeError( - typeName->location(), - "Library types cannot be used as mapping keys." - ); - } - else if (typeName->annotation().type->category() != Type::Category::Enum) - m_errorReporter.fatalTypeError( - typeName->location(), - "Only elementary types, contract types or enums are allowed as mapping keys." - ); - } - else - solAssert(dynamic_cast(&_mapping.keyType()), ""); - - TypePointer keyType = _mapping.keyType().annotation().type; - TypePointer valueType = _mapping.valueType().annotation().type; - - // Convert key type to memory. - keyType = TypeProvider::withLocationIfReference(DataLocation::Memory, keyType); - - // Convert value type to storage reference. - valueType = TypeProvider::withLocationIfReference(DataLocation::Storage, valueType); - _mapping.annotation().type = TypeProvider::mapping(keyType, valueType); -} - -void ReferencesResolver::endVisit(ArrayTypeName const& _typeName) -{ - TypePointer baseType = _typeName.baseType().annotation().type; - if (!baseType) - { - solAssert(!m_errorReporter.errors().empty(), ""); - return; - } - if (baseType->storageBytes() == 0) - fatalTypeError(_typeName.baseType().location(), "Illegal base type of storage size zero for array."); - if (Expression const* length = _typeName.length()) - { - TypePointer& lengthTypeGeneric = length->annotation().type; - if (!lengthTypeGeneric) - lengthTypeGeneric = ConstantEvaluator(m_errorReporter).evaluate(*length); - RationalNumberType const* lengthType = dynamic_cast(lengthTypeGeneric); - if (!lengthType || !lengthType->mobileType()) - fatalTypeError(length->location(), "Invalid array length, expected integer literal or constant expression."); - else if (lengthType->isZero()) - fatalTypeError(length->location(), "Array with zero length specified."); - else if (lengthType->isFractional()) - fatalTypeError(length->location(), "Array with fractional length specified."); - else if (lengthType->isNegative()) - fatalTypeError(length->location(), "Array with negative length specified."); - else - _typeName.annotation().type = TypeProvider::array(DataLocation::Storage, baseType, lengthType->literalValue(nullptr)); - } - else - _typeName.annotation().type = TypeProvider::array(DataLocation::Storage, baseType); } bool ReferencesResolver::visit(InlineAssembly const& _inlineAssembly) @@ -321,115 +181,6 @@ bool ReferencesResolver::visit(Return const& _return) return true; } -void ReferencesResolver::endVisit(VariableDeclaration const& _variable) -{ - if (_variable.annotation().type) - return; - - if (_variable.isConstant() && !_variable.isStateVariable()) - m_errorReporter.declarationError(_variable.location(), "The \"constant\" keyword can only be used for state variables."); - if (_variable.immutable() && !_variable.isStateVariable()) - m_errorReporter.declarationError(_variable.location(), "The \"immutable\" keyword can only be used for state variables."); - - if (!_variable.typeName()) - { - // This can still happen in very unusual cases where a developer uses constructs, such as - // `var a;`, however, such code will have generated errors already. - // However, we cannot blindingly solAssert() for that here, as the TypeChecker (which is - // invoking ReferencesResolver) is generating it, so the error is most likely(!) generated - // after this step. - return; - } - using Location = VariableDeclaration::Location; - Location varLoc = _variable.referenceLocation(); - DataLocation typeLoc = DataLocation::Memory; - - set allowedDataLocations = _variable.allowedDataLocations(); - if (!allowedDataLocations.count(varLoc)) - { - auto locationToString = [](VariableDeclaration::Location _location) -> string - { - switch (_location) - { - case Location::Memory: return "\"memory\""; - case Location::Storage: return "\"storage\""; - case Location::CallData: return "\"calldata\""; - case Location::Unspecified: return "none"; - } - return {}; - }; - - string errorString; - if (!_variable.hasReferenceOrMappingType()) - errorString = "Data location can only be specified for array, struct or mapping types"; - else - { - errorString = "Data location must be " + - util::joinHumanReadable( - allowedDataLocations | boost::adaptors::transformed(locationToString), - ", ", - " or " - ); - if (_variable.isCallableOrCatchParameter()) - errorString += - " for " + - string(_variable.isReturnParameter() ? "return " : "") + - "parameter in" + - string(_variable.isExternalCallableParameter() ? " external" : "") + - " function"; - else - errorString += " for variable"; - } - errorString += ", but " + locationToString(varLoc) + " was given."; - typeError(_variable.location(), errorString); - - solAssert(!allowedDataLocations.empty(), ""); - varLoc = *allowedDataLocations.begin(); - } - - // Find correct data location. - if (_variable.isEventParameter()) - { - solAssert(varLoc == Location::Unspecified, ""); - typeLoc = DataLocation::Memory; - } - else if (_variable.isStateVariable()) - { - solAssert(varLoc == Location::Unspecified, ""); - typeLoc = (_variable.isConstant() || _variable.immutable()) ? DataLocation::Memory : DataLocation::Storage; - } - else if ( - dynamic_cast(_variable.scope()) || - dynamic_cast(_variable.scope()) - ) - // The actual location will later be changed depending on how the type is used. - typeLoc = DataLocation::Storage; - else - switch (varLoc) - { - case Location::Memory: - typeLoc = DataLocation::Memory; - break; - case Location::Storage: - typeLoc = DataLocation::Storage; - break; - case Location::CallData: - typeLoc = DataLocation::CallData; - break; - case Location::Unspecified: - solAssert(!_variable.hasReferenceOrMappingType(), "Data location not properly set."); - } - - TypePointer type = _variable.typeName()->annotation().type; - if (auto ref = dynamic_cast(type)) - { - bool isPointer = !_variable.isStateVariable(); - type = TypeProvider::withLocation(ref, typeLoc, isPointer); - } - - _variable.annotation().type = type; -} - void ReferencesResolver::operator()(yul::FunctionDefinition const& _function) { bool wasInsideFunction = m_yulInsideFunction; @@ -514,18 +265,6 @@ void ReferencesResolver::operator()(yul::VariableDeclaration const& _varDecl) visit(*_varDecl.value); } -void ReferencesResolver::typeError(SourceLocation const& _location, string const& _description) -{ - m_errorOccurred = true; - m_errorReporter.typeError(_location, _description); -} - -void ReferencesResolver::fatalTypeError(SourceLocation const& _location, string const& _description) -{ - m_errorOccurred = true; - m_errorReporter.fatalTypeError(_location, _description); -} - void ReferencesResolver::declarationError(SourceLocation const& _location, string const& _description) { m_errorOccurred = true; diff --git a/libsolidity/analysis/ReferencesResolver.h b/libsolidity/analysis/ReferencesResolver.h index 488562f21565..2a4daf3db15f 100644 --- a/libsolidity/analysis/ReferencesResolver.h +++ b/libsolidity/analysis/ReferencesResolver.h @@ -76,29 +76,18 @@ class ReferencesResolver: private ASTConstVisitor, private yul::ASTWalker void endVisit(ForStatement const& _for) override; void endVisit(VariableDeclarationStatement const& _varDeclStatement) override; bool visit(Identifier const& _identifier) override; - bool visit(ElementaryTypeName const& _typeName) override; bool visit(FunctionDefinition const& _functionDefinition) override; void endVisit(FunctionDefinition const& _functionDefinition) override; bool visit(ModifierDefinition const& _modifierDefinition) override; void endVisit(ModifierDefinition const& _modifierDefinition) override; void endVisit(UserDefinedTypeName const& _typeName) override; - void endVisit(FunctionTypeName const& _typeName) override; - void endVisit(Mapping const& _mapping) override; - void endVisit(ArrayTypeName const& _typeName) override; bool visit(InlineAssembly const& _inlineAssembly) override; bool visit(Return const& _return) override; - void endVisit(VariableDeclaration const& _variable) override; void operator()(yul::FunctionDefinition const& _function) override; void operator()(yul::Identifier const& _identifier) override; void operator()(yul::VariableDeclaration const& _varDecl) override; - /// Adds a new error to the list of errors. - void typeError(langutil::SourceLocation const& _location, std::string const& _description); - - /// Adds a new error to the list of errors and throws to abort reference resolving. - void fatalTypeError(langutil::SourceLocation const& _location, std::string const& _description); - /// Adds a new error to the list of errors. void declarationError(langutil::SourceLocation const& _location, std::string const& _description); diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index b2e49b052635..76e805c27d67 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -2776,11 +2776,57 @@ bool TypeChecker::visit(IndexRangeAccess const& _access) return false; } +vector TypeChecker::cleanOverloadedDeclarations( + Identifier const& _identifier, + vector const& _candidates +) +{ + solAssert(_candidates.size() > 1, ""); + vector uniqueDeclarations; + + for (Declaration const* declaration: _candidates) + { + solAssert(declaration, ""); + // the declaration is functionDefinition, eventDefinition or a VariableDeclaration while declarations > 1 + solAssert( + dynamic_cast(declaration) || + dynamic_cast(declaration) || + dynamic_cast(declaration) || + dynamic_cast(declaration), + "Found overloading involving something not a function, event or a (magic) variable." + ); + + FunctionTypePointer functionType {declaration->functionType(false)}; + if (!functionType) + functionType = declaration->functionType(true); + solAssert(functionType, "Failed to determine the function type of the overloaded."); + + for (TypePointer parameter: functionType->parameterTypes() + functionType->returnParameterTypes()) + if (!parameter) + m_errorReporter.fatalDeclarationError(_identifier.location(), "Function type can not be used in this context."); + + if (uniqueDeclarations.end() == find_if( + uniqueDeclarations.begin(), + uniqueDeclarations.end(), + [&](Declaration const* d) + { + FunctionType const* newFunctionType = d->functionType(false); + if (!newFunctionType) + newFunctionType = d->functionType(true); + return newFunctionType && functionType->hasEqualParameterTypes(*newFunctionType); + } + )) + uniqueDeclarations.push_back(declaration); + } + return uniqueDeclarations; +} + bool TypeChecker::visit(Identifier const& _identifier) { IdentifierAnnotation& annotation = _identifier.annotation(); if (!annotation.referencedDeclaration) { + annotation.overloadedDeclarations = cleanOverloadedDeclarations(_identifier, annotation.candidateDeclarations); if (!annotation.arguments) { // The identifier should be a public state variable shadowing other functions diff --git a/libsolidity/analysis/TypeChecker.h b/libsolidity/analysis/TypeChecker.h index a26ab81bdcfa..5aaba15f7aa2 100644 --- a/libsolidity/analysis/TypeChecker.h +++ b/libsolidity/analysis/TypeChecker.h @@ -154,6 +154,11 @@ class TypeChecker: private ASTConstVisitor /// @returns the referenced declaration and throws on error. Declaration const& dereference(UserDefinedTypeName const& _typeName) const; + std::vector cleanOverloadedDeclarations( + Identifier const& _reference, + std::vector const& _candidates + ); + /// Runs type checks on @a _expression to infer its type and then checks that it is implicitly /// convertible to @a _expectedType. bool expectType(Expression const& _expression, Type const& _expectedType); diff --git a/libsolidity/ast/ASTAnnotations.h b/libsolidity/ast/ASTAnnotations.h index 021a742f5417..9f7edf84ec93 100644 --- a/libsolidity/ast/ASTAnnotations.h +++ b/libsolidity/ast/ASTAnnotations.h @@ -248,6 +248,8 @@ struct IdentifierAnnotation: ExpressionAnnotation { /// Referenced declaration, set at latest during overload resolution stage. Declaration const* referencedDeclaration = nullptr; + /// List of possible declarations it could refer to (can contain duplicates). + std::vector candidateDeclarations; /// List of possible declarations it could refer to. std::vector overloadedDeclarations; }; diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index fe0268fb1aa0..08e44531d697 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -348,6 +349,11 @@ bool CompilerStack::analyze() } + DeclarationTypeChecker declarationTypeChecker(m_errorReporter, m_evmVersion); + for (Source const* source: m_sourceOrder) + if (source->ast && !declarationTypeChecker.check(*source->ast)) + return false; + // Next, we check inheritance, overrides, function collisions and other things at // contract or function level. // This also calculates whether a contract is abstract, which is needed by the diff --git a/test/libsolidity/Assembly.cpp b/test/libsolidity/Assembly.cpp index f762849e838c..2af81c932d19 100644 --- a/test/libsolidity/Assembly.cpp +++ b/test/libsolidity/Assembly.cpp @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -60,6 +61,7 @@ evmasm::AssemblyItems compileContract(std::shared_ptr _sourceCode) map> scopes; GlobalContext globalContext; NameAndTypeResolver resolver(globalContext, solidity::test::CommonOptions::get().evmVersion(), scopes, errorReporter); + DeclarationTypeChecker declarationTypeChecker(errorReporter, solidity::test::CommonOptions::get().evmVersion()); solAssert(Error::containsOnlyWarnings(errorReporter.errors()), ""); resolver.registerDeclarations(*sourceUnit); for (ASTPointer const& node: sourceUnit->nodes()) @@ -69,6 +71,12 @@ evmasm::AssemblyItems compileContract(std::shared_ptr _sourceCode) if (!Error::containsOnlyWarnings(errorReporter.errors())) return AssemblyItems(); } + for (ASTPointer const& node: sourceUnit->nodes()) + { + BOOST_REQUIRE_NO_THROW(declarationTypeChecker.check(*node)); + if (!Error::containsOnlyWarnings(errorReporter.errors())) + return AssemblyItems(); + } for (ASTPointer const& node: sourceUnit->nodes()) if (ContractDefinition* contract = dynamic_cast(node.get())) { diff --git a/test/libsolidity/SolidityExpressionCompiler.cpp b/test/libsolidity/SolidityExpressionCompiler.cpp index 5dfefc0b721d..875a042fd49f 100644 --- a/test/libsolidity/SolidityExpressionCompiler.cpp +++ b/test/libsolidity/SolidityExpressionCompiler.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -118,10 +119,12 @@ bytes compileFirstExpression( map> scopes; NameAndTypeResolver resolver(globalContext, solidity::test::CommonOptions::get().evmVersion(), scopes, errorReporter); resolver.registerDeclarations(*sourceUnit); - for (ASTPointer const& node: sourceUnit->nodes()) if (ContractDefinition* contract = dynamic_cast(node.get())) BOOST_REQUIRE_MESSAGE(resolver.resolveNamesAndTypes(*contract), "Resolving names failed"); + DeclarationTypeChecker declarationTypeChecker(errorReporter, solidity::test::CommonOptions::get().evmVersion()); + for (ASTPointer const& node: sourceUnit->nodes()) + BOOST_REQUIRE(declarationTypeChecker.check(*node)); for (ASTPointer const& node: sourceUnit->nodes()) if (ContractDefinition* contract = dynamic_cast(node.get())) { diff --git a/test/libsolidity/syntaxTests/array/length/array_length_cannot_be_constant_function_parameter.sol b/test/libsolidity/syntaxTests/array/length/array_length_cannot_be_constant_function_parameter.sol index 5932814019df..270f1908f8f4 100644 --- a/test/libsolidity/syntaxTests/array/length/array_length_cannot_be_constant_function_parameter.sol +++ b/test/libsolidity/syntaxTests/array/length/array_length_cannot_be_constant_function_parameter.sol @@ -6,3 +6,4 @@ contract C { // ---- // DeclarationError: (28-45): The "constant" keyword can only be used for state variables. // TypeError: (69-72): Invalid array length, expected integer literal or constant expression. +// TypeError: (64-75): Data location must be "storage" or "memory" for variable, but none was given. diff --git a/test/libsolidity/syntaxTests/functionTypes/function_type_struct_undefined_member.sol b/test/libsolidity/syntaxTests/functionTypes/function_type_struct_undefined_member.sol index ca08afe5b5f8..586753b0bfaa 100644 --- a/test/libsolidity/syntaxTests/functionTypes/function_type_struct_undefined_member.sol +++ b/test/libsolidity/syntaxTests/functionTypes/function_type_struct_undefined_member.sol @@ -8,4 +8,3 @@ library L } // ---- // DeclarationError: (32-35): Identifier not found or not unique. -// TypeError: (63-76): Internal type cannot be used for external function type. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/150_array_with_nonconstant_length.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/150_array_with_nonconstant_length.sol index 49a1851c0f3d..678e7e42b4ca 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/150_array_with_nonconstant_length.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/150_array_with_nonconstant_length.sol @@ -3,3 +3,4 @@ contract c { } // ---- // TypeError: (51-52): Invalid array length, expected integer literal or constant expression. +// TypeError: (45-55): Data location must be "storage" or "memory" for variable, but none was given. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/151_array_with_negative_length.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/151_array_with_negative_length.sol index b87160b0f545..ab736e7daed4 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/151_array_with_negative_length.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/151_array_with_negative_length.sol @@ -3,3 +3,4 @@ contract c { } // ---- // TypeError: (51-53): Array with negative length specified. +// TypeError: (45-56): Data location must be "storage" or "memory" for variable, but none was given. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/318_invalid_array_declaration_with_rational.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/318_invalid_array_declaration_with_rational.sol index 3dd779ec3045..7e18c5fe7cd9 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/318_invalid_array_declaration_with_rational.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/318_invalid_array_declaration_with_rational.sol @@ -5,3 +5,4 @@ contract test { } // ---- // TypeError: (55-58): Array with fractional length specified. +// TypeError: (50-61): Data location must be "storage" or "memory" for variable, but none was given. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/319_invalid_array_declaration_with_signed_fixed_type.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/319_invalid_array_declaration_with_signed_fixed_type.sol index 83f0950db3ae..8aef0ac23796 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/319_invalid_array_declaration_with_signed_fixed_type.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/319_invalid_array_declaration_with_signed_fixed_type.sol @@ -5,3 +5,4 @@ contract test { } // ---- // TypeError: (55-65): Invalid array length, expected integer literal or constant expression. +// TypeError: (50-68): Data location must be "storage" or "memory" for variable, but none was given. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/320_invalid_array_declaration_with_unsigned_fixed_type.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/320_invalid_array_declaration_with_unsigned_fixed_type.sol index 26d5a85e987b..dfd145f8e272 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/320_invalid_array_declaration_with_unsigned_fixed_type.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/320_invalid_array_declaration_with_unsigned_fixed_type.sol @@ -5,3 +5,4 @@ contract test { } // ---- // TypeError: (55-66): Invalid array length, expected integer literal or constant expression. +// TypeError: (50-69): Data location must be "storage" or "memory" for variable, but none was given. diff --git a/test/libsolidity/syntaxTests/parsing/address_invalid_state_mutability.sol b/test/libsolidity/syntaxTests/parsing/address_invalid_state_mutability.sol index 606f5cd0e5c3..4d4a5676f039 100644 --- a/test/libsolidity/syntaxTests/parsing/address_invalid_state_mutability.sol +++ b/test/libsolidity/syntaxTests/parsing/address_invalid_state_mutability.sol @@ -18,9 +18,9 @@ contract C { // TypeError: (33-45): Address types can only be payable or non-payable. // TypeError: (52-64): Address types can only be payable or non-payable. // TypeError: (89-101): Address types can only be payable or non-payable. +// TypeError: (138-150): Address types can only be payable or non-payable. +// TypeError: (156-168): Address types can only be payable or non-payable. // TypeError: (195-207): Address types can only be payable or non-payable. // TypeError: (236-248): Address types can only be payable or non-payable. // TypeError: (300-312): Address types can only be payable or non-payable. // TypeError: (352-364): Address types can only be payable or non-payable. -// TypeError: (138-150): Address types can only be payable or non-payable. -// TypeError: (156-168): Address types can only be payable or non-payable. diff --git a/test/libsolidity/syntaxTests/structs/member_type_eq_name.sol b/test/libsolidity/syntaxTests/structs/member_type_eq_name.sol index 50f0eb365dc0..4aa5ad742d78 100644 --- a/test/libsolidity/syntaxTests/structs/member_type_eq_name.sol +++ b/test/libsolidity/syntaxTests/structs/member_type_eq_name.sol @@ -4,4 +4,3 @@ contract C { } // ---- // TypeError: (25-26): Name has to refer to a struct, enum or contract. -// TypeError: (53-61): Internal type cannot be used for external function type. diff --git a/test/libsolidity/syntaxTests/structs/member_type_func.sol b/test/libsolidity/syntaxTests/structs/member_type_func.sol index 709a40e267d7..1a47c1996006 100644 --- a/test/libsolidity/syntaxTests/structs/member_type_func.sol +++ b/test/libsolidity/syntaxTests/structs/member_type_func.sol @@ -5,4 +5,3 @@ contract C { } // ---- // TypeError: (50-51): Name has to refer to a struct, enum or contract. -// TypeError: (78-86): Internal type cannot be used for external function type. diff --git a/test/libsolidity/syntaxTests/tupleAssignments/tuple_in_tuple_long.sol b/test/libsolidity/syntaxTests/tupleAssignments/tuple_in_tuple_long.sol index 2fa48cfb1b5a..f5d8fb1ba05d 100644 --- a/test/libsolidity/syntaxTests/tupleAssignments/tuple_in_tuple_long.sol +++ b/test/libsolidity/syntaxTests/tupleAssignments/tuple_in_tuple_long.sol @@ -1,12 +1,11 @@ contract C { - function f() { + function f() public { (((((((((((,2),)),)),),))=4))); } } // ---- -// SyntaxError: (15-69): No visibility specified. Did you intend to add "public"? -// TypeError: (46-47): Expression has to be an lvalue. -// TypeError: (60-61): Type int_const 4 is not implicitly convertible to expected type tuple(tuple(tuple(tuple(tuple(,int_const 2),),),),). -// TypeError: (37-61): Tuple component cannot be empty. -// TypeError: (36-62): Tuple component cannot be empty. -// TypeError: (35-63): Tuple component cannot be empty. +// TypeError: (53-54): Expression has to be an lvalue. +// TypeError: (67-68): Type int_const 4 is not implicitly convertible to expected type tuple(tuple(tuple(tuple(tuple(,int_const 2),),),),). +// TypeError: (44-68): Tuple component cannot be empty. +// TypeError: (43-69): Tuple component cannot be empty. +// TypeError: (42-70): Tuple component cannot be empty. diff --git a/test/libsolidity/syntaxTests/types/var_decl_val_mismatch.sol b/test/libsolidity/syntaxTests/types/var_decl_val_mismatch.sol index fbefafa61390..08e0005c287d 100644 --- a/test/libsolidity/syntaxTests/types/var_decl_val_mismatch.sol +++ b/test/libsolidity/syntaxTests/types/var_decl_val_mismatch.sol @@ -1,6 +1,6 @@ contract n { - fallback() + fallback() external { // Used to cause a segfault var (x,y) = (1); @@ -12,5 +12,4 @@ contract n } } // ---- -// SyntaxError: (14-129): No visibility specified. Did you intend to add "external"? -// TypeError: (60-75): Different number of components on the left hand side (2) than on the right hand side (1). +// TypeError: (69-84): Different number of components on the left hand side (2) than on the right hand side (1). From c47714f0ba69ad463d87e3500001e915ed7cb20d Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Wed, 8 Apr 2020 13:24:04 +0200 Subject: [PATCH 023/126] Update ASTJSON tests that fail to compile. --- test/libsolidity/ASTJSON/assembly/nested_functions.json | 8 ++++---- .../ASTJSON/assembly/nested_functions_legacy.json | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/libsolidity/ASTJSON/assembly/nested_functions.json b/test/libsolidity/ASTJSON/assembly/nested_functions.json index 73022e005040..f52cbfc084c3 100644 --- a/test/libsolidity/ASTJSON/assembly/nested_functions.json +++ b/test/libsolidity/ASTJSON/assembly/nested_functions.json @@ -131,8 +131,8 @@ "storageLocation": "default", "typeDescriptions": { - "typeIdentifier": "t_uint256", - "typeString": "uint256" + "typeIdentifier": null, + "typeString": null }, "typeName": { @@ -142,8 +142,8 @@ "src": "49:4:1", "typeDescriptions": { - "typeIdentifier": "t_uint256", - "typeString": "uint256" + "typeIdentifier": null, + "typeString": null } }, "value": null, diff --git a/test/libsolidity/ASTJSON/assembly/nested_functions_legacy.json b/test/libsolidity/ASTJSON/assembly/nested_functions_legacy.json index e2b4926f78ac..c0164e3f7745 100644 --- a/test/libsolidity/ASTJSON/assembly/nested_functions_legacy.json +++ b/test/libsolidity/ASTJSON/assembly/nested_functions_legacy.json @@ -83,7 +83,7 @@ "scope": 7, "stateVariable": false, "storageLocation": "default", - "type": "uint256", + "type": null, "value": null, "visibility": "internal" }, @@ -93,7 +93,7 @@ "attributes": { "name": "uint", - "type": "uint256" + "type": null }, "id": 2, "name": "ElementaryTypeName", From 760ae7f5840ee744713bed8e70b008b86de50f30 Mon Sep 17 00:00:00 2001 From: a3d4 Date: Fri, 10 Apr 2020 02:16:22 +0200 Subject: [PATCH 024/126] Update evmone version to v0.4 in docs and scripts --- .circleci/osx_install_dependencies.sh | 6 +++--- docs/contributing.rst | 2 +- test/Common.h | 6 +++--- test/evmc/README.md | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/osx_install_dependencies.sh b/.circleci/osx_install_dependencies.sh index 59e7c2234029..2409cbad7da6 100755 --- a/.circleci/osx_install_dependencies.sh +++ b/.circleci/osx_install_dependencies.sh @@ -52,8 +52,8 @@ then rm -rf z3-4.8.7-x64-osx-10.14.6 # evmone - wget https://github.com/ethereum/evmone/releases/download/v0.3.0/evmone-0.3.0-darwin-x86_64.tar.gz - tar xzpf evmone-0.3.0-darwin-x86_64.tar.gz -C /usr/local - rm -f evmone-0.3.0-darwin-x86_64.tar.gz + wget https://github.com/ethereum/evmone/releases/download/v0.4.0/evmone-0.4.0-darwin-x86_64.tar.gz + tar xzpf evmone-0.4.0-darwin-x86_64.tar.gz -C /usr/local + rm -f evmone-0.4.0-darwin-x86_64.tar.gz fi diff --git a/docs/contributing.rst b/docs/contributing.rst index 571a298f23da..98a01fd6dc15 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -107,7 +107,7 @@ starting from the current directory. The required file is called ``libevmone.so` ``evmone.dll`` on Windows systems and ``libevmone.dylib`` on macOS. If it is not found, tests that use it are skipped. These tests are ``libsolididty/semanticTests``, ``libsolidity/GasCosts``, ``libsolidity/SolidityEndToEndTest``, part of the soltest suite. To run all tests, download the library from -`GitHub `_ +`GitHub `_ and place it in the project root path or inside the ``deps`` folder. If the ``libz3`` library is not installed on your system, you should disable the diff --git a/test/Common.h b/test/Common.h index a65c95539328..513e7154bc21 100644 --- a/test/Common.h +++ b/test/Common.h @@ -29,13 +29,13 @@ namespace solidity::test #ifdef _WIN32 static constexpr auto evmoneFilename = "evmone.dll"; -static constexpr auto evmoneDownloadLink = "https://github.com/ethereum/evmone/releases/download/v0.3.0/evmone-0.3.0-windows-amd64.zip"; +static constexpr auto evmoneDownloadLink = "https://github.com/ethereum/evmone/releases/download/v0.4.1/evmone-0.4.1-windows-amd64.zip"; #elif defined(__APPLE__) static constexpr auto evmoneFilename = "libevmone.dylib"; -static constexpr auto evmoneDownloadLink = "https://github.com/ethereum/evmone/releases/download/v0.3.0/evmone-0.3.0-darwin-x86_64.tar.gz"; +static constexpr auto evmoneDownloadLink = "https://github.com/ethereum/evmone/releases/download/v0.4.1/evmone-0.4.1-darwin-x86_64.tar.gz"; #else static constexpr auto evmoneFilename = "libevmone.so"; -static constexpr auto evmoneDownloadLink = "https://github.com/ethereum/evmone/releases/download/v0.3.0/evmone-0.3.0-linux-x86_64.tar.gz"; +static constexpr auto evmoneDownloadLink = "https://github.com/ethereum/evmone/releases/download/v0.4.1/evmone-0.4.1-linux-x86_64.tar.gz"; #endif diff --git a/test/evmc/README.md b/test/evmc/README.md index 37417222c0ce..436a47e684ed 100644 --- a/test/evmc/README.md +++ b/test/evmc/README.md @@ -1,3 +1,3 @@ # EVMC -This is an import of [EVMC](https://github.com/ethereum/evmc) version [7.0.0](https://github.com/ethereum/evmc/releases/tag/v7.0.0). +This is an import of [EVMC](https://github.com/ethereum/evmc) version [7.1.0](https://github.com/ethereum/evmc/releases/tag/v7.1.0). From aac7a1e43458fcb8378e05fb168ea25bbd6c24bc Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Tue, 31 Mar 2020 22:04:29 -0500 Subject: [PATCH 025/126] Apply modernize-pass-by-value. --- libevmasm/ExpressionClasses.cpp | 12 +- libevmasm/GasMeter.h | 7 +- libevmasm/KnownState.h | 3 +- liblangutil/CharStream.h | 5 +- liblangutil/Exceptions.h | 9 +- liblangutil/SemVerHandler.h | 6 +- libsolidity/analysis/ConstantEvaluator.h | 4 +- libsolidity/analysis/ControlFlowGraph.h | 5 +- libsolidity/analysis/ViewPureChecker.cpp | 3 +- libsolidity/ast/AST.cpp | 6 +- libsolidity/ast/AST.h | 279 +++++++++++---------- libsolidity/ast/ASTJsonConverter.cpp | 3 +- libsolidity/ast/ASTVisitor.h | 6 +- libsolidity/ast/AsmJsonImporter.h | 4 +- libsolidity/ast/Types.cpp | 5 +- libsolidity/ast/Types.h | 31 +-- libsolidity/formal/SMTLib2Interface.cpp | 5 +- libsolidity/formal/SMTLib2Interface.h | 2 +- libsolidity/interface/CompilerStack.cpp | 5 +- libsolidity/interface/CompilerStack.h | 2 +- libsolidity/interface/StandardCompiler.cpp | 2 +- libsolidity/interface/StandardCompiler.h | 7 +- libsolutil/Common.h | 3 +- libyul/AsmAnalysis.h | 9 +- libyul/backends/evm/EVMCodeTransform.cpp | 7 +- libyul/backends/evm/EVMCodeTransform.h | 2 +- libyul/optimiser/FullInliner.h | 5 +- test/tools/isoltest.cpp | 13 +- tools/solidityUpgrade/UpgradeChange.h | 5 +- 29 files changed, 243 insertions(+), 212 deletions(-) diff --git a/libevmasm/ExpressionClasses.cpp b/libevmasm/ExpressionClasses.cpp index 644f4a274a4e..62780b19f166 100644 --- a/libevmasm/ExpressionClasses.cpp +++ b/libevmasm/ExpressionClasses.cpp @@ -22,15 +22,17 @@ */ #include -#include -#include -#include -#include -#include #include #include #include +#include +#include + +#include +#include +#include + using namespace std; using namespace solidity; using namespace solidity::evmasm; diff --git a/libevmasm/GasMeter.h b/libevmasm/GasMeter.h index 64b772c6e8fd..a6c0f29d9137 100644 --- a/libevmasm/GasMeter.h +++ b/libevmasm/GasMeter.h @@ -28,6 +28,7 @@ #include #include +#include namespace solidity::evmasm { @@ -119,7 +120,7 @@ class GasMeter struct GasConsumption { GasConsumption(unsigned _value = 0, bool _infinite = false): value(_value), isInfinite(_infinite) {} - GasConsumption(u256 _value, bool _infinite = false): value(_value), isInfinite(_infinite) {} + GasConsumption(u256 _value, bool _infinite = false): value(std::move(_value)), isInfinite(_infinite) {} static GasConsumption infinite() { return GasConsumption(0, true); } GasConsumption& operator+=(GasConsumption const& _other); @@ -133,8 +134,8 @@ class GasMeter }; /// Constructs a new gas meter given the current state. - GasMeter(std::shared_ptr const& _state, langutil::EVMVersion _evmVersion, u256 const& _largestMemoryAccess = 0): - m_state(_state), m_evmVersion(_evmVersion), m_largestMemoryAccess(_largestMemoryAccess) {} + GasMeter(std::shared_ptr _state, langutil::EVMVersion _evmVersion, u256 _largestMemoryAccess = 0): + m_state(std::move(_state)), m_evmVersion(_evmVersion), m_largestMemoryAccess(std::move(_largestMemoryAccess)) {} /// @returns an upper bound on the gas consumed by the given instruction and updates /// the state. diff --git a/libevmasm/KnownState.h b/libevmasm/KnownState.h index a9cc4f201868..b19585036aac 100644 --- a/libevmasm/KnownState.h +++ b/libevmasm/KnownState.h @@ -23,6 +23,7 @@ #pragma once +#include #include #include #include @@ -83,7 +84,7 @@ class KnownState explicit KnownState( std::shared_ptr _expressionClasses = std::make_shared() - ): m_expressionClasses(_expressionClasses) + ): m_expressionClasses(std::move(_expressionClasses)) { } diff --git a/liblangutil/CharStream.h b/liblangutil/CharStream.h index eb9b1f12bb13..6962864b979f 100644 --- a/liblangutil/CharStream.h +++ b/liblangutil/CharStream.h @@ -55,6 +55,7 @@ #include #include #include +#include namespace solidity::langutil { @@ -68,8 +69,8 @@ class CharStream { public: CharStream() = default; - explicit CharStream(std::string const& _source, std::string const& name): - m_source(_source), m_name(name) {} + explicit CharStream(std::string _source, std::string name): + m_source(std::move(_source)), m_name(std::move(name)) {} int position() const { return m_position; } bool isPastEndOfInput(size_t _charsForward = 0) const { return (m_position + _charsForward) >= m_source.size(); } diff --git a/liblangutil/Exceptions.h b/liblangutil/Exceptions.h index 575847ed97a5..b8688afa6b39 100644 --- a/liblangutil/Exceptions.h +++ b/liblangutil/Exceptions.h @@ -22,15 +22,16 @@ #pragma once -#include -#include -#include -#include #include #include #include #include +#include +#include +#include +#include + namespace solidity::langutil { class Error; diff --git a/liblangutil/SemVerHandler.h b/liblangutil/SemVerHandler.h index f5cdf860dfdb..d3dfd5a5b0a3 100644 --- a/liblangutil/SemVerHandler.h +++ b/liblangutil/SemVerHandler.h @@ -23,7 +23,9 @@ #pragma once #include + #include +#include #include namespace solidity::langutil @@ -80,8 +82,8 @@ struct SemVerMatchExpression class SemVerMatchExpressionParser { public: - SemVerMatchExpressionParser(std::vector const& _tokens, std::vector const& _literals): - m_tokens(_tokens), m_literals(_literals) + SemVerMatchExpressionParser(std::vector _tokens, std::vector _literals): + m_tokens(std::move(_tokens)), m_literals(std::move(_literals)) {} SemVerMatchExpression parse(); diff --git a/libsolidity/analysis/ConstantEvaluator.h b/libsolidity/analysis/ConstantEvaluator.h index e92113bb8885..f07e6783d342 100644 --- a/libsolidity/analysis/ConstantEvaluator.h +++ b/libsolidity/analysis/ConstantEvaluator.h @@ -24,6 +24,8 @@ #include +#include + namespace solidity::langutil { class ErrorReporter; @@ -47,7 +49,7 @@ class ConstantEvaluator: private ASTConstVisitor ): m_errorReporter(_errorReporter), m_depth(_newDepth), - m_types(_types) + m_types(std::move(_types)) { } diff --git a/libsolidity/analysis/ControlFlowGraph.h b/libsolidity/analysis/ControlFlowGraph.h index 93e2e3b89b7f..4d11ef8f6bb2 100644 --- a/libsolidity/analysis/ControlFlowGraph.h +++ b/libsolidity/analysis/ControlFlowGraph.h @@ -26,6 +26,7 @@ #include #include #include +#include #include namespace solidity::frontend @@ -48,8 +49,8 @@ class VariableOccurrence Assignment, InlineAssembly }; - VariableOccurrence(VariableDeclaration const& _declaration, Kind _kind, std::optional const& _occurrence = {}): - m_declaration(_declaration), m_occurrenceKind(_kind), m_occurrence(_occurrence) + VariableOccurrence(VariableDeclaration const& _declaration, Kind _kind, std::optional _occurrence = {}): + m_declaration(_declaration), m_occurrenceKind(_kind), m_occurrence(std::move(_occurrence)) { } diff --git a/libsolidity/analysis/ViewPureChecker.cpp b/libsolidity/analysis/ViewPureChecker.cpp index 2468b03389a1..57eeb429c713 100644 --- a/libsolidity/analysis/ViewPureChecker.cpp +++ b/libsolidity/analysis/ViewPureChecker.cpp @@ -23,6 +23,7 @@ #include #include +#include #include using namespace std; @@ -41,7 +42,7 @@ class AssemblyViewPureChecker std::function _reportMutability ): m_dialect(_dialect), - m_reportMutability(_reportMutability) {} + m_reportMutability(std::move(_reportMutability)) {} void operator()(yul::Literal const&) {} void operator()(yul::Identifier const&) {} diff --git a/libsolidity/ast/AST.cpp b/libsolidity/ast/AST.cpp index d9425937f12e..0d4a75190d50 100644 --- a/libsolidity/ast/AST.cpp +++ b/libsolidity/ast/AST.cpp @@ -28,16 +28,18 @@ #include #include + #include #include +#include using namespace std; using namespace solidity; using namespace solidity::frontend; -ASTNode::ASTNode(int64_t _id, SourceLocation const& _location): +ASTNode::ASTNode(int64_t _id, SourceLocation _location): m_id(_id), - m_location(_location) + m_location(std::move(_location)) { } diff --git a/libsolidity/ast/AST.h b/libsolidity/ast/AST.h index c0f7e6dc97c7..706d57684353 100644 --- a/libsolidity/ast/AST.h +++ b/libsolidity/ast/AST.h @@ -38,6 +38,7 @@ #include #include #include +#include #include namespace solidity::yul @@ -82,7 +83,7 @@ class ASTNode: private boost::noncopyable using SourceLocation = langutil::SourceLocation; - explicit ASTNode(int64_t _id, SourceLocation const& _location); + explicit ASTNode(int64_t _id, SourceLocation _location); virtual ~ASTNode() {} /// @returns an identifier of this AST node that is unique for a single compilation run. @@ -155,8 +156,8 @@ std::vector ASTNode::filteredNodes(std::vector> co class SourceUnit: public ASTNode { public: - SourceUnit(int64_t _id, SourceLocation const& _location, std::vector> const& _nodes): - ASTNode(_id, _location), m_nodes(_nodes) {} + SourceUnit(int64_t _id, SourceLocation const& _location, std::vector> _nodes): + ASTNode(_id, _location), m_nodes(std::move(_nodes)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -224,10 +225,10 @@ class Declaration: public ASTNode, public Scopable Declaration( int64_t _id, SourceLocation const& _location, - ASTPointer const& _name, + ASTPointer _name, Visibility _visibility = Visibility::Default ): - ASTNode(_id, _location), m_name(_name), m_visibility(_visibility) {} + ASTNode(_id, _location), m_name(std::move(_name)), m_visibility(_visibility) {} /// @returns the declared name. ASTString const& name() const { return *m_name; } @@ -275,9 +276,9 @@ class PragmaDirective: public ASTNode PragmaDirective( int64_t _id, SourceLocation const& _location, - std::vector const& _tokens, - std::vector const& _literals - ): ASTNode(_id, _location), m_tokens(_tokens), m_literals(_literals) + std::vector _tokens, + std::vector _literals + ): ASTNode(_id, _location), m_tokens(std::move(_tokens)), m_literals(std::move(_literals)) {} void accept(ASTVisitor& _visitor) override; @@ -318,12 +319,12 @@ class ImportDirective: public Declaration ImportDirective( int64_t _id, SourceLocation const& _location, - ASTPointer const& _path, + ASTPointer _path, ASTPointer const& _unitAlias, SymbolAliasList _symbolAliases ): Declaration(_id, _location, _unitAlias), - m_path(_path), + m_path(std::move(_path)), m_symbolAliases(move(_symbolAliases)) { } @@ -372,8 +373,8 @@ class StructuredDocumentation: public ASTNode StructuredDocumentation( int64_t _id, SourceLocation const& _location, - ASTPointer const& _text - ): ASTNode(_id, _location), m_text(_text) + ASTPointer _text + ): ASTNode(_id, _location), m_text(std::move(_text)) {} void accept(ASTVisitor& _visitor) override; @@ -394,7 +395,7 @@ class Documented { public: virtual ~Documented() = default; - explicit Documented(ASTPointer const& _documentation): m_documentation(_documentation) {} + explicit Documented(ASTPointer _documentation): m_documentation(std::move(_documentation)) {} /// @return A shared pointer of an ASTString. /// Can contain a nullptr in which case indicates absence of documentation @@ -411,7 +412,7 @@ class StructurallyDocumented { public: virtual ~StructurallyDocumented() = default; - explicit StructurallyDocumented(ASTPointer const& _documentation): m_documentation(_documentation) {} + explicit StructurallyDocumented(ASTPointer _documentation): m_documentation(std::move(_documentation)) {} /// @return A shared pointer of a FormalDocumentation. /// Can contain a nullptr in which case indicates absence of documentation @@ -453,15 +454,15 @@ class ContractDefinition: public Declaration, public StructurallyDocumented SourceLocation const& _location, ASTPointer const& _name, ASTPointer const& _documentation, - std::vector> const& _baseContracts, - std::vector> const& _subNodes, + std::vector> _baseContracts, + std::vector> _subNodes, ContractKind _contractKind = ContractKind::Contract, bool _abstract = false ): Declaration(_id, _location, _name), StructurallyDocumented(_documentation), - m_baseContracts(_baseContracts), - m_subNodes(_subNodes), + m_baseContracts(std::move(_baseContracts)), + m_subNodes(std::move(_subNodes)), m_contractKind(_contractKind), m_abstract(_abstract) {} @@ -538,10 +539,10 @@ class InheritanceSpecifier: public ASTNode InheritanceSpecifier( int64_t _id, SourceLocation const& _location, - ASTPointer const& _baseName, + ASTPointer _baseName, std::unique_ptr>> _arguments ): - ASTNode(_id, _location), m_baseName(_baseName), m_arguments(std::move(_arguments)) {} + ASTNode(_id, _location), m_baseName(std::move(_baseName)), m_arguments(std::move(_arguments)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -568,10 +569,10 @@ class UsingForDirective: public ASTNode UsingForDirective( int64_t _id, SourceLocation const& _location, - ASTPointer const& _libraryName, - ASTPointer const& _typeName + ASTPointer _libraryName, + ASTPointer _typeName ): - ASTNode(_id, _location), m_libraryName(_libraryName), m_typeName(_typeName) {} + ASTNode(_id, _location), m_libraryName(std::move(_libraryName)), m_typeName(std::move(_typeName)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -592,9 +593,9 @@ class StructDefinition: public Declaration int64_t _id, SourceLocation const& _location, ASTPointer const& _name, - std::vector> const& _members + std::vector> _members ): - Declaration(_id, _location, _name), m_members(_members) {} + Declaration(_id, _location, _name), m_members(std::move(_members)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -619,9 +620,9 @@ class EnumDefinition: public Declaration int64_t _id, SourceLocation const& _location, ASTPointer const& _name, - std::vector> const& _members + std::vector> _members ): - Declaration(_id, _location, _name), m_members(_members) {} + Declaration(_id, _location, _name), m_members(std::move(_members)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -664,9 +665,9 @@ class ParameterList: public ASTNode ParameterList( int64_t _id, SourceLocation const& _location, - std::vector> const& _parameters + std::vector> _parameters ): - ASTNode(_id, _location), m_parameters(_parameters) {} + ASTNode(_id, _location), m_parameters(std::move(_parameters)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -688,15 +689,15 @@ class CallableDeclaration: public Declaration, public VariableScope SourceLocation const& _location, ASTPointer const& _name, Visibility _visibility, - ASTPointer const& _parameters, + ASTPointer _parameters, bool _isVirtual = false, - ASTPointer const& _overrides = nullptr, - ASTPointer const& _returnParameters = ASTPointer() + ASTPointer _overrides = nullptr, + ASTPointer _returnParameters = ASTPointer() ): Declaration(_id, _location, _name, _visibility), - m_parameters(_parameters), - m_overrides(_overrides), - m_returnParameters(_returnParameters), + m_parameters(std::move(_parameters)), + m_overrides(std::move(_overrides)), + m_returnParameters(std::move(_returnParameters)), m_isVirtual(_isVirtual) { } @@ -740,10 +741,10 @@ class OverrideSpecifier: public ASTNode OverrideSpecifier( int64_t _id, SourceLocation const& _location, - std::vector> const& _overrides + std::vector> _overrides ): ASTNode(_id, _location), - m_overrides(_overrides) + m_overrides(std::move(_overrides)) { } @@ -771,7 +772,7 @@ class FunctionDefinition: public CallableDeclaration, public StructurallyDocumen ASTPointer const& _overrides, ASTPointer const& _documentation, ASTPointer const& _parameters, - std::vector> const& _modifiers, + std::vector> _modifiers, ASTPointer const& _returnParameters, ASTPointer const& _body ): @@ -780,7 +781,7 @@ class FunctionDefinition: public CallableDeclaration, public StructurallyDocumen ImplementationOptional(_body != nullptr), m_stateMutability(_stateMutability), m_kind(_kind), - m_functionModifiers(_modifiers), + m_functionModifiers(std::move(_modifiers)), m_body(_body) { solAssert(_kind == Token::Constructor || _kind == Token::Function || _kind == Token::Fallback || _kind == Token::Receive, ""); @@ -869,23 +870,23 @@ class VariableDeclaration: public Declaration VariableDeclaration( int64_t _id, SourceLocation const& _location, - ASTPointer const& _type, + ASTPointer _type, ASTPointer const& _name, ASTPointer _value, Visibility _visibility, bool _isStateVar = false, bool _isIndexed = false, Mutability _mutability = Mutability::Mutable, - ASTPointer const& _overrides = nullptr, + ASTPointer _overrides = nullptr, Location _referenceLocation = Location::Unspecified ): Declaration(_id, _location, _name, _visibility), - m_typeName(_type), - m_value(_value), + m_typeName(std::move(_type)), + m_value(std::move(_value)), m_isStateVariable(_isStateVar), m_isIndexed(_isIndexed), m_mutability(_mutability), - m_overrides(_overrides), + m_overrides(std::move(_overrides)), m_location(_referenceLocation) {} @@ -977,11 +978,11 @@ class ModifierDefinition: public CallableDeclaration, public StructurallyDocumen ASTPointer const& _parameters, bool _isVirtual, ASTPointer const& _overrides, - ASTPointer const& _body + ASTPointer _body ): CallableDeclaration(_id, _location, _name, Visibility::Internal, _parameters, _isVirtual, _overrides), StructurallyDocumented(_documentation), - m_body(_body) + m_body(std::move(_body)) { } @@ -1015,10 +1016,10 @@ class ModifierInvocation: public ASTNode ModifierInvocation( int64_t _id, SourceLocation const& _location, - ASTPointer const& _name, + ASTPointer _name, std::unique_ptr>> _arguments ): - ASTNode(_id, _location), m_modifierName(_name), m_arguments(std::move(_arguments)) {} + ASTNode(_id, _location), m_modifierName(std::move(_name)), m_arguments(std::move(_arguments)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1159,8 +1160,8 @@ class ElementaryTypeName: public TypeName class UserDefinedTypeName: public TypeName { public: - UserDefinedTypeName(int64_t _id, SourceLocation const& _location, std::vector const& _namePath): - TypeName(_id, _location), m_namePath(_namePath) {} + UserDefinedTypeName(int64_t _id, SourceLocation const& _location, std::vector _namePath): + TypeName(_id, _location), m_namePath(std::move(_namePath)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1181,12 +1182,12 @@ class FunctionTypeName: public TypeName FunctionTypeName( int64_t _id, SourceLocation const& _location, - ASTPointer const& _parameterTypes, - ASTPointer const& _returnTypes, + ASTPointer _parameterTypes, + ASTPointer _returnTypes, Visibility _visibility, StateMutability _stateMutability ): - TypeName(_id, _location), m_parameterTypes(_parameterTypes), m_returnTypes(_returnTypes), + TypeName(_id, _location), m_parameterTypes(std::move(_parameterTypes)), m_returnTypes(std::move(_returnTypes)), m_visibility(_visibility), m_stateMutability(_stateMutability) {} void accept(ASTVisitor& _visitor) override; @@ -1220,10 +1221,10 @@ class Mapping: public TypeName Mapping( int64_t _id, SourceLocation const& _location, - ASTPointer const& _keyType, - ASTPointer const& _valueType + ASTPointer _keyType, + ASTPointer _valueType ): - TypeName(_id, _location), m_keyType(_keyType), m_valueType(_valueType) {} + TypeName(_id, _location), m_keyType(std::move(_keyType)), m_valueType(std::move(_valueType)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1244,10 +1245,10 @@ class ArrayTypeName: public TypeName ArrayTypeName( int64_t _id, SourceLocation const& _location, - ASTPointer const& _baseType, - ASTPointer const& _length + ASTPointer _baseType, + ASTPointer _length ): - TypeName(_id, _location), m_baseType(_baseType), m_length(_length) {} + TypeName(_id, _location), m_baseType(std::move(_baseType)), m_length(std::move(_length)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1291,9 +1292,9 @@ class InlineAssembly: public Statement SourceLocation const& _location, ASTPointer const& _docString, yul::Dialect const& _dialect, - std::shared_ptr const& _operations + std::shared_ptr _operations ): - Statement(_id, _location, _docString), m_dialect(_dialect), m_operations(_operations) {} + Statement(_id, _location, _docString), m_dialect(_dialect), m_operations(std::move(_operations)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1317,9 +1318,9 @@ class Block: public Statement, public Scopable int64_t _id, SourceLocation const& _location, ASTPointer const& _docString, - std::vector> const& _statements + std::vector> _statements ): - Statement(_id, _location, _docString), m_statements(_statements) {} + Statement(_id, _location, _docString), m_statements(std::move(_statements)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1359,14 +1360,14 @@ class IfStatement: public Statement int64_t _id, SourceLocation const& _location, ASTPointer const& _docString, - ASTPointer const& _condition, - ASTPointer const& _trueBody, - ASTPointer const& _falseBody + ASTPointer _condition, + ASTPointer _trueBody, + ASTPointer _falseBody ): Statement(_id, _location, _docString), - m_condition(_condition), - m_trueBody(_trueBody), - m_falseBody(_falseBody) + m_condition(std::move(_condition)), + m_trueBody(std::move(_trueBody)), + m_falseBody(std::move(_falseBody)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1393,14 +1394,14 @@ class TryCatchClause: public ASTNode, public Scopable TryCatchClause( int64_t _id, SourceLocation const& _location, - ASTPointer const& _errorName, - ASTPointer const& _parameters, - ASTPointer const& _block + ASTPointer _errorName, + ASTPointer _parameters, + ASTPointer _block ): ASTNode(_id, _location), - m_errorName(_errorName), - m_parameters(_parameters), - m_block(_block) + m_errorName(std::move(_errorName)), + m_parameters(std::move(_parameters)), + m_block(std::move(_block)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1438,12 +1439,12 @@ class TryStatement: public Statement int64_t _id, SourceLocation const& _location, ASTPointer const& _docString, - ASTPointer const& _externalCall, - std::vector> const& _clauses + ASTPointer _externalCall, + std::vector> _clauses ): Statement(_id, _location, _docString), - m_externalCall(_externalCall), - m_clauses(_clauses) + m_externalCall(std::move(_externalCall)), + m_clauses(std::move(_clauses)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1476,11 +1477,11 @@ class WhileStatement: public BreakableStatement int64_t _id, SourceLocation const& _location, ASTPointer const& _docString, - ASTPointer const& _condition, - ASTPointer const& _body, + ASTPointer _condition, + ASTPointer _body, bool _isDoWhile ): - BreakableStatement(_id, _location, _docString), m_condition(_condition), m_body(_body), + BreakableStatement(_id, _location, _docString), m_condition(std::move(_condition)), m_body(std::move(_body)), m_isDoWhile(_isDoWhile) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1505,16 +1506,16 @@ class ForStatement: public BreakableStatement, public Scopable int64_t _id, SourceLocation const& _location, ASTPointer const& _docString, - ASTPointer const& _initExpression, - ASTPointer const& _conditionExpression, - ASTPointer const& _loopExpression, - ASTPointer const& _body + ASTPointer _initExpression, + ASTPointer _conditionExpression, + ASTPointer _loopExpression, + ASTPointer _body ): BreakableStatement(_id, _location, _docString), - m_initExpression(_initExpression), - m_condExpression(_conditionExpression), - m_loopExpression(_loopExpression), - m_body(_body) + m_initExpression(std::move(_initExpression)), + m_condExpression(std::move(_conditionExpression)), + m_loopExpression(std::move(_loopExpression)), + m_body(std::move(_body)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1563,7 +1564,7 @@ class Return: public Statement SourceLocation const& _location, ASTPointer const& _docString, ASTPointer _expression - ): Statement(_id, _location, _docString), m_expression(_expression) {} + ): Statement(_id, _location, _docString), m_expression(std::move(_expression)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1597,9 +1598,9 @@ class EmitStatement: public Statement int64_t _id, SourceLocation const& _location, ASTPointer const& _docString, - ASTPointer const& _functionCall + ASTPointer _functionCall ): - Statement(_id, _location, _docString), m_eventCall(_functionCall) {} + Statement(_id, _location, _docString), m_eventCall(std::move(_functionCall)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1624,10 +1625,10 @@ class VariableDeclarationStatement: public Statement int64_t _id, SourceLocation const& _location, ASTPointer const& _docString, - std::vector> const& _variables, - ASTPointer const& _initialValue + std::vector> _variables, + ASTPointer _initialValue ): - Statement(_id, _location, _docString), m_variables(_variables), m_initialValue(_initialValue) {} + Statement(_id, _location, _docString), m_variables(std::move(_variables)), m_initialValue(std::move(_initialValue)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1656,7 +1657,7 @@ class ExpressionStatement: public Statement ASTPointer const& _docString, ASTPointer _expression ): - Statement(_id, _location, _docString), m_expression(_expression) {} + Statement(_id, _location, _docString), m_expression(std::move(_expression)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1690,14 +1691,14 @@ class Conditional: public Expression Conditional( int64_t _id, SourceLocation const& _location, - ASTPointer const& _condition, - ASTPointer const& _trueExpression, - ASTPointer const& _falseExpression + ASTPointer _condition, + ASTPointer _trueExpression, + ASTPointer _falseExpression ): Expression(_id, _location), - m_condition(_condition), - m_trueExpression(_trueExpression), - m_falseExpression(_falseExpression) + m_condition(std::move(_condition)), + m_trueExpression(std::move(_trueExpression)), + m_falseExpression(std::move(_falseExpression)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1720,14 +1721,14 @@ class Assignment: public Expression Assignment( int64_t _id, SourceLocation const& _location, - ASTPointer const& _leftHandSide, + ASTPointer _leftHandSide, Token _assignmentOperator, - ASTPointer const& _rightHandSide + ASTPointer _rightHandSide ): Expression(_id, _location), - m_leftHandSide(_leftHandSide), + m_leftHandSide(std::move(_leftHandSide)), m_assigmentOperator(_assignmentOperator), - m_rightHandSide(_rightHandSide) + m_rightHandSide(std::move(_rightHandSide)) { solAssert(TokenTraits::isAssignmentOp(_assignmentOperator), ""); } @@ -1758,11 +1759,11 @@ class TupleExpression: public Expression TupleExpression( int64_t _id, SourceLocation const& _location, - std::vector> const& _components, + std::vector> _components, bool _isArray ): Expression(_id, _location), - m_components(_components), + m_components(std::move(_components)), m_isArray(_isArray) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1786,12 +1787,12 @@ class UnaryOperation: public Expression int64_t _id, SourceLocation const& _location, Token _operator, - ASTPointer const& _subExpression, + ASTPointer _subExpression, bool _isPrefix ): Expression(_id, _location), m_operator(_operator), - m_subExpression(_subExpression), + m_subExpression(std::move(_subExpression)), m_isPrefix(_isPrefix) { solAssert(TokenTraits::isUnaryOp(_operator), ""); @@ -1819,11 +1820,11 @@ class BinaryOperation: public Expression BinaryOperation( int64_t _id, SourceLocation const& _location, - ASTPointer const& _left, + ASTPointer _left, Token _operator, - ASTPointer const& _right + ASTPointer _right ): - Expression(_id, _location), m_left(_left), m_operator(_operator), m_right(_right) + Expression(_id, _location), m_left(std::move(_left)), m_operator(_operator), m_right(std::move(_right)) { solAssert(TokenTraits::isBinaryOp(_operator) || TokenTraits::isCompareOp(_operator), ""); } @@ -1851,11 +1852,11 @@ class FunctionCall: public Expression FunctionCall( int64_t _id, SourceLocation const& _location, - ASTPointer const& _expression, - std::vector> const& _arguments, - std::vector> const& _names + ASTPointer _expression, + std::vector> _arguments, + std::vector> _names ): - Expression(_id, _location), m_expression(_expression), m_arguments(_arguments), m_names(_names) {} + Expression(_id, _location), m_expression(std::move(_expression)), m_arguments(std::move(_arguments)), m_names(std::move(_names)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1881,11 +1882,11 @@ class FunctionCallOptions: public Expression FunctionCallOptions( int64_t _id, SourceLocation const& _location, - ASTPointer const& _expression, - std::vector> const& _options, - std::vector> const& _names + ASTPointer _expression, + std::vector> _options, + std::vector> _names ): - Expression(_id, _location), m_expression(_expression), m_options(_options), m_names(_names) {} + Expression(_id, _location), m_expression(std::move(_expression)), m_options(std::move(_options)), m_names(std::move(_names)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1910,9 +1911,9 @@ class NewExpression: public Expression NewExpression( int64_t _id, SourceLocation const& _location, - ASTPointer const& _typeName + ASTPointer _typeName ): - Expression(_id, _location), m_typeName(_typeName) {} + Expression(_id, _location), m_typeName(std::move(_typeName)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1932,9 +1933,9 @@ class MemberAccess: public Expression int64_t _id, SourceLocation const& _location, ASTPointer _expression, - ASTPointer const& _memberName + ASTPointer _memberName ): - Expression(_id, _location), m_expression(_expression), m_memberName(_memberName) {} + Expression(_id, _location), m_expression(std::move(_expression)), m_memberName(std::move(_memberName)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; Expression const& expression() const { return *m_expression; } @@ -1956,10 +1957,10 @@ class IndexAccess: public Expression IndexAccess( int64_t _id, SourceLocation const& _location, - ASTPointer const& _base, - ASTPointer const& _index + ASTPointer _base, + ASTPointer _index ): - Expression(_id, _location), m_base(_base), m_index(_index) {} + Expression(_id, _location), m_base(std::move(_base)), m_index(std::move(_index)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -1980,11 +1981,11 @@ class IndexRangeAccess: public Expression IndexRangeAccess( int64_t _id, SourceLocation const& _location, - ASTPointer const& _base, - ASTPointer const& _start, - ASTPointer const& _end + ASTPointer _base, + ASTPointer _start, + ASTPointer _end ): - Expression(_id, _location), m_base(_base), m_start(_start), m_end(_end) {} + Expression(_id, _location), m_base(std::move(_base)), m_start(std::move(_start)), m_end(std::move(_end)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -2017,9 +2018,9 @@ class Identifier: public PrimaryExpression Identifier( int64_t _id, SourceLocation const& _location, - ASTPointer const& _name + ASTPointer _name ): - PrimaryExpression(_id, _location), m_name(_name) {} + PrimaryExpression(_id, _location), m_name(std::move(_name)) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; @@ -2042,10 +2043,10 @@ class ElementaryTypeNameExpression: public PrimaryExpression ElementaryTypeNameExpression( int64_t _id, SourceLocation const& _location, - ASTPointer const& _type + ASTPointer _type ): PrimaryExpression(_id, _location), - m_type(_type) + m_type(std::move(_type)) { } void accept(ASTVisitor& _visitor) override; @@ -2081,10 +2082,10 @@ class Literal: public PrimaryExpression int64_t _id, SourceLocation const& _location, Token _token, - ASTPointer const& _value, + ASTPointer _value, SubDenomination _sub = SubDenomination::None ): - PrimaryExpression(_id, _location), m_token(_token), m_value(_value), m_subDenomination(_sub) {} + PrimaryExpression(_id, _location), m_token(_token), m_value(std::move(_value)), m_subDenomination(_sub) {} void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; diff --git a/libsolidity/ast/ASTJsonConverter.cpp b/libsolidity/ast/ASTJsonConverter.cpp index 81802907f552..78f1dba7462a 100644 --- a/libsolidity/ast/ASTJsonConverter.cpp +++ b/libsolidity/ast/ASTJsonConverter.cpp @@ -34,6 +34,7 @@ #include #include +#include #include #include @@ -45,7 +46,7 @@ namespace solidity::frontend ASTJsonConverter::ASTJsonConverter(bool _legacy, map _sourceIndices): m_legacy(_legacy), - m_sourceIndices(_sourceIndices) + m_sourceIndices(std::move(_sourceIndices)) { } diff --git a/libsolidity/ast/ASTVisitor.h b/libsolidity/ast/ASTVisitor.h index db67dc392137..52e90b0d75b1 100644 --- a/libsolidity/ast/ASTVisitor.h +++ b/libsolidity/ast/ASTVisitor.h @@ -23,9 +23,11 @@ #pragma once #include + #include #include #include +#include namespace solidity::frontend { @@ -299,7 +301,7 @@ class SimpleASTVisitor: public ASTConstVisitor SimpleASTVisitor( std::function _onVisit, std::function _onEndVisit - ): m_onVisit(_onVisit), m_onEndVisit(_onEndVisit) {} + ): m_onVisit(std::move(_onVisit)), m_onEndVisit(std::move(_onEndVisit)) {} protected: bool visitNode(ASTNode const& _n) override { return m_onVisit ? m_onVisit(_n) : true; } @@ -329,7 +331,7 @@ class ASTReduce: public ASTConstVisitor ASTReduce( std::function _onNode, std::function _onEdge - ): m_onNode(_onNode), m_onEdge(_onEdge) + ): m_onNode(std::move(_onNode)), m_onEdge(std::move(_onEdge)) { } diff --git a/libsolidity/ast/AsmJsonImporter.h b/libsolidity/ast/AsmJsonImporter.h index e7f4821c2501..498b4e489bc8 100644 --- a/libsolidity/ast/AsmJsonImporter.h +++ b/libsolidity/ast/AsmJsonImporter.h @@ -26,6 +26,8 @@ #include #include +#include + namespace solidity::frontend { @@ -35,7 +37,7 @@ namespace solidity::frontend class AsmJsonImporter { public: - explicit AsmJsonImporter(std::string _sourceName) : m_sourceName(_sourceName) {} + explicit AsmJsonImporter(std::string _sourceName) : m_sourceName(std::move(_sourceName)) {} yul::Block createBlock(Json::Value const& _node); private: diff --git a/libsolidity/ast/Types.cpp b/libsolidity/ast/Types.cpp index 2238e86f37b0..59eeb18ee6ed 100644 --- a/libsolidity/ast/Types.cpp +++ b/libsolidity/ast/Types.cpp @@ -43,6 +43,7 @@ #include #include +#include using namespace std; using namespace solidity; @@ -1253,8 +1254,8 @@ StringLiteralType::StringLiteralType(Literal const& _literal): { } -StringLiteralType::StringLiteralType(string const& _value): - m_value{_value} +StringLiteralType::StringLiteralType(string _value): + m_value{std::move(_value)} { } diff --git a/libsolidity/ast/Types.h b/libsolidity/ast/Types.h index d0e774cb7cc7..d07c47dde183 100644 --- a/libsolidity/ast/Types.h +++ b/libsolidity/ast/Types.h @@ -38,6 +38,7 @@ #include #include #include +#include namespace solidity::frontend { @@ -92,8 +93,8 @@ class MemberList public: struct Member { - Member(std::string const& _name, Type const* _type, Declaration const* _declaration = nullptr): - name(_name), + Member(std::string _name, Type const* _type, Declaration const* _declaration = nullptr): + name(std::move(_name)), type(_type), declaration(_declaration) { @@ -106,7 +107,7 @@ class MemberList using MemberMap = std::vector; - explicit MemberList(MemberMap const& _members): m_memberTypes(_members) {} + explicit MemberList(MemberMap _members): m_memberTypes(std::move(_members)) {} void combine(MemberList const& _other); TypePointer memberType(std::string const& _name) const @@ -520,8 +521,8 @@ class FixedPointType: public Type class RationalNumberType: public Type { public: - explicit RationalNumberType(rational const& _value, Type const* _compatibleBytesType = nullptr): - m_value(_value), m_compatibleBytesType(_compatibleBytesType) + explicit RationalNumberType(rational _value, Type const* _compatibleBytesType = nullptr): + m_value(std::move(_value)), m_compatibleBytesType(_compatibleBytesType) {} Category category() const override { return Category::RationalNumber; } @@ -543,7 +544,7 @@ class RationalNumberType: public Type /// @returns the smallest integer type that can hold the value or an empty pointer if not possible. IntegerType const* integerType() const; - /// @returns the smallest fixed type that can hold the value or incurs the least precision loss, + /// @returns the smallest fixed type that can hold the value or incurs the least precision loss, /// unless the value was truncated, then a suitable type will be chosen to indicate such event. /// If the integer part does not fit, returns an empty pointer. FixedPointType const* fixedPointType() const; @@ -582,7 +583,7 @@ class StringLiteralType: public Type { public: explicit StringLiteralType(Literal const& _literal); - explicit StringLiteralType(std::string const& _value); + explicit StringLiteralType(std::string _value); Category category() const override { return Category::StringLiteral; } @@ -744,11 +745,11 @@ class ArrayType: public ReferenceType } /// Constructor for a fixed-size array type ("type[20]") - ArrayType(DataLocation _location, Type const* _baseType, u256 const& _length): + ArrayType(DataLocation _location, Type const* _baseType, u256 _length): ReferenceType(_location), m_baseType(copyForLocationIfReference(_baseType)), m_hasDynamicLength(false), - m_length(_length) + m_length(std::move(_length)) {} Category category() const override { return Category::Array; } @@ -1131,8 +1132,8 @@ class FunctionType: public Type /// Detailed constructor, use with care. FunctionType( - TypePointers const& _parameterTypes, - TypePointers const& _returnParameterTypes, + TypePointers _parameterTypes, + TypePointers _returnParameterTypes, strings _parameterNames = strings(), strings _returnParameterNames = strings(), Kind _kind = Kind::Internal, @@ -1144,10 +1145,10 @@ class FunctionType: public Type bool _saltSet = false, bool _bound = false ): - m_parameterTypes(_parameterTypes), - m_returnParameterTypes(_returnParameterTypes), - m_parameterNames(_parameterNames), - m_returnParameterNames(_returnParameterNames), + m_parameterTypes(std::move(_parameterTypes)), + m_returnParameterTypes(std::move(_returnParameterTypes)), + m_parameterNames(std::move(_parameterNames)), + m_returnParameterNames(std::move(_returnParameterNames)), m_kind(_kind), m_stateMutability(_stateMutability), m_arbitraryParameters(_arbitraryParameters), diff --git a/libsolidity/formal/SMTLib2Interface.cpp b/libsolidity/formal/SMTLib2Interface.cpp index 0c6b8715de40..76527baeb40c 100644 --- a/libsolidity/formal/SMTLib2Interface.cpp +++ b/libsolidity/formal/SMTLib2Interface.cpp @@ -29,6 +29,7 @@ #include #include #include +#include using namespace std; using namespace solidity; @@ -38,10 +39,10 @@ using namespace solidity::frontend::smt; SMTLib2Interface::SMTLib2Interface( map const& _queryResponses, - ReadCallback::Callback const& _smtCallback + ReadCallback::Callback _smtCallback ): m_queryResponses(_queryResponses), - m_smtCallback(_smtCallback) + m_smtCallback(std::move(_smtCallback)) { reset(); } diff --git a/libsolidity/formal/SMTLib2Interface.h b/libsolidity/formal/SMTLib2Interface.h index e3ebd8b5eca8..5310f7e9576d 100644 --- a/libsolidity/formal/SMTLib2Interface.h +++ b/libsolidity/formal/SMTLib2Interface.h @@ -39,7 +39,7 @@ class SMTLib2Interface: public SolverInterface, public boost::noncopyable public: explicit SMTLib2Interface( std::map const& _queryResponses, - ReadCallback::Callback const& _smtCallback + ReadCallback::Callback _smtCallback ); void reset() override; diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 08e44531d697..17348c24278f 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -68,6 +68,7 @@ #include #include +#include using namespace std; using namespace solidity; @@ -79,8 +80,8 @@ using solidity::util::toHex; static int g_compilerStackCounts = 0; -CompilerStack::CompilerStack(ReadCallback::Callback const& _readFile): - m_readFile{_readFile}, +CompilerStack::CompilerStack(ReadCallback::Callback _readFile): + m_readFile{std::move(_readFile)}, m_enabledSMTSolvers{smt::SMTSolverChoice::All()}, m_generateIR{false}, m_generateEwasm{false}, diff --git a/libsolidity/interface/CompilerStack.h b/libsolidity/interface/CompilerStack.h index b0ea0603f6b0..8bc8828c6305 100644 --- a/libsolidity/interface/CompilerStack.h +++ b/libsolidity/interface/CompilerStack.h @@ -108,7 +108,7 @@ class CompilerStack: boost::noncopyable /// Creates a new compiler stack. /// @param _readFile callback used to read files for import statements. Must return /// and must not emit exceptions. - explicit CompilerStack(ReadCallback::Callback const& _readFile = ReadCallback::Callback()); + explicit CompilerStack(ReadCallback::Callback _readFile = ReadCallback::Callback()); ~CompilerStack(); diff --git a/libsolidity/interface/StandardCompiler.cpp b/libsolidity/interface/StandardCompiler.cpp index fc2b513002d0..1b35040aa75c 100644 --- a/libsolidity/interface/StandardCompiler.cpp +++ b/libsolidity/interface/StandardCompiler.cpp @@ -591,7 +591,7 @@ boost::variant StandardCompile )); else { - ret.sources[sourceName] = result.responseOrErrorMessage; + ret.sources[sourceName] = result.responseOrErrorMessage; found = true; break; } diff --git a/libsolidity/interface/StandardCompiler.h b/libsolidity/interface/StandardCompiler.h index b37025576adc..885417d8bd56 100644 --- a/libsolidity/interface/StandardCompiler.h +++ b/libsolidity/interface/StandardCompiler.h @@ -24,8 +24,9 @@ #include -#include #include +#include +#include namespace solidity::frontend { @@ -40,8 +41,8 @@ class StandardCompiler: boost::noncopyable /// Creates a new StandardCompiler. /// @param _readFile callback used to read files for import statements. Must return /// and must not emit exceptions. - explicit StandardCompiler(ReadCallback::Callback const& _readFile = ReadCallback::Callback()): - m_readFile(_readFile) + explicit StandardCompiler(ReadCallback::Callback _readFile = ReadCallback::Callback()): + m_readFile(std::move(_readFile)) { } diff --git a/libsolutil/Common.h b/libsolutil/Common.h index 35bd37fe889a..f2d9c1d84371 100644 --- a/libsolutil/Common.h +++ b/libsolutil/Common.h @@ -47,6 +47,7 @@ #include #include +#include #include #include #include @@ -120,7 +121,7 @@ inline std::ostream& operator<<(std::ostream& os, bytes const& _bytes) class ScopeGuard { public: - explicit ScopeGuard(std::function _f): m_f(_f) {} + explicit ScopeGuard(std::function _f): m_f(std::move(_f)) {} ~ScopeGuard() { m_f(); } private: diff --git a/libyul/AsmAnalysis.h b/libyul/AsmAnalysis.h index 853f9f654f34..09ca73ee2bb1 100644 --- a/libyul/AsmAnalysis.h +++ b/libyul/AsmAnalysis.h @@ -34,6 +34,7 @@ #include #include #include +#include namespace solidity::langutil { @@ -58,14 +59,14 @@ class AsmAnalyzer AsmAnalysisInfo& _analysisInfo, langutil::ErrorReporter& _errorReporter, Dialect const& _dialect, - ExternalIdentifierAccess::Resolver const& _resolver = ExternalIdentifierAccess::Resolver(), - std::set const& _dataNames = {} + ExternalIdentifierAccess::Resolver _resolver = ExternalIdentifierAccess::Resolver(), + std::set _dataNames = {} ): - m_resolver(_resolver), + m_resolver(std::move(_resolver)), m_info(_analysisInfo), m_errorReporter(_errorReporter), m_dialect(_dialect), - m_dataNames(_dataNames) + m_dataNames(std::move(_dataNames)) { if (EVMDialect const* evmDialect = dynamic_cast(&m_dialect)) m_evmVersion = evmDialect->evmVersion(); diff --git a/libyul/backends/evm/EVMCodeTransform.cpp b/libyul/backends/evm/EVMCodeTransform.cpp index cd24db738fa0..6ca55821d2d4 100644 --- a/libyul/backends/evm/EVMCodeTransform.cpp +++ b/libyul/backends/evm/EVMCodeTransform.cpp @@ -29,6 +29,7 @@ #include +#include #include using namespace std; @@ -100,7 +101,7 @@ CodeTransform::CodeTransform( EVMDialect const& _dialect, BuiltinContext& _builtinContext, bool _evm15, - ExternalIdentifierAccess const& _identifierAccess, + ExternalIdentifierAccess _identifierAccess, bool _useNamedLabelsForFunctions, shared_ptr _context ): @@ -111,8 +112,8 @@ CodeTransform::CodeTransform( m_allowStackOpt(_allowStackOpt), m_evm15(_evm15), m_useNamedLabelsForFunctions(_useNamedLabelsForFunctions), - m_identifierAccess(_identifierAccess), - m_context(_context) + m_identifierAccess(std::move(_identifierAccess)), + m_context(std::move(_context)) { if (!m_context) { diff --git a/libyul/backends/evm/EVMCodeTransform.h b/libyul/backends/evm/EVMCodeTransform.h index eae870815066..6447aa7b1d13 100644 --- a/libyul/backends/evm/EVMCodeTransform.h +++ b/libyul/backends/evm/EVMCodeTransform.h @@ -152,7 +152,7 @@ class CodeTransform EVMDialect const& _dialect, BuiltinContext& _builtinContext, bool _evm15, - ExternalIdentifierAccess const& _identifierAccess, + ExternalIdentifierAccess _identifierAccess, bool _useNamedLabelsForFunctions, std::shared_ptr _context ); diff --git a/libyul/optimiser/FullInliner.h b/libyul/optimiser/FullInliner.h index 81c608de8f1e..1c1ad0d95355 100644 --- a/libyul/optimiser/FullInliner.h +++ b/libyul/optimiser/FullInliner.h @@ -31,6 +31,7 @@ #include #include +#include namespace solidity::yul { @@ -147,10 +148,10 @@ class BodyCopier: public ASTCopier public: BodyCopier( NameDispenser& _nameDispenser, - std::map const& _variableReplacements + std::map _variableReplacements ): m_nameDispenser(_nameDispenser), - m_variableReplacements(_variableReplacements) + m_variableReplacements(std::move(_variableReplacements)) {} using ASTCopier::operator (); diff --git a/test/tools/isoltest.cpp b/test/tools/isoltest.cpp index ad689ffcf6c4..7a3c63cff77c 100644 --- a/test/tools/isoltest.cpp +++ b/test/tools/isoltest.cpp @@ -31,10 +31,11 @@ #include #include -#include #include +#include #include #include +#include #if defined(_WIN32) #include @@ -71,7 +72,7 @@ struct TestStats class TestFilter { public: - explicit TestFilter(string const& _filter): m_filter(_filter) + explicit TestFilter(string _filter): m_filter(std::move(_filter)) { string filter{m_filter}; @@ -97,14 +98,14 @@ class TestTool TestTool( TestCreator _testCaseCreator, TestOptions const& _options, - fs::path const& _path, - string const& _name + fs::path _path, + string _name ): m_testCaseCreator(_testCaseCreator), m_options(_options), m_filter(TestFilter{_options.testFilter}), - m_path(_path), - m_name(_name) + m_path(std::move(_path)), + m_name(std::move(_name)) {} enum class Result diff --git a/tools/solidityUpgrade/UpgradeChange.h b/tools/solidityUpgrade/UpgradeChange.h index 4d279fa60905..19f08c6a0ec7 100644 --- a/tools/solidityUpgrade/UpgradeChange.h +++ b/tools/solidityUpgrade/UpgradeChange.h @@ -8,7 +8,7 @@ solidity is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License @@ -21,6 +21,7 @@ #include #include +#include namespace solidity::tools { @@ -50,7 +51,7 @@ class UpgradeChange : m_location(_location), m_source(_location.source->source()), - m_patch(_patch), + m_patch(std::move(_patch)), m_level(_level) {} ~UpgradeChange() {} From 3b83365b42d7a8b4f9ed603beb30f72ecc4ea467 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 16 Mar 2020 11:47:39 +0100 Subject: [PATCH 026/126] [libsolidity] TryStatement: Adding AST accessors for success/structured/fail clauses. --- libsolidity/ast/AST.cpp | 22 ++++++++++++++++++++++ libsolidity/ast/AST.h | 4 ++++ 2 files changed, 26 insertions(+) diff --git a/libsolidity/ast/AST.cpp b/libsolidity/ast/AST.cpp index d9425937f12e..badb59f35313 100644 --- a/libsolidity/ast/AST.cpp +++ b/libsolidity/ast/AST.cpp @@ -756,3 +756,25 @@ string Literal::getChecksummedAddress() const address.insert(address.begin(), 40 - address.size(), '0'); return util::getChecksummedAddress(address); } + +TryCatchClause const* TryStatement::successClause() const +{ + solAssert(m_clauses.size() > 0, ""); + return m_clauses[0].get(); +} + +TryCatchClause const* TryStatement::structuredClause() const +{ + for (size_t i = 1; i < m_clauses.size(); ++i) + if (m_clauses[i]->errorName() == "Error") + return m_clauses[i].get(); + return nullptr; +} + +TryCatchClause const* TryStatement::fallbackClause() const +{ + for (size_t i = 1; i < m_clauses.size(); ++i) + if (m_clauses[i]->errorName().empty()) + return m_clauses[i].get(); + return nullptr; +} diff --git a/libsolidity/ast/AST.h b/libsolidity/ast/AST.h index c0f7e6dc97c7..0ef56bbde1b9 100644 --- a/libsolidity/ast/AST.h +++ b/libsolidity/ast/AST.h @@ -1451,6 +1451,10 @@ class TryStatement: public Statement Expression const& externalCall() const { return *m_externalCall; } std::vector> const& clauses() const { return m_clauses; } + TryCatchClause const* successClause() const; + TryCatchClause const* structuredClause() const; + TryCatchClause const* fallbackClause() const; + private: ASTPointer m_externalCall; std::vector> m_clauses; From d00d3c45b116fc021335cb60ad07406c26244311 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 10 Feb 2020 11:58:36 +0100 Subject: [PATCH 027/126] [Sol2Yul] Implements codegen part for try/catch statements. --- libsolidity/CMakeLists.txt | 2 + libsolidity/codegen/ContractCompiler.cpp | 38 +-- libsolidity/codegen/ExpressionCompiler.cpp | 35 +-- libsolidity/codegen/ReturnInfo.cpp | 55 +++++ libsolidity/codegen/ReturnInfo.h | 47 ++++ libsolidity/codegen/YulUtilFunctions.cpp | 81 +++++++ libsolidity/codegen/YulUtilFunctions.h | 15 ++ .../codegen/ir/IRGenerationContext.cpp | 12 + libsolidity/codegen/ir/IRGenerationContext.h | 4 + .../codegen/ir/IRGeneratorForStatements.cpp | 224 +++++++++++++----- .../codegen/ir/IRGeneratorForStatements.h | 16 ++ .../semanticTests/tryCatch/lowLevel.sol | 1 + .../semanticTests/tryCatch/nested.sol | 1 + .../tryCatch/return_function.sol | 2 + .../semanticTests/tryCatch/simple.sol | 7 +- .../semanticTests/tryCatch/simple_notuple.sol | 19 ++ .../semanticTests/tryCatch/structured.sol | 1 + .../tryCatch/structuredAndLowLevel.sol | 1 + .../semanticTests/tryCatch/super_trivial.sol | 18 ++ 19 files changed, 456 insertions(+), 123 deletions(-) create mode 100644 libsolidity/codegen/ReturnInfo.cpp create mode 100644 libsolidity/codegen/ReturnInfo.h create mode 100644 test/libsolidity/semanticTests/tryCatch/simple_notuple.sol create mode 100644 test/libsolidity/semanticTests/tryCatch/super_trivial.sol diff --git a/libsolidity/CMakeLists.txt b/libsolidity/CMakeLists.txt index 7e07b88079ec..fe1bbad7aeac 100644 --- a/libsolidity/CMakeLists.txt +++ b/libsolidity/CMakeLists.txt @@ -75,6 +75,8 @@ set(sources codegen/LValue.h codegen/MultiUseYulFunctionCollector.h codegen/MultiUseYulFunctionCollector.cpp + codegen/ReturnInfo.h + codegen/ReturnInfo.cpp codegen/YulUtilFunctions.h codegen/YulUtilFunctions.cpp codegen/ir/IRGenerator.cpp diff --git a/libsolidity/codegen/ContractCompiler.cpp b/libsolidity/codegen/ContractCompiler.cpp index a599de6a47f8..18de2ba840e1 100644 --- a/libsolidity/codegen/ContractCompiler.cpp +++ b/libsolidity/codegen/ContractCompiler.cpp @@ -949,43 +949,7 @@ void ContractCompiler::handleCatch(vector> const& _ca // Try to decode the error message. // If this fails, leaves 0 on the stack, otherwise the pointer to the data string. - m_context << u256(0); - m_context.appendInlineAssembly( - util::Whiskers(R"({ - data := mload(0x40) - mstore(data, 0) - for {} 1 {} { - if lt(returndatasize(), 0x44) { data := 0 break } - returndatacopy(0, 0, 4) - let sig := - if iszero(eq(sig, 0x)) { data := 0 break } - returndatacopy(data, 4, sub(returndatasize(), 4)) - let offset := mload(data) - if or( - gt(offset, 0xffffffffffffffff), - gt(add(offset, 0x24), returndatasize()) - ) { - data := 0 - break - } - let msg := add(data, offset) - let length := mload(msg) - if gt(length, 0xffffffffffffffff) { data := 0 break } - let end := add(add(msg, 0x20), length) - if gt(end, add(data, returndatasize())) { data := 0 break } - mstore(0x40, and(add(end, 0x1f), not(0x1f))) - data := msg - break - } - })") - ("ErrorSignature", errorHash) - ("getSig", - m_context.evmVersion().hasBitwiseShifting() ? - "shr(224, mload(0))" : - "div(mload(0), " + (u256(1) << 224).str() + ")" - ).render(), - {"data"} - ); + m_context.callYulFunction(m_context.utilFunctions().tryDecodeErrorMessageFunction(), 0, 1); m_context << Instruction::DUP1; AssemblyItem decodeSuccessTag = m_context.appendConditionalJump(); m_context << Instruction::POP; diff --git a/libsolidity/codegen/ExpressionCompiler.cpp b/libsolidity/codegen/ExpressionCompiler.cpp index a02497561279..354a2409924f 100644 --- a/libsolidity/codegen/ExpressionCompiler.cpp +++ b/libsolidity/codegen/ExpressionCompiler.cpp @@ -22,12 +22,14 @@ #include -#include -#include +#include #include #include #include +#include +#include + #include #include #include @@ -2185,30 +2187,11 @@ void ExpressionCompiler::appendExternalFunctionCall( solAssert(!_functionType.isBareCall(), ""); } - bool haveReturndatacopy = m_context.evmVersion().supportsReturndata(); - unsigned retSize = 0; - bool dynamicReturnSize = false; - TypePointers returnTypes; - if (!returnSuccessConditionAndReturndata) - { - if (haveReturndatacopy) - returnTypes = _functionType.returnParameterTypes(); - else - returnTypes = _functionType.returnParameterTypesWithoutDynamicTypes(); - - for (auto const& retType: returnTypes) - if (retType->isDynamicallyEncoded()) - { - solAssert(haveReturndatacopy, ""); - dynamicReturnSize = true; - retSize = 0; - break; - } - else if (retType->decodingType()) - retSize += retType->decodingType()->calldataEncodedSize(); - else - retSize += retType->calldataEncodedSize(); - } + ReturnInfo const returnInfo{m_context.evmVersion(), _functionType}; + bool const haveReturndatacopy = m_context.evmVersion().supportsReturndata(); + unsigned const retSize = returnInfo.estimatedReturnSize; + bool const dynamicReturnSize = returnInfo.dynamicReturnSize; + TypePointers const& returnTypes = returnInfo.returnTypes; // Evaluate arguments. TypePointers argumentTypes; diff --git a/libsolidity/codegen/ReturnInfo.cpp b/libsolidity/codegen/ReturnInfo.cpp new file mode 100644 index 000000000000..b522addd4d25 --- /dev/null +++ b/libsolidity/codegen/ReturnInfo.cpp @@ -0,0 +1,55 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include +#include + +using namespace solidity::frontend; +using namespace solidity::langutil; + +ReturnInfo::ReturnInfo(EVMVersion const& _evmVersion, FunctionType const& _functionType) +{ + FunctionType::Kind const funKind = _functionType.kind(); + bool const haveReturndatacopy = _evmVersion.supportsReturndata(); + bool const returnSuccessConditionAndReturndata = + funKind == FunctionType::Kind::BareCall || + funKind == FunctionType::Kind::BareDelegateCall || + funKind == FunctionType::Kind::BareStaticCall; + + if (!returnSuccessConditionAndReturndata) + { + if (haveReturndatacopy) + returnTypes = _functionType.returnParameterTypes(); + else + returnTypes = _functionType.returnParameterTypesWithoutDynamicTypes(); + + for (auto const& retType: returnTypes) + if (retType->isDynamicallyEncoded()) + { + solAssert(haveReturndatacopy, ""); + dynamicReturnSize = true; + estimatedReturnSize = 0; + break; + } + else if (retType->decodingType()) + estimatedReturnSize += retType->decodingType()->calldataEncodedSize(); + else + estimatedReturnSize += retType->calldataEncodedSize(); + } +} diff --git a/libsolidity/codegen/ReturnInfo.h b/libsolidity/codegen/ReturnInfo.h new file mode 100644 index 000000000000..01aa308620ff --- /dev/null +++ b/libsolidity/codegen/ReturnInfo.h @@ -0,0 +1,47 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +/** + * Component that computes information relevant during decoding an external function + * call's return values. + */ +#pragma once + +#include +#include + +namespace solidity::frontend +{ + +/** + * Computes and holds information relevant during decoding an external function + * call's return values. + */ +struct ReturnInfo +{ + ReturnInfo(langutil::EVMVersion const& _evmVersion, FunctionType const& _functionType); + + /// Vector of TypePointer, for each return variable. Dynamic types are already replaced if required. + TypePointers returnTypes = {}; + + /// Boolean, indicating whether or not return size is only known at runtime. + bool dynamicReturnSize = false; + + /// Contains the at compile time estimated return size. + unsigned estimatedReturnSize = 0; +}; + +} diff --git a/libsolidity/codegen/YulUtilFunctions.cpp b/libsolidity/codegen/YulUtilFunctions.cpp index 28a4f69fbbf0..2f14ae744a8e 100644 --- a/libsolidity/codegen/YulUtilFunctions.cpp +++ b/libsolidity/codegen/YulUtilFunctions.cpp @@ -2252,3 +2252,84 @@ string YulUtilFunctions::revertReasonIfDebug(string const& _message) { return revertReasonIfDebug(m_revertStrings, _message); } + +string YulUtilFunctions::tryDecodeErrorMessageFunction() +{ + string const functionName = "try_decode_error_message"; + + return m_functionCollector.createFunction(functionName, [&]() { + return util::Whiskers(R"( + function () -> ret { + if lt(returndatasize(), 0x44) { leave } + + returndatacopy(0, 0, 4) + let sig := (mload(0)) + if iszero(eq(sig, 0x)) { leave } + + let data := mload() + returndatacopy(data, 4, sub(returndatasize(), 4)) + + let offset := mload(data) + if or( + gt(offset, 0xffffffffffffffff), + gt(add(offset, 0x24), returndatasize()) + ) { + leave + } + + let msg := add(data, offset) + let length := mload(msg) + if gt(length, 0xffffffffffffffff) { leave } + + let end := add(add(msg, 0x20), length) + if gt(end, add(data, returndatasize())) { leave } + + mstore(, add(add(msg, 0x20), (length))) + ret := msg + } + )") + ("functionName", functionName) + ("shr224", shiftRightFunction(224)) + ("ErrorSignature", FixedHash<4>(util::keccak256("Error(string)")).hex()) + ("freeMemoryPointer", to_string(CompilerUtils::freeMemoryPointer)) + ("roundUp", roundUpFunction()) + .render(); + }); +} + +string YulUtilFunctions::extractReturndataFunction() +{ + string const functionName = "extract_returndata"; + + return m_functionCollector.createFunction(functionName, [&]() { + return util::Whiskers(R"( + function () -> data { + + switch returndatasize() + case 0 { + data := () + } + default { + // allocate some memory into data of size returndatasize() + PADDING + data := ((add(returndatasize(), 0x20))) + + // store array length into the front + mstore(data, returndatasize()) + + // append to data + returndatacopy(add(data, 0x20), 0, returndatasize()) + } + + data := () + + } + )") + ("functionName", functionName) + ("supportsReturndata", m_evmVersion.supportsReturndata()) + ("allocate", allocationFunction()) + ("roundUp", roundUpFunction()) + ("emptyArray", zeroValueFunction(*TypeProvider::bytesMemory())) + .render(); + }); +} + diff --git a/libsolidity/codegen/YulUtilFunctions.h b/libsolidity/codegen/YulUtilFunctions.h index b902f9f645fc..9550b9e89e32 100644 --- a/libsolidity/codegen/YulUtilFunctions.h +++ b/libsolidity/codegen/YulUtilFunctions.h @@ -22,6 +22,7 @@ #include +#include #include #include @@ -322,6 +323,20 @@ class YulUtilFunctions static std::string revertReasonIfDebug(RevertStrings revertStrings, std::string const& _message = ""); std::string revertReasonIfDebug(std::string const& _message = ""); + + /// Returns the name of a function that decodes an error message. + /// signature: () -> arrayPtr + /// + /// Returns a newly allocated `bytes memory` array containing the decoded error message + /// or 0 on failure. + std::string tryDecodeErrorMessageFunction(); + + + /// Returns a function name that returns a newly allocated `bytes` array that contains the return data. + /// + /// If returndatacopy() is not supported by the underlying target, a empty array will be returned instead. + std::string extractReturndataFunction(); + private: /// Special case of conversionFunction - handles everything that does not /// use exactly one variable to hold the value. diff --git a/libsolidity/codegen/ir/IRGenerationContext.cpp b/libsolidity/codegen/ir/IRGenerationContext.cpp index 26f6f030961a..45cb1ce14c18 100644 --- a/libsolidity/codegen/ir/IRGenerationContext.cpp +++ b/libsolidity/codegen/ir/IRGenerationContext.cpp @@ -87,6 +87,17 @@ string IRGenerationContext::newYulVariable() return "_" + to_string(++m_varCounter); } +string IRGenerationContext::trySuccessConditionVariable(Expression const& _expression) const +{ + // NB: The TypeChecker already ensured that the Expression is of type FunctionCall. + solAssert( + static_cast(_expression.annotation()).tryCall, + "Parameter must be a FunctionCall with tryCall-annotation set." + ); + + return "trySuccessCondition_" + to_string(_expression.id()); +} + string IRGenerationContext::internalDispatch(size_t _in, size_t _out) { string funName = "dispatch_internal_in_" + to_string(_in) + "_out_" + to_string(_out); @@ -141,3 +152,4 @@ std::string IRGenerationContext::revertReasonIfDebug(std::string const& _message { return YulUtilFunctions::revertReasonIfDebug(m_revertStrings, _message); } + diff --git a/libsolidity/codegen/ir/IRGenerationContext.h b/libsolidity/codegen/ir/IRGenerationContext.h index b0fc92cdb8c3..0dac6d65a5a4 100644 --- a/libsolidity/codegen/ir/IRGenerationContext.h +++ b/libsolidity/codegen/ir/IRGenerationContext.h @@ -99,6 +99,10 @@ class IRGenerationContext RevertStrings revertStrings() const { return m_revertStrings; } + /// @returns the variable name that can be used to inspect the success or failure of an external + /// function call that was invoked as part of the try statement. + std::string trySuccessConditionVariable(Expression const& _expression) const; + private: langutil::EVMVersion m_evmVersion; RevertStrings m_revertStrings; diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index 57c8cb152903..a6858d8541a4 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -41,6 +42,7 @@ #include #include +#include #include using namespace std; @@ -1260,39 +1262,15 @@ void IRGeneratorForStatements::appendExternalFunctionCall( _arguments.size() == funType.parameterTypes().size(), "" ); solUnimplementedAssert(!funType.bound(), ""); - FunctionType::Kind funKind = funType.kind(); + FunctionType::Kind const funKind = funType.kind(); solAssert(funKind != FunctionType::Kind::BareStaticCall || m_context.evmVersion().hasStaticCall(), ""); solAssert(funKind != FunctionType::Kind::BareCallCode, "Callcode has been removed."); - bool returnSuccessConditionAndReturndata = funKind == FunctionType::Kind::BareCall || funKind == FunctionType::Kind::BareDelegateCall || funKind == FunctionType::Kind::BareStaticCall; - bool isDelegateCall = funKind == FunctionType::Kind::BareDelegateCall || funKind == FunctionType::Kind::DelegateCall; - bool useStaticCall = funKind == FunctionType::Kind::BareStaticCall || (funType.stateMutability() <= StateMutability::View && m_context.evmVersion().hasStaticCall()); + bool const isDelegateCall = funKind == FunctionType::Kind::BareDelegateCall || funKind == FunctionType::Kind::DelegateCall; + bool const useStaticCall = funKind == FunctionType::Kind::BareStaticCall || (funType.stateMutability() <= StateMutability::View && m_context.evmVersion().hasStaticCall()); - bool haveReturndatacopy = m_context.evmVersion().supportsReturndata(); - unsigned estimatedReturnSize = 0; - bool dynamicReturnSize = false; - TypePointers returnTypes; - if (!returnSuccessConditionAndReturndata) - { - if (haveReturndatacopy) - returnTypes = funType.returnParameterTypes(); - else - returnTypes = funType.returnParameterTypesWithoutDynamicTypes(); - - for (auto const& retType: returnTypes) - if (retType->isDynamicallyEncoded()) - { - solAssert(haveReturndatacopy, ""); - dynamicReturnSize = true; - estimatedReturnSize = 0; - break; - } - else if (retType->decodingType()) - estimatedReturnSize += retType->decodingType()->calldataEncodedSize(); - else - estimatedReturnSize += retType->calldataEncodedSize(); - } + ReturnInfo const returnInfo{m_context.evmVersion(), funType}; TypePointers argumentTypes; vector argumentStrings; @@ -1311,8 +1289,8 @@ void IRGeneratorForStatements::appendExternalFunctionCall( // (which we would have to subtract from the gas left) // We could also just use MLOAD; POP right before the gas calculation, but the optimizer // would remove that, so we use MSTORE here. - if (!funType.gasSet() && estimatedReturnSize > 0) - m_code << "mstore(add(" << freeMemory() << ", " << to_string(estimatedReturnSize) << "), 0)\n"; + if (!funType.gasSet() && returnInfo.estimatedReturnSize > 0) + m_code << "mstore(add(" << freeMemory() << ", " << to_string(returnInfo.estimatedReturnSize) << "), 0)\n"; } ABIFunctions abi(m_context.evmVersion(), m_context.revertStrings(), m_context.functionCollector()); @@ -1323,30 +1301,61 @@ void IRGeneratorForStatements::appendExternalFunctionCall( if iszero(extcodesize(
)) { revert(0, 0) } + // storage for arguments and returned data let := - mstore(, ()) let := (add(, 4) ) - let := (,
, , , sub(, ), , ) - if iszero() { () } - - - returndatacopy(, 0, returndatasize()) - - - mstore(, add(, and(add(, 0x1f), not(0x1f)))) - let := (, add(, )) + let := (,
, , , sub(, ), , ) + + if iszero() { () } + + let + if { + + // copy dynamic return data out + returndatacopy(, 0, returndatasize()) + + + // update freeMemoryPointer according to dynamic return size + mstore(, add(, ())) + + // decode return parameters from external try-call into retVars + := (, add(, )) + } )"); templ("pos", m_context.newYulVariable()); templ("end", m_context.newYulVariable()); - templ("result", m_context.newYulVariable()); + if (_functionCall.annotation().tryCall) + templ("success", m_context.trySuccessConditionVariable(_functionCall)); + else + templ("success", m_context.newYulVariable()); templ("freeMemory", freeMemory()); - templ("freeMemoryPointer", to_string(CompilerUtils::freeMemoryPointer)); templ("shl28", m_utils.shiftLeftFunction(8 * (32 - 4))); templ("funId", IRVariable(_functionCall.expression()).part("functionIdentifier").name()); templ("address", IRVariable(_functionCall.expression()).part("address").name()); + // Always use the actual return length, and not our calculated expected length, if returndatacopy is supported. + // This ensures it can catch badly formatted input from external calls. + if (m_context.evmVersion().supportsReturndata()) + templ("returnSize", "returndatasize()"); + else + templ("returnSize", to_string(returnInfo.estimatedReturnSize)); + + templ("reservedReturnSize", returnInfo.dynamicReturnSize ? "0" : to_string(returnInfo.estimatedReturnSize)); + + string const retVars = IRVariable(_functionCall).commaSeparatedList(); + templ("retVars", retVars); + templ("hasRetVars", !retVars.empty()); + solAssert(retVars.empty() == returnInfo.returnTypes.empty(), ""); + + templ("roundUp", m_utils.roundUpFunction()); + templ("abiDecode", abi.tupleDecoder(returnInfo.returnTypes, true)); + templ("dynamicReturnSize", returnInfo.dynamicReturnSize); + templ("freeMemoryPointer", to_string(CompilerUtils::freeMemoryPointer)); + + templ("noTryCall", !_functionCall.annotation().tryCall); + // If the function takes arbitrary parameters or is a bare call, copy dynamic length data in place. // Move arguments to memory, will not update the free memory pointer (but will update the memory // pointer on the stack). @@ -1401,24 +1410,9 @@ void IRGeneratorForStatements::appendExternalFunctionCall( templ("forwardingRevert", m_utils.forwardingRevertFunction()); - solUnimplementedAssert(!returnSuccessConditionAndReturndata, ""); solUnimplementedAssert(funKind != FunctionType::Kind::RIPEMD160, ""); solUnimplementedAssert(funKind != FunctionType::Kind::ECRecover, ""); - templ("dynamicReturnSize", dynamicReturnSize); - // Always use the actual return length, and not our calculated expected length, if returndatacopy is supported. - // This ensures it can catch badly formatted input from external calls. - if (haveReturndatacopy) - templ("returnSize", "returndatasize()"); - else - templ("returnSize", to_string(estimatedReturnSize)); - - templ("reservedReturnSize", dynamicReturnSize ? "0" : to_string(estimatedReturnSize)); - - templ("abiDecode", abi.tupleDecoder(returnTypes, true)); - templ("returns", !returnTypes.empty()); - templ("retVars", IRVariable(_functionCall).commaSeparatedList()); - m_code << templ.render(); } @@ -1749,3 +1743,119 @@ Type const& IRGeneratorForStatements::type(Expression const& _expression) solAssert(_expression.annotation().type, "Type of expression not set."); return *_expression.annotation().type; } + +bool IRGeneratorForStatements::visit(TryStatement const& _tryStatement) +{ + Expression const& externalCall = _tryStatement.externalCall(); + externalCall.accept(*this); + + m_code << "switch iszero(" << m_context.trySuccessConditionVariable(externalCall) << ")\n"; + + m_code << "case 0 { // success case\n"; + TryCatchClause const& successClause = *_tryStatement.clauses().front(); + if (successClause.parameters()) + { + size_t i = 0; + for (ASTPointer const& varDecl: successClause.parameters()->parameters()) + { + solAssert(varDecl, ""); + define(m_context.addLocalVariable(*varDecl), + successClause.parameters()->parameters().size() == 1 ? + IRVariable(externalCall) : + IRVariable(externalCall).tupleComponent(i++) + ); + } + } + + successClause.block().accept(*this); + m_code << "}\n"; + + m_code << "default { // failure case\n"; + handleCatch(_tryStatement); + m_code << "}\n"; + + return false; +} + +void IRGeneratorForStatements::handleCatch(TryStatement const& _tryStatement) +{ + if (_tryStatement.structuredClause()) + handleCatchStructuredAndFallback(*_tryStatement.structuredClause(), _tryStatement.fallbackClause()); + else if (_tryStatement.fallbackClause()) + handleCatchFallback(*_tryStatement.fallbackClause()); + else + rethrow(); +} + +void IRGeneratorForStatements::handleCatchStructuredAndFallback( + TryCatchClause const& _structured, + TryCatchClause const* _fallback +) +{ + solAssert( + _structured.parameters() && + _structured.parameters()->parameters().size() == 1 && + _structured.parameters()->parameters().front() && + *_structured.parameters()->parameters().front()->annotation().type == *TypeProvider::stringMemory(), + "" + ); + solAssert(m_context.evmVersion().supportsReturndata(), ""); + + // Try to decode the error message. + // If this fails, leaves 0 on the stack, otherwise the pointer to the data string. + string const dataVariable = m_context.newYulVariable(); + + m_code << "let " << dataVariable << " := " << m_utils.tryDecodeErrorMessageFunction() << "()\n"; + m_code << "switch iszero(" << dataVariable << ") \n"; + m_code << "case 0 { // decoding success\n"; + if (_structured.parameters()) + { + solAssert(_structured.parameters()->parameters().size() == 1, ""); + IRVariable const& var = m_context.addLocalVariable(*_structured.parameters()->parameters().front()); + define(var) << dataVariable << "\n"; + } + _structured.accept(*this); + m_code << "}\n"; + m_code << "default { // decoding failure\n"; + if (_fallback) + handleCatchFallback(*_fallback); + else + rethrow(); + m_code << "}\n"; +} + +void IRGeneratorForStatements::handleCatchFallback(TryCatchClause const& _fallback) +{ + if (_fallback.parameters()) + { + solAssert(m_context.evmVersion().supportsReturndata(), ""); + solAssert( + _fallback.parameters()->parameters().size() == 1 && + _fallback.parameters()->parameters().front() && + *_fallback.parameters()->parameters().front()->annotation().type == *TypeProvider::bytesMemory(), + "" + ); + + VariableDeclaration const& paramDecl = *_fallback.parameters()->parameters().front(); + define(m_context.addLocalVariable(paramDecl)) << m_utils.extractReturndataFunction() << "()\n"; + } + _fallback.accept(*this); +} + +void IRGeneratorForStatements::rethrow() +{ + if (m_context.evmVersion().supportsReturndata()) + m_code << R"( + returndatacopy(0, 0, returndatasize()) + revert(0, returndatasize()) + )"s; + else + m_code << "revert(0, 0) // rethrow\n"s; +} + +bool IRGeneratorForStatements::visit(TryCatchClause const& _clause) +{ + _clause.block().accept(*this); + return false; +} + diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.h b/libsolidity/codegen/ir/IRGeneratorForStatements.h index 710961cd768b..553e81dcc1f2 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.h +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.h @@ -24,6 +24,8 @@ #include #include +#include + namespace solidity::frontend { @@ -70,7 +72,21 @@ class IRGeneratorForStatements: public ASTConstVisitor void endVisit(Identifier const& _identifier) override; bool visit(Literal const& _literal) override; + bool visit(TryStatement const& _tryStatement) override; + bool visit(TryCatchClause const& _tryCatchClause) override; + private: + /// Handles all catch cases of a try statement, except the success-case. + void handleCatch(TryStatement const& _tryStatement); + void handleCatchStructuredAndFallback( + TryCatchClause const& _structured, + TryCatchClause const* _fallback + ); + void handleCatchFallback(TryCatchClause const& _fallback); + + /// Generates code to rethrow an exception. + void rethrow(); + /// Appends code to call an external function with the given arguments. /// All involved expressions have already been visited. void appendExternalFunctionCall( diff --git a/test/libsolidity/semanticTests/tryCatch/lowLevel.sol b/test/libsolidity/semanticTests/tryCatch/lowLevel.sol index ac275522d9e5..c707cca59f76 100644 --- a/test/libsolidity/semanticTests/tryCatch/lowLevel.sol +++ b/test/libsolidity/semanticTests/tryCatch/lowLevel.sol @@ -12,6 +12,7 @@ contract C { } } // ==== +// compileViaYul: also // EVMVersion: >=byzantium // ---- // f(bool): true -> 1, 2, 96, 0 diff --git a/test/libsolidity/semanticTests/tryCatch/nested.sol b/test/libsolidity/semanticTests/tryCatch/nested.sol index 7c56a81b78d1..03e6ad55322d 100644 --- a/test/libsolidity/semanticTests/tryCatch/nested.sol +++ b/test/libsolidity/semanticTests/tryCatch/nested.sol @@ -26,6 +26,7 @@ contract C { } // ==== // EVMVersion: >=byzantium +// compileViaYul: also // ---- // f(bool,bool): true, true -> 1, 2, 96, 7, "success" // f(bool,bool): true, false -> 12, 0, 96, 7, "failure" diff --git a/test/libsolidity/semanticTests/tryCatch/return_function.sol b/test/libsolidity/semanticTests/tryCatch/return_function.sol index 82d5dc821218..b9c2d903fa01 100644 --- a/test/libsolidity/semanticTests/tryCatch/return_function.sol +++ b/test/libsolidity/semanticTests/tryCatch/return_function.sol @@ -13,5 +13,7 @@ contract C { } function fun() public pure {} } +// ==== +// compileViaYul: also // ---- // f() -> 0x1, 0xfdd67305928fcac8d213d1e47bfa6165cd0b87b946644cd0000000000000000, 9 diff --git a/test/libsolidity/semanticTests/tryCatch/simple.sol b/test/libsolidity/semanticTests/tryCatch/simple.sol index b7f36701695e..1f12614a73d0 100644 --- a/test/libsolidity/semanticTests/tryCatch/simple.sol +++ b/test/libsolidity/semanticTests/tryCatch/simple.sol @@ -1,10 +1,10 @@ contract C { - function g(bool b) public pure returns (uint, uint) { + function g(bool b) public pure returns (uint x, uint y) { require(b); return (1, 2); } - function f(bool b) public returns (uint x, uint y) { - try this.g(b) returns (uint a, uint b) { + function f(bool flag) public view returns (uint x, uint y) { + try this.g(flag) returns (uint a, uint b) { (x, y) = (a, b); } catch { (x, y) = (9, 10); @@ -13,6 +13,7 @@ contract C { } // ==== // EVMVersion: >=byzantium +// compileViaYul: also // ---- // f(bool): true -> 1, 2 // f(bool): false -> 9, 10 diff --git a/test/libsolidity/semanticTests/tryCatch/simple_notuple.sol b/test/libsolidity/semanticTests/tryCatch/simple_notuple.sol new file mode 100644 index 000000000000..7b58006f482e --- /dev/null +++ b/test/libsolidity/semanticTests/tryCatch/simple_notuple.sol @@ -0,0 +1,19 @@ +contract C { + function g(bool b) public pure returns (uint x) { + require(b); + return 13; + } + function f(bool flag) public view returns (uint x) { + try this.g(flag) returns (uint a) { + x = a; + } catch { + x = 9; + } + } +} +// ==== +// EVMVersion: >=byzantium +// compileViaYul: also +// ---- +// f(bool): true -> 13 +// f(bool): false -> 9 diff --git a/test/libsolidity/semanticTests/tryCatch/structured.sol b/test/libsolidity/semanticTests/tryCatch/structured.sol index e5aa238b30da..e4c04932aceb 100644 --- a/test/libsolidity/semanticTests/tryCatch/structured.sol +++ b/test/libsolidity/semanticTests/tryCatch/structured.sol @@ -14,6 +14,7 @@ contract C { } // ==== // EVMVersion: >=byzantium +// compileViaYul: also // ---- // f(bool): true -> 1, 2, 0x60, 7, "success" // f(bool): false -> 0, 0, 0x60, 7, "message" diff --git a/test/libsolidity/semanticTests/tryCatch/structuredAndLowLevel.sol b/test/libsolidity/semanticTests/tryCatch/structuredAndLowLevel.sol index 8a8cb3d1b62f..91542d7ba250 100644 --- a/test/libsolidity/semanticTests/tryCatch/structuredAndLowLevel.sol +++ b/test/libsolidity/semanticTests/tryCatch/structuredAndLowLevel.sol @@ -18,6 +18,7 @@ contract C { } // ==== // EVMVersion: >=byzantium +// compileViaYul: also // ---- // f(bool): true -> 1, 2, 96, 7, "success" // f(bool): false -> 99, 0, 96, 82, "message longer than 32 bytes 32 ", "bytes 32 bytes 32 bytes 32 bytes", " 32 bytes 32 bytes" diff --git a/test/libsolidity/semanticTests/tryCatch/super_trivial.sol b/test/libsolidity/semanticTests/tryCatch/super_trivial.sol new file mode 100644 index 000000000000..9c0340522485 --- /dev/null +++ b/test/libsolidity/semanticTests/tryCatch/super_trivial.sol @@ -0,0 +1,18 @@ +contract C { + function g(bool x) external pure { + require(x); + } + function f(bool x) public returns (uint) { + try this.g(x) { + return 1; + } catch { + return 2; + } + } +} +// ==== +// EVMVersion: >=byzantium +// compileViaYul: also +// ---- +// f(bool): true -> 1 +// f(bool): false -> 2 From c6c2d7174db08aed6c2ed595f903e83653d75880 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Wed, 15 Apr 2020 09:46:41 +0200 Subject: [PATCH 028/126] Use latest docker image for chk_proofs and t_ems_solcjs CI runs. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 23da90b8f35a..b1eb45af4b90 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -338,7 +338,7 @@ jobs: chk_proofs: docker: - - image: buildpack-deps:disco + - image: buildpack-deps:latest environment: TERM: xterm steps: @@ -661,7 +661,7 @@ jobs: t_ems_solcjs: docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1904 + - image: buildpack-deps:latest environment: TERM: xterm steps: From 0c5c93f3c2c2c3492e1718674232cd129e6fb4b6 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Wed, 15 Apr 2020 09:41:36 +0200 Subject: [PATCH 029/126] Fix tuple assignments for empty tuples in Yul IR. --- libsolidity/codegen/YulUtilFunctions.cpp | 5 +++-- .../codegen/ir/IRGeneratorForStatements.cpp | 13 ++++++++----- .../semanticTests/types/nested_tuples.sol | 12 ++++++++++++ .../semanticTests/types/nested_tuples_noyul.sol | 17 ----------------- 4 files changed, 23 insertions(+), 24 deletions(-) delete mode 100644 test/libsolidity/semanticTests/types/nested_tuples_noyul.sol diff --git a/libsolidity/codegen/YulUtilFunctions.cpp b/libsolidity/codegen/YulUtilFunctions.cpp index 2f14ae744a8e..e520c586fe9a 100644 --- a/libsolidity/codegen/YulUtilFunctions.cpp +++ b/libsolidity/codegen/YulUtilFunctions.cpp @@ -2079,7 +2079,7 @@ string YulUtilFunctions::conversionFunctionSpecial(Type const& _from, Type const { conversions += suffixedVariableNameList("converted", destStackSize, destStackSize + toComponent->sizeOnStack()) + - " := " + + (toComponent->sizeOnStack() > 0 ? " := " : "") + conversionFunction(*fromComponent, *toComponent) + "(" + suffixedVariableNameList("value", sourceStackSize, sourceStackSize + fromComponent->sizeOnStack()) + @@ -2089,12 +2089,13 @@ string YulUtilFunctions::conversionFunctionSpecial(Type const& _from, Type const sourceStackSize += fromComponent->sizeOnStack(); } return Whiskers(R"( - function () -> { + function () { } )") ("functionName", functionName) ("values", suffixedVariableNameList("value", 0, sourceStackSize)) + ("arrow", destStackSize > 0 ? "->" : "") ("converted", suffixedVariableNameList("converted", 0, destStackSize)) ("conversions", conversions) .render(); diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index a6858d8541a4..5046925fde33 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -1465,14 +1465,17 @@ void IRGeneratorForStatements::declareAssign(IRVariable const& _lhs, IRVariable else m_code << (_declare ? "let ": "") << _lhs.part(stackItemName).name() << " := " << _rhs.part(stackItemName).name() << "\n"; else - m_code << - (_declare ? "let ": "") << - _lhs.commaSeparatedList() << - " := " << - m_context.utils().conversionFunction(_rhs.type(), _lhs.type()) << + { + if (_lhs.type().sizeOnStack() > 0) + m_code << + (_declare ? "let ": "") << + _lhs.commaSeparatedList() << + " := "; + m_code << m_context.utils().conversionFunction(_rhs.type(), _lhs.type()) << "(" << _rhs.commaSeparatedList() << ")\n"; + } } IRVariable IRGeneratorForStatements::zeroValue(Type const& _type, bool _splitFunctionTypes) diff --git a/test/libsolidity/semanticTests/types/nested_tuples.sol b/test/libsolidity/semanticTests/types/nested_tuples.sol index d9737a807832..b94d4437071e 100644 --- a/test/libsolidity/semanticTests/types/nested_tuples.sol +++ b/test/libsolidity/semanticTests/types/nested_tuples.sol @@ -15,6 +15,16 @@ contract test { (((, a),)) = ((1, 2), 3); return a; } + function f3() public returns(int) { + int a = 3; + ((, ), ) = ((7, 8), 9); + return a; + } + function f4() public returns(int) { + int a; + (a, ) = (4, (8, 16, 32)); + return a; + } } // ==== // compileViaYul: also @@ -22,3 +32,5 @@ contract test { // f0() -> 2, true // f1() -> 1 // f2() -> 2 +// f3() -> 3 +// f4() -> 4 diff --git a/test/libsolidity/semanticTests/types/nested_tuples_noyul.sol b/test/libsolidity/semanticTests/types/nested_tuples_noyul.sol deleted file mode 100644 index 1eff9e2b9ac3..000000000000 --- a/test/libsolidity/semanticTests/types/nested_tuples_noyul.sol +++ /dev/null @@ -1,17 +0,0 @@ -contract test { - function f3() public returns(int) { - int a = 3; - ((, ), ) = ((7, 8), 9); - return a; - } - function f4() public returns(int) { - int a; - (a, ) = (4, (8, 16, 32)); - return a; - } -} -// ==== -// compileViaYul: false -// ---- -// f3() -> 3 -// f4() -> 4 From b86c9275057813722c26954466d3813980e8fc25 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Wed, 15 Apr 2020 13:58:37 +0200 Subject: [PATCH 030/126] Disallow virtual and override for constructors. --- Changelog.md | 1 + libsolidity/analysis/TypeChecker.cpp | 4 ++++ .../syntaxTests/constructor/constructor_override.sol | 5 +++++ .../syntaxTests/constructor/constructor_virtual.sol | 5 +++++ 4 files changed, 15 insertions(+) create mode 100644 test/libsolidity/syntaxTests/constructor/constructor_override.sol create mode 100644 test/libsolidity/syntaxTests/constructor/constructor_virtual.sol diff --git a/Changelog.md b/Changelog.md index eb428c1edec0..cdc3ac010701 100644 --- a/Changelog.md +++ b/Changelog.md @@ -7,6 +7,7 @@ Compiler Features: Bugfixes: + * Type Checker: Disallow ``virtual`` and ``override`` for constructors. diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index 76e805c27d67..30842a11725e 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -1796,6 +1796,10 @@ void TypeChecker::typeCheckReceiveFunction(FunctionDefinition const& _function) void TypeChecker::typeCheckConstructor(FunctionDefinition const& _function) { solAssert(_function.isConstructor(), ""); + if (_function.markedVirtual()) + m_errorReporter.typeError(_function.location(), "Constructors cannot be virtual."); + if (_function.overrides()) + m_errorReporter.typeError(_function.location(), "Constructors cannot override."); if (!_function.returnParameters().empty()) m_errorReporter.typeError(_function.returnParameterList()->location(), "Non-empty \"returns\" directive for constructor."); if (_function.stateMutability() != StateMutability::NonPayable && _function.stateMutability() != StateMutability::Payable) diff --git a/test/libsolidity/syntaxTests/constructor/constructor_override.sol b/test/libsolidity/syntaxTests/constructor/constructor_override.sol new file mode 100644 index 000000000000..48203a27d31c --- /dev/null +++ b/test/libsolidity/syntaxTests/constructor/constructor_override.sol @@ -0,0 +1,5 @@ +contract C { + constructor() override public {} +} +// ---- +// TypeError: (17-49): Constructors cannot override. diff --git a/test/libsolidity/syntaxTests/constructor/constructor_virtual.sol b/test/libsolidity/syntaxTests/constructor/constructor_virtual.sol new file mode 100644 index 000000000000..cd692dbcdc56 --- /dev/null +++ b/test/libsolidity/syntaxTests/constructor/constructor_virtual.sol @@ -0,0 +1,5 @@ +contract C { + constructor() virtual public {} +} +// ---- +// TypeError: (17-48): Constructors cannot be virtual. From bca43586c6e00d9ac2b01fe6204d4efe0eda3a17 Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Tue, 14 Apr 2020 23:25:56 +0200 Subject: [PATCH 031/126] [SMTChecker] Remove redundant CHC constraints --- libsolidity/formal/CHC.cpp | 27 ++++++++----------- libsolidity/formal/CHC.h | 1 + libsolidity/formal/SMTEncoder.cpp | 4 ++- .../types/function_type_nested.sol | 1 - .../types/function_type_nested_return.sol | 2 -- .../types/tuple_return_branch.sol | 2 -- 6 files changed, 15 insertions(+), 22 deletions(-) diff --git a/libsolidity/formal/CHC.cpp b/libsolidity/formal/CHC.cpp index 5067e1396156..5599ac4d9ec0 100644 --- a/libsolidity/formal/CHC.cpp +++ b/libsolidity/formal/CHC.cpp @@ -130,15 +130,10 @@ bool CHC::visit(ContractDefinition const& _contract) clearIndices(&_contract); - auto errorFunctionSort = make_shared( - vector(), - smt::SortProvider::boolSort - ); - string suffix = _contract.name() + "_" + to_string(_contract.id()); - m_errorPredicate = createSymbolicBlock(errorFunctionSort, "error_" + suffix); + m_errorPredicate = createSymbolicBlock(arity0FunctionSort(), "error_" + suffix); m_constructorSummaryPredicate = createSymbolicBlock(constructorSort(), "summary_constructor_" + suffix); - m_implicitConstructorPredicate = createSymbolicBlock(interfaceSort(), "implicit_constructor_" + suffix); + m_implicitConstructorPredicate = createSymbolicBlock(arity0FunctionSort(), "implicit_constructor_" + suffix); auto stateExprs = currentStateVariables(); setCurrentBlock(*m_interfaces.at(m_currentContract), &stateExprs); @@ -148,15 +143,7 @@ bool CHC::visit(ContractDefinition const& _contract) void CHC::endVisit(ContractDefinition const& _contract) { - for (auto const& var: m_stateVariables) - { - solAssert(m_context.knownVariable(*var), ""); - auto const& symbVar = m_context.variable(*var); - symbVar->resetIndex(); - m_context.setZeroValue(*var); - symbVar->increaseIndex(); - } - auto implicitConstructor = (*m_implicitConstructorPredicate)(initialStateVariables()); + auto implicitConstructor = (*m_implicitConstructorPredicate)({}); connectBlocks(genesis(), implicitConstructor); m_currentBlock = implicitConstructor; m_context.addAssertion(m_error.currentValue() == 0); @@ -682,6 +669,14 @@ smt::SortPointer CHC::interfaceSort(ContractDefinition const& _contract) ); } +smt::SortPointer CHC::arity0FunctionSort() +{ + return make_shared( + vector(), + smt::SortProvider::boolSort + ); +} + /// A function in the symbolic CFG requires: /// - Index of failed assertion. 0 means no assertion failed. /// - 2 sets of state variables: diff --git a/libsolidity/formal/CHC.h b/libsolidity/formal/CHC.h index 4f601a2d51b9..7fc752451c9d 100644 --- a/libsolidity/formal/CHC.h +++ b/libsolidity/formal/CHC.h @@ -106,6 +106,7 @@ class CHC: public SMTEncoder smt::SortPointer constructorSort(); smt::SortPointer interfaceSort(); static smt::SortPointer interfaceSort(ContractDefinition const& _const); + smt::SortPointer arity0FunctionSort(); smt::SortPointer sort(FunctionDefinition const& _function); smt::SortPointer sort(ASTNode const* _block); /// @returns the sort of a predicate that represents the summary of _function in the scope of _contract. diff --git a/libsolidity/formal/SMTEncoder.cpp b/libsolidity/formal/SMTEncoder.cpp index 67a4885072a6..b12712db6905 100644 --- a/libsolidity/formal/SMTEncoder.cpp +++ b/libsolidity/formal/SMTEncoder.cpp @@ -107,6 +107,9 @@ void SMTEncoder::endVisit(ContractDefinition const& _contract) solAssert(m_currentContract == &_contract, ""); m_currentContract = nullptr; + + if (m_callStack.empty()) + m_context.popSolver(); } void SMTEncoder::endVisit(VariableDeclaration const& _varDecl) @@ -662,7 +665,6 @@ void SMTEncoder::initFunction(FunctionDefinition const& _function) { solAssert(m_callStack.empty(), ""); solAssert(m_currentContract, ""); - m_context.reset(); m_context.pushSolver(); m_pathConditions.clear(); pushCallStack({&_function, nullptr}); diff --git a/test/libsolidity/smtCheckerTests/types/function_type_nested.sol b/test/libsolidity/smtCheckerTests/types/function_type_nested.sol index 5eb6373159c9..90d101f4dbc5 100644 --- a/test/libsolidity/smtCheckerTests/types/function_type_nested.sol +++ b/test/libsolidity/smtCheckerTests/types/function_type_nested.sol @@ -18,5 +18,4 @@ contract C { // Warning: (212-219): Assertion checker does not yet implement this type of function call. // Warning: (255-257): Internal error: Expression undefined for SMT solver. // Warning: (255-257): Assertion checker does not yet implement type function (function (uint256)) -// Warning: (212-214): Assertion checker does not yet implement type function (function (uint256)) // Warning: (212-219): Assertion checker does not yet implement this type of function call. diff --git a/test/libsolidity/smtCheckerTests/types/function_type_nested_return.sol b/test/libsolidity/smtCheckerTests/types/function_type_nested_return.sol index 2e974fef8665..d9a10370b51c 100644 --- a/test/libsolidity/smtCheckerTests/types/function_type_nested_return.sol +++ b/test/libsolidity/smtCheckerTests/types/function_type_nested_return.sol @@ -22,6 +22,4 @@ contract C { // Warning: (284-291): Assertion checker does not yet implement this type of function call. // Warning: (327-329): Internal error: Expression undefined for SMT solver. // Warning: (327-329): Assertion checker does not yet implement type function (function (uint256)) -// Warning: (284-286): Assertion checker does not yet implement type function (function (uint256)) -// Warning: (287-288): Assertion checker does not yet support this global variable. // Warning: (284-291): Assertion checker does not yet implement this type of function call. diff --git a/test/libsolidity/smtCheckerTests/types/tuple_return_branch.sol b/test/libsolidity/smtCheckerTests/types/tuple_return_branch.sol index 1a266aaa7dd5..b6ee9fa3a83d 100644 --- a/test/libsolidity/smtCheckerTests/types/tuple_return_branch.sol +++ b/test/libsolidity/smtCheckerTests/types/tuple_return_branch.sol @@ -19,7 +19,5 @@ contract C { // Warning: (137-141): Assertion checker does not yet implement type struct C.S memory // Warning: (137-141): Assertion checker does not yet implement this expression. // Warning: (193-203): Assertion checker does not yet support the type of this variable. -// Warning: (137-138): Assertion checker does not yet implement type type(struct C.S storage pointer) -// Warning: (137-141): Assertion checker does not yet implement type struct C.S memory // Warning: (137-141): Assertion checker does not yet implement this expression. // Warning: (227-228): Assertion checker does not yet implement type struct C.S memory From 06adbc2c5954900929d277cf52671004726b3cc7 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 16 Apr 2020 00:33:32 +0200 Subject: [PATCH 032/126] Implement bit operations. --- .../codegen/ir/IRGeneratorForStatements.cpp | 9 +++++++++ .../expressions/bit_operators.sol | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 test/libsolidity/semanticTests/expressions/bit_operators.sol diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index 5046925fde33..2a9c6a9ea888 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -1537,6 +1537,15 @@ string IRGeneratorForStatements::binaryOperation( case Token::Mod: fun = m_utils.checkedIntModFunction(*type); break; + case Token::BitOr: + fun = "or"; + break; + case Token::BitXor: + fun = "xor"; + break; + case Token::BitAnd: + fun = "and"; + break; default: break; } diff --git a/test/libsolidity/semanticTests/expressions/bit_operators.sol b/test/libsolidity/semanticTests/expressions/bit_operators.sol new file mode 100644 index 000000000000..32ec16bf6bc7 --- /dev/null +++ b/test/libsolidity/semanticTests/expressions/bit_operators.sol @@ -0,0 +1,19 @@ +contract test { + uint8 x; + uint v; + function f() public returns (uint x, uint y, uint z) { + uint16 a; + uint32 b; + assembly { + a := 0x0f0f0f0f0f + b := 0xff0fff0fff + } + x = a & b; + y = a | b; + z = a ^ b; + } +} +// ==== +// compileViaYul: also +// ---- +// f() -> 3855, 268374015, 268370160 From f271dfa2e26ba5bf3656e1b98110a3ac074750f8 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 16 Apr 2020 13:42:58 +0200 Subject: [PATCH 033/126] Yul codegen for `.selector`. --- .../codegen/ir/IRGeneratorForStatements.cpp | 25 ++++++++++++++++--- .../builtinFunctions/function_types_sig.sol | 2 ++ .../function_selector_via_contract_name.sol | 2 ++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index 2a9c6a9ea888..0100056cddfd 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -842,11 +842,18 @@ void IRGeneratorForStatements::endVisit(MemberAccess const& _memberAccess) case Type::Category::Function: if (member == "selector") { - solUnimplementedAssert( - dynamic_cast(*_memberAccess.expression().annotation().type).kind() == - FunctionType::Kind::External, "" + FunctionType const& functionType = dynamic_cast( + *_memberAccess.expression().annotation().type ); - define(IRVariable{_memberAccess}, IRVariable(_memberAccess.expression()).part("functionIdentifier")); + if (functionType.kind() == FunctionType::Kind::External) + define(IRVariable{_memberAccess}, IRVariable(_memberAccess.expression()).part("functionIdentifier")); + else if (functionType.kind() == FunctionType::Kind::Declaration) + { + solAssert(functionType.hasDeclaration(), ""); + define(IRVariable{_memberAccess}) << formatNumber(functionType.externalIdentifier() << 224) << "\n"; + } + else + solAssert(false, "Invalid use of .selector"); } else if (member == "address") { @@ -973,6 +980,16 @@ void IRGeneratorForStatements::endVisit(MemberAccess const& _memberAccess) solAssert(false, "Illegal fixed bytes member."); break; } + case Type::Category::TypeType: + { + TypeType const& type = dynamic_cast(*_memberAccess.expression().annotation().type); + solUnimplementedAssert(type.actualType()->category() == Type::Category::Contract, ""); + FunctionType const* funType = dynamic_cast(_memberAccess.annotation().type); + solUnimplementedAssert(funType, ""); + solUnimplementedAssert(funType->kind() == FunctionType::Kind::Declaration, ""); + // Nothing to do for declaration. + break; + } default: solAssert(false, "Member access to unknown type."); } diff --git a/test/libsolidity/semanticTests/builtinFunctions/function_types_sig.sol b/test/libsolidity/semanticTests/builtinFunctions/function_types_sig.sol index 2e771032bc7e..e753cbbe99e7 100644 --- a/test/libsolidity/semanticTests/builtinFunctions/function_types_sig.sol +++ b/test/libsolidity/semanticTests/builtinFunctions/function_types_sig.sol @@ -19,6 +19,8 @@ contract C { return this.x.selector; } } +// ==== +// compileViaYul: also // ---- // f() -> 0x26121ff000000000000000000000000000000000000000000000000000000000 // g() -> 0x26121ff000000000000000000000000000000000000000000000000000000000 diff --git a/test/libsolidity/semanticTests/functionSelector/function_selector_via_contract_name.sol b/test/libsolidity/semanticTests/functionSelector/function_selector_via_contract_name.sol index 0a9e6d3ffd14..b6a666d5b9a2 100644 --- a/test/libsolidity/semanticTests/functionSelector/function_selector_via_contract_name.sol +++ b/test/libsolidity/semanticTests/functionSelector/function_selector_via_contract_name.sol @@ -15,6 +15,8 @@ contract C { return (a.f.selector, a.g.selector, b.f.selector, b.g.selector); } } +// ==== +// compileViaYul: also // ---- // test1() -> left(0x26121ff0), left(0xe420264a), left(0x26121ff0), left(0xe420264a) // test2() -> left(0x26121ff0), left(0xe420264a), left(0x26121ff0), left(0xe420264a) From 1b4e06605dd38cefdbfc19e0643c6570674cd731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sun, 1 Mar 2020 00:25:00 +0100 Subject: [PATCH 034/126] OptimiserSuite: Extract the code for repeating a sequence into runSequenceUntilStable() - Define a constant for the maximum number of repeats of the optimisation string --- libyul/optimiser/Suite.cpp | 239 ++++++++++++++++--------------------- libyul/optimiser/Suite.h | 7 ++ 2 files changed, 113 insertions(+), 133 deletions(-) diff --git a/libyul/optimiser/Suite.cpp b/libyul/optimiser/Suite.cpp index a2fec4cc4979..bc35d096d366 100644 --- a/libyul/optimiser/Suite.cpp +++ b/libyul/optimiser/Suite.cpp @@ -113,150 +113,105 @@ void OptimiserSuite::run( // None of the above can make stack problems worse. - size_t codeSize = 0; - for (size_t rounds = 0; rounds < 12; ++rounds) - { - { - size_t newSize = CodeSize::codeSizeIncludingFunctions(ast); - if (newSize == codeSize) - break; - codeSize = newSize; - } + suite.runSequenceUntilStable({ + // Turn into SSA and simplify + ExpressionSplitter::name, + SSATransform::name, + RedundantAssignEliminator::name, + RedundantAssignEliminator::name, + ExpressionSimplifier::name, + CommonSubexpressionEliminator::name, + LoadResolver::name, + LoopInvariantCodeMotion::name, - { - // Turn into SSA and simplify - suite.runSequence({ - ExpressionSplitter::name, - SSATransform::name, - RedundantAssignEliminator::name, - RedundantAssignEliminator::name, - ExpressionSimplifier::name, - CommonSubexpressionEliminator::name, - LoadResolver::name, - LoopInvariantCodeMotion::name - }, ast); - } + // perform structural simplification + CommonSubexpressionEliminator::name, + ConditionalSimplifier::name, + LiteralRematerialiser::name, + ConditionalUnsimplifier::name, + StructuralSimplifier::name, + LiteralRematerialiser::name, + ForLoopConditionOutOfBody::name, + ControlFlowSimplifier::name, + StructuralSimplifier::name, + ControlFlowSimplifier::name, + BlockFlattener::name, + DeadCodeEliminator::name, + ForLoopConditionIntoBody::name, + UnusedPruner::name, + CircularReferencesPruner::name, - { - // perform structural simplification - suite.runSequence({ - CommonSubexpressionEliminator::name, - ConditionalSimplifier::name, - LiteralRematerialiser::name, - ConditionalUnsimplifier::name, - StructuralSimplifier::name, - LiteralRematerialiser::name, - ForLoopConditionOutOfBody::name, - ControlFlowSimplifier::name, - StructuralSimplifier::name, - ControlFlowSimplifier::name, - BlockFlattener::name, - DeadCodeEliminator::name, - ForLoopConditionIntoBody::name, - UnusedPruner::name, - CircularReferencesPruner::name - }, ast); - } + // simplify again + LoadResolver::name, + CommonSubexpressionEliminator::name, + UnusedPruner::name, + CircularReferencesPruner::name, - { - // simplify again - suite.runSequence({ - LoadResolver::name, - CommonSubexpressionEliminator::name, - UnusedPruner::name, - CircularReferencesPruner::name, - }, ast); - } + // reverse SSA + SSAReverser::name, + CommonSubexpressionEliminator::name, + UnusedPruner::name, + CircularReferencesPruner::name, - { - // reverse SSA - suite.runSequence({ - SSAReverser::name, - CommonSubexpressionEliminator::name, - UnusedPruner::name, - CircularReferencesPruner::name, - - ExpressionJoiner::name, - ExpressionJoiner::name, - }, ast); - } + ExpressionJoiner::name, + ExpressionJoiner::name, // should have good "compilability" property here. - { - // run functional expression inliner - suite.runSequence({ - ExpressionInliner::name, - UnusedPruner::name, - CircularReferencesPruner::name, - }, ast); - } + // run functional expression inliner + ExpressionInliner::name, + UnusedPruner::name, + CircularReferencesPruner::name, - { - // Prune a bit more in SSA - suite.runSequence({ - ExpressionSplitter::name, - SSATransform::name, - RedundantAssignEliminator::name, - UnusedPruner::name, - CircularReferencesPruner::name, - RedundantAssignEliminator::name, - UnusedPruner::name, - CircularReferencesPruner::name, - }, ast); - } + // Prune a bit more in SSA + ExpressionSplitter::name, + SSATransform::name, + RedundantAssignEliminator::name, + UnusedPruner::name, + CircularReferencesPruner::name, + RedundantAssignEliminator::name, + UnusedPruner::name, + CircularReferencesPruner::name, - { - // Turn into SSA again and simplify - suite.runSequence({ - ExpressionSplitter::name, - SSATransform::name, - RedundantAssignEliminator::name, - RedundantAssignEliminator::name, - CommonSubexpressionEliminator::name, - LoadResolver::name, - }, ast); - } + // Turn into SSA again and simplify + ExpressionSplitter::name, + SSATransform::name, + RedundantAssignEliminator::name, + RedundantAssignEliminator::name, + CommonSubexpressionEliminator::name, + LoadResolver::name, - { - // run full inliner - suite.runSequence({ - FunctionGrouper::name, - EquivalentFunctionCombiner::name, - FullInliner::name, - BlockFlattener::name - }, ast); - } + // run full inliner + FunctionGrouper::name, + EquivalentFunctionCombiner::name, + FullInliner::name, + BlockFlattener::name, - { - // SSA plus simplify - suite.runSequence({ - ConditionalSimplifier::name, - LiteralRematerialiser::name, - ConditionalUnsimplifier::name, - CommonSubexpressionEliminator::name, - SSATransform::name, - RedundantAssignEliminator::name, - RedundantAssignEliminator::name, - LoadResolver::name, - ExpressionSimplifier::name, - LiteralRematerialiser::name, - ForLoopConditionOutOfBody::name, - StructuralSimplifier::name, - BlockFlattener::name, - DeadCodeEliminator::name, - ControlFlowSimplifier::name, - CommonSubexpressionEliminator::name, - SSATransform::name, - RedundantAssignEliminator::name, - RedundantAssignEliminator::name, - ForLoopConditionIntoBody::name, - UnusedPruner::name, - CircularReferencesPruner::name, - CommonSubexpressionEliminator::name, - }, ast); - } - } + // SSA plus simplify + ConditionalSimplifier::name, + LiteralRematerialiser::name, + ConditionalUnsimplifier::name, + CommonSubexpressionEliminator::name, + SSATransform::name, + RedundantAssignEliminator::name, + RedundantAssignEliminator::name, + LoadResolver::name, + ExpressionSimplifier::name, + LiteralRematerialiser::name, + ForLoopConditionOutOfBody::name, + StructuralSimplifier::name, + BlockFlattener::name, + DeadCodeEliminator::name, + ControlFlowSimplifier::name, + CommonSubexpressionEliminator::name, + SSATransform::name, + RedundantAssignEliminator::name, + RedundantAssignEliminator::name, + ForLoopConditionIntoBody::name, + UnusedPruner::name, + CircularReferencesPruner::name, + CommonSubexpressionEliminator::name, + }, ast); // Make source short and pretty. @@ -453,3 +408,21 @@ void OptimiserSuite::runSequence(std::vector const& _steps, Block& _ast) } } } + +void OptimiserSuite::runSequenceUntilStable( + std::vector const& _steps, + Block& _ast, + size_t maxRounds +) +{ + size_t codeSize = 0; + for (size_t rounds = 0; rounds < maxRounds; ++rounds) + { + size_t newSize = CodeSize::codeSizeIncludingFunctions(_ast); + if (newSize == codeSize) + break; + codeSize = newSize; + + runSequence(_steps, _ast); + } +} diff --git a/libyul/optimiser/Suite.h b/libyul/optimiser/Suite.h index 6ce6b485c71d..66f67dd825be 100644 --- a/libyul/optimiser/Suite.h +++ b/libyul/optimiser/Suite.h @@ -45,6 +45,8 @@ struct Object; class OptimiserSuite { public: + static constexpr size_t MaxRounds = 12; + enum class Debug { None, @@ -60,6 +62,11 @@ class OptimiserSuite ); void runSequence(std::vector const& _steps, Block& _ast); + void runSequenceUntilStable( + std::vector const& _steps, + Block& _ast, + size_t maxRounds = MaxRounds + ); static std::map> const& allSteps(); static std::map const& stepNameToAbbreviationMap(); From 9d7df5db6983284872193237059d689f24194aec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sun, 1 Mar 2020 00:28:12 +0100 Subject: [PATCH 035/126] OptimiserSuite: Add a variant of runSequence() that works with a string of abbreviations --- libyul/optimiser/Suite.cpp | 59 ++++++++++++++++++++++++++++++++++++-- libyul/optimiser/Suite.h | 1 + 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/libyul/optimiser/Suite.cpp b/libyul/optimiser/Suite.cpp index bc35d096d366..d41320dd470d 100644 --- a/libyul/optimiser/Suite.cpp +++ b/libyul/optimiser/Suite.cpp @@ -67,6 +67,8 @@ #include +#include + using namespace std; using namespace solidity; using namespace solidity::yul; @@ -243,8 +245,8 @@ void OptimiserSuite::run( // This is a tuning parameter, but actually just prevents infinite loops. size_t stackCompressorMaxIterations = 16; - suite.runSequence({ - FunctionGrouper::name + suite.runSequence(vector{ + FunctionGrouper::name, }, ast); // We ignore the return value because we will get a much better error // message once we perform code generation. @@ -384,6 +386,59 @@ map const& OptimiserSuite::stepAbbreviationToNameMap() return lookupTable; } +void OptimiserSuite::runSequence(string const& _stepAbbreviations, Block& _ast) +{ + string input = _stepAbbreviations; + boost::remove_erase(input, ' '); + boost::remove_erase(input, '\n'); + + bool insideLoop = false; + for (char abbreviation: input) + switch (abbreviation) + { + case '(': + assertThrow(!insideLoop, OptimizerException, "Nested parentheses not supported"); + insideLoop = true; + break; + case ')': + assertThrow(insideLoop, OptimizerException, "Unbalanced parenthesis"); + insideLoop = false; + break; + default: + assertThrow( + stepAbbreviationToNameMap().find(abbreviation) != stepAbbreviationToNameMap().end(), + OptimizerException, + "Invalid optimisation step abbreviation" + ); + } + assertThrow(!insideLoop, OptimizerException, "Unbalanced parenthesis"); + + auto abbreviationsToSteps = [](string const& _sequence) -> vector + { + vector steps; + for (char abbreviation: _sequence) + steps.emplace_back(stepAbbreviationToNameMap().at(abbreviation)); + return steps; + }; + + // The sequence has now been validated and must consist of pairs of segments that look like this: `aaa(bbb)` + // `aaa` or `(bbb)` can be empty. For example we consider a sequence like `fgo(aaf)Oo` to have + // four segments, the last of which is an empty parenthesis. + size_t currentPairStart = 0; + while (currentPairStart < input.size()) + { + size_t openingParenthesis = input.find('(', currentPairStart); + size_t closingParenthesis = input.find(')', openingParenthesis); + size_t firstCharInside = (openingParenthesis == string::npos ? input.size() : openingParenthesis + 1); + yulAssert((openingParenthesis == string::npos) == (closingParenthesis == string::npos), ""); + + runSequence(abbreviationsToSteps(input.substr(currentPairStart, openingParenthesis - currentPairStart)), _ast); + runSequenceUntilStable(abbreviationsToSteps(input.substr(firstCharInside, closingParenthesis - firstCharInside)), _ast); + + currentPairStart = (closingParenthesis == string::npos ? input.size() : closingParenthesis + 1); + } +} + void OptimiserSuite::runSequence(std::vector const& _steps, Block& _ast) { unique_ptr copy; diff --git a/libyul/optimiser/Suite.h b/libyul/optimiser/Suite.h index 66f67dd825be..3b0ed0163a5b 100644 --- a/libyul/optimiser/Suite.h +++ b/libyul/optimiser/Suite.h @@ -62,6 +62,7 @@ class OptimiserSuite ); void runSequence(std::vector const& _steps, Block& _ast); + void runSequence(std::string const& _stepAbbreviations, Block& _ast); void runSequenceUntilStable( std::vector const& _steps, Block& _ast, From 9db7d2bde9ee27afe2e7e8121c960f211ee106a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sun, 1 Mar 2020 01:02:45 +0100 Subject: [PATCH 036/126] OptimiserSuite: Replace full step names in run() with abbreviations --- libyul/optimiser/Suite.cpp | 184 +++++-------------------------------- 1 file changed, 22 insertions(+), 162 deletions(-) diff --git a/libyul/optimiser/Suite.cpp b/libyul/optimiser/Suite.cpp index d41320dd470d..d1c6a6dd4cc4 100644 --- a/libyul/optimiser/Suite.cpp +++ b/libyul/optimiser/Suite.cpp @@ -93,161 +93,30 @@ void OptimiserSuite::run( OptimiserSuite suite(_dialect, reservedIdentifiers, Debug::None, ast); - suite.runSequence({ - VarDeclInitializer::name, - FunctionHoister::name, - BlockFlattener::name, - ForLoopInitRewriter::name, - DeadCodeEliminator::name, - FunctionGrouper::name, - EquivalentFunctionCombiner::name, - UnusedPruner::name, - CircularReferencesPruner::name, - BlockFlattener::name, - ControlFlowSimplifier::name, - LiteralRematerialiser::name, - ConditionalUnsimplifier::name, - StructuralSimplifier::name, - ControlFlowSimplifier::name, - ForLoopConditionIntoBody::name, - BlockFlattener::name - }, ast); - - // None of the above can make stack problems worse. - - suite.runSequenceUntilStable({ - // Turn into SSA and simplify - ExpressionSplitter::name, - SSATransform::name, - RedundantAssignEliminator::name, - RedundantAssignEliminator::name, - ExpressionSimplifier::name, - CommonSubexpressionEliminator::name, - LoadResolver::name, - LoopInvariantCodeMotion::name, - - // perform structural simplification - CommonSubexpressionEliminator::name, - ConditionalSimplifier::name, - LiteralRematerialiser::name, - ConditionalUnsimplifier::name, - StructuralSimplifier::name, - LiteralRematerialiser::name, - ForLoopConditionOutOfBody::name, - ControlFlowSimplifier::name, - StructuralSimplifier::name, - ControlFlowSimplifier::name, - BlockFlattener::name, - DeadCodeEliminator::name, - ForLoopConditionIntoBody::name, - UnusedPruner::name, - CircularReferencesPruner::name, - - // simplify again - LoadResolver::name, - CommonSubexpressionEliminator::name, - UnusedPruner::name, - CircularReferencesPruner::name, - - // reverse SSA - SSAReverser::name, - CommonSubexpressionEliminator::name, - UnusedPruner::name, - CircularReferencesPruner::name, - - ExpressionJoiner::name, - ExpressionJoiner::name, - - // should have good "compilability" property here. - - // run functional expression inliner - ExpressionInliner::name, - UnusedPruner::name, - CircularReferencesPruner::name, - - // Prune a bit more in SSA - ExpressionSplitter::name, - SSATransform::name, - RedundantAssignEliminator::name, - UnusedPruner::name, - CircularReferencesPruner::name, - RedundantAssignEliminator::name, - UnusedPruner::name, - CircularReferencesPruner::name, - - // Turn into SSA again and simplify - ExpressionSplitter::name, - SSATransform::name, - RedundantAssignEliminator::name, - RedundantAssignEliminator::name, - CommonSubexpressionEliminator::name, - LoadResolver::name, - - // run full inliner - FunctionGrouper::name, - EquivalentFunctionCombiner::name, - FullInliner::name, - BlockFlattener::name, - - // SSA plus simplify - ConditionalSimplifier::name, - LiteralRematerialiser::name, - ConditionalUnsimplifier::name, - CommonSubexpressionEliminator::name, - SSATransform::name, - RedundantAssignEliminator::name, - RedundantAssignEliminator::name, - LoadResolver::name, - ExpressionSimplifier::name, - LiteralRematerialiser::name, - ForLoopConditionOutOfBody::name, - StructuralSimplifier::name, - BlockFlattener::name, - DeadCodeEliminator::name, - ControlFlowSimplifier::name, - CommonSubexpressionEliminator::name, - SSATransform::name, - RedundantAssignEliminator::name, - RedundantAssignEliminator::name, - ForLoopConditionIntoBody::name, - UnusedPruner::name, - CircularReferencesPruner::name, - CommonSubexpressionEliminator::name, - }, ast); - - // Make source short and pretty. - - suite.runSequence({ - ExpressionJoiner::name, - Rematerialiser::name, - UnusedPruner::name, - CircularReferencesPruner::name, - ExpressionJoiner::name, - UnusedPruner::name, - CircularReferencesPruner::name, - ExpressionJoiner::name, - UnusedPruner::name, - CircularReferencesPruner::name, - - SSAReverser::name, - CommonSubexpressionEliminator::name, - LiteralRematerialiser::name, - ForLoopConditionOutOfBody::name, - CommonSubexpressionEliminator::name, - UnusedPruner::name, - CircularReferencesPruner::name, - - ExpressionJoiner::name, - Rematerialiser::name, - UnusedPruner::name, - CircularReferencesPruner::name, - }, ast); + suite.runSequence( + "dhfoDgvulfnTUtnIf" // None of these can make stack problems worse + "(" + "xarrscLM" // Turn into SSA and simplify + "cCTUtTOntnfDIul" // Perform structural simplification + "Lcul" // Simplify again + "Vcul jj" // Reverse SSA + + // should have good "compilability" property here. + + "eul" // Run functional expression inliner + "xarulrul" // Prune a bit more in SSA + "xarrcL" // Turn into SSA again and simplify + "gvif" // Run full inliner + "CTUcarrLsTOtfDncarrIulc" // SSA plus simplify + ")" + "jmuljuljul VcTOcul jmul", // Make source short and pretty + ast + ); // This is a tuning parameter, but actually just prevents infinite loops. size_t stackCompressorMaxIterations = 16; - suite.runSequence(vector{ - FunctionGrouper::name, - }, ast); + suite.runSequence("g", ast); + // We ignore the return value because we will get a much better error // message once we perform code generation. StackCompressor::run( @@ -256,16 +125,7 @@ void OptimiserSuite::run( _optimizeStackAllocation, stackCompressorMaxIterations ); - suite.runSequence({ - BlockFlattener::name, - DeadCodeEliminator::name, - ControlFlowSimplifier::name, - LiteralRematerialiser::name, - ForLoopConditionOutOfBody::name, - CommonSubexpressionEliminator::name, - - FunctionGrouper::name, - }, ast); + suite.runSequence("fDnTOc g", ast); if (EVMDialect const* dialect = dynamic_cast(&_dialect)) { From df1809f8dabcbad040d77cbf965584eaedfdb5da Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Tue, 14 Apr 2020 16:36:37 +0200 Subject: [PATCH 037/126] Annotate struct definitions with a recursive flag. --- .../analysis/DeclarationTypeChecker.cpp | 120 +++++++++---- libsolidity/analysis/DeclarationTypeChecker.h | 6 +- libsolidity/analysis/TypeChecker.cpp | 8 + libsolidity/ast/AST.cpp | 5 +- libsolidity/ast/AST.h | 2 +- libsolidity/ast/ASTAnnotations.h | 6 + libsolidity/ast/Types.cpp | 164 +++++++++--------- libsolidity/ast/Types.h | 11 +- libsolutil/Algorithms.h | 4 + test/libsolidity/SolidityTypes.cpp | 1 + .../recursive_struct_function_pointer.sol | 10 ++ .../function_type_argument_external.sol | 1 - .../mapping/function_type_return_external.sol | 1 - 13 files changed, 208 insertions(+), 131 deletions(-) create mode 100644 test/libsolidity/syntaxTests/structs/recursion/recursive_struct_function_pointer.sol diff --git a/libsolidity/analysis/DeclarationTypeChecker.cpp b/libsolidity/analysis/DeclarationTypeChecker.cpp index 12e84d53db0c..71fd52f1447d 100644 --- a/libsolidity/analysis/DeclarationTypeChecker.cpp +++ b/libsolidity/analysis/DeclarationTypeChecker.cpp @@ -31,40 +31,86 @@ using namespace solidity::frontend; bool DeclarationTypeChecker::visit(ElementaryTypeName const& _typeName) { - if (!_typeName.annotation().type) + if (_typeName.annotation().type) + return false; + + _typeName.annotation().type = TypeProvider::fromElementaryTypeName(_typeName.typeName()); + if (_typeName.stateMutability().has_value()) { - _typeName.annotation().type = TypeProvider::fromElementaryTypeName(_typeName.typeName()); - if (_typeName.stateMutability().has_value()) + // for non-address types this was already caught by the parser + solAssert(_typeName.annotation().type->category() == Type::Category::Address, ""); + switch (*_typeName.stateMutability()) { - // for non-address types this was already caught by the parser - solAssert(_typeName.annotation().type->category() == Type::Category::Address, ""); - switch (*_typeName.stateMutability()) - { - case StateMutability::Payable: - _typeName.annotation().type = TypeProvider::payableAddress(); - break; - case StateMutability::NonPayable: - _typeName.annotation().type = TypeProvider::address(); - break; - default: - typeError( - _typeName.location(), - "Address types can only be payable or non-payable." - ); - break; - } + case StateMutability::Payable: + _typeName.annotation().type = TypeProvider::payableAddress(); + break; + case StateMutability::NonPayable: + _typeName.annotation().type = TypeProvider::address(); + break; + default: + typeError( + _typeName.location(), + "Address types can only be payable or non-payable." + ); + break; } } return true; } +bool DeclarationTypeChecker::visit(StructDefinition const& _struct) +{ + if (_struct.annotation().recursive.has_value()) + { + if (!m_currentStructsSeen.empty() && *_struct.annotation().recursive) + m_recursiveStructSeen = true; + return false; + } + + if (m_currentStructsSeen.count(&_struct)) + { + _struct.annotation().recursive = true; + m_recursiveStructSeen = true; + return false; + } + + bool previousRecursiveStructSeen = m_recursiveStructSeen; + bool hasRecursiveChild = false; + + m_currentStructsSeen.insert(&_struct); + + for (auto const& _member: _struct.members()) + { + m_recursiveStructSeen = false; + _member->accept(*this); + if (m_recursiveStructSeen) + hasRecursiveChild = true; + } + + if (!_struct.annotation().recursive.has_value()) + _struct.annotation().recursive = hasRecursiveChild; + m_recursiveStructSeen = previousRecursiveStructSeen || *_struct.annotation().recursive; + m_currentStructsSeen.erase(&_struct); + if (m_currentStructsSeen.empty()) + m_recursiveStructSeen = false; + + return false; +} + void DeclarationTypeChecker::endVisit(UserDefinedTypeName const& _typeName) { + if (_typeName.annotation().type) + return; + Declaration const* declaration = _typeName.annotation().referencedDeclaration; solAssert(declaration, ""); if (StructDefinition const* structDef = dynamic_cast(declaration)) + { + if (!m_insideFunctionType && !m_currentStructsSeen.empty()) + structDef->accept(*this); _typeName.annotation().type = TypeProvider::structType(*structDef, DataLocation::Storage); + } else if (EnumDefinition const* enumDef = dynamic_cast(declaration)) _typeName.annotation().type = TypeProvider::enumType(*enumDef); else if (ContractDefinition const* contract = dynamic_cast(declaration)) @@ -75,8 +121,17 @@ void DeclarationTypeChecker::endVisit(UserDefinedTypeName const& _typeName) fatalTypeError(_typeName.location(), "Name has to refer to a struct, enum or contract."); } } -void DeclarationTypeChecker::endVisit(FunctionTypeName const& _typeName) +bool DeclarationTypeChecker::visit(FunctionTypeName const& _typeName) { + if (_typeName.annotation().type) + return false; + + bool previousInsideFunctionType = m_insideFunctionType; + m_insideFunctionType = true; + _typeName.parameterTypeList()->accept(*this); + _typeName.returnParameterTypeList()->accept(*this); + m_insideFunctionType = previousInsideFunctionType; + switch (_typeName.visibility()) { case Visibility::Internal: @@ -84,30 +139,22 @@ void DeclarationTypeChecker::endVisit(FunctionTypeName const& _typeName) break; default: fatalTypeError(_typeName.location(), "Invalid visibility, can only be \"external\" or \"internal\"."); - return; + return false; } if (_typeName.isPayable() && _typeName.visibility() != Visibility::External) { fatalTypeError(_typeName.location(), "Only external function types can be payable."); - return; + return false; } - - if (_typeName.visibility() == Visibility::External) - for (auto const& t: _typeName.parameterTypes() + _typeName.returnParameterTypes()) - { - solAssert(t->annotation().type, "Type not set for parameter."); - if (!t->annotation().type->interfaceType(false).get()) - { - fatalTypeError(t->location(), "Internal type cannot be used for external function type."); - return; - } - } - _typeName.annotation().type = TypeProvider::function(_typeName); + return false; } void DeclarationTypeChecker::endVisit(Mapping const& _mapping) { + if (_mapping.annotation().type) + return; + if (auto const* typeName = dynamic_cast(&_mapping.keyType())) { if (auto const* contractType = dynamic_cast(typeName->annotation().type)) @@ -140,6 +187,9 @@ void DeclarationTypeChecker::endVisit(Mapping const& _mapping) void DeclarationTypeChecker::endVisit(ArrayTypeName const& _typeName) { + if (_typeName.annotation().type) + return; + TypePointer baseType = _typeName.baseType().annotation().type; if (!baseType) { diff --git a/libsolidity/analysis/DeclarationTypeChecker.h b/libsolidity/analysis/DeclarationTypeChecker.h index 6878359ab3dc..a3f2c0c8fd6c 100644 --- a/libsolidity/analysis/DeclarationTypeChecker.h +++ b/libsolidity/analysis/DeclarationTypeChecker.h @@ -53,10 +53,11 @@ class DeclarationTypeChecker: private ASTConstVisitor bool visit(ElementaryTypeName const& _typeName) override; void endVisit(UserDefinedTypeName const& _typeName) override; - void endVisit(FunctionTypeName const& _typeName) override; + bool visit(FunctionTypeName const& _typeName) override; void endVisit(Mapping const& _mapping) override; void endVisit(ArrayTypeName const& _typeName) override; void endVisit(VariableDeclaration const& _variable) override; + bool visit(StructDefinition const& _struct) override; /// Adds a new error to the list of errors. void typeError(langutil::SourceLocation const& _location, std::string const& _description); @@ -67,6 +68,9 @@ class DeclarationTypeChecker: private ASTConstVisitor langutil::ErrorReporter& m_errorReporter; bool m_errorOccurred = false; langutil::EVMVersion m_evmVersion; + bool m_insideFunctionType = false; + bool m_recursiveStructSeen = false; + std::set m_currentStructsSeen; }; } diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index 76e805c27d67..dba8aaab9e3d 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -631,7 +631,15 @@ void TypeChecker::endVisit(FunctionTypeName const& _funType) { FunctionType const& fun = dynamic_cast(*_funType.annotation().type); if (fun.kind() == FunctionType::Kind::External) + { + for (auto const& t: _funType.parameterTypes() + _funType.returnParameterTypes()) + { + solAssert(t->annotation().type, "Type not set for parameter."); + if (!t->annotation().type->interfaceType(false).get()) + m_errorReporter.typeError(t->location(), "Internal type cannot be used for external function type."); + } solAssert(fun.interfaceType(false), "External function type uses internal types."); + } } bool TypeChecker::visit(InlineAssembly const& _inlineAssembly) diff --git a/libsolidity/ast/AST.cpp b/libsolidity/ast/AST.cpp index d615babf5e9c..10825217ccbe 100644 --- a/libsolidity/ast/AST.cpp +++ b/libsolidity/ast/AST.cpp @@ -257,12 +257,13 @@ TypeNameAnnotation& TypeName::annotation() const TypePointer StructDefinition::type() const { + solAssert(annotation().recursive.has_value(), "Requested struct type before DeclarationTypeChecker."); return TypeProvider::typeType(TypeProvider::structType(*this, DataLocation::Storage)); } -TypeDeclarationAnnotation& StructDefinition::annotation() const +StructDeclarationAnnotation& StructDefinition::annotation() const { - return initAnnotation(); + return initAnnotation(); } TypePointer EnumValue::type() const diff --git a/libsolidity/ast/AST.h b/libsolidity/ast/AST.h index 04067329db0b..bc4fed110f50 100644 --- a/libsolidity/ast/AST.h +++ b/libsolidity/ast/AST.h @@ -607,7 +607,7 @@ class StructDefinition: public Declaration bool isVisibleInDerivedContracts() const override { return true; } bool isVisibleViaContractTypeAccess() const override { return true; } - TypeDeclarationAnnotation& annotation() const override; + StructDeclarationAnnotation& annotation() const override; private: std::vector> m_members; diff --git a/libsolidity/ast/ASTAnnotations.h b/libsolidity/ast/ASTAnnotations.h index 9f7edf84ec93..126a40b43278 100644 --- a/libsolidity/ast/ASTAnnotations.h +++ b/libsolidity/ast/ASTAnnotations.h @@ -128,6 +128,12 @@ struct TypeDeclarationAnnotation: DeclarationAnnotation std::string canonicalName; }; +struct StructDeclarationAnnotation: TypeDeclarationAnnotation +{ + /// Whether the struct is recursive. Will be filled in by the DeclarationTypeChecker. + std::optional recursive; +}; + struct ContractDefinitionAnnotation: TypeDeclarationAnnotation, StructurallyDocumentedAnnotation { /// List of functions without a body. Can also contain functions from base classes. diff --git a/libsolidity/ast/Types.cpp b/libsolidity/ast/Types.cpp index 59eeb18ee6ed..554ff5015547 100644 --- a/libsolidity/ast/Types.cpp +++ b/libsolidity/ast/Types.cpp @@ -2175,93 +2175,107 @@ MemberList::MemberMap StructType::nativeMembers(ContractDefinition const*) const TypeResult StructType::interfaceType(bool _inLibrary) const { - if (_inLibrary && m_interfaceType_library.has_value()) - return *m_interfaceType_library; - - if (!_inLibrary && m_interfaceType.has_value()) - return *m_interfaceType; - - TypeResult result{TypePointer{}}; - - m_recursive = false; - - auto visitor = [&]( - StructDefinition const& _struct, - util::CycleDetector& _cycleDetector, - size_t /*_depth*/ - ) + if (!_inLibrary) { - // Check that all members have interface types. - // Return an error if at least one struct member does not have a type. - // This might happen, for example, if the type of the member does not exist. - for (ASTPointer const& variable: _struct.members()) + if (!m_interfaceType.has_value()) { - // If the struct member does not have a type return false. - // A TypeError is expected in this case. - if (!variable->annotation().type) + if (recursive()) + m_interfaceType = TypeResult::err("Recursive type not allowed for public or external contract functions."); + else { - result = TypeResult::err("Invalid type!"); - return; - } - - Type const* memberType = variable->annotation().type; - - while (dynamic_cast(memberType)) - memberType = dynamic_cast(memberType)->baseType(); - - if (StructType const* innerStruct = dynamic_cast(memberType)) - if ( - innerStruct->m_recursive == true || - _cycleDetector.run(innerStruct->structDefinition()) - ) + TypeResult result{TypePointer{}}; + for (ASTPointer const& member: m_struct.members()) { - m_recursive = true; - if (_inLibrary && location() == DataLocation::Storage) - continue; - else + if (!member->annotation().type) { - result = TypeResult::err("Recursive structs can only be passed as storage pointers to libraries, not as memory objects to contract functions."); - return; + result = TypeResult::err("Invalid type!"); + break; + } + auto interfaceType = member->annotation().type->interfaceType(false); + if (!interfaceType.get()) + { + solAssert(!interfaceType.message().empty(), "Expected detailed error message!"); + result = interfaceType; + break; } } - - auto iType = memberType->interfaceType(_inLibrary); - if (!iType.get()) - { - solAssert(!iType.message().empty(), "Expected detailed error message!"); - result = iType; - return; + if (result.message().empty()) + m_interfaceType = TypeProvider::withLocation(this, DataLocation::Memory, true); + else + m_interfaceType = result; } } - }; + return *m_interfaceType; + } + else if (m_interfaceType_library.has_value()) + return *m_interfaceType_library; - m_recursive = m_recursive.value() || (util::CycleDetector(visitor).run(structDefinition()) != nullptr); + TypeResult result{TypePointer{}}; - std::string const recursiveErrMsg = "Recursive type not allowed for public or external contract functions."; + util::BreadthFirstSearch breadthFirstSearch{{&m_struct}}; + breadthFirstSearch.run( + [&](StructDefinition const* _struct, auto&& _addChild) { + // Check that all members have interface types. + // Return an error if at least one struct member does not have a type. + // This might happen, for example, if the type of the member does not exist. + for (ASTPointer const& variable: _struct->members()) + { + // If the struct member does not have a type return false. + // A TypeError is expected in this case. + if (!variable->annotation().type) + { + result = TypeResult::err("Invalid type!"); + breadthFirstSearch.abort(); + return; + } - if (_inLibrary) - { - if (!result.message().empty()) - m_interfaceType_library = result; - else if (location() == DataLocation::Storage) - m_interfaceType_library = this; - else - m_interfaceType_library = TypeProvider::withLocation(this, DataLocation::Memory, true); + Type const* memberType = variable->annotation().type; - if (m_recursive.value()) - m_interfaceType = TypeResult::err(recursiveErrMsg); + while (dynamic_cast(memberType)) + memberType = dynamic_cast(memberType)->baseType(); - return *m_interfaceType_library; - } + if (StructType const* innerStruct = dynamic_cast(memberType)) + { + if (innerStruct->recursive() && !(_inLibrary && location() == DataLocation::Storage)) + { + result = TypeResult::err( + "Recursive structs can only be passed as storage pointers to libraries, not as memory objects to contract functions." + ); + breadthFirstSearch.abort(); + return; + } + else + _addChild(&innerStruct->structDefinition()); + } + else + { + auto iType = memberType->interfaceType(_inLibrary); + if (!iType.get()) + { + solAssert(!iType.message().empty(), "Expected detailed error message!"); + result = iType; + breadthFirstSearch.abort(); + return; + } + } + } + } + ); - if (m_recursive.value()) - m_interfaceType = TypeResult::err(recursiveErrMsg); - else if (!result.message().empty()) - m_interfaceType = result; + if (!result.message().empty()) + return result; + + if (location() == DataLocation::Storage) + m_interfaceType_library = this; else - m_interfaceType = TypeProvider::withLocation(this, DataLocation::Memory, true); + m_interfaceType_library = TypeProvider::withLocation(this, DataLocation::Memory, true); + return *m_interfaceType_library; +} - return *m_interfaceType; +bool StructType::recursive() const +{ + solAssert(m_struct.annotation().recursive.has_value(), "Called StructType::recursive() before DeclarationTypeChecker."); + return *m_struct.annotation().recursive; } std::unique_ptr StructType::copyForLocation(DataLocation _location, bool _isPointer) const @@ -2644,21 +2658,11 @@ FunctionType::FunctionType(FunctionTypeName const& _typeName): for (auto const& t: _typeName.parameterTypes()) { solAssert(t->annotation().type, "Type not set for parameter."); - if (m_kind == Kind::External) - solAssert( - t->annotation().type->interfaceType(false).get(), - "Internal type used as parameter for external function." - ); m_parameterTypes.push_back(t->annotation().type); } for (auto const& t: _typeName.returnParameterTypes()) { solAssert(t->annotation().type, "Type not set for return parameter."); - if (m_kind == Kind::External) - solAssert( - t->annotation().type->interfaceType(false).get(), - "Internal type used as return parameter for external function." - ); m_returnParameterTypes.push_back(t->annotation().type); } diff --git a/libsolidity/ast/Types.h b/libsolidity/ast/Types.h index d07c47dde183..7ab9f7d912ab 100644 --- a/libsolidity/ast/Types.h +++ b/libsolidity/ast/Types.h @@ -934,15 +934,7 @@ class StructType: public ReferenceType Type const* encodingType() const override; TypeResult interfaceType(bool _inLibrary) const override; - bool recursive() const - { - if (m_recursive.has_value()) - return m_recursive.value(); - - interfaceType(false); - - return m_recursive.value(); - } + bool recursive() const; std::unique_ptr copyForLocation(DataLocation _location, bool _isPointer) const override; @@ -971,7 +963,6 @@ class StructType: public ReferenceType // Caches for interfaceType(bool) mutable std::optional m_interfaceType; mutable std::optional m_interfaceType_library; - mutable std::optional m_recursive; }; /** diff --git a/libsolutil/Algorithms.h b/libsolutil/Algorithms.h index b9028f19b2f9..3897d65d2498 100644 --- a/libsolutil/Algorithms.h +++ b/libsolutil/Algorithms.h @@ -114,6 +114,10 @@ struct BreadthFirstSearch } return *this; } + void abort() + { + verticesToTraverse.clear(); + } std::set verticesToTraverse; std::set visited{}; diff --git a/test/libsolidity/SolidityTypes.cpp b/test/libsolidity/SolidityTypes.cpp index c6cd25c5691c..b21c8f2e11ac 100644 --- a/test/libsolidity/SolidityTypes.cpp +++ b/test/libsolidity/SolidityTypes.cpp @@ -184,6 +184,7 @@ BOOST_AUTO_TEST_CASE(type_identifiers) BOOST_CHECK_EQUAL(ContractType(c, true).identifier(), "t_super$_MyContract$$$_$2"); StructDefinition s(++id, {}, make_shared("Struct"), {}); + s.annotation().recursive = false; BOOST_CHECK_EQUAL(s.type()->identifier(), "t_type$_t_struct$_Struct_$3_storage_ptr_$"); EnumDefinition e(++id, {}, make_shared("Enum"), {}); diff --git a/test/libsolidity/syntaxTests/structs/recursion/recursive_struct_function_pointer.sol b/test/libsolidity/syntaxTests/structs/recursion/recursive_struct_function_pointer.sol new file mode 100644 index 000000000000..dc40ae3b2682 --- /dev/null +++ b/test/libsolidity/syntaxTests/structs/recursion/recursive_struct_function_pointer.sol @@ -0,0 +1,10 @@ +pragma experimental ABIEncoderV2; +contract C { + struct S { + uint a; + function() external returns (S memory) sub; + } + function f() public pure returns (S memory) { + } +} +// ---- diff --git a/test/libsolidity/syntaxTests/types/mapping/function_type_argument_external.sol b/test/libsolidity/syntaxTests/types/mapping/function_type_argument_external.sol index 34f957019e2f..8638baf85080 100644 --- a/test/libsolidity/syntaxTests/types/mapping/function_type_argument_external.sol +++ b/test/libsolidity/syntaxTests/types/mapping/function_type_argument_external.sol @@ -4,4 +4,3 @@ contract C { } // ---- // TypeError: (37-64): Data location must be "memory" for parameter in function, but "storage" was given. -// TypeError: (37-64): Internal type cannot be used for external function type. diff --git a/test/libsolidity/syntaxTests/types/mapping/function_type_return_external.sol b/test/libsolidity/syntaxTests/types/mapping/function_type_return_external.sol index aed9b387834d..b9bd5bc3f6a2 100644 --- a/test/libsolidity/syntaxTests/types/mapping/function_type_return_external.sol +++ b/test/libsolidity/syntaxTests/types/mapping/function_type_return_external.sol @@ -4,4 +4,3 @@ contract C { } // ---- // TypeError: (57-84): Data location must be "memory" for return parameter in function, but "storage" was given. -// TypeError: (57-84): Internal type cannot be used for external function type. From b744a568013d84ba8f9b4ecdc0d23d972d165b3c Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Tue, 14 Apr 2020 19:03:46 +0200 Subject: [PATCH 038/126] Refactoring of errors and fixes for various ICEs. --- libsolidity/analysis/TypeChecker.cpp | 19 +++----- libsolidity/ast/AST.cpp | 20 ++++++--- libsolidity/ast/Types.cpp | 43 +++++++++++++++++-- libsolidity/ast/Types.h | 11 +++-- .../array/length/not_too_large.sol | 1 + .../array/length/parameter_too_large.sol | 2 +- .../length/parameter_too_large_multidim.sol | 8 ++-- .../parameter_too_large_multidim_ABIv2.sol | 4 +- .../large_array_in_memory_struct.sol | 17 ++++++++ .../large_array_in_memory_struct_2.sol | 12 ++++++ .../memory_mapping_array.sol | 7 +++ .../recursive_struct_memory.sol | 13 ++++++ .../207_no_mappings_in_memory_array.sol | 2 +- ...too_large_arrays_for_calldata_external.sol | 2 +- ...too_large_arrays_for_calldata_internal.sol | 2 +- ...8_too_large_arrays_for_calldata_public.sol | 2 +- 16 files changed, 127 insertions(+), 38 deletions(-) create mode 100644 test/libsolidity/syntaxTests/iceRegressionTests/large_array_in_memory_struct.sol create mode 100644 test/libsolidity/syntaxTests/iceRegressionTests/large_array_in_memory_struct_2.sol create mode 100644 test/libsolidity/syntaxTests/iceRegressionTests/memory_mapping_array.sol create mode 100644 test/libsolidity/syntaxTests/iceRegressionTests/recursive_struct_memory.sol diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index dba8aaab9e3d..920b453d626f 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -518,19 +518,14 @@ bool TypeChecker::visit(VariableDeclaration const& _variable) m_errorReporter.typeError(_variable.location(), "Internal or recursive type is not allowed for public state variables."); } - switch (varType->category()) + if (auto referenceType = dynamic_cast(varType)) { - case Type::Category::Array: - if (auto arrayType = dynamic_cast(varType)) - if ( - ((arrayType->location() == DataLocation::Memory) || - (arrayType->location() == DataLocation::CallData)) && - !arrayType->validForCalldata() - ) - m_errorReporter.typeError(_variable.location(), "Array is too large to be encoded."); - break; - default: - break; + auto result = referenceType->validForLocation(referenceType->location()); + if (!result) + { + solAssert(!result.message().empty(), "Expected detailed error message"); + m_errorReporter.typeError(_variable.location(), result.message()); + } } return false; diff --git a/libsolidity/ast/AST.cpp b/libsolidity/ast/AST.cpp index 10825217ccbe..e65431d60f0e 100644 --- a/libsolidity/ast/AST.cpp +++ b/libsolidity/ast/AST.cpp @@ -617,12 +617,20 @@ set VariableDeclaration::allowedDataLocations() c else if (isLocalVariable()) { solAssert(typeName(), ""); - solAssert(typeName()->annotation().type, "Can only be called after reference resolution"); - if (typeName()->annotation().type->category() == Type::Category::Mapping) - return set{ Location::Storage }; - else - // TODO: add Location::Calldata once implemented for local variables. - return set{ Location::Memory, Location::Storage }; + auto getDataLocations = [](TypePointer _type, auto&& _recursion) -> set { + solAssert(_type, "Can only be called after reference resolution"); + switch (_type->category()) + { + case Type::Category::Array: + return _recursion(dynamic_cast(_type)->baseType(), _recursion); + case Type::Category::Mapping: + return set{ Location::Storage }; + default: + // TODO: add Location::Calldata once implemented for local variables. + return set{ Location::Memory, Location::Storage }; + } + }; + return getDataLocations(typeName()->annotation().type, getDataLocations); } else // Struct members etc. diff --git a/libsolidity/ast/Types.cpp b/libsolidity/ast/Types.cpp index 554ff5015547..7f6577ea3407 100644 --- a/libsolidity/ast/Types.cpp +++ b/libsolidity/ast/Types.cpp @@ -1649,12 +1649,35 @@ bool ArrayType::operator==(Type const& _other) const return isDynamicallySized() || length() == other.length(); } -bool ArrayType::validForCalldata() const +BoolResult ArrayType::validForLocation(DataLocation _loc) const { if (auto arrayBaseType = dynamic_cast(baseType())) - if (!arrayBaseType->validForCalldata()) - return false; - return isDynamicallySized() || unlimitedStaticCalldataSize(true) <= numeric_limits::max(); + { + BoolResult result = arrayBaseType->validForLocation(_loc); + if (!result) + return result; + } + if (isDynamicallySized()) + return true; + switch (_loc) + { + case DataLocation::Memory: + { + bigint size = bigint(length()) * m_baseType->memoryHeadSize(); + if (size >= numeric_limits::max()) + return BoolResult::err("Type too large for memory."); + break; + } + case DataLocation::CallData: + { + if (unlimitedStaticCalldataSize(true) >= numeric_limits::max()) + return BoolResult::err("Type too large for calldata."); + break; + } + case DataLocation::Storage: + break; + } + return true; } bigint ArrayType::unlimitedStaticCalldataSize(bool _padded) const @@ -2272,6 +2295,18 @@ TypeResult StructType::interfaceType(bool _inLibrary) const return *m_interfaceType_library; } +BoolResult StructType::validForLocation(DataLocation _loc) const +{ + for (auto const& member: m_struct.members()) + if (auto referenceType = dynamic_cast(member->annotation().type)) + { + BoolResult result = referenceType->validForLocation(_loc); + if (!result) + return result; + } + return true; +} + bool StructType::recursive() const { solAssert(m_struct.annotation().recursive.has_value(), "Called StructType::recursive() before DeclarationTypeChecker."); diff --git a/libsolidity/ast/Types.h b/libsolidity/ast/Types.h index 7ab9f7d912ab..c1f66df4d52d 100644 --- a/libsolidity/ast/Types.h +++ b/libsolidity/ast/Types.h @@ -706,6 +706,9 @@ class ReferenceType: public Type /// never change the contents of the original value. bool isPointer() const; + /// @returns true if this is valid to be stored in data location _loc + virtual BoolResult validForLocation(DataLocation _loc) const = 0; + bool operator==(ReferenceType const& _other) const { return location() == _other.location() && isPointer() == _other.isPointer(); @@ -772,8 +775,7 @@ class ArrayType: public ReferenceType TypePointer decodingType() const override; TypeResult interfaceType(bool _inLibrary) const override; - /// @returns true if this is valid to be stored in calldata - bool validForCalldata() const; + BoolResult validForLocation(DataLocation _loc) const override; /// @returns true if this is a byte array or a string bool isByteArray() const { return m_arrayKind != ArrayKind::Ordinary; } @@ -827,8 +829,7 @@ class ArraySliceType: public ReferenceType bool canLiveOutsideStorage() const override { return m_arrayType.canLiveOutsideStorage(); } std::string toString(bool _short) const override; - /// @returns true if this is valid to be stored in calldata - bool validForCalldata() const { return m_arrayType.validForCalldata(); } + BoolResult validForLocation(DataLocation _loc) const override { return m_arrayType.validForLocation(_loc); } ArrayType const& arrayType() const { return m_arrayType; } u256 memoryDataSize() const override { solAssert(false, ""); } @@ -934,6 +935,8 @@ class StructType: public ReferenceType Type const* encodingType() const override; TypeResult interfaceType(bool _inLibrary) const override; + BoolResult validForLocation(DataLocation _loc) const override; + bool recursive() const; std::unique_ptr copyForLocation(DataLocation _location, bool _isPointer) const override; diff --git a/test/libsolidity/syntaxTests/array/length/not_too_large.sol b/test/libsolidity/syntaxTests/array/length/not_too_large.sol index deb10c5b1a62..0ba74d7f859b 100644 --- a/test/libsolidity/syntaxTests/array/length/not_too_large.sol +++ b/test/libsolidity/syntaxTests/array/length/not_too_large.sol @@ -7,3 +7,4 @@ contract C { } } // ---- +// TypeError: (226-234): Type too large for memory. diff --git a/test/libsolidity/syntaxTests/array/length/parameter_too_large.sol b/test/libsolidity/syntaxTests/array/length/parameter_too_large.sol index 02e0a7cc6dbc..7c7cf4fa0f8d 100644 --- a/test/libsolidity/syntaxTests/array/length/parameter_too_large.sol +++ b/test/libsolidity/syntaxTests/array/length/parameter_too_large.sol @@ -2,4 +2,4 @@ contract C { function f(bytes32[1263941234127518272] memory) public pure {} } // ---- -// TypeError: (26-61): Array is too large to be encoded. +// TypeError: (26-61): Type too large for memory. diff --git a/test/libsolidity/syntaxTests/array/length/parameter_too_large_multidim.sol b/test/libsolidity/syntaxTests/array/length/parameter_too_large_multidim.sol index 5f96ecd564dc..4954cbe9490c 100644 --- a/test/libsolidity/syntaxTests/array/length/parameter_too_large_multidim.sol +++ b/test/libsolidity/syntaxTests/array/length/parameter_too_large_multidim.sol @@ -2,10 +2,8 @@ contract C { function f(bytes32[1263941234127518272][500] memory) public pure {} function f(uint[2**30][] memory) public pure {} function f(uint[2**30][2**30][] memory) public pure {} - function f(uint[2**16][2**16][] memory) public pure {} } // ---- -// TypeError: (26-66): Array is too large to be encoded. -// TypeError: (96-116): Array is too large to be encoded. -// TypeError: (146-173): Array is too large to be encoded. -// TypeError: (203-230): Array is too large to be encoded. +// TypeError: (26-66): Type too large for memory. +// TypeError: (96-116): Type too large for memory. +// TypeError: (146-173): Type too large for memory. diff --git a/test/libsolidity/syntaxTests/array/length/parameter_too_large_multidim_ABIv2.sol b/test/libsolidity/syntaxTests/array/length/parameter_too_large_multidim_ABIv2.sol index d376bca9aeb6..26a683aa7b32 100644 --- a/test/libsolidity/syntaxTests/array/length/parameter_too_large_multidim_ABIv2.sol +++ b/test/libsolidity/syntaxTests/array/length/parameter_too_large_multidim_ABIv2.sol @@ -5,5 +5,5 @@ contract C { function f(uint[2**30][2**30][][] memory) public pure {} } // ---- -// TypeError: (61-101): Array is too large to be encoded. -// TypeError: (131-160): Array is too large to be encoded. +// TypeError: (61-101): Type too large for memory. +// TypeError: (131-160): Type too large for memory. diff --git a/test/libsolidity/syntaxTests/iceRegressionTests/large_array_in_memory_struct.sol b/test/libsolidity/syntaxTests/iceRegressionTests/large_array_in_memory_struct.sol new file mode 100644 index 000000000000..273a75a5b777 --- /dev/null +++ b/test/libsolidity/syntaxTests/iceRegressionTests/large_array_in_memory_struct.sol @@ -0,0 +1,17 @@ +contract C { + struct X { bytes31 [ 3 ] x1 ; + uint x2 ; + } + struct S { uint256 [ ] [ 0.425781 ether ] s1 ; + uint [ 2 ** 0xFF ] [ 2 ** 0x42 ] s2 ; + X s3 ; + mapping ( uint => address payable ) c ; + uint [ 9 hours ** 16 ] d ; + string s ; + } + function f ( ) public { function ( function ( bytes9 , uint ) external pure returns ( uint ) , uint ) external pure returns ( uint ) [ 3 ] memory s2 ; + S memory s ; + } +} +// ---- +// TypeError: (530-540): Type too large for memory. diff --git a/test/libsolidity/syntaxTests/iceRegressionTests/large_array_in_memory_struct_2.sol b/test/libsolidity/syntaxTests/iceRegressionTests/large_array_in_memory_struct_2.sol new file mode 100644 index 000000000000..ada721c54626 --- /dev/null +++ b/test/libsolidity/syntaxTests/iceRegressionTests/large_array_in_memory_struct_2.sol @@ -0,0 +1,12 @@ +contract C { + struct R { uint[10][10] y; } + struct S { uint a; uint b; R d; uint[20][20][2999999999999999999999999990] c; } + function f() public pure { + C.S memory y; + C.S[10] memory z; + y.a < 2; + z; y; + } +} +// ---- +// TypeError: (169-181): Type too large for memory. diff --git a/test/libsolidity/syntaxTests/iceRegressionTests/memory_mapping_array.sol b/test/libsolidity/syntaxTests/iceRegressionTests/memory_mapping_array.sol new file mode 100644 index 000000000000..a9a574132c42 --- /dev/null +++ b/test/libsolidity/syntaxTests/iceRegressionTests/memory_mapping_array.sol @@ -0,0 +1,7 @@ + contract C { + function h ( bool flag ) public returns ( bool c ) { + mapping ( string => uint24 ) [ 1 ] memory val ; + } +} +// ---- +// TypeError: (91-136): Data location must be "storage" for variable, but "memory" was given. diff --git a/test/libsolidity/syntaxTests/iceRegressionTests/recursive_struct_memory.sol b/test/libsolidity/syntaxTests/iceRegressionTests/recursive_struct_memory.sol new file mode 100644 index 000000000000..fdb652b0677f --- /dev/null +++ b/test/libsolidity/syntaxTests/iceRegressionTests/recursive_struct_memory.sol @@ -0,0 +1,13 @@ +contract Test { + struct RecursiveStruct { + address payable d ; + mapping ( uint => address payable ) c ; + mapping ( uint => address payable [ ] ) d ; + } + function func ( ) private pure { + RecursiveStruct [ 1 ] memory val ; + val ; + } +} +// ---- +// DeclarationError: (157-198): Identifier already declared. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/207_no_mappings_in_memory_array.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/207_no_mappings_in_memory_array.sol index 5220ee22bf6d..cd2d2f2fcaab 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/207_no_mappings_in_memory_array.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/207_no_mappings_in_memory_array.sol @@ -4,4 +4,4 @@ contract C { } } // ---- -// TypeError: (47-77): Type mapping(uint256 => uint256)[] memory is only valid in storage. +// TypeError: (47-77): Data location must be "storage" for variable, but "memory" was given. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/476_too_large_arrays_for_calldata_external.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/476_too_large_arrays_for_calldata_external.sol index 78c38aaf2150..80bb632c956a 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/476_too_large_arrays_for_calldata_external.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/476_too_large_arrays_for_calldata_external.sol @@ -3,4 +3,4 @@ contract C { } } // ---- -// TypeError: (28-56): Array is too large to be encoded. +// TypeError: (28-56): Type too large for calldata. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/477_too_large_arrays_for_calldata_internal.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/477_too_large_arrays_for_calldata_internal.sol index 7578246ee447..5159f57ea216 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/477_too_large_arrays_for_calldata_internal.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/477_too_large_arrays_for_calldata_internal.sol @@ -3,4 +3,4 @@ contract C { } } // ---- -// TypeError: (28-54): Array is too large to be encoded. +// TypeError: (28-54): Type too large for memory. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/478_too_large_arrays_for_calldata_public.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/478_too_large_arrays_for_calldata_public.sol index 2831b6fbb9ca..de42bad1e958 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/478_too_large_arrays_for_calldata_public.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/478_too_large_arrays_for_calldata_public.sol @@ -3,4 +3,4 @@ contract C { } } // ---- -// TypeError: (28-54): Array is too large to be encoded. +// TypeError: (28-54): Type too large for memory. From 6f06154eb5a0e9380bcc7775fafbb946f026397e Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Wed, 15 Apr 2020 11:37:39 +0200 Subject: [PATCH 039/126] Move direct struct recursion check to detect recursion in global structs. --- .../analysis/DeclarationTypeChecker.cpp | 37 ++++++++++++++++++- libsolidity/analysis/DeclarationTypeChecker.h | 3 ++ libsolidity/analysis/TypeChecker.cpp | 33 ----------------- libsolidity/analysis/TypeChecker.h | 1 - .../recursive_struct_forward_reference.sol | 2 +- .../syntaxTests/structs/struct_var_member.sol | 7 ++++ .../types/global_struct_recursive.sol | 8 ++++ 7 files changed, 54 insertions(+), 37 deletions(-) create mode 100644 test/libsolidity/syntaxTests/structs/struct_var_member.sol create mode 100644 test/libsolidity/syntaxTests/types/global_struct_recursive.sol diff --git a/libsolidity/analysis/DeclarationTypeChecker.cpp b/libsolidity/analysis/DeclarationTypeChecker.cpp index 71fd52f1447d..193ad83bdc2a 100644 --- a/libsolidity/analysis/DeclarationTypeChecker.cpp +++ b/libsolidity/analysis/DeclarationTypeChecker.cpp @@ -23,6 +23,8 @@ #include +#include + #include using namespace std; @@ -79,10 +81,12 @@ bool DeclarationTypeChecker::visit(StructDefinition const& _struct) m_currentStructsSeen.insert(&_struct); - for (auto const& _member: _struct.members()) + for (auto const& member: _struct.members()) { m_recursiveStructSeen = false; - _member->accept(*this); + member->accept(*this); + solAssert(member->annotation().type, ""); + solAssert(member->annotation().type->canBeStored(), "Type cannot be used in struct."); if (m_recursiveStructSeen) hasRecursiveChild = true; } @@ -94,6 +98,29 @@ bool DeclarationTypeChecker::visit(StructDefinition const& _struct) if (m_currentStructsSeen.empty()) m_recursiveStructSeen = false; + // Check direct recursion, fatal error if detected. + auto visitor = [&](StructDefinition const& _struct, auto& _cycleDetector, size_t _depth) + { + if (_depth >= 256) + fatalDeclarationError(_struct.location(), "Struct definition exhausting cyclic dependency validator."); + + for (ASTPointer const& member: _struct.members()) + { + Type const* memberType = member->annotation().type; + while (auto arrayType = dynamic_cast(memberType)) + { + if (arrayType->isDynamicallySized()) + break; + memberType = arrayType->baseType(); + } + if (auto structType = dynamic_cast(memberType)) + if (_cycleDetector.run(structType->structDefinition())) + return; + } + }; + if (util::CycleDetector(visitor).run(_struct) != nullptr) + fatalTypeError(_struct.location(), "Recursive struct definition."); + return false; } @@ -342,6 +369,12 @@ void DeclarationTypeChecker::fatalTypeError(SourceLocation const& _location, str m_errorReporter.fatalTypeError(_location, _description); } +void DeclarationTypeChecker::fatalDeclarationError(SourceLocation const& _location, string const& _description) +{ + m_errorOccurred = true; + m_errorReporter.fatalDeclarationError(_location, _description); +} + bool DeclarationTypeChecker::check(ASTNode const& _node) { _node.accept(*this); diff --git a/libsolidity/analysis/DeclarationTypeChecker.h b/libsolidity/analysis/DeclarationTypeChecker.h index a3f2c0c8fd6c..b704c4fd203d 100644 --- a/libsolidity/analysis/DeclarationTypeChecker.h +++ b/libsolidity/analysis/DeclarationTypeChecker.h @@ -65,6 +65,9 @@ class DeclarationTypeChecker: private ASTConstVisitor /// Adds a new error to the list of errors and throws to abort reference resolving. void fatalTypeError(langutil::SourceLocation const& _location, std::string const& _description); + /// Adds a new error to the list of errors and throws to abort reference resolving. + void fatalDeclarationError(langutil::SourceLocation const& _location, std::string const& _description); + langutil::ErrorReporter& m_errorReporter; bool m_errorOccurred = false; langutil::EVMVersion m_evmVersion; diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index 920b453d626f..b17f5b7730ee 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -289,39 +289,6 @@ void TypeChecker::endVisit(UsingForDirective const& _usingFor) m_errorReporter.fatalTypeError(_usingFor.libraryName().location(), "Library name expected."); } -bool TypeChecker::visit(StructDefinition const& _struct) -{ - for (ASTPointer const& member: _struct.members()) - solAssert(type(*member)->canBeStored(), "Type cannot be used in struct."); - - // Check recursion, fatal error if detected. - auto visitor = [&](StructDefinition const& _struct, CycleDetector& _cycleDetector, size_t _depth) - { - if (_depth >= 256) - m_errorReporter.fatalDeclarationError(_struct.location(), "Struct definition exhausting cyclic dependency validator."); - - for (ASTPointer const& member: _struct.members()) - { - Type const* memberType = type(*member); - while (auto arrayType = dynamic_cast(memberType)) - { - if (arrayType->isDynamicallySized()) - break; - memberType = arrayType->baseType(); - } - if (auto structType = dynamic_cast(memberType)) - if (_cycleDetector.run(structType->structDefinition())) - return; - } - }; - if (CycleDetector(visitor).run(_struct) != nullptr) - m_errorReporter.fatalTypeError(_struct.location(), "Recursive struct definition."); - - ASTNode::listAccept(_struct.members(), *this); - - return false; -} - bool TypeChecker::visit(FunctionDefinition const& _function) { bool isLibraryFunction = _function.inContractKind() == ContractKind::Library; diff --git a/libsolidity/analysis/TypeChecker.h b/libsolidity/analysis/TypeChecker.h index 5aaba15f7aa2..585f7d95f47f 100644 --- a/libsolidity/analysis/TypeChecker.h +++ b/libsolidity/analysis/TypeChecker.h @@ -112,7 +112,6 @@ class TypeChecker: private ASTConstVisitor void endVisit(InheritanceSpecifier const& _inheritance) override; void endVisit(UsingForDirective const& _usingFor) override; - bool visit(StructDefinition const& _struct) override; bool visit(FunctionDefinition const& _function) override; bool visit(VariableDeclaration const& _variable) override; /// We need to do this manually because we want to pass the bases of the current contract in diff --git a/test/libsolidity/syntaxTests/structs/recursion/recursive_struct_forward_reference.sol b/test/libsolidity/syntaxTests/structs/recursion/recursive_struct_forward_reference.sol index 6e1ad1cf37c7..573f227444a4 100644 --- a/test/libsolidity/syntaxTests/structs/recursion/recursive_struct_forward_reference.sol +++ b/test/libsolidity/syntaxTests/structs/recursion/recursive_struct_forward_reference.sol @@ -4,7 +4,7 @@ contract C { function f(Data.S memory a) public {} } contract Data { - struct S { S x; } + struct S { S[] x; } } // ---- // TypeError: (63-78): Recursive type not allowed for public or external contract functions. diff --git a/test/libsolidity/syntaxTests/structs/struct_var_member.sol b/test/libsolidity/syntaxTests/structs/struct_var_member.sol new file mode 100644 index 000000000000..04a274bcb232 --- /dev/null +++ b/test/libsolidity/syntaxTests/structs/struct_var_member.sol @@ -0,0 +1,7 @@ +contract C { + struct S { + var x; + } +} +// ---- +// ParserError: (27-30): Expected explicit type name. diff --git a/test/libsolidity/syntaxTests/types/global_struct_recursive.sol b/test/libsolidity/syntaxTests/types/global_struct_recursive.sol new file mode 100644 index 000000000000..dc4becaae5f6 --- /dev/null +++ b/test/libsolidity/syntaxTests/types/global_struct_recursive.sol @@ -0,0 +1,8 @@ +struct s1 { s2 x; } +struct s2 { s1 y; } + +contract C { + // whatever +} +// ---- +// TypeError: (0-19): Recursive struct definition. From 3a5a4c6505d787d0e617bb97ae3cbe1aca538947 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Wed, 15 Apr 2020 12:05:23 +0200 Subject: [PATCH 040/126] Another ICE test. --- .../identifier_collision_return_declare.sol | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 test/libsolidity/syntaxTests/iceRegressionTests/identifier_collision_return_declare.sol diff --git a/test/libsolidity/syntaxTests/iceRegressionTests/identifier_collision_return_declare.sol b/test/libsolidity/syntaxTests/iceRegressionTests/identifier_collision_return_declare.sol new file mode 100644 index 000000000000..9df2f3f0b79a --- /dev/null +++ b/test/libsolidity/syntaxTests/iceRegressionTests/identifier_collision_return_declare.sol @@ -0,0 +1,5 @@ +contract C { + function ( uint ) external returns ( a [ ] calldata ) public a = ( 1 / 2 ) ; +} +// ---- +// TypeError: (58-59): Name has to refer to a struct, enum or contract. From 6093982606afbfdff1267a283072d2613ed287fe Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Wed, 15 Apr 2020 16:00:01 +0200 Subject: [PATCH 041/126] Review suggestions. --- libsolidity/analysis/DeclarationTypeChecker.cpp | 2 +- libsolidity/ast/AST.cpp | 4 ++-- libsolidity/ast/ASTAnnotations.h | 6 +++++- .../large_struct_array.sol} | 0 .../types/cyclic_dependency_check_on_struct_exhausted.sol | 2 +- 5 files changed, 9 insertions(+), 5 deletions(-) rename test/libsolidity/syntaxTests/{array/length/not_too_large.sol => iceRegressionTests/large_struct_array.sol} (100%) diff --git a/libsolidity/analysis/DeclarationTypeChecker.cpp b/libsolidity/analysis/DeclarationTypeChecker.cpp index 193ad83bdc2a..0c53732d614d 100644 --- a/libsolidity/analysis/DeclarationTypeChecker.cpp +++ b/libsolidity/analysis/DeclarationTypeChecker.cpp @@ -102,7 +102,7 @@ bool DeclarationTypeChecker::visit(StructDefinition const& _struct) auto visitor = [&](StructDefinition const& _struct, auto& _cycleDetector, size_t _depth) { if (_depth >= 256) - fatalDeclarationError(_struct.location(), "Struct definition exhausting cyclic dependency validator."); + fatalDeclarationError(_struct.location(), "Struct definition exhausts cyclic dependency validator."); for (ASTPointer const& member: _struct.members()) { diff --git a/libsolidity/ast/AST.cpp b/libsolidity/ast/AST.cpp index e65431d60f0e..474461dada99 100644 --- a/libsolidity/ast/AST.cpp +++ b/libsolidity/ast/AST.cpp @@ -617,7 +617,7 @@ set VariableDeclaration::allowedDataLocations() c else if (isLocalVariable()) { solAssert(typeName(), ""); - auto getDataLocations = [](TypePointer _type, auto&& _recursion) -> set { + auto dataLocations = [](TypePointer _type, auto&& _recursion) -> set { solAssert(_type, "Can only be called after reference resolution"); switch (_type->category()) { @@ -630,7 +630,7 @@ set VariableDeclaration::allowedDataLocations() c return set{ Location::Memory, Location::Storage }; } }; - return getDataLocations(typeName()->annotation().type, getDataLocations); + return dataLocations(typeName()->annotation().type, dataLocations); } else // Struct members etc. diff --git a/libsolidity/ast/ASTAnnotations.h b/libsolidity/ast/ASTAnnotations.h index 126a40b43278..230f18e5ee53 100644 --- a/libsolidity/ast/ASTAnnotations.h +++ b/libsolidity/ast/ASTAnnotations.h @@ -130,7 +130,11 @@ struct TypeDeclarationAnnotation: DeclarationAnnotation struct StructDeclarationAnnotation: TypeDeclarationAnnotation { - /// Whether the struct is recursive. Will be filled in by the DeclarationTypeChecker. + /// Whether the struct is recursive, i.e. if the struct (recursively) contains a member that involves a struct of the same + /// type, either in a dynamic array, as member of another struct or inside a mapping. + /// Only cases in which the recursive occurrence is within a dynamic array or a mapping are valid, while direct + /// recursion immediately raises an error. + /// Will be filled in by the DeclarationTypeChecker. std::optional recursive; }; diff --git a/test/libsolidity/syntaxTests/array/length/not_too_large.sol b/test/libsolidity/syntaxTests/iceRegressionTests/large_struct_array.sol similarity index 100% rename from test/libsolidity/syntaxTests/array/length/not_too_large.sol rename to test/libsolidity/syntaxTests/iceRegressionTests/large_struct_array.sol diff --git a/test/libsolidity/syntaxTests/types/cyclic_dependency_check_on_struct_exhausted.sol b/test/libsolidity/syntaxTests/types/cyclic_dependency_check_on_struct_exhausted.sol index db0ff4af660e..027db9754292 100644 --- a/test/libsolidity/syntaxTests/types/cyclic_dependency_check_on_struct_exhausted.sol +++ b/test/libsolidity/syntaxTests/types/cyclic_dependency_check_on_struct_exhausted.sol @@ -257,4 +257,4 @@ contract Main { struct JW { int i; } } // ---- -// DeclarationError: (6091-6111): Struct definition exhausting cyclic dependency validator. +// DeclarationError: (6091-6111): Struct definition exhausts cyclic dependency validator. From f6d1cee06b28f6af772e66e2aa6e4b403aef2530 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Thu, 16 Apr 2020 16:29:23 +0200 Subject: [PATCH 042/126] Ensure that public callable parameters are valid for calldata. --- libsolidity/analysis/TypeChecker.cpp | 2 ++ libsolidity/ast/AST.cpp | 12 ++++++++++++ libsolidity/ast/AST.h | 2 ++ libsolutil/Result.h | 2 +- .../array/length/parameter_too_large_multidim.sol | 2 ++ 5 files changed, 19 insertions(+), 1 deletion(-) diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index b17f5b7730ee..3cc56c114971 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -488,6 +488,8 @@ bool TypeChecker::visit(VariableDeclaration const& _variable) if (auto referenceType = dynamic_cast(varType)) { auto result = referenceType->validForLocation(referenceType->location()); + if (result && _variable.isPublicCallableParameter()) + result = referenceType->validForLocation(DataLocation::CallData); if (!result) { solAssert(!result.message().empty(), "Expected detailed error message"); diff --git a/libsolidity/ast/AST.cpp b/libsolidity/ast/AST.cpp index 474461dada99..ecb24de4cc34 100644 --- a/libsolidity/ast/AST.cpp +++ b/libsolidity/ast/AST.cpp @@ -559,6 +559,18 @@ bool VariableDeclaration::isExternalCallableParameter() const return false; } +bool VariableDeclaration::isPublicCallableParameter() const +{ + if (!isCallableOrCatchParameter()) + return false; + + if (auto const* callable = dynamic_cast(scope())) + if (callable->visibility() == Visibility::Public) + return !isReturnParameter(); + + return false; +} + bool VariableDeclaration::isInternalCallableParameter() const { if (!isCallableOrCatchParameter()) diff --git a/libsolidity/ast/AST.h b/libsolidity/ast/AST.h index bc4fed110f50..0b24acb6cdd9 100644 --- a/libsolidity/ast/AST.h +++ b/libsolidity/ast/AST.h @@ -914,6 +914,8 @@ class VariableDeclaration: public Declaration /// @returns true if this variable is a parameter (not return parameter) of an external function. /// This excludes parameters of external function type names. bool isExternalCallableParameter() const; + /// @returns true if this variable is a parameter (not return parameter) of a public function. + bool isPublicCallableParameter() const; /// @returns true if this variable is a parameter or return parameter of an internal function /// or a function type of internal visibility. bool isInternalCallableParameter() const; diff --git a/libsolutil/Result.h b/libsolutil/Result.h index a8150a82555a..a2c471097f90 100644 --- a/libsolutil/Result.h +++ b/libsolutil/Result.h @@ -36,7 +36,7 @@ namespace solidity::util /// template -class Result +class [[nodiscard]] Result { public: /// Constructs a result with _value and an empty message. diff --git a/test/libsolidity/syntaxTests/array/length/parameter_too_large_multidim.sol b/test/libsolidity/syntaxTests/array/length/parameter_too_large_multidim.sol index 4954cbe9490c..f862b01cd28c 100644 --- a/test/libsolidity/syntaxTests/array/length/parameter_too_large_multidim.sol +++ b/test/libsolidity/syntaxTests/array/length/parameter_too_large_multidim.sol @@ -2,8 +2,10 @@ contract C { function f(bytes32[1263941234127518272][500] memory) public pure {} function f(uint[2**30][] memory) public pure {} function f(uint[2**30][2**30][] memory) public pure {} + function f(uint[2**16][2**16][] memory) public pure {} } // ---- // TypeError: (26-66): Type too large for memory. // TypeError: (96-116): Type too large for memory. // TypeError: (146-173): Type too large for memory. +// TypeError: (203-230): Type too large for calldata. From 150497c12aa436b8d1b76946623c644e007931ee Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Thu, 16 Apr 2020 17:51:18 +0200 Subject: [PATCH 043/126] Stricter bounds for memory arrays. --- libsolidity/ast/Types.cpp | 17 ++++++++++++++++- .../array/length/local_memory_too_large.sol | 14 ++++++++++++++ .../length/parameter_too_large_multidim.sol | 2 +- 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 test/libsolidity/syntaxTests/array/length/local_memory_too_large.sol diff --git a/libsolidity/ast/Types.cpp b/libsolidity/ast/Types.cpp index 7f6577ea3407..8f1e6c910a22 100644 --- a/libsolidity/ast/Types.cpp +++ b/libsolidity/ast/Types.cpp @@ -1663,7 +1663,22 @@ BoolResult ArrayType::validForLocation(DataLocation _loc) const { case DataLocation::Memory: { - bigint size = bigint(length()) * m_baseType->memoryHeadSize(); + bigint size = bigint(length()); + auto type = m_baseType; + while (auto arrayType = dynamic_cast(type)) + { + if (arrayType->isDynamicallySized()) + break; + else + { + size *= arrayType->length(); + type = arrayType->baseType(); + } + } + if (type->isDynamicallySized()) + size *= type->memoryHeadSize(); + else + size *= type->memoryDataSize(); if (size >= numeric_limits::max()) return BoolResult::err("Type too large for memory."); break; diff --git a/test/libsolidity/syntaxTests/array/length/local_memory_too_large.sol b/test/libsolidity/syntaxTests/array/length/local_memory_too_large.sol new file mode 100644 index 000000000000..9e0d6d62d5c9 --- /dev/null +++ b/test/libsolidity/syntaxTests/array/length/local_memory_too_large.sol @@ -0,0 +1,14 @@ +contract C { + function f() public pure + { + bytes32[1263941234127518272][500] memory x; + uint[2**30][] memory y; + uint[2**30][2**30][] memory z; + uint[2**16][2**16][] memory w; + } +} +// ---- +// TypeError: (48-90): Type too large for memory. +// TypeError: (96-118): Type too large for memory. +// TypeError: (124-153): Type too large for memory. +// TypeError: (159-188): Type too large for memory. diff --git a/test/libsolidity/syntaxTests/array/length/parameter_too_large_multidim.sol b/test/libsolidity/syntaxTests/array/length/parameter_too_large_multidim.sol index f862b01cd28c..737c99feb8ac 100644 --- a/test/libsolidity/syntaxTests/array/length/parameter_too_large_multidim.sol +++ b/test/libsolidity/syntaxTests/array/length/parameter_too_large_multidim.sol @@ -8,4 +8,4 @@ contract C { // TypeError: (26-66): Type too large for memory. // TypeError: (96-116): Type too large for memory. // TypeError: (146-173): Type too large for memory. -// TypeError: (203-230): Type too large for calldata. +// TypeError: (203-230): Type too large for memory. From 1e38985fef4cce8dec3e3d8da54e92dd458dc8ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 16 Apr 2020 18:05:54 +0200 Subject: [PATCH 044/126] command-line help: Clarify that --optimize-yul no longer does anything --- solc/CommandLineInterface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index 90c2931c56ab..0338b234f8cf 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -753,7 +753,7 @@ Allowed options)", "Set for how many contract runs to optimize." "Lower values will optimize more for initial deployment cost, higher values will optimize more for high-frequency usage." ) - (g_strOptimizeYul.c_str(), "Enable Yul optimizer in Solidity. Legacy option: the yul optimizer is enabled as part of the general --optimize option.") + (g_strOptimizeYul.c_str(), "Legacy option, ignored. Use the general --optimize to enable Yul optimizer.") (g_strNoOptimizeYul.c_str(), "Disable Yul optimizer in Solidity.") (g_argPrettyJson.c_str(), "Output JSON in pretty format. Currently it only works with the combined JSON output.") ( From 004be1788f8492c3fd2280d3e6c80204115fb91a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 16 Apr 2020 18:12:53 +0200 Subject: [PATCH 045/126] command-line help: Move optimizer options to a separate section --- solc/CommandLineInterface.cpp | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index 0338b234f8cf..3c1445e449d4 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -746,15 +746,6 @@ Allowed options)", "Select desired EVM version. Either homestead, tangerineWhistle, spuriousDragon, " "byzantium, constantinople, petersburg, istanbul (default) or berlin." ) - (g_argOptimize.c_str(), "Enable bytecode optimizer.") - ( - g_argOptimizeRuns.c_str(), - po::value()->value_name("n")->default_value(200), - "Set for how many contract runs to optimize." - "Lower values will optimize more for initial deployment cost, higher values will optimize more for high-frequency usage." - ) - (g_strOptimizeYul.c_str(), "Legacy option, ignored. Use the general --optimize to enable Yul optimizer.") - (g_strNoOptimizeYul.c_str(), "Disable Yul optimizer in Solidity.") (g_argPrettyJson.c_str(), "Output JSON in pretty format. Currently it only works with the combined JSON output.") ( g_argLibraries.c_str(), @@ -834,6 +825,18 @@ Allowed options)", (g_argOldReporter.c_str(), "Enables old diagnostics reporter.") (g_argErrorRecovery.c_str(), "Enables additional parser error recovery.") (g_argIgnoreMissingFiles.c_str(), "Ignore missing files."); + po::options_description optimizerOptions("Optimizer options"); + optimizerOptions.add_options() + (g_argOptimize.c_str(), "Enable bytecode optimizer.") + ( + g_argOptimizeRuns.c_str(), + po::value()->value_name("n")->default_value(200), + "Set for how many contract runs to optimize." + "Lower values will optimize more for initial deployment cost, higher values will optimize more for high-frequency usage." + ) + (g_strOptimizeYul.c_str(), "Legacy option, ignored. Use the general --optimize to enable Yul optimizer.") + (g_strNoOptimizeYul.c_str(), "Disable Yul optimizer in Solidity."); + desc.add(optimizerOptions); po::options_description outputComponents("Output Components"); outputComponents.add_options() (g_argAstJson.c_str(), "AST of all source files in JSON format.") From 56af85ef39acc9a9ff64ce74167ab27d3df2a2e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 16 Apr 2020 18:20:28 +0200 Subject: [PATCH 046/126] command-line help: Minor whitespace and text tweaks --- solc/CommandLineInterface.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index 3c1445e449d4..79da752893f4 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -778,8 +778,8 @@ Allowed options)", ) ( g_argImportAst.c_str(), - "Import ASTs to be compiled, assumes input holds the AST in compact JSON format." - " Supported Inputs is the output of the standard-json or the one produced by --combined-json ast,compact-format" + "Import ASTs to be compiled, assumes input holds the AST in compact JSON format. " + "Supported Inputs is the output of the --standard-json or the one produced by --combined-json ast,compact-format" ) ( @@ -831,7 +831,7 @@ Allowed options)", ( g_argOptimizeRuns.c_str(), po::value()->value_name("n")->default_value(200), - "Set for how many contract runs to optimize." + "Set for how many contract runs to optimize. " "Lower values will optimize more for initial deployment cost, higher values will optimize more for high-frequency usage." ) (g_strOptimizeYul.c_str(), "Legacy option, ignored. Use the general --optimize to enable Yul optimizer.") From 3754a86ab203095c880e93aae3a12bf28be1fbcb Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Wed, 8 Apr 2020 17:08:49 -0500 Subject: [PATCH 047/126] Add support for interfaceID. --- Changelog.md | 2 +- docs/units-and-global-variables.rst | 9 +++ libsolidity/analysis/TypeChecker.cpp | 2 + libsolidity/analysis/ViewPureChecker.cpp | 1 + libsolidity/ast/AST.cpp | 16 +++-- libsolidity/ast/AST.h | 6 +- libsolidity/ast/Types.cpp | 4 +- libsolidity/codegen/ExpressionCompiler.cpp | 9 +++ .../codegen/ir/IRGeneratorForStatements.cpp | 13 ++++ .../semanticTests/interfaceID/homer.sol | 36 ++++++++++ .../interfaceID/homer_interfaceId.sol | 36 ++++++++++ .../interfaceID/interfaceId_events.sol | 21 ++++++ .../semanticTests/interfaceID/interfaces.sol | 67 +++++++++++++++++++ .../semanticTests/interfaceID/lisa.sol | 47 +++++++++++++ .../interfaceID/lisa_interfaceId.sol | 47 +++++++++++++ 15 files changed, 304 insertions(+), 12 deletions(-) create mode 100644 test/libsolidity/semanticTests/interfaceID/homer.sol create mode 100644 test/libsolidity/semanticTests/interfaceID/homer_interfaceId.sol create mode 100644 test/libsolidity/semanticTests/interfaceID/interfaceId_events.sol create mode 100644 test/libsolidity/semanticTests/interfaceID/interfaces.sol create mode 100644 test/libsolidity/semanticTests/interfaceID/lisa.sol create mode 100644 test/libsolidity/semanticTests/interfaceID/lisa_interfaceId.sol diff --git a/Changelog.md b/Changelog.md index 2ddb74055953..607a0f02b65c 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,11 +4,11 @@ Important Bugfixes: * Fix tuple assignments with components occupying multiple stack slots and different stack size on left- and right-hand-side. Language Features: + * Add support for EIP 165 interface identifiers with `type(I).interfaceId`. Compiler Features: - Bugfixes: * AST export: Export `immutable` property in the field `mutability`. * SMTChecker: Fix internal error in the CHC engine when calling inherited functions internally. diff --git a/docs/units-and-global-variables.rst b/docs/units-and-global-variables.rst index 6c625e997c2d..caf3bebacbe9 100644 --- a/docs/units-and-global-variables.rst +++ b/docs/units-and-global-variables.rst @@ -310,3 +310,12 @@ available for a contract type ``C``: regular calls. The same restrictions as with ``.creationCode`` also apply for this property. + +In addition to the properties above, the following properties are available +for an interface type ``I``: + +``type(I).interfaceId``: + A ``bytes4`` value containing the `EIP-165 `_ + interface identifier of the given interface ``I``. This identifier is defined as the ``XOR`` of all + function selectors defined within the interface itself - excluding all inherited functions. + diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index 2bdc706a2539..a252189dd38f 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -2617,6 +2617,8 @@ bool TypeChecker::visit(MemberAccess const& _memberAccess) } else if (magicType->kind() == MagicType::Kind::MetaType && memberName == "name") annotation.isPure = true; + else if (magicType->kind() == MagicType::Kind::MetaType && memberName == "interfaceId") + annotation.isPure = true; } return false; diff --git a/libsolidity/analysis/ViewPureChecker.cpp b/libsolidity/analysis/ViewPureChecker.cpp index 2468b03389a1..8fe35b1bc7c0 100644 --- a/libsolidity/analysis/ViewPureChecker.cpp +++ b/libsolidity/analysis/ViewPureChecker.cpp @@ -355,6 +355,7 @@ void ViewPureChecker::endVisit(MemberAccess const& _memberAccess) {MagicType::Kind::MetaType, "creationCode"}, {MagicType::Kind::MetaType, "runtimeCode"}, {MagicType::Kind::MetaType, "name"}, + {MagicType::Kind::MetaType, "interfaceId"}, }; set static const payableMembers{ {MagicType::Kind::Message, "value"} diff --git a/libsolidity/ast/AST.cpp b/libsolidity/ast/AST.cpp index d9425937f12e..740317b15436 100644 --- a/libsolidity/ast/AST.cpp +++ b/libsolidity/ast/AST.cpp @@ -96,9 +96,9 @@ bool ContractDefinition::derivesFrom(ContractDefinition const& _base) const return util::contains(annotation().linearizedBaseContracts, &_base); } -map, FunctionTypePointer> ContractDefinition::interfaceFunctions() const +map, FunctionTypePointer> ContractDefinition::interfaceFunctions(bool _includeInheritedFunctions) const { - auto exportedFunctionList = interfaceFunctionList(); + auto exportedFunctionList = interfaceFunctionList(_includeInheritedFunctions); map, FunctionTypePointer> exportedFunctions; for (auto const& it: exportedFunctionList) @@ -174,14 +174,16 @@ vector const& ContractDefinition::interfaceEvents() cons return *m_interfaceEvents; } -vector, FunctionTypePointer>> const& ContractDefinition::interfaceFunctionList() const +vector, FunctionTypePointer>> const& ContractDefinition::interfaceFunctionList(bool _includeInheritedFunctions) const { - if (!m_interfaceFunctionList) + if (!m_interfaceFunctionList[_includeInheritedFunctions]) { set signaturesSeen; - m_interfaceFunctionList = make_unique, FunctionTypePointer>>>(); + m_interfaceFunctionList[_includeInheritedFunctions] = make_unique, FunctionTypePointer>>>(); for (ContractDefinition const* contract: annotation().linearizedBaseContracts) { + if (_includeInheritedFunctions == false && contract != this) + continue; vector functions; for (FunctionDefinition const* f: contract->definedFunctions()) if (f->isPartOfExternalInterface()) @@ -199,12 +201,12 @@ vector, FunctionTypePointer>> const& ContractDefinition: { signaturesSeen.insert(functionSignature); util::FixedHash<4> hash(util::keccak256(functionSignature)); - m_interfaceFunctionList->emplace_back(hash, fun); + m_interfaceFunctionList[_includeInheritedFunctions]->emplace_back(hash, fun); } } } } - return *m_interfaceFunctionList; + return *m_interfaceFunctionList[_includeInheritedFunctions]; } TypePointer ContractDefinition::type() const diff --git a/libsolidity/ast/AST.h b/libsolidity/ast/AST.h index c0f7e6dc97c7..684c86906ea4 100644 --- a/libsolidity/ast/AST.h +++ b/libsolidity/ast/AST.h @@ -488,8 +488,8 @@ class ContractDefinition: public Declaration, public StructurallyDocumented /// @returns a map of canonical function signatures to FunctionDefinitions /// as intended for use by the ABI. - std::map, FunctionTypePointer> interfaceFunctions() const; - std::vector, FunctionTypePointer>> const& interfaceFunctionList() const; + std::map, FunctionTypePointer> interfaceFunctions(bool _includeInheritedFunctions = true) const; + std::vector, FunctionTypePointer>> const& interfaceFunctionList(bool _includeInheritedFunctions = true) const; /// @returns a list of all declarations in this contract std::vector declarations() const { return filteredNodes(m_subNodes); } @@ -528,7 +528,7 @@ class ContractDefinition: public Declaration, public StructurallyDocumented ContractKind m_contractKind; bool m_abstract{false}; - mutable std::unique_ptr, FunctionTypePointer>>> m_interfaceFunctionList; + mutable std::unique_ptr, FunctionTypePointer>>> m_interfaceFunctionList[2]; mutable std::unique_ptr> m_interfaceEvents; }; diff --git a/libsolidity/ast/Types.cpp b/libsolidity/ast/Types.cpp index 2238e86f37b0..279666b7db3e 100644 --- a/libsolidity/ast/Types.cpp +++ b/libsolidity/ast/Types.cpp @@ -3719,7 +3719,9 @@ MemberList::MemberMap MagicType::nativeMembers(ContractDefinition const*) const {"name", TypeProvider::stringMemory()}, }); else - return {}; + return MemberList::MemberMap({ + {"interfaceId", TypeProvider::fixedBytes(4)}, + }); } } solAssert(false, "Unknown kind of magic."); diff --git a/libsolidity/codegen/ExpressionCompiler.cpp b/libsolidity/codegen/ExpressionCompiler.cpp index a02497561279..ab4e120c072a 100644 --- a/libsolidity/codegen/ExpressionCompiler.cpp +++ b/libsolidity/codegen/ExpressionCompiler.cpp @@ -1580,6 +1580,15 @@ bool ExpressionCompiler::visit(MemberAccess const& _memberAccess) m_context << Instruction::DUP1 << u256(32) << Instruction::ADD; utils().storeStringData(contract.name()); } + else if (member == "interfaceId") + { + TypePointer arg = dynamic_cast(*_memberAccess.expression().annotation().type).typeArgument(); + ContractDefinition const& contract = dynamic_cast(*arg).contractDefinition(); + uint64_t result{0}; + for (auto const& function: contract.interfaceFunctionList(false)) + result ^= fromBigEndian(function.first.ref()); + m_context << (u256{result} << (256 - 32)); + } else if ((set{"encode", "encodePacked", "encodeWithSelector", "encodeWithSignature", "decode"}).count(member)) { // no-op diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index 57c8cb152903..d14d587531a6 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -754,6 +754,10 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall) } break; } + case FunctionType::Kind::MetaType: + { + break; + } default: solUnimplemented("FunctionKind " + toString(static_cast(functionType->kind())) + " not yet implemented"); } @@ -904,6 +908,15 @@ void IRGeneratorForStatements::endVisit(MemberAccess const& _memberAccess) { solUnimplementedAssert(false, ""); } + else if (member == "interfaceId") + { + TypePointer arg = dynamic_cast(*_memberAccess.expression().annotation().type).typeArgument(); + ContractDefinition const& contract = dynamic_cast(*arg).contractDefinition(); + uint64_t result{0}; + for (auto const& function: contract.interfaceFunctionList(false)) + result ^= fromBigEndian(function.first.ref()); + define(_memberAccess) << formatNumber(u256{result} << (256 - 32)) << "\n"; + } else if (set{"encode", "encodePacked", "encodeWithSelector", "encodeWithSignature", "decode"}.count(member)) { // no-op diff --git a/test/libsolidity/semanticTests/interfaceID/homer.sol b/test/libsolidity/semanticTests/interfaceID/homer.sol new file mode 100644 index 000000000000..243cba0b4edd --- /dev/null +++ b/test/libsolidity/semanticTests/interfaceID/homer.sol @@ -0,0 +1,36 @@ +interface ERC165 { + /// @notice Query if a contract implements an interface + /// @param interfaceID The interface identifier, as specified in ERC-165 + /// @dev Interface identification is specified in ERC-165. This function + /// uses less than 30,000 gas. + /// @return `true` if the contract implements `interfaceID` and + /// `interfaceID` is not 0xffffffff, `false` otherwise + function supportsInterface(bytes4 interfaceID) external view returns (bool); +} + +interface Simpson { + function is2D() external returns (bool); + function skinColor() external returns (string memory); +} + +contract Homer is ERC165, Simpson { + function supportsInterface(bytes4 interfaceID) external view override returns (bool) { + return + interfaceID == this.supportsInterface.selector || // ERC165 + interfaceID == this.is2D.selector ^ this.skinColor.selector; // Simpson + } + + function is2D() external override returns (bool) { + return true; + } + + function skinColor() external override returns (string memory) { + return "yellow"; + } +} + +// ---- +// supportsInterface(bytes4): left(0x01ffc9a0) -> false +// supportsInterface(bytes4): left(0x01ffc9a7) -> true +// supportsInterface(bytes4): left(0x73b6b492) -> true +// supportsInterface(bytes4): left(0x70b6b492) -> false diff --git a/test/libsolidity/semanticTests/interfaceID/homer_interfaceId.sol b/test/libsolidity/semanticTests/interfaceID/homer_interfaceId.sol new file mode 100644 index 000000000000..d2fa2e821e59 --- /dev/null +++ b/test/libsolidity/semanticTests/interfaceID/homer_interfaceId.sol @@ -0,0 +1,36 @@ +interface ERC165 { + /// @notice Query if a contract implements an interface + /// @param interfaceID The interface identifier, as specified in ERC-165 + /// @dev Interface identification is specified in ERC-165. This function + /// uses less than 30,000 gas. + /// @return `true` if the contract implements `interfaceID` and + /// `interfaceID` is not 0xffffffff, `false` otherwise + function supportsInterface(bytes4 interfaceID) external view returns (bool); +} + +interface Simpson { + function is2D() external returns (bool); + function skinColor() external returns (string memory); +} + +contract Homer is ERC165, Simpson { + function supportsInterface(bytes4 interfaceID) external view override returns (bool) { + return + interfaceID == type(ERC165).interfaceId || + interfaceID == type(Simpson).interfaceId; + } + + function is2D() external override returns (bool) { + return true; + } + + function skinColor() external override returns (string memory) { + return "yellow"; + } +} + +// ---- +// supportsInterface(bytes4): left(0x01ffc9a0) -> false +// supportsInterface(bytes4): left(0x01ffc9a7) -> true +// supportsInterface(bytes4): left(0x73b6b492) -> true +// supportsInterface(bytes4): left(0x70b6b492) -> false diff --git a/test/libsolidity/semanticTests/interfaceID/interfaceId_events.sol b/test/libsolidity/semanticTests/interfaceID/interfaceId_events.sol new file mode 100644 index 000000000000..09cb6c0d4f76 --- /dev/null +++ b/test/libsolidity/semanticTests/interfaceID/interfaceId_events.sol @@ -0,0 +1,21 @@ +interface HelloWorld { + function hello() external pure; + function world(int) external pure; +} + +interface HelloWorldWithEvent { + event Event(); + function hello() external pure; + function world(int) external pure; +} + +contract Test { + bytes4 public hello_world = type(HelloWorld).interfaceId; + bytes4 public hello_world_with_event = type(HelloWorldWithEvent).interfaceId; +} + +// ==== +// compileViaYul: also +// ---- +// hello_world() -> left(0xc6be8b58) +// hello_world_with_event() -> left(0xc6be8b58) diff --git a/test/libsolidity/semanticTests/interfaceID/interfaces.sol b/test/libsolidity/semanticTests/interfaceID/interfaces.sol new file mode 100644 index 000000000000..ba63cfb12483 --- /dev/null +++ b/test/libsolidity/semanticTests/interfaceID/interfaces.sol @@ -0,0 +1,67 @@ +interface HelloWorld { + function hello() external pure; + function world(int) external pure; +} + +interface HelloWorldDerived is HelloWorld { + function other() external pure; +} + +interface ERC165 { + /// @notice Query if a contract implements an interface + /// @param interfaceID The interface identifier, as specified in ERC-165 + /// @dev Interface identification is specified in ERC-165. This function + /// uses less than 30,000 gas. + /// @return `true` if the contract implements `interfaceID` and + /// `interfaceID` is not 0xffffffff, `false` otherwise + function supportsInterface(bytes4 interfaceID) external view returns (bool); +} + +contract Test { + bytes4 public ghello_world_interfaceId = type(HelloWorld).interfaceId; + bytes4 public ERC165_interfaceId = type(ERC165).interfaceId; + + function hello() public pure returns (bytes4 data){ + HelloWorld i; + return i.hello.selector; + } + + function world() public pure returns (bytes4 data){ + HelloWorld i; + return i.world.selector; + } + + function hello_world() public pure returns (bytes4 data){ + // HelloWorld i; + // return i.hello.selector ^ i.world.selector; // = 0xc6be8b58 + return 0xc6be8b58; + } + + function hello_world_interfaceId() public pure returns (bytes4 data){ + return type(HelloWorld).interfaceId; + } + + function other() public pure returns (bytes4 data){ + HelloWorldDerived i; + return i.other.selector; + } + + function hello_world_derived_interfaceId() public pure returns (bytes4 data){ + return type(HelloWorldDerived).interfaceId; + } +} + +// ==== +// compileViaYul: also +// ---- +// hello() -> left(0x19ff1d21) +// world() -> left(0xdf419679) +// +// ERC165_interfaceId() -> left(0x01ffc9a7) +// +// hello_world() -> left(0xc6be8b58) +// hello_world_interfaceId() -> left(0xc6be8b58) +// ghello_world_interfaceId() -> left(0xc6be8b58) +// +// other() -> left(0x85295877) +// hello_world_derived_interfaceId() -> left(0x85295877) diff --git a/test/libsolidity/semanticTests/interfaceID/lisa.sol b/test/libsolidity/semanticTests/interfaceID/lisa.sol new file mode 100644 index 000000000000..ea844dd819b9 --- /dev/null +++ b/test/libsolidity/semanticTests/interfaceID/lisa.sol @@ -0,0 +1,47 @@ +interface ERC165 { + /// @notice Query if a contract implements an interface + /// @param interfaceID The interface identifier, as specified in ERC-165 + /// @dev Interface identification is specified in ERC-165. This function + /// uses less than 30,000 gas. + /// @return `true` if the contract implements `interfaceID` and + /// `interfaceID` is not 0xffffffff, `false` otherwise + function supportsInterface(bytes4 interfaceID) external view returns (bool); +} + +contract ERC165MappingImplementation is ERC165 { + /// @dev You must not set element 0xffffffff to true + mapping(bytes4 => bool) internal supportedInterfaces; + + constructor() internal { + supportedInterfaces[this.supportsInterface.selector] = true; + } + + function supportsInterface(bytes4 interfaceID) external view override returns (bool) { + return supportedInterfaces[interfaceID]; + } +} + +interface Simpson { + function is2D() external returns (bool); + function skinColor() external returns (string memory); +} + +contract Lisa is ERC165MappingImplementation, Simpson { + constructor() public { + supportedInterfaces[this.is2D.selector ^ this.skinColor.selector] = true; + } + + function is2D() external override returns (bool) { + return true; + } + + function skinColor() external override returns (string memory) { + return "yellow"; + } +} + +// ---- +// supportsInterface(bytes4): left(0x01ffc9a0) -> false +// supportsInterface(bytes4): left(0x01ffc9a7) -> true +// supportsInterface(bytes4): left(0x73b6b492) -> true +// supportsInterface(bytes4): left(0x70b6b492) -> false diff --git a/test/libsolidity/semanticTests/interfaceID/lisa_interfaceId.sol b/test/libsolidity/semanticTests/interfaceID/lisa_interfaceId.sol new file mode 100644 index 000000000000..9c9dd5eb7fc7 --- /dev/null +++ b/test/libsolidity/semanticTests/interfaceID/lisa_interfaceId.sol @@ -0,0 +1,47 @@ +interface ERC165 { + /// @notice Query if a contract implements an interface + /// @param interfaceID The interface identifier, as specified in ERC-165 + /// @dev Interface identification is specified in ERC-165. This function + /// uses less than 30,000 gas. + /// @return `true` if the contract implements `interfaceID` and + /// `interfaceID` is not 0xffffffff, `false` otherwise + function supportsInterface(bytes4 interfaceID) external view returns (bool); +} + +contract ERC165MappingImplementation is ERC165 { + /// @dev You must not set element 0xffffffff to true + mapping(bytes4 => bool) internal supportedInterfaces; + + constructor() internal { + supportedInterfaces[this.supportsInterface.selector] = true; + } + + function supportsInterface(bytes4 interfaceID) external view override returns (bool) { + return supportedInterfaces[interfaceID]; + } +} + +interface Simpson { + function is2D() external returns (bool); + function skinColor() external returns (string memory); +} + +contract Lisa is ERC165MappingImplementation, Simpson { + constructor() public { + supportedInterfaces[type(Simpson).interfaceId] = true; + } + + function is2D() external override returns (bool) { + return true; + } + + function skinColor() external override returns (string memory) { + return "yellow"; + } +} + +// ---- +// supportsInterface(bytes4): left(0x01ffc9a0) -> false +// supportsInterface(bytes4): left(0x01ffc9a7) -> true +// supportsInterface(bytes4): left(0x73b6b492) -> true +// supportsInterface(bytes4): left(0x70b6b492) -> false From 45f22e3ff4f7a2e1bcb6e44a9cd8227f5841006a Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Thu, 16 Apr 2020 09:30:24 +0200 Subject: [PATCH 048/126] Add functional map and fold generic functions --- libsolidity/formal/CHC.cpp | 87 +++++++++++++++----------------------- libsolutil/CommonData.h | 31 ++++++++++++++ 2 files changed, 66 insertions(+), 52 deletions(-) diff --git a/libsolidity/formal/CHC.cpp b/libsolidity/formal/CHC.cpp index 5067e1396156..9d8bf16d4899 100644 --- a/libsolidity/formal/CHC.cpp +++ b/libsolidity/formal/CHC.cpp @@ -31,6 +31,7 @@ using namespace std; using namespace solidity; +using namespace solidity::util; using namespace solidity::langutil; using namespace solidity::frontend; @@ -643,19 +644,19 @@ set CHC::transactionAssertions(ASTNode const* vector CHC::stateVariablesIncludingInheritedAndPrivate(ContractDefinition const& _contract) { - vector stateVars; - for (auto const& contract: _contract.annotation().linearizedBaseContracts) - for (auto var: contract->stateVariables()) - stateVars.push_back(var); - return stateVars; + return fold( + _contract.annotation().linearizedBaseContracts, + vector{}, + [](auto&& _acc, auto _contract) { return _acc + _contract->stateVariables(); } + ); } vector CHC::stateSorts(ContractDefinition const& _contract) { - vector stateSorts; - for (auto const& var: stateVariablesIncludingInheritedAndPrivate(_contract)) - stateSorts.push_back(smt::smtSortAbstractFunction(*var->type())); - return stateSorts; + return applyMap( + stateVariablesIncludingInheritedAndPrivate(_contract), + [](auto _var) { return smt::smtSortAbstractFunction(*_var->type()); } + ); } smt::SortPointer CHC::constructorSort() @@ -695,12 +696,9 @@ smt::SortPointer CHC::interfaceSort(ContractDefinition const& _contract) /// - 1 set of output variables smt::SortPointer CHC::sort(FunctionDefinition const& _function) { - vector inputSorts; - for (auto const& var: _function.parameters()) - inputSorts.push_back(smt::smtSortAbstractFunction(*var->type())); - vector outputSorts; - for (auto const& var: _function.returnParameters()) - outputSorts.push_back(smt::smtSortAbstractFunction(*var->type())); + auto smtSort = [](auto _var) { return smt::smtSortAbstractFunction(*_var->type()); }; + auto inputSorts = applyMap(_function.parameters(), smtSort); + auto outputSorts = applyMap(_function.returnParameters(), smtSort); return make_shared( vector{smt::SortProvider::intSort} + m_stateSorts + inputSorts + m_stateSorts + inputSorts + outputSorts, smt::SortProvider::boolSort @@ -715,11 +713,9 @@ smt::SortPointer CHC::sort(ASTNode const* _node) auto fSort = dynamic_pointer_cast(sort(*m_currentFunction)); solAssert(fSort, ""); - vector varSorts; - for (auto const& var: m_currentFunction->localVariables()) - varSorts.push_back(smt::smtSortAbstractFunction(*var->type())); + auto smtSort = [](auto _var) { return smt::smtSortAbstractFunction(*_var->type()); }; return make_shared( - fSort->domain + varSorts, + fSort->domain + applyMap(m_currentFunction->localVariables(), smtSort), smt::SortProvider::boolSort ); } @@ -729,11 +725,9 @@ smt::SortPointer CHC::summarySort(FunctionDefinition const& _function, ContractD auto stateVariables = stateVariablesIncludingInheritedAndPrivate(_contract); auto sorts = stateSorts(_contract); - vector inputSorts, outputSorts; - for (auto const& var: _function.parameters()) - inputSorts.push_back(smt::smtSortAbstractFunction(*var->type())); - for (auto const& var: _function.returnParameters()) - outputSorts.push_back(smt::smtSortAbstractFunction(*var->type())); + auto smtSort = [](auto _var) { return smt::smtSortAbstractFunction(*_var->type()); }; + auto inputSorts = applyMap(_function.parameters(), smtSort); + auto outputSorts = applyMap(_function.returnParameters(), smtSort); return make_shared( vector{smt::SortProvider::intSort} + sorts + inputSorts + sorts + outputSorts, smt::SortProvider::boolSort @@ -769,9 +763,10 @@ void CHC::defineInterfacesAndSummaries(SourceUnit const& _source) smt::Expression CHC::interface() { - vector paramExprs; - for (auto const& var: m_stateVariables) - paramExprs.push_back(m_context.variable(*var)->currentValue()); + auto paramExprs = applyMap( + m_stateVariables, + [this](auto _var) { return m_context.variable(*_var)->currentValue(); } + ); return (*m_interfaces.at(m_currentContract))(paramExprs); } @@ -803,11 +798,9 @@ smt::Expression CHC::summary(FunctionDefinition const& _function) vector args{m_error.currentValue()}; auto contract = _function.annotation().contract; args += contract->isLibrary() ? stateVariablesAtIndex(0, *contract) : initialStateVariables(); - for (auto const& var: _function.parameters()) - args.push_back(m_context.variable(*var)->valueAtIndex(0)); + args += applyMap(_function.parameters(), [this](auto _var) { return valueAtIndex(*_var, 0); }); args += contract->isLibrary() ? stateVariablesAtIndex(1, *contract) : currentStateVariables(); - for (auto const& var: _function.returnParameters()) - args.push_back(m_context.variable(*var)->currentValue()); + args += applyMap(_function.returnParameters(), [this](auto _var) { return currentValue(*_var); }); return (*m_summaries.at(m_currentContract).at(&_function))(args); } @@ -854,27 +847,21 @@ vector CHC::initialStateVariables() vector CHC::stateVariablesAtIndex(int _index) { solAssert(m_currentContract, ""); - vector exprs; - for (auto const& var: m_stateVariables) - exprs.push_back(m_context.variable(*var)->valueAtIndex(_index)); - return exprs; + return applyMap(m_stateVariables, [&](auto _var) { return valueAtIndex(*_var, _index); }); } vector CHC::stateVariablesAtIndex(int _index, ContractDefinition const& _contract) { - vector exprs; - for (auto const& var: stateVariablesIncludingInheritedAndPrivate(_contract)) - exprs.push_back(m_context.variable(*var)->valueAtIndex(_index)); - return exprs; + return applyMap( + stateVariablesIncludingInheritedAndPrivate(_contract), + [&](auto _var) { return valueAtIndex(*_var, _index); } + ); } vector CHC::currentStateVariables() { solAssert(m_currentContract, ""); - vector exprs; - for (auto const& var: m_stateVariables) - exprs.push_back(m_context.variable(*var)->currentValue()); - return exprs; + return applyMap(m_stateVariables, [this](auto _var) { return currentValue(*_var); }); } vector CHC::currentFunctionVariables() @@ -886,9 +873,7 @@ vector CHC::currentFunctionVariables() initInputExprs.push_back(m_context.variable(*var)->valueAtIndex(0)); mutableInputExprs.push_back(m_context.variable(*var)->currentValue()); } - vector returnExprs; - for (auto const& var: m_currentFunction->returnParameters()) - returnExprs.push_back(m_context.variable(*var)->currentValue()); + auto returnExprs = applyMap(m_currentFunction->returnParameters(), [this](auto _var) { return currentValue(*_var); }); return vector{m_error.currentValue()} + initialStateVariables() + initInputExprs + @@ -899,11 +884,10 @@ vector CHC::currentFunctionVariables() vector CHC::currentBlockVariables() { - vector paramExprs; if (m_currentFunction) - for (auto const& var: m_currentFunction->localVariables()) - paramExprs.push_back(m_context.variable(*var)->currentValue()); - return currentFunctionVariables() + paramExprs; + return currentFunctionVariables() + applyMap(m_currentFunction->localVariables(), [this](auto _var) { return currentValue(*_var); }); + + return currentFunctionVariables(); } string CHC::predicateName(ASTNode const* _node, ContractDefinition const* _contract) @@ -958,8 +942,7 @@ smt::Expression CHC::predicate(FunctionCall const& _funCall) m_context.variable(*param)->increaseIndex(); else createVariable(*param); - for (auto const& var: function->returnParameters()) - args.push_back(m_context.variable(*var)->currentValue()); + args += applyMap(function->returnParameters(), [this](auto _var) { return currentValue(*_var); }); if (contract->isLibrary()) return (*m_summaries.at(contract).at(function))(args); diff --git a/libsolutil/CommonData.h b/libsolutil/CommonData.h index feed0925f295..bd1133511dfb 100644 --- a/libsolutil/CommonData.h +++ b/libsolutil/CommonData.h @@ -34,6 +34,7 @@ #include #include #include +#include /// Operators need to stay in the global namespace. @@ -141,6 +142,36 @@ inline std::multiset& operator-=(std::multiset& _a, C const& _b) namespace solidity::util { +/// Functional map. +/// Returns a container _oc applying @param _op to each element in @param _c. +/// By default _oc is a vector. +/// If another return type is desired, an empty contained of that type +/// is given as @param _oc. +template())) +>>> +auto applyMap(Container const& _c, Callable&& _op, OutputContainer _oc = OutputContainer{}) +{ + std::transform(std::begin(_c), std::end(_c), std::inserter(_oc, std::end(_oc)), _op); + return _oc; +} + +/// Functional fold. +/// Given a container @param _c, an initial value @param _acc, +/// and a binary operator @param _binaryOp(T, U), accumulate +/// the elements of _c over _acc. +/// Note that has a similar function `accumulate` which +/// until C++20 does *not* std::move the partial accumulated. +template +auto fold(C const& _c, T _acc, Callable&& _binaryOp) +{ + for (auto const& e: _c) + _acc = _binaryOp(std::move(_acc), e); + return _acc; +} + template T convertContainer(U const& _from) { From 1f28f79ae609fc36cc1758fbb0fb0bae475ea28c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 10 Apr 2020 12:07:53 +0200 Subject: [PATCH 049/126] IRGenerator::generate(): Remove duplicate call to setMostDerivedContract() - The same call happens in resetContext(_contract) called immediately before --- libsolidity/codegen/ir/IRGenerator.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libsolidity/codegen/ir/IRGenerator.cpp b/libsolidity/codegen/ir/IRGenerator.cpp index 6d8d15f61984..97a284c8c197 100644 --- a/libsolidity/codegen/ir/IRGenerator.cpp +++ b/libsolidity/codegen/ir/IRGenerator.cpp @@ -110,7 +110,6 @@ string IRGenerator::generate(ContractDefinition const& _contract) t("functions", m_context.functionCollector().requestedFunctions()); resetContext(_contract); - m_context.setMostDerivedContract(_contract); t("RuntimeObject", runtimeObjectName(_contract)); t("dispatch", dispatchRoutine(_contract)); for (auto const* contract: _contract.annotation().linearizedBaseContracts) From c7947c1af689984a462122ed6452bec39f1a2ec7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 10 Apr 2020 19:16:44 +0200 Subject: [PATCH 050/126] Implement lazy function generation using function generation queue --- .../codegen/MultiUseYulFunctionCollector.cpp | 1 + .../codegen/MultiUseYulFunctionCollector.h | 5 +++ .../codegen/ir/IRGenerationContext.cpp | 26 ++++++++++++--- libsolidity/codegen/ir/IRGenerationContext.h | 20 ++++++++++- libsolidity/codegen/ir/IRGenerator.cpp | 30 +++++++++-------- libsolidity/codegen/ir/IRGenerator.h | 3 ++ .../codegen/ir/IRGeneratorForStatements.cpp | 5 ++- .../output.json | 2 -- .../standard_ir_requested/output.json | 5 --- .../yul_string_format_ascii/output.json | 29 ---------------- .../output.json | 17 ---------- .../output.json | 29 ---------------- .../yul_string_format_ascii_long/output.json | 33 ------------------- .../yul_string_format_hex/output.json | 29 ---------------- .../internal_virtual_function_calls.sol | 21 ++++++++++++ 15 files changed, 91 insertions(+), 164 deletions(-) create mode 100644 test/libsolidity/semanticTests/virtualFunctions/internal_virtual_function_calls.sol diff --git a/libsolidity/codegen/MultiUseYulFunctionCollector.cpp b/libsolidity/codegen/MultiUseYulFunctionCollector.cpp index 2be3d29ae6b3..e8994ac04bc3 100644 --- a/libsolidity/codegen/MultiUseYulFunctionCollector.cpp +++ b/libsolidity/codegen/MultiUseYulFunctionCollector.cpp @@ -34,6 +34,7 @@ string MultiUseYulFunctionCollector::requestedFunctions() { string result; for (auto const& f: m_requestedFunctions) + // std::map guarantees ascending order when iterating through its keys. result += f.second; m_requestedFunctions.clear(); return result; diff --git a/libsolidity/codegen/MultiUseYulFunctionCollector.h b/libsolidity/codegen/MultiUseYulFunctionCollector.h index d839a31be873..031d08bdef36 100644 --- a/libsolidity/codegen/MultiUseYulFunctionCollector.h +++ b/libsolidity/codegen/MultiUseYulFunctionCollector.h @@ -41,10 +41,15 @@ class MultiUseYulFunctionCollector std::string createFunction(std::string const& _name, std::function const& _creator); /// @returns concatenation of all generated functions. + /// Guarantees that the order of functions in the generated code is deterministic and + /// platform-independent. /// Clears the internal list, i.e. calling it again will result in an /// empty return value. std::string requestedFunctions(); + /// @returns true IFF a function with the specified name has already been collected. + bool contains(std::string const& _name) const { return m_requestedFunctions.count(_name) > 0; } + private: /// Map from function name to code for a multi-use function. std::map m_requestedFunctions; diff --git a/libsolidity/codegen/ir/IRGenerationContext.cpp b/libsolidity/codegen/ir/IRGenerationContext.cpp index 45cb1ce14c18..c3b896ee114b 100644 --- a/libsolidity/codegen/ir/IRGenerationContext.cpp +++ b/libsolidity/codegen/ir/IRGenerationContext.cpp @@ -32,6 +32,25 @@ using namespace solidity; using namespace solidity::util; using namespace solidity::frontend; +string IRGenerationContext::enqueueFunctionForCodeGeneration(FunctionDefinition const& _function) +{ + string name = functionName(_function); + + if (!m_functions.contains(name)) + m_functionGenerationQueue.insert(&_function); + + return name; +} + +FunctionDefinition const* IRGenerationContext::dequeueFunctionForCodeGeneration() +{ + solAssert(!m_functionGenerationQueue.empty(), ""); + + FunctionDefinition const* result = *m_functionGenerationQueue.begin(); + m_functionGenerationQueue.erase(m_functionGenerationQueue.begin()); + return result; +} + ContractDefinition const& IRGenerationContext::mostDerivedContract() const { solAssert(m_mostDerivedContract, "Most derived contract requested but not set."); @@ -77,11 +96,6 @@ string IRGenerationContext::functionName(VariableDeclaration const& _varDecl) return "getter_fun_" + _varDecl.name() + "_" + to_string(_varDecl.id()); } -string IRGenerationContext::virtualFunctionName(FunctionDefinition const& _functionDeclaration) -{ - return functionName(_functionDeclaration.resolveVirtual(mostDerivedContract())); -} - string IRGenerationContext::newYulVariable() { return "_" + to_string(++m_varCounter); @@ -137,6 +151,8 @@ string IRGenerationContext::internalDispatch(size_t _in, size_t _out) { "funID", to_string(function->id()) }, { "name", functionName(*function)} }); + + enqueueFunctionForCodeGeneration(*function); } templ("cases", move(functions)); return templ.render(); diff --git a/libsolidity/codegen/ir/IRGenerationContext.h b/libsolidity/codegen/ir/IRGenerationContext.h index 0dac6d65a5a4..acbe6250ba43 100644 --- a/libsolidity/codegen/ir/IRGenerationContext.h +++ b/libsolidity/codegen/ir/IRGenerationContext.h @@ -30,6 +30,7 @@ #include +#include #include #include #include @@ -61,6 +62,15 @@ class IRGenerationContext MultiUseYulFunctionCollector& functionCollector() { return m_functions; } + /// Adds a Solidity function to the function generation queue and returns the name of the + /// corresponding Yul function. + std::string enqueueFunctionForCodeGeneration(FunctionDefinition const& _function); + + /// Pops one item from the function generation queue. Must not be called if the queue is empty. + FunctionDefinition const* dequeueFunctionForCodeGeneration(); + + bool functionGenerationQueueEmpty() { return m_functionGenerationQueue.empty(); } + /// Sets the most derived contract (the one currently being compiled)> void setMostDerivedContract(ContractDefinition const& _mostDerivedContract) { @@ -82,7 +92,6 @@ class IRGenerationContext std::string functionName(FunctionDefinition const& _function); std::string functionName(VariableDeclaration const& _varDecl); - std::string virtualFunctionName(FunctionDefinition const& _functionDeclaration); std::string newYulVariable(); @@ -113,6 +122,15 @@ class IRGenerationContext std::map> m_stateVariables; MultiUseYulFunctionCollector m_functions; size_t m_varCounter = 0; + + /// Function definitions queued for code generation. They're the Solidity functions whose calls + /// were discovered by the IR generator during AST traversal. + /// Note that the queue gets filled in a lazy way - new definitions can be added while the + /// collected ones get removed and traversed. + /// The order and duplicates are irrelevant here (hence std::set rather than std::queue) as + /// long as the order of Yul functions in the generated code is deterministic and the same on + /// all platforms - which is a property guaranteed by MultiUseYulFunctionCollector. + std::set m_functionGenerationQueue; }; } diff --git a/libsolidity/codegen/ir/IRGenerator.cpp b/libsolidity/codegen/ir/IRGenerator.cpp index 97a284c8c197..7499d3909e57 100644 --- a/libsolidity/codegen/ir/IRGenerator.cpp +++ b/libsolidity/codegen/ir/IRGenerator.cpp @@ -101,20 +101,13 @@ string IRGenerator::generate(ContractDefinition const& _contract) t("memoryInit", memoryInit()); t("constructor", constructorCode(_contract)); t("deploy", deployCode(_contract)); - // We generate code for all functions and rely on the optimizer to remove them again - // TODO it would probably be better to only generate functions when internalDispatch or - // virtualFunctionName is called - same below. - for (auto const* contract: _contract.annotation().linearizedBaseContracts) - for (auto const* fun: contract->definedFunctions()) - generateFunction(*fun); + generateQueuedFunctions(); t("functions", m_context.functionCollector().requestedFunctions()); resetContext(_contract); t("RuntimeObject", runtimeObjectName(_contract)); t("dispatch", dispatchRoutine(_contract)); - for (auto const* contract: _contract.annotation().linearizedBaseContracts) - for (auto const* fun: contract->definedFunctions()) - generateFunction(*fun); + generateQueuedFunctions(); t("runtimeFunctions", m_context.functionCollector().requestedFunctions()); return t.render(); } @@ -126,6 +119,13 @@ string IRGenerator::generate(Block const& _block) return generator.code(); } +void IRGenerator::generateQueuedFunctions() +{ + while (!m_context.functionGenerationQueueEmpty()) + // NOTE: generateFunction() may modify function generation queue + generateFunction(*m_context.dequeueFunctionForCodeGeneration()); +} + string IRGenerator::generateFunction(FunctionDefinition const& _function) { string functionName = m_context.functionName(_function); @@ -290,7 +290,7 @@ string IRGenerator::constructorCode(ContractDefinition const& _contract) t("assignToParams", paramVars == 0 ? "" : "let " + suffixedVariableNameList("param_", 0, paramVars) + " := "); t("params", suffixedVariableNameList("param_", 0, paramVars)); t("abiDecode", abiFunctions.tupleDecoder(constructor->functionType(false)->parameterTypes(), true)); - t("constructorName", m_context.functionName(*constructor)); + t("constructorName", m_context.enqueueFunctionForCodeGeneration(*constructor)); out << t.render(); } @@ -369,7 +369,7 @@ string IRGenerator::dispatchRoutine(ContractDefinition const& _contract) templ["retParams"] = suffixedVariableNameList("ret_", retVars, 0); if (FunctionDefinition const* funDef = dynamic_cast(&type->declaration())) - templ["function"] = generateFunction(*funDef); + templ["function"] = m_context.enqueueFunctionForCodeGeneration(*funDef); else if (VariableDeclaration const* varDecl = dynamic_cast(&type->declaration())) templ["function"] = generateGetter(*varDecl); else @@ -385,14 +385,14 @@ string IRGenerator::dispatchRoutine(ContractDefinition const& _contract) string fallbackCode; if (!fallback->isPayable()) fallbackCode += callValueCheck(); - fallbackCode += generateFunction(*fallback) + "() stop()"; + fallbackCode += m_context.enqueueFunctionForCodeGeneration(*fallback) + "() stop()"; t("fallback", fallbackCode); } else t("fallback", "revert(0, 0)"); if (FunctionDefinition const* etherReceiver = _contract.receiveFunction()) - t("receiveEther", generateFunction(*etherReceiver) + "() stop()"); + t("receiveEther", m_context.enqueueFunctionForCodeGeneration(*etherReceiver) + "() stop()"); else t("receiveEther", ""); return t.render(); @@ -412,6 +412,10 @@ string IRGenerator::memoryInit() void IRGenerator::resetContext(ContractDefinition const& _contract) { + solAssert( + m_context.functionGenerationQueueEmpty(), + "Reset function generation queue while it still had functions." + ); solAssert( m_context.functionCollector().requestedFunctions().empty(), "Reset context while it still had functions." diff --git a/libsolidity/codegen/ir/IRGenerator.h b/libsolidity/codegen/ir/IRGenerator.h index e0c1dcd4f9e2..c4035141ce3c 100644 --- a/libsolidity/codegen/ir/IRGenerator.h +++ b/libsolidity/codegen/ir/IRGenerator.h @@ -56,6 +56,9 @@ class IRGenerator std::string generate(ContractDefinition const& _contract); std::string generate(Block const& _block); + /// Generates code for all the functions from the function generation queue. + /// The resulting code is stored in the function collector in IRGenerationContext. + void generateQueuedFunctions(); /// Generates code for and returns the name of the function. std::string generateFunction(FunctionDefinition const& _function); /// Generates a getter for the given declaration and returns its name diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index 2a9c6a9ea888..b72ddd9768f3 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -577,7 +577,9 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall) if (auto functionDef = dynamic_cast(identifier->annotation().referencedDeclaration)) { define(_functionCall) << - m_context.virtualFunctionName(*functionDef) << + m_context.enqueueFunctionForCodeGeneration( + functionDef->resolveVirtual(m_context.mostDerivedContract()) + ) << "(" << joinHumanReadable(args) << ")\n"; @@ -586,6 +588,7 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall) } define(_functionCall) << + // NOTE: internalDispatch() takes care of adding the function to function generation queue m_context.internalDispatch( TupleType(functionType->parameterTypes()).sizeOnStack(), TupleType(functionType->returnParameterTypes()).sizeOnStack() diff --git a/test/cmdlineTests/standard_irOptimized_requested/output.json b/test/cmdlineTests/standard_irOptimized_requested/output.json index 87e73dee7905..78ebe59c3186 100644 --- a/test/cmdlineTests/standard_irOptimized_requested/output.json +++ b/test/cmdlineTests/standard_irOptimized_requested/output.json @@ -10,8 +10,6 @@ object \"C_6\" { mstore(64, 128) codecopy(0, dataoffset(\"C_6_deployed\"), datasize(\"C_6_deployed\")) return(0, datasize(\"C_6_deployed\")) - function fun_f_5() - { } } object \"C_6_deployed\" { code { diff --git a/test/cmdlineTests/standard_ir_requested/output.json b/test/cmdlineTests/standard_ir_requested/output.json index cb89a6911d0b..2ceee9da0258 100644 --- a/test/cmdlineTests/standard_ir_requested/output.json +++ b/test/cmdlineTests/standard_ir_requested/output.json @@ -18,11 +18,6 @@ object \"C_6\" { return(0, datasize(\"C_6_deployed\")) - function fun_f_5() { - - - } - } object \"C_6_deployed\" { code { diff --git a/test/cmdlineTests/yul_string_format_ascii/output.json b/test/cmdlineTests/yul_string_format_ascii/output.json index 24a2a10aa056..0e88aa5aa4cb 100644 --- a/test/cmdlineTests/yul_string_format_ascii/output.json +++ b/test/cmdlineTests/yul_string_format_ascii/output.json @@ -18,35 +18,6 @@ object \"C_10\" { return(0, datasize(\"C_10_deployed\")) - function allocateMemory(size) -> memPtr { - memPtr := mload(64) - let newFreePtr := add(memPtr, size) - // protect against overflow - if or(gt(newFreePtr, 0xffffffffffffffff), lt(newFreePtr, memPtr)) { revert(0, 0) } - mstore(64, newFreePtr) - } - - function convert_t_stringliteral_9f0adad0a59b05d2e04a1373342b10b9eb16c57c164c8a3bfcbf46dccee39a21_to_t_string_memory_ptr() -> converted { - converted := allocateMemory(64) - mstore(converted, 6) - - mstore(add(converted, 32), \"abcabc\") - - } - - function fun_f_9() -> vloc__4_mpos { - let zero_value_for_type_t_string_memory_ptr_1_mpos := zero_value_for_split_t_string_memory_ptr() - vloc__4_mpos := zero_value_for_type_t_string_memory_ptr_1_mpos - - vloc__4_mpos := convert_t_stringliteral_9f0adad0a59b05d2e04a1373342b10b9eb16c57c164c8a3bfcbf46dccee39a21_to_t_string_memory_ptr() - leave - - } - - function zero_value_for_split_t_string_memory_ptr() -> ret { - ret := 96 - } - } object \"C_10_deployed\" { code { diff --git a/test/cmdlineTests/yul_string_format_ascii_bytes32/output.json b/test/cmdlineTests/yul_string_format_ascii_bytes32/output.json index 8028e5dd6e08..256b1e4e14a8 100644 --- a/test/cmdlineTests/yul_string_format_ascii_bytes32/output.json +++ b/test/cmdlineTests/yul_string_format_ascii_bytes32/output.json @@ -18,23 +18,6 @@ object \"C_10\" { return(0, datasize(\"C_10_deployed\")) - function convert_t_stringliteral_9f0adad0a59b05d2e04a1373342b10b9eb16c57c164c8a3bfcbf46dccee39a21_to_t_bytes32() -> converted { - converted := 0x6162636162630000000000000000000000000000000000000000000000000000 - } - - function fun_f_9() -> vloc__4 { - let zero_value_for_type_t_bytes32_1 := zero_value_for_split_t_bytes32() - vloc__4 := zero_value_for_type_t_bytes32_1 - - vloc__4 := convert_t_stringliteral_9f0adad0a59b05d2e04a1373342b10b9eb16c57c164c8a3bfcbf46dccee39a21_to_t_bytes32() - leave - - } - - function zero_value_for_split_t_bytes32() -> ret { - ret := 0 - } - } object \"C_10_deployed\" { code { diff --git a/test/cmdlineTests/yul_string_format_ascii_bytes32_from_number/output.json b/test/cmdlineTests/yul_string_format_ascii_bytes32_from_number/output.json index a9588d145575..abb929c1923f 100644 --- a/test/cmdlineTests/yul_string_format_ascii_bytes32_from_number/output.json +++ b/test/cmdlineTests/yul_string_format_ascii_bytes32_from_number/output.json @@ -18,35 +18,6 @@ object \"C_10\" { return(0, datasize(\"C_10_deployed\")) - function cleanup_t_rational_1633837924_by_1(value) -> cleaned { - cleaned := value - } - - function convert_t_rational_1633837924_by_1_to_t_bytes4(value) -> converted { - converted := shift_left_224(cleanup_t_rational_1633837924_by_1(value)) - } - - function fun_f_9() -> vloc__4 { - let zero_value_for_type_t_bytes4_1 := zero_value_for_split_t_bytes4() - vloc__4 := zero_value_for_type_t_bytes4_1 - - let expr_6 := 0x61626364 - vloc__4 := convert_t_rational_1633837924_by_1_to_t_bytes4(expr_6) - leave - - } - - function shift_left_224(value) -> newValue { - newValue := - - shl(224, value) - - } - - function zero_value_for_split_t_bytes4() -> ret { - ret := 0 - } - } object \"C_10_deployed\" { code { diff --git a/test/cmdlineTests/yul_string_format_ascii_long/output.json b/test/cmdlineTests/yul_string_format_ascii_long/output.json index 552712154dcc..991f9d0fd4b3 100644 --- a/test/cmdlineTests/yul_string_format_ascii_long/output.json +++ b/test/cmdlineTests/yul_string_format_ascii_long/output.json @@ -18,39 +18,6 @@ object \"C_10\" { return(0, datasize(\"C_10_deployed\")) - function allocateMemory(size) -> memPtr { - memPtr := mload(64) - let newFreePtr := add(memPtr, size) - // protect against overflow - if or(gt(newFreePtr, 0xffffffffffffffff), lt(newFreePtr, memPtr)) { revert(0, 0) } - mstore(64, newFreePtr) - } - - function convert_t_stringliteral_d6604f85ac07e2b33103a620b3d3d75b0473c7214912beded67b9b624d41c571_to_t_string_memory_ptr() -> converted { - converted := allocateMemory(128) - mstore(converted, 85) - - mstore(add(converted, 32), \"abcdabcdcafecafeabcdabcdcafecafe\") - - mstore(add(converted, 64), \"ffffzzzzoooo0123456789,.<,>.?:;'\") - - mstore(add(converted, 96), \"[{]}|`~!@#$%^&*()-_=+\") - - } - - function fun_f_9() -> vloc__4_mpos { - let zero_value_for_type_t_string_memory_ptr_1_mpos := zero_value_for_split_t_string_memory_ptr() - vloc__4_mpos := zero_value_for_type_t_string_memory_ptr_1_mpos - - vloc__4_mpos := convert_t_stringliteral_d6604f85ac07e2b33103a620b3d3d75b0473c7214912beded67b9b624d41c571_to_t_string_memory_ptr() - leave - - } - - function zero_value_for_split_t_string_memory_ptr() -> ret { - ret := 96 - } - } object \"C_10_deployed\" { code { diff --git a/test/cmdlineTests/yul_string_format_hex/output.json b/test/cmdlineTests/yul_string_format_hex/output.json index 48c1146faeac..ff84b1c323db 100644 --- a/test/cmdlineTests/yul_string_format_hex/output.json +++ b/test/cmdlineTests/yul_string_format_hex/output.json @@ -18,35 +18,6 @@ object \"C_10\" { return(0, datasize(\"C_10_deployed\")) - function cleanup_t_rational_2864434397_by_1(value) -> cleaned { - cleaned := value - } - - function convert_t_rational_2864434397_by_1_to_t_bytes4(value) -> converted { - converted := shift_left_224(cleanup_t_rational_2864434397_by_1(value)) - } - - function fun_f_9() -> vloc__4 { - let zero_value_for_type_t_bytes4_1 := zero_value_for_split_t_bytes4() - vloc__4 := zero_value_for_type_t_bytes4_1 - - let expr_6 := 0xaabbccdd - vloc__4 := convert_t_rational_2864434397_by_1_to_t_bytes4(expr_6) - leave - - } - - function shift_left_224(value) -> newValue { - newValue := - - shl(224, value) - - } - - function zero_value_for_split_t_bytes4() -> ret { - ret := 0 - } - } object \"C_10_deployed\" { code { diff --git a/test/libsolidity/semanticTests/virtualFunctions/internal_virtual_function_calls.sol b/test/libsolidity/semanticTests/virtualFunctions/internal_virtual_function_calls.sol new file mode 100644 index 000000000000..b06a89c4663c --- /dev/null +++ b/test/libsolidity/semanticTests/virtualFunctions/internal_virtual_function_calls.sol @@ -0,0 +1,21 @@ +contract Base { + function f() public returns (uint256 i) { + return g(); + } + + function g() internal virtual returns (uint256 i) { + return 1; + } +} + + +contract Derived is Base { + function g() internal override returns (uint256 i) { + return 2; + } +} + +// ==== +// compileViaYul: also +// ---- +// f() -> 2 From 3180e6c5a2eab45085a7544d8abee5b28a79f49e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 17 Apr 2020 15:20:37 +0200 Subject: [PATCH 051/126] cmdlineTests.sh: Fix missing details in some error messages - As far as I know $STDERR is not a standard shell variable and it's not defined in the file either. --- test/cmdlineTests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index 9001c02f4817..37f2e326cca3 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -214,7 +214,7 @@ printTask "Testing unknown options..." then echo "Passed" else - printError "Incorrect response to unknown options: $STDERR" + printError "Incorrect response to unknown options: $output" exit 1 fi ) @@ -385,7 +385,7 @@ SOLTMPDIR=$(mktemp -d) # This should fail if [[ !("$output" =~ "No input files given") || ($result == 0) ]] then - printError "Incorrect response to empty input arg list: $STDERR" + printError "Incorrect response to empty input arg list: $output" exit 1 fi From a80b03208133023b3d2de1d32d3ce5a653f54104 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 17 Apr 2020 14:32:38 +0200 Subject: [PATCH 052/126] Fix inconsistent indentation in scripts/ - NOT reindenting all files. Just choosing one style in files that were using multiple. --- scripts/ASTImportTest.sh | 14 +++--- scripts/build_emscripten.sh | 4 +- scripts/docs_version_pragma_check.sh | 2 +- scripts/install_deps.cmake | 6 +-- scripts/isolate_tests.py | 36 +++++++-------- scripts/release.bat | 2 +- scripts/release_ppa.sh | 10 ++--- scripts/report_errors.sh | 20 ++++----- scripts/soltest.sh | 2 +- scripts/tests.sh | 66 ++++++++++++++-------------- 10 files changed, 81 insertions(+), 81 deletions(-) diff --git a/scripts/ASTImportTest.sh b/scripts/ASTImportTest.sh index b3a1b6724e8c..c23b3dc346a7 100755 --- a/scripts/ASTImportTest.sh +++ b/scripts/ASTImportTest.sh @@ -21,8 +21,8 @@ UNCOMPILABLE=0 TESTED=0 if [ $(ls | wc -l) -ne 0 ]; then - echo "Test directory not empty. Skipping!" - exit -1 + echo "Test directory not empty. Skipping!" + exit -1 fi # function tests whether exporting and importing again leaves the JSON ast unchanged @@ -40,11 +40,11 @@ function testImportExportEquivalence { $SOLC --import-ast --combined-json ast,compact-format --pretty-json expected.json > obtained.json 2> /dev/null if [ $? -ne 0 ] then - # For investigating, use exit 1 here so the script stops at the - # first failing test - # exit 1 - FAILED=$((FAILED + 1)) - return 1 + # For investigating, use exit 1 here so the script stops at the + # first failing test + # exit 1 + FAILED=$((FAILED + 1)) + return 1 fi DIFF="$(diff expected.json obtained.json)" if [ "$DIFF" != "" ] diff --git a/scripts/build_emscripten.sh b/scripts/build_emscripten.sh index b457b741d801..cf6cbdd790b9 100755 --- a/scripts/build_emscripten.sh +++ b/scripts/build_emscripten.sh @@ -29,9 +29,9 @@ set -e if test -z "$1"; then - BUILD_DIR="emscripten_build" + BUILD_DIR="emscripten_build" else - BUILD_DIR="$1" + BUILD_DIR="$1" fi docker run -v $(pwd):/root/project -w /root/project trzeci/emscripten:sdk-tag-1.39.3-64bit \ diff --git a/scripts/docs_version_pragma_check.sh b/scripts/docs_version_pragma_check.sh index c56bdc6c98c7..67accc68da73 100755 --- a/scripts/docs_version_pragma_check.sh +++ b/scripts/docs_version_pragma_check.sh @@ -184,4 +184,4 @@ SOLTMPDIR=$(mktemp -d) done ) rm -rf "$SOLTMPDIR" -echo "Done." \ No newline at end of file +echo "Done." diff --git a/scripts/install_deps.cmake b/scripts/install_deps.cmake index 0cb0ed6211b2..d90d4ec32b0d 100644 --- a/scripts/install_deps.cmake +++ b/scripts/install_deps.cmake @@ -43,7 +43,7 @@ function(download_and_unpack PACKAGE_URL DST_DIR) if (STATUS) message("Unpacking ${FILE_NAME} to ${DST_DIR}") execute_process(COMMAND ${CMAKE_COMMAND} -E tar -xf ${DST_FILE} - WORKING_DIRECTORY ${DST_DIR}) + WORKING_DIRECTORY ${DST_DIR}) endif() endfunction(download_and_unpack) @@ -59,8 +59,8 @@ function(create_package NAME DIR) set(PACKAGE_FILE "${PACKAGES_DIR}/${NAME}.tar.gz") execute_process(COMMAND ${CMAKE_COMMAND} -E - tar -czf ${PACKAGE_FILE} ${TOP_FILES} - WORKING_DIRECTORY ${DIR}) + tar -czf ${PACKAGE_FILE} ${TOP_FILES} + WORKING_DIRECTORY ${DIR}) endfunction(create_package) # Downloads the source code of the package and unpacks it to dedicated 'src' diff --git a/scripts/isolate_tests.py b/scripts/isolate_tests.py index 3a41d54c9099..9dea81aa56e8 100755 --- a/scripts/isolate_tests.py +++ b/scripts/isolate_tests.py @@ -20,17 +20,17 @@ def extract_test_cases(path): tests = [] for l in lines: - if inside: - if l.strip().endswith(')' + delimiter + '";'): - inside = False + if inside: + if l.strip().endswith(')' + delimiter + '";'): + inside = False + else: + tests[-1] += l + '\n' else: - tests[-1] += l + '\n' - else: - m = re.search(r'R"([^(]*)\($', l.strip()) - if m: - inside = True - delimiter = m.group(1) - tests += [''] + m = re.search(r'R"([^(]*)\($', l.strip()) + if m: + inside = True + delimiter = m.group(1) + tests += [''] return tests @@ -75,20 +75,20 @@ def write_cases(f, tests): open(sol_filename, mode='w', encoding='utf8').write(remainder) def extract_and_write(f, path): - if docs: - cases = extract_docs_cases(path) + if docs: + cases = extract_docs_cases(path) + else: + if f.endswith('.sol'): + cases = [open(path, mode='r', encoding='utf8').read()] else: - if f.endswith('.sol'): - cases = [open(path, mode='r', encoding='utf8').read()] - else: - cases = extract_test_cases(path) - write_cases(f, cases) + cases = extract_test_cases(path) + write_cases(f, cases) if __name__ == '__main__': path = sys.argv[1] docs = False if len(sys.argv) > 2 and sys.argv[2] == 'docs': - docs = True + docs = True if isfile(path): extract_and_write(path, path) diff --git a/scripts/release.bat b/scripts/release.bat index cfd0c838e074..caa56fc9a561 100644 --- a/scripts/release.bat +++ b/scripts/release.bat @@ -31,7 +31,7 @@ set VERSION=%2 set "DLLS=MSVC_DLLS_NOT_FOUND" FOR /d %%d IN ("C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Redist\MSVC\*" - "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Redist\MSVC\*") DO set "DLLS=%%d\x86\Microsoft.VC141.CRT\msvc*.dll" + "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Redist\MSVC\*") DO set "DLLS=%%d\x86\Microsoft.VC141.CRT\msvc*.dll" 7z a solidity-windows.zip ^ .\build\solc\%CONFIGURATION%\solc.exe .\build\test\%CONFIGURATION%\soltest.exe ^ diff --git a/scripts/release_ppa.sh b/scripts/release_ppa.sh index 49634a977365..d84b3fd3653f 100755 --- a/scripts/release_ppa.sh +++ b/scripts/release_ppa.sh @@ -86,16 +86,16 @@ else if [ $distribution = focal ] then SMTDEPENDENCY="libz3-dev, - libcvc4-dev, - " + libcvc4-dev, + " elif [ $distribution = disco ] then SMTDEPENDENCY="libz3-static-dev, - libcvc4-dev, - " + libcvc4-dev, + " else SMTDEPENDENCY="libz3-static-dev, - " + " fi CMAKE_OPTIONS="" fi diff --git a/scripts/report_errors.sh b/scripts/report_errors.sh index 18a3c266d8ef..296feb84b40f 100755 --- a/scripts/report_errors.sh +++ b/scripts/report_errors.sh @@ -39,11 +39,11 @@ function post_error_to_github FORMATTED_ERROR_MSG=$(echo $ESCAPED_ERROR_MSG | sed 's/\$/\\n/g' | tr -d '\n') curl --request POST \ - --url $GITHUB_API_URL \ - --header 'accept: application/vnd.github.v3+json' \ - --header 'content-type: application/json' \ - -u stackenbotten:$GITHUB_ACCESS_TOKEN \ - --data "{\"body\": \"There was an error when running \`$CIRCLE_JOB\` for commit \`$CIRCLE_SHA1\`:\n\`\`\`\n$FORMATTED_ERROR_MSG\n\`\`\`\nPlease check that your changes are working as intended.\"}" + --url $GITHUB_API_URL \ + --header 'accept: application/vnd.github.v3+json' \ + --header 'content-type: application/json' \ + -u stackenbotten:$GITHUB_ACCESS_TOKEN \ + --data "{\"body\": \"There was an error when running \`$CIRCLE_JOB\` for commit \`$CIRCLE_SHA1\`:\n\`\`\`\n$FORMATTED_ERROR_MSG\n\`\`\`\nPlease check that your changes are working as intended.\"}" post_review_comment_to_github } @@ -60,11 +60,11 @@ function post_review_comment_to_github ERROR_LINE=$(echo $line | grep -oE "[0-9]*") curl --request POST \ - --url $GITHUB_API_URL \ - --header 'accept: application/vnd.github.v3+json, application/vnd.github.comfort-fade-preview+json' \ - --header 'content-type: application/json' \ - -u stackenbotten:$GITHUB_ACCESS_TOKEN \ - --data "{\"commit_id\": \"$CIRCLE_SHA1\", \"path\": \"$ERROR_PATH\", \"line\": $ERROR_LINE, \"side\": \"RIGHT\", \"body\": \"Coding style error\"}" + --url $GITHUB_API_URL \ + --header 'accept: application/vnd.github.v3+json, application/vnd.github.comfort-fade-preview+json' \ + --header 'content-type: application/json' \ + -u stackenbotten:$GITHUB_ACCESS_TOKEN \ + --data "{\"commit_id\": \"$CIRCLE_SHA1\", \"path\": \"$ERROR_PATH\", \"line\": $ERROR_LINE, \"side\": \"RIGHT\", \"body\": \"Coding style error\"}" done < $ERROR_LOG } diff --git a/scripts/soltest.sh b/scripts/soltest.sh index d1c41404ff47..4ae08d5a4add 100755 --- a/scripts/soltest.sh +++ b/scripts/soltest.sh @@ -9,7 +9,7 @@ SOLTEST_OPTIONS= SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-build} usage() { - echo 2>&1 " + echo 2>&1 " Usage: $0 [options] [soltest-options] Runs BOOST C++ unit test program, soltest. diff --git a/scripts/tests.sh b/scripts/tests.sh index 266c3184ebd0..17da957d2732 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -88,43 +88,43 @@ fi # and homestead / byzantium VM for optimize in "" "--optimize" do - for vm in $EVM_VERSIONS - do - FORCE_ABIV2_RUNS="no" - if [[ "$vm" == "istanbul" ]] - then - FORCE_ABIV2_RUNS="no yes" # run both in istanbul - fi - for abiv2 in $FORCE_ABIV2_RUNS + for vm in $EVM_VERSIONS do - force_abiv2_flag="" - if [[ "$abiv2" == "yes" ]] - then - force_abiv2_flag="--abiencoderv2 --optimize-yul" - fi - printTask "--> Running tests using "$optimize" --evm-version "$vm" $force_abiv2_flag..." - - log="" - if [ -n "$log_directory" ] + FORCE_ABIV2_RUNS="no" + if [[ "$vm" == "istanbul" ]] then - if [ -n "$optimize" ] - then - log=--logger=JUNIT,error,$log_directory/opt_$vm.xml $testargs - else - log=--logger=JUNIT,error,$log_directory/noopt_$vm.xml $testargs_no_opt - fi + FORCE_ABIV2_RUNS="no yes" # run both in istanbul fi - - set +e - "$REPO_ROOT"/${SOLIDITY_BUILD_DIR}/test/soltest --show-progress $log -- --testpath "$REPO_ROOT"/test "$optimize" --evm-version "$vm" $SMT_FLAGS $force_abiv2_flag - - if test "0" -ne "$?"; then - exit 1 - fi - set -e - + for abiv2 in $FORCE_ABIV2_RUNS + do + force_abiv2_flag="" + if [[ "$abiv2" == "yes" ]] + then + force_abiv2_flag="--abiencoderv2 --optimize-yul" + fi + printTask "--> Running tests using "$optimize" --evm-version "$vm" $force_abiv2_flag..." + + log="" + if [ -n "$log_directory" ] + then + if [ -n "$optimize" ] + then + log=--logger=JUNIT,error,$log_directory/opt_$vm.xml $testargs + else + log=--logger=JUNIT,error,$log_directory/noopt_$vm.xml $testargs_no_opt + fi + fi + + set +e + "$REPO_ROOT"/${SOLIDITY_BUILD_DIR}/test/soltest --show-progress $log -- --testpath "$REPO_ROOT"/test "$optimize" --evm-version "$vm" $SMT_FLAGS $force_abiv2_flag + + if test "0" -ne "$?"; then + exit 1 + fi + set -e + + done done - done done if [[ -n $CMDLINE_PID ]] && ! wait $CMDLINE_PID From 3e65bcfd7febd180bf83d4bd91ab625ed5d69ad6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 17 Apr 2020 15:02:40 +0200 Subject: [PATCH 053/126] Allow SOLIDITY_BUILD_DIR outside of REPO_ROOT in scripts that respect this variable --- Changelog.md | 3 +++ scripts/ASTImportTest.sh | 4 ++-- scripts/docs_version_pragma_check.sh | 2 +- scripts/soltest.sh | 6 +++--- scripts/tests.sh | 4 ++-- test/cmdlineTests.sh | 10 +++++----- 6 files changed, 16 insertions(+), 13 deletions(-) diff --git a/Changelog.md b/Changelog.md index cdc3ac010701..303fc0a23c80 100644 --- a/Changelog.md +++ b/Changelog.md @@ -9,6 +9,9 @@ Compiler Features: Bugfixes: * Type Checker: Disallow ``virtual`` and ``override`` for constructors. +Build System: + * soltest.sh: ``SOLIDITY_BUILD_DIR`` is no longer relative to ``REPO_ROOT`` to allow for build directories outside of the source tree. + ### 0.6.6 (2020-04-09) diff --git a/scripts/ASTImportTest.sh b/scripts/ASTImportTest.sh index c23b3dc346a7..6f8d8b06e1ca 100755 --- a/scripts/ASTImportTest.sh +++ b/scripts/ASTImportTest.sh @@ -5,8 +5,8 @@ # and exporting it again. The second JSON should be identical to the first REPO_ROOT=$(readlink -f "$(dirname "$0")"/..) -SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-build} -SOLC=${REPO_ROOT}/${SOLIDITY_BUILD_DIR}/solc/solc +SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-${REPO_ROOT}/build} +SOLC=${SOLIDITY_BUILD_DIR}/solc/solc SPLITSOURCES=${REPO_ROOT}/scripts/splitSources.py SYNTAXTESTS_DIR="${REPO_ROOT}/test/libsolidity/syntaxTests" diff --git a/scripts/docs_version_pragma_check.sh b/scripts/docs_version_pragma_check.sh index 67accc68da73..520be3e5e53c 100755 --- a/scripts/docs_version_pragma_check.sh +++ b/scripts/docs_version_pragma_check.sh @@ -28,7 +28,7 @@ set -e ## GLOBAL VARIABLES REPO_ROOT=$(cd $(dirname "$0")/.. && pwd) -SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-build} +SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-${REPO_ROOT}/build} source "${REPO_ROOT}/scripts/common.sh" source "${REPO_ROOT}/scripts/common_cmdline.sh" diff --git a/scripts/soltest.sh b/scripts/soltest.sh index 4ae08d5a4add..bb4c48691bd8 100755 --- a/scripts/soltest.sh +++ b/scripts/soltest.sh @@ -6,7 +6,7 @@ USE_DEBUGGER=0 DEBUGGER="gdb --args" BOOST_OPTIONS= SOLTEST_OPTIONS= -SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-build} +SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-${REPO_ROOT}/build} usage() { echo 2>&1 " @@ -23,7 +23,7 @@ Options: Important environment variables: -SOLIDITY_BUILD_DIR: Sets directory under the repository root of where test/soltest should be found. +SOLIDITY_BUILD_DIR: Sets directory where test/soltest should be found. The default is \"${SOLIDITY_BUILD_DIR}\". " } @@ -64,4 +64,4 @@ if [ "$USE_DEBUGGER" -ne "0" ]; then DEBUG_PREFIX=${DEBUGGER} fi -exec ${DEBUG_PREFIX} ${REPO_ROOT}/${SOLIDITY_BUILD_DIR}/test/soltest ${BOOST_OPTIONS} -- --testpath ${REPO_ROOT}/test ${SOLTEST_OPTIONS} +exec ${DEBUG_PREFIX} ${SOLIDITY_BUILD_DIR}/test/soltest ${BOOST_OPTIONS} -- --testpath ${REPO_ROOT}/test ${SOLTEST_OPTIONS} diff --git a/scripts/tests.sh b/scripts/tests.sh index 17da957d2732..c0601a12da0c 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -29,7 +29,7 @@ set -e REPO_ROOT="$(dirname "$0")/.." -SOLIDITY_BUILD_DIR="${SOLIDITY_BUILD_DIR:-build}" +SOLIDITY_BUILD_DIR="${SOLIDITY_BUILD_DIR:-${REPO_ROOT}/build}" source "${REPO_ROOT}/scripts/common.sh" @@ -116,7 +116,7 @@ do fi set +e - "$REPO_ROOT"/${SOLIDITY_BUILD_DIR}/test/soltest --show-progress $log -- --testpath "$REPO_ROOT"/test "$optimize" --evm-version "$vm" $SMT_FLAGS $force_abiv2_flag + "${SOLIDITY_BUILD_DIR}"/test/soltest --show-progress $log -- --testpath "$REPO_ROOT"/test "$optimize" --evm-version "$vm" $SMT_FLAGS $force_abiv2_flag if test "0" -ne "$?"; then exit 1 diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index 37f2e326cca3..e99969808a9c 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -31,19 +31,19 @@ set -e ## GLOBAL VARIABLES REPO_ROOT=$(cd $(dirname "$0")/.. && pwd) -SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-build} +SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-${REPO_ROOT}/build} source "${REPO_ROOT}/scripts/common.sh" source "${REPO_ROOT}/scripts/common_cmdline.sh" case "$OSTYPE" in msys) - SOLC="$REPO_ROOT/${SOLIDITY_BUILD_DIR}/solc/Release/solc.exe" + SOLC="${SOLIDITY_BUILD_DIR}/solc/Release/solc.exe" # prevents msys2 path translation for a remapping test export MSYS2_ARG_CONV_EXCL="=" ;; *) - SOLC="$REPO_ROOT/${SOLIDITY_BUILD_DIR}/solc/solc" + SOLC="${SOLIDITY_BUILD_DIR}/solc/solc" ;; esac echo "${SOLC}" @@ -431,8 +431,8 @@ SOLTMPDIR=$(mktemp -d) "$REPO_ROOT"/scripts/isolate_tests.py "$REPO_ROOT"/test/ "$REPO_ROOT"/scripts/isolate_tests.py "$REPO_ROOT"/docs/ docs - echo *.sol | xargs -P 4 -n 50 "$REPO_ROOT"/${SOLIDITY_BUILD_DIR}/test/tools/solfuzzer --quiet --input-files - echo *.sol | xargs -P 4 -n 50 "$REPO_ROOT"/${SOLIDITY_BUILD_DIR}/test/tools/solfuzzer --without-optimizer --quiet --input-files + echo *.sol | xargs -P 4 -n 50 "${SOLIDITY_BUILD_DIR}/test/tools/solfuzzer" --quiet --input-files + echo *.sol | xargs -P 4 -n 50 "${SOLIDITY_BUILD_DIR}/test/tools/solfuzzer" --without-optimizer --quiet --input-files ) rm -rf "$SOLTMPDIR" From 4760b8589de0803b12dc26c36d5f6213565e2cd1 Mon Sep 17 00:00:00 2001 From: hrkrshnn Date: Thu, 9 Apr 2020 16:18:57 +0530 Subject: [PATCH 054/126] Replaced all instances of lValueRequested to willBeWrittenTo --- libsolidity/analysis/ControlFlowBuilder.cpp | 2 +- libsolidity/analysis/ImmutableValidator.cpp | 2 +- libsolidity/analysis/TypeChecker.cpp | 6 +++--- libsolidity/analysis/ViewPureChecker.cpp | 8 ++++---- libsolidity/ast/ASTAnnotations.h | 2 +- libsolidity/ast/ASTJsonConverter.cpp | 2 +- libsolidity/codegen/ExpressionCompiler.cpp | 6 +++--- libsolidity/codegen/ExpressionCompiler.h | 2 +- .../codegen/ir/IRGeneratorForStatements.cpp | 14 +++++++------- libsolidity/codegen/ir/IRGeneratorForStatements.h | 2 +- libsolidity/formal/SMTEncoder.cpp | 4 ++-- libsolidity/formal/VariableUsage.cpp | 6 +++--- 12 files changed, 28 insertions(+), 28 deletions(-) diff --git a/libsolidity/analysis/ControlFlowBuilder.cpp b/libsolidity/analysis/ControlFlowBuilder.cpp index f4b6b5d5fdee..a0c557a3ef14 100644 --- a/libsolidity/analysis/ControlFlowBuilder.cpp +++ b/libsolidity/analysis/ControlFlowBuilder.cpp @@ -591,7 +591,7 @@ bool ControlFlowBuilder::visit(Identifier const& _identifier) if (auto const* variableDeclaration = dynamic_cast(_identifier.annotation().referencedDeclaration)) m_currentNode->variableOccurrences.emplace_back( *variableDeclaration, - static_cast(_identifier).annotation().lValueRequested ? + static_cast(_identifier).annotation().willBeWrittenTo ? VariableOccurrence::Kind::Assignment : VariableOccurrence::Kind::Access, _identifier.location() diff --git a/libsolidity/analysis/ImmutableValidator.cpp b/libsolidity/analysis/ImmutableValidator.cpp index 6e0d0f6c796b..b03d314ebdb2 100644 --- a/libsolidity/analysis/ImmutableValidator.cpp +++ b/libsolidity/analysis/ImmutableValidator.cpp @@ -160,7 +160,7 @@ void ImmutableValidator::analyseVariableReference(VariableDeclaration const& _va if (!_variableReference.isStateVariable() || !_variableReference.immutable()) return; - if (_expression.annotation().lValueRequested && _expression.annotation().lValueOfOrdinaryAssignment) + if (_expression.annotation().willBeWrittenTo && _expression.annotation().lValueOfOrdinaryAssignment) { if (!m_currentConstructor) m_errorReporter.typeError( diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index 30842a11725e..033461c7f58d 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -1303,7 +1303,7 @@ bool TypeChecker::visit(Conditional const& _conditional) _conditional.trueExpression().annotation().isPure && _conditional.falseExpression().annotation().isPure; - if (_conditional.annotation().lValueRequested) + if (_conditional.annotation().willBeWrittenTo) m_errorReporter.typeError( _conditional.location(), "Conditional expression as left value is not supported yet." @@ -1399,7 +1399,7 @@ bool TypeChecker::visit(TupleExpression const& _tuple) vector> const& components = _tuple.components(); TypePointers types; - if (_tuple.annotation().lValueRequested) + if (_tuple.annotation().willBeWrittenTo) { if (_tuple.isInlineArray()) m_errorReporter.fatalTypeError(_tuple.location(), "Inline array type cannot be declared as LValue."); @@ -3050,7 +3050,7 @@ bool TypeChecker::expectType(Expression const& _expression, Type const& _expecte void TypeChecker::requireLValue(Expression const& _expression, bool _ordinaryAssignment) { - _expression.annotation().lValueRequested = true; + _expression.annotation().willBeWrittenTo = true; _expression.annotation().lValueOfOrdinaryAssignment = _ordinaryAssignment; _expression.accept(*this); diff --git a/libsolidity/analysis/ViewPureChecker.cpp b/libsolidity/analysis/ViewPureChecker.cpp index 57eeb429c713..5381fd16672a 100644 --- a/libsolidity/analysis/ViewPureChecker.cpp +++ b/libsolidity/analysis/ViewPureChecker.cpp @@ -197,7 +197,7 @@ void ViewPureChecker::endVisit(Identifier const& _identifier) StateMutability mutability = StateMutability::Pure; - bool writes = _identifier.annotation().lValueRequested; + bool writes = _identifier.annotation().willBeWrittenTo; if (VariableDeclaration const* varDecl = dynamic_cast(declaration)) { if (varDecl->isStateVariable() && !varDecl->isConstant()) @@ -332,7 +332,7 @@ bool ViewPureChecker::visit(MemberAccess const& _memberAccess) void ViewPureChecker::endVisit(MemberAccess const& _memberAccess) { StateMutability mutability = StateMutability::Pure; - bool writes = _memberAccess.annotation().lValueRequested; + bool writes = _memberAccess.annotation().willBeWrittenTo; ASTString const& member = _memberAccess.memberName(); switch (_memberAccess.expression().annotation().type->category()) @@ -402,7 +402,7 @@ void ViewPureChecker::endVisit(IndexAccess const& _indexAccess) solAssert(_indexAccess.annotation().type->category() == Type::Category::TypeType, ""); else { - bool writes = _indexAccess.annotation().lValueRequested; + bool writes = _indexAccess.annotation().willBeWrittenTo; if (_indexAccess.baseExpression().annotation().type->dataStoredIn(DataLocation::Storage)) reportMutability(writes ? StateMutability::NonPayable : StateMutability::View, _indexAccess.location()); } @@ -410,7 +410,7 @@ void ViewPureChecker::endVisit(IndexAccess const& _indexAccess) void ViewPureChecker::endVisit(IndexRangeAccess const& _indexRangeAccess) { - bool writes = _indexRangeAccess.annotation().lValueRequested; + bool writes = _indexRangeAccess.annotation().willBeWrittenTo; if (_indexRangeAccess.baseExpression().annotation().type->dataStoredIn(DataLocation::Storage)) reportMutability(writes ? StateMutability::NonPayable : StateMutability::View, _indexRangeAccess.location()); } diff --git a/libsolidity/ast/ASTAnnotations.h b/libsolidity/ast/ASTAnnotations.h index 9f7edf84ec93..492a8b68dace 100644 --- a/libsolidity/ast/ASTAnnotations.h +++ b/libsolidity/ast/ASTAnnotations.h @@ -234,7 +234,7 @@ struct ExpressionAnnotation: ASTAnnotation /// Whether it is an LValue (i.e. something that can be assigned to). bool isLValue = false; /// Whether the expression is used in a context where the LValue is actually required. - bool lValueRequested = false; + bool willBeWrittenTo = false; /// Whether the expression is an lvalue that is only assigned. /// Would be false for --, ++, delete, +=, -=, .... bool lValueOfOrdinaryAssignment = false; diff --git a/libsolidity/ast/ASTJsonConverter.cpp b/libsolidity/ast/ASTJsonConverter.cpp index 78f1dba7462a..c9d3e1389c93 100644 --- a/libsolidity/ast/ASTJsonConverter.cpp +++ b/libsolidity/ast/ASTJsonConverter.cpp @@ -182,7 +182,7 @@ void ASTJsonConverter::appendExpressionAttributes( make_pair("isConstant", _annotation.isConstant), make_pair("isPure", _annotation.isPure), make_pair("isLValue", _annotation.isLValue), - make_pair("lValueRequested", _annotation.lValueRequested), + make_pair("lValueRequested", _annotation.willBeWrittenTo), make_pair("argumentTypes", typePointerToJson(_annotation.arguments)) }; _attributes += exprAttributes; diff --git a/libsolidity/codegen/ExpressionCompiler.cpp b/libsolidity/codegen/ExpressionCompiler.cpp index 354a2409924f..b41950fef1af 100644 --- a/libsolidity/codegen/ExpressionCompiler.cpp +++ b/libsolidity/codegen/ExpressionCompiler.cpp @@ -348,15 +348,15 @@ bool ExpressionCompiler::visit(TupleExpression const& _tuple) if (component) { component->accept(*this); - if (_tuple.annotation().lValueRequested) + if (_tuple.annotation().willBeWrittenTo) { solAssert(!!m_currentLValue, ""); lvalues.push_back(move(m_currentLValue)); } } - else if (_tuple.annotation().lValueRequested) + else if (_tuple.annotation().willBeWrittenTo) lvalues.push_back(unique_ptr()); - if (_tuple.annotation().lValueRequested) + if (_tuple.annotation().willBeWrittenTo) { if (_tuple.components().size() == 1) m_currentLValue = move(lvalues[0]); diff --git a/libsolidity/codegen/ExpressionCompiler.h b/libsolidity/codegen/ExpressionCompiler.h index f75fbc8ab75b..0216b260bfed 100644 --- a/libsolidity/codegen/ExpressionCompiler.h +++ b/libsolidity/codegen/ExpressionCompiler.h @@ -148,7 +148,7 @@ void ExpressionCompiler::setLValue(Expression const& _expression, Arguments cons { solAssert(!m_currentLValue, "Current LValue not reset before trying to set new one."); std::unique_ptr lvalue = std::make_unique(m_context, _arguments...); - if (_expression.annotation().lValueRequested) + if (_expression.annotation().willBeWrittenTo) m_currentLValue = move(lvalue); else lvalue->retrieveValue(_expression.location(), true); diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index 2a9c6a9ea888..26527978c895 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -257,14 +257,14 @@ bool IRGeneratorForStatements::visit(TupleExpression const& _tuple) solUnimplementedAssert(false, ""); else { - bool lValueRequested = _tuple.annotation().lValueRequested; - if (lValueRequested) + bool willBeWrittenTo = _tuple.annotation().willBeWrittenTo; + if (willBeWrittenTo) solAssert(!m_currentLValue, ""); if (_tuple.components().size() == 1) { solAssert(_tuple.components().front(), ""); _tuple.components().front()->accept(*this); - if (lValueRequested) + if (willBeWrittenTo) solAssert(!!m_currentLValue, ""); else define(_tuple, *_tuple.components().front()); @@ -276,7 +276,7 @@ bool IRGeneratorForStatements::visit(TupleExpression const& _tuple) if (auto const& component = _tuple.components()[i]) { component->accept(*this); - if (lValueRequested) + if (willBeWrittenTo) { solAssert(!!m_currentLValue, ""); lvalues.emplace_back(std::move(m_currentLValue)); @@ -285,10 +285,10 @@ bool IRGeneratorForStatements::visit(TupleExpression const& _tuple) else define(IRVariable(_tuple).tupleComponent(i), *component); } - else if (lValueRequested) + else if (willBeWrittenTo) lvalues.emplace_back(); - if (_tuple.annotation().lValueRequested) + if (_tuple.annotation().willBeWrittenTo) m_currentLValue.emplace(IRLValue{ *_tuple.annotation().type, IRLValue::Tuple{std::move(lvalues)} @@ -1694,7 +1694,7 @@ void IRGeneratorForStatements::setLValue(Expression const& _expression, IRLValue { solAssert(!m_currentLValue, ""); - if (_expression.annotation().lValueRequested) + if (_expression.annotation().willBeWrittenTo) { m_currentLValue.emplace(std::move(_lvalue)); solAssert(!_lvalue.type.dataStoredIn(DataLocation::CallData), ""); diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.h b/libsolidity/codegen/ir/IRGeneratorForStatements.h index 553e81dcc1f2..810087a2d7f5 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.h +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.h @@ -139,7 +139,7 @@ class IRGeneratorForStatements: public ASTConstVisitor /// @returns a fresh IR variable containing the value of the lvalue @a _lvalue. IRVariable readFromLValue(IRLValue const& _lvalue); - /// Stores the given @a _lvalue in m_currentLValue, if it will be written to (lValueRequested). Otherwise + /// Stores the given @a _lvalue in m_currentLValue, if it will be written to (willBeWrittenTo). Otherwise /// defines the expression @a _expression by reading the value from @a _lvalue. void setLValue(Expression const& _expression, IRLValue _lvalue); void generateLoop( diff --git a/libsolidity/formal/SMTEncoder.cpp b/libsolidity/formal/SMTEncoder.cpp index 67a4885072a6..cbd7e931b762 100644 --- a/libsolidity/formal/SMTEncoder.cpp +++ b/libsolidity/formal/SMTEncoder.cpp @@ -467,7 +467,7 @@ void SMTEncoder::endVisit(UnaryOperation const& _op) { solAssert(smt::isInteger(_op.annotation().type->category()), ""); - solAssert(_op.subExpression().annotation().lValueRequested, ""); + solAssert(_op.subExpression().annotation().willBeWrittenTo, ""); if (auto identifier = dynamic_cast(&_op.subExpression())) { auto decl = identifierToVariable(*identifier); @@ -704,7 +704,7 @@ void SMTEncoder::visitGasLeft(FunctionCall const& _funCall) void SMTEncoder::endVisit(Identifier const& _identifier) { - if (_identifier.annotation().lValueRequested) + if (_identifier.annotation().willBeWrittenTo) { // Will be translated as part of the node that requested the lvalue. } diff --git a/libsolidity/formal/VariableUsage.cpp b/libsolidity/formal/VariableUsage.cpp index 62e60e3f7d2e..1871c09085fa 100644 --- a/libsolidity/formal/VariableUsage.cpp +++ b/libsolidity/formal/VariableUsage.cpp @@ -40,15 +40,15 @@ set VariableUsage::touchedVariables(ASTNode const& _ void VariableUsage::endVisit(Identifier const& _identifier) { - if (_identifier.annotation().lValueRequested) + if (_identifier.annotation().willBeWrittenTo) checkIdentifier(_identifier); } void VariableUsage::endVisit(IndexAccess const& _indexAccess) { - if (_indexAccess.annotation().lValueRequested) + if (_indexAccess.annotation().willBeWrittenTo) { - /// identifier.annotation().lValueRequested == false, that's why we + /// identifier.annotation().willBeWrittenTo == false, that's why we /// need to check that before. auto identifier = dynamic_cast(SMTEncoder::leftmostBase(_indexAccess)); if (identifier) From 5aa2107877b419b0f62d8c288658046686ea3dee Mon Sep 17 00:00:00 2001 From: chriseth Date: Mon, 20 Apr 2020 13:26:39 +0200 Subject: [PATCH 055/126] Changelog entries for declaration type checker. --- Changelog.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changelog.md b/Changelog.md index cdc3ac010701..e51b23e325c8 100644 --- a/Changelog.md +++ b/Changelog.md @@ -8,6 +8,8 @@ Compiler Features: Bugfixes: * Type Checker: Disallow ``virtual`` and ``override`` for constructors. + * Type Checker: Fix several internal errors by performing size and recursiveness checks of types before the full type checking. + * Type Checker: Perform recursiveness check on structs declared at the file level. From ea5b64ca9e8d9354ec55384c0f382b3ee2a7d43f Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Mon, 20 Apr 2020 14:28:38 +0200 Subject: [PATCH 056/126] Fuzzer: Permit linking of libraries in compilation framework --- test/tools/ossfuzz/abiV2FuzzerCommon.cpp | 7 +++++-- test/tools/ossfuzz/abiV2FuzzerCommon.h | 5 +++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/test/tools/ossfuzz/abiV2FuzzerCommon.cpp b/test/tools/ossfuzz/abiV2FuzzerCommon.cpp index f30a8517ab76..c9109ecdbbb7 100644 --- a/test/tools/ossfuzz/abiV2FuzzerCommon.cpp +++ b/test/tools/ossfuzz/abiV2FuzzerCommon.cpp @@ -9,13 +9,16 @@ SolidityCompilationFramework::SolidityCompilationFramework(langutil::EVMVersion solidity::bytes SolidityCompilationFramework::compileContract( std::string const& _sourceCode, - std::string const& _contractName + std::string const& _contractName, + std::map const& _libraryAddresses, + frontend::OptimiserSettings _optimization ) { std::string sourceCode = _sourceCode; m_compiler.setSources({{"", sourceCode}}); + m_compiler.setLibraries(_libraryAddresses); m_compiler.setEVMVersion(m_evmVersion); - m_compiler.setOptimiserSettings(m_optimiserSettings); + m_compiler.setOptimiserSettings(_optimization); if (!m_compiler.compile()) { langutil::SourceReferenceFormatter formatter(std::cerr); diff --git a/test/tools/ossfuzz/abiV2FuzzerCommon.h b/test/tools/ossfuzz/abiV2FuzzerCommon.h index 3094fbdc1b96..e18458b77662 100644 --- a/test/tools/ossfuzz/abiV2FuzzerCommon.h +++ b/test/tools/ossfuzz/abiV2FuzzerCommon.h @@ -23,12 +23,13 @@ class SolidityCompilationFramework } bytes compileContract( std::string const& _sourceCode, - std::string const& _contractName = {} + std::string const& _contractName, + std::map const& _libraryAddresses = {}, + frontend::OptimiserSettings _optimization = frontend::OptimiserSettings::minimal() ); protected: frontend::CompilerStack m_compiler; langutil::EVMVersion m_evmVersion; - frontend::OptimiserSettings m_optimiserSettings = frontend::OptimiserSettings::none(); }; } From 1ada2a52fb40a6cd4e8b4890243a419d79083a9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 11 Mar 2020 03:49:16 +0100 Subject: [PATCH 057/126] [yul-phaser] Mutations: Add two-point and uniform crossover operators --- test/yulPhaser/Mutations.cpp | 179 +++++++++++++++++++++++++++++++++- tools/yulPhaser/Mutations.cpp | 135 +++++++++++++++++++++++++ tools/yulPhaser/Mutations.h | 18 ++++ 3 files changed, 331 insertions(+), 1 deletion(-) diff --git a/test/yulPhaser/Mutations.cpp b/test/yulPhaser/Mutations.cpp index 33c623f84a56..27e57be08f11 100644 --- a/test/yulPhaser/Mutations.cpp +++ b/test/yulPhaser/Mutations.cpp @@ -18,15 +18,17 @@ #include #include - #include +#include + #include #include #include using namespace std; +using namespace solidity::util; namespace solidity::phaser::test { @@ -434,6 +436,181 @@ BOOST_AUTO_TEST_CASE(fixedPointCrossover_should_always_use_position_zero_as_spli BOOST_CHECK(crossover10(empty, splittable) == splittable); } +BOOST_AUTO_TEST_CASE(randomTwoPointCrossover_should_swap_chromosome_parts_between_two_random_points) +{ + function crossover = randomTwoPointCrossover(); + + SimulationRNG::reset(1); + Chromosome result1 = crossover(Chromosome("aaaaaaaaaa"), Chromosome("cccccc")); + BOOST_TEST(result1 == Chromosome("aaacccaaaa")); + + SimulationRNG::reset(1); + Chromosome result2 = crossover(Chromosome("cccccc"), Chromosome("aaaaaaaaaa")); + BOOST_TEST(result2 == Chromosome("cccaaa")); +} + +BOOST_AUTO_TEST_CASE(symmetricRandomTwoPointCrossover_should_swap_chromosome_parts_at_random_point) +{ + function crossover = symmetricRandomTwoPointCrossover(); + + SimulationRNG::reset(1); + tuple result1 = crossover(Chromosome("aaaaaaaaaa"), Chromosome("cccccc")); + tuple expectedPair1 = {Chromosome("aaacccaaaa"), Chromosome("cccaaa")}; + BOOST_TEST(result1 == expectedPair1); + + tuple result2 = crossover(Chromosome("cccccc"), Chromosome("aaaaaaaaaa")); + tuple expectedPair2 = {Chromosome("ccccca"), Chromosome("aaaaacaaaa")}; + BOOST_TEST(result2 == expectedPair2); +} + +BOOST_AUTO_TEST_CASE(randomTwoPointCrossover_should_only_consider_points_available_on_both_chromosomes) +{ + function crossover = randomTwoPointCrossover(); + + for (size_t i = 0; i < 30; ++i) + { + Chromosome result1 = crossover(Chromosome("aaa"), Chromosome("TTTTTTTTTTTTTTTTTTTT")); + Chromosome result2 = crossover(Chromosome("TTTTTTTTTTTTTTTTTTTT"), Chromosome("aaa")); + BOOST_TEST(( + result1 == Chromosome("aaa") || + result1 == Chromosome("Taa") || + result1 == Chromosome("TTa") || + result1 == Chromosome("TTT") || + result1 == Chromosome("aTa") || + result1 == Chromosome("aTT") || + result1 == Chromosome("aaT") + )); + BOOST_TEST(( + result2 == Chromosome("TTTTTTTTTTTTTTTTTTTT") || + result2 == Chromosome("aTTTTTTTTTTTTTTTTTTT") || + result2 == Chromosome("aaTTTTTTTTTTTTTTTTTT") || + result2 == Chromosome("aaaTTTTTTTTTTTTTTTTT") || + result2 == Chromosome("TaTTTTTTTTTTTTTTTTTT") || + result2 == Chromosome("TaaTTTTTTTTTTTTTTTTT") || + result2 == Chromosome("TTaTTTTTTTTTTTTTTTTT") + )); + } +} + +BOOST_AUTO_TEST_CASE(uniformCrossover_should_swap_randomly_selected_genes) +{ + function crossover = uniformCrossover(0.7); + + SimulationRNG::reset(1); + Chromosome result1 = crossover(Chromosome("aaaaaaaaaa"), Chromosome("cccccc")); + BOOST_TEST(result1 == Chromosome("caaacc")); + + SimulationRNG::reset(1); + Chromosome result2 = crossover(Chromosome("cccccc"), Chromosome("aaaaaaaaaa")); + BOOST_TEST(result2 == Chromosome("acccaaaaaa")); +} + +BOOST_AUTO_TEST_CASE(symmetricUniformCrossover_should_swap_randomly_selected_genes) +{ + function crossover = symmetricUniformCrossover(0.7); + + SimulationRNG::reset(1); + tuple result1 = crossover(Chromosome("aaaaaaaaaa"), Chromosome("cccccc")); + tuple expectedPair1 = {Chromosome("caaacc"), Chromosome("acccaaaaaa")}; + BOOST_TEST(result1 == expectedPair1); + + tuple result2 = crossover(Chromosome("cccccc"), Chromosome("aaaaaaaaaa")); + tuple expectedPair2 = {Chromosome("caaaaaaaaa"), Chromosome("accccc")}; + BOOST_TEST(result2 == expectedPair2); +} + +BOOST_AUTO_TEST_CASE(uniformCrossover_should_only_consider_points_available_on_both_chromosomes) +{ + function crossover = uniformCrossover(0.7); + + set expectedPatterns = { + "TTTTTTTTTTTTTTTTTTTT", + "aTTTTTTTTTTTTTTTTTTT", + "TaTTTTTTTTTTTTTTTTTT", + "TTaTTTTTTTTTTTTTTTTT", + "aaTTTTTTTTTTTTTTTTTT", + "TaaTTTTTTTTTTTTTTTTT", + "aTaTTTTTTTTTTTTTTTTT", + "aaaTTTTTTTTTTTTTTTTT", + "aaa", + "Taa", + "aTa", + "aaT", + "TTa", + "aTT", + "TaT", + "TTT", + }; + + for (size_t i = 0; i < 30; ++i) + { + Chromosome result1 = crossover(Chromosome("aaa"), Chromosome("TTTTTTTTTTTTTTTTTTTT")); + Chromosome result2 = crossover(Chromosome("TTTTTTTTTTTTTTTTTTTT"), Chromosome("aaa")); + BOOST_TEST(expectedPatterns.count(toString(result1)) == 1); + BOOST_TEST(expectedPatterns.count(toString(result2)) == 1); + } +} + +BOOST_AUTO_TEST_CASE(uniformCrossover_should_not_swap_anything_if_chance_is_zero) +{ + BOOST_TEST(uniformCrossover(0.0)(Chromosome("aaaaaaaaaa"), Chromosome("cccccc")) == Chromosome("aaaaaaaaaa")); + BOOST_TEST(uniformCrossover(0.0)(Chromosome("cccccc"), Chromosome("aaaaaaaaaa")) == Chromosome("cccccc")); +} + +BOOST_AUTO_TEST_CASE(uniformCrossover_should_swap_whole_chromosomes_if_chance_is_one) +{ + BOOST_TEST(uniformCrossover(1.0)(Chromosome("aaaaaaaaaa"), Chromosome("cccccc")) == Chromosome("cccccc")); + BOOST_TEST(uniformCrossover(1.0)(Chromosome("cccccc"), Chromosome("aaaaaaaaaa")) == Chromosome("aaaaaaaaaa")); +} + +BOOST_AUTO_TEST_CASE(uniformCrossover_should_swap_genes_with_uniform_probability) +{ + constexpr size_t operationCount = 1000; + constexpr double swapChance = 0.8; + constexpr double relativeTolerance = 0.05; + double const expectedValue = swapChance; + double const variance = swapChance * (1 - swapChance); + + function crossover = uniformCrossover(swapChance); + Chromosome chromosome1("aaaaaaaaaa"); + Chromosome chromosome2("cccccccccc"); + + vector bernoulliTrials; + for (size_t i = 0; i < operationCount; ++i) + { + string genes = toString(crossover(chromosome1, chromosome2)); + for (size_t j = 0; j < chromosome1.length(); ++j) + bernoulliTrials.push_back(static_cast(genes[j] == 'c')); + } + + BOOST_TEST(abs(mean(bernoulliTrials) - expectedValue) < expectedValue * relativeTolerance); + BOOST_TEST(abs(meanSquaredError(bernoulliTrials, expectedValue) - variance) < variance * relativeTolerance); +} + +BOOST_AUTO_TEST_CASE(uniformCrossover_should_swap_tail_with_uniform_probability) +{ + constexpr size_t operationCount = 1000; + constexpr double swapChance = 0.3; + constexpr double relativeTolerance = 0.05; + double const expectedValue = swapChance; + double const variance = swapChance * (1 - swapChance); + + function crossover = uniformCrossover(swapChance); + Chromosome chromosome1("aaaaa"); + Chromosome chromosome2("cccccccccc"); + + vector bernoulliTrials; + for (size_t i = 0; i < operationCount; ++i) + { + string genes = toString(crossover(chromosome1, chromosome2)); + BOOST_REQUIRE(genes.size() == 5 || genes.size() == 10); + bernoulliTrials.push_back(static_cast(genes.size() == 10)); + } + + BOOST_TEST(abs(mean(bernoulliTrials) - expectedValue) < expectedValue * relativeTolerance); + BOOST_TEST(abs(meanSquaredError(bernoulliTrials, expectedValue) - variance) < variance * relativeTolerance); +} + BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() diff --git a/tools/yulPhaser/Mutations.cpp b/tools/yulPhaser/Mutations.cpp index 98689a810216..7313cd45b851 100644 --- a/tools/yulPhaser/Mutations.cpp +++ b/tools/yulPhaser/Mutations.cpp @@ -180,3 +180,138 @@ function phaser::fixedPointCrossover(double _crossoverPoint) return get<0>(fixedPointSwap(_chromosome1, _chromosome2, concretePoint)); }; } + +namespace +{ + +ChromosomePair fixedTwoPointSwap( + Chromosome const& _chromosome1, + Chromosome const& _chromosome2, + size_t _crossoverPoint1, + size_t _crossoverPoint2 +) +{ + assert(_crossoverPoint1 <= _chromosome1.length()); + assert(_crossoverPoint1 <= _chromosome2.length()); + assert(_crossoverPoint2 <= _chromosome1.length()); + assert(_crossoverPoint2 <= _chromosome2.length()); + + size_t lowPoint = min(_crossoverPoint1, _crossoverPoint2); + size_t highPoint = max(_crossoverPoint1, _crossoverPoint2); + + auto begin1 = _chromosome1.optimisationSteps().begin(); + auto begin2 = _chromosome2.optimisationSteps().begin(); + auto end1 = _chromosome1.optimisationSteps().end(); + auto end2 = _chromosome2.optimisationSteps().end(); + + return { + Chromosome( + vector(begin1, begin1 + lowPoint) + + vector(begin2 + lowPoint, begin2 + highPoint) + + vector(begin1 + highPoint, end1) + ), + Chromosome( + vector(begin2, begin2 + lowPoint) + + vector(begin1 + lowPoint, begin1 + highPoint) + + vector(begin2 + highPoint, end2) + ), + }; +} + +} + +function phaser::randomTwoPointCrossover() +{ + return [=](Chromosome const& _chromosome1, Chromosome const& _chromosome2) + { + size_t minLength = min(_chromosome1.length(), _chromosome2.length()); + + // Don't use position 0 (because this just swaps the values) unless it's the only choice. + size_t minPoint = (minLength > 0 ? 1 : 0); + assert(minPoint <= minLength); + + size_t randomPoint1 = SimulationRNG::uniformInt(minPoint, minLength); + size_t randomPoint2 = SimulationRNG::uniformInt(randomPoint1, minLength); + return get<0>(fixedTwoPointSwap(_chromosome1, _chromosome2, randomPoint1, randomPoint2)); + }; +} + +function phaser::symmetricRandomTwoPointCrossover() +{ + return [=](Chromosome const& _chromosome1, Chromosome const& _chromosome2) + { + size_t minLength = min(_chromosome1.length(), _chromosome2.length()); + + // Don't use position 0 (because this just swaps the values) unless it's the only choice. + size_t minPoint = (minLength > 0 ? 1 : 0); + assert(minPoint <= minLength); + + size_t randomPoint1 = SimulationRNG::uniformInt(minPoint, minLength); + size_t randomPoint2 = SimulationRNG::uniformInt(randomPoint1, minLength); + return fixedTwoPointSwap(_chromosome1, _chromosome2, randomPoint1, randomPoint2); + }; +} + +namespace +{ + +ChromosomePair uniformSwap(Chromosome const& _chromosome1, Chromosome const& _chromosome2, double _swapChance) +{ + vector steps1; + vector steps2; + + size_t minLength = min(_chromosome1.length(), _chromosome2.length()); + for (size_t i = 0; i < minLength; ++i) + if (SimulationRNG::bernoulliTrial(_swapChance)) + { + steps1.push_back(_chromosome2.optimisationSteps()[i]); + steps2.push_back(_chromosome1.optimisationSteps()[i]); + } + else + { + steps1.push_back(_chromosome1.optimisationSteps()[i]); + steps2.push_back(_chromosome2.optimisationSteps()[i]); + } + + auto begin1 = _chromosome1.optimisationSteps().begin(); + auto begin2 = _chromosome2.optimisationSteps().begin(); + auto end1 = _chromosome1.optimisationSteps().end(); + auto end2 = _chromosome2.optimisationSteps().end(); + + bool swapTail = SimulationRNG::bernoulliTrial(_swapChance); + if (_chromosome1.length() > minLength) + { + if (swapTail) + steps2.insert(steps2.end(), begin1 + minLength, end1); + else + steps1.insert(steps1.end(), begin1 + minLength, end1); + } + + if (_chromosome2.length() > minLength) + { + if (swapTail) + steps1.insert(steps1.end(), begin2 + minLength, end2); + else + steps2.insert(steps2.end(), begin2 + minLength, end2); + } + + return {Chromosome(steps1), Chromosome(steps2)}; +} + +} + +function phaser::uniformCrossover(double _swapChance) +{ + return [=](Chromosome const& _chromosome1, Chromosome const& _chromosome2) + { + return get<0>(uniformSwap(_chromosome1, _chromosome2, _swapChance)); + }; +} + +function phaser::symmetricUniformCrossover(double _swapChance) +{ + return [=](Chromosome const& _chromosome1, Chromosome const& _chromosome2) + { + return uniformSwap(_chromosome1, _chromosome2, _swapChance); + }; +} diff --git a/tools/yulPhaser/Mutations.h b/tools/yulPhaser/Mutations.h index d545aa0934fa..04e7050afda5 100644 --- a/tools/yulPhaser/Mutations.h +++ b/tools/yulPhaser/Mutations.h @@ -80,4 +80,22 @@ std::function symmetricRandomPointCrossover(); /// unless there is no other choice (i.e. one of the chromosomes is empty). std::function fixedPointCrossover(double _crossoverPoint); +/// Creates a crossover operator that randomly selects two points between 0 and 1 and swaps genes +/// from the resulting interval. The interval may be empty in which case no genes are swapped. +std::function randomTwoPointCrossover(); + +/// Symmetric version of @a randomTwoPointCrossover(). Creates an operator that returns a pair +/// containing both possible results for the same crossover points. +std::function symmetricRandomTwoPointCrossover(); + +/// Creates a crossover operator that goes over the length of the shorter chromosomes and for +/// each gene independently decides whether to swap it or not (with probability given by +/// @a _swapChance). The tail of the longer chromosome (the part that's past the length of the +/// shorter one) is treated as a single gene and can potentially be swapped too. +std::function uniformCrossover(double _swapChance); + +/// Symmetric version of @a uniformCrossover(). Creates an operator that returns a pair +/// containing both possible results for the same set or swap decisions. +std::function symmetricUniformCrossover(double _swapChance); + } From ad89b477c86cd4f9ac632260a87549a6b7f132c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 11 Mar 2020 03:38:31 +0100 Subject: [PATCH 058/126] [yul-phaser] GeneticAlgorithms: Add methods for selecting a crossover operator --- tools/yulPhaser/GeneticAlgorithms.cpp | 39 +++++++++++++++++++++++++++ tools/yulPhaser/GeneticAlgorithms.h | 20 ++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/tools/yulPhaser/GeneticAlgorithms.cpp b/tools/yulPhaser/GeneticAlgorithms.cpp index 701bdf28c137..7dcc44ed48df 100644 --- a/tools/yulPhaser/GeneticAlgorithms.cpp +++ b/tools/yulPhaser/GeneticAlgorithms.cpp @@ -21,8 +21,47 @@ #include using namespace std; +using namespace solidity; using namespace solidity::phaser; +function phaser::buildCrossoverOperator( + CrossoverChoice _choice, + optional _uniformCrossoverSwapChance +) +{ + switch (_choice) + { + case CrossoverChoice::SinglePoint: + return randomPointCrossover(); + case CrossoverChoice::TwoPoint: + return randomTwoPointCrossover(); + case CrossoverChoice::Uniform: + assert(_uniformCrossoverSwapChance.has_value()); + return uniformCrossover(_uniformCrossoverSwapChance.value()); + default: + assertThrow(false, solidity::util::Exception, "Invalid CrossoverChoice value."); + }; +} + +function phaser::buildSymmetricCrossoverOperator( + CrossoverChoice _choice, + optional _uniformCrossoverSwapChance +) +{ + switch (_choice) + { + case CrossoverChoice::SinglePoint: + return symmetricRandomPointCrossover(); + case CrossoverChoice::TwoPoint: + return symmetricRandomTwoPointCrossover(); + case CrossoverChoice::Uniform: + assert(_uniformCrossoverSwapChance.has_value()); + return symmetricUniformCrossover(_uniformCrossoverSwapChance.value()); + default: + assertThrow(false, solidity::util::Exception, "Invalid CrossoverChoice value."); + }; +} + Population RandomAlgorithm::runNextRound(Population _population) { RangeSelection elite(0.0, m_options.elitePoolSize); diff --git a/tools/yulPhaser/GeneticAlgorithms.h b/tools/yulPhaser/GeneticAlgorithms.h index 0d1f0375f12a..22ab62c048ae 100644 --- a/tools/yulPhaser/GeneticAlgorithms.h +++ b/tools/yulPhaser/GeneticAlgorithms.h @@ -20,11 +20,31 @@ #pragma once +#include #include +#include + namespace solidity::phaser { +enum class CrossoverChoice +{ + SinglePoint, + TwoPoint, + Uniform, +}; + +std::function buildCrossoverOperator( + CrossoverChoice _choice, + std::optional _uniformCrossoverSwapChance +); + +std::function buildSymmetricCrossoverOperator( + CrossoverChoice _choice, + std::optional _uniformCrossoverSwapChance +); + /** * Abstract base class for genetic algorithms. * The main feature is the @a runNextRound() method that executes one round of the algorithm, From d9e2735361f2fa85bbc8be4f3f5def2a34f4fa14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 11 Mar 2020 03:39:29 +0100 Subject: [PATCH 059/126] [yul-phaser] Add options for selecting crossover operator used by the algorithms --- test/yulPhaser/GeneticAlgorithms.cpp | 10 ++++++++++ test/yulPhaser/Phaser.cpp | 7 +++++++ tools/yulPhaser/GeneticAlgorithms.cpp | 12 ++++++++++-- tools/yulPhaser/GeneticAlgorithms.h | 8 +++++++- tools/yulPhaser/Phaser.cpp | 26 ++++++++++++++++++++++++++ tools/yulPhaser/Phaser.h | 8 ++++++++ 6 files changed, 68 insertions(+), 3 deletions(-) diff --git a/test/yulPhaser/GeneticAlgorithms.cpp b/test/yulPhaser/GeneticAlgorithms.cpp index 1b902d97878b..d7ef8db57c94 100644 --- a/test/yulPhaser/GeneticAlgorithms.cpp +++ b/test/yulPhaser/GeneticAlgorithms.cpp @@ -51,6 +51,8 @@ class ClassicGeneticAlgorithmFixture: public GeneticAlgorithmFixture /* mutationChance = */ 0.0, /* deletionChance = */ 0.0, /* additionChance = */ 0.0, + /* CrossoverChoice = */ CrossoverChoice::SinglePoint, + /* uniformCrossoverSwapChance= */ 0.5, }; }; @@ -113,6 +115,8 @@ BOOST_FIXTURE_TEST_CASE(runNextRound_should_preserve_elite_and_regenerate_rest_o /* deletionVsAdditionChance = */ 1.0, /* percentGenesToRandomise = */ 0.0, /* percentGenesToAddOrDelete = */ 1.0, + /* CrossoverChoice = */ CrossoverChoice::SinglePoint, + /* uniformCrossoverSwapChance= */ 0.5, }; GenerationalElitistWithExclusivePools algorithm(options); @@ -133,6 +137,8 @@ BOOST_FIXTURE_TEST_CASE(runNextRound_should_not_replace_elite_with_worse_individ /* deletionVsAdditionChance = */ 0.0, /* percentGenesToRandomise = */ 0.0, /* percentGenesToAddOrDelete = */ 1.0, + /* CrossoverChoice = */ CrossoverChoice::SinglePoint, + /* uniformCrossoverSwapChance= */ 0.5, }; GenerationalElitistWithExclusivePools algorithm(options); @@ -152,6 +158,8 @@ BOOST_FIXTURE_TEST_CASE(runNextRound_should_generate_individuals_in_the_crossove /* deletionVsAdditionChance = */ 0.5, /* percentGenesToRandomise = */ 1.0, /* percentGenesToAddOrDelete = */ 1.0, + /* CrossoverChoice = */ CrossoverChoice::SinglePoint, + /* uniformCrossoverSwapChance= */ 0.5, }; GenerationalElitistWithExclusivePools algorithm(options); @@ -179,6 +187,8 @@ BOOST_FIXTURE_TEST_CASE(runNextRound_should_generate_individuals_in_the_crossove /* deletionVsAdditionChance = */ 0.0, /* percentGenesToRandomise = */ 0.0, /* percentGenesToAddOrDelete = */ 0.0, + /* CrossoverChoice = */ CrossoverChoice::SinglePoint, + /* uniformCrossoverSwapChance= */ 0.5, }; GenerationalElitistWithExclusivePools algorithm(options); diff --git a/test/yulPhaser/Phaser.cpp b/test/yulPhaser/Phaser.cpp index c365ce310501..c381ba623cbe 100644 --- a/test/yulPhaser/Phaser.cpp +++ b/test/yulPhaser/Phaser.cpp @@ -45,6 +45,8 @@ class GeneticAlgorithmFactoryFixture /* algorithm = */ Algorithm::Random, /* minChromosomeLength = */ 50, /* maxChromosomeLength = */ 100, + /* CrossoverChoice = */ CrossoverChoice::Uniform, + /* uniformCrossoverSwapChance = */ 0.5, /* randomElitePoolSize = */ 0.5, /* gewepMutationPoolSize = */ 0.1, /* gewepCrossoverPoolSize = */ 0.1, @@ -121,6 +123,9 @@ BOOST_FIXTURE_TEST_CASE(build_should_select_the_right_algorithm_and_pass_the_opt auto gewepAlgorithm = dynamic_cast(algorithm2.get()); BOOST_REQUIRE(gewepAlgorithm != nullptr); + BOOST_TEST(gewepAlgorithm->options().crossover == m_options.crossover); + BOOST_TEST(gewepAlgorithm->options().uniformCrossoverSwapChance.has_value()); + BOOST_TEST(gewepAlgorithm->options().uniformCrossoverSwapChance.value() == m_options.uniformCrossoverSwapChance); BOOST_TEST(gewepAlgorithm->options().mutationPoolSize == m_options.gewepMutationPoolSize); BOOST_TEST(gewepAlgorithm->options().crossoverPoolSize == m_options.gewepCrossoverPoolSize); BOOST_TEST(gewepAlgorithm->options().randomisationChance == m_options.gewepRandomisationChance); @@ -134,6 +139,8 @@ BOOST_FIXTURE_TEST_CASE(build_should_select_the_right_algorithm_and_pass_the_opt auto classicAlgorithm = dynamic_cast(algorithm3.get()); BOOST_REQUIRE(classicAlgorithm != nullptr); + BOOST_TEST(classicAlgorithm->options().uniformCrossoverSwapChance.has_value()); + BOOST_TEST(classicAlgorithm->options().uniformCrossoverSwapChance.value() == m_options.uniformCrossoverSwapChance); BOOST_TEST(classicAlgorithm->options().elitePoolSize == m_options.classicElitePoolSize); BOOST_TEST(classicAlgorithm->options().crossoverChance == m_options.classicCrossoverChance); BOOST_TEST(classicAlgorithm->options().mutationChance == m_options.classicMutationChance); diff --git a/tools/yulPhaser/GeneticAlgorithms.cpp b/tools/yulPhaser/GeneticAlgorithms.cpp index 7dcc44ed48df..1da31645fd2e 100644 --- a/tools/yulPhaser/GeneticAlgorithms.cpp +++ b/tools/yulPhaser/GeneticAlgorithms.cpp @@ -96,7 +96,10 @@ Population GenerationalElitistWithExclusivePools::runNextRound(Population _popul geneAddition(m_options.percentGenesToAddOrDelete) ) ); - std::function crossoverOperator = randomPointCrossover(); + std::function crossoverOperator = buildCrossoverOperator( + m_options.crossover, + m_options.uniformCrossoverSwapChance + ); return _population.select(elitePool) + @@ -111,10 +114,15 @@ Population ClassicGeneticAlgorithm::runNextRound(Population _population) Population selectedPopulation = select(_population, rest.individuals().size()); + std::function crossoverOperator = buildSymmetricCrossoverOperator( + m_options.crossover, + m_options.uniformCrossoverSwapChance + ); + Population crossedPopulation = Population::combine( selectedPopulation.symmetricCrossoverWithRemainder( PairsFromRandomSubset(m_options.crossoverChance), - symmetricRandomPointCrossover() + crossoverOperator ) ); diff --git a/tools/yulPhaser/GeneticAlgorithms.h b/tools/yulPhaser/GeneticAlgorithms.h index 22ab62c048ae..96c0277380ef 100644 --- a/tools/yulPhaser/GeneticAlgorithms.h +++ b/tools/yulPhaser/GeneticAlgorithms.h @@ -130,6 +130,8 @@ class GenerationalElitistWithExclusivePools: public GeneticAlgorithm double deletionVsAdditionChance; ///< The chance of choosing @a geneDeletion as the mutation if randomisation was not chosen. double percentGenesToRandomise; ///< The chance of any given gene being mutated in gene randomisation. double percentGenesToAddOrDelete; ///< The chance of a gene being added (or deleted) in gene addition (or deletion). + CrossoverChoice crossover; ///< The crossover operator to use. + std::optional uniformCrossoverSwapChance; ///< Chance of a pair of genes being swapped in uniform crossover. bool isValid() const { @@ -140,6 +142,7 @@ class GenerationalElitistWithExclusivePools: public GeneticAlgorithm 0 <= deletionVsAdditionChance && deletionVsAdditionChance <= 1.0 && 0 <= percentGenesToRandomise && percentGenesToRandomise <= 1.0 && 0 <= percentGenesToAddOrDelete && percentGenesToAddOrDelete <= 1.0 && + 0 <= uniformCrossoverSwapChance && uniformCrossoverSwapChance <= 1.0 && mutationPoolSize + crossoverPoolSize <= 1.0 ); } @@ -185,6 +188,8 @@ class ClassicGeneticAlgorithm: public GeneticAlgorithm double mutationChance; ///< The chance of a particular gene being randomised in @a geneRandomisation mutation. double deletionChance; ///< The chance of a particular gene being deleted in @a geneDeletion mutation. double additionChance; ///< The chance of a particular gene being added in @a geneAddition mutation. + CrossoverChoice crossover; ///< The crossover operator to use + std::optional uniformCrossoverSwapChance; ///< Chance of a pair of genes being swapped in uniform crossover. bool isValid() const { @@ -193,7 +198,8 @@ class ClassicGeneticAlgorithm: public GeneticAlgorithm 0 <= crossoverChance && crossoverChance <= 1.0 && 0 <= mutationChance && mutationChance <= 1.0 && 0 <= deletionChance && deletionChance <= 1.0 && - 0 <= additionChance && additionChance <= 1.0 + 0 <= additionChance && additionChance <= 1.0 && + 0 <= uniformCrossoverSwapChance && uniformCrossoverSwapChance <= 1.0 ); } }; diff --git a/tools/yulPhaser/Phaser.cpp b/tools/yulPhaser/Phaser.cpp index 6f7be3257d2a..ebfdbb8975c7 100644 --- a/tools/yulPhaser/Phaser.cpp +++ b/tools/yulPhaser/Phaser.cpp @@ -78,6 +78,14 @@ map const MetricAggregatorChoiceToStringMap = }; map const StringToMetricAggregatorChoiceMap = invertMap(MetricAggregatorChoiceToStringMap); +map const CrossoverChoiceToStringMap = +{ + {CrossoverChoice::SinglePoint, "single-point"}, + {CrossoverChoice::TwoPoint, "two-point"}, + {CrossoverChoice::Uniform, "uniform"}, +}; +map const StringToCrossoverChoiceMap = invertMap(CrossoverChoiceToStringMap); + } istream& phaser::operator>>(istream& _inputStream, PhaserMode& _phaserMode) { return deserializeChoice(_inputStream, _phaserMode, StringToPhaserModeMap); } @@ -88,6 +96,8 @@ istream& phaser::operator>>(istream& _inputStream, MetricChoice& _metric) { retu ostream& phaser::operator<<(ostream& _outputStream, MetricChoice _metric) { return serializeChoice(_outputStream, _metric, MetricChoiceToStringMap); } istream& phaser::operator>>(istream& _inputStream, MetricAggregatorChoice& _aggregator) { return deserializeChoice(_inputStream, _aggregator, StringToMetricAggregatorChoiceMap); } ostream& phaser::operator<<(ostream& _outputStream, MetricAggregatorChoice _aggregator) { return serializeChoice(_outputStream, _aggregator, MetricAggregatorChoiceToStringMap); } +istream& phaser::operator>>(istream& _inputStream, CrossoverChoice& _crossover) { return deserializeChoice(_inputStream, _crossover, StringToCrossoverChoiceMap); } +ostream& phaser::operator<<(ostream& _outputStream, CrossoverChoice _crossover) { return serializeChoice(_outputStream, _crossover, CrossoverChoiceToStringMap); } GeneticAlgorithmFactory::Options GeneticAlgorithmFactory::Options::fromCommandLine(po::variables_map const& _arguments) { @@ -95,6 +105,8 @@ GeneticAlgorithmFactory::Options GeneticAlgorithmFactory::Options::fromCommandLi _arguments["algorithm"].as(), _arguments["min-chromosome-length"].as(), _arguments["max-chromosome-length"].as(), + _arguments["crossover"].as(), + _arguments["uniform-crossover-swap-chance"].as(), _arguments.count("random-elite-pool-size") > 0 ? _arguments["random-elite-pool-size"].as() : optional{}, @@ -155,6 +167,8 @@ unique_ptr GeneticAlgorithmFactory::build( /* deletionVsAdditionChance = */ _options.gewepDeletionVsAdditionChance, /* percentGenesToRandomise = */ percentGenesToRandomise, /* percentGenesToAddOrDelete = */ percentGenesToAddOrDelete, + /* crossover = */ _options.crossover, + /* uniformCrossoverSwapChance = */ _options.uniformCrossoverSwapChance, }); } case Algorithm::Classic: @@ -165,6 +179,8 @@ unique_ptr GeneticAlgorithmFactory::build( /* mutationChance = */ _options.classicMutationChance, /* deletionChance = */ _options.classicDeletionChance, /* additionChance = */ _options.classicAdditionChance, + /* crossover = */ _options.crossover, + /* uniformCrossoverSwapChance = */ _options.uniformCrossoverSwapChance, }); } default: @@ -451,6 +467,16 @@ Phaser::CommandLineDescription Phaser::buildCommandLineDescription() po::value()->value_name("")->default_value(30), "Maximum length of randomly generated chromosomes." ) + ( + "crossover", + po::value()->value_name("")->default_value(CrossoverChoice::SinglePoint), + "Type of the crossover operator to use." + ) + ( + "uniform-crossover-swap-chance", + po::value()->value_name("")->default_value(0.5), + "Chance of two genes being swapped between chromosomes in uniform crossover." + ) ; keywordDescription.add(algorithmDescription); diff --git a/tools/yulPhaser/Phaser.h b/tools/yulPhaser/Phaser.h index 2896dd090879..9c1c5ef751f5 100644 --- a/tools/yulPhaser/Phaser.h +++ b/tools/yulPhaser/Phaser.h @@ -22,6 +22,7 @@ #pragma once #include +#include #include @@ -83,6 +84,8 @@ std::istream& operator>>(std::istream& _inputStream, solidity::phaser::MetricCho std::ostream& operator<<(std::ostream& _outputStream, solidity::phaser::MetricChoice _metric); std::istream& operator>>(std::istream& _inputStream, solidity::phaser::MetricAggregatorChoice& _aggregator); std::ostream& operator<<(std::ostream& _outputStream, solidity::phaser::MetricAggregatorChoice _aggregator); +std::istream& operator>>(std::istream& _inputStream, solidity::phaser::CrossoverChoice& _crossover); +std::ostream& operator<<(std::ostream& _outputStream, solidity::phaser::CrossoverChoice _crossover); /** * Builds and validates instances of @a GeneticAlgorithm and its derived classes. @@ -95,13 +98,18 @@ class GeneticAlgorithmFactory Algorithm algorithm; size_t minChromosomeLength; size_t maxChromosomeLength; + CrossoverChoice crossover; + double uniformCrossoverSwapChance; + std::optional randomElitePoolSize; + double gewepMutationPoolSize; double gewepCrossoverPoolSize; double gewepRandomisationChance; double gewepDeletionVsAdditionChance; std::optional gewepGenesToRandomise; std::optional gewepGenesToAddOrDelete; + double classicElitePoolSize; double classicCrossoverChance; double classicMutationChance; From c2effd4e983a45c2274a2104c99b8527330fa1a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 16 Apr 2020 11:36:05 +0200 Subject: [PATCH 060/126] [yul-phaser] Mutations: Minor style tweak, missing space in the ternary operator --- tools/yulPhaser/Mutations.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/yulPhaser/Mutations.cpp b/tools/yulPhaser/Mutations.cpp index 7313cd45b851..2b988e62206c 100644 --- a/tools/yulPhaser/Mutations.cpp +++ b/tools/yulPhaser/Mutations.cpp @@ -145,7 +145,7 @@ function phaser::randomPointCrossover() size_t minLength = min(_chromosome1.length(), _chromosome2.length()); // Don't use position 0 (because this just swaps the values) unless it's the only choice. - size_t minPoint = (minLength > 0? 1 : 0); + size_t minPoint = (minLength > 0 ? 1 : 0); assert(minPoint <= minLength); size_t randomPoint = SimulationRNG::uniformInt(minPoint, minLength); @@ -160,7 +160,7 @@ function phaser::symmetricRandomPointCrossover() size_t minLength = min(_chromosome1.length(), _chromosome2.length()); // Don't use position 0 (because this just swaps the values) unless it's the only choice. - size_t minPoint = (minLength > 0? 1 : 0); + size_t minPoint = (minLength > 0 ? 1 : 0); assert(minPoint <= minLength); size_t randomPoint = SimulationRNG::uniformInt(minPoint, minLength); From c59854c4bb32ecf34e8196a1a3de528ccf01c4a2 Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Mon, 20 Apr 2020 20:23:45 +0400 Subject: [PATCH 061/126] docs: add dark mode to documentation --- docs/_static/css/dark.css | 622 ++++++++++++++++++++++++++++++++++ docs/_static/css/toggle.css | 77 +++++ docs/_static/js/toggle.js | 38 +++ docs/_templates/versions.html | 36 ++ docs/conf.py | 6 +- 5 files changed, 778 insertions(+), 1 deletion(-) create mode 100644 docs/_static/css/dark.css create mode 100644 docs/_static/css/toggle.css create mode 100644 docs/_static/js/toggle.js create mode 100644 docs/_templates/versions.html diff --git a/docs/_static/css/dark.css b/docs/_static/css/dark.css new file mode 100644 index 000000000000..f9b445898879 --- /dev/null +++ b/docs/_static/css/dark.css @@ -0,0 +1,622 @@ +/* links */ + +a, +a:visited { + color: #aaddff; +} + + +/* code directives */ + +.method dt, +.class dt, +.data dt, +.attribute dt, +.function dt, +.classmethod dt, +.exception dt, +.descclassname, +.descname { + background-color: #2d2d2d !important; +} + +.rst-content dl:not(.docutils) dt { + color: #aaddff; + background-color: #2d2d2d; + border-top: solid 3px #525252; + border-left: solid 3px #525252; +} + +em.property { + color: #888888; +} + + +/* tables */ + +.rst-content table.docutils thead { + color: #ddd; +} + +.rst-content table.docutils td { + border: 0px; +} + +.rst-content table.docutils:not(.field-list) tr:nth-child(2n-1) td { + background-color: #5a5a5a; +} + + +/* inlined code highlights */ + +.xref, +.py-meth, +.rst-content a code { + color: #aaddff !important; + font-weight: normal !important; +} + +.rst-content code { + color: #eee !important; + font-weight: normal !important; +} + +code.literal { + background-color: #2d2d2d !important; + border: 1px solid #6d6d6d !important; +} + +code.docutils.literal.notranslate { + color: #ddd; +} + + +/* notes, warnings, hints */ + +.hint .admonition-title { + background: #2aa87c !important; +} + +.warning .admonition-title { + background: #cc4444 !important; +} + +.admonition-title { + background: #3a7ca8 !important; +} + +.admonition, +.note { + background-color: #2d2d2d !important; +} + + +/* table of contents */ + +.wy-nav-content-wrap { + background-color: rgba(0, 0, 0, 0.6) !important; +} + +.sidebar { + background-color: #191919 !important; +} + +.sidebar-title { + background-color: #2b2b2b !important; +} + +.wy-menu-vertical a { + color: #ddd; +} + +.wy-menu-vertical code.docutils.literal.notranslate { + color: #404040; + background: none !important; + border: none !important; +} + +.wy-nav-content { + background: #3c3c3c; + color: #dddddd; +} + +.wy-menu-vertical li.on a, +.wy-menu-vertical li.current>a { + background: #a3a3a3; + border-bottom: 0px !important; + border-top: 0px !important; +} + +.wy-menu-vertical li.current { + background: #b3b3b3; +} + +.toc-backref { + color: grey !important; +} + +.highlight .hll { + background-color: #49483e +} + +.highlight { + background: #222; + color: #f8f8f2 +} + +.highlight .c { + color: #888 +} + + +/* Comment */ + +.highlight .err { + color: #960050; + background-color: #1e0010 +} + + +/* Error */ + +.highlight .k { + color: #66d9ef +} + + +/* Keyword */ + +.highlight .l { + color: #ae81ff +} + + +/* Literal */ + +.highlight .n { + color: #f8f8f2 +} + + +/* Name */ + +.highlight .o { + color: #f92672 +} + + +/* Operator */ + +.highlight .p { + color: #f8f8f2 +} + + +/* Punctuation */ + +.highlight .ch { + color: #888 +} + + +/* Comment.Hashbang */ + +.highlight .cm { + color: #888 +} + + +/* Comment.Multiline */ + +.highlight .cp { + color: #888 +} + + +/* Comment.Preproc */ + +.highlight .cpf { + color: #888 +} + + +/* Comment.PreprocFile */ + +.highlight .c1 { + color: #888 +} + + +/* Comment.Single */ + +.highlight .cs { + color: #888 +} + + +/* Comment.Special */ + +.highlight .gd { + color: #f92672 +} + + +/* Generic.Deleted */ + +.highlight .ge { + font-style: italic +} + + +/* Generic.Emph */ + +.highlight .gi { + color: #a6e22e +} + + +/* Generic.Inserted */ + +.highlight .gs { + font-weight: bold +} + + +/* Generic.Strong */ + +.highlight .gu { + color: #888 +} + + +/* Generic.Subheading */ + +.highlight .kc { + color: #66d9ef +} + + +/* Keyword.Constant */ + +.highlight .kd { + color: #66d9ef +} + + +/* Keyword.Declaration */ + +.highlight .kn { + color: #f92672 +} + + +/* Keyword.Namespace */ + +.highlight .kp { + color: #66d9ef +} + + +/* Keyword.Pseudo */ + +.highlight .kr { + color: #66d9ef +} + + +/* Keyword.Reserved */ + +.highlight .kt { + color: #66d9ef +} + + +/* Keyword.Type */ + +.highlight .ld { + color: #e6db74 +} + + +/* Literal.Date */ + +.highlight .m { + color: #ae81ff +} + + +/* Literal.Number */ + +.highlight .s { + color: #e6db74 +} + + +/* Literal.String */ + +.highlight .na { + color: #a6e22e +} + + +/* Name.Attribute */ + +.highlight .nb { + color: #f8f8f2 +} + + +/* Name.Builtin */ + +.highlight .nc { + color: #a6e22e +} + + +/* Name.Class */ + +.highlight .no { + color: #66d9ef +} + + +/* Name.Constant */ + +.highlight .nd { + color: #a6e22e +} + + +/* Name.Decorator */ + +.highlight .ni { + color: #f8f8f2 +} + + +/* Name.Entity */ + +.highlight .ne { + color: #a6e22e +} + + +/* Name.Exception */ + +.highlight .nf { + color: #a6e22e +} + + +/* Name.Function */ + +.highlight .nl { + color: #f8f8f2 +} + + +/* Name.Label */ + +.highlight .nn { + color: #f8f8f2 +} + + +/* Name.Namespace */ + +.highlight .nx { + color: #a6e22e +} + + +/* Name.Other */ + +.highlight .py { + color: #f8f8f2 +} + + +/* Name.Property */ + +.highlight .nt { + color: #f92672 +} + + +/* Name.Tag */ + +.highlight .nv { + color: #f8f8f2 +} + + +/* Name.Variable */ + +.highlight .ow { + color: #f92672 +} + + +/* Operator.Word */ + +.highlight .w { + color: #f8f8f2 +} + + +/* Text.Whitespace */ + +.highlight .mb { + color: #ae81ff +} + + +/* Literal.Number.Bin */ + +.highlight .mf { + color: #ae81ff +} + + +/* Literal.Number.Float */ + +.highlight .mh { + color: #ae81ff +} + + +/* Literal.Number.Hex */ + +.highlight .mi { + color: #ae81ff +} + + +/* Literal.Number.Integer */ + +.highlight .mo { + color: #ae81ff +} + + +/* Literal.Number.Oct */ + +.highlight .sa { + color: #e6db74 +} + + +/* Literal.String.Affix */ + +.highlight .sb { + color: #e6db74 +} + + +/* Literal.String.Backtick */ + +.highlight .sc { + color: #e6db74 +} + + +/* Literal.String.Char */ + +.highlight .dl { + color: #e6db74 +} + + +/* Literal.String.Delimiter */ + +.highlight .sd { + color: #e6db74 +} + + +/* Literal.String.Doc */ + +.highlight .s2 { + color: #e6db74 +} + + +/* Literal.String.Double */ + +.highlight .se { + color: #ae81ff +} + + +/* Literal.String.Escape */ + +.highlight .sh { + color: #e6db74 +} + + +/* Literal.String.Heredoc */ + +.highlight .si { + color: #e6db74 +} + + +/* Literal.String.Interpol */ + +.highlight .sx { + color: #e6db74 +} + + +/* Literal.String.Other */ + +.highlight .sr { + color: #e6db74 +} + + +/* Literal.String.Regex */ + +.highlight .s1 { + color: #e6db74 +} + + +/* Literal.String.Single */ + +.highlight .ss { + color: #e6db74 +} + + +/* Literal.String.Symbol */ + +.highlight .bp { + color: #f8f8f2 +} + + +/* Name.Builtin.Pseudo */ + +.highlight .fm { + color: #a6e22e +} + + +/* Name.Function.Magic */ + +.highlight .vc { + color: #f8f8f2 +} + + +/* Name.Variable.Class */ + +.highlight .vg { + color: #f8f8f2 +} + + +/* Name.Variable.Global */ + +.highlight .vi { + color: #f8f8f2 +} + + +/* Name.Variable.Instance */ + +.highlight .vm { + color: #f8f8f2 +} + + +/* Name.Variable.Magic */ + +.highlight .il { + color: #ae81ff +} + + +/* Literal.Number.Integer.Long */ \ No newline at end of file diff --git a/docs/_static/css/toggle.css b/docs/_static/css/toggle.css new file mode 100644 index 000000000000..ebbd0658a1f1 --- /dev/null +++ b/docs/_static/css/toggle.css @@ -0,0 +1,77 @@ +input[type=checkbox] { + visibility: hidden; + height: 0; + width: 0; + margin: 0; +} + +.rst-versions .rst-current-version { + padding: 10px; + display: flex; + justify-content: space-between; +} + +.rst-versions .rst-current-version .fa-book, +.rst-versions .rst-current-version .fa-v, +.rst-versions .rst-current-version .fa-caret-down { + height: 24px; + line-height: 24px; + vertical-align: middle; +} + +.rst-versions .rst-current-version .fa-element { + width: 80px; + text-align: center; +} + +.rst-versions .rst-current-version .fa-book { + text-align: left; +} + +.rst-versions .rst-current-version .fa-v { + color: #27AE60; + text-align: right; +} + +label { + margin: 0 auto; + display: inline-block; + justify-content: center; + align-items: right; + border-radius: 100px; + position: relative; + cursor: pointer; + text-indent: -9999px; + width: 50px; + height: 21px; + background: #000; +} + +label:after { + border-radius: 50%; + position: absolute; + content: ''; + background: #fff; + width: 15px; + height: 15px; + top: 3px; + left: 3px; + transition: ease-in-out 200ms; +} + +input:checked+label { + background: #3a7ca8; +} + +input:checked+label:after { + left: calc(100% - 5px); + transform: translateX(-100%); +} + +html.transition, +html.transition *, +html.transition *:before, +html.transition *:after { + transition: ease-in-out 200ms !important; + transition-delay: 0 !important; +} \ No newline at end of file diff --git a/docs/_static/js/toggle.js b/docs/_static/js/toggle.js new file mode 100644 index 000000000000..f46a3a6662cc --- /dev/null +++ b/docs/_static/js/toggle.js @@ -0,0 +1,38 @@ +document.addEventListener('DOMContentLoaded', function() { + + function toggleCssMode(isDay) { + var mode = (isDay ? "Day" : "Night"); + localStorage.setItem("css-mode", mode); + + var daysheet = $('link[href="_static/pygments.css"]')[0].sheet; + daysheet.disabled = !isDay; + + var nightsheet = $('link[href="_static/css/dark.css"]')[0]; + if (!isDay && nightsheet === undefined) { + var element = document.createElement("link"); + element.setAttribute("rel", "stylesheet"); + element.setAttribute("type", "text/css"); + element.setAttribute("href", "_static/css/dark.css"); + document.getElementsByTagName("head")[0].appendChild(element); + return; + } + if (nightsheet !== undefined) { + nightsheet.sheet.disabled = isDay; + } + } + + var initial = localStorage.getItem("css-mode") != "Night"; + var checkbox = document.querySelector('input[name=mode]'); + + toggleCssMode(initial); + checkbox.checked = initial; + + checkbox.addEventListener('change', function() { + document.documentElement.classList.add('transition'); + window.setTimeout(() => { + document.documentElement.classList.remove('transition'); + }, 1000) + toggleCssMode(this.checked); + }) + +}); \ No newline at end of file diff --git a/docs/_templates/versions.html b/docs/_templates/versions.html new file mode 100644 index 000000000000..f680b6506d58 --- /dev/null +++ b/docs/_templates/versions.html @@ -0,0 +1,36 @@ +{# Add rst-badge after rst-versions for small badge style. #} +
+ + RTD + + + + + + + v: {{ current_version }} + + +
+
+
{{ _('Versions') }}
{% for slug, url in versions %} +
{{ slug }}
+ {% endfor %} +
+
+
{{ _('Downloads') }}
{% for type, url in downloads %} +
{{ type }}
+ {% endfor %} +
+
+ {# Translators: The phrase "Read the Docs" is not translated #} +
{{ _('On Read the Docs') }}
+
+ {{ _('Project Home') }} +
+
+ {{ _('Builds') }} +
+
+
+
\ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index c8d60690b755..9a9d574fc66a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -146,10 +146,14 @@ def setup(sphinx): # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] +html_css_files = ["css/toggle.css"] + +html_js_files = ["js/toggle.js"] + # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +html_extra_path = ["_static/css"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. From d59706e5f5da3e68012a810ec5f31a44eb13f712 Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Mon, 20 Apr 2020 20:24:31 +0400 Subject: [PATCH 062/126] docs: fix formatting errors and style inconsistencies --- docs/contracts/functions.rst | 10 ++-- docs/contracts/visibility-and-getters.rst | 8 +-- docs/examples/micropayment.rst | 2 +- docs/installing-solidity.rst | 10 ++-- docs/introduction-to-smart-contracts.rst | 6 +-- docs/natspec-format.rst | 4 +- docs/style-guide.rst | 14 +++--- docs/types.rst | 4 +- docs/types/value-types.rst | 2 +- docs/units-and-global-variables.rst | 59 +++++++++++++---------- docs/using-the-compiler.rst | 5 +- 11 files changed, 68 insertions(+), 56 deletions(-) diff --git a/docs/contracts/functions.rst b/docs/contracts/functions.rst index 8b9e7df6439f..7a0ec2e08559 100644 --- a/docs/contracts/functions.rst +++ b/docs/contracts/functions.rst @@ -195,7 +195,7 @@ In addition to the list of state modifying statements explained above, the follo } } -Pure functions are able to use the `revert()` and `require()` functions to revert +Pure functions are able to use the ``revert()`` and ``require()`` functions to revert potential state changes when an :ref:`error occurs `. Reverting a state change is not considered a "state modification", as only changes to the @@ -235,9 +235,9 @@ A contract can have at most one ``receive`` function, declared using ``receive() external payable { ... }`` (without the ``function`` keyword). This function cannot have arguments, cannot return anything and must have -``external`` visibility and ``payable`` state mutability. It is executed on a +``external`` visibility and ``payable`` state mutability. It is executed on a call to the contract with empty calldata. This is the function that is executed -on plain Ether transfers (e.g. via `.send()` or `.transfer()`). If no such +on plain Ether transfers (e.g. via ``.send()`` or ``.transfer()``). If no such function exists, but a payable :ref:`fallback function ` exists, the fallback function will be called on a plain Ether transfer. If neither a receive Ether nor a payable fallback function is present, the @@ -245,7 +245,7 @@ contract cannot receive Ether through regular transactions and throws an exception. In the worst case, the fallback function can only rely on 2300 gas being -available (for example when `send` or `transfer` is used), leaving little +available (for example when ``send`` or ``transfer`` is used), leaving little room to perform other operations except basic logging. The following operations will consume more gas than the 2300 gas stipend: @@ -265,7 +265,7 @@ will consume more gas than the 2300 gas stipend: .. warning:: A contract without a receive Ether function can receive Ether as a - recipient of a `coinbase transaction` (aka `miner block reward`) + recipient of a *coinbase transaction* (aka *miner block reward*) or as a destination of a ``selfdestruct``. A contract cannot react to such Ether transfers and thus also diff --git a/docs/contracts/visibility-and-getters.rst b/docs/contracts/visibility-and-getters.rst index 5ef105784b08..04af8a26f2e7 100644 --- a/docs/contracts/visibility-and-getters.rst +++ b/docs/contracts/visibility-and-getters.rst @@ -16,7 +16,7 @@ Functions have to be specified as being ``external``, ``public``, ``internal`` or ``private``. For state variables, ``external`` is not possible. -``external``: +``external`` External functions are part of the contract interface, which means they can be called from other contracts and via transactions. An external function ``f`` cannot be called @@ -25,18 +25,18 @@ For state variables, ``external`` is not possible. they receive large arrays of data, because the data is not copied from calldata to memory. -``public``: +``public`` Public functions are part of the contract interface and can be either called internally or via messages. For public state variables, an automatic getter function (see below) is generated. -``internal``: +``internal`` Those functions and state variables can only be accessed internally (i.e. from within the current contract or contracts deriving from it), without using ``this``. -``private``: +``private`` Private functions and state variables are only visible for the contract they are defined in and not in derived contracts. diff --git a/docs/examples/micropayment.rst b/docs/examples/micropayment.rst index b4f1f6c6b813..ab69fd296be6 100644 --- a/docs/examples/micropayment.rst +++ b/docs/examples/micropayment.rst @@ -454,7 +454,7 @@ The recipient should verify each message using the following process: We'll use the `ethereumjs-util `_ library to write this verification. The final step can be done a number of ways, -and we use JavaScript. The following code borrows the `constructMessage` function from the signing **JavaScript code** above: +and we use JavaScript. The following code borrows the ``constructMessage`` function from the signing **JavaScript code** above: :: diff --git a/docs/installing-solidity.rst b/docs/installing-solidity.rst index d02651a2e692..27a20b3bc537 100644 --- a/docs/installing-solidity.rst +++ b/docs/installing-solidity.rst @@ -35,11 +35,11 @@ or if you require more compilation options. npm / Node.js ============= -Use `npm` for a convenient and portable way to install `solcjs`, a Solidity compiler. The +Use ``npm`` for a convenient and portable way to install ``solcjs``, a Solidity compiler. The `solcjs` program has fewer features than the ways to access the compiler described further down this page. The :ref:`commandline-compiler` documentation assumes you are using -the full-featured compiler, `solc`. The usage of `solcjs` is documented inside its own +the full-featured compiler, ``solc``. The usage of ``solcjs`` is documented inside its own `repository `_. Note: The solc-js project is derived from the C++ @@ -53,10 +53,10 @@ Please refer to the solc-js repository for instructions. .. note:: - The commandline executable is named `solcjs`. + The commandline executable is named ``solcjs``. - The comandline options of `solcjs` are not compatible with `solc` and tools (such as `geth`) - expecting the behaviour of `solc` will not work with `solcjs`. + The comandline options of ``solcjs`` are not compatible with ``solc`` and tools (such as ``geth``) + expecting the behaviour of ``solc`` will not work with ``solcjs``. Docker ====== diff --git a/docs/introduction-to-smart-contracts.rst b/docs/introduction-to-smart-contracts.rst index a5f056493b6f..e3e38b92c697 100644 --- a/docs/introduction-to-smart-contracts.rst +++ b/docs/introduction-to-smart-contracts.rst @@ -323,7 +323,7 @@ Every account has a persistent key-value store mapping 256-bit words to 256-bit words called **storage**. Furthermore, every account has a **balance** in -Ether (in "Wei" to be exact, `1 ether` is `10**18 wei`) which can be modified by sending transactions that +Ether (in "Wei" to be exact, ``1 ether`` is ``10**18 wei``) which can be modified by sending transactions that include Ether. .. index:: ! transaction @@ -520,9 +520,9 @@ idea, but it is potentially dangerous, as if someone sends Ether to removed contracts, the Ether is forever lost. .. warning:: - Even if a contract is removed by "selfdestruct", it is still part of the + Even if a contract is removed by ``selfdestruct``, it is still part of the history of the blockchain and probably retained by most Ethereum nodes. - So using "selfdestruct" is not the same as deleting data from a hard disk. + So using ``selfdestruct`` is not the same as deleting data from a hard disk. .. note:: Even if a contract's code does not contain a call to ``selfdestruct``, diff --git a/docs/natspec-format.rst b/docs/natspec-format.rst index ae429580a3ce..1776925ee947 100644 --- a/docs/natspec-format.rst +++ b/docs/natspec-format.rst @@ -73,8 +73,8 @@ Tags All tags are optional. The following table explains the purpose of each NatSpec tag and where it may be used. As a special case, if no tags are -used then the Solidity compiler will interpret a `///` or `/**` comment -in the same way as if it were tagged with `@notice`. +used then the Solidity compiler will interpret a ``///`` or ``/**`` comment +in the same way as if it were tagged with ``@notice``. =========== =============================================================================== ============================= Tag Context diff --git a/docs/style-guide.rst b/docs/style-guide.rst index c637c8379cb2..c2990824d812 100644 --- a/docs/style-guide.rst +++ b/docs/style-guide.rst @@ -24,8 +24,11 @@ solidity code. The goal of this guide is *consistency*. A quote from python's `pep8 `_ captures this concept well. +.. note:: + A style guide is about consistency. Consistency with this style guide is important. Consistency within a project is more important. Consistency within one module or function is most important. - But most importantly: know when to be inconsistent -- sometimes the style guide just doesn't apply. When in doubt, use your best judgement. Look at other examples and decide what looks best. And don't hesitate to ask! + + But most importantly: **know when to be inconsistent** -- sometimes the style guide just doesn't apply. When in doubt, use your best judgement. Look at other examples and decide what looks best. And don't hesitate to ask! *********** @@ -383,8 +386,7 @@ No:: function spam(uint i , Coin coin) public ; -More than one space around an assignment or other operator to align with - another: +More than one space around an assignment or other operator to align with another: Yes:: @@ -996,7 +998,7 @@ Contract and Library Names * Contract and library names should also match their filenames. * If a contract file includes multiple contracts and/or libraries, then the filename should match the *core contract*. This is not recommended however if it can be avoided. -As shown in the example below, if the contract name is `Congress` and the library name is `Owned`, then their associated filenames should be `Congress.sol` and `Owned.sol`. +As shown in the example below, if the contract name is ``Congress`` and the library name is ``Owned``, then their associated filenames should be ``Congress.sol`` and ``Owned.sol``. Yes:: @@ -1132,8 +1134,8 @@ Solidity contracts can have a form of comments that are the basis of the Ethereum Natural Language Specification Format. Add comments above functions or contracts following `doxygen `_ notation -of one or multiple lines starting with `///` or a -multiline comment starting with `/**` and ending with `*/`. +of one or multiple lines starting with ``///`` or a +multiline comment starting with ``/**`` and ending with ``*/``. For example, the contract from `a simple smart contract `_ with the comments added looks like the one below:: diff --git a/docs/types.rst b/docs/types.rst index b9c06f6c89c9..d929a274f202 100644 --- a/docs/types.rst +++ b/docs/types.rst @@ -16,7 +16,7 @@ operators. For a quick reference of the various operators, see :ref:`order`. The concept of "undefined" or "null" values does not exist in Solidity, but newly declared variables always have a :ref:`default value` dependent on its type. To handle any unexpected values, you should use the :ref:`revert function` to revert the whole transaction, or return a -tuple with a second `bool` value denoting success. +tuple with a second ``bool`` value denoting success. .. include:: types/value-types.rst @@ -26,4 +26,4 @@ tuple with a second `bool` value denoting success. .. include:: types/operators.rst -.. include:: types/conversion.rst \ No newline at end of file +.. include:: types/conversion.rst diff --git a/docs/types/value-types.rst b/docs/types/value-types.rst index 80b9d548cb4a..0301a8ad5633 100644 --- a/docs/types/value-types.rst +++ b/docs/types/value-types.rst @@ -332,7 +332,7 @@ the :ref:`address type
`. Before version 0.5.0, contracts directly derived from the address type and there was no distinction between ``address`` and ``address payable``. -If you declare a local variable of contract type (`MyContract c`), you can call +If you declare a local variable of contract type (``MyContract c``), you can call functions on that contract. Take care to assign it from somewhere that is the same contract type. diff --git a/docs/units-and-global-variables.rst b/docs/units-and-global-variables.rst index 6c625e997c2d..75ed7546f1f4 100644 --- a/docs/units-and-global-variables.rst +++ b/docs/units-and-global-variables.rst @@ -140,15 +140,19 @@ Error Handling See the dedicated section on :ref:`assert and require` for more details on error handling and when to use which function. -``assert(bool condition)``: +``assert(bool condition)`` causes an invalid opcode and thus state change reversion if the condition is not met - to be used for internal errors. -``require(bool condition)``: + +``require(bool condition)`` reverts if the condition is not met - to be used for errors in inputs or external components. -``require(bool condition, string memory message)``: + +``require(bool condition, string memory message)`` reverts if the condition is not met - to be used for errors in inputs or external components. Also provides an error message. -``revert()``: + +``revert()`` abort execution and revert state changes -``revert(string memory reason)``: + +``revert(string memory reason)`` abort execution and revert state changes, providing an explanatory string .. index:: keccak256, ripemd160, sha256, ecrecover, addmod, mulmod, cryptography, @@ -156,32 +160,32 @@ more details on error handling and when to use which function. Mathematical and Cryptographic Functions ---------------------------------------- -``addmod(uint x, uint y, uint k) returns (uint)``: +``addmod(uint x, uint y, uint k) returns (uint)`` compute ``(x + y) % k`` where the addition is performed with arbitrary precision and does not wrap around at ``2**256``. Assert that ``k != 0`` starting from version 0.5.0. -``mulmod(uint x, uint y, uint k) returns (uint)``: +``mulmod(uint x, uint y, uint k) returns (uint)`` compute ``(x * y) % k`` where the multiplication is performed with arbitrary precision and does not wrap around at ``2**256``. Assert that ``k != 0`` starting from version 0.5.0. -``keccak256(bytes memory) returns (bytes32)``: +``keccak256(bytes memory) returns (bytes32)`` compute the Keccak-256 hash of the input .. note:: There used to be an alias for ``keccak256`` called ``sha3``, which was removed in version 0.5.0. -``sha256(bytes memory) returns (bytes32)``: +``sha256(bytes memory) returns (bytes32)`` compute the SHA-256 hash of the input -``ripemd160(bytes memory) returns (bytes20)``: +``ripemd160(bytes memory) returns (bytes20)`` compute RIPEMD-160 hash of the input -``ecrecover(bytes32 hash, uint8 v, bytes32 r, bytes32 s) returns (address)``: +``ecrecover(bytes32 hash, uint8 v, bytes32 r, bytes32 s) returns (address)`` recover the address associated with the public key from elliptic curve signature or return zero on error. The function parameters correspond to ECDSA values of the signature: - ``r`` = first 32 bytes of signature - ``s`` = second 32 bytes of signature - ``v`` = final 1 byte of signature + * ``r`` = first 32 bytes of signature + * ``s`` = second 32 bytes of signature + * ``v`` = final 1 byte of signature ``ecrecover`` returns an ``address``, and not an ``address payable``. See :ref:`address payable
` for conversion, in case you need to transfer funds to the recovered address. @@ -209,17 +213,22 @@ Mathematical and Cryptographic Functions Members of Address Types ------------------------ -``
.balance`` (``uint256``): +``
.balance`` (``uint256``) balance of the :ref:`address` in Wei -``
.transfer(uint256 amount)``: + +``
.transfer(uint256 amount)`` send given amount of Wei to :ref:`address`, reverts on failure, forwards 2300 gas stipend, not adjustable -``
.send(uint256 amount) returns (bool)``: + +``
.send(uint256 amount) returns (bool)`` send given amount of Wei to :ref:`address`, returns ``false`` on failure, forwards 2300 gas stipend, not adjustable -``
.call(bytes memory) returns (bool, bytes memory)``: + +``
.call(bytes memory) returns (bool, bytes memory)`` issue low-level ``CALL`` with the given payload, returns success condition and return data, forwards all available gas, adjustable -``
.delegatecall(bytes memory) returns (bool, bytes memory)``: + +``
.delegatecall(bytes memory) returns (bool, bytes memory)`` issue low-level ``DELEGATECALL`` with the given payload, returns success condition and return data, forwards all available gas, adjustable -``
.staticcall(bytes memory) returns (bool, bytes memory)``: + +``
.staticcall(bytes memory) returns (bool, bytes memory)`` issue low-level ``STATICCALL`` with the given payload, returns success condition and return data, forwards all available gas, adjustable For more information, see the section on :ref:`address`. @@ -258,10 +267,10 @@ For more information, see the section on :ref:`address`. Contract Related ---------------- -``this`` (current contract's type): +``this`` (current contract's type) the current contract, explicitly convertible to :ref:`address` -``selfdestruct(address payable recipient)``: +``selfdestruct(address payable recipient)`` Destroy the current contract, sending its funds to the given :ref:`address` and end execution. Note that ``selfdestruct`` has some peculiarities inherited from the EVM: @@ -290,10 +299,10 @@ type ``X``. Currently, there is limited support for this feature, but it might be expanded in the future. The following properties are available for a contract type ``C``: -``type(C).name``: +``type(C).name`` The name of the contract. -``type(C).creationCode``: +``type(C).creationCode`` Memory byte array that contains the creation bytecode of the contract. This can be used in inline assembly to build custom creation routines, especially by using the ``create2`` opcode. @@ -301,7 +310,7 @@ available for a contract type ``C``: derived contract. It causes the bytecode to be included in the bytecode of the call site and thus circular references like that are not possible. -``type(C).runtimeCode``: +``type(C).runtimeCode`` Memory byte array that contains the runtime bytecode of the contract. This is the code that is usually deployed by the constructor of ``C``. If ``C`` has a constructor that uses inline assembly, this might be diff --git a/docs/using-the-compiler.rst b/docs/using-the-compiler.rst index 02218592e687..ea573ed70989 100644 --- a/docs/using-the-compiler.rst +++ b/docs/using-the-compiler.rst @@ -106,7 +106,8 @@ Target options Below is a list of target EVM versions and the compiler-relevant changes introduced at each version. Backward compatibility is not guaranteed between each version. -- ``homestead`` (oldest version) +- ``homestead`` + - (oldest version) - ``tangerineWhistle`` - Gas cost for access to other accounts increased, relevant for gas estimation and the optimizer. - All gas sent by default for external calls, previously a certain amount had to be retained. @@ -692,7 +693,7 @@ Review changes The command above applies all changes as shown below. Please review them carefully. -.. code-block:: none +.. code-block:: solidity pragma solidity >=0.6.0 <0.7.0; From 606153ba71debbf44e54f89fe6c275bdbd665293 Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Mon, 20 Apr 2020 16:14:22 +0200 Subject: [PATCH 063/126] Add optimizer rules for repeated and --- test/formal/opcodes.py | 3 +++ test/formal/repeated_and.py | 39 +++++++++++++++++++++++++++++++++++++ test/formal/repeated_or.py | 39 +++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 test/formal/repeated_and.py create mode 100644 test/formal/repeated_or.py diff --git a/test/formal/opcodes.py b/test/formal/opcodes.py index cdd60ddaefc8..e65ecef29d37 100644 --- a/test/formal/opcodes.py +++ b/test/formal/opcodes.py @@ -42,6 +42,9 @@ def ISZERO(x): def AND(x, y): return x & y +def OR(x, y): + return x | y + def SHL(x, y): return y << x diff --git a/test/formal/repeated_and.py b/test/formal/repeated_and.py new file mode 100644 index 000000000000..2e8431b3ce00 --- /dev/null +++ b/test/formal/repeated_and.py @@ -0,0 +1,39 @@ +from rule import Rule +from opcodes import * + +""" +Rule: +AND(AND(X, Y), Y) -> AND(X, Y) +AND(Y, AND(X, Y)) -> AND(X, Y) +AND(AND(Y, X), Y) -> AND(Y, X) +AND(Y, AND(Y, X)) -> AND(Y, X) +Requirements: +""" + +rule = Rule() + +n_bits = 256 + +# Input vars +X = BitVec('X', n_bits) +Y = BitVec('Y', n_bits) + +# Constants +BitWidth = BitVecVal(n_bits, n_bits) + +# Requirements + +# Non optimized result +nonopt_1 = AND(AND(X, Y), Y) +nonopt_2 = AND(Y, AND(X, Y)) +nonopt_3 = AND(AND(Y, X), Y) +nonopt_4 = AND(Y, AND(Y, X)) + +# Optimized result +opt_1 = AND(X, Y) +opt_2 = AND(Y, X) + +rule.check(nonopt_1, opt_1) +rule.check(nonopt_2, opt_1) +rule.check(nonopt_3, opt_2) +rule.check(nonopt_4, opt_2) diff --git a/test/formal/repeated_or.py b/test/formal/repeated_or.py new file mode 100644 index 000000000000..c1b2ebd098d6 --- /dev/null +++ b/test/formal/repeated_or.py @@ -0,0 +1,39 @@ +from rule import Rule +from opcodes import * + +""" +Rule: +OR(OR(X, Y), Y) -> OR(X, Y) +OR(Y, OR(X, Y)) -> OR(X, Y) +OR(OR(Y, X), Y) -> OR(Y, X) +OR(Y, OR(Y, X)) -> OR(Y, X) +Requirements: +""" + +rule = Rule() + +n_bits = 256 + +# Input vars +X = BitVec('X', n_bits) +Y = BitVec('Y', n_bits) + +# Constants +BitWidth = BitVecVal(n_bits, n_bits) + +# Requirements + +# Non optimized result +nonopt_1 = OR(OR(X, Y), Y) +nonopt_2 = OR(Y, OR(X, Y)) +nonopt_3 = OR(OR(Y, X), Y) +nonopt_4 = OR(Y, OR(Y, X)) + +# Optimized result +opt_1 = OR(X, Y) +opt_2 = OR(Y, X) + +rule.check(nonopt_1, opt_1) +rule.check(nonopt_2, opt_1) +rule.check(nonopt_3, opt_2) +rule.check(nonopt_4, opt_2) From 2daa52f18cb104109cd3cf176008f8c85bf5c9b7 Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Wed, 22 Apr 2020 10:20:52 +0200 Subject: [PATCH 064/126] Fix CI pip version --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b1eb45af4b90..bfadb1139278 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -347,8 +347,8 @@ jobs: name: Z3 python deps command: | apt-get -qq update - apt-get -qy install python-pip - pip install --user z3-solver + apt-get -qy install python3-pip + pip3 install --user z3-solver - run: *run_proofs chk_docs_pragma_min_version: From 4908101ad757f0ec6ef54abd6929f7d5cf128992 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 16 Apr 2020 14:21:12 +0200 Subject: [PATCH 065/126] Yul IR generation for member access to type types. --- .../codegen/ir/IRGeneratorForStatements.cpp | 125 ++++++++++++++---- .../codegen/ir/IRGeneratorForStatements.h | 5 + ...ract_enums_with_explicit_contract_name.sol | 2 + .../semanticTests/enums/using_enums.sol | 2 + .../enums/using_inherited_enum.sol | 3 +- .../using_inherited_enum_excplicitly.sol | 3 +- .../intheritance/explicit_base_class.sol | 3 +- .../intheritance/inherited_function.sol | 3 +- .../state_variable_local_variable_mixture.sol | 3 +- .../state_variable_under_contract_name.sol | 3 +- 10 files changed, 117 insertions(+), 35 deletions(-) diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index c1b4c8b216a6..e375739d0c83 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -985,12 +985,74 @@ void IRGeneratorForStatements::endVisit(MemberAccess const& _memberAccess) } case Type::Category::TypeType: { - TypeType const& type = dynamic_cast(*_memberAccess.expression().annotation().type); - solUnimplementedAssert(type.actualType()->category() == Type::Category::Contract, ""); - FunctionType const* funType = dynamic_cast(_memberAccess.annotation().type); - solUnimplementedAssert(funType, ""); - solUnimplementedAssert(funType->kind() == FunctionType::Kind::Declaration, ""); - // Nothing to do for declaration. + Type const& actualType = *dynamic_cast( + *_memberAccess.expression().annotation().type + ).actualType(); + + if (actualType.category() == Type::Category::Contract) + { + if (auto const* variable = dynamic_cast(_memberAccess.annotation().referencedDeclaration)) + handleVariableReference(*variable, _memberAccess); + else if (auto const* funType = dynamic_cast(_memberAccess.annotation().type)) + { + switch (funType->kind()) + { + case FunctionType::Kind::Declaration: + break; + case FunctionType::Kind::Internal: + if (auto const* function = dynamic_cast(_memberAccess.annotation().referencedDeclaration)) + define(_memberAccess) << to_string(function->id()) << "\n"; + else + solAssert(false, "Function not found in member access"); + break; + case FunctionType::Kind::Event: + solAssert( + dynamic_cast(_memberAccess.annotation().referencedDeclaration), + "Event not found" + ); + // the call will do the resolving + break; + case FunctionType::Kind::DelegateCall: + define(IRVariable(_memberAccess).part("address"), _memberAccess.expression()); + define(IRVariable(_memberAccess).part("functionIdentifier")) << formatNumber(funType->externalIdentifier()) << "\n"; + break; + case FunctionType::Kind::External: + case FunctionType::Kind::Creation: + case FunctionType::Kind::Send: + case FunctionType::Kind::BareCall: + case FunctionType::Kind::BareCallCode: + case FunctionType::Kind::BareDelegateCall: + case FunctionType::Kind::BareStaticCall: + case FunctionType::Kind::Transfer: + case FunctionType::Kind::Log0: + case FunctionType::Kind::Log1: + case FunctionType::Kind::Log2: + case FunctionType::Kind::Log3: + case FunctionType::Kind::Log4: + case FunctionType::Kind::ECRecover: + case FunctionType::Kind::SHA256: + case FunctionType::Kind::RIPEMD160: + default: + solAssert(false, "unsupported member function"); + } + } + else if (dynamic_cast(_memberAccess.annotation().type)) + { + // no-op + } + else + // The old code generator had a generic "else" case here + // without any specific code being generated, + // but it would still be better to have an exhaustive list. + solAssert(false, ""); + } + else if (EnumType const* enumType = dynamic_cast(&actualType)) + define(_memberAccess) << to_string(enumType->memberValue(_memberAccess.memberName())) << "\n"; + else + // The old code generator had a generic "else" case here + // without any specific code being generated, + // but it would still be better to have an exhaustive list. + solAssert(false, ""); break; } default: @@ -1208,28 +1270,7 @@ void IRGeneratorForStatements::endVisit(Identifier const& _identifier) else if (FunctionDefinition const* functionDef = dynamic_cast(declaration)) define(_identifier) << to_string(functionDef->resolveVirtual(m_context.mostDerivedContract()).id()) << "\n"; else if (VariableDeclaration const* varDecl = dynamic_cast(declaration)) - { - // TODO for the constant case, we have to be careful: - // If the value is visited twice, `defineExpression` is called twice on - // the same expression. - solUnimplementedAssert(!varDecl->isConstant(), ""); - solUnimplementedAssert(!varDecl->immutable(), ""); - if (m_context.isLocalVariable(*varDecl)) - setLValue(_identifier, IRLValue{ - *varDecl->annotation().type, - IRLValue::Stack{m_context.localVariable(*varDecl)} - }); - else if (m_context.isStateVariable(*varDecl)) - setLValue(_identifier, IRLValue{ - *varDecl->annotation().type, - IRLValue::Storage{ - toCompactHexWithPrefix(m_context.storageLocationOfVariable(*varDecl).first), - m_context.storageLocationOfVariable(*varDecl).second - } - }); - else - solAssert(false, "Invalid variable kind."); - } + handleVariableReference(*varDecl, _identifier); else if (auto contract = dynamic_cast(declaration)) { solUnimplementedAssert(!contract->isLibrary(), "Libraries not yet supported."); @@ -1271,6 +1312,33 @@ bool IRGeneratorForStatements::visit(Literal const& _literal) return false; } +void IRGeneratorForStatements::handleVariableReference( + VariableDeclaration const& _variable, + Expression const& _referencingExpression +) +{ + // TODO for the constant case, we have to be careful: + // If the value is visited twice, `defineExpression` is called twice on + // the same expression. + solUnimplementedAssert(!_variable.isConstant(), ""); + solUnimplementedAssert(!_variable.immutable(), ""); + if (m_context.isLocalVariable(_variable)) + setLValue(_referencingExpression, IRLValue{ + *_variable.annotation().type, + IRLValue::Stack{m_context.localVariable(_variable)} + }); + else if (m_context.isStateVariable(_variable)) + setLValue(_referencingExpression, IRLValue{ + *_variable.annotation().type, + IRLValue::Storage{ + toCompactHexWithPrefix(m_context.storageLocationOfVariable(_variable).first), + m_context.storageLocationOfVariable(_variable).second + } + }); + else + solAssert(false, "Invalid variable kind."); +} + void IRGeneratorForStatements::appendExternalFunctionCall( FunctionCall const& _functionCall, vector> const& _arguments @@ -1890,4 +1958,3 @@ bool IRGeneratorForStatements::visit(TryCatchClause const& _clause) _clause.block().accept(*this); return false; } - diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.h b/libsolidity/codegen/ir/IRGeneratorForStatements.h index 810087a2d7f5..c0cf11ba0b98 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.h +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.h @@ -87,6 +87,11 @@ class IRGeneratorForStatements: public ASTConstVisitor /// Generates code to rethrow an exception. void rethrow(); + void handleVariableReference( + VariableDeclaration const& _variable, + Expression const& _referencingExpression + ); + /// Appends code to call an external function with the given arguments. /// All involved expressions have already been visited. void appendExternalFunctionCall( diff --git a/test/libsolidity/semanticTests/enums/using_contract_enums_with_explicit_contract_name.sol b/test/libsolidity/semanticTests/enums/using_contract_enums_with_explicit_contract_name.sol index 5d60a746650e..c6830abf8320 100644 --- a/test/libsolidity/semanticTests/enums/using_contract_enums_with_explicit_contract_name.sol +++ b/test/libsolidity/semanticTests/enums/using_contract_enums_with_explicit_contract_name.sol @@ -6,5 +6,7 @@ contract test { } } +// ==== +// compileViaYul: also // ---- // answer() -> 1 diff --git a/test/libsolidity/semanticTests/enums/using_enums.sol b/test/libsolidity/semanticTests/enums/using_enums.sol index ae497f40997f..883726f7c24d 100644 --- a/test/libsolidity/semanticTests/enums/using_enums.sol +++ b/test/libsolidity/semanticTests/enums/using_enums.sol @@ -12,5 +12,7 @@ contract test { ActionChoices choices; } +// ==== +// compileViaYul: also // ---- // getChoice() -> 2 diff --git a/test/libsolidity/semanticTests/enums/using_inherited_enum.sol b/test/libsolidity/semanticTests/enums/using_inherited_enum.sol index c39bfe064a1e..187ec962ecad 100644 --- a/test/libsolidity/semanticTests/enums/using_inherited_enum.sol +++ b/test/libsolidity/semanticTests/enums/using_inherited_enum.sol @@ -8,6 +8,7 @@ contract test is base { _ret = Choice.B; } } - +// ==== +// compileViaYul: also // ---- // answer() -> 1 diff --git a/test/libsolidity/semanticTests/enums/using_inherited_enum_excplicitly.sol b/test/libsolidity/semanticTests/enums/using_inherited_enum_excplicitly.sol index 0be3f80d4810..3bd2b0cea727 100644 --- a/test/libsolidity/semanticTests/enums/using_inherited_enum_excplicitly.sol +++ b/test/libsolidity/semanticTests/enums/using_inherited_enum_excplicitly.sol @@ -8,6 +8,7 @@ contract test is base { _ret = base.Choice.B; } } - +// ==== +// compileViaYul: also // ---- // answer() -> 1 diff --git a/test/libsolidity/semanticTests/intheritance/explicit_base_class.sol b/test/libsolidity/semanticTests/intheritance/explicit_base_class.sol index 44553b3f1adb..7481a0441e11 100644 --- a/test/libsolidity/semanticTests/intheritance/explicit_base_class.sol +++ b/test/libsolidity/semanticTests/intheritance/explicit_base_class.sol @@ -21,7 +21,8 @@ contract Derived is Base { return 3; } } - +// ==== +// compileViaYul: also // ---- // g() -> 3 // f() -> 1 diff --git a/test/libsolidity/semanticTests/intheritance/inherited_function.sol b/test/libsolidity/semanticTests/intheritance/inherited_function.sol index 23f9bee371ac..fc01100de029 100644 --- a/test/libsolidity/semanticTests/intheritance/inherited_function.sol +++ b/test/libsolidity/semanticTests/intheritance/inherited_function.sol @@ -14,6 +14,7 @@ contract B is A { return A.f(); } } - +// ==== +// compileViaYul: also // ---- // g() -> 1 diff --git a/test/libsolidity/semanticTests/various/state_variable_local_variable_mixture.sol b/test/libsolidity/semanticTests/various/state_variable_local_variable_mixture.sol index 585914c804b9..6f712839c86a 100644 --- a/test/libsolidity/semanticTests/various/state_variable_local_variable_mixture.sol +++ b/test/libsolidity/semanticTests/various/state_variable_local_variable_mixture.sol @@ -6,6 +6,7 @@ contract A { x = A.y; } } - +// ==== +// compileViaYul: also // ---- // a() -> 2 diff --git a/test/libsolidity/semanticTests/various/state_variable_under_contract_name.sol b/test/libsolidity/semanticTests/various/state_variable_under_contract_name.sol index 6bef6f85a650..2a2c9b975c9d 100644 --- a/test/libsolidity/semanticTests/various/state_variable_under_contract_name.sol +++ b/test/libsolidity/semanticTests/various/state_variable_under_contract_name.sol @@ -5,6 +5,7 @@ contract Scope { stateVar = Scope.stateVar; } } - +// ==== +// compileViaYul: also // ---- // getStateVar() -> 42 From fe383fbd7afbea44f99f069f56e4b40ad6952bc0 Mon Sep 17 00:00:00 2001 From: Alex Beregszaszi Date: Wed, 22 Apr 2020 12:03:10 +0100 Subject: [PATCH 066/126] Add missing blog URLs to bugs.json --- docs/bugs.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/bugs.json b/docs/bugs.json index 42a5277b11f8..56bae3aaffff 100644 --- a/docs/bugs.json +++ b/docs/bugs.json @@ -11,6 +11,7 @@ "name": "MemoryArrayCreationOverflow", "summary": "The creation of very large memory arrays can result in overlapping memory regions and thus memory corruption.", "description": "No runtime overflow checks were performed for the length of memory arrays during creation. In cases for which the memory size of an array in bytes, i.e. the array length times 32, is larger than 2^256-1, the memory allocation will overflow, potentially resulting in overlapping memory areas. The length of the array is still stored correctly, so copying or iterating over such an array will result in out-of-gas.", + "link": "https://solidity.ethereum.org/2020/04/06/memory-creation-overflow-bug/", "introduced": "0.2.0", "fixed": "0.6.5", "severity": "low" @@ -73,6 +74,7 @@ "name": "SignedArrayStorageCopy", "summary": "Assigning an array of signed integers to a storage array of different type can lead to data corruption in that array.", "description": "In two's complement, negative integers have their higher order bits set. In order to fit into a shared storage slot, these have to be set to zero. When a conversion is done at the same time, the bits to set to zero were incorrectly determined from the source and not the target type. This means that such copy operations can lead to incorrect values being stored.", + "link": "https://blog.ethereum.org/2019/06/25/solidity-storage-array-bugs/", "introduced": "0.4.7", "fixed": "0.5.10", "severity": "low/medium" @@ -81,6 +83,7 @@ "name": "ABIEncoderV2StorageArrayWithMultiSlotElement", "summary": "Storage arrays containing structs or other statically-sized arrays are not read properly when directly encoded in external function calls or in abi.encode*.", "description": "When storage arrays whose elements occupy more than a single storage slot are directly encoded in external function calls or using abi.encode*, their elements are read in an overlapping manner, i.e. the element pointer is not properly advanced between reads. This is not a problem when the storage data is first copied to a memory variable or if the storage array only contains value types or dynamically-sized arrays.", + "link": "https://blog.ethereum.org/2019/06/25/solidity-storage-array-bugs/", "introduced": "0.4.16", "fixed": "0.5.10", "severity": "low", From 5329da93fb2af94b2dc763a39640a38a093ed2f6 Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Mon, 6 Apr 2020 17:26:59 +0200 Subject: [PATCH 067/126] [Sol2Yul] Adding support for constructors with parameters in case of inheritance --- libsolidity/codegen/YulUtilFunctions.cpp | 36 ++++ libsolidity/codegen/YulUtilFunctions.h | 7 + libsolidity/codegen/ir/IRGenerator.cpp | 175 +++++++++++++----- libsolidity/codegen/ir/IRGenerator.h | 19 +- .../codegen/ir/IRGeneratorForStatements.cpp | 8 + .../codegen/ir/IRGeneratorForStatements.h | 3 + .../standard_eWasm_requested/output.json | 85 +++++++-- .../output.json | 4 + .../standard_ir_requested/output.json | 12 +- .../yul_string_format_ascii/output.json | 12 +- .../output.json | 12 +- .../output.json | 12 +- .../yul_string_format_ascii_long/output.json | 12 +- .../yul_string_format_hex/output.json | 12 +- .../constructor_ihneritance_init_order_2.sol | 14 ++ .../constructor_inheritance_init_order.sol | 17 ++ ...ructor_with_params_diamond_inheritance.sol | 28 +++ .../constructor_with_params_inheritance.sol | 18 ++ .../constructor_with_params_inheritance_2.sol | 15 ++ .../state_variables_init_order.sol | 14 ++ .../state_variables_init_order_2.sol | 18 ++ .../state_variables_init_order_3.sol | 29 +++ 22 files changed, 478 insertions(+), 84 deletions(-) create mode 100644 test/libsolidity/semanticTests/constructor_ihneritance_init_order_2.sol create mode 100644 test/libsolidity/semanticTests/constructor_inheritance_init_order.sol create mode 100644 test/libsolidity/semanticTests/constructor_with_params_diamond_inheritance.sol create mode 100644 test/libsolidity/semanticTests/constructor_with_params_inheritance.sol create mode 100644 test/libsolidity/semanticTests/constructor_with_params_inheritance_2.sol create mode 100644 test/libsolidity/semanticTests/state_variables_init_order.sol create mode 100644 test/libsolidity/semanticTests/state_variables_init_order_2.sol create mode 100644 test/libsolidity/semanticTests/state_variables_init_order_3.sol diff --git a/libsolidity/codegen/YulUtilFunctions.cpp b/libsolidity/codegen/YulUtilFunctions.cpp index e520c586fe9a..ab449a730b4e 100644 --- a/libsolidity/codegen/YulUtilFunctions.cpp +++ b/libsolidity/codegen/YulUtilFunctions.cpp @@ -2334,3 +2334,39 @@ string YulUtilFunctions::extractReturndataFunction() }); } +string YulUtilFunctions::copyConstructorArgumentsToMemoryFunction( + ContractDefinition const& _contract, + string const& _creationObjectName +) +{ + string functionName = "copy_arguments_for_constructor_" + + toString(_contract.constructor()->id()) + + "_object_" + + _contract.name() + + "_" + + toString(_contract.id()); + + return m_functionCollector.createFunction(functionName, [&]() { + string returnParams = suffixedVariableNameList("ret_param_",0, _contract.constructor()->parameters().size()); + ABIFunctions abiFunctions(m_evmVersion, m_revertStrings, m_functionCollector); + + return util::Whiskers(R"( + function () -> { + let programSize := datasize("") + let argSize := sub(codesize(), programSize) + + let memoryDataOffset := (argSize) + codecopy(memoryDataOffset, programSize, argSize) + + := (memoryDataOffset, add(memoryDataOffset, argSize)) + } + )") + ("functionName", functionName) + ("retParams", returnParams) + ("object", _creationObjectName) + ("allocate", allocationFunction()) + ("abiDecode", abiFunctions.tupleDecoder(FunctionType(*_contract.constructor()).parameterTypes(), true)) + .render(); + }); +} + diff --git a/libsolidity/codegen/YulUtilFunctions.h b/libsolidity/codegen/YulUtilFunctions.h index 9550b9e89e32..b16eb067281d 100644 --- a/libsolidity/codegen/YulUtilFunctions.h +++ b/libsolidity/codegen/YulUtilFunctions.h @@ -337,6 +337,13 @@ class YulUtilFunctions /// If returndatacopy() is not supported by the underlying target, a empty array will be returned instead. std::string extractReturndataFunction(); + /// @returns function name that returns constructor arguments copied to memory + /// signature: () -> arguments + std::string copyConstructorArgumentsToMemoryFunction( + ContractDefinition const& _contract, + std::string const& _creationObjectName + ); + private: /// Special case of conversionFunction - handles everything that does not /// use exactly one variable to hold the value. diff --git a/libsolidity/codegen/ir/IRGenerator.cpp b/libsolidity/codegen/ir/IRGenerator.cpp index 7499d3909e57..92631ddc241a 100644 --- a/libsolidity/codegen/ir/IRGenerator.cpp +++ b/libsolidity/codegen/ir/IRGenerator.cpp @@ -81,7 +81,9 @@ string IRGenerator::generate(ContractDefinition const& _contract) object "" { code { - + + let := () + () } @@ -99,8 +101,25 @@ string IRGenerator::generate(ContractDefinition const& _contract) t("CreationObject", creationObjectName(_contract)); t("memoryInit", memoryInit()); - t("constructor", constructorCode(_contract)); + + FunctionDefinition const* constructor = _contract.constructor(); + t("callValueCheck", !constructor || !constructor->isPayable() ? callValueCheck() : ""); + vector constructorParams; + if (constructor && !constructor->parameters().empty()) + { + for (size_t i = 0; i < constructor->parameters().size(); ++i) + constructorParams.emplace_back(m_context.newYulVariable()); + t( + "copyConstructorArguments", + m_utils.copyConstructorArgumentsToMemoryFunction(_contract, creationObjectName(_contract)) + ); + } + t("constructorParams", joinHumanReadable(constructorParams)); + t("constructorHasParams", !constructorParams.empty()); + t("implicitConstructor", implicitConstructorName(_contract)); + t("deploy", deployCode(_contract)); + generateImplicitConstructors(_contract); generateQueuedFunctions(); t("functions", m_context.functionCollector().requestedFunctions()); @@ -238,64 +257,115 @@ string IRGenerator::generateInitialAssignment(VariableDeclaration const& _varDec return generator.code(); } -string IRGenerator::constructorCode(ContractDefinition const& _contract) +pair> IRGenerator::evaluateConstructorArguments( + ContractDefinition const& _contract +) { - // Initialization of state variables in base-to-derived order. - solAssert(!_contract.isLibrary(), "Tried to initialize state variables of library."); - - using boost::adaptors::reverse; - - ostringstream out; - - FunctionDefinition const* constructor = _contract.constructor(); - if (constructor && !constructor->isPayable()) - out << callValueCheck(); + map constructorParams; + vector>const *>> baseConstructorArguments; + + for (ASTPointer const& base: _contract.baseContracts()) + if (FunctionDefinition const* baseConstructor = dynamic_cast( + base->name().annotation().referencedDeclaration + )->constructor(); baseConstructor && base->arguments()) + baseConstructorArguments.emplace_back( + dynamic_cast(baseConstructor->scope()), + base->arguments() + ); + + if (FunctionDefinition const* constructor = _contract.constructor()) + for (auto const& modifier: constructor->modifiers()) + if (FunctionDefinition const* baseConstructor = dynamic_cast( + modifier->name()->annotation().referencedDeclaration + )->constructor(); baseConstructor && modifier->arguments()) + baseConstructorArguments.emplace_back( + dynamic_cast(baseConstructor->scope()), + modifier->arguments() + ); - for (ContractDefinition const* contract: reverse(_contract.annotation().linearizedBaseContracts)) + IRGeneratorForStatements generator{m_context, m_utils}; + for (auto&& [baseContract, arguments]: baseConstructorArguments) { - out << - "\n// Begin state variable initialization for contract \"" << - contract->name() << - "\" (" << - contract->stateVariables().size() << - " variables)\n"; - - IRGeneratorForStatements generator{m_context, m_utils}; - for (VariableDeclaration const* variable: contract->stateVariables()) - if (!variable->isConstant() && !variable->immutable()) - generator.initializeStateVar(*variable); - out << generator.code(); - - out << "// End state variable initialization for contract \"" << contract->name() << "\".\n"; + solAssert(baseContract && arguments, ""); + if (baseContract->constructor() && !arguments->empty()) + { + vector params; + for (size_t i = 0; i < arguments->size(); ++i) + params.emplace_back(generator.evaluateExpression( + *(arguments->at(i)), + *(baseContract->constructor()->parameters()[i]->type()) + ).commaSeparatedList()); + constructorParams[baseContract] = joinHumanReadable(params); + } } - if (constructor) - { - ABIFunctions abiFunctions(m_evmVersion, m_context.revertStrings(), m_context.functionCollector()); - unsigned paramVars = make_shared(constructor->functionType(false)->parameterTypes())->sizeOnStack(); - - Whiskers t(R"X( - let programSize := datasize("") - let argSize := sub(codesize(), programSize) + return {generator.code(), constructorParams}; +} - let memoryDataOffset := (argSize) - codecopy(memoryDataOffset, programSize, argSize) +string IRGenerator::initStateVariables(ContractDefinition const& _contract) +{ + IRGeneratorForStatements generator{m_context, m_utils}; + for (VariableDeclaration const* variable: _contract.stateVariables()) + if (!variable->isConstant() && !variable->immutable()) + generator.initializeStateVar(*variable); - (memoryDataOffset, add(memoryDataOffset, argSize)) + return generator.code(); +} - () - )X"); - t("object", creationObjectName(_contract)); - t("allocate", m_utils.allocationFunction()); - t("assignToParams", paramVars == 0 ? "" : "let " + suffixedVariableNameList("param_", 0, paramVars) + " := "); - t("params", suffixedVariableNameList("param_", 0, paramVars)); - t("abiDecode", abiFunctions.tupleDecoder(constructor->functionType(false)->parameterTypes(), true)); - t("constructorName", m_context.enqueueFunctionForCodeGeneration(*constructor)); +void IRGenerator::generateImplicitConstructors(ContractDefinition const& _contract) +{ + auto listAllParams = [&]( + map const& baseParams) -> string + { + vector params; + for (ContractDefinition const* contract: _contract.annotation().linearizedBaseContracts) + if (baseParams.count(contract)) + params.emplace_back(baseParams.at(contract)); + return joinHumanReadable(params); + }; + + map baseConstructorParams; + for (size_t i = 0; i < _contract.annotation().linearizedBaseContracts.size(); ++i) + { + ContractDefinition const* contract = _contract.annotation().linearizedBaseContracts[i]; + baseConstructorParams.erase(contract); + + m_context.functionCollector().createFunction(implicitConstructorName(*contract), [&]() { + Whiskers t(R"( + function () { + + () + + + } + )"); + string params; + if (contract->constructor()) + for (ASTPointer const& varDecl: contract->constructor()->parameters()) + params += (params.empty() ? "" : ", ") + m_context.addLocalVariable(*varDecl).commaSeparatedList(); + t("params", params); + string baseParamsString = listAllParams(baseConstructorParams); + t("baseParams", baseParamsString); + t("comma", !params.empty() && !baseParamsString.empty() ? ", " : ""); + t("functionName", implicitConstructorName(*contract)); + pair> evaluatedArgs = evaluateConstructorArguments(*contract); + baseConstructorParams.insert(evaluatedArgs.second.begin(), evaluatedArgs.second.end()); + t("evalBaseArguments", evaluatedArgs.first); + if (i < _contract.annotation().linearizedBaseContracts.size() - 1) + { + t("hasNextConstructor", true); + ContractDefinition const* nextContract = _contract.annotation().linearizedBaseContracts[i + 1]; + t("nextConstructor", implicitConstructorName(*nextContract)); + t("nextParams", listAllParams(baseConstructorParams)); + } + else + t("hasNextConstructor", false); + t("initStateVariables", initStateVariables(*contract)); + t("userDefinedConstructorBody", contract->constructor() ? generate(contract->constructor()->body()) : ""); - out << t.render(); + return t.render(); + }); } - - return out.str(); } string IRGenerator::deployCode(ContractDefinition const& _contract) @@ -323,6 +393,11 @@ string IRGenerator::runtimeObjectName(ContractDefinition const& _contract) return _contract.name() + "_" + to_string(_contract.id()) + "_deployed"; } +string IRGenerator::implicitConstructorName(ContractDefinition const& _contract) +{ + return "constructor_" + _contract.name() + "_" + to_string(_contract.id()); +} + string IRGenerator::dispatchRoutine(ContractDefinition const& _contract) { Whiskers t(R"X( diff --git a/libsolidity/codegen/ir/IRGenerator.h b/libsolidity/codegen/ir/IRGenerator.h index c4035141ce3c..fde382cd71dc 100644 --- a/libsolidity/codegen/ir/IRGenerator.h +++ b/libsolidity/codegen/ir/IRGenerator.h @@ -67,12 +67,29 @@ class IRGenerator /// Generates code that assigns the initial value of the respective type. std::string generateInitialAssignment(VariableDeclaration const& _varDecl); - std::string constructorCode(ContractDefinition const& _contract); + /// Generates implicit constructors for all contracts in the inheritance hierarchy of + /// @a _contract + /// If there are user defined constructors, their body will be included in implicit constructors body. + void generateImplicitConstructors(ContractDefinition const& _contract); + + /// Evaluates constructor's arguments for all base contracts (listed in inheritance specifiers) of + /// @a _contract + /// @returns Pair of expressions needed to evaluate params and list of parameters in a map contract -> params + std::pair> evaluateConstructorArguments( + ContractDefinition const& _contract + ); + + /// Initializes state variables of + /// @a _contract + /// @returns Source code to initialize state variables + std::string initStateVariables(ContractDefinition const& _contract); + std::string deployCode(ContractDefinition const& _contract); std::string callValueCheck(); std::string creationObjectName(ContractDefinition const& _contract); std::string runtimeObjectName(ContractDefinition const& _contract); + std::string implicitConstructorName(ContractDefinition const& _contract); std::string dispatchRoutine(ContractDefinition const& _contract); diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index c1b4c8b216a6..71f87aa525fb 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -169,6 +169,14 @@ void IRGeneratorForStatements::initializeLocalVar(VariableDeclaration const& _va assign(m_context.localVariable(_varDecl), zero); } +IRVariable IRGeneratorForStatements::evaluateExpression(Expression const& _expression, Type const& _targetType) +{ + _expression.accept(*this); + IRVariable variable{m_context.newYulVariable(), _targetType}; + define(variable, _expression); + return variable; +} + void IRGeneratorForStatements::endVisit(VariableDeclarationStatement const& _varDeclStatement) { if (Expression const* expression = _varDeclStatement.initialValue()) diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.h b/libsolidity/codegen/ir/IRGeneratorForStatements.h index 810087a2d7f5..4b245e9c8c05 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.h +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.h @@ -51,6 +51,9 @@ class IRGeneratorForStatements: public ASTConstVisitor /// Generates code to initialize the given local variable. void initializeLocalVar(VariableDeclaration const& _varDecl); + /// Calculates expression's value and returns variable where it was stored + IRVariable evaluateExpression(Expression const& _expression, Type const& _to); + void endVisit(VariableDeclarationStatement const& _variableDeclaration) override; bool visit(Conditional const& _conditional) override; bool visit(Assignment const& _assignment) override; diff --git a/test/cmdlineTests/standard_eWasm_requested/output.json b/test/cmdlineTests/standard_eWasm_requested/output.json index 0d796bce5bae..a67539f62f49 100644 --- a/test/cmdlineTests/standard_eWasm_requested/output.json +++ b/test/cmdlineTests/standard_eWasm_requested/output.json @@ -1,6 +1,8 @@ {"contracts":{"A":{"C":{"ewasm":{"wast":"(module ;; sub-module \"C_2_deployed\" will be encoded as custom section in binary here, but is skipped in text mode. (import \"ethereum\" \"codeCopy\" (func $eth.codeCopy (param i32 i32 i32))) + (import \"ethereum\" \"revert\" (func $eth.revert (param i32 i32))) + (import \"ethereum\" \"getCallValue\" (func $eth.getCallValue (param i32))) (import \"ethereum\" \"finish\" (func $eth.finish (param i32 i32))) (memory $memory (export \"memory\") 1) (export \"main\" (func $main)) @@ -9,27 +11,30 @@ (local $_1 i64) (local $p i64) (local $r i64) - (local $hi i64) - (local $hi_1 i64) - (local $y i64) - (local $hi_2 i64) (local $_2 i64) + (local $z1 i64) + (local $z2 i64) + (local $z3 i64) + (local $_3 i64) (local.set $_1 (i64.const 0)) (local.set $p (call $u256_to_i32 (local.get $_1) (local.get $_1) (local.get $_1) (i64.const 64))) (local.set $r (i64.extend_i32_u (i32.add (i32.wrap_i64 (local.get $p)) (i32.wrap_i64 (i64.const 64))))) (if (i64.ne (i64.extend_i32_u (i32.lt_u (i32.wrap_i64 (local.get $r)) (i32.wrap_i64 (local.get $p)))) (i64.const 0)) (then (unreachable))) - (local.set $hi (i64.shl (call $endian_swap_16 (local.get $_1)) (i64.const 16))) - (local.set $hi_1 (i64.shl (i64.or (local.get $hi) (call $endian_swap_16 (i64.shr_u (local.get $_1) (i64.const 16)))) (i64.const 32))) - (local.set $y (i64.or (local.get $hi_1) (call $endian_swap_32 (i64.shr_u (local.get $_1) (i64.const 32))))) - (i64.store (i32.wrap_i64 (local.get $r)) (local.get $y)) - (i64.store (i32.wrap_i64 (i64.extend_i32_u (i32.add (i32.wrap_i64 (local.get $r)) (i32.wrap_i64 (i64.const 8))))) (local.get $y)) - (i64.store (i32.wrap_i64 (i64.extend_i32_u (i32.add (i32.wrap_i64 (local.get $r)) (i32.wrap_i64 (i64.const 16))))) (local.get $y)) - (local.set $hi_2 (i64.shl (call $endian_swap_32 (i64.const 128)) (i64.const 32))) - (i64.store (i32.wrap_i64 (i64.extend_i32_u (i32.add (i32.wrap_i64 (local.get $r)) (i32.wrap_i64 (i64.const 24))))) (i64.or (local.get $hi_2) (call $endian_swap_32 (i64.shr_u (i64.const 128) (i64.const 32))))) - (local.set $_2 (datasize \"C_2_deployed\")) - (call $eth.codeCopy (i32.wrap_i64 (call $to_internal_i32ptr (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_1))) (i32.wrap_i64 (call $u256_to_i32 (local.get $_1) (local.get $_1) (local.get $_1) (dataoffset \"C_2_deployed\"))) (i32.wrap_i64 (call $u256_to_i32 (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_2)))) - (call $eth.finish (i32.wrap_i64 (call $to_internal_i32ptr (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_1))) (i32.wrap_i64 (call $u256_to_i32 (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_2)))) + (local.set $_2 (call $endian_swap (local.get $_1))) + (i64.store (i32.wrap_i64 (local.get $r)) (local.get $_2)) + (i64.store (i32.wrap_i64 (i64.extend_i32_u (i32.add (i32.wrap_i64 (local.get $r)) (i32.wrap_i64 (i64.const 8))))) (local.get $_2)) + (i64.store (i32.wrap_i64 (i64.extend_i32_u (i32.add (i32.wrap_i64 (local.get $r)) (i32.wrap_i64 (i64.const 16))))) (local.get $_2)) + (i64.store (i32.wrap_i64 (i64.extend_i32_u (i32.add (i32.wrap_i64 (local.get $r)) (i32.wrap_i64 (i64.const 24))))) (call $endian_swap (i64.const 128))) + (call $eth.getCallValue (i32.wrap_i64 (i64.const 0))) + (local.set $z1 (call $endian_swap (i64.load (i32.wrap_i64 (i64.const 0))))) + (local.set $z2 (call $endian_swap (i64.load (i32.wrap_i64 (i64.extend_i32_u (i32.add (i32.wrap_i64 (i64.const 0)) (i32.wrap_i64 (i64.const 8)))))))) + (local.set $z3 (call $endian_swap (i64.load (i32.wrap_i64 (i64.extend_i32_u (i32.add (i32.wrap_i64 (i64.const 0)) (i32.wrap_i64 (i64.const 16)))))))) + (if (i64.ne (i64.extend_i32_u (i32.eqz (i32.wrap_i64 (i64.extend_i32_u (i64.eqz (i64.or (i64.or (local.get $z1) (local.get $z2)) (i64.or (local.get $z3) (call $endian_swap (i64.load (i32.wrap_i64 (i64.extend_i32_u (i32.add (i32.wrap_i64 (i64.const 0)) (i32.wrap_i64 (i64.const 24)))))))))))))) (i64.const 0)) (then + (call $revert (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_1)))) + (local.set $_3 (datasize \"C_2_deployed\")) + (call $codecopy (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_1) (dataoffset \"C_2_deployed\") (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_3)) + (call $return (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_1) (local.get $_3)) ) (func $u256_to_i32 @@ -62,6 +67,22 @@ (local.get $r) ) +(func $codecopy + (param $x1 i64) + (param $x2 i64) + (param $x3 i64) + (param $x4 i64) + (param $y1 i64) + (param $y2 i64) + (param $y3 i64) + (param $y4 i64) + (param $z1 i64) + (param $z2 i64) + (param $z3 i64) + (param $z4 i64) + (call $eth.codeCopy (i32.wrap_i64 (call $to_internal_i32ptr (local.get $x1) (local.get $x2) (local.get $x3) (local.get $x4))) (i32.wrap_i64 (call $u256_to_i32 (local.get $y1) (local.get $y2) (local.get $y3) (local.get $y4))) (i32.wrap_i64 (call $u256_to_i32 (local.get $z1) (local.get $z2) (local.get $z3) (local.get $z4)))) +) + (func $endian_swap_16 (param $x i64) (result i64) @@ -80,5 +101,39 @@ (local.get $y) ) +(func $endian_swap + (param $x i64) + (result i64) + (local $y i64) + (local $hi i64) + (local.set $hi (i64.shl (call $endian_swap_32 (local.get $x)) (i64.const 32))) + (local.set $y (i64.or (local.get $hi) (call $endian_swap_32 (i64.shr_u (local.get $x) (i64.const 32))))) + (local.get $y) +) + +(func $return + (param $x1 i64) + (param $x2 i64) + (param $x3 i64) + (param $x4 i64) + (param $y1 i64) + (param $y2 i64) + (param $y3 i64) + (param $y4 i64) + (call $eth.finish (i32.wrap_i64 (call $to_internal_i32ptr (local.get $x1) (local.get $x2) (local.get $x3) (local.get $x4))) (i32.wrap_i64 (call $u256_to_i32 (local.get $y1) (local.get $y2) (local.get $y3) (local.get $y4)))) +) + +(func $revert + (param $x1 i64) + (param $x2 i64) + (param $x3 i64) + (param $x4 i64) + (param $y1 i64) + (param $y2 i64) + (param $y3 i64) + (param $y4 i64) + (call $eth.revert (i32.wrap_i64 (call $to_internal_i32ptr (local.get $x1) (local.get $x2) (local.get $x3) (local.get $x4))) (i32.wrap_i64 (call $u256_to_i32 (local.get $y1) (local.get $y2) (local.get $y3) (local.get $y4)))) +) + ) "}}}},"sources":{"A":{"id":0}}} diff --git a/test/cmdlineTests/standard_irOptimized_requested/output.json b/test/cmdlineTests/standard_irOptimized_requested/output.json index 78ebe59c3186..ba3cb8b2ad93 100644 --- a/test/cmdlineTests/standard_irOptimized_requested/output.json +++ b/test/cmdlineTests/standard_irOptimized_requested/output.json @@ -8,8 +8,12 @@ object \"C_6\" { code { mstore(64, 128) + if callvalue() { revert(0, 0) } + constructor_C_6() codecopy(0, dataoffset(\"C_6_deployed\"), datasize(\"C_6_deployed\")) return(0, datasize(\"C_6_deployed\")) + function constructor_C_6() + { } } object \"C_6_deployed\" { code { diff --git a/test/cmdlineTests/standard_ir_requested/output.json b/test/cmdlineTests/standard_ir_requested/output.json index 2ceee9da0258..99fcc85b9b97 100644 --- a/test/cmdlineTests/standard_ir_requested/output.json +++ b/test/cmdlineTests/standard_ir_requested/output.json @@ -9,15 +9,21 @@ object \"C_6\" { code { mstore(64, 128) + if callvalue() { revert(0, 0) } - // Begin state variable initialization for contract \"C\" (0 variables) - // End state variable initialization for contract \"C\". - + constructor_C_6() codecopy(0, dataoffset(\"C_6_deployed\"), datasize(\"C_6_deployed\")) return(0, datasize(\"C_6_deployed\")) + function constructor_C_6() { + + + + + } + } object \"C_6_deployed\" { code { diff --git a/test/cmdlineTests/yul_string_format_ascii/output.json b/test/cmdlineTests/yul_string_format_ascii/output.json index 0e88aa5aa4cb..7c5c7a0beef4 100644 --- a/test/cmdlineTests/yul_string_format_ascii/output.json +++ b/test/cmdlineTests/yul_string_format_ascii/output.json @@ -9,15 +9,21 @@ object \"C_10\" { code { mstore(64, 128) + if callvalue() { revert(0, 0) } - // Begin state variable initialization for contract \"C\" (0 variables) - // End state variable initialization for contract \"C\". - + constructor_C_10() codecopy(0, dataoffset(\"C_10_deployed\"), datasize(\"C_10_deployed\")) return(0, datasize(\"C_10_deployed\")) + function constructor_C_10() { + + + + + } + } object \"C_10_deployed\" { code { diff --git a/test/cmdlineTests/yul_string_format_ascii_bytes32/output.json b/test/cmdlineTests/yul_string_format_ascii_bytes32/output.json index 256b1e4e14a8..763be945ef8f 100644 --- a/test/cmdlineTests/yul_string_format_ascii_bytes32/output.json +++ b/test/cmdlineTests/yul_string_format_ascii_bytes32/output.json @@ -9,15 +9,21 @@ object \"C_10\" { code { mstore(64, 128) + if callvalue() { revert(0, 0) } - // Begin state variable initialization for contract \"C\" (0 variables) - // End state variable initialization for contract \"C\". - + constructor_C_10() codecopy(0, dataoffset(\"C_10_deployed\"), datasize(\"C_10_deployed\")) return(0, datasize(\"C_10_deployed\")) + function constructor_C_10() { + + + + + } + } object \"C_10_deployed\" { code { diff --git a/test/cmdlineTests/yul_string_format_ascii_bytes32_from_number/output.json b/test/cmdlineTests/yul_string_format_ascii_bytes32_from_number/output.json index abb929c1923f..55b35f138658 100644 --- a/test/cmdlineTests/yul_string_format_ascii_bytes32_from_number/output.json +++ b/test/cmdlineTests/yul_string_format_ascii_bytes32_from_number/output.json @@ -9,15 +9,21 @@ object \"C_10\" { code { mstore(64, 128) + if callvalue() { revert(0, 0) } - // Begin state variable initialization for contract \"C\" (0 variables) - // End state variable initialization for contract \"C\". - + constructor_C_10() codecopy(0, dataoffset(\"C_10_deployed\"), datasize(\"C_10_deployed\")) return(0, datasize(\"C_10_deployed\")) + function constructor_C_10() { + + + + + } + } object \"C_10_deployed\" { code { diff --git a/test/cmdlineTests/yul_string_format_ascii_long/output.json b/test/cmdlineTests/yul_string_format_ascii_long/output.json index 991f9d0fd4b3..289846431a59 100644 --- a/test/cmdlineTests/yul_string_format_ascii_long/output.json +++ b/test/cmdlineTests/yul_string_format_ascii_long/output.json @@ -9,15 +9,21 @@ object \"C_10\" { code { mstore(64, 128) + if callvalue() { revert(0, 0) } - // Begin state variable initialization for contract \"C\" (0 variables) - // End state variable initialization for contract \"C\". - + constructor_C_10() codecopy(0, dataoffset(\"C_10_deployed\"), datasize(\"C_10_deployed\")) return(0, datasize(\"C_10_deployed\")) + function constructor_C_10() { + + + + + } + } object \"C_10_deployed\" { code { diff --git a/test/cmdlineTests/yul_string_format_hex/output.json b/test/cmdlineTests/yul_string_format_hex/output.json index ff84b1c323db..726c6ccf2c91 100644 --- a/test/cmdlineTests/yul_string_format_hex/output.json +++ b/test/cmdlineTests/yul_string_format_hex/output.json @@ -9,15 +9,21 @@ object \"C_10\" { code { mstore(64, 128) + if callvalue() { revert(0, 0) } - // Begin state variable initialization for contract \"C\" (0 variables) - // End state variable initialization for contract \"C\". - + constructor_C_10() codecopy(0, dataoffset(\"C_10_deployed\"), datasize(\"C_10_deployed\")) return(0, datasize(\"C_10_deployed\")) + function constructor_C_10() { + + + + + } + } object \"C_10_deployed\" { code { diff --git a/test/libsolidity/semanticTests/constructor_ihneritance_init_order_2.sol b/test/libsolidity/semanticTests/constructor_ihneritance_init_order_2.sol new file mode 100644 index 000000000000..db5f4556039e --- /dev/null +++ b/test/libsolidity/semanticTests/constructor_ihneritance_init_order_2.sol @@ -0,0 +1,14 @@ +contract A { + uint x = 42; + function f() public returns(uint256) { + return x; + } +} +contract B is A { + uint public y = f(); +} +// ==== +// compileViaYul: true +// ---- +// constructor() -> +// y() -> 42 diff --git a/test/libsolidity/semanticTests/constructor_inheritance_init_order.sol b/test/libsolidity/semanticTests/constructor_inheritance_init_order.sol new file mode 100644 index 000000000000..898edf6ad954 --- /dev/null +++ b/test/libsolidity/semanticTests/constructor_inheritance_init_order.sol @@ -0,0 +1,17 @@ +contract A { + uint x; + constructor() public { + x = 42; + } + function f() public returns(uint256) { + return x; + } +} +contract B is A { + uint public y = f(); +} +// ==== +// compileViaYul: true +// ---- +// constructor() -> +// y() -> 42 diff --git a/test/libsolidity/semanticTests/constructor_with_params_diamond_inheritance.sol b/test/libsolidity/semanticTests/constructor_with_params_diamond_inheritance.sol new file mode 100644 index 000000000000..0ec3479638ba --- /dev/null +++ b/test/libsolidity/semanticTests/constructor_with_params_diamond_inheritance.sol @@ -0,0 +1,28 @@ +contract A { + uint public i; + uint public k; + + constructor(uint newI, uint newK) public { + i = newI; + k = newK; + } +} +abstract contract B is A { + uint public j; + constructor(uint newJ) public { + j = newJ; + } +} +contract C is A { + constructor(uint newI, uint newK) A(newI, newK) public {} +} +contract D is B, C { + constructor(uint newI, uint newK) B(newI) C(newI, newK + 1) public {} +} +// ==== +// compileViaYul: also +// ---- +// constructor(): 2, 0 -> +// i() -> 2 +// j() -> 2 +// k() -> 1 diff --git a/test/libsolidity/semanticTests/constructor_with_params_inheritance.sol b/test/libsolidity/semanticTests/constructor_with_params_inheritance.sol new file mode 100644 index 000000000000..3419a9c25a93 --- /dev/null +++ b/test/libsolidity/semanticTests/constructor_with_params_inheritance.sol @@ -0,0 +1,18 @@ +contract C { + uint public i; + uint public k; + + constructor(uint newI, uint newK) public { + i = newI; + k = newK; + } +} +contract D is C { + constructor(uint newI, uint newK) C(newI, newK + 1) public {} +} +// ==== +// compileViaYul: also +// ---- +// constructor(): 2, 0 -> +// i() -> 2 +// k() -> 1 diff --git a/test/libsolidity/semanticTests/constructor_with_params_inheritance_2.sol b/test/libsolidity/semanticTests/constructor_with_params_inheritance_2.sol new file mode 100644 index 000000000000..f2c39e75edca --- /dev/null +++ b/test/libsolidity/semanticTests/constructor_with_params_inheritance_2.sol @@ -0,0 +1,15 @@ +contract C { + uint public i; + uint public k; + + constructor(uint newI, uint newK) public { + i = newI; + k = newK; + } +} +contract D is C(2, 1) {} +// ==== +// compileViaYul: also +// ---- +// i() -> 2 +// k() -> 1 diff --git a/test/libsolidity/semanticTests/state_variables_init_order.sol b/test/libsolidity/semanticTests/state_variables_init_order.sol new file mode 100644 index 000000000000..ef63a38f964e --- /dev/null +++ b/test/libsolidity/semanticTests/state_variables_init_order.sol @@ -0,0 +1,14 @@ +contract A { + uint public x = 0; + uint y = f(); + function f() public returns (uint256) { + ++x; + return 42; + } +} +contract B is A { +} +// ==== +// compileViaYul: also +// ---- +// x() -> 1 \ No newline at end of file diff --git a/test/libsolidity/semanticTests/state_variables_init_order_2.sol b/test/libsolidity/semanticTests/state_variables_init_order_2.sol new file mode 100644 index 000000000000..9bdf9c70cfff --- /dev/null +++ b/test/libsolidity/semanticTests/state_variables_init_order_2.sol @@ -0,0 +1,18 @@ +contract A { + uint public x = 0; + uint y = f(); + function f() public returns (uint256) { + ++x; + return 42; + } +} +contract B is A { + uint public z; + constructor() public { + z = x; + } +} +// ==== +// compileViaYul: also +// ---- +// z() -> 1 \ No newline at end of file diff --git a/test/libsolidity/semanticTests/state_variables_init_order_3.sol b/test/libsolidity/semanticTests/state_variables_init_order_3.sol new file mode 100644 index 000000000000..81a5a693d4cb --- /dev/null +++ b/test/libsolidity/semanticTests/state_variables_init_order_3.sol @@ -0,0 +1,29 @@ +contract A { + uint public a = 42; + uint public b; + uint public c; + constructor(uint x) public { + b = a; + a = x; + } + function f(uint x) public returns (uint256) { c = x * 3; return 23; } +} +contract B is A { + uint public d = f(a); + uint public e = b; + uint public b_a; + uint public b_b; + uint public b_c; + constructor() public A(17) { b_a = a; b_b = b; b_c = c; } +} +// ==== +// compileViaYul: true +// ---- +// a() -> 17 +// b() -> 42 +// c() -> 51 +// b_a() -> 17 +// b_b() -> 42 +// b_c() -> 51 +// d() -> 23 +// e() -> 42 From 83c9e82099cda58806ec638864e25c8f90d92dce Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Wed, 22 Apr 2020 19:57:00 +0200 Subject: [PATCH 068/126] Fix ICE with fixed point --- Changelog.md | 1 + libsolidity/formal/BMC.cpp | 5 ++++- libsolidity/formal/SMTEncoder.cpp | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 07ab24363173..50a3e4d1ec62 100644 --- a/Changelog.md +++ b/Changelog.md @@ -7,6 +7,7 @@ Compiler Features: Bugfixes: + * SMTChecker: Fix internal error when fixed points are used. * Type Checker: Disallow ``virtual`` and ``override`` for constructors. * Type Checker: Fix several internal errors by performing size and recursiveness checks of types before the full type checking. * Type Checker: Perform recursiveness check on structs declared at the file level. diff --git a/libsolidity/formal/BMC.cpp b/libsolidity/formal/BMC.cpp index 1b4e8174a001..13a19694b194 100644 --- a/libsolidity/formal/BMC.cpp +++ b/libsolidity/formal/BMC.cpp @@ -304,7 +304,10 @@ void BMC::endVisit(UnaryOperation const& _op) { SMTEncoder::endVisit(_op); - if (_op.annotation().type->category() == Type::Category::RationalNumber) + if ( + _op.annotation().type->category() == Type::Category::RationalNumber || + _op.annotation().type->category() == Type::Category::FixedPoint + ) return; switch (_op.getOperator()) diff --git a/libsolidity/formal/SMTEncoder.cpp b/libsolidity/formal/SMTEncoder.cpp index 820af022ec30..12c9971dc222 100644 --- a/libsolidity/formal/SMTEncoder.cpp +++ b/libsolidity/formal/SMTEncoder.cpp @@ -457,6 +457,9 @@ void SMTEncoder::endVisit(UnaryOperation const& _op) createExpr(_op); + if (_op.annotation().type->category() == Type::Category::FixedPoint) + return; + switch (_op.getOperator()) { case Token::Not: // ! From 1cb68b1be7881274d521435a6ef7dc14e472b5d2 Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Wed, 22 Apr 2020 20:15:01 +0200 Subject: [PATCH 069/126] Add internal function calls to CHC docs --- docs/security-considerations.rst | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/docs/security-considerations.rst b/docs/security-considerations.rst index b4737049351b..cf706fb7b100 100644 --- a/docs/security-considerations.rst +++ b/docs/security-considerations.rst @@ -491,7 +491,8 @@ Horn clauses, where the lifecycle of the contract is represented by a loop that can visit every public/external function non-deterministically. This way, the behavior of the entire contract over an unbounded number of transactions is taken into account when analyzing any function. Loops are fully supported -by this engine. Function calls are currently unsupported. +by this engine. Internal function calls are supported, but external function +calls are currently unsupported. The CHC engine is much more powerful than BMC in terms of what it can prove, and might require more computing resources. @@ -505,10 +506,16 @@ erasing knowledge or using a non-precise type). If it determines that a verification target is safe, it is indeed safe, that is, there are no false negatives (unless there is a bug in the SMTChecker). -Function calls to the same contract (or base contracts) are inlined when -possible, that is, when their implementation is available. -Calls to functions in other contracts are not inlined even if their code is +In the BMC engine, function calls to the same contract (or base contracts) are +inlined when possible, that is, when their implementation is available. Calls +to functions in other contracts are not inlined even if their code is available, since we cannot guarantee that the actual deployed code is the same. + +The CHC engine creates nonlinear Horn clauses that use summaries of the called +functions to support internal function calls. The same approach can and will be +used for external function calls, but the latter requires more work regarding +the entire state of the blockchain and is still unimplemented. + Complex pure functions are abstracted by an uninterpreted function (UF) over the arguments. @@ -519,11 +526,14 @@ the arguments. +-----------------------------------+--------------------------------------+ |``require`` |Assumption | +-----------------------------------+--------------------------------------+ -|internal |Inline function call | +|internal |BMC: Inline function call | +| |CHC: Function summaries | +-----------------------------------+--------------------------------------+ -|external |Inline function call | -| |Erase knowledge about state variables | -| |and local storage references | +|external |BMC: Inline function call or | +| |erase knowledge about state variables | +| |and local storage references. | +| |CHC: Function summaries and erase | +| |state knowledge. | +-----------------------------------+--------------------------------------+ |``gasleft``, ``blockhash``, |Abstracted with UF | |``keccak256``, ``ecrecover`` | | @@ -534,8 +544,8 @@ the arguments. |implementation (external or | | |complex) | | +-----------------------------------+--------------------------------------+ -|external functions without |Unsupported | -|implementation | | +|external functions without |BMC: Unsupported | +|implementation |CHC: Nondeterministic summary | +-----------------------------------+--------------------------------------+ |others |Currently unsupported | +-----------------------------------+--------------------------------------+ From b191139f2a299da4a036beb416d2637dd01195e6 Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Wed, 22 Apr 2020 20:41:24 +0200 Subject: [PATCH 070/126] Fix undefined behavior with nullptr --- libsolidity/formal/Z3Interface.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libsolidity/formal/Z3Interface.cpp b/libsolidity/formal/Z3Interface.cpp index f175e45d25e2..a84f39bb1ff5 100644 --- a/libsolidity/formal/Z3Interface.cpp +++ b/libsolidity/formal/Z3Interface.cpp @@ -237,8 +237,8 @@ z3::sort Z3Interface::z3Sort(Sort const& _sort) z3::func_decl tupleConstructor = m_context.tuple_sort( tupleSort.name.c_str(), tupleSort.members.size(), - &cMembers[0], - &sorts[0], + cMembers.data(), + sorts.data(), projs ); return tupleConstructor.range(); From cfe368611655194b0bb1be5d7513b606135f4454 Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Wed, 22 Apr 2020 18:54:38 +0200 Subject: [PATCH 071/126] Fix internal error when using array slices --- Changelog.md | 1 + libsolidity/formal/SymbolicTypes.cpp | 13 ++++++++++--- .../smtCheckerTests/operators/slices_1.sol | 13 +++++++++++++ .../smtCheckerTests/typecast/slice_to_bytes.sol | 13 +++++++++++++ 4 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 test/libsolidity/smtCheckerTests/operators/slices_1.sol create mode 100644 test/libsolidity/smtCheckerTests/typecast/slice_to_bytes.sol diff --git a/Changelog.md b/Changelog.md index 50a3e4d1ec62..f37d9e09623e 100644 --- a/Changelog.md +++ b/Changelog.md @@ -8,6 +8,7 @@ Compiler Features: Bugfixes: * SMTChecker: Fix internal error when fixed points are used. + * SMTChecker: Fix internal error when using array slices. * Type Checker: Disallow ``virtual`` and ``override`` for constructors. * Type Checker: Fix several internal errors by performing size and recursiveness checks of types before the full type checking. * Type Checker: Perform recursiveness check on structs declared at the file level. diff --git a/libsolidity/formal/SymbolicTypes.cpp b/libsolidity/formal/SymbolicTypes.cpp index 0364a04580c3..b8da5da0cd5e 100644 --- a/libsolidity/formal/SymbolicTypes.cpp +++ b/libsolidity/formal/SymbolicTypes.cpp @@ -69,8 +69,14 @@ SortPointer smtSort(frontend::Type const& _type) } else { - solAssert(isArray(_type.category()), ""); - auto arrayType = dynamic_cast(&_type); + frontend::ArrayType const* arrayType = nullptr; + if (auto const* arr = dynamic_cast(&_type)) + arrayType = arr; + else if (auto const* slice = dynamic_cast(&_type)) + arrayType = &slice->arrayType(); + else + solAssert(false, ""); + solAssert(arrayType, ""); return make_shared(SortProvider::intSort, smtSortAbstractFunction(*arrayType->baseType())); } @@ -297,7 +303,8 @@ bool isMapping(frontend::Type::Category _category) bool isArray(frontend::Type::Category _category) { return _category == frontend::Type::Category::Array || - _category == frontend::Type::Category::StringLiteral; + _category == frontend::Type::Category::StringLiteral || + _category == frontend::Type::Category::ArraySlice; } bool isTuple(frontend::Type::Category _category) diff --git a/test/libsolidity/smtCheckerTests/operators/slices_1.sol b/test/libsolidity/smtCheckerTests/operators/slices_1.sol new file mode 100644 index 000000000000..c723472a86b4 --- /dev/null +++ b/test/libsolidity/smtCheckerTests/operators/slices_1.sol @@ -0,0 +1,13 @@ +pragma experimental SMTChecker; + +contract C { + function f(bytes calldata x) external pure { + x[:18726387213]; + x[18726387213:]; + x[18726387213:111111111111111111]; + } +} +// ---- +// Warning: (94-109): Assertion checker does not yet implement this expression. +// Warning: (113-128): Assertion checker does not yet implement this expression. +// Warning: (132-165): Assertion checker does not yet implement this expression. diff --git a/test/libsolidity/smtCheckerTests/typecast/slice_to_bytes.sol b/test/libsolidity/smtCheckerTests/typecast/slice_to_bytes.sol new file mode 100644 index 000000000000..1fb74bac9cec --- /dev/null +++ b/test/libsolidity/smtCheckerTests/typecast/slice_to_bytes.sol @@ -0,0 +1,13 @@ +pragma experimental SMTChecker; + +contract C { + function f(bytes calldata x) external pure { + bytes(x[:18726387213]); + bytes(x[18726387213:]); + bytes(x[18726387213:111111111111111111]); + } +} +// ---- +// Warning: (100-115): Assertion checker does not yet implement this expression. +// Warning: (126-141): Assertion checker does not yet implement this expression. +// Warning: (152-185): Assertion checker does not yet implement this expression. From 615668bfb4dd5fbe7df0619b537f17e680e2b6fb Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 23 Apr 2020 01:35:21 +0200 Subject: [PATCH 072/126] Explain nonpayable Fixes https://github.com/ethereum/solidity/issues/8736 --- docs/abi-spec.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/abi-spec.rst b/docs/abi-spec.rst index 59636acb6038..ec31654dd710 100644 --- a/docs/abi-spec.rst +++ b/docs/abi-spec.rst @@ -507,13 +507,17 @@ A function description is a JSON object with the fields: - ``outputs``: an array of objects similar to ``inputs``. - ``stateMutability``: a string with one of the following values: ``pure`` (:ref:`specified to not read blockchain state `), ``view`` (:ref:`specified to not modify the blockchain - state `), ``nonpayable`` (function does not accept Ether) and ``payable`` (function accepts Ether). + state `), ``nonpayable`` (function does not accept Ether - the default) and ``payable`` (function accepts Ether). Constructor and fallback function never have ``name`` or ``outputs``. Fallback function doesn't have ``inputs`` either. .. note:: Sending non-zero Ether to non-payable function will revert the transaction. +.. note:: + The state mutability ``nonpayable`` is reflected in Solidity by not specifying + a state mutability modifier at all. + An event description is a JSON object with fairly similar fields: - ``type``: always ``"event"`` From 9538024c81368d866788a360ab81e527c57a38bd Mon Sep 17 00:00:00 2001 From: a3d4 Date: Thu, 23 Apr 2020 05:48:02 +0200 Subject: [PATCH 073/126] Fix #8711, #8277 --- Changelog.md | 1 + libsolidity/analysis/TypeChecker.cpp | 2 +- .../tupleAssignments/empty_tuples_lhs.sol | 24 +++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 test/libsolidity/syntaxTests/tupleAssignments/empty_tuples_lhs.sol diff --git a/Changelog.md b/Changelog.md index 50a3e4d1ec62..30f0e3368b94 100644 --- a/Changelog.md +++ b/Changelog.md @@ -10,6 +10,7 @@ Bugfixes: * SMTChecker: Fix internal error when fixed points are used. * Type Checker: Disallow ``virtual`` and ``override`` for constructors. * Type Checker: Fix several internal errors by performing size and recursiveness checks of types before the full type checking. + * Type Checker: Fix internal error when assigning to empty tuples. * Type Checker: Perform recursiveness check on structs declared at the file level. Build System: diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index 17add7a66696..db88f20bf123 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -1289,7 +1289,7 @@ void TypeChecker::checkExpressionAssignment(Type const& _type, Expression const& if (auto const* tupleExpression = dynamic_cast(&_expression)) { auto const* tupleType = dynamic_cast(&_type); - auto const& types = tupleType && tupleExpression->components().size() > 1 ? tupleType->components() : vector { &_type }; + auto const& types = tupleType && tupleExpression->components().size() != 1 ? tupleType->components() : vector { &_type }; solAssert( tupleExpression->components().size() == types.size() || m_errorReporter.hasErrors(), diff --git a/test/libsolidity/syntaxTests/tupleAssignments/empty_tuples_lhs.sol b/test/libsolidity/syntaxTests/tupleAssignments/empty_tuples_lhs.sol new file mode 100644 index 000000000000..390a3636aaf7 --- /dev/null +++ b/test/libsolidity/syntaxTests/tupleAssignments/empty_tuples_lhs.sol @@ -0,0 +1,24 @@ +contract C { + function f0() public { (()) = 2; } + + function f1() public pure { (()) = (); } + + //#8711 + function f2() internal pure returns (uint, uint) { return () = f2(); } + + //#8277 + function f3()public{return()=();} + + //#8277 + function f4 ( bytes32 hash , uint8 v , bytes32 r , bytes32 s , uint blockExpired , bytes32 salt ) public returns ( address ) { + require ( ( ( ) ) |= keccak256 ( abi . encodePacked ( blockExpired , salt ) ) ) ; + return ecrecover ( hash , v , r , s ) ; + } +} +// ---- +// TypeError: (47-48): Type int_const 2 is not implicitly convertible to expected type tuple(). +// TypeError: (178-182): Type tuple(uint256,uint256) is not implicitly convertible to expected type tuple(). +// TypeError: (166-182): Different number of arguments in return statement than in returns declaration. +// TypeError: (399-466): Compound assignment is not allowed for tuple types. +// TypeError: (410-466): Type bytes32 is not implicitly convertible to expected type tuple(). +// TypeError: (389-396): No matching declaration found after argument-dependent lookup. From c4bc77874bf029f60daecbba959e196708ff4124 Mon Sep 17 00:00:00 2001 From: a3d4 Date: Thu, 23 Apr 2020 07:02:04 +0200 Subject: [PATCH 074/126] Disallow empty tuples on the left hand side --- libsolidity/analysis/TypeChecker.cpp | 3 +++ .../syntaxTests/tupleAssignments/empty_tuples_lhs.sol | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index db88f20bf123..f0248538e823 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -1288,6 +1288,9 @@ void TypeChecker::checkExpressionAssignment(Type const& _type, Expression const& { if (auto const* tupleExpression = dynamic_cast(&_expression)) { + if (tupleExpression->components().empty()) + m_errorReporter.typeError(_expression.location(), "Empty tuple on the left hand side."); + auto const* tupleType = dynamic_cast(&_type); auto const& types = tupleType && tupleExpression->components().size() != 1 ? tupleType->components() : vector { &_type }; diff --git a/test/libsolidity/syntaxTests/tupleAssignments/empty_tuples_lhs.sol b/test/libsolidity/syntaxTests/tupleAssignments/empty_tuples_lhs.sol index 390a3636aaf7..a771f630ace5 100644 --- a/test/libsolidity/syntaxTests/tupleAssignments/empty_tuples_lhs.sol +++ b/test/libsolidity/syntaxTests/tupleAssignments/empty_tuples_lhs.sol @@ -16,9 +16,14 @@ contract C { } } // ---- +// TypeError: (41-43): Empty tuple on the left hand side. // TypeError: (47-48): Type int_const 2 is not implicitly convertible to expected type tuple(). +// TypeError: (86-88): Empty tuple on the left hand side. +// TypeError: (173-175): Empty tuple on the left hand side. // TypeError: (178-182): Type tuple(uint256,uint256) is not implicitly convertible to expected type tuple(). // TypeError: (166-182): Different number of arguments in return statement than in returns declaration. +// TypeError: (229-231): Empty tuple on the left hand side. +// TypeError: (401-404): Empty tuple on the left hand side. // TypeError: (399-466): Compound assignment is not allowed for tuple types. // TypeError: (410-466): Type bytes32 is not implicitly convertible to expected type tuple(). // TypeError: (389-396): No matching declaration found after argument-dependent lookup. From f78414b3337af41d00aabc2784b8d5b072be6d6d Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Wed, 22 Apr 2020 15:06:42 +0200 Subject: [PATCH 075/126] Disallow statements containing empty blocks e.g., empty if, for, function definition --- test/tools/ossfuzz/protoToYul.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/test/tools/ossfuzz/protoToYul.cpp b/test/tools/ossfuzz/protoToYul.cpp index 5414405c6eaf..5589043de2e1 100644 --- a/test/tools/ossfuzz/protoToYul.cpp +++ b/test/tools/ossfuzz/protoToYul.cpp @@ -1302,19 +1302,23 @@ void ProtoConverter::visit(Statement const& _x) visit(_x.assignment()); break; case Statement::kIfstmt: - visit(_x.ifstmt()); + if (_x.ifstmt().if_body().statements_size() > 0) + visit(_x.ifstmt()); break; case Statement::kStorageFunc: visit(_x.storage_func()); break; case Statement::kBlockstmt: - visit(_x.blockstmt()); + if (_x.blockstmt().statements_size() > 0) + visit(_x.blockstmt()); break; case Statement::kForstmt: - visit(_x.forstmt()); + if (_x.forstmt().for_body().statements_size() > 0) + visit(_x.forstmt()); break; case Statement::kBoundedforstmt: - visit(_x.boundedforstmt()); + if (_x.boundedforstmt().for_body().statements_size() > 0) + visit(_x.boundedforstmt()); break; case Statement::kSwitchstmt: visit(_x.switchstmt()); @@ -1346,8 +1350,9 @@ void ProtoConverter::visit(Statement const& _x) visit(_x.functioncall()); break; case Statement::kFuncdef: - if (!m_inForInitScope) - visit(_x.funcdef()); + if (_x.funcdef().block().statements_size() > 0) + if (!m_inForInitScope) + visit(_x.funcdef()); break; case Statement::kPop: visit(_x.pop()); @@ -1524,7 +1529,7 @@ void ProtoConverter::visit(Block const& _x) // scope belongs to for-init (in which function declarations // are forbidden). for (auto const& statement: _x.statements()) - if (statement.has_funcdef() && !m_inForInitScope) + if (statement.has_funcdef() && statement.funcdef().block().statements_size() > 0 && !m_inForInitScope) registerFunction(&statement.funcdef()); if (_x.statements_size() > 0) From 92059fa84848ce7f78d93a8af720bef034b74fde Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Thu, 23 Apr 2020 10:40:57 +0200 Subject: [PATCH 076/126] Use Spacer option to improve performance of constant arrays --- libsolidity/formal/Z3CHCInterface.cpp | 2 ++ .../functions/functions_recursive_indirect.sol | 1 - ...for_loop_array_assignment_storage_memory.sol | 4 +--- ...or_loop_array_assignment_storage_storage.sol | 1 - ...le_loop_array_assignment_storage_storage.sol | 6 ++---- .../operators/delete_array_index_2d.sol | 1 + .../smtCheckerTests/types/unused_mapping.sol | 17 +++++++++++++++++ 7 files changed, 23 insertions(+), 9 deletions(-) create mode 100644 test/libsolidity/smtCheckerTests/types/unused_mapping.sol diff --git a/libsolidity/formal/Z3CHCInterface.cpp b/libsolidity/formal/Z3CHCInterface.cpp index c64da2edcc81..dcfd19c8617c 100644 --- a/libsolidity/formal/Z3CHCInterface.cpp +++ b/libsolidity/formal/Z3CHCInterface.cpp @@ -43,6 +43,8 @@ Z3CHCInterface::Z3CHCInterface(): p.set("fp.spacer.mbqi", false); // Ground pobs by using values from a model. p.set("fp.spacer.ground_pobs", false); + // Limits array reasoning, good for constant arrays. + p.set("fp.spacer.weak_abs", false); m_solver.set(p); } diff --git a/test/libsolidity/smtCheckerTests/functions/functions_recursive_indirect.sol b/test/libsolidity/smtCheckerTests/functions/functions_recursive_indirect.sol index 5d3292992ad3..7cb7b22b1aaa 100644 --- a/test/libsolidity/smtCheckerTests/functions/functions_recursive_indirect.sol +++ b/test/libsolidity/smtCheckerTests/functions/functions_recursive_indirect.sol @@ -22,4 +22,3 @@ contract C } } // ---- -// Warning: (130-144): Error trying to invoke SMT solver. diff --git a/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_memory.sol b/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_memory.sol index 0b301505b4b0..d7d88424a2e9 100644 --- a/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_memory.sol +++ b/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_memory.sol @@ -12,12 +12,10 @@ contract LoopFor2 { b[i] = i + 1; c[i] = b[i]; } - // This is safe but too hard to solve currently. assert(b[0] == c[0]); assert(a[0] == 900); assert(b[0] == 900); } } // ---- -// Warning: (316-336): Assertion violation happens here -// Warning: (363-382): Assertion violation happens here +// Warning: (312-331): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_storage.sol b/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_storage.sol index 333273781373..a90053024d47 100644 --- a/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_storage.sol +++ b/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_storage.sol @@ -19,6 +19,5 @@ contract LoopFor2 { } } // ---- -// Warning: (317-337): Assertion violation happens here // Warning: (341-360): Assertion violation happens here // Warning: (364-383): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/loops/while_loop_array_assignment_storage_storage.sol b/test/libsolidity/smtCheckerTests/loops/while_loop_array_assignment_storage_storage.sol index 92d1ded3ee46..7670ddb6d88b 100644 --- a/test/libsolidity/smtCheckerTests/loops/while_loop_array_assignment_storage_storage.sol +++ b/test/libsolidity/smtCheckerTests/loops/while_loop_array_assignment_storage_storage.sol @@ -14,13 +14,11 @@ contract LoopFor2 { c[i] = b[i]; ++i; } - // Fails as false positive. assert(b[0] == c[0]); assert(a[0] == 900); assert(b[0] == 900); } } // ---- -// Warning: (296-316): Assertion violation happens here -// Warning: (320-339): Assertion violation happens here -// Warning: (343-362): Assertion violation happens here +// Warning: (290-309): Assertion violation happens here +// Warning: (313-332): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol b/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol index 8a1ba5e6ef90..cb5a0799fea1 100644 --- a/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol +++ b/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol @@ -16,4 +16,5 @@ contract C // ==== // SMTSolvers: z3 // ---- +// Warning: (174-194): Error trying to invoke SMT solver. // Warning: (174-194): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/types/unused_mapping.sol b/test/libsolidity/smtCheckerTests/types/unused_mapping.sol new file mode 100644 index 000000000000..f12cd41de5fc --- /dev/null +++ b/test/libsolidity/smtCheckerTests/types/unused_mapping.sol @@ -0,0 +1,17 @@ +pragma experimental SMTChecker; + +contract C { + uint x; + uint y; + mapping (address => bool) public never_used; + + function inc() public { + require(x < 10); + require(y < 10); + + if(x == 0) x = 0; // noop state var read + x++; + y++; + assert(y == x); + } +} From b864fe1c43b9bb85fb7ab9cc41f7251d2fa5ef66 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 23 Apr 2020 12:13:26 +0200 Subject: [PATCH 077/126] Remove unnecessary move. --- libsolidity/codegen/ir/IRGeneratorForStatements.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index e375739d0c83..711f0f1f6825 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -1069,7 +1069,7 @@ bool IRGeneratorForStatements::visit(InlineAssembly const& _inlineAsm) solAssert(holds_alternative(modified), ""); // Do not provide dialect so that we get the full type information. - m_code << yul::AsmPrinter()(std::get(std::move(modified))) << "\n"; + m_code << yul::AsmPrinter()(std::get(modified)) << "\n"; return false; } From 35eae96a7f93a16d939f348e22d33ad112007e7d Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 23 Apr 2020 13:16:10 +0200 Subject: [PATCH 078/126] Move helper up and avoid trailing spaces. --- test/TestCase.cpp | 13 +++++++++++++ test/TestCase.h | 2 ++ test/libsolidity/ABIJsonTest.cpp | 15 --------------- test/libsolidity/ABIJsonTest.h | 2 -- test/libyul/EwasmTranslationTest.cpp | 8 -------- test/libyul/EwasmTranslationTest.h | 1 - test/libyul/YulInterpreterTest.cpp | 8 -------- test/libyul/YulInterpreterTest.h | 1 - test/libyul/YulOptimizerTest.cpp | 8 -------- test/libyul/YulOptimizerTest.h | 1 - 10 files changed, 15 insertions(+), 44 deletions(-) diff --git a/test/TestCase.cpp b/test/TestCase.cpp index f952e40bdaa4..e0897ae47c47 100644 --- a/test/TestCase.cpp +++ b/test/TestCase.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -60,6 +61,18 @@ void TestCase::expect(string::iterator& _it, string::iterator _end, string::valu ++_it; } +void TestCase::printIndented(ostream& _stream, string const& _output, string const& _linePrefix) const +{ + stringstream output(_output); + string line; + while (getline(output, line)) + if (line.empty()) + // Avoid trailing spaces. + _stream << boost::trim_right_copy(_linePrefix) << endl; + else + _stream << _linePrefix << line << endl; +} + EVMVersionRestrictedTestCase::EVMVersionRestrictedTestCase(string const& _filename): TestCase(_filename) { diff --git a/test/TestCase.h b/test/TestCase.h index 0add6947b6b4..8ec3086fe93b 100644 --- a/test/TestCase.h +++ b/test/TestCase.h @@ -93,6 +93,8 @@ class TestCase ++_it; } + void printIndented(std::ostream& _stream, std::string const& _output, std::string const& _linePrefix = "") const; + TestCaseReader m_reader; bool m_shouldRun = true; }; diff --git a/test/libsolidity/ABIJsonTest.cpp b/test/libsolidity/ABIJsonTest.cpp index e7a0cce6dd7c..d144e2ee7d0c 100644 --- a/test/libsolidity/ABIJsonTest.cpp +++ b/test/libsolidity/ABIJsonTest.cpp @@ -26,8 +26,6 @@ #include #include -#include - #include using namespace std; @@ -85,16 +83,3 @@ void ABIJsonTest::printUpdatedExpectations(ostream& _stream, string const& _line { printIndented(_stream, m_obtainedResult, _linePrefix); } - -void ABIJsonTest::printIndented(ostream& _stream, string const& _output, string const& _linePrefix) const -{ - stringstream output(_output); - string line; - while (getline(output, line)) - if (line.empty()) - // Avoid trailing spaces. - _stream << boost::trim_right_copy(_linePrefix) << endl; - else - _stream << _linePrefix << line << endl; -} - diff --git a/test/libsolidity/ABIJsonTest.h b/test/libsolidity/ABIJsonTest.h index 9193dc750b2a..3a7cd397cb72 100644 --- a/test/libsolidity/ABIJsonTest.h +++ b/test/libsolidity/ABIJsonTest.h @@ -41,8 +41,6 @@ class ABIJsonTest: public TestCase void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override; private: - void printIndented(std::ostream& _stream, std::string const& _output, std::string const& _linePrefix = "") const; - std::string m_source; std::string m_expectation; std::string m_obtainedResult; diff --git a/test/libyul/EwasmTranslationTest.cpp b/test/libyul/EwasmTranslationTest.cpp index a545b6124f5b..46ede42213fa 100644 --- a/test/libyul/EwasmTranslationTest.cpp +++ b/test/libyul/EwasmTranslationTest.cpp @@ -94,14 +94,6 @@ void EwasmTranslationTest::printUpdatedExpectations(ostream& _stream, string con printIndented(_stream, m_obtainedResult, _linePrefix); } -void EwasmTranslationTest::printIndented(ostream& _stream, string const& _output, string const& _linePrefix) const -{ - stringstream output(_output); - string line; - while (getline(output, line)) - _stream << _linePrefix << line << endl; -} - bool EwasmTranslationTest::parse(ostream& _stream, string const& _linePrefix, bool const _formatted) { AssemblyStack stack( diff --git a/test/libyul/EwasmTranslationTest.h b/test/libyul/EwasmTranslationTest.h index 68b640032639..f9c6e1d3cb68 100644 --- a/test/libyul/EwasmTranslationTest.h +++ b/test/libyul/EwasmTranslationTest.h @@ -46,7 +46,6 @@ class EwasmTranslationTest: public solidity::frontend::test::EVMVersionRestricte void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override; private: - void printIndented(std::ostream& _stream, std::string const& _output, std::string const& _linePrefix = "") const; bool parse(std::ostream& _stream, std::string const& _linePrefix, bool const _formatted); std::string interpret(); diff --git a/test/libyul/YulInterpreterTest.cpp b/test/libyul/YulInterpreterTest.cpp index 937ffb57e7d3..69e2c8d10add 100644 --- a/test/libyul/YulInterpreterTest.cpp +++ b/test/libyul/YulInterpreterTest.cpp @@ -82,14 +82,6 @@ void YulInterpreterTest::printUpdatedExpectations(ostream& _stream, string const printIndented(_stream, m_obtainedResult, _linePrefix); } -void YulInterpreterTest::printIndented(ostream& _stream, string const& _output, string const& _linePrefix) const -{ - stringstream output(_output); - string line; - while (getline(output, line)) - _stream << _linePrefix << line << endl; -} - bool YulInterpreterTest::parse(ostream& _stream, string const& _linePrefix, bool const _formatted) { AssemblyStack stack( diff --git a/test/libyul/YulInterpreterTest.h b/test/libyul/YulInterpreterTest.h index 5fb6d02b32a2..ec60dbaf4d3d 100644 --- a/test/libyul/YulInterpreterTest.h +++ b/test/libyul/YulInterpreterTest.h @@ -51,7 +51,6 @@ class YulInterpreterTest: public solidity::frontend::test::EVMVersionRestrictedT void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override; private: - void printIndented(std::ostream& _stream, std::string const& _output, std::string const& _linePrefix = "") const; bool parse(std::ostream& _stream, std::string const& _linePrefix, bool const _formatted); std::string interpret(); diff --git a/test/libyul/YulOptimizerTest.cpp b/test/libyul/YulOptimizerTest.cpp index 9458174edb34..2a5209b25853 100644 --- a/test/libyul/YulOptimizerTest.cpp +++ b/test/libyul/YulOptimizerTest.cpp @@ -375,14 +375,6 @@ void YulOptimizerTest::printUpdatedExpectations(ostream& _stream, string const& printIndented(_stream, m_obtainedResult, _linePrefix); } -void YulOptimizerTest::printIndented(ostream& _stream, string const& _output, string const& _linePrefix) const -{ - stringstream output(_output); - string line; - while (getline(output, line)) - _stream << _linePrefix << line << endl; -} - bool YulOptimizerTest::parse(ostream& _stream, string const& _linePrefix, bool const _formatted) { ErrorList errors; diff --git a/test/libyul/YulOptimizerTest.h b/test/libyul/YulOptimizerTest.h index 99e239e3b90f..374ea9b19226 100644 --- a/test/libyul/YulOptimizerTest.h +++ b/test/libyul/YulOptimizerTest.h @@ -60,7 +60,6 @@ class YulOptimizerTest: public solidity::frontend::test::EVMVersionRestrictedTes void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override; private: - void printIndented(std::ostream& _stream, std::string const& _output, std::string const& _linePrefix = "") const; bool parse(std::ostream& _stream, std::string const& _linePrefix, bool const _formatted); void disambiguate(); void updateContext(); From f82f16729053d00d3457234e5b50f2d0dd8d7c17 Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 22 Apr 2020 10:49:44 +0200 Subject: [PATCH 079/126] Yul formatting: Reduce multiple consecutive empty lines to a single one. --- libyul/Utilities.cpp | 7 +++++++ test/cmdlineTests/standard_ir_requested/output.json | 6 ------ test/cmdlineTests/yul_string_format_ascii/output.json | 8 -------- .../yul_string_format_ascii_bytes32/output.json | 5 ----- .../output.json | 5 ----- .../cmdlineTests/yul_string_format_ascii_long/output.json | 8 -------- test/cmdlineTests/yul_string_format_hex/output.json | 5 ----- 7 files changed, 7 insertions(+), 37 deletions(-) diff --git a/libyul/Utilities.cpp b/libyul/Utilities.cpp index dfe72b05e258..8789381b7ae1 100644 --- a/libyul/Utilities.cpp +++ b/libyul/Utilities.cpp @@ -59,6 +59,13 @@ string solidity::yul::reindent(string const& _code) for (string& line: lines) boost::trim(line); + // Reduce multiple consecutive empty lines. + lines = fold(lines, vector{}, [](auto&& _lines, auto&& _line) { + if (!(_line.empty() && !_lines.empty() && _lines.back().empty())) + _lines.emplace_back(std::move(_line)); + return std::move(_lines); + }); + stringstream out; int depth = 0; diff --git a/test/cmdlineTests/standard_ir_requested/output.json b/test/cmdlineTests/standard_ir_requested/output.json index 99fcc85b9b97..2e7aef192331 100644 --- a/test/cmdlineTests/standard_ir_requested/output.json +++ b/test/cmdlineTests/standard_ir_requested/output.json @@ -16,12 +16,8 @@ object \"C_6\" { codecopy(0, dataoffset(\"C_6_deployed\"), datasize(\"C_6_deployed\")) return(0, datasize(\"C_6_deployed\")) - function constructor_C_6() { - - - } } @@ -50,7 +46,6 @@ object \"C_6\" { if iszero(calldatasize()) { } revert(0, 0) - function abi_decode_tuple_(headStart, dataEnd) { if slt(sub(dataEnd, headStart), 0) { revert(0, 0) } @@ -71,7 +66,6 @@ object \"C_6\" { function fun_f_5() { - } function shift_right_224_unsigned(value) -> newValue { diff --git a/test/cmdlineTests/yul_string_format_ascii/output.json b/test/cmdlineTests/yul_string_format_ascii/output.json index 7c5c7a0beef4..0ca1c28dcb8a 100644 --- a/test/cmdlineTests/yul_string_format_ascii/output.json +++ b/test/cmdlineTests/yul_string_format_ascii/output.json @@ -16,12 +16,8 @@ object \"C_10\" { codecopy(0, dataoffset(\"C_10_deployed\"), datasize(\"C_10_deployed\")) return(0, datasize(\"C_10_deployed\")) - function constructor_C_10() { - - - } } @@ -50,7 +46,6 @@ object \"C_10\" { if iszero(calldatasize()) { } revert(0, 0) - function abi_decode_tuple_(headStart, dataEnd) { if slt(sub(dataEnd, headStart), 0) { revert(0, 0) } @@ -81,11 +76,8 @@ object \"C_10\" { function array_length_t_string_memory_ptr(value) -> length { - length := mload(value) - - } function array_storeLengthForEncoding_t_string_memory_ptr_fromStack(pos, length) -> updated_pos { diff --git a/test/cmdlineTests/yul_string_format_ascii_bytes32/output.json b/test/cmdlineTests/yul_string_format_ascii_bytes32/output.json index 763be945ef8f..062a56a548b0 100644 --- a/test/cmdlineTests/yul_string_format_ascii_bytes32/output.json +++ b/test/cmdlineTests/yul_string_format_ascii_bytes32/output.json @@ -16,12 +16,8 @@ object \"C_10\" { codecopy(0, dataoffset(\"C_10_deployed\"), datasize(\"C_10_deployed\")) return(0, datasize(\"C_10_deployed\")) - function constructor_C_10() { - - - } } @@ -50,7 +46,6 @@ object \"C_10\" { if iszero(calldatasize()) { } revert(0, 0) - function abi_decode_tuple_(headStart, dataEnd) { if slt(sub(dataEnd, headStart), 0) { revert(0, 0) } diff --git a/test/cmdlineTests/yul_string_format_ascii_bytes32_from_number/output.json b/test/cmdlineTests/yul_string_format_ascii_bytes32_from_number/output.json index 55b35f138658..6779e68c08f3 100644 --- a/test/cmdlineTests/yul_string_format_ascii_bytes32_from_number/output.json +++ b/test/cmdlineTests/yul_string_format_ascii_bytes32_from_number/output.json @@ -16,12 +16,8 @@ object \"C_10\" { codecopy(0, dataoffset(\"C_10_deployed\"), datasize(\"C_10_deployed\")) return(0, datasize(\"C_10_deployed\")) - function constructor_C_10() { - - - } } @@ -50,7 +46,6 @@ object \"C_10\" { if iszero(calldatasize()) { } revert(0, 0) - function abi_decode_tuple_(headStart, dataEnd) { if slt(sub(dataEnd, headStart), 0) { revert(0, 0) } diff --git a/test/cmdlineTests/yul_string_format_ascii_long/output.json b/test/cmdlineTests/yul_string_format_ascii_long/output.json index 289846431a59..4b4b11347c10 100644 --- a/test/cmdlineTests/yul_string_format_ascii_long/output.json +++ b/test/cmdlineTests/yul_string_format_ascii_long/output.json @@ -16,12 +16,8 @@ object \"C_10\" { codecopy(0, dataoffset(\"C_10_deployed\"), datasize(\"C_10_deployed\")) return(0, datasize(\"C_10_deployed\")) - function constructor_C_10() { - - - } } @@ -50,7 +46,6 @@ object \"C_10\" { if iszero(calldatasize()) { } revert(0, 0) - function abi_decode_tuple_(headStart, dataEnd) { if slt(sub(dataEnd, headStart), 0) { revert(0, 0) } @@ -81,11 +76,8 @@ object \"C_10\" { function array_length_t_string_memory_ptr(value) -> length { - length := mload(value) - - } function array_storeLengthForEncoding_t_string_memory_ptr_fromStack(pos, length) -> updated_pos { diff --git a/test/cmdlineTests/yul_string_format_hex/output.json b/test/cmdlineTests/yul_string_format_hex/output.json index 726c6ccf2c91..d011fdade45a 100644 --- a/test/cmdlineTests/yul_string_format_hex/output.json +++ b/test/cmdlineTests/yul_string_format_hex/output.json @@ -16,12 +16,8 @@ object \"C_10\" { codecopy(0, dataoffset(\"C_10_deployed\"), datasize(\"C_10_deployed\")) return(0, datasize(\"C_10_deployed\")) - function constructor_C_10() { - - - } } @@ -50,7 +46,6 @@ object \"C_10\" { if iszero(calldatasize()) { } revert(0, 0) - function abi_decode_tuple_(headStart, dataEnd) { if slt(sub(dataEnd, headStart), 0) { revert(0, 0) } From e2e32d372ffb378fc2115cbf5b67ec80f77192c5 Mon Sep 17 00:00:00 2001 From: hrkrshnn Date: Wed, 15 Apr 2020 16:12:15 +0530 Subject: [PATCH 080/126] virtual modifiers (in Abstract contracts) allow empty bodies --- libsolidity/analysis/ContractLevelChecker.cpp | 65 ++++++++----------- libsolidity/analysis/ContractLevelChecker.h | 3 +- libsolidity/analysis/ImmutableValidator.cpp | 3 +- libsolidity/analysis/OverrideChecker.cpp | 12 +++- libsolidity/analysis/OverrideChecker.h | 2 + libsolidity/analysis/SyntaxChecker.cpp | 2 +- libsolidity/analysis/TypeChecker.cpp | 6 ++ libsolidity/analysis/TypeChecker.h | 1 + libsolidity/ast/AST.h | 9 +-- libsolidity/ast/ASTAnnotations.h | 4 +- libsolidity/ast/ASTJsonConverter.cpp | 4 +- libsolidity/ast/ASTJsonImporter.cpp | 2 +- libsolidity/ast/AST_accept.h | 6 +- libsolidity/ast/Types.h | 1 - libsolidity/formal/SMTEncoder.cpp | 3 +- libsolidity/parsing/Parser.cpp | 17 +++-- .../SolidityNameAndTypeResolution.cpp | 12 ++-- tools/solidityUpgrade/Upgrade060.cpp | 2 +- 18 files changed, 87 insertions(+), 67 deletions(-) diff --git a/libsolidity/analysis/ContractLevelChecker.cpp b/libsolidity/analysis/ContractLevelChecker.cpp index 27323c544930..18051f99bb49 100644 --- a/libsolidity/analysis/ContractLevelChecker.cpp +++ b/libsolidity/analysis/ContractLevelChecker.cpp @@ -55,7 +55,7 @@ bool ContractLevelChecker::check(ContractDefinition const& _contract) checkDuplicateEvents(_contract); m_overrideChecker.check(_contract); checkBaseConstructorArguments(_contract); - checkAbstractFunctions(_contract); + checkAbstractDefinitions(_contract); checkExternalTypeClashes(_contract); checkHashCollisions(_contract); checkLibraryRequirements(_contract); @@ -156,55 +156,43 @@ void ContractLevelChecker::findDuplicateDefinitions(map> const } } -void ContractLevelChecker::checkAbstractFunctions(ContractDefinition const& _contract) +void ContractLevelChecker::checkAbstractDefinitions(ContractDefinition const& _contract) { - // Mapping from name to function definition (exactly one per argument type equality class) and - // flag to indicate whether it is fully implemented. - using FunTypeAndFlag = std::pair; - map> functions; + // Collects functions, static variable getters and modifiers. If they + // override (unimplemented) base class ones, they are replaced. + set proxies; - auto registerFunction = [&](Declaration const& _declaration, FunctionTypePointer const& _type, bool _implemented) + auto registerProxy = [&proxies](OverrideProxy const& _overrideProxy) { - auto& overloads = functions[_declaration.name()]; - auto it = find_if(overloads.begin(), overloads.end(), [&](FunTypeAndFlag const& _funAndFlag) - { - return _type->hasEqualParameterTypes(*_funAndFlag.first); - }); - if (it == overloads.end()) - overloads.emplace_back(_type, _implemented); - else if (_implemented) - it->second = true; + // Overwrite an existing proxy, if it exists. + if (!_overrideProxy.unimplemented()) + proxies.erase(_overrideProxy); + + proxies.insert(_overrideProxy); }; - // Search from base to derived, collect all functions and update - // the 'implemented' flag. + // Search from base to derived, collect all functions and modifiers and + // update proxies. for (ContractDefinition const* contract: boost::adaptors::reverse(_contract.annotation().linearizedBaseContracts)) { for (VariableDeclaration const* v: contract->stateVariables()) if (v->isPartOfExternalInterface()) - registerFunction(*v, TypeProvider::function(*v), true); + registerProxy(OverrideProxy(v)); for (FunctionDefinition const* function: contract->definedFunctions()) if (!function->isConstructor()) - registerFunction( - *function, - TypeProvider::function(*function)->asCallableFunction(false), - function->isImplemented() - ); + registerProxy(OverrideProxy(function)); + + for (ModifierDefinition const* modifier: contract->functionModifiers()) + registerProxy(OverrideProxy(modifier)); } // Set to not fully implemented if at least one flag is false. - // Note that `_contract.annotation().unimplementedFunctions` has already been + // Note that `_contract.annotation().unimplementedDeclarations` has already been // pre-filled by `checkBaseConstructorArguments`. - for (auto const& it: functions) - for (auto const& funAndFlag: it.second) - if (!funAndFlag.second) - { - FunctionDefinition const* function = dynamic_cast(&funAndFlag.first->declaration()); - solAssert(function, ""); - _contract.annotation().unimplementedFunctions.push_back(function); - break; - } + for (auto const& proxy: proxies) + if (proxy.unimplemented()) + _contract.annotation().unimplementedDeclarations.push_back(proxy.declaration()); if (_contract.abstract()) { @@ -221,15 +209,16 @@ void ContractLevelChecker::checkAbstractFunctions(ContractDefinition const& _con if ( _contract.contractKind() == ContractKind::Contract && !_contract.abstract() && - !_contract.annotation().unimplementedFunctions.empty() + !_contract.annotation().unimplementedDeclarations.empty() ) { SecondarySourceLocation ssl; - for (auto function: _contract.annotation().unimplementedFunctions) - ssl.append("Missing implementation:", function->location()); + for (auto declaration: _contract.annotation().unimplementedDeclarations) + ssl.append("Missing implementation: ", declaration->location()); m_errorReporter.typeError(_contract.location(), ssl, "Contract \"" + _contract.annotation().canonicalName + "\" should be marked as abstract."); + } } @@ -277,7 +266,7 @@ void ContractLevelChecker::checkBaseConstructorArguments(ContractDefinition cons if (FunctionDefinition const* constructor = contract->constructor()) if (contract != &_contract && !constructor->parameters().empty()) if (!_contract.annotation().baseConstructorArguments.count(constructor)) - _contract.annotation().unimplementedFunctions.push_back(constructor); + _contract.annotation().unimplementedDeclarations.push_back(constructor); } void ContractLevelChecker::annotateBaseConstructorArguments( diff --git a/libsolidity/analysis/ContractLevelChecker.h b/libsolidity/analysis/ContractLevelChecker.h index 919c35c07d9f..736b3d3d7a29 100644 --- a/libsolidity/analysis/ContractLevelChecker.h +++ b/libsolidity/analysis/ContractLevelChecker.h @@ -61,7 +61,8 @@ class ContractLevelChecker void checkDuplicateEvents(ContractDefinition const& _contract); template void findDuplicateDefinitions(std::map> const& _definitions, std::string _message); - void checkAbstractFunctions(ContractDefinition const& _contract); + /// Checks for unimplemented functions and modifiers. + void checkAbstractDefinitions(ContractDefinition const& _contract); /// Checks that the base constructor arguments are properly provided. /// Fills the list of unimplemented functions in _contract's annotations. void checkBaseConstructorArguments(ContractDefinition const& _contract); diff --git a/libsolidity/analysis/ImmutableValidator.cpp b/libsolidity/analysis/ImmutableValidator.cpp index b03d314ebdb2..598f43e50830 100644 --- a/libsolidity/analysis/ImmutableValidator.cpp +++ b/libsolidity/analysis/ImmutableValidator.cpp @@ -148,7 +148,8 @@ bool ImmutableValidator::analyseCallable(CallableDeclaration const& _callableDec funcDef->body().accept(*this); } else if (ModifierDefinition const* modDef = dynamic_cast(&_callableDeclaration)) - modDef->body().accept(*this); + if (modDef->isImplemented()) + modDef->body().accept(*this); m_currentConstructor = prevConstructor; diff --git a/libsolidity/analysis/OverrideChecker.cpp b/libsolidity/analysis/OverrideChecker.cpp index 8367f9f589ba..91b45d33c184 100644 --- a/libsolidity/analysis/OverrideChecker.cpp +++ b/libsolidity/analysis/OverrideChecker.cpp @@ -327,6 +327,16 @@ ModifierType const* OverrideProxy::modifierType() const }, m_item); } + +Declaration const* OverrideProxy::declaration() const +{ + return std::visit(GenericVisitor{ + [&](FunctionDefinition const* _function) -> Declaration const* { return _function; }, + [&](VariableDeclaration const* _variable) -> Declaration const* { return _variable; }, + [&](ModifierDefinition const* _modifier) -> Declaration const* { return _modifier; } + }, m_item); +} + SourceLocation const& OverrideProxy::location() const { return std::visit(GenericVisitor{ @@ -365,7 +375,7 @@ bool OverrideProxy::unimplemented() const { return std::visit(GenericVisitor{ [&](FunctionDefinition const* _item) { return !_item->isImplemented(); }, - [&](ModifierDefinition const*) { return false; }, + [&](ModifierDefinition const* _item) { return !_item->isImplemented(); }, [&](VariableDeclaration const*) { return false; } }, m_item); } diff --git a/libsolidity/analysis/OverrideChecker.h b/libsolidity/analysis/OverrideChecker.h index d4b96663c3dc..9d77760282a4 100644 --- a/libsolidity/analysis/OverrideChecker.h +++ b/libsolidity/analysis/OverrideChecker.h @@ -85,6 +85,8 @@ class OverrideProxy FunctionType const* functionType() const; ModifierType const* modifierType() const; + Declaration const* declaration() const; + langutil::SourceLocation const& location() const; std::string astNodeName() const; diff --git a/libsolidity/analysis/SyntaxChecker.cpp b/libsolidity/analysis/SyntaxChecker.cpp index ba973bffa110..c9ad29d332b8 100644 --- a/libsolidity/analysis/SyntaxChecker.cpp +++ b/libsolidity/analysis/SyntaxChecker.cpp @@ -141,7 +141,7 @@ bool SyntaxChecker::visit(ModifierDefinition const&) void SyntaxChecker::endVisit(ModifierDefinition const& _modifier) { - if (!m_placeholderFound) + if (_modifier.isImplemented() && !m_placeholderFound) m_errorReporter.syntaxError(_modifier.body().location(), "Modifier body does not contain '_'."); m_placeholderFound = false; } diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index 250607fe08fa..a0155c9ceea4 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -289,6 +289,12 @@ void TypeChecker::endVisit(UsingForDirective const& _usingFor) m_errorReporter.fatalTypeError(_usingFor.libraryName().location(), "Library name expected."); } +void TypeChecker::endVisit(ModifierDefinition const& _modifier) +{ + if (!_modifier.isImplemented() && !_modifier.virtualSemantics()) + m_errorReporter.typeError(_modifier.location(), "Modifiers without implementation must be marked virtual."); +} + bool TypeChecker::visit(FunctionDefinition const& _function) { bool isLibraryFunction = _function.inContractKind() == ContractKind::Library; diff --git a/libsolidity/analysis/TypeChecker.h b/libsolidity/analysis/TypeChecker.h index 585f7d95f47f..5c327bf2e920 100644 --- a/libsolidity/analysis/TypeChecker.h +++ b/libsolidity/analysis/TypeChecker.h @@ -112,6 +112,7 @@ class TypeChecker: private ASTConstVisitor void endVisit(InheritanceSpecifier const& _inheritance) override; void endVisit(UsingForDirective const& _usingFor) override; + void endVisit(ModifierDefinition const& _modifier) override; bool visit(FunctionDefinition const& _function) override; bool visit(VariableDeclaration const& _variable) override; /// We need to do this manually because we want to pass the bases of the current contract in diff --git a/libsolidity/ast/AST.h b/libsolidity/ast/AST.h index 926d4e74aa9a..3c79912c2441 100644 --- a/libsolidity/ast/AST.h +++ b/libsolidity/ast/AST.h @@ -969,7 +969,7 @@ class VariableDeclaration: public Declaration /** * Definition of a function modifier. */ -class ModifierDefinition: public CallableDeclaration, public StructurallyDocumented +class ModifierDefinition: public CallableDeclaration, public StructurallyDocumented, public ImplementationOptional { public: ModifierDefinition( @@ -980,18 +980,19 @@ class ModifierDefinition: public CallableDeclaration, public StructurallyDocumen ASTPointer const& _parameters, bool _isVirtual, ASTPointer const& _overrides, - ASTPointer _body + ASTPointer const& _body ): CallableDeclaration(_id, _location, _name, Visibility::Internal, _parameters, _isVirtual, _overrides), StructurallyDocumented(_documentation), - m_body(std::move(_body)) + ImplementationOptional(_body != nullptr), + m_body(_body) { } void accept(ASTVisitor& _visitor) override; void accept(ASTConstVisitor& _visitor) const override; - Block const& body() const { return *m_body; } + Block const& body() const { solAssert(m_body, ""); return *m_body; } TypePointer type() const override; diff --git a/libsolidity/ast/ASTAnnotations.h b/libsolidity/ast/ASTAnnotations.h index 2589103cb3f9..0ed54897a9e0 100644 --- a/libsolidity/ast/ASTAnnotations.h +++ b/libsolidity/ast/ASTAnnotations.h @@ -140,8 +140,8 @@ struct StructDeclarationAnnotation: TypeDeclarationAnnotation struct ContractDefinitionAnnotation: TypeDeclarationAnnotation, StructurallyDocumentedAnnotation { - /// List of functions without a body. Can also contain functions from base classes. - std::vector unimplementedFunctions; + /// List of functions and modifiers without a body. Can also contain functions from base classes. + std::vector unimplementedDeclarations; /// List of all (direct and indirect) base contracts in order from derived to /// base, including the contract itself. std::vector linearizedBaseContracts; diff --git a/libsolidity/ast/ASTJsonConverter.cpp b/libsolidity/ast/ASTJsonConverter.cpp index c9d3e1389c93..9f61184bea40 100644 --- a/libsolidity/ast/ASTJsonConverter.cpp +++ b/libsolidity/ast/ASTJsonConverter.cpp @@ -272,7 +272,7 @@ bool ASTJsonConverter::visit(ContractDefinition const& _node) make_pair("documentation", _node.documentation() ? toJson(*_node.documentation()) : Json::nullValue), make_pair("contractKind", contractKind(_node.contractKind())), make_pair("abstract", _node.abstract()), - make_pair("fullyImplemented", _node.annotation().unimplementedFunctions.empty()), + make_pair("fullyImplemented", _node.annotation().unimplementedDeclarations.empty()), make_pair("linearizedBaseContracts", getContainerIds(_node.annotation().linearizedBaseContracts)), make_pair("baseContracts", toJson(_node.baseContracts())), make_pair("contractDependencies", getContainerIds(_node.annotation().contractDependencies, true)), @@ -407,7 +407,7 @@ bool ASTJsonConverter::visit(ModifierDefinition const& _node) make_pair("parameters", toJson(_node.parameterList())), make_pair("virtual", _node.markedVirtual()), make_pair("overrides", _node.overrides() ? toJson(*_node.overrides()) : Json::nullValue), - make_pair("body", toJson(_node.body())) + make_pair("body", _node.isImplemented() ? toJson(_node.body()) : Json::nullValue) }; if (!_node.annotation().baseFunctions.empty()) attributes.emplace_back(make_pair("baseModifiers", getContainerIds(_node.annotation().baseFunctions, true))); diff --git a/libsolidity/ast/ASTJsonImporter.cpp b/libsolidity/ast/ASTJsonImporter.cpp index 7d388877bb3c..e4538059d2fd 100644 --- a/libsolidity/ast/ASTJsonImporter.cpp +++ b/libsolidity/ast/ASTJsonImporter.cpp @@ -453,7 +453,7 @@ ASTPointer ASTJsonImporter::createModifierDefinition(Json::V createParameterList(member(_node, "parameters")), memberAsBool(_node, "virtual"), _node["overrides"].isNull() ? nullptr : createOverrideSpecifier(member(_node, "overrides")), - createBlock(member(_node, "body")) + _node["body"].isNull() ? nullptr: createBlock(member(_node, "body")) ); } diff --git a/libsolidity/ast/AST_accept.h b/libsolidity/ast/AST_accept.h index bc0123d513d7..e597ce46b305 100644 --- a/libsolidity/ast/AST_accept.h +++ b/libsolidity/ast/AST_accept.h @@ -288,7 +288,8 @@ void ModifierDefinition::accept(ASTVisitor& _visitor) m_parameters->accept(_visitor); if (m_overrides) m_overrides->accept(_visitor); - m_body->accept(_visitor); + if (m_body) + m_body->accept(_visitor); } _visitor.endVisit(*this); } @@ -302,7 +303,8 @@ void ModifierDefinition::accept(ASTConstVisitor& _visitor) const m_parameters->accept(_visitor); if (m_overrides) m_overrides->accept(_visitor); - m_body->accept(_visitor); + if (m_body) + m_body->accept(_visitor); } _visitor.endVisit(*this); } diff --git a/libsolidity/ast/Types.h b/libsolidity/ast/Types.h index c1f66df4d52d..974bae4a3978 100644 --- a/libsolidity/ast/Types.h +++ b/libsolidity/ast/Types.h @@ -1390,7 +1390,6 @@ class ModifierType: public Type std::string richIdentifier() const override; bool operator==(Type const& _other) const override; std::string toString(bool _short) const override; - protected: std::vector> makeStackItems() const override { return {}; } private: diff --git a/libsolidity/formal/SMTEncoder.cpp b/libsolidity/formal/SMTEncoder.cpp index 12c9971dc222..9003d26ce94c 100644 --- a/libsolidity/formal/SMTEncoder.cpp +++ b/libsolidity/formal/SMTEncoder.cpp @@ -194,7 +194,8 @@ void SMTEncoder::inlineModifierInvocation(ModifierInvocation const* _invocation, pushCallStack({_definition, _invocation}); if (auto modifier = dynamic_cast(_definition)) { - modifier->body().accept(*this); + if (modifier->isImplemented()) + modifier->body().accept(*this); popCallStack(); } else if (auto function = dynamic_cast(_definition)) diff --git a/libsolidity/parsing/Parser.cpp b/libsolidity/parsing/Parser.cpp index befd3c696d7d..c6da9a55b492 100644 --- a/libsolidity/parsing/Parser.cpp +++ b/libsolidity/parsing/Parser.cpp @@ -595,13 +595,13 @@ ASTPointer Parser::parseFunctionDefinition() ASTPointer block; nodeFactory.markEndPosition(); - if (m_scanner->currentToken() != Token::Semicolon) + if (m_scanner->currentToken() == Token::Semicolon) + m_scanner->next(); + else { block = parseBlock(); nodeFactory.setEndPositionFromNode(block); } - else - m_scanner->next(); // just consume the ';' return nodeFactory.createNode( name, header.visibility, @@ -851,9 +851,16 @@ ASTPointer Parser::parseModifierDefinition() break; } + ASTPointer block; + nodeFactory.markEndPosition(); + if (m_scanner->currentToken() != Token::Semicolon) + { + block = parseBlock(); + nodeFactory.setEndPositionFromNode(block); + } + else + m_scanner->next(); // just consume the ';' - ASTPointer block = parseBlock(); - nodeFactory.setEndPositionFromNode(block); return nodeFactory.createNode(name, documentation, parameters, isVirtual, overrides, block); } diff --git a/test/libsolidity/SolidityNameAndTypeResolution.cpp b/test/libsolidity/SolidityNameAndTypeResolution.cpp index b66b5aad6274..26d9ca98997a 100644 --- a/test/libsolidity/SolidityNameAndTypeResolution.cpp +++ b/test/libsolidity/SolidityNameAndTypeResolution.cpp @@ -52,7 +52,7 @@ BOOST_AUTO_TEST_CASE(function_no_implementation) std::vector> nodes = sourceUnit->nodes(); ContractDefinition* contract = dynamic_cast(nodes[1].get()); BOOST_REQUIRE(contract); - BOOST_CHECK(!contract->annotation().unimplementedFunctions.empty()); + BOOST_CHECK(!contract->annotation().unimplementedDeclarations.empty()); BOOST_CHECK(!contract->definedFunctions()[0]->isImplemented()); } @@ -68,10 +68,10 @@ BOOST_AUTO_TEST_CASE(abstract_contract) ContractDefinition* base = dynamic_cast(nodes[1].get()); ContractDefinition* derived = dynamic_cast(nodes[2].get()); BOOST_REQUIRE(base); - BOOST_CHECK(!base->annotation().unimplementedFunctions.empty()); + BOOST_CHECK(!base->annotation().unimplementedDeclarations.empty()); BOOST_CHECK(!base->definedFunctions()[0]->isImplemented()); BOOST_REQUIRE(derived); - BOOST_CHECK(derived->annotation().unimplementedFunctions.empty()); + BOOST_CHECK(derived->annotation().unimplementedDeclarations.empty()); BOOST_CHECK(derived->definedFunctions()[0]->isImplemented()); } @@ -87,9 +87,9 @@ BOOST_AUTO_TEST_CASE(abstract_contract_with_overload) ContractDefinition* base = dynamic_cast(nodes[1].get()); ContractDefinition* derived = dynamic_cast(nodes[2].get()); BOOST_REQUIRE(base); - BOOST_CHECK(!base->annotation().unimplementedFunctions.empty()); + BOOST_CHECK(!base->annotation().unimplementedDeclarations.empty()); BOOST_REQUIRE(derived); - BOOST_CHECK(!derived->annotation().unimplementedFunctions.empty()); + BOOST_CHECK(!derived->annotation().unimplementedDeclarations.empty()); } BOOST_AUTO_TEST_CASE(implement_abstract_via_constructor) @@ -104,7 +104,7 @@ BOOST_AUTO_TEST_CASE(implement_abstract_via_constructor) BOOST_CHECK_EQUAL(nodes.size(), 3); ContractDefinition* derived = dynamic_cast(nodes[2].get()); BOOST_REQUIRE(derived); - BOOST_CHECK(!derived->annotation().unimplementedFunctions.empty()); + BOOST_CHECK(!derived->annotation().unimplementedDeclarations.empty()); } BOOST_AUTO_TEST_CASE(function_canonical_signature) diff --git a/tools/solidityUpgrade/Upgrade060.cpp b/tools/solidityUpgrade/Upgrade060.cpp index 9ed7cbc74ae5..f5fc4bdf54d7 100644 --- a/tools/solidityUpgrade/Upgrade060.cpp +++ b/tools/solidityUpgrade/Upgrade060.cpp @@ -99,7 +99,7 @@ inline string appendVirtual(FunctionDefinition const& _function) void AbstractContract::endVisit(ContractDefinition const& _contract) { - bool isFullyImplemented = _contract.annotation().unimplementedFunctions.empty(); + bool isFullyImplemented = _contract.annotation().unimplementedDeclarations.empty(); if ( !isFullyImplemented && From 48ff9fd4d68461e9a130d50b79183b889b4d8608 Mon Sep 17 00:00:00 2001 From: hrkrshnn Date: Wed, 15 Apr 2020 15:29:47 +0530 Subject: [PATCH 081/126] Tests, Changelog and updated grammar --- Changelog.md | 1 + docs/Solidity.g4 | 2 +- .../modifiers/function_modifier_empty.sol | 17 ++++++++++ .../modifiers/empty_modifier_body.sol | 14 +++++++++ .../modifiers/empty_modifier_err.sol | 10 ++++++ .../unimplemented_function_and_modifier.sol | 31 +++++++++++++++++++ 6 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 test/libsolidity/semanticTests/modifiers/function_modifier_empty.sol create mode 100644 test/libsolidity/syntaxTests/modifiers/empty_modifier_body.sol create mode 100644 test/libsolidity/syntaxTests/modifiers/empty_modifier_err.sol create mode 100644 test/libsolidity/syntaxTests/modifiers/unimplemented_function_and_modifier.sol diff --git a/Changelog.md b/Changelog.md index 55f063755a07..4baf84ebbbe3 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,7 @@ Language Features: * Add support for EIP 165 interface identifiers with `type(I).interfaceId`. + * Allow virtual modifiers inside abstract contracts to have empty body. Compiler Features: diff --git a/docs/Solidity.g4 b/docs/Solidity.g4 index 8721f47a4dd8..a415c19f97ff 100644 --- a/docs/Solidity.g4 +++ b/docs/Solidity.g4 @@ -68,7 +68,7 @@ structDefinition '{' ( variableDeclaration ';' (variableDeclaration ';')* )? '}' ; modifierDefinition - : 'modifier' identifier parameterList? ( VirtualKeyword | overrideSpecifier )* block ; + : 'modifier' identifier parameterList? ( VirtualKeyword | overrideSpecifier )* ( ';' | block ) ; functionDefinition : functionDescriptor parameterList modifierList returnParameters? ( ';' | block ) ; diff --git a/test/libsolidity/semanticTests/modifiers/function_modifier_empty.sol b/test/libsolidity/semanticTests/modifiers/function_modifier_empty.sol new file mode 100644 index 000000000000..94a352587338 --- /dev/null +++ b/test/libsolidity/semanticTests/modifiers/function_modifier_empty.sol @@ -0,0 +1,17 @@ +abstract contract A { + function f() public mod returns (bool r) { + return true; + } + + modifier mod virtual; +} + + +contract C is A { + modifier mod override { + if (false) _; + } +} + +// ---- +// f() -> false diff --git a/test/libsolidity/syntaxTests/modifiers/empty_modifier_body.sol b/test/libsolidity/syntaxTests/modifiers/empty_modifier_body.sol new file mode 100644 index 000000000000..3ca8c8933143 --- /dev/null +++ b/test/libsolidity/syntaxTests/modifiers/empty_modifier_body.sol @@ -0,0 +1,14 @@ +abstract contract A { modifier mod(uint a) virtual;} +contract B is A { modifier mod(uint a) override { _; } } + +abstract contract C { + modifier m virtual; + function f() m public { + + } +} +contract D is C { + modifier m override { + _; + } +} diff --git a/test/libsolidity/syntaxTests/modifiers/empty_modifier_err.sol b/test/libsolidity/syntaxTests/modifiers/empty_modifier_err.sol new file mode 100644 index 000000000000..77a29903f585 --- /dev/null +++ b/test/libsolidity/syntaxTests/modifiers/empty_modifier_err.sol @@ -0,0 +1,10 @@ +contract A {modifier m virtual;} + +abstract contract B {modifier m virtual;} +contract C is B { } + +abstract contract D {modifier m;} +// ---- +// TypeError: (0-32): Contract "A" should be marked as abstract. +// TypeError: (76-95): Contract "C" should be marked as abstract. +// TypeError: (118-129): Modifiers without implementation must be marked virtual. diff --git a/test/libsolidity/syntaxTests/modifiers/unimplemented_function_and_modifier.sol b/test/libsolidity/syntaxTests/modifiers/unimplemented_function_and_modifier.sol new file mode 100644 index 000000000000..aff392e8b8b8 --- /dev/null +++ b/test/libsolidity/syntaxTests/modifiers/unimplemented_function_and_modifier.sol @@ -0,0 +1,31 @@ +abstract contract A { + function foo() public virtual; + function foo(uint x) virtual public returns(uint); + modifier mod() virtual; +} + +contract B is A { + function foo(uint x) override public returns(uint) {return x;} + modifier mod() override { _; } +} + +contract C is A { + function foo() public override {} + modifier mod() override { _; } +} + +contract D is A { + function foo() public override {} + function foo(uint x) override public returns(uint) {return x;} +} + +/* No errors */ +contract E is A { + function foo() public override {} + function foo(uint x) override public returns(uint) {return x;} + modifier mod() override { _;} +} +// ---- +// TypeError: (137-254): Contract "B" should be marked as abstract. +// TypeError: (256-344): Contract "C" should be marked as abstract. +// TypeError: (346-466): Contract "D" should be marked as abstract. From d136e7dc95eb91644242308a4d58d0e79a82823a Mon Sep 17 00:00:00 2001 From: chriseth Date: Mon, 20 Apr 2020 15:47:58 +0200 Subject: [PATCH 082/126] Rules for optimizing idempotency for bitwise operations. --- Changelog.md | 1 + libevmasm/RuleList.h | 23 +++++++++++++++++++ .../expressionSimplifier/idempotency.yul | 15 ++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 test/libyul/yulOptimizerTests/expressionSimplifier/idempotency.yul diff --git a/Changelog.md b/Changelog.md index 4baf84ebbbe3..6a7042e6feb3 100644 --- a/Changelog.md +++ b/Changelog.md @@ -6,6 +6,7 @@ Language Features: Compiler Features: + * Optimizer: Simplify repeated AND and OR operations. Bugfixes: * SMTChecker: Fix internal error when fixed points are used. diff --git a/libevmasm/RuleList.h b/libevmasm/RuleList.h index 812c6700258e..cd6eda81d9df 100644 --- a/libevmasm/RuleList.h +++ b/libevmasm/RuleList.h @@ -232,6 +232,28 @@ std::vector> simplificationRuleListPart4( }; } +template +std::vector> simplificationRuleListPart4_5( + Pattern, + Pattern, + Pattern, + Pattern X, + Pattern Y +) +{ + using Builtins = typename Pattern::Builtins; + return std::vector>{ + // idempotent operations + {Builtins::AND(Builtins::AND(X, Y), Y), [=]{ return Builtins::AND(X, Y); }, true}, + {Builtins::AND(Y, Builtins::AND(X, Y)), [=]{ return Builtins::AND(X, Y); }, true}, + {Builtins::AND(Builtins::AND(Y, X), Y), [=]{ return Builtins::AND(Y, X); }, true}, + {Builtins::AND(Y, Builtins::AND(Y, X)), [=]{ return Builtins::AND(Y, X); }, true}, + {Builtins::OR(Builtins::OR(X, Y), Y), [=]{ return Builtins::OR(X, Y); }, true}, + {Builtins::OR(Y, Builtins::OR(X, Y)), [=]{ return Builtins::OR(X, Y); }, true}, + {Builtins::OR(Builtins::OR(Y, X), Y), [=]{ return Builtins::OR(Y, X); }, true}, + {Builtins::OR(Y, Builtins::OR(Y, X)), [=]{ return Builtins::OR(Y, X); }, true}, + }; +} template std::vector> simplificationRuleListPart5( @@ -663,6 +685,7 @@ std::vector> simplificationRuleList( rules += simplificationRuleListPart2(A, B, C, W, X); rules += simplificationRuleListPart3(A, B, C, W, X); rules += simplificationRuleListPart4(A, B, C, W, X); + rules += simplificationRuleListPart4_5(A, B, C, W, X); rules += simplificationRuleListPart5(A, B, C, W, X); rules += simplificationRuleListPart6(A, B, C, W, X); rules += simplificationRuleListPart7(A, B, C, W, X); diff --git a/test/libyul/yulOptimizerTests/expressionSimplifier/idempotency.yul b/test/libyul/yulOptimizerTests/expressionSimplifier/idempotency.yul new file mode 100644 index 000000000000..78d51db73903 --- /dev/null +++ b/test/libyul/yulOptimizerTests/expressionSimplifier/idempotency.yul @@ -0,0 +1,15 @@ +{ + let x := calldataload(0) + let z := calldataload(1) + let t := and(and(x, z), x) + let w := or(or(x, z), x) +} +// ---- +// step: expressionSimplifier +// +// { +// let x := calldataload(0) +// let z := calldataload(1) +// let t := and(x, z) +// let w := or(x, z) +// } From d429d20b0b9f55d07b60c82f557e353dabd6c677 Mon Sep 17 00:00:00 2001 From: chriseth Date: Mon, 20 Apr 2020 21:43:24 +0200 Subject: [PATCH 083/126] Restructure documentation and improve TOC. --- docs/cheatsheet.rst | 190 ++++++ docs/grammar.rst | 6 + docs/index.rst | 40 +- docs/internals/layout_in_calldata.rst | 13 + docs/internals/layout_in_memory.rst | 39 ++ docs/internals/layout_in_storage.rst | 359 ++++++++++++ docs/internals/optimiser.rst | 71 +++ docs/internals/source_mappings.rst | 62 ++ docs/internals/variable_cleanup.rst | 47 ++ docs/miscellaneous.rst | 814 -------------------------- docs/solidity-in-depth.rst | 22 - 11 files changed, 823 insertions(+), 840 deletions(-) create mode 100644 docs/cheatsheet.rst create mode 100644 docs/grammar.rst create mode 100644 docs/internals/layout_in_calldata.rst create mode 100644 docs/internals/layout_in_memory.rst create mode 100644 docs/internals/layout_in_storage.rst create mode 100644 docs/internals/optimiser.rst create mode 100644 docs/internals/source_mappings.rst create mode 100644 docs/internals/variable_cleanup.rst delete mode 100644 docs/miscellaneous.rst delete mode 100644 docs/solidity-in-depth.rst diff --git a/docs/cheatsheet.rst b/docs/cheatsheet.rst new file mode 100644 index 000000000000..26d7480b9322 --- /dev/null +++ b/docs/cheatsheet.rst @@ -0,0 +1,190 @@ +********** +Cheatsheet +********** + +.. index:: precedence + +.. _order: + +Order of Precedence of Operators +================================ + +The following is the order of precedence for operators, listed in order of evaluation. + ++------------+-------------------------------------+--------------------------------------------+ +| Precedence | Description | Operator | ++============+=====================================+============================================+ +| *1* | Postfix increment and decrement | ``++``, ``--`` | ++ +-------------------------------------+--------------------------------------------+ +| | New expression | ``new `` | ++ +-------------------------------------+--------------------------------------------+ +| | Array subscripting | ``[]`` | ++ +-------------------------------------+--------------------------------------------+ +| | Member access | ``.`` | ++ +-------------------------------------+--------------------------------------------+ +| | Function-like call | ``()`` | ++ +-------------------------------------+--------------------------------------------+ +| | Parentheses | ``()`` | ++------------+-------------------------------------+--------------------------------------------+ +| *2* | Prefix increment and decrement | ``++``, ``--`` | ++ +-------------------------------------+--------------------------------------------+ +| | Unary minus | ``-`` | ++ +-------------------------------------+--------------------------------------------+ +| | Unary operations | ``delete`` | ++ +-------------------------------------+--------------------------------------------+ +| | Logical NOT | ``!`` | ++ +-------------------------------------+--------------------------------------------+ +| | Bitwise NOT | ``~`` | ++------------+-------------------------------------+--------------------------------------------+ +| *3* | Exponentiation | ``**`` | ++------------+-------------------------------------+--------------------------------------------+ +| *4* | Multiplication, division and modulo | ``*``, ``/``, ``%`` | ++------------+-------------------------------------+--------------------------------------------+ +| *5* | Addition and subtraction | ``+``, ``-`` | ++------------+-------------------------------------+--------------------------------------------+ +| *6* | Bitwise shift operators | ``<<``, ``>>`` | ++------------+-------------------------------------+--------------------------------------------+ +| *7* | Bitwise AND | ``&`` | ++------------+-------------------------------------+--------------------------------------------+ +| *8* | Bitwise XOR | ``^`` | ++------------+-------------------------------------+--------------------------------------------+ +| *9* | Bitwise OR | ``|`` | ++------------+-------------------------------------+--------------------------------------------+ +| *10* | Inequality operators | ``<``, ``>``, ``<=``, ``>=`` | ++------------+-------------------------------------+--------------------------------------------+ +| *11* | Equality operators | ``==``, ``!=`` | ++------------+-------------------------------------+--------------------------------------------+ +| *12* | Logical AND | ``&&`` | ++------------+-------------------------------------+--------------------------------------------+ +| *13* | Logical OR | ``||`` | ++------------+-------------------------------------+--------------------------------------------+ +| *14* | Ternary operator | `` ? : `` | ++ +-------------------------------------+--------------------------------------------+ +| | Assignment operators | ``=``, ``|=``, ``^=``, ``&=``, ``<<=``, | +| | | ``>>=``, ``+=``, ``-=``, ``*=``, ``/=``, | +| | | ``%=`` | ++------------+-------------------------------------+--------------------------------------------+ +| *15* | Comma operator | ``,`` | ++------------+-------------------------------------+--------------------------------------------+ + +.. index:: assert, block, coinbase, difficulty, number, block;number, timestamp, block;timestamp, msg, data, gas, sender, value, now, gas price, origin, revert, require, keccak256, ripemd160, sha256, ecrecover, addmod, mulmod, cryptography, this, super, selfdestruct, balance, send + +Global Variables +================ + +- ``abi.decode(bytes memory encodedData, (...)) returns (...)``: :ref:`ABI `-decodes + the provided data. The types are given in parentheses as second argument. + Example: ``(uint a, uint[2] memory b, bytes memory c) = abi.decode(data, (uint, uint[2], bytes))`` +- ``abi.encode(...) returns (bytes memory)``: :ref:`ABI `-encodes the given arguments +- ``abi.encodePacked(...) returns (bytes memory)``: Performs :ref:`packed encoding ` of + the given arguments. Note that this encoding can be ambiguous! +- ``abi.encodeWithSelector(bytes4 selector, ...) returns (bytes memory)``: :ref:`ABI `-encodes + the given arguments starting from the second and prepends the given four-byte selector +- ``abi.encodeWithSignature(string memory signature, ...) returns (bytes memory)``: Equivalent + to ``abi.encodeWithSelector(bytes4(keccak256(bytes(signature)), ...)``` +- ``block.coinbase`` (``address payable``): current block miner's address +- ``block.difficulty`` (``uint``): current block difficulty +- ``block.gaslimit`` (``uint``): current block gaslimit +- ``block.number`` (``uint``): current block number +- ``block.timestamp`` (``uint``): current block timestamp +- ``gasleft() returns (uint256)``: remaining gas +- ``msg.data`` (``bytes``): complete calldata +- ``msg.sender`` (``address payable``): sender of the message (current call) +- ``msg.value`` (``uint``): number of wei sent with the message +- ``now`` (``uint``): current block timestamp (alias for ``block.timestamp``) +- ``tx.gasprice`` (``uint``): gas price of the transaction +- ``tx.origin`` (``address payable``): sender of the transaction (full call chain) +- ``assert(bool condition)``: abort execution and revert state changes if condition is ``false`` (use for internal error) +- ``require(bool condition)``: abort execution and revert state changes if condition is ``false`` (use + for malformed input or error in external component) +- ``require(bool condition, string memory message)``: abort execution and revert state changes if + condition is ``false`` (use for malformed input or error in external component). Also provide error message. +- ``revert()``: abort execution and revert state changes +- ``revert(string memory message)``: abort execution and revert state changes providing an explanatory string +- ``blockhash(uint blockNumber) returns (bytes32)``: hash of the given block - only works for 256 most recent blocks +- ``keccak256(bytes memory) returns (bytes32)``: compute the Keccak-256 hash of the input +- ``sha256(bytes memory) returns (bytes32)``: compute the SHA-256 hash of the input +- ``ripemd160(bytes memory) returns (bytes20)``: compute the RIPEMD-160 hash of the input +- ``ecrecover(bytes32 hash, uint8 v, bytes32 r, bytes32 s) returns (address)``: recover address associated with + the public key from elliptic curve signature, return zero on error +- ``addmod(uint x, uint y, uint k) returns (uint)``: compute ``(x + y) % k`` where the addition is performed with + arbitrary precision and does not wrap around at ``2**256``. Assert that ``k != 0`` starting from version 0.5.0. +- ``mulmod(uint x, uint y, uint k) returns (uint)``: compute ``(x * y) % k`` where the multiplication is performed + with arbitrary precision and does not wrap around at ``2**256``. Assert that ``k != 0`` starting from version 0.5.0. +- ``this`` (current contract's type): the current contract, explicitly convertible to ``address`` or ``address payable`` +- ``super``: the contract one level higher in the inheritance hierarchy +- ``selfdestruct(address payable recipient)``: destroy the current contract, sending its funds to the given address +- ``
.balance`` (``uint256``): balance of the :ref:`address` in Wei +- ``
.send(uint256 amount) returns (bool)``: send given amount of Wei to :ref:`address`, + returns ``false`` on failure +- ``
.transfer(uint256 amount)``: send given amount of Wei to :ref:`address`, throws on failure +- ``type(C).name`` (``string``): the name of the contract +- ``type(C).creationCode`` (``bytes memory``): creation bytecode of the given contract, see :ref:`Type Information`. +- ``type(C).runtimeCode`` (``bytes memory``): runtime bytecode of the given contract, see :ref:`Type Information`. + +.. note:: + Do not rely on ``block.timestamp``, ``now`` and ``blockhash`` as a source of randomness, + unless you know what you are doing. + + Both the timestamp and the block hash can be influenced by miners to some degree. + Bad actors in the mining community can for example run a casino payout function on a chosen hash + and just retry a different hash if they did not receive any money. + + The current block timestamp must be strictly larger than the timestamp of the last block, + but the only guarantee is that it will be somewhere between the timestamps of two + consecutive blocks in the canonical chain. + +.. note:: + The block hashes are not available for all blocks for scalability reasons. + You can only access the hashes of the most recent 256 blocks, all other + values will be zero. + +.. note:: + In version 0.5.0, the following aliases were removed: ``suicide`` as alias for ``selfdestruct``, + ``msg.gas`` as alias for ``gasleft``, ``block.blockhash`` as alias for ``blockhash`` and + ``sha3`` as alias for ``keccak256``. + +.. index:: visibility, public, private, external, internal + +Function Visibility Specifiers +============================== + +:: + + function myFunction() returns (bool) { + return true; + } + +- ``public``: visible externally and internally (creates a :ref:`getter function` for storage/state variables) +- ``private``: only visible in the current contract +- ``external``: only visible externally (only for functions) - i.e. can only be message-called (via ``this.func``) +- ``internal``: only visible internally + + +.. index:: modifiers, pure, view, payable, constant, anonymous, indexed + +Modifiers +========= + +- ``pure`` for functions: Disallows modification or access of state. +- ``view`` for functions: Disallows modification of state. +- ``payable`` for functions: Allows them to receive Ether together with a call. +- ``constant`` for state variables: Disallows assignment (except initialisation), does not occupy storage slot. +- ``immutable`` for state variables: Allows exactly one assignment at construction time and is constant afterwards. Is stored in code. +- ``anonymous`` for events: Does not store event signature as topic. +- ``indexed`` for event parameters: Stores the parameter as topic. +- ``virtual`` for functions and modifiers: Allows the function's or modifier's + behaviour to be changed in derived contracts. +- ``override``: States that this function, modifier or public state variable changes + the behaviour of a function or modifier in a base contract. + +Reserved Keywords +================= + +These keywords are reserved in Solidity. They might become part of the syntax in the future: + +``after``, ``alias``, ``apply``, ``auto``, ``case``, ``copyof``, ``default``, +``define``, ``final``, ``immutable``, ``implements``, ``in``, ``inline``, ``let``, ``macro``, ``match``, +``mutable``, ``null``, ``of``, ``partial``, ``promise``, ``reference``, ``relocatable``, +``sealed``, ``sizeof``, ``static``, ``supports``, ``switch``, ``typedef``, ``typeof``, +``unchecked``. diff --git a/docs/grammar.rst b/docs/grammar.rst new file mode 100644 index 000000000000..c7a22dd7e92c --- /dev/null +++ b/docs/grammar.rst @@ -0,0 +1,6 @@ +**************** +Language Grammar +**************** + +.. literalinclude:: Solidity.g4 + :language: antlr diff --git a/docs/index.rst b/docs/index.rst index 6610b521728b..89976a8d4e91 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,7 +36,7 @@ If you are new to the concept of smart contracts we recommend you start with :ref:`an example smart contract ` written in Solidity. When you are ready for more detail, we recommend you read the :doc:`"Solidity by Example" ` and -:doc:`"Solidity in Depth" ` sections to learn the core concepts of the language. +"Language Description" sections to learn the core concepts of the language. For further reading, try :ref:`the basics of blockchains ` and details of the :ref:`Ethereum Virtual Machine `. @@ -88,17 +88,49 @@ Contents .. toctree:: :maxdepth: 2 + :caption: Basics introduction-to-smart-contracts.rst installing-solidity.rst solidity-by-example.rst - solidity-in-depth.rst + +.. toctree:: + :maxdepth: 2 + :caption: Language Description + + layout-of-source-files.rst + structure-of-a-contract.rst + types.rst + units-and-global-variables.rst + control-structures.rst + contracts.rst + assembly.rst + cheatsheet.rst + grammar.rst + +.. toctree:: + :maxdepth: 2 + :caption: Internals + + internals/layout_in_storage.rst + internals/layout_in_memory.rst + internals/layout_in_calldata.rst + internals/variable_cleanup.rst + internals/source_mappings.rst + internals/optimiser.rst + metadata.rst + abi-spec.rst + +.. toctree:: + :maxdepth: 2 + :caption: Additional Material + + 050-breaking-changes.rst + 060-breaking-changes.rst natspec-format.rst security-considerations.rst resources.rst using-the-compiler.rst - metadata.rst - abi-spec.rst yul.rst style-guide.rst common-patterns.rst diff --git a/docs/internals/layout_in_calldata.rst b/docs/internals/layout_in_calldata.rst new file mode 100644 index 000000000000..cfabaf99fa0d --- /dev/null +++ b/docs/internals/layout_in_calldata.rst @@ -0,0 +1,13 @@ +******************* +Layout of Call Data +******************* + +The input data for a function call is assumed to be in the format defined by the :ref:`ABI +specification `. Among others, the ABI specification requires arguments to be padded to multiples of 32 +bytes. The internal function calls use a different convention. + +Arguments for the constructor of a contract are directly appended at the end of the +contract's code, also in ABI encoding. The constructor will access them through a hard-coded offset, and +not by using the ``codesize`` opcode, since this of course changes when appending +data to the code. + diff --git a/docs/internals/layout_in_memory.rst b/docs/internals/layout_in_memory.rst new file mode 100644 index 000000000000..34c3035eb393 --- /dev/null +++ b/docs/internals/layout_in_memory.rst @@ -0,0 +1,39 @@ + +.. index: memory layout + +**************** +Layout in Memory +**************** + +Solidity reserves four 32-byte slots, with specific byte ranges (inclusive of endpoints) being used as follows: + +- ``0x00`` - ``0x3f`` (64 bytes): scratch space for hashing methods +- ``0x40`` - ``0x5f`` (32 bytes): currently allocated memory size (aka. free memory pointer) +- ``0x60`` - ``0x7f`` (32 bytes): zero slot + +Scratch space can be used between statements (i.e. within inline assembly). The zero slot +is used as initial value for dynamic memory arrays and should never be written to +(the free memory pointer points to ``0x80`` initially). + +Solidity always places new objects at the free memory pointer and +memory is never freed (this might change in the future). + +Elements in memory arrays in Solidity always occupy multiples of 32 bytes (this +is even true for ``byte[]``, but not for ``bytes`` and ``string``). +Multi-dimensional memory arrays are pointers to memory arrays. The length of a +dynamic array is stored at the first slot of the array and followed by the array +elements. + +.. warning:: + There are some operations in Solidity that need a temporary memory area + larger than 64 bytes and therefore will not fit into the scratch space. + They will be placed where the free memory points to, but given their + short lifetime, the pointer is not updated. The memory may or may not + be zeroed out. Because of this, one should not expect the free memory + to point to zeroed out memory. + + While it may seem like a good idea to use ``msize`` to arrive at a + definitely zeroed out memory area, using such a pointer non-temporarily + without updating the free memory pointer can have unexpected results. + +.. index: calldata layout \ No newline at end of file diff --git a/docs/internals/layout_in_storage.rst b/docs/internals/layout_in_storage.rst new file mode 100644 index 000000000000..0101556ef836 --- /dev/null +++ b/docs/internals/layout_in_storage.rst @@ -0,0 +1,359 @@ +.. index:: storage, state variable, mapping + +************************************ +Layout of State Variables in Storage +************************************ + +.. _storage-inplace-encoding: + +Statically-sized variables (everything except mapping and dynamically-sized +array types) are laid out contiguously in storage starting from position ``0``. +Multiple, contiguous items that need less than 32 bytes are packed into a single +storage slot if possible, according to the following rules: + +- The first item in a storage slot is stored lower-order aligned. +- Elementary types use only as many bytes as are necessary to store them. +- If an elementary type does not fit the remaining part of a storage slot, it is moved to the next storage slot. +- Structs and array data always start a new slot and occupy whole slots + (but items inside a struct or array are packed tightly according to these rules). + +For contracts that use inheritance, the ordering of state variables is determined by the +C3-linearized order of contracts starting with the most base-ward contract. If allowed +by the above rules, state variables from different contracts do share the same storage slot. + +The elements of structs and arrays are stored after each other, just as if they were given explicitly. + +.. warning:: + When using elements that are smaller than 32 bytes, your contract's gas usage may be higher. + This is because the EVM operates on 32 bytes at a time. Therefore, if the element is smaller + than that, the EVM must use more operations in order to reduce the size of the element from 32 + bytes to the desired size. + + It is only beneficial to use reduced-size arguments if you are dealing with storage values + because the compiler will pack multiple elements into one storage slot, and thus, combine + multiple reads or writes into a single operation. When dealing with function arguments or memory + values, there is no inherent benefit because the compiler does not pack these values. + + Finally, in order to allow the EVM to optimize for this, ensure that you try to order your + storage variables and ``struct`` members such that they can be packed tightly. For example, + declaring your storage variables in the order of ``uint128, uint128, uint256`` instead of + ``uint128, uint256, uint128``, as the former will only take up two slots of storage whereas the + latter will take up three. + +.. note:: + The layout of state variables in storage is considered to be part of the external interface + of Solidity due to the fact that storage pointers can be passed to libraries. This means that + any change to the rules outlined in this section is considered a breaking change + of the language and due to its critical nature should be considered very carefully before + being executed. + + +Mappings and Dynamic Arrays +=========================== + +.. _storage-hashed-encoding: + +Due to their unpredictable size, mapping and dynamically-sized array types use a Keccak-256 hash +computation to find the starting position of the value or the array data. +These starting positions are always full stack slots. + +The mapping or the dynamic array itself occupies a slot in storage at some position ``p`` +according to the above rule (or by recursively applying this rule for +mappings of mappings or arrays of arrays). For dynamic arrays, +this slot stores the number of elements in the array (byte arrays and +strings are an exception, see :ref:`below `). +For mappings, the slot is unused (but it is needed so that two equal mappings after each other will use a different +hash distribution). Array data is located at ``keccak256(p)`` and the value corresponding to a mapping key +``k`` is located at ``keccak256(k . p)`` where ``.`` is concatenation. If the value is again a +non-elementary type, the positions are found by adding an offset of ``keccak256(k . p)``. + +So for the following contract snippet +the position of ``data[4][9].b`` is at ``keccak256(uint256(9) . keccak256(uint256(4) . uint256(1))) + 1``:: + + + pragma solidity >=0.4.0 <0.7.0; + + + contract C { + struct S { uint a; uint b; } + uint x; + mapping(uint => mapping(uint => S)) data; + } + +.. _bytes-and-string: + +``bytes`` and ``string`` +------------------------ + +``bytes`` and ``string`` are encoded identically. For short byte arrays, they store their data in the same +slot where the length is also stored. In particular: if the data is at most ``31`` bytes long, it is stored +in the higher-order bytes (left aligned) and the lowest-order byte stores ``length * 2``. +For byte arrays that store data which is ``32`` or more bytes long, the main slot stores ``length * 2 + 1`` and the data is +stored as usual in ``keccak256(slot)``. This means that you can distinguish a short array from a long array +by checking if the lowest bit is set: short (not set) and long (set). + +.. note:: + Handling invalidly encoded slots is currently not supported but may be added in the future. + +JSON Output +=========== + +.. _storage-layout-top-level: + +The storage layout of a contract can be requested via +the :ref:`standard JSON interface `. The output is a JSON object containing two keys, +``storage`` and ``types``. The ``storage`` object is an array where each +element has the following form: + + +.. code:: + + + { + "astId": 2, + "contract": "fileA:A", + "label": "x", + "offset": 0, + "slot": "0", + "type": "t_uint256" + } + +The example above is the storage layout of ``contract A { uint x; }`` from source unit ``fileA`` +and + +- ``astId`` is the id of the AST node of the state variable's declaration +- ``contract`` is the name of the contract including its path as prefix +- ``label`` is the name of the state variable +- ``offset`` is the offset in bytes within the storage slot according to the encoding +- ``slot`` is the storage slot where the state variable resides or starts. This + number may be very large and therefore its JSON value is represented as a + string. +- ``type`` is an identifier used as key to the variable's type information (described in the following) + +The given ``type``, in this case ``t_uint256`` represents an element in +``types``, which has the form: + + +.. code:: + + { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32", + } + +where + +- ``encoding`` how the data is encoded in storage, where the possible values are: + + - ``inplace``: data is laid out contiguously in storage (see :ref:`above `). + - ``mapping``: Keccak-256 hash-based method (see :ref:`above `). + - ``dynamic_array``: Keccak-256 hash-based method (see :ref:`above `). + - ``bytes``: single slot or Keccak-256 hash-based depending on the data size (see :ref:`above `). + +- ``label`` is the canonical type name. +- ``numberOfBytes`` is the number of used bytes (as a decimal string). + Note that if ``numberOfBytes > 32`` this means that more than one slot is used. + +Some types have extra information besides the four above. Mappings contain +its ``key`` and ``value`` types (again referencing an entry in this mapping +of types), arrays have its ``base`` type, and structs list their ``members`` in +the same format as the top-level ``storage`` (see :ref:`above +`). + +.. note :: + The JSON output format of a contract's storage layout is still considered experimental + and is subject to change in non-breaking releases of Solidity. + +The following example shows a contract and its storage layout, containing +value and reference types, types that are encoded packed, and nested types. + + +.. code:: + + pragma solidity >=0.4.0 <0.7.0; + contract A { + struct S { + uint128 a; + uint128 b; + uint[2] staticArray; + uint[] dynArray; + } + + uint x; + uint y; + S s; + address addr; + mapping (uint => mapping (address => bool)) map; + uint[] array; + string s1; + bytes b1; + } + +.. code:: + + "storageLayout": { + "storage": [ + { + "astId": 14, + "contract": "fileA:A", + "label": "x", + "offset": 0, + "slot": "0", + "type": "t_uint256" + }, + { + "astId": 16, + "contract": "fileA:A", + "label": "y", + "offset": 0, + "slot": "1", + "type": "t_uint256" + }, + { + "astId": 18, + "contract": "fileA:A", + "label": "s", + "offset": 0, + "slot": "2", + "type": "t_struct(S)12_storage" + }, + { + "astId": 20, + "contract": "fileA:A", + "label": "addr", + "offset": 0, + "slot": "6", + "type": "t_address" + }, + { + "astId": 26, + "contract": "fileA:A", + "label": "map", + "offset": 0, + "slot": "7", + "type": "t_mapping(t_uint256,t_mapping(t_address,t_bool))" + }, + { + "astId": 29, + "contract": "fileA:A", + "label": "array", + "offset": 0, + "slot": "8", + "type": "t_array(t_uint256)dyn_storage" + }, + { + "astId": 31, + "contract": "fileA:A", + "label": "s1", + "offset": 0, + "slot": "9", + "type": "t_string_storage" + }, + { + "astId": 33, + "contract": "fileA:A", + "label": "b1", + "offset": 0, + "slot": "10", + "type": "t_bytes_storage" + } + ], + "types": { + "t_address": { + "encoding": "inplace", + "label": "address", + "numberOfBytes": "20" + }, + "t_array(t_uint256)2_storage": { + "base": "t_uint256", + "encoding": "inplace", + "label": "uint256[2]", + "numberOfBytes": "64" + }, + "t_array(t_uint256)dyn_storage": { + "base": "t_uint256", + "encoding": "dynamic_array", + "label": "uint256[]", + "numberOfBytes": "32" + }, + "t_bool": { + "encoding": "inplace", + "label": "bool", + "numberOfBytes": "1" + }, + "t_bytes_storage": { + "encoding": "bytes", + "label": "bytes", + "numberOfBytes": "32" + }, + "t_mapping(t_address,t_bool)": { + "encoding": "mapping", + "key": "t_address", + "label": "mapping(address => bool)", + "numberOfBytes": "32", + "value": "t_bool" + }, + "t_mapping(t_uint256,t_mapping(t_address,t_bool))": { + "encoding": "mapping", + "key": "t_uint256", + "label": "mapping(uint256 => mapping(address => bool))", + "numberOfBytes": "32", + "value": "t_mapping(t_address,t_bool)" + }, + "t_string_storage": { + "encoding": "bytes", + "label": "string", + "numberOfBytes": "32" + }, + "t_struct(S)12_storage": { + "encoding": "inplace", + "label": "struct A.S", + "members": [ + { + "astId": 2, + "contract": "fileA:A", + "label": "a", + "offset": 0, + "slot": "0", + "type": "t_uint128" + }, + { + "astId": 4, + "contract": "fileA:A", + "label": "b", + "offset": 16, + "slot": "0", + "type": "t_uint128" + }, + { + "astId": 8, + "contract": "fileA:A", + "label": "staticArray", + "offset": 0, + "slot": "1", + "type": "t_array(t_uint256)2_storage" + }, + { + "astId": 11, + "contract": "fileA:A", + "label": "dynArray", + "offset": 0, + "slot": "3", + "type": "t_array(t_uint256)dyn_storage" + } + ], + "numberOfBytes": "128" + }, + "t_uint128": { + "encoding": "inplace", + "label": "uint128", + "numberOfBytes": "16" + }, + "t_uint256": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + } + } + } diff --git a/docs/internals/optimiser.rst b/docs/internals/optimiser.rst new file mode 100644 index 000000000000..a66291f01f84 --- /dev/null +++ b/docs/internals/optimiser.rst @@ -0,0 +1,71 @@ +.. index:: optimizer, common subexpression elimination, constant propagation + +************* +The Optimiser +************* + +This section discusses the optimiser that was first added to Solidity, +which operates on opcode streams. For information on the new Yul-based optimiser, +please see the `readme on github `_. + +The Solidity optimiser operates on assembly. It splits the sequence of instructions into basic blocks +at ``JUMPs`` and ``JUMPDESTs``. Inside these blocks, the optimiser +analyses the instructions and records every modification to the stack, +memory, or storage as an expression which consists of an instruction and +a list of arguments which are pointers to other expressions. The optimiser +uses a component called "CommonSubexpressionEliminator" that amongst other +tasks, finds expressions that are always equal (on every input) and combines +them into an expression class. The optimiser first tries to find each new +expression in a list of already known expressions. If this does not work, +it simplifies the expression according to rules like +``constant + constant = sum_of_constants`` or ``X * 1 = X``. Since this is +a recursive process, we can also apply the latter rule if the second factor +is a more complex expression where we know that it always evaluates to one. +Modifications to storage and memory locations have to erase knowledge about +storage and memory locations which are not known to be different. If we first +write to location x and then to location y and both are input variables, the +second could overwrite the first, so we do not know what is stored at x after +we wrote to y. If simplification of the expression x - y evaluates to a +non-zero constant, we know that we can keep our knowledge about what is stored at x. + +After this process, we know which expressions have to be on the stack at +the end, and have a list of modifications to memory and storage. This information +is stored together with the basic blocks and is used to link them. Furthermore, +knowledge about the stack, storage and memory configuration is forwarded to +the next block(s). If we know the targets of all ``JUMP`` and ``JUMPI`` instructions, +we can build a complete control flow graph of the program. If there is only +one target we do not know (this can happen as in principle, jump targets can +be computed from inputs), we have to erase all knowledge about the input state +of a block as it can be the target of the unknown ``JUMP``. If the optimiser +finds a ``JUMPI`` whose condition evaluates to a constant, it transforms it +to an unconditional jump. + +As the last step, the code in each block is re-generated. The optimiser creates +a dependency graph from the expressions on the stack at the end of the block, +and it drops every operation that is not part of this graph. It generates code +that applies the modifications to memory and storage in the order they were +made in the original code (dropping modifications which were found not to be +needed). Finally, it generates all values that are required to be on the +stack in the correct place. + +These steps are applied to each basic block and the newly generated code +is used as replacement if it is smaller. If a basic block is split at a +``JUMPI`` and during the analysis, the condition evaluates to a constant, +the ``JUMPI`` is replaced depending on the value of the constant. Thus code like + +:: + + uint x = 7; + data[7] = 9; + if (data[x] != x + 2) + return 2; + else + return 1; + +still simplifies to code which you can compile even though the instructions contained +a jump in the beginning of the process: + +:: + + data[7] = 9; + return 1; diff --git a/docs/internals/source_mappings.rst b/docs/internals/source_mappings.rst new file mode 100644 index 000000000000..d650031c712c --- /dev/null +++ b/docs/internals/source_mappings.rst @@ -0,0 +1,62 @@ +.. index:: source mappings + +*************** +Source Mappings +*************** + +As part of the AST output, the compiler provides the range of the source +code that is represented by the respective node in the AST. This can be +used for various purposes ranging from static analysis tools that report +errors based on the AST and debugging tools that highlight local variables +and their uses. + +Furthermore, the compiler can also generate a mapping from the bytecode +to the range in the source code that generated the instruction. This is again +important for static analysis tools that operate on bytecode level and +for displaying the current position in the source code inside a debugger +or for breakpoint handling. This mapping also contains other information, +like the jump type and the modifier depth (see below). + +Both kinds of source mappings use integer identifiers to refer to source files. +The identifier of a source file is stored in +``output['sources'][sourceName]['id']`` where ``output`` is the output of the +standard-json compiler interface parsed as JSON. + +.. note :: + In the case of instructions that are not associated with any particular source file, + the source mapping assigns an integer identifier of ``-1``. This may happen for + bytecode sections stemming from compiler-generated inline assembly statements. + +The source mappings inside the AST use the following +notation: + +``s:l:f`` + +Where ``s`` is the byte-offset to the start of the range in the source file, +``l`` is the length of the source range in bytes and ``f`` is the source +index mentioned above. + +The encoding in the source mapping for the bytecode is more complicated: +It is a list of ``s:l:f:j:m`` separated by ``;``. Each of these +elements corresponds to an instruction, i.e. you cannot use the byte offset +but have to use the instruction offset (push instructions are longer than a single byte). +The fields ``s``, ``l`` and ``f`` are as above. ``j`` can be either +``i``, ``o`` or ``-`` signifying whether a jump instruction goes into a +function, returns from a function or is a regular jump as part of e.g. a loop. +The last field, ``m``, is an integer that denotes the "modifier depth". This depth +is increased whenever the placeholder statement (``_``) is entered in a modifier +and decreased when it is left again. This allows debuggers to track tricky cases +like the same modifier being used twice or multiple placeholder statements being +used in a single modifier. + +In order to compress these source mappings especially for bytecode, the +following rules are used: + + - If a field is empty, the value of the preceding element is used. + - If a ``:`` is missing, all following fields are considered empty. + +This means the following source mappings represent the same information: + +``1:2:1;1:9:1;2:1:2;2:1:2;2:1:2`` + +``1:2:1;:9;2:1:2;;`` diff --git a/docs/internals/variable_cleanup.rst b/docs/internals/variable_cleanup.rst new file mode 100644 index 000000000000..1718fc66b5e9 --- /dev/null +++ b/docs/internals/variable_cleanup.rst @@ -0,0 +1,47 @@ +.. index: variable cleanup + +********************* +Cleaning Up Variables +********************* + +When a value is shorter than 256 bit, in some cases the remaining bits +must be cleaned. +The Solidity compiler is designed to clean such remaining bits before any operations +that might be adversely affected by the potential garbage in the remaining bits. +For example, before writing a value to memory, the remaining bits need +to be cleared because the memory contents can be used for computing +hashes or sent as the data of a message call. Similarly, before +storing a value in the storage, the remaining bits need to be cleaned +because otherwise the garbled value can be observed. + +On the other hand, we do not clean the bits if the immediately +following operation is not affected. For instance, since any non-zero +value is considered ``true`` by ``JUMPI`` instruction, we do not clean +the boolean values before they are used as the condition for +``JUMPI``. + +In addition to the design principle above, the Solidity compiler +cleans input data when it is loaded onto the stack. + +Different types have different rules for cleaning up invalid values: + ++---------------+---------------+-------------------+ +|Type |Valid Values |Invalid Values Mean| ++===============+===============+===================+ +|enum of n |0 until n - 1 |exception | +|members | | | ++---------------+---------------+-------------------+ +|bool |0 or 1 |1 | ++---------------+---------------+-------------------+ +|signed integers|sign-extended |currently silently | +| |word |wraps; in the | +| | |future exceptions | +| | |will be thrown | +| | | | +| | | | ++---------------+---------------+-------------------+ +|unsigned |higher bits |currently silently | +|integers |zeroed |wraps; in the | +| | |future exceptions | +| | |will be thrown | ++---------------+---------------+-------------------+ diff --git a/docs/miscellaneous.rst b/docs/miscellaneous.rst deleted file mode 100644 index 3c30fec86d1a..000000000000 --- a/docs/miscellaneous.rst +++ /dev/null @@ -1,814 +0,0 @@ -############# -Miscellaneous -############# - -.. index:: storage, state variable, mapping - -************************************ -Layout of State Variables in Storage -************************************ - -.. _storage-inplace-encoding: - -Statically-sized variables (everything except mapping and dynamically-sized -array types) are laid out contiguously in storage starting from position ``0``. -Multiple, contiguous items that need less than 32 bytes are packed into a single -storage slot if possible, according to the following rules: - -- The first item in a storage slot is stored lower-order aligned. -- Elementary types use only as many bytes as are necessary to store them. -- If an elementary type does not fit the remaining part of a storage slot, it is moved to the next storage slot. -- Structs and array data always start a new slot and occupy whole slots - (but items inside a struct or array are packed tightly according to these rules). - -For contracts that use inheritance, the ordering of state variables is determined by the -C3-linearized order of contracts starting with the most base-ward contract. If allowed -by the above rules, state variables from different contracts do share the same storage slot. - -The elements of structs and arrays are stored after each other, just as if they were given explicitly. - -.. warning:: - When using elements that are smaller than 32 bytes, your contract's gas usage may be higher. - This is because the EVM operates on 32 bytes at a time. Therefore, if the element is smaller - than that, the EVM must use more operations in order to reduce the size of the element from 32 - bytes to the desired size. - - It is only beneficial to use reduced-size arguments if you are dealing with storage values - because the compiler will pack multiple elements into one storage slot, and thus, combine - multiple reads or writes into a single operation. When dealing with function arguments or memory - values, there is no inherent benefit because the compiler does not pack these values. - - Finally, in order to allow the EVM to optimize for this, ensure that you try to order your - storage variables and ``struct`` members such that they can be packed tightly. For example, - declaring your storage variables in the order of ``uint128, uint128, uint256`` instead of - ``uint128, uint256, uint128``, as the former will only take up two slots of storage whereas the - latter will take up three. - -.. note:: - The layout of state variables in storage is considered to be part of the external interface - of Solidity due to the fact that storage pointers can be passed to libraries. This means that - any change to the rules outlined in this section is considered a breaking change - of the language and due to its critical nature should be considered very carefully before - being executed. - - -Mappings and Dynamic Arrays -=========================== - -.. _storage-hashed-encoding: - -Due to their unpredictable size, mapping and dynamically-sized array types use a Keccak-256 hash -computation to find the starting position of the value or the array data. -These starting positions are always full stack slots. - -The mapping or the dynamic array itself occupies a slot in storage at some position ``p`` -according to the above rule (or by recursively applying this rule for -mappings of mappings or arrays of arrays). For dynamic arrays, -this slot stores the number of elements in the array (byte arrays and -strings are an exception, see :ref:`below `). -For mappings, the slot is unused (but it is needed so that two equal mappings after each other will use a different -hash distribution). Array data is located at ``keccak256(p)`` and the value corresponding to a mapping key -``k`` is located at ``keccak256(k . p)`` where ``.`` is concatenation. If the value is again a -non-elementary type, the positions are found by adding an offset of ``keccak256(k . p)``. - -So for the following contract snippet -the position of ``data[4][9].b`` is at ``keccak256(uint256(9) . keccak256(uint256(4) . uint256(1))) + 1``:: - - - pragma solidity >=0.4.0 <0.7.0; - - - contract C { - struct S { uint a; uint b; } - uint x; - mapping(uint => mapping(uint => S)) data; - } - -.. _bytes-and-string: - -``bytes`` and ``string`` ------------------------- - -``bytes`` and ``string`` are encoded identically. For short byte arrays, they store their data in the same -slot where the length is also stored. In particular: if the data is at most ``31`` bytes long, it is stored -in the higher-order bytes (left aligned) and the lowest-order byte stores ``length * 2``. -For byte arrays that store data which is ``32`` or more bytes long, the main slot stores ``length * 2 + 1`` and the data is -stored as usual in ``keccak256(slot)``. This means that you can distinguish a short array from a long array -by checking if the lowest bit is set: short (not set) and long (set). - -.. note:: - Handling invalidly encoded slots is currently not supported but may be added in the future. - -JSON Output -=========== - -.. _storage-layout-top-level: - -The storage layout of a contract can be requested via -the :ref:`standard JSON interface `. The output is a JSON object containing two keys, -``storage`` and ``types``. The ``storage`` object is an array where each -element has the following form: - - -.. code:: - - - { - "astId": 2, - "contract": "fileA:A", - "label": "x", - "offset": 0, - "slot": "0", - "type": "t_uint256" - } - -The example above is the storage layout of ``contract A { uint x; }`` from source unit ``fileA`` -and - -- ``astId`` is the id of the AST node of the state variable's declaration -- ``contract`` is the name of the contract including its path as prefix -- ``label`` is the name of the state variable -- ``offset`` is the offset in bytes within the storage slot according to the encoding -- ``slot`` is the storage slot where the state variable resides or starts. This - number may be very large and therefore its JSON value is represented as a - string. -- ``type`` is an identifier used as key to the variable's type information (described in the following) - -The given ``type``, in this case ``t_uint256`` represents an element in -``types``, which has the form: - - -.. code:: - - { - "encoding": "inplace", - "label": "uint256", - "numberOfBytes": "32", - } - -where - -- ``encoding`` how the data is encoded in storage, where the possible values are: - - - ``inplace``: data is laid out contiguously in storage (see :ref:`above `). - - ``mapping``: Keccak-256 hash-based method (see :ref:`above `). - - ``dynamic_array``: Keccak-256 hash-based method (see :ref:`above `). - - ``bytes``: single slot or Keccak-256 hash-based depending on the data size (see :ref:`above `). - -- ``label`` is the canonical type name. -- ``numberOfBytes`` is the number of used bytes (as a decimal string). - Note that if ``numberOfBytes > 32`` this means that more than one slot is used. - -Some types have extra information besides the four above. Mappings contain -its ``key`` and ``value`` types (again referencing an entry in this mapping -of types), arrays have its ``base`` type, and structs list their ``members`` in -the same format as the top-level ``storage`` (see :ref:`above -`). - -.. note :: - The JSON output format of a contract's storage layout is still considered experimental - and is subject to change in non-breaking releases of Solidity. - -The following example shows a contract and its storage layout, containing -value and reference types, types that are encoded packed, and nested types. - - -.. code:: - - pragma solidity >=0.4.0 <0.7.0; - contract A { - struct S { - uint128 a; - uint128 b; - uint[2] staticArray; - uint[] dynArray; - } - - uint x; - uint y; - S s; - address addr; - mapping (uint => mapping (address => bool)) map; - uint[] array; - string s1; - bytes b1; - } - -.. code:: - - "storageLayout": { - "storage": [ - { - "astId": 14, - "contract": "fileA:A", - "label": "x", - "offset": 0, - "slot": "0", - "type": "t_uint256" - }, - { - "astId": 16, - "contract": "fileA:A", - "label": "y", - "offset": 0, - "slot": "1", - "type": "t_uint256" - }, - { - "astId": 18, - "contract": "fileA:A", - "label": "s", - "offset": 0, - "slot": "2", - "type": "t_struct(S)12_storage" - }, - { - "astId": 20, - "contract": "fileA:A", - "label": "addr", - "offset": 0, - "slot": "6", - "type": "t_address" - }, - { - "astId": 26, - "contract": "fileA:A", - "label": "map", - "offset": 0, - "slot": "7", - "type": "t_mapping(t_uint256,t_mapping(t_address,t_bool))" - }, - { - "astId": 29, - "contract": "fileA:A", - "label": "array", - "offset": 0, - "slot": "8", - "type": "t_array(t_uint256)dyn_storage" - }, - { - "astId": 31, - "contract": "fileA:A", - "label": "s1", - "offset": 0, - "slot": "9", - "type": "t_string_storage" - }, - { - "astId": 33, - "contract": "fileA:A", - "label": "b1", - "offset": 0, - "slot": "10", - "type": "t_bytes_storage" - } - ], - "types": { - "t_address": { - "encoding": "inplace", - "label": "address", - "numberOfBytes": "20" - }, - "t_array(t_uint256)2_storage": { - "base": "t_uint256", - "encoding": "inplace", - "label": "uint256[2]", - "numberOfBytes": "64" - }, - "t_array(t_uint256)dyn_storage": { - "base": "t_uint256", - "encoding": "dynamic_array", - "label": "uint256[]", - "numberOfBytes": "32" - }, - "t_bool": { - "encoding": "inplace", - "label": "bool", - "numberOfBytes": "1" - }, - "t_bytes_storage": { - "encoding": "bytes", - "label": "bytes", - "numberOfBytes": "32" - }, - "t_mapping(t_address,t_bool)": { - "encoding": "mapping", - "key": "t_address", - "label": "mapping(address => bool)", - "numberOfBytes": "32", - "value": "t_bool" - }, - "t_mapping(t_uint256,t_mapping(t_address,t_bool))": { - "encoding": "mapping", - "key": "t_uint256", - "label": "mapping(uint256 => mapping(address => bool))", - "numberOfBytes": "32", - "value": "t_mapping(t_address,t_bool)" - }, - "t_string_storage": { - "encoding": "bytes", - "label": "string", - "numberOfBytes": "32" - }, - "t_struct(S)12_storage": { - "encoding": "inplace", - "label": "struct A.S", - "members": [ - { - "astId": 2, - "contract": "fileA:A", - "label": "a", - "offset": 0, - "slot": "0", - "type": "t_uint128" - }, - { - "astId": 4, - "contract": "fileA:A", - "label": "b", - "offset": 16, - "slot": "0", - "type": "t_uint128" - }, - { - "astId": 8, - "contract": "fileA:A", - "label": "staticArray", - "offset": 0, - "slot": "1", - "type": "t_array(t_uint256)2_storage" - }, - { - "astId": 11, - "contract": "fileA:A", - "label": "dynArray", - "offset": 0, - "slot": "3", - "type": "t_array(t_uint256)dyn_storage" - } - ], - "numberOfBytes": "128" - }, - "t_uint128": { - "encoding": "inplace", - "label": "uint128", - "numberOfBytes": "16" - }, - "t_uint256": { - "encoding": "inplace", - "label": "uint256", - "numberOfBytes": "32" - } - } - } - -.. index: memory layout - -**************** -Layout in Memory -**************** - -Solidity reserves four 32-byte slots, with specific byte ranges (inclusive of endpoints) being used as follows: - -- ``0x00`` - ``0x3f`` (64 bytes): scratch space for hashing methods -- ``0x40`` - ``0x5f`` (32 bytes): currently allocated memory size (aka. free memory pointer) -- ``0x60`` - ``0x7f`` (32 bytes): zero slot - -Scratch space can be used between statements (i.e. within inline assembly). The zero slot -is used as initial value for dynamic memory arrays and should never be written to -(the free memory pointer points to ``0x80`` initially). - -Solidity always places new objects at the free memory pointer and -memory is never freed (this might change in the future). - -Elements in memory arrays in Solidity always occupy multiples of 32 bytes (this -is even true for ``byte[]``, but not for ``bytes`` and ``string``). -Multi-dimensional memory arrays are pointers to memory arrays. The length of a -dynamic array is stored at the first slot of the array and followed by the array -elements. - -.. warning:: - There are some operations in Solidity that need a temporary memory area - larger than 64 bytes and therefore will not fit into the scratch space. - They will be placed where the free memory points to, but given their - short lifetime, the pointer is not updated. The memory may or may not - be zeroed out. Because of this, one should not expect the free memory - to point to zeroed out memory. - - While it may seem like a good idea to use ``msize`` to arrive at a - definitely zeroed out memory area, using such a pointer non-temporarily - without updating the free memory pointer can have unexpected results. - -.. index: calldata layout - -******************* -Layout of Call Data -******************* - -The input data for a function call is assumed to be in the format defined by the :ref:`ABI -specification `. Among others, the ABI specification requires arguments to be padded to multiples of 32 -bytes. The internal function calls use a different convention. - -Arguments for the constructor of a contract are directly appended at the end of the -contract's code, also in ABI encoding. The constructor will access them through a hard-coded offset, and -not by using the ``codesize`` opcode, since this of course changes when appending -data to the code. - - -.. index: variable cleanup - -********************************* -Internals - Cleaning Up Variables -********************************* - -When a value is shorter than 256 bit, in some cases the remaining bits -must be cleaned. -The Solidity compiler is designed to clean such remaining bits before any operations -that might be adversely affected by the potential garbage in the remaining bits. -For example, before writing a value to memory, the remaining bits need -to be cleared because the memory contents can be used for computing -hashes or sent as the data of a message call. Similarly, before -storing a value in the storage, the remaining bits need to be cleaned -because otherwise the garbled value can be observed. - -On the other hand, we do not clean the bits if the immediately -following operation is not affected. For instance, since any non-zero -value is considered ``true`` by ``JUMPI`` instruction, we do not clean -the boolean values before they are used as the condition for -``JUMPI``. - -In addition to the design principle above, the Solidity compiler -cleans input data when it is loaded onto the stack. - -Different types have different rules for cleaning up invalid values: - -+---------------+---------------+-------------------+ -|Type |Valid Values |Invalid Values Mean| -+===============+===============+===================+ -|enum of n |0 until n - 1 |exception | -|members | | | -+---------------+---------------+-------------------+ -|bool |0 or 1 |1 | -+---------------+---------------+-------------------+ -|signed integers|sign-extended |currently silently | -| |word |wraps; in the | -| | |future exceptions | -| | |will be thrown | -| | | | -| | | | -+---------------+---------------+-------------------+ -|unsigned |higher bits |currently silently | -|integers |zeroed |wraps; in the | -| | |future exceptions | -| | |will be thrown | -+---------------+---------------+-------------------+ - -.. index:: optimizer, common subexpression elimination, constant propagation - -************************* -Internals - The Optimiser -************************* - -This section discusses the optimiser that was first added to Solidity, -which operates on opcode streams. For information on the new Yul-based optimiser, -please see the `readme on github `_. - -The Solidity optimiser operates on assembly. It splits the sequence of instructions into basic blocks -at ``JUMPs`` and ``JUMPDESTs``. Inside these blocks, the optimiser -analyses the instructions and records every modification to the stack, -memory, or storage as an expression which consists of an instruction and -a list of arguments which are pointers to other expressions. The optimiser -uses a component called "CommonSubexpressionEliminator" that amongst other -tasks, finds expressions that are always equal (on every input) and combines -them into an expression class. The optimiser first tries to find each new -expression in a list of already known expressions. If this does not work, -it simplifies the expression according to rules like -``constant + constant = sum_of_constants`` or ``X * 1 = X``. Since this is -a recursive process, we can also apply the latter rule if the second factor -is a more complex expression where we know that it always evaluates to one. -Modifications to storage and memory locations have to erase knowledge about -storage and memory locations which are not known to be different. If we first -write to location x and then to location y and both are input variables, the -second could overwrite the first, so we do not know what is stored at x after -we wrote to y. If simplification of the expression x - y evaluates to a -non-zero constant, we know that we can keep our knowledge about what is stored at x. - -After this process, we know which expressions have to be on the stack at -the end, and have a list of modifications to memory and storage. This information -is stored together with the basic blocks and is used to link them. Furthermore, -knowledge about the stack, storage and memory configuration is forwarded to -the next block(s). If we know the targets of all ``JUMP`` and ``JUMPI`` instructions, -we can build a complete control flow graph of the program. If there is only -one target we do not know (this can happen as in principle, jump targets can -be computed from inputs), we have to erase all knowledge about the input state -of a block as it can be the target of the unknown ``JUMP``. If the optimiser -finds a ``JUMPI`` whose condition evaluates to a constant, it transforms it -to an unconditional jump. - -As the last step, the code in each block is re-generated. The optimiser creates -a dependency graph from the expressions on the stack at the end of the block, -and it drops every operation that is not part of this graph. It generates code -that applies the modifications to memory and storage in the order they were -made in the original code (dropping modifications which were found not to be -needed). Finally, it generates all values that are required to be on the -stack in the correct place. - -These steps are applied to each basic block and the newly generated code -is used as replacement if it is smaller. If a basic block is split at a -``JUMPI`` and during the analysis, the condition evaluates to a constant, -the ``JUMPI`` is replaced depending on the value of the constant. Thus code like - -:: - - uint x = 7; - data[7] = 9; - if (data[x] != x + 2) - return 2; - else - return 1; - -still simplifies to code which you can compile even though the instructions contained -a jump in the beginning of the process: - -:: - - data[7] = 9; - return 1; - -.. index:: source mappings - -*************** -Source Mappings -*************** - -As part of the AST output, the compiler provides the range of the source -code that is represented by the respective node in the AST. This can be -used for various purposes ranging from static analysis tools that report -errors based on the AST and debugging tools that highlight local variables -and their uses. - -Furthermore, the compiler can also generate a mapping from the bytecode -to the range in the source code that generated the instruction. This is again -important for static analysis tools that operate on bytecode level and -for displaying the current position in the source code inside a debugger -or for breakpoint handling. This mapping also contains other information, -like the jump type and the modifier depth (see below). - -Both kinds of source mappings use integer identifiers to refer to source files. -The identifier of a source file is stored in -``output['sources'][sourceName]['id']`` where ``output`` is the output of the -standard-json compiler interface parsed as JSON. - -.. note :: - In the case of instructions that are not associated with any particular source file, - the source mapping assigns an integer identifier of ``-1``. This may happen for - bytecode sections stemming from compiler-generated inline assembly statements. - -The source mappings inside the AST use the following -notation: - -``s:l:f`` - -Where ``s`` is the byte-offset to the start of the range in the source file, -``l`` is the length of the source range in bytes and ``f`` is the source -index mentioned above. - -The encoding in the source mapping for the bytecode is more complicated: -It is a list of ``s:l:f:j:m`` separated by ``;``. Each of these -elements corresponds to an instruction, i.e. you cannot use the byte offset -but have to use the instruction offset (push instructions are longer than a single byte). -The fields ``s``, ``l`` and ``f`` are as above. ``j`` can be either -``i``, ``o`` or ``-`` signifying whether a jump instruction goes into a -function, returns from a function or is a regular jump as part of e.g. a loop. -The last field, ``m``, is an integer that denotes the "modifier depth". This depth -is increased whenever the placeholder statement (``_``) is entered in a modifier -and decreased when it is left again. This allows debuggers to track tricky cases -like the same modifier being used twice or multiple placeholder statements being -used in a single modifier. - -In order to compress these source mappings especially for bytecode, the -following rules are used: - - - If a field is empty, the value of the preceding element is used. - - If a ``:`` is missing, all following fields are considered empty. - -This means the following source mappings represent the same information: - -``1:2:1;1:9:1;2:1:2;2:1:2;2:1:2`` - -``1:2:1;:9;2:1:2;;`` - -*************** -Tips and Tricks -*************** - -* Use ``delete`` on arrays to delete all its elements. -* Use shorter types for struct elements and sort them such that short types are - grouped together. This can lower the gas costs as multiple ``SSTORE`` operations - might be combined into a single (``SSTORE`` costs 5000 or 20000 gas, so this is - what you want to optimise). Use the gas price estimator (with optimiser enabled) to check! -* Make your state variables public - the compiler creates :ref:`getters ` for you automatically. -* If you end up checking conditions on input or state a lot at the beginning of your functions, try using :ref:`modifiers`. -* Initialize storage structs with a single assignment: ``x = MyStruct({a: 1, b: 2});`` - -.. note:: - If the storage struct has tightly packed properties, initialize it with separate - assignments: ``x.a = 1; x.b = 2;``. In this way it will be easier for the - optimizer to update storage in one go, thus making assignment cheaper. - -********** -Cheatsheet -********** - -.. index:: precedence - -.. _order: - -Order of Precedence of Operators -================================ - -The following is the order of precedence for operators, listed in order of evaluation. - -+------------+-------------------------------------+--------------------------------------------+ -| Precedence | Description | Operator | -+============+=====================================+============================================+ -| *1* | Postfix increment and decrement | ``++``, ``--`` | -+ +-------------------------------------+--------------------------------------------+ -| | New expression | ``new `` | -+ +-------------------------------------+--------------------------------------------+ -| | Array subscripting | ``[]`` | -+ +-------------------------------------+--------------------------------------------+ -| | Member access | ``.`` | -+ +-------------------------------------+--------------------------------------------+ -| | Function-like call | ``()`` | -+ +-------------------------------------+--------------------------------------------+ -| | Parentheses | ``()`` | -+------------+-------------------------------------+--------------------------------------------+ -| *2* | Prefix increment and decrement | ``++``, ``--`` | -+ +-------------------------------------+--------------------------------------------+ -| | Unary minus | ``-`` | -+ +-------------------------------------+--------------------------------------------+ -| | Unary operations | ``delete`` | -+ +-------------------------------------+--------------------------------------------+ -| | Logical NOT | ``!`` | -+ +-------------------------------------+--------------------------------------------+ -| | Bitwise NOT | ``~`` | -+------------+-------------------------------------+--------------------------------------------+ -| *3* | Exponentiation | ``**`` | -+------------+-------------------------------------+--------------------------------------------+ -| *4* | Multiplication, division and modulo | ``*``, ``/``, ``%`` | -+------------+-------------------------------------+--------------------------------------------+ -| *5* | Addition and subtraction | ``+``, ``-`` | -+------------+-------------------------------------+--------------------------------------------+ -| *6* | Bitwise shift operators | ``<<``, ``>>`` | -+------------+-------------------------------------+--------------------------------------------+ -| *7* | Bitwise AND | ``&`` | -+------------+-------------------------------------+--------------------------------------------+ -| *8* | Bitwise XOR | ``^`` | -+------------+-------------------------------------+--------------------------------------------+ -| *9* | Bitwise OR | ``|`` | -+------------+-------------------------------------+--------------------------------------------+ -| *10* | Inequality operators | ``<``, ``>``, ``<=``, ``>=`` | -+------------+-------------------------------------+--------------------------------------------+ -| *11* | Equality operators | ``==``, ``!=`` | -+------------+-------------------------------------+--------------------------------------------+ -| *12* | Logical AND | ``&&`` | -+------------+-------------------------------------+--------------------------------------------+ -| *13* | Logical OR | ``||`` | -+------------+-------------------------------------+--------------------------------------------+ -| *14* | Ternary operator | `` ? : `` | -+ +-------------------------------------+--------------------------------------------+ -| | Assignment operators | ``=``, ``|=``, ``^=``, ``&=``, ``<<=``, | -| | | ``>>=``, ``+=``, ``-=``, ``*=``, ``/=``, | -| | | ``%=`` | -+------------+-------------------------------------+--------------------------------------------+ -| *15* | Comma operator | ``,`` | -+------------+-------------------------------------+--------------------------------------------+ - -.. index:: assert, block, coinbase, difficulty, number, block;number, timestamp, block;timestamp, msg, data, gas, sender, value, now, gas price, origin, revert, require, keccak256, ripemd160, sha256, ecrecover, addmod, mulmod, cryptography, this, super, selfdestruct, balance, send - -Global Variables -================ - -- ``abi.decode(bytes memory encodedData, (...)) returns (...)``: :ref:`ABI `-decodes - the provided data. The types are given in parentheses as second argument. - Example: ``(uint a, uint[2] memory b, bytes memory c) = abi.decode(data, (uint, uint[2], bytes))`` -- ``abi.encode(...) returns (bytes memory)``: :ref:`ABI `-encodes the given arguments -- ``abi.encodePacked(...) returns (bytes memory)``: Performs :ref:`packed encoding ` of - the given arguments. Note that this encoding can be ambiguous! -- ``abi.encodeWithSelector(bytes4 selector, ...) returns (bytes memory)``: :ref:`ABI `-encodes - the given arguments starting from the second and prepends the given four-byte selector -- ``abi.encodeWithSignature(string memory signature, ...) returns (bytes memory)``: Equivalent - to ``abi.encodeWithSelector(bytes4(keccak256(bytes(signature)), ...)``` -- ``block.coinbase`` (``address payable``): current block miner's address -- ``block.difficulty`` (``uint``): current block difficulty -- ``block.gaslimit`` (``uint``): current block gaslimit -- ``block.number`` (``uint``): current block number -- ``block.timestamp`` (``uint``): current block timestamp -- ``gasleft() returns (uint256)``: remaining gas -- ``msg.data`` (``bytes``): complete calldata -- ``msg.sender`` (``address payable``): sender of the message (current call) -- ``msg.value`` (``uint``): number of wei sent with the message -- ``now`` (``uint``): current block timestamp (alias for ``block.timestamp``) -- ``tx.gasprice`` (``uint``): gas price of the transaction -- ``tx.origin`` (``address payable``): sender of the transaction (full call chain) -- ``assert(bool condition)``: abort execution and revert state changes if condition is ``false`` (use for internal error) -- ``require(bool condition)``: abort execution and revert state changes if condition is ``false`` (use - for malformed input or error in external component) -- ``require(bool condition, string memory message)``: abort execution and revert state changes if - condition is ``false`` (use for malformed input or error in external component). Also provide error message. -- ``revert()``: abort execution and revert state changes -- ``revert(string memory message)``: abort execution and revert state changes providing an explanatory string -- ``blockhash(uint blockNumber) returns (bytes32)``: hash of the given block - only works for 256 most recent blocks -- ``keccak256(bytes memory) returns (bytes32)``: compute the Keccak-256 hash of the input -- ``sha256(bytes memory) returns (bytes32)``: compute the SHA-256 hash of the input -- ``ripemd160(bytes memory) returns (bytes20)``: compute the RIPEMD-160 hash of the input -- ``ecrecover(bytes32 hash, uint8 v, bytes32 r, bytes32 s) returns (address)``: recover address associated with - the public key from elliptic curve signature, return zero on error -- ``addmod(uint x, uint y, uint k) returns (uint)``: compute ``(x + y) % k`` where the addition is performed with - arbitrary precision and does not wrap around at ``2**256``. Assert that ``k != 0`` starting from version 0.5.0. -- ``mulmod(uint x, uint y, uint k) returns (uint)``: compute ``(x * y) % k`` where the multiplication is performed - with arbitrary precision and does not wrap around at ``2**256``. Assert that ``k != 0`` starting from version 0.5.0. -- ``this`` (current contract's type): the current contract, explicitly convertible to ``address`` or ``address payable`` -- ``super``: the contract one level higher in the inheritance hierarchy -- ``selfdestruct(address payable recipient)``: destroy the current contract, sending its funds to the given address -- ``
.balance`` (``uint256``): balance of the :ref:`address` in Wei -- ``
.send(uint256 amount) returns (bool)``: send given amount of Wei to :ref:`address`, - returns ``false`` on failure -- ``
.transfer(uint256 amount)``: send given amount of Wei to :ref:`address`, throws on failure -- ``type(C).name`` (``string``): the name of the contract -- ``type(C).creationCode`` (``bytes memory``): creation bytecode of the given contract, see :ref:`Type Information`. -- ``type(C).runtimeCode`` (``bytes memory``): runtime bytecode of the given contract, see :ref:`Type Information`. - -.. note:: - Do not rely on ``block.timestamp``, ``now`` and ``blockhash`` as a source of randomness, - unless you know what you are doing. - - Both the timestamp and the block hash can be influenced by miners to some degree. - Bad actors in the mining community can for example run a casino payout function on a chosen hash - and just retry a different hash if they did not receive any money. - - The current block timestamp must be strictly larger than the timestamp of the last block, - but the only guarantee is that it will be somewhere between the timestamps of two - consecutive blocks in the canonical chain. - -.. note:: - The block hashes are not available for all blocks for scalability reasons. - You can only access the hashes of the most recent 256 blocks, all other - values will be zero. - -.. note:: - In version 0.5.0, the following aliases were removed: ``suicide`` as alias for ``selfdestruct``, - ``msg.gas`` as alias for ``gasleft``, ``block.blockhash`` as alias for ``blockhash`` and - ``sha3`` as alias for ``keccak256``. - -.. index:: visibility, public, private, external, internal - -Function Visibility Specifiers -============================== - -:: - - function myFunction() returns (bool) { - return true; - } - -- ``public``: visible externally and internally (creates a :ref:`getter function` for storage/state variables) -- ``private``: only visible in the current contract -- ``external``: only visible externally (only for functions) - i.e. can only be message-called (via ``this.func``) -- ``internal``: only visible internally - - -.. index:: modifiers, pure, view, payable, constant, anonymous, indexed - -Modifiers -========= - -- ``pure`` for functions: Disallows modification or access of state. -- ``view`` for functions: Disallows modification of state. -- ``payable`` for functions: Allows them to receive Ether together with a call. -- ``constant`` for state variables: Disallows assignment (except initialisation), does not occupy storage slot. -- ``immutable`` for state variables: Allows exactly one assignment at construction time and is constant afterwards. Is stored in code. -- ``anonymous`` for events: Does not store event signature as topic. -- ``indexed`` for event parameters: Stores the parameter as topic. -- ``virtual`` for functions and modifiers: Allows the function's or modifier's - behaviour to be changed in derived contracts. -- ``override``: States that this function, modifier or public state variable changes - the behaviour of a function or modifier in a base contract. - -Reserved Keywords -================= - -These keywords are reserved in Solidity. They might become part of the syntax in the future: - -``after``, ``alias``, ``apply``, ``auto``, ``case``, ``copyof``, ``default``, -``define``, ``final``, ``immutable``, ``implements``, ``in``, ``inline``, ``let``, ``macro``, ``match``, -``mutable``, ``null``, ``of``, ``partial``, ``promise``, ``reference``, ``relocatable``, -``sealed``, ``sizeof``, ``static``, ``supports``, ``switch``, ``typedef``, ``typeof``, -``unchecked``. - -Language Grammar -================ - -.. literalinclude:: Solidity.g4 - :language: antlr diff --git a/docs/solidity-in-depth.rst b/docs/solidity-in-depth.rst deleted file mode 100644 index 111a100d7e95..000000000000 --- a/docs/solidity-in-depth.rst +++ /dev/null @@ -1,22 +0,0 @@ -################# -Solidity in Depth -################# - -This section should provide you with all you need to know about Solidity. -If something is missing here, please contact us on -`Gitter `_ or create a pull request on -`Github `_. - -.. toctree:: - :maxdepth: 2 - - layout-of-source-files.rst - structure-of-a-contract.rst - types.rst - units-and-global-variables.rst - control-structures.rst - contracts.rst - assembly.rst - miscellaneous.rst - 050-breaking-changes.rst - 060-breaking-changes.rst From e65a5a562e69607756e12e8796b3705afebff074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 23 Apr 2020 21:16:41 +0200 Subject: [PATCH 084/126] IRGenerationContext::internalDispatch(): Fix code generated when the function called via pointer does not return anything --- .../codegen/ir/IRGenerationContext.cpp | 3 ++- ...function_returning_nothing_via_pointer.sol | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 test/libsolidity/semanticTests/functionCall/call_function_returning_nothing_via_pointer.sol diff --git a/libsolidity/codegen/ir/IRGenerationContext.cpp b/libsolidity/codegen/ir/IRGenerationContext.cpp index c3b896ee114b..0410d883577d 100644 --- a/libsolidity/codegen/ir/IRGenerationContext.cpp +++ b/libsolidity/codegen/ir/IRGenerationContext.cpp @@ -122,7 +122,7 @@ string IRGenerationContext::internalDispatch(size_t _in, size_t _out) <#cases> case { - := () + () } default { invalid() } @@ -133,6 +133,7 @@ string IRGenerationContext::internalDispatch(size_t _in, size_t _out) YulUtilFunctions utils(m_evmVersion, m_revertStrings, m_functions); templ("in", suffixedVariableNameList("in_", 0, _in)); templ("arrow", _out > 0 ? "->" : ""); + templ("assignment_op", _out > 0 ? ":=" : ""); templ("out", suffixedVariableNameList("out_", 0, _out)); vector> functions; for (auto const& contract: mostDerivedContract().annotation().linearizedBaseContracts) diff --git a/test/libsolidity/semanticTests/functionCall/call_function_returning_nothing_via_pointer.sol b/test/libsolidity/semanticTests/functionCall/call_function_returning_nothing_via_pointer.sol new file mode 100644 index 000000000000..569d185bdc2a --- /dev/null +++ b/test/libsolidity/semanticTests/functionCall/call_function_returning_nothing_via_pointer.sol @@ -0,0 +1,19 @@ +contract test { + bool public flag = false; + + function f0() public { + flag = true; + } + + function f() public returns (bool) { + function() internal x = f0; + x(); + return flag; + } +} + +// ==== +// compileViaYul: also +// ---- +// f() -> true +// flag() -> true From aafa4583df7a9989be7d8350ddb92bfa81f8426c Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Thu, 23 Apr 2020 17:14:03 -0500 Subject: [PATCH 085/126] [Sol - Yul] Add support for built-in gasleft(). --- libsolidity/codegen/ir/IRGeneratorForStatements.cpp | 5 +++++ test/libsolidity/semanticTests/various/gasleft_decrease.sol | 2 ++ 2 files changed, 7 insertions(+) diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index c572d5d8aa13..908e1d48c63b 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -771,6 +771,11 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall) { break; } + case FunctionType::Kind::GasLeft: + { + define(_functionCall) << "gas()\n"; + break; + } default: solUnimplemented("FunctionKind " + toString(static_cast(functionType->kind())) + " not yet implemented"); } diff --git a/test/libsolidity/semanticTests/various/gasleft_decrease.sol b/test/libsolidity/semanticTests/various/gasleft_decrease.sol index 8de56296e0b7..b6251205e2d2 100644 --- a/test/libsolidity/semanticTests/various/gasleft_decrease.sol +++ b/test/libsolidity/semanticTests/various/gasleft_decrease.sol @@ -15,6 +15,8 @@ contract C { } } +// ==== +// compileViaYul: also // ---- // f() -> true // g() -> true From 8717c073a67d1a7f44cb61a78df15e6be5de1927 Mon Sep 17 00:00:00 2001 From: a3d4 Date: Thu, 23 Apr 2020 08:23:01 +0200 Subject: [PATCH 086/126] Fix leftpad in SourceReferenceFormatterHuman --- liblangutil/SourceReferenceFormatterHuman.cpp | 25 +++-- test/cmdlineTests/message_format/err | 50 +++++++++ test/cmdlineTests/message_format/input.sol | 103 ++++++++++++++++++ test/cmdlineTests/too_long_line/err | 2 +- .../too_long_line_both_sides_short/err | 2 +- test/cmdlineTests/too_long_line_edge_in/err | 2 +- test/cmdlineTests/too_long_line_edge_out/err | 2 +- .../cmdlineTests/too_long_line_left_short/err | 2 +- test/cmdlineTests/too_long_line_multiline/err | 2 +- .../too_long_line_right_short/err | 2 +- 10 files changed, 173 insertions(+), 19 deletions(-) create mode 100644 test/cmdlineTests/message_format/err create mode 100644 test/cmdlineTests/message_format/input.sol diff --git a/liblangutil/SourceReferenceFormatterHuman.cpp b/liblangutil/SourceReferenceFormatterHuman.cpp index 669fa200748f..430883996349 100644 --- a/liblangutil/SourceReferenceFormatterHuman.cpp +++ b/liblangutil/SourceReferenceFormatterHuman.cpp @@ -70,35 +70,36 @@ void SourceReferenceFormatterHuman::printSourceLocation(SourceReference const& _ if (_ref.sourceName.empty()) return; // Nothing we can print here - int const leftpad = static_cast(log10(max(_ref.position.line, 1))) + 1; - - // line 0: source name - frameColored() << string(leftpad, ' ') << "--> "; - if (_ref.position.line < 0) { + frameColored() << "--> "; m_stream << _ref.sourceName << "\n"; return; // No line available, nothing else to print } - m_stream << _ref.sourceName << ":" << (_ref.position.line + 1) << ":" << (_ref.position.column + 1) << ":" << '\n'; + string line = std::to_string(_ref.position.line + 1); // one-based line number as string + string leftpad = string(line.size(), ' '); + + // line 0: source name + frameColored() << leftpad << "--> "; + m_stream << _ref.sourceName << ":" << line << ":" << (_ref.position.column + 1) << ":" << '\n'; if (!_ref.multiline) { int const locationLength = _ref.endColumn - _ref.startColumn; // line 1: - m_stream << string(leftpad, ' '); + m_stream << leftpad; frameColored() << " |" << '\n'; // line 2: - frameColored() << (_ref.position.line + 1) << " | "; + frameColored() << line << " | "; m_stream << _ref.text.substr(0, _ref.startColumn); highlightColored() << _ref.text.substr(_ref.startColumn, locationLength); m_stream << _ref.text.substr(_ref.endColumn) << '\n'; // line 3: - m_stream << string(leftpad, ' '); + m_stream << leftpad; frameColored() << " | "; for_each( _ref.text.cbegin(), @@ -110,16 +111,16 @@ void SourceReferenceFormatterHuman::printSourceLocation(SourceReference const& _ else { // line 1: - m_stream << string(leftpad, ' '); + m_stream << leftpad; frameColored() << " |" << '\n'; // line 2: - frameColored() << (_ref.position.line + 1) << " | "; + frameColored() << line << " | "; m_stream << _ref.text.substr(0, _ref.startColumn); highlightColored() << _ref.text.substr(_ref.startColumn) << '\n'; // line 3: - frameColored() << string(leftpad, ' ') << " | "; + frameColored() << leftpad << " | "; m_stream << string(_ref.startColumn, ' '); diagColored() << "^ (Relevant source part starts here and spans across multiple lines).\n"; } diff --git a/test/cmdlineTests/message_format/err b/test/cmdlineTests/message_format/err new file mode 100644 index 000000000000..eee8267e25f3 --- /dev/null +++ b/test/cmdlineTests/message_format/err @@ -0,0 +1,50 @@ +Warning: Source file does not specify required compiler version! +--> message_format/input.sol + +Warning: Unused local variable. + --> message_format/input.sol:9:27: + | +9 | function f() public { int x; } + | ^^^^^ + +Warning: Unused local variable. + --> message_format/input.sol:10:27: + | +10 | function g() public { int x; } + | ^^^^^ + +Warning: Unused local variable. + --> message_format/input.sol:99:14: + | +99 | /**/ int a; /**/ + | ^^^^^ + +Warning: Unused local variable. + --> message_format/input.sol:100:14: + | +100 | /**/ int b; /**/ + | ^^^^^ + +Warning: Unused local variable. + --> message_format/input.sol:101:14: + | +101 | /**/ int c; /**/ + | ^^^^^ + +Warning: Function state mutability can be restricted to pure + --> message_format/input.sol:9:5: + | +9 | function f() public { int x; } + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Warning: Function state mutability can be restricted to pure + --> message_format/input.sol:10:5: + | +10 | function g() public { int x; } + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Warning: Function state mutability can be restricted to pure + --> message_format/input.sol:11:5: + | +11 | function h() public { + | ^ (Relevant source part starts here and spans across multiple lines). diff --git a/test/cmdlineTests/message_format/input.sol b/test/cmdlineTests/message_format/input.sol new file mode 100644 index 000000000000..6f854f96af69 --- /dev/null +++ b/test/cmdlineTests/message_format/input.sol @@ -0,0 +1,103 @@ +// checks that error messages around power-or-10 lines are formatted correctly + + + + + + +contract C { + function f() public { int x; } + function g() public { int x; } + function h() public { + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + /**/ int a; /**/ + /**/ int b; /**/ + /**/ int c; /**/ + } +} diff --git a/test/cmdlineTests/too_long_line/err b/test/cmdlineTests/too_long_line/err index 87612fe59196..6e51190371ad 100644 --- a/test/cmdlineTests/too_long_line/err +++ b/test/cmdlineTests/too_long_line/err @@ -1,5 +1,5 @@ Warning: Source file does not specify required compiler version! - --> too_long_line/input.sol +--> too_long_line/input.sol Error: Identifier not found or not unique. --> too_long_line/input.sol:2:164: diff --git a/test/cmdlineTests/too_long_line_both_sides_short/err b/test/cmdlineTests/too_long_line_both_sides_short/err index 81f44930f2a8..34bd25d28a07 100644 --- a/test/cmdlineTests/too_long_line_both_sides_short/err +++ b/test/cmdlineTests/too_long_line_both_sides_short/err @@ -1,5 +1,5 @@ Warning: Source file does not specify required compiler version! - --> too_long_line_both_sides_short/input.sol +--> too_long_line_both_sides_short/input.sol Error: Identifier not found or not unique. --> too_long_line_both_sides_short/input.sol:2:15: diff --git a/test/cmdlineTests/too_long_line_edge_in/err b/test/cmdlineTests/too_long_line_edge_in/err index b00bec7a1c81..2988bc44e578 100644 --- a/test/cmdlineTests/too_long_line_edge_in/err +++ b/test/cmdlineTests/too_long_line_edge_in/err @@ -1,5 +1,5 @@ Warning: Source file does not specify required compiler version! - --> too_long_line_edge_in/input.sol +--> too_long_line_edge_in/input.sol Error: Identifier not found or not unique. --> too_long_line_edge_in/input.sol:2:36: diff --git a/test/cmdlineTests/too_long_line_edge_out/err b/test/cmdlineTests/too_long_line_edge_out/err index 3de913b7d95d..a85faa44fe82 100644 --- a/test/cmdlineTests/too_long_line_edge_out/err +++ b/test/cmdlineTests/too_long_line_edge_out/err @@ -1,5 +1,5 @@ Warning: Source file does not specify required compiler version! - --> too_long_line_edge_out/input.sol +--> too_long_line_edge_out/input.sol Error: Identifier not found or not unique. --> too_long_line_edge_out/input.sol:2:37: diff --git a/test/cmdlineTests/too_long_line_left_short/err b/test/cmdlineTests/too_long_line_left_short/err index a6ddfca81590..aa36b1068fe2 100644 --- a/test/cmdlineTests/too_long_line_left_short/err +++ b/test/cmdlineTests/too_long_line_left_short/err @@ -1,5 +1,5 @@ Warning: Source file does not specify required compiler version! - --> too_long_line_left_short/input.sol +--> too_long_line_left_short/input.sol Error: Identifier not found or not unique. --> too_long_line_left_short/input.sol:2:15: diff --git a/test/cmdlineTests/too_long_line_multiline/err b/test/cmdlineTests/too_long_line_multiline/err index 5a2655174714..2aa801623c39 100644 --- a/test/cmdlineTests/too_long_line_multiline/err +++ b/test/cmdlineTests/too_long_line_multiline/err @@ -5,4 +5,4 @@ Error: No visibility specified. Did you intend to add "public"? | ^ (Relevant source part starts here and spans across multiple lines). Warning: Source file does not specify required compiler version! - --> too_long_line_multiline/input.sol +--> too_long_line_multiline/input.sol diff --git a/test/cmdlineTests/too_long_line_right_short/err b/test/cmdlineTests/too_long_line_right_short/err index f5801e616c7f..d2d8f3980abd 100644 --- a/test/cmdlineTests/too_long_line_right_short/err +++ b/test/cmdlineTests/too_long_line_right_short/err @@ -1,5 +1,5 @@ Warning: Source file does not specify required compiler version! - --> too_long_line_right_short/input.sol +--> too_long_line_right_short/input.sol Error: Identifier not found or not unique. --> too_long_line_right_short/input.sol:2:164: From 523460da077585430094654a09f760aca6fa0aa0 Mon Sep 17 00:00:00 2001 From: a3d4 Date: Fri, 24 Apr 2020 00:51:15 +0200 Subject: [PATCH 087/126] Prevent coloring irrelevant whitespaces --- liblangutil/SourceReferenceFormatterHuman.cpp | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/liblangutil/SourceReferenceFormatterHuman.cpp b/liblangutil/SourceReferenceFormatterHuman.cpp index 430883996349..ca947314e60e 100644 --- a/liblangutil/SourceReferenceFormatterHuman.cpp +++ b/liblangutil/SourceReferenceFormatterHuman.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include using namespace std; @@ -72,8 +71,8 @@ void SourceReferenceFormatterHuman::printSourceLocation(SourceReference const& _ if (_ref.position.line < 0) { - frameColored() << "--> "; - m_stream << _ref.sourceName << "\n"; + frameColored() << "-->"; + m_stream << ' ' << _ref.sourceName << '\n'; return; // No line available, nothing else to print } @@ -81,48 +80,55 @@ void SourceReferenceFormatterHuman::printSourceLocation(SourceReference const& _ string leftpad = string(line.size(), ' '); // line 0: source name - frameColored() << leftpad << "--> "; - m_stream << _ref.sourceName << ":" << line << ":" << (_ref.position.column + 1) << ":" << '\n'; + m_stream << leftpad; + frameColored() << "-->"; + m_stream << ' ' << _ref.sourceName << ':' << line << ':' << (_ref.position.column + 1) << ":\n"; if (!_ref.multiline) { int const locationLength = _ref.endColumn - _ref.startColumn; // line 1: - m_stream << leftpad; - frameColored() << " |" << '\n'; + m_stream << leftpad << ' '; + frameColored() << '|'; + m_stream << '\n'; // line 2: - frameColored() << line << " | "; - m_stream << _ref.text.substr(0, _ref.startColumn); + frameColored() << line << " |"; + m_stream << ' ' << _ref.text.substr(0, _ref.startColumn); highlightColored() << _ref.text.substr(_ref.startColumn, locationLength); m_stream << _ref.text.substr(_ref.endColumn) << '\n'; // line 3: - m_stream << leftpad; - frameColored() << " | "; + m_stream << leftpad << ' '; + frameColored() << '|'; + m_stream << ' '; for_each( _ref.text.cbegin(), _ref.text.cbegin() + _ref.startColumn, [this](char ch) { m_stream << (ch == '\t' ? '\t' : ' '); } ); - diagColored() << string(locationLength, '^') << '\n'; + diagColored() << string(locationLength, '^'); + m_stream << '\n'; } else { // line 1: - m_stream << leftpad; - frameColored() << " |" << '\n'; + m_stream << leftpad << ' '; + frameColored() << '|'; + m_stream << '\n'; // line 2: - frameColored() << line << " | "; - m_stream << _ref.text.substr(0, _ref.startColumn); + frameColored() << line << " |"; + m_stream << ' ' << _ref.text.substr(0, _ref.startColumn); highlightColored() << _ref.text.substr(_ref.startColumn) << '\n'; // line 3: - frameColored() << leftpad << " | "; - m_stream << string(_ref.startColumn, ' '); - diagColored() << "^ (Relevant source part starts here and spans across multiple lines).\n"; + m_stream << leftpad << ' '; + frameColored() << '|'; + m_stream << ' ' << string(_ref.startColumn, ' '); + diagColored() << "^ (Relevant source part starts here and spans across multiple lines)."; + m_stream << '\n'; } } From 059d0bdebbe45edffb676b19ff49a112325e5b99 Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Fri, 24 Apr 2020 11:55:58 +0200 Subject: [PATCH 088/126] Revert "Use Spacer option to improve performance of constant arrays" This reverts commit 92059fa84848ce7f78d93a8af720bef034b74fde. --- libsolidity/formal/Z3CHCInterface.cpp | 2 -- .../functions/functions_recursive_indirect.sol | 1 + ...for_loop_array_assignment_storage_memory.sol | 4 +++- ...or_loop_array_assignment_storage_storage.sol | 1 + ...le_loop_array_assignment_storage_storage.sol | 6 ++++-- .../operators/delete_array_index_2d.sol | 1 - .../smtCheckerTests/types/unused_mapping.sol | 17 ----------------- 7 files changed, 9 insertions(+), 23 deletions(-) delete mode 100644 test/libsolidity/smtCheckerTests/types/unused_mapping.sol diff --git a/libsolidity/formal/Z3CHCInterface.cpp b/libsolidity/formal/Z3CHCInterface.cpp index dcfd19c8617c..c64da2edcc81 100644 --- a/libsolidity/formal/Z3CHCInterface.cpp +++ b/libsolidity/formal/Z3CHCInterface.cpp @@ -43,8 +43,6 @@ Z3CHCInterface::Z3CHCInterface(): p.set("fp.spacer.mbqi", false); // Ground pobs by using values from a model. p.set("fp.spacer.ground_pobs", false); - // Limits array reasoning, good for constant arrays. - p.set("fp.spacer.weak_abs", false); m_solver.set(p); } diff --git a/test/libsolidity/smtCheckerTests/functions/functions_recursive_indirect.sol b/test/libsolidity/smtCheckerTests/functions/functions_recursive_indirect.sol index 7cb7b22b1aaa..5d3292992ad3 100644 --- a/test/libsolidity/smtCheckerTests/functions/functions_recursive_indirect.sol +++ b/test/libsolidity/smtCheckerTests/functions/functions_recursive_indirect.sol @@ -22,3 +22,4 @@ contract C } } // ---- +// Warning: (130-144): Error trying to invoke SMT solver. diff --git a/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_memory.sol b/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_memory.sol index d7d88424a2e9..0b301505b4b0 100644 --- a/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_memory.sol +++ b/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_memory.sol @@ -12,10 +12,12 @@ contract LoopFor2 { b[i] = i + 1; c[i] = b[i]; } + // This is safe but too hard to solve currently. assert(b[0] == c[0]); assert(a[0] == 900); assert(b[0] == 900); } } // ---- -// Warning: (312-331): Assertion violation happens here +// Warning: (316-336): Assertion violation happens here +// Warning: (363-382): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_storage.sol b/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_storage.sol index a90053024d47..333273781373 100644 --- a/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_storage.sol +++ b/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_storage.sol @@ -19,5 +19,6 @@ contract LoopFor2 { } } // ---- +// Warning: (317-337): Assertion violation happens here // Warning: (341-360): Assertion violation happens here // Warning: (364-383): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/loops/while_loop_array_assignment_storage_storage.sol b/test/libsolidity/smtCheckerTests/loops/while_loop_array_assignment_storage_storage.sol index 7670ddb6d88b..92d1ded3ee46 100644 --- a/test/libsolidity/smtCheckerTests/loops/while_loop_array_assignment_storage_storage.sol +++ b/test/libsolidity/smtCheckerTests/loops/while_loop_array_assignment_storage_storage.sol @@ -14,11 +14,13 @@ contract LoopFor2 { c[i] = b[i]; ++i; } + // Fails as false positive. assert(b[0] == c[0]); assert(a[0] == 900); assert(b[0] == 900); } } // ---- -// Warning: (290-309): Assertion violation happens here -// Warning: (313-332): Assertion violation happens here +// Warning: (296-316): Assertion violation happens here +// Warning: (320-339): Assertion violation happens here +// Warning: (343-362): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol b/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol index cb5a0799fea1..8a1ba5e6ef90 100644 --- a/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol +++ b/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol @@ -16,5 +16,4 @@ contract C // ==== // SMTSolvers: z3 // ---- -// Warning: (174-194): Error trying to invoke SMT solver. // Warning: (174-194): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/types/unused_mapping.sol b/test/libsolidity/smtCheckerTests/types/unused_mapping.sol deleted file mode 100644 index f12cd41de5fc..000000000000 --- a/test/libsolidity/smtCheckerTests/types/unused_mapping.sol +++ /dev/null @@ -1,17 +0,0 @@ -pragma experimental SMTChecker; - -contract C { - uint x; - uint y; - mapping (address => bool) public never_used; - - function inc() public { - require(x < 10); - require(y < 10); - - if(x == 0) x = 0; // noop state var read - x++; - y++; - assert(y == x); - } -} From 397ea18b788b0b1277c305a13a14a0b3b3bdce36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Mon, 20 Apr 2020 16:21:57 +0200 Subject: [PATCH 089/126] IRVariable: Fix improperly wrapped docstring --- libsolidity/codegen/ir/IRVariable.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libsolidity/codegen/ir/IRVariable.h b/libsolidity/codegen/ir/IRVariable.h index 8967f55fa8e8..ba4b1b93ad81 100644 --- a/libsolidity/codegen/ir/IRVariable.h +++ b/libsolidity/codegen/ir/IRVariable.h @@ -29,8 +29,7 @@ class Expression; /** * An IRVariable refers to a set of yul variables that correspond to the stack layout of a Solidity variable or expression - * of a specific S - * olidity type. If the Solidity type occupies a single stack slot, the IRVariable refers to a single yul variable. + * of a specific Solidity type. If the Solidity type occupies a single stack slot, the IRVariable refers to a single yul variable. * Otherwise the set of yul variables it refers to is (recursively) determined by @see ``Type::stackItems()``. * For example, an IRVariable referring to a dynamically sized calldata array will consist of two parts named * ``offset`` and ``length``, whereas an IRVariable referring to a statically sized calldata type, a storage reference From 56a85d6cb32df4e0d137d74a178c2bcc0bf6319c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Mon, 20 Apr 2020 23:21:09 +0200 Subject: [PATCH 090/126] IRGeneratorForStatements: Handle internal calls to functions from specific base contracts as static calls rather than calls via pointers --- .../codegen/ir/IRGeneratorForStatements.cpp | 68 +++++++++++++------ .../functionCall/base_base_overload.sol | 2 + .../functionCall/base_overload.sol | 2 + 3 files changed, 52 insertions(+), 20 deletions(-) diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index 908e1d48c63b..c7a873e60aac 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -579,32 +579,60 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall) else args.emplace_back(convert(*arguments[i], *parameterTypes[i]).commaSeparatedList()); - if (auto identifier = dynamic_cast(&_functionCall.expression())) + optional functionDef; + if (auto memberAccess = dynamic_cast(&_functionCall.expression())) { solAssert(!functionType->bound(), ""); - if (auto functionDef = dynamic_cast(identifier->annotation().referencedDeclaration)) + + functionDef = dynamic_cast(memberAccess->annotation().referencedDeclaration); + if (functionDef.value() != nullptr) + solAssert(functionType->declaration() == *memberAccess->annotation().referencedDeclaration, ""); + else { - define(_functionCall) << - m_context.enqueueFunctionForCodeGeneration( - functionDef->resolveVirtual(m_context.mostDerivedContract()) - ) << - "(" << - joinHumanReadable(args) << - ")\n"; - return; + solAssert(dynamic_cast(memberAccess->annotation().referencedDeclaration), ""); + solAssert(!functionType->hasDeclaration(), ""); } } + else if (auto identifier = dynamic_cast(&_functionCall.expression())) + { + solAssert(!functionType->bound(), ""); - define(_functionCall) << - // NOTE: internalDispatch() takes care of adding the function to function generation queue - m_context.internalDispatch( - TupleType(functionType->parameterTypes()).sizeOnStack(), - TupleType(functionType->returnParameterTypes()).sizeOnStack() - ) << - "(" << - IRVariable(_functionCall.expression()).part("functionIdentifier").name() << - joinHumanReadablePrefixed(args) << - ")\n"; + if (auto unresolvedFunctionDef = dynamic_cast(identifier->annotation().referencedDeclaration)) + { + functionDef = &unresolvedFunctionDef->resolveVirtual(m_context.mostDerivedContract()); + solAssert(functionType->declaration() == *identifier->annotation().referencedDeclaration, ""); + } + else + { + functionDef = nullptr; + solAssert(dynamic_cast(identifier->annotation().referencedDeclaration), ""); + solAssert(!functionType->hasDeclaration(), ""); + } + } + else + // Not a simple expression like x or A.x + functionDef = nullptr; + + solAssert(functionDef.has_value(), ""); + solAssert(functionDef.value() == nullptr || functionDef.value()->isImplemented(), ""); + + if (functionDef.value() != nullptr) + define(_functionCall) << + m_context.enqueueFunctionForCodeGeneration(*functionDef.value()) << + "(" << + joinHumanReadable(args) << + ")\n"; + else + define(_functionCall) << + // NOTE: internalDispatch() takes care of adding the function to function generation queue + m_context.internalDispatch( + TupleType(functionType->parameterTypes()).sizeOnStack(), + TupleType(functionType->returnParameterTypes()).sizeOnStack() + ) << + "(" << + IRVariable(_functionCall.expression()).part("functionIdentifier").name() << + joinHumanReadablePrefixed(args) << + ")\n"; break; } case FunctionType::Kind::External: diff --git a/test/libsolidity/semanticTests/functionCall/base_base_overload.sol b/test/libsolidity/semanticTests/functionCall/base_base_overload.sol index adf1200eedec..c1ba1c1ffaed 100644 --- a/test/libsolidity/semanticTests/functionCall/base_base_overload.sol +++ b/test/libsolidity/semanticTests/functionCall/base_base_overload.sol @@ -34,6 +34,8 @@ contract Child is Base { BaseBase.init(c, d); } } +// ==== +// compileViaYul: also // ---- // x() -> 0 // y() -> 0 diff --git a/test/libsolidity/semanticTests/functionCall/base_overload.sol b/test/libsolidity/semanticTests/functionCall/base_overload.sol index 924431ec53f5..be4a73809121 100644 --- a/test/libsolidity/semanticTests/functionCall/base_overload.sol +++ b/test/libsolidity/semanticTests/functionCall/base_overload.sol @@ -18,6 +18,8 @@ contract Child is Base { Base.init(c, d); } } +// ==== +// compileViaYul: also // ---- // x() -> 0 // y() -> 0 From 64bce597a164c917261c618be355ad47976617ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Mon, 20 Apr 2020 16:22:39 +0200 Subject: [PATCH 091/126] IRGenerator: Enable code generation for libraries --- libsolidity/codegen/ir/IRGenerator.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libsolidity/codegen/ir/IRGenerator.cpp b/libsolidity/codegen/ir/IRGenerator.cpp index 92631ddc241a..68a6f2a9d2f9 100644 --- a/libsolidity/codegen/ir/IRGenerator.cpp +++ b/libsolidity/codegen/ir/IRGenerator.cpp @@ -75,15 +75,15 @@ pair IRGenerator::run(ContractDefinition const& _contract) string IRGenerator::generate(ContractDefinition const& _contract) { - solUnimplementedAssert(!_contract.isLibrary(), "Libraries not yet implemented."); - Whiskers t(R"( object "" { code { + let := () () + } @@ -101,6 +101,7 @@ string IRGenerator::generate(ContractDefinition const& _contract) t("CreationObject", creationObjectName(_contract)); t("memoryInit", memoryInit()); + t("notLibrary", !_contract.isLibrary()); FunctionDefinition const* constructor = _contract.constructor(); t("callValueCheck", !constructor || !constructor->isPayable() ? callValueCheck() : ""); From d3da8782002f596c49c95eec98872f92515bc636 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 23 Apr 2020 15:59:42 +0200 Subject: [PATCH 092/126] Enable internal library calls --- libsolidity/codegen/ir/IRGenerationContext.cpp | 5 +++++ .../codegen/ir/IRGeneratorForStatements.cpp | 14 +++++++++++--- .../inherited_function_from_a_library.sol | 2 ++ .../libraries/internal_library_function.sol | 2 ++ .../internal_library_function_calling_private.sol | 2 ++ .../semanticTests/libraries/stub_internal.sol | 2 ++ 6 files changed, 24 insertions(+), 3 deletions(-) diff --git a/libsolidity/codegen/ir/IRGenerationContext.cpp b/libsolidity/codegen/ir/IRGenerationContext.cpp index 0410d883577d..aa6ec7619c09 100644 --- a/libsolidity/codegen/ir/IRGenerationContext.cpp +++ b/libsolidity/codegen/ir/IRGenerationContext.cpp @@ -135,6 +135,11 @@ string IRGenerationContext::internalDispatch(size_t _in, size_t _out) templ("arrow", _out > 0 ? "->" : ""); templ("assignment_op", _out > 0 ? ":=" : ""); templ("out", suffixedVariableNameList("out_", 0, _out)); + + // UNIMPLEMENTED: Internal library calls via pointers are not implemented yet. + // We're not generating code for internal library functions here even though it's possible + // to call them via pointers. Right now such calls end up triggering the `default` case in + // the switch above. vector> functions; for (auto const& contract: mostDerivedContract().annotation().linearizedBaseContracts) for (FunctionDefinition const* function: contract->definedFunctions()) diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index c7a873e60aac..08a5fa900b74 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -567,6 +567,14 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall) arguments.push_back(callArguments[std::distance(callArgumentNames.begin(), it)]); } + if (auto memberAccess = dynamic_cast(&_functionCall.expression())) + if (auto expressionType = dynamic_cast(memberAccess->expression().annotation().type)) + if (auto contractType = dynamic_cast(expressionType->actualType())) + solUnimplementedAssert( + !contractType->contractDefinition().isLibrary() || functionType->kind() == FunctionType::Kind::Internal, + "Only internal function calls implemented for libraries" + ); + solUnimplementedAssert(!functionType->bound(), ""); switch (functionType->kind()) { @@ -582,7 +590,7 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall) optional functionDef; if (auto memberAccess = dynamic_cast(&_functionCall.expression())) { - solAssert(!functionType->bound(), ""); + solUnimplementedAssert(!functionType->bound(), "Internal calls to bound functions are not yet implemented for libraries and not allowed for contracts"); functionDef = dynamic_cast(memberAccess->annotation().referencedDeclaration); if (functionDef.value() != nullptr) @@ -1325,9 +1333,9 @@ void IRGeneratorForStatements::endVisit(Identifier const& _identifier) define(_identifier) << to_string(functionDef->resolveVirtual(m_context.mostDerivedContract()).id()) << "\n"; else if (VariableDeclaration const* varDecl = dynamic_cast(declaration)) handleVariableReference(*varDecl, _identifier); - else if (auto contract = dynamic_cast(declaration)) + else if (dynamic_cast(declaration)) { - solUnimplementedAssert(!contract->isLibrary(), "Libraries not yet supported."); + // no-op } else if (dynamic_cast(declaration)) { diff --git a/test/libsolidity/semanticTests/intheritance/inherited_function_from_a_library.sol b/test/libsolidity/semanticTests/intheritance/inherited_function_from_a_library.sol index 42d4f711c9b9..3756a4e97cf7 100644 --- a/test/libsolidity/semanticTests/intheritance/inherited_function_from_a_library.sol +++ b/test/libsolidity/semanticTests/intheritance/inherited_function_from_a_library.sol @@ -15,5 +15,7 @@ contract B { } } +// ==== +// compileViaYul: also // ---- // g() -> 1 diff --git a/test/libsolidity/semanticTests/libraries/internal_library_function.sol b/test/libsolidity/semanticTests/libraries/internal_library_function.sol index a3c8a870018e..8e8d561e9b27 100644 --- a/test/libsolidity/semanticTests/libraries/internal_library_function.sol +++ b/test/libsolidity/semanticTests/libraries/internal_library_function.sol @@ -17,5 +17,7 @@ contract C { } } +// ==== +// compileViaYul: also // ---- // f() -> 2 diff --git a/test/libsolidity/semanticTests/libraries/internal_library_function_calling_private.sol b/test/libsolidity/semanticTests/libraries/internal_library_function_calling_private.sol index 2283c30ff5fc..e9806aa4f96c 100644 --- a/test/libsolidity/semanticTests/libraries/internal_library_function_calling_private.sol +++ b/test/libsolidity/semanticTests/libraries/internal_library_function_calling_private.sol @@ -22,5 +22,7 @@ contract C { } } +// ==== +// compileViaYul: also // ---- // f() -> 2 diff --git a/test/libsolidity/semanticTests/libraries/stub_internal.sol b/test/libsolidity/semanticTests/libraries/stub_internal.sol index 075b8dbb22dc..4634d4e7af5e 100644 --- a/test/libsolidity/semanticTests/libraries/stub_internal.sol +++ b/test/libsolidity/semanticTests/libraries/stub_internal.sol @@ -6,6 +6,8 @@ contract C { return L.f(v); } } +// ==== +// compileViaYul: also // ---- // g(uint256): 1 -> 1 // g(uint256): 2 -> 4 From 172b6c245ff6a2be2bcb0a46d48acbd62a439c2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 24 Apr 2020 10:30:19 +0200 Subject: [PATCH 093/126] cmdlineTests.sh: Fix the script not removing all temporary files it creates in /tmp - The script was leaving hundreds of loose `tmp.XXXXXX` and `tmp.XXXXXX.bak` files in `/tmp` after each run - There's a trap handler that removes them but it's being registered multiple times in a loop and only the last one actually runs when the script exits. It's still useful because it removes the remaining files from the most recent iteration but on its own it's not enough to clean up everything. --- test/cmdlineTests.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index e99969808a9c..077aa5c19e92 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -126,7 +126,7 @@ function test_solc_behaviour() # Remove trailing empty lines. Needs a line break to make OSX sed happy. sed -i.bak -e '1{/^$/d }' "$stderr_path" - rm "$stderr_path.bak" + rm "$stderr_path.bak" "$stdout_path.bak" fi # Remove path to cpp file sed -i.bak -e 's/^\(Exception while assembling:\).*/\1/' "$stderr_path" @@ -175,6 +175,8 @@ function test_solc_behaviour() exit 1 fi fi + + rm -f "$stdout_path" "$stderr_path" } From 55eda85a302e477519a3ae3ed5a3baf75d8a4be1 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Fri, 24 Apr 2020 08:06:15 -0500 Subject: [PATCH 094/126] docs/cheatsheet.rst: Add ``type(I).interfaceId`` description. --- docs/cheatsheet.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/cheatsheet.rst b/docs/cheatsheet.rst index 26d7480b9322..5d6169a48251 100644 --- a/docs/cheatsheet.rst +++ b/docs/cheatsheet.rst @@ -121,6 +121,7 @@ Global Variables - ``type(C).name`` (``string``): the name of the contract - ``type(C).creationCode`` (``bytes memory``): creation bytecode of the given contract, see :ref:`Type Information`. - ``type(C).runtimeCode`` (``bytes memory``): runtime bytecode of the given contract, see :ref:`Type Information`. +- ``type(I).interfaceId`` (``bytes4``): value containing the EIP-165 interface identifier of the given interface, see :ref:`Type Information`. .. note:: Do not rely on ``block.timestamp``, ``now`` and ``blockhash`` as a source of randomness, From 5a515240acd90372b47d9b5b451caa1dffdef3ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 16 Apr 2020 17:13:56 +0200 Subject: [PATCH 095/126] OptimiserSuite: Use brackets instead of parentheses as syntax for repeating abbreviation sequences - We want to start accepting abbreviation sequences on the command line and parentheses would always have to be escaped in that context. - There wasn't any important reason behind choosing () rather than [] or {} and it still isn't too late to switch. --- libyul/optimiser/Suite.cpp | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/libyul/optimiser/Suite.cpp b/libyul/optimiser/Suite.cpp index d1c6a6dd4cc4..4a175eb023e2 100644 --- a/libyul/optimiser/Suite.cpp +++ b/libyul/optimiser/Suite.cpp @@ -95,7 +95,7 @@ void OptimiserSuite::run( suite.runSequence( "dhfoDgvulfnTUtnIf" // None of these can make stack problems worse - "(" + "[" "xarrscLM" // Turn into SSA and simplify "cCTUtTOntnfDIul" // Perform structural simplification "Lcul" // Simplify again @@ -108,7 +108,7 @@ void OptimiserSuite::run( "xarrcL" // Turn into SSA again and simplify "gvif" // Run full inliner "CTUcarrLsTOtfDncarrIulc" // SSA plus simplify - ")" + "]" "jmuljuljul VcTOcul jmul", // Make source short and pretty ast ); @@ -256,12 +256,12 @@ void OptimiserSuite::runSequence(string const& _stepAbbreviations, Block& _ast) for (char abbreviation: input) switch (abbreviation) { - case '(': - assertThrow(!insideLoop, OptimizerException, "Nested parentheses not supported"); + case '[': + assertThrow(!insideLoop, OptimizerException, "Nested brackets not supported"); insideLoop = true; break; - case ')': - assertThrow(insideLoop, OptimizerException, "Unbalanced parenthesis"); + case ']': + assertThrow(insideLoop, OptimizerException, "Unbalanced bracket"); insideLoop = false; break; default: @@ -271,7 +271,7 @@ void OptimiserSuite::runSequence(string const& _stepAbbreviations, Block& _ast) "Invalid optimisation step abbreviation" ); } - assertThrow(!insideLoop, OptimizerException, "Unbalanced parenthesis"); + assertThrow(!insideLoop, OptimizerException, "Unbalanced bracket"); auto abbreviationsToSteps = [](string const& _sequence) -> vector { @@ -281,21 +281,21 @@ void OptimiserSuite::runSequence(string const& _stepAbbreviations, Block& _ast) return steps; }; - // The sequence has now been validated and must consist of pairs of segments that look like this: `aaa(bbb)` - // `aaa` or `(bbb)` can be empty. For example we consider a sequence like `fgo(aaf)Oo` to have - // four segments, the last of which is an empty parenthesis. + // The sequence has now been validated and must consist of pairs of segments that look like this: `aaa[bbb]` + // `aaa` or `[bbb]` can be empty. For example we consider a sequence like `fgo[aaf]Oo` to have + // four segments, the last of which is an empty bracket. size_t currentPairStart = 0; while (currentPairStart < input.size()) { - size_t openingParenthesis = input.find('(', currentPairStart); - size_t closingParenthesis = input.find(')', openingParenthesis); - size_t firstCharInside = (openingParenthesis == string::npos ? input.size() : openingParenthesis + 1); - yulAssert((openingParenthesis == string::npos) == (closingParenthesis == string::npos), ""); + size_t openingBracket = input.find('[', currentPairStart); + size_t closingBracket = input.find(']', openingBracket); + size_t firstCharInside = (openingBracket == string::npos ? input.size() : openingBracket + 1); + yulAssert((openingBracket == string::npos) == (closingBracket == string::npos), ""); - runSequence(abbreviationsToSteps(input.substr(currentPairStart, openingParenthesis - currentPairStart)), _ast); - runSequenceUntilStable(abbreviationsToSteps(input.substr(firstCharInside, closingParenthesis - firstCharInside)), _ast); + runSequence(abbreviationsToSteps(input.substr(currentPairStart, openingBracket - currentPairStart)), _ast); + runSequenceUntilStable(abbreviationsToSteps(input.substr(firstCharInside, closingBracket - firstCharInside)), _ast); - currentPairStart = (closingParenthesis == string::npos ? input.size() : closingParenthesis + 1); + currentPairStart = (closingBracket == string::npos ? input.size() : closingBracket + 1); } } From e2c0e6331c8c9896c007e536bb5e88ca76e09097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 24 Apr 2020 14:13:34 +0200 Subject: [PATCH 096/126] OptimiserSuite: Define NonStepAbbreviations and use it for extra sanity checks --- libyul/optimiser/Suite.cpp | 11 +++++++++++ libyul/optimiser/Suite.h | 4 ++++ 2 files changed, 15 insertions(+) diff --git a/libyul/optimiser/Suite.cpp b/libyul/optimiser/Suite.cpp index 4a175eb023e2..1a0628b00a2c 100644 --- a/libyul/optimiser/Suite.cpp +++ b/libyul/optimiser/Suite.cpp @@ -67,6 +67,7 @@ #include +#include #include using namespace std; @@ -235,6 +236,12 @@ map const& OptimiserSuite::stepNameToAbbreviationMap() {VarDeclInitializer::name, 'd'}, }; yulAssert(lookupTable.size() == allSteps().size(), ""); + yulAssert(( + util::convertContainer>(string(NonStepAbbreviations)) - + util::convertContainer>(lookupTable | boost::adaptors::map_values) + ).size() == string(NonStepAbbreviations).size(), + "Step abbreviation conflicts with a character reserved for another syntactic element" + ); return lookupTable; } @@ -265,6 +272,10 @@ void OptimiserSuite::runSequence(string const& _stepAbbreviations, Block& _ast) insideLoop = false; break; default: + yulAssert( + string(NonStepAbbreviations).find(abbreviation) == string::npos, + "Unhandled syntactic element in the abbreviation sequence" + ); assertThrow( stepAbbreviationToNameMap().find(abbreviation) != stepAbbreviationToNameMap().end(), OptimizerException, diff --git a/libyul/optimiser/Suite.h b/libyul/optimiser/Suite.h index 3b0ed0163a5b..e8a6c7a56e4e 100644 --- a/libyul/optimiser/Suite.h +++ b/libyul/optimiser/Suite.h @@ -47,6 +47,10 @@ class OptimiserSuite public: static constexpr size_t MaxRounds = 12; + /// Special characters that do not represent optimiser steps but are allowed in abbreviation sequences. + /// Some of them (like whitespace) are ignored, others (like brackets) are a part of the syntax. + static constexpr char NonStepAbbreviations[] = " \n[]"; + enum class Debug { None, From 69b79f848b1faab99def4ebc066ac854857bde93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 24 Apr 2020 14:15:45 +0200 Subject: [PATCH 097/126] OptimiserSuite: Allow validating the optimisation sequence without executing it - Create a separate validateSequence() that can be used independently. - Tweak the exception messages a bit to be usable as command-line errors --- libyul/optimiser/Suite.cpp | 28 ++++++++++++++++++---------- libyul/optimiser/Suite.h | 4 ++++ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/libyul/optimiser/Suite.cpp b/libyul/optimiser/Suite.cpp index 1a0628b00a2c..338acd0e8606 100644 --- a/libyul/optimiser/Suite.cpp +++ b/libyul/optimiser/Suite.cpp @@ -253,22 +253,21 @@ map const& OptimiserSuite::stepAbbreviationToNameMap() return lookupTable; } -void OptimiserSuite::runSequence(string const& _stepAbbreviations, Block& _ast) +void OptimiserSuite::validateSequence(string const& _stepAbbreviations) { - string input = _stepAbbreviations; - boost::remove_erase(input, ' '); - boost::remove_erase(input, '\n'); - bool insideLoop = false; - for (char abbreviation: input) + for (char abbreviation: _stepAbbreviations) switch (abbreviation) { + case ' ': + case '\n': + break; case '[': - assertThrow(!insideLoop, OptimizerException, "Nested brackets not supported"); + assertThrow(!insideLoop, OptimizerException, "Nested brackets are not supported"); insideLoop = true; break; case ']': - assertThrow(insideLoop, OptimizerException, "Unbalanced bracket"); + assertThrow(insideLoop, OptimizerException, "Unbalanced brackets"); insideLoop = false; break; default: @@ -279,10 +278,19 @@ void OptimiserSuite::runSequence(string const& _stepAbbreviations, Block& _ast) assertThrow( stepAbbreviationToNameMap().find(abbreviation) != stepAbbreviationToNameMap().end(), OptimizerException, - "Invalid optimisation step abbreviation" + "'"s + abbreviation + "' is not a valid step abbreviation" ); } - assertThrow(!insideLoop, OptimizerException, "Unbalanced bracket"); + assertThrow(!insideLoop, OptimizerException, "Unbalanced brackets"); +} + +void OptimiserSuite::runSequence(string const& _stepAbbreviations, Block& _ast) +{ + validateSequence(_stepAbbreviations); + + string input = _stepAbbreviations; + boost::remove_erase(input, ' '); + boost::remove_erase(input, '\n'); auto abbreviationsToSteps = [](string const& _sequence) -> vector { diff --git a/libyul/optimiser/Suite.h b/libyul/optimiser/Suite.h index e8a6c7a56e4e..36df60ca69d6 100644 --- a/libyul/optimiser/Suite.h +++ b/libyul/optimiser/Suite.h @@ -65,6 +65,10 @@ class OptimiserSuite std::set const& _externallyUsedIdentifiers = {} ); + /// Ensures that specified sequence of step abbreviations is well-formed and can be executed. + /// @throw OptimizerException if the sequence is invalid + static void validateSequence(std::string const& _stepAbbreviations); + void runSequence(std::vector const& _steps, Block& _ast); void runSequence(std::string const& _stepAbbreviations, Block& _ast); void runSequenceUntilStable( From c41a832f6504815ce423a1b6f6e9c5f1719058ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 24 Apr 2020 14:19:36 +0200 Subject: [PATCH 098/126] Move the default optimisation steps from OptimiserSuite to OptimiserSettings - Now it's a mandatory parameter in OptimiserSuite::run() --- libsolidity/codegen/CompilerContext.cpp | 1 + libsolidity/interface/OptimiserSettings.h | 25 ++++++++++++++++++++++ libyul/AssemblyStack.cpp | 3 ++- libyul/optimiser/Suite.cpp | 26 ++++++----------------- libyul/optimiser/Suite.h | 1 + test/libyul/YulOptimizerTest.cpp | 4 +++- 6 files changed, 39 insertions(+), 21 deletions(-) diff --git a/libsolidity/codegen/CompilerContext.cpp b/libsolidity/codegen/CompilerContext.cpp index 2b4a888ee4d1..2d81e833b432 100644 --- a/libsolidity/codegen/CompilerContext.cpp +++ b/libsolidity/codegen/CompilerContext.cpp @@ -502,6 +502,7 @@ void CompilerContext::optimizeYul(yul::Object& _object, yul::EVMDialect const& _ &meter, _object, _optimiserSettings.optimizeStackAllocation, + _optimiserSettings.yulOptimiserSteps, _externalIdentifiers ); diff --git a/libsolidity/interface/OptimiserSettings.h b/libsolidity/interface/OptimiserSettings.h index 16aae9493bf1..cd82290af073 100644 --- a/libsolidity/interface/OptimiserSettings.h +++ b/libsolidity/interface/OptimiserSettings.h @@ -23,12 +23,31 @@ #pragma once #include +#include namespace solidity::frontend { struct OptimiserSettings { + static char constexpr DefaultYulOptimiserSteps[] = + "dhfoDgvulfnTUtnIf" // None of these can make stack problems worse + "[" + "xarrscLM" // Turn into SSA and simplify + "cCTUtTOntnfDIul" // Perform structural simplification + "Lcul" // Simplify again + "Vcul jj" // Reverse SSA + + // should have good "compilability" property here. + + "eul" // Run functional expression inliner + "xarulrul" // Prune a bit more in SSA + "xarrcL" // Turn into SSA again and simplify + "gvif" // Run full inliner + "CTUcarrLsTOtfDncarrIulc" // SSA plus simplify + "]" + "jmuljuljul VcTOcul jmul"; // Make source short and pretty + /// No optimisations at all - not recommended. static OptimiserSettings none() { @@ -74,6 +93,7 @@ struct OptimiserSettings runConstantOptimiser == _other.runConstantOptimiser && optimizeStackAllocation == _other.optimizeStackAllocation && runYulOptimiser == _other.runYulOptimiser && + yulOptimiserSteps == _other.yulOptimiserSteps && expectedExecutionsPerDeployment == _other.expectedExecutionsPerDeployment; } @@ -95,6 +115,11 @@ struct OptimiserSettings bool optimizeStackAllocation = false; /// Yul optimiser with default settings. Will only run on certain parts of the code for now. bool runYulOptimiser = false; + /// Sequence of optimisation steps to be performed by Yul optimiser. + /// Note that there are some hard-coded steps in the optimiser and you cannot disable + /// them just by setting this to an empty string. Set @a runYulOptimiser to false if you want + /// no optimisations. + std::string yulOptimiserSteps = DefaultYulOptimiserSteps; /// This specifies an estimate on how often each opcode in this assembly will be executed, /// i.e. use a small value to optimise for size and a large value to optimise for runtime gas usage. size_t expectedExecutionsPerDeployment = 200; diff --git a/libyul/AssemblyStack.cpp b/libyul/AssemblyStack.cpp index e64420bd91fd..3988bc3fd1dd 100644 --- a/libyul/AssemblyStack.cpp +++ b/libyul/AssemblyStack.cpp @@ -183,7 +183,8 @@ void AssemblyStack::optimize(Object& _object, bool _isCreation) dialect, meter.get(), _object, - m_optimiserSettings.optimizeStackAllocation + m_optimiserSettings.optimizeStackAllocation, + m_optimiserSettings.yulOptimiserSteps ); } diff --git a/libyul/optimiser/Suite.cpp b/libyul/optimiser/Suite.cpp index 338acd0e8606..9f2511e7232c 100644 --- a/libyul/optimiser/Suite.cpp +++ b/libyul/optimiser/Suite.cpp @@ -79,6 +79,7 @@ void OptimiserSuite::run( GasMeter const* _meter, Object& _object, bool _optimizeStackAllocation, + string const& _optimisationSequence, set const& _externallyUsedIdentifiers ) { @@ -94,25 +95,12 @@ void OptimiserSuite::run( OptimiserSuite suite(_dialect, reservedIdentifiers, Debug::None, ast); - suite.runSequence( - "dhfoDgvulfnTUtnIf" // None of these can make stack problems worse - "[" - "xarrscLM" // Turn into SSA and simplify - "cCTUtTOntnfDIul" // Perform structural simplification - "Lcul" // Simplify again - "Vcul jj" // Reverse SSA - - // should have good "compilability" property here. - - "eul" // Run functional expression inliner - "xarulrul" // Prune a bit more in SSA - "xarrcL" // Turn into SSA again and simplify - "gvif" // Run full inliner - "CTUcarrLsTOtfDncarrIulc" // SSA plus simplify - "]" - "jmuljuljul VcTOcul jmul", // Make source short and pretty - ast - ); + // Some steps depend on properties ensured by FunctionHoister, FunctionGrouper and + // ForLoopInitRewriter. Run them first to be able to run arbitrary sequences safely. + suite.runSequence("fgo", ast); + + // Now the user-supplied part + suite.runSequence(_optimisationSequence, ast); // This is a tuning parameter, but actually just prevents infinite loops. size_t stackCompressorMaxIterations = 16; diff --git a/libyul/optimiser/Suite.h b/libyul/optimiser/Suite.h index 36df60ca69d6..790b2f9ad037 100644 --- a/libyul/optimiser/Suite.h +++ b/libyul/optimiser/Suite.h @@ -62,6 +62,7 @@ class OptimiserSuite GasMeter const* _meter, Object& _object, bool _optimizeStackAllocation, + std::string const& _optimisationSequence, std::set const& _externallyUsedIdentifiers = {} ); diff --git a/test/libyul/YulOptimizerTest.cpp b/test/libyul/YulOptimizerTest.cpp index 2a5209b25853..042d528dbedf 100644 --- a/test/libyul/YulOptimizerTest.cpp +++ b/test/libyul/YulOptimizerTest.cpp @@ -74,6 +74,8 @@ #include +#include + #include #include @@ -342,7 +344,7 @@ TestCase::TestResult YulOptimizerTest::run(ostream& _stream, string const& _line yul::Object obj; obj.code = m_ast; obj.analysisInfo = m_analysisInfo; - OptimiserSuite::run(*m_dialect, &meter, obj, true); + OptimiserSuite::run(*m_dialect, &meter, obj, true, solidity::frontend::OptimiserSettings::DefaultYulOptimiserSteps); } else { From 35cc64e33d6773ec30dbccfb4eaeff18238a71fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 24 Apr 2020 14:24:48 +0200 Subject: [PATCH 099/126] Add --yul-optimizations option to the command-line interface --- solc/CommandLineInterface.cpp | 30 ++++++++++++++-- test/cmdlineTests/yul_optimizer_steps/args | 1 + .../yul_optimizer_steps/input.sol | 6 ++++ test/cmdlineTests/yul_optimizer_steps/output | 34 +++++++++++++++++++ .../yul_optimizer_steps_disabled/args | 1 + .../yul_optimizer_steps_disabled/err | 1 + .../yul_optimizer_steps_disabled/exit | 1 + .../yul_optimizer_steps_disabled/input.sol | 6 ++++ .../args | 1 + .../err | 1 + .../exit | 1 + .../input.sol | 6 ++++ .../yul_optimizer_steps_invalid_nesting/args | 1 + .../yul_optimizer_steps_invalid_nesting/err | 1 + .../yul_optimizer_steps_invalid_nesting/exit | 1 + .../input.sol | 6 ++++ .../args | 1 + .../err | 1 + .../exit | 1 + .../input.sol | 6 ++++ 20 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 test/cmdlineTests/yul_optimizer_steps/args create mode 100644 test/cmdlineTests/yul_optimizer_steps/input.sol create mode 100644 test/cmdlineTests/yul_optimizer_steps/output create mode 100644 test/cmdlineTests/yul_optimizer_steps_disabled/args create mode 100644 test/cmdlineTests/yul_optimizer_steps_disabled/err create mode 100644 test/cmdlineTests/yul_optimizer_steps_disabled/exit create mode 100644 test/cmdlineTests/yul_optimizer_steps_disabled/input.sol create mode 100644 test/cmdlineTests/yul_optimizer_steps_invalid_abbreviation/args create mode 100644 test/cmdlineTests/yul_optimizer_steps_invalid_abbreviation/err create mode 100644 test/cmdlineTests/yul_optimizer_steps_invalid_abbreviation/exit create mode 100644 test/cmdlineTests/yul_optimizer_steps_invalid_abbreviation/input.sol create mode 100644 test/cmdlineTests/yul_optimizer_steps_invalid_nesting/args create mode 100644 test/cmdlineTests/yul_optimizer_steps_invalid_nesting/err create mode 100644 test/cmdlineTests/yul_optimizer_steps_invalid_nesting/exit create mode 100644 test/cmdlineTests/yul_optimizer_steps_invalid_nesting/input.sol create mode 100644 test/cmdlineTests/yul_optimizer_steps_unbalanced_bracket/args create mode 100644 test/cmdlineTests/yul_optimizer_steps_unbalanced_bracket/err create mode 100644 test/cmdlineTests/yul_optimizer_steps_unbalanced_bracket/exit create mode 100644 test/cmdlineTests/yul_optimizer_steps_unbalanced_bracket/input.sol diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index 79da752893f4..da14c58f9fdf 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -37,6 +37,7 @@ #include #include +#include #include #include @@ -145,6 +146,7 @@ static string const g_strOpcodes = "opcodes"; static string const g_strOptimize = "optimize"; static string const g_strOptimizeRuns = "optimize-runs"; static string const g_strOptimizeYul = "optimize-yul"; +static string const g_strYulOptimizations = "yul-optimizations"; static string const g_strOutputDir = "output-dir"; static string const g_strOverwrite = "overwrite"; static string const g_strRevertStrings = "revert-strings"; @@ -781,7 +783,6 @@ Allowed options)", "Import ASTs to be compiled, assumes input holds the AST in compact JSON format. " "Supported Inputs is the output of the --standard-json or the one produced by --combined-json ast,compact-format" ) - ( g_argAssemble.c_str(), "Switch to assembly mode, ignoring all options except --machine, --yul-dialect and --optimize and assumes input is assembly." @@ -835,7 +836,12 @@ Allowed options)", "Lower values will optimize more for initial deployment cost, higher values will optimize more for high-frequency usage." ) (g_strOptimizeYul.c_str(), "Legacy option, ignored. Use the general --optimize to enable Yul optimizer.") - (g_strNoOptimizeYul.c_str(), "Disable Yul optimizer in Solidity."); + (g_strNoOptimizeYul.c_str(), "Disable Yul optimizer in Solidity.") + ( + g_strYulOptimizations.c_str(), + po::value()->value_name("steps"), + "Forces yul optimizer to use the specified sequence of optimization steps instead of the built-in one." + ); desc.add(optimizerOptions); po::options_description outputComponents("Output Components"); outputComponents.add_options() @@ -1168,6 +1174,26 @@ bool CommandLineInterface::processInput() settings.expectedExecutionsPerDeployment = m_args[g_argOptimizeRuns].as(); if (m_args.count(g_strNoOptimizeYul)) settings.runYulOptimiser = false; + if (m_args.count(g_strYulOptimizations)) + { + if (!settings.runYulOptimiser) + { + serr() << "--" << g_strYulOptimizations << " is invalid if Yul optimizer is disabled" << endl; + return false; + } + + try + { + yul::OptimiserSuite::validateSequence(m_args[g_strYulOptimizations].as()); + } + catch (yul::OptimizerException const& _exception) + { + serr() << "Invalid optimizer step sequence in --" << g_strYulOptimizations << ": " << _exception.what() << endl; + return false; + } + + settings.yulOptimiserSteps = m_args[g_strYulOptimizations].as(); + } settings.optimizeStackAllocation = settings.runYulOptimiser; m_compiler->setOptimiserSettings(settings); diff --git a/test/cmdlineTests/yul_optimizer_steps/args b/test/cmdlineTests/yul_optimizer_steps/args new file mode 100644 index 000000000000..1f84565de294 --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps/args @@ -0,0 +1 @@ +--ir-optimized --optimize --yul-optimizations dhfoDgvulfnTUtnIf diff --git a/test/cmdlineTests/yul_optimizer_steps/input.sol b/test/cmdlineTests/yul_optimizer_steps/input.sol new file mode 100644 index 000000000000..f787d2fcf147 --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps/input.sol @@ -0,0 +1,6 @@ +pragma solidity >=0.0; + +contract C +{ + constructor() public {} +} diff --git a/test/cmdlineTests/yul_optimizer_steps/output b/test/cmdlineTests/yul_optimizer_steps/output new file mode 100644 index 000000000000..847c731e9388 --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps/output @@ -0,0 +1,34 @@ +Optimized IR: +/******************************************************* + * WARNING * + * Solidity to Yul compilation is still EXPERIMENTAL * + * It can result in LOSS OF FUNDS or worse * + * !USE AT YOUR OWN RISK! * + *******************************************************/ + +object "C_6" { + code { + { + mstore(64, 128) + if callvalue() { revert(0, 0) } + codecopy(0, dataoffset("C_6_deployed"), datasize("C_6_deployed")) + return(0, datasize("C_6_deployed")) + } + } + object "C_6_deployed" { + code { + { + mstore(64, 128) + if iszero(lt(calldatasize(), 4)) + { + let selector := shift_right_224_unsigned(calldataload(0)) + pop(selector) + } + pop(iszero(calldatasize())) + revert(0, 0) + } + function shift_right_224_unsigned(value) -> newValue + { newValue := shr(224, value) } + } + } +} diff --git a/test/cmdlineTests/yul_optimizer_steps_disabled/args b/test/cmdlineTests/yul_optimizer_steps_disabled/args new file mode 100644 index 000000000000..449d8191c131 --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps_disabled/args @@ -0,0 +1 @@ +--ir-optimized --yul-optimizations dhfoDgvulfnTUtnIf diff --git a/test/cmdlineTests/yul_optimizer_steps_disabled/err b/test/cmdlineTests/yul_optimizer_steps_disabled/err new file mode 100644 index 000000000000..ab77d4e1c1e2 --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps_disabled/err @@ -0,0 +1 @@ +--yul-optimizations is invalid if Yul optimizer is disabled diff --git a/test/cmdlineTests/yul_optimizer_steps_disabled/exit b/test/cmdlineTests/yul_optimizer_steps_disabled/exit new file mode 100644 index 000000000000..d00491fd7e5b --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps_disabled/exit @@ -0,0 +1 @@ +1 diff --git a/test/cmdlineTests/yul_optimizer_steps_disabled/input.sol b/test/cmdlineTests/yul_optimizer_steps_disabled/input.sol new file mode 100644 index 000000000000..363a4c72182c --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps_disabled/input.sol @@ -0,0 +1,6 @@ +pragma solidity >=0.0; + +contract C +{ + function f() public pure {} +} diff --git a/test/cmdlineTests/yul_optimizer_steps_invalid_abbreviation/args b/test/cmdlineTests/yul_optimizer_steps_invalid_abbreviation/args new file mode 100644 index 000000000000..e10815d812c9 --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps_invalid_abbreviation/args @@ -0,0 +1 @@ +--ir-optimized --optimize --yul-optimizations abcdefg{hijklmno}pqr[st]uvwxyz diff --git a/test/cmdlineTests/yul_optimizer_steps_invalid_abbreviation/err b/test/cmdlineTests/yul_optimizer_steps_invalid_abbreviation/err new file mode 100644 index 000000000000..b66c6c60f81e --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps_invalid_abbreviation/err @@ -0,0 +1 @@ +Invalid optimizer step sequence in --yul-optimizations: 'b' is not a valid step abbreviation diff --git a/test/cmdlineTests/yul_optimizer_steps_invalid_abbreviation/exit b/test/cmdlineTests/yul_optimizer_steps_invalid_abbreviation/exit new file mode 100644 index 000000000000..d00491fd7e5b --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps_invalid_abbreviation/exit @@ -0,0 +1 @@ +1 diff --git a/test/cmdlineTests/yul_optimizer_steps_invalid_abbreviation/input.sol b/test/cmdlineTests/yul_optimizer_steps_invalid_abbreviation/input.sol new file mode 100644 index 000000000000..363a4c72182c --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps_invalid_abbreviation/input.sol @@ -0,0 +1,6 @@ +pragma solidity >=0.0; + +contract C +{ + function f() public pure {} +} diff --git a/test/cmdlineTests/yul_optimizer_steps_invalid_nesting/args b/test/cmdlineTests/yul_optimizer_steps_invalid_nesting/args new file mode 100644 index 000000000000..724fe4724a43 --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps_invalid_nesting/args @@ -0,0 +1 @@ +--ir-optimized --optimize --yul-optimizations a[a][aa[aa]]a diff --git a/test/cmdlineTests/yul_optimizer_steps_invalid_nesting/err b/test/cmdlineTests/yul_optimizer_steps_invalid_nesting/err new file mode 100644 index 000000000000..2f6d9ff47330 --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps_invalid_nesting/err @@ -0,0 +1 @@ +Invalid optimizer step sequence in --yul-optimizations: Nested brackets are not supported diff --git a/test/cmdlineTests/yul_optimizer_steps_invalid_nesting/exit b/test/cmdlineTests/yul_optimizer_steps_invalid_nesting/exit new file mode 100644 index 000000000000..d00491fd7e5b --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps_invalid_nesting/exit @@ -0,0 +1 @@ +1 diff --git a/test/cmdlineTests/yul_optimizer_steps_invalid_nesting/input.sol b/test/cmdlineTests/yul_optimizer_steps_invalid_nesting/input.sol new file mode 100644 index 000000000000..363a4c72182c --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps_invalid_nesting/input.sol @@ -0,0 +1,6 @@ +pragma solidity >=0.0; + +contract C +{ + function f() public pure {} +} diff --git a/test/cmdlineTests/yul_optimizer_steps_unbalanced_bracket/args b/test/cmdlineTests/yul_optimizer_steps_unbalanced_bracket/args new file mode 100644 index 000000000000..d0f68f8c4c04 --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps_unbalanced_bracket/args @@ -0,0 +1 @@ +--ir-optimized --optimize --yul-optimizations a[a][ diff --git a/test/cmdlineTests/yul_optimizer_steps_unbalanced_bracket/err b/test/cmdlineTests/yul_optimizer_steps_unbalanced_bracket/err new file mode 100644 index 000000000000..1da42b74300a --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps_unbalanced_bracket/err @@ -0,0 +1 @@ +Invalid optimizer step sequence in --yul-optimizations: Unbalanced brackets diff --git a/test/cmdlineTests/yul_optimizer_steps_unbalanced_bracket/exit b/test/cmdlineTests/yul_optimizer_steps_unbalanced_bracket/exit new file mode 100644 index 000000000000..d00491fd7e5b --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps_unbalanced_bracket/exit @@ -0,0 +1 @@ +1 diff --git a/test/cmdlineTests/yul_optimizer_steps_unbalanced_bracket/input.sol b/test/cmdlineTests/yul_optimizer_steps_unbalanced_bracket/input.sol new file mode 100644 index 000000000000..363a4c72182c --- /dev/null +++ b/test/cmdlineTests/yul_optimizer_steps_unbalanced_bracket/input.sol @@ -0,0 +1,6 @@ +pragma solidity >=0.0; + +contract C +{ + function f() public pure {} +} From c8b612536f084e4726849ccd566ba40b4ba0c494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 24 Apr 2020 14:26:31 +0200 Subject: [PATCH 100/126] Add yulDetails.optimizerSteps to the standard JSON interface --- libsolidity/interface/CompilerStack.cpp | 1 + libsolidity/interface/StandardCompiler.cpp | 32 ++++++++++++++++++- .../input.json | 21 ++++++++++++ .../output.json | 1 + .../input.json | 21 ++++++++++++ .../output.json | 1 + .../input.json | 21 ++++++++++++ .../output.json | 1 + .../input.json | 21 ++++++++++++ .../output.json | 1 + .../input.json | 21 ++++++++++++ .../output.json | 1 + test/libsolidity/StandardCompiler.cpp | 10 +++++- 13 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps/input.json create mode 100644 test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps/output.json create mode 100644 test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_invalid_abbreviation/input.json create mode 100644 test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_invalid_abbreviation/output.json create mode 100644 test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_invalid_nesting/input.json create mode 100644 test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_invalid_nesting/output.json create mode 100644 test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_type/input.json create mode 100644 test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_type/output.json create mode 100644 test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_unbalanced_bracket/input.json create mode 100644 test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_unbalanced_bracket/output.json diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 17348c24278f..16422320cf5d 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -1265,6 +1265,7 @@ string CompilerStack::createMetadata(Contract const& _contract) const { details["yulDetails"] = Json::objectValue; details["yulDetails"]["stackAllocation"] = m_optimiserSettings.optimizeStackAllocation; + details["yulDetails"]["optimizerSteps"] = m_optimiserSettings.yulOptimiserSteps; } meta["settings"]["optimizer"]["details"] = std::move(details); diff --git a/libsolidity/interface/StandardCompiler.cpp b/libsolidity/interface/StandardCompiler.cpp index 1b35040aa75c..15989dcf32b3 100644 --- a/libsolidity/interface/StandardCompiler.cpp +++ b/libsolidity/interface/StandardCompiler.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -402,6 +403,33 @@ std::optional checkOptimizerDetail(Json::Value const& _details, std return {}; } +std::optional checkOptimizerDetailSteps(Json::Value const& _details, std::string const& _name, string& _setting) +{ + if (_details.isMember(_name)) + { + if (_details[_name].isString()) + { + try + { + yul::OptimiserSuite::validateSequence(_details[_name].asString()); + } + catch (yul::OptimizerException const& _exception) + { + return formatFatalError( + "JSONError", + "Invalid optimizer step sequence in \"settings.optimizer.details." + _name + "\": " + _exception.what() + ); + } + + _setting = _details[_name].asString(); + } + else + return formatFatalError("JSONError", "\"settings.optimizer.details." + _name + "\" must be a string"); + + } + return {}; +} + std::optional checkMetadataKeys(Json::Value const& _input) { if (_input.isObject()) @@ -511,10 +539,12 @@ boost::variant parseOptimizerSettings(Json::Valu if (!settings.runYulOptimiser) return formatFatalError("JSONError", "\"Providing yulDetails requires Yul optimizer to be enabled."); - if (auto result = checkKeys(details["yulDetails"], {"stackAllocation"}, "settings.optimizer.details.yulDetails")) + if (auto result = checkKeys(details["yulDetails"], {"stackAllocation", "optimizerSteps"}, "settings.optimizer.details.yulDetails")) return *result; if (auto error = checkOptimizerDetail(details["yulDetails"], "stackAllocation", settings.optimizeStackAllocation)) return *error; + if (auto error = checkOptimizerDetailSteps(details["yulDetails"], "optimizerSteps", settings.yulOptimiserSteps)) + return *error; } } return { std::move(settings) }; diff --git a/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps/input.json b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps/input.json new file mode 100644 index 000000000000..00919a864d77 --- /dev/null +++ b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps/input.json @@ -0,0 +1,21 @@ +{ + "language": "Solidity", + "sources": + { + "A": + { + "content": "pragma solidity >=0.0; contract C { function f() public pure {} }" + } + }, + "settings": + { + "optimizer": { + "details": { + "yul": true, + "yulDetails": { + "optimizerSteps": "dhfoDgvulfnTUtnIf\n[ xarrscLM\n]\njmuljuljul VcTOcul jmul" + } + } + } + } +} diff --git a/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps/output.json b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps/output.json new file mode 100644 index 000000000000..59b90c8cc98e --- /dev/null +++ b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps/output.json @@ -0,0 +1 @@ +{"sources":{"A":{"id":0}}} diff --git a/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_invalid_abbreviation/input.json b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_invalid_abbreviation/input.json new file mode 100644 index 000000000000..25b3c4bb486d --- /dev/null +++ b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_invalid_abbreviation/input.json @@ -0,0 +1,21 @@ +{ + "language": "Solidity", + "sources": + { + "A": + { + "content": "pragma solidity >=0.0; contract C { function f() public pure {} }" + } + }, + "settings": + { + "optimizer": { + "details": { + "yul": true, + "yulDetails": { + "optimizerSteps": "abcdefg{hijklmno}pqr[st]uvwxyz" + } + } + } + } +} diff --git a/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_invalid_abbreviation/output.json b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_invalid_abbreviation/output.json new file mode 100644 index 000000000000..84711c002226 --- /dev/null +++ b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_invalid_abbreviation/output.json @@ -0,0 +1 @@ +{"errors":[{"component":"general","formattedMessage":"Invalid optimizer step sequence in \"settings.optimizer.details.optimizerSteps\": 'b' is not a valid step abbreviation","message":"Invalid optimizer step sequence in \"settings.optimizer.details.optimizerSteps\": 'b' is not a valid step abbreviation","severity":"error","type":"JSONError"}]} diff --git a/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_invalid_nesting/input.json b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_invalid_nesting/input.json new file mode 100644 index 000000000000..c322913d9fd6 --- /dev/null +++ b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_invalid_nesting/input.json @@ -0,0 +1,21 @@ +{ + "language": "Solidity", + "sources": + { + "A": + { + "content": "pragma solidity >=0.0; contract C { function f() public pure {} }" + } + }, + "settings": + { + "optimizer": { + "details": { + "yul": true, + "yulDetails": { + "optimizerSteps": "a[a][aa[aa]]a" + } + } + } + } +} diff --git a/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_invalid_nesting/output.json b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_invalid_nesting/output.json new file mode 100644 index 000000000000..409fd755f23b --- /dev/null +++ b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_invalid_nesting/output.json @@ -0,0 +1 @@ +{"errors":[{"component":"general","formattedMessage":"Invalid optimizer step sequence in \"settings.optimizer.details.optimizerSteps\": Nested brackets are not supported","message":"Invalid optimizer step sequence in \"settings.optimizer.details.optimizerSteps\": Nested brackets are not supported","severity":"error","type":"JSONError"}]} diff --git a/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_type/input.json b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_type/input.json new file mode 100644 index 000000000000..c02aa6bb32a5 --- /dev/null +++ b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_type/input.json @@ -0,0 +1,21 @@ +{ + "language": "Solidity", + "sources": + { + "A": + { + "content": "pragma solidity >=0.0; contract C { function f() public pure {} }" + } + }, + "settings": + { + "optimizer": { + "details": { + "yul": true, + "yulDetails": { + "optimizerSteps": 42 + } + } + } + } +} diff --git a/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_type/output.json b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_type/output.json new file mode 100644 index 000000000000..d18de5299872 --- /dev/null +++ b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_type/output.json @@ -0,0 +1 @@ +{"errors":[{"component":"general","formattedMessage":"\"settings.optimizer.details.optimizerSteps\" must be a string","message":"\"settings.optimizer.details.optimizerSteps\" must be a string","severity":"error","type":"JSONError"}]} diff --git a/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_unbalanced_bracket/input.json b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_unbalanced_bracket/input.json new file mode 100644 index 000000000000..d6e1e0dc7b1f --- /dev/null +++ b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_unbalanced_bracket/input.json @@ -0,0 +1,21 @@ +{ + "language": "Solidity", + "sources": + { + "A": + { + "content": "pragma solidity >=0.0; contract C { function f() public pure {} }" + } + }, + "settings": + { + "optimizer": { + "details": { + "yul": true, + "yulDetails": { + "optimizerSteps": "a[a][" + } + } + } + } +} diff --git a/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_unbalanced_bracket/output.json b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_unbalanced_bracket/output.json new file mode 100644 index 000000000000..d1843709b470 --- /dev/null +++ b/test/cmdlineTests/standard_optimizer_yulDetails_optimiserSteps_unbalanced_bracket/output.json @@ -0,0 +1 @@ +{"errors":[{"component":"general","formattedMessage":"Invalid optimizer step sequence in \"settings.optimizer.details.optimizerSteps\": Unbalanced brackets","message":"Invalid optimizer step sequence in \"settings.optimizer.details.optimizerSteps\": Unbalanced brackets","severity":"error","type":"JSONError"}]} diff --git a/test/libsolidity/StandardCompiler.cpp b/test/libsolidity/StandardCompiler.cpp index df2aecbf5d7b..ce9a29268581 100644 --- a/test/libsolidity/StandardCompiler.cpp +++ b/test/libsolidity/StandardCompiler.cpp @@ -21,11 +21,15 @@ #include #include +#include #include #include #include +#include #include +#include + using namespace std; using namespace solidity::evmasm; @@ -1058,8 +1062,12 @@ BOOST_AUTO_TEST_CASE(optimizer_settings_details_different) BOOST_CHECK(optimizer["details"]["peephole"].asBool() == true); BOOST_CHECK(optimizer["details"]["yul"].asBool() == true); BOOST_CHECK(optimizer["details"]["yulDetails"].isObject()); - BOOST_CHECK(optimizer["details"]["yulDetails"].getMemberNames() == vector{"stackAllocation"}); + BOOST_CHECK( + util::convertContainer>(optimizer["details"]["yulDetails"].getMemberNames()) == + (set{"stackAllocation", "optimizerSteps"}) + ); BOOST_CHECK(optimizer["details"]["yulDetails"]["stackAllocation"].asBool() == true); + BOOST_CHECK(optimizer["details"]["yulDetails"]["optimizerSteps"].asString() == OptimiserSettings::DefaultYulOptimiserSteps); BOOST_CHECK_EQUAL(optimizer["details"].getMemberNames().size(), 8); BOOST_CHECK(optimizer["runs"].asUInt() == 600); } From e19d8d1fa376e0478f26c0daeb93e2cbfbddc55b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 20 Mar 2020 18:54:10 +0100 Subject: [PATCH 101/126] [yul-phaser] GeneticAlgorithm::runNextRound(): Fix outdated docstring --- tools/yulPhaser/GeneticAlgorithms.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/yulPhaser/GeneticAlgorithms.h b/tools/yulPhaser/GeneticAlgorithms.h index 96c0277380ef..f6c809a8b6de 100644 --- a/tools/yulPhaser/GeneticAlgorithms.h +++ b/tools/yulPhaser/GeneticAlgorithms.h @@ -58,8 +58,8 @@ class GeneticAlgorithm GeneticAlgorithm& operator=(GeneticAlgorithm const&) = delete; virtual ~GeneticAlgorithm() = default; - /// The method that actually implements the algorithm. Should use @a m_population as input and - /// replace it with the updated state after the round. + /// The method that actually implements the algorithm. Should accept the current population in + /// @a _population and return the updated one after the round. virtual Population runNextRound(Population _population) = 0; }; From 424edecd21ed56582e2b9721cd73a10717f99d3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 20 Mar 2020 18:14:28 +0100 Subject: [PATCH 102/126] [yul-phaser] Phaser: List all available values of enum options in --help --- tools/yulPhaser/Phaser.cpp | 49 ++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/tools/yulPhaser/Phaser.cpp b/tools/yulPhaser/Phaser.cpp index ebfdbb8975c7..47f988ce9cb1 100644 --- a/tools/yulPhaser/Phaser.cpp +++ b/tools/yulPhaser/Phaser.cpp @@ -437,8 +437,15 @@ Phaser::CommandLineDescription Phaser::buildCommandLineDescription() ( "mode", po::value()->value_name("")->default_value(PhaserMode::RunAlgorithm), - "Mode of operation. The default is to run the algorithm but you can also tell phaser " - "to do something else with its parameters, e.g. just print the optimised programs and exit." + ( + "Mode of operation. The default is to run the algorithm but you can also tell phaser " + "to do something else with its parameters, e.g. just print the optimised programs and exit.\n" + "\n" + "AVAILABLE MODES:\n" + "* " + toString(PhaserMode::RunAlgorithm) + "\n" + + "* " + toString(PhaserMode::PrintOptimisedPrograms) + "\n" + + "* " + toString(PhaserMode::PrintOptimisedASTs) + ).c_str() ) ; keywordDescription.add(generalDescription); @@ -448,7 +455,14 @@ Phaser::CommandLineDescription Phaser::buildCommandLineDescription() ( "algorithm", po::value()->value_name("")->default_value(Algorithm::GEWEP), - "Algorithm" + ( + "Algorithm\n" + "\n" + "AVAILABLE ALGORITHMS:\n" + "* " + toString(Algorithm::GEWEP) + "\n" + + "* " + toString(Algorithm::Classic) + "\n" + + "* " + toString(Algorithm::Random) + ).c_str() ) ( "no-randomise-duplicates", @@ -470,7 +484,14 @@ Phaser::CommandLineDescription Phaser::buildCommandLineDescription() ( "crossover", po::value()->value_name("")->default_value(CrossoverChoice::SinglePoint), - "Type of the crossover operator to use." + ( + "Type of the crossover operator to use.\n" + "\n" + "AVAILABLE CROSSOVER OPERATORS:\n" + "* " + toString(CrossoverChoice::SinglePoint) + "\n" + + "* " + toString(CrossoverChoice::TwoPoint) + "\n" + + "* " + toString(CrossoverChoice::Uniform) + ).c_str() ) ( "uniform-crossover-swap-chance", @@ -590,13 +611,27 @@ Phaser::CommandLineDescription Phaser::buildCommandLineDescription() ( "metric", po::value()->value_name("")->default_value(MetricChoice::RelativeCodeSize), - "Metric used to evaluate the fitness of a chromosome." + ( + "Metric used to evaluate the fitness of a chromosome.\n" + "\n" + "AVAILABLE METRICS:\n" + "* " + toString(MetricChoice::CodeSize) + "\n" + + "* " + toString(MetricChoice::RelativeCodeSize) + ).c_str() ) ( "metric-aggregator", po::value()->value_name("")->default_value(MetricAggregatorChoice::Average), - "Operator used to combine multiple fitness metric obtained by evaluating a chromosome " - "separately for each input program." + ( + "Operator used to combine multiple fitness metric obtained by evaluating a chromosome " + "separately for each input program.\n" + "\n" + "AVAILABLE METRIC AGGREGATORS:\n" + "* " + toString(MetricAggregatorChoice::Average) + "\n" + + "* " + toString(MetricAggregatorChoice::Sum) + "\n" + + "* " + toString(MetricAggregatorChoice::Maximum) + "\n" + + "* " + toString(MetricAggregatorChoice::Minimum) + ).c_str() ) ( "relative-metric-scale", From 35395a4b9c7dd9db514aa3a4d13173b82f426480 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 20 Mar 2020 18:15:44 +0100 Subject: [PATCH 103/126] [yul-phaser] Phaser: Missing word in --metric-aggregator option description --- tools/yulPhaser/Phaser.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/yulPhaser/Phaser.cpp b/tools/yulPhaser/Phaser.cpp index 47f988ce9cb1..8e34a756bb8b 100644 --- a/tools/yulPhaser/Phaser.cpp +++ b/tools/yulPhaser/Phaser.cpp @@ -623,8 +623,8 @@ Phaser::CommandLineDescription Phaser::buildCommandLineDescription() "metric-aggregator", po::value()->value_name("")->default_value(MetricAggregatorChoice::Average), ( - "Operator used to combine multiple fitness metric obtained by evaluating a chromosome " - "separately for each input program.\n" + "Operator used to combine multiple fitness metric values obtained by evaluating a " + "chromosome separately for each input program.\n" "\n" "AVAILABLE METRIC AGGREGATORS:\n" "* " + toString(MetricAggregatorChoice::Average) + "\n" + From 163e35dd23b228d6c620f4717ff73fa972851d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 20 Mar 2020 18:44:45 +0100 Subject: [PATCH 104/126] [yul-phaser] Tweak default values according to experiment results - Long chromosomes in the intial population are better. Set minimum and maximum to 100. - The classic algorithm does not work well without elite. 50% performed better but I think it might be too large. Let's set it to 25%. - Switch to uniform crossover since this is what was used in most experiments and performed well. --- tools/yulPhaser/Phaser.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/yulPhaser/Phaser.cpp b/tools/yulPhaser/Phaser.cpp index 8e34a756bb8b..d5b3bdda7599 100644 --- a/tools/yulPhaser/Phaser.cpp +++ b/tools/yulPhaser/Phaser.cpp @@ -473,17 +473,17 @@ Phaser::CommandLineDescription Phaser::buildCommandLineDescription() ) ( "min-chromosome-length", - po::value()->value_name("")->default_value(12), + po::value()->value_name("")->default_value(100), "Minimum length of randomly generated chromosomes." ) ( "max-chromosome-length", - po::value()->value_name("")->default_value(30), + po::value()->value_name("")->default_value(100), "Maximum length of randomly generated chromosomes." ) ( "crossover", - po::value()->value_name("")->default_value(CrossoverChoice::SinglePoint), + po::value()->value_name("")->default_value(CrossoverChoice::Uniform), ( "Type of the crossover operator to use.\n" "\n" @@ -542,7 +542,7 @@ Phaser::CommandLineDescription Phaser::buildCommandLineDescription() classicGeneticAlgorithmDescription.add_options() ( "classic-elite-pool-size", - po::value()->value_name("")->default_value(0), + po::value()->value_name("")->default_value(0.25), "Percentage of population to regenerate using mutations in each round." ) ( From ee915008bdb29249f9a1301c367d7b69804df84a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 20 Mar 2020 07:47:13 +0100 Subject: [PATCH 105/126] [yul-phaser] README --- tools/yulPhaser/README.md | 91 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 tools/yulPhaser/README.md diff --git a/tools/yulPhaser/README.md b/tools/yulPhaser/README.md new file mode 100644 index 000000000000..abc2a0e47337 --- /dev/null +++ b/tools/yulPhaser/README.md @@ -0,0 +1,91 @@ +## yul-phaser +`yul-phaser` is an internal tool for finding good sequences of [optimisation steps](/libyul/optimiser/README.md) for Yul optimiser. + +### How it works +The space of possible solutions to this problem (usually referred to as _phase-ordering problem_) is extremely large and there may even be no single sequence that produces optimal results for all possible programs. + +The tool uses genetic algorithms to find sequences that result in better programs than others and to iteratively refine them. +The input is a set of one or more [Yul](/docs/yul.rst) programs and each sequence is applied to all of these programs. +Optimised programs are given numeric scores according to the selected metric. + +Optimisation step sequences are presented in an abbreviated form - as strings of letters where each character represents one step. +The abbreviations are defined in [`OptimiserSuite::stepNameToAbbreviationMap()`](/libyul/optimiser/Suite.cpp#L388-L423). + +### How to use it +The application has sensible defaults for most parameters. +An invocation can be as simple as: + +``` bash +tools/yul-phaser ../test/libyul/yulOptimizerTests/fullSuite/*.yul \ + --random-population 100 +``` + +This assumes that you have a working copy of the Solidity repository and you're in the build directory within that working copy. + +Run `yul-phaser --help` for a full list of available options. + +#### Restarting from a previous state +`yul-phaser` can save the list of sequences found after each round: + +``` bash +tools/yul-phaser *.yul \ + --random-population 100 \ + --population-autosave /tmp/population.txt +``` + +If you stop the application, you can later use the file to continue the search from the point you left off: + +``` bash +tools/yul-phaser *.yul \ + --population-from-file /tmp/population.txt \ + --population-autosave /tmp/population.txt +``` + +#### Analysing a sequence +Apart from running the genetic algorithm, `yul-phaser` can also provide useful information about a particular sequence. + +For example, to see the value of a particular metric for a given sequence and program run: +``` bash +tools/yul-phaser *.yul \ + --show-initial-population \ + --rounds 0 \ + --metric code-size \ + --metric-aggregator sum \ + --population +``` + +You can also easily see program code after being optimised using that sequence: +``` bash +tools/yul-phaser *.yul \ + --rounds 0 \ + --mode print-optimised-programs \ + --population +``` + +#### Using output from Solidity compiler +`yul-phaser` can process the intermediate representation produced by `solc`: + +``` bash +solc/solc \ + --ir \ + --no-optimize-yul \ + --output-dir +``` + +After running this command you'll find one or more .yul files in the output directory. +These files contain whole Yul objects rather than just raw Yul programs but `yul-phaser` is prepared to handle them. + +### How to choose good parameters +Choosing good parameters for a genetic algorithm is not a trivial task but phaser's defaults are generally enough to find a sequence that gives results comparable or better than one hand-crafted by an experienced developer for a given set of programs. +The difficult part is providing a fairly representative set of input files. +If the files you give don't need certain optimisations the tool will find sequences that don't use these optimisations and perform badly for programs that could benefit from them. +If all the provided files greatly benefit from a specific optimisation, the sequence may not work well for programs that do not. + +We have conducted [a set of rough experiments](https://github.com/ethereum/solidity/issues/7806#issuecomment-598644491) to evaluate some combinations of parameter values. +The conclusions were used to adjust the defaults but you might still benefit from some general observations: + +1. The algorithm that performed the best was `GEWEP`. +2. Using longer sequences in the initial population yields better results. The algorithm is good at removing superfluous steps. +3. Preserving the top sequences from previous rounds improves results. Elite should contain at least a few individuals, especially when using the `classic` algorithm. +4. Don't set mutation/deletion/addition chance too high. It makes results worse because it destroys the good patterns preserved by crossover. Values around 1-5% seem to work best. +5. Keep the algorithm running for 1000 rounds or more. It usually finds good sequences faster than that but it can shorten them significantly if you let it run longer. This is especially important when starting with long sequences. From 2fa26f4e92b1f297697d814c1c079d02e59f362e Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Fri, 24 Apr 2020 17:03:41 -0500 Subject: [PATCH 106/126] [Sol - Yul] Add support for built-in selfdestruct(..). --- libsolidity/codegen/ir/IRGeneratorForStatements.cpp | 6 ++++++ test/libsolidity/SolidityEndToEndTest.cpp | 10 ++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index 08a5fa900b74..1dd6dcdd18a5 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -812,6 +812,12 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall) define(_functionCall) << "gas()\n"; break; } + case FunctionType::Kind::Selfdestruct: + { + solAssert(arguments.size() == 1, ""); + define(_functionCall) << "selfdestruct(" << expressionAsType(*arguments.front(), *parameterTypes.front()) << ")\n"; + break; + } default: solUnimplemented("FunctionKind " + toString(static_cast(functionType->kind())) + " not yet implemented"); } diff --git a/test/libsolidity/SolidityEndToEndTest.cpp b/test/libsolidity/SolidityEndToEndTest.cpp index 69e645494cec..a0e59170b6e9 100644 --- a/test/libsolidity/SolidityEndToEndTest.cpp +++ b/test/libsolidity/SolidityEndToEndTest.cpp @@ -1238,11 +1238,13 @@ BOOST_AUTO_TEST_CASE(selfdestruct) } )"; u256 amount(130); - compileAndRun(sourceCode, amount); u160 address(23); - ABI_CHECK(callContractFunction("a(address)", address), bytes()); - BOOST_CHECK(!addressHasCode(m_contractAddress)); - BOOST_CHECK_EQUAL(balanceAt(address), amount); + ALSO_VIA_YUL( + compileAndRun(sourceCode, amount); + ABI_CHECK(callContractFunction("a(address)", address), bytes()); + BOOST_CHECK(!addressHasCode(m_contractAddress)); + BOOST_CHECK_EQUAL(balanceAt(address), amount); + ) } BOOST_AUTO_TEST_CASE(keccak256) From 66edaf43f4330a4a88de867bec2209f6e63f57a7 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Fri, 24 Apr 2020 16:33:43 -0500 Subject: [PATCH 107/126] [Sol - Yul] Add support for built-in logN(). --- .../codegen/ir/IRGeneratorForStatements.cpp | 27 ++++ test/libsolidity/SolidityEndToEndTest.cpp | 146 ++++++++++-------- 2 files changed, 108 insertions(+), 65 deletions(-) diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index 1dd6dcdd18a5..d8cb8f49c184 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -818,6 +818,33 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall) define(_functionCall) << "selfdestruct(" << expressionAsType(*arguments.front(), *parameterTypes.front()) << ")\n"; break; } + case FunctionType::Kind::Log0: + case FunctionType::Kind::Log1: + case FunctionType::Kind::Log2: + case FunctionType::Kind::Log3: + case FunctionType::Kind::Log4: + { + unsigned logNumber = int(functionType->kind()) - int(FunctionType::Kind::Log0); + solAssert(arguments.size() == logNumber + 1, ""); + ABIFunctions abi(m_context.evmVersion(), m_context.revertStrings(), m_context.functionCollector()); + string indexedArgs; + for (unsigned arg = 0; arg < logNumber; ++arg) + indexedArgs += ", " + expressionAsType(*arguments[arg + 1], *(parameterTypes[arg + 1])); + Whiskers templ(R"({ + let := + let := (, ) + (, sub(, ) ) + })"); + templ("pos", m_context.newYulVariable()); + templ("end", m_context.newYulVariable()); + templ("freeMemory", freeMemory()); + templ("encode", abi.tupleEncoder({arguments.front()->annotation().type},{parameterTypes.front()})); + templ("nonIndexedArgs", IRVariable(*arguments.front()).commaSeparatedList()); + templ("log", "log" + to_string(logNumber)); + templ("indexedArgs", indexedArgs); + m_code << templ.render(); + break; + } default: solUnimplemented("FunctionKind " + toString(static_cast(functionType->kind())) + " not yet implemented"); } diff --git a/test/libsolidity/SolidityEndToEndTest.cpp b/test/libsolidity/SolidityEndToEndTest.cpp index a0e59170b6e9..153d6a8f428b 100644 --- a/test/libsolidity/SolidityEndToEndTest.cpp +++ b/test/libsolidity/SolidityEndToEndTest.cpp @@ -1126,12 +1126,14 @@ BOOST_AUTO_TEST_CASE(log0) } } )"; - compileAndRun(sourceCode); - callContractFunction("a()"); - BOOST_REQUIRE_EQUAL(numLogs(), 1); - BOOST_CHECK_EQUAL(logAddress(0), m_contractAddress); - BOOST_CHECK_EQUAL(h256(logData(0)), h256(u256(1))); - BOOST_CHECK_EQUAL(numLogTopics(0), 0); + ALSO_VIA_YUL( + compileAndRun(sourceCode); + callContractFunction("a()"); + BOOST_REQUIRE_EQUAL(numLogs(), 1); + BOOST_CHECK_EQUAL(logAddress(0), m_contractAddress); + BOOST_CHECK_EQUAL(h256(logData(0)), h256(u256(1))); + BOOST_CHECK_EQUAL(numLogTopics(0), 0); + ) } BOOST_AUTO_TEST_CASE(log1) @@ -1143,13 +1145,15 @@ BOOST_AUTO_TEST_CASE(log1) } } )"; - compileAndRun(sourceCode); - callContractFunction("a()"); - BOOST_REQUIRE_EQUAL(numLogs(), 1); - BOOST_CHECK_EQUAL(logAddress(0), m_contractAddress); - BOOST_CHECK_EQUAL(h256(logData(0)), h256(u256(1))); - BOOST_REQUIRE_EQUAL(numLogTopics(0), 1); - BOOST_CHECK_EQUAL(logTopic(0, 0), h256(u256(2))); + ALSO_VIA_YUL( + compileAndRun(sourceCode); + callContractFunction("a()"); + BOOST_REQUIRE_EQUAL(numLogs(), 1); + BOOST_CHECK_EQUAL(logAddress(0), m_contractAddress); + BOOST_CHECK_EQUAL(h256(logData(0)), h256(u256(1))); + BOOST_REQUIRE_EQUAL(numLogTopics(0), 1); + BOOST_CHECK_EQUAL(logTopic(0, 0), h256(u256(2))); + ) } BOOST_AUTO_TEST_CASE(log2) @@ -1161,14 +1165,16 @@ BOOST_AUTO_TEST_CASE(log2) } } )"; - compileAndRun(sourceCode); - callContractFunction("a()"); - BOOST_REQUIRE_EQUAL(numLogs(), 1); - BOOST_CHECK_EQUAL(logAddress(0), m_contractAddress); - BOOST_CHECK_EQUAL(h256(logData(0)), h256(u256(1))); - BOOST_REQUIRE_EQUAL(numLogTopics(0), 2); - for (unsigned i = 0; i < 2; ++i) - BOOST_CHECK_EQUAL(logTopic(0, i), h256(u256(i + 2))); + ALSO_VIA_YUL( + compileAndRun(sourceCode); + callContractFunction("a()"); + BOOST_REQUIRE_EQUAL(numLogs(), 1); + BOOST_CHECK_EQUAL(logAddress(0), m_contractAddress); + BOOST_CHECK_EQUAL(h256(logData(0)), h256(u256(1))); + BOOST_REQUIRE_EQUAL(numLogTopics(0), 2); + for (unsigned i = 0; i < 2; ++i) + BOOST_CHECK_EQUAL(logTopic(0, i), h256(u256(i + 2))); + ) } BOOST_AUTO_TEST_CASE(log3) @@ -1180,14 +1186,16 @@ BOOST_AUTO_TEST_CASE(log3) } } )"; - compileAndRun(sourceCode); - callContractFunction("a()"); - BOOST_REQUIRE_EQUAL(numLogs(), 1); - BOOST_CHECK_EQUAL(logAddress(0), m_contractAddress); - BOOST_CHECK_EQUAL(h256(logData(0)), h256(u256(1))); - BOOST_REQUIRE_EQUAL(numLogTopics(0), 3); - for (unsigned i = 0; i < 3; ++i) - BOOST_CHECK_EQUAL(logTopic(0, i), h256(u256(i + 2))); + ALSO_VIA_YUL( + compileAndRun(sourceCode); + callContractFunction("a()"); + BOOST_REQUIRE_EQUAL(numLogs(), 1); + BOOST_CHECK_EQUAL(logAddress(0), m_contractAddress); + BOOST_CHECK_EQUAL(h256(logData(0)), h256(u256(1))); + BOOST_REQUIRE_EQUAL(numLogTopics(0), 3); + for (unsigned i = 0; i < 3; ++i) + BOOST_CHECK_EQUAL(logTopic(0, i), h256(u256(i + 2))); + ) } BOOST_AUTO_TEST_CASE(log4) @@ -1199,14 +1207,16 @@ BOOST_AUTO_TEST_CASE(log4) } } )"; - compileAndRun(sourceCode); - callContractFunction("a()"); - BOOST_REQUIRE_EQUAL(numLogs(), 1); - BOOST_CHECK_EQUAL(logAddress(0), m_contractAddress); - BOOST_CHECK_EQUAL(h256(logData(0)), h256(u256(1))); - BOOST_REQUIRE_EQUAL(numLogTopics(0), 4); - for (unsigned i = 0; i < 4; ++i) - BOOST_CHECK_EQUAL(logTopic(0, i), h256(u256(i + 2))); + ALSO_VIA_YUL( + compileAndRun(sourceCode); + callContractFunction("a()"); + BOOST_REQUIRE_EQUAL(numLogs(), 1); + BOOST_CHECK_EQUAL(logAddress(0), m_contractAddress); + BOOST_CHECK_EQUAL(h256(logData(0)), h256(u256(1))); + BOOST_REQUIRE_EQUAL(numLogTopics(0), 4); + for (unsigned i = 0; i < 4; ++i) + BOOST_CHECK_EQUAL(logTopic(0, i), h256(u256(i + 2))); + ) } BOOST_AUTO_TEST_CASE(log_in_constructor) @@ -1218,12 +1228,14 @@ BOOST_AUTO_TEST_CASE(log_in_constructor) } } )"; - compileAndRun(sourceCode); - BOOST_REQUIRE_EQUAL(numLogs(), 1); - BOOST_CHECK_EQUAL(logAddress(0), m_contractAddress); - BOOST_CHECK_EQUAL(h256(logData(0)), h256(u256(1))); - BOOST_REQUIRE_EQUAL(numLogTopics(0), 1); - BOOST_CHECK_EQUAL(logTopic(0, 0), h256(u256(2))); + ALSO_VIA_YUL( + compileAndRun(sourceCode); + BOOST_REQUIRE_EQUAL(numLogs(), 1); + BOOST_CHECK_EQUAL(logAddress(0), m_contractAddress); + BOOST_CHECK_EQUAL(h256(logData(0)), h256(u256(1))); + BOOST_REQUIRE_EQUAL(numLogTopics(0), 1); + BOOST_CHECK_EQUAL(logTopic(0, 0), h256(u256(2))); + ) } BOOST_AUTO_TEST_CASE(selfdestruct) @@ -1776,20 +1788,22 @@ BOOST_AUTO_TEST_CASE(event) } } )"; - compileAndRun(sourceCode); - u256 value(18); - u256 id(0x1234); - for (bool manually: {true, false}) - { - callContractFunctionWithValue("deposit(bytes32,bool)", value, id, manually); - BOOST_REQUIRE_EQUAL(numLogs(), 1); - BOOST_CHECK_EQUAL(logAddress(0), m_contractAddress); - BOOST_CHECK_EQUAL(h256(logData(0)), h256(u256(value))); - BOOST_REQUIRE_EQUAL(numLogTopics(0), 3); - BOOST_CHECK_EQUAL(logTopic(0, 0), util::keccak256(string("Deposit(address,bytes32,uint256)"))); - BOOST_CHECK_EQUAL(logTopic(0, 1), h256(m_sender, h256::AlignRight)); - BOOST_CHECK_EQUAL(logTopic(0, 2), h256(id)); - } + ALSO_VIA_YUL( + compileAndRun(sourceCode); + u256 value(18); + u256 id(0x1234); + for (bool manually: {true, false}) + { + callContractFunctionWithValue("deposit(bytes32,bool)", value, id, manually); + BOOST_REQUIRE_EQUAL(numLogs(), 1); + BOOST_CHECK_EQUAL(logAddress(0), m_contractAddress); + BOOST_CHECK_EQUAL(h256(logData(0)), h256(u256(value))); + BOOST_REQUIRE_EQUAL(numLogTopics(0), 3); + BOOST_CHECK_EQUAL(logTopic(0, 0), util::keccak256(string("Deposit(address,bytes32,uint256)"))); + BOOST_CHECK_EQUAL(logTopic(0, 1), h256(m_sender, h256::AlignRight)); + BOOST_CHECK_EQUAL(logTopic(0, 2), h256(id)); + } + ) } BOOST_AUTO_TEST_CASE(event_emit) @@ -1803,7 +1817,7 @@ BOOST_AUTO_TEST_CASE(event_emit) } )"; ALSO_VIA_YUL( - compileAndRun(sourceCode); + compileAndRun(sourceCode); u256 value(18); u256 id(0x1234); callContractFunctionWithValue("deposit(bytes32)", value, id); @@ -1852,13 +1866,15 @@ BOOST_AUTO_TEST_CASE(event_access_through_base_name_emit) } } )"; - compileAndRun(sourceCode); - callContractFunction("f()"); - BOOST_REQUIRE_EQUAL(numLogs(), 1); - BOOST_CHECK_EQUAL(logAddress(0), m_contractAddress); - BOOST_CHECK(logData(0).empty()); - BOOST_REQUIRE_EQUAL(numLogTopics(0), 1); - BOOST_CHECK_EQUAL(logTopic(0, 0), util::keccak256(string("x()"))); + ALSO_VIA_YUL( + compileAndRun(sourceCode); + callContractFunction("f()"); + BOOST_REQUIRE_EQUAL(numLogs(), 1); + BOOST_CHECK_EQUAL(logAddress(0), m_contractAddress); + BOOST_CHECK(logData(0).empty()); + BOOST_REQUIRE_EQUAL(numLogTopics(0), 1); + BOOST_CHECK_EQUAL(logTopic(0, 0), util::keccak256(string("x()"))); + ); } BOOST_AUTO_TEST_CASE(events_with_same_name) From a481ea719f901071a04a9282e102a39d6c7c4b5d Mon Sep 17 00:00:00 2001 From: Noel Maersk Date: Sun, 26 Apr 2020 11:36:05 +0300 Subject: [PATCH 108/126] docs: use Yul lexer to highlight Yul code segments. Many commits squashed; turns out that with the combination of: * Python v2.7, * Sphinx v1.8.5, and * Pygments v2.3.1 versions (old!) used in the CI, the only viable approach is: * to use `code-block` directives with explicit language specification, * to provide no file-local default using `highlight`, and * to set language as `none` for grammar specifications. Underlying are the following issues (again, for the old versions listed above): * Generic RST `code` doesn't work when language is `none`: Warning, treated as error: /root/project/docs/yul.rst:430:Cannot analyze code. No Pygments lexer found for "none". Additionally, it might be trying to fall back to the default (Solidity) if left unspecified. * If a file-local default is specified using `highlight`, then `code-block` _must_ also provide a language: Warning, treated as error: /root/project/docs/yul.rst:185:Error in "code-block" directive: 1 argument(s) required, 0 supplied. * Sphinx seems to try the file-local default "yul" (specified with `highlight`) on `code` marked having language `json`: Warning, treated as error: /root/project/docs/yul.rst:130:Could not lex literal_block as "yul". Highlighting skipped. * The only well-lexed highlighter for two of the three grammar specifications is `peg`, but it was added in Pygments v2.6. One of the grammars - in the "Formal Specification" section, the one after "We will use a destructuring notation for the AST nodes." - _must_ be left unhighlighted, with language set to `none`: all lexers do really poorly. ... And one should never, ever start treating warnings as mere warnings, without having exhausted all other options. Otherwise, it's a slippery slope, - and look where that brought Gandhi: to being a strawman in every lousy argument to be had!.. --- docs/conf.py | 3 ++- docs/requirements.txt | 2 +- docs/yul.rst | 38 +++++++++++++++++++------------------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9a9d574fc66a..1ff7947224f2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -17,7 +17,7 @@ import os import re -from pygments_lexer_solidity import SolidityLexer +from pygments_lexer_solidity import SolidityLexer, YulLexer # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -27,6 +27,7 @@ def setup(sphinx): thisdir = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, thisdir + '/utils') sphinx.add_lexer('Solidity', SolidityLexer()) + sphinx.add_lexer('Yul', YulLexer()) sphinx.add_stylesheet('css/custom.css') diff --git a/docs/requirements.txt b/docs/requirements.txt index 5fba631c7da9..8f67f9594fa0 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ sphinx_rtd_theme>=0.3.1 -pygments-lexer-solidity>=0.3.1 +pygments-lexer-solidity>=0.5.1 diff --git a/docs/yul.rst b/docs/yul.rst index c6fb28986d22..2fcae1175612 100644 --- a/docs/yul.rst +++ b/docs/yul.rst @@ -70,7 +70,7 @@ The following example program is written in the EVM dialect and computes exponen It can be compiled using ``solc --strict-assembly``. The builtin functions ``mul`` and ``div`` compute product and division, respectively. -.. code:: +.. code-block:: yul { function power(base, exponent) -> result @@ -91,7 +91,7 @@ It is also possible to implement the same function using a for-loop instead of with recursion. Here, ``lt(a, b)`` computes whether ``a`` is less than ``b``. less-than comparison. -.. code:: +.. code-block:: yul { function power(base, exponent) -> result @@ -115,7 +115,7 @@ This will use the :ref:`Yul object notation ` so that it is possible to code as data to deploy contracts. This Yul mode is available for the commandline compiler (use ``--strict-assembly``) and for the :ref:`standard-json interface `: -:: +.. code-block:: json { "language": "Yul", @@ -180,14 +180,14 @@ bitwise ``and`` with the string "abc" is computed. The final value is assigned to a local variable called ``x``. Strings are stored left-aligned and cannot be longer than 32 bytes. -.. code:: +.. code-block:: yul let x := and("abc", add(3, 2)) Unless it is the default type, the type of a literal has to be specified after a colon: -.. code:: +.. code-block:: yul let x := and("abc":uint32, add(3:uint256, 2:uint256)) @@ -201,7 +201,7 @@ If the function returns a single value, it can be directly used inside an expression again. If it returns multiple values, they have to be assigned to local variables. -.. code:: +.. code-block:: yul mstore(0x80, add(mload(0x80), 3)) // Here, the user-defined function `f` returns @@ -242,7 +242,7 @@ Future dialects migh introduce specific types for such pointers. When a variable is referenced, its current value is copied. For the EVM, this translates to a ``DUP`` instruction. -.. code:: +.. code-block:: yul { let zero := 0 @@ -260,7 +260,7 @@ you denote that following a colon. You can also declare multiple variables in one statement when you assign from a function call that returns multiple values. -.. code:: +.. code-block:: yul { let zero:uint32 := 0:uint32 @@ -283,7 +283,7 @@ values have to match. If you want to assign the values returned from a function that has multiple return parameters, you have to provide multiple variables. -.. code:: +.. code-block:: yul let v := 0 // re-assign v @@ -301,7 +301,7 @@ The if statement can be used for conditionally executing code. No "else" block can be defined. Consider using "switch" instead (see below) if you need multiple alternatives. -.. code:: +.. code-block:: yul if eq(value, 0) { revert(0, 0) } @@ -317,7 +317,7 @@ Contrary to other programming languages, for safety reasons, control flow does not continue from one case to the next. There can be a fallback or default case called ``default`` which is taken if none of the literal constants matches. -.. code:: +.. code-block:: yul { let x := 0 @@ -349,7 +349,7 @@ or skip to the post-part, respectively. The following example computes the sum of an area in memory. -.. code:: +.. code-block:: yul { let x := 0 @@ -361,7 +361,7 @@ The following example computes the sum of an area in memory. For loops can also be used as a replacement for while loops: Simply leave the initialization and post-iteration parts empty. -.. code:: +.. code-block:: yul { let x := 0 @@ -404,7 +404,7 @@ the current yul function. The following example implements the power function by square-and-multiply. -.. code:: +.. code-block:: yul { function power(base, exponent) -> result { @@ -425,7 +425,7 @@ Specification of Yul This chapter describes Yul code formally. Yul code is usually placed inside Yul objects, which are explained in their own chapter. -Grammar:: +.. code-block:: none Block = '{' Statement* '}' Statement = @@ -588,7 +588,7 @@ For an identifier ``v``, let ``$v`` be the name of the identifier. We will use a destructuring notation for the AST nodes. -.. code:: +.. code-block:: none E(G, L, <{St1, ..., Stn}>: Block) = let G1, L1, mode = E(G, L, St1, ..., Stn) @@ -915,7 +915,7 @@ Hex strings can be used to specify data in hex encoding, regular strings in native encoding. For code, ``datacopy`` will access its assembled binary representation. -Grammar:: +.. code-block:: none Object = 'object' StringLiteral '{' Code ( Object | Data )* '}' Code = 'code' Block @@ -927,7 +927,7 @@ Above, ``Block`` refers to ``Block`` in the Yul code grammar explained in the pr An example Yul Object is shown below: -.. code:: +.. code-block:: yul // A contract consists of a single object with sub-objects representing // the code to be deployed or other contracts it can create. @@ -1010,7 +1010,7 @@ for more details about its internals. If you want to use Solidity in stand-alone Yul mode, you activate the optimizer using ``--optimize``: -:: +.. code-block:: sh solc --strict-assembly --optimize From aa8107f45a33ca195f85d68f043958279b52b038 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 23 Apr 2020 16:29:17 +0200 Subject: [PATCH 109/126] Conditional strings for Whiskers. --- docs/contributing.rst | 5 +++++ libsolutil/Whiskers.cpp | 30 ++++++++++++++++++++++++------ libsolutil/Whiskers.h | 3 +++ test/libsolutil/Whiskers.cpp | 13 +++++++++++++ 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 98a01fd6dc15..395439018b7f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -356,6 +356,11 @@ by as many concatenations of its contents as there were sets of variables suppli each time replacing any ```` items by their respective value. Top-level variables can also be used inside such areas. +There are also conditionals of the form ``......``, where template replacements +continue recursively either in the first or the second segment depending on the value of the boolean +parameter ``name``. If ``......`` is used, then the check is whether +the string parameter ``name`` is non-empty. + .. _documentation-style: Documentation Style Guide diff --git a/libsolutil/Whiskers.cpp b/libsolutil/Whiskers.cpp index 208719d430c0..aebe55c69c4c 100644 --- a/libsolutil/Whiskers.cpp +++ b/libsolutil/Whiskers.cpp @@ -132,7 +132,11 @@ string Whiskers::replace( map> const& _listParameters ) { - static regex listOrTag("<(" + paramRegex() + ")>|<#(" + paramRegex() + ")>((?:.|\\r|\\n)*?)|<\\?(" + paramRegex() + ")>((?:.|\\r|\\n)*?)(((?:.|\\r|\\n)*?))?"); + static regex listOrTag( + "<(" + paramRegex() + ")>|" + "<#(" + paramRegex() + ")>((?:.|\\r|\\n)*?)|" + "<\\?(\\+?" + paramRegex() + ")>((?:.|\\r|\\n)*?)(((?:.|\\r|\\n)*?))?" + ); return regex_replace(_template, listOrTag, [&](match_results _match) -> string { string tagName(_match[1]); @@ -164,12 +168,26 @@ string Whiskers::replace( else { assertThrow(!conditionName.empty(), WhiskersError, ""); - assertThrow( - _conditions.count(conditionName), - WhiskersError, "Condition parameter " + conditionName + " not set." - ); + bool conditionValue = false; + if (conditionName[0] == '+') + { + string tag = conditionName.substr(1); + assertThrow( + _parameters.count(tag), + WhiskersError, "Tag " + tag + " used as condition but was not set." + ); + conditionValue = !_parameters.at(tag).empty(); + } + else + { + assertThrow( + _conditions.count(conditionName), + WhiskersError, "Condition parameter " + conditionName + " not set." + ); + conditionValue = _conditions.at(conditionName); + } return replace( - _conditions.at(conditionName) ? _match[5] : _match[7], + conditionValue ? _match[5] : _match[7], _parameters, _conditions, _listParameters diff --git a/libsolutil/Whiskers.h b/libsolutil/Whiskers.h index 0165429fb4d8..5eb00dce556c 100644 --- a/libsolutil/Whiskers.h +++ b/libsolutil/Whiskers.h @@ -59,6 +59,9 @@ DEV_SIMPLE_EXCEPTION(WhiskersError); * - Condition parameter: ......, where "" is optional * replaced (and recursively expanded) by the first part if the condition is true * and by the second (or empty string if missing) if the condition is false + * - Conditional string parameter: ...... + * Works similar to a conditional parameter where the checked condition is + * that the regular (string) parameter called "name" is non-empty. * - List parameter: <#list>... * The part between the tags is repeated as often as values are provided * in the mapping. Each list element can have its own parameter -> value mapping. diff --git a/test/libsolutil/Whiskers.cpp b/test/libsolutil/Whiskers.cpp index 3f2a487335cd..093b0acb69fd 100644 --- a/test/libsolutil/Whiskers.cpp +++ b/test/libsolutil/Whiskers.cpp @@ -114,6 +114,19 @@ BOOST_AUTO_TEST_CASE(conditional_plus_list) BOOST_CHECK_EQUAL(m.render(), " - ab - "); } +BOOST_AUTO_TEST_CASE(string_as_conditional) +{ + string templ = "+-"; + BOOST_CHECK_EQUAL(Whiskers(templ)("b", "abc").render(), "+abc"); + BOOST_CHECK_EQUAL(Whiskers(templ)("b", "").render(), "-"); +} + +BOOST_AUTO_TEST_CASE(string_as_conditional_wrong) +{ + string templ = "+"; + BOOST_CHECK_EQUAL(Whiskers(templ)("b", "abc").render(), "+abc"); +} + BOOST_AUTO_TEST_CASE(complicated_replacement) { string templ = "a x \n >."; From dda883b585c2f00ac60c3729feaa9a8fda782b22 Mon Sep 17 00:00:00 2001 From: chriseth Date: Mon, 20 Apr 2020 23:26:00 +0200 Subject: [PATCH 110/126] IR generation for sha256. --- .../codegen/ir/IRGeneratorForStatements.cpp | 53 +++++++++++++++---- .../builtinFunctions/sha256_empty.sol | 3 +- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index d8cb8f49c184..2edad9329333 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -760,6 +760,14 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall) "))\n"; break; } + case FunctionType::Kind::ECRecover: + case FunctionType::Kind::SHA256: + case FunctionType::Kind::RIPEMD160: + { + solAssert(!_functionCall.annotation().tryCall, ""); + appendExternalFunctionCall(_functionCall, arguments); + break; + } case FunctionType::Kind::ArrayPop: { auto const& memberAccessExpression = dynamic_cast(_functionCall.expression()).expression(); @@ -1460,7 +1468,8 @@ void IRGeneratorForStatements::appendExternalFunctionCall( for (auto const& arg: _arguments) { argumentTypes.emplace_back(&type(*arg)); - argumentStrings.emplace_back(IRVariable(*arg).commaSeparatedList()); + if (IRVariable(*arg).type().sizeOnStack() > 0) + argumentStrings.emplace_back(IRVariable(*arg).commaSeparatedList()); } string argumentString = argumentStrings.empty() ? ""s : (", " + joinHumanReadable(argumentStrings)); @@ -1478,7 +1487,6 @@ void IRGeneratorForStatements::appendExternalFunctionCall( ABIFunctions abi(m_context.evmVersion(), m_context.revertStrings(), m_context.functionCollector()); - solUnimplementedAssert(!funType.isBareCall(), ""); Whiskers templ(R"( if iszero(extcodesize(
)) { revert(0, 0) } @@ -1486,8 +1494,18 @@ void IRGeneratorForStatements::appendExternalFunctionCall( // storage for arguments and returned data let := - mstore(, ()) - let := (add(, 4) ) + + + mstore(, ()) + + let := ( + + + + add(, 4) + + + ) let := (,
, , , sub(, ), , ) @@ -1509,14 +1527,25 @@ void IRGeneratorForStatements::appendExternalFunctionCall( )"); templ("pos", m_context.newYulVariable()); templ("end", m_context.newYulVariable()); + templ("bareCall", funType.isBareCall()); if (_functionCall.annotation().tryCall) templ("success", m_context.trySuccessConditionVariable(_functionCall)); else templ("success", m_context.newYulVariable()); templ("freeMemory", freeMemory()); templ("shl28", m_utils.shiftLeftFunction(8 * (32 - 4))); - templ("funId", IRVariable(_functionCall.expression()).part("functionIdentifier").name()); - templ("address", IRVariable(_functionCall.expression()).part("address").name()); + + if (!funType.isBareCall()) + templ("funId", IRVariable(_functionCall.expression()).part("functionIdentifier").name()); + + if (funKind == FunctionType::Kind::ECRecover) + templ("address", "1"); + else if (funKind == FunctionType::Kind::SHA256) + templ("address", "2"); + else if (funKind == FunctionType::Kind::RIPEMD160) + templ("address", "3"); + else + templ("address", IRVariable(_functionCall.expression()).part("address").name()); // Always use the actual return length, and not our calculated expected length, if returndatacopy is supported. // This ensures it can catch badly formatted input from external calls. @@ -1548,9 +1577,15 @@ void IRGeneratorForStatements::appendExternalFunctionCall( // but all parameters of ecrecover are value types anyway. encodeInPlace = false; bool encodeForLibraryCall = funKind == FunctionType::Kind::DelegateCall; - solUnimplementedAssert(!encodeInPlace, ""); - solUnimplementedAssert(funType.padArguments(), ""); - templ("encodeArgs", abi.tupleEncoder(argumentTypes, funType.parameterTypes(), encodeForLibraryCall)); + + solUnimplementedAssert(encodeInPlace == !funType.padArguments(), ""); + if (encodeInPlace) + { + solUnimplementedAssert(!encodeForLibraryCall, ""); + templ("encodeArgs", abi.tupleEncoderPacked(argumentTypes, funType.parameterTypes())); + } + else + templ("encodeArgs", abi.tupleEncoder(argumentTypes, funType.parameterTypes(), encodeForLibraryCall)); templ("argumentString", argumentString); // Output data will replace input data, unless we have ECRecover (then, output diff --git a/test/libsolidity/semanticTests/builtinFunctions/sha256_empty.sol b/test/libsolidity/semanticTests/builtinFunctions/sha256_empty.sol index 69b9e15f535c..ededa2fabf20 100644 --- a/test/libsolidity/semanticTests/builtinFunctions/sha256_empty.sol +++ b/test/libsolidity/semanticTests/builtinFunctions/sha256_empty.sol @@ -3,6 +3,7 @@ contract C { return sha256(""); } } - +// ==== +// compileViaYul: also // ---- // f() -> 0xe3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 From aa0a69b47f9d307c29213320c6471211ff3646ad Mon Sep 17 00:00:00 2001 From: Alex Beregszaszi Date: Mon, 27 Apr 2020 12:03:44 +0100 Subject: [PATCH 111/126] IRGenerator: include assertion for FunctionType::Kind::Declaration --- libsolidity/codegen/ir/IRGeneratorForStatements.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index d8cb8f49c184..a356b4e8ce47 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -578,6 +578,9 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall) solUnimplementedAssert(!functionType->bound(), ""); switch (functionType->kind()) { + case FunctionType::Kind::Declaration: + solAssert(false, "Attempted to generate code for calling a function definition."); + break; case FunctionType::Kind::Internal: { vector args; From 7280ed716a1bdd6bbdea26aba9a83fc3ff36879e Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Tue, 21 Apr 2020 13:06:27 +0200 Subject: [PATCH 112/126] Yul proto spec: Add multiple variable declaration statement --- test/tools/ossfuzz/protoToYul.cpp | 77 +++++++++++++++++++++++++++++++ test/tools/ossfuzz/protoToYul.h | 1 + test/tools/ossfuzz/yulProto.proto | 5 ++ 3 files changed, 83 insertions(+) diff --git a/test/tools/ossfuzz/protoToYul.cpp b/test/tools/ossfuzz/protoToYul.cpp index 5589043de2e1..eaf7d6652139 100644 --- a/test/tools/ossfuzz/protoToYul.cpp +++ b/test/tools/ossfuzz/protoToYul.cpp @@ -400,6 +400,80 @@ void ProtoConverter::visit(VarDecl const& _x) } } +void ProtoConverter::visit(MultiVarDecl const& _x) +{ + m_output << "let "; + vector varNames; + // We support up to 4 variables in a single + // declaration statement. + unsigned numVars = _x.num_vars() % 3 + 2; + string delimiter = ""; + for (unsigned i = 0; i < numVars; i++) + { + string varName = newVarName(); + varNames.push_back(varName); + m_output << delimiter << varName; + if (i == 0) + delimiter = ", "; + } + m_output << "\n"; + + // If we are inside a for-init block, there are two places + // where the visited vardecl may have been defined: + // - directly inside the for-init block + // - inside a block within the for-init block + // In the latter case, we don't scope extend. + if (m_inFunctionDef) + { + // Variables declared directly in for-init block + // are tracked separately because their scope + // extends beyond the block they are defined in + // to the rest of the for-loop statement. + if (m_inForInitScope && m_forInitScopeExtEnabled) + { + yulAssert( + !m_funcForLoopInitVars.empty() && !m_funcForLoopInitVars.back().empty(), + "Proto fuzzer: Invalid operation" + ); + for (auto const& varName: varNames) + m_funcForLoopInitVars.back().back().push_back(varName); + } + else + { + yulAssert( + !m_funcVars.empty() && !m_funcVars.back().empty(), + "Proto fuzzer: Invalid operation" + ); + for (auto const& varName: varNames) + m_funcVars.back().back().push_back(varName); + } + + } + else + { + if (m_inForInitScope && m_forInitScopeExtEnabled) + { + yulAssert( + !m_globalForLoopInitVars.empty(), + "Proto fuzzer: Invalid operation" + ); + + for (auto const& varName: varNames) + m_globalForLoopInitVars.back().push_back(varName); + } + else + { + yulAssert( + !m_globalVars.empty(), + "Proto fuzzer: Invalid operation" + ); + + for (auto const& varName: varNames) + m_globalVars.back().push_back(varName); + } + } +} + void ProtoConverter::visit(TypedVarDecl const& _x) { string varName = newVarName(); @@ -1361,6 +1435,9 @@ void ProtoConverter::visit(Statement const& _x) if (m_inFunctionDef) visit(_x.leave()); break; + case Statement::kMultidecl: + visit(_x.multidecl()); + break; case Statement::STMT_ONEOF_NOT_SET: break; } diff --git a/test/tools/ossfuzz/protoToYul.h b/test/tools/ossfuzz/protoToYul.h index d16a9981af6c..34cff65f8d48 100644 --- a/test/tools/ossfuzz/protoToYul.h +++ b/test/tools/ossfuzz/protoToYul.h @@ -77,6 +77,7 @@ class ProtoConverter void visit(VarRef const&); void visit(Expression const&); void visit(VarDecl const&); + void visit(MultiVarDecl const&); void visit(TypedVarDecl const&); void visit(UnaryOp const&); void visit(AssignmentStatement const&); diff --git a/test/tools/ossfuzz/yulProto.proto b/test/tools/ossfuzz/yulProto.proto index c08151b2e351..3180b421a0ce 100644 --- a/test/tools/ossfuzz/yulProto.proto +++ b/test/tools/ossfuzz/yulProto.proto @@ -21,6 +21,10 @@ message VarDecl { required Expression expr = 1; } +message MultiVarDecl { + required uint32 num_vars = 1; +} + message LowLevelCall { enum Type { CALL = 0; @@ -373,6 +377,7 @@ message Statement { FunctionDef funcdef = 16; PopStmt pop = 17; LeaveStmt leave = 18; + MultiVarDecl multidecl = 19; } } From 62e5ccec90866c2a69c1575840450a856c8e2fb0 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Wed, 22 Apr 2020 11:17:15 +0200 Subject: [PATCH 113/126] Minor code factoring to clarify for init scope extension --- test/tools/ossfuzz/protoToYul.cpp | 124 +++++++++++------------------- test/tools/ossfuzz/protoToYul.h | 5 ++ 2 files changed, 49 insertions(+), 80 deletions(-) diff --git a/test/tools/ossfuzz/protoToYul.cpp b/test/tools/ossfuzz/protoToYul.cpp index eaf7d6652139..b9b805b7351e 100644 --- a/test/tools/ossfuzz/protoToYul.cpp +++ b/test/tools/ossfuzz/protoToYul.cpp @@ -344,62 +344,80 @@ void ProtoConverter::visit(BinaryOp const& _x) m_output << ")"; } -void ProtoConverter::visit(VarDecl const& _x) +void ProtoConverter::scopeVariables(vector const& _varNames) { - string varName = newVarName(); - m_output << "let " << varName << " := "; - visit(_x.expr()); - m_output << "\n"; // If we are inside a for-init block, there are two places // where the visited vardecl may have been defined: // - directly inside the for-init block // - inside a block within the for-init block - // In the latter case, we don't scope extend. + // In the latter case, we don't scope extend. The flag + // m_forInitScopeExtEnabled (= true) indicates whether we are directly + // inside a for-init block e.g., for { let x } or (= false) inside a + // nested for-init block e.g., for { { let x } } + bool forInitScopeExtendVariable = m_inForInitScope && m_forInitScopeExtEnabled; + + // There are four cases that are tackled here + // Case 1. We are inside a function definition and the variable declaration's + // scope needs to be extended. + // Case 2. We are inside a function definition but scope extension is disabled + // Case 3. We are inside global scope and scope extension is required + // Case 4. We are inside global scope but scope extension is disabled if (m_inFunctionDef) { // Variables declared directly in for-init block // are tracked separately because their scope // extends beyond the block they are defined in // to the rest of the for-loop statement. - if (m_inForInitScope && m_forInitScopeExtEnabled) + // Case 1 + if (forInitScopeExtendVariable) { yulAssert( - !m_funcForLoopInitVars.empty() && !m_funcForLoopInitVars.back().empty(), - "Proto fuzzer: Invalid operation" - ); - m_funcForLoopInitVars.back().back().push_back(varName); + !m_funcForLoopInitVars.empty() && !m_funcForLoopInitVars.back().empty(), "Proto fuzzer: Invalid operation"); + for (auto const& varName: _varNames) + m_funcForLoopInitVars.back().back().push_back(varName); } + // Case 2 else { yulAssert( !m_funcVars.empty() && !m_funcVars.back().empty(), "Proto fuzzer: Invalid operation" ); - m_funcVars.back().back().push_back(varName); + for (auto const& varName: _varNames) + m_funcVars.back().back().push_back(varName); } - } + // If m_inFunctionDef is false, we are in global scope else { - if (m_inForInitScope && m_forInitScopeExtEnabled) + // Case 3 + if (forInitScopeExtendVariable) { - yulAssert( - !m_globalForLoopInitVars.empty(), - "Proto fuzzer: Invalid operation" - ); - m_globalForLoopInitVars.back().push_back(varName); + yulAssert(!m_globalForLoopInitVars.empty(), "Proto fuzzer: Invalid operation"); + + for (auto const& varName: _varNames) + m_globalForLoopInitVars.back().push_back(varName); } + // Case 4 else { - yulAssert( - !m_globalVars.empty(), - "Proto fuzzer: Invalid operation" - ); - m_globalVars.back().push_back(varName); + yulAssert(!m_globalVars.empty(), "Proto fuzzer: Invalid operation"); + + for (auto const& varName: _varNames) + m_globalVars.back().push_back(varName); } } } +void ProtoConverter::visit(VarDecl const& _x) +{ + string varName = newVarName(); + m_output << "let " << varName << " := "; + visit(_x.expr()); + m_output << "\n"; + scopeVariables({varName}); +} + void ProtoConverter::visit(MultiVarDecl const& _x) { m_output << "let "; @@ -417,61 +435,7 @@ void ProtoConverter::visit(MultiVarDecl const& _x) delimiter = ", "; } m_output << "\n"; - - // If we are inside a for-init block, there are two places - // where the visited vardecl may have been defined: - // - directly inside the for-init block - // - inside a block within the for-init block - // In the latter case, we don't scope extend. - if (m_inFunctionDef) - { - // Variables declared directly in for-init block - // are tracked separately because their scope - // extends beyond the block they are defined in - // to the rest of the for-loop statement. - if (m_inForInitScope && m_forInitScopeExtEnabled) - { - yulAssert( - !m_funcForLoopInitVars.empty() && !m_funcForLoopInitVars.back().empty(), - "Proto fuzzer: Invalid operation" - ); - for (auto const& varName: varNames) - m_funcForLoopInitVars.back().back().push_back(varName); - } - else - { - yulAssert( - !m_funcVars.empty() && !m_funcVars.back().empty(), - "Proto fuzzer: Invalid operation" - ); - for (auto const& varName: varNames) - m_funcVars.back().back().push_back(varName); - } - - } - else - { - if (m_inForInitScope && m_forInitScopeExtEnabled) - { - yulAssert( - !m_globalForLoopInitVars.empty(), - "Proto fuzzer: Invalid operation" - ); - - for (auto const& varName: varNames) - m_globalForLoopInitVars.back().push_back(varName); - } - else - { - yulAssert( - !m_globalVars.empty(), - "Proto fuzzer: Invalid operation" - ); - - for (auto const& varName: varNames) - m_globalVars.back().push_back(varName); - } - } + scopeVariables(varNames); } void ProtoConverter::visit(TypedVarDecl const& _x) @@ -1824,7 +1788,7 @@ void ProtoConverter::createFunctionDefAndCall( yulAssert( !m_inForInitScope, - "Proto fuzzer: Trying to create function call inside for-init block" + "Proto fuzzer: Trying to create function call inside a for-init block" ); if (_x.force_call()) createFunctionCall(funcName, _numInParams, _numOutParams); diff --git a/test/tools/ossfuzz/protoToYul.h b/test/tools/ossfuzz/protoToYul.h index 34cff65f8d48..8ef5584fa052 100644 --- a/test/tools/ossfuzz/protoToYul.h +++ b/test/tools/ossfuzz/protoToYul.h @@ -236,6 +236,11 @@ class ProtoConverter /// @return A vector of strings containing the printed variable names. std::vector createVars(unsigned _startIdx, unsigned _endIdx); + /// Manages scope of yul variables + /// @param _varNames is a list of yul variable names whose scope needs + /// to be tracked according to yul scoping rules. + void scopeVariables(std::vector const& _varNames); + /// Print the yul syntax to make a call to a function named @a _funcName to /// the output stream. /// @param _funcName Name of the function to be called From 5113af1df06378d32962efd0f57d2104ee629cea Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Mon, 27 Apr 2020 13:52:44 +0200 Subject: [PATCH 114/126] Update test/tools/ossfuzz/protoToYul.h Co-Authored-By: Leonardo --- test/tools/ossfuzz/protoToYul.cpp | 15 +++++++++------ test/tools/ossfuzz/protoToYul.h | 22 +++++++++++----------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/test/tools/ossfuzz/protoToYul.cpp b/test/tools/ossfuzz/protoToYul.cpp index b9b805b7351e..8d5ced0a85f2 100644 --- a/test/tools/ossfuzz/protoToYul.cpp +++ b/test/tools/ossfuzz/protoToYul.cpp @@ -372,7 +372,9 @@ void ProtoConverter::scopeVariables(vector const& _varNames) if (forInitScopeExtendVariable) { yulAssert( - !m_funcForLoopInitVars.empty() && !m_funcForLoopInitVars.back().empty(), "Proto fuzzer: Invalid operation"); + !m_funcForLoopInitVars.empty() && !m_funcForLoopInitVars.back().empty(), + "Proto fuzzer: Invalid operation" + ); for (auto const& varName: _varNames) m_funcForLoopInitVars.back().back().push_back(varName); } @@ -697,7 +699,7 @@ void ProtoConverter::visit(CopyFunc const& _x) CopyFunc_CopyType type = _x.ct(); // datacopy() is valid only if we are inside - // a yul object. + // a Yul object. if (type == CopyFunc::DATA && !m_isObject) return; @@ -1131,7 +1133,8 @@ void ProtoConverter::visit(ForStmt const& _x) { yulAssert( !m_funcForLoopInitVars.empty() && !m_funcForLoopInitVars.back().empty(), - "Proto fuzzer: Invalid data structure"); + "Proto fuzzer: Invalid data structure" + ); // Remove variables in for-init m_funcForLoopInitVars.back().pop_back(); } @@ -1647,7 +1650,7 @@ void ProtoConverter::fillFunctionCallInput(unsigned _numInParams) { // Throw a 4-sided dice to choose whether to populate function input // argument from a pseudo-randomly chosen slot in one of the following - // locations: calldata, memory, storage, or yul optimizer dictionary. + // locations: calldata, memory, storage, or Yul optimizer dictionary. unsigned diceValue = counter() % 4; // Pseudo-randomly choose one of the first ten 32-byte // aligned slots. @@ -1878,7 +1881,7 @@ void ProtoConverter::visit(Program const& _x) // Record EVM Version m_evmVersion = evmVersionMapping(_x.ver()); - // Program is either a yul object or a block of + // Program is either a Yul object or a block of // statements. switch (_x.program_oneof_case()) { @@ -1895,7 +1898,7 @@ void ProtoConverter::visit(Program const& _x) visit(_x.obj()); break; case Program::PROGRAM_ONEOF_NOT_SET: - // {} is a trivial yul program + // {} is a trivial Yul program m_output << "{}"; break; } diff --git a/test/tools/ossfuzz/protoToYul.h b/test/tools/ossfuzz/protoToYul.h index 8ef5584fa052..9f7240f5eeb0 100644 --- a/test/tools/ossfuzz/protoToYul.h +++ b/test/tools/ossfuzz/protoToYul.h @@ -68,7 +68,7 @@ class ProtoConverter void visit(BinaryOp const&); /// Visits a basic block optionally adding @a _funcParams to scope. - /// @param _block Reference to a basic block of yul statements. + /// @param _block Reference to a basic block of Yul statements. /// @param _funcParams List of function parameter names, defaults to /// an empty vector. void visit(Block const& _block); @@ -197,7 +197,7 @@ class ProtoConverter /// false otherwise bool functionValid(FunctionCall_Returns _type, unsigned _numOutParams); - /// Converts protobuf function call to a yul function call and appends + /// Converts protobuf function call to a Yul function call and appends /// it to output stream. /// @param _x Protobuf function call /// @param _name Function name @@ -211,7 +211,7 @@ class ProtoConverter bool _newLine = true ); - /// Prints a yul formatted variable declaration statement to the output + /// Prints a Yul formatted variable declaration statement to the output /// stream. /// Example 1: createVarDecls(0, 1, true) returns {"x_0"} and prints /// let x_0 := @@ -236,26 +236,26 @@ class ProtoConverter /// @return A vector of strings containing the printed variable names. std::vector createVars(unsigned _startIdx, unsigned _endIdx); - /// Manages scope of yul variables - /// @param _varNames is a list of yul variable names whose scope needs - /// to be tracked according to yul scoping rules. + /// Manages scope of Yul variables + /// @param _varNames is a list of Yul variable names whose scope needs + /// to be tracked according to Yul scoping rules. void scopeVariables(std::vector const& _varNames); - /// Print the yul syntax to make a call to a function named @a _funcName to + /// Print the Yul syntax to make a call to a function named @a _funcName to /// the output stream. /// @param _funcName Name of the function to be called /// @param _numInParams Number of input parameters in function signature /// @param _numOutParams Number of output parameters in function signature void createFunctionCall(std::string _funcName, unsigned _numInParams, unsigned _numOutParams); - /// Print the yul syntax to pass input arguments to a function that has + /// Print the Yul syntax to pass input arguments to a function that has /// @a _numInParams number of input parameters to the output stream. /// The input arguments are pseudo-randomly chosen from calldata, memory, - /// storage, or the yul optimizer hex dictionary. + /// storage, or the Yul optimizer hex dictionary. /// @param _numInParams Number of input arguments to fill void fillFunctionCallInput(unsigned _numInParams); - /// Print the yul syntax to save values returned by a function call + /// Print the Yul syntax to save values returned by a function call /// to the output stream. The values are either stored to memory or /// storage based on a simulated coin flip. The saved location is /// decided pseudo-randomly. @@ -272,7 +272,7 @@ class ProtoConverter /// Build a tree of objects that contains the object/data /// identifiers that are in scope in a given object. - /// @param _x root object of the yul protobuf specification. + /// @param _x root object of the Yul protobuf specification. void buildObjectScopeTree(Object const& _x); /// Returns a pseudo-random dictionary token. From d0bed502601777b0db3e99aacb25630d4a42f7b6 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Tue, 31 Mar 2020 13:05:58 -0500 Subject: [PATCH 115/126] [ci] add chk_shellscripts --- .circleci/config.yml | 6 +++ scripts/chk_shellscripts/chk_shellscripts.sh | 27 +++++++++++ scripts/chk_shellscripts/ignore.txt | 50 ++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100755 scripts/chk_shellscripts/chk_shellscripts.sh create mode 100644 scripts/chk_shellscripts/ignore.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index bfadb1139278..d3bc4a0682b5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -287,9 +287,15 @@ jobs: - image: buildpack-deps:disco steps: - checkout + - run: + name: Install shellcheck + command: apt -q update && apt install -y shellcheck - run: name: Check for C++ coding style command: ./scripts/check_style.sh + - run: + name: checking shell scripts + command: ./scripts/chk_shellscripts/chk_shellscripts.sh chk_pylint: docker: diff --git a/scripts/chk_shellscripts/chk_shellscripts.sh b/scripts/chk_shellscripts/chk_shellscripts.sh new file mode 100755 index 000000000000..8eccc551727d --- /dev/null +++ b/scripts/chk_shellscripts/chk_shellscripts.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -e + +REPO_ROOT="$(dirname "$0")"/../.. +REPO_ROOT=$(realpath "${REPO_ROOT}") +IGNORE_FILENAME="ignore.txt" +IGNORE_FILE="${REPO_ROOT}/scripts/chk_shellscripts/${IGNORE_FILENAME}" + +FOUND_FILES_TMP=$(mktemp) +IGNORE_FILES_TMP=$(mktemp) +trap 'rm -f ${FOUND_FILES_TMP} ; rm -f ${IGNORE_FILES_TMP}' EXIT + +sort < "${IGNORE_FILE}" >"${IGNORE_FILES_TMP}" +cd "${REPO_ROOT}" +find . -type f -name "*.sh" | sort >"${FOUND_FILES_TMP}" + +SHELLCHECK=${SHELLCHECK:-"$(command -v -- shellcheck)"} +if [ ! -f "${SHELLCHECK}" ]; then + echo "error: shellcheck '${SHELLCHECK}' not found." + exit 1 +fi + +FILES=$(join -v2 "${IGNORE_FILES_TMP}" "${FOUND_FILES_TMP}") + +# shellcheck disable=SC2086 +"${SHELLCHECK}" ${FILES[*]} diff --git a/scripts/chk_shellscripts/ignore.txt b/scripts/chk_shellscripts/ignore.txt new file mode 100644 index 000000000000..0986077641a8 --- /dev/null +++ b/scripts/chk_shellscripts/ignore.txt @@ -0,0 +1,50 @@ +./test/docsCodeStyle.sh +./test/cmdlineTests.sh +./test/externalTests.sh +./test/externalTests/common.sh +./test/externalTests/gnosis.sh +./test/externalTests/zeppelin.sh +./test/externalTests/colony.sh +./test/externalTests/solc-js/solc-js.sh +./scripts/common.sh +./scripts/isoltest.sh +./scripts/get_version.sh +./scripts/soltest.sh +./scripts/test_emscripten.sh +./scripts/wasm-rebuild/docker-scripts/rebuild_tags.sh +./scripts/wasm-rebuild/docker-scripts/rebuild_current.sh +./scripts/wasm-rebuild/docker-scripts/genbytecode.sh +./scripts/wasm-rebuild/docker-scripts/patch.sh +./scripts/wasm-rebuild/rebuild.sh +./scripts/build_emscripten.sh +./scripts/travis-emscripten/build_emscripten.sh +./scripts/travis-emscripten/install_deps.sh +./scripts/travis-emscripten/publish_binary.sh +./scripts/docker_build.sh +./scripts/docs_version_pragma_check.sh +./scripts/uniqueErrors.sh +./scripts/report_errors.sh +./scripts/tests.sh +./scripts/docker_deploy.sh +./scripts/bytecodecompare/storebytecode.sh +./scripts/deps-ppa/static_z3.sh +./scripts/ASTImportTest.sh +./scripts/install_static_z3.sh +./scripts/install_obsolete_jsoncpp_1_7_4.sh +./scripts/install_deps.sh +./scripts/build.sh +./scripts/check_style.sh +./scripts/run_proofs.sh +./scripts/common_cmdline.sh +./scripts/docker_deploy_manual.sh +./scripts/endToEndExtraction/create_traces.sh +./scripts/release.sh +./scripts/download_ossfuzz_corpus.sh +./scripts/release_ppa.sh +./scripts/install_cmake.sh +./scripts/release_emscripten.sh +./scripts/create_source_tarball.sh +./scripts/docs.sh +./.circleci/soltest.sh +./.circleci/osx_install_dependencies.sh +./.circleci/soltest_all.sh From 9d06dd070d3a8980042443ba3142867a179da206 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 27 Apr 2020 15:26:25 +0200 Subject: [PATCH 116/126] CircleCI: Change from Ubuntu Disco (19.04) to Ubuntu Focal (20.04) due to support EOL --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bfadb1139278..bc0c3cf02d17 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -284,7 +284,7 @@ jobs: chk_coding_style: docker: - - image: buildpack-deps:disco + - image: buildpack-deps:focal steps: - checkout - run: From 3d772edc7a18451f16d71aebecb9acbd84461eea Mon Sep 17 00:00:00 2001 From: ssi91 Date: Fri, 17 Apr 2020 03:29:40 +0700 Subject: [PATCH 117/126] handle file prefix add the description to the changelog fix: use the right method to search in string follow the codestyle using tabs delete redundant declaration implement the handling inline Update Changelog.md Co-Authored-By: Leonardo --- Changelog.md | 1 + solc/CommandLineInterface.cpp | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 6a7042e6feb3..ec45f9da7df4 100644 --- a/Changelog.md +++ b/Changelog.md @@ -15,6 +15,7 @@ Bugfixes: * Type Checker: Fix several internal errors by performing size and recursiveness checks of types before the full type checking. * Type Checker: Fix internal error when assigning to empty tuples. * Type Checker: Perform recursiveness check on structs declared at the file level. + * Standard Json Input: Fix error when using prefix ``file://`` in the field ``urls``. Build System: * soltest.sh: ``SOLIDITY_BUILD_DIR`` is no longer relative to ``REPO_ROOT`` to allow for build directories outside of the source tree. diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index da14c58f9fdf..537b2096baf4 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -953,7 +953,10 @@ bool CommandLineInterface::processInput() "ReadFile callback used as callback kind " + _kind )); - auto path = boost::filesystem::path(_path); + string validPath = _path; + if (validPath.find("file://") == 0) + validPath.erase(0, 7); + auto path = boost::filesystem::path(validPath); auto canonicalPath = boost::filesystem::weakly_canonical(path); bool isAllowed = false; for (auto const& allowedDir: m_allowedDirectories) From f3f729549d09d7c28b4af5695cb9eed19e6726d6 Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Thu, 9 Apr 2020 21:59:17 +0200 Subject: [PATCH 118/126] [Sol->Yul] Enabling creation function call --- libsolidity/codegen/YulUtilFunctions.cpp | 28 ++++++ libsolidity/codegen/YulUtilFunctions.h | 7 ++ .../codegen/ir/IRGenerationContext.cpp | 15 +++ libsolidity/codegen/ir/IRGenerationContext.h | 15 ++- libsolidity/codegen/ir/IRGenerator.cpp | 43 +++++---- libsolidity/codegen/ir/IRGenerator.h | 12 ++- .../codegen/ir/IRGeneratorForStatements.cpp | 54 ++++++++++- libsolidity/interface/CompilerStack.cpp | 8 +- .../ir_compiler_inheritance_nosubobjects/args | 1 + .../input.sol | 7 ++ .../output | 55 +++++++++++ test/cmdlineTests/ir_compiler_subobjects/args | 1 + test/cmdlineTests/ir_compiler_subobjects/err | 5 + .../ir_compiler_subobjects/input.sol | 8 ++ .../ir_compiler_subobjects/output | 96 +++++++++++++++++++ .../standard_ir_requested/output.json | 2 + .../yul_string_format_ascii/output.json | 2 + .../output.json | 2 + .../output.json | 2 + .../yul_string_format_ascii_long/output.json | 2 + .../yul_string_format_hex/output.json | 2 + .../array/fixed_arrays_as_return_type.sol | 2 + .../array/function_array_cross_calls.sol | 2 + .../creation_function_call_no_args.sol | 15 +++ .../creation_function_call_with_args.sol | 20 ++++ .../creation_function_call_with_salt.sol | 21 ++++ .../gas_and_value_brace_syntax.sol | 4 +- .../interface_inheritance_conversions.sol | 2 + .../address_overload_resolution.sol | 2 + ...d_function_calldata_calldata_interface.sol | 2 + ...ted_function_calldata_memory_interface.sol | 2 + .../various/contract_binary_dependencies.sol | 2 + .../various/external_types_in_calls.sol | 2 + .../semanticTests/various/senders_balance.sol | 2 + .../various/staticcall_for_view_and_pure.sol | 1 + .../various/write_storage_external.sol | 2 + 36 files changed, 420 insertions(+), 28 deletions(-) create mode 100644 test/cmdlineTests/ir_compiler_inheritance_nosubobjects/args create mode 100644 test/cmdlineTests/ir_compiler_inheritance_nosubobjects/input.sol create mode 100644 test/cmdlineTests/ir_compiler_inheritance_nosubobjects/output create mode 100644 test/cmdlineTests/ir_compiler_subobjects/args create mode 100644 test/cmdlineTests/ir_compiler_subobjects/err create mode 100644 test/cmdlineTests/ir_compiler_subobjects/input.sol create mode 100644 test/cmdlineTests/ir_compiler_subobjects/output create mode 100644 test/libsolidity/semanticTests/functionCall/creation_function_call_no_args.sol create mode 100644 test/libsolidity/semanticTests/functionCall/creation_function_call_with_args.sol create mode 100644 test/libsolidity/semanticTests/functionCall/creation_function_call_with_salt.sol diff --git a/libsolidity/codegen/YulUtilFunctions.cpp b/libsolidity/codegen/YulUtilFunctions.cpp index ab449a730b4e..75d297b3474f 100644 --- a/libsolidity/codegen/YulUtilFunctions.cpp +++ b/libsolidity/codegen/YulUtilFunctions.cpp @@ -1337,6 +1337,34 @@ string YulUtilFunctions::allocationFunction() }); } +string YulUtilFunctions::allocationTemporaryMemoryFunction() +{ + string functionName = "allocateTemporaryMemory"; + return m_functionCollector.createFunction(functionName, [&]() { + return Whiskers(R"( + function () -> memPtr { + memPtr := mload() + } + )") + ("freeMemoryPointer", to_string(CompilerUtils::freeMemoryPointer)) + ("functionName", functionName) + .render(); + }); +} + +string YulUtilFunctions::releaseTemporaryMemoryFunction() +{ + string functionName = "releaseTemporaryMemory"; + return m_functionCollector.createFunction(functionName, [&](){ + return Whiskers(R"( + function () { + } + )") + ("functionName", functionName) + .render(); + }); +} + string YulUtilFunctions::zeroMemoryArrayFunction(ArrayType const& _type) { if (_type.baseType()->hasSimpleZeroValueInMemory()) diff --git a/libsolidity/codegen/YulUtilFunctions.h b/libsolidity/codegen/YulUtilFunctions.h index b16eb067281d..4f7556f5fbb0 100644 --- a/libsolidity/codegen/YulUtilFunctions.h +++ b/libsolidity/codegen/YulUtilFunctions.h @@ -253,6 +253,13 @@ class YulUtilFunctions /// Return value: pointer std::string allocationFunction(); + /// @returns the name of the function that allocates temporary memory with predefined size + /// Return value: pointer + std::string allocationTemporaryMemoryFunction(); + + /// @returns the name of the function that releases previously allocated temporary memory + std::string releaseTemporaryMemoryFunction(); + /// @returns the name of a function that zeroes an array. /// signature: (dataStart, dataSizeInBytes) -> std::string zeroMemoryArrayFunction(ArrayType const& _type); diff --git a/libsolidity/codegen/ir/IRGenerationContext.cpp b/libsolidity/codegen/ir/IRGenerationContext.cpp index aa6ec7619c09..43e9f95e2932 100644 --- a/libsolidity/codegen/ir/IRGenerationContext.cpp +++ b/libsolidity/codegen/ir/IRGenerationContext.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -96,6 +97,15 @@ string IRGenerationContext::functionName(VariableDeclaration const& _varDecl) return "getter_fun_" + _varDecl.name() + "_" + to_string(_varDecl.id()); } +string IRGenerationContext::creationObjectName(ContractDefinition const& _contract) const +{ + return _contract.name() + "_" + toString(_contract.id()); +} +string IRGenerationContext::runtimeObjectName(ContractDefinition const& _contract) const +{ + return _contract.name() + "_" + toString(_contract.id()) + "_deployed"; +} + string IRGenerationContext::newYulVariable() { return "_" + to_string(++m_varCounter); @@ -170,6 +180,11 @@ YulUtilFunctions IRGenerationContext::utils() return YulUtilFunctions(m_evmVersion, m_revertStrings, m_functions); } +ABIFunctions IRGenerationContext::abiFunctions() +{ + return ABIFunctions(m_evmVersion, m_revertStrings, m_functions); +} + std::string IRGenerationContext::revertReasonIfDebug(std::string const& _message) { return YulUtilFunctions::revertReasonIfDebug(m_revertStrings, _message); diff --git a/libsolidity/codegen/ir/IRGenerationContext.h b/libsolidity/codegen/ir/IRGenerationContext.h index acbe6250ba43..0a07f2f29ffb 100644 --- a/libsolidity/codegen/ir/IRGenerationContext.h +++ b/libsolidity/codegen/ir/IRGenerationContext.h @@ -20,6 +20,7 @@ #pragma once +#include #include #include #include @@ -38,11 +39,8 @@ namespace solidity::frontend { -class ContractDefinition; -class VariableDeclaration; -class FunctionDefinition; -class Expression; class YulUtilFunctions; +class ABIFunctions; /** * Class that contains contextual information during IR generation. @@ -93,6 +91,9 @@ class IRGenerationContext std::string functionName(FunctionDefinition const& _function); std::string functionName(VariableDeclaration const& _varDecl); + std::string creationObjectName(ContractDefinition const& _contract) const; + std::string runtimeObjectName(ContractDefinition const& _contract) const; + std::string newYulVariable(); std::string internalDispatch(size_t _in, size_t _out); @@ -102,6 +103,8 @@ class IRGenerationContext langutil::EVMVersion evmVersion() const { return m_evmVersion; }; + ABIFunctions abiFunctions(); + /// @returns code that stores @param _message for revert reason /// if m_revertStrings is debug. std::string revertReasonIfDebug(std::string const& _message = ""); @@ -112,6 +115,8 @@ class IRGenerationContext /// function call that was invoked as part of the try statement. std::string trySuccessConditionVariable(Expression const& _expression) const; + std::set& subObjectsCreated() { return m_subObjects; } + private: langutil::EVMVersion m_evmVersion; RevertStrings m_revertStrings; @@ -131,6 +136,8 @@ class IRGenerationContext /// long as the order of Yul functions in the generated code is deterministic and the same on /// all platforms - which is a property guaranteed by MultiUseYulFunctionCollector. std::set m_functionGenerationQueue; + + std::set m_subObjects; }; } diff --git a/libsolidity/codegen/ir/IRGenerator.cpp b/libsolidity/codegen/ir/IRGenerator.cpp index 68a6f2a9d2f9..aed373c1018f 100644 --- a/libsolidity/codegen/ir/IRGenerator.cpp +++ b/libsolidity/codegen/ir/IRGenerator.cpp @@ -48,9 +48,12 @@ using namespace solidity; using namespace solidity::util; using namespace solidity::frontend; -pair IRGenerator::run(ContractDefinition const& _contract) +pair IRGenerator::run( + ContractDefinition const& _contract, + map const& _otherYulSources +) { - string const ir = yul::reindent(generate(_contract)); + string const ir = yul::reindent(generate(_contract, _otherYulSources)); yul::AssemblyStack asmStack(m_evmVersion, yul::AssemblyStack::Language::StrictAssembly, m_optimiserSettings); if (!asmStack.parseAndAnalyze("", ir)) @@ -73,8 +76,19 @@ pair IRGenerator::run(ContractDefinition const& _contract) return {warning + ir, warning + asmStack.print()}; } -string IRGenerator::generate(ContractDefinition const& _contract) +string IRGenerator::generate( + ContractDefinition const& _contract, + map const& _otherYulSources +) { + auto subObjectSources = [&_otherYulSources](std::set const& subObjects) -> string + { + std::string subObjectsSources; + for (ContractDefinition const* subObject: subObjects) + subObjectsSources += _otherYulSources.at(subObject); + return subObjectsSources; + }; + Whiskers t(R"( object "" { code { @@ -93,13 +107,15 @@ string IRGenerator::generate(ContractDefinition const& _contract) } + } + } )"); resetContext(_contract); - t("CreationObject", creationObjectName(_contract)); + t("CreationObject", m_context.creationObjectName(_contract)); t("memoryInit", memoryInit()); t("notLibrary", !_contract.isLibrary()); @@ -112,7 +128,7 @@ string IRGenerator::generate(ContractDefinition const& _contract) constructorParams.emplace_back(m_context.newYulVariable()); t( "copyConstructorArguments", - m_utils.copyConstructorArgumentsToMemoryFunction(_contract, creationObjectName(_contract)) + m_utils.copyConstructorArgumentsToMemoryFunction(_contract, m_context.creationObjectName(_contract)) ); } t("constructorParams", joinHumanReadable(constructorParams)); @@ -123,12 +139,14 @@ string IRGenerator::generate(ContractDefinition const& _contract) generateImplicitConstructors(_contract); generateQueuedFunctions(); t("functions", m_context.functionCollector().requestedFunctions()); + t("subObjects", subObjectSources(m_context.subObjectsCreated())); resetContext(_contract); - t("RuntimeObject", runtimeObjectName(_contract)); + t("RuntimeObject", m_context.runtimeObjectName(_contract)); t("dispatch", dispatchRoutine(_contract)); generateQueuedFunctions(); t("runtimeFunctions", m_context.functionCollector().requestedFunctions()); + t("runtimeSubObjects", subObjectSources(m_context.subObjectsCreated())); return t.render(); } @@ -313,6 +331,7 @@ string IRGenerator::initStateVariables(ContractDefinition const& _contract) return generator.code(); } + void IRGenerator::generateImplicitConstructors(ContractDefinition const& _contract) { auto listAllParams = [&]( @@ -375,7 +394,7 @@ string IRGenerator::deployCode(ContractDefinition const& _contract) codecopy(0, dataoffset(""), datasize("")) return(0, datasize("")) )X"); - t("object", runtimeObjectName(_contract)); + t("object", m_context.runtimeObjectName(_contract)); return t.render(); } @@ -384,16 +403,6 @@ string IRGenerator::callValueCheck() return "if callvalue() { revert(0, 0) }"; } -string IRGenerator::creationObjectName(ContractDefinition const& _contract) -{ - return _contract.name() + "_" + to_string(_contract.id()); -} - -string IRGenerator::runtimeObjectName(ContractDefinition const& _contract) -{ - return _contract.name() + "_" + to_string(_contract.id()) + "_deployed"; -} - string IRGenerator::implicitConstructorName(ContractDefinition const& _contract) { return "constructor_" + _contract.name() + "_" + to_string(_contract.id()); diff --git a/libsolidity/codegen/ir/IRGenerator.h b/libsolidity/codegen/ir/IRGenerator.h index fde382cd71dc..a4298a902b47 100644 --- a/libsolidity/codegen/ir/IRGenerator.h +++ b/libsolidity/codegen/ir/IRGenerator.h @@ -50,10 +50,16 @@ class IRGenerator /// Generates and returns the IR code, in unoptimized and optimized form /// (or just pretty-printed, depending on the optimizer settings). - std::pair run(ContractDefinition const& _contract); + std::pair run( + ContractDefinition const& _contract, + std::map const& _otherYulSources + ); private: - std::string generate(ContractDefinition const& _contract); + std::string generate( + ContractDefinition const& _contract, + std::map const& _otherYulSources + ); std::string generate(Block const& _block); /// Generates code for all the functions from the function generation queue. @@ -87,8 +93,6 @@ class IRGenerator std::string deployCode(ContractDefinition const& _contract); std::string callValueCheck(); - std::string creationObjectName(ContractDefinition const& _contract); - std::string runtimeObjectName(ContractDefinition const& _contract); std::string implicitConstructorName(ContractDefinition const& _contract); std::string dispatchRoutine(ContractDefinition const& _contract); diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index a5706bd9c2cd..252ddffa700e 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -849,11 +849,63 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall) templ("pos", m_context.newYulVariable()); templ("end", m_context.newYulVariable()); templ("freeMemory", freeMemory()); - templ("encode", abi.tupleEncoder({arguments.front()->annotation().type},{parameterTypes.front()})); + templ("encode", abi.tupleEncoder({arguments.front()->annotation().type}, {parameterTypes.front()})); templ("nonIndexedArgs", IRVariable(*arguments.front()).commaSeparatedList()); templ("log", "log" + to_string(logNumber)); templ("indexedArgs", indexedArgs); m_code << templ.render(); + + break; + } + case FunctionType::Kind::Creation: + { + solAssert(!functionType->gasSet(), "Gas limit set for contract creation."); + solAssert( + functionType->returnParameterTypes().size() == 1, + "Constructor should return only one type" + ); + + TypePointers argumentTypes; + string constructorParams; + for (ASTPointer const& arg: arguments) + { + argumentTypes.push_back(arg->annotation().type); + constructorParams += ", " + IRVariable{*arg}.commaSeparatedList(); + } + + ContractDefinition const* contract = + &dynamic_cast(*functionType->returnParameterTypes().front()).contractDefinition(); + m_context.subObjectsCreated().insert(contract); + + Whiskers t(R"( + let := () + let := add(, datasize("")) + if or(gt(, 0xffffffffffffffff), lt(, )) { revert(0, 0) } + datacopy(, dataoffset(""), datasize("")) + := () + + let := create2(, , sub(, ), ) + + let := create(, , sub(, )) + + () + )"); + t("memPos", m_context.newYulVariable()); + t("memEnd", m_context.newYulVariable()); + t("allocateTemporaryMemory", m_utils.allocationTemporaryMemoryFunction()); + t("releaseTemporaryMemory", m_utils.releaseTemporaryMemoryFunction()); + t("object", m_context.creationObjectName(*contract)); + t("abiEncode", + m_context.abiFunctions().tupleEncoder(argumentTypes, functionType->parameterTypes(),false) + ); + t("constructorParams", constructorParams); + t("value", functionType->valueSet() ? IRVariable(_functionCall.expression()).part("value").name() : "0"); + t("saltSet", functionType->saltSet()); + if (functionType->saltSet()) + t("salt", IRVariable(_functionCall.expression()).part("salt").name()); + t("retVars", IRVariable(_functionCall).commaSeparatedList()); + m_code << t.render(); + break; } default: diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 16422320cf5d..0af68bda7767 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -1129,15 +1129,21 @@ void CompilerStack::generateIR(ContractDefinition const& _contract) if (!_contract.canBeDeployed()) return; + map otherYulSources; + Contract& compiledContract = m_contracts.at(_contract.fullyQualifiedName()); if (!compiledContract.yulIR.empty()) return; + string dependenciesSource; for (auto const* dependency: _contract.annotation().contractDependencies) + { generateIR(*dependency); + otherYulSources.emplace(dependency, m_contracts.at(dependency->fullyQualifiedName()).yulIR); + } IRGenerator generator(m_evmVersion, m_revertStrings, m_optimiserSettings); - tie(compiledContract.yulIR, compiledContract.yulIROptimized) = generator.run(_contract); + tie(compiledContract.yulIR, compiledContract.yulIROptimized) = generator.run(_contract, otherYulSources); } void CompilerStack::generateEwasm(ContractDefinition const& _contract) diff --git a/test/cmdlineTests/ir_compiler_inheritance_nosubobjects/args b/test/cmdlineTests/ir_compiler_inheritance_nosubobjects/args new file mode 100644 index 000000000000..cae21e7207e0 --- /dev/null +++ b/test/cmdlineTests/ir_compiler_inheritance_nosubobjects/args @@ -0,0 +1 @@ +--ir-optimized --optimize \ No newline at end of file diff --git a/test/cmdlineTests/ir_compiler_inheritance_nosubobjects/input.sol b/test/cmdlineTests/ir_compiler_inheritance_nosubobjects/input.sol new file mode 100644 index 000000000000..69f553304338 --- /dev/null +++ b/test/cmdlineTests/ir_compiler_inheritance_nosubobjects/input.sol @@ -0,0 +1,7 @@ +pragma solidity >=0.6.0; + +contract C { + constructor() public {} +} +contract D is C { +} \ No newline at end of file diff --git a/test/cmdlineTests/ir_compiler_inheritance_nosubobjects/output b/test/cmdlineTests/ir_compiler_inheritance_nosubobjects/output new file mode 100644 index 000000000000..fd372367a055 --- /dev/null +++ b/test/cmdlineTests/ir_compiler_inheritance_nosubobjects/output @@ -0,0 +1,55 @@ +Optimized IR: +/******************************************************* + * WARNING * + * Solidity to Yul compilation is still EXPERIMENTAL * + * It can result in LOSS OF FUNDS or worse * + * !USE AT YOUR OWN RISK! * + *******************************************************/ + +object "C_6" { + code { + { + mstore(64, 128) + if callvalue() { revert(0, 0) } + let _1 := datasize("C_6_deployed") + codecopy(0, dataoffset("C_6_deployed"), _1) + return(0, _1) + } + } + object "C_6_deployed" { + code { + { + mstore(64, 128) + revert(0, 0) + } + } + } +} + +Optimized IR: +/******************************************************* + * WARNING * + * Solidity to Yul compilation is still EXPERIMENTAL * + * It can result in LOSS OF FUNDS or worse * + * !USE AT YOUR OWN RISK! * + *******************************************************/ + +object "D_9" { + code { + { + mstore(64, 128) + if callvalue() { revert(0, 0) } + let _1 := datasize("D_9_deployed") + codecopy(0, dataoffset("D_9_deployed"), _1) + return(0, _1) + } + } + object "D_9_deployed" { + code { + { + mstore(64, 128) + revert(0, 0) + } + } + } +} diff --git a/test/cmdlineTests/ir_compiler_subobjects/args b/test/cmdlineTests/ir_compiler_subobjects/args new file mode 100644 index 000000000000..cae21e7207e0 --- /dev/null +++ b/test/cmdlineTests/ir_compiler_subobjects/args @@ -0,0 +1 @@ +--ir-optimized --optimize \ No newline at end of file diff --git a/test/cmdlineTests/ir_compiler_subobjects/err b/test/cmdlineTests/ir_compiler_subobjects/err new file mode 100644 index 000000000000..a28ad19665ba --- /dev/null +++ b/test/cmdlineTests/ir_compiler_subobjects/err @@ -0,0 +1,5 @@ +Warning: Unused local variable. + --> ir_compiler_subobjects/input.sol:6:9: + | +6 | C c = new C(); + | ^^^ diff --git a/test/cmdlineTests/ir_compiler_subobjects/input.sol b/test/cmdlineTests/ir_compiler_subobjects/input.sol new file mode 100644 index 000000000000..e0fdbbc136fb --- /dev/null +++ b/test/cmdlineTests/ir_compiler_subobjects/input.sol @@ -0,0 +1,8 @@ +pragma solidity >=0.6.0; + +contract C {} +contract D { + function f() public { + C c = new C(); + } +} diff --git a/test/cmdlineTests/ir_compiler_subobjects/output b/test/cmdlineTests/ir_compiler_subobjects/output new file mode 100644 index 000000000000..6b5f8e6778bd --- /dev/null +++ b/test/cmdlineTests/ir_compiler_subobjects/output @@ -0,0 +1,96 @@ +Optimized IR: +/******************************************************* + * WARNING * + * Solidity to Yul compilation is still EXPERIMENTAL * + * It can result in LOSS OF FUNDS or worse * + * !USE AT YOUR OWN RISK! * + *******************************************************/ + +object "C_2" { + code { + { + mstore(64, 128) + if callvalue() { revert(0, 0) } + let _1 := datasize("C_2_deployed") + codecopy(0, dataoffset("C_2_deployed"), _1) + return(0, _1) + } + } + object "C_2_deployed" { + code { + { + mstore(64, 128) + revert(0, 0) + } + } + } +} + +Optimized IR: +/******************************************************* + * WARNING * + * Solidity to Yul compilation is still EXPERIMENTAL * + * It can result in LOSS OF FUNDS or worse * + * !USE AT YOUR OWN RISK! * + *******************************************************/ + +object "D_13" { + code { + { + mstore(64, 128) + if callvalue() { revert(0, 0) } + let _1 := datasize("D_13_deployed") + codecopy(0, dataoffset("D_13_deployed"), _1) + return(0, _1) + } + } + object "D_13_deployed" { + code { + { + mstore(64, 128) + if iszero(lt(calldatasize(), 4)) + { + let _1 := 0 + if eq(0x26121ff0, shr(224, calldataload(_1))) + { + if callvalue() { revert(_1, _1) } + if slt(add(calldatasize(), not(3)), _1) { revert(_1, _1) } + let _2 := datasize("C_2") + let _3 := add(128, _2) + if or(gt(_3, 0xffffffffffffffff), lt(_3, 128)) { revert(_1, _1) } + datacopy(128, dataoffset("C_2"), _2) + pop(create(_1, 128, _2)) + return(allocateMemory(_1), _1) + } + } + revert(0, 0) + } + function allocateMemory(size) -> memPtr + { + memPtr := mload(64) + let newFreePtr := add(memPtr, size) + if or(gt(newFreePtr, 0xffffffffffffffff), lt(newFreePtr, memPtr)) { revert(0, 0) } + mstore(64, newFreePtr) + } + } + object "C_2" { + code { + { + mstore(64, 128) + if callvalue() { revert(0, 0) } + let _1 := datasize("C_2_deployed") + codecopy(0, dataoffset("C_2_deployed"), _1) + return(0, _1) + } + } + object "C_2_deployed" { + code { + { + mstore(64, 128) + revert(0, 0) + } + } + } + } + } +} diff --git a/test/cmdlineTests/standard_ir_requested/output.json b/test/cmdlineTests/standard_ir_requested/output.json index 2e7aef192331..bf9b20ed82a3 100644 --- a/test/cmdlineTests/standard_ir_requested/output.json +++ b/test/cmdlineTests/standard_ir_requested/output.json @@ -76,7 +76,9 @@ object \"C_6\" { } } + } + } "}}},"sources":{"A":{"id":0}}} diff --git a/test/cmdlineTests/yul_string_format_ascii/output.json b/test/cmdlineTests/yul_string_format_ascii/output.json index 0ca1c28dcb8a..81fa10ca70d7 100644 --- a/test/cmdlineTests/yul_string_format_ascii/output.json +++ b/test/cmdlineTests/yul_string_format_ascii/output.json @@ -131,7 +131,9 @@ object \"C_10\" { } } + } + } "}}},"sources":{"A":{"id":0}}} diff --git a/test/cmdlineTests/yul_string_format_ascii_bytes32/output.json b/test/cmdlineTests/yul_string_format_ascii_bytes32/output.json index 062a56a548b0..a171f87b59ce 100644 --- a/test/cmdlineTests/yul_string_format_ascii_bytes32/output.json +++ b/test/cmdlineTests/yul_string_format_ascii_bytes32/output.json @@ -99,7 +99,9 @@ object \"C_10\" { } } + } + } "}}},"sources":{"A":{"id":0}}} diff --git a/test/cmdlineTests/yul_string_format_ascii_bytes32_from_number/output.json b/test/cmdlineTests/yul_string_format_ascii_bytes32_from_number/output.json index 6779e68c08f3..3dccf690dc22 100644 --- a/test/cmdlineTests/yul_string_format_ascii_bytes32_from_number/output.json +++ b/test/cmdlineTests/yul_string_format_ascii_bytes32_from_number/output.json @@ -111,7 +111,9 @@ object \"C_10\" { } } + } + } "}}},"sources":{"A":{"id":0}}} diff --git a/test/cmdlineTests/yul_string_format_ascii_long/output.json b/test/cmdlineTests/yul_string_format_ascii_long/output.json index 4b4b11347c10..77360e5a4631 100644 --- a/test/cmdlineTests/yul_string_format_ascii_long/output.json +++ b/test/cmdlineTests/yul_string_format_ascii_long/output.json @@ -135,7 +135,9 @@ object \"C_10\" { } } + } + } "}}},"sources":{"A":{"id":0}}} diff --git a/test/cmdlineTests/yul_string_format_hex/output.json b/test/cmdlineTests/yul_string_format_hex/output.json index d011fdade45a..81853dadbe2b 100644 --- a/test/cmdlineTests/yul_string_format_hex/output.json +++ b/test/cmdlineTests/yul_string_format_hex/output.json @@ -111,7 +111,9 @@ object \"C_10\" { } } + } + } "}}},"sources":{"A":{"id":0}}} diff --git a/test/libsolidity/semanticTests/array/fixed_arrays_as_return_type.sol b/test/libsolidity/semanticTests/array/fixed_arrays_as_return_type.sol index 61d1a33f55d2..c09d397ee8c8 100644 --- a/test/libsolidity/semanticTests/array/fixed_arrays_as_return_type.sol +++ b/test/libsolidity/semanticTests/array/fixed_arrays_as_return_type.sol @@ -17,5 +17,7 @@ contract B { } } +// ==== +// compileViaYul: also // ---- // f() -> 2, 3, 4, 5, 6, 1000, 1001, 1002, 1003, 1004 diff --git a/test/libsolidity/semanticTests/array/function_array_cross_calls.sol b/test/libsolidity/semanticTests/array/function_array_cross_calls.sol index 64f8c74028fb..d812400694dd 100644 --- a/test/libsolidity/semanticTests/array/function_array_cross_calls.sol +++ b/test/libsolidity/semanticTests/array/function_array_cross_calls.sol @@ -41,5 +41,7 @@ contract C { } } +// ==== +// compileViaYul: also // ---- // test() -> 5, 6, 7 diff --git a/test/libsolidity/semanticTests/functionCall/creation_function_call_no_args.sol b/test/libsolidity/semanticTests/functionCall/creation_function_call_no_args.sol new file mode 100644 index 000000000000..eac910f05411 --- /dev/null +++ b/test/libsolidity/semanticTests/functionCall/creation_function_call_no_args.sol @@ -0,0 +1,15 @@ +contract C { + uint public i; + constructor() public { + i = 2; + } +} +contract D { + function f() public returns (uint r) { + return new C().i(); + } +} +// ==== +// compileViaYul: also +// ---- +// f() -> 2 diff --git a/test/libsolidity/semanticTests/functionCall/creation_function_call_with_args.sol b/test/libsolidity/semanticTests/functionCall/creation_function_call_with_args.sol new file mode 100644 index 000000000000..32c2910e277b --- /dev/null +++ b/test/libsolidity/semanticTests/functionCall/creation_function_call_with_args.sol @@ -0,0 +1,20 @@ +contract C { + uint public i; + constructor(uint newI) public { + i = newI; + } +} +contract D { + C c; + constructor(uint v) public { + c = new C(v); + } + function f() public returns (uint r) { + return c.i(); + } +} +// ==== +// compileViaYul: also +// ---- +// constructor(): 2 -> +// f() -> 2 diff --git a/test/libsolidity/semanticTests/functionCall/creation_function_call_with_salt.sol b/test/libsolidity/semanticTests/functionCall/creation_function_call_with_salt.sol new file mode 100644 index 000000000000..3bb9765a9f46 --- /dev/null +++ b/test/libsolidity/semanticTests/functionCall/creation_function_call_with_salt.sol @@ -0,0 +1,21 @@ +contract C { + uint public i; + constructor(uint newI) public { + i = newI; + } +} +contract D { + C c; + constructor(uint v) public { + c = new C{salt: "abc"}(v); + } + function f() public returns (uint r) { + return c.i(); + } +} +// ==== +// EVMVersion: >=constantinople +// compileViaYul: also +// ---- +// constructor(): 2 -> +// f() -> 2 diff --git a/test/libsolidity/semanticTests/functionCall/gas_and_value_brace_syntax.sol b/test/libsolidity/semanticTests/functionCall/gas_and_value_brace_syntax.sol index dbd524deb744..46c922a9cbf1 100644 --- a/test/libsolidity/semanticTests/functionCall/gas_and_value_brace_syntax.sol +++ b/test/libsolidity/semanticTests/functionCall/gas_and_value_brace_syntax.sol @@ -36,8 +36,10 @@ contract test { } } +// ==== +// compileViaYul: also // ---- // constructor(), 20 wei -> // sendAmount(uint256): 5 -> 5 // outOfGas() -> FAILURE # call to helper should not succeed but amount should be transferred anyway # -// checkState() -> false, 15 \ No newline at end of file +// checkState() -> false, 15 diff --git a/test/libsolidity/semanticTests/interface_inheritance_conversions.sol b/test/libsolidity/semanticTests/interface_inheritance_conversions.sol index 4241afe812b0..62213211a5aa 100644 --- a/test/libsolidity/semanticTests/interface_inheritance_conversions.sol +++ b/test/libsolidity/semanticTests/interface_inheritance_conversions.sol @@ -33,6 +33,8 @@ contract C { } } +// ==== +// compileViaYul: also // ---- // convertParent() -> 1 // convertSubA() -> 1, 2 diff --git a/test/libsolidity/semanticTests/intheritance/address_overload_resolution.sol b/test/libsolidity/semanticTests/intheritance/address_overload_resolution.sol index 9b68cc154c4d..640fcce09545 100644 --- a/test/libsolidity/semanticTests/intheritance/address_overload_resolution.sol +++ b/test/libsolidity/semanticTests/intheritance/address_overload_resolution.sol @@ -19,6 +19,8 @@ contract D { } } +// ==== +// compileViaYul: also // ---- // f() -> 1 // g() -> 5 diff --git a/test/libsolidity/semanticTests/intheritance/inherited_function_calldata_calldata_interface.sol b/test/libsolidity/semanticTests/intheritance/inherited_function_calldata_calldata_interface.sol index 9812ca5208d8..cb89901a42e3 100644 --- a/test/libsolidity/semanticTests/intheritance/inherited_function_calldata_calldata_interface.sol +++ b/test/libsolidity/semanticTests/intheritance/inherited_function_calldata_calldata_interface.sol @@ -21,5 +21,7 @@ contract B { } } +// ==== +// compileViaYul: also // ---- // g() -> 42 diff --git a/test/libsolidity/semanticTests/intheritance/inherited_function_calldata_memory_interface.sol b/test/libsolidity/semanticTests/intheritance/inherited_function_calldata_memory_interface.sol index 9a2c1a1e7d07..febaf9f0431a 100644 --- a/test/libsolidity/semanticTests/intheritance/inherited_function_calldata_memory_interface.sol +++ b/test/libsolidity/semanticTests/intheritance/inherited_function_calldata_memory_interface.sol @@ -21,5 +21,7 @@ contract B { } } +// ==== +// compileViaYul: also // ---- // g() -> 42 diff --git a/test/libsolidity/semanticTests/various/contract_binary_dependencies.sol b/test/libsolidity/semanticTests/various/contract_binary_dependencies.sol index f1220d351e4c..aa3734b3aad7 100644 --- a/test/libsolidity/semanticTests/various/contract_binary_dependencies.sol +++ b/test/libsolidity/semanticTests/various/contract_binary_dependencies.sol @@ -16,5 +16,7 @@ contract C { } } +// ==== +// compileViaYul: also // ---- // constructor() -> diff --git a/test/libsolidity/semanticTests/various/external_types_in_calls.sol b/test/libsolidity/semanticTests/various/external_types_in_calls.sol index 9906bd52bc33..0e65511c16b2 100644 --- a/test/libsolidity/semanticTests/various/external_types_in_calls.sol +++ b/test/libsolidity/semanticTests/various/external_types_in_calls.sol @@ -23,6 +23,8 @@ contract C { } } +// ==== +// compileViaYul: also // ---- // test() -> 9, 7 // t2() -> 9 diff --git a/test/libsolidity/semanticTests/various/senders_balance.sol b/test/libsolidity/semanticTests/various/senders_balance.sol index 0c84352b6237..7da77aab2af6 100644 --- a/test/libsolidity/semanticTests/various/senders_balance.sol +++ b/test/libsolidity/semanticTests/various/senders_balance.sol @@ -15,6 +15,8 @@ contract D { } } +// ==== +// compileViaYul: also // ---- // constructor(), 27 wei -> // f() -> 27 diff --git a/test/libsolidity/semanticTests/various/staticcall_for_view_and_pure.sol b/test/libsolidity/semanticTests/various/staticcall_for_view_and_pure.sol index 96c5419bea6b..c2f045082fab 100644 --- a/test/libsolidity/semanticTests/various/staticcall_for_view_and_pure.sol +++ b/test/libsolidity/semanticTests/various/staticcall_for_view_and_pure.sol @@ -32,6 +32,7 @@ contract D { } } // ==== +// compileViaYul: also // EVMVersion: >=byzantium // ---- // f() -> 0x1 # This should work, next should throw # diff --git a/test/libsolidity/semanticTests/various/write_storage_external.sol b/test/libsolidity/semanticTests/various/write_storage_external.sol index 0bbe522484da..7ed94b6f8cfe 100644 --- a/test/libsolidity/semanticTests/various/write_storage_external.sol +++ b/test/libsolidity/semanticTests/various/write_storage_external.sol @@ -34,6 +34,8 @@ contract D { } } +// ==== +// compileViaYul: also // ---- // f() -> 3 // g() -> 8 From 4327434d079c0d76a501d4050e365afcbfbb329e Mon Sep 17 00:00:00 2001 From: yoni206 Date: Tue, 28 Apr 2020 09:37:08 -0700 Subject: [PATCH 119/126] Adding bit-vector NOT operation to the opcodes. --- test/formal/opcodes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/formal/opcodes.py b/test/formal/opcodes.py index e65ecef29d37..589bd9520bb5 100644 --- a/test/formal/opcodes.py +++ b/test/formal/opcodes.py @@ -45,6 +45,9 @@ def AND(x, y): def OR(x, y): return x | y +def NOT(x): + return ~(x) + def SHL(x, y): return y << x From 8973732b741e1a3d16e0f7c883aa1c995c0d2909 Mon Sep 17 00:00:00 2001 From: Jason Cobb Date: Mon, 27 Apr 2020 15:01:01 -0400 Subject: [PATCH 120/126] Compare categories instead of types in ContractCompiler(740) --- libsolidity/codegen/ContractCompiler.cpp | 5 ++++- .../constants/asm_address_constant_regression.sol | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 test/libsolidity/semanticTests/constants/asm_address_constant_regression.sol diff --git a/libsolidity/codegen/ContractCompiler.cpp b/libsolidity/codegen/ContractCompiler.cpp index 18de2ba840e1..d095c1eac6f8 100644 --- a/libsolidity/codegen/ContractCompiler.cpp +++ b/libsolidity/codegen/ContractCompiler.cpp @@ -733,7 +733,10 @@ bool ContractCompiler::visit(InlineAssembly const& _inlineAssembly) { case Type::Category::Bool: case Type::Category::Address: - solAssert(*type == *variable->annotation().type, ""); + // Either both the literal and the variable are bools, or they are both addresses. + // If they are both bools, comparing category is the same as comparing the types. + // If they are both addresses, compare category so that payable/nonpayable is not compared. + solAssert(type->category() == variable->annotation().type->category(), ""); value = type->literalValue(literal); break; case Type::Category::StringLiteral: diff --git a/test/libsolidity/semanticTests/constants/asm_address_constant_regression.sol b/test/libsolidity/semanticTests/constants/asm_address_constant_regression.sol new file mode 100644 index 000000000000..730b04746250 --- /dev/null +++ b/test/libsolidity/semanticTests/constants/asm_address_constant_regression.sol @@ -0,0 +1,12 @@ +// Test for regression of https://github.com/ethereum/solidity/issues/8406 + +contract C { + address constant e = 0x1212121212121212121212121000002134593163; + + function f() public returns (byte z) { + assembly { z := e } + } +} + +// ---- +// f() -> 0x00 From 3bd15655cb1d11a82ee6e3db95bf02827f2e6eac Mon Sep 17 00:00:00 2001 From: a3d4 Date: Tue, 28 Apr 2020 17:15:55 +0200 Subject: [PATCH 121/126] Type Checker: Fix internal error when applying unary operators to tuples with empty components --- Changelog.md | 1 + libsolidity/analysis/TypeChecker.cpp | 14 ++++++------- .../declaration_bitnot_tuple.sol | 9 +++++++++ .../declaration_dec_tuple.sol | 9 +++++++++ .../declaration_delete_tuple.sol | 10 ++++++++++ .../declaration_inc_tuple.sol | 9 +++++++++ .../declaration_unary_tuple.sol | 20 +++++++++++++++++++ 7 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_bitnot_tuple.sol create mode 100644 test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_dec_tuple.sol create mode 100644 test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_delete_tuple.sol create mode 100644 test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_inc_tuple.sol create mode 100644 test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_unary_tuple.sol diff --git a/Changelog.md b/Changelog.md index ec45f9da7df4..a901e8606937 100644 --- a/Changelog.md +++ b/Changelog.md @@ -14,6 +14,7 @@ Bugfixes: * Type Checker: Disallow ``virtual`` and ``override`` for constructors. * Type Checker: Fix several internal errors by performing size and recursiveness checks of types before the full type checking. * Type Checker: Fix internal error when assigning to empty tuples. + * Type Checker: Fix internal error when applying unary operators to tuples with empty components. * Type Checker: Perform recursiveness check on structs declared at the file level. * Standard Json Input: Fix error when using prefix ``file://`` in the field ``urls``. diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index a0155c9ceea4..b4030c0c4e18 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -1478,13 +1478,13 @@ bool TypeChecker::visit(UnaryOperation const& _operation) TypePointer t = type(_operation.subExpression())->unaryOperatorResult(op); if (!t) { - m_errorReporter.typeError( - _operation.location(), - "Unary operator " + - string(TokenTraits::toString(op)) + - " cannot be applied to type " + - subExprType->toString() - ); + string description = "Unary operator " + string(TokenTraits::toString(op)) + " cannot be applied to type " + subExprType->toString(); + if (modifying) + // Cannot just report the error, ignore the unary operator, and continue, + // because the sub-expression was already processed with requireLValue() + m_errorReporter.fatalTypeError(_operation.location(), description); + else + m_errorReporter.typeError(_operation.location(), description); t = subExprType; } _operation.annotation().type = t; diff --git a/test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_bitnot_tuple.sol b/test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_bitnot_tuple.sol new file mode 100644 index 000000000000..56877ecdc95b --- /dev/null +++ b/test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_bitnot_tuple.sol @@ -0,0 +1,9 @@ +contract C +{ + function f() public + { + int x = ~(0,); + } +} +// ---- +// TypeError: (60-64): Tuple component cannot be empty. diff --git a/test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_dec_tuple.sol b/test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_dec_tuple.sol new file mode 100644 index 000000000000..1612006424e0 --- /dev/null +++ b/test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_dec_tuple.sol @@ -0,0 +1,9 @@ +contract C +{ + function f() public + { + int x = --(,); + } +} +// ---- +// TypeError: (59-64): Unary operator -- cannot be applied to type tuple(,) diff --git a/test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_delete_tuple.sol b/test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_delete_tuple.sol new file mode 100644 index 000000000000..0cdcee5cc3c3 --- /dev/null +++ b/test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_delete_tuple.sol @@ -0,0 +1,10 @@ +contract C +{ + function f() public + { + int x = delete (,0); + } +} +// ---- +// TypeError: (68-69): Expression has to be an lvalue. +// TypeError: (59-70): Unary operator delete cannot be applied to type tuple(,int_const 0) diff --git a/test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_inc_tuple.sol b/test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_inc_tuple.sol new file mode 100644 index 000000000000..fe5d5e5ce298 --- /dev/null +++ b/test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_inc_tuple.sol @@ -0,0 +1,9 @@ +contract C +{ + function f() public + { + (int x) = ++(,); + } +} +// ---- +// TypeError: (61-66): Unary operator ++ cannot be applied to type tuple(,) diff --git a/test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_unary_tuple.sol b/test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_unary_tuple.sol new file mode 100644 index 000000000000..6f40f1ed1cb8 --- /dev/null +++ b/test/libsolidity/syntaxTests/iceRegressionTests/declarationUnaryTuple/declaration_unary_tuple.sol @@ -0,0 +1,20 @@ +contract C +{ + function f() public + { + int x = +(0, 0); + int y = -(0, 0); + (int z) = ~(0, 0); + (int t) = !(0, 0); + } +} +// ---- +// SyntaxError: (59-66): Use of unary + is disallowed. +// TypeError: (59-66): Unary operator + cannot be applied to type tuple(int_const 0,int_const 0) +// TypeError: (51-66): Different number of components on the left hand side (1) than on the right hand side (2). +// TypeError: (84-91): Unary operator - cannot be applied to type tuple(int_const 0,int_const 0) +// TypeError: (76-91): Different number of components on the left hand side (1) than on the right hand side (2). +// TypeError: (111-118): Unary operator ~ cannot be applied to type tuple(int_const 0,int_const 0) +// TypeError: (101-118): Different number of components on the left hand side (1) than on the right hand side (2). +// TypeError: (138-145): Unary operator ! cannot be applied to type tuple(int_const 0,int_const 0) +// TypeError: (128-145): Different number of components on the left hand side (1) than on the right hand side (2). From d1fd6782bc3796ae16a7f58f4698d9d597ca9b1e Mon Sep 17 00:00:00 2001 From: hrkrshnn Date: Thu, 30 Apr 2020 19:41:31 +0530 Subject: [PATCH 122/126] Note on abstract contract about overriding implemented functions --- docs/contracts/abstract-contracts.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/contracts/abstract-contracts.rst b/docs/contracts/abstract-contracts.rst index 7b9f8325613b..4571f45b2c85 100644 --- a/docs/contracts/abstract-contracts.rst +++ b/docs/contracts/abstract-contracts.rst @@ -52,3 +52,8 @@ facilitating patterns like the `Template method Date: Fri, 1 May 2020 00:14:45 +0800 Subject: [PATCH 123/126] fix(docs/mapping-types): fix wrong demo code. get index before push, so the access of this index won't beyond the boundery. --- docs/types/mapping-types.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/types/mapping-types.rst b/docs/types/mapping-types.rst index f69d25b0f6ee..d84da8ec5a4f 100644 --- a/docs/types/mapping-types.rst +++ b/docs/types/mapping-types.rst @@ -138,8 +138,8 @@ the ``sum`` function iterates over to sum all the values. if (keyIndex > 0) return true; else { - self.keys.push(); keyIndex = self.keys.length; + self.keys.push(); self.data[key].keyIndex = keyIndex + 1; self.keys[keyIndex].key = key; self.size++; From a9f4d1401059146a70ab04f087c2f3b5aa84e228 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Thu, 30 Apr 2020 10:17:29 -0500 Subject: [PATCH 124/126] [Sol - Yul] Add support for send(..) & transfer(..) --- .../codegen/ir/IRGeneratorForStatements.cpp | 38 +++++++++++++++++++ test/libsolidity/SolidityEndToEndTest.cpp | 37 ++++++++++-------- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index 252ddffa700e..755110c27b0a 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -730,6 +730,16 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall) break; } + case FunctionType::Kind::Revert: + { + solAssert(arguments.size() == parameterTypes.size(), ""); + if (arguments.empty()) + m_code << "revert(0, 0)\n"; + else + solUnimplementedAssert(false, ""); + + break; + } // Array creation using new case FunctionType::Kind::ObjectCreation: { @@ -908,6 +918,34 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall) break; } + case FunctionType::Kind::Send: + case FunctionType::Kind::Transfer: + { + solAssert(arguments.size() == 1 && parameterTypes.size() == 1, ""); + string address{IRVariable(_functionCall.expression()).part("address").name()}; + string value{expressionAsType(*arguments[0], *(parameterTypes[0]))}; + Whiskers templ(R"( + let := 0 + if iszero() { := } + let := call(,
, , 0, 0, 0, 0) + + if iszero() { () } + + )"); + templ("gas", m_context.newYulVariable()); + templ("callStipend", toString(evmasm::GasCosts::callStipend)); + templ("address", address); + templ("value", value); + if (functionType->kind() == FunctionType::Kind::Transfer) + templ("success", m_context.newYulVariable()); + else + templ("success", IRVariable(_functionCall).commaSeparatedList()); + templ("isTransfer", functionType->kind() == FunctionType::Kind::Transfer); + templ("forwardingRevert", m_utils.forwardingRevertFunction()); + m_code << templ.render(); + + break; + } default: solUnimplemented("FunctionKind " + toString(static_cast(functionType->kind())) + " not yet implemented"); } diff --git a/test/libsolidity/SolidityEndToEndTest.cpp b/test/libsolidity/SolidityEndToEndTest.cpp index 153d6a8f428b..83e7ef95da2c 100644 --- a/test/libsolidity/SolidityEndToEndTest.cpp +++ b/test/libsolidity/SolidityEndToEndTest.cpp @@ -50,6 +50,7 @@ using namespace solidity::langutil; #define ALSO_VIA_YUL(CODE) \ { \ { CODE } \ + reset(); \ m_compileViaYul = true; \ { CODE } \ } @@ -1058,11 +1059,13 @@ BOOST_AUTO_TEST_CASE(send_ether) } } )"; - u256 amount(130); - compileAndRun(sourceCode, amount + 1); - u160 address(23); - ABI_CHECK(callContractFunction("a(address,uint256)", address, amount), encodeArgs(1)); - BOOST_CHECK_EQUAL(balanceAt(address), amount); + ALSO_VIA_YUL( + u256 amount(250); + compileAndRun(sourceCode, amount + 1); + u160 address(23); + ABI_CHECK(callContractFunction("a(address,uint256)", address, amount), encodeArgs(1)); + BOOST_CHECK_EQUAL(balanceAt(address), amount); + ) } BOOST_AUTO_TEST_CASE(transfer_ether) @@ -1088,17 +1091,19 @@ BOOST_AUTO_TEST_CASE(transfer_ether) } } )"; - compileAndRun(sourceCode, 0, "B"); - u160 const nonPayableRecipient = m_contractAddress; - compileAndRun(sourceCode, 0, "C"); - u160 const oogRecipient = m_contractAddress; - compileAndRun(sourceCode, 20, "A"); - u160 payableRecipient(23); - ABI_CHECK(callContractFunction("a(address,uint256)", payableRecipient, 10), encodeArgs(10)); - BOOST_CHECK_EQUAL(balanceAt(payableRecipient), 10); - BOOST_CHECK_EQUAL(balanceAt(m_contractAddress), 10); - ABI_CHECK(callContractFunction("b(address,uint256)", nonPayableRecipient, 10), encodeArgs()); - ABI_CHECK(callContractFunction("b(address,uint256)", oogRecipient, 10), encodeArgs()); + ALSO_VIA_YUL( + compileAndRun(sourceCode, 0, "B"); + u160 const nonPayableRecipient = m_contractAddress; + compileAndRun(sourceCode, 0, "C"); + u160 const oogRecipient = m_contractAddress; + compileAndRun(sourceCode, 20, "A"); + u160 payableRecipient(23); + ABI_CHECK(callContractFunction("a(address,uint256)", payableRecipient, 10), encodeArgs(10)); + BOOST_CHECK_EQUAL(balanceAt(payableRecipient), 10); + BOOST_CHECK_EQUAL(balanceAt(m_contractAddress), 10); + ABI_CHECK(callContractFunction("b(address,uint256)", nonPayableRecipient, 10), encodeArgs()); + ABI_CHECK(callContractFunction("b(address,uint256)", oogRecipient, 10), encodeArgs()); + ) } BOOST_AUTO_TEST_CASE(uncalled_blockhash) From d7180c052269bb652691bc27f5bb6aabe8ceec80 Mon Sep 17 00:00:00 2001 From: chriseth Date: Mon, 4 May 2020 12:57:27 +0200 Subject: [PATCH 125/126] Prepare 0.6.7. --- Changelog.md | 4 ++-- docs/bugs_by_version.json | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Changelog.md b/Changelog.md index a901e8606937..13c506f7f29c 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,4 +1,4 @@ -### 0.6.7 (unreleased) +### 0.6.7 (2020-05-04) Language Features: * Add support for EIP 165 interface identifiers with `type(I).interfaceId`. @@ -7,6 +7,7 @@ Language Features: Compiler Features: * Optimizer: Simplify repeated AND and OR operations. + * Standard Json Input: Support the prefix ``file://`` in the field ``urls``. Bugfixes: * SMTChecker: Fix internal error when fixed points are used. @@ -16,7 +17,6 @@ Bugfixes: * Type Checker: Fix internal error when assigning to empty tuples. * Type Checker: Fix internal error when applying unary operators to tuples with empty components. * Type Checker: Perform recursiveness check on structs declared at the file level. - * Standard Json Input: Fix error when using prefix ``file://`` in the field ``urls``. Build System: * soltest.sh: ``SOLIDITY_BUILD_DIR`` is no longer relative to ``REPO_ROOT`` to allow for build directories outside of the source tree. diff --git a/docs/bugs_by_version.json b/docs/bugs_by_version.json index 7ccd0c66ab2c..4c27e90d95a6 100644 --- a/docs/bugs_by_version.json +++ b/docs/bugs_by_version.json @@ -1085,5 +1085,9 @@ "0.6.6": { "bugs": [], "released": "2020-04-09" + }, + "0.6.7": { + "bugs": [], + "released": "2020-05-04" } } \ No newline at end of file From 682f53fc82bfa2fdcfbdba12c7940a673a0461f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Mon, 4 May 2020 12:47:29 +0200 Subject: [PATCH 126/126] Changelog: Add an entry for --yul-optimizations option --- Changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Changelog.md b/Changelog.md index 13c506f7f29c..21d45156c13b 100644 --- a/Changelog.md +++ b/Changelog.md @@ -8,6 +8,7 @@ Language Features: Compiler Features: * Optimizer: Simplify repeated AND and OR operations. * Standard Json Input: Support the prefix ``file://`` in the field ``urls``. + * Add option to specify optimization steps to be performed by Yul optimizer with `--yul-optimizations` in the commandline interface or `optimizer.details.yulDetails.optimizerSteps` in standard-json. Bugfixes: * SMTChecker: Fix internal error when fixed points are used.