From e8bb212ec67fdbb87b47e0ac858bfb69a666d7aa Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Fri, 14 Feb 2020 21:59:06 +0100 Subject: [PATCH 01/92] Add hour and minute to bytecode repo directory names --- appveyor.yml | 2 +- scripts/bytecodecompare/storebytecode.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 4b66e027e897..d7211e14b4b1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -64,7 +64,7 @@ build_script: - msbuild solidity.sln /p:Configuration=%CONFIGURATION% /m:%NUMBER_OF_PROCESSORS% /v:minimal - cd %APPVEYOR_BUILD_FOLDER% - scripts\release.bat %CONFIGURATION% 2017 - - ps: $bytecodedir = git show -s --format="%cd-%H" --date=short + - ps: $bytecodedir = git show -s --format="%cd-%H" --date="format:%Y-%m-%d-%H-%M" test_script: - cd %APPVEYOR_BUILD_FOLDER%\build\test\%CONFIGURATION% diff --git a/scripts/bytecodecompare/storebytecode.sh b/scripts/bytecodecompare/storebytecode.sh index 451490e0c66f..f2b3a38972db 100755 --- a/scripts/bytecodecompare/storebytecode.sh +++ b/scripts/bytecodecompare/storebytecode.sh @@ -126,7 +126,7 @@ EOF git config user.email "chris@ethereum.org" git clean -f -d -x - DIRNAME=$(cd "$REPO_ROOT" && git show -s --format="%cd-%H" --date=short) + DIRNAME=$(cd "$REPO_ROOT" && git show -s --format="%cd-%H" --date="format:%Y-%m-%d-%H-%M") mkdir -p "$DIRNAME" REPORT="$DIRNAME/$ZIP_SUFFIX.txt" cp ../report.txt "$REPORT" From 9641d93167105d1a3bc26c51c72e49e5392e6616 Mon Sep 17 00:00:00 2001 From: Chris Chinchilla Date: Thu, 5 Dec 2019 16:58:03 +0100 Subject: [PATCH 02/92] Clarify array copying semantics --- docs/control-structures.rst | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/docs/control-structures.rst b/docs/control-structures.rst index 248641e3414e..407d2523c188 100644 --- a/docs/control-structures.rst +++ b/docs/control-structures.rst @@ -343,18 +343,8 @@ i.e. the following is not valid: ``(x, uint y) = (1, 2);`` Complications for Arrays and Structs ------------------------------------ -The semantics of assignments are a bit more complicated for -non-value types like arrays and structs. -Assigning *to* a state variable always creates an independent -copy. On the other hand, assigning to a local variable creates -an independent copy only for elementary types, i.e. static -types that fit into 32 bytes. If structs or arrays (including -``bytes`` and ``string``) are assigned from a state variable -to a local variable, the local variable holds a reference to -the original state variable. A second assignment to the local -variable does not modify the state but only changes the -reference. Assignments to members (or elements) of the local -variable *do* change the state. +The semantics of assignments are more complicated for non-value types like arrays and structs, +including ``bytes`` and ``string``, see :ref:`Data location and assignment behaviour ` for details. In the example below the call to ``g(x)`` has no effect on ``x`` because it creates an independent copy of the storage value in memory. However, ``h(x)`` successfully modifies ``x`` From 9dd9a68c08f3d6aa04388ad797a807aaa30b0dff Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 18 Feb 2020 16:42:02 +0100 Subject: [PATCH 03/92] Set version to 0.6.4. --- CMakeLists.txt | 2 +- Changelog.md | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ab3e374180f..4278770f1dee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ include(EthPolicy) eth_policy() # project name and version should be set after cmake_policy CMP0048 -set(PROJECT_VERSION "0.6.3") +set(PROJECT_VERSION "0.6.4") project(solidity VERSION ${PROJECT_VERSION} LANGUAGES C CXX) include(TestBigEndian) diff --git a/Changelog.md b/Changelog.md index f00d555e958e..c83b26e830e2 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,15 @@ +### 0.6.4 (unreleased) + +Language Features: + + +Compiler Features: + + +Bugfixes: + + + ### 0.6.3 (2020-02-18) Language Features: @@ -6,7 +18,6 @@ Language Features: * Report source locations for structured documentation errors. - Compiler Features: * AST: Add a new node for doxygen-style, structured documentation that can be received by contract, function, event and modifier definitions. * Code Generator: Use ``calldatacopy`` instead of ``codecopy`` to zero out memory past input. From b3f595ce9d8b83438602b04cd30b314914a58c9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 6 Feb 2020 07:10:59 +0100 Subject: [PATCH 04/92] [yul-phaser] Population: Remove unused include for --- tools/yulPhaser/Population.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index a28db27a4582..43158ac431dc 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -21,7 +21,6 @@ #include #include -#include #include using namespace std; From 71dcbf9df53c3073a1201fe5b0e8a113b59b191b Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 14:25:27 +0100 Subject: [PATCH 05/92] [yul-phaser] Population: Remove ambiguous default from one of the constructors --- tools/yulPhaser/Population.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 276db83ffcd8..983e40dd88e4 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -67,7 +67,7 @@ class Population friend std::ostream& operator<<(std::ostream& _stream, Population const& _population); private: - explicit Population(Program _program, std::vector _individuals = {}): + explicit Population(Program _program, std::vector _individuals): m_program{std::move(_program)}, m_individuals{std::move(_individuals)} {} From 837ea96da72ec0fd0caf7e43518269e4fab1117c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 15 Feb 2020 00:09:43 +0100 Subject: [PATCH 06/92] [yul-phaser] Move stripWhitespace() from Program tests to Common --- test/yulPhaser/Common.cpp | 8 ++++++++ test/yulPhaser/Common.h | 5 +++++ test/yulPhaser/CommonTest.cpp | 14 ++++++++++++++ test/yulPhaser/Program.cpp | 9 ++------- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/test/yulPhaser/Common.cpp b/test/yulPhaser/Common.cpp index 3cef4ff2bc85..c3653eaec804 100644 --- a/test/yulPhaser/Common.cpp +++ b/test/yulPhaser/Common.cpp @@ -19,6 +19,8 @@ #include +#include + using namespace std; using namespace solidity; using namespace solidity::yul; @@ -32,3 +34,9 @@ map phaser::test::enumerateOptmisationSteps() return stepIndices; } + +string phaser::test::stripWhitespace(string const& input) +{ + regex whitespaceRegex("\\s+"); + return regex_replace(input, whitespaceRegex, ""); +} diff --git a/test/yulPhaser/Common.h b/test/yulPhaser/Common.h index a3dd24bce4e5..3dcf63288423 100644 --- a/test/yulPhaser/Common.h +++ b/test/yulPhaser/Common.h @@ -42,6 +42,11 @@ namespace solidity::phaser::test /// integers. std::map enumerateOptmisationSteps(); +// STRING UTILITIES + +/// Returns the input string with all the whitespace characters (spaces, line endings, etc.) removed. +std::string stripWhitespace(std::string const& input); + // STATISTICAL UTILITIES /// Calculates the mean value of a series of samples given in a vector. diff --git a/test/yulPhaser/CommonTest.cpp b/test/yulPhaser/CommonTest.cpp index d7268c669d72..cdcba6f5341f 100644 --- a/test/yulPhaser/CommonTest.cpp +++ b/test/yulPhaser/CommonTest.cpp @@ -51,6 +51,20 @@ BOOST_AUTO_TEST_CASE(enumerateOptimisationSteps_should_assing_indices_to_all_ava } } +BOOST_AUTO_TEST_CASE(stripWhitespace_should_remove_all_whitespace_characters_from_a_string) +{ + BOOST_TEST(stripWhitespace("") == ""); + BOOST_TEST(stripWhitespace(" ") == ""); + BOOST_TEST(stripWhitespace(" \n\t\v ") == ""); + + BOOST_TEST(stripWhitespace("abc") == "abc"); + BOOST_TEST(stripWhitespace(" abc") == "abc"); + BOOST_TEST(stripWhitespace("abc ") == "abc"); + BOOST_TEST(stripWhitespace(" a b c ") == "abc"); + BOOST_TEST(stripWhitespace(" a b\tc\n") == "abc"); + BOOST_TEST(stripWhitespace(" a b \n\n c \n\t\v") == "abc"); +} + BOOST_AUTO_TEST_CASE(mean_should_calculate_statistical_mean) { BOOST_TEST(mean({0}) == 0.0); diff --git a/test/yulPhaser/Program.cpp b/test/yulPhaser/Program.cpp index 44c2bfa3c6e5..7d9c81cd3427 100644 --- a/test/yulPhaser/Program.cpp +++ b/test/yulPhaser/Program.cpp @@ -15,6 +15,8 @@ along with solidity. If not, see . */ +#include + #include #include @@ -30,7 +32,6 @@ #include #include -#include #include using namespace std; @@ -51,12 +52,6 @@ namespace else return _block; } - - string stripWhitespace(string const& input) - { - regex whitespaceRegex("\\s+"); - return regex_replace(input, whitespaceRegex, ""); - } } namespace solidity::phaser::test From 38f79a17618037af6b0248f48ec928e6e0a2b22a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 14 Feb 2020 19:41:43 +0100 Subject: [PATCH 07/92] [yul-phaser] Common: Add chromosomeLengths() --- test/yulPhaser/Common.cpp | 9 +++++++++ test/yulPhaser/Common.h | 6 ++++++ test/yulPhaser/CommonTest.cpp | 15 +++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/test/yulPhaser/Common.cpp b/test/yulPhaser/Common.cpp index c3653eaec804..38c6e445b995 100644 --- a/test/yulPhaser/Common.cpp +++ b/test/yulPhaser/Common.cpp @@ -25,6 +25,15 @@ using namespace std; using namespace solidity; using namespace solidity::yul; +vector phaser::test::chromosomeLengths(Population const& _population) +{ + vector lengths; + for (auto const& individual: _population.individuals()) + lengths.push_back(individual.chromosome.length()); + + return lengths; +} + map phaser::test::enumerateOptmisationSteps() { map stepIndices; diff --git a/test/yulPhaser/Common.h b/test/yulPhaser/Common.h index 3dcf63288423..c4da6c99c933 100644 --- a/test/yulPhaser/Common.h +++ b/test/yulPhaser/Common.h @@ -28,6 +28,8 @@ #pragma once +#include + #include #include #include @@ -37,6 +39,10 @@ namespace solidity::phaser::test { // CHROMOSOME AND POPULATION HELPERS + +/// Returns a vector containing lengths of all chromosomes in the population (in the same order). +std::vector chromosomeLengths(Population const& _population); + /// Assigns indices from 0 to N to all optimisation steps available in the OptimiserSuite. /// This is a convenience helper to make it easier to test their distribution with tools made for /// integers. diff --git a/test/yulPhaser/CommonTest.cpp b/test/yulPhaser/CommonTest.cpp index cdcba6f5341f..be686446f1e2 100644 --- a/test/yulPhaser/CommonTest.cpp +++ b/test/yulPhaser/CommonTest.cpp @@ -19,11 +19,14 @@ #include +#include + #include #include using namespace std; +using namespace solidity::langutil; using namespace solidity::yul; using namespace boost::test_tools; @@ -33,6 +36,18 @@ namespace solidity::phaser::test BOOST_AUTO_TEST_SUITE(Phaser) BOOST_AUTO_TEST_SUITE(CommonTest) +BOOST_AUTO_TEST_CASE(chromosomeLengths_should_return_lengths_of_all_chromosomes_in_a_population) +{ + CharStream sourceStream("{}", ""); + auto program = Program::load(sourceStream); + + Population population1(program, {Chromosome(), Chromosome("a"), Chromosome("aa"), Chromosome("aaa")}); + BOOST_TEST((chromosomeLengths(population1) == vector{0, 1, 2, 3})); + + Population population2(program); + BOOST_TEST((chromosomeLengths(population2) == vector{})); +} + BOOST_AUTO_TEST_CASE(enumerateOptimisationSteps_should_assing_indices_to_all_available_optimisation_steps) { map stepsAndAbbreviations = OptimiserSuite::stepNameToAbbreviationMap(); From d22c59aa0e6c3e8dfb46e27802111a61cac8478e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 8 Feb 2020 01:51:04 +0100 Subject: [PATCH 08/92] [yul-phaser] Chromosome: Add a constructor that reads steps from an abbreviation string --- test/yulPhaser/Chromosome.cpp | 30 +++++++++++++++++++++++++++++- test/yulPhaser/Population.cpp | 10 +++++----- tools/yulPhaser/Chromosome.cpp | 6 ++++++ tools/yulPhaser/Chromosome.h | 1 + 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/test/yulPhaser/Chromosome.cpp b/test/yulPhaser/Chromosome.cpp index 13f2f6811175..5865c501d635 100644 --- a/test/yulPhaser/Chromosome.cpp +++ b/test/yulPhaser/Chromosome.cpp @@ -21,8 +21,18 @@ #include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include #include #include @@ -39,6 +49,24 @@ namespace solidity::phaser::test BOOST_AUTO_TEST_SUITE(Phaser) BOOST_AUTO_TEST_SUITE(ChromosomeTest) +BOOST_AUTO_TEST_CASE(constructor_should_convert_from_string_to_optimisation_steps) +{ + vector expectedSteps{ + ConditionalSimplifier::name, + FunctionHoister::name, + RedundantAssignEliminator::name, + ForLoopConditionOutOfBody::name, + Rematerialiser::name, + ForLoopConditionOutOfBody::name, + ExpressionSimplifier::name, + ForLoopInitRewriter::name, + LoopInvariantCodeMotion::name, + ExpressionInliner::name + }; + + BOOST_TEST(Chromosome("ChrOmOsoMe").optimisationSteps() == expectedSteps); +} + BOOST_AUTO_TEST_CASE(makeRandom_should_create_chromosome_with_random_optimisation_steps) { constexpr uint32_t numSteps = 1000; diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index d2c11a886c72..17907333dae5 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -142,11 +142,11 @@ BOOST_AUTO_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse) stringstream output; CharStream sourceStream(sampleSourceCode, current_test_case().p_name); vector chromosomes = { - Chromosome({StructuralSimplifier::name}), - Chromosome({BlockFlattener::name}), - Chromosome({SSAReverser::name}), - Chromosome({UnusedPruner::name}), - Chromosome({StructuralSimplifier::name, BlockFlattener::name}), + Chromosome(vector{StructuralSimplifier::name}), + Chromosome(vector{BlockFlattener::name}), + Chromosome(vector{SSAReverser::name}), + Chromosome(vector{UnusedPruner::name}), + Chromosome(vector{StructuralSimplifier::name, BlockFlattener::name}), }; auto program = Program::load(sourceStream); Population population(program, chromosomes); diff --git a/tools/yulPhaser/Chromosome.cpp b/tools/yulPhaser/Chromosome.cpp index ae30a87e9582..832c2cf804b8 100644 --- a/tools/yulPhaser/Chromosome.cpp +++ b/tools/yulPhaser/Chromosome.cpp @@ -36,6 +36,12 @@ ostream& operator<<(ostream& _stream, Chromosome const& _chromosome); } +Chromosome::Chromosome(string const& _optimisationSteps) +{ + for (char abbreviation: _optimisationSteps) + m_optimisationSteps.push_back(OptimiserSuite::stepAbbreviationToNameMap().at(abbreviation)); +} + Chromosome Chromosome::makeRandom(size_t _length) { vector steps; diff --git a/tools/yulPhaser/Chromosome.h b/tools/yulPhaser/Chromosome.h index bf91dbd305fc..7d8f1b912dfa 100644 --- a/tools/yulPhaser/Chromosome.h +++ b/tools/yulPhaser/Chromosome.h @@ -42,6 +42,7 @@ class Chromosome Chromosome() = default; explicit Chromosome(std::vector _optimisationSteps): m_optimisationSteps(std::move(_optimisationSteps)) {} + explicit Chromosome(std::string const& _optimisationSteps); static Chromosome makeRandom(size_t _length); size_t length() const { return m_optimisationSteps.size(); } From e771f00971bf61a170be677c0f46b9dc78933dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 13 Feb 2020 22:25:25 +0100 Subject: [PATCH 09/92] [yul-phaser] Population: Extract Program construction in tests into a fixture --- test/yulPhaser/Population.cpp | 90 ++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 43 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index 17907333dae5..15297f7d72e0 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -53,40 +53,50 @@ namespace } } +class PopulationFixture +{ +protected: + PopulationFixture(): + m_sourceStream(SampleSourceCode, ""), + m_program(Program::load(m_sourceStream)) {} + + static constexpr char SampleSourceCode[] = + "{\n" + " let factor := 13\n" + " {\n" + " if factor\n" + " {\n" + " let variable := add(1, 2)\n" + " }\n" + " let result := factor\n" + " }\n" + " let something := 6\n" + " {\n" + " {\n" + " {\n" + " let value := 15\n" + " }\n" + " }\n" + " }\n" + " let something_else := mul(mul(something, 1), add(factor, 0))\n" + " if 1 { let x := 1 }\n" + " if 0 { let y := 2 }\n" + "}\n"; + + CharStream m_sourceStream; + Program m_program; +}; + BOOST_AUTO_TEST_SUITE(Phaser) BOOST_AUTO_TEST_SUITE(PopulationTest) -string const& sampleSourceCode = - "{\n" - " let factor := 13\n" - " {\n" - " if factor\n" - " {\n" - " let variable := add(1, 2)\n" - " }\n" - " let result := factor\n" - " }\n" - " let something := 6\n" - " {\n" - " {\n" - " {\n" - " let value := 15\n" - " }\n" - " }\n" - " }\n" - " let something_else := mul(mul(something, 1), add(factor, 0))\n" - " if 1 { let x := 1 }\n" - " if 0 { let y := 2 }\n" - "}\n"; - -BOOST_AUTO_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitness) +BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitness, PopulationFixture) { - CharStream sourceStream(sampleSourceCode, current_test_case().p_name); vector chromosomes = { Chromosome::makeRandom(5), Chromosome::makeRandom(10), }; - Population population(Program::load(sourceStream), chromosomes); + Population population(m_program, chromosomes); BOOST_TEST(population.individuals().size() == 2); BOOST_TEST(population.individuals()[0].chromosome == chromosomes[0]); @@ -96,12 +106,10 @@ BOOST_AUTO_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitness BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); } -BOOST_AUTO_TEST_CASE(makeRandom_should_return_population_with_random_chromosomes) +BOOST_FIXTURE_TEST_CASE(makeRandom_should_return_population_with_random_chromosomes, PopulationFixture) { - CharStream sourceStream(sampleSourceCode, current_test_case().p_name); - auto program = Program::load(sourceStream); - auto population1 = Population::makeRandom(program, 100); - auto population2 = Population::makeRandom(program, 100); + auto population1 = Population::makeRandom(m_program, 100); + auto population2 = Population::makeRandom(m_program, 100); BOOST_TEST(population1.individuals().size() == 100); BOOST_TEST(population2.individuals().size() == 100); @@ -117,19 +125,17 @@ BOOST_AUTO_TEST_CASE(makeRandom_should_return_population_with_random_chromosomes BOOST_TEST(numMatchingPositions < 10); } -BOOST_AUTO_TEST_CASE(makeRandom_should_not_compute_fitness) +BOOST_FIXTURE_TEST_CASE(makeRandom_should_not_compute_fitness, PopulationFixture) { - CharStream sourceStream(sampleSourceCode, current_test_case().p_name); - auto population = Population::makeRandom(Program::load(sourceStream), 5); + auto population = Population::makeRandom(m_program, 5); BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); } -BOOST_AUTO_TEST_CASE(run_should_evaluate_fitness) +BOOST_FIXTURE_TEST_CASE(run_should_evaluate_fitness, PopulationFixture) { stringstream output; - CharStream sourceStream(sampleSourceCode, current_test_case().p_name); - auto population = Population::makeRandom(Program::load(sourceStream), 5); + auto population = Population::makeRandom(m_program, 5); assert(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); population.run(1, output); @@ -137,10 +143,9 @@ BOOST_AUTO_TEST_CASE(run_should_evaluate_fitness) BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessSet)); } -BOOST_AUTO_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse) +BOOST_FIXTURE_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse, PopulationFixture) { stringstream output; - CharStream sourceStream(sampleSourceCode, current_test_case().p_name); vector chromosomes = { Chromosome(vector{StructuralSimplifier::name}), Chromosome(vector{BlockFlattener::name}), @@ -148,12 +153,11 @@ BOOST_AUTO_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse) Chromosome(vector{UnusedPruner::name}), Chromosome(vector{StructuralSimplifier::name, BlockFlattener::name}), }; - auto program = Program::load(sourceStream); - Population population(program, chromosomes); + Population population(m_program, chromosomes); size_t initialTopFitness[2] = { - Population::measureFitness(chromosomes[0], program), - Population::measureFitness(chromosomes[1], program), + Population::measureFitness(chromosomes[0], m_program), + Population::measureFitness(chromosomes[1], m_program), }; for (int i = 0; i < 6; ++i) From 806891f494ed447d71fc781923b4c98fc6640f61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 6 Feb 2020 06:19:55 +0100 Subject: [PATCH 10/92] [yul-phaser] Population: Customizable chromosome length in makeRandom() --- test/yulPhaser/Population.cpp | 59 +++++++++++++++++++++++++++++++--- tools/yulPhaser/Population.cpp | 24 ++++++++++++-- tools/yulPhaser/Population.h | 16 +++++++-- tools/yulPhaser/main.cpp | 7 +++- 4 files changed, 96 insertions(+), 10 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index 15297f7d72e0..15bb48d0481a 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -15,6 +15,8 @@ along with solidity. If not, see . */ +#include + #include #include #include @@ -106,10 +108,59 @@ BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitn BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); } +BOOST_FIXTURE_TEST_CASE(makeRandom_should_get_chromosome_lengths_from_specified_generator, PopulationFixture) +{ + size_t chromosomeCount = 30; + size_t maxLength = 5; + assert(chromosomeCount % maxLength == 0); + + auto nextLength = [counter = 0, maxLength]() mutable { return counter++ % maxLength; }; + auto population = Population::makeRandom(m_program, chromosomeCount, nextLength); + + // We can't rely on the order since the population sorts its chromosomes immediately but + // we can check the number of occurrences of each length. + for (size_t length = 0; length < maxLength; ++length) + BOOST_TEST( + count_if( + population.individuals().begin(), + population.individuals().end(), + [&length](auto const& individual) { return individual.chromosome.length() == length; } + ) == chromosomeCount / maxLength + ); +} + +BOOST_FIXTURE_TEST_CASE(makeRandom_should_get_chromosome_lengths_from_specified_range, PopulationFixture) +{ + auto population = Population::makeRandom(m_program, 100, 5, 10); + BOOST_TEST(all_of( + population.individuals().begin(), + population.individuals().end(), + [](auto const& individual){ return 5 <= individual.chromosome.length() && individual.chromosome.length() <= 10; } + )); +} + +BOOST_FIXTURE_TEST_CASE(makeRandom_should_use_random_chromosome_length, PopulationFixture) +{ + SimulationRNG::reset(1); + constexpr int populationSize = 200; + constexpr int minLength = 5; + constexpr int maxLength = 10; + constexpr double relativeTolerance = 0.05; + + auto population = Population::makeRandom(m_program, populationSize, minLength, maxLength); + vector samples = chromosomeLengths(population); + + const double expectedValue = (maxLength + minLength) / 2.0; + const double variance = ((maxLength - minLength + 1) * (maxLength - minLength + 1) - 1) / 12.0; + + BOOST_TEST(abs(mean(samples) - expectedValue) < expectedValue * relativeTolerance); + BOOST_TEST(abs(meanSquaredError(samples, expectedValue) - variance) < variance * relativeTolerance); +} + BOOST_FIXTURE_TEST_CASE(makeRandom_should_return_population_with_random_chromosomes, PopulationFixture) { - auto population1 = Population::makeRandom(m_program, 100); - auto population2 = Population::makeRandom(m_program, 100); + auto population1 = Population::makeRandom(m_program, 100, 30, 30); + auto population2 = Population::makeRandom(m_program, 100, 30, 30); BOOST_TEST(population1.individuals().size() == 100); BOOST_TEST(population2.individuals().size() == 100); @@ -127,7 +178,7 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_return_population_with_random_chromoso BOOST_FIXTURE_TEST_CASE(makeRandom_should_not_compute_fitness, PopulationFixture) { - auto population = Population::makeRandom(m_program, 5); + auto population = Population::makeRandom(m_program, 3, 5, 10); BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); } @@ -135,7 +186,7 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_not_compute_fitness, PopulationFixture BOOST_FIXTURE_TEST_CASE(run_should_evaluate_fitness, PopulationFixture) { stringstream output; - auto population = Population::makeRandom(m_program, 5); + auto population = Population::makeRandom(m_program, 5, 5, 10); assert(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); population.run(1, output); diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index 43158ac431dc..c51c3cb0c74a 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -55,15 +55,33 @@ Population::Population(Program _program, vector const& _chromosomes) m_individuals.push_back({chromosome}); } -Population Population::makeRandom(Program _program, size_t _size) +Population Population::makeRandom( + Program _program, + size_t _size, + function _chromosomeLengthGenerator +) { vector individuals; for (size_t i = 0; i < _size; ++i) - individuals.push_back({Chromosome::makeRandom(randomChromosomeLength())}); + individuals.push_back({Chromosome::makeRandom(_chromosomeLengthGenerator())}); return Population(move(_program), individuals); } +Population Population::makeRandom( + Program _program, + size_t _size, + size_t _minChromosomeLength, + size_t _maxChromosomeLength +) +{ + return makeRandom( + move(_program), + _size, + std::bind(uniformChromosomeLength, _minChromosomeLength, _maxChromosomeLength) + ); +} + size_t Population::measureFitness(Chromosome const& _chromosome, Program const& _program) { Program programCopy = _program; @@ -130,6 +148,6 @@ void Population::randomizeWorstChromosomes( auto individual = _individuals.begin() + (_individuals.size() - _count); for (; individual != _individuals.end(); ++individual) { - *individual = {Chromosome::makeRandom(randomChromosomeLength())}; + *individual = {Chromosome::makeRandom(binomialChromosomeLength(MaxChromosomeLength))}; } } diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 983e40dd88e4..c06e0c42ac31 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -55,13 +55,25 @@ class Population static constexpr size_t MaxChromosomeLength = 30; explicit Population(Program _program, std::vector const& _chromosomes = {}); - static Population makeRandom(Program _program, size_t _size); + + static Population makeRandom( + Program _program, + size_t _size, + std::function _chromosomeLengthGenerator + ); + static Population makeRandom( + Program _program, + size_t _size, + size_t _minChromosomeLength, + size_t _maxChromosomeLength + ); void run(std::optional _numRounds, std::ostream& _outputStream); std::vector const& individuals() const { return m_individuals; } - static size_t randomChromosomeLength() { return SimulationRNG::binomialInt(MaxChromosomeLength, 0.5); } + static size_t uniformChromosomeLength(size_t _min, size_t _max) { return SimulationRNG::uniformInt(_min, _max); } + static size_t binomialChromosomeLength(size_t _max) { return SimulationRNG::binomialInt(_max, 0.5); } static size_t measureFitness(Chromosome const& _chromosome, Program const& _program); friend std::ostream& operator<<(std::ostream& _stream, Population const& _population); diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp index b5c522ce7414..ed846dc4f923 100644 --- a/tools/yulPhaser/main.cpp +++ b/tools/yulPhaser/main.cpp @@ -28,6 +28,7 @@ #include #include +#include #include using namespace std; @@ -70,7 +71,11 @@ CharStream loadSource(string const& _sourcePath) void runAlgorithm(string const& _sourcePath) { CharStream sourceCode = loadSource(_sourcePath); - auto population = Population::makeRandom(Program::load(sourceCode), 10); + auto population = Population::makeRandom( + Program::load(sourceCode), + 10, + bind(Population::binomialChromosomeLength, Population::MaxChromosomeLength) + ); population.run(nullopt, cout); } From 823e715902d6167231f30716aeaaaafa97593807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 14 Feb 2020 06:07:28 +0100 Subject: [PATCH 11/92] [yul-phaser] Population+Chromosome: Better tests for makeRandom() --- test/yulPhaser/Chromosome.cpp | 42 ++++++++++++++++++++--------------- test/yulPhaser/Population.cpp | 27 ++++++++++++---------- 2 files changed, 39 insertions(+), 30 deletions(-) diff --git a/test/yulPhaser/Chromosome.cpp b/test/yulPhaser/Chromosome.cpp index 5865c501d635..23de34a2dce3 100644 --- a/test/yulPhaser/Chromosome.cpp +++ b/test/yulPhaser/Chromosome.cpp @@ -67,25 +67,31 @@ BOOST_AUTO_TEST_CASE(constructor_should_convert_from_string_to_optimisation_step BOOST_TEST(Chromosome("ChrOmOsoMe").optimisationSteps() == expectedSteps); } -BOOST_AUTO_TEST_CASE(makeRandom_should_create_chromosome_with_random_optimisation_steps) +BOOST_AUTO_TEST_CASE(makeRandom_should_return_different_chromosome_each_time) { - constexpr uint32_t numSteps = 1000; - - auto chromosome1 = Chromosome::makeRandom(numSteps); - auto chromosome2 = Chromosome::makeRandom(numSteps); - BOOST_CHECK_EQUAL(chromosome1.length(), numSteps); - BOOST_CHECK_EQUAL(chromosome2.length(), numSteps); - - multiset steps1; - multiset steps2; - for (auto const& step: chromosome1.optimisationSteps()) - steps1.insert(step); - for (auto const& step: chromosome2.optimisationSteps()) - steps2.insert(step); - - // Check if steps are different and also if they're not just a permutation of the same set. - // Technically they could be the same and still random but the probability is infinitesimally low. - BOOST_TEST(steps1 != steps2); + SimulationRNG::reset(1); + for (size_t i = 0; i < 10; ++i) + BOOST_TEST(Chromosome::makeRandom(100) != Chromosome::makeRandom(100)); +} + +BOOST_AUTO_TEST_CASE(makeRandom_should_use_every_possible_step_with_the_same_probability) +{ + SimulationRNG::reset(1); + constexpr int samplesPerStep = 100; + constexpr double relativeTolerance = 0.01; + + map stepIndices = enumerateOptmisationSteps(); + auto chromosome = Chromosome::makeRandom(stepIndices.size() * samplesPerStep); + + vector samples; + for (auto& step: chromosome.optimisationSteps()) + samples.push_back(stepIndices.at(step)); + + const double expectedValue = (stepIndices.size() - 1) / 2.0; + const double variance = (stepIndices.size() * stepIndices.size() - 1) / 12.0; + + BOOST_TEST(abs(mean(samples) - expectedValue) < expectedValue * relativeTolerance); + BOOST_TEST(abs(meanSquaredError(samples, expectedValue) - variance) < variance * relativeTolerance); } BOOST_AUTO_TEST_CASE(constructor_should_store_optimisation_steps) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index 15bb48d0481a..d241117de79b 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -159,21 +159,24 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_use_random_chromosome_length, Populati BOOST_FIXTURE_TEST_CASE(makeRandom_should_return_population_with_random_chromosomes, PopulationFixture) { - auto population1 = Population::makeRandom(m_program, 100, 30, 30); - auto population2 = Population::makeRandom(m_program, 100, 30, 30); + SimulationRNG::reset(1); + constexpr int populationSize = 100; + constexpr int chromosomeLength = 30; + constexpr double relativeTolerance = 0.01; + + map stepIndices = enumerateOptmisationSteps(); + auto population = Population::makeRandom(m_program, populationSize, chromosomeLength, chromosomeLength); - BOOST_TEST(population1.individuals().size() == 100); - BOOST_TEST(population2.individuals().size() == 100); + vector samples; + for (auto& individual: population.individuals()) + for (auto& step: individual.chromosome.optimisationSteps()) + samples.push_back(stepIndices.at(step)); - int numMatchingPositions = 0; - for (size_t i = 0; i < 100; ++i) - if (population1.individuals()[i].chromosome == population2.individuals()[i].chromosome) - ++numMatchingPositions; + const double expectedValue = (stepIndices.size() - 1) / 2.0; + const double variance = (stepIndices.size() * stepIndices.size() - 1) / 12.0; - // Assume that the results are random if there are no more than 10 identical chromosomes on the - // same positions. One duplicate is very unlikely but still possible after billions of runs - // (especially for short chromosomes). For ten the probability is so small that we can ignore it. - BOOST_TEST(numMatchingPositions < 10); + BOOST_TEST(abs(mean(samples) - expectedValue) < expectedValue * relativeTolerance); + BOOST_TEST(abs(meanSquaredError(samples, expectedValue) - variance) < variance * relativeTolerance); } BOOST_FIXTURE_TEST_CASE(makeRandom_should_not_compute_fitness, PopulationFixture) From 40a666953878957dd746ab43835bf8e246173346 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Tue, 11 Feb 2020 19:20:01 +0100 Subject: [PATCH 12/92] [yul-phaser] Population: Extract a function for comparing fitness of individuals - Mostly for readability and convenience. This significantly shortens calls to sort(). - I could define it as Individual::operator< instead but it would be inconsistent with operator== because it does not compare the chromosomes, only fitness. It could result in an unintuitive situation where (a <= b <= a) does not necessarily imply (a == b). --- test/yulPhaser/Population.cpp | 18 ++++++++++++++++++ tools/yulPhaser/Population.cpp | 14 ++++++++------ tools/yulPhaser/Population.h | 3 +++ 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index d241117de79b..695d6b397255 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -92,6 +92,24 @@ class PopulationFixture BOOST_AUTO_TEST_SUITE(Phaser) BOOST_AUTO_TEST_SUITE(PopulationTest) +BOOST_AUTO_TEST_CASE(isFitter_should_use_fitness_as_the_main_criterion) +{ + BOOST_TEST(isFitter(Individual{Chromosome("a"), 5}, Individual{Chromosome("a"), 10})); + BOOST_TEST(!isFitter(Individual{Chromosome("a"), 10}, Individual{Chromosome("a"), 5})); + + BOOST_TEST(isFitter(Individual{Chromosome("aaa"), 5}, Individual{Chromosome("aaaaa"), 10})); + BOOST_TEST(!isFitter(Individual{Chromosome("aaaaa"), 10}, Individual{Chromosome("aaa"), 5})); + + BOOST_TEST(isFitter(Individual{Chromosome("aaaaa"), 5}, Individual{Chromosome("aaa"), 10})); + BOOST_TEST(!isFitter(Individual{Chromosome("aaa"), 10}, Individual{Chromosome("aaaaa"), 5})); +} + +BOOST_AUTO_TEST_CASE(isFitter_should_return_false_for_identical_individuals) +{ + BOOST_TEST(!isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("a"), 3})); + BOOST_TEST(!isFitter(Individual{Chromosome("acT"), 0}, Individual{Chromosome("acT"), 0})); +} + BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitness, PopulationFixture) { vector chromosomes = { diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index c51c3cb0c74a..c4b529a91c4e 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -48,6 +48,13 @@ ostream& phaser::operator<<(ostream& _stream, Individual const& _individual) return _stream; } +bool phaser::isFitter(Individual const& a, Individual const& b) +{ + assert(a.fitness.has_value() && b.fitness.has_value()); + + return a.fitness.value() < b.fitness.value(); +} + Population::Population(Program _program, vector const& _chromosomes): m_program{move(_program)} { @@ -128,12 +135,7 @@ void Population::doSelection() { assert(all_of(m_individuals.begin(), m_individuals.end(), [](auto& i){ return i.fitness.has_value(); })); - sort( - m_individuals.begin(), - m_individuals.end(), - [](auto const& a, auto const& b){ return a.fitness.value() < b.fitness.value(); } - ); - + sort(m_individuals.begin(), m_individuals.end(), isFitter); randomizeWorstChromosomes(m_individuals, m_individuals.size() / 2); } diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index c06e0c42ac31..0c6c69be7392 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -40,6 +40,9 @@ struct Individual friend std::ostream& operator<<(std::ostream& _stream, Individual const& _individual); }; +/// Determines which individual is better by comparing fitness values. +bool isFitter(Individual const& a, Individual const& b); + /** * Represents a changing set of individuals undergoing a genetic algorithm. * Each round of the algorithm involves mutating existing individuals, evaluating their fitness From ecb30c670fef1a6eeace327336d91d214b79ac07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Tue, 11 Feb 2020 22:25:17 +0100 Subject: [PATCH 13/92] [yul-phaser] Population: Make ordering of individuals with same fitness deterministic and prioritise shorter chromosomes - Before this change the order of chromosomes with the same fitness in a population depended on the initial order set when the population was first created. Now it only depends on the individual. - The length comparison is not strictly necessary (lexicographical order covers that) but it makes the intention clear and the comparison slightly faster when chromosomes have different lengths. --- test/yulPhaser/Population.cpp | 12 ++++++++++++ tools/yulPhaser/Population.cpp | 9 ++++++++- tools/yulPhaser/Population.h | 4 +++- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index 695d6b397255..bb2f4cc0561e 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -104,6 +104,18 @@ BOOST_AUTO_TEST_CASE(isFitter_should_use_fitness_as_the_main_criterion) BOOST_TEST(!isFitter(Individual{Chromosome("aaa"), 10}, Individual{Chromosome("aaaaa"), 5})); } +BOOST_AUTO_TEST_CASE(isFitter_should_use_alphabetical_order_when_fitness_is_the_same) +{ + BOOST_TEST(isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("c"), 3})); + BOOST_TEST(!isFitter(Individual{Chromosome("c"), 3}, Individual{Chromosome("a"), 3})); + + BOOST_TEST(isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("aa"), 3})); + BOOST_TEST(!isFitter(Individual{Chromosome("aa"), 3}, Individual{Chromosome("a"), 3})); + + BOOST_TEST(isFitter(Individual{Chromosome("T"), 3}, Individual{Chromosome("a"), 3})); + BOOST_TEST(!isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("T"), 3})); +} + BOOST_AUTO_TEST_CASE(isFitter_should_return_false_for_identical_individuals) { BOOST_TEST(!isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("a"), 3})); diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index c4b529a91c4e..890e13b43563 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -19,6 +19,8 @@ #include +#include + #include #include #include @@ -26,6 +28,7 @@ using namespace std; using namespace solidity; using namespace solidity::langutil; +using namespace solidity::util; using namespace solidity::phaser; namespace solidity::phaser @@ -52,7 +55,11 @@ bool phaser::isFitter(Individual const& a, Individual const& b) { assert(a.fitness.has_value() && b.fitness.has_value()); - return a.fitness.value() < b.fitness.value(); + return ( + (a.fitness.value() < b.fitness.value()) || + (a.fitness.value() == b.fitness.value() && a.chromosome.length() < b.chromosome.length()) || + (a.fitness.value() == b.fitness.value() && a.chromosome.length() == b.chromosome.length() && toString(a.chromosome) < toString(b.chromosome)) + ); } Population::Population(Program _program, vector const& _chromosomes): diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 0c6c69be7392..43f8643e1440 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -40,7 +40,9 @@ struct Individual friend std::ostream& operator<<(std::ostream& _stream, Individual const& _individual); }; -/// Determines which individual is better by comparing fitness values. +/// Determines which individual is better by comparing fitness values. If fitness is the same +/// takes into account all the other properties of the individual to make the comparison +/// deterministic as long as the individuals are not equal. bool isFitter(Individual const& a, Individual const& b); /** From d9c5e2dc9f8b967430d75c0c276ca94d4c98535d Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 16:51:43 +0100 Subject: [PATCH 14/92] [yul-phaser] Population: Add operator+() --- test/yulPhaser/Population.cpp | 9 +++++++++ tools/yulPhaser/Population.cpp | 8 ++++++++ tools/yulPhaser/Population.h | 12 ++++++++++++ 3 files changed, 29 insertions(+) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index bb2f4cc0561e..be90382c7758 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -261,6 +261,15 @@ BOOST_FIXTURE_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse, Po } } +BOOST_FIXTURE_TEST_CASE(plus_operator_should_add_two_populations, PopulationFixture) +{ + BOOST_CHECK_EQUAL( + Population(m_program, {Chromosome("ac"), Chromosome("cx")}) + + Population(m_program, {Chromosome("g"), Chromosome("h"), Chromosome("iI")}), + Population(m_program, {Chromosome("ac"), Chromosome("cx"), Chromosome("g"), Chromosome("h"), Chromosome("iI")}) + ); +} + BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index 890e13b43563..e8a302683af7 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -19,6 +19,7 @@ #include +#include #include #include @@ -117,6 +118,13 @@ void Population::run(optional _numRounds, ostream& _outputStream) } } +Population operator+(Population _a, Population _b) +{ + assert(toString(_a.m_program) == toString(_b.m_program)); + + return Population(_a.m_program, move(_a.m_individuals) + move(_b.m_individuals)); +} + ostream& phaser::operator<<(ostream& _stream, Population const& _population) { auto individual = _population.m_individuals.begin(); diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 43f8643e1440..7171a7045dc8 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -28,6 +28,17 @@ namespace solidity::phaser { +class Population; + +} + +// This operator+ must be declared in the global namespace. Otherwise it would shadow global +// operator+ overloads from CommonData.h (e.g. the one for vector) in the namespace it was declared in. +solidity::phaser::Population operator+(solidity::phaser::Population _a, solidity::phaser::Population _b); + +namespace solidity::phaser +{ + /** * Information describing the state of an individual member of the population during the course * of the genetic algorithm. @@ -74,6 +85,7 @@ class Population ); void run(std::optional _numRounds, std::ostream& _outputStream); + friend Population (::operator+)(Population _a, Population _b); std::vector const& individuals() const { return m_individuals; } From 31d8d5930af52b258d6def957484271fc22fb003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Tue, 11 Feb 2020 11:25:00 +0100 Subject: [PATCH 15/92] [yul-phaser] Population: Equality operators for populations and individuals --- tools/yulPhaser/Population.cpp | 7 +++++++ tools/yulPhaser/Population.h | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index e8a302683af7..a0b64999b829 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -125,6 +125,13 @@ Population operator+(Population _a, Population _b) return Population(_a.m_program, move(_a.m_individuals) + move(_b.m_individuals)); } +bool Population::operator==(Population const& _other) const +{ + // TODO: Comparing programs is pretty heavy but it's just a stopgap. It will soon be replaced + // by a comparison of fitness metric associated with the population (once metrics are introduced). + return m_individuals == _other.m_individuals && toString(m_program) == toString(_other.m_program); +} + ostream& phaser::operator<<(ostream& _stream, Population const& _population) { auto individual = _population.m_individuals.begin(); diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 7171a7045dc8..466c8f6f7757 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -48,6 +48,9 @@ struct Individual Chromosome chromosome; std::optional fitness = std::nullopt; + bool operator==(Individual const& _other) const { return fitness == _other.fitness && chromosome == _other.chromosome; } + bool operator!=(Individual const& _other) const { return !(*this == _other); } + friend std::ostream& operator<<(std::ostream& _stream, Individual const& _individual); }; @@ -93,6 +96,9 @@ class Population static size_t binomialChromosomeLength(size_t _max) { return SimulationRNG::binomialInt(_max, 0.5); } static size_t measureFitness(Chromosome const& _chromosome, Program const& _program); + bool operator==(Population const& _other) const; + bool operator!=(Population const& _other) const { return !(*this == _other); } + friend std::ostream& operator<<(std::ostream& _stream, Population const& _population); private: From 4a29726f76ecc8375b06b64a5c53de1f4d50af26 Mon Sep 17 00:00:00 2001 From: a3d4 Date: Tue, 11 Feb 2020 01:47:57 +0100 Subject: [PATCH 16/92] Adjusted solc path and fixed remapping tests in cmdlineTests.sh under mingw64. --- test/cmdlineTests.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index 754cd0bac24a..fd03754072f8 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -33,7 +33,19 @@ set -e REPO_ROOT=$(cd $(dirname "$0")/.. && pwd) SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-build} source "${REPO_ROOT}/scripts/common.sh" -SOLC="$REPO_ROOT/${SOLIDITY_BUILD_DIR}/solc/solc" + +case "$OSTYPE" in + msys) + SOLC="$REPO_ROOT/${SOLIDITY_BUILD_DIR}/solc/Release/solc.exe" + + # prevents msys2 path translation for a remapping test + export MSYS2_ARG_CONV_EXCL="=" + ;; + *) + SOLC="$REPO_ROOT/${SOLIDITY_BUILD_DIR}/solc/solc" + ;; +esac + INTERACTIVE=true if ! tty -s || [ "$CI" ] then From 096129fbc47fa07c6189b415e7515e171fd6964a Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 14:56:55 +0100 Subject: [PATCH 17/92] [yul-phaser] Base class for fitness metrics --- test/CMakeLists.txt | 1 + tools/CMakeLists.txt | 2 ++ tools/yulPhaser/FitnessMetrics.cpp | 18 +++++++++++ tools/yulPhaser/FitnessMetrics.h | 48 ++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+) create mode 100644 tools/yulPhaser/FitnessMetrics.cpp create mode 100644 tools/yulPhaser/FitnessMetrics.h diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 37a78287d51a..d7ac67114c53 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -151,6 +151,7 @@ set(yul_phaser_sources # My current workaround is just to include its source files here but this introduces # unnecessary duplication. Create a library or find a way to reuse the list in both places. ../tools/yulPhaser/Chromosome.cpp + ../tools/yulPhaser/FitnessMetrics.cpp ../tools/yulPhaser/Population.cpp ../tools/yulPhaser/Program.cpp ../tools/yulPhaser/SimulationRNG.cpp diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index cd33c4006069..d3101ad7ba9c 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -17,6 +17,8 @@ add_executable(yul-phaser yulPhaser/main.cpp yulPhaser/Population.h yulPhaser/Population.cpp + yulPhaser/FitnessMetrics.h + yulPhaser/FitnessMetrics.cpp yulPhaser/Chromosome.h yulPhaser/Chromosome.cpp yulPhaser/Program.h diff --git a/tools/yulPhaser/FitnessMetrics.cpp b/tools/yulPhaser/FitnessMetrics.cpp new file mode 100644 index 000000000000..a1014476b47a --- /dev/null +++ b/tools/yulPhaser/FitnessMetrics.cpp @@ -0,0 +1,18 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include diff --git a/tools/yulPhaser/FitnessMetrics.h b/tools/yulPhaser/FitnessMetrics.h new file mode 100644 index 000000000000..712ae1ab54c0 --- /dev/null +++ b/tools/yulPhaser/FitnessMetrics.h @@ -0,0 +1,48 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +/** + * Contains an abstract base class representing a fitness metric and its concrete implementations. + */ + +#pragma once + +#include + +#include + +namespace solidity::phaser +{ + +/** + * Abstract base class for fitness metrics. + * + * The main feature is the @a evaluate() method that can tell how good a given chromosome is. + * The lower the value, the better the fitness is. The result should be deterministic and depend + * only on the chromosome and metric's state (which is constant). + */ +class FitnessMetric +{ +public: + FitnessMetric() = default; + FitnessMetric(FitnessMetric const&) = delete; + FitnessMetric& operator=(FitnessMetric const&) = delete; + virtual ~FitnessMetric() = default; + + virtual size_t evaluate(Chromosome const& _chromosome) const = 0; +}; + +} From 2238919c76b258d53079ae9a085e632d7db87fcf Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 14:58:06 +0100 Subject: [PATCH 18/92] [yul-phaser] Add ProgramSize metric --- test/CMakeLists.txt | 1 + test/yulPhaser/FitnessMetrics.cpp | 80 ++++++++++++++++++++++++++++++ tools/yulPhaser/FitnessMetrics.cpp | 10 ++++ tools/yulPhaser/FitnessMetrics.h | 16 ++++++ 4 files changed, 107 insertions(+) create mode 100644 test/yulPhaser/FitnessMetrics.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d7ac67114c53..3b2fef6f00ef 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -143,6 +143,7 @@ set(yul_phaser_sources yulPhaser/Common.cpp yulPhaser/CommonTest.cpp yulPhaser/Chromosome.cpp + yulPhaser/FitnessMetrics.cpp yulPhaser/Population.cpp yulPhaser/Program.cpp yulPhaser/SimulationRNG.cpp diff --git a/test/yulPhaser/FitnessMetrics.cpp b/test/yulPhaser/FitnessMetrics.cpp new file mode 100644 index 000000000000..7d3c3ab528ef --- /dev/null +++ b/test/yulPhaser/FitnessMetrics.cpp @@ -0,0 +1,80 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include +#include + +#include + +#include + +using namespace std; +using namespace solidity::langutil; +using namespace solidity::yul; + +namespace solidity::phaser::test +{ + +class FitnessMetricFixture +{ +protected: + FitnessMetricFixture(): + m_sourceStream(SampleSourceCode, ""), + m_program(Program::load(m_sourceStream)) {} + + static constexpr char SampleSourceCode[] = + "{\n" + " function foo() -> result\n" + " {\n" + " let x := 1\n" + " result := 15\n" + " }\n" + " function bar() -> result\n" + " {\n" + " result := 15\n" + " }\n" + " mstore(foo(), bar())\n" + "}\n"; + + CharStream m_sourceStream; + Program m_program; +}; + +BOOST_AUTO_TEST_SUITE(Phaser) +BOOST_AUTO_TEST_SUITE(FitnessMetricsTest) +BOOST_AUTO_TEST_SUITE(ProgramSizeTest) + +BOOST_FIXTURE_TEST_CASE(evaluate_should_compute_size_of_the_optimised_program, FitnessMetricFixture) +{ + Chromosome chromosome(vector{UnusedPruner::name, EquivalentFunctionCombiner::name}); + + Program optimisedProgram = m_program; + optimisedProgram.optimise(chromosome.optimisationSteps()); + assert(m_program.codeSize() != optimisedProgram.codeSize()); + + BOOST_TEST(ProgramSize(m_program).evaluate(chromosome) != m_program.codeSize()); + BOOST_TEST(ProgramSize(m_program).evaluate(chromosome) == optimisedProgram.codeSize()); +} + +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() + +} + diff --git a/tools/yulPhaser/FitnessMetrics.cpp b/tools/yulPhaser/FitnessMetrics.cpp index a1014476b47a..27e2e309440c 100644 --- a/tools/yulPhaser/FitnessMetrics.cpp +++ b/tools/yulPhaser/FitnessMetrics.cpp @@ -16,3 +16,13 @@ */ #include + +using namespace std; +using namespace solidity::phaser; + +size_t ProgramSize::evaluate(Chromosome const& _chromosome) const +{ + Program programCopy = m_program; + programCopy.optimise(_chromosome.optimisationSteps()); + return programCopy.codeSize(); +} diff --git a/tools/yulPhaser/FitnessMetrics.h b/tools/yulPhaser/FitnessMetrics.h index 712ae1ab54c0..025b0ccec1a6 100644 --- a/tools/yulPhaser/FitnessMetrics.h +++ b/tools/yulPhaser/FitnessMetrics.h @@ -21,6 +21,7 @@ #pragma once #include +#include #include @@ -45,4 +46,19 @@ class FitnessMetric virtual size_t evaluate(Chromosome const& _chromosome) const = 0; }; +/** + * Fitness metric based on the size of a specific program after applying the optimisations from the + * chromosome to it. + */ +class ProgramSize: public FitnessMetric +{ +public: + ProgramSize(Program _program): m_program(std::move(_program)) {} + + size_t evaluate(Chromosome const& _chromosome) const override; + +private: + Program m_program; +}; + } From 751caf0ed3a5c7838bbaaec90426cfd85b8b80bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 15 Feb 2020 01:40:18 +0100 Subject: [PATCH 19/92] [yul-phaser] Common: Add ChromosomeLengthMetric --- test/yulPhaser/Common.h | 14 ++++++++++++++ test/yulPhaser/CommonTest.cpp | 17 ++++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/test/yulPhaser/Common.h b/test/yulPhaser/Common.h index c4da6c99c933..63ca45c5083a 100644 --- a/test/yulPhaser/Common.h +++ b/test/yulPhaser/Common.h @@ -28,6 +28,8 @@ #pragma once +#include +#include #include #include @@ -38,6 +40,18 @@ namespace solidity::phaser::test { +/** + * Fitness metric that only takes into account the number of optimisation steps in the chromosome. + * Recommended for use in tests because it's much faster than ProgramSize metric and it's very + * easy to guess the result at a glance. + */ +class ChromosomeLengthMetric: public FitnessMetric +{ +public: + using FitnessMetric::FitnessMetric; + size_t evaluate(Chromosome const& _chromosome) const override { return _chromosome.length(); } +}; + // CHROMOSOME AND POPULATION HELPERS /// Returns a vector containing lengths of all chromosomes in the population (in the same order). diff --git a/test/yulPhaser/CommonTest.cpp b/test/yulPhaser/CommonTest.cpp index be686446f1e2..feded8292965 100644 --- a/test/yulPhaser/CommonTest.cpp +++ b/test/yulPhaser/CommonTest.cpp @@ -19,14 +19,11 @@ #include -#include - #include #include using namespace std; -using namespace solidity::langutil; using namespace solidity::yul; using namespace boost::test_tools; @@ -36,15 +33,21 @@ namespace solidity::phaser::test BOOST_AUTO_TEST_SUITE(Phaser) BOOST_AUTO_TEST_SUITE(CommonTest) +BOOST_AUTO_TEST_CASE(ChromosomeLengthMetric_evaluate_should_return_chromosome_length) +{ + BOOST_TEST(ChromosomeLengthMetric{}.evaluate(Chromosome()) == 0); + BOOST_TEST(ChromosomeLengthMetric{}.evaluate(Chromosome("a")) == 1); + BOOST_TEST(ChromosomeLengthMetric{}.evaluate(Chromosome("aaaaa")) == 5); +} + BOOST_AUTO_TEST_CASE(chromosomeLengths_should_return_lengths_of_all_chromosomes_in_a_population) { - CharStream sourceStream("{}", ""); - auto program = Program::load(sourceStream); + shared_ptr fitnessMetric = make_shared(); - Population population1(program, {Chromosome(), Chromosome("a"), Chromosome("aa"), Chromosome("aaa")}); + Population population1(fitnessMetric, {Chromosome(), Chromosome("a"), Chromosome("aa"), Chromosome("aaa")}); BOOST_TEST((chromosomeLengths(population1) == vector{0, 1, 2, 3})); - Population population2(program); + Population population2(fitnessMetric); BOOST_TEST((chromosomeLengths(population2) == vector{})); } From 930a9918a61b0240f6183e820ce0b68f3dfc0cb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 5 Feb 2020 19:24:46 +0100 Subject: [PATCH 20/92] [yul-phaser] ProgramSize: Add an option to repeat the optimisation sequence several times --- test/yulPhaser/FitnessMetrics.cpp | 33 ++++++++++++++++++++++++++++++ tools/yulPhaser/FitnessMetrics.cpp | 4 +++- tools/yulPhaser/FitnessMetrics.h | 5 ++++- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/test/yulPhaser/FitnessMetrics.cpp b/test/yulPhaser/FitnessMetrics.cpp index 7d3c3ab528ef..58561806dcba 100644 --- a/test/yulPhaser/FitnessMetrics.cpp +++ b/test/yulPhaser/FitnessMetrics.cpp @@ -72,6 +72,39 @@ BOOST_FIXTURE_TEST_CASE(evaluate_should_compute_size_of_the_optimised_program, F BOOST_TEST(ProgramSize(m_program).evaluate(chromosome) == optimisedProgram.codeSize()); } +BOOST_FIXTURE_TEST_CASE(evaluate_should_repeat_the_optimisation_specified_number_of_times, FitnessMetricFixture) +{ + Chromosome chromosome(vector{UnusedPruner::name, EquivalentFunctionCombiner::name}); + + Program programOptimisedOnce = m_program; + programOptimisedOnce.optimise(chromosome.optimisationSteps()); + Program programOptimisedTwice = programOptimisedOnce; + programOptimisedTwice.optimise(chromosome.optimisationSteps()); + assert(m_program.codeSize() != programOptimisedOnce.codeSize()); + assert(m_program.codeSize() != programOptimisedTwice.codeSize()); + assert(programOptimisedOnce.codeSize() != programOptimisedTwice.codeSize()); + + ProgramSize metric(m_program, 2); + + BOOST_TEST(metric.evaluate(chromosome) != m_program.codeSize()); + BOOST_TEST(metric.evaluate(chromosome) != programOptimisedOnce.codeSize()); + BOOST_TEST(metric.evaluate(chromosome) == programOptimisedTwice.codeSize()); +} + +BOOST_FIXTURE_TEST_CASE(evaluate_should_not_optimise_if_number_of_repetitions_is_zero, FitnessMetricFixture) +{ + Chromosome chromosome(vector{UnusedPruner::name, EquivalentFunctionCombiner::name}); + + Program optimisedProgram = m_program; + optimisedProgram.optimise(chromosome.optimisationSteps()); + assert(m_program.codeSize() != optimisedProgram.codeSize()); + + ProgramSize metric(m_program, 0); + + BOOST_TEST(metric.evaluate(chromosome) == m_program.codeSize()); + BOOST_TEST(metric.evaluate(chromosome) != optimisedProgram.codeSize()); +} + BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() diff --git a/tools/yulPhaser/FitnessMetrics.cpp b/tools/yulPhaser/FitnessMetrics.cpp index 27e2e309440c..be7ea549729e 100644 --- a/tools/yulPhaser/FitnessMetrics.cpp +++ b/tools/yulPhaser/FitnessMetrics.cpp @@ -23,6 +23,8 @@ using namespace solidity::phaser; size_t ProgramSize::evaluate(Chromosome const& _chromosome) const { Program programCopy = m_program; - programCopy.optimise(_chromosome.optimisationSteps()); + for (size_t i = 0; i < m_repetitionCount; ++i) + programCopy.optimise(_chromosome.optimisationSteps()); + return programCopy.codeSize(); } diff --git a/tools/yulPhaser/FitnessMetrics.h b/tools/yulPhaser/FitnessMetrics.h index 025b0ccec1a6..7fac5f0804de 100644 --- a/tools/yulPhaser/FitnessMetrics.h +++ b/tools/yulPhaser/FitnessMetrics.h @@ -53,12 +53,15 @@ class FitnessMetric class ProgramSize: public FitnessMetric { public: - ProgramSize(Program _program): m_program(std::move(_program)) {} + explicit ProgramSize(Program _program, size_t _repetitionCount = 1): + m_program(std::move(_program)), + m_repetitionCount(_repetitionCount) {} size_t evaluate(Chromosome const& _chromosome) const override; private: Program m_program; + size_t m_repetitionCount; }; } From 41f36f421db46bbfde1e1a754bbb37267d768c96 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 15:15:17 +0100 Subject: [PATCH 21/92] [yul-phaser] Population: Extract conversion from chromosomes to individuals into a separate function --- tools/yulPhaser/Population.cpp | 18 +++++++++++------- tools/yulPhaser/Population.h | 9 ++++++++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index a0b64999b829..02d356bb177e 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -63,13 +63,6 @@ bool phaser::isFitter(Individual const& a, Individual const& b) ); } -Population::Population(Program _program, vector const& _chromosomes): - m_program{move(_program)} -{ - for (auto const& chromosome: _chromosomes) - m_individuals.push_back({chromosome}); -} - Population Population::makeRandom( Program _program, size_t _size, @@ -175,3 +168,14 @@ void Population::randomizeWorstChromosomes( *individual = {Chromosome::makeRandom(binomialChromosomeLength(MaxChromosomeLength))}; } } + +vector Population::chromosomesToIndividuals( + vector _chromosomes +) +{ + vector individuals; + for (auto& chromosome: _chromosomes) + individuals.push_back({move(chromosome)}); + + return individuals; +} diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 466c8f6f7757..ec384ecc8323 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -73,7 +73,11 @@ class Population public: static constexpr size_t MaxChromosomeLength = 30; - explicit Population(Program _program, std::vector const& _chromosomes = {}); + explicit Population(Program _program, std::vector _chromosomes = {}): + Population( + std::move(_program), + chromosomesToIndividuals(std::move(_chromosomes)) + ) {} static Population makeRandom( Program _program, @@ -114,6 +118,9 @@ class Population std::vector& _individuals, size_t _count ); + static std::vector chromosomesToIndividuals( + std::vector _chromosomes + ); Program m_program; std::vector m_individuals; From 85074e7a8a7956e4fc2f926f6cd4630301cd8f54 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 16:04:17 +0100 Subject: [PATCH 22/92] [yul-phaser] Population: Extract sorting from doSelection() into sortIndividuals() in Population class --- tools/yulPhaser/Population.cpp | 12 +++++++++--- tools/yulPhaser/Population.h | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index 02d356bb177e..dbd406613b44 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -148,9 +148,7 @@ void Population::doEvaluation() void Population::doSelection() { - assert(all_of(m_individuals.begin(), m_individuals.end(), [](auto& i){ return i.fitness.has_value(); })); - - sort(m_individuals.begin(), m_individuals.end(), isFitter); + m_individuals = sortedIndividuals(move(m_individuals)); randomizeWorstChromosomes(m_individuals, m_individuals.size() / 2); } @@ -179,3 +177,11 @@ vector Population::chromosomesToIndividuals( return individuals; } + +vector Population::sortedIndividuals(vector _individuals) +{ + assert(all_of(_individuals.begin(), _individuals.end(), [](auto& i){ return i.fitness.has_value(); })); + + sort(_individuals.begin(), _individuals.end(), isFitter); + return _individuals; +} diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index ec384ecc8323..1e8a116b8064 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -121,6 +121,7 @@ class Population static std::vector chromosomesToIndividuals( std::vector _chromosomes ); + static std::vector sortedIndividuals(std::vector _individuals); Program m_program; std::vector m_individuals; From 66fdc1c374dd26d937d0f8eff9e96a6af84324a3 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 15:58:35 +0100 Subject: [PATCH 23/92] [yul-phaser] Population: Store fitness metric rather than program directly - In the console app use ProgramSize metric when creating the population. --- test/yulPhaser/Population.cpp | 56 +++++++++------------------------- tools/yulPhaser/Population.cpp | 31 ++++++++----------- tools/yulPhaser/Population.h | 27 ++++++++-------- tools/yulPhaser/main.cpp | 4 ++- 4 files changed, 45 insertions(+), 73 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index be90382c7758..4fb796c3e98b 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -58,35 +58,7 @@ namespace class PopulationFixture { protected: - PopulationFixture(): - m_sourceStream(SampleSourceCode, ""), - m_program(Program::load(m_sourceStream)) {} - - static constexpr char SampleSourceCode[] = - "{\n" - " let factor := 13\n" - " {\n" - " if factor\n" - " {\n" - " let variable := add(1, 2)\n" - " }\n" - " let result := factor\n" - " }\n" - " let something := 6\n" - " {\n" - " {\n" - " {\n" - " let value := 15\n" - " }\n" - " }\n" - " }\n" - " let something_else := mul(mul(something, 1), add(factor, 0))\n" - " if 1 { let x := 1 }\n" - " if 0 { let y := 2 }\n" - "}\n"; - - CharStream m_sourceStream; - Program m_program; + shared_ptr m_fitnessMetric = make_shared(); }; BOOST_AUTO_TEST_SUITE(Phaser) @@ -128,7 +100,7 @@ BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitn Chromosome::makeRandom(5), Chromosome::makeRandom(10), }; - Population population(m_program, chromosomes); + Population population(m_fitnessMetric, chromosomes); BOOST_TEST(population.individuals().size() == 2); BOOST_TEST(population.individuals()[0].chromosome == chromosomes[0]); @@ -145,7 +117,7 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_get_chromosome_lengths_from_specified_ assert(chromosomeCount % maxLength == 0); auto nextLength = [counter = 0, maxLength]() mutable { return counter++ % maxLength; }; - auto population = Population::makeRandom(m_program, chromosomeCount, nextLength); + auto population = Population::makeRandom(m_fitnessMetric, chromosomeCount, nextLength); // We can't rely on the order since the population sorts its chromosomes immediately but // we can check the number of occurrences of each length. @@ -161,7 +133,7 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_get_chromosome_lengths_from_specified_ BOOST_FIXTURE_TEST_CASE(makeRandom_should_get_chromosome_lengths_from_specified_range, PopulationFixture) { - auto population = Population::makeRandom(m_program, 100, 5, 10); + auto population = Population::makeRandom(m_fitnessMetric, 100, 5, 10); BOOST_TEST(all_of( population.individuals().begin(), population.individuals().end(), @@ -177,7 +149,7 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_use_random_chromosome_length, Populati constexpr int maxLength = 10; constexpr double relativeTolerance = 0.05; - auto population = Population::makeRandom(m_program, populationSize, minLength, maxLength); + auto population = Population::makeRandom(m_fitnessMetric, populationSize, minLength, maxLength); vector samples = chromosomeLengths(population); const double expectedValue = (maxLength + minLength) / 2.0; @@ -195,7 +167,7 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_return_population_with_random_chromoso constexpr double relativeTolerance = 0.01; map stepIndices = enumerateOptmisationSteps(); - auto population = Population::makeRandom(m_program, populationSize, chromosomeLength, chromosomeLength); + auto population = Population::makeRandom(m_fitnessMetric, populationSize, chromosomeLength, chromosomeLength); vector samples; for (auto& individual: population.individuals()) @@ -211,7 +183,7 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_return_population_with_random_chromoso BOOST_FIXTURE_TEST_CASE(makeRandom_should_not_compute_fitness, PopulationFixture) { - auto population = Population::makeRandom(m_program, 3, 5, 10); + auto population = Population::makeRandom(m_fitnessMetric, 3, 5, 10); BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); } @@ -219,7 +191,7 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_not_compute_fitness, PopulationFixture BOOST_FIXTURE_TEST_CASE(run_should_evaluate_fitness, PopulationFixture) { stringstream output; - auto population = Population::makeRandom(m_program, 5, 5, 10); + auto population = Population::makeRandom(m_fitnessMetric, 5, 5, 10); assert(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); population.run(1, output); @@ -237,11 +209,11 @@ BOOST_FIXTURE_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse, Po Chromosome(vector{UnusedPruner::name}), Chromosome(vector{StructuralSimplifier::name, BlockFlattener::name}), }; - Population population(m_program, chromosomes); + Population population(m_fitnessMetric, chromosomes); size_t initialTopFitness[2] = { - Population::measureFitness(chromosomes[0], m_program), - Population::measureFitness(chromosomes[1], m_program), + m_fitnessMetric->evaluate(chromosomes[0]), + m_fitnessMetric->evaluate(chromosomes[1]), }; for (int i = 0; i < 6; ++i) @@ -264,9 +236,9 @@ BOOST_FIXTURE_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse, Po BOOST_FIXTURE_TEST_CASE(plus_operator_should_add_two_populations, PopulationFixture) { BOOST_CHECK_EQUAL( - Population(m_program, {Chromosome("ac"), Chromosome("cx")}) + - Population(m_program, {Chromosome("g"), Chromosome("h"), Chromosome("iI")}), - Population(m_program, {Chromosome("ac"), Chromosome("cx"), Chromosome("g"), Chromosome("h"), Chromosome("iI")}) + Population(m_fitnessMetric, {Chromosome("ac"), Chromosome("cx")}) + + Population(m_fitnessMetric, {Chromosome("g"), Chromosome("h"), Chromosome("iI")}), + Population(m_fitnessMetric, {Chromosome("ac"), Chromosome("cx"), Chromosome("g"), Chromosome("h"), Chromosome("iI")}) ); } diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index dbd406613b44..58309299aa22 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -17,7 +17,6 @@ #include -#include #include #include @@ -64,7 +63,7 @@ bool phaser::isFitter(Individual const& a, Individual const& b) } Population Population::makeRandom( - Program _program, + shared_ptr _fitnessMetric, size_t _size, function _chromosomeLengthGenerator ) @@ -73,30 +72,23 @@ Population Population::makeRandom( for (size_t i = 0; i < _size; ++i) individuals.push_back({Chromosome::makeRandom(_chromosomeLengthGenerator())}); - return Population(move(_program), individuals); + return Population(move(_fitnessMetric), move(individuals)); } Population Population::makeRandom( - Program _program, + shared_ptr _fitnessMetric, size_t _size, size_t _minChromosomeLength, size_t _maxChromosomeLength ) { return makeRandom( - move(_program), + move(_fitnessMetric), _size, std::bind(uniformChromosomeLength, _minChromosomeLength, _maxChromosomeLength) ); } -size_t Population::measureFitness(Chromosome const& _chromosome, Program const& _program) -{ - Program programCopy = _program; - programCopy.optimise(_chromosome.optimisationSteps()); - return programCopy.codeSize(); -} - void Population::run(optional _numRounds, ostream& _outputStream) { doEvaluation(); @@ -113,16 +105,19 @@ void Population::run(optional _numRounds, ostream& _outputStream) Population operator+(Population _a, Population _b) { - assert(toString(_a.m_program) == toString(_b.m_program)); + // This operator is meant to be used only with populations sharing the same metric (and, to make + // things simple, "the same" here means the same exact object in memory). + assert(_a.m_fitnessMetric == _b.m_fitnessMetric); - return Population(_a.m_program, move(_a.m_individuals) + move(_b.m_individuals)); + return Population(_a.m_fitnessMetric, move(_a.m_individuals) + move(_b.m_individuals)); } bool Population::operator==(Population const& _other) const { - // TODO: Comparing programs is pretty heavy but it's just a stopgap. It will soon be replaced - // by a comparison of fitness metric associated with the population (once metrics are introduced). - return m_individuals == _other.m_individuals && toString(m_program) == toString(_other.m_program); + // We consider populations identical only if they share the same exact instance of the metric. + // It might be possible to define some notion of equality for metric objects but it would + // be an overkill since mixing populations using different metrics is not a common use case. + return m_individuals == _other.m_individuals && m_fitnessMetric == _other.m_fitnessMetric; } ostream& phaser::operator<<(ostream& _stream, Population const& _population) @@ -143,7 +138,7 @@ void Population::doEvaluation() { for (auto& individual: m_individuals) if (!individual.fitness.has_value()) - individual.fitness = measureFitness(individual.chromosome, m_program); + individual.fitness = m_fitnessMetric->evaluate(individual.chromosome); } void Population::doSelection() diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 1e8a116b8064..5387cea281fd 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -18,7 +18,7 @@ #pragma once #include -#include +#include #include #include @@ -64,28 +64,31 @@ bool isFitter(Individual const& a, Individual const& b); * Each round of the algorithm involves mutating existing individuals, evaluating their fitness * and selecting the best ones for the next round. * - * An individual is a sequence of optimiser steps represented by a @a Chromosome instance. The whole - * population is associated with a fixed Yul program. By applying the steps to the @a Program - * instance the class can compute fitness of the individual. + * An individual is a sequence of optimiser steps represented by a @a Chromosome instance. + * Individuals are stored together with a fitness value that can be computed by the fitness metric + * associated with the population. */ class Population { public: static constexpr size_t MaxChromosomeLength = 30; - explicit Population(Program _program, std::vector _chromosomes = {}): + explicit Population( + std::shared_ptr _fitnessMetric, + std::vector _chromosomes = {} + ): Population( - std::move(_program), + std::move(_fitnessMetric), chromosomesToIndividuals(std::move(_chromosomes)) ) {} static Population makeRandom( - Program _program, + std::shared_ptr _fitnessMetric, size_t _size, std::function _chromosomeLengthGenerator ); static Population makeRandom( - Program _program, + std::shared_ptr _fitnessMetric, size_t _size, size_t _minChromosomeLength, size_t _maxChromosomeLength @@ -94,11 +97,11 @@ class Population void run(std::optional _numRounds, std::ostream& _outputStream); friend Population (::operator+)(Population _a, Population _b); + std::shared_ptr fitnessMetric() const { return m_fitnessMetric; } std::vector const& individuals() const { return m_individuals; } static size_t uniformChromosomeLength(size_t _min, size_t _max) { return SimulationRNG::uniformInt(_min, _max); } static size_t binomialChromosomeLength(size_t _max) { return SimulationRNG::binomialInt(_max, 0.5); } - static size_t measureFitness(Chromosome const& _chromosome, Program const& _program); bool operator==(Population const& _other) const; bool operator!=(Population const& _other) const { return !(*this == _other); } @@ -106,8 +109,8 @@ class Population friend std::ostream& operator<<(std::ostream& _stream, Population const& _population); private: - explicit Population(Program _program, std::vector _individuals): - m_program{std::move(_program)}, + explicit Population(std::shared_ptr _fitnessMetric, std::vector _individuals): + m_fitnessMetric(std::move(_fitnessMetric)), m_individuals{std::move(_individuals)} {} void doMutation(); @@ -123,7 +126,7 @@ class Population ); static std::vector sortedIndividuals(std::vector _individuals); - Program m_program; + std::shared_ptr m_fitnessMetric; std::vector m_individuals; }; diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp index ed846dc4f923..f8226e1b9cc7 100644 --- a/tools/yulPhaser/main.cpp +++ b/tools/yulPhaser/main.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -71,8 +72,9 @@ CharStream loadSource(string const& _sourcePath) void runAlgorithm(string const& _sourcePath) { CharStream sourceCode = loadSource(_sourcePath); + shared_ptr fitnessMetric = make_shared(Program::load(sourceCode)); auto population = Population::makeRandom( - Program::load(sourceCode), + fitnessMetric, 10, bind(Population::binomialChromosomeLength, Population::MaxChromosomeLength) ); From 76842ac3fdd63fed559ccbbdf070c41dc2b0df41 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 16:18:53 +0100 Subject: [PATCH 24/92] [yul-phaser] Population: Evaluate fitness immediately when an individual is added or modified - This removes the explicit evaluation phase. - Fitness is no longer optional in Individual --- test/yulPhaser/Population.cpp | 42 +++++++------------------------- tools/yulPhaser/Population.cpp | 44 +++++++++++++--------------------- tools/yulPhaser/Population.h | 10 ++++---- 3 files changed, 32 insertions(+), 64 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index 4fb796c3e98b..1b8e76da08fc 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -42,19 +42,6 @@ using namespace boost::unit_test::framework; namespace solidity::phaser::test { -namespace -{ - bool fitnessNotSet(Individual const& individual) - { - return !individual.fitness.has_value(); - } - - bool fitnessSet(Individual const& individual) - { - return individual.fitness.has_value(); - } -} - class PopulationFixture { protected: @@ -94,7 +81,7 @@ BOOST_AUTO_TEST_CASE(isFitter_should_return_false_for_identical_individuals) BOOST_TEST(!isFitter(Individual{Chromosome("acT"), 0}, Individual{Chromosome("acT"), 0})); } -BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitness, PopulationFixture) +BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_and_compute_fitness, PopulationFixture) { vector chromosomes = { Chromosome::makeRandom(5), @@ -106,8 +93,8 @@ BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_and_not_compute_fitn BOOST_TEST(population.individuals()[0].chromosome == chromosomes[0]); BOOST_TEST(population.individuals()[1].chromosome == chromosomes[1]); - auto fitnessNotSet = [](auto const& individual){ return !individual.fitness.has_value(); }; - BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); + BOOST_TEST(population.individuals()[0].fitness == m_fitnessMetric->evaluate(population.individuals()[0].chromosome)); + BOOST_TEST(population.individuals()[1].fitness == m_fitnessMetric->evaluate(population.individuals()[1].chromosome)); } BOOST_FIXTURE_TEST_CASE(makeRandom_should_get_chromosome_lengths_from_specified_generator, PopulationFixture) @@ -181,22 +168,13 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_return_population_with_random_chromoso BOOST_TEST(abs(meanSquaredError(samples, expectedValue) - variance) < variance * relativeTolerance); } -BOOST_FIXTURE_TEST_CASE(makeRandom_should_not_compute_fitness, PopulationFixture) +BOOST_FIXTURE_TEST_CASE(makeRandom_should_compute_fitness, PopulationFixture) { auto population = Population::makeRandom(m_fitnessMetric, 3, 5, 10); - BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); -} - -BOOST_FIXTURE_TEST_CASE(run_should_evaluate_fitness, PopulationFixture) -{ - stringstream output; - auto population = Population::makeRandom(m_fitnessMetric, 5, 5, 10); - assert(all_of(population.individuals().begin(), population.individuals().end(), fitnessNotSet)); - - population.run(1, output); - - BOOST_TEST(all_of(population.individuals().begin(), population.individuals().end(), fitnessSet)); + BOOST_TEST(population.individuals()[0].fitness == m_fitnessMetric->evaluate(population.individuals()[0].chromosome)); + BOOST_TEST(population.individuals()[1].fitness == m_fitnessMetric->evaluate(population.individuals()[1].chromosome)); + BOOST_TEST(population.individuals()[2].fitness == m_fitnessMetric->evaluate(population.individuals()[2].chromosome)); } BOOST_FIXTURE_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse, PopulationFixture) @@ -220,12 +198,10 @@ BOOST_FIXTURE_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse, Po { population.run(1, output); BOOST_TEST(population.individuals().size() == 5); - BOOST_TEST(fitnessSet(population.individuals()[0])); - BOOST_TEST(fitnessSet(population.individuals()[1])); size_t currentTopFitness[2] = { - population.individuals()[0].fitness.value(), - population.individuals()[1].fitness.value(), + population.individuals()[0].fitness, + population.individuals()[1].fitness, }; BOOST_TEST(currentTopFitness[0] <= initialTopFitness[0]); BOOST_TEST(currentTopFitness[1] <= initialTopFitness[1]); diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index 58309299aa22..d258fbf3dba9 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -41,11 +41,7 @@ ostream& operator<<(ostream& _stream, Population const& _population); ostream& phaser::operator<<(ostream& _stream, Individual const& _individual) { - _stream << "Fitness: "; - if (_individual.fitness.has_value()) - _stream << _individual.fitness.value(); - else - _stream << ""; + _stream << "Fitness: " << _individual.fitness; _stream << ", optimisations: " << _individual.chromosome; return _stream; @@ -53,12 +49,10 @@ ostream& phaser::operator<<(ostream& _stream, Individual const& _individual) bool phaser::isFitter(Individual const& a, Individual const& b) { - assert(a.fitness.has_value() && b.fitness.has_value()); - return ( - (a.fitness.value() < b.fitness.value()) || - (a.fitness.value() == b.fitness.value() && a.chromosome.length() < b.chromosome.length()) || - (a.fitness.value() == b.fitness.value() && a.chromosome.length() == b.chromosome.length() && toString(a.chromosome) < toString(b.chromosome)) + (a.fitness < b.fitness) || + (a.fitness == b.fitness && a.chromosome.length() < b.chromosome.length()) || + (a.fitness == b.fitness && a.chromosome.length() == b.chromosome.length() && toString(a.chromosome) < toString(b.chromosome)) ); } @@ -68,11 +62,11 @@ Population Population::makeRandom( function _chromosomeLengthGenerator ) { - vector individuals; + vector chromosomes; for (size_t i = 0; i < _size; ++i) - individuals.push_back({Chromosome::makeRandom(_chromosomeLengthGenerator())}); + chromosomes.push_back(Chromosome::makeRandom(_chromosomeLengthGenerator())); - return Population(move(_fitnessMetric), move(individuals)); + return Population(move(_fitnessMetric), move(chromosomes)); } Population Population::makeRandom( @@ -91,12 +85,10 @@ Population Population::makeRandom( void Population::run(optional _numRounds, ostream& _outputStream) { - doEvaluation(); for (size_t round = 0; !_numRounds.has_value() || round < _numRounds.value(); ++round) { doMutation(); doSelection(); - doEvaluation(); _outputStream << "---------- ROUND " << round << " ----------" << endl; _outputStream << *this; @@ -134,20 +126,14 @@ void Population::doMutation() // TODO: Implement mutation and crossover } -void Population::doEvaluation() -{ - for (auto& individual: m_individuals) - if (!individual.fitness.has_value()) - individual.fitness = m_fitnessMetric->evaluate(individual.chromosome); -} - void Population::doSelection() { m_individuals = sortedIndividuals(move(m_individuals)); - randomizeWorstChromosomes(m_individuals, m_individuals.size() / 2); + randomizeWorstChromosomes(*m_fitnessMetric, m_individuals, m_individuals.size() / 2); } void Population::randomizeWorstChromosomes( + FitnessMetric const& _fitnessMetric, vector& _individuals, size_t _count ) @@ -158,25 +144,29 @@ void Population::randomizeWorstChromosomes( auto individual = _individuals.begin() + (_individuals.size() - _count); for (; individual != _individuals.end(); ++individual) { - *individual = {Chromosome::makeRandom(binomialChromosomeLength(MaxChromosomeLength))}; + auto chromosome = Chromosome::makeRandom(binomialChromosomeLength(MaxChromosomeLength)); + size_t fitness = _fitnessMetric.evaluate(chromosome); + *individual = {move(chromosome), fitness}; } } vector Population::chromosomesToIndividuals( + FitnessMetric const& _fitnessMetric, vector _chromosomes ) { vector individuals; for (auto& chromosome: _chromosomes) - individuals.push_back({move(chromosome)}); + { + size_t fitness = _fitnessMetric.evaluate(chromosome); + individuals.push_back({move(chromosome), fitness}); + } return individuals; } vector Population::sortedIndividuals(vector _individuals) { - assert(all_of(_individuals.begin(), _individuals.end(), [](auto& i){ return i.fitness.has_value(); })); - sort(_individuals.begin(), _individuals.end(), isFitter); return _individuals; } diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 5387cea281fd..0dbac3235531 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -46,7 +46,7 @@ namespace solidity::phaser struct Individual { Chromosome chromosome; - std::optional fitness = std::nullopt; + size_t fitness; bool operator==(Individual const& _other) const { return fitness == _other.fitness && chromosome == _other.chromosome; } bool operator!=(Individual const& _other) const { return !(*this == _other); } @@ -67,6 +67,7 @@ bool isFitter(Individual const& a, Individual const& b); * An individual is a sequence of optimiser steps represented by a @a Chromosome instance. * Individuals are stored together with a fitness value that can be computed by the fitness metric * associated with the population. + * The fitness is computed using the metric as soon as an individual is inserted into the population. */ class Population { @@ -78,8 +79,8 @@ class Population std::vector _chromosomes = {} ): Population( - std::move(_fitnessMetric), - chromosomesToIndividuals(std::move(_chromosomes)) + _fitnessMetric, + chromosomesToIndividuals(*_fitnessMetric, std::move(_chromosomes)) ) {} static Population makeRandom( @@ -114,14 +115,15 @@ class Population m_individuals{std::move(_individuals)} {} void doMutation(); - void doEvaluation(); void doSelection(); static void randomizeWorstChromosomes( + FitnessMetric const& _fitnessMetric, std::vector& _individuals, size_t _count ); static std::vector chromosomesToIndividuals( + FitnessMetric const& _fitnessMetric, std::vector _chromosomes ); static std::vector sortedIndividuals(std::vector _individuals); From cef01c961a02e1af9efbca41eb10d76b582ef911 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 16:49:50 +0100 Subject: [PATCH 25/92] [yul-phaser] Population: Keep the individuals always sorted --- test/yulPhaser/Population.cpp | 16 ++++++++++------ tools/yulPhaser/Population.cpp | 2 +- tools/yulPhaser/Population.h | 5 ++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index 1b8e76da08fc..c4b127888d5e 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -81,20 +81,24 @@ BOOST_AUTO_TEST_CASE(isFitter_should_return_false_for_identical_individuals) BOOST_TEST(!isFitter(Individual{Chromosome("acT"), 0}, Individual{Chromosome("acT"), 0})); } -BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_and_compute_fitness, PopulationFixture) +BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_compute_fitness_and_sort_chromosomes, PopulationFixture) { vector chromosomes = { Chromosome::makeRandom(5), + Chromosome::makeRandom(15), Chromosome::makeRandom(10), }; Population population(m_fitnessMetric, chromosomes); - BOOST_TEST(population.individuals().size() == 2); - BOOST_TEST(population.individuals()[0].chromosome == chromosomes[0]); - BOOST_TEST(population.individuals()[1].chromosome == chromosomes[1]); + vector const& individuals = population.individuals(); - BOOST_TEST(population.individuals()[0].fitness == m_fitnessMetric->evaluate(population.individuals()[0].chromosome)); - BOOST_TEST(population.individuals()[1].fitness == m_fitnessMetric->evaluate(population.individuals()[1].chromosome)); + BOOST_TEST(individuals.size() == 3); + BOOST_TEST(individuals[0].fitness == 5); + BOOST_TEST(individuals[1].fitness == 10); + BOOST_TEST(individuals[2].fitness == 15); + BOOST_TEST(individuals[0].chromosome == chromosomes[0]); + BOOST_TEST(individuals[1].chromosome == chromosomes[2]); + BOOST_TEST(individuals[2].chromosome == chromosomes[1]); } BOOST_FIXTURE_TEST_CASE(makeRandom_should_get_chromosome_lengths_from_specified_generator, PopulationFixture) diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index d258fbf3dba9..923e75e6a7e1 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -128,8 +128,8 @@ void Population::doMutation() void Population::doSelection() { - m_individuals = sortedIndividuals(move(m_individuals)); randomizeWorstChromosomes(*m_fitnessMetric, m_individuals, m_individuals.size() / 2); + m_individuals = sortedIndividuals(move(m_individuals)); } void Population::randomizeWorstChromosomes( diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index 0dbac3235531..bd58c397a897 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -65,8 +65,7 @@ bool isFitter(Individual const& a, Individual const& b); * and selecting the best ones for the next round. * * An individual is a sequence of optimiser steps represented by a @a Chromosome instance. - * Individuals are stored together with a fitness value that can be computed by the fitness metric - * associated with the population. + * Individuals are always ordered by their fitness (based on @_fitnessMetric and @a isFitter()). * The fitness is computed using the metric as soon as an individual is inserted into the population. */ class Population @@ -112,7 +111,7 @@ class Population private: explicit Population(std::shared_ptr _fitnessMetric, std::vector _individuals): m_fitnessMetric(std::move(_fitnessMetric)), - m_individuals{std::move(_individuals)} {} + m_individuals{sortedIndividuals(std::move(_individuals))} {} void doMutation(); void doSelection(); From 2291cf78ac5afa42598facb7f8e1f8b2cf99558d Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 17:02:32 +0100 Subject: [PATCH 26/92] [yul-phaser] Population: Add constructors to Individual to simplify initialization --- test/yulPhaser/Population.cpp | 28 ++++++++++++++-------------- tools/yulPhaser/Population.cpp | 5 +---- tools/yulPhaser/Population.h | 7 +++++++ 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index c4b127888d5e..55c346575850 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -53,32 +53,32 @@ BOOST_AUTO_TEST_SUITE(PopulationTest) BOOST_AUTO_TEST_CASE(isFitter_should_use_fitness_as_the_main_criterion) { - BOOST_TEST(isFitter(Individual{Chromosome("a"), 5}, Individual{Chromosome("a"), 10})); - BOOST_TEST(!isFitter(Individual{Chromosome("a"), 10}, Individual{Chromosome("a"), 5})); + BOOST_TEST(isFitter(Individual(Chromosome("a"), 5), Individual(Chromosome("a"), 10))); + BOOST_TEST(!isFitter(Individual(Chromosome("a"), 10), Individual(Chromosome("a"), 5))); - BOOST_TEST(isFitter(Individual{Chromosome("aaa"), 5}, Individual{Chromosome("aaaaa"), 10})); - BOOST_TEST(!isFitter(Individual{Chromosome("aaaaa"), 10}, Individual{Chromosome("aaa"), 5})); + BOOST_TEST(isFitter(Individual(Chromosome("aaa"), 5), Individual(Chromosome("aaaaa"), 10))); + BOOST_TEST(!isFitter(Individual(Chromosome("aaaaa"), 10), Individual(Chromosome("aaa"), 5))); - BOOST_TEST(isFitter(Individual{Chromosome("aaaaa"), 5}, Individual{Chromosome("aaa"), 10})); - BOOST_TEST(!isFitter(Individual{Chromosome("aaa"), 10}, Individual{Chromosome("aaaaa"), 5})); + BOOST_TEST(isFitter(Individual(Chromosome("aaaaa"), 5), Individual(Chromosome("aaa"), 10))); + BOOST_TEST(!isFitter(Individual(Chromosome("aaa"), 10), Individual(Chromosome("aaaaa"), 5))); } BOOST_AUTO_TEST_CASE(isFitter_should_use_alphabetical_order_when_fitness_is_the_same) { - BOOST_TEST(isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("c"), 3})); - BOOST_TEST(!isFitter(Individual{Chromosome("c"), 3}, Individual{Chromosome("a"), 3})); + BOOST_TEST(isFitter(Individual(Chromosome("a"), 3), Individual(Chromosome("c"), 3))); + BOOST_TEST(!isFitter(Individual(Chromosome("c"), 3), Individual(Chromosome("a"), 3))); - BOOST_TEST(isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("aa"), 3})); - BOOST_TEST(!isFitter(Individual{Chromosome("aa"), 3}, Individual{Chromosome("a"), 3})); + BOOST_TEST(isFitter(Individual(Chromosome("a"), 3), Individual(Chromosome("aa"), 3))); + BOOST_TEST(!isFitter(Individual(Chromosome("aa"), 3), Individual(Chromosome("a"), 3))); - BOOST_TEST(isFitter(Individual{Chromosome("T"), 3}, Individual{Chromosome("a"), 3})); - BOOST_TEST(!isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("T"), 3})); + BOOST_TEST(isFitter(Individual(Chromosome("T"), 3), Individual(Chromosome("a"), 3))); + BOOST_TEST(!isFitter(Individual(Chromosome("a"), 3), Individual(Chromosome("T"), 3))); } BOOST_AUTO_TEST_CASE(isFitter_should_return_false_for_identical_individuals) { - BOOST_TEST(!isFitter(Individual{Chromosome("a"), 3}, Individual{Chromosome("a"), 3})); - BOOST_TEST(!isFitter(Individual{Chromosome("acT"), 0}, Individual{Chromosome("acT"), 0})); + BOOST_TEST(!isFitter(Individual(Chromosome("a"), 3), Individual(Chromosome("a"), 3))); + BOOST_TEST(!isFitter(Individual(Chromosome("acT"), 0), Individual(Chromosome("acT"), 0))); } BOOST_FIXTURE_TEST_CASE(constructor_should_copy_chromosomes_compute_fitness_and_sort_chromosomes, PopulationFixture) diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index 923e75e6a7e1..2fed8b90d500 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -157,10 +157,7 @@ vector Population::chromosomesToIndividuals( { vector individuals; for (auto& chromosome: _chromosomes) - { - size_t fitness = _fitnessMetric.evaluate(chromosome); - individuals.push_back({move(chromosome), fitness}); - } + individuals.emplace_back(move(chromosome), _fitnessMetric); return individuals; } diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index bd58c397a897..bd15674d693b 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -48,6 +48,13 @@ struct Individual Chromosome chromosome; size_t fitness; + Individual(Chromosome _chromosome, size_t _fitness): + chromosome(std::move(_chromosome)), + fitness(_fitness) {} + Individual(Chromosome _chromosome, FitnessMetric const& _fitnessMetric): + chromosome(std::move(_chromosome)), + fitness(_fitnessMetric.evaluate(chromosome)) {} + bool operator==(Individual const& _other) const { return fitness == _other.fitness && chromosome == _other.chromosome; } bool operator!=(Individual const& _other) const { return !(*this == _other); } From e8192e9aa34315b6bf29d4ee7a4e8ed095154c9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 5 Feb 2020 19:25:15 +0100 Subject: [PATCH 27/92] [yul-phaser] main: Set the number of optimisation sequence repetitions to 5 --- tools/yulPhaser/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp index f8226e1b9cc7..eea43e5fb76c 100644 --- a/tools/yulPhaser/main.cpp +++ b/tools/yulPhaser/main.cpp @@ -72,7 +72,7 @@ CharStream loadSource(string const& _sourcePath) void runAlgorithm(string const& _sourcePath) { CharStream sourceCode = loadSource(_sourcePath); - shared_ptr fitnessMetric = make_shared(Program::load(sourceCode)); + shared_ptr fitnessMetric = make_shared(Program::load(sourceCode), 5); auto population = Population::makeRandom( fitnessMetric, 10, From 98fb71f03f9013ba9f1d32e8ae0167579c2f1962 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Thu, 20 Feb 2020 11:34:49 +0530 Subject: [PATCH 28/92] circleci: Use custom pipeline parameters per docker image revision in config --- .circleci/README.md | 2 +- .circleci/config.yml | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.circleci/README.md b/.circleci/README.md index 466aaa76a189..b48adba24ca5 100644 --- a/.circleci/README.md +++ b/.circleci/README.md @@ -11,7 +11,7 @@ docker build -t ethereum/solidity-buildpack-deps:ubuntu1904- -f Docker docker push ethereum/solidity-buildpack-deps:ubuntu1904- ``` -The current revision is stored in a [circle ci pipeline parameter](https://github.com/CircleCI-Public/api-preview-docs/blob/master/docs/pipeline-parameters.md#pipeline-parameters) called `docker-image-rev`. Please update the value assigned to this parameter at the time of a docker image update. Please verify that the value assigned to the parameter matches the revision part of the docker image tag (`` in the docker build/push snippet shown above). Otherwise, the docker image used by circle ci and the one actually pushed to docker hub will differ. +The current revisions per docker image are stored in [circle ci pipeline parameters](https://github.com/CircleCI-Public/api-preview-docs/blob/master/docs/pipeline-parameters.md#pipeline-parameters) called `-docker-image-rev` (e.g., `ubuntu-1904-docker-image-rev`). Please update the value assigned to the parameter(s) corresponding to the docker image(s) being updated at the time of the update. Please verify that the value assigned to the parameter matches the revision part of the docker image tag (`` in the docker build/push snippet shown above). Otherwise, the docker image used by circle ci and the one actually pushed to docker hub will differ. Once the docker image has been built and pushed to Dockerhub, you can find it at: diff --git a/.circleci/config.yml b/.circleci/config.yml index 1009d8217c1c..3d95b0a73eb3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,13 @@ # - ems: Emscripten version: 2.1 parameters: - docker-image-rev: + ubuntu-1804-docker-image-rev: + type: string + default: "4" + ubuntu-1904-docker-image-rev: + type: string + default: "4" + ubuntu-1904-clang-docker-image-rev: type: string default: "4" @@ -115,7 +121,7 @@ defaults: - test_ubuntu1904_clang: &test_ubuntu1904_clang docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1904-clang-<< pipeline.parameters.docker-image-rev >> + - image: ethereum/solidity-buildpack-deps:ubuntu1904-clang-<< pipeline.parameters.ubuntu-1904-clang-docker-image-rev >> steps: - checkout - attach_workspace: @@ -126,7 +132,7 @@ defaults: - test_ubuntu1904: &test_ubuntu1904 docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1904-<< pipeline.parameters.docker-image-rev >> + - image: ethereum/solidity-buildpack-deps:ubuntu1904-<< pipeline.parameters.ubuntu-1904-docker-image-rev >> steps: - checkout - attach_workspace: @@ -312,7 +318,7 @@ jobs: b_ubu_clang: &build_ubuntu1904_clang docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1904-clang-<< pipeline.parameters.docker-image-rev >> + - image: ethereum/solidity-buildpack-deps:ubuntu1904-clang-<< pipeline.parameters.ubuntu-1904-clang-docker-image-rev >> environment: CC: clang CXX: clang++ @@ -324,7 +330,7 @@ jobs: b_ubu: &build_ubuntu1904 docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1904-<< pipeline.parameters.docker-image-rev >> + - image: ethereum/solidity-buildpack-deps:ubuntu1904-<< pipeline.parameters.ubuntu-1904-docker-image-rev >> steps: - checkout - run: *run_build @@ -339,7 +345,7 @@ jobs: b_ubu18: &build_ubuntu1804 docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1804-<< pipeline.parameters.docker-image-rev >> + - image: ethereum/solidity-buildpack-deps:ubuntu1804-<< pipeline.parameters.ubuntu-1804-docker-image-rev >> environment: CMAKE_OPTIONS: -DCMAKE_CXX_FLAGS=-O2 CMAKE_BUILD_TYPE: RelWithDebugInfo @@ -546,7 +552,7 @@ jobs: b_docs: docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1904-<< pipeline.parameters.docker-image-rev >> + - image: ethereum/solidity-buildpack-deps:ubuntu1904-<< pipeline.parameters.ubuntu-1904-docker-image-rev >> steps: - checkout - run: *setup_prerelease_commit_hash @@ -571,7 +577,7 @@ jobs: t_ubu_cli: &t_ubu_cli docker: - - image: ethereum/solidity-buildpack-deps:ubuntu1904-<< pipeline.parameters.docker-image-rev >> + - image: ethereum/solidity-buildpack-deps:ubuntu1904-<< pipeline.parameters.ubuntu-1904-docker-image-rev >> environment: TERM: xterm steps: From dd9009eba6cc6f901123174a5932090a611c73ba Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Tue, 18 Feb 2020 12:57:48 +0100 Subject: [PATCH 29/92] TestFileParser: Adding new keyword wei for expressing function value --- Changelog.md | 3 +- test/libsolidity/SemanticTest.cpp | 6 +-- .../semanticTests/functionCall/value_test.sol | 8 +++ .../inlineAssembly/selfbalance.sol | 2 +- .../receive/empty_calldata_calls_receive.sol | 3 +- .../semanticTests/revertStrings/transfer.sol | 2 +- .../libsolidity/semanticTests/smoke/basic.sol | 3 +- .../semanticTests/smoke/constructor.sol | 2 +- .../semanticTests/smoke/fallback.sol | 4 +- test/libsolidity/util/BytesUtils.cpp | 2 +- test/libsolidity/util/SoltestTypes.h | 33 +++++++++++- test/libsolidity/util/TestFileParser.cpp | 16 ++++-- test/libsolidity/util/TestFileParser.h | 3 +- test/libsolidity/util/TestFileParserTests.cpp | 50 +++++++++++++------ test/libsolidity/util/TestFunctionCall.cpp | 4 +- .../util/TestFunctionCallTests.cpp | 26 +++++----- 16 files changed, 117 insertions(+), 50 deletions(-) create mode 100644 test/libsolidity/semanticTests/functionCall/value_test.sol diff --git a/Changelog.md b/Changelog.md index c83b26e830e2..9981ea25fd99 100644 --- a/Changelog.md +++ b/Changelog.md @@ -7,7 +7,7 @@ Compiler Features: Bugfixes: - + * isoltest: Added new keyword `wei` to express function value in semantic tests ### 0.6.3 (2020-02-18) @@ -31,7 +31,6 @@ Bugfixes: * Type Checker: Make invalid calls to uncallable types fatal errors instead of regular. - ### 0.6.2 (2020-01-27) Language Features: diff --git a/test/libsolidity/SemanticTest.cpp b/test/libsolidity/SemanticTest.cpp index 35ea80bd9509..b1ef9372aa4a 100644 --- a/test/libsolidity/SemanticTest.cpp +++ b/test/libsolidity/SemanticTest.cpp @@ -133,7 +133,7 @@ TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePref else { if (test.call().isConstructor) - deploy("", test.call().value, test.call().arguments.rawBytes(), libraries); + deploy("", test.call().value.value, test.call().arguments.rawBytes(), libraries); else soltestAssert(deploy("", 0, bytes(), libraries), "Failed to deploy contract."); constructed = true; @@ -151,7 +151,7 @@ TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePref { bytes output; if (test.call().useCallWithoutSignature) - output = callLowLevel(test.call().arguments.rawBytes(), test.call().value); + output = callLowLevel(test.call().arguments.rawBytes(), test.call().value.value); else { soltestAssert( @@ -161,7 +161,7 @@ TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePref output = callContractFunctionWithValueNoEncoding( test.call().signature, - test.call().value, + test.call().value.value, test.call().arguments.rawBytes() ); } diff --git a/test/libsolidity/semanticTests/functionCall/value_test.sol b/test/libsolidity/semanticTests/functionCall/value_test.sol new file mode 100644 index 000000000000..582cda9f6d91 --- /dev/null +++ b/test/libsolidity/semanticTests/functionCall/value_test.sol @@ -0,0 +1,8 @@ +contract C { + function f() public payable returns (uint) { + return msg.value; + } +} +// ---- +// f(), 1 ether -> 1000000000000000000 +// f(), 1 wei -> 1 \ No newline at end of file diff --git a/test/libsolidity/semanticTests/inlineAssembly/selfbalance.sol b/test/libsolidity/semanticTests/inlineAssembly/selfbalance.sol index 8a0a5caae55b..b16678833981 100644 --- a/test/libsolidity/semanticTests/inlineAssembly/selfbalance.sol +++ b/test/libsolidity/semanticTests/inlineAssembly/selfbalance.sol @@ -9,4 +9,4 @@ contract C { // EVMVersion: >=istanbul // compileViaYul: also // ---- -// f(), 254 ether -> 254 +// f(), 254 wei -> 254 diff --git a/test/libsolidity/semanticTests/receive/empty_calldata_calls_receive.sol b/test/libsolidity/semanticTests/receive/empty_calldata_calls_receive.sol index 42f56413946f..ce04cd73745b 100644 --- a/test/libsolidity/semanticTests/receive/empty_calldata_calls_receive.sol +++ b/test/libsolidity/semanticTests/receive/empty_calldata_calls_receive.sol @@ -6,7 +6,8 @@ contract A { // x() -> 0 // () // x() -> 1 -// (), 1 ether +// (), 1 wei // x() -> 2 +// x(), 1 wei -> FAILURE // (): hex"00" -> FAILURE // (), 1 ether: hex"00" -> FAILURE diff --git a/test/libsolidity/semanticTests/revertStrings/transfer.sol b/test/libsolidity/semanticTests/revertStrings/transfer.sol index fcf971a5c927..0d766bd1edc2 100644 --- a/test/libsolidity/semanticTests/revertStrings/transfer.sol +++ b/test/libsolidity/semanticTests/revertStrings/transfer.sol @@ -21,7 +21,7 @@ contract C { // EVMVersion: >=byzantium // revertStrings: debug // ---- -// (), 10 ether -> +// (), 10 wei -> // g() -> 10 // f() -> FAILURE, hex"08c379a0", 0x20, 10, "no_receive" // h() -> FAILURE diff --git a/test/libsolidity/semanticTests/smoke/basic.sol b/test/libsolidity/semanticTests/smoke/basic.sol index 403b13c7bef9..892ee8702d58 100644 --- a/test/libsolidity/semanticTests/smoke/basic.sol +++ b/test/libsolidity/semanticTests/smoke/basic.sol @@ -30,7 +30,8 @@ contract C { } // ---- // d() -> -// e(), 1 ether -> 1 +// e(), 1 wei -> 1 +// e(), 1 ether -> 1000000000000000000 // f(uint256): 3 -> 3, 3 // g() -> 2, 3 // h(uint256,uint256): 1, -2 -> 3 diff --git a/test/libsolidity/semanticTests/smoke/constructor.sol b/test/libsolidity/semanticTests/smoke/constructor.sol index 9aac2ed7b484..e3bee950f076 100644 --- a/test/libsolidity/semanticTests/smoke/constructor.sol +++ b/test/libsolidity/semanticTests/smoke/constructor.sol @@ -11,7 +11,7 @@ contract C { } } // ---- -// constructor(), 2 ether: 3 -> +// constructor(), 2 wei: 3 -> // state() -> 3 // balance() -> 2 // update(uint256): 4 diff --git a/test/libsolidity/semanticTests/smoke/fallback.sol b/test/libsolidity/semanticTests/smoke/fallback.sol index 72bc27ac7015..ff0e65bffeee 100644 --- a/test/libsolidity/semanticTests/smoke/fallback.sol +++ b/test/libsolidity/semanticTests/smoke/fallback.sol @@ -16,8 +16,8 @@ contract A { // data() -> 2 // externalData() -> 0x20, 2, left(0x42ef) // balance() -> 0 -// (), 1 ether +// (), 1 wei // balance() -> 1 -// (), 2 ether: hex"fefe" +// (), 2 wei: hex"fefe" // balance() -> 2 // externalData() -> 0x20, 2, left(0xfefe) \ No newline at end of file diff --git a/test/libsolidity/util/BytesUtils.cpp b/test/libsolidity/util/BytesUtils.cpp index 6377fbe2e724..0c5518afb60e 100644 --- a/test/libsolidity/util/BytesUtils.cpp +++ b/test/libsolidity/util/BytesUtils.cpp @@ -225,7 +225,7 @@ string BytesUtils::formatRawBytes( { bytes byteRange{it, it + static_cast(parameter.abiType.size)}; - os << _linePrefix << byteRange; + os << _linePrefix << formatBytes(byteRange, parameter.abiType); if (¶meter != ¶meters.back()) os << endl; diff --git a/test/libsolidity/util/SoltestTypes.h b/test/libsolidity/util/SoltestTypes.h index 0fec97eb1080..4c1cb01fd112 100644 --- a/test/libsolidity/util/SoltestTypes.h +++ b/test/libsolidity/util/SoltestTypes.h @@ -50,6 +50,7 @@ namespace solidity::frontend::test T(Identifier, "identifier", 0) \ /* type keywords */ \ K(Ether, "ether", 0) \ + K(Wei, "wei", 0) \ K(Hex, "hex", 0) \ K(Boolean, "boolean", 0) \ /* special keywords */ \ @@ -229,6 +230,32 @@ struct FunctionCallArgs } }; +/// Units that can be used to express function value +enum class FunctionValueUnit +{ + Wei, + Ether +}; + +/// Holds value along with unit it was expressed in originally. +/// Value should be always converted to wei, no meter on which unit it was originally +struct FunctionValue +{ + u256 value; + FunctionValueUnit unit = FunctionValueUnit::Wei; +}; + +inline bool operator==(FunctionValue const& _a, FunctionValue const& _b) +{ + return _a.value == _b.value; +} + +inline std::ostream& operator<<(std::ostream& _os, FunctionValue const& _v) +{ + _os << _v.value << (_v.unit == FunctionValueUnit::Wei ? " wei" : " ether"); + return _os; +} + /** * Represents a function call read from an input stream. It contains the signature, the * arguments, an optional ether value and an expected execution result. @@ -237,8 +264,10 @@ struct FunctionCall { /// Signature of the function call, e.g. `f(uint256, uint256)`. std::string signature; - /// Optional `ether` value that can be send with the call. - u256 value; + /// Optional value that can be sent with the call. + /// Value is expressed in wei, smallest unit of ether + /// Value has a field unit which represents denomination on which value was expressed originally + FunctionValue value; /// Object that holds all function parameters in their `bytes` /// representations given by the contract ABI. FunctionCallArgs arguments; diff --git a/test/libsolidity/util/TestFileParser.cpp b/test/libsolidity/util/TestFileParser.cpp index 4be96c88d452..50ca68f632c6 100644 --- a/test/libsolidity/util/TestFileParser.cpp +++ b/test/libsolidity/util/TestFileParser.cpp @@ -88,6 +88,7 @@ vector TestFileParser::parseFunctionCall tie(call.signature, call.useCallWithoutSignature) = parseFunctionSignature(); if (accept(Token::Comma, true)) call.value = parseFunctionCallValue(); + if (accept(Token::Colon, true)) call.arguments = parseFunctionCallArguments(); @@ -199,13 +200,19 @@ pair TestFileParser::parseFunctionSignature() return {signature, !hasName}; } -u256 TestFileParser::parseFunctionCallValue() +FunctionValue TestFileParser::parseFunctionCallValue() { try { - u256 value{parseDecimalNumber()}; - expect(Token::Ether); - return value; + u256 value{ parseDecimalNumber() }; + Token token = m_scanner.currentToken(); + if (token != Token::Ether && token != Token::Wei) + throw Error(Error::Type::ParserError, "Invalid value unit provided. Coins can be wei or ether."); + + m_scanner.scanNextToken(); + + FunctionValueUnit unit = token == Token::Wei ? FunctionValueUnit::Wei : FunctionValueUnit::Ether; + return { (unit == FunctionValueUnit::Wei ? u256(1) : exp256(u256(10), u256(18))) * value, unit }; } catch (std::exception const&) { @@ -467,6 +474,7 @@ void TestFileParser::Scanner::scanNextToken() if (_literal == "true") return TokenDesc{Token::Boolean, _literal}; if (_literal == "false") return TokenDesc{Token::Boolean, _literal}; if (_literal == "ether") return TokenDesc{Token::Ether, _literal}; + if (_literal == "wei") return TokenDesc{Token::Wei, _literal}; if (_literal == "left") return TokenDesc{Token::Left, _literal}; if (_literal == "library") return TokenDesc{Token::Library, _literal}; if (_literal == "right") return TokenDesc{Token::Right, _literal}; diff --git a/test/libsolidity/util/TestFileParser.h b/test/libsolidity/util/TestFileParser.h index a10bb1114ee2..502279d2dbb1 100644 --- a/test/libsolidity/util/TestFileParser.h +++ b/test/libsolidity/util/TestFileParser.h @@ -39,6 +39,7 @@ namespace solidity::frontend::test * // f(uint256, uint256): 1, 1 # Signature and comma-separated list of arguments # * // -> 1, 1 # Expected result value # * // g(), 2 ether # (Optional) Ether to be send with the call # + * // g(), 1 wei # (Optional) Wei to be sent with the call # * // -> 2, 3 * // h(uint256), 1 ether: 42 * // -> FAILURE # If REVERT or other EVM failure was detected # @@ -134,7 +135,7 @@ class TestFileParser /// Parses the optional ether value that can be passed alongside the /// function call arguments. Throws an InvalidEtherValueEncoding exception /// if given value cannot be converted to `u256`. - u256 parseFunctionCallValue(); + FunctionValue parseFunctionCallValue(); /// Parses a comma-separated list of arguments passed with a function call. /// Does not check for a potential mismatch between the signature and the number diff --git a/test/libsolidity/util/TestFileParserTests.cpp b/test/libsolidity/util/TestFileParserTests.cpp index c7e4cb22a9f4..82d885b9e5c6 100644 --- a/test/libsolidity/util/TestFileParserTests.cpp +++ b/test/libsolidity/util/TestFileParserTests.cpp @@ -51,7 +51,7 @@ void testFunctionCall( bool _failure = true, bytes _arguments = bytes{}, bytes _expectations = bytes{}, - u256 _value = 0, + FunctionValue _value = { 0 }, string _argumentComment = "", string _expectationComment = "", vector _rawArguments = vector{}, @@ -143,7 +143,7 @@ BOOST_AUTO_TEST_CASE(call_arguments_comments_success) false, fmt::encodeArgs(1, 1), fmt::encodeArgs(), - 0, + {0}, " Comment on the parameters. ", " This call should not return a value, but still succeed. " ); @@ -154,7 +154,7 @@ BOOST_AUTO_TEST_CASE(call_arguments_comments_success) false, fmt::encodeArgs(), fmt::encodeArgs(1), - 0, + {0}, " Comment on no parameters. ", " This comment should be parsed. " ); @@ -176,7 +176,7 @@ BOOST_AUTO_TEST_CASE(simple_single_line_call_comment_success) false, fmt::encodeArgs(1), fmt::encodeArgs(), - 0, + {0}, "", " f(uint256) does not return a value. " ); @@ -275,7 +275,7 @@ BOOST_AUTO_TEST_CASE(call_comments) false, fmt::encodeArgs(), fmt::encodeArgs(1), - 0, + {0}, " Parameter comment ", " Expectation comment " ); @@ -286,7 +286,7 @@ BOOST_AUTO_TEST_CASE(call_comments) false, fmt::encodeArgs(), fmt::encodeArgs(1), - 0, + {0}, " Parameter comment ", " Expectation comment " ); @@ -295,7 +295,7 @@ BOOST_AUTO_TEST_CASE(call_comments) BOOST_AUTO_TEST_CASE(call_arguments) { char const* source = R"( - // f(uint256), 314 ether: 5 # optional ether value # + // f(uint256), 314 wei: 5 # optional wei value # // -> 4 )"; auto const calls = parse(source); @@ -307,7 +307,27 @@ BOOST_AUTO_TEST_CASE(call_arguments) false, fmt::encodeArgs(5), fmt::encodeArgs(4), - 314, + {314}, + " optional wei value " + ); +} + +BOOST_AUTO_TEST_CASE(call_arguments_ether) +{ + char const* source = R"( + // f(uint256), 1 ether: 5 # optional ether value # + // -> 4 + )"; + auto const calls = parse(source); + BOOST_REQUIRE_EQUAL(calls.size(), 1); + testFunctionCall( + calls.at(0), + Mode::MultiLine, + "f(uint256)", + false, + fmt::encodeArgs(5), + fmt::encodeArgs(4), + {exp256(u256(10), u256(18)) , FunctionValueUnit::Ether}, " optional ether value " ); } @@ -521,7 +541,7 @@ BOOST_AUTO_TEST_CASE(call_arguments_tuple_of_tuples) false, fmt::encodeArgs(), fmt::encodeArgs(), - 0, + {0}, " f(S memory s, uint256 b) " ); } @@ -565,7 +585,7 @@ BOOST_AUTO_TEST_CASE(call_arguments_mismatch) false, fmt::encodeArgs(1, 2), fmt::encodeArgs(1), - 0, + {0}, " This only throws at runtime " ); } @@ -594,7 +614,7 @@ BOOST_AUTO_TEST_CASE(call_multiple_arguments) BOOST_AUTO_TEST_CASE(call_multiple_arguments_mixed_format) { char const* source = R"( - // test(uint256, uint256), 314 ether: + // test(uint256, uint256), 314 wei: // 1, -2 // -> -1, 2 )"; @@ -607,7 +627,7 @@ BOOST_AUTO_TEST_CASE(call_multiple_arguments_mixed_format) false, fmt::encodeArgs(1, -2), fmt::encodeArgs(-1, 2), - 314 + {314} ); } @@ -668,7 +688,7 @@ BOOST_AUTO_TEST_CASE(call_raw_arguments) false, fmt::encodeArgs(1, -2, -3), fmt::encodeArgs(), - 0, + {0}, "", "", {"1", "-2", "-3"} @@ -899,7 +919,7 @@ BOOST_AUTO_TEST_CASE(constructor) false, {}, {}, - 0, + {0}, "", "", {}, @@ -921,7 +941,7 @@ BOOST_AUTO_TEST_CASE(library) false, {}, {}, - 0, + {0}, "", "", {}, diff --git a/test/libsolidity/util/TestFunctionCall.cpp b/test/libsolidity/util/TestFunctionCall.cpp index 22dc82aa802d..43f19034986b 100644 --- a/test/libsolidity/util/TestFunctionCall.cpp +++ b/test/libsolidity/util/TestFunctionCall.cpp @@ -63,8 +63,8 @@ string TestFunctionCall::format( /// Formats the function signature. This is the same independent from the display-mode. stream << _linePrefix << newline << ws << m_call.signature; - if (m_call.value > u256(0)) - stream << comma << ws << m_call.value << ws << ether; + if (m_call.value.value > u256(0)) + stream << comma << ws << m_call.value.value << ws << ether; if (!m_call.arguments.rawBytes().empty()) { string output = formatRawParameters(m_call.arguments.parameters, _linePrefix); diff --git a/test/libsolidity/util/TestFunctionCallTests.cpp b/test/libsolidity/util/TestFunctionCallTests.cpp index f3f977bbf7af..7b34cbe8ecd5 100644 --- a/test/libsolidity/util/TestFunctionCallTests.cpp +++ b/test/libsolidity/util/TestFunctionCallTests.cpp @@ -40,7 +40,7 @@ BOOST_AUTO_TEST_CASE(format_unsigned_singleline) Parameter param{expectedBytes, "1", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(uint8)", 0, arguments, expectations}; + FunctionCall call{"f(uint8)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -60,7 +60,7 @@ BOOST_AUTO_TEST_CASE(format_unsigned_singleline_signed_encoding) Parameter param{expectedBytes, "1", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(uint8)", 0, arguments, expectations}; + FunctionCall call{"f(uint8)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -80,7 +80,7 @@ BOOST_AUTO_TEST_CASE(format_unsigned_multiline) Parameter result{expectedBytes, "1", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{result}, false, string{}}; FunctionCallArgs arguments{vector{}, string{}}; - FunctionCall call{"f(uint8)", 0, arguments, expectations}; + FunctionCall call{"f(uint8)", {0}, arguments, expectations}; call.omitsArrow = false; call.displayMode = FunctionCall::DisplayMode::MultiLine; TestFunctionCall test{call}; @@ -95,7 +95,7 @@ BOOST_AUTO_TEST_CASE(format_multiple_unsigned_singleline) Parameter param{expectedBytes, "1", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param, param}, false, string{}}; FunctionCallArgs arguments{vector{param, param}, string{}}; - FunctionCall call{"f(uint8, uint8)", 0, arguments, expectations}; + FunctionCall call{"f(uint8, uint8)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -109,7 +109,7 @@ BOOST_AUTO_TEST_CASE(format_signed_singleline) Parameter param{expectedBytes, "-1", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(int8)", 0, arguments, expectations}; + FunctionCall call{"f(int8)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -129,7 +129,7 @@ BOOST_AUTO_TEST_CASE(format_hex_singleline) Parameter param{expectedBytes, "0x31", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(bytes32)", 0, arguments, expectations}; + FunctionCall call{"f(bytes32)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -151,7 +151,7 @@ BOOST_AUTO_TEST_CASE(format_hex_string_singleline) Parameter param{expectedBytes, "hex\"4200ef\"", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(string)", 0, arguments, expectations}; + FunctionCall call{"f(string)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -165,7 +165,7 @@ BOOST_AUTO_TEST_CASE(format_bool_true_singleline) Parameter param{expectedBytes, "true", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(bool)", 0, arguments, expectations}; + FunctionCall call{"f(bool)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -186,7 +186,7 @@ BOOST_AUTO_TEST_CASE(format_bool_false_singleline) Parameter param{expectedBytes, "false", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(bool)", 0, arguments, expectations}; + FunctionCall call{"f(bool)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -200,7 +200,7 @@ BOOST_AUTO_TEST_CASE(format_bool_left_singleline) Parameter param{expectedBytes, "left(false)", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(bool)", 0, arguments, expectations}; + FunctionCall call{"f(bool)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -215,7 +215,7 @@ BOOST_AUTO_TEST_CASE(format_hex_number_right_singleline) Parameter param{expectedBytes, "right(0x42)", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(bool)", 0, arguments, expectations}; + FunctionCall call{"f(bool)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -229,7 +229,7 @@ BOOST_AUTO_TEST_CASE(format_empty_byte_range) Parameter param{expectedBytes, "1", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{param}, false, string{}}; FunctionCallArgs arguments{vector{}, string{}}; - FunctionCall call{"f()", 0, arguments, expectations}; + FunctionCall call{"f()", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; @@ -243,7 +243,7 @@ BOOST_AUTO_TEST_CASE(format_failure_singleline) Parameter param{expectedBytes, "1", abiType, FormatInfo{}}; FunctionCallExpectations expectations{vector{}, true, string{}}; FunctionCallArgs arguments{vector{param}, string{}}; - FunctionCall call{"f(uint8)", 0, arguments, expectations}; + FunctionCall call{"f(uint8)", {0}, arguments, expectations}; call.omitsArrow = false; TestFunctionCall test{call}; From 8b6bfabfee72561cb95b0b0125389d9c412e0e85 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Thu, 20 Feb 2020 15:25:20 +0530 Subject: [PATCH 30/92] Docker: Upgrade libprotobuf-mutator inside clang docker image --- .circleci/config.yml | 2 +- .circleci/docker/Dockerfile.clang.ubuntu1904 | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d95b0a73eb3..f8f142412f9d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ parameters: default: "4" ubuntu-1904-clang-docker-image-rev: type: string - default: "4" + default: "5" defaults: diff --git a/.circleci/docker/Dockerfile.clang.ubuntu1904 b/.circleci/docker/Dockerfile.clang.ubuntu1904 index 14c9b1339ec0..cf790c84ea79 100644 --- a/.circleci/docker/Dockerfile.clang.ubuntu1904 +++ b/.circleci/docker/Dockerfile.clang.ubuntu1904 @@ -51,9 +51,10 @@ ENV CC clang ENV CXX clang++ # Boost -RUN git clone --recursive -b boost-1.69.0 https://github.com/boostorg/boost.git \ +RUN git clone -b boost-1.69.0 https://github.com/boostorg/boost.git \ /usr/src/boost; \ cd /usr/src/boost; \ + git submodule update --init --recursive; \ ./bootstrap.sh --with-toolset=clang --prefix=/usr; \ ./b2 toolset=clang headers; \ ./b2 toolset=clang variant=release \ @@ -76,7 +77,7 @@ RUN set -ex; \ git clone https://github.com/google/libprotobuf-mutator.git \ /usr/src/libprotobuf-mutator; \ cd /usr/src/libprotobuf-mutator; \ - git checkout d1fe8a7d8ae18f3d454f055eba5213c291986f21; \ + git checkout 3521f47a2828da9ace403e4ecc4aece1a84feb36; \ mkdir build; \ cd build; \ cmake .. -GNinja -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON \ From 1f51716227853d222a0c970c31365939f66c1e73 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 16 Jan 2020 18:56:05 +0100 Subject: [PATCH 31/92] Type checking for Yul. --- libyul/AsmAnalysis.cpp | 110 +++++++++++++++++++++++++++++------------ libyul/AsmAnalysis.h | 9 +++- libyul/Dialect.cpp | 9 ++++ libyul/Dialect.h | 7 ++- 4 files changed, 101 insertions(+), 34 deletions(-) diff --git a/libyul/AsmAnalysis.cpp b/libyul/AsmAnalysis.cpp index 89db52d8d4bd..91f8181003ec 100644 --- a/libyul/AsmAnalysis.cpp +++ b/libyul/AsmAnalysis.cpp @@ -82,7 +82,6 @@ AsmAnalysisInfo AsmAnalyzer::analyzeStrictAssertCorrect(Dialect const& _dialect, vector AsmAnalyzer::operator()(Literal const& _literal) { expectValidType(_literal.type, _literal.location); - if (_literal.kind == LiteralKind::String && _literal.value.str().size() > 32) typeError( _literal.location, @@ -93,6 +92,13 @@ vector AsmAnalyzer::operator()(Literal const& _literal) else if (_literal.kind == LiteralKind::Boolean) yulAssert(_literal.value == "true"_yulstring || _literal.value == "false"_yulstring, ""); + if (!m_dialect.validTypeForLiteral(_literal.kind, _literal.value, _literal.type)) + typeError( + _literal.location, + "Invalid type \"" + _literal.type.str() + "\" for literal \"" + _literal.value.str() + "\"." + ); + + return {_literal.type}; } @@ -179,7 +185,8 @@ void AsmAnalyzer::operator()(Assignment const& _assignment) ); for (size_t i = 0; i < numVariables; ++i) - checkAssignment(_assignment.variableNames[i]); + if (i < types.size()) + checkAssignment(_assignment.variableNames[i], types[i]); } void AsmAnalyzer::operator()(VariableDeclaration const& _varDecl) @@ -193,6 +200,9 @@ void AsmAnalyzer::operator()(VariableDeclaration const& _varDecl) yul::IdentifierContext::VariableDeclaration, m_currentScope->insideFunction() ); + for (auto const& variable: _varDecl.variables) + expectValidType(variable.type, variable.location); + if (_varDecl.value) { vector types = std::visit(*this, *_varDecl.value); @@ -204,15 +214,25 @@ void AsmAnalyzer::operator()(VariableDeclaration const& _varDecl) to_string(types.size()) + " values." ); + + for (size_t i = 0; i < _varDecl.variables.size(); ++i) + { + YulString givenType = m_dialect.defaultType; + if (i < types.size()) + givenType = types[i]; + TypedName const& variable = _varDecl.variables[i]; + if (variable.type != givenType) + typeError( + variable.location, + "Assigning value of type \"" + givenType.str() + "\" to variable of type \"" + variable.type.str() + "." + ); + } } for (TypedName const& variable: _varDecl.variables) - { - expectValidType(variable.type, variable.location); m_activeVariables.insert(&std::get( m_currentScope->identifiers.at(variable.name)) ); - } } void AsmAnalyzer::operator()(FunctionDefinition const& _funDef) @@ -291,6 +311,11 @@ vector AsmAnalyzer::operator()(FunctionCall const& _funCall) ); } } + std::reverse(argTypes.begin(), argTypes.end()); + + if (parameterTypes && parameterTypes->size() == argTypes.size()) + for (size_t i = 0; i < parameterTypes->size(); ++i) + expectType((*parameterTypes)[i], argTypes[i], locationOf(_funCall.arguments[i])); if (m_success) { @@ -306,7 +331,7 @@ vector AsmAnalyzer::operator()(FunctionCall const& _funCall) void AsmAnalyzer::operator()(If const& _if) { - expectExpression(*_if.condition); + expectBoolExpression(*_if.condition); (*this)(_if.body); } @@ -315,32 +340,15 @@ void AsmAnalyzer::operator()(Switch const& _switch) { yulAssert(_switch.expression, ""); - expectExpression(*_switch.expression); - - YulString caseType; - bool mismatchingTypes = false; - for (auto const& _case: _switch.cases) - if (_case.value) - { - if (caseType.empty()) - caseType = _case.value->type; - else if (caseType != _case.value->type) - { - mismatchingTypes = true; - break; - } - } - if (mismatchingTypes) - m_errorReporter.typeError( - _switch.location, - "Switch cases have non-matching types." - ); + YulString valueType = expectExpression(*_switch.expression); set cases; for (auto const& _case: _switch.cases) { if (_case.value) { + expectType(valueType, _case.value->type, _case.value->location); + // We cannot use "expectExpression" here because *_case.value is not an // Expression and would be converted to an Expression otherwise. (*this)(*_case.value); @@ -366,8 +374,7 @@ void AsmAnalyzer::operator()(ForLoop const& _for) // condition, the body and the post part inside. m_currentScope = &scope(&_for.pre); - expectExpression(*_for.condition); - + expectBoolExpression(*_for.condition); // backup outer for-loop & create new state auto outerForLoop = m_currentForLoop; m_currentForLoop = &_for; @@ -403,10 +410,24 @@ YulString AsmAnalyzer::expectExpression(Expression const& _expr) return types.empty() ? m_dialect.defaultType : types.front(); } -void AsmAnalyzer::checkAssignment(Identifier const& _variable) +void AsmAnalyzer::expectBoolExpression(Expression const& _expr) +{ + YulString type = expectExpression(_expr); + if (type != m_dialect.boolType) + typeError(locationOf(_expr), + "Expected a value of boolean type \"" + + m_dialect.boolType.str() + + "\" but got \"" + + type.str() + + "\"" + ); +} + +void AsmAnalyzer::checkAssignment(Identifier const& _variable, YulString _valueType) { yulAssert(!_variable.name.empty(), ""); size_t numErrorsBefore = m_errorReporter.errors().size(); + YulString const* variableType = nullptr; bool found = false; if (Scope::Identifier const* var = m_currentScope->lookup(_variable.name)) { @@ -418,6 +439,8 @@ void AsmAnalyzer::checkAssignment(Identifier const& _variable) _variable.location, "Variable " + _variable.name.str() + " used before it was declared." ); + else + variableType = &std::get(*var).type; found = true; } else if (m_resolver) @@ -427,6 +450,7 @@ void AsmAnalyzer::checkAssignment(Identifier const& _variable) if (variableSize != size_t(-1)) { found = true; + variableType = &m_dialect.defaultType; yulAssert(variableSize == 1, "Invalid stack size of external reference."); } } @@ -438,6 +462,17 @@ void AsmAnalyzer::checkAssignment(Identifier const& _variable) if (numErrorsBefore == m_errorReporter.errors().size()) declarationError(_variable.location, "Variable not found or variable not lvalue."); } + if (variableType && *variableType != _valueType) + typeError(_variable.location, + "Assigning a value of type \"" + + _valueType.str() + + "\" to a variable of type \"" + + variableType->str() + + "\"." + ); + + if (m_success) + yulAssert(variableType, ""); } Scope& AsmAnalyzer::scope(Block const* _block) @@ -447,15 +482,28 @@ Scope& AsmAnalyzer::scope(Block const* _block) yulAssert(scopePtr, "Scope requested but not present."); return *scopePtr; } + void AsmAnalyzer::expectValidType(YulString _type, SourceLocation const& _location) { - if (!_type.empty() && !m_dialect.types.count(_type)) - m_errorReporter.typeError( + if (!m_dialect.types.count(_type)) + typeError( _location, "\"" + _type.str() + "\" is not a valid type (user defined types are not yet supported)." ); } +void AsmAnalyzer::expectType(YulString _expectedType, YulString _givenType, SourceLocation const& _location) +{ + if (_expectedType != _givenType) + typeError(_location, + "Expected a value of type \"" + + _expectedType.str() + + "\" but got \"" + + _givenType.str() + + "\"" + ); +} + bool AsmAnalyzer::warnOnInstructions(std::string const& _instructionIdentifier, langutil::SourceLocation const& _location) { auto const builtin = EVMDialect::strictAssemblyForEVM(EVMVersion{}).builtin(YulString(_instructionIdentifier)); diff --git a/libyul/AsmAnalysis.h b/libyul/AsmAnalysis.h index dded6d5ab2be..853f9f654f34 100644 --- a/libyul/AsmAnalysis.h +++ b/libyul/AsmAnalysis.h @@ -96,13 +96,18 @@ class AsmAnalyzer /// Visits the expression, expects that it evaluates to exactly one value and /// returns the type. Reports errors on errors and returns the default type. YulString expectExpression(Expression const& _expr); + /// Vists the expression and expects it to return a single boolean value. + /// Reports an error otherwise. + void expectBoolExpression(Expression const& _expr); bool expectDeposit(int _deposit, int _oldHeight, langutil::SourceLocation const& _location); - /// Verifies that a variable to be assigned to exists and can be assigned to. - void checkAssignment(Identifier const& _variable); + /// Verifies that a variable to be assigned to exists, can be assigned to + /// and has the same type as the value. + void checkAssignment(Identifier const& _variable, YulString _valueType); Scope& scope(Block const* _block); void expectValidType(YulString _type, langutil::SourceLocation const& _location); + void expectType(YulString _expectedType, YulString _givenType, langutil::SourceLocation const& _location); bool warnOnInstructions(evmasm::Instruction _instr, langutil::SourceLocation const& _location); bool warnOnInstructions(std::string const& _instrIdentifier, langutil::SourceLocation const& _location); diff --git a/libyul/Dialect.cpp b/libyul/Dialect.cpp index 6bc056a759a0..4d5cccd41512 100644 --- a/libyul/Dialect.cpp +++ b/libyul/Dialect.cpp @@ -19,10 +19,19 @@ */ #include +#include using namespace solidity::yul; using namespace std; +bool Dialect::validTypeForLiteral(LiteralKind _kind, YulString, YulString _type) const +{ + if (_kind == LiteralKind::Boolean) + return _type == boolType; + else + return true; +} + Dialect const& Dialect::yulDeprecated() { static unique_ptr dialect; diff --git a/libyul/Dialect.h b/libyul/Dialect.h index c25ba1ac3569..3d261fdeefc9 100644 --- a/libyul/Dialect.h +++ b/libyul/Dialect.h @@ -33,6 +33,7 @@ namespace solidity::yul class YulString; using Type = YulString; +enum class LiteralKind; struct BuiltinFunction { @@ -52,7 +53,7 @@ struct Dialect: boost::noncopyable YulString defaultType; /// Type used for the literals "true" and "false". YulString boolType; - std::set types; + std::set types = {{}}; /// @returns the builtin function of the given name or a nullptr if it is not a builtin function. virtual BuiltinFunction const* builtin(YulString /*_name*/) const { return nullptr; } @@ -61,6 +62,10 @@ struct Dialect: boost::noncopyable virtual BuiltinFunction const* equalityFunction() const { return nullptr; } virtual BuiltinFunction const* booleanNegationFunction() const { return nullptr; } + /// Check whether the given type is legal for the given literal value. + /// Should only be called if the type exists in the dialect at all. + virtual bool validTypeForLiteral(LiteralKind _kind, YulString _value, YulString _type) const; + virtual std::set fixedFunctionNames() const { return {}; } Dialect() = default; From 6eec968365e22c28975dce970ddd93223316f41d Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 22 Jan 2020 20:11:33 +0100 Subject: [PATCH 32/92] Test updates. --- test/libyul/Inliner.cpp | 2 +- test/libyul/Parser.cpp | 22 ++----------------- .../disambiguator/for_statement.yul | 7 ++++-- 3 files changed, 8 insertions(+), 23 deletions(-) diff --git a/test/libyul/Inliner.cpp b/test/libyul/Inliner.cpp index e564ac2b8779..de47f48854db 100644 --- a/test/libyul/Inliner.cpp +++ b/test/libyul/Inliner.cpp @@ -85,7 +85,7 @@ BOOST_AUTO_TEST_CASE(simple_inside_structures) BOOST_CHECK_EQUAL(inlinableFunctions("{" "function g(a:u256) -> b:u256 { b := a }" "for {" - "} 1:u256 {" + "} true {" "function f() -> x:u256 { x := g(2:u256) }" "}" "{" diff --git a/test/libyul/Parser.cpp b/test/libyul/Parser.cpp index 8ec5512587a1..020f3d4c4430 100644 --- a/test/libyul/Parser.cpp +++ b/test/libyul/Parser.cpp @@ -245,16 +245,6 @@ BOOST_AUTO_TEST_CASE(optional_types) BOOST_CHECK(successParse("{ function f(a:u256) -> b {} }")); } -BOOST_AUTO_TEST_CASE(invalid_types) -{ - /// testing invalid literal - /// NOTE: these will need to change when types are compared - CHECK_ERROR("{ let x:bool := 1:invalid }", TypeError, "\"invalid\" is not a valid type (user defined types are not yet supported)."); - /// testing invalid variable declaration - CHECK_ERROR("{ let x:invalid := 1:bool }", TypeError, "\"invalid\" is not a valid type (user defined types are not yet supported)."); - CHECK_ERROR("{ function f(a:invalid) {} }", TypeError, "\"invalid\" is not a valid type (user defined types are not yet supported)."); -} - BOOST_AUTO_TEST_CASE(number_literals) { BOOST_CHECK(successParse("{ let x:u256 := 1:u256 }")); @@ -268,7 +258,7 @@ BOOST_AUTO_TEST_CASE(builtin_types) { BOOST_CHECK(successParse("{ let x:bool := true:bool }")); BOOST_CHECK(successParse("{ let x:u8 := 1:u8 }")); - BOOST_CHECK(successParse("{ let x:s8 := 1:u8 }")); + BOOST_CHECK(successParse("{ let x:s8 := 1:s8 }")); BOOST_CHECK(successParse("{ let x:u32 := 1:u32 }")); BOOST_CHECK(successParse("{ let x:s32 := 1:s32 }")); BOOST_CHECK(successParse("{ let x:u64 := 1:u64 }")); @@ -495,15 +485,7 @@ BOOST_AUTO_TEST_CASE(if_statement_invalid) { CHECK_ERROR("{ if let x:u256 {} }", ParserError, "Literal or identifier expected."); CHECK_ERROR("{ if true:bool let x:u256 := 3:u256 }", ParserError, "Expected '{' but got reserved keyword 'let'"); - // TODO change this to an error once we check types. - BOOST_CHECK(successParse("{ if 42:u256 { } }")); -} - -BOOST_AUTO_TEST_CASE(switch_case_types) -{ - CHECK_ERROR("{ switch 0:u256 case 0:u256 {} case 1:u32 {} }", TypeError, "Switch cases have non-matching types."); - // The following should be an error in the future, but this is not yet detected. - BOOST_CHECK(successParse("{ switch 0:u256 case 0:u32 {} case 1:u32 {} }")); + CHECK_ERROR("{ if 42:u256 { } }", TypeError, "Expected a value of boolean type"); } BOOST_AUTO_TEST_CASE(switch_duplicate_case) diff --git a/test/libyul/yulOptimizerTests/disambiguator/for_statement.yul b/test/libyul/yulOptimizerTests/disambiguator/for_statement.yul index ce79ef5e6bd9..c6b246ad0f4f 100644 --- a/test/libyul/yulOptimizerTests/disambiguator/for_statement.yul +++ b/test/libyul/yulOptimizerTests/disambiguator/for_statement.yul @@ -1,7 +1,8 @@ { { let a:u256, b:u256 } { - for { let a:u256 } a { a := a } { + function eq(x: u256, y: u256) -> z: bool {} + for { let a:u256 } eq(a, a) { a := a } { let b:u256 := a } } @@ -13,7 +14,9 @@ // { // { let a, b } // { -// for { let a_1 } a_1 { a_1 := a_1 } +// function eq(x, y) -> z:bool +// { } +// for { let a_1 } eq(a_1, a_1) { a_1 := a_1 } // { let b_2 := a_1 } // } // } From 9140bc52c44a32d828e51ac480159c7fd1c4a448 Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 19 Feb 2020 16:30:35 +0100 Subject: [PATCH 33/92] Tests for invalid types. --- test/libyul/yulSyntaxTests/invalid_type.yul | 7 +++++++ test/libyul/yulSyntaxTests/invalid_type2.yul | 8 ++++++++ test/libyul/yulSyntaxTests/invalid_type3.yul | 8 ++++++++ test/libyul/yulSyntaxTests/invalid_type4.yul | 9 +++++++++ 4 files changed, 32 insertions(+) create mode 100644 test/libyul/yulSyntaxTests/invalid_type.yul create mode 100644 test/libyul/yulSyntaxTests/invalid_type2.yul create mode 100644 test/libyul/yulSyntaxTests/invalid_type3.yul create mode 100644 test/libyul/yulSyntaxTests/invalid_type4.yul diff --git a/test/libyul/yulSyntaxTests/invalid_type.yul b/test/libyul/yulSyntaxTests/invalid_type.yul new file mode 100644 index 000000000000..ec16ebf01427 --- /dev/null +++ b/test/libyul/yulSyntaxTests/invalid_type.yul @@ -0,0 +1,7 @@ +{ + let x: invalidType +} +// ==== +// dialect: evmTyped +// ---- +// TypeError: (10-24): "invalidType" is not a valid type (user defined types are not yet supported). diff --git a/test/libyul/yulSyntaxTests/invalid_type2.yul b/test/libyul/yulSyntaxTests/invalid_type2.yul new file mode 100644 index 000000000000..04593e666bf0 --- /dev/null +++ b/test/libyul/yulSyntaxTests/invalid_type2.yul @@ -0,0 +1,8 @@ +{ + let x := 1:invalidType +} +// ==== +// dialect: evmTyped +// ---- +// TypeError: (15-28): "invalidType" is not a valid type (user defined types are not yet supported). +// TypeError: (10-11): Assigning value of type "invalidType" to variable of type "u256. diff --git a/test/libyul/yulSyntaxTests/invalid_type3.yul b/test/libyul/yulSyntaxTests/invalid_type3.yul new file mode 100644 index 000000000000..cf92ce7b89a4 --- /dev/null +++ b/test/libyul/yulSyntaxTests/invalid_type3.yul @@ -0,0 +1,8 @@ +{ + function f(a: invalidType) -> b: invalidType {} +} +// ==== +// dialect: evmTyped +// ---- +// TypeError: (17-31): "invalidType" is not a valid type (user defined types are not yet supported). +// TypeError: (36-50): "invalidType" is not a valid type (user defined types are not yet supported). diff --git a/test/libyul/yulSyntaxTests/invalid_type4.yul b/test/libyul/yulSyntaxTests/invalid_type4.yul new file mode 100644 index 000000000000..faf217a88a57 --- /dev/null +++ b/test/libyul/yulSyntaxTests/invalid_type4.yul @@ -0,0 +1,9 @@ +{ + switch 1 + case 8: invalidType {} +} +// ==== +// dialect: evmTyped +// ---- +// TypeError: (24-38): Expected a value of type "u256" but got "invalidType" +// TypeError: (24-38): "invalidType" is not a valid type (user defined types are not yet supported). From 2ee748b7f57bcd5696e8b778329a987d41020135 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 6 Feb 2020 19:44:14 +0100 Subject: [PATCH 34/92] Add tests --- test/libyul/yulSyntaxTests/assignment_fail.yul | 12 ++++++++++++ test/libyul/yulSyntaxTests/for_loop_condition.yul | 7 +++++++ .../yulSyntaxTests/for_loop_condition_fail.yul | 8 ++++++++ .../yulSyntaxTests/for_loop_condition_fail_ewasm.yul | 8 ++++++++ test/libyul/yulSyntaxTests/type_check_cases.yul | 8 ++++++++ test/libyul/yulSyntaxTests/type_check_cases_fail.yul | 10 ++++++++++ .../type_check_cases_fail_evmtyped.yul | 10 ++++++++++ .../yulSyntaxTests/type_check_if_condition.yul | 7 +++++++ .../yulSyntaxTests/type_check_if_condition_fail.yul | 8 ++++++++ .../yulSyntaxTests/user_defined_functions_fail.yul | 12 ++++++++++++ .../yulSyntaxTests/user_defined_functions_fine.yul | 10 ++++++++++ 11 files changed, 100 insertions(+) create mode 100644 test/libyul/yulSyntaxTests/assignment_fail.yul create mode 100644 test/libyul/yulSyntaxTests/for_loop_condition.yul create mode 100644 test/libyul/yulSyntaxTests/for_loop_condition_fail.yul create mode 100644 test/libyul/yulSyntaxTests/for_loop_condition_fail_ewasm.yul create mode 100644 test/libyul/yulSyntaxTests/type_check_cases.yul create mode 100644 test/libyul/yulSyntaxTests/type_check_cases_fail.yul create mode 100644 test/libyul/yulSyntaxTests/type_check_cases_fail_evmtyped.yul create mode 100644 test/libyul/yulSyntaxTests/type_check_if_condition.yul create mode 100644 test/libyul/yulSyntaxTests/type_check_if_condition_fail.yul create mode 100644 test/libyul/yulSyntaxTests/user_defined_functions_fail.yul create mode 100644 test/libyul/yulSyntaxTests/user_defined_functions_fine.yul diff --git a/test/libyul/yulSyntaxTests/assignment_fail.yul b/test/libyul/yulSyntaxTests/assignment_fail.yul new file mode 100644 index 000000000000..59c18bfd3d39 --- /dev/null +++ b/test/libyul/yulSyntaxTests/assignment_fail.yul @@ -0,0 +1,12 @@ +{ + let x:u256 + let y := x + let z:bool + z := y + y := z +} +// ==== +// dialect: evmTyped +// ---- +// TypeError: (51-52): Assigning a value of type "u256" to a variable of type "bool". +// TypeError: (62-63): Assigning a value of type "bool" to a variable of type "u256". diff --git a/test/libyul/yulSyntaxTests/for_loop_condition.yul b/test/libyul/yulSyntaxTests/for_loop_condition.yul new file mode 100644 index 000000000000..72ed7b71f15f --- /dev/null +++ b/test/libyul/yulSyntaxTests/for_loop_condition.yul @@ -0,0 +1,7 @@ +{ + let x:bool + for {} x {} {} +} +// ==== +// dialect: evmTyped +// ---- diff --git a/test/libyul/yulSyntaxTests/for_loop_condition_fail.yul b/test/libyul/yulSyntaxTests/for_loop_condition_fail.yul new file mode 100644 index 000000000000..903f5471a4fb --- /dev/null +++ b/test/libyul/yulSyntaxTests/for_loop_condition_fail.yul @@ -0,0 +1,8 @@ +{ + let x + for {} x {} {} +} +// ==== +// dialect: evmTyped +// ---- +// TypeError: (23-24): Expected a value of boolean type "bool" but got "u256" diff --git a/test/libyul/yulSyntaxTests/for_loop_condition_fail_ewasm.yul b/test/libyul/yulSyntaxTests/for_loop_condition_fail_ewasm.yul new file mode 100644 index 000000000000..cbe266f9ffe8 --- /dev/null +++ b/test/libyul/yulSyntaxTests/for_loop_condition_fail_ewasm.yul @@ -0,0 +1,8 @@ +{ + let x + for {} x {} {} +} +// ==== +// dialect: ewasm +// ---- +// TypeError: (23-24): Expected a value of boolean type "i32" but got "i64" diff --git a/test/libyul/yulSyntaxTests/type_check_cases.yul b/test/libyul/yulSyntaxTests/type_check_cases.yul new file mode 100644 index 000000000000..4e09457c89f5 --- /dev/null +++ b/test/libyul/yulSyntaxTests/type_check_cases.yul @@ -0,0 +1,8 @@ +{ + switch 7:i64 + case 0:i64 {} + case 2:i64 {} +} +// ==== +// dialect: ewasm +// ---- diff --git a/test/libyul/yulSyntaxTests/type_check_cases_fail.yul b/test/libyul/yulSyntaxTests/type_check_cases_fail.yul new file mode 100644 index 000000000000..c799bec4b288 --- /dev/null +++ b/test/libyul/yulSyntaxTests/type_check_cases_fail.yul @@ -0,0 +1,10 @@ +{ + switch 7:i32 + case 0:i64 {} + case 2:i64 {} +} +// ==== +// dialect: ewasm +// ---- +// TypeError: (28-33): Expected a value of type "i32" but got "i64" +// TypeError: (46-51): Expected a value of type "i32" but got "i64" diff --git a/test/libyul/yulSyntaxTests/type_check_cases_fail_evmtyped.yul b/test/libyul/yulSyntaxTests/type_check_cases_fail_evmtyped.yul new file mode 100644 index 000000000000..c1db211f015c --- /dev/null +++ b/test/libyul/yulSyntaxTests/type_check_cases_fail_evmtyped.yul @@ -0,0 +1,10 @@ +{ + switch 7 + case true:bool {} + case true:bool {} +} +// ==== +// dialect: evmTyped +// ---- +// TypeError: (24-33): Expected a value of type "u256" but got "bool" +// TypeError: (46-55): Expected a value of type "u256" but got "bool" diff --git a/test/libyul/yulSyntaxTests/type_check_if_condition.yul b/test/libyul/yulSyntaxTests/type_check_if_condition.yul new file mode 100644 index 000000000000..eaccf63f6408 --- /dev/null +++ b/test/libyul/yulSyntaxTests/type_check_if_condition.yul @@ -0,0 +1,7 @@ +{ + let x:i32 + if x {} +} +// ==== +// dialect: ewasm +// ---- diff --git a/test/libyul/yulSyntaxTests/type_check_if_condition_fail.yul b/test/libyul/yulSyntaxTests/type_check_if_condition_fail.yul new file mode 100644 index 000000000000..a79d50e342df --- /dev/null +++ b/test/libyul/yulSyntaxTests/type_check_if_condition_fail.yul @@ -0,0 +1,8 @@ +{ + let x:i64 + if x {} +} +// ==== +// dialect: ewasm +// ---- +// TypeError: (23-24): Expected a value of boolean type "i32" but got "i64" diff --git a/test/libyul/yulSyntaxTests/user_defined_functions_fail.yul b/test/libyul/yulSyntaxTests/user_defined_functions_fail.yul new file mode 100644 index 000000000000..324f937d4374 --- /dev/null +++ b/test/libyul/yulSyntaxTests/user_defined_functions_fail.yul @@ -0,0 +1,12 @@ +{ + function f(a:u256, b:u256, c:bool) -> r:bool, t { + r := lt(a, b) + t := bool_to_u256(not(c)) + } + let x, y: bool := f(1, 2: u256, true) +} +// ==== +// dialect: evmTyped +// ---- +// TypeError: (126-127): Assigning value of type "bool" to variable of type "u256. +// TypeError: (129-136): Assigning value of type "u256" to variable of type "bool. diff --git a/test/libyul/yulSyntaxTests/user_defined_functions_fine.yul b/test/libyul/yulSyntaxTests/user_defined_functions_fine.yul new file mode 100644 index 000000000000..e24ea923ec75 --- /dev/null +++ b/test/libyul/yulSyntaxTests/user_defined_functions_fine.yul @@ -0,0 +1,10 @@ +{ + function f(a:u256, b:u256, c:bool) -> r:bool, t { + r := lt(a, b) + t := bool_to_u256(not(c)) + } + let x: bool, y: u256 := f(1, 2: u256, true) +} +// ==== +// dialect: evmTyped +// ---- From 915cb65106f1f1006974df752842f011b74a3ede Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 11 Feb 2020 21:57:52 +0100 Subject: [PATCH 35/92] Collect types together with names. --- libyul/CMakeLists.txt | 2 + libyul/optimiser/TypeInfo.cpp | 98 +++++++++++++++++++++++++++++++++++ libyul/optimiser/TypeInfo.h | 61 ++++++++++++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 libyul/optimiser/TypeInfo.cpp create mode 100644 libyul/optimiser/TypeInfo.h diff --git a/libyul/CMakeLists.txt b/libyul/CMakeLists.txt index 12d2866c2745..65269e9e0b28 100644 --- a/libyul/CMakeLists.txt +++ b/libyul/CMakeLists.txt @@ -154,6 +154,8 @@ add_library(yul optimiser/Suite.h optimiser/SyntacticalEquality.cpp optimiser/SyntacticalEquality.h + optimiser/TypeInfo.cpp + optimiser/TypeInfo.h optimiser/UnusedPruner.cpp optimiser/UnusedPruner.h optimiser/VarDeclInitializer.cpp diff --git a/libyul/optimiser/TypeInfo.cpp b/libyul/optimiser/TypeInfo.cpp new file mode 100644 index 000000000000..4b5715baa9d2 --- /dev/null +++ b/libyul/optimiser/TypeInfo.cpp @@ -0,0 +1,98 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +/** + * Helper class that keeps track of the types while performing optimizations. + */ + +#include + +#include + +#include +#include + +#include + +using namespace std; +using namespace solidity::yul; +using namespace solidity::util; + +class TypeInfo::TypeCollector: public ASTWalker +{ +public: + explicit TypeCollector(Block const& _block) + { + (*this)(_block); + } + + using ASTWalker::operator(); + void operator()(VariableDeclaration const& _varDecl) override + { + for (auto const& var: _varDecl.variables) + variableTypes[var.name] = var.type; + } + void operator()(FunctionDefinition const& _funDef) override + { + ASTWalker::operator()(_funDef); + + auto& funType = functionTypes[_funDef.name]; + for (auto const arg: _funDef.parameters) + { + funType.parameters.emplace_back(arg.type); + variableTypes[arg.name] = arg.type; + } + for (auto const ret: _funDef.returnVariables) + { + funType.returns.emplace_back(ret.type); + variableTypes[ret.name] = ret.type; + } + } + + std::map variableTypes; + std::map functionTypes; +}; + + +TypeInfo::TypeInfo(Dialect const& _dialect, Block const& _ast): + m_dialect(_dialect) +{ + TypeCollector types(_ast); + m_functionTypes = std::move(types.functionTypes); + m_variableTypes = std::move(types.variableTypes); +} + +YulString TypeInfo::typeOf(Expression const& _expression) const +{ + return std::visit(GenericVisitor{ + [&](FunctionCall const& _funCall) { + YulString name = _funCall.functionName.name; + vector const* retTypes = nullptr; + if (BuiltinFunction const* fun = m_dialect.builtin(name)) + retTypes = &fun->returns; + else + retTypes = &m_functionTypes.at(name).returns; + yulAssert(retTypes && retTypes->size() == 1, "Call to typeOf for non-single-value expression."); + return retTypes->front(); + }, + [&](Identifier const& _identifier) { + return m_variableTypes.at(_identifier.name); + }, + [&](Literal const& _literal) { + return _literal.type; + } + }, _expression); +} diff --git a/libyul/optimiser/TypeInfo.h b/libyul/optimiser/TypeInfo.h new file mode 100644 index 000000000000..deb537ea3216 --- /dev/null +++ b/libyul/optimiser/TypeInfo.h @@ -0,0 +1,61 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +/** + * Helper class that keeps track of the types while performing optimizations. + */ +#pragma once + +#include +#include + +#include +#include + +namespace solidity::yul +{ +struct Dialect; + +/** + * Helper class that keeps track of the types while performing optimizations. + * + * Only works on disambiguated sources! + */ +class TypeInfo +{ +public: + TypeInfo(Dialect const& _dialect, Block const& _ast); + + void setVariableType(YulString _name, YulString _type) { m_variableTypes[_name] = _type; } + + /// @returns the type of an expression that is assumed to return exactly one value. + YulString typeOf(Expression const& _expression) const; + +private: + class TypeCollector; + + struct FunctionType + { + std::vector parameters; + std::vector returns; + }; + + Dialect const& m_dialect; + std::map m_variableTypes; + std::map m_functionTypes; +}; + +} From b9b36cd89e81265d3e243a17bb504af0b47da495 Mon Sep 17 00:00:00 2001 From: chriseth Date: Fri, 14 Feb 2020 15:33:54 +0100 Subject: [PATCH 36/92] Properly assign types in ExpressionSplitter. --- libyul/optimiser/ExpressionSplitter.cpp | 10 ++++- libyul/optimiser/ExpressionSplitter.h | 13 ++++-- .../expressionSplitter/typed.yul | 42 +++++++++++++++++++ 3 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 test/libyul/yulOptimizerTests/expressionSplitter/typed.yul diff --git a/libyul/optimiser/ExpressionSplitter.cpp b/libyul/optimiser/ExpressionSplitter.cpp index d95e11a81ec6..7b2ac14f6235 100644 --- a/libyul/optimiser/ExpressionSplitter.cpp +++ b/libyul/optimiser/ExpressionSplitter.cpp @@ -23,11 +23,13 @@ #include #include +#include #include #include #include +#include #include @@ -39,7 +41,8 @@ using namespace solidity::langutil; void ExpressionSplitter::run(OptimiserStepContext& _context, Block& _ast) { - ExpressionSplitter{_context.dialect, _context.dispenser}(_ast); + TypeInfo typeInfo(_context.dialect, _ast); + ExpressionSplitter{_context.dialect, _context.dispenser, typeInfo}(_ast); } void ExpressionSplitter::operator()(FunctionCall& _funCall) @@ -103,10 +106,13 @@ void ExpressionSplitter::outlineExpression(Expression& _expr) SourceLocation location = locationOf(_expr); YulString var = m_nameDispenser.newName({}); + YulString type = m_typeInfo.typeOf(_expr); m_statementsToPrefix.emplace_back(VariableDeclaration{ location, - {{TypedName{location, var, {}}}}, + {{TypedName{location, var, type}}}, make_unique(std::move(_expr)) }); _expr = Identifier{location, var}; + m_typeInfo.setVariableType(var, type); } + diff --git a/libyul/optimiser/ExpressionSplitter.h b/libyul/optimiser/ExpressionSplitter.h index dfd640ac3470..107be25905d4 100644 --- a/libyul/optimiser/ExpressionSplitter.h +++ b/libyul/optimiser/ExpressionSplitter.h @@ -30,9 +30,9 @@ namespace solidity::yul { -class NameCollector; struct Dialect; struct OptimiserStepContext; +class TypeInfo; /** * Optimiser component that modifies an AST in place, turning complex @@ -68,8 +68,14 @@ class ExpressionSplitter: public ASTModifier void operator()(Block& _block) override; private: - explicit ExpressionSplitter(Dialect const& _dialect, NameDispenser& _nameDispenser): - m_dialect(_dialect), m_nameDispenser(_nameDispenser) + explicit ExpressionSplitter( + Dialect const& _dialect, + NameDispenser& _nameDispenser, + TypeInfo& _typeInfo + ): + m_dialect(_dialect), + m_nameDispenser(_nameDispenser), + m_typeInfo(_typeInfo) { } /// Replaces the expression by a variable if it is a function call or functional @@ -82,6 +88,7 @@ class ExpressionSplitter: public ASTModifier std::vector m_statementsToPrefix; Dialect const& m_dialect; NameDispenser& m_nameDispenser; + TypeInfo& m_typeInfo; }; } diff --git a/test/libyul/yulOptimizerTests/expressionSplitter/typed.yul b/test/libyul/yulOptimizerTests/expressionSplitter/typed.yul new file mode 100644 index 000000000000..37f40f5e68e4 --- /dev/null +++ b/test/libyul/yulOptimizerTests/expressionSplitter/typed.yul @@ -0,0 +1,42 @@ +{ + function fun(x: i32, y) -> t: i32, z: i32 { + z := i32.add(x, i32.add(z, z)) + + } + i64.store(i32.load(5:i32), i64.load(8:i32)) + let i := 0 + for {} i32.eqz(i32.load(9:i32)) { i := i64.add(i, 1) } { + let f: i32, g: i32 := fun(i32.load(1:i32), i64.load(i32.load(0: i32))) + } +} +// ==== +// dialect: ewasm +// step: expressionSplitter +// ---- +// { +// function fun(x:i32, y) -> t:i32, z:i32 +// { +// let _1:i32 := i32.add(z, z) +// z := i32.add(x, _1) +// } +// let _2:i32 := 8:i32 +// let _3 := i64.load(_2) +// let _4:i32 := 5:i32 +// let _5:i32 := i32.load(_4) +// i64.store(_5, _3) +// let i := 0 +// for { } +// i32.eqz(i32.load(9:i32)) +// { +// let _6 := 1 +// i := i64.add(i, _6) +// } +// { +// let _7:i32 := 0:i32 +// let _8:i32 := i32.load(_7) +// let _9 := i64.load(_8) +// let _10:i32 := 1:i32 +// let _11:i32 := i32.load(_10) +// let f:i32, g:i32 := fun(_11, _9) +// } +// } From e728cd76b6cfb5f2a19206dd24808799203c68f5 Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 19 Feb 2020 00:18:13 +0100 Subject: [PATCH 37/92] Introduce typed named functions to dialect. --- libyul/Dialect.h | 4 ++-- libyul/backends/evm/EVMDialect.cpp | 22 ++++++++++++++++++++++ libyul/backends/evm/EVMDialect.h | 8 ++++++-- libyul/backends/wasm/WasmDialect.cpp | 19 ++++++++++++++++++- libyul/backends/wasm/WasmDialect.h | 12 ++++++------ 5 files changed, 54 insertions(+), 11 deletions(-) diff --git a/libyul/Dialect.h b/libyul/Dialect.h index 3d261fdeefc9..4a1aff0fcb91 100644 --- a/libyul/Dialect.h +++ b/libyul/Dialect.h @@ -58,8 +58,8 @@ struct Dialect: boost::noncopyable /// @returns the builtin function of the given name or a nullptr if it is not a builtin function. virtual BuiltinFunction const* builtin(YulString /*_name*/) const { return nullptr; } - virtual BuiltinFunction const* discardFunction() const { return nullptr; } - virtual BuiltinFunction const* equalityFunction() const { return nullptr; } + virtual BuiltinFunction const* discardFunction(YulString /* _type */) const { return nullptr; } + virtual BuiltinFunction const* equalityFunction(YulString /* _type */) const { return nullptr; } virtual BuiltinFunction const* booleanNegationFunction() const { return nullptr; } /// Check whether the given type is legal for the given literal value. diff --git a/libyul/backends/evm/EVMDialect.cpp b/libyul/backends/evm/EVMDialect.cpp index 39fc65632dda..27a2b85e3eaf 100644 --- a/libyul/backends/evm/EVMDialect.cpp +++ b/libyul/backends/evm/EVMDialect.cpp @@ -290,6 +290,28 @@ EVMDialectTyped::EVMDialectTyped(langutil::EVMVersion _evmVersion, bool _objectA m_functions["u256_to_bool"_yulstring].returns = {"bool"_yulstring}; } +BuiltinFunctionForEVM const* EVMDialectTyped::discardFunction(YulString _type) const +{ + if (_type == "bool"_yulstring) + return builtin("popbool"_yulstring); + else + { + yulAssert(_type == defaultType, ""); + return builtin("pop"_yulstring); + } +} + +BuiltinFunctionForEVM const* EVMDialectTyped::equalityFunction(YulString _type) const +{ + if (_type == "bool"_yulstring) + return nullptr; + else + { + yulAssert(_type == defaultType, ""); + return builtin("eq"_yulstring); + } +} + EVMDialectTyped const& EVMDialectTyped::instance(langutil::EVMVersion _version) { static map> dialects; diff --git a/libyul/backends/evm/EVMDialect.h b/libyul/backends/evm/EVMDialect.h index 40842fa57e63..2141ab98f5da 100644 --- a/libyul/backends/evm/EVMDialect.h +++ b/libyul/backends/evm/EVMDialect.h @@ -68,8 +68,8 @@ struct EVMDialect: public Dialect /// @returns the builtin function of the given name or a nullptr if it is not a builtin function. BuiltinFunctionForEVM const* builtin(YulString _name) const override; - BuiltinFunctionForEVM const* discardFunction() const override { return builtin("pop"_yulstring); } - BuiltinFunctionForEVM const* equalityFunction() const override { return builtin("eq"_yulstring); } + BuiltinFunctionForEVM const* discardFunction(YulString /*_type*/) const override { return builtin("pop"_yulstring); } + BuiltinFunctionForEVM const* equalityFunction(YulString /*_type*/) const override { return builtin("eq"_yulstring); } BuiltinFunctionForEVM const* booleanNegationFunction() const override { return builtin("iszero"_yulstring); } static EVMDialect const& strictAssemblyForEVM(langutil::EVMVersion _version); @@ -102,6 +102,10 @@ struct EVMDialectTyped: public EVMDialect /// Constructor, should only be used internally. Use the factory function below. EVMDialectTyped(langutil::EVMVersion _evmVersion, bool _objectAccess); + BuiltinFunctionForEVM const* discardFunction(YulString _type) const override; + BuiltinFunctionForEVM const* equalityFunction(YulString _type) const override; + BuiltinFunctionForEVM const* booleanNegationFunction() const override { return builtin("not"_yulstring); } + static EVMDialectTyped const& instance(langutil::EVMVersion _version); }; diff --git a/libyul/backends/wasm/WasmDialect.cpp b/libyul/backends/wasm/WasmDialect.cpp index 947326306d96..7f2149adcfdd 100644 --- a/libyul/backends/wasm/WasmDialect.cpp +++ b/libyul/backends/wasm/WasmDialect.cpp @@ -91,8 +91,9 @@ WasmDialect::WasmDialect() m_functions["i64.load"_yulstring].sideEffects.sideEffectFreeIfNoMSize = true; // Drop is actually overloaded for all types, but Yul does not support that. - // We could introduce "i32.drop". + // Because of that, we introduce "i32.drop". addFunction("drop", {i64}, {}); + addFunction("i32.drop", {i32}, {}); addFunction("nop", {}, {}); addFunction("unreachable", {}, {}, false); @@ -114,6 +115,22 @@ BuiltinFunction const* WasmDialect::builtin(YulString _name) const return nullptr; } +BuiltinFunction const* WasmDialect::discardFunction(YulString _type) const +{ + if (_type == "i32"_yulstring) + return builtin("i32.drop"_yulstring); + yulAssert(_type == "i64"_yulstring, ""); + return builtin("drop"_yulstring); +} + +BuiltinFunction const* WasmDialect::equalityFunction(YulString _type) const +{ + if (_type == "i32"_yulstring) + return builtin("i32.eq"_yulstring); + yulAssert(_type == "i64"_yulstring, ""); + return builtin("i64.eq"_yulstring); +} + WasmDialect const& WasmDialect::instance() { static std::unique_ptr dialect; diff --git a/libyul/backends/wasm/WasmDialect.h b/libyul/backends/wasm/WasmDialect.h index cf1a76d9dd6c..1de65dfdc898 100644 --- a/libyul/backends/wasm/WasmDialect.h +++ b/libyul/backends/wasm/WasmDialect.h @@ -35,19 +35,19 @@ struct Object; /** * Yul dialect for Wasm as a backend. * - * Builtin functions are a subset of the wasm instructions, always implicitly assuming - * unsigned 64 bit types. + * Builtin functions are a subset of the wasm instructions. + * + * There is a builtin function `i32.drop` that takes an i32, while `drop` takes i64. * - * !This is subject to changes! */ struct WasmDialect: public Dialect { WasmDialect(); BuiltinFunction const* builtin(YulString _name) const override; - BuiltinFunction const* discardFunction() const override { return builtin("drop"_yulstring); } - BuiltinFunction const* equalityFunction() const override { return builtin("i64.eq"_yulstring); } - BuiltinFunction const* booleanNegationFunction() const override { return builtin("i64.eqz"_yulstring); } + BuiltinFunction const* discardFunction(YulString _type) const override; + BuiltinFunction const* equalityFunction(YulString _type) const override; + BuiltinFunction const* booleanNegationFunction() const override { return builtin("i32.eqz"_yulstring); } std::set fixedFunctionNames() const override { return {"main"_yulstring}; } From e75cace78d47b8f95a30e25b02db77c5dc9c508a Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 19 Feb 2020 00:36:50 +0100 Subject: [PATCH 38/92] Properly assign types in control flow simplifier. --- libyul/optimiser/ControlFlowSimplifier.cpp | 123 +++++++++++---------- libyul/optimiser/ControlFlowSimplifier.h | 10 +- 2 files changed, 73 insertions(+), 60 deletions(-) diff --git a/libyul/optimiser/ControlFlowSimplifier.cpp b/libyul/optimiser/ControlFlowSimplifier.cpp index 8af8e6bb02ed..1640fa1b09cd 100644 --- a/libyul/optimiser/ControlFlowSimplifier.cpp +++ b/libyul/optimiser/ControlFlowSimplifier.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -38,14 +39,13 @@ namespace ExpressionStatement makeDiscardCall( langutil::SourceLocation const& _location, - Dialect const& _dialect, + BuiltinFunction const& _discardFunction, Expression&& _expression ) { - yulAssert(_dialect.discardFunction(), "No discard function available."); return {_location, FunctionCall{ _location, - Identifier{_location, _dialect.discardFunction()->name}, + Identifier{_location, _discardFunction.name}, {std::move(_expression)} }}; } @@ -74,62 +74,12 @@ void removeEmptyCasesFromSwitch(Switch& _switchStmt) ); } -OptionalStatements reduceNoCaseSwitch(Dialect const& _dialect, Switch& _switchStmt) -{ - yulAssert(_switchStmt.cases.empty(), "Expected no case!"); - if (!_dialect.discardFunction()) - return {}; - - auto loc = locationOf(*_switchStmt.expression); - - return make_vector(makeDiscardCall( - loc, - _dialect, - std::move(*_switchStmt.expression) - )); -} - -OptionalStatements reduceSingleCaseSwitch(Dialect const& _dialect, Switch& _switchStmt) -{ - yulAssert(_switchStmt.cases.size() == 1, "Expected only one case!"); - - auto& switchCase = _switchStmt.cases.front(); - auto loc = locationOf(*_switchStmt.expression); - if (switchCase.value) - { - if (!_dialect.equalityFunction()) - return {}; - return make_vector(If{ - std::move(_switchStmt.location), - make_unique(FunctionCall{ - loc, - Identifier{loc, _dialect.equalityFunction()->name}, - {std::move(*switchCase.value), std::move(*_switchStmt.expression)} - }), - std::move(switchCase.body) - }); - } - else - { - if (!_dialect.discardFunction()) - return {}; - - return make_vector( - makeDiscardCall( - loc, - _dialect, - std::move(*_switchStmt.expression) - ), - std::move(switchCase.body) - ); - } -} - } void ControlFlowSimplifier::run(OptimiserStepContext& _context, Block& _ast) { - ControlFlowSimplifier{_context.dialect}(_ast); + TypeInfo typeInfo(_context.dialect, _ast); + ControlFlowSimplifier{_context.dialect, typeInfo}(_ast); } void ControlFlowSimplifier::operator()(Block& _block) @@ -194,12 +144,12 @@ void ControlFlowSimplifier::simplify(std::vector& _statements) GenericVisitor visitor{ VisitorFallback{}, [&](If& _ifStmt) -> OptionalStatements { - if (_ifStmt.body.statements.empty() && m_dialect.discardFunction()) + if (_ifStmt.body.statements.empty() && m_dialect.discardFunction(m_dialect.boolType)) { OptionalStatements s = vector{}; s->emplace_back(makeDiscardCall( _ifStmt.location, - m_dialect, + *m_dialect.discardFunction(m_dialect.boolType), std::move(*_ifStmt.condition) )); return s; @@ -211,9 +161,9 @@ void ControlFlowSimplifier::simplify(std::vector& _statements) removeEmptyCasesFromSwitch(_switchStmt); if (_switchStmt.cases.empty()) - return reduceNoCaseSwitch(m_dialect, _switchStmt); + return reduceNoCaseSwitch(_switchStmt); else if (_switchStmt.cases.size() == 1) - return reduceSingleCaseSwitch(m_dialect, _switchStmt); + return reduceSingleCaseSwitch(_switchStmt); return {}; } @@ -231,3 +181,58 @@ void ControlFlowSimplifier::simplify(std::vector& _statements) } ); } + +OptionalStatements ControlFlowSimplifier::reduceNoCaseSwitch(Switch& _switchStmt) const +{ + yulAssert(_switchStmt.cases.empty(), "Expected no case!"); + BuiltinFunction const* discardFunction = + m_dialect.discardFunction(m_typeInfo.typeOf(*_switchStmt.expression)); + if (!discardFunction) + return {}; + + auto loc = locationOf(*_switchStmt.expression); + + return make_vector(makeDiscardCall( + loc, + *discardFunction, + std::move(*_switchStmt.expression) + )); +} + +OptionalStatements ControlFlowSimplifier::reduceSingleCaseSwitch(Switch& _switchStmt) const +{ + yulAssert(_switchStmt.cases.size() == 1, "Expected only one case!"); + + auto& switchCase = _switchStmt.cases.front(); + auto loc = locationOf(*_switchStmt.expression); + YulString type = m_typeInfo.typeOf(*_switchStmt.expression); + if (switchCase.value) + { + if (!m_dialect.equalityFunction(type)) + return {}; + return make_vector(If{ + std::move(_switchStmt.location), + make_unique(FunctionCall{ + loc, + Identifier{loc, m_dialect.equalityFunction(type)->name}, + {std::move(*switchCase.value), std::move(*_switchStmt.expression)} + }), + std::move(switchCase.body) + }); + } + else + { + if (!m_dialect.discardFunction(type)) + return {}; + + return make_vector( + makeDiscardCall( + loc, + *m_dialect.discardFunction(type), + std::move(*_switchStmt.expression) + ), + std::move(switchCase.body) + ); + } +} + diff --git a/libyul/optimiser/ControlFlowSimplifier.h b/libyul/optimiser/ControlFlowSimplifier.h index 5713f12a1dc3..f8ea1af1ec24 100644 --- a/libyul/optimiser/ControlFlowSimplifier.h +++ b/libyul/optimiser/ControlFlowSimplifier.h @@ -23,6 +23,7 @@ namespace solidity::yul { struct Dialect; struct OptimiserStepContext; +class TypeInfo; /** * Simplifies several control-flow structures: @@ -61,11 +62,18 @@ class ControlFlowSimplifier: public ASTModifier void visit(Statement& _st) override; private: - ControlFlowSimplifier(Dialect const& _dialect): m_dialect(_dialect) {} + ControlFlowSimplifier(Dialect const& _dialect, TypeInfo const& _typeInfo): + m_dialect(_dialect), + m_typeInfo(_typeInfo) + {} void simplify(std::vector& _statements); + std::optional> reduceNoCaseSwitch(Switch& _switchStmt) const; + std::optional> reduceSingleCaseSwitch(Switch& _switchStmt) const; + Dialect const& m_dialect; + TypeInfo const& m_typeInfo; size_t m_numBreakStatements = 0; size_t m_numContinueStatements = 0; }; From a52c9af5b9fc26877f14b0fc451ff4b3e3cec6e4 Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Tue, 18 Feb 2020 16:11:21 +0100 Subject: [PATCH 39/92] Adding vardecl optimization for boolean types --- libyul/Dialect.cpp | 8 +++++++ libyul/Dialect.h | 3 +++ libyul/optimiser/UnusedPruner.cpp | 4 ++-- libyul/optimiser/VarDeclInitializer.cpp | 12 ++++++---- libyul/optimiser/VarDeclInitializer.h | 7 +++++- test/libyul/YulOptimizerTest.cpp | 2 ++ .../varDeclInitializer/typed.yul | 23 +++++++++++++++++++ 7 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 test/libyul/yulOptimizerTests/varDeclInitializer/typed.yul diff --git a/libyul/Dialect.cpp b/libyul/Dialect.cpp index 4d5cccd41512..407fdea2fc4f 100644 --- a/libyul/Dialect.cpp +++ b/libyul/Dialect.cpp @@ -23,6 +23,14 @@ using namespace solidity::yul; using namespace std; +using namespace solidity::langutil; + +Literal Dialect::zeroLiteralForType(solidity::yul::YulString _type) const +{ + if (_type == boolType && _type != defaultType) + return {SourceLocation{}, LiteralKind::Boolean, "false"_yulstring, _type}; + return {SourceLocation{}, LiteralKind::Number, "0"_yulstring, _type}; +} bool Dialect::validTypeForLiteral(LiteralKind _kind, YulString, YulString _type) const { diff --git a/libyul/Dialect.h b/libyul/Dialect.h index 4a1aff0fcb91..c137791ee9c3 100644 --- a/libyul/Dialect.h +++ b/libyul/Dialect.h @@ -34,6 +34,7 @@ namespace solidity::yul class YulString; using Type = YulString; enum class LiteralKind; +struct Literal; struct BuiltinFunction { @@ -66,6 +67,8 @@ struct Dialect: boost::noncopyable /// Should only be called if the type exists in the dialect at all. virtual bool validTypeForLiteral(LiteralKind _kind, YulString _value, YulString _type) const; + virtual Literal zeroLiteralForType(YulString _type) const; + virtual std::set fixedFunctionNames() const { return {}; } Dialect() = default; diff --git a/libyul/optimiser/UnusedPruner.cpp b/libyul/optimiser/UnusedPruner.cpp index b7925790a96d..9dc7cedc33cc 100644 --- a/libyul/optimiser/UnusedPruner.cpp +++ b/libyul/optimiser/UnusedPruner.cpp @@ -100,10 +100,10 @@ void UnusedPruner::operator()(Block& _block) subtractReferences(ReferencesCounter::countReferences(*varDecl.value)); statement = Block{std::move(varDecl.location), {}}; } - else if (varDecl.variables.size() == 1 && m_dialect.discardFunction()) + else if (varDecl.variables.size() == 1 && m_dialect.discardFunction(varDecl.variables.front().type)) statement = ExpressionStatement{varDecl.location, FunctionCall{ varDecl.location, - {varDecl.location, m_dialect.discardFunction()->name}, + {varDecl.location, m_dialect.discardFunction(varDecl.variables.front().type)->name}, {*std::move(varDecl.value)} }}; } diff --git a/libyul/optimiser/VarDeclInitializer.cpp b/libyul/optimiser/VarDeclInitializer.cpp index bad601f0e13e..66c5b45056be 100644 --- a/libyul/optimiser/VarDeclInitializer.cpp +++ b/libyul/optimiser/VarDeclInitializer.cpp @@ -20,6 +20,7 @@ #include #include +#include using namespace std; using namespace solidity; @@ -32,14 +33,14 @@ void VarDeclInitializer::operator()(Block& _block) using OptionalStatements = std::optional>; util::GenericVisitor visitor{ util::VisitorFallback{}, - [](VariableDeclaration& _varDecl) -> OptionalStatements + [this](VariableDeclaration& _varDecl) -> OptionalStatements { if (_varDecl.value) return {}; - Literal zero{{}, LiteralKind::Number, YulString{"0"}, {}}; + if (_varDecl.variables.size() == 1) { - _varDecl.value = make_unique(std::move(zero)); + _varDecl.value = make_unique(m_dialect.zeroLiteralForType(_varDecl.variables.front().type)); return {}; } else @@ -47,7 +48,10 @@ void VarDeclInitializer::operator()(Block& _block) OptionalStatements ret{vector{}}; langutil::SourceLocation loc{std::move(_varDecl.location)}; for (auto& var: _varDecl.variables) - ret->emplace_back(VariableDeclaration{loc, {std::move(var)}, make_unique(zero)}); + { + unique_ptr expr = make_unique(m_dialect.zeroLiteralForType(var.type)); + ret->emplace_back(VariableDeclaration{loc, {std::move(var)}, std::move(expr)}); + } return ret; } } diff --git a/libyul/optimiser/VarDeclInitializer.h b/libyul/optimiser/VarDeclInitializer.h index 556ce081b874..75bb1493c8ee 100644 --- a/libyul/optimiser/VarDeclInitializer.h +++ b/libyul/optimiser/VarDeclInitializer.h @@ -34,9 +34,14 @@ class VarDeclInitializer: public ASTModifier { public: static constexpr char const* name{"VarDeclInitializer"}; - static void run(OptimiserStepContext&, Block& _ast) { VarDeclInitializer{}(_ast); } + static void run(OptimiserStepContext& _ctx, Block& _ast) { VarDeclInitializer{_ctx.dialect}(_ast); } void operator()(Block& _block) override; + +private: + explicit VarDeclInitializer(Dialect const& _dialect): m_dialect(_dialect) {} + + Dialect const& m_dialect; }; } diff --git a/test/libyul/YulOptimizerTest.cpp b/test/libyul/YulOptimizerTest.cpp index 3ac745d6fd67..699a767d07bf 100644 --- a/test/libyul/YulOptimizerTest.cpp +++ b/test/libyul/YulOptimizerTest.cpp @@ -111,6 +111,8 @@ YulOptimizerTest::YulOptimizerTest(string const& _filename) m_dialect = &WasmDialect::instance(); else if (dialectName == "evm") m_dialect = &EVMDialect::strictAssemblyForEVMObjects(solidity::test::CommonOptions::get().evmVersion()); + else if (dialectName == "evmTyped") + m_dialect = &EVMDialectTyped::instance(solidity::test::CommonOptions::get().evmVersion()); else BOOST_THROW_EXCEPTION(runtime_error("Invalid dialect " + dialectName)); diff --git a/test/libyul/yulOptimizerTests/varDeclInitializer/typed.yul b/test/libyul/yulOptimizerTests/varDeclInitializer/typed.yul new file mode 100644 index 000000000000..912246bea0db --- /dev/null +++ b/test/libyul/yulOptimizerTests/varDeclInitializer/typed.yul @@ -0,0 +1,23 @@ +{ + let a1 + let a2: bool + let b1, b2: bool + function f(a:u256, b:u256, c:bool) -> r:bool, t { + let x1: bool, x2 + } +} +// ==== +// dialect: evmTyped +// step: varDeclInitializer +// ---- +// { +// let a1 := 0 +// let a2:bool := false +// let b1 := 0 +// let b2:bool := false +// function f(a, b, c:bool) -> r:bool, t +// { +// let x1:bool := false +// let x2 := 0 +// } +// } From a52305d3bd8cb1792175e8333560569cb566a3e1 Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 19 Feb 2020 16:39:59 +0100 Subject: [PATCH 40/92] Use bool type in conditional simplifier and for loop condition into body. --- libyul/optimiser/ConditionalSimplifier.cpp | 7 +------ libyul/optimiser/ForLoopConditionIntoBody.cpp | 6 +++--- .../conditionalSimplifier/add_correct_type.yul | 18 ++++++++++++++++++ .../add_correct_type_wasm.yul | 18 ++++++++++++++++++ .../forLoopConditionIntoBody/cond_types.yul | 2 +- .../forLoopConditionIntoBody/empty_body.yul | 2 +- .../forLoopConditionIntoBody/nested.yul | 6 +++--- .../forLoopConditionIntoBody/simple.yul | 2 +- .../fullSuite/no_move_loop_orig.yul | 2 +- 9 files changed, 47 insertions(+), 16 deletions(-) create mode 100644 test/libyul/yulOptimizerTests/conditionalSimplifier/add_correct_type.yul create mode 100644 test/libyul/yulOptimizerTests/conditionalSimplifier/add_correct_type_wasm.yul diff --git a/libyul/optimiser/ConditionalSimplifier.cpp b/libyul/optimiser/ConditionalSimplifier.cpp index 858ac8999da5..453d2d9ce6ef 100644 --- a/libyul/optimiser/ConditionalSimplifier.cpp +++ b/libyul/optimiser/ConditionalSimplifier.cpp @@ -77,12 +77,7 @@ void ConditionalSimplifier::operator()(Block& _block) Assignment{ location, {Identifier{location, condition}}, - make_unique(Literal{ - location, - LiteralKind::Number, - "0"_yulstring, - {} - }) + make_unique(m_dialect.zeroLiteralForType(m_dialect.boolType)) } ); } diff --git a/libyul/optimiser/ForLoopConditionIntoBody.cpp b/libyul/optimiser/ForLoopConditionIntoBody.cpp index a45b33635285..22c3e76dce08 100644 --- a/libyul/optimiser/ForLoopConditionIntoBody.cpp +++ b/libyul/optimiser/ForLoopConditionIntoBody.cpp @@ -56,9 +56,9 @@ void ForLoopConditionIntoBody::operator()(ForLoop& _forLoop) _forLoop.condition = make_unique( Literal { loc, - LiteralKind::Number, - "1"_yulstring, - {} + LiteralKind::Boolean, + "true"_yulstring, + m_dialect.boolType } ); } diff --git a/test/libyul/yulOptimizerTests/conditionalSimplifier/add_correct_type.yul b/test/libyul/yulOptimizerTests/conditionalSimplifier/add_correct_type.yul new file mode 100644 index 000000000000..f0b46072f14c --- /dev/null +++ b/test/libyul/yulOptimizerTests/conditionalSimplifier/add_correct_type.yul @@ -0,0 +1,18 @@ +{ + let y:bool := false + for {} true { } { + if y { break } + } +} +// ==== +// dialect: yul +// step: conditionalSimplifier +// ---- +// { +// let y:bool := false +// for { } true { } +// { +// if y { break } +// y := false +// } +// } diff --git a/test/libyul/yulOptimizerTests/conditionalSimplifier/add_correct_type_wasm.yul b/test/libyul/yulOptimizerTests/conditionalSimplifier/add_correct_type_wasm.yul new file mode 100644 index 000000000000..ab523246e33d --- /dev/null +++ b/test/libyul/yulOptimizerTests/conditionalSimplifier/add_correct_type_wasm.yul @@ -0,0 +1,18 @@ +{ + let y:i32 := 0:i32 + for {} true { } { + if y { break } + } +} +// ==== +// dialect: ewasm +// step: conditionalSimplifier +// ---- +// { +// let y:i32 := 0:i32 +// for { } true { } +// { +// if y { break } +// y := false +// } +// } diff --git a/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/cond_types.yul b/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/cond_types.yul index 9b401e6384d4..95e171475f2e 100644 --- a/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/cond_types.yul +++ b/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/cond_types.yul @@ -19,7 +19,7 @@ // { } // for { } a { } // { } -// for { } 1 { } +// for { } true { } // { // if iszero(add(a, a)) { break } // } diff --git a/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/empty_body.yul b/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/empty_body.yul index 6a570750f7b3..6f1ad5458a5b 100644 --- a/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/empty_body.yul +++ b/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/empty_body.yul @@ -5,7 +5,7 @@ // step: forLoopConditionIntoBody // ---- // { -// for { let a := 1 } 1 { a := add(a, 1) } +// for { let a := 1 } true { a := add(a, 1) } // { // if iszero(iszero(eq(a, 10))) { break } // } diff --git a/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/nested.yul b/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/nested.yul index 257467d28289..292284a65c91 100644 --- a/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/nested.yul +++ b/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/nested.yul @@ -19,16 +19,16 @@ // { // let random := 42 // for { -// for { let a := 1 } 1 { } +// for { let a := 1 } true { } // { // if iszero(iszero(eq(a, 10))) { break } // a := add(a, 1) // } // let b := 1 // } -// 1 +// true // { -// for { let c := 1 } 1 { c := add(c, 1) } +// for { let c := 1 } true { c := add(c, 1) } // { // if iszero(iszero(eq(c, 2))) { break } // b := add(b, 1) diff --git a/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/simple.yul b/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/simple.yul index 0e3c45ffe2b7..f470264627b1 100644 --- a/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/simple.yul +++ b/test/libyul/yulOptimizerTests/forLoopConditionIntoBody/simple.yul @@ -9,7 +9,7 @@ // ---- // { // let random := 42 -// for { let a := 1 } 1 { a := add(a, 1) } +// for { let a := 1 } true { a := add(a, 1) } // { // if iszero(iszero(eq(a, 10))) { break } // a := add(a, 1) diff --git a/test/libyul/yulOptimizerTests/fullSuite/no_move_loop_orig.yul b/test/libyul/yulOptimizerTests/fullSuite/no_move_loop_orig.yul index 77c3840215be..4efe81d899e3 100644 --- a/test/libyul/yulOptimizerTests/fullSuite/no_move_loop_orig.yul +++ b/test/libyul/yulOptimizerTests/fullSuite/no_move_loop_orig.yul @@ -14,7 +14,7 @@ // { // let _1 := iszero(caller()) // for { } -// 1 +// true // { // for { } iszero(_1) { } // { } From 6b272faec003a5177acbf70cb0e8b4a4c3357c2f Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 19 Feb 2020 16:56:53 +0100 Subject: [PATCH 41/92] Some wasm related type fixes. --- libyul/backends/wasm/EVMToEwasmTranslator.cpp | 8 ++++++-- libyul/backends/wasm/WordSizeTransform.h | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/libyul/backends/wasm/EVMToEwasmTranslator.cpp b/libyul/backends/wasm/EVMToEwasmTranslator.cpp index 19bd98da79f7..a5b88d053736 100644 --- a/libyul/backends/wasm/EVMToEwasmTranslator.cpp +++ b/libyul/backends/wasm/EVMToEwasmTranslator.cpp @@ -1251,8 +1251,12 @@ Object EVMToEwasmTranslator::run(Object const& _object) AsmAnalyzer analyzer(*ret.analysisInfo, errorReporter, WasmDialect::instance(), {}, _object.dataNames()); if (!analyzer.analyze(*ret.code)) { - // TODO the errors here are "wrong" because they have invalid source references! - string message; + string message = "Invalid code generated after EVM to wasm translation.\n"; + message += "Note that the source locations in the errors below will reference the original, not the translated code.\n"; + message += "Translated code:\n"; + message += "----------------------------------\n"; + message += ret.toString(&WasmDialect::instance()); + message += "----------------------------------\n"; for (auto const& err: errors) message += langutil::SourceReferenceFormatter::formatErrorInformation(*err); yulAssert(false, message); diff --git a/libyul/backends/wasm/WordSizeTransform.h b/libyul/backends/wasm/WordSizeTransform.h index 71e526b7fbcc..2e4db8446e17 100644 --- a/libyul/backends/wasm/WordSizeTransform.h +++ b/libyul/backends/wasm/WordSizeTransform.h @@ -53,7 +53,7 @@ namespace solidity::yul * take four times the parameters and each of type u64. * In addition, it uses a single other builtin function called `or_bool` that * takes four u64 parameters and is supposed to return the logical disjunction - * of them as a u64 value. If this name is already used somewhere, it is renamed. + * of them as a i32 value. If this name is already used somewhere, it is renamed. * * Prerequisite: Disambiguator, ForLoopConditionIntoBody, ExpressionSplitter */ From bddbcbe6a4a78efa3e40d86eac4657c593bb5d31 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 20 Feb 2020 16:17:09 +0100 Subject: [PATCH 42/92] Use bool type in word size transform. --- libyul/backends/wasm/EVMToEwasmTranslator.cpp | 2 +- libyul/backends/wasm/WordSizeTransform.cpp | 47 ++++++++++++++----- libyul/backends/wasm/WordSizeTransform.h | 14 ++---- test/libyul/YulOptimizerTest.cpp | 2 +- .../wordSizeTransform/switch_3.yul | 8 ++-- .../wordSizeTransform/switch_4.yul | 12 ++--- .../wordSizeTransform/switch_5.yul | 2 +- 7 files changed, 52 insertions(+), 35 deletions(-) diff --git a/libyul/backends/wasm/EVMToEwasmTranslator.cpp b/libyul/backends/wasm/EVMToEwasmTranslator.cpp index a5b88d053736..dc69d913aa2e 100644 --- a/libyul/backends/wasm/EVMToEwasmTranslator.cpp +++ b/libyul/backends/wasm/EVMToEwasmTranslator.cpp @@ -1235,7 +1235,7 @@ Object EVMToEwasmTranslator::run(Object const& _object) MainFunction{}(ast); ForLoopConditionIntoBody::run(context, ast); ExpressionSplitter::run(context, ast); - WordSizeTransform::run(m_dialect, WasmDialect::instance().defaultType, ast, nameDispenser); + WordSizeTransform::run(m_dialect, WasmDialect::instance(), ast, nameDispenser); NameDisplacer{nameDispenser, m_polyfillFunctions}(ast); for (auto const& st: m_polyfill->statements) diff --git a/libyul/backends/wasm/WordSizeTransform.cpp b/libyul/backends/wasm/WordSizeTransform.cpp index 9ebf3173836a..eec04fae62c4 100644 --- a/libyul/backends/wasm/WordSizeTransform.cpp +++ b/libyul/backends/wasm/WordSizeTransform.cpp @@ -45,7 +45,7 @@ void WordSizeTransform::operator()(FunctionCall& _fc) if (fun->literalArguments) { for (Expression& arg: _fc.arguments) - get(arg).type = m_defaultType; + get(arg).type = m_targetDialect.defaultType; return; } @@ -106,12 +106,17 @@ void WordSizeTransform::operator()(Block& _block) for (int i = 0; i < 3; i++) ret.push_back(VariableDeclaration{ varDecl.location, - {TypedName{varDecl.location, newLhs[i], m_defaultType}}, - make_unique(Literal{locationOf(*varDecl.value), LiteralKind::Number, "0"_yulstring, m_defaultType}) + {TypedName{varDecl.location, newLhs[i], m_targetDialect.defaultType}}, + make_unique(Literal{ + locationOf(*varDecl.value), + LiteralKind::Number, + "0"_yulstring, + m_targetDialect.defaultType + }) }); ret.push_back(VariableDeclaration{ varDecl.location, - {TypedName{varDecl.location, newLhs[3], m_defaultType}}, + {TypedName{varDecl.location, newLhs[3], m_targetDialect.defaultType}}, std::move(varDecl.value) }); return {std::move(ret)}; @@ -133,7 +138,7 @@ void WordSizeTransform::operator()(Block& _block) ret.push_back( VariableDeclaration{ varDecl.location, - {TypedName{varDecl.location, newLhs[i], m_defaultType}}, + {TypedName{varDecl.location, newLhs[i], m_targetDialect.defaultType}}, std::move(newRhs[i]) } ); @@ -163,7 +168,12 @@ void WordSizeTransform::operator()(Block& _block) ret.push_back(Assignment{ assignment.location, {Identifier{assignment.location, newLhs[i]}}, - make_unique(Literal{locationOf(*assignment.value), LiteralKind::Number, "0"_yulstring, m_defaultType}) + make_unique(Literal{ + locationOf(*assignment.value), + LiteralKind::Number, + "0"_yulstring, + m_targetDialect.defaultType + }) }); ret.push_back(Assignment{ assignment.location, @@ -209,14 +219,25 @@ void WordSizeTransform::operator()(Block& _block) void WordSizeTransform::run( Dialect const& _inputDialect, - YulString _targetDefaultType, + Dialect const& _targetDialect, Block& _ast, NameDispenser& _nameDispenser ) { // Free the name `or_bool`. NameDisplacer{_nameDispenser, {"or_bool"_yulstring}}(_ast); - WordSizeTransform{_inputDialect, _nameDispenser, _targetDefaultType}(_ast); + WordSizeTransform{_inputDialect, _targetDialect, _nameDispenser}(_ast); +} + +WordSizeTransform::WordSizeTransform( + Dialect const& _inputDialect, + Dialect const& _targetDialect, + NameDispenser& _nameDispenser +): + m_inputDialect(_inputDialect), + m_targetDialect(_targetDialect), + m_nameDispenser(_nameDispenser) +{ } void WordSizeTransform::rewriteVarDeclList(TypedNameList& _nameList) @@ -227,7 +248,7 @@ void WordSizeTransform::rewriteVarDeclList(TypedNameList& _nameList) { TypedNameList ret; for (auto newName: generateU64IdentifierNames(_n.name)) - ret.emplace_back(TypedName{_n.location, newName, m_defaultType}); + ret.emplace_back(TypedName{_n.location, newName, m_targetDialect.defaultType}); return ret; } ); @@ -291,7 +312,7 @@ vector WordSizeTransform::handleSwitchInternal( for (auto& c: cases) { - Literal label{_location, LiteralKind::Number, YulString(c.first.str()), m_defaultType}; + Literal label{_location, LiteralKind::Number, YulString(c.first.str()), m_targetDialect.defaultType}; ret.cases.emplace_back(Case{ c.second.front().location, make_unique(std::move(label)), @@ -312,7 +333,7 @@ vector WordSizeTransform::handleSwitchInternal( Assignment{ _location, {{_location, _runDefaultFlag}}, - make_unique(Literal{_location, LiteralKind::Number, "1"_yulstring, m_defaultType}) + make_unique(Literal{_location, LiteralKind::Boolean, "true"_yulstring, m_targetDialect.boolType}) } )} }); @@ -337,7 +358,7 @@ std::vector WordSizeTransform::handleSwitch(Switch& _switch) _switch.cases.pop_back(); ret.emplace_back(VariableDeclaration{ _switch.location, - {TypedName{_switch.location, runDefaultFlag, m_defaultType}}, + {TypedName{_switch.location, runDefaultFlag, m_targetDialect.boolType}}, {} }); } @@ -392,7 +413,7 @@ array, 4> WordSizeTransform::expandValue(Expression const lit.location, LiteralKind::Number, YulString(currentVal.str()), - m_defaultType + m_targetDialect.defaultType } ); } diff --git a/libyul/backends/wasm/WordSizeTransform.h b/libyul/backends/wasm/WordSizeTransform.h index 2e4db8446e17..67dfe886439f 100644 --- a/libyul/backends/wasm/WordSizeTransform.h +++ b/libyul/backends/wasm/WordSizeTransform.h @@ -69,7 +69,7 @@ class WordSizeTransform: public ASTModifier static void run( Dialect const& _inputDialect, - YulString _targetDefaultType, + Dialect const& _targetDialect, Block& _ast, NameDispenser& _nameDispenser ); @@ -77,13 +77,9 @@ class WordSizeTransform: public ASTModifier private: explicit WordSizeTransform( Dialect const& _inputDialect, - NameDispenser& _nameDispenser, - YulString _defaultType - ): - m_inputDialect(_inputDialect), - m_nameDispenser(_nameDispenser), - m_defaultType(_defaultType) - { } + Dialect const& _targetDialect, + NameDispenser& _nameDispenser + ); void rewriteVarDeclList(std::vector&); void rewriteIdentifierList(std::vector&); @@ -103,8 +99,8 @@ class WordSizeTransform: public ASTModifier std::vector expandValueToVector(Expression const& _e); Dialect const& m_inputDialect; + Dialect const& m_targetDialect; NameDispenser& m_nameDispenser; - YulString m_defaultType; /// maps original u256 variable's name to corresponding u64 variables' names std::map> m_variableMapping; }; diff --git a/test/libyul/YulOptimizerTest.cpp b/test/libyul/YulOptimizerTest.cpp index 699a767d07bf..e488460fccda 100644 --- a/test/libyul/YulOptimizerTest.cpp +++ b/test/libyul/YulOptimizerTest.cpp @@ -359,7 +359,7 @@ TestCase::TestResult YulOptimizerTest::run(ostream& _stream, string const& _line { disambiguate(); ExpressionSplitter::run(*m_context, *m_ast); - WordSizeTransform::run(*m_dialect, ""_yulstring, *m_ast, *m_nameDispenser); + WordSizeTransform::run(*m_dialect, *m_dialect, *m_ast, *m_nameDispenser); } else if (m_optimizerStep == "fullSuite") { diff --git a/test/libyul/yulOptimizerTests/wordSizeTransform/switch_3.yul b/test/libyul/yulOptimizerTests/wordSizeTransform/switch_3.yul index 50a36f61fbe4..617eebb6416a 100644 --- a/test/libyul/yulOptimizerTests/wordSizeTransform/switch_3.yul +++ b/test/libyul/yulOptimizerTests/wordSizeTransform/switch_3.yul @@ -67,13 +67,13 @@ // let _10_3 := 3 // sstore(_10_0, _10_1, _10_2, _10_3, _9_0, _9_1, _9_2, _9_3) // } -// default { run_default := 1 } +// default { run_default := true } // } -// default { run_default := 1 } +// default { run_default := true } // } -// default { run_default := 1 } +// default { run_default := true } // } -// default { run_default := 1 } +// default { run_default := true } // if run_default // { // let _11_0 := 0 diff --git a/test/libyul/yulOptimizerTests/wordSizeTransform/switch_4.yul b/test/libyul/yulOptimizerTests/wordSizeTransform/switch_4.yul index 106941302be4..20f81d614a4d 100644 --- a/test/libyul/yulOptimizerTests/wordSizeTransform/switch_4.yul +++ b/test/libyul/yulOptimizerTests/wordSizeTransform/switch_4.yul @@ -45,9 +45,9 @@ // let _8_3 := 2 // sstore(_8_0, _8_1, _8_2, _8_3, _7_0, _7_1, _7_2, _7_3) // } -// default { run_default := 1 } +// default { run_default := true } // } -// default { run_default := 1 } +// default { run_default := true } // } // case 536870912 { // switch _2_2 @@ -75,13 +75,13 @@ // let _10_3 := 3 // sstore(_10_0, _10_1, _10_2, _10_3, _9_0, _9_1, _9_2, _9_3) // } -// default { run_default := 1 } +// default { run_default := true } // } -// default { run_default := 1 } +// default { run_default := true } // } -// default { run_default := 1 } +// default { run_default := true } // } -// default { run_default := 1 } +// default { run_default := true } // if run_default // { // let _11_0 := 0 diff --git a/test/libyul/yulOptimizerTests/wordSizeTransform/switch_5.yul b/test/libyul/yulOptimizerTests/wordSizeTransform/switch_5.yul index eb05e0a6c750..2a42adefe333 100644 --- a/test/libyul/yulOptimizerTests/wordSizeTransform/switch_5.yul +++ b/test/libyul/yulOptimizerTests/wordSizeTransform/switch_5.yul @@ -13,7 +13,7 @@ // let _2_0, _2_1, _2_2, _2_3 := calldataload(_1_0, _1_1, _1_2, _1_3) // let run_default // switch _2_0 -// default { run_default := 1 } +// default { run_default := true } // if run_default // { // let _3_0 := 0 From 2efda4129baa3b545f3417587e6cc605f145b41e Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Wed, 19 Feb 2020 15:36:05 +0100 Subject: [PATCH 43/92] Adding test for multi return values including bool in evmTyped dialect. Calling zeroLiteralForType from inliner --- libyul/optimiser/FullInliner.cpp | 11 +++++----- libyul/optimiser/FullInliner.h | 11 ++++++---- .../fullInliner/multi_return_typed.yul | 22 +++++++++++++++++++ 3 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 test/libyul/yulOptimizerTests/fullInliner/multi_return_typed.yul diff --git a/libyul/optimiser/FullInliner.cpp b/libyul/optimiser/FullInliner.cpp index 4b72e4ad9695..32036184af31 100644 --- a/libyul/optimiser/FullInliner.cpp +++ b/libyul/optimiser/FullInliner.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -41,11 +42,11 @@ using namespace solidity::yul; void FullInliner::run(OptimiserStepContext& _context, Block& _ast) { - FullInliner{_ast, _context.dispenser}.run(); + FullInliner{_ast, _context.dispenser, _context.dialect}.run(); } -FullInliner::FullInliner(Block& _ast, NameDispenser& _dispenser): - m_ast(_ast), m_nameDispenser(_dispenser) +FullInliner::FullInliner(Block& _ast, NameDispenser& _dispenser, Dialect const& _dialect): + m_ast(_ast), m_nameDispenser(_dispenser), m_dialect(_dialect) { // Determine constants SSAValueTracker tracker; @@ -139,7 +140,7 @@ void FullInliner::updateCodeSize(FunctionDefinition const& _fun) void FullInliner::handleBlock(YulString _currentFunctionName, Block& _block) { - InlineModifier{*this, m_nameDispenser, _currentFunctionName}(_block); + InlineModifier{*this, m_nameDispenser, _currentFunctionName, m_dialect}(_block); } bool FullInliner::recursive(FunctionDefinition const& _fun) const @@ -198,7 +199,7 @@ vector InlineModifier::performInline(Statement& _statement, FunctionC if (_value) varDecl.value = make_unique(std::move(*_value)); else - varDecl.value = make_unique(Literal{{}, LiteralKind::Number, YulString{"0"}, {}}); + varDecl.value = make_unique(m_dialect.zeroLiteralForType(varDecl.variables.front().type)); newStatements.emplace_back(std::move(varDecl)); }; diff --git a/libyul/optimiser/FullInliner.h b/libyul/optimiser/FullInliner.h index 788eea99bebb..81c608de8f1e 100644 --- a/libyul/optimiser/FullInliner.h +++ b/libyul/optimiser/FullInliner.h @@ -69,7 +69,7 @@ class FullInliner: public ASTModifier { public: static constexpr char const* name{"FullInliner"}; - static void run(OptimiserStepContext&, Block& _ast); + static void run(OptimiserStepContext& _context, Block& _ast); /// Inlining heuristic. /// @param _callSite the name of the function in which the function call is located. @@ -89,7 +89,7 @@ class FullInliner: public ASTModifier void tentativelyUpdateCodeSize(YulString _function, YulString _callSite); private: - FullInliner(Block& _ast, NameDispenser& _dispenser); + FullInliner(Block& _ast, NameDispenser& _dispenser, Dialect const& _dialect); void run(); void updateCodeSize(FunctionDefinition const& _fun); @@ -108,6 +108,7 @@ class FullInliner: public ASTModifier std::set m_constants; std::map m_functionSizes; NameDispenser& m_nameDispenser; + Dialect const& m_dialect; }; /** @@ -117,10 +118,11 @@ class FullInliner: public ASTModifier class InlineModifier: public ASTModifier { public: - InlineModifier(FullInliner& _driver, NameDispenser& _nameDispenser, YulString _functionName): + InlineModifier(FullInliner& _driver, NameDispenser& _nameDispenser, YulString _functionName, Dialect const& _dialect): m_currentFunction(std::move(_functionName)), m_driver(_driver), - m_nameDispenser(_nameDispenser) + m_nameDispenser(_nameDispenser), + m_dialect(_dialect) { } void operator()(Block& _block) override; @@ -132,6 +134,7 @@ class InlineModifier: public ASTModifier YulString m_currentFunction; FullInliner& m_driver; NameDispenser& m_nameDispenser; + Dialect const& m_dialect; }; /** diff --git a/test/libyul/yulOptimizerTests/fullInliner/multi_return_typed.yul b/test/libyul/yulOptimizerTests/fullInliner/multi_return_typed.yul new file mode 100644 index 000000000000..f9f01a193c0b --- /dev/null +++ b/test/libyul/yulOptimizerTests/fullInliner/multi_return_typed.yul @@ -0,0 +1,22 @@ +{ + function f(a: u256) -> x: bool, y:u256 { + y := mul(a, a) + } + let r: bool, s: u256 := f(mload(3)) +} +// ==== +// dialect: evmTyped +// step: fullInliner +// ---- +// { +// { +// let a_3 := mload(3) +// let x_4:bool := false +// let y_5 := 0 +// y_5 := mul(a_3, a_3) +// let r:bool := x_4 +// let s := y_5 +// } +// function f(a) -> x:bool, y +// { y := mul(a, a) } +// } From c8915972046cc45edc37f1803a09c807df5b2140 Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Wed, 19 Feb 2020 17:34:08 +0100 Subject: [PATCH 44/92] Adding ssa type check and test for that one --- libyul/optimiser/SSATransform.cpp | 38 ++++++++++++----- libyul/optimiser/TypeInfo.cpp | 5 +++ libyul/optimiser/TypeInfo.h | 3 ++ .../yulOptimizerTests/ssaTransform/typed.yul | 41 +++++++++++++++++++ .../ssaTransform/typed_for.yul | 25 +++++++++++ .../ssaTransform/typed_switch.yul | 25 +++++++++++ 6 files changed, 127 insertions(+), 10 deletions(-) create mode 100644 test/libyul/yulOptimizerTests/ssaTransform/typed.yul create mode 100644 test/libyul/yulOptimizerTests/ssaTransform/typed_for.yul create mode 100644 test/libyul/yulOptimizerTests/ssaTransform/typed_switch.yul diff --git a/libyul/optimiser/SSATransform.cpp b/libyul/optimiser/SSATransform.cpp index 91454b9eeea5..948a0e6a69ff 100644 --- a/libyul/optimiser/SSATransform.cpp +++ b/libyul/optimiser/SSATransform.cpp @@ -27,6 +27,8 @@ #include +#include + using namespace std; using namespace solidity; using namespace solidity::yul; @@ -42,8 +44,14 @@ namespace class IntroduceSSA: public ASTModifier { public: - explicit IntroduceSSA(NameDispenser& _nameDispenser, set const& _variablesToReplace): - m_nameDispenser(_nameDispenser), m_variablesToReplace(_variablesToReplace) + explicit IntroduceSSA( + NameDispenser& _nameDispenser, + set const& _variablesToReplace, + TypeInfo& _typeInfo + ): + m_nameDispenser(_nameDispenser), + m_variablesToReplace(_variablesToReplace), + m_typeInfo(_typeInfo) { } void operator()(Block& _block) override; @@ -51,6 +59,7 @@ class IntroduceSSA: public ASTModifier private: NameDispenser& m_nameDispenser; set const& m_variablesToReplace; + TypeInfo const& m_typeInfo; }; @@ -83,10 +92,10 @@ void IntroduceSSA::operator()(Block& _block) { YulString oldName = var.name; YulString newName = m_nameDispenser.newName(oldName); - newVariables.emplace_back(TypedName{loc, newName, {}}); + newVariables.emplace_back(TypedName{loc, newName, var.type}); statements.emplace_back(VariableDeclaration{ loc, - {TypedName{loc, oldName, {}}}, + {TypedName{loc, oldName, var.type}}, make_unique(Identifier{loc, newName}) }); } @@ -110,7 +119,11 @@ void IntroduceSSA::operator()(Block& _block) { YulString oldName = var.name; YulString newName = m_nameDispenser.newName(oldName); - newVariables.emplace_back(TypedName{loc, newName, {}}); + newVariables.emplace_back(TypedName{ + loc, + newName, + m_typeInfo.typeOfVariable(oldName) + }); statements.emplace_back(Assignment{ loc, {Identifier{loc, oldName}}, @@ -136,9 +149,12 @@ class IntroduceControlFlowSSA: public ASTModifier public: explicit IntroduceControlFlowSSA( NameDispenser& _nameDispenser, - set const& _variablesToReplace + set const& _variablesToReplace, + TypeInfo const& _typeInfo ): - m_nameDispenser(_nameDispenser), m_variablesToReplace(_variablesToReplace) + m_nameDispenser(_nameDispenser), + m_variablesToReplace(_variablesToReplace), + m_typeInfo(_typeInfo) { } void operator()(FunctionDefinition& _function) override; @@ -153,6 +169,7 @@ class IntroduceControlFlowSSA: public ASTModifier set m_variablesInScope; /// Set of variables that do not have a specific value. set m_variablesToReassign; + TypeInfo const& m_typeInfo; }; void IntroduceControlFlowSSA::operator()(FunctionDefinition& _function) @@ -221,7 +238,7 @@ void IntroduceControlFlowSSA::operator()(Block& _block) YulString newName = m_nameDispenser.newName(toReassign); toPrepend.emplace_back(VariableDeclaration{ locationOf(_s), - {TypedName{locationOf(_s), newName, {}}}, + {TypedName{locationOf(_s), newName, m_typeInfo.typeOfVariable(toReassign)}}, make_unique(Identifier{locationOf(_s), toReassign}) }); assignedVariables.insert(toReassign); @@ -375,10 +392,11 @@ void PropagateValues::operator()(Block& _block) void SSATransform::run(OptimiserStepContext& _context, Block& _ast) { + TypeInfo typeInfo(_context.dialect, _ast); Assignments assignments; assignments(_ast); - IntroduceSSA{_context.dispenser, assignments.names()}(_ast); - IntroduceControlFlowSSA{_context.dispenser, assignments.names()}(_ast); + IntroduceSSA{_context.dispenser, assignments.names(), typeInfo}(_ast); + IntroduceControlFlowSSA{_context.dispenser, assignments.names(), typeInfo}(_ast); PropagateValues{assignments.names()}(_ast); } diff --git a/libyul/optimiser/TypeInfo.cpp b/libyul/optimiser/TypeInfo.cpp index 4b5715baa9d2..ea2d81a8347b 100644 --- a/libyul/optimiser/TypeInfo.cpp +++ b/libyul/optimiser/TypeInfo.cpp @@ -96,3 +96,8 @@ YulString TypeInfo::typeOf(Expression const& _expression) const } }, _expression); } + +YulString TypeInfo::typeOfVariable(YulString _name) const +{ + return m_variableTypes.at(_name); +} diff --git a/libyul/optimiser/TypeInfo.h b/libyul/optimiser/TypeInfo.h index deb537ea3216..48c6c1fba820 100644 --- a/libyul/optimiser/TypeInfo.h +++ b/libyul/optimiser/TypeInfo.h @@ -44,6 +44,9 @@ class TypeInfo /// @returns the type of an expression that is assumed to return exactly one value. YulString typeOf(Expression const& _expression) const; + /// \returns the type of variable + YulString typeOfVariable(YulString _name) const; + private: class TypeCollector; diff --git a/test/libyul/yulOptimizerTests/ssaTransform/typed.yul b/test/libyul/yulOptimizerTests/ssaTransform/typed.yul new file mode 100644 index 000000000000..f9d48a22730b --- /dev/null +++ b/test/libyul/yulOptimizerTests/ssaTransform/typed.yul @@ -0,0 +1,41 @@ +{ + let b:bool := true + let c:bool := false + c := b + b := false + + let a:u256 := 1 + a := add(a, 1) + if c { + a := add(a, 1) + } + a := add(a, 1) + mstore(a, 1) +} +// ==== +// dialect: evmTyped +// step: ssaTransform +// ---- +// { +// let b_1:bool := true +// let b:bool := b_1 +// let c_2:bool := false +// let c:bool := c_2 +// let c_3:bool := b_1 +// c := c_3 +// let b_4:bool := false +// b := b_4 +// let a_5 := 1 +// let a := a_5 +// let a_6 := add(a_5, 1) +// a := a_6 +// if c_3 +// { +// let a_7 := add(a_6, 1) +// a := a_7 +// } +// let a_9 := a +// let a_8 := add(a_9, 1) +// a := a_8 +// mstore(a_8, 1) +// } diff --git a/test/libyul/yulOptimizerTests/ssaTransform/typed_for.yul b/test/libyul/yulOptimizerTests/ssaTransform/typed_for.yul new file mode 100644 index 000000000000..685c42a24ba9 --- /dev/null +++ b/test/libyul/yulOptimizerTests/ssaTransform/typed_for.yul @@ -0,0 +1,25 @@ +{ + let b:bool := true + let c:bool := false + for {} b {} { + c := true + } + let d: bool := c +} +// ==== +// dialect: evmTyped +// step: ssaTransform +// ---- +// { +// let b:bool := true +// let c_1:bool := false +// let c:bool := c_1 +// for { } b { } +// { +// let c_3:bool := c +// let c_2:bool := true +// c := c_2 +// } +// let c_4:bool := c +// let d:bool := c_4 +// } diff --git a/test/libyul/yulOptimizerTests/ssaTransform/typed_switch.yul b/test/libyul/yulOptimizerTests/ssaTransform/typed_switch.yul new file mode 100644 index 000000000000..7d61b1031da8 --- /dev/null +++ b/test/libyul/yulOptimizerTests/ssaTransform/typed_switch.yul @@ -0,0 +1,25 @@ +{ + let b:bool := true + let c:bool := false + switch b + case true { c := true} + case false { } + let d: bool := c +} +// ==== +// dialect: evmTyped +// step: ssaTransform +// ---- +// { +// let b:bool := true +// let c_1:bool := false +// let c:bool := c_1 +// switch b +// case true { +// let c_2:bool := true +// c := c_2 +// } +// case false { } +// let c_3:bool := c +// let d:bool := c_3 +// } From 8524e3f48d17f1c063c984d7341b6565c497a7d5 Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Mon, 24 Feb 2020 18:54:23 +0100 Subject: [PATCH 45/92] Standard-JSON-Interface: Fix a bug (#8371) related to empty filenames and imports. --- Changelog.md | 1 + libsolidity/interface/CompilerStack.cpp | 1 - test/cmdlineTests/standard_empty_file_name/exit | 1 + test/cmdlineTests/standard_empty_file_name/input.json | 10 ++++++++++ test/cmdlineTests/standard_empty_file_name/output.json | 4 ++++ 5 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 test/cmdlineTests/standard_empty_file_name/exit create mode 100644 test/cmdlineTests/standard_empty_file_name/input.json create mode 100644 test/cmdlineTests/standard_empty_file_name/output.json diff --git a/Changelog.md b/Changelog.md index 9981ea25fd99..10c32d884907 100644 --- a/Changelog.md +++ b/Changelog.md @@ -8,6 +8,7 @@ Compiler Features: Bugfixes: * isoltest: Added new keyword `wei` to express function value in semantic tests + * Standard-JSON-Interface: Fix a bug related to empty filenames and imports. ### 0.6.3 (2020-02-18) diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 89ab13818192..6cf044c1c5ee 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -1005,7 +1005,6 @@ void CompilerStack::resolveImports() if (ImportDirective const* import = dynamic_cast(node.get())) { string const& path = import->annotation().absolutePath; - solAssert(!path.empty(), ""); solAssert(m_sources.count(path), ""); import->annotation().sourceUnit = m_sources[path].ast.get(); toposort(&m_sources[path]); diff --git a/test/cmdlineTests/standard_empty_file_name/exit b/test/cmdlineTests/standard_empty_file_name/exit new file mode 100644 index 000000000000..573541ac9702 --- /dev/null +++ b/test/cmdlineTests/standard_empty_file_name/exit @@ -0,0 +1 @@ +0 diff --git a/test/cmdlineTests/standard_empty_file_name/input.json b/test/cmdlineTests/standard_empty_file_name/input.json new file mode 100644 index 000000000000..95c2cdd307aa --- /dev/null +++ b/test/cmdlineTests/standard_empty_file_name/input.json @@ -0,0 +1,10 @@ +{ + "language": "Solidity", + "sources": + { + "": + { + "content": "pragma solidity >=0.0; import {A} from \".\";" + } + } +} diff --git a/test/cmdlineTests/standard_empty_file_name/output.json b/test/cmdlineTests/standard_empty_file_name/output.json new file mode 100644 index 000000000000..b31ceb152a8d --- /dev/null +++ b/test/cmdlineTests/standard_empty_file_name/output.json @@ -0,0 +1,4 @@ +{"errors":[{"component":"general","formattedMessage":":1:24: DeclarationError: Declaration \"A\" not found in \"\" (referenced as \".\"). +pragma solidity >=0.0; import {A} from \".\"; + ^------------------^ +","message":"Declaration \"A\" not found in \"\" (referenced as \".\").","severity":"error","type":"DeclarationError"}],"sources":{}} From 24eb39ca1146905efaf2f44807e35eec2035500d Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Thu, 20 Feb 2020 20:02:28 +0530 Subject: [PATCH 46/92] Docker: Add image identical to one used by ossfuzz builder --- .circleci/config.yml | 32 ++++-- .../Dockerfile.ubuntu1604.clang.ossfuzz | 101 ++++++++++++++++++ ...ubuntu1904 => Dockerfile.ubuntu1904.clang} | 0 cmake/toolchains/libfuzzer.cmake | 6 +- 4 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 .circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz rename .circleci/docker/{Dockerfile.clang.ubuntu1904 => Dockerfile.ubuntu1904.clang} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index f8f142412f9d..8bee0b83b957 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,6 +16,9 @@ parameters: ubuntu-1904-clang-docker-image-rev: type: string default: "5" + ubuntu-1604-clang-ossfuzz-docker-image-rev: + type: string + default: "1" defaults: @@ -119,6 +122,17 @@ defaults: name: command line tests command: ./test/cmdlineTests.sh + - test_ubuntu1604_clang: &test_ubuntu1604_clang + docker: + - image: ethereum/solidity-buildpack-deps:ubuntu1604-clang-ossfuzz-<< pipeline.parameters.ubuntu-1604-clang-ossfuzz-docker-image-rev >> + steps: + - checkout + - attach_workspace: + at: build + - run: *run_soltest + - store_test_results: *store_test_results + - store_artifacts: *artifacts_test_results + - test_ubuntu1904_clang: &test_ubuntu1904_clang docker: - image: ethereum/solidity-buildpack-deps:ubuntu1904-clang-<< pipeline.parameters.ubuntu-1904-clang-docker-image-rev >> @@ -166,6 +180,11 @@ defaults: requires: - b_ubu + - workflow_ubuntu1604_clang: &workflow_ubuntu1604_clang + <<: *workflow_trigger_on_tags + requires: + - b_ubu_ossfuzz + - workflow_ubuntu1904_clang: &workflow_ubuntu1904_clang <<: *workflow_trigger_on_tags requires: @@ -196,7 +215,7 @@ defaults: requires: - b_ems - - workflow_ubuntu1904_ossfuzz: &workflow_ubuntu1904_ossfuzz + - workflow_ubuntu1604_ossfuzz: &workflow_ubuntu1604_ossfuzz <<: *workflow_trigger_on_tags requires: - b_ubu_ossfuzz @@ -397,12 +416,13 @@ jobs: - checkout - run: *run_build - b_ubu_ossfuzz: - <<: *build_ubuntu1904_clang + b_ubu_ossfuzz: &build_ubuntu1604_clang + docker: + - image: ethereum/solidity-buildpack-deps:ubuntu1604-clang-ossfuzz-<< pipeline.parameters.ubuntu-1604-clang-ossfuzz-docker-image-rev >> environment: - TERM: xterm CC: clang CXX: clang++ + TERM: xterm CMAKE_OPTIONS: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchains/libfuzzer.cmake steps: - checkout @@ -411,7 +431,7 @@ jobs: - persist_to_workspace: *artifacts_executables_ossfuzz t_ubu_ossfuzz: &t_ubu_ossfuzz - <<: *test_ubuntu1904_clang + <<: *test_ubuntu1604_clang steps: - checkout - attach_workspace: @@ -799,7 +819,7 @@ workflows: jobs: # OSSFUZZ builds and (regression) tests - b_ubu_ossfuzz: *workflow_trigger_on_tags - - t_ubu_ossfuzz: *workflow_ubuntu1904_ossfuzz + - t_ubu_ossfuzz: *workflow_ubuntu1604_ossfuzz # Code Coverage enabled build and tests - b_ubu_codecov: *workflow_trigger_on_tags diff --git a/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz b/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz new file mode 100644 index 000000000000..8e2ee01b0668 --- /dev/null +++ b/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz @@ -0,0 +1,101 @@ +# vim:syntax=dockerfile +#------------------------------------------------------------------------------ +# Dockerfile for building and testing Solidity Compiler on CI +# Target: Ubuntu 16.04 (Xenial Xerus) ossfuzz Clang variant +# URL: https://hub.docker.com/r/ethereum/solidity-buildpack-deps +# +# This file is part of solidity. +# +# solidity is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# solidity is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with solidity. If not, see +# +# (c) 2016-2019 solidity contributors. +#------------------------------------------------------------------------------ +FROM gcr.io/oss-fuzz-base/base-clang as base + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update; \ + apt-get -qqy install --no-install-recommends \ + build-essential \ + software-properties-common \ + ninja-build git wget \ + libbz2-dev zlib1g-dev git; \ + apt-get install -qy python-pip python-sphinx; + +# Install cmake 3.14 (minimum requirement is cmake 3.10) +RUN wget https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.sh; \ + chmod +x cmake-3.14.5-Linux-x86_64.sh; \ + ./cmake-3.14.5-Linux-x86_64.sh --skip-license --prefix="/usr" + +FROM base AS libraries + +# Boost +RUN git clone -b boost-1.69.0 https://github.com/boostorg/boost.git \ + /usr/src/boost; \ + cd /usr/src/boost; \ + git submodule update --init --recursive; \ + ./bootstrap.sh --with-toolset=clang --prefix=/usr; \ + ./b2 toolset=clang cxxflags="-stdlib=libc++" linkflags="-stdlib=libc++" headers; \ + ./b2 toolset=clang cxxflags="-stdlib=libc++" linkflags="-stdlib=libc++" \ + link=static variant=release runtime-link=static \ + system filesystem unit_test_framework program_options \ + install -j $(($(nproc)/2)); \ + rm -rf /usr/src/boost + +# Z3 +RUN git clone --depth 1 -b z3-4.8.7 https://github.com/Z3Prover/z3.git \ + /usr/src/z3; \ + cd /usr/src/z3; \ + mkdir build; \ + cd build; \ + LDFLAGS=$CXXFLAGS cmake -DZ3_BUILD_LIBZ3_SHARED=OFF -DCMAKE_INSTALL_PREFIX=/usr \ + -DCMAKE_BUILD_TYPE=Release ..; \ + make libz3 -j; \ + make install; \ + rm -rf /usr/src/z3 + +# OSSFUZZ: libprotobuf-mutator +RUN set -ex; \ + git clone https://github.com/google/libprotobuf-mutator.git \ + /usr/src/libprotobuf-mutator; \ + cd /usr/src/libprotobuf-mutator; \ + git checkout 3521f47a2828da9ace403e4ecc4aece1a84feb36; \ + mkdir build; \ + cd build; \ + cmake .. -GNinja -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON \ + -DLIB_PROTO_MUTATOR_TESTING=OFF -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="/usr"; \ + ninja; \ + cp -vpr external.protobuf/bin/* /usr/bin/; \ + cp -vpr external.protobuf/include/* /usr/include/; \ + cp -vpr external.protobuf/lib/* /usr/lib/; \ + ninja install/strip; \ + rm -rf /usr/src/libprotobuf-mutator + +# EVMONE +RUN set -ex; \ + cd /usr/src; \ + git clone --branch="v0.4.0" --recurse-submodules https://github.com/ethereum/evmone.git; \ + cd evmone; \ + mkdir build; \ + cd build; \ + cmake -G Ninja -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX="/usr" ..; \ + ninja; \ + ninja install/strip; \ + rm -rf /usr/src/evmone + +FROM base +COPY --from=libraries /usr/lib /usr/lib +COPY --from=libraries /usr/bin /usr/bin +COPY --from=libraries /usr/include /usr/include diff --git a/.circleci/docker/Dockerfile.clang.ubuntu1904 b/.circleci/docker/Dockerfile.ubuntu1904.clang similarity index 100% rename from .circleci/docker/Dockerfile.clang.ubuntu1904 rename to .circleci/docker/Dockerfile.ubuntu1904.clang diff --git a/cmake/toolchains/libfuzzer.cmake b/cmake/toolchains/libfuzzer.cmake index 53c36d3c90c5..a734df975ec4 100644 --- a/cmake/toolchains/libfuzzer.cmake +++ b/cmake/toolchains/libfuzzer.cmake @@ -8,4 +8,8 @@ set(OSSFUZZ ON CACHE BOOL "Enable fuzzer build" FORCE) # Use libfuzzer as the fuzzing back-end set(LIB_FUZZING_ENGINE "-fsanitize=fuzzer" CACHE STRING "Use libfuzzer back-end" FORCE) # clang/libfuzzer specific flags for UBSan instrumentation -set(CMAKE_CXX_FLAGS "-O1 -gline-tables-only -fsanitize=undefined -fsanitize=fuzzer-no-link -stdlib=libstdc++" CACHE STRING "Custom compilation flags" FORCE) +set(CMAKE_CXX_FLAGS "-O1 -fno-omit-frame-pointer -gline-tables-only -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -I /usr/local/include/c++/v1 -fsanitize=undefined -fsanitize=fuzzer-no-link -stdlib=libc++" CACHE STRING "Custom compilation flags" FORCE) +# Link statically against boost libraries +set(BOOST_FOUND ON CACHE BOOL "" FORCE) +set(Boost_USE_STATIC_LIBS ON CACHE BOOL "Link against static Boost libraries" FORCE) +set(Boost_USE_STATIC_RUNTIME ON CACHE BOOL "Link against static Boost runtime library" FORCE) From 45041e5d3a155782150ca2a09debb1d4cbfd7eae Mon Sep 17 00:00:00 2001 From: chriseth Date: Mon, 24 Feb 2020 15:53:32 +0100 Subject: [PATCH 47/92] Allow access to ``_slot`` for local storage pointer variables. --- Changelog.md | 1 + docs/assembly.rst | 6 +++- libsolidity/analysis/TypeChecker.cpp | 16 +++++++-- libsolidity/codegen/ContractCompiler.cpp | 2 +- .../inlineAssembly/slot_access.sol | 33 +++++++++++++++++++ .../storage_reference_assignment.sol | 3 +- .../storage_reference_assignment_statevar.sol | 12 +++++++ .../inlineAssembly/storage_slot_assign.yul | 12 +++++++ 8 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 test/libsolidity/semanticTests/inlineAssembly/slot_access.sol create mode 100644 test/libsolidity/syntaxTests/inlineAssembly/storage_reference_assignment_statevar.sol create mode 100644 test/libsolidity/syntaxTests/inlineAssembly/storage_slot_assign.yul diff --git a/Changelog.md b/Changelog.md index 9981ea25fd99..6a829845a837 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,6 +1,7 @@ ### 0.6.4 (unreleased) Language Features: + * Inline Assembly: Allow assigning to `_slot` of local storage variable pointers. Compiler Features: diff --git a/docs/assembly.rst b/docs/assembly.rst index 67c1ab6f603d..7e498cd5afa1 100644 --- a/docs/assembly.rst +++ b/docs/assembly.rst @@ -172,6 +172,11 @@ Assignments are possible to assembly-local variables and to function-local variables. Take care that when you assign to variables that point to memory or storage, you will only change the pointer and not the data. +You can assign to the ``_slot`` part of a local storage variable pointer. +For these (structs, arrays or mappings), the ``_offset`` part is always zero. +It is not possible to assign to the ``_slot`` or ``_offset`` part of a state variable, +though. + Things to Avoid @@ -225,4 +230,3 @@ first slot of the array and followed by the array elements. Statically-sized memory arrays do not have a length field, but it might be added later to allow better convertibility between statically- and dynamically-sized arrays, so do not rely on this. - diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index e41425c6e594..2f9526f879fd 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -677,10 +677,20 @@ bool TypeChecker::visit(InlineAssembly const& _inlineAssembly) m_errorReporter.typeError(_identifier.location, "The suffixes _offset and _slot can only be used on storage variables."); return size_t(-1); } - else if (_context != yul::IdentifierContext::RValue) + else if (_context == yul::IdentifierContext::LValue) { - m_errorReporter.typeError(_identifier.location, "Storage variables cannot be assigned to."); - return size_t(-1); + if (var->isStateVariable()) + { + m_errorReporter.typeError(_identifier.location, "State variables cannot be assigned to - you have to use \"sstore()\"."); + return size_t(-1); + } + else if (ref->second.isOffset) + { + m_errorReporter.typeError(_identifier.location, "Only _slot can be assigned to."); + return size_t(-1); + } + else + solAssert(ref->second.isSlot, ""); } } else if (!var->isConstant() && var->isStateVariable()) diff --git a/libsolidity/codegen/ContractCompiler.cpp b/libsolidity/codegen/ContractCompiler.cpp index 298437b1deee..a99aefb5f3ce 100644 --- a/libsolidity/codegen/ContractCompiler.cpp +++ b/libsolidity/codegen/ContractCompiler.cpp @@ -781,7 +781,7 @@ bool ContractCompiler::visit(InlineAssembly const& _inlineAssembly) else { // lvalue context - solAssert(!ref->second.isOffset && !ref->second.isSlot, ""); + solAssert(!ref->second.isOffset, ""); auto variable = dynamic_cast(decl); solAssert( !!variable && m_context.isLocalVariable(variable), diff --git a/test/libsolidity/semanticTests/inlineAssembly/slot_access.sol b/test/libsolidity/semanticTests/inlineAssembly/slot_access.sol new file mode 100644 index 000000000000..4f3be74897bd --- /dev/null +++ b/test/libsolidity/semanticTests/inlineAssembly/slot_access.sol @@ -0,0 +1,33 @@ +contract C { + struct S { + uint a; + uint b; + } + + mapping(uint => S) public mappingAccess; + + function data() internal view returns (S storage _data) { + // We need to assign it from somewhere, otherwise we would + // get an "uninitialized access" error. + _data = mappingAccess[20]; + + bytes32 slot = keccak256(abi.encode(uint(1), uint(0))); + assembly { + _data_slot := slot + } + } + + function set(uint x) public { + data().a = x; + } + + function get() public view returns (uint) { + return data().a; + } +} +// ---- +// get() -> 0 +// mappingAccess(uint256): 1 -> 0, 0 +// set(uint256): 4 +// get() -> 4 +// mappingAccess(uint256): 1 -> 4, 0 diff --git a/test/libsolidity/syntaxTests/inlineAssembly/storage_reference_assignment.sol b/test/libsolidity/syntaxTests/inlineAssembly/storage_reference_assignment.sol index 164265781e23..91362d719241 100644 --- a/test/libsolidity/syntaxTests/inlineAssembly/storage_reference_assignment.sol +++ b/test/libsolidity/syntaxTests/inlineAssembly/storage_reference_assignment.sol @@ -9,5 +9,4 @@ contract C { } } // ---- -// TypeError: (114-120): Storage variables cannot be assigned to. -// TypeError: (138-146): Storage variables cannot be assigned to. +// TypeError: (138-146): Only _slot can be assigned to. diff --git a/test/libsolidity/syntaxTests/inlineAssembly/storage_reference_assignment_statevar.sol b/test/libsolidity/syntaxTests/inlineAssembly/storage_reference_assignment_statevar.sol new file mode 100644 index 000000000000..d37008774fdf --- /dev/null +++ b/test/libsolidity/syntaxTests/inlineAssembly/storage_reference_assignment_statevar.sol @@ -0,0 +1,12 @@ +contract C { + uint[] x; + fallback() external { + assembly { + x_slot := 1 + x_offset := 2 + } + } +} +// ---- +// TypeError: (84-90): State variables cannot be assigned to - you have to use "sstore()". +// TypeError: (108-116): State variables cannot be assigned to - you have to use "sstore()". diff --git a/test/libsolidity/syntaxTests/inlineAssembly/storage_slot_assign.yul b/test/libsolidity/syntaxTests/inlineAssembly/storage_slot_assign.yul new file mode 100644 index 000000000000..91362d719241 --- /dev/null +++ b/test/libsolidity/syntaxTests/inlineAssembly/storage_slot_assign.yul @@ -0,0 +1,12 @@ +contract C { + uint[] x; + fallback() external { + uint[] storage y = x; + assembly { + y_slot := 1 + y_offset := 2 + } + } +} +// ---- +// TypeError: (138-146): Only _slot can be assigned to. From 26bae6b4599cc00122fa0c3a38937f4f6395e454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 14 Feb 2020 20:37:49 +0100 Subject: [PATCH 48/92] [yul-phaser] Common: Add countSubstringOccurrences() --- test/yulPhaser/Common.cpp | 15 +++++++++++++++ test/yulPhaser/Common.h | 4 ++++ test/yulPhaser/CommonTest.cpp | 19 +++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/test/yulPhaser/Common.cpp b/test/yulPhaser/Common.cpp index 38c6e445b995..93aa432fa996 100644 --- a/test/yulPhaser/Common.cpp +++ b/test/yulPhaser/Common.cpp @@ -49,3 +49,18 @@ string phaser::test::stripWhitespace(string const& input) regex whitespaceRegex("\\s+"); return regex_replace(input, whitespaceRegex, ""); } + +size_t phaser::test::countSubstringOccurrences(string const& _inputString, string const& _substring) +{ + assert(_substring.size() > 0); + + size_t count = 0; + size_t lastOccurrence = 0; + while ((lastOccurrence = _inputString.find(_substring, lastOccurrence)) != string::npos) + { + ++count; + lastOccurrence += _substring.size(); + } + + return count; +} diff --git a/test/yulPhaser/Common.h b/test/yulPhaser/Common.h index 63ca45c5083a..e73260d7adb7 100644 --- a/test/yulPhaser/Common.h +++ b/test/yulPhaser/Common.h @@ -67,6 +67,10 @@ std::map enumerateOptmisationSteps(); /// Returns the input string with all the whitespace characters (spaces, line endings, etc.) removed. std::string stripWhitespace(std::string const& input); +/// Counts the number of times one strinng can be found inside another. Only non-overlapping +/// occurrences are counted. +size_t countSubstringOccurrences(std::string const& _inputString, std::string const& _substring); + // STATISTICAL UTILITIES /// Calculates the mean value of a series of samples given in a vector. diff --git a/test/yulPhaser/CommonTest.cpp b/test/yulPhaser/CommonTest.cpp index feded8292965..9bcab992314f 100644 --- a/test/yulPhaser/CommonTest.cpp +++ b/test/yulPhaser/CommonTest.cpp @@ -83,6 +83,25 @@ BOOST_AUTO_TEST_CASE(stripWhitespace_should_remove_all_whitespace_characters_fro BOOST_TEST(stripWhitespace(" a b \n\n c \n\t\v") == "abc"); } +BOOST_AUTO_TEST_CASE(countSubstringOccurrences_should_count_non_overlapping_substring_occurrences_in_a_string) +{ + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "a") == 6); + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "aa") == 2); + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "aaa") == 2); + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "aaab") == 1); + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "b") == 2); + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "d") == 1); + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "cdc") == 1); + + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "x") == 0); + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "aaaa") == 0); + BOOST_TEST(countSubstringOccurrences("aaabcdcbaaa", "dcd") == 0); + + BOOST_TEST(countSubstringOccurrences("", "a") == 0); + BOOST_TEST(countSubstringOccurrences("", "aa") == 0); + BOOST_TEST(countSubstringOccurrences("a", "aa") == 0); +} + BOOST_AUTO_TEST_CASE(mean_should_calculate_statistical_mean) { BOOST_TEST(mean({0}) == 0.0); From 11bdf358df078d9fb69136b79a788164f4636b22 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 14:56:55 +0100 Subject: [PATCH 49/92] [yul-phaser] Base class for genetic algorithms --- test/CMakeLists.txt | 2 + test/yulPhaser/GeneticAlgorithms.cpp | 96 +++++++++++++++++++++++++++ tools/CMakeLists.txt | 2 + tools/yulPhaser/GeneticAlgorithms.cpp | 32 +++++++++ tools/yulPhaser/GeneticAlgorithms.h | 66 ++++++++++++++++++ 5 files changed, 198 insertions(+) create mode 100644 test/yulPhaser/GeneticAlgorithms.cpp create mode 100644 tools/yulPhaser/GeneticAlgorithms.cpp create mode 100644 tools/yulPhaser/GeneticAlgorithms.h diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3b2fef6f00ef..70d547a0b1d0 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -144,6 +144,7 @@ set(yul_phaser_sources yulPhaser/CommonTest.cpp yulPhaser/Chromosome.cpp yulPhaser/FitnessMetrics.cpp + yulPhaser/GeneticAlgorithms.cpp yulPhaser/Population.cpp yulPhaser/Program.cpp yulPhaser/SimulationRNG.cpp @@ -153,6 +154,7 @@ set(yul_phaser_sources # unnecessary duplication. Create a library or find a way to reuse the list in both places. ../tools/yulPhaser/Chromosome.cpp ../tools/yulPhaser/FitnessMetrics.cpp + ../tools/yulPhaser/GeneticAlgorithms.cpp ../tools/yulPhaser/Population.cpp ../tools/yulPhaser/Program.cpp ../tools/yulPhaser/SimulationRNG.cpp diff --git a/test/yulPhaser/GeneticAlgorithms.cpp b/test/yulPhaser/GeneticAlgorithms.cpp new file mode 100644 index 000000000000..3528f2be08fa --- /dev/null +++ b/test/yulPhaser/GeneticAlgorithms.cpp @@ -0,0 +1,96 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include +#include +#include +#include + +#include + +#include + +#include +#include + +#include + +using namespace std; +using namespace boost::unit_test::framework; +using namespace boost::test_tools; +using namespace solidity::langutil; +using namespace solidity::util; + +namespace solidity::phaser::test +{ + +class DummyAlgorithm: public GeneticAlgorithm +{ +public: + using GeneticAlgorithm::GeneticAlgorithm; + void runNextRound() override { ++m_currentRound; } + + size_t m_currentRound = 0; +}; + +class GeneticAlgorithmFixture +{ +protected: + shared_ptr m_fitnessMetric = make_shared(); + output_test_stream m_output; +}; + +BOOST_AUTO_TEST_SUITE(Phaser) +BOOST_AUTO_TEST_SUITE(GeneticAlgorithmsTest) +BOOST_AUTO_TEST_SUITE(GeneticAlgorithmTest) + +BOOST_FIXTURE_TEST_CASE(run_should_call_runNextRound_once_per_round, GeneticAlgorithmFixture) +{ + DummyAlgorithm algorithm(Population(m_fitnessMetric), m_output); + + BOOST_TEST(algorithm.m_currentRound == 0); + algorithm.run(10); + BOOST_TEST(algorithm.m_currentRound == 10); + algorithm.run(3); + BOOST_TEST(algorithm.m_currentRound == 13); +} + +BOOST_FIXTURE_TEST_CASE(run_should_print_the_top_chromosome, GeneticAlgorithmFixture) +{ + // run() is allowed to print more but should at least print the first one + + DummyAlgorithm algorithm( + // NOTE: Chromosomes chosen so that they're not substrings of each other and are not + // words likely to appear in the output in normal circumstances. + Population(m_fitnessMetric, {Chromosome("fcCUnDve"), Chromosome("jsxIOo"), Chromosome("ighTLM")}), + m_output + ); + + BOOST_TEST(m_output.is_empty()); + algorithm.run(1); + BOOST_TEST(countSubstringOccurrences(m_output.str(), toString(algorithm.population().individuals()[0].chromosome)) == 1); + algorithm.run(3); + BOOST_TEST(countSubstringOccurrences(m_output.str(), toString(algorithm.population().individuals()[0].chromosome)) == 4); +} + +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() + +} diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index d3101ad7ba9c..601b249e6002 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -15,6 +15,8 @@ install(TARGETS solidity-upgrade DESTINATION "${CMAKE_INSTALL_BINDIR}") add_executable(yul-phaser yulPhaser/main.cpp + yulPhaser/GeneticAlgorithms.h + yulPhaser/GeneticAlgorithms.cpp yulPhaser/Population.h yulPhaser/Population.cpp yulPhaser/FitnessMetrics.h diff --git a/tools/yulPhaser/GeneticAlgorithms.cpp b/tools/yulPhaser/GeneticAlgorithms.cpp new file mode 100644 index 000000000000..6e69565f4ee6 --- /dev/null +++ b/tools/yulPhaser/GeneticAlgorithms.cpp @@ -0,0 +1,32 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +using namespace std; +using namespace solidity::phaser; + +void GeneticAlgorithm::run(optional _numRounds) +{ + for (size_t round = 0; !_numRounds.has_value() || round < _numRounds.value(); ++round) + { + runNextRound(); + + m_outputStream << "---------- ROUND " << round << " ----------" << endl; + m_outputStream << m_population; + } +} diff --git a/tools/yulPhaser/GeneticAlgorithms.h b/tools/yulPhaser/GeneticAlgorithms.h new file mode 100644 index 000000000000..c2700416b6e2 --- /dev/null +++ b/tools/yulPhaser/GeneticAlgorithms.h @@ -0,0 +1,66 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +/** + * Contains an abstract base class representing a genetic algorithm and its concrete implementations. + */ + +#pragma once + +#include + +#include +#include + +namespace solidity::phaser +{ + +/** + * Abstract base class for genetic algorithms. + * + * The main feature is the @a run() method that executes the algorithm, updating the internal + * population during each round and printing the results to the stream provided to the constructor. + * + * Derived classes can provide specific methods for updating the population by implementing + * the @a runNextRound() method. + */ +class GeneticAlgorithm +{ +public: + GeneticAlgorithm(Population _initialPopulation, std::ostream& _outputStream): + m_population(std::move(_initialPopulation)), + m_outputStream(_outputStream) {} + + GeneticAlgorithm(GeneticAlgorithm const&) = delete; + GeneticAlgorithm& operator=(GeneticAlgorithm const&) = delete; + virtual ~GeneticAlgorithm() = default; + + Population const& population() const { return m_population; } + + void run(std::optional _numRounds = std::nullopt); + + /// The method that actually implements the algorithm. Should use @a m_population as input and + /// replace it with the updated state after the round. + virtual void runNextRound() = 0; + +protected: + Population m_population; + +private: + std::ostream& m_outputStream; +}; + +} From 3c41bfbc4e22b7a34dee6886803ebd60ca4cdc10 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 14:56:55 +0100 Subject: [PATCH 50/92] [yul-phaser] Base class for selections --- tools/CMakeLists.txt | 2 ++ tools/yulPhaser/Selections.cpp | 18 ++++++++++++ tools/yulPhaser/Selections.h | 52 ++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 tools/yulPhaser/Selections.cpp create mode 100644 tools/yulPhaser/Selections.h diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 601b249e6002..21c0b8c913a6 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -23,6 +23,8 @@ add_executable(yul-phaser yulPhaser/FitnessMetrics.cpp yulPhaser/Chromosome.h yulPhaser/Chromosome.cpp + yulPhaser/Selections.h + yulPhaser/Selections.cpp yulPhaser/Program.h yulPhaser/Program.cpp yulPhaser/SimulationRNG.h diff --git a/tools/yulPhaser/Selections.cpp b/tools/yulPhaser/Selections.cpp new file mode 100644 index 000000000000..19451f45c8ea --- /dev/null +++ b/tools/yulPhaser/Selections.cpp @@ -0,0 +1,18 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include diff --git a/tools/yulPhaser/Selections.h b/tools/yulPhaser/Selections.h new file mode 100644 index 000000000000..1155c6708ed2 --- /dev/null +++ b/tools/yulPhaser/Selections.h @@ -0,0 +1,52 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +/** + * Contains an abstract base class representing a selection of elements from a collection + * and its concrete implementations. + */ + +#pragma once + +#include + +namespace solidity::phaser +{ + +/** + * Abstract base class for selections of elements from a collection. + * + * An instance of this class represents a specific method of selecting a set of elements from + * containers of arbitrary sizes. The set of selected elements is always a subset of the container + * but may indicate the same element more than once. The selection may or may not be fixed - it's + * up to a specific implementation whether subsequent calls for the same container produce the same + * indices or not. + * + * Derived classes are meant to override the @a materialise() method. + * This method is expected to produce indices of selected elements given the size of the collection. + */ +class Selection +{ +public: + Selection() = default; + Selection(Selection const&) = delete; + Selection& operator=(Selection const&) = delete; + virtual ~Selection() = default; + + virtual std::vector materialise(size_t _poolSize) const = 0; +}; + +} From 83b8ab8012d4a0d33e5e65812b46d63d77611a2e Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 14:57:29 +0100 Subject: [PATCH 51/92] [yul-phaser] Add RangeSelection, MosaicSelection and RandomSelection classes --- test/CMakeLists.txt | 2 + test/yulPhaser/Selections.cpp | 206 +++++++++++++++++++++++++++++++++ tools/yulPhaser/Selections.cpp | 42 +++++++ tools/yulPhaser/Selections.h | 69 +++++++++++ 4 files changed, 319 insertions(+) create mode 100644 test/yulPhaser/Selections.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 70d547a0b1d0..3ea080686771 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -147,6 +147,7 @@ set(yul_phaser_sources yulPhaser/GeneticAlgorithms.cpp yulPhaser/Population.cpp yulPhaser/Program.cpp + yulPhaser/Selections.cpp yulPhaser/SimulationRNG.cpp # FIXME: yul-phaser is not a library so I can't just add it to target_link_libraries(). @@ -157,6 +158,7 @@ set(yul_phaser_sources ../tools/yulPhaser/GeneticAlgorithms.cpp ../tools/yulPhaser/Population.cpp ../tools/yulPhaser/Program.cpp + ../tools/yulPhaser/Selections.cpp ../tools/yulPhaser/SimulationRNG.cpp ) detect_stray_source_files("${yul_phaser_sources}" "yulPhaser/") diff --git a/test/yulPhaser/Selections.cpp b/test/yulPhaser/Selections.cpp new file mode 100644 index 000000000000..ce870ce80273 --- /dev/null +++ b/test/yulPhaser/Selections.cpp @@ -0,0 +1,206 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include +#include + +#include + +#include + +#include +#include + +using namespace std; + +namespace solidity::phaser::test +{ + +BOOST_AUTO_TEST_SUITE(Phaser) +BOOST_AUTO_TEST_SUITE(SelectionsTest) +BOOST_AUTO_TEST_SUITE(RangeSelectionTest) + +BOOST_AUTO_TEST_CASE(materialise) +{ + BOOST_TEST(RangeSelection(0.0, 1.0).materialise(10) == vector({0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + BOOST_TEST(RangeSelection(0.0, 0.1).materialise(10) == vector({0})); + BOOST_TEST(RangeSelection(0.0, 0.2).materialise(10) == vector({0, 1})); + BOOST_TEST(RangeSelection(0.0, 0.7).materialise(10) == vector({0, 1, 2, 3, 4, 5, 6})); + + BOOST_TEST(RangeSelection(0.9, 1.0).materialise(10) == vector({ 9})); + BOOST_TEST(RangeSelection(0.8, 1.0).materialise(10) == vector({ 8, 9})); + BOOST_TEST(RangeSelection(0.5, 1.0).materialise(10) == vector({ 5, 6, 7, 8, 9})); + + BOOST_TEST(RangeSelection(0.3, 0.6).materialise(10) == vector({ 3, 4, 5 })); + BOOST_TEST(RangeSelection(0.2, 0.7).materialise(10) == vector({ 2, 3, 4, 5, 6 })); + BOOST_TEST(RangeSelection(0.4, 0.7).materialise(10) == vector({ 4, 5, 6 })); + + BOOST_TEST(RangeSelection(0.4, 0.7).materialise(5) == vector({2, 3})); +} + +BOOST_AUTO_TEST_CASE(materialise_should_round_indices) +{ + BOOST_TEST(RangeSelection(0.01, 0.99).materialise(10) == vector({0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + BOOST_TEST(RangeSelection(0.04, 0.96).materialise(10) == vector({0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + BOOST_TEST(RangeSelection(0.05, 0.95).materialise(10) == vector({ 1, 2, 3, 4, 5, 6, 7, 8, 9})); + BOOST_TEST(RangeSelection(0.06, 0.94).materialise(10) == vector({ 1, 2, 3, 4, 5, 6, 7, 8 })); +} + +BOOST_AUTO_TEST_CASE(materialise_should_handle_empty_collections) +{ + BOOST_TEST(RangeSelection(0.0, 0.0).materialise(0).empty()); + BOOST_TEST(RangeSelection(0.0, 1.0).materialise(0).empty()); + BOOST_TEST(RangeSelection(0.5, 1.0).materialise(0).empty()); + BOOST_TEST(RangeSelection(0.0, 0.5).materialise(0).empty()); + BOOST_TEST(RangeSelection(0.2, 0.7).materialise(0).empty()); +} + +BOOST_AUTO_TEST_CASE(materialise_should_handle_empty_selection_ranges) +{ + BOOST_TEST(RangeSelection(0.0, 0.0).materialise(1).empty()); + BOOST_TEST(RangeSelection(1.0, 1.0).materialise(1).empty()); + + BOOST_TEST(RangeSelection(0.0, 0.0).materialise(100).empty()); + BOOST_TEST(RangeSelection(1.0, 1.0).materialise(100).empty()); + BOOST_TEST(RangeSelection(0.5, 0.5).materialise(100).empty()); + + BOOST_TEST(RangeSelection(0.45, 0.54).materialise(10).empty()); + BOOST_TEST(!RangeSelection(0.45, 0.54).materialise(100).empty()); + BOOST_TEST(RangeSelection(0.045, 0.054).materialise(100).empty()); +} + +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE(MosaicSelectionTest) + +BOOST_AUTO_TEST_CASE(materialise) +{ + BOOST_TEST(MosaicSelection({1}, 0.5).materialise(4) == vector({1, 1})); + BOOST_TEST(MosaicSelection({1}, 1.0).materialise(4) == vector({1, 1, 1, 1})); + BOOST_TEST(MosaicSelection({1}, 2.0).materialise(4) == vector({1, 1, 1, 1, 1, 1, 1, 1})); + BOOST_TEST(MosaicSelection({1}, 1.0).materialise(2) == vector({1, 1})); + + BOOST_TEST(MosaicSelection({0, 1}, 0.5).materialise(4) == vector({0, 1})); + BOOST_TEST(MosaicSelection({0, 1}, 1.0).materialise(4) == vector({0, 1, 0, 1})); + BOOST_TEST(MosaicSelection({0, 1}, 2.0).materialise(4) == vector({0, 1, 0, 1, 0, 1, 0, 1})); + BOOST_TEST(MosaicSelection({0, 1}, 1.0).materialise(2) == vector({0, 1})); + + BOOST_TEST(MosaicSelection({3, 2, 1, 0}, 0.5).materialise(4) == vector({3, 2})); + BOOST_TEST(MosaicSelection({3, 2, 1, 0}, 1.0).materialise(4) == vector({3, 2, 1, 0})); + BOOST_TEST(MosaicSelection({3, 2, 1, 0}, 2.0).materialise(4) == vector({3, 2, 1, 0, 3, 2, 1, 0})); + BOOST_TEST(MosaicSelection({1, 0, 1, 0}, 1.0).materialise(2) == vector({1, 0})); +} + +BOOST_AUTO_TEST_CASE(materialise_should_round_indices) +{ + BOOST_TEST(MosaicSelection({4, 3, 2, 1, 0}, 0.49).materialise(5) == vector({4, 3})); + BOOST_TEST(MosaicSelection({4, 3, 2, 1, 0}, 0.50).materialise(5) == vector({4, 3, 2})); + BOOST_TEST(MosaicSelection({4, 3, 2, 1, 0}, 0.51).materialise(5) == vector({4, 3, 2})); +} + +BOOST_AUTO_TEST_CASE(materialise_should_handle_empty_collections) +{ + BOOST_TEST(MosaicSelection({1}, 1.0).materialise(0).empty()); + BOOST_TEST(MosaicSelection({1, 3}, 2.0).materialise(0).empty()); + BOOST_TEST(MosaicSelection({5, 4, 3, 2}, 0.5).materialise(0).empty()); +} + +BOOST_AUTO_TEST_CASE(materialise_should_handle_empty_selections) +{ + BOOST_TEST(MosaicSelection({1}, 0.0).materialise(8).empty()); + BOOST_TEST(MosaicSelection({1, 3}, 0.0).materialise(8).empty()); + BOOST_TEST(MosaicSelection({5, 4, 3, 2}, 0.0).materialise(8).empty()); +} + +BOOST_AUTO_TEST_CASE(materialise_should_clamp_indices_at_collection_size) +{ + BOOST_TEST(MosaicSelection({4, 3, 2, 1, 0}, 1.0).materialise(4) == vector({3, 3, 2, 1})); + BOOST_TEST(MosaicSelection({4, 3, 2, 1, 0}, 2.0).materialise(3) == vector({2, 2, 2, 1, 0, 2})); + BOOST_TEST(MosaicSelection({4, 3, 2, 1, 0}, 1.0).materialise(1) == vector({0})); + BOOST_TEST(MosaicSelection({4, 3, 2, 1, 0}, 7.0).materialise(1) == vector({0, 0, 0, 0, 0, 0, 0})); +} + +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE(RandomSelectionTest) + +BOOST_AUTO_TEST_CASE(materialise_should_return_random_values_with_equal_probabilities) +{ + constexpr int collectionSize = 10; + constexpr int selectionSize = 100; + constexpr double relativeTolerance = 0.1; + constexpr double expectedValue = (collectionSize - 1) / 2.0; + constexpr double variance = (collectionSize * collectionSize - 1) / 12.0; + + SimulationRNG::reset(1); + vector samples = RandomSelection(selectionSize).materialise(collectionSize); + + BOOST_TEST(abs(mean(samples) - expectedValue) < expectedValue * relativeTolerance); + BOOST_TEST(abs(meanSquaredError(samples, expectedValue) - variance) < variance * relativeTolerance); +} + +BOOST_AUTO_TEST_CASE(materialise_should_return_only_values_that_can_be_used_as_collection_indices) +{ + const size_t collectionSize = 200; + + vector indices = RandomSelection(0.5).materialise(collectionSize); + + BOOST_TEST(indices.size() == 100); + BOOST_TEST(all_of(indices.begin(), indices.end(), [&](auto const& index){ return index <= collectionSize; })); +} + +BOOST_AUTO_TEST_CASE(materialise_should_return_number_of_indices_thats_a_fraction_of_collection_size) +{ + BOOST_TEST(RandomSelection(0.0).materialise(10).size() == 0); + BOOST_TEST(RandomSelection(0.3).materialise(10).size() == 3); + BOOST_TEST(RandomSelection(0.5).materialise(10).size() == 5); + BOOST_TEST(RandomSelection(0.7).materialise(10).size() == 7); + BOOST_TEST(RandomSelection(1.0).materialise(10).size() == 10); +} + +BOOST_AUTO_TEST_CASE(materialise_should_support_number_of_indices_bigger_than_collection_size) +{ + BOOST_TEST(RandomSelection(2.0).materialise(5).size() == 10); + BOOST_TEST(RandomSelection(1.5).materialise(10).size() == 15); + BOOST_TEST(RandomSelection(10.0).materialise(10).size() == 100); +} + +BOOST_AUTO_TEST_CASE(materialise_should_round_the_number_of_indices_to_the_nearest_integer) +{ + BOOST_TEST(RandomSelection(0.49).materialise(3).size() == 1); + BOOST_TEST(RandomSelection(0.50).materialise(3).size() == 2); + BOOST_TEST(RandomSelection(0.51).materialise(3).size() == 2); + + BOOST_TEST(RandomSelection(1.51).materialise(3).size() == 5); + + BOOST_TEST(RandomSelection(0.01).materialise(2).size() == 0); + BOOST_TEST(RandomSelection(0.01).materialise(3).size() == 0); +} + +BOOST_AUTO_TEST_CASE(materialise_should_return_no_indices_if_collection_is_empty) +{ + BOOST_TEST(RandomSelection(0.0).materialise(0).empty()); + BOOST_TEST(RandomSelection(0.5).materialise(0).empty()); + BOOST_TEST(RandomSelection(1.0).materialise(0).empty()); + BOOST_TEST(RandomSelection(2.0).materialise(0).empty()); +} + +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() + +} diff --git a/tools/yulPhaser/Selections.cpp b/tools/yulPhaser/Selections.cpp index 19451f45c8ea..abc080fdee6a 100644 --- a/tools/yulPhaser/Selections.cpp +++ b/tools/yulPhaser/Selections.cpp @@ -16,3 +16,45 @@ */ #include + +#include + +#include + +using namespace std; +using namespace solidity::phaser; + +vector RangeSelection::materialise(size_t _poolSize) const +{ + size_t beginIndex = static_cast(round(_poolSize * m_startPercent)); + size_t endIndex = static_cast(round(_poolSize * m_endPercent)); + vector selection; + + for (size_t i = beginIndex; i < endIndex; ++i) + selection.push_back(i); + + return selection; +} + +vector MosaicSelection::materialise(size_t _poolSize) const +{ + size_t count = static_cast(round(_poolSize * m_selectionSize)); + + vector selection; + for (size_t i = 0; i < count; ++i) + selection.push_back(min(m_pattern[i % m_pattern.size()], _poolSize - 1)); + + return selection; +} + +vector RandomSelection::materialise(size_t _poolSize) const +{ + size_t count = static_cast(round(_poolSize * m_selectionSize)); + + vector selection; + for (size_t i = 0; i < count; ++i) + selection.push_back(SimulationRNG::uniformInt(0, _poolSize - 1)); + + return selection; +} + diff --git a/tools/yulPhaser/Selections.h b/tools/yulPhaser/Selections.h index 1155c6708ed2..46d975bbd5e8 100644 --- a/tools/yulPhaser/Selections.h +++ b/tools/yulPhaser/Selections.h @@ -21,6 +21,7 @@ #pragma once +#include #include namespace solidity::phaser @@ -49,4 +50,72 @@ class Selection virtual std::vector materialise(size_t _poolSize) const = 0; }; +/** + * A selection that selects a contiguous slice of the container. Start and end of this part are + * specified as percentages of its size. + */ +class RangeSelection: public Selection +{ +public: + explicit RangeSelection(double _startPercent = 0.0, double _endPercent = 1.0): + m_startPercent(_startPercent), + m_endPercent(_endPercent) + { + assert(0 <= m_startPercent && m_startPercent <= m_endPercent && m_endPercent <= 1.0); + } + + std::vector materialise(size_t _poolSize) const override; + +private: + double m_startPercent; + double m_endPercent; +}; + +/** + * A selection that selects elements at specific, fixed positions indicated by a repeating "pattern". + * If the positions in the pattern exceed the size of the container, they are capped at the maximum + * available position. Always selects as many elements as the size of the container multiplied by + * @a _selectionSize (unless the container is empty). + * + * E.g. if the pattern is {0, 9} and collection size is 5, the selection will materialise into + * {0, 4, 0, 4, 0}. If the size is 3, it will be {0, 2, 0}. + */ +class MosaicSelection: public Selection +{ +public: + explicit MosaicSelection(std::vector _pattern, double _selectionSize = 1.0): + m_pattern(move(_pattern)), + m_selectionSize(_selectionSize) + { + assert(m_pattern.size() > 0 || _selectionSize == 0.0); + } + + std::vector materialise(size_t _poolSize) const override; + +private: + std::vector m_pattern; + double m_selectionSize; +}; + +/** + * A selection that randomly selects elements from a container. The resulting set of indices may + * contain duplicates and is different on each call to @a materialise(). Always selects as many + * elements as the size of the container multiplied by @a _selectionSize (unless the container is + * empty). + */ +class RandomSelection: public Selection +{ +public: + explicit RandomSelection(double _selectionSize): + m_selectionSize(_selectionSize) + { + assert(_selectionSize >= 0); + } + + std::vector materialise(size_t _poolSize) const override; + +private: + double m_selectionSize; +}; + } From 4665b7a7e4d5fe5d4135a5ea67b06a7f1c7e5814 Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 16:55:50 +0100 Subject: [PATCH 52/92] [yul-phaser] Population: Add select() method --- test/yulPhaser/Population.cpp | 36 ++++++++++++++++++++++++++++++++++ tools/yulPhaser/Population.cpp | 10 ++++++++++ tools/yulPhaser/Population.h | 3 +++ 3 files changed, 49 insertions(+) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index 55c346575850..dfee5f74ab69 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -222,6 +223,41 @@ BOOST_FIXTURE_TEST_CASE(plus_operator_should_add_two_populations, PopulationFixt ); } +BOOST_FIXTURE_TEST_CASE(select_should_return_population_containing_individuals_indicated_by_selection, PopulationFixture) +{ + Population population(m_fitnessMetric, {Chromosome("a"), Chromosome("c"), Chromosome("g"), Chromosome("h")}); + RangeSelection selection(0.25, 0.75); + assert(selection.materialise(population.individuals().size()) == (vector{1, 2})); + + BOOST_TEST( + population.select(selection) == + Population(m_fitnessMetric, {population.individuals()[1].chromosome, population.individuals()[2].chromosome}) + ); +} + +BOOST_FIXTURE_TEST_CASE(select_should_include_duplicates_if_selection_contains_duplicates, PopulationFixture) +{ + Population population(m_fitnessMetric, {Chromosome("a"), Chromosome("c")}); + MosaicSelection selection({0, 1}, 2.0); + assert(selection.materialise(population.individuals().size()) == (vector{0, 1, 0, 1})); + + BOOST_TEST(population.select(selection) == Population(m_fitnessMetric, { + population.individuals()[0].chromosome, + population.individuals()[1].chromosome, + population.individuals()[0].chromosome, + population.individuals()[1].chromosome, + })); +} + +BOOST_FIXTURE_TEST_CASE(select_should_return_empty_population_if_selection_is_empty, PopulationFixture) +{ + Population population(m_fitnessMetric, {Chromosome("a"), Chromosome("c")}); + RangeSelection selection(0.0, 0.0); + assert(selection.materialise(population.individuals().size()).empty()); + + BOOST_TEST(population.select(selection).individuals().empty()); +} + BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index 2fed8b90d500..12acbe417655 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -17,6 +17,7 @@ #include +#include #include #include @@ -95,6 +96,15 @@ void Population::run(optional _numRounds, ostream& _outputStream) } } +Population Population::select(Selection const& _selection) const +{ + vector selectedIndividuals; + for (size_t i: _selection.materialise(m_individuals.size())) + selectedIndividuals.emplace_back(m_individuals[i]); + + return Population(m_fitnessMetric, selectedIndividuals); +} + Population operator+(Population _a, Population _b) { // This operator is meant to be used only with populations sharing the same metric (and, to make diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index bd15674d693b..f66efc9a0d19 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -39,6 +39,8 @@ solidity::phaser::Population operator+(solidity::phaser::Population _a, solidity namespace solidity::phaser { +class Selection; + /** * Information describing the state of an individual member of the population during the course * of the genetic algorithm. @@ -102,6 +104,7 @@ class Population ); void run(std::optional _numRounds, std::ostream& _outputStream); + Population select(Selection const& _selection) const; friend Population (::operator+)(Population _a, Population _b); std::shared_ptr fitnessMetric() const { return m_fitnessMetric; } From 67fbafab8fddb42e0d84271bcac5ecce95835c9d Mon Sep 17 00:00:00 2001 From: cameel Date: Wed, 5 Feb 2020 14:58:48 +0100 Subject: [PATCH 53/92] [yul-phaser] Add RandomAlgorithm --- test/yulPhaser/GeneticAlgorithms.cpp | 43 +++++++++++++++++++++++ tools/yulPhaser/GeneticAlgorithms.cpp | 18 ++++++++++ tools/yulPhaser/GeneticAlgorithms.h | 49 +++++++++++++++++++++++++++ 3 files changed, 110 insertions(+) diff --git a/test/yulPhaser/GeneticAlgorithms.cpp b/test/yulPhaser/GeneticAlgorithms.cpp index 3528f2be08fa..aaa0a0b05d34 100644 --- a/test/yulPhaser/GeneticAlgorithms.cpp +++ b/test/yulPhaser/GeneticAlgorithms.cpp @@ -89,6 +89,49 @@ BOOST_FIXTURE_TEST_CASE(run_should_print_the_top_chromosome, GeneticAlgorithmFix BOOST_TEST(countSubstringOccurrences(m_output.str(), toString(algorithm.population().individuals()[0].chromosome)) == 4); } +BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE(RandomAlgorithmTest) + +BOOST_FIXTURE_TEST_CASE(runNextRound_should_preserve_elite_and_randomise_rest_of_population, GeneticAlgorithmFixture) +{ + auto population = Population::makeRandom(m_fitnessMetric, 4, 3, 3) + Population::makeRandom(m_fitnessMetric, 4, 5, 5); + RandomAlgorithm algorithm(population, m_output, {0.5, 1, 1}); + assert((chromosomeLengths(algorithm.population()) == vector{3, 3, 3, 3, 5, 5, 5, 5})); + + algorithm.runNextRound(); + BOOST_TEST((chromosomeLengths(algorithm.population()) == vector{1, 1, 1, 1, 3, 3, 3, 3})); +} + +BOOST_FIXTURE_TEST_CASE(runNextRound_should_not_replace_elite_with_worse_individuals, GeneticAlgorithmFixture) +{ + auto population = Population::makeRandom(m_fitnessMetric, 4, 3, 3) + Population::makeRandom(m_fitnessMetric, 4, 5, 5); + RandomAlgorithm algorithm(population, m_output, {0.5, 7, 7}); + assert((chromosomeLengths(algorithm.population()) == vector{3, 3, 3, 3, 5, 5, 5, 5})); + + algorithm.runNextRound(); + BOOST_TEST((chromosomeLengths(algorithm.population()) == vector{3, 3, 3, 3, 7, 7, 7, 7})); +} + +BOOST_FIXTURE_TEST_CASE(runNextRound_should_replace_all_chromosomes_if_zero_size_elite, GeneticAlgorithmFixture) +{ + auto population = Population::makeRandom(m_fitnessMetric, 4, 3, 3) + Population::makeRandom(m_fitnessMetric, 4, 5, 5); + RandomAlgorithm algorithm(population, m_output, {0.0, 1, 1}); + assert((chromosomeLengths(algorithm.population()) == vector{3, 3, 3, 3, 5, 5, 5, 5})); + + algorithm.runNextRound(); + BOOST_TEST((chromosomeLengths(algorithm.population()) == vector{1, 1, 1, 1, 1, 1, 1, 1})); +} + +BOOST_FIXTURE_TEST_CASE(runNextRound_should_not_replace_any_chromosomes_if_whole_population_is_the_elite, GeneticAlgorithmFixture) +{ + auto population = Population::makeRandom(m_fitnessMetric, 4, 3, 3) + Population::makeRandom(m_fitnessMetric, 4, 5, 5); + RandomAlgorithm algorithm(population, m_output, {1.0, 1, 1}); + assert((chromosomeLengths(algorithm.population()) == vector{3, 3, 3, 3, 5, 5, 5, 5})); + + algorithm.runNextRound(); + BOOST_TEST((chromosomeLengths(algorithm.population()) == vector{3, 3, 3, 3, 5, 5, 5, 5})); +} + BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END() diff --git a/tools/yulPhaser/GeneticAlgorithms.cpp b/tools/yulPhaser/GeneticAlgorithms.cpp index 6e69565f4ee6..756a410f7663 100644 --- a/tools/yulPhaser/GeneticAlgorithms.cpp +++ b/tools/yulPhaser/GeneticAlgorithms.cpp @@ -16,6 +16,7 @@ */ #include +#include using namespace std; using namespace solidity::phaser; @@ -30,3 +31,20 @@ void GeneticAlgorithm::run(optional _numRounds) m_outputStream << m_population; } } + +void RandomAlgorithm::runNextRound() +{ + RangeSelection elite(0.0, m_options.elitePoolSize); + + Population elitePopulation = m_population.select(elite); + size_t replacementCount = m_population.individuals().size() - elitePopulation.individuals().size(); + + m_population = + move(elitePopulation) + + Population::makeRandom( + m_population.fitnessMetric(), + replacementCount, + m_options.minChromosomeLength, + m_options.maxChromosomeLength + ); +} diff --git a/tools/yulPhaser/GeneticAlgorithms.h b/tools/yulPhaser/GeneticAlgorithms.h index c2700416b6e2..a7600d8942c0 100644 --- a/tools/yulPhaser/GeneticAlgorithms.h +++ b/tools/yulPhaser/GeneticAlgorithms.h @@ -63,4 +63,53 @@ class GeneticAlgorithm std::ostream& m_outputStream; }; +/** + * Completely random genetic algorithm, + * + * The algorithm simply replaces the worst chromosomes with entirely new ones, generated + * randomly and not based on any member of the current population. Only a constant proportion of the + * chromosomes (the elite) is preserved in each round. + * + * Preserves the size of the population. You can use @a elitePoolSize to make the algorithm + * generational (replacing most members in each round) or steady state (replacing only one member). + * Both versions are equivalent in terms of the outcome but the generational one converges in a + * smaller number of rounds while the steady state one does less work per round. This may matter + * in case of metrics that take a long time to compute though in case of this particular + * algorithm the same result could also be achieved by simply making the population smaller. + */ +class RandomAlgorithm: public GeneticAlgorithm +{ +public: + struct Options + { + double elitePoolSize; ///< Percentage of the population treated as the elite + size_t minChromosomeLength; ///< Minimum length of newly generated chromosomes + size_t maxChromosomeLength; ///< Maximum length of newly generated chromosomes + + bool isValid() const + { + return ( + 0 <= elitePoolSize && elitePoolSize <= 1.0 && + minChromosomeLength <= maxChromosomeLength + ); + } + }; + + explicit RandomAlgorithm( + Population _initialPopulation, + std::ostream& _outputStream, + Options const& _options + ): + GeneticAlgorithm(_initialPopulation, _outputStream), + m_options(_options) + { + assert(_options.isValid()); + } + + void runNextRound() override; + +private: + Options m_options; +}; + } From e1b8b64f05a5b6efd860e59aabd6ac5ded6aaa9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 6 Feb 2020 07:59:08 +0100 Subject: [PATCH 54/92] [yul-phaser] Population: Remove no longer used methods for running algorithm steps - They have been superseded by objects from GeneticAlgorithms.h --- test/yulPhaser/Population.cpp | 32 -------------------------- tools/yulPhaser/Population.cpp | 41 ---------------------------------- tools/yulPhaser/Population.h | 19 +++++----------- 3 files changed, 5 insertions(+), 87 deletions(-) diff --git a/test/yulPhaser/Population.cpp b/test/yulPhaser/Population.cpp index dfee5f74ab69..98532c9dceee 100644 --- a/test/yulPhaser/Population.cpp +++ b/test/yulPhaser/Population.cpp @@ -182,38 +182,6 @@ BOOST_FIXTURE_TEST_CASE(makeRandom_should_compute_fitness, PopulationFixture) BOOST_TEST(population.individuals()[2].fitness == m_fitnessMetric->evaluate(population.individuals()[2].chromosome)); } -BOOST_FIXTURE_TEST_CASE(run_should_not_make_fitness_of_top_chromosomes_worse, PopulationFixture) -{ - stringstream output; - vector chromosomes = { - Chromosome(vector{StructuralSimplifier::name}), - Chromosome(vector{BlockFlattener::name}), - Chromosome(vector{SSAReverser::name}), - Chromosome(vector{UnusedPruner::name}), - Chromosome(vector{StructuralSimplifier::name, BlockFlattener::name}), - }; - Population population(m_fitnessMetric, chromosomes); - - size_t initialTopFitness[2] = { - m_fitnessMetric->evaluate(chromosomes[0]), - m_fitnessMetric->evaluate(chromosomes[1]), - }; - - for (int i = 0; i < 6; ++i) - { - population.run(1, output); - BOOST_TEST(population.individuals().size() == 5); - - size_t currentTopFitness[2] = { - population.individuals()[0].fitness, - population.individuals()[1].fitness, - }; - BOOST_TEST(currentTopFitness[0] <= initialTopFitness[0]); - BOOST_TEST(currentTopFitness[1] <= initialTopFitness[1]); - BOOST_TEST(currentTopFitness[0] <= currentTopFitness[1]); - } -} - BOOST_FIXTURE_TEST_CASE(plus_operator_should_add_two_populations, PopulationFixture) { BOOST_CHECK_EQUAL( diff --git a/tools/yulPhaser/Population.cpp b/tools/yulPhaser/Population.cpp index 12acbe417655..b39f5dad5d9a 100644 --- a/tools/yulPhaser/Population.cpp +++ b/tools/yulPhaser/Population.cpp @@ -84,18 +84,6 @@ Population Population::makeRandom( ); } -void Population::run(optional _numRounds, ostream& _outputStream) -{ - for (size_t round = 0; !_numRounds.has_value() || round < _numRounds.value(); ++round) - { - doMutation(); - doSelection(); - - _outputStream << "---------- ROUND " << round << " ----------" << endl; - _outputStream << *this; - } -} - Population Population::select(Selection const& _selection) const { vector selectedIndividuals; @@ -131,35 +119,6 @@ ostream& phaser::operator<<(ostream& _stream, Population const& _population) return _stream; } -void Population::doMutation() -{ - // TODO: Implement mutation and crossover -} - -void Population::doSelection() -{ - randomizeWorstChromosomes(*m_fitnessMetric, m_individuals, m_individuals.size() / 2); - m_individuals = sortedIndividuals(move(m_individuals)); -} - -void Population::randomizeWorstChromosomes( - FitnessMetric const& _fitnessMetric, - vector& _individuals, - size_t _count -) -{ - assert(_individuals.size() >= _count); - // ASSUMPTION: _individuals is sorted in ascending order - - auto individual = _individuals.begin() + (_individuals.size() - _count); - for (; individual != _individuals.end(); ++individual) - { - auto chromosome = Chromosome::makeRandom(binomialChromosomeLength(MaxChromosomeLength)); - size_t fitness = _fitnessMetric.evaluate(chromosome); - *individual = {move(chromosome), fitness}; - } -} - vector Population::chromosomesToIndividuals( FitnessMetric const& _fitnessMetric, vector _chromosomes diff --git a/tools/yulPhaser/Population.h b/tools/yulPhaser/Population.h index f66efc9a0d19..f4f42346de22 100644 --- a/tools/yulPhaser/Population.h +++ b/tools/yulPhaser/Population.h @@ -69,19 +69,19 @@ struct Individual bool isFitter(Individual const& a, Individual const& b); /** - * Represents a changing set of individuals undergoing a genetic algorithm. - * Each round of the algorithm involves mutating existing individuals, evaluating their fitness - * and selecting the best ones for the next round. + * Represents a snapshot of a population undergoing a genetic algorithm. Consists of a set of + * chromosomes with associated fitness values. * * An individual is a sequence of optimiser steps represented by a @a Chromosome instance. * Individuals are always ordered by their fitness (based on @_fitnessMetric and @a isFitter()). * The fitness is computed using the metric as soon as an individual is inserted into the population. + * + * The population is immutable. Selections, mutations and crossover work by producing a new + * instance and copying the individuals. */ class Population { public: - static constexpr size_t MaxChromosomeLength = 30; - explicit Population( std::shared_ptr _fitnessMetric, std::vector _chromosomes = {} @@ -103,7 +103,6 @@ class Population size_t _maxChromosomeLength ); - void run(std::optional _numRounds, std::ostream& _outputStream); Population select(Selection const& _selection) const; friend Population (::operator+)(Population _a, Population _b); @@ -123,14 +122,6 @@ class Population m_fitnessMetric(std::move(_fitnessMetric)), m_individuals{sortedIndividuals(std::move(_individuals))} {} - void doMutation(); - void doSelection(); - - static void randomizeWorstChromosomes( - FitnessMetric const& _fitnessMetric, - std::vector& _individuals, - size_t _count - ); static std::vector chromosomesToIndividuals( FitnessMetric const& _fitnessMetric, std::vector _chromosomes From 4aac7d167387e65ca53b1ce9bac9809b30d6eaf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Thu, 13 Feb 2020 20:35:50 +0100 Subject: [PATCH 55/92] [yul-phaser] main: Switch from using Population::run() to RandomAlgorithm --- tools/yulPhaser/main.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp index eea43e5fb76c..d3ea9d71321c 100644 --- a/tools/yulPhaser/main.cpp +++ b/tools/yulPhaser/main.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -29,7 +30,6 @@ #include #include -#include #include using namespace std; @@ -71,14 +71,26 @@ CharStream loadSource(string const& _sourcePath) void runAlgorithm(string const& _sourcePath) { + constexpr size_t minChromosomeLength = 12; + constexpr size_t maxChromosomeLength = 30; + CharStream sourceCode = loadSource(_sourcePath); shared_ptr fitnessMetric = make_shared(Program::load(sourceCode), 5); auto population = Population::makeRandom( fitnessMetric, 10, - bind(Population::binomialChromosomeLength, Population::MaxChromosomeLength) + minChromosomeLength, + maxChromosomeLength ); - population.run(nullopt, cout); + RandomAlgorithm( + population, + cout, + { + /* elitePoolSize = */ 0.5, + /* minChromosomeLength = */ minChromosomeLength, + /* maxChromosomeLength = */ maxChromosomeLength, + } + ).run(); } CommandLineParsingResult parseCommandLine(int argc, char** argv) From 3aaca31c31e80151f4b3208194a76a329f38e79b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 5 Feb 2020 19:25:15 +0100 Subject: [PATCH 56/92] [yul-phaser] main: Change the number of chromosomes preserved in every round of the algorithm to 1 --- tools/yulPhaser/main.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp index d3ea9d71321c..26a2ea00527e 100644 --- a/tools/yulPhaser/main.cpp +++ b/tools/yulPhaser/main.cpp @@ -71,6 +71,7 @@ CharStream loadSource(string const& _sourcePath) void runAlgorithm(string const& _sourcePath) { + constexpr size_t populationSize = 10; constexpr size_t minChromosomeLength = 12; constexpr size_t maxChromosomeLength = 30; @@ -78,7 +79,7 @@ void runAlgorithm(string const& _sourcePath) shared_ptr fitnessMetric = make_shared(Program::load(sourceCode), 5); auto population = Population::makeRandom( fitnessMetric, - 10, + populationSize, minChromosomeLength, maxChromosomeLength ); @@ -86,7 +87,7 @@ void runAlgorithm(string const& _sourcePath) population, cout, { - /* elitePoolSize = */ 0.5, + /* elitePoolSize = */ 1.0 / populationSize, /* minChromosomeLength = */ minChromosomeLength, /* maxChromosomeLength = */ maxChromosomeLength, } From 66d733fbacd66ce91c448873d6fe0ab831fb5f0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 5 Feb 2020 19:25:15 +0100 Subject: [PATCH 57/92] [yul-phaser] main: Increase the number of chromosomes in the initial population to 20 --- tools/yulPhaser/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/yulPhaser/main.cpp b/tools/yulPhaser/main.cpp index 26a2ea00527e..321ebe0881e9 100644 --- a/tools/yulPhaser/main.cpp +++ b/tools/yulPhaser/main.cpp @@ -71,7 +71,7 @@ CharStream loadSource(string const& _sourcePath) void runAlgorithm(string const& _sourcePath) { - constexpr size_t populationSize = 10; + constexpr size_t populationSize = 20; constexpr size_t minChromosomeLength = 12; constexpr size_t maxChromosomeLength = 30; From 40cae442d5ca32e6803e101839448b85b65d2d46 Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 25 Feb 2020 17:55:14 +0100 Subject: [PATCH 58/92] Fix re-printing value expectations. --- test/libsolidity/util/SoltestTypes.h | 13 +------------ test/libsolidity/util/TestFileParserTests.cpp | 3 ++- test/libsolidity/util/TestFunctionCall.cpp | 10 +++++++++- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/test/libsolidity/util/SoltestTypes.h b/test/libsolidity/util/SoltestTypes.h index 4c1cb01fd112..26b92d5356cb 100644 --- a/test/libsolidity/util/SoltestTypes.h +++ b/test/libsolidity/util/SoltestTypes.h @@ -238,24 +238,13 @@ enum class FunctionValueUnit }; /// Holds value along with unit it was expressed in originally. -/// Value should be always converted to wei, no meter on which unit it was originally +/// @a value is always in wei - it is converted back when stringifying again. struct FunctionValue { u256 value; FunctionValueUnit unit = FunctionValueUnit::Wei; }; -inline bool operator==(FunctionValue const& _a, FunctionValue const& _b) -{ - return _a.value == _b.value; -} - -inline std::ostream& operator<<(std::ostream& _os, FunctionValue const& _v) -{ - _os << _v.value << (_v.unit == FunctionValueUnit::Wei ? " wei" : " ether"); - return _os; -} - /** * Represents a function call read from an input stream. It contains the signature, the * arguments, an optional ether value and an expected execution result. diff --git a/test/libsolidity/util/TestFileParserTests.cpp b/test/libsolidity/util/TestFileParserTests.cpp index 82d885b9e5c6..f074fc1bbb93 100644 --- a/test/libsolidity/util/TestFileParserTests.cpp +++ b/test/libsolidity/util/TestFileParserTests.cpp @@ -64,7 +64,8 @@ void testFunctionCall( ABI_CHECK(_call.arguments.rawBytes(), _arguments); ABI_CHECK(_call.expectations.rawBytes(), _expectations); BOOST_REQUIRE_EQUAL(_call.displayMode, _mode); - BOOST_REQUIRE_EQUAL(_call.value, _value); + BOOST_REQUIRE_EQUAL(_call.value.value, _value.value); + BOOST_REQUIRE_EQUAL(size_t(_call.value.unit), size_t(_value.unit)); BOOST_REQUIRE_EQUAL(_call.arguments.comment, _argumentComment); BOOST_REQUIRE_EQUAL(_call.expectations.comment, _expectationComment); diff --git a/test/libsolidity/util/TestFunctionCall.cpp b/test/libsolidity/util/TestFunctionCall.cpp index 43f19034986b..f961c615c5e5 100644 --- a/test/libsolidity/util/TestFunctionCall.cpp +++ b/test/libsolidity/util/TestFunctionCall.cpp @@ -52,6 +52,7 @@ string TestFunctionCall::format( string comma = formatToken(Token::Comma); string comment = formatToken(Token::Comment); string ether = formatToken(Token::Ether); + string wei = formatToken(Token::Wei); string newline = formatToken(Token::Newline); string failure = formatToken(Token::Failure); @@ -64,7 +65,14 @@ string TestFunctionCall::format( /// Formats the function signature. This is the same independent from the display-mode. stream << _linePrefix << newline << ws << m_call.signature; if (m_call.value.value > u256(0)) - stream << comma << ws << m_call.value.value << ws << ether; + { + if (m_call.value.unit == FunctionValueUnit::Ether) + stream << comma << ws << (m_call.value.value / exp256(10, 18)) << ws << ether; + else if (m_call.value.unit == FunctionValueUnit::Wei) + stream << comma << ws << m_call.value.value << ws << wei; + else + soltestAssert(false, ""); + } if (!m_call.arguments.rawBytes().empty()) { string output = formatRawParameters(m_call.arguments.parameters, _linePrefix); From ec083c487837ebc409cda9a7531b843fe8a437fa Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Mon, 24 Feb 2020 18:11:20 +0100 Subject: [PATCH 59/92] Adding source location support to AssemblyStack and thus debugging Yul sources --- Changelog.md | 1 + libevmasm/AssemblyItem.cpp | 91 ++++++++++++++++++ libevmasm/AssemblyItem.h | 9 +- liblangutil/Scanner.h | 1 + libsolidity/interface/CompilerStack.cpp | 95 +------------------ libsolidity/interface/CompilerStack.h | 3 - libsolidity/interface/StandardCompiler.cpp | 2 +- libyul/AssemblyStack.cpp | 6 ++ libyul/AssemblyStack.h | 3 + test/cmdlineTests/standard_yul/output.json | 2 +- .../standard_yul_object/output.json | 2 +- .../standard_yul_object_name/output.json | 2 +- .../standard_yul_optimized/output.json | 2 +- test/libyul/ObjectCompilerTest.cpp | 3 + test/libyul/objectCompiler/data.yul | 1 + test/libyul/objectCompiler/datacopy.yul | 1 + .../libyul/objectCompiler/dataoffset_code.yul | 1 + .../libyul/objectCompiler/dataoffset_data.yul | 1 + .../libyul/objectCompiler/dataoffset_self.yul | 1 + test/libyul/objectCompiler/datasize_code.yul | 1 + test/libyul/objectCompiler/datasize_data.yul | 1 + test/libyul/objectCompiler/datasize_self.yul | 1 + .../libyul/objectCompiler/function_series.yul | 1 + .../libyul/objectCompiler/namedObjectCode.yul | 1 + .../objectCompiler/nested_optimizer.yul | 1 + test/libyul/objectCompiler/simple.yul | 1 + .../objectCompiler/simple_optimizer.yul | 1 + test/libyul/objectCompiler/subObject.yul | 1 + test/libyul/objectCompiler/subSubObject.yul | 1 + 29 files changed, 136 insertions(+), 101 deletions(-) diff --git a/Changelog.md b/Changelog.md index 9981ea25fd99..d1f0b76fa4f0 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,6 +4,7 @@ Language Features: Compiler Features: + * AssemblyStack: Support for source locations (source mappings) and thus debugging Yul sources. Bugfixes: diff --git a/libevmasm/AssemblyItem.cpp b/libevmasm/AssemblyItem.cpp index b9b6c4acd942..c238a3a3c741 100644 --- a/libevmasm/AssemblyItem.cpp +++ b/libevmasm/AssemblyItem.cpp @@ -19,12 +19,14 @@ #include #include +#include #include using namespace std; using namespace solidity; using namespace solidity::evmasm; +using namespace solidity::langutil; static_assert(sizeof(size_t) <= 8, "size_t must be at most 64-bits wide"); @@ -281,3 +283,92 @@ ostream& solidity::evmasm::operator<<(ostream& _out, AssemblyItem const& _item) } return _out; } + +std::string AssemblyItem::computeSourceMapping( + AssemblyItems const& _items, + map const& _sourceIndicesMap +) +{ + string ret; + + int prevStart = -1; + int prevLength = -1; + int prevSourceIndex = -1; + size_t prevModifierDepth = -1; + char prevJump = 0; + for (auto const& item: _items) + { + if (!ret.empty()) + ret += ";"; + + SourceLocation const& location = item.location(); + int length = location.start != -1 && location.end != -1 ? location.end - location.start : -1; + int sourceIndex = + location.source && _sourceIndicesMap.count(location.source->name()) ? + _sourceIndicesMap.at(location.source->name()) : + -1; + char jump = '-'; + if (item.getJumpType() == evmasm::AssemblyItem::JumpType::IntoFunction) + jump = 'i'; + else if (item.getJumpType() == evmasm::AssemblyItem::JumpType::OutOfFunction) + jump = 'o'; + size_t modifierDepth = item.m_modifierDepth; + + unsigned components = 5; + if (modifierDepth == prevModifierDepth) + { + components--; + if (jump == prevJump) + { + components--; + if (sourceIndex == prevSourceIndex) + { + components--; + if (length == prevLength) + { + components--; + if (location.start == prevStart) + components--; + } + } + } + } + + if (components-- > 0) + { + if (location.start != prevStart) + ret += to_string(location.start); + if (components-- > 0) + { + ret += ':'; + if (length != prevLength) + ret += to_string(length); + if (components-- > 0) + { + ret += ':'; + if (sourceIndex != prevSourceIndex) + ret += to_string(sourceIndex); + if (components-- > 0) + { + ret += ':'; + if (jump != prevJump) + ret += jump; + if (components-- > 0) + { + ret += ':'; + if (modifierDepth != prevModifierDepth) + ret += to_string(modifierDepth); + } + } + } + } + } + + prevStart = location.start; + prevLength = length; + prevSourceIndex = sourceIndex; + prevJump = jump; + prevModifierDepth = modifierDepth; + } + return ret; +} diff --git a/libevmasm/AssemblyItem.h b/libevmasm/AssemblyItem.h index 31f175103175..e506a9fdb3e0 100644 --- a/libevmasm/AssemblyItem.h +++ b/libevmasm/AssemblyItem.h @@ -48,6 +48,8 @@ enum AssemblyItemType { }; class Assembly; +class AssemblyItem; +using AssemblyItems = std::vector; class AssemblyItem { @@ -122,6 +124,11 @@ class AssemblyItem } bool operator!=(Instruction _instr) const { return !operator==(_instr); } + static std::string computeSourceMapping( + AssemblyItems const& _items, + std::map const& _sourceIndicesMap + ); + /// @returns an upper bound for the number of bytes required by this item, assuming that /// the value of a jump tag takes @a _addressLength bytes. unsigned bytesRequired(unsigned _addressLength) const; @@ -157,8 +164,6 @@ class AssemblyItem mutable std::shared_ptr m_pushedValue; }; -using AssemblyItems = std::vector; - inline size_t bytesRequired(AssemblyItems const& _items, size_t _addressLength) { size_t size = 0; diff --git a/liblangutil/Scanner.h b/liblangutil/Scanner.h index 8f6cdab875e1..5b365f134918 100644 --- a/liblangutil/Scanner.h +++ b/liblangutil/Scanner.h @@ -98,6 +98,7 @@ class Scanner std::string const& source() const noexcept { return m_source->source(); } std::shared_ptr charStream() noexcept { return m_source; } + std::shared_ptr charStream() const noexcept { return m_source; } /// Resets the scanner as if newly constructed with _source as input. void reset(CharStream _source); diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 89ab13818192..8509d512f028 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -565,7 +565,7 @@ string const* CompilerStack::sourceMapping(string const& _contractName) const if (!c.sourceMapping) { if (auto items = assemblyItems(_contractName)) - c.sourceMapping = make_unique(computeSourceMapping(*items)); + c.sourceMapping = make_unique(evmasm::AssemblyItem::computeSourceMapping(*items, sourceIndices())); } return c.sourceMapping.get(); } @@ -579,7 +579,9 @@ string const* CompilerStack::runtimeSourceMapping(string const& _contractName) c if (!c.runtimeSourceMapping) { if (auto items = runtimeAssemblyItems(_contractName)) - c.runtimeSourceMapping = make_unique(computeSourceMapping(*items)); + c.runtimeSourceMapping = make_unique( + evmasm::AssemblyItem::computeSourceMapping(*items, sourceIndices()) + ); } return c.runtimeSourceMapping.get(); } @@ -1390,95 +1392,6 @@ bytes CompilerStack::createCBORMetadata(string const& _metadata, bool _experimen return encoder.serialise(); } -string CompilerStack::computeSourceMapping(evmasm::AssemblyItems const& _items) const -{ - if (m_stackState != CompilationSuccessful) - BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Compilation was not successful.")); - - string ret; - map sourceIndicesMap = sourceIndices(); - int prevStart = -1; - int prevLength = -1; - int prevSourceIndex = -1; - size_t prevModifierDepth = -1; - char prevJump = 0; - for (auto const& item: _items) - { - if (!ret.empty()) - ret += ";"; - - SourceLocation const& location = item.location(); - int length = location.start != -1 && location.end != -1 ? location.end - location.start : -1; - int sourceIndex = - location.source && sourceIndicesMap.count(location.source->name()) ? - sourceIndicesMap.at(location.source->name()) : - -1; - char jump = '-'; - if (item.getJumpType() == evmasm::AssemblyItem::JumpType::IntoFunction) - jump = 'i'; - else if (item.getJumpType() == evmasm::AssemblyItem::JumpType::OutOfFunction) - jump = 'o'; - size_t modifierDepth = item.m_modifierDepth; - - unsigned components = 5; - if (modifierDepth == prevModifierDepth) - { - components--; - if (jump == prevJump) - { - components--; - if (sourceIndex == prevSourceIndex) - { - components--; - if (length == prevLength) - { - components--; - if (location.start == prevStart) - components--; - } - } - } - } - - if (components-- > 0) - { - if (location.start != prevStart) - ret += to_string(location.start); - if (components-- > 0) - { - ret += ':'; - if (length != prevLength) - ret += to_string(length); - if (components-- > 0) - { - ret += ':'; - if (sourceIndex != prevSourceIndex) - ret += to_string(sourceIndex); - if (components-- > 0) - { - ret += ':'; - if (jump != prevJump) - ret += jump; - if (components-- > 0) - { - ret += ':'; - if (modifierDepth != prevModifierDepth) - ret += to_string(modifierDepth); - } - } - } - } - } - - prevStart = location.start; - prevLength = length; - prevSourceIndex = sourceIndex; - prevJump = jump; - prevModifierDepth = modifierDepth; - } - return ret; -} - namespace { diff --git a/libsolidity/interface/CompilerStack.h b/libsolidity/interface/CompilerStack.h index 099375879497..b0ea0603f6b0 100644 --- a/libsolidity/interface/CompilerStack.h +++ b/libsolidity/interface/CompilerStack.h @@ -401,9 +401,6 @@ class CompilerStack: boost::noncopyable /// @returns the metadata CBOR for the given serialised metadata JSON. bytes createCBORMetadata(std::string const& _metadata, bool _experimentalMode); - /// @returns the computer source mapping string. - std::string computeSourceMapping(evmasm::AssemblyItems const& _items) const; - /// @returns the contract ABI as a JSON object. /// This will generate the JSON object and store it in the Contract object if it is not present yet. Json::Value const& contractABI(Contract const&) const; diff --git a/libsolidity/interface/StandardCompiler.cpp b/libsolidity/interface/StandardCompiler.cpp index 9f837a996a51..77e00df057a8 100644 --- a/libsolidity/interface/StandardCompiler.cpp +++ b/libsolidity/interface/StandardCompiler.cpp @@ -1081,7 +1081,7 @@ Json::Value StandardCompiler::compileYul(InputsAndSettings _inputsAndSettings) { "evm.bytecode", "evm.bytecode.object", "evm.bytecode.opcodes", "evm.bytecode.sourceMap", "evm.bytecode.linkReferences" }, wildcardMatchesExperimental )) - output["contracts"][sourceName][contractName]["evm"]["bytecode"] = collectEVMObject(*object.bytecode, nullptr); + output["contracts"][sourceName][contractName]["evm"]["bytecode"] = collectEVMObject(*object.bytecode, object.sourceMappings.get()); if (isArtifactRequested(_inputsAndSettings.outputSelection, sourceName, contractName, "irOptimized", wildcardMatchesExperimental)) output["contracts"][sourceName][contractName]["irOptimized"] = stack.print(); diff --git a/libyul/AssemblyStack.cpp b/libyul/AssemblyStack.cpp index 9f7cc57a5945..7c0eca27621c 100644 --- a/libyul/AssemblyStack.cpp +++ b/libyul/AssemblyStack.cpp @@ -204,6 +204,12 @@ MachineAssemblyObject AssemblyStack::assemble(Machine _machine) const compileEVM(adapter, false, m_optimiserSettings.optimizeStackAllocation); object.bytecode = make_shared(assembly.assemble()); object.assembly = assembly.assemblyString(); + object.sourceMappings = make_unique( + evmasm::AssemblyItem::computeSourceMapping( + assembly.items(), + {{scanner().charStream() ? scanner().charStream()->name() : "", 0}} + ) + ); return object; } case Machine::EVM15: diff --git a/libyul/AssemblyStack.h b/libyul/AssemblyStack.h index 1104d6b0504d..60efd96749fc 100644 --- a/libyul/AssemblyStack.h +++ b/libyul/AssemblyStack.h @@ -48,6 +48,7 @@ struct MachineAssemblyObject { std::shared_ptr bytecode; std::string assembly; + std::unique_ptr sourceMappings; }; /* @@ -114,6 +115,8 @@ class AssemblyStack std::shared_ptr m_parserResult; langutil::ErrorList m_errors; langutil::ErrorReporter m_errorReporter; + + std::unique_ptr m_sourceMappings; }; } diff --git a/test/cmdlineTests/standard_yul/output.json b/test/cmdlineTests/standard_yul/output.json index b2c711e7bf63..ac6c8f7cbf2c 100644 --- a/test/cmdlineTests/standard_yul/output.json +++ b/test/cmdlineTests/standard_yul/output.json @@ -14,7 +14,7 @@ sstore /* \"A\":0:42 */ pop -","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":""}},"ir":"object \"object\" { +","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":"sourceMap removed"}},"ir":"object \"object\" { code { let x := mload(0) sstore(add(x, 0), 0) diff --git a/test/cmdlineTests/standard_yul_object/output.json b/test/cmdlineTests/standard_yul_object/output.json index f7bf6c1b8e18..946e5773b68a 100644 --- a/test/cmdlineTests/standard_yul_object/output.json +++ b/test/cmdlineTests/standard_yul_object/output.json @@ -13,7 +13,7 @@ pop stop data_4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45 616263 -","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":""}},"ir":"object \"NamedObject\" { +","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":"sourceMap removed"}},"ir":"object \"NamedObject\" { code { let x := dataoffset(\"DataName\") sstore(add(x, 0), 0) diff --git a/test/cmdlineTests/standard_yul_object_name/output.json b/test/cmdlineTests/standard_yul_object_name/output.json index 92b4fab5bb66..2fdd9bc11d12 100644 --- a/test/cmdlineTests/standard_yul_object_name/output.json +++ b/test/cmdlineTests/standard_yul_object_name/output.json @@ -22,7 +22,7 @@ sub_0: assembly { /* \"A\":137:149 */ revert } -","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":""}},"ir":"object \"NamedObject\" { +","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":"sourceMap removed"}},"ir":"object \"NamedObject\" { code { let x := dataoffset(\"DataName\") sstore(add(x, 0), 0) diff --git a/test/cmdlineTests/standard_yul_optimized/output.json b/test/cmdlineTests/standard_yul_optimized/output.json index c5e1067f9384..532ff00d9190 100644 --- a/test/cmdlineTests/standard_yul_optimized/output.json +++ b/test/cmdlineTests/standard_yul_optimized/output.json @@ -5,7 +5,7 @@ mload /* \"A\":20:40 */ sstore -","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":""}},"ir":"object \"object\" { +","bytecode":{"linkReferences":{},"object":"bytecode removed","opcodes":"opcodes removed","sourceMap":"sourceMap removed"}},"ir":"object \"object\" { code { let x := mload(0) sstore(add(x, 0), 0) diff --git a/test/libyul/ObjectCompilerTest.cpp b/test/libyul/ObjectCompilerTest.cpp index 7472b124f6ed..f1b284042e9f 100644 --- a/test/libyul/ObjectCompilerTest.cpp +++ b/test/libyul/ObjectCompilerTest.cpp @@ -74,6 +74,7 @@ TestCase::TestResult ObjectCompilerTest::run(ostream& _stream, string const& _li MachineAssemblyObject obj = stack.assemble(AssemblyStack::Machine::EVM); solAssert(obj.bytecode, ""); + solAssert(obj.sourceMappings, ""); m_obtainedResult = "Assembly:\n" + obj.assembly; if (obj.bytecode->bytecode.empty()) @@ -84,6 +85,8 @@ TestCase::TestResult ObjectCompilerTest::run(ostream& _stream, string const& _li toHex(obj.bytecode->bytecode) + "\nOpcodes: " + boost::trim_copy(evmasm::disassemble(obj.bytecode->bytecode)) + + "\nSourceMappings:" + + (obj.sourceMappings->empty() ? "" : " " + *obj.sourceMappings) + "\n"; if (m_expectation != m_obtainedResult) diff --git a/test/libyul/objectCompiler/data.yul b/test/libyul/objectCompiler/data.yul index daa22d21c8bf..611715b6535d 100644 --- a/test/libyul/objectCompiler/data.yul +++ b/test/libyul/objectCompiler/data.yul @@ -9,3 +9,4 @@ object "a" { // data_acaf3289d7b601cbd114fb36c4d29c85bbfd5e133f14cb355c3fd8d99367964f 48656c6c6f2c20576f726c6421 // Bytecode: fe // Opcodes: INVALID +// SourceMappings: diff --git a/test/libyul/objectCompiler/datacopy.yul b/test/libyul/objectCompiler/datacopy.yul index 2259e5dc971a..1a725bab0fb0 100644 --- a/test/libyul/objectCompiler/datacopy.yul +++ b/test/libyul/objectCompiler/datacopy.yul @@ -46,3 +46,4 @@ object "a" { // } // Bytecode: 600b600d600039600b6000f3fe6000600055600d600052fe // Opcodes: PUSH1 0xB PUSH1 0xD PUSH1 0x0 CODECOPY PUSH1 0xB PUSH1 0x0 RETURN INVALID PUSH1 0x0 PUSH1 0x0 SSTORE PUSH1 0xD PUSH1 0x0 MSTORE INVALID +// SourceMappings: 26:47:0:-:0;;35:1;26:47;78:26;85:1;78:26 diff --git a/test/libyul/objectCompiler/dataoffset_code.yul b/test/libyul/objectCompiler/dataoffset_code.yul index 725267f27eda..ed6242090ad8 100644 --- a/test/libyul/objectCompiler/dataoffset_code.yul +++ b/test/libyul/objectCompiler/dataoffset_code.yul @@ -27,3 +27,4 @@ object "a" { // } // Bytecode: 6006600055fe6008600055fe // Opcodes: PUSH1 0x6 PUSH1 0x0 SSTORE INVALID PUSH1 0x8 PUSH1 0x0 SSTORE INVALID +// SourceMappings: 22:28:0:-:0;29:1;22:28 diff --git a/test/libyul/objectCompiler/dataoffset_data.yul b/test/libyul/objectCompiler/dataoffset_data.yul index 9a0a461dc92c..6848ce9b9d0c 100644 --- a/test/libyul/objectCompiler/dataoffset_data.yul +++ b/test/libyul/objectCompiler/dataoffset_data.yul @@ -14,3 +14,4 @@ object "a" { // data_acaf3289d7b601cbd114fb36c4d29c85bbfd5e133f14cb355c3fd8d99367964f 48656c6c6f2c20576f726c6421 // Bytecode: 6006600055fe48656c6c6f2c20576f726c6421 // Opcodes: PUSH1 0x6 PUSH1 0x0 SSTORE INVALID 0x48 PUSH6 0x6C6C6F2C2057 PUSH16 0x726C6421000000000000000000000000 +// SourceMappings: 22:30:0:-:0;29:1;22:30 diff --git a/test/libyul/objectCompiler/dataoffset_self.yul b/test/libyul/objectCompiler/dataoffset_self.yul index b774073530b4..6709bc98db64 100644 --- a/test/libyul/objectCompiler/dataoffset_self.yul +++ b/test/libyul/objectCompiler/dataoffset_self.yul @@ -14,3 +14,4 @@ object "a" { // data_acaf3289d7b601cbd114fb36c4d29c85bbfd5e133f14cb355c3fd8d99367964f 48656c6c6f2c20576f726c6421 // Bytecode: 6000600055fe // Opcodes: PUSH1 0x0 PUSH1 0x0 SSTORE INVALID +// SourceMappings: 22:26:0:-:0;29:1;22:26 diff --git a/test/libyul/objectCompiler/datasize_code.yul b/test/libyul/objectCompiler/datasize_code.yul index cff68515015f..c48b1eba9b93 100644 --- a/test/libyul/objectCompiler/datasize_code.yul +++ b/test/libyul/objectCompiler/datasize_code.yul @@ -27,3 +27,4 @@ object "a" { // } // Bytecode: 6006600055fe // Opcodes: PUSH1 0x6 PUSH1 0x0 SSTORE INVALID +// SourceMappings: 22:26:0:-:0;29:1;22:26 diff --git a/test/libyul/objectCompiler/datasize_data.yul b/test/libyul/objectCompiler/datasize_data.yul index f83414697dc8..191d162dfe67 100644 --- a/test/libyul/objectCompiler/datasize_data.yul +++ b/test/libyul/objectCompiler/datasize_data.yul @@ -14,3 +14,4 @@ object "a" { // data_acaf3289d7b601cbd114fb36c4d29c85bbfd5e133f14cb355c3fd8d99367964f 48656c6c6f2c20576f726c6421 // Bytecode: 600d600055fe // Opcodes: PUSH1 0xD PUSH1 0x0 SSTORE INVALID +// SourceMappings: 22:28:0:-:0;29:1;22:28 diff --git a/test/libyul/objectCompiler/datasize_self.yul b/test/libyul/objectCompiler/datasize_self.yul index 4579fe67c502..cfd96555a905 100644 --- a/test/libyul/objectCompiler/datasize_self.yul +++ b/test/libyul/objectCompiler/datasize_self.yul @@ -14,3 +14,4 @@ object "a" { // data_acaf3289d7b601cbd114fb36c4d29c85bbfd5e133f14cb355c3fd8d99367964f 48656c6c6f2c20576f726c6421 // Bytecode: 6006600055fe // Opcodes: PUSH1 0x6 PUSH1 0x0 SSTORE INVALID +// SourceMappings: 22:24:0:-:0;29:1;22:24 diff --git a/test/libyul/objectCompiler/function_series.yul b/test/libyul/objectCompiler/function_series.yul index 88d2e171f62a..90dc88753fe2 100644 --- a/test/libyul/objectCompiler/function_series.yul +++ b/test/libyul/objectCompiler/function_series.yul @@ -28,3 +28,4 @@ object "Contract" { // sstore // Bytecode: 6009565b5b565b5b565b6001600055 // Opcodes: PUSH1 0x9 JUMP JUMPDEST JUMPDEST JUMP JUMPDEST JUMPDEST JUMP JUMPDEST PUSH1 0x1 PUSH1 0x0 SSTORE +// SourceMappings: 33:15:0:-:0;;;46:2;;53:15;66:2;;;83:1;80;73:12 diff --git a/test/libyul/objectCompiler/namedObjectCode.yul b/test/libyul/objectCompiler/namedObjectCode.yul index 4fc6891c33a5..b13bdae5e0fd 100644 --- a/test/libyul/objectCompiler/namedObjectCode.yul +++ b/test/libyul/objectCompiler/namedObjectCode.yul @@ -11,3 +11,4 @@ object "a" { // sstore // Bytecode: 6001600055 // Opcodes: PUSH1 0x1 PUSH1 0x0 SSTORE +// SourceMappings: 32:1:0:-:0;29;22:12 diff --git a/test/libyul/objectCompiler/nested_optimizer.yul b/test/libyul/objectCompiler/nested_optimizer.yul index 1d5e568f1a38..e4512f03dfc7 100644 --- a/test/libyul/objectCompiler/nested_optimizer.yul +++ b/test/libyul/objectCompiler/nested_optimizer.yul @@ -38,3 +38,4 @@ object "a" { // } // Bytecode: 600060003555fe // Opcodes: PUSH1 0x0 PUSH1 0x0 CALLDATALOAD SSTORE INVALID +// SourceMappings: 48:1:0:-:0;;35:15;107:20 diff --git a/test/libyul/objectCompiler/simple.yul b/test/libyul/objectCompiler/simple.yul index d41b527cb369..3e7d1859f337 100644 --- a/test/libyul/objectCompiler/simple.yul +++ b/test/libyul/objectCompiler/simple.yul @@ -11,3 +11,4 @@ // sstore // Bytecode: 6001600055 // Opcodes: PUSH1 0x1 PUSH1 0x0 SSTORE +// SourceMappings: 14:1:0:-:0;11;4:12 diff --git a/test/libyul/objectCompiler/simple_optimizer.yul b/test/libyul/objectCompiler/simple_optimizer.yul index 3d00e45d3c17..40234a1db56d 100644 --- a/test/libyul/objectCompiler/simple_optimizer.yul +++ b/test/libyul/objectCompiler/simple_optimizer.yul @@ -17,3 +17,4 @@ // sstore // Bytecode: 600060003555 // Opcodes: PUSH1 0x0 PUSH1 0x0 CALLDATALOAD SSTORE +// SourceMappings: 26:1:0:-:0;;13:15;79:20 diff --git a/test/libyul/objectCompiler/subObject.yul b/test/libyul/objectCompiler/subObject.yul index 98ea4d07944c..f99b071af5cb 100644 --- a/test/libyul/objectCompiler/subObject.yul +++ b/test/libyul/objectCompiler/subObject.yul @@ -19,3 +19,4 @@ object "a" { // } // Bytecode: fe // Opcodes: INVALID +// SourceMappings: diff --git a/test/libyul/objectCompiler/subSubObject.yul b/test/libyul/objectCompiler/subSubObject.yul index 5e01f6ddd711..a36f97619e94 100644 --- a/test/libyul/objectCompiler/subSubObject.yul +++ b/test/libyul/objectCompiler/subSubObject.yul @@ -37,3 +37,4 @@ object "a" { // } // Bytecode: fe // Opcodes: INVALID +// SourceMappings: From 9815a618b007fc95a9b8ff7f2df733ee47793fad Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Wed, 26 Feb 2020 11:52:40 +0100 Subject: [PATCH 60/92] Docker: install curl in ubuntu16.04 ossfuzz base image --- .circleci/config.yml | 2 +- .circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8bee0b83b957..0319ca1529fe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ parameters: default: "5" ubuntu-1604-clang-ossfuzz-docker-image-rev: type: string - default: "1" + default: "2" defaults: diff --git a/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz b/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz index 8e2ee01b0668..f7dfc7001bdf 100644 --- a/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz +++ b/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz @@ -30,7 +30,7 @@ RUN apt-get update; \ build-essential \ software-properties-common \ ninja-build git wget \ - libbz2-dev zlib1g-dev git; \ + libbz2-dev zlib1g-dev git curl; \ apt-get install -qy python-pip python-sphinx; # Install cmake 3.14 (minimum requirement is cmake 3.10) From d1b6a4a649557b0740c860d2b0e4106bc4a8e8c7 Mon Sep 17 00:00:00 2001 From: Erik Kundt Date: Wed, 26 Feb 2020 21:37:52 +0100 Subject: [PATCH 61/92] Fixes raw bytes warning in semantic test framework. --- test/libsolidity/util/BytesUtils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/libsolidity/util/BytesUtils.cpp b/test/libsolidity/util/BytesUtils.cpp index 0c5518afb60e..6377fbe2e724 100644 --- a/test/libsolidity/util/BytesUtils.cpp +++ b/test/libsolidity/util/BytesUtils.cpp @@ -225,7 +225,7 @@ string BytesUtils::formatRawBytes( { bytes byteRange{it, it + static_cast(parameter.abiType.size)}; - os << _linePrefix << formatBytes(byteRange, parameter.abiType); + os << _linePrefix << byteRange; if (¶meter != ¶meters.back()) os << endl; From bc3261936471b0cc4f7b2c9ae1be3179d52ca71d Mon Sep 17 00:00:00 2001 From: chriseth Date: Wed, 26 Feb 2020 15:50:34 +0100 Subject: [PATCH 62/92] Enable optimized IR output via the commandline. --- Changelog.md | 1 + solc/CommandLineInterface.cpp | 66 ++++++++++++++++++++++------------- solc/CommandLineInterface.h | 1 + 3 files changed, 44 insertions(+), 24 deletions(-) diff --git a/Changelog.md b/Changelog.md index f151365aefa7..9f99cea33aff 100644 --- a/Changelog.md +++ b/Changelog.md @@ -6,6 +6,7 @@ Language Features: Compiler Features: * AssemblyStack: Support for source locations (source mappings) and thus debugging Yul sources. + * Commandline Interface: Enable output of experimental optimized IR via ``--ir-optimized``. Bugfixes: diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index ccb192ba1bdc..e91f6446cc87 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -127,6 +127,7 @@ static string const g_strInterface = "interface"; static string const g_strYul = "yul"; static string const g_strYulDialect = "yul-dialect"; static string const g_strIR = "ir"; +static string const g_strIROptimized = "ir-optimized"; static string const g_strIPFS = "ipfs"; static string const g_strLicense = "license"; static string const g_strLibraries = "libraries"; @@ -190,6 +191,7 @@ static string const g_argImportAst = g_strImportAst; static string const g_argInputFile = g_strInputFile; static string const g_argYul = g_strYul; static string const g_argIR = g_strIR; +static string const g_argIROptimized = g_strIROptimized; static string const g_argEwasm = g_strEwasm; static string const g_argLibraries = g_strLibraries; static string const g_argLink = g_strLink; @@ -336,36 +338,50 @@ void CommandLineInterface::handleOpcode(string const& _contract) void CommandLineInterface::handleIR(string const& _contractName) { - if (m_args.count(g_argIR)) + if (!m_args.count(g_argIR)) + return; + + if (m_args.count(g_argOutputDir)) + createFile(m_compiler->filesystemFriendlyName(_contractName) + ".yul", m_compiler->yulIR(_contractName)); + else { - if (m_args.count(g_argOutputDir)) - createFile(m_compiler->filesystemFriendlyName(_contractName) + ".yul", m_compiler->yulIR(_contractName)); - else - { - sout() << "IR:" << endl; - sout() << m_compiler->yulIR(_contractName) << endl; - } + sout() << "IR:" << endl; + sout() << m_compiler->yulIR(_contractName) << endl; + } +} + +void CommandLineInterface::handleIROptimized(string const& _contractName) +{ + if (!m_args.count(g_argIROptimized)) + return; + + if (m_args.count(g_argOutputDir)) + createFile(m_compiler->filesystemFriendlyName(_contractName) + "_opt.yul", m_compiler->yulIROptimized(_contractName)); + else + { + sout() << "Optimized IR:" << endl; + sout() << m_compiler->yulIROptimized(_contractName) << endl; } } void CommandLineInterface::handleEwasm(string const& _contractName) { - if (m_args.count(g_argEwasm)) + if (!m_args.count(g_argEwasm)) + return; + + if (m_args.count(g_argOutputDir)) { - if (m_args.count(g_argOutputDir)) - { - createFile(m_compiler->filesystemFriendlyName(_contractName) + ".wast", m_compiler->ewasm(_contractName)); - createFile( - m_compiler->filesystemFriendlyName(_contractName) + ".wasm", - asString(m_compiler->ewasmObject(_contractName).bytecode) - ); - } - else - { - sout() << "Ewasm text:" << endl; - sout() << m_compiler->ewasm(_contractName) << endl; - sout() << "Ewasm binary (hex): " << m_compiler->ewasmObject(_contractName).toHex() << endl; - } + createFile(m_compiler->filesystemFriendlyName(_contractName) + ".wast", m_compiler->ewasm(_contractName)); + createFile( + m_compiler->filesystemFriendlyName(_contractName) + ".wasm", + asString(m_compiler->ewasmObject(_contractName).bytecode) + ); + } + else + { + sout() << "Ewasm text:" << endl; + sout() << m_compiler->ewasm(_contractName) << endl; + sout() << "Ewasm binary (hex): " << m_compiler->ewasmObject(_contractName).toHex() << endl; } } @@ -812,6 +828,7 @@ Allowed options)", (g_argBinaryRuntime.c_str(), "Binary of the runtime part of the contracts in hex.") (g_argAbi.c_str(), "ABI specification of the contracts.") (g_argIR.c_str(), "Intermediate Representation (IR) of all contracts (EXPERIMENTAL).") + (g_argIROptimized.c_str(), "Optimized intermediate Representation (IR) of all contracts (EXPERIMENTAL).") (g_argEwasm.c_str(), "Ewasm text representation of all contracts (EXPERIMENTAL).") (g_argSignatureHashes.c_str(), "Function signature hashes of the contracts.") (g_argNatspecUser.c_str(), "Natspec user documentation of all contracts.") @@ -1123,7 +1140,7 @@ bool CommandLineInterface::processInput() m_compiler->setRevertStringBehaviour(m_revertStrings); // TODO: Perhaps we should not compile unless requested - m_compiler->enableIRGeneration(m_args.count(g_argIR)); + m_compiler->enableIRGeneration(m_args.count(g_argIR) || m_args.count(g_argIROptimized)); m_compiler->enableEwasmGeneration(m_args.count(g_argEwasm)); OptimiserSettings settings = m_args.count(g_argOptimize) ? OptimiserSettings::standard() : OptimiserSettings::minimal(); @@ -1631,6 +1648,7 @@ void CommandLineInterface::outputCompilationResults() handleBytecode(contract); handleIR(contract); + handleIROptimized(contract); handleEwasm(contract); handleSignatureHashes(contract); handleMetadata(contract); diff --git a/solc/CommandLineInterface.h b/solc/CommandLineInterface.h index 553f9e69b5dc..417501907b1d 100644 --- a/solc/CommandLineInterface.h +++ b/solc/CommandLineInterface.h @@ -65,6 +65,7 @@ class CommandLineInterface void handleBinary(std::string const& _contract); void handleOpcode(std::string const& _contract); void handleIR(std::string const& _contract); + void handleIROptimized(std::string const& _contract); void handleEwasm(std::string const& _contract); void handleBytecode(std::string const& _contract); void handleSignatureHashes(std::string const& _contract); From ed02aae1d9cad7514df6131ed326dc33f87923f2 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Fri, 28 Feb 2020 15:41:22 +0100 Subject: [PATCH 63/92] Update solidity fuzzing dictionary with >0.6.0 keywords --- test/tools/ossfuzz/config/solidity.dict | 13 ++++++++++++- test/tools/ossfuzz/config/strict_assembly.dict | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/test/tools/ossfuzz/config/solidity.dict b/test/tools/ossfuzz/config/solidity.dict index 9fe773870a5c..8190e7c55105 100644 --- a/test/tools/ossfuzz/config/solidity.dict +++ b/test/tools/ossfuzz/config/solidity.dict @@ -119,7 +119,7 @@ "bytes8 " "bytes9 " "constant " -"constructor " +"constructor() " "continue;" "contract " "delete " @@ -212,3 +212,14 @@ "|" "}" "~" +"override" +"virtual" +" is " +"receive() " +"fallback() " +"catch Error() {}" +"catch (bytes memory ) {}" +"{value: 1, gas: 2}" +"{salt: "salt", value: 10}" +"leave" +"a[1:2]" diff --git a/test/tools/ossfuzz/config/strict_assembly.dict b/test/tools/ossfuzz/config/strict_assembly.dict index 4415c87f4203..7f4fe4b5b5e3 100644 --- a/test/tools/ossfuzz/config/strict_assembly.dict +++ b/test/tools/ossfuzz/config/strict_assembly.dict @@ -89,3 +89,4 @@ "xor(" "{" "}" +"leave" From f10c6500b2378e652c5e7e91f255501d9acec4ba Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 25 Feb 2020 18:25:19 +0100 Subject: [PATCH 64/92] Immutable is not reserved anymore. --- liblangutil/Token.h | 2 +- test/libsolidity/SolidityParser.cpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/liblangutil/Token.h b/liblangutil/Token.h index ed6cff3e16f9..d9d4471cb372 100644 --- a/liblangutil/Token.h +++ b/liblangutil/Token.h @@ -166,6 +166,7 @@ namespace solidity::langutil K(Indexed, "indexed", 0) \ K(Interface, "interface", 0) \ K(Internal, "internal", 0) \ + K(Immutable, "immutable", 0) \ K(Import, "import", 0) \ K(Is, "is", 0) \ K(Library, "library", 0) \ @@ -243,7 +244,6 @@ namespace solidity::langutil K(Default, "default", 0) \ K(Define, "define", 0) \ K(Final, "final", 0) \ - K(Immutable, "immutable", 0) \ K(Implements, "implements", 0) \ K(In, "in", 0) \ K(Inline, "inline", 0) \ diff --git a/test/libsolidity/SolidityParser.cpp b/test/libsolidity/SolidityParser.cpp index 3f70f9160eff..192a268983b7 100644 --- a/test/libsolidity/SolidityParser.cpp +++ b/test/libsolidity/SolidityParser.cpp @@ -540,7 +540,6 @@ BOOST_AUTO_TEST_CASE(keyword_is_reserved) "default", "define", "final", - "immutable", "implements", "in", "inline", From 1488a1ceb807633beef3349ad785ee98be45742d Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 25 Feb 2020 18:42:16 +0100 Subject: [PATCH 65/92] Refactor isConstant to add "immutable". --- libsolidity/ast/AST.cpp | 2 +- libsolidity/ast/AST.h | 10 ++++++---- libsolidity/ast/ASTJsonImporter.cpp | 8 +++++++- libsolidity/parsing/Parser.cpp | 6 +++--- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/libsolidity/ast/AST.cpp b/libsolidity/ast/AST.cpp index 90084f9c490d..6b03c7b586e9 100644 --- a/libsolidity/ast/AST.cpp +++ b/libsolidity/ast/AST.cpp @@ -390,7 +390,7 @@ DeclarationAnnotation& Declaration::annotation() const bool VariableDeclaration::isLValue() const { // Constant declared variables are Read-Only - if (m_isConstant) + if (isConstant()) return false; // External function arguments of reference type are Read-Only if (isExternalCallableParameter() && dynamic_cast(type())) diff --git a/libsolidity/ast/AST.h b/libsolidity/ast/AST.h index 336c221c6767..adb957cfec5b 100644 --- a/libsolidity/ast/AST.h +++ b/libsolidity/ast/AST.h @@ -814,6 +814,7 @@ class VariableDeclaration: public Declaration { public: enum Location { Unspecified, Storage, Memory, CallData }; + enum class Constantness { Mutable, Immutable, Constant }; VariableDeclaration( int64_t _id, @@ -824,7 +825,7 @@ class VariableDeclaration: public Declaration Visibility _visibility, bool _isStateVar = false, bool _isIndexed = false, - bool _isConstant = false, + Constantness _constantness = Constantness::Mutable, ASTPointer const& _overrides = nullptr, Location _referenceLocation = Location::Unspecified ): @@ -833,7 +834,7 @@ class VariableDeclaration: public Declaration m_value(_value), m_isStateVariable(_isStateVar), m_isIndexed(_isIndexed), - m_isConstant(_isConstant), + m_constantness(_constantness), m_overrides(_overrides), m_location(_referenceLocation) {} @@ -877,7 +878,7 @@ class VariableDeclaration: public Declaration bool hasReferenceOrMappingType() const; bool isStateVariable() const { return m_isStateVariable; } bool isIndexed() const { return m_isIndexed; } - bool isConstant() const { return m_isConstant; } + bool isConstant() const { return m_constantness == Constantness::Constant; } ASTPointer const& overrides() const { return m_overrides; } Location referenceLocation() const { return m_location; } /// @returns a set of allowed storage locations for the variable. @@ -904,7 +905,8 @@ class VariableDeclaration: public Declaration ASTPointer m_value; bool m_isStateVariable = false; ///< Whether or not this is a contract state variable bool m_isIndexed = false; ///< Whether this is an indexed variable (used by events). - bool m_isConstant = false; ///< Whether the variable is a compile-time constant. + /// Whether the variable is "constant", "immutable" or non-marked (mutable). + Constantness m_constantness = Constantness::Mutable; ASTPointer m_overrides; ///< Contains the override specifier node Location m_location = Location::Unspecified; ///< Location of the variable if it is of reference type. }; diff --git a/libsolidity/ast/ASTJsonImporter.cpp b/libsolidity/ast/ASTJsonImporter.cpp index aa28e3f17f06..26a7c7b7ec2a 100644 --- a/libsolidity/ast/ASTJsonImporter.cpp +++ b/libsolidity/ast/ASTJsonImporter.cpp @@ -411,6 +411,12 @@ ASTPointer ASTJsonImporter::createVariableDeclaration(Json: { astAssert(_node["name"].isString(), "Expected 'name' to be a string!"); + VariableDeclaration::Constantness constantness{}; + if (memberAsBool(_node, "constant")) + constantness = VariableDeclaration::Constantness::Constant; + else + constantness = VariableDeclaration::Constantness::Mutable; + return createASTNode( _node, nullOrCast(member(_node, "typeName")), @@ -419,7 +425,7 @@ ASTPointer ASTJsonImporter::createVariableDeclaration(Json: visibility(_node), memberAsBool(_node, "stateVariable"), _node.isMember("indexed") ? memberAsBool(_node, "indexed") : false, - memberAsBool(_node, "constant"), + constantness, _node["overrides"].isNull() ? nullptr : createOverrideSpecifier(member(_node, "overrides")), location(_node) ); diff --git a/libsolidity/parsing/Parser.cpp b/libsolidity/parsing/Parser.cpp index 99aa6e2a1ed7..edf43377f67c 100644 --- a/libsolidity/parsing/Parser.cpp +++ b/libsolidity/parsing/Parser.cpp @@ -695,7 +695,7 @@ ASTPointer Parser::parseVariableDeclaration( ); bool isIndexed = false; - bool isDeclaredConst = false; + VariableDeclaration::Constantness constantness = VariableDeclaration::Constantness::Mutable; ASTPointer overrides = nullptr; Visibility visibility(Visibility::Default); VariableDeclaration::Location location = VariableDeclaration::Location::Unspecified; @@ -731,7 +731,7 @@ ASTPointer Parser::parseVariableDeclaration( if (_options.allowIndexed && token == Token::Indexed) isIndexed = true; else if (token == Token::Constant) - isDeclaredConst = true; + constantness = VariableDeclaration::Constantness::Constant; else if (_options.allowLocationSpecifier && TokenTraits::isLocationSpecifier(token)) { if (location != VariableDeclaration::Location::Unspecified) @@ -790,7 +790,7 @@ ASTPointer Parser::parseVariableDeclaration( visibility, _options.isStateVariable, isIndexed, - isDeclaredConst, + constantness, overrides, location ); From 90fa56c7191d4353d8cdbc778c0f71a4459f4a56 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Mon, 2 Mar 2020 16:32:30 +0100 Subject: [PATCH 66/92] Allow use of yul util functions in legacy code generation. --- libsolidity/codegen/ABIFunctions.cpp | 11 +--------- libsolidity/codegen/ABIFunctions.h | 7 ------- libsolidity/codegen/CompilerContext.cpp | 14 +++++++++++++ libsolidity/codegen/CompilerContext.h | 20 ++++++++++++++++++- libsolidity/codegen/ContractCompiler.cpp | 8 ++++---- .../codegen/MultiUseYulFunctionCollector.cpp | 12 +++++++++-- .../codegen/MultiUseYulFunctionCollector.h | 19 ++++++++++++++++-- libsolidity/codegen/ir/IRGenerator.cpp | 6 +++--- 8 files changed, 68 insertions(+), 29 deletions(-) diff --git a/libsolidity/codegen/ABIFunctions.cpp b/libsolidity/codegen/ABIFunctions.cpp index 0ad044a54df1..b69f47346b5d 100644 --- a/libsolidity/codegen/ABIFunctions.cpp +++ b/libsolidity/codegen/ABIFunctions.cpp @@ -240,13 +240,6 @@ string ABIFunctions::tupleDecoder(TypePointers const& _types, bool _fromMemory) }); } -pair> ABIFunctions::requestedFunctions() -{ - std::set empty; - swap(empty, m_externallyUsedFunctions); - return make_pair(m_functionCollector->requestedFunctions(), std::move(empty)); -} - string ABIFunctions::EncodingOptions::toFunctionNameSuffix() const { string suffix; @@ -1504,9 +1497,7 @@ string ABIFunctions::createFunction(string const& _name, function con string ABIFunctions::createExternallyUsedFunction(string const& _name, function const& _creator) { - string name = createFunction(_name, _creator); - m_externallyUsedFunctions.insert(name); - return name; + return m_functionCollector->createExternallyUsedFunction(_name, _creator); } size_t ABIFunctions::headSize(TypePointers const& _targetTypes) diff --git a/libsolidity/codegen/ABIFunctions.h b/libsolidity/codegen/ABIFunctions.h index ce3efe6eac37..74bfdf7fa11e 100644 --- a/libsolidity/codegen/ABIFunctions.h +++ b/libsolidity/codegen/ABIFunctions.h @@ -104,12 +104,6 @@ class ABIFunctions /// stack slot, it takes exactly that number of values. std::string tupleDecoder(TypePointers const& _types, bool _fromMemory = false); - /// @returns concatenation of all generated functions and a set of the - /// externally used functions. - /// Clears the internal list, i.e. calling it again will result in an - /// empty return value. - std::pair> requestedFunctions(); - private: struct EncodingOptions { @@ -260,7 +254,6 @@ class ABIFunctions langutil::EVMVersion m_evmVersion; RevertStrings const m_revertStrings; std::shared_ptr m_functionCollector; - std::set m_externallyUsedFunctions; YulUtilFunctions m_utils; }; diff --git a/libsolidity/codegen/CompilerContext.cpp b/libsolidity/codegen/CompilerContext.cpp index 8c1099855bed..f65ccae52e7e 100644 --- a/libsolidity/codegen/CompilerContext.cpp +++ b/libsolidity/codegen/CompilerContext.cpp @@ -94,6 +94,20 @@ void CompilerContext::callLowLevelFunction( *this << retTag.tag(); } +void CompilerContext::callYulUtilFunction( + string const& _name, + unsigned _inArgs, + unsigned _outArgs +) +{ + m_functionCollector->markAsExternallyUsed(_name); + auto retTag = pushNewTag(); + CompilerUtils(*this).moveIntoStack(_inArgs); + appendJumpTo(namedTag(_name)); + adjustStackOffset(int(_outArgs) - 1 - _inArgs); + *this << retTag.tag(); +} + evmasm::AssemblyItem CompilerContext::lowLevelFunctionTag( string const& _name, unsigned _inArgs, diff --git a/libsolidity/codegen/CompilerContext.h b/libsolidity/codegen/CompilerContext.h index c37afbfa5234..d9393e23ad3c 100644 --- a/libsolidity/codegen/CompilerContext.h +++ b/libsolidity/codegen/CompilerContext.h @@ -65,7 +65,8 @@ class CompilerContext m_evmVersion(_evmVersion), m_revertStrings(_revertStrings), m_runtimeContext(_runtimeContext), - m_abiFunctions(m_evmVersion, m_revertStrings) + m_abiFunctions(m_evmVersion, m_revertStrings, m_functionCollector), + m_yulUtilFunctions(m_evmVersion, m_revertStrings, m_functionCollector) { if (m_runtimeContext) m_runtimeSub = size_t(m_asm->newSub(m_runtimeContext->m_asm).data()); @@ -131,6 +132,14 @@ class CompilerContext unsigned _outArgs, std::function const& _generator ); + + /// Appends a call to a yul util function and registers the function as externally used. + void callYulUtilFunction( + std::string const& _name, + unsigned _inArgs, + unsigned _outArgs + ); + /// Returns the tag of the named low-level function and inserts the generator into the /// list of low-level-functions to be generated, unless it already exists. /// Note that the generator should not assume that objects are still alive when it is called, @@ -144,6 +153,11 @@ class CompilerContext /// Generates the code for missing low-level functions, i.e. calls the generators passed above. void appendMissingLowLevelFunctions(); ABIFunctions& abiFunctions() { return m_abiFunctions; } + YulUtilFunctions& utilFunctions() { return m_yulUtilFunctions; } + std::pair> requestedYulFunctions() + { + return m_functionCollector->requestedFunctions(); + } ModifierDefinition const& resolveVirtualFunctionModifier(ModifierDefinition const& _modifier) const; /// Returns the distance of the given local variable from the bottom of the stack (of the current function). @@ -355,8 +369,12 @@ class CompilerContext size_t m_runtimeSub = -1; /// An index of low-level function labels by name. std::map m_lowLevelFunctions; + // Collector for yul functions. + std::shared_ptr m_functionCollector = std::make_shared(); /// Container for ABI functions to be generated. ABIFunctions m_abiFunctions; + /// Container for Yul Util functions to be generated. + YulUtilFunctions m_yulUtilFunctions; /// The queue of low-level functions to generate. std::queue>> m_lowLevelFunctionGenerationQueue; }; diff --git a/libsolidity/codegen/ContractCompiler.cpp b/libsolidity/codegen/ContractCompiler.cpp index a99aefb5f3ce..47187b1c88ff 100644 --- a/libsolidity/codegen/ContractCompiler.cpp +++ b/libsolidity/codegen/ContractCompiler.cpp @@ -1267,12 +1267,12 @@ void ContractCompiler::appendMissingFunctions() solAssert(m_context.nextFunctionToCompile() != function, "Compiled the wrong function?"); } m_context.appendMissingLowLevelFunctions(); - auto abiFunctions = m_context.abiFunctions().requestedFunctions(); - if (!abiFunctions.first.empty()) + auto yulFunctions = m_context.requestedYulFunctions(); + if (!yulFunctions.first.empty()) m_context.appendInlineAssembly( - "{" + move(abiFunctions.first) + "}", + "{" + move(yulFunctions.first) + "}", {}, - abiFunctions.second, + yulFunctions.second, true, m_optimiserSettings ); diff --git a/libsolidity/codegen/MultiUseYulFunctionCollector.cpp b/libsolidity/codegen/MultiUseYulFunctionCollector.cpp index 2be3d29ae6b3..8af292eda99e 100644 --- a/libsolidity/codegen/MultiUseYulFunctionCollector.cpp +++ b/libsolidity/codegen/MultiUseYulFunctionCollector.cpp @@ -30,13 +30,15 @@ using namespace std; using namespace solidity; using namespace solidity::frontend; -string MultiUseYulFunctionCollector::requestedFunctions() +pair> MultiUseYulFunctionCollector::requestedFunctions() { string result; for (auto const& f: m_requestedFunctions) result += f.second; m_requestedFunctions.clear(); - return result; + std::set empty; + swap(empty, m_externallyUsedFunctions); + return make_pair(result, std::move(empty)); } string MultiUseYulFunctionCollector::createFunction(string const& _name, function const& _creator) @@ -50,3 +52,9 @@ string MultiUseYulFunctionCollector::createFunction(string const& _name, functio } return _name; } + +string MultiUseYulFunctionCollector::createExternallyUsedFunction(string const& _name, function const& _creator) +{ + m_externallyUsedFunctions.insert(_name); + return createFunction(_name, _creator); +} diff --git a/libsolidity/codegen/MultiUseYulFunctionCollector.h b/libsolidity/codegen/MultiUseYulFunctionCollector.h index d839a31be873..a1c733274b21 100644 --- a/libsolidity/codegen/MultiUseYulFunctionCollector.h +++ b/libsolidity/codegen/MultiUseYulFunctionCollector.h @@ -23,6 +23,7 @@ #include #include +#include #include namespace solidity::frontend @@ -40,14 +41,28 @@ class MultiUseYulFunctionCollector /// cases. std::string createFunction(std::string const& _name, std::function const& _creator); - /// @returns concatenation of all generated functions. + /// Helper function that uses @a _creator to create a function and add it to + /// @a m_requestedFunctions if it has not been created yet and returns @a _name in both + /// cases. + std::string createExternallyUsedFunction(std::string const& _name, std::function const& _creator); + + /// Manually mark a function as externally used. + void markAsExternallyUsed(std::string const& _name) + { + m_externallyUsedFunctions.insert(_name); + } + + /// @returns concatenation of all generated functions and a set of the + /// externally used functions. /// Clears the internal list, i.e. calling it again will result in an /// empty return value. - std::string requestedFunctions(); + std::pair> requestedFunctions(); private: /// Map from function name to code for a multi-use function. std::map m_requestedFunctions; + // Set of externally used functions. + std::set m_externallyUsedFunctions; }; } diff --git a/libsolidity/codegen/ir/IRGenerator.cpp b/libsolidity/codegen/ir/IRGenerator.cpp index 9c2d8807c88b..0f2ffaf35b95 100644 --- a/libsolidity/codegen/ir/IRGenerator.cpp +++ b/libsolidity/codegen/ir/IRGenerator.cpp @@ -107,7 +107,7 @@ string IRGenerator::generate(ContractDefinition const& _contract) for (auto const* contract: _contract.annotation().linearizedBaseContracts) for (auto const* fun: contract->definedFunctions()) generateFunction(*fun); - t("functions", m_context.functionCollector()->requestedFunctions()); + t("functions", m_context.functionCollector()->requestedFunctions().first); resetContext(_contract); m_context.setInheritanceHierarchy(_contract.annotation().linearizedBaseContracts); @@ -116,7 +116,7 @@ string IRGenerator::generate(ContractDefinition const& _contract) for (auto const* contract: _contract.annotation().linearizedBaseContracts) for (auto const* fun: contract->definedFunctions()) generateFunction(*fun); - t("runtimeFunctions", m_context.functionCollector()->requestedFunctions()); + t("runtimeFunctions", m_context.functionCollector()->requestedFunctions().first); return t.render(); } @@ -383,7 +383,7 @@ string IRGenerator::memoryInit() void IRGenerator::resetContext(ContractDefinition const& _contract) { solAssert( - m_context.functionCollector()->requestedFunctions().empty(), + m_context.functionCollector()->requestedFunctions().first.empty(), "Reset context while it still had functions." ); m_context = IRGenerationContext(m_evmVersion, m_context.revertStrings(), m_optimiserSettings); From 96a230af507187aa42d849a95f45cb551a61cd80 Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Mon, 2 Mar 2020 22:10:15 +0100 Subject: [PATCH 67/92] [SMTChecker] Fix ICEs with tuples --- Changelog.md | 1 + libsolidity/formal/SMTEncoder.cpp | 15 +++++++++++---- .../smtCheckerTests/types/function_in_tuple_1.sol | 10 ++++++++++ .../smtCheckerTests/types/function_in_tuple_2.sol | 10 ++++++++++ .../types/tuple_single_element_1.sol | 9 +++++++++ .../types/tuple_single_element_2.sol | 9 +++++++++ .../types/tuple_single_non_tuple_element.sol | 9 +++++++++ 7 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 test/libsolidity/smtCheckerTests/types/function_in_tuple_1.sol create mode 100644 test/libsolidity/smtCheckerTests/types/function_in_tuple_2.sol create mode 100644 test/libsolidity/smtCheckerTests/types/tuple_single_element_1.sol create mode 100644 test/libsolidity/smtCheckerTests/types/tuple_single_element_2.sol create mode 100644 test/libsolidity/smtCheckerTests/types/tuple_single_non_tuple_element.sol diff --git a/Changelog.md b/Changelog.md index f151365aefa7..fc5d3d0d3eb8 100644 --- a/Changelog.md +++ b/Changelog.md @@ -11,6 +11,7 @@ Compiler Features: Bugfixes: * isoltest: Added new keyword `wei` to express function value in semantic tests * Standard-JSON-Interface: Fix a bug related to empty filenames and imports. + * SMTChecker: Fix internal errors when analysing tuples. ### 0.6.3 (2020-02-18) diff --git a/libsolidity/formal/SMTEncoder.cpp b/libsolidity/formal/SMTEncoder.cpp index ac81a5d878ca..3b2f48816589 100644 --- a/libsolidity/formal/SMTEncoder.cpp +++ b/libsolidity/formal/SMTEncoder.cpp @@ -407,19 +407,26 @@ void SMTEncoder::endVisit(TupleExpression const& _tuple) auto const& symbTuple = dynamic_pointer_cast(m_context.expression(_tuple)); solAssert(symbTuple, ""); auto const& symbComponents = symbTuple->components(); - auto const& tupleComponents = _tuple.components(); - solAssert(symbComponents.size() == _tuple.components().size(), ""); + auto const* tupleComponents = &_tuple.components(); + while (tupleComponents->size() == 1) + { + auto innerTuple = dynamic_pointer_cast(tupleComponents->front()); + solAssert(innerTuple, ""); + tupleComponents = &innerTuple->components(); + } + solAssert(symbComponents.size() == tupleComponents->size(), ""); for (unsigned i = 0; i < symbComponents.size(); ++i) { auto sComponent = symbComponents.at(i); - auto tComponent = tupleComponents.at(i); + auto tComponent = tupleComponents->at(i); if (sComponent && tComponent) { if (auto varDecl = identifierToVariable(*tComponent)) m_context.addAssertion(sComponent->currentValue() == currentValue(*varDecl)); else { - solAssert(m_context.knownExpression(*tComponent), ""); + if (!m_context.knownExpression(*tComponent)) + createExpr(*tComponent); m_context.addAssertion(sComponent->currentValue() == expr(*tComponent)); } } diff --git a/test/libsolidity/smtCheckerTests/types/function_in_tuple_1.sol b/test/libsolidity/smtCheckerTests/types/function_in_tuple_1.sol new file mode 100644 index 000000000000..4d1d954f7b38 --- /dev/null +++ b/test/libsolidity/smtCheckerTests/types/function_in_tuple_1.sol @@ -0,0 +1,10 @@ +pragma experimental SMTChecker; + +contract K { + function f() public pure { + (abi.encode, 2); + } +} +// ---- +// Warning: (76-91): Statement has no effect. +// Warning: (77-80): Assertion checker does not yet implement type abi diff --git a/test/libsolidity/smtCheckerTests/types/function_in_tuple_2.sol b/test/libsolidity/smtCheckerTests/types/function_in_tuple_2.sol new file mode 100644 index 000000000000..5b9c2f41c6d3 --- /dev/null +++ b/test/libsolidity/smtCheckerTests/types/function_in_tuple_2.sol @@ -0,0 +1,10 @@ +pragma experimental SMTChecker; + +contract K { + function f() public pure { + (abi.encode, ""); + } +} +// ---- +// Warning: (76-92): Statement has no effect. +// Warning: (77-80): Assertion checker does not yet implement type abi diff --git a/test/libsolidity/smtCheckerTests/types/tuple_single_element_1.sol b/test/libsolidity/smtCheckerTests/types/tuple_single_element_1.sol new file mode 100644 index 000000000000..f14ca9bc1579 --- /dev/null +++ b/test/libsolidity/smtCheckerTests/types/tuple_single_element_1.sol @@ -0,0 +1,9 @@ +pragma experimental SMTChecker; + +contract C { + function f() public pure { + (("", 2)); + } +} +// ---- +// Warning: (76-85): Statement has no effect. diff --git a/test/libsolidity/smtCheckerTests/types/tuple_single_element_2.sol b/test/libsolidity/smtCheckerTests/types/tuple_single_element_2.sol new file mode 100644 index 000000000000..29b9c974775a --- /dev/null +++ b/test/libsolidity/smtCheckerTests/types/tuple_single_element_2.sol @@ -0,0 +1,9 @@ +pragma experimental SMTChecker; + +contract C { + function f() public pure { + (("", "")); + } +} +// ---- +// Warning: (76-86): Statement has no effect. diff --git a/test/libsolidity/smtCheckerTests/types/tuple_single_non_tuple_element.sol b/test/libsolidity/smtCheckerTests/types/tuple_single_non_tuple_element.sol new file mode 100644 index 000000000000..2a99a552e5b1 --- /dev/null +++ b/test/libsolidity/smtCheckerTests/types/tuple_single_non_tuple_element.sol @@ -0,0 +1,9 @@ +pragma experimental SMTChecker; + +contract C { + function f() public pure { + (2); + } +} +// ---- +// Warning: (76-79): Statement has no effect. From 3bee348525de032cdc510dc62b7366c90cf13d89 Mon Sep 17 00:00:00 2001 From: Leonardo Alt Date: Tue, 11 Feb 2020 22:52:35 -0300 Subject: [PATCH 68/92] Change CHC encoding to functions forest instead of explicit CFG --- libsolidity/formal/CHC.cpp | 387 +++++++++++++----- libsolidity/formal/CHC.h | 77 +++- libsolidity/formal/EncodingContext.h | 1 + libsolidity/formal/ModelChecker.cpp | 4 +- libsolidity/formal/ModelChecker.h | 6 +- libsolidity/formal/Z3Interface.h | 6 +- .../functions/library_constant.sol | 26 ++ .../functions/library_constant_2.sol | 9 + .../invariants/loop_nested.sol | 5 + .../invariants/loop_nested_for.sol | 5 + ...r_loop_array_assignment_storage_memory.sol | 6 +- ..._loop_array_assignment_storage_storage.sol | 8 +- .../operators/delete_array_index_2d.sol | 4 +- 13 files changed, 427 insertions(+), 117 deletions(-) create mode 100644 test/libsolidity/smtCheckerTests/functions/library_constant.sol create mode 100644 test/libsolidity/smtCheckerTests/functions/library_constant_2.sol diff --git a/libsolidity/formal/CHC.cpp b/libsolidity/formal/CHC.cpp index 25201474b6d2..af50b27226e8 100644 --- a/libsolidity/formal/CHC.cpp +++ b/libsolidity/formal/CHC.cpp @@ -27,6 +27,8 @@ #include +#include + using namespace std; using namespace solidity; using namespace solidity::langutil; @@ -75,16 +77,39 @@ void CHC::analyze(SourceUnit const& _source) m_context.setAssertionAccumulation(false); m_variableUsage.setFunctionInlining(false); + resetSourceAnalysis(); + auto boolSort = make_shared(smt::Kind::Bool); auto genesisSort = make_shared( vector(), boolSort ); m_genesisPredicate = createSymbolicBlock(genesisSort, "genesis"); - auto genesis = (*m_genesisPredicate)({}); - addRule(genesis, genesis.name); - - _source.accept(*this); + addRule(genesis(), "genesis"); + + set sources; + sources.insert(&_source); + for (auto const& source: _source.referencedSourceUnits(true)) + sources.insert(source); + for (auto const* source: sources) + defineInterfacesAndSummaries(*source); + for (auto const* source: sources) + source->accept(*this); + + for (auto const& [scope, target]: m_verificationTargets) + { + auto assertions = transactionAssertions(scope); + for (auto const* assertion: assertions) + { + createErrorBlock(); + connectBlocks(target.value, error(), target.constraints && (target.errorId == assertion->id())); + auto [result, model] = query(error(), assertion->location()); + // This should be fine but it's a bug in the old compiler + (void)model; + if (result == smt::CheckResult::UNSATISFIABLE) + m_safeAssertions.insert(assertion); + } + } } vector CHC::unhandledQueries() const @@ -97,26 +122,15 @@ vector CHC::unhandledQueries() const bool CHC::visit(ContractDefinition const& _contract) { - if (!shouldVisit(_contract)) - return false; - - reset(); + resetContractAnalysis(); initContract(_contract); - m_stateVariables = _contract.stateVariablesIncludingInherited(); - - for (auto const& var: m_stateVariables) - // SMT solvers do not support function types as arguments. - if (var->type()->category() == Type::Category::Function) - m_stateSorts.push_back(make_shared(smt::Kind::Int)); - else - m_stateSorts.push_back(smt::smtSort(*var->type())); + m_stateVariables = stateVariablesIncludingInheritedAndPrivate(_contract); + m_stateSorts = stateSorts(_contract); clearIndices(&_contract); - string suffix = _contract.name() + "_" + to_string(_contract.id()); - m_interfacePredicate = createSymbolicBlock(interfaceSort(), "interface_" + suffix); // TODO create static instances for Bool/Int sorts in SolverInterface. auto boolSort = make_shared(smt::Kind::Bool); @@ -125,10 +139,12 @@ bool CHC::visit(ContractDefinition const& _contract) boolSort ); + string suffix = _contract.name() + "_" + to_string(_contract.id()); m_errorPredicate = createSymbolicBlock(errorFunctionSort, "error_" + suffix); - m_constructorPredicate = createSymbolicBlock(constructorSort(), "implicit_constructor_" + to_string(_contract.id())); + m_constructorSummaryPredicate = createSymbolicBlock(constructorSort(), "summary_constructor_" + suffix); + m_implicitConstructorPredicate = createSymbolicBlock(interfaceSort(), "implicit_constructor_" + suffix); auto stateExprs = currentStateVariables(); - setCurrentBlock(*m_interfacePredicate, &stateExprs); + setCurrentBlock(*m_interfaces.at(m_currentContract), &stateExprs); SMTEncoder::visit(_contract); return false; @@ -136,33 +152,33 @@ bool CHC::visit(ContractDefinition const& _contract) void CHC::endVisit(ContractDefinition const& _contract) { - if (!shouldVisit(_contract)) - return; - for (auto const& var: m_stateVariables) { solAssert(m_context.knownVariable(*var), ""); + auto const& symbVar = m_context.variable(*var); + symbVar->resetIndex(); m_context.setZeroValue(*var); + symbVar->increaseIndex(); } - auto genesisPred = (*m_genesisPredicate)({}); - auto implicitConstructor = (*m_constructorPredicate)(currentStateVariables()); - connectBlocks(genesisPred, implicitConstructor); + auto implicitConstructor = (*m_implicitConstructorPredicate)(initialStateVariables()); + connectBlocks(genesis(), implicitConstructor); m_currentBlock = implicitConstructor; + m_context.addAssertion(m_error.currentValue() == 0); if (auto constructor = _contract.constructor()) constructor->accept(*this); else inlineConstructorHierarchy(_contract); - connectBlocks(m_currentBlock, interface()); + auto summary = predicate(*m_constructorSummaryPredicate, vector{m_error.currentValue()} + currentStateVariables()); + connectBlocks(m_currentBlock, summary); - for (unsigned i = 0; i < m_verificationTargets.size(); ++i) - { - auto const& target = m_verificationTargets.at(i); - auto errorAppl = error(i + 1); - if (query(errorAppl, target->location())) - m_safeAssertions.insert(target); - } + clearIndices(m_currentContract, nullptr); + auto stateExprs = vector{m_error.currentValue()} + currentStateVariables(); + setCurrentBlock(*m_constructorSummaryPredicate, &stateExprs); + + addVerificationTarget(m_currentContract, m_currentBlock, smt::Expression(true), m_error.currentValue()); + connectBlocks(m_currentBlock, interface(), m_error.currentValue() == 0); SMTEncoder::endVisit(_contract); } @@ -182,7 +198,7 @@ bool CHC::visit(FunctionDefinition const& _function) return false; } - solAssert(!m_currentFunction, "Inlining internal function calls not yet implemented"); + solAssert(!m_currentFunction, "Function inlining should not happen in CHC."); m_currentFunction = &_function; initFunction(_function); @@ -193,7 +209,17 @@ bool CHC::visit(FunctionDefinition const& _function) auto functionPred = predicate(*functionEntryBlock, currentFunctionVariables()); auto bodyPred = predicate(*bodyBlock); - connectBlocks(m_currentBlock, functionPred); + if (_function.isConstructor()) + connectBlocks(m_currentBlock, functionPred); + else + connectBlocks(genesis(), functionPred); + + m_context.addAssertion(m_error.currentValue() == 0); + for (auto const* var: m_stateVariables) + m_context.addAssertion(m_context.variable(*var)->valueAtIndex(0) == currentValue(*var)); + for (auto const& var: _function.parameters()) + m_context.addAssertion(m_context.variable(*var)->valueAtIndex(0) == currentValue(*var)); + connectBlocks(functionPred, bodyPred); setCurrentBlock(*bodyBlock); @@ -225,18 +251,30 @@ void CHC::endVisit(FunctionDefinition const& _function) // This is done in endVisit(ContractDefinition). if (_function.isConstructor()) { - auto constructorExit = createSymbolicBlock(interfaceSort(), "constructor_exit_" + to_string(_function.id())); - connectBlocks(m_currentBlock, predicate(*constructorExit, currentStateVariables())); + string suffix = m_currentContract->name() + "_" + to_string(m_currentContract->id()); + auto constructorExit = createSymbolicBlock(constructorSort(), "constructor_exit_" + suffix); + connectBlocks(m_currentBlock, predicate(*constructorExit, vector{m_error.currentValue()} + currentStateVariables())); + clearIndices(m_currentContract, m_currentFunction); - auto stateExprs = currentStateVariables(); + auto stateExprs = vector{m_error.currentValue()} + currentStateVariables(); setCurrentBlock(*constructorExit, &stateExprs); } else { - connectBlocks(m_currentBlock, interface()); - clearIndices(m_currentContract, m_currentFunction); - auto stateExprs = currentStateVariables(); - setCurrentBlock(*m_interfacePredicate, &stateExprs); + auto assertionError = m_error.currentValue(); + auto sum = summary(_function); + connectBlocks(m_currentBlock, sum); + + auto iface = interface(); + + auto stateExprs = initialStateVariables(); + setCurrentBlock(*m_interfaces.at(m_currentContract), &stateExprs); + + if (_function.isPublic()) + { + addVerificationTarget(&_function, m_currentBlock, sum, assertionError); + connectBlocks(m_currentBlock, iface, sum && (assertionError == 0)); + } } m_currentFunction = nullptr; } @@ -468,12 +506,23 @@ void CHC::visitAssert(FunctionCall const& _funCall) solAssert(args.size() == 1, ""); solAssert(args.front()->annotation().type->category() == Type::Category::Bool, ""); - createErrorBlock(); + solAssert(m_currentContract, ""); + solAssert(m_currentFunction, ""); + if (m_currentFunction->isConstructor()) + m_functionAssertions[m_currentContract].insert(&_funCall); + else + m_functionAssertions[m_currentFunction].insert(&_funCall); + + auto previousError = m_error.currentValue(); + m_error.increaseIndex(); - smt::Expression assertNeg = !(m_context.expression(*args.front())->currentValue()); - connectBlocks(m_currentBlock, error(), currentPathConditions() && assertNeg); + connectBlocks( + m_currentBlock, + m_currentFunction->isConstructor() ? summary(*m_currentContract) : summary(*m_currentFunction), + currentPathConditions() && !m_context.expression(*args.front())->currentValue() && (m_error.currentValue() == _funCall.id()) + ); - m_verificationTargets.push_back(&_funCall); + m_context.addAssertion(m_error.currentValue() == previousError); } void CHC::unknownFunctionCall(FunctionCall const&) @@ -488,15 +537,23 @@ void CHC::unknownFunctionCall(FunctionCall const&) m_unknownFunctionCallSeen = true; } -void CHC::reset() +void CHC::resetSourceAnalysis() { - m_stateSorts.clear(); - m_stateVariables.clear(); m_verificationTargets.clear(); m_safeAssertions.clear(); + m_functionAssertions.clear(); + m_callGraph.clear(); + m_summaries.clear(); +} + +void CHC::resetContractAnalysis() +{ + m_stateSorts.clear(); + m_stateVariables.clear(); m_unknownFunctionCallSeen = false; m_breakDest = nullptr; m_continueDest = nullptr; + m_error.resetIndex(); } void CHC::eraseKnowledge() @@ -521,17 +578,6 @@ void CHC::clearIndices(ContractDefinition const* _contract, FunctionDefinition c } } - -bool CHC::shouldVisit(ContractDefinition const& _contract) const -{ - if ( - _contract.isLibrary() || - _contract.isInterface() - ) - return false; - return true; -} - bool CHC::shouldVisit(FunctionDefinition const& _function) const { if ( @@ -547,7 +593,8 @@ void CHC::setCurrentBlock( vector const* _arguments ) { - m_context.popSolver(); + if (m_context.solverStackHeigh() > 0) + m_context.popSolver(); solAssert(m_currentContract, ""); clearIndices(m_currentContract, m_currentFunction); m_context.pushSolver(); @@ -557,10 +604,42 @@ void CHC::setCurrentBlock( m_currentBlock = predicate(_block); } +set CHC::transactionAssertions(ASTNode const* _txRoot) +{ + set assertions; + solidity::util::BreadthFirstSearch{{_txRoot}}.run([&](auto const* function, auto&& _addChild) { + assertions.insert(m_functionAssertions[function].begin(), m_functionAssertions[function].end()); + for (auto const* called: m_callGraph[function]) + _addChild(called); + }); + return assertions; +} + +vector CHC::stateVariablesIncludingInheritedAndPrivate(ContractDefinition const& _contract) +{ + vector stateVars; + for (auto const& contract: _contract.annotation().linearizedBaseContracts) + for (auto var: contract->stateVariables()) + stateVars.push_back(var); + return stateVars; +} + +vector CHC::stateSorts(ContractDefinition const& _contract) +{ + vector stateSorts; + for (auto const& var: stateVariablesIncludingInheritedAndPrivate(_contract)) + stateSorts.push_back(smt::smtSortAbstractFunction(*var->type())); + return stateSorts; +} + smt::SortPointer CHC::constructorSort() { - // TODO this will change once we support function calls. - return interfaceSort(); + auto boolSort = make_shared(smt::Kind::Bool); + auto intSort = make_shared(smt::Kind::Int); + return make_shared( + vector{intSort} + m_stateSorts, + boolSort + ); } smt::SortPointer CHC::interfaceSort() @@ -572,20 +651,38 @@ smt::SortPointer CHC::interfaceSort() ); } +smt::SortPointer CHC::interfaceSort(ContractDefinition const& _contract) +{ + auto boolSort = make_shared(smt::Kind::Bool); + return make_shared( + stateSorts(_contract), + boolSort + ); +} + +/// A function in the symbolic CFG requires: +/// - Index of failed assertion. 0 means no assertion failed. +/// - 2 sets of state variables: +/// - State variables at the beginning of the current function, immutable +/// - Current state variables +/// At the beginning of the function these must equal set 1 +/// - 2 sets of input variables: +/// - Input variables at the beginning of the current function, immutable +/// - Current input variables +/// At the beginning of the function these must equal set 1 +/// - 1 set of output variables smt::SortPointer CHC::sort(FunctionDefinition const& _function) { auto boolSort = make_shared(smt::Kind::Bool); - vector varSorts; - for (auto const& var: _function.parameters() + _function.returnParameters()) - { - // SMT solvers do not support function types as arguments. - if (var->type()->category() == Type::Category::Function) - varSorts.push_back(make_shared(smt::Kind::Int)); - else - varSorts.push_back(smt::smtSort(*var->type())); - } + auto intSort = make_shared(smt::Kind::Int); + vector inputSorts; + for (auto const& var: _function.parameters()) + inputSorts.push_back(smt::smtSortAbstractFunction(*var->type())); + vector outputSorts; + for (auto const& var: _function.returnParameters()) + outputSorts.push_back(smt::smtSortAbstractFunction(*var->type())); return make_shared( - m_stateSorts + varSorts, + vector{intSort} + m_stateSorts + inputSorts + m_stateSorts + inputSorts + outputSorts, boolSort ); } @@ -601,19 +698,31 @@ smt::SortPointer CHC::sort(ASTNode const* _node) auto boolSort = make_shared(smt::Kind::Bool); vector varSorts; for (auto const& var: m_currentFunction->localVariables()) - { - // SMT solvers do not support function types as arguments. - if (var->type()->category() == Type::Category::Function) - varSorts.push_back(make_shared(smt::Kind::Int)); - else - varSorts.push_back(smt::smtSort(*var->type())); - } + varSorts.push_back(smt::smtSortAbstractFunction(*var->type())); return make_shared( fSort->domain + varSorts, boolSort ); } +smt::SortPointer CHC::summarySort(FunctionDefinition const& _function, ContractDefinition const& _contract) +{ + auto stateVariables = stateVariablesIncludingInheritedAndPrivate(_contract); + auto sorts = stateSorts(_contract); + + auto boolSort = make_shared(smt::Kind::Bool); + auto intSort = make_shared(smt::Kind::Int); + vector inputSorts, outputSorts; + for (auto const& var: _function.parameters()) + inputSorts.push_back(smt::smtSortAbstractFunction(*var->type())); + for (auto const& var: _function.returnParameters()) + outputSorts.push_back(smt::smtSortAbstractFunction(*var->type())); + return make_shared( + vector{intSort} + sorts + inputSorts + sorts + outputSorts, + boolSort + ); +} + unique_ptr CHC::createSymbolicBlock(smt::SortPointer _sort, string const& _name) { auto block = make_unique( @@ -625,12 +734,33 @@ unique_ptr CHC::createSymbolicBlock(smt::SortPoin return block; } +void CHC::defineInterfacesAndSummaries(SourceUnit const& _source) +{ + for (auto const& node: _source.nodes()) + if (auto const* contract = dynamic_cast(node.get())) + for (auto const* base: contract->annotation().linearizedBaseContracts) + { + string suffix = base->name() + "_" + to_string(base->id()); + m_interfaces[base] = createSymbolicBlock(interfaceSort(*base), "interface_" + suffix); + for (auto const* var: stateVariablesIncludingInheritedAndPrivate(*base)) + if (!m_context.knownVariable(*var)) + createVariable(*var); + for (auto const* function: base->definedFunctions()) + m_summaries[contract].emplace(function, createSummaryBlock(*function, *contract)); + } +} + smt::Expression CHC::interface() { vector paramExprs; for (auto const& var: m_stateVariables) paramExprs.push_back(m_context.variable(*var)->currentValue()); - return (*m_interfacePredicate)(paramExprs); + return (*m_interfaces.at(m_currentContract))(paramExprs); +} + +smt::Expression CHC::interface(ContractDefinition const& _contract) +{ + return (*m_interfaces.at(&_contract))(stateVariablesAtIndex(0, _contract)); } smt::Expression CHC::error() @@ -643,6 +773,27 @@ smt::Expression CHC::error(unsigned _idx) return m_errorPredicate->functionValueAtIndex(_idx)({}); } +smt::Expression CHC::summary(ContractDefinition const&) +{ + return (*m_constructorSummaryPredicate)( + vector{m_error.currentValue()} + + currentStateVariables() + ); +} + +smt::Expression CHC::summary(FunctionDefinition const& _function) +{ + vector args{m_error.currentValue()}; + auto contract = _function.annotation().contract; + args += contract->isLibrary() ? stateVariablesAtIndex(0, *contract) : initialStateVariables(); + for (auto const& var: _function.parameters()) + args.push_back(m_context.variable(*var)->valueAtIndex(0)); + args += contract->isLibrary() ? stateVariablesAtIndex(1, *contract) : currentStateVariables(); + for (auto const& var: _function.returnParameters()) + args.push_back(m_context.variable(*var)->currentValue()); + return (*m_summaries.at(m_currentContract).at(&_function))(args); +} + unique_ptr CHC::createBlock(ASTNode const* _node, string const& _prefix) { return createSymbolicBlock(sort(_node), @@ -653,6 +804,15 @@ unique_ptr CHC::createBlock(ASTNode const* _node, predicateName(_node)); } +unique_ptr CHC::createSummaryBlock(FunctionDefinition const& _function, ContractDefinition const& _contract) +{ + return createSymbolicBlock(summarySort(_function, _contract), + "summary_" + + uniquePrefix() + + "_" + + predicateName(&_function, &_contract)); +} + void CHC::createErrorBlock() { solAssert(m_errorPredicate, ""); @@ -669,6 +829,28 @@ void CHC::connectBlocks(smt::Expression const& _from, smt::Expression const& _to addRule(edge, _from.name + "_to_" + _to.name); } +vector CHC::initialStateVariables() +{ + return stateVariablesAtIndex(0); +} + +vector CHC::stateVariablesAtIndex(int _index) +{ + solAssert(m_currentContract, ""); + vector exprs; + for (auto const& var: m_stateVariables) + exprs.push_back(m_context.variable(*var)->valueAtIndex(_index)); + return exprs; +} + +vector CHC::stateVariablesAtIndex(int _index, ContractDefinition const& _contract) +{ + vector exprs; + for (auto const& var: stateVariablesIncludingInheritedAndPrivate(_contract)) + exprs.push_back(m_context.variable(*var)->valueAtIndex(_index)); + return exprs; +} + vector CHC::currentStateVariables() { solAssert(m_currentContract, ""); @@ -680,11 +862,22 @@ vector CHC::currentStateVariables() vector CHC::currentFunctionVariables() { - vector paramExprs; - if (m_currentFunction) - for (auto const& var: m_currentFunction->parameters() + m_currentFunction->returnParameters()) - paramExprs.push_back(m_context.variable(*var)->currentValue()); - return currentStateVariables() + paramExprs; + vector initInputExprs; + vector mutableInputExprs; + for (auto const& var: m_currentFunction->parameters()) + { + initInputExprs.push_back(m_context.variable(*var)->valueAtIndex(0)); + mutableInputExprs.push_back(m_context.variable(*var)->currentValue()); + } + vector returnExprs; + for (auto const& var: m_currentFunction->returnParameters()) + returnExprs.push_back(m_context.variable(*var)->currentValue()); + return vector{m_error.currentValue()} + + initialStateVariables() + + initInputExprs + + currentStateVariables() + + mutableInputExprs + + returnExprs; } vector CHC::currentBlockVariables() @@ -696,7 +889,7 @@ vector CHC::currentBlockVariables() return currentFunctionVariables() + paramExprs; } -string CHC::predicateName(ASTNode const* _node) +string CHC::predicateName(ASTNode const* _node, ContractDefinition const* _contract) { string prefix; if (auto funDef = dynamic_cast(_node)) @@ -705,7 +898,12 @@ string CHC::predicateName(ASTNode const* _node) if (!funDef->name().empty()) prefix += "_" + funDef->name() + "_"; } - return prefix + to_string(_node->id()); + else if (m_currentFunction && !m_currentFunction->name().empty()) + prefix += m_currentFunction->name(); + + auto contract = _contract ? _contract : m_currentContract; + solAssert(contract, ""); + return prefix + "_" + to_string(_node->id()) + "_" + to_string(contract->id()); } smt::Expression CHC::predicate(smt::SymbolicFunctionVariable const& _block) @@ -726,7 +924,7 @@ void CHC::addRule(smt::Expression const& _rule, string const& _ruleName) m_interface->addRule(_rule, _ruleName); } -bool CHC::query(smt::Expression const& _query, langutil::SourceLocation const& _location) +pair> CHC::query(smt::Expression const& _query, langutil::SourceLocation const& _location) { smt::CheckResult result; vector values; @@ -736,7 +934,7 @@ bool CHC::query(smt::Expression const& _query, langutil::SourceLocation const& _ case smt::CheckResult::SATISFIABLE: break; case smt::CheckResult::UNSATISFIABLE: - return true; + break; case smt::CheckResult::UNKNOWN: break; case smt::CheckResult::CONFLICTING: @@ -746,7 +944,12 @@ bool CHC::query(smt::Expression const& _query, langutil::SourceLocation const& _ m_outerErrorReporter.warning(_location, "Error trying to invoke SMT solver."); break; } - return false; + return {result, values}; +} + +void CHC::addVerificationTarget(ASTNode const* _scope, smt::Expression _from, smt::Expression _constraints, smt::Expression _errorId) +{ + m_verificationTargets.emplace(_scope, CHCVerificationTarget{{VerificationTarget::Type::Assert, _from, _constraints}, _errorId}); } string CHC::uniquePrefix() diff --git a/libsolidity/formal/CHC.h b/libsolidity/formal/CHC.h index 6f912fbd87b3..889c21b6aa35 100644 --- a/libsolidity/formal/CHC.h +++ b/libsolidity/formal/CHC.h @@ -79,22 +79,38 @@ class CHC: public SMTEncoder void unknownFunctionCall(FunctionCall const& _funCall); //@} + struct IdCompare + { + bool operator()(ASTNode const* lhs, ASTNode const* rhs) const + { + return lhs->id() < rhs->id(); + } + }; + /// Helpers. //@{ - void reset(); + void resetSourceAnalysis(); + void resetContractAnalysis(); void eraseKnowledge(); void clearIndices(ContractDefinition const* _contract, FunctionDefinition const* _function = nullptr) override; - bool shouldVisit(ContractDefinition const& _contract) const; bool shouldVisit(FunctionDefinition const& _function) const; void setCurrentBlock(smt::SymbolicFunctionVariable const& _block, std::vector const* _arguments = nullptr); + std::set transactionAssertions(ASTNode const* _txRoot); + static std::vector stateVariablesIncludingInheritedAndPrivate(ContractDefinition const& _contract); //@} /// Sort helpers. //@{ + static std::vector stateSorts(ContractDefinition const& _contract); smt::SortPointer constructorSort(); smt::SortPointer interfaceSort(); + static smt::SortPointer interfaceSort(ContractDefinition const& _const); smt::SortPointer sort(FunctionDefinition const& _function); smt::SortPointer sort(ASTNode const* _block); + /// @returns the sort of a predicate that represents the summary of _function in the scope of _contract. + /// The _contract is also needed because the same function might be in many contracts due to inheritance, + /// where the sort changes because the set of state variables might change. + static smt::SortPointer summarySort(FunctionDefinition const& _function, ContractDefinition const& _contract); //@} /// Predicate helpers. @@ -102,14 +118,24 @@ class CHC: public SMTEncoder /// @returns a new block of given _sort and _name. std::unique_ptr createSymbolicBlock(smt::SortPointer _sort, std::string const& _name); + /// Creates summary predicates for all functions of all contracts + /// in a given _source. + void defineInterfacesAndSummaries(SourceUnit const& _source); + + /// Genesis predicate. + smt::Expression genesis() { return (*m_genesisPredicate)({}); } /// Interface predicate over current variables. smt::Expression interface(); + smt::Expression interface(ContractDefinition const& _contract); /// Error predicate over current variables. smt::Expression error(); smt::Expression error(unsigned _idx); /// Creates a block for the given _node. std::unique_ptr createBlock(ASTNode const* _node, std::string const& _prefix = ""); + /// Creates a call block for the given function _function from contract _contract. + /// The contract is needed here because of inheritance. + std::unique_ptr createSummaryBlock(FunctionDefinition const& _function, ContractDefinition const& _contract); /// Creates a new error block to be used by an assertion. /// Also registers the predicate. @@ -117,6 +143,11 @@ class CHC: public SMTEncoder void connectBlocks(smt::Expression const& _from, smt::Expression const& _to, smt::Expression const& _constraints = smt::Expression(true)); + /// @returns the symbolic values of the state variables at the beginning + /// of the current transaction. + std::vector initialStateVariables(); + std::vector stateVariablesAtIndex(int _index); + std::vector stateVariablesAtIndex(int _index, ContractDefinition const& _contract); /// @returns the current symbolic values of the current state variables. std::vector currentStateVariables(); @@ -128,19 +159,26 @@ class CHC: public SMTEncoder std::vector currentBlockVariables(); /// @returns the predicate name for a given node. - std::string predicateName(ASTNode const* _node); + std::string predicateName(ASTNode const* _node, ContractDefinition const* _contract = nullptr); /// @returns a predicate application over the current scoped variables. smt::Expression predicate(smt::SymbolicFunctionVariable const& _block); /// @returns a predicate application over @param _arguments. smt::Expression predicate(smt::SymbolicFunctionVariable const& _block, std::vector const& _arguments); + /// @returns a predicate that defines a constructor summary. + smt::Expression summary(ContractDefinition const& _contract); + /// @returns a predicate that defines a function summary. + smt::Expression summary(FunctionDefinition const& _function); //@} /// Solver related. //@{ /// Adds Horn rule to the solver. void addRule(smt::Expression const& _rule, std::string const& _ruleName); - /// @returns true if query is unsatisfiable (safe). - bool query(smt::Expression const& _query, langutil::SourceLocation const& _location); + /// @returns if query is unsatisfiable (safe). + /// @returns otherwise. + std::pair> query(smt::Expression const& _query, langutil::SourceLocation const& _location); + + void addVerificationTarget(ASTNode const* _scope, smt::Expression _from, smt::Expression _constraints, smt::Expression _errorId); //@} /// Misc. @@ -157,15 +195,29 @@ class CHC: public SMTEncoder /// Implicit constructor predicate. /// Explicit constructors are handled as functions. - std::unique_ptr m_constructorPredicate; + std::unique_ptr m_implicitConstructorPredicate; + + /// Constructor summary predicate, exists after the constructor + /// (implicit or explicit) and before the interface. + std::unique_ptr m_constructorSummaryPredicate; /// Artificial Interface predicate. /// Single entry block for all functions. - std::unique_ptr m_interfacePredicate; + std::map> m_interfaces; /// Artificial Error predicate. /// Single error block for all assertions. std::unique_ptr m_errorPredicate; + + /// Function predicates. + std::map>> m_summaries; + + smt::SymbolicIntVariable m_error{ + TypeProvider::uint256(), + TypeProvider::uint256(), + "error", + m_context + }; //@} /// Variables. @@ -180,7 +232,12 @@ class CHC: public SMTEncoder /// Verification targets. //@{ - std::vector m_verificationTargets; + struct CHCVerificationTarget: VerificationTarget + { + smt::Expression errorId; + }; + + std::map m_verificationTargets; /// Assertions proven safe. std::set m_safeAssertions; @@ -190,6 +247,10 @@ class CHC: public SMTEncoder //@{ FunctionDefinition const* m_currentFunction = nullptr; + std::map, IdCompare> m_callGraph; + + std::map, IdCompare> m_functionAssertions; + /// The current block. smt::Expression m_currentBlock = smt::Expression(true); diff --git a/libsolidity/formal/EncodingContext.h b/libsolidity/formal/EncodingContext.h index 9b75065f6617..648f7dc33d2e 100644 --- a/libsolidity/formal/EncodingContext.h +++ b/libsolidity/formal/EncodingContext.h @@ -140,6 +140,7 @@ class EncodingContext void pushSolver(); void popSolver(); void addAssertion(Expression const& _e); + unsigned solverStackHeigh() { return m_assertions.size(); } const SolverInterface* solver() { solAssert(m_solver, ""); diff --git a/libsolidity/formal/ModelChecker.cpp b/libsolidity/formal/ModelChecker.cpp index c69849a06b48..ebc1a875131c 100644 --- a/libsolidity/formal/ModelChecker.cpp +++ b/libsolidity/formal/ModelChecker.cpp @@ -29,9 +29,9 @@ ModelChecker::ModelChecker( ReadCallback::Callback const& _smtCallback, smt::SMTSolverChoice _enabledSolvers ): + m_context(), m_bmc(m_context, _errorReporter, _smtlib2Responses, _smtCallback, _enabledSolvers), - m_chc(m_context, _errorReporter, _smtlib2Responses, _smtCallback, _enabledSolvers), - m_context() + m_chc(m_context, _errorReporter, _smtlib2Responses, _smtCallback, _enabledSolvers) { } diff --git a/libsolidity/formal/ModelChecker.h b/libsolidity/formal/ModelChecker.h index ac19ba2706c9..546f2df86e97 100644 --- a/libsolidity/formal/ModelChecker.h +++ b/libsolidity/formal/ModelChecker.h @@ -62,14 +62,14 @@ class ModelChecker static smt::SMTSolverChoice availableSolvers(); private: + /// Stores the context of the encoding. + smt::EncodingContext m_context; + /// Bounded Model Checker engine. BMC m_bmc; /// Constrained Horn Clauses engine. CHC m_chc; - - /// Stores the context of the encoding. - smt::EncodingContext m_context; }; } diff --git a/libsolidity/formal/Z3Interface.h b/libsolidity/formal/Z3Interface.h index a6247a541fc9..3791af4c55ab 100644 --- a/libsolidity/formal/Z3Interface.h +++ b/libsolidity/formal/Z3Interface.h @@ -58,11 +58,11 @@ class Z3Interface: public SolverInterface, public boost::noncopyable z3::sort z3Sort(smt::Sort const& _sort); z3::sort_vector z3Sort(std::vector const& _sorts); - std::map m_constants; - std::map m_functions; - z3::context m_context; z3::solver m_solver; + + std::map m_constants; + std::map m_functions; }; } diff --git a/test/libsolidity/smtCheckerTests/functions/library_constant.sol b/test/libsolidity/smtCheckerTests/functions/library_constant.sol new file mode 100644 index 000000000000..e32b8565b29b --- /dev/null +++ b/test/libsolidity/smtCheckerTests/functions/library_constant.sol @@ -0,0 +1,26 @@ +pragma experimental SMTChecker; + +library l1 { + + uint private constant TON = 1000; + function f1() public pure { + assert(TON == 1000); + assert(TON == 2000); + } + function f2(uint x, uint y) internal pure returns (uint) { + return x + y; + } +} + +contract C { + function f(uint x) public pure { + uint z = l1.f2(x, 1); + assert(z == x + 1); + } +} +// ---- +// Warning: (136-155): Assertion violation happens here +// Warning: (229-234): Overflow (resulting value larger than 2**256 - 1) happens here +// Warning: (300-302): Assertion checker does not yet implement type type(library l1) +// Warning: (229-234): Overflow (resulting value larger than 2**256 - 1) happens here +// Warning: (327-332): Overflow (resulting value larger than 2**256 - 1) happens here diff --git a/test/libsolidity/smtCheckerTests/functions/library_constant_2.sol b/test/libsolidity/smtCheckerTests/functions/library_constant_2.sol new file mode 100644 index 000000000000..9c04e6f9ab03 --- /dev/null +++ b/test/libsolidity/smtCheckerTests/functions/library_constant_2.sol @@ -0,0 +1,9 @@ +pragma experimental SMTChecker; + +library l1 { + + uint private constant TON = 1000; + function f1() public pure { + assert(TON == 1000); + } +} diff --git a/test/libsolidity/smtCheckerTests/invariants/loop_nested.sol b/test/libsolidity/smtCheckerTests/invariants/loop_nested.sol index 867403b61482..1fb380649f38 100644 --- a/test/libsolidity/smtCheckerTests/invariants/loop_nested.sol +++ b/test/libsolidity/smtCheckerTests/invariants/loop_nested.sol @@ -17,3 +17,8 @@ contract Simple { } // ==== // SMTSolvers: z3 +// ---- +// Warning: (172-187): Error trying to invoke SMT solver. +// Warning: (195-209): Error trying to invoke SMT solver. +// Warning: (172-187): Assertion violation happens here +// Warning: (195-209): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/invariants/loop_nested_for.sol b/test/libsolidity/smtCheckerTests/invariants/loop_nested_for.sol index 9cfd4569fd8a..e7fde4d5aa28 100644 --- a/test/libsolidity/smtCheckerTests/invariants/loop_nested_for.sol +++ b/test/libsolidity/smtCheckerTests/invariants/loop_nested_for.sol @@ -14,3 +14,8 @@ contract Simple { } // ==== // SMTSolvers: z3 +// ---- +// Warning: (164-179): Error trying to invoke SMT solver. +// Warning: (187-201): Error trying to invoke SMT solver. +// Warning: (164-179): Assertion violation happens here +// Warning: (187-201): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_memory.sol b/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_memory.sol index 9e7a83866831..0b301505b4b0 100644 --- a/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_memory.sol +++ b/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_memory.sol @@ -12,12 +12,12 @@ contract LoopFor2 { b[i] = i + 1; c[i] = b[i]; } + // This is safe but too hard to solve currently. assert(b[0] == c[0]); assert(a[0] == 900); assert(b[0] == 900); } } -// ==== -// SMTSolvers: z3 // ---- -// Warning: (312-331): Assertion violation happens here +// Warning: (316-336): Assertion violation happens here +// Warning: (363-382): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_storage.sol b/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_storage.sol index 8146a7f19634..333273781373 100644 --- a/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_storage.sol +++ b/test/libsolidity/smtCheckerTests/loops/for_loop_array_assignment_storage_storage.sol @@ -12,13 +12,13 @@ contract LoopFor2 { b[i] = i + 1; c[i] = b[i]; } + // This is safe but too hard to prove currently. assert(b[0] == c[0]); assert(a[0] == 900); assert(b[0] == 900); } } -// ==== -// SMTSolvers: z3 // ---- -// Warning: (290-309): Assertion violation happens here -// Warning: (313-332): Assertion violation happens here +// Warning: (317-337): Assertion violation happens here +// Warning: (341-360): Assertion violation happens here +// Warning: (364-383): Assertion violation happens here diff --git a/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol b/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol index 06ef66be96c6..9cd7de9e67ba 100644 --- a/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol +++ b/test/libsolidity/smtCheckerTests/operators/delete_array_index_2d.sol @@ -13,5 +13,5 @@ contract C assert(a[1][1] == 0); } } -// ==== -// SMTSolvers: z3 +// ---- +// Warning: (184-204): Assertion violation happens here From 24d6e6295e75ee9da587ced7471df9e89713b21b Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Mon, 2 Mar 2020 17:20:49 +0100 Subject: [PATCH 69/92] Reuse the mechanism for abi functions and move tracking of used functions to CompilerContext --- libsolidity/codegen/ABIFunctions.cpp | 11 +++-------- libsolidity/codegen/ABIFunctions.h | 6 ------ libsolidity/codegen/CompilerContext.cpp | 9 ++++++++- libsolidity/codegen/CompilerContext.h | 11 +++++++---- libsolidity/codegen/CompilerUtils.cpp | 16 +++------------- .../codegen/MultiUseYulFunctionCollector.cpp | 12 ++---------- .../codegen/MultiUseYulFunctionCollector.h | 19 ++----------------- libsolidity/codegen/ir/IRGenerator.cpp | 6 +++--- 8 files changed, 28 insertions(+), 62 deletions(-) diff --git a/libsolidity/codegen/ABIFunctions.cpp b/libsolidity/codegen/ABIFunctions.cpp index b69f47346b5d..b6d690898100 100644 --- a/libsolidity/codegen/ABIFunctions.cpp +++ b/libsolidity/codegen/ABIFunctions.cpp @@ -55,7 +55,7 @@ string ABIFunctions::tupleEncoder( functionName += t->identifier() + "_"; functionName += options.toFunctionNameSuffix(); - return createExternallyUsedFunction(functionName, [&]() { + return createFunction(functionName, [&]() { // Note that the values are in reverse due to the difference in calling semantics. Whiskers templ(R"( function (headStart ) -> tail { @@ -121,7 +121,7 @@ string ABIFunctions::tupleEncoderPacked( functionName += t->identifier() + "_"; functionName += options.toFunctionNameSuffix(); - return createExternallyUsedFunction(functionName, [&]() { + return createFunction(functionName, [&]() { solAssert(!_givenTypes.empty(), ""); // Note that the values are in reverse due to the difference in calling semantics. @@ -173,7 +173,7 @@ string ABIFunctions::tupleDecoder(TypePointers const& _types, bool _fromMemory) if (_fromMemory) functionName += "_fromMemory"; - return createExternallyUsedFunction(functionName, [&]() { + return createFunction(functionName, [&]() { TypePointers decodingTypes; for (auto const& t: _types) decodingTypes.emplace_back(t->decodingType()); @@ -1495,11 +1495,6 @@ string ABIFunctions::createFunction(string const& _name, function con return m_functionCollector->createFunction(_name, _creator); } -string ABIFunctions::createExternallyUsedFunction(string const& _name, function const& _creator) -{ - return m_functionCollector->createExternallyUsedFunction(_name, _creator); -} - size_t ABIFunctions::headSize(TypePointers const& _targetTypes) { size_t headSize = 0; diff --git a/libsolidity/codegen/ABIFunctions.h b/libsolidity/codegen/ABIFunctions.h index 74bfdf7fa11e..1099e3593fe7 100644 --- a/libsolidity/codegen/ABIFunctions.h +++ b/libsolidity/codegen/ABIFunctions.h @@ -31,7 +31,6 @@ #include #include -#include #include namespace solidity::frontend @@ -233,11 +232,6 @@ class ABIFunctions /// cases. std::string createFunction(std::string const& _name, std::function const& _creator); - /// Helper function that uses @a _creator to create a function and add it to - /// @a m_requestedFunctions if it has not been created yet and returns @a _name in both - /// cases. Also adds it to the list of externally used functions. - std::string createExternallyUsedFunction(std::string const& _name, std::function const& _creator); - /// @returns the size of the static part of the encoding of the given types. static size_t headSize(TypePointers const& _targetTypes); diff --git a/libsolidity/codegen/CompilerContext.cpp b/libsolidity/codegen/CompilerContext.cpp index f65ccae52e7e..ee12b9b3e90b 100644 --- a/libsolidity/codegen/CompilerContext.cpp +++ b/libsolidity/codegen/CompilerContext.cpp @@ -100,7 +100,7 @@ void CompilerContext::callYulUtilFunction( unsigned _outArgs ) { - m_functionCollector->markAsExternallyUsed(_name); + m_externallyUsedFunctions.insert(_name); auto retTag = pushNewTag(); CompilerUtils(*this).moveIntoStack(_inArgs); appendJumpTo(namedTag(_name)); @@ -147,6 +147,13 @@ void CompilerContext::appendMissingLowLevelFunctions() } } +pair> CompilerContext::requestedYulFunctions() +{ + set empty; + swap(empty, m_externallyUsedFunctions); + return make_pair(m_functionCollector->requestedFunctions(), std::move(empty)); +} + void CompilerContext::addVariable( VariableDeclaration const& _declaration, unsigned _offsetToCurrent diff --git a/libsolidity/codegen/CompilerContext.h b/libsolidity/codegen/CompilerContext.h index d9393e23ad3c..6d554da825c3 100644 --- a/libsolidity/codegen/CompilerContext.h +++ b/libsolidity/codegen/CompilerContext.h @@ -154,10 +154,11 @@ class CompilerContext void appendMissingLowLevelFunctions(); ABIFunctions& abiFunctions() { return m_abiFunctions; } YulUtilFunctions& utilFunctions() { return m_yulUtilFunctions; } - std::pair> requestedYulFunctions() - { - return m_functionCollector->requestedFunctions(); - } + /// @returns concatenation of all generated functions and a set of the + /// externally used functions. + /// Clears the internal list, i.e. calling it again will result in an + /// empty return value. + std::pair> requestedYulFunctions(); ModifierDefinition const& resolveVirtualFunctionModifier(ModifierDefinition const& _modifier) const; /// Returns the distance of the given local variable from the bottom of the stack (of the current function). @@ -371,6 +372,8 @@ class CompilerContext std::map m_lowLevelFunctions; // Collector for yul functions. std::shared_ptr m_functionCollector = std::make_shared(); + /// Set of externally used yul functions. + std::set m_externallyUsedFunctions; /// Container for ABI functions to be generated. ABIFunctions m_abiFunctions; /// Container for Yul Util functions to be generated. diff --git a/libsolidity/codegen/CompilerUtils.cpp b/libsolidity/codegen/CompilerUtils.cpp index 4c75081ffbc6..a0d38eae55e0 100644 --- a/libsolidity/codegen/CompilerUtils.cpp +++ b/libsolidity/codegen/CompilerUtils.cpp @@ -595,31 +595,21 @@ void CompilerUtils::abiEncodeV2( // stack: <$value0> <$value1> ... <$value(n-1)> <$headStart> - auto ret = m_context.pushNewTag(); - moveIntoStack(sizeOnStack(_givenTypes) + 1); - string encoderName = _padToWordBoundaries ? m_context.abiFunctions().tupleEncoder(_givenTypes, _targetTypes, _encodeAsLibraryTypes) : m_context.abiFunctions().tupleEncoderPacked(_givenTypes, _targetTypes); - m_context.appendJumpTo(m_context.namedTag(encoderName)); - m_context.adjustStackOffset(-int(sizeOnStack(_givenTypes)) - 1); - m_context << ret.tag(); + m_context.callYulUtilFunction(encoderName, sizeOnStack(_givenTypes) + 1, 1); } void CompilerUtils::abiDecodeV2(TypePointers const& _parameterTypes, bool _fromMemory) { // stack: [stack top] - auto ret = m_context.pushNewTag(); - moveIntoStack(2); - // stack: [stack top] m_context << Instruction::DUP2 << Instruction::ADD; m_context << Instruction::SWAP1; - // stack: + // stack: string decoderName = m_context.abiFunctions().tupleDecoder(_parameterTypes, _fromMemory); - m_context.appendJumpTo(m_context.namedTag(decoderName)); - m_context.adjustStackOffset(int(sizeOnStack(_parameterTypes)) - 3); - m_context << ret.tag(); + m_context.callYulUtilFunction(decoderName, 2, sizeOnStack(_parameterTypes)); } void CompilerUtils::zeroInitialiseMemoryArray(ArrayType const& _type) diff --git a/libsolidity/codegen/MultiUseYulFunctionCollector.cpp b/libsolidity/codegen/MultiUseYulFunctionCollector.cpp index 8af292eda99e..2be3d29ae6b3 100644 --- a/libsolidity/codegen/MultiUseYulFunctionCollector.cpp +++ b/libsolidity/codegen/MultiUseYulFunctionCollector.cpp @@ -30,15 +30,13 @@ using namespace std; using namespace solidity; using namespace solidity::frontend; -pair> MultiUseYulFunctionCollector::requestedFunctions() +string MultiUseYulFunctionCollector::requestedFunctions() { string result; for (auto const& f: m_requestedFunctions) result += f.second; m_requestedFunctions.clear(); - std::set empty; - swap(empty, m_externallyUsedFunctions); - return make_pair(result, std::move(empty)); + return result; } string MultiUseYulFunctionCollector::createFunction(string const& _name, function const& _creator) @@ -52,9 +50,3 @@ string MultiUseYulFunctionCollector::createFunction(string const& _name, functio } return _name; } - -string MultiUseYulFunctionCollector::createExternallyUsedFunction(string const& _name, function const& _creator) -{ - m_externallyUsedFunctions.insert(_name); - return createFunction(_name, _creator); -} diff --git a/libsolidity/codegen/MultiUseYulFunctionCollector.h b/libsolidity/codegen/MultiUseYulFunctionCollector.h index a1c733274b21..d839a31be873 100644 --- a/libsolidity/codegen/MultiUseYulFunctionCollector.h +++ b/libsolidity/codegen/MultiUseYulFunctionCollector.h @@ -23,7 +23,6 @@ #include #include -#include #include namespace solidity::frontend @@ -41,28 +40,14 @@ class MultiUseYulFunctionCollector /// cases. std::string createFunction(std::string const& _name, std::function const& _creator); - /// Helper function that uses @a _creator to create a function and add it to - /// @a m_requestedFunctions if it has not been created yet and returns @a _name in both - /// cases. - std::string createExternallyUsedFunction(std::string const& _name, std::function const& _creator); - - /// Manually mark a function as externally used. - void markAsExternallyUsed(std::string const& _name) - { - m_externallyUsedFunctions.insert(_name); - } - - /// @returns concatenation of all generated functions and a set of the - /// externally used functions. + /// @returns concatenation of all generated functions. /// Clears the internal list, i.e. calling it again will result in an /// empty return value. - std::pair> requestedFunctions(); + std::string requestedFunctions(); private: /// Map from function name to code for a multi-use function. std::map m_requestedFunctions; - // Set of externally used functions. - std::set m_externallyUsedFunctions; }; } diff --git a/libsolidity/codegen/ir/IRGenerator.cpp b/libsolidity/codegen/ir/IRGenerator.cpp index 0f2ffaf35b95..9c2d8807c88b 100644 --- a/libsolidity/codegen/ir/IRGenerator.cpp +++ b/libsolidity/codegen/ir/IRGenerator.cpp @@ -107,7 +107,7 @@ string IRGenerator::generate(ContractDefinition const& _contract) for (auto const* contract: _contract.annotation().linearizedBaseContracts) for (auto const* fun: contract->definedFunctions()) generateFunction(*fun); - t("functions", m_context.functionCollector()->requestedFunctions().first); + t("functions", m_context.functionCollector()->requestedFunctions()); resetContext(_contract); m_context.setInheritanceHierarchy(_contract.annotation().linearizedBaseContracts); @@ -116,7 +116,7 @@ string IRGenerator::generate(ContractDefinition const& _contract) for (auto const* contract: _contract.annotation().linearizedBaseContracts) for (auto const* fun: contract->definedFunctions()) generateFunction(*fun); - t("runtimeFunctions", m_context.functionCollector()->requestedFunctions().first); + t("runtimeFunctions", m_context.functionCollector()->requestedFunctions()); return t.render(); } @@ -383,7 +383,7 @@ string IRGenerator::memoryInit() void IRGenerator::resetContext(ContractDefinition const& _contract) { solAssert( - m_context.functionCollector()->requestedFunctions().first.empty(), + m_context.functionCollector()->requestedFunctions().empty(), "Reset context while it still had functions." ); m_context = IRGenerationContext(m_evmVersion, m_context.revertStrings(), m_optimiserSettings); From 561e5d9b27439e56cbd136abecfde5727594155b Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Mon, 2 Mar 2020 17:23:58 +0100 Subject: [PATCH 70/92] Rename variables and review suggestion. --- libsolidity/codegen/CompilerContext.cpp | 10 +++++----- libsolidity/codegen/CompilerContext.h | 14 +++++++------- libsolidity/codegen/CompilerUtils.cpp | 4 ++-- libsolidity/codegen/ContractCompiler.cpp | 8 ++++---- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/libsolidity/codegen/CompilerContext.cpp b/libsolidity/codegen/CompilerContext.cpp index ee12b9b3e90b..1d1eb92b9cbc 100644 --- a/libsolidity/codegen/CompilerContext.cpp +++ b/libsolidity/codegen/CompilerContext.cpp @@ -94,14 +94,14 @@ void CompilerContext::callLowLevelFunction( *this << retTag.tag(); } -void CompilerContext::callYulUtilFunction( +void CompilerContext::callYulFunction( string const& _name, unsigned _inArgs, unsigned _outArgs ) { - m_externallyUsedFunctions.insert(_name); - auto retTag = pushNewTag(); + m_externallyUsedYulFunctions.insert(_name); + auto const retTag = pushNewTag(); CompilerUtils(*this).moveIntoStack(_inArgs); appendJumpTo(namedTag(_name)); adjustStackOffset(int(_outArgs) - 1 - _inArgs); @@ -150,8 +150,8 @@ void CompilerContext::appendMissingLowLevelFunctions() pair> CompilerContext::requestedYulFunctions() { set empty; - swap(empty, m_externallyUsedFunctions); - return make_pair(m_functionCollector->requestedFunctions(), std::move(empty)); + swap(empty, m_externallyUsedYulFunctions); + return make_pair(m_yulFunctionCollector->requestedFunctions(), std::move(empty)); } void CompilerContext::addVariable( diff --git a/libsolidity/codegen/CompilerContext.h b/libsolidity/codegen/CompilerContext.h index 6d554da825c3..76c72770d6b5 100644 --- a/libsolidity/codegen/CompilerContext.h +++ b/libsolidity/codegen/CompilerContext.h @@ -65,8 +65,8 @@ class CompilerContext m_evmVersion(_evmVersion), m_revertStrings(_revertStrings), m_runtimeContext(_runtimeContext), - m_abiFunctions(m_evmVersion, m_revertStrings, m_functionCollector), - m_yulUtilFunctions(m_evmVersion, m_revertStrings, m_functionCollector) + m_abiFunctions(m_evmVersion, m_revertStrings, m_yulFunctionCollector), + m_yulUtilFunctions(m_evmVersion, m_revertStrings, m_yulFunctionCollector) { if (m_runtimeContext) m_runtimeSub = size_t(m_asm->newSub(m_runtimeContext->m_asm).data()); @@ -133,8 +133,8 @@ class CompilerContext std::function const& _generator ); - /// Appends a call to a yul util function and registers the function as externally used. - void callYulUtilFunction( + /// Appends a call to a yul function and registers the function as externally used. + void callYulFunction( std::string const& _name, unsigned _inArgs, unsigned _outArgs @@ -370,10 +370,10 @@ class CompilerContext size_t m_runtimeSub = -1; /// An index of low-level function labels by name. std::map m_lowLevelFunctions; - // Collector for yul functions. - std::shared_ptr m_functionCollector = std::make_shared(); + /// Collector for yul functions. + std::shared_ptr m_yulFunctionCollector = std::make_shared(); /// Set of externally used yul functions. - std::set m_externallyUsedFunctions; + std::set m_externallyUsedYulFunctions; /// Container for ABI functions to be generated. ABIFunctions m_abiFunctions; /// Container for Yul Util functions to be generated. diff --git a/libsolidity/codegen/CompilerUtils.cpp b/libsolidity/codegen/CompilerUtils.cpp index a0d38eae55e0..b39633377052 100644 --- a/libsolidity/codegen/CompilerUtils.cpp +++ b/libsolidity/codegen/CompilerUtils.cpp @@ -599,7 +599,7 @@ void CompilerUtils::abiEncodeV2( _padToWordBoundaries ? m_context.abiFunctions().tupleEncoder(_givenTypes, _targetTypes, _encodeAsLibraryTypes) : m_context.abiFunctions().tupleEncoderPacked(_givenTypes, _targetTypes); - m_context.callYulUtilFunction(encoderName, sizeOnStack(_givenTypes) + 1, 1); + m_context.callYulFunction(encoderName, sizeOnStack(_givenTypes) + 1, 1); } void CompilerUtils::abiDecodeV2(TypePointers const& _parameterTypes, bool _fromMemory) @@ -609,7 +609,7 @@ void CompilerUtils::abiDecodeV2(TypePointers const& _parameterTypes, bool _fromM m_context << Instruction::SWAP1; // stack: string decoderName = m_context.abiFunctions().tupleDecoder(_parameterTypes, _fromMemory); - m_context.callYulUtilFunction(decoderName, 2, sizeOnStack(_parameterTypes)); + m_context.callYulFunction(decoderName, 2, sizeOnStack(_parameterTypes)); } void CompilerUtils::zeroInitialiseMemoryArray(ArrayType const& _type) diff --git a/libsolidity/codegen/ContractCompiler.cpp b/libsolidity/codegen/ContractCompiler.cpp index 47187b1c88ff..8b9061af3d2a 100644 --- a/libsolidity/codegen/ContractCompiler.cpp +++ b/libsolidity/codegen/ContractCompiler.cpp @@ -1267,12 +1267,12 @@ void ContractCompiler::appendMissingFunctions() solAssert(m_context.nextFunctionToCompile() != function, "Compiled the wrong function?"); } m_context.appendMissingLowLevelFunctions(); - auto yulFunctions = m_context.requestedYulFunctions(); - if (!yulFunctions.first.empty()) + auto [yulFunctions, externallyUsedYulFunctions] = m_context.requestedYulFunctions(); + if (!yulFunctions.empty()) m_context.appendInlineAssembly( - "{" + move(yulFunctions.first) + "}", + "{" + move(yulFunctions) + "}", {}, - yulFunctions.second, + externallyUsedYulFunctions, true, m_optimiserSettings ); From 495abee7693810b012032741e0723c55e8a26024 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 3 Mar 2020 12:21:16 +0100 Subject: [PATCH 71/92] [test] Fixes ExecutionFramework providing different contract addresses when running test cases for both, old and new yul codegen. --- test/ExecutionFramework.cpp | 8 ++++++-- test/ExecutionFramework.h | 2 ++ test/libsolidity/SemanticTest.cpp | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/test/ExecutionFramework.cpp b/test/ExecutionFramework.cpp index cc54e7a7b478..170f5638c6f3 100644 --- a/test/ExecutionFramework.cpp +++ b/test/ExecutionFramework.cpp @@ -53,12 +53,16 @@ ExecutionFramework::ExecutionFramework(langutil::EVMVersion _evmVersion): m_optimiserSettings = solidity::frontend::OptimiserSettings::full(); else if (solidity::test::CommonOptions::get().optimize) m_optimiserSettings = solidity::frontend::OptimiserSettings::standard(); - m_evmHost->reset(); + reset(); +} + +void ExecutionFramework::reset() +{ + m_evmHost->reset(); for (size_t i = 0; i < 10; i++) m_evmHost->accounts[EVMHost::convertToEVMC(account(i))].balance = EVMHost::convertToEVMC(u256(1) << 100); - } std::pair ExecutionFramework::compareAndCreateMessage( diff --git a/test/ExecutionFramework.h b/test/ExecutionFramework.h index 6a3ffc0fbc75..e55a12aa6528 100644 --- a/test/ExecutionFramework.h +++ b/test/ExecutionFramework.h @@ -251,6 +251,8 @@ class ExecutionFramework } protected: + void reset(); + void sendMessage(bytes const& _data, bool _isCreation, u256 const& _value = 0); void sendEther(Address const& _to, u256 const& _value); size_t currentTimestamp(); diff --git a/test/libsolidity/SemanticTest.cpp b/test/libsolidity/SemanticTest.cpp index b1ef9372aa4a..b2c4628daf81 100644 --- a/test/libsolidity/SemanticTest.cpp +++ b/test/libsolidity/SemanticTest.cpp @@ -101,6 +101,7 @@ TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePref { for(bool compileViaYul: set{!m_runWithoutYul, m_runWithYul}) { + reset(); bool success = true; m_compileViaYul = compileViaYul; From 58c6b9070530d91ca1c7851a55b106dba3a8be99 Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Tue, 3 Mar 2020 12:15:59 +0100 Subject: [PATCH 72/92] Deprecated warning for .value() and .gas() on function and constructror calls --- Changelog.md | 1 + docs/types/value-types.rst | 2 +- libsolidity/analysis/TypeChecker.cpp | 18 +++++++++++++++++- .../semanticTests/tryCatch/assert.sol | 2 +- .../semanticTests/tryCatch/trivial.sol | 2 +- .../types/function_type_members.sol | 3 +++ .../constructor/constructor_payable.sol | 10 ++++++++++ .../functionCalls/calloptions_repeated.sol | 1 + .../new_with_invalid_calloptions.sol | 8 ++++---- .../functionTypes/call_gas_on_function.sol | 7 +++++++ .../call_value_library_function.sol | 10 ++++++++++ ...call_value_on_non_payable_function_type.sol | 2 +- .../call_value_on_payable_function_type.sol | 2 +- ...ue_options_on_non_payable_function_type.sol | 8 ++++++++ .../warn_deprecate_gas_function.sol | 8 ++++++++ .../warn_deprecate_value_constructor.sol | 11 +++++++++++ .../warn_deprecate_value_function.sol | 8 ++++++++ .../348_unused_return_value_call_value.sol | 3 +++ .../361_calling_payable.sol | 9 +++++++-- .../362_calling_nonpayable.sol | 6 ++++-- ...not_warn_msg_value_in_internal_function.sol | 1 + ...unspecified_encoding_internal_functions.sol | 1 + .../viewPureChecker/gas_value_without_call.sol | 13 +++++++++++++ .../gas_with_call_nonpayable.sol | 15 +++++++++++++-- .../viewPureChecker/staticcall_gas_view.sol | 4 ++++ .../value_with_call_nonpayable.sol | 13 +++++++++++-- 26 files changed, 150 insertions(+), 18 deletions(-) create mode 100644 test/libsolidity/syntaxTests/constructor/constructor_payable.sol create mode 100644 test/libsolidity/syntaxTests/functionTypes/call_gas_on_function.sol create mode 100644 test/libsolidity/syntaxTests/functionTypes/call_value_library_function.sol create mode 100644 test/libsolidity/syntaxTests/functionTypes/call_value_options_on_non_payable_function_type.sol create mode 100644 test/libsolidity/syntaxTests/functionTypes/warn_deprecate_gas_function.sol create mode 100644 test/libsolidity/syntaxTests/functionTypes/warn_deprecate_value_constructor.sol create mode 100644 test/libsolidity/syntaxTests/functionTypes/warn_deprecate_value_function.sol diff --git a/Changelog.md b/Changelog.md index f151365aefa7..9a84d3012ea9 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,7 @@ Language Features: * Inline Assembly: Allow assigning to `_slot` of local storage variable pointers. + * General: Deprecated `value(...)` and `gas(...)` in favor of `{value: ...}` and `{gas: ...}` Compiler Features: diff --git a/docs/types/value-types.rst b/docs/types/value-types.rst index 9dae379124eb..8984a5b8ed97 100644 --- a/docs/types/value-types.rst +++ b/docs/types/value-types.rst @@ -646,7 +646,7 @@ External (or public) functions have the following members: Example that shows how to use the members:: pragma solidity >=0.4.16 <0.7.0; - + // This will report a warning contract Example { function f() public payable returns (bytes4) { diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index 2f9526f879fd..f713ea289234 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -2312,7 +2312,11 @@ bool TypeChecker::visit(FunctionCallOptions const& _functionCallOptions) else if (!expressionFunctionType->isPayable()) m_errorReporter.typeError( _functionCallOptions.location(), - "Cannot set option \"value\" on a non-payable function type." + kind == FunctionType::Kind::Creation ? + "Cannot set option \"value\", since the constructor of " + + expressionFunctionType->returnParameterTypes().front()->toString() + + " is not payable." : + "Cannot set option \"value\" on a non-payable function type." ); else { @@ -2522,12 +2526,24 @@ bool TypeChecker::visit(MemberAccess const& _memberAccess) annotation.type = possibleMembers.front().type; if (auto funType = dynamic_cast(annotation.type)) + { solAssert( !funType->bound() || exprType->isImplicitlyConvertibleTo(*funType->selfType()), "Function \"" + memberName + "\" cannot be called on an object of type " + exprType->toString() + " (expected " + funType->selfType()->toString() + ")." ); + if ( + dynamic_cast(exprType) && + !annotation.referencedDeclaration && + (memberName == "value" || memberName == "gas") + ) + m_errorReporter.warning( + _memberAccess.location(), + "Using \"." + memberName + "(...)\" is deprecated. Use \"{" + memberName + ": ...}\" instead." + ); + } + if (auto const* structType = dynamic_cast(exprType)) annotation.isLValue = !structType->dataStoredIn(DataLocation::CallData); else if (exprType->category() == Type::Category::Array) diff --git a/test/libsolidity/semanticTests/tryCatch/assert.sol b/test/libsolidity/semanticTests/tryCatch/assert.sol index 367ab287bcd0..8b6a7b99692e 100644 --- a/test/libsolidity/semanticTests/tryCatch/assert.sol +++ b/test/libsolidity/semanticTests/tryCatch/assert.sol @@ -4,7 +4,7 @@ contract C { } function f(bool x) public returns (uint) { // Set the gas to make this work on pre-byzantium VMs - try this.g.gas(8000)(x) { + try this.g{gas: 8000}(x) { return 1; } catch { return 2; diff --git a/test/libsolidity/semanticTests/tryCatch/trivial.sol b/test/libsolidity/semanticTests/tryCatch/trivial.sol index e2f8739bd7d0..d43477e994ab 100644 --- a/test/libsolidity/semanticTests/tryCatch/trivial.sol +++ b/test/libsolidity/semanticTests/tryCatch/trivial.sol @@ -4,7 +4,7 @@ contract C { } function f(bool x) public returns (uint) { // Set the gas to make this work on pre-byzantium VMs - try this.g.gas(8000)(x) { + try this.g{gas: 8000}(x) { return 1; } catch { return 2; diff --git a/test/libsolidity/smtCheckerTests/types/function_type_members.sol b/test/libsolidity/smtCheckerTests/types/function_type_members.sol index 2296fa798581..923767dd7366 100644 --- a/test/libsolidity/smtCheckerTests/types/function_type_members.sol +++ b/test/libsolidity/smtCheckerTests/types/function_type_members.sol @@ -3,9 +3,12 @@ contract C { function f(function(uint) external payable g) internal { g.selector; g.gas(2).value(3)(4); + g{gas: 2, value: 3}(4); } } // ---- +// Warning: (122-127): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. +// Warning: (122-136): Using ".value(...)" is deprecated. Use "{value: ...}" instead. // Warning: (108-118): Assertion checker does not yet support this expression. // Warning: (122-130): Assertion checker does not yet implement this type of function call. // Warning: (122-139): Assertion checker does not yet implement this type of function call. diff --git a/test/libsolidity/syntaxTests/constructor/constructor_payable.sol b/test/libsolidity/syntaxTests/constructor/constructor_payable.sol new file mode 100644 index 000000000000..e5c6ac28d5b4 --- /dev/null +++ b/test/libsolidity/syntaxTests/constructor/constructor_payable.sol @@ -0,0 +1,10 @@ +contract C { + constructor() public payable { } +} + +contract D { + function createC() public returns (C) { + C c = (new C){value: 1}(); + return c; + } +} diff --git a/test/libsolidity/syntaxTests/functionCalls/calloptions_repeated.sol b/test/libsolidity/syntaxTests/functionCalls/calloptions_repeated.sol index 2ffe4b3edffe..181b325b05f4 100644 --- a/test/libsolidity/syntaxTests/functionCalls/calloptions_repeated.sol +++ b/test/libsolidity/syntaxTests/functionCalls/calloptions_repeated.sol @@ -15,6 +15,7 @@ contract C { // TypeError: (78-110): Option "gas" has already been set. // TypeError: (120-154): Option "gas" has already been set. // TypeError: (164-198): Option "value" has already been set. +// Warning: (208-222): Using ".value(...)" is deprecated. Use "{value: ...}" instead. // TypeError: (208-242): Option "value" has already been set. // TypeError: (252-293): Option "value" has already been set. // TypeError: (252-293): Option "gas" has already been set. diff --git a/test/libsolidity/syntaxTests/functionCalls/new_with_invalid_calloptions.sol b/test/libsolidity/syntaxTests/functionCalls/new_with_invalid_calloptions.sol index a8cabe0de185..29798dde6a83 100644 --- a/test/libsolidity/syntaxTests/functionCalls/new_with_invalid_calloptions.sol +++ b/test/libsolidity/syntaxTests/functionCalls/new_with_invalid_calloptions.sol @@ -14,14 +14,14 @@ contract C { // ==== // EVMVersion: >=constantinople // ---- -// TypeError: (64-98): Cannot set option "value" on a non-payable function type. +// TypeError: (64-98): Cannot set option "value", since the constructor of contract D is not payable. // TypeError: (64-98): Function call option "gas" cannot be used with "new". // TypeError: (102-123): Unknown call option "slt". Valid options are "salt", "value" and "gas". -// TypeError: (102-123): Cannot set option "value" on a non-payable function type. +// TypeError: (102-123): Cannot set option "value", since the constructor of contract D is not payable. // TypeError: (127-139): Unknown call option "val". Valid options are "salt", "value" and "gas". // TypeError: (143-172): Duplicate option "salt". -// TypeError: (176-199): Cannot set option "value" on a non-payable function type. -// TypeError: (176-199): Cannot set option "value" on a non-payable function type. +// TypeError: (176-199): Cannot set option "value", since the constructor of contract D is not payable. +// TypeError: (176-199): Cannot set option "value", since the constructor of contract D is not payable. // TypeError: (203-220): Unknown call option "random". Valid options are "salt", "value" and "gas". // TypeError: (224-242): Unknown call option "what". Valid options are "salt", "value" and "gas". // TypeError: (246-259): Function call option "gas" cannot be used with "new". diff --git a/test/libsolidity/syntaxTests/functionTypes/call_gas_on_function.sol b/test/libsolidity/syntaxTests/functionTypes/call_gas_on_function.sol new file mode 100644 index 000000000000..381fb6ea0a67 --- /dev/null +++ b/test/libsolidity/syntaxTests/functionTypes/call_gas_on_function.sol @@ -0,0 +1,7 @@ +contract C { + function (uint) external returns (uint) x; + function f() public { + x{gas: 2}(1); + } +} + diff --git a/test/libsolidity/syntaxTests/functionTypes/call_value_library_function.sol b/test/libsolidity/syntaxTests/functionTypes/call_value_library_function.sol new file mode 100644 index 000000000000..97eebfd233d1 --- /dev/null +++ b/test/libsolidity/syntaxTests/functionTypes/call_value_library_function.sol @@ -0,0 +1,10 @@ +library L { + function value(function()internal a, uint256 b) internal {} +} +contract C { + using L for function()internal; + function f() public { + function()internal x; + x.value(42); + } +} diff --git a/test/libsolidity/syntaxTests/functionTypes/call_value_on_non_payable_function_type.sol b/test/libsolidity/syntaxTests/functionTypes/call_value_on_non_payable_function_type.sol index 5efdf2407f13..822d3eb28e5b 100644 --- a/test/libsolidity/syntaxTests/functionTypes/call_value_on_non_payable_function_type.sol +++ b/test/libsolidity/syntaxTests/functionTypes/call_value_on_non_payable_function_type.sol @@ -1,7 +1,7 @@ contract C { function (uint) external returns (uint) x; function f() public { - x.value(2)(); + x.value(2)(1); } } // ---- diff --git a/test/libsolidity/syntaxTests/functionTypes/call_value_on_payable_function_type.sol b/test/libsolidity/syntaxTests/functionTypes/call_value_on_payable_function_type.sol index ca2a01964d7c..1fd0e7188a15 100644 --- a/test/libsolidity/syntaxTests/functionTypes/call_value_on_payable_function_type.sol +++ b/test/libsolidity/syntaxTests/functionTypes/call_value_on_payable_function_type.sol @@ -1,6 +1,6 @@ contract C { function (uint) external payable returns (uint) x; function f() public { - x.value(2)(1); + x{value: 2}(1); } } diff --git a/test/libsolidity/syntaxTests/functionTypes/call_value_options_on_non_payable_function_type.sol b/test/libsolidity/syntaxTests/functionTypes/call_value_options_on_non_payable_function_type.sol new file mode 100644 index 000000000000..30f36f219281 --- /dev/null +++ b/test/libsolidity/syntaxTests/functionTypes/call_value_options_on_non_payable_function_type.sol @@ -0,0 +1,8 @@ +contract C { + function (uint) external returns (uint) x; + function g() public { + x{value: 2}(1); + } +} +// ---- +// TypeError: (94-105): Cannot set option "value" on a non-payable function type. diff --git a/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_gas_function.sol b/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_gas_function.sol new file mode 100644 index 000000000000..36d42e3499ec --- /dev/null +++ b/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_gas_function.sol @@ -0,0 +1,8 @@ +contract C { + function (uint) external payable returns (uint) x; + function f() public { + x.gas(2)(1); + } +} +// ---- +// Warning: (102-107): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. diff --git a/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_value_constructor.sol b/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_value_constructor.sol new file mode 100644 index 000000000000..28a573c93ee4 --- /dev/null +++ b/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_value_constructor.sol @@ -0,0 +1,11 @@ +contract C { + constructor() payable public {} +} +contract D { + function createC() public returns (C) { + C c = (new C).value(2)(); + return c; + } +} +// ---- +// Warning: (122-135): Using ".value(...)" is deprecated. Use "{value: ...}" instead. diff --git a/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_value_function.sol b/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_value_function.sol new file mode 100644 index 000000000000..854ea28fafaa --- /dev/null +++ b/test/libsolidity/syntaxTests/functionTypes/warn_deprecate_value_function.sol @@ -0,0 +1,8 @@ +contract C { + function (uint) external payable returns (uint) x; + function f() public { + x.value(2)(1); + } +} +// ---- +// Warning: (102-109): Using ".value(...)" is deprecated. Use "{value: ...}" instead. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/348_unused_return_value_call_value.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/348_unused_return_value_call_value.sol index 1ac7c6f33a70..dffa55fdfff9 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/348_unused_return_value_call_value.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/348_unused_return_value_call_value.sol @@ -1,7 +1,10 @@ contract test { function f() public { address(0x12).call.value(2)("abc"); + address(0x12).call{value: 2}("abc"); } } // ---- +// Warning: (50-74): Using ".value(...)" is deprecated. Use "{value: ...}" instead. // Warning: (50-84): Return value of low-level calls not used. +// Warning: (94-129): Return value of low-level calls not used. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/361_calling_payable.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/361_calling_payable.sol index 8ef4d5799761..a3c5a5f3085e 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/361_calling_payable.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/361_calling_payable.sol @@ -1,6 +1,11 @@ contract receiver { function pay() payable public {} } contract test { - function f() public { (new receiver()).pay.value(10)(); } + function f() public { (new receiver()).pay{value: 10}(); } + function g() public { (new receiver()).pay.value(10)(); } receiver r = new receiver(); - function g() public { r.pay.value(10)(); } + function h() public { r.pay{value: 10}(); } + function i() public { r.pay.value(10)(); } } +// ---- +// Warning: (160-186): Using ".value(...)" is deprecated. Use "{value: ...}" instead. +// Warning: (303-314): Using ".value(...)" is deprecated. Use "{value: ...}" instead. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/362_calling_nonpayable.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/362_calling_nonpayable.sol index 1c04be755e69..6f5706446796 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/362_calling_nonpayable.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/362_calling_nonpayable.sol @@ -1,6 +1,8 @@ contract receiver { function nopay() public {} } contract test { - function f() public { (new receiver()).nopay.value(10)(); } + function f() public { (new receiver()).nopay{value: 10}(); } + function g() public { (new receiver()).nopay.value(10)(); } } // ---- -// TypeError: (91-119): Member "value" is only available for payable functions. +// TypeError: (91-124): Cannot set option "value" on a non-payable function type. +// TypeError: (156-184): Member "value" is only available for payable functions. diff --git a/test/libsolidity/syntaxTests/nameAndTypeResolution/399_does_not_warn_msg_value_in_internal_function.sol b/test/libsolidity/syntaxTests/nameAndTypeResolution/399_does_not_warn_msg_value_in_internal_function.sol index 8492e691e71a..00d8ad4c8fc1 100644 --- a/test/libsolidity/syntaxTests/nameAndTypeResolution/399_does_not_warn_msg_value_in_internal_function.sol +++ b/test/libsolidity/syntaxTests/nameAndTypeResolution/399_does_not_warn_msg_value_in_internal_function.sol @@ -3,3 +3,4 @@ contract C { msg.value; } } + diff --git a/test/libsolidity/syntaxTests/specialFunctions/types_with_unspecified_encoding_internal_functions.sol b/test/libsolidity/syntaxTests/specialFunctions/types_with_unspecified_encoding_internal_functions.sol index a30e428a67c7..ea11703f9079 100644 --- a/test/libsolidity/syntaxTests/specialFunctions/types_with_unspecified_encoding_internal_functions.sol +++ b/test/libsolidity/syntaxTests/specialFunctions/types_with_unspecified_encoding_internal_functions.sol @@ -5,6 +5,7 @@ contract C { } } // ---- +// Warning: (105-115): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. // TypeError: (91-100): This type cannot be encoded. // TypeError: (102-103): This type cannot be encoded. // TypeError: (105-115): This type cannot be encoded. diff --git a/test/libsolidity/syntaxTests/viewPureChecker/gas_value_without_call.sol b/test/libsolidity/syntaxTests/viewPureChecker/gas_value_without_call.sol index 2df3bbe4f142..77dab9af27c1 100644 --- a/test/libsolidity/syntaxTests/viewPureChecker/gas_value_without_call.sol +++ b/test/libsolidity/syntaxTests/viewPureChecker/gas_value_without_call.sol @@ -2,12 +2,25 @@ contract C { function f() external payable {} function g(address a) external pure { a.call.value(42); + a.call{value: 42}; a.call.gas(42); + a.call{gas: 42}; a.staticcall.gas(42); + a.staticcall{gas: 42}; a.delegatecall.gas(42); + a.delegatecall{gas: 42}; } function h() external view { this.f.value(42); + this.f{value: 42}; this.f.gas(42); + this.f{gas: 42}; } } +// ---- +// Warning: (91-103): Using ".value(...)" is deprecated. Use "{value: ...}" instead. +// Warning: (132-142): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. +// Warning: (169-185): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. +// Warning: (218-236): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. +// Warning: (304-316): Using ".value(...)" is deprecated. Use "{value: ...}" instead. +// Warning: (345-355): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. diff --git a/test/libsolidity/syntaxTests/viewPureChecker/gas_with_call_nonpayable.sol b/test/libsolidity/syntaxTests/viewPureChecker/gas_with_call_nonpayable.sol index 0a58a516c6e7..4a0da038ed83 100644 --- a/test/libsolidity/syntaxTests/viewPureChecker/gas_with_call_nonpayable.sol +++ b/test/libsolidity/syntaxTests/viewPureChecker/gas_with_call_nonpayable.sol @@ -1,16 +1,27 @@ contract C { function f(address a) external view returns (bool success) { (success,) = a.call.gas(42)(""); + (success,) = a.call{gas: 42}(""); } function g(address a) external view returns (bool success) { (success,) = a.call.gas(42)(""); + (success,) = a.call{gas: 42}(""); } function h() external payable {} function i() external view { this.h.gas(42)(); } + function j() external view { + this.h{gas: 42}(); + } } // ---- +// Warning: (90-100): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. +// Warning: (226-236): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. +// Warning: (351-361): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. // TypeError: (90-108): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. -// TypeError: (190-208): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. -// TypeError: (279-295): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (125-144): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (226-244): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (261-280): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (351-367): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (404-421): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. diff --git a/test/libsolidity/syntaxTests/viewPureChecker/staticcall_gas_view.sol b/test/libsolidity/syntaxTests/viewPureChecker/staticcall_gas_view.sol index 6f8b31bfce63..2b4821c53aa9 100644 --- a/test/libsolidity/syntaxTests/viewPureChecker/staticcall_gas_view.sol +++ b/test/libsolidity/syntaxTests/viewPureChecker/staticcall_gas_view.sol @@ -3,9 +3,13 @@ contract C { function test(address a) external view returns (bool status) { // This used to incorrectly raise an error about violating the view mutability. (status,) = a.staticcall.gas(42)(""); + (status,) = a.staticcall{gas: 42}(""); this.f.gas(42)(); + this.f{gas: 42}(); } } // ==== // EVMVersion: >=byzantium // ---- +// Warning: (207-223): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. +// Warning: (276-286): Using ".gas(...)" is deprecated. Use "{gas: ...}" instead. diff --git a/test/libsolidity/syntaxTests/viewPureChecker/value_with_call_nonpayable.sol b/test/libsolidity/syntaxTests/viewPureChecker/value_with_call_nonpayable.sol index cf5d885c9a02..6b82f2fab2b8 100644 --- a/test/libsolidity/syntaxTests/viewPureChecker/value_with_call_nonpayable.sol +++ b/test/libsolidity/syntaxTests/viewPureChecker/value_with_call_nonpayable.sol @@ -1,16 +1,25 @@ contract C { function f(address a) external view returns (bool success) { (success,) = a.call.value(42)(""); + (success,) = a.call{value: 42}(""); } function g(address a) external view returns (bool success) { (success,) = a.call.value(42)(""); + (success,) = a.call{value: 42}(""); } function h() external payable {} function i() external view { this.h.value(42)(); + this.h{value: 42}(); } } // ---- +// Warning: (90-102): Using ".value(...)" is deprecated. Use "{value: ...}" instead. +// Warning: (230-242): Using ".value(...)" is deprecated. Use "{value: ...}" instead. +// Warning: (359-371): Using ".value(...)" is deprecated. Use "{value: ...}" instead. // TypeError: (90-110): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. -// TypeError: (192-212): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. -// TypeError: (283-301): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (127-148): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (230-250): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (267-288): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (359-377): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. +// TypeError: (381-400): Function declared as view, but this expression (potentially) modifies the state and thus requires non-payable (the default) or payable. From 919888ddbc470a628676586fcc1b21a7030b1042 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Mon, 2 Mar 2020 16:33:12 +0100 Subject: [PATCH 73/92] Use yul function for calldata tail access, fix checks and add additional revert reason. --- libsolidity/codegen/CompilerUtils.cpp | 56 ++----------------- libsolidity/codegen/YulUtilFunctions.cpp | 9 ++- .../revertStrings/calldata_tail_short.sol | 9 +++ 3 files changed, 21 insertions(+), 53 deletions(-) create mode 100644 test/libsolidity/semanticTests/revertStrings/calldata_tail_short.sol diff --git a/libsolidity/codegen/CompilerUtils.cpp b/libsolidity/codegen/CompilerUtils.cpp index b39633377052..da82c1822fe0 100644 --- a/libsolidity/codegen/CompilerUtils.cpp +++ b/libsolidity/codegen/CompilerUtils.cpp @@ -121,56 +121,12 @@ void CompilerUtils::returnDataToArray() void CompilerUtils::accessCalldataTail(Type const& _type) { - solAssert(_type.dataStoredIn(DataLocation::CallData), ""); - solAssert(_type.isDynamicallyEncoded(), ""); - - unsigned int tailSize = _type.calldataEncodedTailSize(); - solAssert(tailSize > 1, ""); - - // returns the absolute offset of the tail in "base_ref" - m_context.appendInlineAssembly(Whiskers(R"({ - let rel_offset_of_tail := calldataload(ptr_to_tail) - if iszero(slt(rel_offset_of_tail, sub(sub(calldatasize(), base_ref), sub(, 1)))) { } - base_ref := add(base_ref, rel_offset_of_tail) - })") - ("neededLength", toCompactHexWithPrefix(tailSize)) - ("revertString", m_context.revertReasonIfDebug("Invalid calldata tail offset")) - .render(), {"base_ref", "ptr_to_tail"}); - // stack layout: - - if (!_type.isDynamicallySized()) - { - m_context << Instruction::POP; - // stack layout: - solAssert( - _type.category() == Type::Category::Struct || - _type.category() == Type::Category::Array, - "Invalid dynamically encoded base type on tail access." - ); - } - else - { - auto const* arrayType = dynamic_cast(&_type); - solAssert(!!arrayType, "Invalid dynamically sized type."); - unsigned int calldataStride = arrayType->calldataStride(); - solAssert(calldataStride > 0, ""); - - // returns the absolute offset of the tail in "base_ref" - // and the length of the tail in "length" - m_context.appendInlineAssembly( - Whiskers(R"({ - length := calldataload(base_ref) - base_ref := add(base_ref, 0x20) - if gt(length, 0xffffffffffffffff) { } - if sgt(base_ref, sub(calldatasize(), mul(length, ))) { revert(0, 0) } - })") - ("calldataStride", toCompactHexWithPrefix(calldataStride)) - ("revertString", m_context.revertReasonIfDebug("Invalid calldata tail length")) - .render(), - {"base_ref", "length"} - ); - // stack layout: - } + m_context << Instruction::SWAP1; + m_context.callYulFunction( + m_context.utilFunctions().accessCalldataTailFunction(_type), + 2, + _type.isDynamicallySized() ? 2 : 1 + ); } unsigned CompilerUtils::loadFromMemory( diff --git a/libsolidity/codegen/YulUtilFunctions.cpp b/libsolidity/codegen/YulUtilFunctions.cpp index 0472dd588aba..7a5036109e00 100644 --- a/libsolidity/codegen/YulUtilFunctions.cpp +++ b/libsolidity/codegen/YulUtilFunctions.cpp @@ -942,13 +942,13 @@ string YulUtilFunctions::accessCalldataTailFunction(Type const& _type) return Whiskers(R"( function (base_ref, ptr_to_tail) -> addr, length { let rel_offset_of_tail := calldataload(ptr_to_tail) - if iszero(slt(rel_offset_of_tail, sub(sub(calldatasize(), base_ref), sub(, 1)))) { revert(0, 0) } + if iszero(slt(rel_offset_of_tail, sub(sub(calldatasize(), base_ref), sub(, 1)))) { } addr := add(base_ref, rel_offset_of_tail) length := calldataload(addr) - if gt(length, 0xffffffffffffffff) { revert(0, 0) } - if sgt(base_ref, sub(calldatasize(), mul(length, ))) { revert(0, 0) } + if gt(length, 0xffffffffffffffff) { } addr := add(addr, 32) + if sgt(addr, sub(calldatasize(), mul(length, ))) { } } )") @@ -956,6 +956,9 @@ string YulUtilFunctions::accessCalldataTailFunction(Type const& _type) ("dynamicallySized", _type.isDynamicallySized()) ("neededLength", toCompactHexWithPrefix(_type.calldataEncodedTailSize())) ("calldataStride", toCompactHexWithPrefix(_type.isDynamicallySized() ? dynamic_cast(_type).calldataStride() : 0)) + ("invalidCalldataTailOffset", revertReasonIfDebug("Invalid calldata tail offset")) + ("invalidCalldataTailLength", revertReasonIfDebug("Invalid calldata tail length")) + ("shortCalldataTail", revertReasonIfDebug("Calldata tail too short")) .render(); }); } diff --git a/test/libsolidity/semanticTests/revertStrings/calldata_tail_short.sol b/test/libsolidity/semanticTests/revertStrings/calldata_tail_short.sol new file mode 100644 index 000000000000..b6cc5800d42b --- /dev/null +++ b/test/libsolidity/semanticTests/revertStrings/calldata_tail_short.sol @@ -0,0 +1,9 @@ +pragma experimental ABIEncoderV2; +contract C { + function f(uint256[][] calldata x) external { x[0]; } +} +// ==== +// EVMVersion: >=byzantium +// revertStrings: debug +// ---- +// f(uint256[][]): 0x20, 1, 0x20, 2, 0x42 -> FAILURE, hex"08c379a0", 0x20, 23, "Calldata tail too short" From 857ed12b056489eb53131b196bf8ea5e4257c956 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Mon, 2 Mar 2020 18:08:19 +0100 Subject: [PATCH 74/92] Use plain members and references instead of shared pointers for MultiUseYulFunctionCollector --- libsolidity/codegen/ABIFunctions.cpp | 2 +- libsolidity/codegen/ABIFunctions.h | 6 +- libsolidity/codegen/CompilerContext.cpp | 2 +- libsolidity/codegen/CompilerContext.h | 2 +- libsolidity/codegen/YulUtilFunctions.cpp | 110 +++++++++--------- libsolidity/codegen/YulUtilFunctions.h | 6 +- .../codegen/ir/IRGenerationContext.cpp | 2 +- libsolidity/codegen/ir/IRGenerationContext.h | 7 +- libsolidity/codegen/ir/IRGenerator.cpp | 13 +-- 9 files changed, 74 insertions(+), 76 deletions(-) diff --git a/libsolidity/codegen/ABIFunctions.cpp b/libsolidity/codegen/ABIFunctions.cpp index b6d690898100..5cca1c1e4065 100644 --- a/libsolidity/codegen/ABIFunctions.cpp +++ b/libsolidity/codegen/ABIFunctions.cpp @@ -1492,7 +1492,7 @@ string ABIFunctions::arrayStoreLengthForEncodingFunction(ArrayType const& _type, string ABIFunctions::createFunction(string const& _name, function const& _creator) { - return m_functionCollector->createFunction(_name, _creator); + return m_functionCollector.createFunction(_name, _creator); } size_t ABIFunctions::headSize(TypePointers const& _targetTypes) diff --git a/libsolidity/codegen/ABIFunctions.h b/libsolidity/codegen/ABIFunctions.h index 1099e3593fe7..e760917ac619 100644 --- a/libsolidity/codegen/ABIFunctions.h +++ b/libsolidity/codegen/ABIFunctions.h @@ -57,11 +57,11 @@ class ABIFunctions explicit ABIFunctions( langutil::EVMVersion _evmVersion, RevertStrings _revertStrings, - std::shared_ptr _functionCollector = std::make_shared() + MultiUseYulFunctionCollector& _functionCollector ): m_evmVersion(_evmVersion), m_revertStrings(_revertStrings), - m_functionCollector(std::move(_functionCollector)), + m_functionCollector(_functionCollector), m_utils(_evmVersion, m_revertStrings, m_functionCollector) {} @@ -247,7 +247,7 @@ class ABIFunctions langutil::EVMVersion m_evmVersion; RevertStrings const m_revertStrings; - std::shared_ptr m_functionCollector; + MultiUseYulFunctionCollector& m_functionCollector; YulUtilFunctions m_utils; }; diff --git a/libsolidity/codegen/CompilerContext.cpp b/libsolidity/codegen/CompilerContext.cpp index 1d1eb92b9cbc..cb195c49fa73 100644 --- a/libsolidity/codegen/CompilerContext.cpp +++ b/libsolidity/codegen/CompilerContext.cpp @@ -151,7 +151,7 @@ pair> CompilerContext::requestedYulFunctions() { set empty; swap(empty, m_externallyUsedYulFunctions); - return make_pair(m_yulFunctionCollector->requestedFunctions(), std::move(empty)); + return {m_yulFunctionCollector.requestedFunctions(), std::move(empty)}; } void CompilerContext::addVariable( diff --git a/libsolidity/codegen/CompilerContext.h b/libsolidity/codegen/CompilerContext.h index 76c72770d6b5..8c577523916f 100644 --- a/libsolidity/codegen/CompilerContext.h +++ b/libsolidity/codegen/CompilerContext.h @@ -371,7 +371,7 @@ class CompilerContext /// An index of low-level function labels by name. std::map m_lowLevelFunctions; /// Collector for yul functions. - std::shared_ptr m_yulFunctionCollector = std::make_shared(); + MultiUseYulFunctionCollector m_yulFunctionCollector; /// Set of externally used yul functions. std::set m_externallyUsedYulFunctions; /// Container for ABI functions to be generated. diff --git a/libsolidity/codegen/YulUtilFunctions.cpp b/libsolidity/codegen/YulUtilFunctions.cpp index 0472dd588aba..bcb309c99817 100644 --- a/libsolidity/codegen/YulUtilFunctions.cpp +++ b/libsolidity/codegen/YulUtilFunctions.cpp @@ -39,7 +39,7 @@ using namespace solidity::frontend; string YulUtilFunctions::combineExternalFunctionIdFunction() { string functionName = "combine_external_function_id"; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (addr, selector) -> combined { combined := (or((addr), and(selector, 0xffffffff))) @@ -55,7 +55,7 @@ string YulUtilFunctions::combineExternalFunctionIdFunction() string YulUtilFunctions::splitExternalFunctionIdFunction() { string functionName = "split_external_function_id"; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (combined) -> addr, selector { combined := (combined) @@ -73,7 +73,7 @@ string YulUtilFunctions::splitExternalFunctionIdFunction() string YulUtilFunctions::copyToMemoryFunction(bool _fromCalldata) { string functionName = "copy_" + string(_fromCalldata ? "calldata" : "memory") + "_to_memory"; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { if (_fromCalldata) { return Whiskers(R"( @@ -116,7 +116,7 @@ string YulUtilFunctions::requireOrAssertFunction(bool _assert, Type const* _mess solAssert(!_assert || !_messageType, "Asserts can't have messages!"); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { if (!_messageType) return Whiskers(R"( function (condition) { @@ -166,7 +166,7 @@ string YulUtilFunctions::requireOrAssertFunction(bool _assert, Type const* _mess string YulUtilFunctions::leftAlignFunction(Type const& _type) { string functionName = string("leftAlign_") + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers templ(R"( function (value) -> aligned { @@ -228,7 +228,7 @@ string YulUtilFunctions::shiftLeftFunction(size_t _numBits) solAssert(_numBits < 256, ""); string functionName = "shift_left_" + to_string(_numBits); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (value) -> newValue { @@ -251,7 +251,7 @@ string YulUtilFunctions::shiftLeftFunction(size_t _numBits) string YulUtilFunctions::shiftLeftFunctionDynamic() { string functionName = "shift_left_dynamic"; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (bits, value) -> newValue { @@ -277,7 +277,7 @@ string YulUtilFunctions::shiftRightFunction(size_t _numBits) // the opcodes SAR and SDIV behave differently with regards to rounding! string functionName = "shift_right_" + to_string(_numBits) + "_unsigned"; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (value) -> newValue { @@ -303,7 +303,7 @@ string YulUtilFunctions::shiftRightFunctionDynamic() // the opcodes SAR and SDIV behave differently with regards to rounding! string const functionName = "shift_right_unsigned_dynamic"; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (bits, value) -> newValue { @@ -328,7 +328,7 @@ string YulUtilFunctions::updateByteSliceFunction(size_t _numBytes, size_t _shift size_t numBits = _numBytes * 8; size_t shiftBits = _shiftBytes * 8; string functionName = "update_byte_slice_" + to_string(_numBytes) + "_shift_" + to_string(_shiftBytes); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (value, toInsert) -> result { @@ -350,7 +350,7 @@ string YulUtilFunctions::updateByteSliceFunctionDynamic(size_t _numBytes) solAssert(_numBytes <= 32, ""); size_t numBits = _numBytes * 8; string functionName = "update_byte_slice_dynamic" + to_string(_numBytes); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (value, shiftBytes, toInsert) -> result { @@ -371,7 +371,7 @@ string YulUtilFunctions::updateByteSliceFunctionDynamic(size_t _numBytes) string YulUtilFunctions::roundUpFunction() { string functionName = "round_up_to_mul_of_32"; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (value) -> result { @@ -389,7 +389,7 @@ string YulUtilFunctions::overflowCheckedIntAddFunction(IntegerType const& _type) // TODO: Consider to add a special case for unsigned 256-bit integers // and use the following instead: // sum := add(x, y) if lt(sum, x) { revert(0, 0) } - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (x, y) -> sum { @@ -416,7 +416,7 @@ string YulUtilFunctions::overflowCheckedIntAddFunction(IntegerType const& _type) string YulUtilFunctions::overflowCheckedIntMulFunction(IntegerType const& _type) { string functionName = "checked_mul_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return // Multiplication by zero could be treated separately and directly return zero. Whiskers(R"( @@ -448,7 +448,7 @@ string YulUtilFunctions::overflowCheckedIntMulFunction(IntegerType const& _type) string YulUtilFunctions::overflowCheckedIntDivFunction(IntegerType const& _type) { string functionName = "checked_div_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (x, y) -> r { @@ -473,7 +473,7 @@ string YulUtilFunctions::overflowCheckedIntDivFunction(IntegerType const& _type) string YulUtilFunctions::checkedIntModFunction(IntegerType const& _type) { string functionName = "checked_mod_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (x, y) -> r { @@ -490,7 +490,7 @@ string YulUtilFunctions::checkedIntModFunction(IntegerType const& _type) string YulUtilFunctions::overflowCheckedIntSubFunction(IntegerType const& _type) { string functionName = "checked_sub_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { return Whiskers(R"( function (x, y) -> diff { @@ -516,7 +516,7 @@ string YulUtilFunctions::overflowCheckedIntSubFunction(IntegerType const& _type) string YulUtilFunctions::arrayLengthFunction(ArrayType const& _type) { string functionName = "array_length_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers w(R"( function (value) -> length { @@ -564,7 +564,7 @@ std::string YulUtilFunctions::resizeDynamicArrayFunction(ArrayType const& _type) solUnimplementedAssert(_type.baseType()->storageSize() == 1, ""); string functionName = "resize_array_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (array, newLen) { if gt(newLen, ) { @@ -604,7 +604,7 @@ string YulUtilFunctions::storageArrayPopFunction(ArrayType const& _type) solUnimplementedAssert(_type.baseType()->storageBytes() <= 32, "Base type is not yet implemented."); string functionName = "array_pop_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (array) { let oldLen := (array) @@ -632,7 +632,7 @@ string YulUtilFunctions::storageArrayPushFunction(ArrayType const& _type) solUnimplementedAssert(_type.baseType()->storageBytes() <= 32, "Base type is not yet implemented."); string functionName = "array_push_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (array, value) { let oldLen := (array) @@ -659,7 +659,7 @@ string YulUtilFunctions::storageArrayPushZeroFunction(ArrayType const& _type) solUnimplementedAssert(_type.baseType()->storageBytes() <= 32, "Base type is not yet implemented."); string functionName = "array_push_zero_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (array) -> slot, offset { let oldLen := (array) @@ -684,7 +684,7 @@ string YulUtilFunctions::clearStorageRangeFunction(Type const& _type) solAssert(_type.storageBytes() >= 32, "Expected smaller value for storage bytes"); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (start, end) { for {} lt(start, end) { start := add(start, ) } @@ -715,7 +715,7 @@ string YulUtilFunctions::clearStorageArrayFunction(ArrayType const& _type) string functionName = "clear_storage_array_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (slot) { @@ -745,7 +745,7 @@ string YulUtilFunctions::clearStorageArrayFunction(ArrayType const& _type) string YulUtilFunctions::arrayConvertLengthToSize(ArrayType const& _type) { string functionName = "array_convert_length_to_size_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Type const& baseType = *_type.baseType(); switch (_type.location()) @@ -798,7 +798,7 @@ string YulUtilFunctions::arrayAllocationSizeFunction(ArrayType const& _type) { solAssert(_type.dataStoredIn(DataLocation::Memory), ""); string functionName = "array_allocation_size_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers w(R"( function (length) -> size { // Make sure we can allocate memory without overflow @@ -825,7 +825,7 @@ string YulUtilFunctions::arrayAllocationSizeFunction(ArrayType const& _type) string YulUtilFunctions::arrayDataAreaFunction(ArrayType const& _type) { string functionName = "array_dataslot_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { // No special processing for calldata arrays, because they are stored as // offset of the data area and length on the stack, so the offset already // points to the data area. @@ -858,7 +858,7 @@ string YulUtilFunctions::storageArrayIndexAccessFunction(ArrayType const& _type) solUnimplementedAssert(_type.baseType()->storageBytes() > 16, ""); string functionName = "storage_array_index_access_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (array, index) -> slot, offset { if iszero(lt(index, (array))) { @@ -886,7 +886,7 @@ string YulUtilFunctions::storageArrayIndexAccessFunction(ArrayType const& _type) string YulUtilFunctions::memoryArrayIndexAccessFunction(ArrayType const& _type) { string functionName = "memory_array_index_access_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (baseRef, index) -> addr { if iszero(lt(index, (baseRef))) { @@ -912,7 +912,7 @@ string YulUtilFunctions::calldataArrayIndexAccessFunction(ArrayType const& _type { solAssert(_type.dataStoredIn(DataLocation::CallData), ""); string functionName = "calldata_array_index_access_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (base_ref, length, index) -> addr, len { if iszero(lt(index, length)) { invalid() } @@ -938,7 +938,7 @@ string YulUtilFunctions::accessCalldataTailFunction(Type const& _type) solAssert(_type.isDynamicallyEncoded(), ""); solAssert(_type.dataStoredIn(DataLocation::CallData), ""); string functionName = "access_calldata_tail_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (base_ref, ptr_to_tail) -> addr, length { let rel_offset_of_tail := calldataload(ptr_to_tail) @@ -966,7 +966,7 @@ string YulUtilFunctions::nextArrayElementFunction(ArrayType const& _type) if (_type.dataStoredIn(DataLocation::Storage)) solAssert(_type.baseType()->storageBytes() > 16, ""); string functionName = "array_nextElement_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers templ(R"( function (ptr) -> next { next := add(ptr, ) @@ -1002,7 +1002,7 @@ string YulUtilFunctions::mappingIndexAccessFunction(MappingType const& _mappingT solAssert(_keyType.sizeOnStack() <= 1, ""); string functionName = "mapping_index_access_" + _mappingType.identifier() + "_of_" + _keyType.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { if (_mappingType.keyType()->isDynamicallySized()) return Whiskers(R"( function (slot ) -> dataSlot { @@ -1050,7 +1050,7 @@ string YulUtilFunctions::readFromStorage(Type const& _type, size_t _offset, bool to_string(_offset) + "_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { solAssert(_type.sizeOnStack() == 1, ""); return Whiskers(R"( function (slot) -> value { @@ -1071,7 +1071,7 @@ string YulUtilFunctions::readFromStorageDynamic(Type const& _type, bool _splitFu string(_splitFunctionTypes ? "split_" : "") + "_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { solAssert(_type.sizeOnStack() == 1, ""); return Whiskers(R"( function (slot, offset) -> value { @@ -1101,7 +1101,7 @@ string YulUtilFunctions::updateStorageValueFunction(Type const& _type, std::opti (_offset.has_value() ? ("offset_" + to_string(*_offset)) : "") + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { if (_type.isValueType()) { solAssert(_type.storageBytes() <= 32, "Invalid storage bytes size."); @@ -1141,7 +1141,7 @@ string YulUtilFunctions::writeToMemoryFunction(Type const& _type) string("write_to_memory_") + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { solAssert(!dynamic_cast(&_type), ""); if (auto ref = dynamic_cast(&_type)) { @@ -1201,7 +1201,7 @@ string YulUtilFunctions::extractFromStorageValueDynamic(Type const& _type, bool "extract_from_storage_value_dynamic" + string(_splitFunctionTypes ? "split_" : "") + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { return Whiskers(R"( function (slot_value, offset) -> value { value := ((mul(offset, 8), slot_value)) @@ -1224,7 +1224,7 @@ string YulUtilFunctions::extractFromStorageValue(Type const& _type, size_t _offs "offset_" + to_string(_offset) + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { return Whiskers(R"( function (slot_value) -> value { value := ((slot_value)) @@ -1243,7 +1243,7 @@ string YulUtilFunctions::cleanupFromStorageFunction(Type const& _type, bool _spl solUnimplementedAssert(!_splitFunctionTypes, ""); string functionName = string("cleanup_from_storage_") + (_splitFunctionTypes ? "split_" : "") + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { Whiskers templ(R"( function (value) -> cleaned { cleaned := @@ -1275,7 +1275,7 @@ string YulUtilFunctions::prepareStoreFunction(Type const& _type) solUnimplementedAssert(_type.category() != Type::Category::Function, ""); string functionName = "prepare_store_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers templ(R"( function (value) -> ret { ret := @@ -1293,7 +1293,7 @@ string YulUtilFunctions::prepareStoreFunction(Type const& _type) string YulUtilFunctions::allocationFunction() { string functionName = "allocateMemory"; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (size) -> memPtr { memPtr := mload() @@ -1314,7 +1314,7 @@ string YulUtilFunctions::allocateMemoryArrayFunction(ArrayType const& _type) solUnimplementedAssert(!_type.isByteArray(), ""); string functionName = "allocate_memory_array_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (length) -> memPtr { memPtr := ((length)) @@ -1341,7 +1341,7 @@ string YulUtilFunctions::conversionFunction(Type const& _from, Type const& _to) _from.identifier() + "_to_" + _to.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers templ(R"( function (value) -> converted { @@ -1519,7 +1519,7 @@ string YulUtilFunctions::conversionFunction(Type const& _from, Type const& _to) string YulUtilFunctions::cleanupFunction(Type const& _type) { string functionName = string("cleanup_") + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers templ(R"( function (value) -> cleaned { @@ -1606,7 +1606,7 @@ string YulUtilFunctions::cleanupFunction(Type const& _type) string YulUtilFunctions::validatorFunction(Type const& _type, bool _revertOnFailure) { string functionName = string("validator_") + (_revertOnFailure ? "revert_" : "assert_") + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers templ(R"( function (value) { if iszero() { } @@ -1667,7 +1667,7 @@ string YulUtilFunctions::packedHashFunction( size_t sizeOnStack = 0; for (Type const* t: _givenTypes) sizeOnStack += t->sizeOnStack(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { Whiskers templ(R"( function () -> hash { let pos := mload() @@ -1688,7 +1688,7 @@ string YulUtilFunctions::forwardingRevertFunction() { bool forward = m_evmVersion.supportsReturndata(); string functionName = "revert_forward_" + to_string(forward); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { if (forward) return Whiskers(R"( function () { @@ -1715,7 +1715,7 @@ std::string YulUtilFunctions::decrementCheckedFunction(Type const& _type) string const functionName = "decrement_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { u256 minintval; // Smallest admissible value to decrement @@ -1743,7 +1743,7 @@ std::string YulUtilFunctions::incrementCheckedFunction(Type const& _type) string const functionName = "increment_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { u256 maxintval; // Biggest admissible value to increment @@ -1774,7 +1774,7 @@ string YulUtilFunctions::negateNumberCheckedFunction(Type const& _type) u256 const minintval = 0 - (u256(1) << (type.numBits() - 1)) + 1; - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (_value) -> ret { if slt(_value, ) { revert(0,0) } @@ -1794,7 +1794,7 @@ string YulUtilFunctions::zeroValueFunction(Type const& _type) string const functionName = "zero_value_for_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function () -> ret { @@ -1810,7 +1810,7 @@ string YulUtilFunctions::storageSetToZeroFunction(Type const& _type) { string const functionName = "storage_set_to_zero_" + _type.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { if (_type.isValueType()) return Whiskers(R"( function (slot, offset) { @@ -1842,7 +1842,7 @@ string YulUtilFunctions::conversionFunctionSpecial(Type const& _from, Type const _from.identifier() + "_to_" + _to.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { if ( auto fromTuple = dynamic_cast(&_from), toTuple = dynamic_cast(&_to); fromTuple && toTuple && fromTuple->components().size() == toTuple->components().size() @@ -1950,7 +1950,7 @@ string YulUtilFunctions::readFromMemoryOrCalldata(Type const& _type, bool _fromC if (_fromCalldata) solAssert(!_type.isDynamicallyEncoded(), ""); - return m_functionCollector->createFunction(functionName, [&] { + return m_functionCollector.createFunction(functionName, [&] { if (auto refType = dynamic_cast(&_type)) { solAssert(refType->sizeOnStack() == 1, ""); diff --git a/libsolidity/codegen/YulUtilFunctions.h b/libsolidity/codegen/YulUtilFunctions.h index 8c92d7b4f1ba..cf50c785f4c2 100644 --- a/libsolidity/codegen/YulUtilFunctions.h +++ b/libsolidity/codegen/YulUtilFunctions.h @@ -47,11 +47,11 @@ class YulUtilFunctions explicit YulUtilFunctions( langutil::EVMVersion _evmVersion, RevertStrings _revertStrings, - std::shared_ptr _functionCollector + MultiUseYulFunctionCollector& _functionCollector ): m_evmVersion(_evmVersion), m_revertStrings(_revertStrings), - m_functionCollector(std::move(_functionCollector)) + m_functionCollector(_functionCollector) {} /// @returns a function that combines the address and selector to a single value @@ -306,7 +306,7 @@ class YulUtilFunctions langutil::EVMVersion m_evmVersion; RevertStrings m_revertStrings; - std::shared_ptr m_functionCollector; + MultiUseYulFunctionCollector& m_functionCollector; }; } diff --git a/libsolidity/codegen/ir/IRGenerationContext.cpp b/libsolidity/codegen/ir/IRGenerationContext.cpp index 43cb6a040b84..7184247f4014 100644 --- a/libsolidity/codegen/ir/IRGenerationContext.cpp +++ b/libsolidity/codegen/ir/IRGenerationContext.cpp @@ -100,7 +100,7 @@ string IRGenerationContext::newYulVariable() string IRGenerationContext::internalDispatch(size_t _in, size_t _out) { string funName = "dispatch_internal_in_" + to_string(_in) + "_out_" + to_string(_out); - return m_functions->createFunction(funName, [&]() { + return m_functions.createFunction(funName, [&]() { Whiskers templ(R"( function (fun ) { switch fun diff --git a/libsolidity/codegen/ir/IRGenerationContext.h b/libsolidity/codegen/ir/IRGenerationContext.h index d4c29ab104df..473b62482d4a 100644 --- a/libsolidity/codegen/ir/IRGenerationContext.h +++ b/libsolidity/codegen/ir/IRGenerationContext.h @@ -56,11 +56,10 @@ class IRGenerationContext ): m_evmVersion(_evmVersion), m_revertStrings(_revertStrings), - m_optimiserSettings(std::move(_optimiserSettings)), - m_functions(std::make_shared()) + m_optimiserSettings(std::move(_optimiserSettings)) {} - std::shared_ptr functionCollector() const { return m_functions; } + MultiUseYulFunctionCollector& functionCollector() { return m_functions; } /// Sets the current inheritance hierarchy from derived to base. void setInheritanceHierarchy(std::vector _hierarchy) @@ -108,7 +107,7 @@ class IRGenerationContext std::map m_localVariables; /// Storage offsets of state variables std::map> m_stateVariables; - std::shared_ptr m_functions; + MultiUseYulFunctionCollector m_functions; size_t m_varCounter = 0; }; diff --git a/libsolidity/codegen/ir/IRGenerator.cpp b/libsolidity/codegen/ir/IRGenerator.cpp index 9c2d8807c88b..8da57605548d 100644 --- a/libsolidity/codegen/ir/IRGenerator.cpp +++ b/libsolidity/codegen/ir/IRGenerator.cpp @@ -107,7 +107,7 @@ string IRGenerator::generate(ContractDefinition const& _contract) for (auto const* contract: _contract.annotation().linearizedBaseContracts) for (auto const* fun: contract->definedFunctions()) generateFunction(*fun); - t("functions", m_context.functionCollector()->requestedFunctions()); + t("functions", m_context.functionCollector().requestedFunctions()); resetContext(_contract); m_context.setInheritanceHierarchy(_contract.annotation().linearizedBaseContracts); @@ -116,7 +116,7 @@ string IRGenerator::generate(ContractDefinition const& _contract) for (auto const* contract: _contract.annotation().linearizedBaseContracts) for (auto const* fun: contract->definedFunctions()) generateFunction(*fun); - t("runtimeFunctions", m_context.functionCollector()->requestedFunctions()); + t("runtimeFunctions", m_context.functionCollector().requestedFunctions()); return t.render(); } @@ -130,7 +130,7 @@ string IRGenerator::generate(Block const& _block) string IRGenerator::generateFunction(FunctionDefinition const& _function) { string functionName = m_context.functionName(_function); - return m_context.functionCollector()->createFunction(functionName, [&]() { + return m_context.functionCollector().createFunction(functionName, [&]() { Whiskers t(R"( function () { @@ -160,7 +160,7 @@ string IRGenerator::generateGetter(VariableDeclaration const& _varDecl) solAssert(_varDecl.isStateVariable(), ""); if (auto const* mappingType = dynamic_cast(type)) - return m_context.functionCollector()->createFunction(functionName, [&]() { + return m_context.functionCollector().createFunction(functionName, [&]() { pair slot_offset = m_context.storageLocationOfVariable(_varDecl); solAssert(slot_offset.second == 0, ""); FunctionType funType(_varDecl); @@ -209,7 +209,7 @@ string IRGenerator::generateGetter(VariableDeclaration const& _varDecl) { solUnimplementedAssert(type->isValueType(), ""); - return m_context.functionCollector()->createFunction(functionName, [&]() { + return m_context.functionCollector().createFunction(functionName, [&]() { pair slot_offset = m_context.storageLocationOfVariable(_varDecl); return Whiskers(R"( @@ -383,11 +383,10 @@ string IRGenerator::memoryInit() void IRGenerator::resetContext(ContractDefinition const& _contract) { solAssert( - m_context.functionCollector()->requestedFunctions().empty(), + m_context.functionCollector().requestedFunctions().empty(), "Reset context while it still had functions." ); m_context = IRGenerationContext(m_evmVersion, m_context.revertStrings(), m_optimiserSettings); - m_utils = YulUtilFunctions(m_evmVersion, m_context.revertStrings(), m_context.functionCollector()); m_context.setInheritanceHierarchy(_contract.annotation().linearizedBaseContracts); for (auto const& var: ContractType(_contract).stateVariables()) From 38b219d140b646705cebf2a92bb2d9b2838a9866 Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Wed, 4 Mar 2020 16:16:30 +0100 Subject: [PATCH 75/92] Throwing stack too deep ICE in case of calling encode with too many arguments instead of invalid opcode --- libsolidity/codegen/CompilerUtils.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libsolidity/codegen/CompilerUtils.cpp b/libsolidity/codegen/CompilerUtils.cpp index 4c75081ffbc6..e958da05e14f 100644 --- a/libsolidity/codegen/CompilerUtils.cpp +++ b/libsolidity/codegen/CompilerUtils.cpp @@ -539,6 +539,10 @@ void CompilerUtils::encodeToMemory( if (targetType->isDynamicallySized() && !_copyDynamicDataInPlace) { // copy tail pointer (=mem_end - mem_start) to memory + solAssert( + (2 + dynPointers) <= 16, + "Stack too deep(" + to_string(2 + dynPointers) + "), try using fewer variables." + ); m_context << dupInstruction(2 + dynPointers) << Instruction::DUP2; m_context << Instruction::SUB; m_context << dupInstruction(2 + dynPointers - thisDynPointer); From 7483c6f13e818c8c75aa2e11014d0d7b7bd38a31 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Thu, 5 Mar 2020 10:37:52 +0100 Subject: [PATCH 76/92] ossfuzz: Update README.md with steps to build fuzzers via docker --- test/tools/ossfuzz/README.md | 68 ++++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/test/tools/ossfuzz/README.md b/test/tools/ossfuzz/README.md index 70469513c4e5..c360eb9a8d7f 100644 --- a/test/tools/ossfuzz/README.md +++ b/test/tools/ossfuzz/README.md @@ -1,8 +1,63 @@ ## Intro -[oss-fuzz][1] is Google's fuzzing infrastructure that performs continuous fuzzing. What this means is that, each and every upstream commit is automatically fetched by the infrastructure and fuzzed. +[oss-fuzz][1] is Google's fuzzing infrastructure that performs continuous fuzzing. What this means is that, each and every upstream commit is automatically fetched by the infrastructure and fuzzed on a daily basis. -## What does this directory contain? +## How to build fuzzers? + +We have multiple fuzzers, some based on string input and others on protobuf input. To build them, please do the following: + +- Create a local docker image from `Dockerfile.ubuntu1604.clang.ossfuzz` in the `.circleci/docker` sub-directory. Please note that this step is going to take at least an hour to complete. Therefore, it is recommended to do it when you are away from computer (and the computer is plugged to power since we do not want a battery drain). + +``` +$ cd .circleci/docker +$ docker build -t solidity-ossfuzz-local -f Dockerfile.ubuntu1604.clang.ossfuzz . +``` + +- Create and login into the docker container sourced from the image built in the previous step from the solidity parent directory + +``` +## Host +$ cd solidity +$ docker run -v `pwd`:/src/solidity -ti solidity-ossfuzz-local /bin/bash +## Docker shell +$ cd /src/solidity +``` + +- Run cmake and build fuzzer harnesses + +``` +## Docker shell +$ cd /src/solidity +$ rm -rf fuzzer-build && mkdir fuzzer-build && cd fuzzer-build +## Compile protobuf C++ bindings +$ protoc --proto_path=../test/tools/ossfuzz yulProto.proto --cpp_out=../test/tools/ossfuzz +$ protoc --proto_path=../test/tools/ossfuzz abiV2Proto.proto --cpp_out=../test/tools/ossfuzz +## Run cmake +$ export CC=clang CXX=clang++ +$ cmake -DCMAKE_TOOLCHAIN_FILE=cmake/toolchains/libfuzzer.cmake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} .. +$ make ossfuzz ossfuzz_proto ossfuzz_abiv2 -j +``` + +## Why the elaborate docker image to build fuzzers? + +For the following reasons: + +- Fuzzing binaries **must** link against libc++ and not libstdc++ + - This is [because][2] (1) MemorySanitizer (which flags uses of uninitialized memory) depends on libc++; and (2) because libc++ is instrumented (to check for memory and type errors) and libstdc++ not, the former may find more bugs. + +- Linking against libc++ requires us to compile everything solidity depends on from source (and link these against libc++ as well) + +- To reproduce the compiler versions used by upstream oss-fuzz bots, we need to reuse their docker image containing the said compiler versions + +- Some fuzzers depend on libprotobuf, libprotobuf-mutator, libevmone etc. which may not be available locally; even if they were they might not be the right versions + +## What is LIB\_FUZZING\_ENGINE? + +oss-fuzz contains multiple fuzzer back-ends i.e., fuzzers. Each back-end may require different linker flags. oss-fuzz builder bot defines the correct linker flags via a bash environment variable called `LIB_FUZZING_ENGINE`. + +For the solidity ossfuzz CI build, we use the libFuzzer back-end. This back-end requires us to manually set the `LIB_FUZZING_ENGINE` to `-fsanitize=fuzzer`. + +## What does the ossfuzz directory contain? To help oss-fuzz do this, we (as project maintainers) need to provide the following: @@ -17,12 +72,5 @@ To be consistent and aid better evaluation of the utility of the fuzzing diction - Incomplete tokens including function calls such as `msg.sender.send()` are abbreviated `.send(` to provide some leeway to the fuzzer to sythesize variants such as `address(this).send()` - Language keywords are suffixed by a whitespace with the exception of those that end a line of code such as `break;` and `continue;` -## What is libFuzzingEngine.a? - -`libFuzzingEngine.a` is an oss-fuzz-related dependency. It is present in the Dockerized environment in which Solidity's oss-fuzz code will be built. - -## Is this directory relevant for routine Solidity CI builds? - -No. This is the reason why the `add_subdirectory(ossfuzz)` cmake directive is nested under the `if (OSSFUZZ)` predicate. `OSSFUZZ` is a solidity-wide cmake option that is invoked by the ossfuzz solidity-builder-bot in order to compile solidity fuzzer binaries. - [1]: https://github.com/google/oss-fuzz +[2]: https://github.com/google/oss-fuzz/issues/1114#issuecomment-360660201 From 420f57aec301e7629721d8220699c7d6239e7333 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Thu, 5 Mar 2020 12:56:14 +0100 Subject: [PATCH 77/92] Fix yul ast import for blocks, switches and string literals. --- Changelog.md | 1 + libsolidity/ast/AsmJsonImporter.cpp | 19 ++- scripts/ASTImportTest.sh | 3 +- .../ASTJSON/assembly/empty_block.json | 95 ++++++++++++++ .../ASTJSON/assembly/empty_block.sol | 7 + .../ASTJSON/assembly/empty_block_legacy.json | 123 ++++++++++++++++++ .../ASTJSON/assembly/switch_default.json | 116 +++++++++++++++++ .../ASTJSON/assembly/switch_default.sol | 7 + .../assembly/switch_default_legacy.json | 123 ++++++++++++++++++ 9 files changed, 488 insertions(+), 6 deletions(-) create mode 100644 test/libsolidity/ASTJSON/assembly/empty_block.json create mode 100644 test/libsolidity/ASTJSON/assembly/empty_block.sol create mode 100644 test/libsolidity/ASTJSON/assembly/empty_block_legacy.json create mode 100644 test/libsolidity/ASTJSON/assembly/switch_default.json create mode 100644 test/libsolidity/ASTJSON/assembly/switch_default.sol create mode 100644 test/libsolidity/ASTJSON/assembly/switch_default_legacy.json diff --git a/Changelog.md b/Changelog.md index d187d252795c..81522f51198f 100644 --- a/Changelog.md +++ b/Changelog.md @@ -13,6 +13,7 @@ Bugfixes: * isoltest: Added new keyword `wei` to express function value in semantic tests * Standard-JSON-Interface: Fix a bug related to empty filenames and imports. * SMTChecker: Fix internal errors when analysing tuples. + * Yul AST Import: correctly import blocks as statements, switch statements and string literals. ### 0.6.3 (2020-02-18) diff --git a/libsolidity/ast/AsmJsonImporter.cpp b/libsolidity/ast/AsmJsonImporter.cpp index a47ea27f76b9..3023888414ac 100644 --- a/libsolidity/ast/AsmJsonImporter.cpp +++ b/libsolidity/ast/AsmJsonImporter.cpp @@ -104,6 +104,8 @@ yul::Statement AsmJsonImporter::createStatement(Json::Value const& _node) return createContinue(_node); else if (nodeType == "Leave") return createLeave(_node); + else if (nodeType == "Block") + return createBlock(_node); else astAssert(false, "Invalid nodeType as statement"); } @@ -158,10 +160,9 @@ yul::Literal AsmJsonImporter::createLiteral(Json::Value const& _node) lit.value = YulString{member(_node, "value").asString()}; lit.type= YulString{member(_node, "type").asString()}; - langutil::Scanner scanner{langutil::CharStream(lit.value.str(), "")}; - if (kind == "number") { + langutil::Scanner scanner{langutil::CharStream(lit.value.str(), "")}; lit.kind = yul::LiteralKind::Number; astAssert( scanner.currentToken() == Token::Number, @@ -170,6 +171,7 @@ yul::Literal AsmJsonImporter::createLiteral(Json::Value const& _node) } else if (kind == "bool") { + langutil::Scanner scanner{langutil::CharStream(lit.value.str(), "")}; lit.kind = yul::LiteralKind::Boolean; astAssert( scanner.currentToken() == Token::TrueLiteral || @@ -180,7 +182,10 @@ yul::Literal AsmJsonImporter::createLiteral(Json::Value const& _node) else if (kind == "string") { lit.kind = yul::LiteralKind::String; - astAssert(scanner.currentToken() == Token::StringLiteral, "Expected string literal!"); + astAssert( + lit.value.str().size() <= 32, + "String literal too long (" + to_string(lit.value.str().size()) + " > 32)" + ); } else solAssert(false, "unknown type of literal"); @@ -268,7 +273,11 @@ yul::If AsmJsonImporter::createIf(Json::Value const& _node) yul::Case AsmJsonImporter::createCase(Json::Value const& _node) { auto caseStatement = createAsmNode(_node); - caseStatement.value = member(_node, "value").asString() == "default" ? nullptr : make_unique(createLiteral(member(_node, "value"))); + auto const& value = member(_node, "value"); + if (value.isString()) + astAssert(value.asString() == "default", "Expected default case"); + else + caseStatement.value = make_unique(createLiteral(value)); caseStatement.body = createBlock(member(_node, "body")); return caseStatement; } @@ -276,7 +285,7 @@ yul::Case AsmJsonImporter::createCase(Json::Value const& _node) yul::Switch AsmJsonImporter::createSwitch(Json::Value const& _node) { auto switchStatement = createAsmNode(_node); - switchStatement.expression = make_unique(createExpression(member(_node, "value"))); + switchStatement.expression = make_unique(createExpression(member(_node, "expression"))); for (auto const& var: member(_node, "cases")) switchStatement.cases.emplace_back(createCase(var)); return switchStatement; diff --git a/scripts/ASTImportTest.sh b/scripts/ASTImportTest.sh index 800c5a5143af..f9172df0c51e 100755 --- a/scripts/ASTImportTest.sh +++ b/scripts/ASTImportTest.sh @@ -10,6 +10,7 @@ SOLC=${REPO_ROOT}/${SOLIDITY_BUILD_DIR}/solc/solc SPLITSOURCES=${REPO_ROOT}/scripts/splitSources.py SYNTAXTESTS_DIR="${REPO_ROOT}/test/libsolidity/syntaxTests" +ASTJSONTESTS_DIR="${REPO_ROOT}/test/libsolidity/ASTJSON" NSOURCES="$(find $SYNTAXTESTS_DIR -type f | wc -l)" # DEV_DIR="${REPO_ROOT}/../tmp/contracts/" @@ -75,7 +76,7 @@ echo "Looking at $NSOURCES .sol files..." WORKINGDIR=$PWD # for solfile in $(find $DEV_DIR -name *.sol) -for solfile in $(find $SYNTAXTESTS_DIR -name *.sol) +for solfile in $(find $SYNTAXTESTS_DIR $ASTJSONTESTS_DIR -name *.sol) do echo -n "." # create a temporary sub-directory diff --git a/test/libsolidity/ASTJSON/assembly/empty_block.json b/test/libsolidity/ASTJSON/assembly/empty_block.json new file mode 100644 index 000000000000..575b1f7f4ce3 --- /dev/null +++ b/test/libsolidity/ASTJSON/assembly/empty_block.json @@ -0,0 +1,95 @@ +{ + "absolutePath": "a", + "exportedSymbols": + { + "C": + [ + 6 + ] + }, + "id": 7, + "nodeType": "SourceUnit", + "nodes": + [ + { + "abstract": false, + "baseContracts": [], + "contractDependencies": [], + "contractKind": "contract", + "documentation": null, + "fullyImplemented": true, + "id": 6, + "linearizedBaseContracts": + [ + 6 + ], + "name": "C", + "nodeType": "ContractDefinition", + "nodes": + [ + { + "body": + { + "id": 4, + "nodeType": "Block", + "src": "42:31:1", + "statements": + [ + { + "AST": + { + "nodeType": "YulBlock", + "src": "61:6:1", + "statements": + [ + { + "nodeType": "YulBlock", + "src": "63:2:1", + "statements": [] + } + ] + }, + "evmVersion": %EVMVERSION%, + "externalReferences": [], + "id": 3, + "nodeType": "InlineAssembly", + "src": "52:15:1" + } + ] + }, + "documentation": null, + "functionSelector": "e2179b8e", + "id": 5, + "implemented": true, + "kind": "function", + "modifiers": [], + "name": "g", + "nodeType": "FunctionDefinition", + "overrides": null, + "parameters": + { + "id": 1, + "nodeType": "ParameterList", + "parameters": [], + "src": "27:2:1" + }, + "returnParameters": + { + "id": 2, + "nodeType": "ParameterList", + "parameters": [], + "src": "42:0:1" + }, + "scope": 6, + "src": "17:56:1", + "stateMutability": "view", + "virtual": false, + "visibility": "public" + } + ], + "scope": 7, + "src": "0:75:1" + } + ], + "src": "0:76:1" +} diff --git a/test/libsolidity/ASTJSON/assembly/empty_block.sol b/test/libsolidity/ASTJSON/assembly/empty_block.sol new file mode 100644 index 000000000000..23c2babbe36d --- /dev/null +++ b/test/libsolidity/ASTJSON/assembly/empty_block.sol @@ -0,0 +1,7 @@ +contract C { + function g() view public { + assembly { {} } + } +} + +// ---- diff --git a/test/libsolidity/ASTJSON/assembly/empty_block_legacy.json b/test/libsolidity/ASTJSON/assembly/empty_block_legacy.json new file mode 100644 index 000000000000..108a76a00342 --- /dev/null +++ b/test/libsolidity/ASTJSON/assembly/empty_block_legacy.json @@ -0,0 +1,123 @@ +{ + "attributes": + { + "absolutePath": "a", + "exportedSymbols": + { + "C": + [ + 6 + ] + } + }, + "children": + [ + { + "attributes": + { + "abstract": false, + "baseContracts": + [ + null + ], + "contractDependencies": + [ + null + ], + "contractKind": "contract", + "documentation": null, + "fullyImplemented": true, + "linearizedBaseContracts": + [ + 6 + ], + "name": "C", + "scope": 7 + }, + "children": + [ + { + "attributes": + { + "documentation": null, + "functionSelector": "e2179b8e", + "implemented": true, + "isConstructor": false, + "kind": "function", + "modifiers": + [ + null + ], + "name": "g", + "overrides": null, + "scope": 6, + "stateMutability": "view", + "virtual": false, + "visibility": "public" + }, + "children": + [ + { + "attributes": + { + "parameters": + [ + null + ] + }, + "children": [], + "id": 1, + "name": "ParameterList", + "src": "27:2:1" + }, + { + "attributes": + { + "parameters": + [ + null + ] + }, + "children": [], + "id": 2, + "name": "ParameterList", + "src": "42:0:1" + }, + { + "children": + [ + { + "attributes": + { + "evmVersion": %EVMVERSION%, + "externalReferences": + [ + null + ], + "operations": "{ { } }" + }, + "children": [], + "id": 3, + "name": "InlineAssembly", + "src": "52:15:1" + } + ], + "id": 4, + "name": "Block", + "src": "42:31:1" + } + ], + "id": 5, + "name": "FunctionDefinition", + "src": "17:56:1" + } + ], + "id": 6, + "name": "ContractDefinition", + "src": "0:75:1" + } + ], + "id": 7, + "name": "SourceUnit", + "src": "0:76:1" +} diff --git a/test/libsolidity/ASTJSON/assembly/switch_default.json b/test/libsolidity/ASTJSON/assembly/switch_default.json new file mode 100644 index 000000000000..c0bf264388ed --- /dev/null +++ b/test/libsolidity/ASTJSON/assembly/switch_default.json @@ -0,0 +1,116 @@ +{ + "absolutePath": "a", + "exportedSymbols": + { + "C": + [ + 6 + ] + }, + "id": 7, + "nodeType": "SourceUnit", + "nodes": + [ + { + "abstract": false, + "baseContracts": [], + "contractDependencies": [], + "contractKind": "contract", + "documentation": null, + "fullyImplemented": true, + "id": 6, + "linearizedBaseContracts": + [ + 6 + ], + "name": "C", + "nodeType": "ContractDefinition", + "nodes": + [ + { + "body": + { + "id": 4, + "nodeType": "Block", + "src": "42:48:1", + "statements": + [ + { + "AST": + { + "nodeType": "YulBlock", + "src": "61:23:1", + "statements": + [ + { + "cases": + [ + { + "body": + { + "nodeType": "YulBlock", + "src": "80:2:1", + "statements": [] + }, + "nodeType": "YulCase", + "src": "72:10:1", + "value": "default" + } + ], + "expression": + { + "kind": "number", + "nodeType": "YulLiteral", + "src": "70:1:1", + "type": "", + "value": "0" + }, + "nodeType": "YulSwitch", + "src": "63:19:1" + } + ] + }, + "evmVersion": %EVMVERSION%, + "externalReferences": [], + "id": 3, + "nodeType": "InlineAssembly", + "src": "52:32:1" + } + ] + }, + "documentation": null, + "functionSelector": "e2179b8e", + "id": 5, + "implemented": true, + "kind": "function", + "modifiers": [], + "name": "g", + "nodeType": "FunctionDefinition", + "overrides": null, + "parameters": + { + "id": 1, + "nodeType": "ParameterList", + "parameters": [], + "src": "27:2:1" + }, + "returnParameters": + { + "id": 2, + "nodeType": "ParameterList", + "parameters": [], + "src": "42:0:1" + }, + "scope": 6, + "src": "17:73:1", + "stateMutability": "view", + "virtual": false, + "visibility": "public" + } + ], + "scope": 7, + "src": "0:92:1" + } + ], + "src": "0:93:1" +} diff --git a/test/libsolidity/ASTJSON/assembly/switch_default.sol b/test/libsolidity/ASTJSON/assembly/switch_default.sol new file mode 100644 index 000000000000..8bfba2c11568 --- /dev/null +++ b/test/libsolidity/ASTJSON/assembly/switch_default.sol @@ -0,0 +1,7 @@ +contract C { + function g() view public { + assembly { switch 0 default {} } + } +} + +// ---- diff --git a/test/libsolidity/ASTJSON/assembly/switch_default_legacy.json b/test/libsolidity/ASTJSON/assembly/switch_default_legacy.json new file mode 100644 index 000000000000..2d5caa0fa446 --- /dev/null +++ b/test/libsolidity/ASTJSON/assembly/switch_default_legacy.json @@ -0,0 +1,123 @@ +{ + "attributes": + { + "absolutePath": "a", + "exportedSymbols": + { + "C": + [ + 6 + ] + } + }, + "children": + [ + { + "attributes": + { + "abstract": false, + "baseContracts": + [ + null + ], + "contractDependencies": + [ + null + ], + "contractKind": "contract", + "documentation": null, + "fullyImplemented": true, + "linearizedBaseContracts": + [ + 6 + ], + "name": "C", + "scope": 7 + }, + "children": + [ + { + "attributes": + { + "documentation": null, + "functionSelector": "e2179b8e", + "implemented": true, + "isConstructor": false, + "kind": "function", + "modifiers": + [ + null + ], + "name": "g", + "overrides": null, + "scope": 6, + "stateMutability": "view", + "virtual": false, + "visibility": "public" + }, + "children": + [ + { + "attributes": + { + "parameters": + [ + null + ] + }, + "children": [], + "id": 1, + "name": "ParameterList", + "src": "27:2:1" + }, + { + "attributes": + { + "parameters": + [ + null + ] + }, + "children": [], + "id": 2, + "name": "ParameterList", + "src": "42:0:1" + }, + { + "children": + [ + { + "attributes": + { + "evmVersion": %EVMVERSION%, + "externalReferences": + [ + null + ], + "operations": "{\n switch 0\n default { }\n}" + }, + "children": [], + "id": 3, + "name": "InlineAssembly", + "src": "52:32:1" + } + ], + "id": 4, + "name": "Block", + "src": "42:48:1" + } + ], + "id": 5, + "name": "FunctionDefinition", + "src": "17:73:1" + } + ], + "id": 6, + "name": "ContractDefinition", + "src": "0:92:1" + } + ], + "id": 7, + "name": "SourceUnit", + "src": "0:93:1" +} From 7f38cbb91d320c56118fc79dec4d4a711c53d2e6 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Fri, 6 Mar 2020 10:44:51 +0100 Subject: [PATCH 78/92] Fix calling unimplemented base function. --- Changelog.md | 1 + libsolidity/analysis/TypeChecker.cpp | 27 +++++++++---------- libsolidity/ast/Types.cpp | 10 ++++++- .../functionCall/call_unimplemented_base.sol | 14 ++++++++++ .../functionCalls/call_unimplemented_base.sol | 8 ++++++ 5 files changed, 45 insertions(+), 15 deletions(-) create mode 100644 test/libsolidity/semanticTests/functionCall/call_unimplemented_base.sol create mode 100644 test/libsolidity/syntaxTests/functionCalls/call_unimplemented_base.sol diff --git a/Changelog.md b/Changelog.md index d187d252795c..693874608fc1 100644 --- a/Changelog.md +++ b/Changelog.md @@ -10,6 +10,7 @@ Compiler Features: Bugfixes: + * Inheritance: Fix incorrect error on calling unimplemented base functions. * isoltest: Added new keyword `wei` to express function value in semantic tests * Standard-JSON-Interface: Fix a bug related to empty filenames and imports. * SMTChecker: Fix internal errors when analysing tuples. diff --git a/libsolidity/analysis/TypeChecker.cpp b/libsolidity/analysis/TypeChecker.cpp index f713ea289234..5cf003bed5b9 100644 --- a/libsolidity/analysis/TypeChecker.cpp +++ b/libsolidity/analysis/TypeChecker.cpp @@ -1703,22 +1703,21 @@ void TypeChecker::typeCheckFunctionCall( if (_functionType->kind() == FunctionType::Kind::Declaration) { - m_errorReporter.typeError( - _functionCall.location(), - "Cannot call function via contract type name." - ); + if ( + m_scope->derivesFrom(*_functionType->declaration().annotation().contract) && + !dynamic_cast(_functionType->declaration()).isImplemented() + ) + m_errorReporter.typeError( + _functionCall.location(), + "Cannot call unimplemented base function." + ); + else + m_errorReporter.typeError( + _functionCall.location(), + "Cannot call function via contract type name." + ); return; } - if (_functionType->kind() == FunctionType::Kind::Internal && _functionType->hasDeclaration()) - if (auto const* functionDefinition = dynamic_cast(&_functionType->declaration())) - // functionDefinition->annotation().contract != m_scope ensures that this is a qualified access, - // e.g. ``A.f();`` instead of a simple function call like ``f();`` (the latter is valid for unimplemented - // functions). - if (functionDefinition->annotation().contract != m_scope && !functionDefinition->isImplemented()) - m_errorReporter.typeError( - _functionCall.location(), - "Cannot call unimplemented base function." - ); // Check for unsupported use of bare static call if ( diff --git a/libsolidity/ast/Types.cpp b/libsolidity/ast/Types.cpp index c4d4c9fd4d90..258088adfe0c 100644 --- a/libsolidity/ast/Types.cpp +++ b/libsolidity/ast/Types.cpp @@ -3471,7 +3471,15 @@ MemberList::MemberMap TypeType::nativeMembers(ContractDefinition const* _current continue; if (!contract.isLibrary() && inDerivingScope && declaration->isVisibleInDerivedContracts()) - members.emplace_back(declaration->name(), declaration->type(), declaration); + { + if ( + auto const* functionDefinition = dynamic_cast(declaration); + functionDefinition && !functionDefinition->isImplemented() + ) + members.emplace_back(declaration->name(), declaration->typeViaContractName(), declaration); + else + members.emplace_back(declaration->name(), declaration->type(), declaration); + } else if ( (contract.isLibrary() && declaration->isVisibleAsLibraryMember()) || declaration->isVisibleViaContractTypeAccess() diff --git a/test/libsolidity/semanticTests/functionCall/call_unimplemented_base.sol b/test/libsolidity/semanticTests/functionCall/call_unimplemented_base.sol new file mode 100644 index 000000000000..3041f0e4b7f8 --- /dev/null +++ b/test/libsolidity/semanticTests/functionCall/call_unimplemented_base.sol @@ -0,0 +1,14 @@ +abstract contract I +{ + function a() internal view virtual returns(uint256); +} +abstract contract V is I +{ + function b() public view returns(uint256) { return a(); } +} +contract C is V +{ + function a() internal view override returns (uint256) { return 42;} +} +// ---- +// b() -> 42 diff --git a/test/libsolidity/syntaxTests/functionCalls/call_unimplemented_base.sol b/test/libsolidity/syntaxTests/functionCalls/call_unimplemented_base.sol new file mode 100644 index 000000000000..82c6eecb9733 --- /dev/null +++ b/test/libsolidity/syntaxTests/functionCalls/call_unimplemented_base.sol @@ -0,0 +1,8 @@ +abstract contract I +{ + function a() internal view virtual returns(uint256); +} +abstract contract V is I +{ + function b() public view returns(uint256) { return a(); } +} From 092827b7ad062fd59ec294a326c6b306c4ce62e3 Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Thu, 5 Mar 2020 19:26:30 +0100 Subject: [PATCH 79/92] Adding sol->yul for f.selector and f.address --- libsolidity/ast/Types.cpp | 5 ++++- .../codegen/ir/IRGeneratorForStatements.cpp | 12 ++++++++++-- .../semanticTests/viaYul/function_address.sol | 17 +++++++++++++++++ .../semanticTests/viaYul/function_selector.sol | 13 +++++++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 test/libsolidity/semanticTests/viaYul/function_address.sol create mode 100644 test/libsolidity/semanticTests/viaYul/function_selector.sol diff --git a/libsolidity/ast/Types.cpp b/libsolidity/ast/Types.cpp index c4d4c9fd4d90..78a504ddc95d 100644 --- a/libsolidity/ast/Types.cpp +++ b/libsolidity/ast/Types.cpp @@ -2924,7 +2924,10 @@ vector> FunctionType::makeStackItems() const { case Kind::External: case Kind::DelegateCall: - slots = {make_tuple("address", TypeProvider::address()), make_tuple("functionIdentifier", TypeProvider::fixedBytes(4))}; + slots = { + make_tuple("address", TypeProvider::address()), + make_tuple("functionIdentifier", TypeProvider::uint(32)) + }; break; case Kind::BareCall: case Kind::BareCallCode: diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index 38d16eb69ca9..b40768be20a9 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -800,11 +800,19 @@ void IRGeneratorForStatements::endVisit(MemberAccess const& _memberAccess) case Type::Category::Function: if (member == "selector") { - solUnimplementedAssert(false, ""); + solUnimplementedAssert( + dynamic_cast(*_memberAccess.expression().annotation().type).kind() == + FunctionType::Kind::External, "" + ); + define(IRVariable{_memberAccess}, IRVariable(_memberAccess.expression()).part("functionIdentifier")); } else if (member == "address") { - solUnimplementedAssert(false, ""); + solUnimplementedAssert( + dynamic_cast(*_memberAccess.expression().annotation().type).kind() == + FunctionType::Kind::External, "" + ); + define(IRVariable{_memberAccess}, IRVariable(_memberAccess.expression()).part("address")); } else solAssert( diff --git a/test/libsolidity/semanticTests/viaYul/function_address.sol b/test/libsolidity/semanticTests/viaYul/function_address.sol new file mode 100644 index 000000000000..402ad1481f3c --- /dev/null +++ b/test/libsolidity/semanticTests/viaYul/function_address.sol @@ -0,0 +1,17 @@ +contract C { + function f() external returns (address) { + return this.f.address; + } + function g() external returns (bool) { + return this.f.address == address(this); + } + function h(function() external a) public returns (address) { + return a.address; + } +} +// ==== +// compileViaYul: also +// ---- +// f() -> 0x0fdd67305928fcac8d213d1e47bfa6165cd0b87b +// g() -> true +// h(function): left(0x1122334400112233445566778899AABBCCDDEEFF42424242) -> 0x1122334400112233445566778899AABBCCDDEEFF diff --git a/test/libsolidity/semanticTests/viaYul/function_selector.sol b/test/libsolidity/semanticTests/viaYul/function_selector.sol new file mode 100644 index 000000000000..40faef1dd411 --- /dev/null +++ b/test/libsolidity/semanticTests/viaYul/function_selector.sol @@ -0,0 +1,13 @@ +contract C { + function f() external returns (bytes4) { + return this.f.selector; + } + function h(function() external a) public returns (bytes4) { + return a.selector; + } +} +// ==== +// compileViaYul: also +// ---- +// f() -> left(0x26121ff0) +// h(function): left(0x1122334400112233445566778899AABBCCDDEEFF42424242) -> left(0x42424242) From cdfb8723892d0b32bfa98de5a558f3f244d8fc9f Mon Sep 17 00:00:00 2001 From: chriseth Date: Sun, 8 Mar 2020 19:15:21 +0100 Subject: [PATCH 80/92] [DOCS] Fix pre-computation of salted address. --- docs/control-structures.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/control-structures.rst b/docs/control-structures.rst index 407d2523c188..d5448d0b30ed 100644 --- a/docs/control-structures.rst +++ b/docs/control-structures.rst @@ -252,7 +252,7 @@ which only need to be created if there is a dispute. /// This complicated expression just tells you how the address /// can be pre-computed. It is just there for illustration. /// You actually only need ``new D{salt: salt}(arg)``. - address predictedAddress = address(bytes20(keccak256(abi.encodePacked( + address predictedAddress = address(uint(keccak256(abi.encodePacked( byte(0xff), address(this), salt, From 2b804017fe25b230e28c8c854a50259e1cb64f8c Mon Sep 17 00:00:00 2001 From: chriseth Date: Mon, 9 Mar 2020 09:42:03 +0100 Subject: [PATCH 81/92] Fix yul links. --- docs/assembly.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/assembly.rst b/docs/assembly.rst index 7e498cd5afa1..f908c0f5769c 100644 --- a/docs/assembly.rst +++ b/docs/assembly.rst @@ -11,7 +11,7 @@ You can interleave Solidity statements with inline assembly in a language close to the one of the Ethereum virtual machine. This gives you more fine-grained control, which is especially useful when you are enhancing the language by writing libraries. -The language used for inline assembly in Solidity is called `Yul `_ +The language used for inline assembly in Solidity is called :ref:`Yul ` and it is documented in its own section. This section will only cover how the inline assembly code can interface with the surrounding Solidity code. @@ -24,7 +24,7 @@ how the inline assembly code can interface with the surrounding Solidity code. An inline assembly block is marked by ``assembly { ... }``, where the code inside -the curly braces is code in the `Yul `_ language. +the curly braces is code in the :ref:`Yul ` language. The inline assembly code can access local Solidity variables as explained below. From c8cbb980026413fb88bedc9cbdafd3cd979da215 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 3 Mar 2020 12:18:26 +0100 Subject: [PATCH 82/92] [Sol2Yul] Fixes appendExternalFunctionCall for argumentStrings.size() == 0. --- libsolidity/codegen/ir/IRGeneratorForStatements.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index 38d16eb69ca9..f35b81f4eff8 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -47,6 +47,7 @@ using namespace std; using namespace solidity; using namespace solidity::util; using namespace solidity::frontend; +using namespace std::string_literals; namespace { @@ -1204,7 +1205,7 @@ void IRGeneratorForStatements::appendExternalFunctionCall( argumentTypes.emplace_back(&type(*arg)); argumentStrings.emplace_back(IRVariable(*arg).commaSeparatedList()); } - string argumentString = ", " + joinHumanReadable(argumentStrings); + string argumentString = argumentStrings.empty() ? ""s : (", " + joinHumanReadable(argumentStrings)); solUnimplementedAssert(funKind != FunctionType::Kind::ECRecover, ""); From 2153a1ef1df8a004bc3fed4c0afca25b19a2ad81 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Mon, 9 Mar 2020 10:27:38 +0100 Subject: [PATCH 83/92] Update test/tools/ossfuzz/README.md Address review comments --- test/tools/ossfuzz/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tools/ossfuzz/README.md b/test/tools/ossfuzz/README.md index c360eb9a8d7f..944595a3f70b 100644 --- a/test/tools/ossfuzz/README.md +++ b/test/tools/ossfuzz/README.md @@ -6,14 +6,14 @@ We have multiple fuzzers, some based on string input and others on protobuf input. To build them, please do the following: -- Create a local docker image from `Dockerfile.ubuntu1604.clang.ossfuzz` in the `.circleci/docker` sub-directory. Please note that this step is going to take at least an hour to complete. Therefore, it is recommended to do it when you are away from computer (and the computer is plugged to power since we do not want a battery drain). +- Create a local docker image from `Dockerfile.ubuntu1604.clang.ossfuzz` in the `.circleci/docker` sub-directory. Please note that this step is likely to take at least an hour to complete. Therefore, it is recommended to do it when you are away from the computer (and the computer is plugged to power since we do not want a battery drain). ``` $ cd .circleci/docker $ docker build -t solidity-ossfuzz-local -f Dockerfile.ubuntu1604.clang.ossfuzz . ``` -- Create and login into the docker container sourced from the image built in the previous step from the solidity parent directory +- Login to the docker container sourced from the image built in the previous step from the solidity parent directory ``` ## Host From e210026e7455f61a93a2b89bdc50f4d2f5c2097a Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 3 Mar 2020 12:20:04 +0100 Subject: [PATCH 84/92] [Sol2Yul] Implements function-to-function cast. --- libsolidity/codegen/YulUtilFunctions.cpp | 29 +++++++++++++++++++ .../viaYul/conversion/function_cast.sol | 23 +++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 test/libsolidity/semanticTests/viaYul/conversion/function_cast.sol diff --git a/libsolidity/codegen/YulUtilFunctions.cpp b/libsolidity/codegen/YulUtilFunctions.cpp index 0472dd588aba..a2af5f777f5c 100644 --- a/libsolidity/codegen/YulUtilFunctions.cpp +++ b/libsolidity/codegen/YulUtilFunctions.cpp @@ -1333,6 +1333,35 @@ string YulUtilFunctions::allocateMemoryArrayFunction(ArrayType const& _type) string YulUtilFunctions::conversionFunction(Type const& _from, Type const& _to) { + if (_from.category() == Type::Category::Function) + { + solAssert(_to.category() == Type::Category::Function, ""); + FunctionType const& fromType = dynamic_cast(_from); + FunctionType const& targetType = dynamic_cast(_to); + solAssert( + fromType.isImplicitlyConvertibleTo(targetType) && + fromType.sizeOnStack() == targetType.sizeOnStack() && + (fromType.kind() == FunctionType::Kind::Internal || fromType.kind() == FunctionType::Kind::External) && + fromType.kind() == targetType.kind(), + "Invalid function type conversion requested." + ); + string const functionName = + "convert_" + + _from.identifier() + + "_to_" + + _to.identifier(); + return m_functionCollector->createFunction(functionName, [&]() { + return Whiskers(R"( + function (addr, functionId) -> outAddr, outFunctionId { + outAddr := addr + outFunctionId := functionId + } + )") + ("functionName", functionName) + .render(); + }); + } + if (_from.sizeOnStack() != 1 || _to.sizeOnStack() != 1) return conversionFunctionSpecial(_from, _to); diff --git a/test/libsolidity/semanticTests/viaYul/conversion/function_cast.sol b/test/libsolidity/semanticTests/viaYul/conversion/function_cast.sol new file mode 100644 index 000000000000..08462c5f8a9d --- /dev/null +++ b/test/libsolidity/semanticTests/viaYul/conversion/function_cast.sol @@ -0,0 +1,23 @@ +contract C { + function f(uint x) public pure returns (uint) { + return 2 * x; + } + function g() public view returns (function (uint) external returns (uint)) { + return this.f; + } + function h(uint x) public returns (uint) { + return this.g()(x) + 1; + } + function t() external view returns ( + function(uint) external returns (uint) a, + function(uint) external view returns (uint) b) { + a = this.f; + b = this.f; + } +} +// ==== +// compileViaYul: also +// ---- +// f(uint256): 2 -> 4 +// h(uint256): 2 -> 5 +// t() -> 0xFDD67305928FCAC8D213D1E47BFA6165CD0B87BB3DE648B0000000000000000, 0xFDD67305928FCAC8D213D1E47BFA6165CD0B87BB3DE648B0000000000000000 From 105d89bea8119ab0dc9780b3fbf0d02a6d0ca737 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 9 Mar 2020 11:42:34 +0100 Subject: [PATCH 85/92] Compilation fix. --- libsolidity/codegen/YulUtilFunctions.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsolidity/codegen/YulUtilFunctions.cpp b/libsolidity/codegen/YulUtilFunctions.cpp index 38ed88f4c7df..83e8272fe0e1 100644 --- a/libsolidity/codegen/YulUtilFunctions.cpp +++ b/libsolidity/codegen/YulUtilFunctions.cpp @@ -1350,7 +1350,7 @@ string YulUtilFunctions::conversionFunction(Type const& _from, Type const& _to) _from.identifier() + "_to_" + _to.identifier(); - return m_functionCollector->createFunction(functionName, [&]() { + return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (addr, functionId) -> outAddr, outFunctionId { outAddr := addr From 37e01a19c0c24f32ecdb978aa7c17db71fd5c769 Mon Sep 17 00:00:00 2001 From: chriseth Date: Sun, 8 Mar 2020 17:27:43 +0100 Subject: [PATCH 86/92] Fix scoping following try/catch. --- Changelog.md | 1 + libsolidity/analysis/ReferencesResolver.cpp | 16 ++++++++++++++++ libsolidity/analysis/ReferencesResolver.h | 2 ++ .../libsolidity/syntaxTests/tryCatch/scoping.sol | 10 ++++++++++ 4 files changed, 29 insertions(+) create mode 100644 test/libsolidity/syntaxTests/tryCatch/scoping.sol diff --git a/Changelog.md b/Changelog.md index a86b3ac59372..92e31b9dcbd1 100644 --- a/Changelog.md +++ b/Changelog.md @@ -12,6 +12,7 @@ Compiler Features: Bugfixes: * Inheritance: Fix incorrect error on calling unimplemented base functions. * isoltest: Added new keyword `wei` to express function value in semantic tests + * Reference Resolver: Fix scoping issue following try/catch statements. * Standard-JSON-Interface: Fix a bug related to empty filenames and imports. * SMTChecker: Fix internal errors when analysing tuples. * Yul AST Import: correctly import blocks as statements, switch statements and string literals. diff --git a/libsolidity/analysis/ReferencesResolver.cpp b/libsolidity/analysis/ReferencesResolver.cpp index 4fe2d01c9c0b..6867f049c50c 100644 --- a/libsolidity/analysis/ReferencesResolver.cpp +++ b/libsolidity/analysis/ReferencesResolver.cpp @@ -67,6 +67,22 @@ void ReferencesResolver::endVisit(Block const& _block) m_resolver.setScope(_block.scope()); } +bool ReferencesResolver::visit(TryCatchClause const& _tryCatchClause) +{ + if (!m_resolveInsideCode) + return false; + m_resolver.setScope(&_tryCatchClause); + return true; +} + +void ReferencesResolver::endVisit(TryCatchClause const& _tryCatchClause) +{ + if (!m_resolveInsideCode) + return; + + m_resolver.setScope(_tryCatchClause.scope()); +} + bool ReferencesResolver::visit(ForStatement const& _for) { if (!m_resolveInsideCode) diff --git a/libsolidity/analysis/ReferencesResolver.h b/libsolidity/analysis/ReferencesResolver.h index c560be31ee3a..488562f21565 100644 --- a/libsolidity/analysis/ReferencesResolver.h +++ b/libsolidity/analysis/ReferencesResolver.h @@ -70,6 +70,8 @@ class ReferencesResolver: private ASTConstVisitor, private yul::ASTWalker bool visit(Block const& _block) override; void endVisit(Block const& _block) override; + bool visit(TryCatchClause const& _tryCatchClause) override; + void endVisit(TryCatchClause const& _tryCatchClause) override; bool visit(ForStatement const& _for) override; void endVisit(ForStatement const& _for) override; void endVisit(VariableDeclarationStatement const& _varDeclStatement) override; diff --git a/test/libsolidity/syntaxTests/tryCatch/scoping.sol b/test/libsolidity/syntaxTests/tryCatch/scoping.sol new file mode 100644 index 000000000000..3c902c336211 --- /dev/null +++ b/test/libsolidity/syntaxTests/tryCatch/scoping.sol @@ -0,0 +1,10 @@ +contract Test { + // This checks a scoping error, + // the variable "a" was not visible + // at the assignment. + function test(address _ext) external { + try Test(_ext).test(_ext) {} catch {} + uint a = 1; + a = 3; + } +} From 29b770c43439addcb29fd79ddbf6bc37dfed0f5e Mon Sep 17 00:00:00 2001 From: a3d4 Date: Tue, 18 Feb 2020 17:13:13 +0100 Subject: [PATCH 87/92] Introduced TestCase::shouldRun(). --- test/TestCase.cpp | 35 ++++++++++++++++++----------- test/TestCase.h | 9 +++++--- test/boostTest.cpp | 3 ++- test/libsolidity/SMTCheckerTest.cpp | 23 ++++++++----------- test/libsolidity/SMTCheckerTest.h | 2 -- test/libsolidity/SemanticTest.cpp | 10 +++------ test/libsolidity/SemanticTest.h | 2 -- test/libyul/SyntaxTest.cpp | 9 +++----- test/libyul/SyntaxTest.h | 5 +---- test/tools/isoltest.cpp | 3 ++- 10 files changed, 48 insertions(+), 53 deletions(-) diff --git a/test/TestCase.cpp b/test/TestCase.cpp index 01dc51e7c601..6b0a16e27cd3 100644 --- a/test/TestCase.cpp +++ b/test/TestCase.cpp @@ -15,6 +15,7 @@ along with solidity. If not, see . */ +#include #include #include @@ -52,14 +53,18 @@ bool TestCase::isTestFilename(boost::filesystem::path const& _filename) !boost::starts_with(_filename.string(), "."); } -bool TestCase::validateSettings(langutil::EVMVersion) +void TestCase::validateSettings() { if (!m_settings.empty()) throw runtime_error( "Unknown setting(s): " + util::joinHumanReadable(m_settings | boost::adaptors::map_keys) ); - return true; +} + +bool TestCase::shouldRun() +{ + return m_shouldRun; } pair, size_t> TestCase::parseSourcesAndSettingsWithLineNumbers(istream& _stream) @@ -157,20 +162,19 @@ void TestCase::expect(string::iterator& _it, string::iterator _end, string::valu ++_it; } -bool EVMVersionRestrictedTestCase::validateSettings(langutil::EVMVersion _evmVersion) +void EVMVersionRestrictedTestCase::validateSettings() { if (!m_settings.count("EVMVersion")) - return true; + return; string versionString = m_settings["EVMVersion"]; m_validatedSettings["EVMVersion"] = versionString; m_settings.erase("EVMVersion"); - if (!TestCase::validateSettings(_evmVersion)) - return false; + TestCase::validateSettings(); if (versionString.empty()) - return true; + return; string comparator; size_t versionBegin = 0; @@ -188,18 +192,23 @@ bool EVMVersionRestrictedTestCase::validateSettings(langutil::EVMVersion _evmVer if (!version) BOOST_THROW_EXCEPTION(runtime_error{"Invalid EVM version: \"" + versionString + "\""}); + langutil::EVMVersion evmVersion = solidity::test::CommonOptions::get().evmVersion(); + bool comparisonResult; if (comparator == ">") - return _evmVersion > version; + comparisonResult = evmVersion > version; else if (comparator == ">=") - return _evmVersion >= version; + comparisonResult = evmVersion >= version; else if (comparator == "<") - return _evmVersion < version; + comparisonResult = evmVersion < version; else if (comparator == "<=") - return _evmVersion <= version; + comparisonResult = evmVersion <= version; else if (comparator == "=") - return _evmVersion == version; + comparisonResult = evmVersion == version; else if (comparator == "!") - return !(_evmVersion == version); + comparisonResult = !(evmVersion == version); else BOOST_THROW_EXCEPTION(runtime_error{"Invalid EVM comparator: \"" + comparator + "\""}); + + if (!comparisonResult) + m_shouldRun = false; } diff --git a/test/TestCase.h b/test/TestCase.h index 927490537e11..d6afff8b8262 100644 --- a/test/TestCase.h +++ b/test/TestCase.h @@ -70,10 +70,12 @@ class TestCase /// Validates the settings, i.e. moves them from m_settings to m_validatedSettings. /// Throws a runtime exception if any setting is left at this class (i.e. unknown setting). + virtual void validateSettings(); + /// Returns true, if the test case is supported in the current environment and false /// otherwise which causes this test to be skipped. /// This might check e.g. for restrictions on the EVM version. - virtual bool validateSettings(langutil::EVMVersion /*_evmVersion*/); + bool shouldRun(); protected: std::pair, std::size_t> parseSourcesAndSettingsWithLineNumbers(std::istream& _file); @@ -102,13 +104,14 @@ class TestCase std::map m_settings; /// Updated settings after validation. std::map m_validatedSettings; + + bool m_shouldRun = true; }; class EVMVersionRestrictedTestCase: public TestCase { public: - /// Returns true, if the test case is supported for EVM version @arg _evmVersion, false otherwise. - bool validateSettings(langutil::EVMVersion _evmVersion) override; + void validateSettings() override; }; } diff --git a/test/boostTest.cpp b/test/boostTest.cpp index 2b5606546eff..3137a50854f5 100644 --- a/test/boostTest.cpp +++ b/test/boostTest.cpp @@ -94,7 +94,8 @@ int registerTests( { stringstream errorStream; auto testCase = _testCaseCreator(config); - if (testCase->validateSettings(solidity::test::CommonOptions::get().evmVersion())) + testCase->validateSettings(); + if (testCase->shouldRun()) switch (testCase->run(errorStream)) { case TestCase::TestResult::Success: diff --git a/test/libsolidity/SMTCheckerTest.cpp b/test/libsolidity/SMTCheckerTest.cpp index 7db668a2d2c9..912349182784 100644 --- a/test/libsolidity/SMTCheckerTest.cpp +++ b/test/libsolidity/SMTCheckerTest.cpp @@ -44,6 +44,15 @@ SMTCheckerTest::SMTCheckerTest(string const& _filename, langutil::EVMVersion _ev } else m_enabledSolvers = smt::SMTSolverChoice::All(); + + auto available = ModelChecker::availableSolvers(); + if (!available.z3) + m_enabledSolvers.z3 = false; + if (!available.cvc4) + m_enabledSolvers.cvc4 = false; + + if (m_enabledSolvers.none()) + m_shouldRun = false; } TestCase::TestResult SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool _formatted) @@ -55,17 +64,3 @@ TestCase::TestResult SMTCheckerTest::run(ostream& _stream, string const& _linePr return printExpectationAndError(_stream, _linePrefix, _formatted) ? TestResult::Success : TestResult::Failure; } - -bool SMTCheckerTest::validateSettings(langutil::EVMVersion _evmVersion) -{ - auto available = ModelChecker::availableSolvers(); - if (!available.z3) - m_enabledSolvers.z3 = false; - if (!available.cvc4) - m_enabledSolvers.cvc4 = false; - - if (m_enabledSolvers.none()) - return false; - - return SyntaxTest::validateSettings(_evmVersion); -} diff --git a/test/libsolidity/SMTCheckerTest.h b/test/libsolidity/SMTCheckerTest.h index ce28409e6806..ad38af25cff0 100644 --- a/test/libsolidity/SMTCheckerTest.h +++ b/test/libsolidity/SMTCheckerTest.h @@ -37,8 +37,6 @@ class SMTCheckerTest: public SyntaxTest TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override; - bool validateSettings(langutil::EVMVersion _evmVersion) override; - protected: /// This is set via option SMTSolvers in the test. /// The possible options are `all`, `z3`, `cvc4`, `none`, diff --git a/test/libsolidity/SemanticTest.cpp b/test/libsolidity/SemanticTest.cpp index b2c4628daf81..97c83a3e5eee 100644 --- a/test/libsolidity/SemanticTest.cpp +++ b/test/libsolidity/SemanticTest.cpp @@ -71,6 +71,9 @@ SemanticTest::SemanticTest(string const& _filename, langutil::EVMVersion _evmVer m_settings.erase("ABIEncoderV1Only"); } + if (m_runWithABIEncoderV1Only && solidity::test::CommonOptions::get().useABIEncoderV2) + m_shouldRun = false; + if (m_settings.count("revertStrings")) { auto revertStrings = revertStringsFromString(m_settings["revertStrings"]); @@ -90,13 +93,6 @@ SemanticTest::SemanticTest(string const& _filename, langutil::EVMVersion _evmVer soltestAssert(!m_tests.empty(), "No tests specified in " + _filename); } -bool SemanticTest::validateSettings(langutil::EVMVersion _evmVersion) -{ - if (m_runWithABIEncoderV1Only && solidity::test::CommonOptions::get().useABIEncoderV2) - return false; - return EVMVersionRestrictedTestCase::validateSettings(_evmVersion); -} - TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _formatted) { for(bool compileViaYul: set{!m_runWithoutYul, m_runWithYul}) diff --git a/test/libsolidity/SemanticTest.h b/test/libsolidity/SemanticTest.h index 5cd8e5cc76f2..0ea486cad327 100644 --- a/test/libsolidity/SemanticTest.h +++ b/test/libsolidity/SemanticTest.h @@ -44,8 +44,6 @@ class SemanticTest: public SolidityExecutionFramework, public EVMVersionRestrict explicit SemanticTest(std::string const& _filename, langutil::EVMVersion _evmVersion); - bool validateSettings(langutil::EVMVersion _evmVersion) override; - TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override; void printSource(std::ostream &_stream, std::string const& _linePrefix = "", bool _formatted = false) const override; void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix = "") const override; diff --git a/test/libyul/SyntaxTest.cpp b/test/libyul/SyntaxTest.cpp index be1b9990b7d0..75e38c844d57 100644 --- a/test/libyul/SyntaxTest.cpp +++ b/test/libyul/SyntaxTest.cpp @@ -114,13 +114,12 @@ void SyntaxTest::parseAndAnalyze() } -bool SyntaxTest::validateSettings(langutil::EVMVersion _evmVersion) +void SyntaxTest::validateSettings() { - if (!CommonSyntaxTest::validateSettings(_evmVersion)) - return false; + CommonSyntaxTest::validateSettings(); if (!m_settings.count("dialect")) - return true; + return; string const dialect = m_settings["dialect"]; m_validatedSettings["dialect"] = dialect; @@ -134,6 +133,4 @@ bool SyntaxTest::validateSettings(langutil::EVMVersion _evmVersion) joinHumanReadable(validDialectNames(), ", ", " and ") + "." }); - - return true; } diff --git a/test/libyul/SyntaxTest.h b/test/libyul/SyntaxTest.h index e355a59329d4..087a6326cab7 100644 --- a/test/libyul/SyntaxTest.h +++ b/test/libyul/SyntaxTest.h @@ -42,10 +42,7 @@ class SyntaxTest: public solidity::test::CommonSyntaxTest /// Validates the settings, i.e. moves them from m_settings to m_validatedSettings. /// Throws a runtime exception if any setting is left at this class (i.e. unknown setting). - /// Returns true, if the test case is supported in the current environment and false - /// otherwise which causes this test to be skipped. - /// This might check e.g. for restrictions on the EVM version. - bool validateSettings(langutil::EVMVersion _evmVersion) override; + void validateSettings() override; protected: void parseAndAnalyze() override; }; diff --git a/test/tools/isoltest.cpp b/test/tools/isoltest.cpp index a8c8748fb45f..e439aba64dd7 100644 --- a/test/tools/isoltest.cpp +++ b/test/tools/isoltest.cpp @@ -161,7 +161,8 @@ TestTool::Result TestTool::process() (AnsiColorized(cout, formatted, {BOLD}) << m_name << ": ").flush(); m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_options.evmVersion()}); - if (m_test->validateSettings(m_options.evmVersion())) + m_test->validateSettings(); + if (m_test->shouldRun()) switch (TestCase::TestResult result = m_test->run(outputMessages, " ", formatted)) { case TestCase::TestResult::Success: From 809e3503baa8b1aa4d2bdcfb81d0ae8213fb1878 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Thu, 5 Mar 2020 10:47:01 +0100 Subject: [PATCH 88/92] Control flow analysis for inline assembly. --- Changelog.md | 1 + libevmasm/SemanticInformation.cpp | 20 ++ libevmasm/SemanticInformation.h | 2 + libsolidity/analysis/ControlFlowAnalyzer.cpp | 10 +- libsolidity/analysis/ControlFlowAnalyzer.h | 4 +- libsolidity/analysis/ControlFlowBuilder.cpp | 186 +++++++++++++++--- libsolidity/analysis/ControlFlowBuilder.h | 25 ++- libsolidity/analysis/ControlFlowGraph.h | 22 ++- libyul/ControlFlowSideEffects.h | 38 ++++ libyul/Dialect.h | 2 + libyul/backends/evm/EVMDialect.cpp | 2 + libyul/backends/wasm/WasmDialect.cpp | 16 +- .../controlFlow/leave_inside_function.sol | 11 ++ .../controlFlow/leave_outside_function.sol | 10 + .../storageReturn/assembly/for_err.sol | 29 +++ .../storageReturn/assembly/for_fine.sol | 15 ++ .../storageReturn/assembly/if_err.sol | 11 ++ .../assembly/returning_function.sol | 13 ++ .../assembly/reverting_function.sol | 13 ++ .../storageReturn/assembly/stub.sol | 10 + .../storageReturn/assembly/switch_err.sol | 27 +++ .../storageReturn/assembly/switch_fine.sol | 25 +++ .../uninitializedAccess/assembly.sol | 2 +- .../assembly/double_revert.sol | 17 ++ .../unreachableCode/assembly/for_break.sol | 13 ++ .../unreachableCode/assembly/for_continue.sol | 12 ++ .../unreachableCode/assembly/return.sol | 17 ++ .../unreachableCode/assembly/revert.sol | 17 ++ test/libyul/Parser.cpp | 2 +- 29 files changed, 526 insertions(+), 46 deletions(-) create mode 100644 libyul/ControlFlowSideEffects.h create mode 100644 test/libsolidity/syntaxTests/controlFlow/leave_inside_function.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/leave_outside_function.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/for_err.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/for_fine.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/if_err.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/returning_function.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/reverting_function.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/stub.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/switch_err.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/switch_fine.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/double_revert.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/for_break.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/for_continue.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/return.sol create mode 100644 test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/revert.sol diff --git a/Changelog.md b/Changelog.md index 19227259587c..0b955c0cb623 100644 --- a/Changelog.md +++ b/Changelog.md @@ -3,6 +3,7 @@ Language Features: * Inline Assembly: Allow assigning to `_slot` of local storage variable pointers. * General: Deprecated `value(...)` and `gas(...)` in favor of `{value: ...}` and `{gas: ...}` + * Inline Assembly: Perform control flow analysis on inline assembly. Allows storage returns to be set in assembly only. Compiler Features: diff --git a/libevmasm/SemanticInformation.cpp b/libevmasm/SemanticInformation.cpp index b7fa069d26e3..4ea9a01aee37 100644 --- a/libevmasm/SemanticInformation.cpp +++ b/libevmasm/SemanticInformation.cpp @@ -155,6 +155,26 @@ bool SemanticInformation::terminatesControlFlow(Instruction _instruction) } } +bool SemanticInformation::reverts(AssemblyItem const& _item) +{ + if (_item.type() != Operation) + return false; + else + return reverts(_item.instruction()); +} + +bool SemanticInformation::reverts(Instruction _instruction) +{ + switch (_instruction) + { + case Instruction::INVALID: + case Instruction::REVERT: + return true; + default: + return false; + } +} + bool SemanticInformation::isDeterministic(AssemblyItem const& _item) { if (_item.type() != Operation) diff --git a/libevmasm/SemanticInformation.h b/libevmasm/SemanticInformation.h index f08c3d73d4b6..39a1b2439418 100644 --- a/libevmasm/SemanticInformation.h +++ b/libevmasm/SemanticInformation.h @@ -47,6 +47,8 @@ struct SemanticInformation static bool altersControlFlow(AssemblyItem const& _item); static bool terminatesControlFlow(AssemblyItem const& _item); static bool terminatesControlFlow(Instruction _instruction); + static bool reverts(AssemblyItem const& _item); + static bool reverts(Instruction _instruction); /// @returns false if the value put on the stack by _item depends on anything else than /// the information in the current block header, memory, storage or stack. static bool isDeterministic(AssemblyItem const& _item); diff --git a/libsolidity/analysis/ControlFlowAnalyzer.cpp b/libsolidity/analysis/ControlFlowAnalyzer.cpp index c8a824c544c2..1a001eb1e92e 100644 --- a/libsolidity/analysis/ControlFlowAnalyzer.cpp +++ b/libsolidity/analysis/ControlFlowAnalyzer.cpp @@ -37,7 +37,7 @@ bool ControlFlowAnalyzer::visit(FunctionDefinition const& _function) { auto const& functionFlow = m_cfg.functionFlow(_function); checkUninitializedAccess(functionFlow.entry, functionFlow.exit); - checkUnreachable(functionFlow.entry, functionFlow.exit, functionFlow.revert); + checkUnreachable(functionFlow.entry, functionFlow.exit, functionFlow.revert, functionFlow.transactionReturn); } return false; } @@ -137,7 +137,7 @@ void ControlFlowAnalyzer::checkUninitializedAccess(CFGNode const* _entry, CFGNod m_errorReporter.typeError( variableOccurrence->occurrence() ? - variableOccurrence->occurrence()->location() : + *variableOccurrence->occurrence() : variableOccurrence->declaration().location(), ssl, string("This variable is of storage pointer type and can be ") + @@ -148,7 +148,7 @@ void ControlFlowAnalyzer::checkUninitializedAccess(CFGNode const* _entry, CFGNod } } -void ControlFlowAnalyzer::checkUnreachable(CFGNode const* _entry, CFGNode const* _exit, CFGNode const* _revert) const +void ControlFlowAnalyzer::checkUnreachable(CFGNode const* _entry, CFGNode const* _exit, CFGNode const* _revert, CFGNode const* _transactionReturn) const { // collect all nodes reachable from the entry point std::set reachable = util::BreadthFirstSearch{{_entry}}.run( @@ -158,10 +158,10 @@ void ControlFlowAnalyzer::checkUnreachable(CFGNode const* _entry, CFGNode const* } ).visited; - // traverse all paths backwards from exit and revert + // traverse all paths backwards from exit, revert and transaction return // and extract (valid) source locations of unreachable nodes into sorted set std::set unreachable; - util::BreadthFirstSearch{{_exit, _revert}}.run( + util::BreadthFirstSearch{{_exit, _revert, _transactionReturn}}.run( [&](CFGNode const* _node, auto&& _addChild) { if (!reachable.count(_node) && _node->location.isValid()) unreachable.insert(_node->location); diff --git a/libsolidity/analysis/ControlFlowAnalyzer.h b/libsolidity/analysis/ControlFlowAnalyzer.h index 24af990ab601..a839c5679f96 100644 --- a/libsolidity/analysis/ControlFlowAnalyzer.h +++ b/libsolidity/analysis/ControlFlowAnalyzer.h @@ -36,9 +36,9 @@ class ControlFlowAnalyzer: private ASTConstVisitor private: /// Checks for uninitialized variable accesses in the control flow between @param _entry and @param _exit. void checkUninitializedAccess(CFGNode const* _entry, CFGNode const* _exit) const; - /// Checks for unreachable code, i.e. code ending in @param _exit or @param _revert + /// Checks for unreachable code, i.e. code ending in @param _exit, @param _revert or @param _transactionReturn /// that can not be reached from @param _entry. - void checkUnreachable(CFGNode const* _entry, CFGNode const* _exit, CFGNode const* _revert) const; + void checkUnreachable(CFGNode const* _entry, CFGNode const* _exit, CFGNode const* _revert, CFGNode const* _transactionReturn) const; CFG const& m_cfg; langutil::ErrorReporter& m_errorReporter; diff --git a/libsolidity/analysis/ControlFlowBuilder.cpp b/libsolidity/analysis/ControlFlowBuilder.cpp index c7504897b71b..f4b6b5d5fdee 100644 --- a/libsolidity/analysis/ControlFlowBuilder.cpp +++ b/libsolidity/analysis/ControlFlowBuilder.cpp @@ -16,6 +16,8 @@ */ #include +#include +#include using namespace solidity; using namespace solidity::langutil; @@ -26,10 +28,12 @@ ControlFlowBuilder::ControlFlowBuilder(CFG::NodeContainer& _nodeContainer, Funct m_nodeContainer(_nodeContainer), m_currentNode(_functionFlow.entry), m_returnNode(_functionFlow.exit), - m_revertNode(_functionFlow.revert) + m_revertNode(_functionFlow.revert), + m_transactionReturnNode(_functionFlow.transactionReturn) { } + unique_ptr ControlFlowBuilder::createFunctionFlow( CFG::NodeContainer& _nodeContainer, FunctionDefinition const& _function @@ -39,6 +43,7 @@ unique_ptr ControlFlowBuilder::createFunctionFlow( functionFlow->entry = _nodeContainer.newNode(); functionFlow->exit = _nodeContainer.newNode(); functionFlow->revert = _nodeContainer.newNode(); + functionFlow->transactionReturn = _nodeContainer.newNode(); ControlFlowBuilder builder(_nodeContainer, *functionFlow); builder.appendControlFlow(_function); @@ -131,17 +136,17 @@ bool ControlFlowBuilder::visit(ForStatement const& _forStatement) if (_forStatement.condition()) appendControlFlow(*_forStatement.condition()); - auto loopExpression = newLabel(); + auto postPart = newLabel(); auto nodes = splitFlow<2>(); auto afterFor = nodes[1]; m_currentNode = nodes[0]; { - BreakContinueScope scope(*this, afterFor, loopExpression); + BreakContinueScope scope(*this, afterFor, postPart); appendControlFlow(_forStatement.body()); } - placeAndConnectLabel(loopExpression); + placeAndConnectLabel(postPart); if (auto expression = _forStatement.loopExpression()) appendControlFlow(*expression); @@ -315,8 +320,7 @@ bool ControlFlowBuilder::visit(FunctionDefinition const& _functionDefinition) appendControlFlow(*returnParameter); m_returnNode->variableOccurrences.emplace_back( *returnParameter, - VariableOccurrence::Kind::Return, - nullptr + VariableOccurrence::Kind::Return ); } @@ -345,7 +349,7 @@ bool ControlFlowBuilder::visit(Return const& _return) m_currentNode->variableOccurrences.emplace_back( *returnParameter, VariableOccurrence::Kind::Assignment, - &_return + _return.location() ); } connect(m_currentNode, m_returnNode); @@ -363,18 +367,158 @@ bool ControlFlowBuilder::visit(FunctionTypeName const& _functionTypeName) bool ControlFlowBuilder::visit(InlineAssembly const& _inlineAssembly) { - solAssert(!!m_currentNode, ""); - visitNode(_inlineAssembly); - for (auto const& ref: _inlineAssembly.annotation().externalReferences) + solAssert(!!m_currentNode && !m_inlineAssembly, ""); + + m_inlineAssembly = &_inlineAssembly; + (*this)(_inlineAssembly.operations()); + m_inlineAssembly = nullptr; + + return false; +} + +void ControlFlowBuilder::visit(yul::Statement const& _statement) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + m_currentNode->location = langutil::SourceLocation::smallestCovering(m_currentNode->location, locationOf(_statement)); + ASTWalker::visit(_statement); +} + +void ControlFlowBuilder::operator()(yul::If const& _if) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + visit(*_if.condition); + + auto nodes = splitFlow<2>(); + m_currentNode = nodes[0]; + (*this)(_if.body); + nodes[0] = m_currentNode; + mergeFlow(nodes, nodes[1]); +} + +void ControlFlowBuilder::operator()(yul::Switch const& _switch) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + visit(*_switch.expression); + + auto beforeSwitch = m_currentNode; + + auto nodes = splitFlow(_switch.cases.size()); + for (size_t i = 0u; i < _switch.cases.size(); ++i) + { + m_currentNode = nodes[i]; + (*this)(_switch.cases[i].body); + nodes[i] = m_currentNode; + } + mergeFlow(nodes); + + bool hasDefault = util::contains_if(_switch.cases, [](yul::Case const& _case) { return !_case.value; }); + if (!hasDefault) + connect(beforeSwitch, m_currentNode); +} + +void ControlFlowBuilder::operator()(yul::ForLoop const& _forLoop) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + + (*this)(_forLoop.pre); + + auto condition = createLabelHere(); + + if (_forLoop.condition) + visit(*_forLoop.condition); + + auto loopExpression = newLabel(); + auto nodes = splitFlow<2>(); + auto afterFor = nodes[1]; + m_currentNode = nodes[0]; + + { + BreakContinueScope scope(*this, afterFor, loopExpression); + (*this)(_forLoop.body); + } + + placeAndConnectLabel(loopExpression); + + (*this)(_forLoop.post); + + connect(m_currentNode, condition); + m_currentNode = afterFor; +} + +void ControlFlowBuilder::operator()(yul::Break const&) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + solAssert(m_breakJump, ""); + connect(m_currentNode, m_breakJump); + m_currentNode = newLabel(); +} + +void ControlFlowBuilder::operator()(yul::Continue const&) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + solAssert(m_continueJump, ""); + connect(m_currentNode, m_continueJump); + m_currentNode = newLabel(); +} + +void ControlFlowBuilder::operator()(yul::Identifier const& _identifier) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + auto const& externalReferences = m_inlineAssembly->annotation().externalReferences; + if (externalReferences.count(&_identifier)) { - if (auto variableDeclaration = dynamic_cast(ref.second.declaration)) + if (auto const* declaration = dynamic_cast(externalReferences.at(&_identifier).declaration)) m_currentNode->variableOccurrences.emplace_back( - *variableDeclaration, - VariableOccurrence::Kind::InlineAssembly, - &_inlineAssembly + *declaration, + VariableOccurrence::Kind::Access, + _identifier.location ); } - return true; +} + +void ControlFlowBuilder::operator()(yul::Assignment const& _assignment) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + visit(*_assignment.value); + auto const& externalReferences = m_inlineAssembly->annotation().externalReferences; + for (auto const& variable: _assignment.variableNames) + if (externalReferences.count(&variable)) + if (auto const* declaration = dynamic_cast(externalReferences.at(&variable).declaration)) + m_currentNode->variableOccurrences.emplace_back( + *declaration, + VariableOccurrence::Kind::Assignment, + variable.location + ); +} + +void ControlFlowBuilder::operator()(yul::FunctionCall const& _functionCall) +{ + using namespace yul; + solAssert(m_currentNode && m_inlineAssembly, ""); + yul::ASTWalker::operator()(_functionCall); + + if (auto const *builtinFunction = m_inlineAssembly->dialect().builtin(_functionCall.functionName.name)) + if (builtinFunction->controlFlowSideEffects.terminates) + { + if (builtinFunction->controlFlowSideEffects.reverts) + connect(m_currentNode, m_revertNode); + else + connect(m_currentNode, m_transactionReturnNode); + m_currentNode = newLabel(); + } +} + +void ControlFlowBuilder::operator()(yul::FunctionDefinition const&) +{ + solAssert(m_currentNode && m_inlineAssembly, ""); + // External references cannot be accessed from within functions, so we can ignore their control flow. + // TODO: we might still want to track if they always revert or return, though. +} + +void ControlFlowBuilder::operator()(yul::Leave const&) +{ + // This has to be implemented, if we ever decide to visit functions. + solUnimplementedAssert(false, ""); } bool ControlFlowBuilder::visit(VariableDeclaration const& _variableDeclaration) @@ -384,8 +528,7 @@ bool ControlFlowBuilder::visit(VariableDeclaration const& _variableDeclaration) m_currentNode->variableOccurrences.emplace_back( _variableDeclaration, - VariableOccurrence::Kind::Declaration, - nullptr + VariableOccurrence::Kind::Declaration ); // Handle declaration with immediate assignment. @@ -393,14 +536,13 @@ bool ControlFlowBuilder::visit(VariableDeclaration const& _variableDeclaration) m_currentNode->variableOccurrences.emplace_back( _variableDeclaration, VariableOccurrence::Kind::Assignment, - _variableDeclaration.value().get() + _variableDeclaration.value()->location() ); // Function arguments are considered to be immediately assigned as well (they are "externally assigned"). else if (_variableDeclaration.isCallableOrCatchParameter() && !_variableDeclaration.isReturnParameter()) m_currentNode->variableOccurrences.emplace_back( _variableDeclaration, - VariableOccurrence::Kind::Assignment, - nullptr + VariableOccurrence::Kind::Assignment ); return true; } @@ -434,7 +576,7 @@ bool ControlFlowBuilder::visit(VariableDeclarationStatement const& _variableDecl m_currentNode->variableOccurrences.emplace_back( *var, VariableOccurrence::Kind::Assignment, - expression + expression ? std::make_optional(expression->location()) : std::optional{} ); } } @@ -452,7 +594,7 @@ bool ControlFlowBuilder::visit(Identifier const& _identifier) static_cast(_identifier).annotation().lValueRequested ? VariableOccurrence::Kind::Assignment : VariableOccurrence::Kind::Access, - &_identifier + _identifier.location() ); return true; diff --git a/libsolidity/analysis/ControlFlowBuilder.h b/libsolidity/analysis/ControlFlowBuilder.h index ecdedfbfefd7..ee1c3ab79af0 100644 --- a/libsolidity/analysis/ControlFlowBuilder.h +++ b/libsolidity/analysis/ControlFlowBuilder.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -30,7 +31,7 @@ namespace solidity::frontend { * Modifiers are not yet applied to the functions. This is done in a second * step in the CFG class. */ -class ControlFlowBuilder: private ASTConstVisitor +class ControlFlowBuilder: private ASTConstVisitor, private yul::ASTWalker { public: static std::unique_ptr createFunctionFlow( @@ -39,7 +40,10 @@ class ControlFlowBuilder: private ASTConstVisitor ); private: - explicit ControlFlowBuilder(CFG::NodeContainer& _nodeContainer, FunctionFlow const& _functionFlow); + explicit ControlFlowBuilder( + CFG::NodeContainer& _nodeContainer, + FunctionFlow const& _functionFlow + ); // Visits for constructing the control flow. bool visit(BinaryOperation const& _operation) override; @@ -62,6 +66,17 @@ class ControlFlowBuilder: private ASTConstVisitor // Visits for filling variable occurrences. bool visit(FunctionTypeName const& _functionTypeName) override; bool visit(InlineAssembly const& _inlineAssembly) override; + void visit(yul::Statement const& _statement) override; + void operator()(yul::If const& _if) override; + void operator()(yul::Switch const& _switch) override; + void operator()(yul::ForLoop const& _for) override; + void operator()(yul::Break const&) override; + void operator()(yul::Continue const&) override; + void operator()(yul::Identifier const& _identifier) override; + void operator()(yul::Assignment const& _assignment) override; + void operator()(yul::FunctionCall const& _functionCall) override; + void operator()(yul::FunctionDefinition const& _functionDefinition) override; + void operator()(yul::Leave const& _leave) override; bool visit(VariableDeclaration const& _variableDeclaration) override; bool visit(VariableDeclarationStatement const& _variableDeclarationStatement) override; bool visit(Identifier const& _identifier) override; @@ -70,6 +85,9 @@ class ControlFlowBuilder: private ASTConstVisitor bool visitNode(ASTNode const&) override; private: + using ASTConstVisitor::visit; + using yul::ASTWalker::visit; + using yul::ASTWalker::operator(); /// Appends the control flow of @a _node to the current control flow. void appendControlFlow(ASTNode const& _node); @@ -136,6 +154,7 @@ class ControlFlowBuilder: private ASTConstVisitor CFGNode* m_currentNode = nullptr; CFGNode* m_returnNode = nullptr; CFGNode* m_revertNode = nullptr; + CFGNode* m_transactionReturnNode = nullptr; /// The current jump destination of break Statements. CFGNode* m_breakJump = nullptr; @@ -145,6 +164,8 @@ class ControlFlowBuilder: private ASTConstVisitor CFGNode* m_placeholderEntry = nullptr; CFGNode* m_placeholderExit = nullptr; + InlineAssembly const* m_inlineAssembly = nullptr; + /// Helper class that replaces the break and continue jump destinations for the /// current scope and restores the originals at the end of the scope. class BreakContinueScope diff --git a/libsolidity/analysis/ControlFlowGraph.h b/libsolidity/analysis/ControlFlowGraph.h index e40bb7623604..93e2e3b89b7f 100644 --- a/libsolidity/analysis/ControlFlowGraph.h +++ b/libsolidity/analysis/ControlFlowGraph.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -33,8 +34,8 @@ namespace solidity::frontend /** * Occurrence of a variable in a block of control flow. * Stores the declaration of the referenced variable, the - * kind of the occurrence and possibly the node at which - * it occurred. + * kind of the occurrence and possibly the source location + * at which it occurred. */ class VariableOccurrence { @@ -47,7 +48,7 @@ class VariableOccurrence Assignment, InlineAssembly }; - VariableOccurrence(VariableDeclaration const& _declaration, Kind _kind, ASTNode const* _occurrence): + VariableOccurrence(VariableDeclaration const& _declaration, Kind _kind, std::optional const& _occurrence = {}): m_declaration(_declaration), m_occurrenceKind(_kind), m_occurrence(_occurrence) { } @@ -57,8 +58,8 @@ class VariableOccurrence { if (m_occurrence && _rhs.m_occurrence) { - if (m_occurrence->id() < _rhs.m_occurrence->id()) return true; - if (_rhs.m_occurrence->id() < m_occurrence->id()) return false; + if (*m_occurrence < *_rhs.m_occurrence) return true; + if (*_rhs.m_occurrence < *m_occurrence) return false; } else if (_rhs.m_occurrence) return true; @@ -74,14 +75,14 @@ class VariableOccurrence VariableDeclaration const& declaration() const { return m_declaration; } Kind kind() const { return m_occurrenceKind; }; - ASTNode const* occurrence() const { return m_occurrence; } + std::optional const& occurrence() const { return m_occurrence; } private: /// Declaration of the occurring variable. VariableDeclaration const& m_declaration; /// Kind of occurrence. Kind m_occurrenceKind = Kind::Access; - /// AST node at which the variable occurred, if available (may be nullptr). - ASTNode const* m_occurrence = nullptr; + /// Source location at which the variable occurred, if available (may be nullptr). + std::optional m_occurrence; }; /** @@ -119,6 +120,10 @@ struct FunctionFlow /// This node is empty does not have any exits, but may have multiple entries /// (e.g. all assert, require, revert and throw statements). CFGNode* revert = nullptr; + /// Transaction return node. Destination node for inline assembly "return" calls. + /// This node is empty and does not have any exits, but may have multiple entries + /// (e.g. all inline assembly return calls). + CFGNode* transactionReturn = nullptr; }; class CFG: private ASTConstVisitor @@ -140,7 +145,6 @@ class CFG: private ASTConstVisitor std::vector> m_nodes; }; private: - langutil::ErrorReporter& m_errorReporter; /// Node container. diff --git a/libyul/ControlFlowSideEffects.h b/libyul/ControlFlowSideEffects.h new file mode 100644 index 000000000000..5621a122b1df --- /dev/null +++ b/libyul/ControlFlowSideEffects.h @@ -0,0 +1,38 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include + +namespace solidity::yul +{ + +/** + * Side effects of code related to control flow. + */ +struct ControlFlowSideEffects +{ + /// If true, this code terminates the control flow. + /// State may or may not be reverted as indicated by the ``reverts`` flag. + bool terminates = false; + /// If true, this code reverts all state changes in the transaction. + /// Whenever this is true, ``terminates`` has to be true as well. + bool reverts = false; +}; + +} diff --git a/libyul/Dialect.h b/libyul/Dialect.h index c137791ee9c3..a2a5961775db 100644 --- a/libyul/Dialect.h +++ b/libyul/Dialect.h @@ -22,6 +22,7 @@ #include #include +#include #include @@ -42,6 +43,7 @@ struct BuiltinFunction std::vector parameters; std::vector returns; SideEffects sideEffects; + ControlFlowSideEffects controlFlowSideEffects; /// If true, this is the msize instruction. bool isMSize = false; /// If true, can only accept literals as arguments and they cannot be moved to variables. diff --git a/libyul/backends/evm/EVMDialect.cpp b/libyul/backends/evm/EVMDialect.cpp index 27a2b85e3eaf..ea6586323ce6 100644 --- a/libyul/backends/evm/EVMDialect.cpp +++ b/libyul/backends/evm/EVMDialect.cpp @@ -52,6 +52,8 @@ pair createEVMFunction( f.parameters.resize(info.args); f.returns.resize(info.ret); f.sideEffects = EVMDialect::sideEffectsOfInstruction(_instruction); + f.controlFlowSideEffects.terminates = evmasm::SemanticInformation::terminatesControlFlow(_instruction); + f.controlFlowSideEffects.reverts = evmasm::SemanticInformation::reverts(_instruction); f.isMSize = _instruction == evmasm::Instruction::MSIZE; f.literalArguments = false; f.instruction = _instruction; diff --git a/libyul/backends/wasm/WasmDialect.cpp b/libyul/backends/wasm/WasmDialect.cpp index 7f2149adcfdd..b8dc9f32f8e1 100644 --- a/libyul/backends/wasm/WasmDialect.cpp +++ b/libyul/backends/wasm/WasmDialect.cpp @@ -99,6 +99,8 @@ WasmDialect::WasmDialect() addFunction("unreachable", {}, {}, false); m_functions["unreachable"_yulstring].sideEffects.invalidatesStorage = false; m_functions["unreachable"_yulstring].sideEffects.invalidatesMemory = false; + m_functions["unreachable"_yulstring].controlFlowSideEffects.terminates = true; + m_functions["unreachable"_yulstring].controlFlowSideEffects.reverts = true; addFunction("datasize", {i64}, {i64}, true, true); addFunction("dataoffset", {i64}, {i64}, true, true); @@ -147,7 +149,12 @@ void WasmDialect::addEthereumExternals() static string const i64{"i64"}; static string const i32{"i32"}; static string const i32ptr{"i32"}; // Uses "i32" on purpose. - struct External { string name; vector parameters; vector returns; }; + struct External { + string name; + vector parameters; + vector returns; + ControlFlowSideEffects controlFlowSideEffects = {}; + }; static vector externals{ {"getAddress", {i32ptr}, {}}, {"getExternalBalance", {i32ptr, i32ptr}, {}}, @@ -175,11 +182,11 @@ void WasmDialect::addEthereumExternals() {"log", {i32ptr, i32, i32, i32ptr, i32ptr, i32ptr, i32ptr}, {}}, {"getBlockNumber", {}, {i64}}, {"getTxOrigin", {i32ptr}, {}}, - {"finish", {i32ptr, i32}, {}}, - {"revert", {i32ptr, i32}, {}}, + {"finish", {i32ptr, i32}, {}, {true, false}}, + {"revert", {i32ptr, i32}, {}, {true, true}}, {"getReturnDataSize", {}, {i32}}, {"returnDataCopy", {i32ptr, i32, i32}, {}}, - {"selfDestruct", {i32ptr}, {}}, + {"selfDestruct", {i32ptr}, {}, {true, false}}, {"getBlockTimestamp", {}, {i64}} }; for (External const& ext: externals) @@ -193,6 +200,7 @@ void WasmDialect::addEthereumExternals() f.returns.emplace_back(YulString(p)); // TODO some of them are side effect free. f.sideEffects = SideEffects::worst(); + f.controlFlowSideEffects = ext.controlFlowSideEffects; f.isMSize = false; f.sideEffects.invalidatesStorage = (ext.name == "storageStore"); f.literalArguments = false; diff --git a/test/libsolidity/syntaxTests/controlFlow/leave_inside_function.sol b/test/libsolidity/syntaxTests/controlFlow/leave_inside_function.sol new file mode 100644 index 000000000000..245106e133f7 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/leave_inside_function.sol @@ -0,0 +1,11 @@ +contract C { + function f() public pure { + assembly { + function f() { + // Make sure this doesn't trigger the unimplemented assertion in the control flow builder. + leave + } + } + } +} +// ---- diff --git a/test/libsolidity/syntaxTests/controlFlow/leave_outside_function.sol b/test/libsolidity/syntaxTests/controlFlow/leave_outside_function.sol new file mode 100644 index 000000000000..772a108e2151 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/leave_outside_function.sol @@ -0,0 +1,10 @@ +contract C { + function f() public pure { + assembly { + // Make sure this doesn't trigger the unimplemented assertion in the control flow builder. + leave + } + } +} +// ---- +// SyntaxError: (178-183): Keyword "leave" can only be used inside a function. diff --git a/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/for_err.sol b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/for_err.sol new file mode 100644 index 000000000000..909d4f333e23 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/for_err.sol @@ -0,0 +1,29 @@ +contract C { + struct S { bool f; } + S s; + function f() internal pure returns (S storage c) { + assembly { + for {} eq(0,0) { c_slot := s_slot } {} + } + } + function g() internal pure returns (S storage c) { + assembly { + for {} eq(0,1) { c_slot := s_slot } {} + } + } + function h() internal pure returns (S storage c) { + assembly { + for {} eq(0,0) {} { c_slot := s_slot } + } + } + function i() internal pure returns (S storage c) { + assembly { + for {} eq(0,1) {} { c_slot := s_slot } + } + } +} +// ---- +// TypeError: (87-98): This variable is of storage pointer type and can be returned without prior assignment, which would lead to undefined behaviour. +// TypeError: (228-239): This variable is of storage pointer type and can be returned without prior assignment, which would lead to undefined behaviour. +// TypeError: (369-380): This variable is of storage pointer type and can be returned without prior assignment, which would lead to undefined behaviour. +// TypeError: (510-521): This variable is of storage pointer type and can be returned without prior assignment, which would lead to undefined behaviour. diff --git a/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/for_fine.sol b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/for_fine.sol new file mode 100644 index 000000000000..ac3532aaf3a4 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/for_fine.sol @@ -0,0 +1,15 @@ +contract C { + struct S { bool f; } + S s; + function f() internal pure returns (S storage c) { + assembly { + for { c_slot := s_slot } iszero(0) {} {} + } + } + function g() internal pure returns (S storage c) { + assembly { + for { c_slot := s_slot } iszero(1) {} {} + } + } +} +// ---- diff --git a/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/if_err.sol b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/if_err.sol new file mode 100644 index 000000000000..2f79ee10f237 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/if_err.sol @@ -0,0 +1,11 @@ +contract C { + struct S { bool f; } + S s; + function f(bool flag) internal pure returns (S storage c) { + assembly { + if flag { c_slot := s_slot } + } + } +} +// ---- +// TypeError: (96-107): This variable is of storage pointer type and can be returned without prior assignment, which would lead to undefined behaviour. diff --git a/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/returning_function.sol b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/returning_function.sol new file mode 100644 index 000000000000..81b6bcfe7268 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/returning_function.sol @@ -0,0 +1,13 @@ +contract C { + struct S { bool f; } + S s; + function f() internal pure returns (S storage c) { + // this should warn about unreachable code, but currently function flow is ignored + assembly { + function f() { return(0, 0) } + f() + c_slot := s_slot + } + } +} +// ---- diff --git a/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/reverting_function.sol b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/reverting_function.sol new file mode 100644 index 000000000000..4619584ef8d2 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/reverting_function.sol @@ -0,0 +1,13 @@ +contract C { + struct S { bool f; } + S s; + function f() internal pure returns (S storage c) { + // this could be allowed, but currently control flow for functions is not analysed + assembly { + function f() { revert(0, 0) } + f() + } + } +} +// ---- +// TypeError: (87-98): This variable is of storage pointer type and can be returned without prior assignment, which would lead to undefined behaviour. diff --git a/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/stub.sol b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/stub.sol new file mode 100644 index 000000000000..e5873f3d1a35 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/stub.sol @@ -0,0 +1,10 @@ +contract C { + struct S { bool f; } + S s; + function f() internal pure returns (S storage c) { + assembly { + c_slot := s_slot + } + } +} +// ---- diff --git a/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/switch_err.sol b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/switch_err.sol new file mode 100644 index 000000000000..0644d4b8f48b --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/switch_err.sol @@ -0,0 +1,27 @@ +contract C { + struct S { bool f; } + S s; + function f(uint256 a) internal pure returns (S storage c) { + assembly { + switch a + case 0 { c_slot := s_slot } + } + } + function g(bool flag) internal pure returns (S storage c) { + assembly { + switch flag + case 0 { c_slot := s_slot } + case 1 { c_slot := s_slot } + } + } + function h(uint256 a) internal pure returns (S storage c) { + assembly { + switch a + case 0 { c_slot := s_slot } + default { return(0,0) } + } + } +} +// ---- +// TypeError: (96-107): This variable is of storage pointer type and can be returned without prior assignment, which would lead to undefined behaviour. +// TypeError: (256-267): This variable is of storage pointer type and can be returned without prior assignment, which would lead to undefined behaviour. diff --git a/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/switch_fine.sol b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/switch_fine.sol new file mode 100644 index 000000000000..9e73ecff1fc5 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/storageReturn/assembly/switch_fine.sol @@ -0,0 +1,25 @@ +contract C { + struct S { bool f; } + S s; + function f(uint256 a) internal pure returns (S storage c) { + assembly { + switch a + default { c_slot := s_slot } + } + } + function g(bool flag) internal pure returns (S storage c) { + assembly { + switch flag + case 0 { c_slot := s_slot } + default { c_slot := s_slot } + } + } + function h(uint256 a) internal pure returns (S storage c) { + assembly { + switch a + case 0 { revert(0, 0) } + default { c_slot := s_slot } + } + } +} +// ---- diff --git a/test/libsolidity/syntaxTests/controlFlow/uninitializedAccess/assembly.sol b/test/libsolidity/syntaxTests/controlFlow/uninitializedAccess/assembly.sol index e73d9cde6918..89a13717e9e8 100644 --- a/test/libsolidity/syntaxTests/controlFlow/uninitializedAccess/assembly.sol +++ b/test/libsolidity/syntaxTests/controlFlow/uninitializedAccess/assembly.sol @@ -6,4 +6,4 @@ contract C { } } // ---- -// TypeError: (92-116): This variable is of storage pointer type and can be accessed without prior assignment, which would lead to undefined behaviour. +// TypeError: (107-113): This variable is of storage pointer type and can be accessed without prior assignment, which would lead to undefined behaviour. diff --git a/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/double_revert.sol b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/double_revert.sol new file mode 100644 index 000000000000..8a441a5b267d --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/double_revert.sol @@ -0,0 +1,17 @@ +contract C { + function f() public pure { + assembly { + revert(0, 0) + revert(0, 0) + } + } + function g() public pure { + assembly { + revert(0, 0) + } + revert(); + } +} +// ---- +// Warning: (100-112): Unreachable code. +// Warning: (222-230): Unreachable code. diff --git a/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/for_break.sol b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/for_break.sol new file mode 100644 index 000000000000..466bf4fae504 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/for_break.sol @@ -0,0 +1,13 @@ +contract C { + function f() public pure { + assembly { + for { let a := 0} lt(a,1) { a := add(a, 1) } { + break + let b := 42 + } + } + } +} +// ---- +// Warning: (103-117): Unreachable code. +// Warning: (160-171): Unreachable code. diff --git a/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/for_continue.sol b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/for_continue.sol new file mode 100644 index 000000000000..be09967f1a00 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/for_continue.sol @@ -0,0 +1,12 @@ +contract C { + function f() public pure { + assembly { + for { let a := 0} lt(a,1) { a := add(a, 1) } { + continue + let b := 42 + } + } + } +} +// ---- +// Warning: (163-174): Unreachable code. diff --git a/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/return.sol b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/return.sol new file mode 100644 index 000000000000..9b13441b45f8 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/return.sol @@ -0,0 +1,17 @@ +contract C { + function f(uint256 y) public pure returns (uint256 x) { + assembly { + return(0, 0) + x := y + } + } + function g(uint256 y) public pure returns (uint256 x) { + assembly { + return(0, 0) + } + x = y; + } +} +// ---- +// Warning: (129-135): Unreachable code. +// Warning: (274-279): Unreachable code. diff --git a/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/revert.sol b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/revert.sol new file mode 100644 index 000000000000..66b6ae382d49 --- /dev/null +++ b/test/libsolidity/syntaxTests/controlFlow/unreachableCode/assembly/revert.sol @@ -0,0 +1,17 @@ +contract C { + function f(uint256 y) public pure returns (uint256 x) { + assembly { + revert(0, 0) + x := y + } + } + function g(uint256 y) public pure returns (uint256 x) { + assembly { + revert(0, 0) + } + x = y; + } +} +// ---- +// Warning: (129-135): Unreachable code. +// Warning: (274-279): Unreachable code. diff --git a/test/libyul/Parser.cpp b/test/libyul/Parser.cpp index 020f3d4c4430..93b5287218ad 100644 --- a/test/libyul/Parser.cpp +++ b/test/libyul/Parser.cpp @@ -539,7 +539,7 @@ BOOST_AUTO_TEST_CASE(builtins_analysis) { return _name == "builtin"_yulstring ? &f : nullptr; } - BuiltinFunction f{"builtin"_yulstring, vector(2), vector(3), {}}; + BuiltinFunction f{"builtin"_yulstring, vector(2), vector(3), {}, {}}; }; SimpleDialect dialect; From d541e222a2ee7194080cbe33edb842664fda6589 Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 10 Mar 2020 09:45:13 +0100 Subject: [PATCH 89/92] Prepare changelog for 0.6.4. --- Changelog.md | 5 ++--- docs/bugs_by_version.json | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Changelog.md b/Changelog.md index 0b955c0cb623..19e89adf5ebf 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,8 +1,8 @@ -### 0.6.4 (unreleased) +### 0.6.4 (2020-03-10) Language Features: - * Inline Assembly: Allow assigning to `_slot` of local storage variable pointers. * General: Deprecated `value(...)` and `gas(...)` in favor of `{value: ...}` and `{gas: ...}` + * Inline Assembly: Allow assigning to `_slot` of local storage variable pointers. * Inline Assembly: Perform control flow analysis on inline assembly. Allows storage returns to be set in assembly only. @@ -13,7 +13,6 @@ Compiler Features: Bugfixes: * Inheritance: Fix incorrect error on calling unimplemented base functions. - * isoltest: Added new keyword `wei` to express function value in semantic tests * Reference Resolver: Fix scoping issue following try/catch statements. * Standard-JSON-Interface: Fix a bug related to empty filenames and imports. * SMTChecker: Fix internal errors when analysing tuples. diff --git a/docs/bugs_by_version.json b/docs/bugs_by_version.json index 3bdc7d9e5427..dc32e2687d33 100644 --- a/docs/bugs_by_version.json +++ b/docs/bugs_by_version.json @@ -888,5 +888,9 @@ "0.6.3": { "bugs": [], "released": "2020-02-18" + }, + "0.6.4": { + "bugs": [], + "released": "2020-03-10" } } \ No newline at end of file From 5d7a37024854990f175c4e93a881b259b6be18b1 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 24 Feb 2020 18:06:58 +0100 Subject: [PATCH 90/92] YulUtilFunctions: convertionFunction() to also handle array string/memory casts. --- libsolidity/codegen/YulUtilFunctions.cpp | 34 +++++++++++++------ .../conversions/string_to_bytes.sol | 9 +++++ 2 files changed, 32 insertions(+), 11 deletions(-) create mode 100644 test/libsolidity/semanticTests/conversions/string_to_bytes.sol diff --git a/libsolidity/codegen/YulUtilFunctions.cpp b/libsolidity/codegen/YulUtilFunctions.cpp index 83e8272fe0e1..9abe860b314c 100644 --- a/libsolidity/codegen/YulUtilFunctions.cpp +++ b/libsolidity/codegen/YulUtilFunctions.cpp @@ -1469,22 +1469,34 @@ string YulUtilFunctions::conversionFunction(Type const& _from, Type const& _to) break; case Type::Category::Array: { - bool equal = _from == _to; - - if (!equal) + if (_from == _to) + body = "converted := value"; + else { ArrayType const& from = dynamic_cast(_from); ArrayType const& to = dynamic_cast(_to); - if (*from.mobileType() == *to.mobileType()) - equal = true; + switch (to.location()) + { + case DataLocation::Storage: + // Other cases are done explicitly in LValue::storeValue, and only possible by assignment. + solAssert( + (to.isPointer() || (from.isByteArray() && to.isByteArray())) && + from.location() == DataLocation::Storage, + "Invalid conversion to storage type." + ); + body = "converted := value"; + break; + case DataLocation::Memory: + // Copy the array to a free position in memory, unless it is already in memory. + solUnimplementedAssert(from.location() == DataLocation::Memory, "Not implemented yet."); + body = "converted := value"; + break; + case DataLocation::CallData: + solUnimplemented("Conversion of calldata types not yet implemented."); + break; + } } - - if (equal) - body = "converted := value"; - else - solUnimplementedAssert(false, "Array conversion not implemented."); - break; } case Type::Category::Struct: diff --git a/test/libsolidity/semanticTests/conversions/string_to_bytes.sol b/test/libsolidity/semanticTests/conversions/string_to_bytes.sol new file mode 100644 index 000000000000..ca258ca26b95 --- /dev/null +++ b/test/libsolidity/semanticTests/conversions/string_to_bytes.sol @@ -0,0 +1,9 @@ +contract C { + function f(string memory s) public pure returns (bytes memory t) { + t = bytes(s); + } +} +// ==== +// compileViaYul: also +// ---- +// f(string): 32, 5, "Hello" -> 32, 5, "Hello" From a3d5af30c6455d12e5f4849f5cfdd18fceb90bf6 Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 10 Mar 2020 12:55:23 +0100 Subject: [PATCH 91/92] Mention ControlFlowSideEffects explicitly to ease the burden on MSVC. --- libyul/backends/wasm/WasmDialect.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/libyul/backends/wasm/WasmDialect.cpp b/libyul/backends/wasm/WasmDialect.cpp index b8dc9f32f8e1..c9817af7668c 100644 --- a/libyul/backends/wasm/WasmDialect.cpp +++ b/libyul/backends/wasm/WasmDialect.cpp @@ -149,11 +149,12 @@ void WasmDialect::addEthereumExternals() static string const i64{"i64"}; static string const i32{"i32"}; static string const i32ptr{"i32"}; // Uses "i32" on purpose. - struct External { + struct External + { string name; vector parameters; vector returns; - ControlFlowSideEffects controlFlowSideEffects = {}; + ControlFlowSideEffects controlFlowSideEffects = ControlFlowSideEffects{}; }; static vector externals{ {"getAddress", {i32ptr}, {}}, @@ -182,11 +183,11 @@ void WasmDialect::addEthereumExternals() {"log", {i32ptr, i32, i32, i32ptr, i32ptr, i32ptr, i32ptr}, {}}, {"getBlockNumber", {}, {i64}}, {"getTxOrigin", {i32ptr}, {}}, - {"finish", {i32ptr, i32}, {}, {true, false}}, - {"revert", {i32ptr, i32}, {}, {true, true}}, + {"finish", {i32ptr, i32}, {}, ControlFlowSideEffects{true, false}}, + {"revert", {i32ptr, i32}, {}, ControlFlowSideEffects{true, true}}, {"getReturnDataSize", {}, {i32}}, {"returnDataCopy", {i32ptr, i32, i32}, {}}, - {"selfDestruct", {i32ptr}, {}, {true, false}}, + {"selfDestruct", {i32ptr}, {}, ControlFlowSideEffects{true, false}}, {"getBlockTimestamp", {}, {i64}} }; for (External const& ext: externals) From 437ab3d24ce33e74eb4be2228d92fc4ddec15554 Mon Sep 17 00:00:00 2001 From: a3d4 Date: Tue, 10 Mar 2020 14:02:16 +0100 Subject: [PATCH 92/92] Fixed ControlFlowBuilder compilation error. --- libsolidity/analysis/ControlFlowBuilder.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsolidity/analysis/ControlFlowBuilder.h b/libsolidity/analysis/ControlFlowBuilder.h index ee1c3ab79af0..2d0910d6a741 100644 --- a/libsolidity/analysis/ControlFlowBuilder.h +++ b/libsolidity/analysis/ControlFlowBuilder.h @@ -76,7 +76,7 @@ class ControlFlowBuilder: private ASTConstVisitor, private yul::ASTWalker void operator()(yul::Assignment const& _assignment) override; void operator()(yul::FunctionCall const& _functionCall) override; void operator()(yul::FunctionDefinition const& _functionDefinition) override; - void operator()(yul::Leave const& _leave) override; + void operator()(yul::Leave const& _leaveStatement) override; bool visit(VariableDeclaration const& _variableDeclaration) override; bool visit(VariableDeclarationStatement const& _variableDeclarationStatement) override; bool visit(Identifier const& _identifier) override;