/* Copyright 2019 Google LLC. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ #ifndef RUY_RUY_TEST_H_ #define RUY_RUY_TEST_H_ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ruy/allocator.h" #include "ruy/context.h" #include "ruy/context_get_ctx.h" #include "ruy/ctx.h" #include "ruy/gtest_wrapper.h" // IWYU pragma: export #include "ruy/matrix.h" // IWYU pragma: export #include "ruy/mul_params.h" // IWYU pragma: export #include "ruy/pack_common.h" #include "ruy/platform.h" #include "ruy/pmu.h" #include "ruy/reference_mul.h" #include "ruy/ruy.h" #include "ruy/size_util.h" #include "ruy/time.h" #ifdef RUY_TEST_EXTERNAL_PATHS #define EIGEN_USE_THREADS #define EIGEN_USE_CUSTOM_THREAD_POOL #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #include "third_party/eigen3/Eigen/Core" #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #pragma GCC diagnostic pop #include "third_party/gemmlowp/public/gemmlowp.h" #include "third_party/lapack/blas.h" #endif #ifdef RUY_PROFILER #include "ruy/profiler/profiler.h" #endif #ifdef __linux__ #include #include #endif namespace ruy { const float kClampRatio = 0.1f; enum class ExternalPath { kNone, kReference, kGemmlowp, kEigen, kEigenTensor, kOpenBlas }; inline std::vector* CoveredPaths() { static std::vector covered_paths; return &covered_paths; } inline const char* PathName(Path path) { #define RUY_PATHNAME_CASE(NAME) \ case Path::NAME: \ return #NAME; switch (path) { RUY_PATHNAME_CASE(kStandardCpp) RUY_PATHNAME_CASE(kInternalStandardCppVariant1) RUY_PATHNAME_CASE(kInternalStandardCppVariant2) RUY_PATHNAME_CASE(kInternalStandardCppVariant3) #if RUY_PLATFORM_NEON RUY_PATHNAME_CASE(kNeon) RUY_PATHNAME_CASE(kNeonDotprod) #elif RUY_PLATFORM_X86 RUY_PATHNAME_CASE(kAvx2Fma) RUY_PATHNAME_CASE(kAvx512) RUY_PATHNAME_CASE(kAvx) #endif default: RUY_CHECK(false); return nullptr; } #undef RUY_PATHNAME_CASE } inline const char* TuningName(Tuning tuning) { #define RUY_SUBPATHNAME_CASE(NAME) \ case Tuning::NAME: \ return #NAME; switch (tuning) { RUY_SUBPATHNAME_CASE(kA55ish) RUY_SUBPATHNAME_CASE(kX1) RUY_SUBPATHNAME_CASE(kGeneric) default: RUY_CHECK(false); return nullptr; } #undef RUY_SUBPATHNAME_CASE } inline const char* PathName(ExternalPath path) { #define RUY_PATHNAME_CASE(NAME) \ case ExternalPath::NAME: \ return #NAME; switch (path) { RUY_PATHNAME_CASE(kReference) RUY_PATHNAME_CASE(kGemmlowp) RUY_PATHNAME_CASE(kEigen) RUY_PATHNAME_CASE(kEigenTensor) RUY_PATHNAME_CASE(kOpenBlas) default: RUY_CHECK(false); return nullptr; } #undef RUY_PATHNAME_CASE } inline std::ostream& operator<<(std::ostream& stream, Path path) { return stream << PathName(path); } inline std::ostream& operator<<(std::ostream& stream, ExternalPath external_path) { return stream << PathName(external_path); } template std::string Join(const ContainerType& container) { if (container.empty()) { return ""; } std::ostringstream stream; auto it = container.begin(); stream << *it++; for (; it != container.end(); ++it) { stream << ", "; stream << *it; } return stream.str(); } struct LogCoveredPathsOnDestruction final { ~LogCoveredPathsOnDestruction() { std::cerr << "Covered paths: " << Join(*CoveredPaths()) << std::endl; // When we know that it would be abnormal for some path not to be covered, // we check it here. Accidentally disabling SIMD paths has occurred in the // past and is one of the biggest performance regressions imaginable. // // TODO: we should be able to require some x86 paths as well, at least // SSE4.2. #if RUY_PLATFORM_ARM // When testing on ARM32 or ARM64, make sure that we covered the NEON path. // NEON is always available on ARM64, and we treat it as always available // also on ARM32. bool found_neon = false; for (const std::string& covered_path : *CoveredPaths()) { if (covered_path == "kNeon") { found_neon = true; } } if (!found_neon) { std::cerr << "Error: we haven't tested the kNeon path as we should have.\n" << std::endl; abort(); } #endif // When testing on ARM64 ChromiumOS emulator, make sure that we covered // the dotprod path. We're getting such coverage at the moment thanks to // using a sufficiently recent emulator, and we don't want to regress that. #if RUY_PLATFORM_ARM_64 && defined RUY_TESTING_ON_CHROMIUMOS bool found_dotprod = false; for (const std::string& covered_path : *CoveredPaths()) { if (covered_path == "kNeonDotprod") { found_dotprod = true; } } if (!found_dotprod) { std::cerr << "Error: we haven't tested the kNeonDotprod path as we should " "have. At the moment, this is required on ChromiumOS as this is " "what we run emulator tests in, that currently supports " "dot-product " "instructions, and we care very much about not regressing that. " "If this test was run in an emulator, please upgrade to a newer " "emulator version. If this test was run on an actual device, and " "you need to be able to run ruy tests on devices not supporting " "dot-product instructions, get in touch with us.\n" << std::endl; abort(); } #endif } static void Singleton() { static LogCoveredPathsOnDestruction singleton; } }; enum class RandomRange { kGeneral, kAvoidMinValue, kOffCenterAvoidMinValue, kReasonableSrcZeroPoint, kReasonableDstZeroPoint, kBias }; template ::value> struct RandomRangeBounds {}; template struct RandomRangeBounds { static Scalar GetMinBound(RandomRange range) { switch (range) { case RandomRange::kGeneral: return -1; case RandomRange::kAvoidMinValue: return -1; case RandomRange::kOffCenterAvoidMinValue: return -1; case RandomRange::kReasonableSrcZeroPoint: return 0; case RandomRange::kReasonableDstZeroPoint: return 0; case RandomRange::kBias: return -1; default: RUY_CHECK(false); return 0; } } static Scalar GetMaxBound(RandomRange range) { switch (range) { case RandomRange::kGeneral: return 1; case RandomRange::kAvoidMinValue: return 1; case RandomRange::kOffCenterAvoidMinValue: return 1; case RandomRange::kReasonableSrcZeroPoint: return 0; case RandomRange::kReasonableDstZeroPoint: return 0; case RandomRange::kBias: return 1; default: RUY_CHECK(false); return 0; } } }; template Scalar WeightedSum(Scalar s1, float weight1, Scalar s2, float weight2) { float sum = s1 * weight1 + s2 * weight2; float clamped = std::min( std::numeric_limits::max(), std::max(std::numeric_limits::lowest(), sum)); return static_cast(clamped); } template Scalar Parametrized(float param) { return WeightedSum(std::numeric_limits::max(), param, std::numeric_limits::lowest(), 1 - param); } template struct RandomRangeBounds { static Scalar GetMinBound(RandomRange range) { static constexpr double offcenteredness = 0.02; // Shift lower limit by about 5 for range of 255. switch (range) { case RandomRange::kGeneral: return std::numeric_limits::lowest(); case RandomRange::kAvoidMinValue: return 1 + std::numeric_limits::lowest(); case RandomRange::kOffCenterAvoidMinValue: return 1 + std::numeric_limits::lowest() + static_cast( offcenteredness * std::numeric_limits::max() - offcenteredness * (std::numeric_limits::lowest() + 1)); case RandomRange::kReasonableSrcZeroPoint: return std::numeric_limits::lowest(); case RandomRange::kReasonableDstZeroPoint: return Parametrized(0.4); case RandomRange::kBias: return std::is_same::value ? static_cast(-10000) : 0; default: RUY_CHECK(false); return 0; } } static Scalar GetMaxBound(RandomRange range) { switch (range) { case RandomRange::kGeneral: return std::numeric_limits::max(); case RandomRange::kAvoidMinValue: return std::numeric_limits::max(); case RandomRange::kOffCenterAvoidMinValue: return std::numeric_limits::max(); case RandomRange::kReasonableSrcZeroPoint: return std::numeric_limits::max(); case RandomRange::kReasonableDstZeroPoint: return Parametrized(0.6); case RandomRange::kBias: return std::is_same::value ? static_cast(10000) : 0; default: RUY_CHECK(false); return 0; } } }; inline std::default_random_engine& global_random_engine() { static std::default_random_engine engine; return engine; } template struct UniformRandomDistribution { UniformRandomDistribution(RandomRange range) : dist(RandomRangeBounds::GetMinBound(range), RandomRangeBounds::GetMaxBound(range)) {} Scalar Get() { return dist(global_random_engine()); } // std::uniform_int_distribution is specified not to support char types, // only short and wider types. MSVC actually generates an error on // std::uniform_int_distribution. using StdDistType = typename std::conditional< std::is_floating_point::value, std::uniform_real_distribution, std::uniform_int_distribution>::type; StdDistType dist; }; template void MakeRandomScalar(UniformRandomDistribution* uniform_dist, Scalar* dst) { *dst = uniform_dist->Get(); } #if defined(__has_feature) #if __has_feature(address_sanitizer) #define RUY_TEST_BUILT_WITH_ASAN #endif #endif // Don't use separate mappings when building with Address Sanitizer, as the // manual mappings could hide actual address errors from ASan (ASan can't see // the actual buffer valid address range inside the manual mapping). #if defined __linux__ && !defined RUY_TEST_BUILT_WITH_ASAN #define RUY_TEST_USE_SEPARATE_MAPPINGS #endif template struct SeparateMappingAllocator { using value_type = T; T* allocate(std::size_t n) { #ifdef RUY_TEST_USE_SEPARATE_MAPPINGS const std::size_t page_size = getpagesize(); std::size_t buffer_size = n * sizeof(T); std::size_t rounded_buffer_size = round_up_pot(buffer_size, page_size); // We are going to map an additional page at the end of our buffer, then // unmap it, to ensure that our buffer's end is the last mapped byte, so as // to catch any overrun. void* mapping = mmap(nullptr, rounded_buffer_size + page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); RUY_CHECK_NE(mapping, MAP_FAILED); int unmap_result = munmap(static_cast(mapping) + rounded_buffer_size, page_size); RUY_CHECK_EQ(unmap_result, 0); // Clearing bytes should be redundant since mmap has do do it already, but // it does not hurt and acts as an assertion checking that we got the above // mapping and unmapping right. std::memset(mapping, 0, rounded_buffer_size); // Compute the offset to make our buffer end at the last mapped byte. std::size_t buffer_offset = rounded_buffer_size - buffer_size; void* buffer = static_cast(static_cast(mapping) + buffer_offset); return static_cast(buffer); #else T* ret = new T[n]; std::memset(ret, 0, n * sizeof(T)); return ret; #endif } void deallocate(T* p, std::size_t n) { #ifdef RUY_TEST_USE_SEPARATE_MAPPINGS // The mapped bytes are the buffer address range, rounded on both ends // to page boundary. const std::size_t page_size = getpagesize(); std::size_t buffer_size = n * sizeof(T); std::size_t rounded_buffer_size = round_up_pot(buffer_size, page_size); std::uintptr_t p_addr = reinterpret_cast(p); void* mapping = reinterpret_cast(p_addr & ~(page_size - 1)); int ret = munmap(mapping, rounded_buffer_size); RUY_CHECK_EQ(ret, 0); #else (void)n; delete[] p; #endif } }; template using SeparateMappingVector = std::vector>; template void MakeRandomVector(UniformRandomDistribution* uniform_dist, int size, std::vector* dst) { dst->resize(size); for (auto& x : *dst) { MakeRandomScalar(uniform_dist, &x); } } template void MakeRandomScalar(RandomRange range, Scalar* dst) { UniformRandomDistribution dist(range); *dst = dist.Get(); if (range == RandomRange::kReasonableDstZeroPoint || range == RandomRange::kReasonableSrcZeroPoint) { if (global_random_engine()() & 1) { *dst = SymmetricZeroPoint(); } } } template void MakeRandomVector(RandomRange range, int size, std::vector* dst) { UniformRandomDistribution dist(range); dst->resize(size); for (auto& x : *dst) { MakeRandomScalar(&dist, &x); } } enum class LayoutStyle { kUnstridedLinear, kLinear }; inline void MakeLayout(int rows, int cols, Order order, LayoutStyle layout_style, Layout* layout) { layout->set_rows(rows); layout->set_cols(cols); layout->set_order(order); const int min_stride = order == Order::kColMajor ? rows : cols; RUY_CHECK(layout_style == LayoutStyle::kUnstridedLinear || layout_style == LayoutStyle::kLinear); if (layout_style == LayoutStyle::kUnstridedLinear) { layout->set_stride(min_stride); } else { layout->set_stride(min_stride + 1); } } template struct StorageMatrix { StorageMatrix() = default; StorageMatrix(const StorageMatrix&) = delete; SeparateMappingVector data; Matrix matrix; }; inline bool IsUnstrided(const Layout& layout) { if (layout.order() == Order::kColMajor) { return layout.stride() == layout.rows(); } else { return layout.stride() == layout.cols(); } } inline bool IsRowMajor(const Layout& layout) { return layout.order() == Order::kRowMajor; } inline bool IsColMajor(const Layout& layout) { return layout.order() == Order::kColMajor; } inline int FlatSize(const Layout& layout) { const int outerdim = layout.order() == Order::kColMajor ? layout.cols() : layout.rows(); return layout.stride() * outerdim; } template void VerifyConsistentFields(const StorageMatrix& storage_matrix) { if (storage_matrix.data.empty()) { RUY_CHECK_EQ(storage_matrix.matrix.data(), nullptr); RUY_CHECK_EQ(storage_matrix.matrix.layout().rows(), 0); RUY_CHECK_EQ(storage_matrix.matrix.layout().cols(), 0); } else { RUY_CHECK_EQ(storage_matrix.matrix.data(), storage_matrix.data.data()); RUY_CHECK_EQ(FlatSize(storage_matrix.matrix.layout()), static_cast(storage_matrix.data.size())); } } template void MakeRandom(int rows, int cols, Order order, Scalar zero_point, LayoutStyle layout_style, RandomRange range, StorageMatrix* storage_matrix) { MakeLayout(rows, cols, order, layout_style, storage_matrix->matrix.mutable_layout()); storage_matrix->matrix.set_zero_point(zero_point); UniformRandomDistribution data_dist(range); MakeRandomVector(&data_dist, FlatSize(storage_matrix->matrix.layout()), &storage_matrix->data); storage_matrix->matrix.set_data(storage_matrix->data.data()); VerifyConsistentFields(*storage_matrix); } template struct TestResult { void operator=(const TestResult&) = delete; void operator=(const TestResult&&) = delete; StorageMatrix storage_matrix; Path path = Path::kNone; Tuning tuning = Tuning::kAuto; ExternalPath external_path = ExternalPath::kNone; float latency; float l1_refill_rate; float l2_refill_rate; float l3_refill_rate; float l1tlb_refill_rate; float l2tlb_refill_rate; float mispred_rate; float frontend_stall_rate; float backend_stall_rate; }; template std::string PathName(const TestResult& result) { std::string pathname; if (result.path != Path::kNone) { pathname.assign(PathName(result.path)); } else if (result.external_path != ExternalPath::kNone) { pathname.assign(PathName(result.external_path)); } else { RUY_CHECK(false); } if (result.tuning != Tuning::kAuto) { pathname.append("/"); pathname.append(TuningName(result.tuning)); } return pathname; } template struct TestSet final { using LhsScalar = tLhsScalar; using RhsScalar = tRhsScalar; using AccumScalar = tAccumScalar; using DstScalar = tDstScalar; using MulParamsType = MulParams; using TestResultType = TestResult; void Run() { MakeZeroPoints(); MakeLhsRhs(); MakeMulParams(); MakeOtherParams(); MakeResultPaths(); Eval(); Verify(); } private: void MakeZeroPoints(); void MakeLhsRhs(); void MakeMulParams(); void MakeResultPaths(); void MakeOtherParams(); void EvalAndVerify(); void Eval(); void Verify(); void EvalResult(TestResultType* result); void EvalRuy(TestResultType* result); void DoMul(TestResultType* result); void Benchmark(TestResultType* result); void VerifyTestResults() const; public: enum class LifeStage { kInitial, kHasZeroPoints, kHasLhsRhs, kHasMulParams, kHasOtherParams, kHasResultPaths, kEvaluated, kFinal }; ~TestSet(); LifeStage life_stage = LifeStage::kInitial; int rows = 0; int cols = 0; int depth = 0; Order lhs_order = Order::kRowMajor; Order rhs_order = Order::kColMajor; Order dst_order = Order::kColMajor; LayoutStyle layout_style = LayoutStyle::kUnstridedLinear; bool use_specified_zero_points = false; LhsScalar lhs_zero_point = 0; RhsScalar rhs_zero_point = 0; DstScalar dst_zero_point = 0; SeparateMappingVector per_channel_multiplier_fixedpoint; SeparateMappingVector per_channel_multiplier_exponent; StorageMatrix lhs; StorageMatrix rhs; MulParamsType mul_params; SeparateMappingVector bias_data; std::vector> results; std::vector paths; std::vector external_paths; bool benchmark = false; bool perchannel = false; int max_num_threads = 0; bool cache_lhs = false; bool cache_rhs = false; }; inline PmuEvents& GlobalPmuEvents() { static PmuEvents pmu; return pmu; } inline Context& GlobalContext() { // Ensure that GlobalPmuEvents is constructed before we create any context. // This ensures that pmu counters are opened before we create any worker // thread, which is necessary to count events from worker threads. GlobalPmuEvents(); static Context context; return context; } template TestSet::~TestSet() { RUY_CHECK_EQ(life_stage, LifeStage::kFinal); LogCoveredPathsOnDestruction::Singleton(); GlobalContext().ClearPrepackedCache(); } #if defined(__has_feature) #if __has_feature(thread_sanitizer) #define RUY_TSAN #endif #if __has_feature(address_sanitizer) #define RUY_ASAN #endif #endif // defined(__has_feature) template void TestSet::DoMul( TestResultType* result) { Mul(lhs.matrix, rhs.matrix, mul_params, &GlobalContext(), &result->storage_matrix.matrix); } template void TestSet::EvalRuy( TestResultType* result) { GlobalContext().set_explicit_tuning(result->tuning); if (max_num_threads) { GlobalContext().set_max_num_threads(max_num_threads); } else if (benchmark) { GlobalContext().set_max_num_threads(1); } else { GlobalContext().set_max_num_threads(1 + global_random_engine()() % 8); } get_ctx(&GlobalContext())->SetRuntimeEnabledPaths(result->path); DoMul(result); // If enabling caching, Mul is stateful, so we run it a second time to get // coverage of these aspects. if (!benchmark && (cache_lhs || cache_rhs)) { DoMul(result); } RUY_CHECK_EQ(GlobalContext().last_used_path(), result->path); GlobalContext().set_explicit_tuning(Tuning::kAuto); GlobalContext().set_max_num_threads(1); } #ifdef RUY_TEST_EXTERNAL_PATHS template void WrapGemmlowp(const Matrix& src, gemmlowp::MatrixMap* dst) { RUY_CHECK(src.layout().order() == (tOrder == gemmlowp::MapOrder::ColMajor ? Order::kColMajor : Order::kRowMajor)); *dst = gemmlowp::MatrixMap( src.data(), src.layout().rows(), src.layout().cols(), src.layout().stride()); } template void WrapGemmlowpMutable(Matrix* src, gemmlowp::MatrixMap* dst) { RUY_CHECK(src->layout().order() == (tOrder == gemmlowp::MapOrder::ColMajor ? Order::kColMajor : Order::kRowMajor)); *dst = gemmlowp::MatrixMap(src->data(), src->layout().rows(), src->layout().cols(), src->layout().stride()); } template struct GemmlowpOrder {}; template <> struct GemmlowpOrder { static constexpr gemmlowp::MapOrder kValue = gemmlowp::MapOrder::ColMajor; }; template <> struct GemmlowpOrder { static constexpr gemmlowp::MapOrder kValue = gemmlowp::MapOrder::RowMajor; }; inline gemmlowp::GemmContext& GlobalGemmlowpContext() { static gemmlowp::GemmContext context; return context; } template void EvalGemmlowp(const Matrix& lhs, const Matrix& rhs, const MulParamsType& mul_params, int max_num_threads, Matrix* dst) { static constexpr gemmlowp::MapOrder kGemmlowpLhsOrder = GemmlowpOrder::kValue; static constexpr gemmlowp::MapOrder kGemmlowpRhsOrder = GemmlowpOrder::kValue; static constexpr gemmlowp::MapOrder kGemmlowpDstOrder = GemmlowpOrder::kValue; gemmlowp::MatrixMap gemmlowp_lhs; gemmlowp::MatrixMap gemmlowp_rhs; gemmlowp::MatrixMap gemmlowp_dst; WrapGemmlowp(lhs, &gemmlowp_lhs); WrapGemmlowp(rhs, &gemmlowp_rhs); WrapGemmlowpMutable(dst, &gemmlowp_dst); gemmlowp::OutputStageScaleInt32ByFixedPointAndExponent quantize_down_stage; quantize_down_stage.result_offset_after_shift = dst->zero_point(); quantize_down_stage.result_fixedpoint_multiplier = mul_params.multiplier_fixedpoint(); quantize_down_stage.result_exponent = mul_params.multiplier_exponent(); gemmlowp::OutputStageScaleInt32ByFixedPointAndExponentPC< gemmlowp::VectorShape::Col> quantize_down_stage_pc; quantize_down_stage_pc.result_offset_after_shift = dst->zero_point(); using ColVectorMap = gemmlowp::VectorMap; quantize_down_stage_pc.result_fixedpoint_multiplier = ColVectorMap( mul_params.multiplier_fixedpoint_perchannel(), lhs.layout().rows()); quantize_down_stage_pc.result_exponent = ColVectorMap( mul_params.multiplier_exponent_perchannel(), lhs.layout().rows()); gemmlowp::OutputStageClamp clamp_stage; clamp_stage.min = mul_params.clamp_min(); clamp_stage.max = mul_params.clamp_max(); using OutputStageSaturatingCast = typename std::conditional< std::is_same::value, gemmlowp::OutputStageSaturatingCastToUint8, gemmlowp::OutputStageSaturatingCastToInt16>::type; OutputStageSaturatingCast saturating_cast_stage; GlobalGemmlowpContext().set_max_num_threads(max_num_threads ? max_num_threads : 1); if (mul_params.bias()) { using ColVectorMap = gemmlowp::VectorMap; gemmlowp::OutputStageBiasAddition bias_add_stage; bias_add_stage.bias_vector = ColVectorMap(mul_params.bias(), dst->layout().rows()); #ifndef GEMMLOWP_SSE4 // gemmlowp perchannel stuff does not build on SSE if (mul_params.multiplier_exponent_perchannel()) { const auto& output_pipeline = std::make_tuple(bias_add_stage, quantize_down_stage_pc, clamp_stage, saturating_cast_stage); gemmlowp::GemmWithOutputPipeline< LhsScalar, DstScalar, gemmlowp::L8R8WithLhsNonzeroBitDepthParams>( &GlobalGemmlowpContext(), gemmlowp_lhs, gemmlowp_rhs, &gemmlowp_dst, -lhs.zero_point(), -rhs.zero_point(), output_pipeline); } else // NOLINT[readability/braces] #endif { const auto& output_pipeline = std::make_tuple(bias_add_stage, quantize_down_stage, clamp_stage, saturating_cast_stage); gemmlowp::GemmWithOutputPipeline< LhsScalar, DstScalar, gemmlowp::L8R8WithLhsNonzeroBitDepthParams>( &GlobalGemmlowpContext(), gemmlowp_lhs, gemmlowp_rhs, &gemmlowp_dst, -lhs.zero_point(), -rhs.zero_point(), output_pipeline); } } else { #ifndef GEMMLOWP_SSE4 // gemmlowp perchannel stuff does not build on SSE if (mul_params.multiplier_exponent_perchannel()) { const auto& output_pipeline = std::make_tuple( quantize_down_stage_pc, clamp_stage, saturating_cast_stage); gemmlowp::GemmWithOutputPipeline< LhsScalar, DstScalar, gemmlowp::L8R8WithLhsNonzeroBitDepthParams>( &GlobalGemmlowpContext(), gemmlowp_lhs, gemmlowp_rhs, &gemmlowp_dst, -lhs.zero_point(), -rhs.zero_point(), output_pipeline); } else // NOLINT[readability/braces] #endif { const auto& output_pipeline = std::make_tuple( quantize_down_stage, clamp_stage, saturating_cast_stage); gemmlowp::GemmWithOutputPipeline< LhsScalar, DstScalar, gemmlowp::L8R8WithLhsNonzeroBitDepthParams>( &GlobalGemmlowpContext(), gemmlowp_lhs, gemmlowp_rhs, &gemmlowp_dst, -lhs.zero_point(), -rhs.zero_point(), output_pipeline); } } } inline constexpr int Mash(Order LhsOrder, Order RhsOrder, Order DstOrder) { return (LhsOrder == Order::kRowMajor ? 4 : 0) + (RhsOrder == Order::kRowMajor ? 2 : 0) + (DstOrder == Order::kRowMajor ? 1 : 0); } template void EvalGemmlowp(const Matrix& lhs, const Matrix& rhs, const MulParamsType& mul_params, int max_num_threads, Matrix* dst) { int index = Mash(lhs.layout().order(), rhs.layout().order(), dst->layout().order()); switch (index) { #define EVALGEMMLOWP_CASE3(LHS, RHS, DST) \ case Mash(LHS, RHS, DST): \ return EvalGemmlowp(lhs, rhs, mul_params, max_num_threads, \ dst); #define EVALGEMMLOWP_CASE2(LHS, RHS) \ EVALGEMMLOWP_CASE3(LHS, RHS, Order::kColMajor) \ EVALGEMMLOWP_CASE3(LHS, RHS, Order::kRowMajor) #define EVALGEMMLOWP_CASE1(LHS) \ EVALGEMMLOWP_CASE2(LHS, Order::kColMajor) \ EVALGEMMLOWP_CASE2(LHS, Order::kRowMajor) EVALGEMMLOWP_CASE1(Order::kColMajor) EVALGEMMLOWP_CASE1(Order::kRowMajor) #undef EVALGEMMLOWP_CASE1 #undef EVALGEMMLOWP_CASE2 #undef EVALGEMMLOWP_CASE3 default: RUY_CHECK(false); } } template struct EigenOrder {}; template <> struct EigenOrder { static constexpr int kValue = Eigen::ColMajor; }; template <> struct EigenOrder { static constexpr int kValue = Eigen::RowMajor; }; template void EvalEigen(const Matrix& lhs, const Matrix& rhs, const MulParamsType& mul_params, int max_num_threads, Matrix* dst) { RUY_CHECK_EQ(lhs.zero_point(), 0); RUY_CHECK_EQ(rhs.zero_point(), 0); RUY_CHECK_EQ(dst->zero_point(), 0); RUY_CHECK_EQ(mul_params.multiplier_fixedpoint(), 0); RUY_CHECK_EQ(mul_params.multiplier_exponent(), 0); static constexpr int kEigenLhsOrder = EigenOrder::kValue; static constexpr int kEigenRhsOrder = EigenOrder::kValue; static constexpr int kEigenDstOrder = EigenOrder::kValue; using EigenLhsType = typename Eigen::Matrix:: template StridedConstMapType>::type; using EigenRhsType = typename Eigen::Matrix:: template StridedConstMapType>::type; using EigenDstType = typename Eigen::Matrix:: template StridedMapType>::type; using EigenBiasType = typename Eigen::Matrix::ConstMapType; EigenLhsType eigen_lhs( lhs.data(), lhs.layout().rows(), lhs.layout().cols(), Eigen::OuterStride(lhs.layout().stride())); EigenRhsType eigen_rhs( rhs.data(), rhs.layout().rows(), rhs.layout().cols(), Eigen::OuterStride(rhs.layout().stride())); EigenDstType eigen_dst( dst->data(), dst->layout().rows(), dst->layout().cols(), Eigen::OuterStride(dst->layout().stride())); Eigen::setNbThreads(max_num_threads ? max_num_threads : 1); if (mul_params.bias()) { EigenBiasType eigen_bias(mul_params.bias(), dst->layout().rows()); if (mul_params.clamp_max() == std::numeric_limits::infinity() && mul_params.clamp_min() == -std::numeric_limits::infinity()) { eigen_dst.noalias() = (eigen_lhs * eigen_rhs).colwise() + eigen_bias; } else { eigen_dst.noalias() = ((eigen_lhs * eigen_rhs).colwise() + eigen_bias) .cwiseMin(mul_params.clamp_max()) .cwiseMax(mul_params.clamp_min()); } } else { if (mul_params.clamp_max() == std::numeric_limits::infinity() && mul_params.clamp_min() == -std::numeric_limits::infinity()) { eigen_dst.noalias() = eigen_lhs * eigen_rhs; } else { eigen_dst.noalias() = (eigen_lhs * eigen_rhs) .cwiseMin(mul_params.clamp_max()) .cwiseMax(mul_params.clamp_min()); } } } template void EvalEigen(const Matrix& lhs, const Matrix& rhs, const MulParamsType& mul_params, int max_num_threads, Matrix* dst) { int index = Mash(lhs.layout().order(), rhs.layout().order(), dst->layout().order()); switch (index) { #define EVALEIGEN_CASE3(LHS, RHS, DST) \ case Mash(LHS, RHS, DST): \ return EvalEigen(lhs, rhs, mul_params, max_num_threads, dst); #define EVALEIGEN_CASE2(LHS, RHS) \ EVALEIGEN_CASE3(LHS, RHS, Order::kColMajor) \ EVALEIGEN_CASE3(LHS, RHS, Order::kRowMajor) #define EVALEIGEN_CASE1(LHS) \ EVALEIGEN_CASE2(LHS, Order::kColMajor) \ EVALEIGEN_CASE2(LHS, Order::kRowMajor) EVALEIGEN_CASE1(Order::kColMajor) EVALEIGEN_CASE1(Order::kRowMajor) #undef EVALEIGEN_CASE1 #undef EVALEIGEN_CASE2 #undef EVALEIGEN_CASE3 default: RUY_CHECK(false); } } template void EvalEigenTensor(const Matrix& lhs, const Matrix& rhs, const MulParamsType& mul_params, int max_num_threads, Matrix* dst) { RUY_CHECK_EQ(lhs.zero_point(), 0); RUY_CHECK_EQ(rhs.zero_point(), 0); RUY_CHECK_EQ(dst->zero_point(), 0); RUY_CHECK_EQ(mul_params.multiplier_fixedpoint(), 0); RUY_CHECK_EQ(mul_params.multiplier_exponent(), 0); // Eigen::TensorMap only supports unstrided layouts RUY_CHECK(IsUnstrided(lhs.layout())); RUY_CHECK(IsUnstrided(rhs.layout())); RUY_CHECK(IsUnstrided(dst->layout())); using TensorLhsType = Eigen::TensorMap>; using TensorRhsType = Eigen::TensorMap>; using TensorDstType = Eigen::TensorMap>; using TensorBiasType = Eigen::TensorMap>; const bool tr = DstOrder == Order::kRowMajor; const auto& contract_lhs = tr ? rhs : lhs; const auto& contract_rhs = tr ? lhs : rhs; TensorLhsType tensor_lhs( contract_lhs.data(), LhsOrder == Order::kColMajor ? contract_lhs.layout().rows() : contract_lhs.layout().cols(), LhsOrder == Order::kColMajor ? contract_lhs.layout().cols() : contract_lhs.layout().rows()); TensorRhsType tensor_rhs( contract_rhs.data(), RhsOrder == Order::kColMajor ? contract_rhs.layout().rows() : contract_rhs.layout().cols(), RhsOrder == Order::kColMajor ? contract_rhs.layout().cols() : contract_rhs.layout().rows()); TensorDstType tensor_dst(dst->data(), DstOrder == Order::kColMajor ? dst->layout().rows() : dst->layout().cols(), DstOrder == Order::kColMajor ? dst->layout().cols() : dst->layout().rows()); using DimPair = typename Eigen::Tensor::DimensionPair; Eigen::array contract_dims{ {DimPair((LhsOrder == Order::kColMajor) ? 1 : 0, (RhsOrder == Order::kColMajor) ? 0 : 1)}}; static Eigen::ThreadPool pool(max_num_threads ? max_num_threads : 1); static Eigen::ThreadPoolDevice device(&pool, pool.NumThreads()); if (mul_params.bias()) { TensorBiasType tensor_bias(mul_params.bias(), dst->layout().rows()); Eigen::array bias_2d_shape{tr ? 1 : dst->layout().rows(), tr ? dst->layout().rows() : 1}; Eigen::array bcast{tr ? dst->layout().cols() : 1, tr ? 1 : dst->layout().cols()}; if (mul_params.clamp_max() == std::numeric_limits::infinity() && mul_params.clamp_min() == -std::numeric_limits::infinity()) { tensor_dst.device(device) = tensor_lhs.contract(tensor_rhs, contract_dims); } else { tensor_dst.device(device) = (tensor_lhs.contract(tensor_rhs, contract_dims) + tensor_bias.reshape(bias_2d_shape).broadcast(bcast)) .cwiseMin(mul_params.clamp_max()) .cwiseMax(mul_params.clamp_min()); } } else { if (mul_params.clamp_max() == std::numeric_limits::infinity() && mul_params.clamp_min() == -std::numeric_limits::infinity()) { tensor_dst.device(device) = tensor_lhs.contract(tensor_rhs, contract_dims); } else { tensor_dst.device(device) = tensor_lhs.contract(tensor_rhs, contract_dims) .cwiseMin(mul_params.clamp_max()) .cwiseMax(mul_params.clamp_min()); } } } template void EvalEigenTensor(const Matrix& lhs, const Matrix& rhs, const MulParamsType& mul_params, int max_num_threads, Matrix* dst) { int index = Mash(lhs.layout().order(), rhs.layout().order(), dst->layout().order()); switch (index) { #define EVALEIGENTENSOR_CASE3(LHS, RHS, DST) \ case Mash(LHS, RHS, DST): \ return EvalEigenTensor(lhs, rhs, mul_params, \ max_num_threads, dst); #define EVALEIGENTENSOR_CASE2(LHS, RHS) \ EVALEIGENTENSOR_CASE3(LHS, RHS, Order::kColMajor) \ EVALEIGENTENSOR_CASE3(LHS, RHS, Order::kRowMajor) #define EVALEIGENTENSOR_CASE1(LHS) \ EVALEIGENTENSOR_CASE2(LHS, Order::kColMajor) \ EVALEIGENTENSOR_CASE2(LHS, Order::kRowMajor) EVALEIGENTENSOR_CASE1(Order::kColMajor) EVALEIGENTENSOR_CASE1(Order::kRowMajor) #undef EVALEIGENTENSOR_CASE1 #undef EVALEIGENTENSOR_CASE2 #undef EVALEIGENTENSOR_CASE3 default: RUY_CHECK(false); } } template struct GenericBlasGemm {}; template <> struct GenericBlasGemm { static void Run(char* transa, char* transb, lapack::integer* m, lapack::integer* n, lapack::integer* k, lapack::doublereal* alpha, lapack::doublereal* a, lapack::integer* lda, lapack::doublereal* b, lapack::integer* ldb, lapack::doublereal* beta, lapack::doublereal* c, lapack::integer* ldc) { dgemm_(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); } }; template <> struct GenericBlasGemm { static void Run(char* transa, char* transb, lapack::integer* m, lapack::integer* n, lapack::integer* k, lapack::real* alpha, lapack::real* a, lapack::integer* lda, lapack::real* b, lapack::integer* ldb, lapack::real* beta, lapack::real* c, lapack::integer* ldc) { sgemm_(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); } }; inline void TransposeLayout(Layout* layout) { layout->set_order((layout->order() == Order::kRowMajor) ? Order::kColMajor : Order::kRowMajor); int tmp_rows = layout->rows(); layout->set_rows(layout->cols()); layout->set_cols(tmp_rows); } template void Transpose(Matrix* matrix) { TransposeLayout(matrix->mutable_layout()); } template void EvalOpenBlas(const Matrix& lhs, const Matrix& rhs, const MulParamsType& mul_params, int max_num_threads, Matrix* dst) { RUY_CHECK_EQ(lhs.zero_point(), 0); RUY_CHECK_EQ(rhs.zero_point(), 0); RUY_CHECK_EQ(dst->zero_point(), 0); RUY_CHECK_EQ(mul_params.multiplier_fixedpoint(), 0); RUY_CHECK_EQ(mul_params.multiplier_exponent(), 0); Matrix gemm_lhs; Matrix gemm_rhs; Matrix gemm_dst; gemm_dst = *dst; // Use Transpose to reduce to the all-column-major case. // Notice that ruy::Matrix merely holds a pointer, does not own data, // so Transpose is cheap -- no actual matrix data is being transposed here. if (dst->layout().order() == Order::kColMajor) { gemm_lhs = lhs; gemm_rhs = rhs; } else { gemm_lhs = rhs; gemm_rhs = lhs; Transpose(&gemm_lhs); Transpose(&gemm_rhs); Transpose(&gemm_dst); } bool transposed_lhs = false; bool transposed_rhs = false; if (gemm_lhs.layout().order() == Order::kRowMajor) { Transpose(&gemm_lhs); transposed_lhs = true; } if (gemm_rhs.layout().order() == Order::kRowMajor) { Transpose(&gemm_rhs); transposed_rhs = true; } RUY_CHECK_EQ(gemm_lhs.layout().order(), Order::kColMajor); RUY_CHECK_EQ(gemm_rhs.layout().order(), Order::kColMajor); RUY_CHECK_EQ(gemm_dst.layout().order(), Order::kColMajor); char transa = transposed_lhs ? 'T' : 'N'; char transb = transposed_rhs ? 'T' : 'N'; int m = gemm_lhs.layout().rows(); int n = gemm_rhs.layout().cols(); int k = gemm_lhs.layout().cols(); float alpha = 1; Scalar* a = gemm_lhs.data(); int lda = gemm_lhs.layout().stride(); Scalar* b = gemm_rhs.data(); int ldb = gemm_rhs.layout().stride(); float beta = 0; Scalar* c = gemm_dst.data(); int ldc = gemm_dst.layout().stride(); GenericBlasGemm::Run(&transa, &transb, &m, &n, &k, &alpha, a, &lda, b, &ldb, &beta, c, &ldc); // BLAS does not allow us to express the bias-addition and clamping, so // we use Eigen for that. using EigenDstType = typename Eigen::Matrix:: template StridedMapType>::type; using EigenBiasType = typename Eigen::Matrix::ConstMapType; EigenDstType eigen_dst( gemm_dst.data(), gemm_dst.layout().rows(), gemm_dst.layout().cols(), Eigen::OuterStride(gemm_dst.layout().stride())); Eigen::setNbThreads(max_num_threads ? max_num_threads : 1); if (mul_params.bias()) { EigenBiasType eigen_bias(mul_params.bias(), dst->layout().rows()); if (mul_params.clamp_max() == std::numeric_limits::infinity() && mul_params.clamp_min() == -std::numeric_limits::infinity()) { eigen_dst.noalias() = eigen_dst.colwise() + eigen_bias; } else { eigen_dst.noalias() = (eigen_dst.colwise() + eigen_bias) .cwiseMin(mul_params.clamp_max()) .cwiseMax(mul_params.clamp_min()); } } else { if (mul_params.clamp_max() == std::numeric_limits::infinity() && mul_params.clamp_min() == -std::numeric_limits::infinity()) { } else { eigen_dst.noalias() = eigen_dst.cwiseMin(mul_params.clamp_max()) .cwiseMax(mul_params.clamp_min()); } } } template struct SupportsGemmlowp { static constexpr bool kValue = std::is_same::value && std::is_same::value; }; template struct UsesSingleScalarType { static constexpr bool kValue = std::is_same::value && std::is_same::value && std::is_same::value; }; template ::value, bool EnableGemmlowp = SupportsGemmlowp::kValue, bool SingleScalarType = UsesSingleScalarType::kValue> struct EvalExternalPathImpl { using DstScalar = typename TestSetType::DstScalar; static void Run(TestSetType*, TestResult*) { RUY_CHECK(false); } }; template struct EvalExternalPathImpl { using DstScalar = typename TestSetType::DstScalar; static void Run(TestSetType* test_set, TestResult* test_result) { if (test_result->external_path == ExternalPath::kEigen) { EvalEigen(test_set->lhs.matrix, test_set->rhs.matrix, test_set->mul_params, test_set->max_num_threads, &test_result->storage_matrix.matrix); } else if (test_result->external_path == ExternalPath::kEigenTensor) { EvalEigenTensor(test_set->lhs.matrix, test_set->rhs.matrix, test_set->mul_params, test_set->max_num_threads, &test_result->storage_matrix.matrix); } else if (test_result->external_path == ExternalPath::kOpenBlas) { EvalOpenBlas(test_set->lhs.matrix, test_set->rhs.matrix, test_set->mul_params, test_set->max_num_threads, &test_result->storage_matrix.matrix); } else { RUY_CHECK(false); } } }; template struct EvalExternalPathImpl { using DstScalar = typename TestSetType::DstScalar; static void Run(TestSetType* test_set, TestResult* test_result) { if (test_result->external_path == ExternalPath::kGemmlowp) { EvalGemmlowp(test_set->lhs.matrix, test_set->rhs.matrix, test_set->mul_params, test_set->max_num_threads, &test_result->storage_matrix.matrix); } else { RUY_CHECK(false); } } }; #endif // RUY_TEST_EXTERNAL_PATHS template void EvalExternalPath( TestSetType* test_set, TestResult* test_result) { if (test_result->external_path == ExternalPath::kReference) { // kReference is special because it's always available (the implementation // is provided by ruy) and supports all cases (quantized and float). ruy::ReferenceMul(test_set->lhs.matrix, test_set->rhs.matrix, test_set->mul_params, &test_result->storage_matrix.matrix); } else { #ifdef RUY_TEST_EXTERNAL_PATHS EvalExternalPathImpl::Run(test_set, test_result); #endif // RUY_TEST_EXTERNAL_PATHS } } template bool Agree(ExternalPath external_path1, const Matrix& matrix1, ExternalPath external_path2, const Matrix& matrix2, int depth) { RUY_CHECK_EQ(matrix1.layout().rows(), matrix2.layout().rows()); RUY_CHECK_EQ(matrix1.layout().cols(), matrix2.layout().cols()); RUY_CHECK_EQ(matrix1.zero_point(), matrix2.zero_point()); const int size = matrix1.layout().rows() * matrix1.layout().cols(); double tolerated_max_diff = 0; double tolerated_mean_diff = 0; const float kSmallestAllowedDifference = 4. * std::numeric_limits::epsilon(); if (std::is_floating_point::value) { double max_abs_val = 0; for (int row = 0; row < matrix1.layout().rows(); row++) { for (int col = 0; col < matrix1.layout().cols(); col++) { max_abs_val = std::max(max_abs_val, std::abs(static_cast(Element(matrix1, row, col)))); max_abs_val = std::max(max_abs_val, std::abs(static_cast(Element(matrix2, row, col)))); } } tolerated_max_diff = max_abs_val * std::numeric_limits::epsilon() * 64 * std::sqrt(static_cast(depth)); if (tolerated_max_diff < kSmallestAllowedDifference) { // Clamp the tolerated max diff to be a bit above machine epsilon if the // calculated value is too small. tolerated_max_diff = kSmallestAllowedDifference; if (external_path1 == ExternalPath::kEigen || external_path2 == ExternalPath::kEigen || external_path1 == ExternalPath::kEigenTensor || external_path2 == ExternalPath::kEigenTensor) { // Make additional allowance for Eigen differences. tolerated_max_diff *= 10.0f; } } tolerated_mean_diff = tolerated_max_diff / std::sqrt(size); } else if (std::is_same::value) { // raw integer case, no rounding, so we can require exactness tolerated_max_diff = 0; tolerated_mean_diff = 0; } else { // quantized case, with rounding errors in downscaling int32 accumulators // to final 8bit or 16bit values. if (external_path1 != ExternalPath::kNone || external_path2 != ExternalPath::kNone) { // In this case, we are comparing against some other library than ruy. // // We may have to tolerate an error of +/- 1 from using different // rounding in fixed-point multiplication, and then again an error of +/- // 1 from using different rounding in right shifts, so the tolerance on // the difference may have to be as large as 2. tolerated_max_diff = 2; } else if (RUY_PLATFORM_ARM) { // All our code paths on ARM (32 and 64-bit) are bit-exact // with the reference code (by design of the reference code). tolerated_max_diff = 0; } else if (RUY_PLATFORM_X86) { // Our reference and ARM paths have diverged from x86 paths in PR #227. // TODO: update the x86 path to adapt to that and reset that tolerance // to 0. tolerated_max_diff = 1; } else { // Other architectures, which we don't have dedicated code paths for // at the moment. // TODO: try resetting that tolerance to 0, since by // definition we're only using non-optimized code paths here. tolerated_max_diff = 1; } // totally empirical tolerated_mean_diff = std::min(1.0, 2.0 * std::pow(size, -0.18)); } double sum_diff = 0; for (int row = 0; row < matrix1.layout().rows(); row++) { for (int col = 0; col < matrix1.layout().cols(); col++) { double elem1 = Element(matrix1, row, col); double elem2 = Element(matrix2, row, col); double diff = elem1 - elem2; sum_diff += diff; // Test (std::abs(diff) > tolerated_max_diff), but also true if diff is // NaN. if (!(std::abs(diff) <= tolerated_max_diff)) { return false; } } } double mean_diff = sum_diff / size; if (std::abs(mean_diff) > tolerated_mean_diff) { return false; } return true; } template bool Agree(ExternalPath external_path1, const StorageMatrix& storage_matrix1, ExternalPath external_path2, const StorageMatrix& storage_matrix2, int depth) { VerifyConsistentFields(storage_matrix1); VerifyConsistentFields(storage_matrix2); return Agree(external_path1, storage_matrix1.matrix, external_path2, storage_matrix2.matrix, depth); } template bool Agree(const TestResult& result1, const TestResult& result2, int depth) { return Agree(result1.external_path, result1.storage_matrix, result2.external_path, result2.storage_matrix, depth); } template void AddTestResultToCluster( TestResult** result, std::vector*>>& clusters, int depth) { bool inserted = false; for (auto& cluster : clusters) { bool agreement = true; // Test for agreement with every result in the cluster. for (auto& other_result : cluster) { agreement &= Agree(**result, *other_result, depth); } if (agreement) { cluster.push_back(*result); inserted = true; } } if (!inserted) { std::vector*> new_results; new_results.push_back(*result); clusters.push_back(new_results); } } template void PrintPathsInAgreement( const std::vector>>& results, int depth) { // A container holding vectors of TestResults, where membership indicates // that all TestResults agree with each other. std::vector*>> clusters; for (const auto& result : results) { TestResult* test_result = result.get(); AddTestResultToCluster(&test_result, clusters, depth); } std::cerr << "Error: Not all paths agree. \n"; for (auto& cluster : clusters) { std::cerr << "These paths all agree with each other: "; for (auto& result : cluster) { std::cerr << PathName(*result) << ", "; } std::cerr << "but disagree with the rest.\n"; } } struct Stats { double median; double mean; double min; double max; }; inline std::string StatsAsString(const Stats& stats) { char buf[256]; snprintf(buf, sizeof(buf), "(median = %g, mean = %g, min = %g, max = %g)", stats.median, stats.mean, stats.min, stats.max); return std::string(buf); } template void GetMatrixStats(const Matrix& matrix, Stats* stats) { double min = std::numeric_limits::infinity(); double max = -std::numeric_limits::infinity(); double sum = 0; std::vector allvals; for (int row = 0; row < matrix.layout().rows(); row++) { for (int col = 0; col < matrix.layout().cols(); col++) { double val = Element(matrix, row, col); min = std::min(min, val); max = std::max(max, val); sum += val; allvals.push_back(val); } } std::sort(allvals.begin(), allvals.end()); stats->min = min; stats->max = max; stats->mean = sum / allvals.size(); stats->median = allvals[allvals.size() / 2]; } struct ErrorAnalysis { Stats stats_good; Stats stats_bad; // The below is to help document departure from bit exactness. It's probably // not going to be relevant to floating-point. std::set error_rows; std::set error_cols; int row_of_first_error = 0; int col_of_first_error = 0; double first_error_good_value = 0; double first_error_bad_value = 0; }; template void AnalyzeTestError(const TestSetType& test_set, int first_bad_result_index, ErrorAnalysis* error_analysis) { const auto& good_matrix = test_set.results[0]->storage_matrix.matrix; const auto& bad_matrix = test_set.results[first_bad_result_index]->storage_matrix.matrix; GetMatrixStats(good_matrix, &error_analysis->stats_good); GetMatrixStats(bad_matrix, &error_analysis->stats_bad); bool found_first_error = false; for (int row = 0; row < good_matrix.layout().rows(); row++) { for (int col = 0; col < good_matrix.layout().cols(); col++) { if (Element(good_matrix, row, col) != Element(bad_matrix, row, col)) { if (!found_first_error) { found_first_error = true; error_analysis->row_of_first_error = row; error_analysis->col_of_first_error = col; error_analysis->first_error_good_value = Element(good_matrix, row, col); error_analysis->first_error_bad_value = Element(bad_matrix, row, col); } error_analysis->error_rows.insert(row); error_analysis->error_cols.insert(col); } } } } template void ComputeReasonableMultiplier( const Matrix& lhs, const Matrix&, double* multiplier) { using LhsScalar = typename TestSetType::LhsScalar; using RhsScalar = typename TestSetType::RhsScalar; using DstScalar = typename TestSetType::DstScalar; if (std::is_floating_point::value || std::is_same::value) { *multiplier = 0; return; } *multiplier = static_cast(std::numeric_limits::max()) / (static_cast(lhs.layout().cols()) * std::numeric_limits::max() * std::numeric_limits::max()); } inline void QuantizeMultiplier(double multiplier_double, std::int32_t* multiplier_fixedpoint, int* multiplier_exponent) { RUY_CHECK_GT(multiplier_double, 0); if (multiplier_double == 0.) { *multiplier_fixedpoint = 0; *multiplier_exponent = 0; return; } const double q = std::frexp(multiplier_double, multiplier_exponent); auto q_fixed = static_cast(std::round(q * (1ll << 31))); RUY_CHECK_LE(q_fixed, (1ll << 31)); if (q_fixed == (1ll << 31)) { q_fixed /= 2; ++*multiplier_exponent; } RUY_CHECK_LE(q_fixed, std::numeric_limits::max()); *multiplier_fixedpoint = static_cast(q_fixed); } template void SwitchMultiplierToPerChannel(TestSetType* test_set) { ChannelDimension channel_dimension = (test_set->benchmark || global_random_engine()() % 2) ? ChannelDimension::kRow : ChannelDimension::kCol; test_set->mul_params.set_channel_dimension(channel_dimension); const int num_channels = channel_dimension == ChannelDimension::kRow ? test_set->rows : test_set->cols; test_set->per_channel_multiplier_fixedpoint.resize(num_channels); test_set->per_channel_multiplier_exponent.resize(num_channels); for (int i = 0; i < num_channels; i++) { // multipliers typically range in [2^30 ; 2^31 - 1]. // Values in [0, 2^30 - 1] are normally unused, but harmless. // Thus a good way to randomize multipliers is to subtract from them // a random value smaller than 2^30 but still significant compared to it. std::int32_t nudged_multiplier = test_set->mul_params.multiplier_fixedpoint() - (global_random_engine()() % (1 << 26)); int nudged_exponent = test_set->mul_params.multiplier_exponent() - 1 + (global_random_engine()() % 4); test_set->per_channel_multiplier_fixedpoint[i] = nudged_multiplier; test_set->per_channel_multiplier_exponent[i] = nudged_exponent; } test_set->mul_params.set_multiplier_fixedpoint(0); test_set->mul_params.set_multiplier_exponent(0); test_set->mul_params.set_multiplier_fixedpoint_perchannel( test_set->per_channel_multiplier_fixedpoint.data()); test_set->mul_params.set_multiplier_exponent_perchannel( test_set->per_channel_multiplier_exponent.data()); } template < typename TestSetType, bool IsApplicable = std::is_same::value && !std::is_same::value> struct MakeSpecMultiplierFieldsImpl {}; template struct MakeSpecMultiplierFieldsImpl { static void Run(TestSetType* test_set) { double multiplier; ComputeReasonableMultiplier(test_set->lhs.matrix, test_set->rhs.matrix, &multiplier); typename TestSetType::AccumScalar multiplier_fixedpoint; int multiplier_exponent; QuantizeMultiplier(multiplier, &multiplier_fixedpoint, &multiplier_exponent); test_set->mul_params.set_multiplier_fixedpoint(multiplier_fixedpoint); test_set->mul_params.set_multiplier_exponent(multiplier_exponent); if (!test_set->benchmark) { test_set->perchannel = global_random_engine()() & 1; } if (test_set->perchannel) { SwitchMultiplierToPerChannel(test_set); } } }; template struct MakeSpecMultiplierFieldsImpl { static void Run(TestSetType*) {} }; template void MakeSpecClampFields(MulParamsType* mul_params) { using DstScalar = typename MulParamsType::DstScalar; if (getenv("BENCHMARK_ONLY_MATMUL")) { if (std::is_floating_point::value) { mul_params->set_clamp_min(-std::numeric_limits::infinity()); mul_params->set_clamp_max(std::numeric_limits::infinity()); } else { mul_params->set_clamp_min(std::numeric_limits::lowest()); mul_params->set_clamp_max(std::numeric_limits::max()); } return; } mul_params->set_clamp_min(std::numeric_limits::lowest() + 1); mul_params->set_clamp_max(std::numeric_limits::max() - 1); } void MakeSpecClampFields(MulParams*) { // Returning raw accumulators, clamping is not supported. } template void TestSet::MakeZeroPoints() { RUY_CHECK_EQ(life_stage, LifeStage::kInitial); if (std::is_same::value || std::is_same::value) { // For now, support for int16 source types is limited to the // symmetric case (zero_point==0) because that appears to be // the case in the initial use cases, and that limits complexity // in thinking about accumulator overflows. // Setting use_specified_zero_points causes the default values 0 to be // used unless explicitly overridden. use_specified_zero_points = true; } if (!benchmark && !use_specified_zero_points) { MakeRandomScalar(RandomRange::kReasonableSrcZeroPoint, &lhs_zero_point); MakeRandomScalar(RandomRange::kReasonableSrcZeroPoint, &rhs_zero_point); // If destination is std::int32_t, no dst_zero_point is necessary. if (std::is_same::value) { dst_zero_point = 0; } else { MakeRandomScalar(RandomRange::kReasonableDstZeroPoint, &dst_zero_point); } } life_stage = LifeStage::kHasZeroPoints; } template void TestSet::MakeLhsRhs() { RUY_CHECK_EQ(life_stage, LifeStage::kHasZeroPoints); MakeRandom(rows, depth, lhs_order, lhs_zero_point, layout_style, RandomRange::kOffCenterAvoidMinValue, &lhs); MakeRandom(depth, cols, rhs_order, rhs_zero_point, layout_style, RandomRange::kGeneral, &rhs); if (!benchmark) { cache_lhs = (global_random_engine()() & 0xf) == 0; cache_rhs = (global_random_engine()() & 0xf) == 0; } if (cache_lhs) { lhs.matrix.set_cache_policy(CachePolicy::kAlwaysCache); } if (cache_rhs) { rhs.matrix.set_cache_policy(CachePolicy::kAlwaysCache); } life_stage = LifeStage::kHasLhsRhs; } template void TestSet::MakeMulParams() { RUY_CHECK_EQ(life_stage, LifeStage::kHasLhsRhs); if (!getenv("BENCHMARK_ONLY_MATMUL") && (benchmark || (global_random_engine()() & 1))) { MakeRandomVector(RandomRange::kBias, std::max(rows, cols), &bias_data); mul_params.set_bias(bias_data.data()); } if (lhs.matrix.zero_point() == std::numeric_limits::lowest() && rhs.matrix.zero_point() == std::numeric_limits::lowest()) { lhs.matrix.set_zero_point(lhs.matrix.zero_point() + 1); } MakeSpecMultiplierFieldsImpl::Run(this); MakeSpecClampFields(&mul_params); life_stage = LifeStage::kHasMulParams; } inline int GetIntEnvVarOrZero(const char* name) { const char* val = getenv(name); if (!val) { return 0; } return std::stoi(val); } inline float GetFloatEnvVarOrZero(const char* name) { const char* val = getenv(name); if (!val) { return 0; } return std::stof(val); } inline int GetHexIntEnvVarOrZero(const char* name) { const char* val = getenv(name); if (!val) { return 0; } return std::stoi(val, nullptr, 16); } inline bool GetBoolEnvVarOrFalse(const char* name) { return static_cast(GetIntEnvVarOrZero(name)); } template void TestSet::MakeOtherParams() { RUY_CHECK_EQ(life_stage, LifeStage::kHasMulParams); if (max_num_threads == 0) { max_num_threads = GetIntEnvVarOrZero("THREADS"); } life_stage = LifeStage::kHasOtherParams; } inline std::vector PathsBitfieldAsVector(Path paths_bitfield) { std::vector result; std::uint32_t remaining_paths = static_cast(paths_bitfield); std::uint32_t test_bit = 1; while (remaining_paths) { if (remaining_paths & test_bit) { result.push_back(static_cast(test_bit)); } remaining_paths &= ~test_bit; test_bit <<= 1; } return result; } inline std::vector EnumerateTuningsForPath(Path path, bool benchmark) { if (benchmark) { return {Tuning::kAuto}; } #if RUY_PLATFORM_ARM if (path == Path::kNeon || path == Path::kNeonDotprod) { return {Tuning::kA55ish, Tuning::kX1, Tuning::kGeneric, Tuning::kAuto}; } #endif (void)path; return {Tuning::kAuto}; } template void TestSet::MakeResultPaths() { RUY_CHECK_EQ(life_stage, LifeStage::kHasOtherParams); Path paths_bitfield = static_cast(GetHexIntEnvVarOrZero("PATHS")); if (paths_bitfield == Path::kNone) { // Use a dummy Context just to perform the resolution of specific runtime // enabled paths. Context context; paths_bitfield = get_ctx(&context)->GetRuntimeEnabledPaths(); } // Disable the internal test-only variants of the StandardCpp path in // benchmarks if (benchmark) { paths_bitfield = paths_bitfield & kAllPaths; } // Disable the internal test-only variants of the StandardCpp path on large // tests. // This constant be large enough to exercise some interesting BlockMap logic, // small enough to avoid large test latency increases from running these // slow code paths on large matrix multiplications. const int kMaxSizeToTestInternalStandardCppVariants = 300; if (rows > kMaxSizeToTestInternalStandardCppVariants || cols > kMaxSizeToTestInternalStandardCppVariants || depth > kMaxSizeToTestInternalStandardCppVariants) { paths_bitfield = paths_bitfield & kAllPaths; } // Trim bits that don't correspond to a compiled path, // to allow specifying e.g. ffff to mean 'all paths' regardless of whether all // those bits exist as actual paths. paths_bitfield = paths_bitfield & kAllPathsIncludingInternalVariants; RUY_CHECK_NE(paths_bitfield, Path::kNone); paths = PathsBitfieldAsVector(paths_bitfield); // kReference is a special 'external path' that's always available. // It can still be disabled by NOEXT. if (!GetBoolEnvVarOrFalse("NOEXT")) { external_paths.push_back(ExternalPath::kReference); } #ifdef RUY_TEST_EXTERNAL_PATHS if (!GetBoolEnvVarOrFalse("NOEXT")) { if (SupportsGemmlowp::kValue) { #ifdef GEMMLOWP_SSE4 const bool gemmlowp_supported = !mul_params.multiplier_fixedpoint_perchannel() && mul_params.channel_dimension() == ChannelDimension::kRow; #else const bool gemmlowp_supported = mul_params.channel_dimension() == ChannelDimension::kRow; #endif if (gemmlowp_supported) { external_paths.push_back(ExternalPath::kGemmlowp); } } if (UsesSingleScalarType::kValue && std::is_floating_point::value) { external_paths.push_back(ExternalPath::kEigen); if (layout_style == LayoutStyle::kUnstridedLinear) { external_paths.push_back(ExternalPath::kEigenTensor); } // We link against a generic BLAS target that only maps to OpenBLAS on specific // architectures. #if RUY_PLATFORM_ARM_32 || RUY_PLATFORM_ARM_64 // OpenBLAS multi-threading is disabled, so avoid mixing single-threaded // and multi-threaded benchmark results. if (max_num_threads == 1 && !getenv("NO_OPENBLAS")) { external_paths.push_back(ExternalPath::kOpenBlas); } #endif } } #endif // RUY_TEST_EXTERNAL_PATHS for (Path path : paths) { for (Tuning tuning : EnumerateTuningsForPath(path, benchmark)) { results.emplace_back(new TestResultType); TestResultType& result = *results.back(); result.path = path; result.tuning = tuning; MakeRandom(rows, cols, dst_order, dst_zero_point, layout_style, RandomRange::kGeneral, &result.storage_matrix); } } for (ExternalPath external_path : external_paths) { results.emplace_back(new TestResultType); TestResultType& result = *results.back(); result.external_path = external_path; MakeRandom(rows, cols, dst_order, dst_zero_point, layout_style, RandomRange::kGeneral, &result.storage_matrix); } life_stage = LifeStage::kHasResultPaths; } template void TestSet::EvalResult( TestResult* result) { RUY_CHECK(result->path != Path::kNone || result->external_path != ExternalPath::kNone); if (result->path != Path::kNone) { EvalRuy(result); } else { EvalExternalPath(this, result); } const std::string& pathname = PathName(*result); if (std::find(CoveredPaths()->begin(), CoveredPaths()->end(), pathname) == CoveredPaths()->end()) { CoveredPaths()->push_back(pathname); } } using f32 = float; using f64 = double; using u8 = std::uint8_t; using i8 = std::int8_t; using u16 = std::uint16_t; using i16 = std::int16_t; using u32 = std::uint32_t; using i32 = std::int32_t; using u64 = std::uint64_t; using i64 = std::int64_t; template const char* TypeName() { return nullptr; } #define RUY_TYPENAME(TYPE) \ template <> \ inline const char* TypeName() { \ return #TYPE; \ } RUY_TYPENAME(f32) RUY_TYPENAME(f64) RUY_TYPENAME(u8) RUY_TYPENAME(i8) RUY_TYPENAME(u16) RUY_TYPENAME(i16) RUY_TYPENAME(u32) RUY_TYPENAME(i32) RUY_TYPENAME(u64) RUY_TYPENAME(i64) #undef RUY_TYPENAME template const char* SymmetryName(const Matrix& matrix) { if (matrix.zero_point() == SymmetricZeroPoint()) { return "symm"; } else { return "asymm"; } } template int StorageSize(const Matrix& matrix) { return sizeof(Scalar) * FlatSize(matrix.layout()); } // Helper that replicates a buffer and gives out pointers to the replicas. // This is useful when one wants to traverse data so that it is cold in cache. // By having a sufficiently large value of num_repeats, one can ensure that the // working set covered by the replicas is greater than the cache size. template class RepeatedBuffer { public: RepeatedBuffer() = default; void Init(const T* elems, int num_elems, int num_repeats) { buffers_.clear(); allocator_.FreeAll(); for (int i = 0; i < num_repeats; i++) { T* p; allocator_.Allocate(num_elems, &p); memcpy(p, elems, num_elems * sizeof(T)); buffers_.push_back(p); } } T* Next() { T* ret = buffers_[current_]; current_ = (current_ + 1) % buffers_.size(); return ret; } private: Allocator allocator_; std::vector buffers_; int current_ = 0; }; template void TestSet::Benchmark( TestResult* result) { const bool cold = getenv("RUY_BENCHMARK_COLD"); LhsScalar* orig_lhs_data = lhs.matrix.data(); RhsScalar* orig_rhs_data = rhs.matrix.data(); DstScalar* orig_dst_data = result->storage_matrix.matrix.data(); int num_matmul_sets = 0; RepeatedBuffer cold_lhs; RepeatedBuffer cold_rhs; RepeatedBuffer cold_dst; if (cold) { const int kWorkingSetSize = 100 << 20; const int each_matmul_set_size = StorageSize(lhs.matrix) + StorageSize(rhs.matrix) + StorageSize(result->storage_matrix.matrix); num_matmul_sets = (kWorkingSetSize + each_matmul_set_size - 1) / each_matmul_set_size; cold_lhs.Init(lhs.matrix.data(), FlatSize(lhs.matrix.layout()), num_matmul_sets); cold_rhs.Init(rhs.matrix.data(), FlatSize(rhs.matrix.layout()), num_matmul_sets); cold_dst.Init(result->storage_matrix.matrix.data(), FlatSize(result->storage_matrix.matrix.layout()), num_matmul_sets); } const bool record_pmu = GetBoolEnvVarOrFalse("RUY_BENCHMARK_PMU"); int repeats = GetIntEnvVarOrZero("RUY_BENCHMARK_REPEATS"); if (!repeats) { repeats = 4; } float benchmark_min_secs = GetFloatEnvVarOrZero("RUY_BENCHMARK_MIN_SECS"); if (!benchmark_min_secs) { benchmark_min_secs = 0.5; } #ifdef RUY_PROFILER { const char* lhstype = TypeName(); const char* lhssymm = SymmetryName(lhs.matrix); const char* rhstype = TypeName(); const char* rhssymm = SymmetryName(rhs.matrix); printf("Profiling path=%s shape=(%dx%dx%d) lhs=(%s,%s) rhs=(%s,%s)\n", PathName(*result).c_str(), rows, depth, cols, lhstype, lhssymm, rhstype, rhssymm); ruy::profiler::ScopeProfile profile; #endif float latency = std::numeric_limits::infinity(); float l1_refill_rate = std::numeric_limits::infinity(); float l2_refill_rate = std::numeric_limits::infinity(); float l3_refill_rate = std::numeric_limits::infinity(); float l1tlb_refill_rate = std::numeric_limits::infinity(); float l2tlb_refill_rate = std::numeric_limits::infinity(); float mispred_rate = std::numeric_limits::infinity(); float frontend_stall_rate = std::numeric_limits::infinity(); float backend_stall_rate = std::numeric_limits::infinity(); for (int repeat = 0; repeat < repeats; repeat++) { auto& pmu_events = GlobalPmuEvents(); if (record_pmu) { pmu_events.StartRecording(); } TimePoint time_start = Now(); TimePoint t = time_start; int iters = 0; int iters_at_a_time = 1; while (ToFloatSeconds(t - time_start) < benchmark_min_secs) { for (int i = 0; i < iters_at_a_time; i++) { if (cold) { lhs.matrix.set_data(cold_lhs.Next()); rhs.matrix.set_data(cold_rhs.Next()); result->storage_matrix.matrix.set_data(cold_dst.Next()); } EvalResult(result); iters++; } iters_at_a_time *= 2; t = Now(); } latency = std::min( latency, static_cast(ToFloatSeconds(t - time_start) / iters)); if (record_pmu) { pmu_events.StopRecording(); const float normalization_factor = 1.0f / (static_cast(iters) * rows * cols * depth); l1_refill_rate = std::min( l1_refill_rate, pmu_events.L1RefillCount() * normalization_factor); l2_refill_rate = std::min( l2_refill_rate, pmu_events.L2RefillCount() * normalization_factor); l3_refill_rate = std::min( l3_refill_rate, pmu_events.L3RefillCount() * normalization_factor); l1tlb_refill_rate = std::min(l1tlb_refill_rate, pmu_events.L1TLBRefillCount() * normalization_factor); l2tlb_refill_rate = std::min(l2tlb_refill_rate, pmu_events.L2TLBRefillCount() * normalization_factor); mispred_rate = std::min(mispred_rate, pmu_events.BranchMispredictionCount() * normalization_factor); frontend_stall_rate = std::min(frontend_stall_rate, pmu_events.FrontendStallCount() * normalization_factor); backend_stall_rate = std::min(backend_stall_rate, pmu_events.BackendStallCount() * normalization_factor); } } result->latency = latency; if (record_pmu) { result->l1_refill_rate = l1_refill_rate; result->l2_refill_rate = l2_refill_rate; result->l3_refill_rate = l3_refill_rate; result->l1tlb_refill_rate = l1tlb_refill_rate; result->l2tlb_refill_rate = l2tlb_refill_rate; result->mispred_rate = mispred_rate; result->frontend_stall_rate = frontend_stall_rate; result->backend_stall_rate = backend_stall_rate; } #ifdef RUY_PROFILER } fflush(stdout); #endif if (cold) { lhs.matrix.set_data(orig_lhs_data); rhs.matrix.set_data(orig_rhs_data); memcpy(orig_dst_data, result->storage_matrix.matrix.data(), StorageSize(result->storage_matrix.matrix)); result->storage_matrix.matrix.set_data(orig_dst_data); } } template void TestSet::Eval() { RUY_CHECK_EQ(life_stage, LifeStage::kHasResultPaths); for (auto& result : results) { if (benchmark) { Benchmark(result.get()); } else { EvalResult(result.get()); } } life_stage = LifeStage::kEvaluated; } template std::string DumpRegion(const Matrix& matrix, int center_row, int center_col) { static constexpr int kRadius = 20; int first_row = std::max(0, center_row - kRadius); int last_row = std::min(matrix.layout().rows() - 1, center_row + kRadius); int first_col = std::max(0, center_col - kRadius); int last_col = std::min(matrix.layout().cols() - 1, center_col + kRadius); std::ostringstream stream; for (int row = first_row; row <= last_row; row++) { for (int col = first_col; col <= last_col; col++) { stream << static_cast(Element(matrix, row, col)) << " "; } stream << "\n"; } return stream.str(); } template void TestSet::VerifyTestResults() const { const int depth = lhs.matrix.layout().cols(); for (int i = 0; i < static_cast(results.size()) - 1; i++) { if (!Agree(*results[i], *results[i + 1], depth)) { PrintPathsInAgreement(results, depth); ErrorAnalysis error_analysis; AnalyzeTestError(*this, i + 1, &error_analysis); std::cerr << "Shape: rows = " << rows << ", cols = " << cols << ", depth = " << depth << std::endl; std::cerr << "Stats of the good result matrix: " << StatsAsString(error_analysis.stats_good) << std::endl; std::cerr << "Stats of the bad result matrix: " << StatsAsString(error_analysis.stats_bad) << std::endl; if (static_cast(error_analysis.error_rows.size()) < rows) { std::cerr << "Rows containing errors: " << Join(error_analysis.error_rows) << std::endl; } else { std::cerr << "Errors found in ALL rows." << std::endl; } if (static_cast(error_analysis.error_cols.size()) < cols) { std::cerr << "Cols containing errors: " << Join(error_analysis.error_cols) << std::endl; } else { std::cerr << "Errors found in ALL cols." << std::endl; } std::cerr << "The first error occurs at row " << error_analysis.row_of_first_error << ", col " << error_analysis.col_of_first_error << std::endl; std::cerr << "Good value: " << error_analysis.first_error_good_value << std::endl; std::cerr << "Bad value : " << error_analysis.first_error_bad_value << std::endl; std::cerr << "Region of Good result matrix around first error:\n\n" << DumpRegion(results[0]->storage_matrix.matrix, error_analysis.row_of_first_error, error_analysis.col_of_first_error) << std::endl; std::cerr << "Region of Bad result matrix around first error:\n\n" << DumpRegion(results[i + 1]->storage_matrix.matrix, error_analysis.row_of_first_error, error_analysis.col_of_first_error) << std::endl; RUY_CHECK(false); } } } template void TestSet::Verify() { RUY_CHECK_EQ(life_stage, LifeStage::kEvaluated); VerifyTestResults(); life_stage = LifeStage::kFinal; } template void TestRCC(int rows, int depth, int cols) { TestSetType test_set; test_set.rows = rows; test_set.depth = depth; test_set.cols = cols; test_set.lhs_order = Order::kRowMajor; test_set.rhs_order = Order::kColMajor; test_set.dst_order = Order::kColMajor; test_set.layout_style = LayoutStyle::kUnstridedLinear; test_set.Run(); } template void TestNonRCC(int rows, int depth, int cols) { TestSetType test_set; test_set.rows = rows; test_set.depth = depth; test_set.cols = cols; test_set.lhs_order = Order::kColMajor; test_set.rhs_order = Order::kColMajor; test_set.dst_order = Order::kColMajor; test_set.layout_style = LayoutStyle::kUnstridedLinear; test_set.Run(); } template void TestLinearAllOrders(int rows, int depth, int cols) { const std::vector orders{Order::kColMajor, Order::kRowMajor}; for (Order lhs_order : orders) { for (Order rhs_order : orders) { for (Order dst_order : orders) { TestSetType test_set; test_set.rows = rows; test_set.depth = depth; test_set.cols = cols; test_set.lhs_order = lhs_order; test_set.rhs_order = rhs_order; test_set.dst_order = dst_order; test_set.layout_style = LayoutStyle::kLinear; test_set.Run(); } } } } } // namespace ruy #endif // RUY_RUY_TEST_H_