diff --git a/nntrainer/layers/pow_layer.cpp b/nntrainer/layers/pow_layer.cpp index 8859738..9f55665 100644 --- a/nntrainer/layers/pow_layer.cpp +++ b/nntrainer/layers/pow_layer.cpp @@ -33,6 +33,11 @@ void PowLayer::forwarding_operation(const Tensor &input, Tensor &hidden) { void PowLayer::calcDerivative(RunLayerContext &context) { float exp = std::get(pow_props).get(); + // The zero-exponent forward is constant, so avoid evaluating x^-1. + if (exp == 0.0f) { + context.getOutgoingDerivative(SINGLE_INOUT_IDX).setZero(); + return; + } context.getOutgoingDerivative(0).copy( context.getIncomingDerivative(SINGLE_INOUT_IDX) .multiply(exp) diff --git a/test/unittest/layers/unittest_layers_pow.cpp b/test/unittest/layers/unittest_layers_pow.cpp index 5ec4737..67cd258 100644 --- a/test/unittest/layers/unittest_layers_pow.cpp +++ b/test/unittest/layers/unittest_layers_pow.cpp @@ -28,3 +28,88 @@ auto semantic_pow_multi = LayerSemanticsParamType( GTEST_PARAMETER_TEST(Pow, LayerSemantics, ::testing::Values(semantic_pow, semantic_pow_multi)); + +#include +#include +#include +#include + +namespace { +struct PowAuditResult { + std::vector value, gradient, input_after, incoming_after; +}; + +PowAuditResult evaluatePowAudit(const std::vector &x, float exponent, + const std::vector &incoming) { + const nntrainer::TensorDim dim({1, 1, 1, (unsigned int)x.size()}); + nntrainer::PowLayer layer; + layer.setProperty({"exponent=" + std::to_string(exponent)}); + nntrainer::InitLayerContext init({dim}, {true}, false, "pow_zero_audit"); + layer.finalize(init); + nntrainer::Var_Grad input(dim, nntrainer::Initializer::NONE, true, true, + "input"); + nntrainer::Var_Grad output(dim, nntrainer::Initializer::NONE, true, true, + "output"); + nntrainer::RunLayerContext context("pow_zero_audit", true, 0.0f, false, 1.0f, + nullptr, false, {}, {&input}, {&output}, + {}); + for (size_t i = 0; i < x.size(); ++i) { + input.getVariableRef().getData()[i] = x[i]; + input.getGradientRef().getData()[i] = 42.0f; + output.getGradientRef().getData()[i] = incoming[i]; + } + layer.forwarding(context, true); + layer.calcDerivative(context); + PowAuditResult result; + for (size_t i = 0; i < x.size(); ++i) { + result.value.push_back(output.getVariableRef().getData()[i]); + result.gradient.push_back(input.getGradientRef().getData()[i]); + result.input_after.push_back(input.getVariableRef().getData()[i]); + result.incoming_after.push_back( + output.getGradientRef().getData()[i]); + } + return result; +} +} // namespace + +TEST(PowZeroExponentAudit, finite_inputs_have_zero_gradient) { + const float tiny = std::numeric_limits::denorm_min(); + const std::vector x = {0.0f, -0.0f, tiny, -tiny, 2.0f, -2.0f}; + const std::vector incoming = {3.0f, -2.0f, 1.0f, 0.0f, 0.5f, 4.0f}; + for (float exponent : {0.0f, -0.0f}) { + const auto result = evaluatePowAudit(x, exponent, incoming); + for (size_t i = 0; i < x.size(); ++i) { + SCOPED_TRACE(i); + EXPECT_FLOAT_EQ(result.value[i], 1.0f); + EXPECT_FLOAT_EQ(result.gradient[i], 0.0f); + EXPECT_EQ(result.input_after[i], x[i]); + EXPECT_EQ(result.incoming_after[i], incoming[i]); + } + } +} + +TEST(PowZeroExponentAudit, backward_matches_constant_forward_difference) { + const float h = 1.0f / 32.0f; + const auto center = evaluatePowAudit({0.0f}, 0.0f, {3.0f}); + const auto plus = evaluatePowAudit({h}, 0.0f, {3.0f}); + const auto minus = evaluatePowAudit({-h}, 0.0f, {3.0f}); + const float difference = 3.0f * (plus.value[0] - minus.value[0]) / (2.0f * h); + EXPECT_FLOAT_EQ(difference, 0.0f); + EXPECT_FLOAT_EQ(center.gradient[0], difference); +} + +TEST(PowZeroExponentAudit, ordinary_exponents_keep_the_chain_rule) { + struct Case { + float x, exponent, incoming, value, gradient; + }; + const std::vector cases = {{0, 1, 3, 0, 3}, + {0, 2, 3, 0, 0}, + {-2, 3, -0.5f, -8, -6}, + {4, 0.5f, 3, 2, 0.75f}, + {2, -1, 3, 0.5f, -0.75f}}; + for (const auto &c : cases) { + const auto r = evaluatePowAudit({c.x}, c.exponent, {c.incoming}); + EXPECT_FLOAT_EQ(r.value[0], c.value); + EXPECT_FLOAT_EQ(r.gradient[0], c.gradient); + } +}