diff --git a/egs/wsj/s5/steps/libs/nnet3/xconfig/basic_layers.py b/egs/wsj/s5/steps/libs/nnet3/xconfig/basic_layers.py
index e18c1359b61..fe66b883d9e 100644
--- a/egs/wsj/s5/steps/libs/nnet3/xconfig/basic_layers.py
+++ b/egs/wsj/s5/steps/libs/nnet3/xconfig/basic_layers.py
@@ -624,7 +624,7 @@ def _generate_config(self):
             # note: by default the LinearComponent uses natural gradient.
             line = ('component name={0}.linear type=LinearComponent '
                     'orthonormal-constraint={1} param-stddev={2} '
-                    'input-dim={3} output-dim={4} max-change=0.75 {5}'
+                    'input-dim={3} output-dim={4} max-change=0.75 use-natural-gradient=false {5}'
                     ''.format(self.name, self.config['orthonormal-constraint'],
                               self.config['orthonormal-constraint'] / math.sqrt(input_dim),
                               input_dim, bottleneck_dim, linear_options))
@@ -637,7 +637,7 @@ def _generate_config(self):
 
 
         line = ('component name={0}.affine'
-                ' type=NaturalGradientAffineComponent'
+                ' type=AffineComponent'
                 ' input-dim={1} output-dim={2} {3}'
                 ''.format(self.name, cur_dim, output_dim, affine_options))
         configs.append(line)
@@ -843,7 +843,7 @@ def _add_components(self, input_desc, input_dim, nonlinearities):
             bottleneck_dim = self.config['bottleneck-dim']
             # note: by default the LinearComponent uses natural gradient.
             line = ('component name={0}.linear type=LinearComponent '
-                    'input-dim={1} orthonormal-constraint=1.0 output-dim={2} {3}'
+                    'input-dim={1} orthonormal-constraint=1.0 output-dim={2} {3} use-natural-gradient=false'
                     ''.format(self.name, input_dim, bottleneck_dim, linear_options))
             configs.append(line)
             line = ('component-node name={0}.linear component={0}.linear input={1}'
@@ -853,7 +853,7 @@ def _add_components(self, input_desc, input_dim, nonlinearities):
             cur_dim = bottleneck_dim
 
 
-        line = ('component name={0}.affine type=NaturalGradientAffineComponent'
+        line = ('component name={0}.affine type=AffineComponent'
                 ' input-dim={1} output-dim={2} {3}'
                 ''.format(self.name, cur_dim, output_dim, affine_options))
         configs.append(line)
diff --git a/egs/wsj/s5/steps/libs/nnet3/xconfig/composite_layers.py b/egs/wsj/s5/steps/libs/nnet3/xconfig/composite_layers.py
index 928ca445ccc..663a2dbcbeb 100644
--- a/egs/wsj/s5/steps/libs/nnet3/xconfig/composite_layers.py
+++ b/egs/wsj/s5/steps/libs/nnet3/xconfig/composite_layers.py
@@ -167,7 +167,7 @@ def _generate_config(self):
         # The first linear layer, from input-dim (spliced x2) to bottleneck-dim
         configs.append('component name={0}.linear type=TdnnComponent input-dim={1} '
                        'output-dim={2} l2-regularize={3} max-change={4} use-bias=false '
-                       'time-offsets={5} orthonormal-constraint=-1.0'.format(
+                       'time-offsets={5} orthonormal-constraint=-1.0 use-natural-gradient=false'.format(
                            name, input_dim, bottleneck_dim, l2_regularize,
                            max_change, time_offsets1))
         configs.append('component-node name={0}.linear component={0}.linear '
@@ -176,7 +176,7 @@ def _generate_config(self):
         # The affine layer, from bottleneck-dim (spliced x2) to output-dim
         configs.append('component name={0}.affine type=TdnnComponent '
                        'input-dim={1} output-dim={2} l2-regularize={3} max-change={4} '
-                       'time-offsets={5}'.format(
+                       'time-offsets={5} use-natural-gradient=false'.format(
                            name, bottleneck_dim, output_dim, l2_regularize,
                            max_change, time_offsets2))
         configs.append('component-node name={0}.affine component={0}.affine '
@@ -291,7 +291,7 @@ def _generate_config(self):
         self_repair_scale = self.config['self-repair-scale']
 
         # The affine layer, from input-dim to big-dim.
-        configs.append('component name={0}.affine type=NaturalGradientAffineComponent '
+        configs.append('component name={0}.affine type=AffineComponent '
                        'input-dim={1} output-dim={2} l2-regularize={3} max-change={4}'.format(
                            name, input_dim, big_dim, l2_regularize, max_change))
         configs.append('component-node name={0}.affine component={0}.affine '
@@ -314,7 +314,7 @@ def _generate_config(self):
         # ("floating" orthonormal constraint).
         configs.append('component name={0}.linear type=LinearComponent '
                        'input-dim={1} output-dim={2} l2-regularize={3} max-change={4} '
-                       'orthonormal-constraint=-1 '.format(
+                       'orthonormal-constraint=-1 use-natural-gradient=false '.format(
                            name, big_dim, small_dim,
                            l2_regularize, max_change))
         configs.append('component-node name={0}.linear component={0}.linear '
diff --git a/egs/wsj/s5/steps/libs/nnet3/xconfig/trivial_layers.py b/egs/wsj/s5/steps/libs/nnet3/xconfig/trivial_layers.py
index 4afea78ad3f..5c146860361 100644
--- a/egs/wsj/s5/steps/libs/nnet3/xconfig/trivial_layers.py
+++ b/egs/wsj/s5/steps/libs/nnet3/xconfig/trivial_layers.py
@@ -328,7 +328,7 @@ def _generate_config(self):
 
         configs = []
         line = ('component name={0} type=LinearComponent input-dim={1} output-dim={2} '
-                '{3}'.format(self.name, input_dim, output_dim, opts))
+                '{3} use-natural-gradient=false'.format(self.name, input_dim, output_dim, opts))
         configs.append(line)
         line = ('component-node name={0} component={0} input={1}'.format(
             self.name, input_desc))