Amazon SageMaker Debugger – 调试机器学习模型

import argparseimport numpy as npimport tensorflow as tfimport random
parser = argparse.ArgumentParser()parser.add_argument('--model_dir', type=str, help="S3 path for the model")parser.add_argument('--lr', type=float, help="Learning Rate", default=0.001)parser.add_argument('--steps', type=int, help="Number of steps to run", default=100)parser.add_argument('--scale', type=float, help="Scaling factor for inputs", default=1.0)
args = parser.parse_args()
with tf.name_scope('initialize'):    # 2-dimensional input sample    x = tf.placeholder(shape=(None, 2), dtype=tf.float32)    # Initial weights: [10, 10]    w = tf.Variable(initial_value=[[10.], [10.]], name='weight1')    # True weights, i.e. the ones we're trying to learn    w0 = [[1], [1.]]with tf.name_scope('multiply'):    # Compute true label    y = tf.matmul(x, w0)    # Compute "predicted" label    y_hat = tf.matmul(x, w)with tf.name_scope('loss'):    # Compute loss    loss = tf.reduce_mean((y_hat - y) ** 2, name="loss")
optimizer = tf.train.AdamOptimizer(args.lr)optimizer_op = optimizer.minimize(loss)
with tf.Session() as sess:    sess.run(tf.global_variables_initializer())    for i in range(args.steps):        x_ = np.random.random((10, 2)) * args.scale        _loss, opt = sess.run([loss, optimizer_op], {x: x_})        print (f'Step={i}, Loss={_loss}')

bad_hyperparameters = {'steps': 10, 'lr': 100, 'scale': 100000000000}
estimator = TensorFlow(    role=sagemaker.get_execution_role(),    base_job_name='debugger-simple-demo',    train_instance_count=1,    train_instance_type='local',    entry_point='script-v1.py',    framework_version='1.13.1',    py_version='py3',    script_mode=True,    hyperparameters=bad_hyperparameters)

import argparseimport numpy as npimport tensorflow as tfimport randomimport smdebug.tensorflow as smd
parser = argparse.ArgumentParser()parser.add_argument('--model_dir', type=str, help="S3 path for the model")parser.add_argument('--lr', type=float, help="Learning Rate", default=0.001 )parser.add_argument('--steps', type=int, help="Number of steps to run", default=100 )parser.add_argument('--scale', type=float, help="Scaling factor for inputs", default=1.0 )parser.add_argument('--debug_path', type=str, default='/opt/ml/output/tensors')parser.add_argument('--debug_frequency', type=int, help="How often to save tensor data", default=10)feature_parser = parser.add_mutually_exclusive_group(required=False)feature_parser.add_argument('--reductions', dest='reductions', action='store_true', help="save reductions of tensors instead of saving full tensors")feature_parser.add_argument('--no_reductions', dest='reductions', action='store_false', help="save full tensors")args = parser.parse_args()args = parser.parse_args()
reduc = smd.ReductionConfig(reductions=['mean'], abs_reductions=['max'], norms=['l1']) if args.reductions else None
hook = smd.SessionHook(out_dir=args.debug_path,                       include_collections=['weights', 'gradients', 'losses'],                       save_config=smd.SaveConfig(save_interval=args.debug_frequency),                       reduction_config=reduc)
with tf.name_scope('initialize'):    # 2-dimensional input sample    x = tf.placeholder(shape=(None, 2), dtype=tf.float32)    # Initial weights: [10, 10]    w = tf.Variable(initial_value=[[10.], [10.]], name='weight1')    # True weights, i.e. the ones we're trying to learn    w0 = [[1], [1.]]with tf.name_scope('multiply'):    # Compute true label    y = tf.matmul(x, w0)    # Compute "predicted" label    y_hat = tf.matmul(x, w)with tf.name_scope('loss'):    # Compute loss    loss = tf.reduce_mean((y_hat - y) ** 2, name="loss")    hook.add_to_collection('losses', loss)
optimizer = tf.train.AdamOptimizer(args.lr)optimizer = hook.wrap_optimizer(optimizer)optimizer_op = optimizer.minimize(loss)
hook.set_mode(smd.modes.TRAIN)
with tf.train.MonitoredSession(hooks=[hook]) as sess:    for i in range(args.steps):        x_ = np.random.random((10, 2)) * args.scale        _loss, opt = sess.run([loss, optimizer_op], {x: x_})        print (f'Step={i}, Loss={_loss}')

bad_hyperparameters = {'steps': 10, 'lr': 100, 'scale': 100000000000, 'debug_frequency': 1}
from sagemaker.debugger import Rule, rule_configsestimator = TensorFlow(    role=sagemaker.get_execution_role(),    base_job_name='debugger-simple-demo',    train_instance_count=1,    train_instance_type='ml.c5.2xlarge',    image_name=cpu_docker_image_name,    entry_point='script-v2.py',    framework_version='1.15',    py_version='py3',    script_mode=True,    hyperparameters=bad_hyperparameters,    rules = [Rule.sagemaker(rule_configs.exploding_tensor())])
estimator.fit()2019-11-27 10:42:02 开始 - 开始训练作业...2019-11-27 10:42:25 开始 - 启动请求的 ML 实例********* Debugger Rule Status *********** ExplodingTensor: InProgress *****************************************

description = client.describe_training_job(TrainingJobName=job_name)print(description['DebugRuleEvaluationStatuses'][0]['RuleConfigurationName'])print(description['DebugRuleEvaluationStatuses'][0]['RuleEvaluationStatus'])
ExplodingTensorIssuesFound

s3_output_path = description["DebugConfig"]["DebugHookConfig"]["S3OutputPath"]trial = create_trial(s3_output_path)

tensor = 'gradients/multiply/MatMul_1_grad/tuple/control_dependency_1:0'for s in list(trial.tensor(tensor).steps()):    print("Value: ", trial.tensor(tensor).step(s).value)
Value:  [[1.1508383e+23] [1.0809098e+23]]Value:  [[1.0278440e+23] [1.1347468e+23]]Value:  [[nan] [nan]]Value:  [[nan] [nan]]Value:  [[nan] [nan]]Value:  [[nan] [nan]]Value:  [[nan] [nan]]Value:  [[nan] [nan]]Value:  [[nan] [nan]]Value:  [[nan] [nan]]

# Compute true labely = tf.matmul(x, w0)# Compute "predicted" labely_hat = tf.matmul(x, w)

x_ = np.random.random((10, 2)) * args.scale
bad_hyperparameters = {'steps': 10, 'lr': 100, 'scale': 100000000000, 'debug_frequency': 1}