tensorboard启动失败
-
运行到get_started.inpynb中以下代码块的时候报错
model = create_model() model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) log_dir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1) model.fit(x=x_train, y=y_train, epochs=5, validation_data=(x_test, y_test), callbacks=[tensorboard_callback])
2023-04-02 14:27:45.323033: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:219] failed to create cublas handle: cublasGetStatusString symbol not found.
2023-04-02 14:27:45.323121: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:222] Failure to initialize cublas may be due to OOM (cublas needs some free memory when you initialize it, and your deep-learning framework may have preallocated more than its fair share), or may be because this binary was not built with support for the GPU in your machine.
2023-04-02 14:27:45.323155: W tensorflow/core/framework/op_kernel.cc:1830] OP_REQUIRES failed at matmul_op_impl.h:621 : INTERNAL: Attempting to perform BLAS operation using StreamExecutor without BLAS support
2023-04-02 14:27:45.323195: I tensorflow/core/common_runtime/executor.cc:1197] [/job:localhost/replica:0/task:0/device:GPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INTERNAL: Attempting to perform BLAS operation using StreamExecutor without BLAS support
[[{{node sequential_2/dense_4/MatMul}}]]InternalError Traceback (most recent call last)
Input In [18], in <cell line: 9>()
6 log_dir = “logs/fit/” + datetime.datetime.now().strftime(“%Y%m%d-%H%M%S”)
7 tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
----> 9 model.fit(x=x_train,
10 y=y_train,
11 epochs=5,
12 validation_data=(x_test, y_test),
13 callbacks=[tensorboard_callback])File /usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py:70, in filter_traceback.<locals>.error_handler(*args, **kwargs)
67 filtered_tb = _process_traceback_frames(e.traceback)
68 # To get the full stack trace, call:
69 #tf.debugging.disable_traceback_filtering()
—> 70 raise e.with_traceback(filtered_tb) from None
71 finally:
72 del filtered_tbFile /usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/execute.py:52, in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
50 try:
51 ctx.ensure_initialized()
—> 52 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
53 inputs, attrs, num_outputs)
54 except core._NotOkStatusException as e:
55 if name is not None:InternalError: Graph execution error:
Detected at node ‘sequential_2/dense_4/MatMul’ defined at (most recent call last):
File “/usr/lib/python3.8/runpy.py”, line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File “/usr/lib/python3.8/runpy.py”, line 87, in _run_code
exec(code, run_globals)
File “/usr/local/lib/python3.8/dist-packages/ipykernel_launcher.py”, line 17, in <module>
app.launch_new_instance()
File “/usr/local/lib/python3.8/dist-packages/traitlets/config/application.py”, line 976, in launch_instance
app.start()
File “/usr/local/lib/python3.8/dist-packages/ipykernel/kernelapp.py”, line 712, in start
self.io_loop.start()
File “/usr/local/lib/python3.8/dist-packages/tornado/platform/asyncio.py”, line 215, in start
self.asyncio_loop.run_forever()
File “/usr/lib/python3.8/asyncio/base_events.py”, line 570, in run_forever
self._run_once()
File “/usr/lib/python3.8/asyncio/base_events.py”, line 1859, in _run_once
handle._run()
File “/usr/lib/python3.8/asyncio/events.py”, line 81, in _run
self._context.run(self._callback, *self._args)
File “/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py”, line 510, in dispatch_queue
await self.process_one()
File “/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py”, line 499, in process_one
await dispatch(*args)
File “/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py”, line 406, in dispatch_shell
await result
File “/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py”, line 730, in execute_request
reply_content = await reply_content
File “/usr/local/lib/python3.8/dist-packages/ipykernel/ipkernel.py”, line 383, in do_execute
res = shell.run_cell(
File “/usr/local/lib/python3.8/dist-packages/ipykernel/zmqshell.py”, line 528, in run_cell
return super().run_cell(*args, **kwargs)
File “/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py”, line 2881, in run_cell
result = self._run_cell(
File “/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py”, line 2936, in _run_cell
return runner(coro)
File “/usr/local/lib/python3.8/dist-packages/IPython/core/async_helpers.py”, line 129, in pseudo_sync_runner
coro.send(None)
File “/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py”, line 3135, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File “/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py”, line 3338, in run_ast_nodes
if await self.run_code(code, result, async=asy):
File “/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py”, line 3398, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File “/tmp/ipykernel_9744/1360389098.py”, line 9, in <cell line: 9>
model.fit(x=x_train,
File “/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py”, line 65, in error_handler
return fn(*args, **kwargs)
File “/usr/local/lib/python3.8/dist-packages/keras/engine/training.py”, line 1685, in fit
tmp_logs = self.train_function(iterator)
File “/usr/local/lib/python3.8/dist-packages/keras/engine/training.py”, line 1284, in train_function
return step_function(self, iterator)
File “/usr/local/lib/python3.8/dist-packages/keras/engine/training.py”, line 1268, in step_function
outputs = model.distribute_strategy.run(run_step, args=(data,))
File “/usr/local/lib/python3.8/dist-packages/keras/engine/training.py”, line 1249, in run_step
outputs = model.train_step(data)
File “/usr/local/lib/python3.8/dist-packages/keras/engine/training.py”, line 1050, in train_step
y_pred = self(x, training=True)
File “/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py”, line 65, in error_handler
return fn(*args, **kwargs)
File “/usr/local/lib/python3.8/dist-packages/keras/engine/training.py”, line 558, in call
return super().call(*args, **kwargs)
File “/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py”, line 65, in error_handler
return fn(*args, **kwargs)
File “/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py”, line 1145, in call
outputs = call_fn(inputs, *args, **kwargs)
File “/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py”, line 96, in error_handler
return fn(*args, **kwargs)
File “/usr/local/lib/python3.8/dist-packages/keras/engine/sequential.py”, line 412, in call
return super().call(inputs, training=training, mask=mask)
File “/usr/local/lib/python3.8/dist-packages/keras/engine/functional.py”, line 512, in call
return self._run_internal_graph(inputs, training=training, mask=mask)
File “/usr/local/lib/python3.8/dist-packages/keras/engine/functional.py”, line 669, in _run_internal_graph
outputs = node.layer(*args, **kwargs)
File “/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py”, line 65, in error_handler
return fn(*args, **kwargs)
File “/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py”, line 1145, in call
outputs = call_fn(inputs, *args, **kwargs)
File “/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py”, line 96, in error_handler
return fn(*args, **kwargs)
File “/usr/local/lib/python3.8/dist-packages/keras/layers/core/dense.py”, line 241, in call
outputs = tf.matmul(a=inputs, b=self.kernel)
Node: ‘sequential_2/dense_4/MatMul’
Attempting to perform BLAS operation using StreamExecutor without BLAS support
[[{{node sequential_2/dense_4/MatMul}}]] [Op:__inference_train_function_2338]