add peek_logged

This commit is contained in:
li-xl 2020-11-30 16:03:10 +08:00
parent 3c7d59ea16
commit 7c0798d057
29 changed files with 37 additions and 28 deletions

View File

@ -58,7 +58,7 @@
// CUDA and CUBLAS functions
#include <helper_functions.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#ifndef min
#define min(a,b) ((a < b) ? a : b)

View File

@ -14,7 +14,7 @@
#include <cuda_runtime.h>
#include <cublas.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#ifdef CUBLAS_API_H_
// cuBLAS API errors

View File

@ -65,7 +65,7 @@
#include <assert.h>
#include <cudnn.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#include "fp16_dev.h"
#include "fp16_emu.h"

View File

@ -1,5 +1,5 @@
#include <cudnn.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
const char *_cudaGetErrorEnum(cudnnStatus_t error) {
return cudnnGetErrorString(error);

View File

@ -9,7 +9,7 @@
#include "init.h"
#include <cuda_runtime.h>
#include <curand.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#include "curand_random_op.h"
#include "curand_warper.h"

View File

@ -10,7 +10,7 @@
#include <cuda_runtime.h>
#include <cublas.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#include <curand.h>
// cuRAND API errors

View File

@ -101,11 +101,17 @@ const char *_cudaGetErrorEnum(NppStatus error);
#endif
#endif
namespace jittor {
extern bool peek_logged;
}
template <typename T>
void peek(T result, char const *const func, const char *const file,
int const line) {
if (result) {
// DEVICE_RESET
if (jittor::peek_logged) return;
jittor::peek_logged = 1;
LOGe << "Peek CUDA error at" << file >> ":" >> line << " code="
>> static_cast<unsigned int>(result) >> "(" << _cudaGetErrorEnum(result) << ")"
<< func;

View File

@ -11,7 +11,7 @@
#include <cuda_runtime.h>
#include <nccl.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
namespace jittor {

View File

@ -12,7 +12,7 @@
#include <nccl.h>
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#include "nccl_warper.h"
#include "ops/op_register.h"
namespace jittor {

View File

@ -12,7 +12,7 @@
#include <nccl.h>
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#include "nccl_warper.h"
#include "ops/op_register.h"
namespace jittor {

View File

@ -12,7 +12,7 @@
#include <nccl.h>
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#include "nccl_warper.h"
#include "ops/op_register.h"
namespace jittor {

View File

@ -13,7 +13,7 @@
// These are CUDA Helper functions for initialization and error checking
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#ifdef _CUFFT_H_

View File

@ -7,7 +7,7 @@
#include <functional>
#ifdef HAS_CUDA
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#include "mem/allocator/cuda_dual_allocator.h"
#include "event_queue.h"
#endif
@ -446,6 +446,8 @@ void Executor::run_sync(vector<Var*> vars, bool device_sync) {
// record trace data
if (PREDICT_BRANCH_NOT_TAKEN(trace_py_var==2)) {
trace_data.record_execution(op, is_fused_op, jkl);
checkCudaErrors(cudaDeviceSynchronize());
}
LOGvvv << "Finished Op(" >> op->name() << rid >>
"/" >> queue.size() >> ") output:" << op->outputs();

View File

@ -9,7 +9,7 @@
#include "mem/allocator.h"
#ifdef HAS_CUDA
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#endif
namespace jittor {

View File

@ -5,7 +5,7 @@
// ***************************************************************
#ifdef HAS_CUDA
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#endif
#include <random>

View File

@ -5,7 +5,7 @@
// ***************************************************************
#ifdef HAS_CUDA
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#include "mem/allocator/cuda_device_allocator.h"
namespace jittor {

View File

@ -9,7 +9,7 @@
#include <mutex>
#include <cstring>
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#include "misc/cuda_flags.h"
#include "var.h"
#include "mem/allocator.h"

View File

@ -5,7 +5,7 @@
// ***************************************************************
#ifdef HAS_CUDA
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#include "mem/allocator/cuda_host_allocator.h"
namespace jittor {

View File

@ -5,7 +5,7 @@
// ***************************************************************
#ifdef HAS_CUDA
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#include "mem/allocator/cuda_managed_allocator.h"
namespace jittor {

View File

@ -5,7 +5,7 @@
// ***************************************************************
#ifdef HAS_CUDA
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#include "mem/allocator.h"
#include "mem/allocator/cuda_dual_allocator.h"
#include "event_queue.h"

View File

@ -5,7 +5,7 @@
// ***************************************************************
#ifdef HAS_CUDA
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#endif
#include <algorithm>
#include "var.h"

View File

@ -10,7 +10,7 @@
#include "ops/copy_op.h"
#ifdef HAS_CUDA
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#include "misc/cuda_flags.h"
#endif

View File

@ -7,7 +7,7 @@
// ***************************************************************
#ifdef HAS_CUDA
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#include <mutex>
#include "misc/cuda_flags.h"
#include "mem/allocator/sfrl_allocator.h"

View File

@ -10,7 +10,7 @@
#include "ops/op_register.h"
#ifdef JIT_cuda
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#endif
#ifndef JIT
#include "misc/stack_vector.h"

View File

@ -12,7 +12,7 @@
#include "ops/binary_op_defs.h"
#ifdef JIT_cuda
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#endif
#else
#include "ops/op_register.h"

View File

@ -10,7 +10,7 @@
#include <dlfcn.h>
#ifdef HAS_CUDA
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#endif
#include "misc/cuda_flags.h"
#include "profiler/profiler.h"

View File

@ -6,7 +6,7 @@
// ***************************************************************
#ifdef HAS_CUDA
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#include "mem/allocator.h"
#include "mem/allocator/cuda_dual_allocator.h"
#include "event_queue.h"

View File

@ -13,7 +13,8 @@
#include "utils/mwsr_list.h"
namespace jittor {
bool peek_logged = 0;
typedef uint32_t uint;
using string = std::string;
using stringstream = std::stringstream;

View File

@ -6,7 +6,7 @@
#include <sstream>
#ifdef HAS_CUDA
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include "helper_cuda.h"
#endif
#include "var_holder.h"
#include "var.h"