mirror of https://github.com/Jittor/Jittor
add naive windows support
This commit is contained in:
parent
a6830de216
commit
4fd355202c
|
|
@ -0,0 +1,10 @@
|
|||
jittor.attention
|
||||
=====================
|
||||
|
||||
这里是Jittor的 数据变换 模块的API文档，您可以通过`from jittor import attention`来获取该模块。
|
||||
|
||||
```eval_rst
|
||||
.. automodule:: jittor.attention
|
||||
:members:
|
||||
:undoc-members:
|
||||
```
|
||||
|
|
@ -8,6 +8,7 @@ import subprocess as sp
|
|||
import os
|
||||
import re
|
||||
import sys
|
||||
import glob
|
||||
import inspect
|
||||
import datetime
|
||||
import threading
|
||||
|
|
@ -49,7 +50,21 @@ def compile(compiler, flags, inputs, output, combind_build=False):
|
|||
run_cmd(cmd)
|
||||
return True
|
||||
link = link_flags
|
||||
base_output = output.split('/')[-1].split('.')[0]
|
||||
base_output = os.path.basename(output).split('.')[0]
|
||||
if os.name == 'nt':
|
||||
# initialize order in windows seems reversed
|
||||
inputs = list(inputs[::-1])
|
||||
# windows need libxxx.a
|
||||
afile = os.path.join(cache_path, f"lib{base_output}.a")
|
||||
link = link + f' -Wl,--export-all-symbols,--out-implib,"{afile}" '
|
||||
if base_output == "jit_utils_core":
|
||||
pass
|
||||
elif base_output == "jittor_core":
|
||||
inputs.append(os.path.join(cache_path, f"libjit_utils_core.a"))
|
||||
else:
|
||||
inputs.append(os.path.join(cache_path, f"libjit_utils_core.a"))
|
||||
inputs.append(os.path.join(cache_path, f"libjittor_core.a"))
|
||||
|
||||
# if output is core, add core_link_flags
|
||||
if output.startswith("jittor_core"):
|
||||
link = link + core_link_flags
|
||||
|
|
@ -58,7 +73,7 @@ def compile(compiler, flags, inputs, output, combind_build=False):
|
|||
obj_files = []
|
||||
new_inputs = []
|
||||
for name in inputs:
|
||||
if name.endswith(".o"):
|
||||
if name[-1] in 'oa':
|
||||
obj_files.append(name)
|
||||
else:
|
||||
new_inputs.append(os.path.join(jittor_path, name))
|
||||
|
|
@ -67,7 +82,7 @@ def compile(compiler, flags, inputs, output, combind_build=False):
|
|||
inputs = new_inputs
|
||||
|
||||
if len(inputs) == 1 or combind_build:
|
||||
cmd = f"{compiler} {' '.join(inputs)} {flags} {link} -o {output}"
|
||||
cmd = f"\"{compiler}\" {' '.join(inputs)} {flags} {link} -o {output}"
|
||||
return do_compile(cmd)
|
||||
# split compile object file and link
|
||||
# remove -l -L flags when compile object files
|
||||
|
|
@ -89,20 +104,21 @@ def compile(compiler, flags, inputs, output, combind_build=False):
|
|||
cmd = cmd.replace("-Ofast", "-O2")
|
||||
cmds.append(cmd)
|
||||
jit_utils.run_cmds(cmds, cache_path, jittor_path, "Compiling "+base_output)
|
||||
cmd = f"{compiler} {' '.join(obj_files)} {flags} {lto_flags} {link} -o {output}"
|
||||
cmd = f"\"{compiler}\" {' '.join(obj_files)} {flags} {lto_flags} {link} -o {output}"
|
||||
return do_compile(cmd)
|
||||
|
||||
def gen_jit_tests():
|
||||
all_src = run_cmd('find -L src/ | grep "cc$"', jittor_path).splitlines()
|
||||
# all_src = run_cmd('find -L src/ | grep "cc$"', jittor_path).splitlines()
|
||||
# all_src = glob.glob(os.path.join(jittor_path,"src","**","*.cc"), recursive=True)
|
||||
all_src = glob.glob(jittor_path+"/src/**/*.cc", recursive=True)
|
||||
jit_declares = []
|
||||
re_def = re.compile("JIT_TEST\\((.*?)\\)")
|
||||
names = set()
|
||||
test_defs = []
|
||||
|
||||
for src_name in all_src:
|
||||
src_name = os.path.join(jittor_path, src_name)
|
||||
with open(src_name) as f:
|
||||
src = f.read()
|
||||
with open(src_name, 'rb') as f:
|
||||
src = f.read().decode('utf8')
|
||||
defs = re_def.findall(src)
|
||||
for name in defs:
|
||||
LOG.vv(f"Find test {name} from {src_name}")
|
||||
|
|
@ -143,7 +159,8 @@ def gen_jit_tests():
|
|||
f.write(jit_src)
|
||||
|
||||
def gen_jit_flags():
|
||||
all_src = run_cmd('find -L src/ | grep "cc$"', jittor_path).splitlines()
|
||||
# all_src = run_cmd('find -L src/ | grep "cc$"', jittor_path).splitlines()
|
||||
all_src = glob.glob(jittor_path+"/src/**/*.cc", recursive=True)
|
||||
jit_declares = []
|
||||
re_def = re.compile("DEFINE_FLAG(_WITH_SETTER)?\\((.*?)\\);", re.DOTALL)
|
||||
|
||||
|
|
@ -151,9 +168,8 @@ def gen_jit_flags():
|
|||
visit = {}
|
||||
|
||||
for src_name in all_src:
|
||||
src_name = os.path.join(jittor_path, src_name)
|
||||
with open(src_name) as f:
|
||||
src = f.read()
|
||||
with open(src_name, 'rb') as f:
|
||||
src = f.read().decode("utf8")
|
||||
defs = re_def.findall(src)
|
||||
for _, args in defs:
|
||||
args = args.split(",")
|
||||
|
|
@ -360,13 +376,13 @@ def gen_jit_op_maker(op_headers, export=False, extra_flags=""):
|
|||
# XxxXxxOp
|
||||
name2 = map(lambda s:s[:1].upper() + s[1:], name.split('_'))
|
||||
name2 = "".join(name2)
|
||||
with open(os.path.join(jittor_path, header), encoding='utf8') as f:
|
||||
with open(header, encoding='utf8') as f:
|
||||
src = f.read()
|
||||
# XxxXxxOp(args)
|
||||
res = re.findall(pybind_attrs_reg + '[^~]('+name2+"\\([^\\n]*\\))", src, re.S)
|
||||
assert len(res) >= 1, "Wrong op args in " + header
|
||||
# registe op
|
||||
cc_name = os.path.join(jittor_path, header[:-2] + ".cc")
|
||||
cc_name = header[:-2] + ".cc"
|
||||
constructors = []
|
||||
for i in range(len(res)):
|
||||
name = 'make_'+func_name+'_'*i
|
||||
|
|
@ -440,10 +456,10 @@ def gen_jit_op_maker(op_headers, export=False, extra_flags=""):
|
|||
vh2v_src = "_op->set_inputs({" + ", ".join(vh2v_src) + "});" + \
|
||||
"".join(more_src)
|
||||
LOG.vvvv(f"Find op: {name2} args: {new_args}")
|
||||
if header.startswith("src/"):
|
||||
jit_headers += f"#include \"{header[4:]}\"\n"
|
||||
else:
|
||||
jit_headers += f"#include \"{header}\"\n"
|
||||
# if header.startswith("src/"):
|
||||
# jit_headers += f"#include \"{header[4:]}\"\n"
|
||||
# else:
|
||||
jit_headers += f"#include \"{header}\"\n"
|
||||
add_src(
|
||||
func_name+'_'*hid,
|
||||
new_args_def,
|
||||
|
|
@ -657,7 +673,7 @@ def compile_custom_ops(
|
|||
|
||||
for name in pyjt_includes:
|
||||
LOG.i("handle pyjt_include", name)
|
||||
bname = name.split("/")[-1].split(".")[0]
|
||||
bname = os.path.basename(name).split(".")[0]
|
||||
gen_src_fname = os.path.join(cache_path, "custom_ops", gen_name+"_"+bname+".cc")
|
||||
pyjt_compiler.compile_single(name, gen_src_fname)
|
||||
builds.insert(1, gen_src_fname)
|
||||
|
|
@ -786,7 +802,10 @@ def check_cache_compile():
|
|||
"src/utils/log.cc",
|
||||
"src/utils/tracer.cc",
|
||||
"src/utils/jit_utils.cc",
|
||||
"src/utils/str_utils.cc",
|
||||
]
|
||||
if os.name == 'nt':
|
||||
files = [ x.replace('/', '\\') for x in files ]
|
||||
global jit_utils_core_files
|
||||
jit_utils_core_files = files
|
||||
recompile = compile(cc_path, cc_flags+f" {opt_flags} ", files, 'jit_utils_core'+extension_suffix, True)
|
||||
|
|
@ -866,13 +885,14 @@ python_path = sys.executable
|
|||
ex_python_path = python_path + '.' + str(sys.version_info.minor)
|
||||
if os.path.isfile(ex_python_path):
|
||||
python_path = ex_python_path
|
||||
py3_config_path = jit_utils.py3_config_path
|
||||
nvcc_path = env_or_try_find('nvcc_path', 'nvcc') or try_find_exe('/usr/local/cuda/bin/nvcc') or try_find_exe('/usr/bin/nvcc')
|
||||
if not nvcc_path:
|
||||
cuda_driver = install_cuda.get_cuda_driver()
|
||||
nvcc_path = install_cuda.install_cuda()
|
||||
if nvcc_path:
|
||||
nvcc_path = try_find_exe(nvcc_path)
|
||||
if nvcc_path is None:
|
||||
nvcc_path = ""
|
||||
gdb_path = try_find_exe('gdb')
|
||||
addr2line_path = try_find_exe('addr2line')
|
||||
has_pybt = check_pybt(gdb_path, python_path)
|
||||
|
|
@ -885,6 +905,20 @@ link_flags = " -lstdc++ -ldl -shared "
|
|||
core_link_flags = ""
|
||||
opt_flags = ""
|
||||
kernel_opt_flags = os.environ.get("kernel_flags", "") + opt_flags + " -fopenmp "
|
||||
if os.name == 'nt':
|
||||
link_flags = link_flags.replace('-ldl', '')
|
||||
py3_link_path = '-L"' + os.path.join(
|
||||
os.path.dirname(sys.executable),
|
||||
"libs"
|
||||
) + f'" -lpython3{sys.version_info.minor} '
|
||||
core_link_flags = py3_link_path
|
||||
link_flags += core_link_flags
|
||||
# link_flags += " -Wl,--unresolved-symbols=ignore-all "
|
||||
# cc_flags += " -Xlinker --allow-shlib-undefined "
|
||||
cc_flags = cc_flags.replace('-std=c++14', '-std=c++17')
|
||||
link_flags += " -fopenmp "
|
||||
kernel_opt_flags += f" {cache_path}\\libjit_utils_core.a "
|
||||
kernel_opt_flags += f" {cache_path}\\libjittor_core.a "
|
||||
|
||||
if ' -O' not in cc_flags:
|
||||
opt_flags += " -O2 "
|
||||
|
|
@ -898,9 +932,9 @@ if os.environ.get("enable_lto") == "1":
|
|||
else:
|
||||
lto_flags = " -flto "
|
||||
|
||||
py_include = run_cmd(py3_config_path+" --includes")
|
||||
py_include = jit_utils.get_py3_include_path()
|
||||
LOG.i(f"py_include: {py_include}")
|
||||
extension_suffix = run_cmd(py3_config_path+" --extension-suffix")
|
||||
extension_suffix = jit_utils.get_py3_extension_suffix()
|
||||
LOG.i(f"extension_suffix: {extension_suffix}")
|
||||
|
||||
make_cache_dir(cache_path)
|
||||
|
|
@ -911,7 +945,7 @@ ck_path = os.path.join(cache_path, "checkpoints")
|
|||
make_cache_dir(ck_path)
|
||||
|
||||
# build cache_compile
|
||||
cc_flags += f" -I{jittor_path}/src "
|
||||
cc_flags += f" -I{os.path.join(jittor_path, 'src')} "
|
||||
cc_flags += py_include
|
||||
check_cache_compile()
|
||||
LOG.v(f"Get cache_compile: {jit_utils.cc}")
|
||||
|
|
@ -943,7 +977,8 @@ if has_cuda:
|
|||
# build core
|
||||
gen_jit_flags()
|
||||
gen_jit_tests()
|
||||
op_headers = run_cmd('find -L src/ops/ | grep "op.h$"', jittor_path).splitlines()
|
||||
op_headers = glob.glob(jittor_path+"/src/ops/**/*op.h", recursive=True)
|
||||
# op_headers = run_cmd('find -L src/ops/ | grep "op.h$"', jittor_path).splitlines()
|
||||
jit_src = gen_jit_op_maker(op_headers)
|
||||
LOG.vvvv(jit_src)
|
||||
with open(os.path.join(cache_path, "gen", "jit_op_maker.h"), 'w') as f:
|
||||
|
|
@ -958,8 +993,10 @@ pyjt_gen_src = pyjt_compiler.compile(cache_path, jittor_path)
|
|||
# 3. op_utils
|
||||
# 4. other
|
||||
files2 = pyjt_gen_src
|
||||
grep_args = '"c[cu]$"' if has_cuda else '"cc$"'
|
||||
files4 = run_cmd('find -L src | grep '+grep_args, jittor_path).splitlines()
|
||||
ext_args = 'c[cu]' if has_cuda else 'cc'
|
||||
files4 = glob.glob(jittor_path+"/src/**/*."+ext_args, recursive=True)
|
||||
files4 = [ f[len(jittor_path)+1:] for f in files4 ]
|
||||
# files4 = run_cmd('find -L src | grep '+grep_args, jittor_path).splitlines()
|
||||
at_beginning = [
|
||||
"src/ops/op_utils.cc",
|
||||
"src/event_queue.cc",
|
||||
|
|
@ -969,16 +1006,14 @@ at_beginning = [
|
|||
at_last = [
|
||||
"src/profiler/profiler.cc",
|
||||
"src/executor.cc",
|
||||
"src/fetcher.cc",
|
||||
]
|
||||
if os.name == 'nt':
|
||||
at_beginning = [ x.replace('/','\\') for x in at_beginning ]
|
||||
at_last = [ x.replace('/','\\') for x in at_last ]
|
||||
for i in range(len(at_beginning)):
|
||||
if at_beginning[i] not in files4:
|
||||
continue
|
||||
files4.remove(at_beginning[i])
|
||||
files4.insert(i, at_beginning[i])
|
||||
for v in at_last:
|
||||
if v not in files4:
|
||||
continue
|
||||
files4.remove(v)
|
||||
files4.append(v)
|
||||
registers = [ name for name in files4 if "register" in name ]
|
||||
|
|
@ -988,22 +1023,24 @@ for file in jit_utils_core_files:
|
|||
files.remove(file)
|
||||
LOG.vv("compile order:", files)
|
||||
|
||||
# manual Link omp using flags(os.RTLD_NOW | os.RTLD_GLOBAL)
|
||||
# if cc_type=="icc":
|
||||
# os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
|
||||
libname = {"clang":"omp", "icc":"iomp5", "g++":"gomp"}[cc_type]
|
||||
libname = ctypes.util.find_library(libname)
|
||||
assert libname is not None, "openmp library not found"
|
||||
ctypes.CDLL(libname, os.RTLD_NOW | os.RTLD_GLOBAL)
|
||||
if os.name != 'nt':
|
||||
# manual Link omp using flags(os.RTLD_NOW | os.RTLD_GLOBAL)
|
||||
# if cc_type=="icc":
|
||||
# os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
|
||||
libname = {"clang":"omp", "icc":"iomp5", "g++":"gomp"}[cc_type]
|
||||
libname = ctypes.util.find_library(libname)
|
||||
assert libname is not None, "openmp library not found"
|
||||
ctypes.CDLL(libname, os.RTLD_NOW | os.RTLD_GLOBAL)
|
||||
|
||||
# get os release
|
||||
with open("/etc/os-release", "r", encoding='utf8') as f:
|
||||
s = f.read().splitlines()
|
||||
os_release = {}
|
||||
for line in s:
|
||||
a = line.split('=')
|
||||
if len(a) != 2: continue
|
||||
os_release[a[0]] = a[1].replace("\"", "")
|
||||
# get os release
|
||||
|
||||
with open("/etc/os-release", "r", encoding='utf8') as f:
|
||||
s = f.read().splitlines()
|
||||
os_release = {}
|
||||
for line in s:
|
||||
a = line.split('=')
|
||||
if len(a) != 2: continue
|
||||
os_release[a[0]] = a[1].replace("\"", "")
|
||||
|
||||
os_type = {
|
||||
"ubuntu": "ubuntu",
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import os
|
|||
from jittor_utils import LOG, run_cmd, simple_timer
|
||||
import json
|
||||
from collections import OrderedDict
|
||||
import glob
|
||||
|
||||
def parse_attrs(s):
|
||||
'''parse @attrs(..., x=y) syntax'''
|
||||
|
|
@ -847,7 +848,7 @@ def compile_src(src, h, basename):
|
|||
return src_code
|
||||
|
||||
def compile_single(head_file_name, src_file_name, src=None):
|
||||
basename = head_file_name.split("/")[-1].split(".")[0]
|
||||
basename = os.path.basename(head_file_name).split(".")[0]
|
||||
if src==None:
|
||||
with open(head_file_name, 'r') as f:
|
||||
src = f.read()
|
||||
|
|
@ -860,22 +861,22 @@ def compile_single(head_file_name, src_file_name, src=None):
|
|||
return True
|
||||
|
||||
def compile(cache_path, jittor_path):
|
||||
headers1 = run_cmd('find -L src/ | grep ".h$"', jittor_path).splitlines()
|
||||
headers2 = run_cmd('find gen/ | grep ".h$"', cache_path).splitlines()
|
||||
headers = [ os.path.join(jittor_path, h) for h in headers1 ] + \
|
||||
[ os.path.join(cache_path, h) for h in headers2 ]
|
||||
headers1 = glob.glob(jittor_path+"/src/**/*.h", recursive=True)
|
||||
headers2 = glob.glob(cache_path+"/gen/**/*.h", recursive=True)
|
||||
headers = headers1 + headers2
|
||||
basenames = []
|
||||
pyjt_names = []
|
||||
for h in headers:
|
||||
with open(h, 'r') as f:
|
||||
src = f.read()
|
||||
|
||||
bh = os.path.basename(h)
|
||||
# jit_op_maker.h merge compile with var_holder.h
|
||||
if h.endswith("src/var_holder.h"): continue
|
||||
if h.endswith("jit_op_maker.h"):
|
||||
if bh == "var_holder.h": continue
|
||||
if bh == "jit_op_maker.h":
|
||||
with open(os.path.join(jittor_path, "src", "var_holder.h"), "r") as f:
|
||||
src = f.read() + src
|
||||
basename = h.split("/")[-1].split(".")[0]
|
||||
basename = bh.split(".")[0]
|
||||
fname = "pyjt_"+basename+".cc"
|
||||
fname = os.path.join(cache_path, "gen", fname)
|
||||
check = compile_single(h, fname, src)
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
|
||||
namespace jittor {
|
||||
|
||||
#ifdef HAS_CUDA
|
||||
EventQueue event_queue;
|
||||
|
||||
void EventQueue::Worker::start() {
|
||||
|
|
@ -25,6 +26,20 @@ void EventQueue::Worker::start() {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
void EventQueue::Worker::stop() {
|
||||
LOGv << "stoping event queue worker...";
|
||||
event_queue.worker.run(nullptr);
|
||||
event_queue.worker.thread.join();
|
||||
LOGv << "stopped event queue worker.";
|
||||
}
|
||||
|
||||
extern vector<void(*)()> cleanup_callback;
|
||||
|
||||
EventQueue::Worker::Worker() : thread(EventQueue::Worker::start) {
|
||||
cleanup_callback.push_back(&EventQueue::Worker::stop);
|
||||
}
|
||||
|
||||
void EventQueue::worker_caller() {
|
||||
int status = OK;
|
||||
try {
|
||||
|
|
@ -39,5 +54,7 @@ void EventQueue::worker_caller() {
|
|||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
} // jittor
|
||||
|
|
@ -12,18 +12,29 @@
|
|||
|
||||
namespace jittor {
|
||||
|
||||
#ifdef HAS_CUDA
|
||||
struct EventQueue {
|
||||
static constexpr int RUNNING = 0;
|
||||
static constexpr int OK = 1;
|
||||
static constexpr int ERROR = 2;
|
||||
typedef void(*Func)();
|
||||
|
||||
list<Func> tasks;
|
||||
std::condition_variable cv;
|
||||
std::mutex mtx;
|
||||
Func func;
|
||||
volatile int run_sync_done;
|
||||
|
||||
struct Worker {
|
||||
Func todo;
|
||||
std::condition_variable cv;
|
||||
std::mutex mtx;
|
||||
std::thread thread = std::thread(Worker::start);
|
||||
std::thread thread;
|
||||
|
||||
static void start();
|
||||
static void stop();
|
||||
|
||||
Worker();
|
||||
|
||||
inline void run(Func func) {
|
||||
{
|
||||
|
|
@ -32,19 +43,8 @@ struct EventQueue {
|
|||
}
|
||||
cv.notify_one();
|
||||
}
|
||||
|
||||
inline ~Worker() {
|
||||
run(nullptr);
|
||||
thread.join();
|
||||
}
|
||||
} worker;
|
||||
|
||||
list<Func> tasks;
|
||||
std::condition_variable cv;
|
||||
std::mutex mtx;
|
||||
Func func;
|
||||
volatile int run_sync_done;
|
||||
|
||||
inline void flush() {
|
||||
list<Func> ts;
|
||||
{
|
||||
|
|
@ -57,7 +57,7 @@ struct EventQueue {
|
|||
|
||||
static void worker_caller();
|
||||
|
||||
int run_sync(Func func) {
|
||||
inline int run_sync(Func func) {
|
||||
// send work to worker and do something by self
|
||||
std::unique_lock<std::mutex> l(mtx);
|
||||
this->func = func;
|
||||
|
|
@ -90,4 +90,6 @@ struct EventQueue {
|
|||
|
||||
extern EventQueue event_queue;
|
||||
|
||||
#endif
|
||||
|
||||
} // jittor
|
||||
|
|
@ -29,10 +29,14 @@ int current_seed;
|
|||
// fron fetch_op.cc
|
||||
extern list<VarPtr> fetcher;
|
||||
extern list<VarPtr> fetcher_to_free;
|
||||
extern vector<void(*)()> cleanup_callback;
|
||||
|
||||
void cleanup() {
|
||||
fetcher_to_free.clear();
|
||||
fetcher.clear();
|
||||
for (auto cb : cleanup_callback)
|
||||
cb();
|
||||
cleanup_callback.clear();
|
||||
}
|
||||
|
||||
static void init_cuda_devices() {
|
||||
|
|
|
|||
|
|
@ -7,7 +7,11 @@
|
|||
#include <fstream>
|
||||
#include <streambuf>
|
||||
#include <stdlib.h>
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <dlfcn.h>
|
||||
#endif
|
||||
|
||||
#include "jit_compiler.h"
|
||||
#include "op.h"
|
||||
|
|
@ -30,16 +34,25 @@ DEFINE_FLAG(int, rewrite_op, 1, "Rewrite source file of jit operator or not");
|
|||
namespace jit_compiler {
|
||||
|
||||
jit_op_entry_t load_jit_lib(string name, string symbol_name="jit_entry") {
|
||||
const char* msg = "";
|
||||
LOGvv << "Opening jit lib:" << name;
|
||||
// void* handle = dlopen(name.c_str(), RTLD_NOW | RTLD_DEEPBIND | RTLD_LOCAL);
|
||||
// RTLD_DEEPBIND and openmp cause segfault
|
||||
void* handle = dlopen(name.c_str(), RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND);
|
||||
CHECK(handle) << "Cannot open library" << name << ":" << dlerror();
|
||||
#ifdef _WIN32
|
||||
void* handle = (void*)LoadLibrary(name.c_str());
|
||||
#else
|
||||
void* handle = dlopen(name.c_str(), RTLD_LAZY | RTLD_DEEPBIND | RTLD_LOCAL);
|
||||
msg = dlerror();
|
||||
#endif
|
||||
|
||||
CHECK(handle) << "Cannot open library" << name << ":" << msg;
|
||||
|
||||
#ifdef _WIN32
|
||||
auto jit_entry = (jit_op_entry_t)GetProcAddress((HINSTANCE)handle, symbol_name.c_str());
|
||||
#else
|
||||
//dlerror();
|
||||
auto jit_entry = (jit_op_entry_t)dlsym(handle, symbol_name.c_str());
|
||||
const char* dlsym_error = dlerror();
|
||||
CHECK(!dlsym_error) << "Loading symbol jit_entry from" << name << "failed:" << dlsym_error;
|
||||
msg = dlerror();
|
||||
#endif
|
||||
CHECK(jit_entry) << "Loading symbol" << symbol_name << "from" << name << "failed:" << msg;
|
||||
|
||||
return jit_entry;
|
||||
}
|
||||
|
|
@ -66,26 +79,30 @@ jit_op_entry_t compile(const string& jit_key, const string& src, const bool is_c
|
|||
// compiler do not allowed filename too long
|
||||
CHECK(cc_path.size());
|
||||
string jit_src_path = Op::get_filename_from_jit_key(jit_key, ".cc");
|
||||
#ifdef _WIN32
|
||||
string jit_lib_path = Op::get_filename_from_jit_key(jit_key, ".dll");
|
||||
#else
|
||||
string jit_lib_path = Op::get_filename_from_jit_key(jit_key, ".so");
|
||||
string other_src = " "+join(jittor_path, "src/op.cc")+" "+
|
||||
join(jittor_path, "src/var.cc")+" ";
|
||||
other_src = "";
|
||||
#endif
|
||||
string other_src;
|
||||
LOGvvv << "Generate" << jit_src_path >> "\n" >> src;
|
||||
if (rewrite_op || !file_exist(jit_src_path))
|
||||
write(jit_src_path, src);
|
||||
string cmd;
|
||||
if (is_cuda_op) {
|
||||
cmd = nvcc_path
|
||||
+ " '" + jit_src_path + "'" + other_src
|
||||
+ " \"" + jit_src_path + "\"" + other_src
|
||||
+ nvcc_flags + extra_flags
|
||||
+ " -o '" + jit_lib_path + "'";
|
||||
+ " -o \"" + jit_lib_path + "\"";
|
||||
} else {
|
||||
cmd = cc_path
|
||||
+ " '" + jit_src_path + "'" + other_src
|
||||
+ " \"" + jit_src_path + "\"" + other_src
|
||||
+ cc_flags + extra_flags
|
||||
+ " -o '" + jit_lib_path + "'";
|
||||
+ " -o \"" + jit_lib_path + "\"";
|
||||
#ifndef _WIN32
|
||||
cmd = python_path+" "+jittor_path+"/utils/asm_tuner.py "
|
||||
"--cc_path=" + cmd;
|
||||
#endif
|
||||
}
|
||||
cache_compile(cmd, cache_path, jittor_path);
|
||||
auto symbol_name = get_symbol_name(jit_key);
|
||||
|
|
|
|||
|
|
@ -4,7 +4,9 @@
|
|||
// This file is subject to the terms and conditions defined in
|
||||
// file 'LICENSE.txt', which is part of this source code package.
|
||||
// ***************************************************************
|
||||
#ifndef _WIN32
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
#include <sstream>
|
||||
#include "jit_key.h"
|
||||
#include "utils/str_utils.h"
|
||||
|
|
@ -28,17 +30,23 @@ static size_t get_buffer_end_page(size_t buffer_end) {
|
|||
}
|
||||
|
||||
JitKey::JitKey() {
|
||||
(void)get_buffer_end_page;
|
||||
#ifndef _WIN32
|
||||
auto buffer_end_page = get_buffer_end_page((size_t)&buffer[buffer_size-1]);
|
||||
LOGvv << "protect page" << (void*)buffer_end_page;
|
||||
ASSERT(0==mprotect((void*)buffer_end_page, page_size, PROT_NONE));
|
||||
// windows assign extern thread_local var cause fault, FIX IT
|
||||
protected_page = buffer_end_page;
|
||||
#endif
|
||||
}
|
||||
|
||||
JitKey::~JitKey() {
|
||||
#ifndef _WIN32
|
||||
auto buffer_end_page = get_buffer_end_page((size_t)&buffer[buffer_size-1]);
|
||||
LOGvv << "un-protect page" << (void*)buffer_end_page;
|
||||
mprotect((void*)buffer_end_page, page_size, PROT_READ|PROT_WRITE|PROT_EXEC);
|
||||
protected_page = 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static void hex_to_dec(string& s) {
|
||||
|
|
|
|||
|
|
@ -165,10 +165,12 @@ inline JK& operator<<(JK& jk, int64 c) {
|
|||
}
|
||||
return jk << JK::hex(c);
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
// win32 cause redefinition error
|
||||
inline JK& operator<<(JK& jk, long long int c) {
|
||||
return jk << (int64)c;
|
||||
}
|
||||
#endif
|
||||
|
||||
inline JK& operator<<(JK& jk, uint64 c) {
|
||||
return jk << JK::hex(c);
|
||||
|
|
|
|||
|
|
@ -9,6 +9,10 @@
|
|||
// ***************************************************************
|
||||
#include <stdio.h>
|
||||
#include <unistd.h>
|
||||
#ifdef _WIN32
|
||||
#include <fileapi.h>
|
||||
#else
|
||||
#endif
|
||||
#include <fcntl.h>
|
||||
#include <errno.h>
|
||||
|
||||
|
|
@ -27,6 +31,11 @@ void set_lock_path(string path) {
|
|||
|
||||
void lock() {
|
||||
ASSERT(lock_fd >= 0);
|
||||
#ifdef _WIN32
|
||||
OVERLAPPED offset = {0, 0, 0, 0, NULL};
|
||||
auto hfile = (HANDLE)_get_osfhandle(lock_fd);
|
||||
ASSERT(LockFileEx(hfile, 2, 0, -0x10000, 0, &offset));
|
||||
#else
|
||||
struct flock lock = {
|
||||
.l_type = F_WRLCK,
|
||||
.l_whence = SEEK_SET,
|
||||
|
|
@ -34,12 +43,18 @@ void lock() {
|
|||
.l_len = 0
|
||||
};
|
||||
ASSERT(fcntl(lock_fd, F_SETLKW, &lock) == 0);
|
||||
#endif
|
||||
_has_lock = 1;
|
||||
LOGvv << "LOCK Pid:" << getpid();
|
||||
}
|
||||
|
||||
void unlock() {
|
||||
ASSERT(lock_fd >= 0);
|
||||
#ifdef _WIN32
|
||||
OVERLAPPED offset = {0, 0, 0, 0, NULL};
|
||||
auto hfile = (HANDLE)_get_osfhandle(lock_fd);
|
||||
ASSERT(UnlockFileEx(hfile, 0, -0x10000, 0, &offset));
|
||||
#else
|
||||
struct flock lock = {
|
||||
.l_type = F_UNLCK,
|
||||
.l_whence = SEEK_SET,
|
||||
|
|
@ -47,6 +62,7 @@ void unlock() {
|
|||
.l_len = 0
|
||||
};
|
||||
ASSERT(fcntl(lock_fd, F_SETLKW, &lock) == 0);
|
||||
#endif
|
||||
_has_lock = 0;
|
||||
LOGvv << "UNLOCK Pid:" << getpid();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,11 @@ AlignedAllocator aligned_allocator;
|
|||
const char* AlignedAllocator::name() const {return "aligned";}
|
||||
|
||||
void* AlignedAllocator::alloc(size_t size, size_t& allocation) {
|
||||
#ifndef _WIN32
|
||||
return aligned_alloc(alignment, size);
|
||||
#else
|
||||
return _aligned_malloc(size, alignment);
|
||||
#endif
|
||||
}
|
||||
|
||||
void AlignedAllocator::free(void* mem_ptr, size_t size, const size_t& allocation) {
|
||||
|
|
|
|||
|
|
@ -6,7 +6,11 @@
|
|||
// ***************************************************************
|
||||
#include <iomanip>
|
||||
#include <algorithm>
|
||||
#ifndef _WIN32
|
||||
#include <sys/sysinfo.h>
|
||||
#else
|
||||
#include <windows.h>
|
||||
#endif
|
||||
|
||||
#include "var.h"
|
||||
#include "op.h"
|
||||
|
|
@ -152,9 +156,15 @@ void display_memory_info(const char* fileline, bool dump_var, bool red_color) {
|
|||
}
|
||||
|
||||
MemInfo::MemInfo() {
|
||||
#ifndef _WIN32
|
||||
struct sysinfo info = {0};
|
||||
sysinfo(&info);
|
||||
total_cpu_ram = info.totalram;
|
||||
#else
|
||||
MEMORYSTATUSEX statex;
|
||||
GlobalMemoryStatusEx (&statex);
|
||||
total_cpu_ram = statex.ullTotalPhys;
|
||||
#endif
|
||||
total_cuda_ram = 0;
|
||||
#ifdef HAS_CUDA
|
||||
cudaDeviceProp prop;
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@
|
|||
#include "mem/allocator/sfrl_allocator.h"
|
||||
#include <iomanip>
|
||||
#include <algorithm>
|
||||
#include <sys/sysinfo.h>
|
||||
#include <sstream>
|
||||
#include "pybind/py_var_tracer.h"
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,8 @@ namespace jittor {
|
|||
m(float32) \
|
||||
m(float64)
|
||||
|
||||
#define map_size(T) {#T, ffs(sizeof(T))-1},
|
||||
#define map_size(T) {#T, __builtin_ffs(sizeof(T))-1},
|
||||
|
||||
unordered_map<string, size_t> dsize_map = {FOR_ALL_TYPES(map_size)};
|
||||
|
||||
// TODO: make all static
|
||||
|
|
|
|||
|
|
@ -6,7 +6,9 @@
|
|||
// ***************************************************************
|
||||
#include <chrono>
|
||||
#include <thread>
|
||||
#ifndef _WIN32
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
#include "common.h"
|
||||
#include "misc/ring_buffer.h"
|
||||
|
||||
|
|
@ -41,7 +43,13 @@ RingBuffer* RingBuffer::make_ring_buffer(uint64 size, bool multiprocess) {
|
|||
uint64 total_size = sizeof(RingBuffer) + size;
|
||||
void* ptr = multiprocess ?
|
||||
// mmap(NULL, total_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0) :
|
||||
mmap(NULL, total_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0) :
|
||||
#ifndef _WIN32
|
||||
mmap(NULL, total_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0)
|
||||
#else
|
||||
// TODO: multiprocess ring buffer in windows
|
||||
(void*)malloc(total_size)
|
||||
#endif
|
||||
:
|
||||
// mmap(NULL, total_size, PROT_READ | PROT_WRITE, MAP_SHARED, -1, 0) :
|
||||
(void*)malloc(total_size);
|
||||
std::memset(ptr, 0, total_size);
|
||||
|
|
@ -55,7 +63,12 @@ void RingBuffer::free_ring_buffer(RingBuffer* rb) {
|
|||
auto is_multiprocess = rb->is_multiprocess;
|
||||
rb->~RingBuffer();
|
||||
if (is_multiprocess) {
|
||||
#ifndef _WIN32
|
||||
munmap(rb, total_size);
|
||||
#else
|
||||
free((void*)rb);
|
||||
#endif
|
||||
(void)total_size;
|
||||
} else {
|
||||
free((void*)rb);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@
|
|||
// ***************************************************************
|
||||
#pragma once
|
||||
#include <pthread.h>
|
||||
#include <sys/mman.h>
|
||||
#include <cstring>
|
||||
#include "common.h"
|
||||
|
||||
|
|
@ -48,10 +47,14 @@ struct RingBuffer {
|
|||
}
|
||||
|
||||
inline ~Cond() {
|
||||
#ifndef _WIN32
|
||||
// a dirty hack
|
||||
// ref: https://stackoverflow.com/questions/20439404/pthread-conditions-and-process-termination
|
||||
// cv.__data.__wrefs = 0;
|
||||
cv.__data = {0};
|
||||
#else
|
||||
// cv.__data = 0;
|
||||
#endif
|
||||
pthread_cond_destroy(&cv);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -201,10 +201,10 @@ string Op::get_filename_from_jit_key(const string& jit_key, const string& suffix
|
|||
string s = iter==jit_key_mapper.end() ? jit_key : iter->second;
|
||||
std::stringstream ss;
|
||||
if (s.size() > 100) {
|
||||
ss << s.substr(0, 90) << "...hash:"
|
||||
ss << s.substr(0, 90) << "...hash_"
|
||||
<< std::hex << std::hash<string>()(s);
|
||||
} else {
|
||||
ss << s << "_hash:" <<
|
||||
ss << s << "_hash_" <<
|
||||
std::hex << std::hash<string>()(s);
|
||||
}
|
||||
s = ss.str();
|
||||
|
|
@ -212,10 +212,14 @@ string Op::get_filename_from_jit_key(const string& jit_key, const string& suffix
|
|||
if (c=='[' || c==']' || c=='<' || c=='>'
|
||||
|| c=='{' || c=='}' || c=='(' || c==')' || c==','
|
||||
|| c=='\n' || c=='\t' || c==' ' || c=='&' || c=='|'
|
||||
|| c=='/')
|
||||
|| c=='/' || c==':')
|
||||
c = '_';
|
||||
}
|
||||
#ifndef _WIN32
|
||||
string filename = cache_path + "/jit/";
|
||||
#else
|
||||
string filename = cache_path + "\\jit\\";
|
||||
#endif
|
||||
filename += s;
|
||||
filename += "_op";
|
||||
filename += suffix;
|
||||
|
|
|
|||
|
|
@ -144,7 +144,7 @@ void GetitemOp::infer_slices(
|
|||
out_shape_j = (slice.stop - slice.start - 1) / slice.step + 1;
|
||||
else
|
||||
out_shape_j = (slice.start - slice.stop - 1) / -slice.step + 1;
|
||||
out_shape_j = std::max(0l, out_shape_j);
|
||||
out_shape_j = std::max((int64)0, out_shape_j);
|
||||
}
|
||||
out_shape.push_back(out_shape_j);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ struct ReshapeOp : Op {
|
|||
/**
|
||||
Returns a tensor with the same data and number of elements as input, but with the specified shape.
|
||||
|
||||
A single dimension may be -1, in which case it’s inferred from the remaining dimensions and the number of elements in input.
|
||||
A single dimension may be -1, in which case it's inferred from the remaining dimensions and the number of elements in input.
|
||||
|
||||
----------------
|
||||
|
||||
|
|
|
|||
|
|
@ -69,11 +69,20 @@ struct SimpleThread {
|
|||
}
|
||||
};
|
||||
|
||||
struct SimpleThreads;
|
||||
extern SimpleThreads threads;
|
||||
extern vector<void(*)()> cleanup_callback;
|
||||
|
||||
struct SimpleThreads {
|
||||
list<SimpleThread> threads;
|
||||
SimpleThreads(int n) {
|
||||
static void stop() {
|
||||
jittor::threads.threads.clear();
|
||||
}
|
||||
void create_threads(int n) {
|
||||
if (threads.size()) return;
|
||||
for (int i=0; i<n; i++)
|
||||
threads.emplace_back(i);
|
||||
cleanup_callback.push_back(&stop);
|
||||
}
|
||||
void wait_all() {
|
||||
for (auto& t : threads) {
|
||||
|
|
@ -105,7 +114,7 @@ struct SimpleThreads {
|
|||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
} threads;
|
||||
|
||||
static int last_compiled_op_num = 0;
|
||||
static int not_compile_window = 0;
|
||||
|
|
@ -190,7 +199,7 @@ void parallel_compile_all_ops(vector<int>& queue, vector<int>& range, FusedOp& f
|
|||
static volatile int has_error;
|
||||
static vector<vector<std::tuple<int,int,void*,string>>> op_entrys(thread_num);
|
||||
// <int,int,void*,string> represents: task id, is_fused_op, entry or context, new_jit_key
|
||||
static SimpleThreads threads(thread_num);
|
||||
threads.create_threads(thread_num);
|
||||
static std::mutex entry_lock;
|
||||
ai = 0;
|
||||
has_error = 0;
|
||||
|
|
|
|||
|
|
@ -8,7 +8,11 @@
|
|||
#include <algorithm>
|
||||
#include <sstream>
|
||||
#include <iomanip>
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <dlfcn.h>
|
||||
#endif
|
||||
#ifdef HAS_CUDA
|
||||
#include <cuda_runtime.h>
|
||||
#include "helper_cuda.h"
|
||||
|
|
@ -57,14 +61,25 @@ void Profiler::stop() {
|
|||
}
|
||||
|
||||
unique_ptr<MemoryChecker>* load_memory_checker(string name) {
|
||||
const char* msg = "";
|
||||
LOGvv << "Opening jit lib:" << name;
|
||||
#ifdef _WIN32
|
||||
void* handle = (void*)LoadLibrary(name.c_str());
|
||||
#else
|
||||
void* handle = dlopen(name.c_str(), RTLD_LAZY | RTLD_DEEPBIND | RTLD_LOCAL);
|
||||
CHECK(handle) << "Cannot open library" << name << ":" << dlerror();
|
||||
msg = dlerror();
|
||||
#endif
|
||||
|
||||
CHECK(handle) << "Cannot open library" << name << ":" << msg;
|
||||
|
||||
#ifdef _WIN32
|
||||
auto mm = (unique_ptr<MemoryChecker>*)GetProcAddress((HINSTANCE)handle, "memory_checker");
|
||||
#else
|
||||
//dlerror();
|
||||
auto mm = (unique_ptr<MemoryChecker>*)dlsym(handle, "memory_checker");
|
||||
const char* dlsym_error = dlerror();
|
||||
CHECK(!dlsym_error) << "Loading symbol memory_checker from" << name << "failed:" << dlsym_error;
|
||||
msg = dlerror();
|
||||
#endif
|
||||
CHECK(!msg) << "Loading symbol memory_checker from" << name << "failed:" << msg;
|
||||
|
||||
return mm;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,11 +7,12 @@
|
|||
// This file is subject to the terms and conditions defined in
|
||||
// file 'LICENSE.txt', which is part of this source code package.
|
||||
// ***************************************************************
|
||||
#define _XOPEN_SOURCE 700
|
||||
#include <fcntl.h> /* open */
|
||||
#include <stdint.h> /* uint64_t */
|
||||
#include <stdio.h> /* printf */
|
||||
#include <stdlib.h> /* size_t */
|
||||
#ifndef _WIN32
|
||||
#define _XOPEN_SOURCE 700
|
||||
#include <fcntl.h> /* open */
|
||||
#include <unistd.h> /* pread, sysconf */
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -76,4 +77,13 @@ int virt_to_phys_user(uintptr_t* paddr, uintptr_t vaddr)
|
|||
return 1;
|
||||
*paddr = (entry.pfn * sysconf(_SC_PAGE_SIZE)) + (vaddr % sysconf(_SC_PAGE_SIZE));
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
int virt_to_phys_user(uintptr_t* paddr, uintptr_t vaddr)
|
||||
{
|
||||
paddr[0] = vaddr;
|
||||
return 1;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@
|
|||
// This file is subject to the terms and conditions defined in
|
||||
// file 'LICENSE.txt', which is part of this source code package.
|
||||
// ***************************************************************
|
||||
#include <dlfcn.h>
|
||||
|
||||
#include "core.h"
|
||||
#include "grad.h"
|
||||
#include "pyjt/py_obj_holder.h"
|
||||
|
|
|
|||
|
|
@ -8,6 +8,34 @@
|
|||
|
||||
namespace jittor {
|
||||
|
||||
NanoString npy2ns[] = {
|
||||
ns_bool,
|
||||
ns_int8, ns_uint8,
|
||||
ns_int16, ns_uint16,
|
||||
ns_int32, ns_uint32,
|
||||
#ifdef _WIN32
|
||||
ns_int32, ns_uint32,
|
||||
#else
|
||||
ns_int64, ns_uint64,
|
||||
#endif
|
||||
ns_int64, ns_uint64,
|
||||
ns_float32, ns_float64, ns_float64,
|
||||
ns_void, ns_void, ns_void,
|
||||
ns_void
|
||||
};
|
||||
|
||||
NPY_TYPES ns2npy[] = {
|
||||
NPY_OBJECT, // placeholder for ns_void
|
||||
NPY_BOOL,
|
||||
#ifdef _WIN32
|
||||
NPY_BYTE, NPY_SHORT, NPY_LONG, NPY_LONGLONG,
|
||||
NPY_UBYTE, NPY_USHORT, NPY_ULONG, NPY_ULONGLONG,
|
||||
#else
|
||||
NPY_BYTE, NPY_SHORT, NPY_INT, NPY_LONGLONG,
|
||||
NPY_UBYTE, NPY_USHORT, NPY_UINT, NPY_ULONGLONG,
|
||||
#endif
|
||||
NPY_FLOAT, NPY_DOUBLE
|
||||
};
|
||||
|
||||
void** PyArray_API;
|
||||
PyTypeObject *PyArray_Type;
|
||||
|
|
|
|||
|
|
@ -38,53 +38,38 @@ struct PyArray_Proxy {
|
|||
int flags;
|
||||
};
|
||||
|
||||
enum NPY_TYPES {
|
||||
NPY_BOOL=0,
|
||||
NPY_BYTE, NPY_UBYTE,
|
||||
NPY_SHORT, NPY_USHORT,
|
||||
NPY_INT, NPY_UINT,
|
||||
NPY_LONG, NPY_ULONG,
|
||||
NPY_LONGLONG, NPY_ULONGLONG,
|
||||
NPY_FLOAT, NPY_DOUBLE, NPY_LONGDOUBLE,
|
||||
NPY_CFLOAT, NPY_CDOUBLE, NPY_CLONGDOUBLE,
|
||||
NPY_OBJECT=17,
|
||||
};
|
||||
|
||||
extern NanoString npy2ns[];
|
||||
extern NPY_TYPES ns2npy[];
|
||||
|
||||
#define NPY_ARRAY_C_CONTIGUOUS 0x0001
|
||||
#define NPY_ARRAY_ALIGNED 0x0100
|
||||
#define NPY_ARRAY_WRITEABLE 0x0400
|
||||
// NPY_ARRAY_C_CONTIGUOUS=1
|
||||
inline bool is_c_style(PyArray_Proxy* obj) { return obj->flags & 1; }
|
||||
inline NanoString get_type_str(PyArray_Proxy* obj) {
|
||||
auto type = obj->descr->type;
|
||||
// bool ?
|
||||
if (type=='?') return ns_bool;
|
||||
// int8 b
|
||||
if (type=='b') return ns_int8;
|
||||
// int16 h
|
||||
if (type=='h') return ns_int16;
|
||||
// int32 i
|
||||
if (type=='i') return ns_int32;
|
||||
// int64 l
|
||||
if (type=='l') return ns_int64;
|
||||
// uint8 B
|
||||
if (type=='B') return ns_uint8;
|
||||
// uint16 H
|
||||
if (type=='H') return ns_uint16;
|
||||
// uint32 I
|
||||
if (type=='I') return ns_uint32;
|
||||
// uint64 L
|
||||
if (type=='L') return ns_uint64;
|
||||
// float32 f
|
||||
if (type=='f') return ns_float32;
|
||||
// float64 d
|
||||
if (type=='d') return ns_float64;
|
||||
LOGf << "Unsupport numpy type char:'" << type << '\'';
|
||||
return ns_float64;
|
||||
NanoString type = ns_void;
|
||||
if (obj->descr->type_num < NPY_OBJECT)
|
||||
type = npy2ns[obj->descr->type_num];
|
||||
CHECK(type != ns_void) << "Numpy type not support, type_num:"
|
||||
<< obj->descr->type_num
|
||||
<< "type_char:" << obj->descr->type;
|
||||
return type;
|
||||
}
|
||||
|
||||
inline int get_typenum(NanoString ns) {
|
||||
if (ns == ns_bool) return 0;
|
||||
if (ns == ns_int8) return 1;
|
||||
if (ns == ns_uint8) return 2;
|
||||
if (ns == ns_int16) return 3;
|
||||
if (ns == ns_uint16) return 4;
|
||||
if (ns == ns_int32) return 5;
|
||||
if (ns == ns_uint32) return 6;
|
||||
if (ns == ns_int64) return 7;
|
||||
if (ns == ns_uint64) return 8;
|
||||
if (ns == ns_float32) return 11;
|
||||
if (ns == ns_float64) return 12;
|
||||
LOGf << ns;
|
||||
return -1;
|
||||
return ns2npy[ns.index()];
|
||||
}
|
||||
|
||||
typedef Py_intptr_t npy_intp;
|
||||
|
|
|
|||
|
|
@ -6,7 +6,11 @@
|
|||
// ***************************************************************
|
||||
#pragma once
|
||||
#include <Python.h>
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <dlfcn.h>
|
||||
#endif
|
||||
#include <vector>
|
||||
#include <list>
|
||||
#include <map>
|
||||
|
|
|
|||
|
|
@ -6,8 +6,12 @@
|
|||
// ***************************************************************
|
||||
#include <fstream>
|
||||
#include <streambuf>
|
||||
#ifdef _WIN32
|
||||
#include <filesystem>
|
||||
#endif
|
||||
#include "misc/hash.h"
|
||||
#include "utils/cache_compile.h"
|
||||
#include "utils/str_utils.h"
|
||||
|
||||
namespace jittor {
|
||||
namespace jit_compiler {
|
||||
|
|
@ -15,7 +19,7 @@ namespace jit_compiler {
|
|||
#ifndef TEST
|
||||
string read_all(const string& fname) {
|
||||
std::ifstream ifs(fname);
|
||||
if (ifs)
|
||||
if (ifs && ifs.good())
|
||||
return string((std::istreambuf_iterator<char>(ifs)),
|
||||
(std::istreambuf_iterator<char>()));
|
||||
return "";
|
||||
|
|
@ -65,7 +69,7 @@ void find_names(string cmd, vector<string>& input_names, string& output_name, ma
|
|||
auto substr = [&](size_t i, size_t j) -> string {
|
||||
string s;
|
||||
for (size_t k=i; k<j; k++)
|
||||
if (cmd[k]!='\'') s += cmd[k];
|
||||
if (cmd[k]!='\'' && cmd[k]!='"') s += cmd[k];
|
||||
return s;
|
||||
};
|
||||
while (i<cmd.size()) {
|
||||
|
|
@ -164,6 +168,29 @@ void process(string src, vector<string>& input_names) {
|
|||
}
|
||||
}
|
||||
|
||||
static inline void check_win_file(const string& name) {
|
||||
#ifdef _WIN32
|
||||
// win32 not allowed so file change when load
|
||||
// but we can rename it
|
||||
if (!file_exist(name)) return;
|
||||
if (!(endswith(name, ".pyd") || endswith(name, ".dll")))
|
||||
return;
|
||||
string new_name = name+".bk";
|
||||
LOGv << "move file" << name << "-> " << new_name;
|
||||
if (file_exist(new_name))
|
||||
std::filesystem::remove(new_name);
|
||||
std::filesystem::rename(name, new_name);
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline bool is_full_path(const string& name) {
|
||||
#ifdef _WIN32
|
||||
return name.size()>=2 && name[1]==':';
|
||||
#else
|
||||
return name.size() && name[0]=='/';
|
||||
#endif
|
||||
}
|
||||
|
||||
bool cache_compile(const string& cmd, const string& cache_path, const string& jittor_path) {
|
||||
vector<string> input_names;
|
||||
map<string,vector<string>> extra;
|
||||
|
|
@ -172,10 +199,12 @@ bool cache_compile(const string& cmd, const string& cache_path, const string& ji
|
|||
string output_cache_key;
|
||||
bool ran = false;
|
||||
output_cache_key = read_all(output_name+".key");
|
||||
string cd_cmd = cache_path.size() ? "cd " + cache_path + " && " + cmd : cmd;
|
||||
// string cd_cmd = cache_path.size() ? "cd " + cache_path + " && " + cmd : cmd;
|
||||
string cd_cmd = cmd;
|
||||
if (output_cache_key.size() == 0) {
|
||||
LOGvv << "Cache key of" << output_name << "not found.";
|
||||
LOGvvv << "Run cmd:" << cmd;
|
||||
check_win_file(output_name);
|
||||
system_with_check(cd_cmd.c_str());
|
||||
ran = true;
|
||||
}
|
||||
|
|
@ -197,7 +226,7 @@ bool cache_compile(const string& cmd, const string& cache_path, const string& ji
|
|||
string full_name;
|
||||
if (name.substr(0, 4) == "jit/" || name.substr(0, 4) == "gen/")
|
||||
full_name = join(cache_path, name);
|
||||
else if (name.size() && name[0]=='/')
|
||||
else if (is_full_path(name))
|
||||
full_name = name;
|
||||
else
|
||||
full_name = join(src_path, name);
|
||||
|
|
@ -225,6 +254,7 @@ bool cache_compile(const string& cmd, const string& cache_path, const string& ji
|
|||
if (output_cache_key.size() != 0 && output_cache_key != cache_key) {
|
||||
LOGvv << "Cache key of" << output_name << "changed.";
|
||||
LOGvvv << "Run cmd:" << cmd;
|
||||
check_win_file(output_name);
|
||||
system_with_check(cd_cmd.c_str());
|
||||
ran = true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,9 @@
|
|||
#endif
|
||||
#ifdef __GNUC__
|
||||
#endif
|
||||
#ifndef _WIN32
|
||||
#include <sys/prctl.h>
|
||||
#endif
|
||||
#include <signal.h>
|
||||
#include <iterator>
|
||||
#include <algorithm>
|
||||
|
|
@ -21,7 +23,9 @@
|
|||
namespace jittor {
|
||||
|
||||
void init_subprocess() {
|
||||
#ifndef _WIN32
|
||||
prctl(PR_SET_PDEATHSIG, SIGKILL);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void __log(
|
||||
|
|
@ -169,14 +173,14 @@ public:
|
|||
}
|
||||
};
|
||||
|
||||
static void ostream_redirect(bool stdout, bool stderr) {
|
||||
if (stdout) {
|
||||
static void ostream_redirect(bool _stdout, bool _stderr) {
|
||||
if (_stdout) {
|
||||
PyObjHolder a(PyImport_ImportModule("sys"));
|
||||
PyObjHolder b(PyObject_GetAttrString(a.obj,"stdout"));
|
||||
auto buf = new pythonbuf(b.obj);
|
||||
std::cout.rdbuf(buf);
|
||||
}
|
||||
if (stderr) {
|
||||
if (_stderr) {
|
||||
PyObjHolder a(PyImport_ImportModule("sys"));
|
||||
PyObjHolder b(PyObject_GetAttrString(a.obj,"stderr"));
|
||||
auto buf = new pythonbuf(b.obj);
|
||||
|
|
|
|||
|
|
@ -11,6 +11,10 @@
|
|||
#include <thread>
|
||||
#include <unordered_map>
|
||||
#include <unistd.h>
|
||||
#ifdef _WIN32
|
||||
#include <wchar.h>
|
||||
#include <windows.h>
|
||||
#endif
|
||||
#include "utils/log.h"
|
||||
#include "utils/mwsr_list.h"
|
||||
#include "utils/str_utils.h"
|
||||
|
|
@ -38,11 +42,19 @@ uint32_t get_tid() {
|
|||
}
|
||||
|
||||
static bool supports_color() {
|
||||
bool term_supports_color = false;
|
||||
#ifdef _WIN32
|
||||
// TODO: windows color not supported yet.
|
||||
term_supports_color = false;
|
||||
#else
|
||||
HANDLE hOut = GetStdHandle(STD_OUTPUT_HANDLE);
|
||||
if (hOut == INVALID_HANDLE_VALUE) return 0;
|
||||
|
||||
DWORD dwMode = 0;
|
||||
if (!GetConsoleMode(hOut, &dwMode)) return 0;
|
||||
|
||||
dwMode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
|
||||
if (!SetConsoleMode(hOut, dwMode)) return 0;
|
||||
return 1;
|
||||
|
||||
#endif
|
||||
bool term_supports_color = false;
|
||||
const char* const term = getenv("TERM");
|
||||
if (term != NULL && term[0] != '\0') {
|
||||
term_supports_color =
|
||||
|
|
@ -57,7 +69,6 @@ static bool supports_color() {
|
|||
!strcmp(term, "linux") ||
|
||||
!strcmp(term, "cygwin");
|
||||
}
|
||||
#endif
|
||||
return term_supports_color;
|
||||
}
|
||||
bool g_supports_color = supports_color();
|
||||
|
|
@ -66,7 +77,8 @@ struct timeval start_tv;
|
|||
|
||||
struct tm get_start_tm() {
|
||||
gettimeofday (&start_tv, NULL);
|
||||
return *localtime(&start_tv.tv_sec);
|
||||
time_t t = start_tv.tv_sec;
|
||||
return *localtime(&t);
|
||||
}
|
||||
|
||||
struct tm start_tm = get_start_tm();
|
||||
|
|
@ -107,7 +119,9 @@ void print_prefix(std::ostream* out) {
|
|||
*out << ' ';
|
||||
}
|
||||
|
||||
#ifdef LOG_ASYNC
|
||||
MWSR_LIST(log, std::ostringstream);
|
||||
#endif
|
||||
DECLARE_FLAG(int, log_sync);
|
||||
|
||||
std::mutex sync_log_m;
|
||||
|
|
@ -157,7 +171,9 @@ void send_log(std::ostringstream&& out) {
|
|||
log_capture(out.str());
|
||||
if (log_silent) return;
|
||||
if (!log_sync) {
|
||||
#if LOG_ASYNC
|
||||
mwsr_list_log::push(move(out));
|
||||
#endif
|
||||
} else {
|
||||
std::lock_guard<std::mutex> lk(sync_log_m);
|
||||
// std::cerr << "[SYNC]";
|
||||
|
|
@ -168,7 +184,9 @@ void send_log(std::ostringstream&& out) {
|
|||
|
||||
void flush_log() {
|
||||
if (!log_sync) {
|
||||
#if LOG_ASYNC
|
||||
mwsr_list_log::flush();
|
||||
#endif
|
||||
} else {
|
||||
std::cerr.flush();
|
||||
}
|
||||
|
|
@ -187,7 +205,15 @@ size_t thread_local protected_page = 0;
|
|||
int segfault_happen = 0;
|
||||
string thread_local thread_name;
|
||||
static int _pid = getpid();
|
||||
vector<void(*)()> cleanup_callback;
|
||||
|
||||
#ifdef _WIN32
|
||||
void handle_signal(int signal) {
|
||||
std::cerr << "Caught SIGNAL " << signal << ", quick exit";
|
||||
std::cerr.flush();
|
||||
abort();
|
||||
}
|
||||
#else
|
||||
void segfault_sigaction(int signal, siginfo_t *si, void *arg) {
|
||||
if (signal == SIGINT) {
|
||||
if (_pid == getpid()) {
|
||||
|
|
@ -218,9 +244,16 @@ void segfault_sigaction(int signal, siginfo_t *si, void *arg) {
|
|||
segfault_happen = 1;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
int register_sigaction() {
|
||||
#ifdef _WIN32
|
||||
signal(SIGINT, handle_signal);
|
||||
signal(SIGTERM, handle_signal);
|
||||
// signal(SIGABRT, handle_signal);
|
||||
signal(SIGSEGV, handle_signal);
|
||||
signal(SIGFPE, handle_signal);
|
||||
#else
|
||||
struct sigaction sa;
|
||||
|
||||
memset(&sa, 0, sizeof(struct sigaction));
|
||||
|
|
@ -237,12 +270,20 @@ int register_sigaction() {
|
|||
sigaction(SIGBUS, &sa, NULL);
|
||||
sigaction(SIGQUIT, &sa, NULL);
|
||||
// sigaction(SIGABRT, &sa, NULL);
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
void log_main() {
|
||||
static int log_init() {
|
||||
register_sigaction();
|
||||
std::atexit(log_exiting);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int _log_init = log_init();
|
||||
|
||||
void log_main() {
|
||||
#ifdef LOG_ASYNC
|
||||
mwsr_list_log::reduce([&](const std::ostringstream& out) {
|
||||
#ifdef TEST_LOG
|
||||
string s = out.str();
|
||||
|
|
@ -253,6 +294,7 @@ void log_main() {
|
|||
}, [&]() {
|
||||
std::cerr.flush();
|
||||
});
|
||||
#endif
|
||||
}
|
||||
|
||||
unordered_map<uint64_t, int> vprefix_map;
|
||||
|
|
@ -260,9 +302,9 @@ void stream_hash(uint64_t& hash, char c) {
|
|||
hash = hash * 257 + (uint8_t)c;
|
||||
}
|
||||
|
||||
DEFINE_FLAG(int, log_sync, 1, "Set log printed synchronously.");
|
||||
DEFINE_FLAG(int, log_silent, 0, "The log will be completely silent.");
|
||||
DEFINE_FLAG(int, log_v, 0, "Verbose level of logging");
|
||||
DEFINE_FLAG(int, log_sync, 1, "Set log printed synchronously.");
|
||||
DEFINE_FLAG_WITH_SETTER(string, log_vprefix, "",
|
||||
"Verbose level of logging prefix\n"
|
||||
"example: log_vprefix='op=1,node=2,executor.cc:38$=1000'");
|
||||
|
|
@ -322,6 +364,90 @@ but you can hot fix it by this command:
|
|||
)";
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
int system_popen(const char *cmd) {
|
||||
HANDLE g_hChildStd_OUT_Rd = NULL;
|
||||
HANDLE g_hChildStd_OUT_Wr = NULL;
|
||||
SECURITY_ATTRIBUTES saAttr;
|
||||
// Set the bInheritHandle flag so pipe handles are inherited.
|
||||
|
||||
saAttr.nLength = sizeof(SECURITY_ATTRIBUTES);
|
||||
saAttr.bInheritHandle = TRUE;
|
||||
saAttr.lpSecurityDescriptor = NULL;
|
||||
|
||||
// Create a pipe for the child process's STDOUT.
|
||||
if (!CreatePipe(&g_hChildStd_OUT_Rd, &g_hChildStd_OUT_Wr, &saAttr, 0))
|
||||
LOGf << "StdoutRd CreatePipe error";
|
||||
// Ensure the read handle to the pipe for STDOUT is not inherited.
|
||||
if (!SetHandleInformation(g_hChildStd_OUT_Rd, HANDLE_FLAG_INHERIT, 0))
|
||||
LOGf << "Stdout SetHandleInformation error";
|
||||
|
||||
// Create the child process.
|
||||
PROCESS_INFORMATION piProcInfo;
|
||||
STARTUPINFO siStartInfo;
|
||||
BOOL bSuccess = FALSE;
|
||||
// Set up members of the PROCESS_INFORMATION structure.
|
||||
ZeroMemory(&piProcInfo, sizeof(PROCESS_INFORMATION));
|
||||
|
||||
// Set up members of the STARTUPINFO structure.
|
||||
// This structure specifies the STDIN and STDOUT handles for redirection.
|
||||
ZeroMemory(&siStartInfo, sizeof(STARTUPINFO));
|
||||
siStartInfo.cb = sizeof(STARTUPINFO);
|
||||
siStartInfo.hStdError = g_hChildStd_OUT_Wr;
|
||||
siStartInfo.hStdOutput = g_hChildStd_OUT_Wr;
|
||||
siStartInfo.dwFlags |= STARTF_USESTDHANDLES;
|
||||
|
||||
// Create the child process.
|
||||
bSuccess = CreateProcess(NULL,
|
||||
(char *)cmd, // command line
|
||||
NULL, // process security attributes
|
||||
NULL, // primary thread security attributes
|
||||
TRUE, // handles are inherited
|
||||
0, // creation flags
|
||||
NULL, // use parent's environment
|
||||
NULL, // use parent's current directory
|
||||
&siStartInfo, // STARTUPINFO pointer
|
||||
&piProcInfo); // receives PROCESS_INFORMATION
|
||||
|
||||
// If an error occurs, exit the application.
|
||||
if (!bSuccess)
|
||||
LOGf << "CreateProcess error";
|
||||
// Close handles to the stdin and stdout pipes no longer needed by the child process.
|
||||
// If they are not explicitly closed, there is no way to recognize that the child process has ended.
|
||||
CloseHandle(g_hChildStd_OUT_Wr);
|
||||
|
||||
DWORD dwRead, dwWritten;
|
||||
CHAR chBuf[BUFSIZ];
|
||||
HANDLE hParentStdOut = GetStdHandle(STD_OUTPUT_HANDLE);
|
||||
|
||||
|
||||
string output;
|
||||
for (;;)
|
||||
{
|
||||
bSuccess = ReadFile(g_hChildStd_OUT_Rd, chBuf, BUFSIZ, &dwRead, NULL);
|
||||
if (!bSuccess || dwRead == 0)
|
||||
break;
|
||||
output += chBuf;
|
||||
bSuccess = WriteFile(hParentStdOut, chBuf,
|
||||
dwRead, &dwWritten, NULL);
|
||||
if (!bSuccess)
|
||||
break;
|
||||
}
|
||||
WaitForSingleObject(piProcInfo.hProcess, INFINITE);
|
||||
DWORD ec;
|
||||
GetExitCodeProcess(piProcInfo.hProcess, &ec);
|
||||
// Close handles to the child process and its primary thread.
|
||||
// Some applications might keep these handles to monitor the status
|
||||
// of the child process, for example.
|
||||
CloseHandle(piProcInfo.hProcess);
|
||||
CloseHandle(piProcInfo.hThread);
|
||||
|
||||
if (ec) {
|
||||
check_cuda_unsupport_version(output);
|
||||
}
|
||||
return ec;
|
||||
}
|
||||
#else
|
||||
int system_popen(const char* cmd) {
|
||||
char buf[BUFSIZ];
|
||||
string cmd2;
|
||||
|
|
@ -345,6 +471,7 @@ int system_popen(const char* cmd) {
|
|||
}
|
||||
return ret;
|
||||
}
|
||||
#endif
|
||||
|
||||
void system_with_check(const char* cmd) {
|
||||
auto ret = system_popen(cmd);
|
||||
|
|
@ -354,18 +481,34 @@ void system_with_check(const char* cmd) {
|
|||
CHECKop(ret,==,0) << "Run cmd failed:" << cmd;
|
||||
}
|
||||
|
||||
#ifdef LOG_ASYNC
|
||||
std::thread log_thread(log_main);
|
||||
#endif
|
||||
|
||||
void log_exiting() {
|
||||
if (exited) return;
|
||||
exited = true;
|
||||
for (auto cb : cleanup_callback)
|
||||
cb();
|
||||
cleanup_callback.clear();
|
||||
#ifdef LOG_ASYNC
|
||||
mwsr_list_log::stop();
|
||||
log_thread.join();
|
||||
#endif
|
||||
}
|
||||
|
||||
} // jittor
|
||||
|
||||
|
||||
void expect_error(std::function<void()> func) {
|
||||
try {
|
||||
func();
|
||||
} catch (...) {
|
||||
return;
|
||||
}
|
||||
LOGf << "Missing error";
|
||||
}
|
||||
|
||||
#ifdef TEST_LOG
|
||||
|
||||
#include <chrono>
|
||||
|
|
|
|||
|
|
@ -38,6 +38,29 @@ extern "C" uint32_t get_tid();
|
|||
extern "C" bool g_supports_color;
|
||||
extern "C" void print_prefix(std::ostream* out);
|
||||
|
||||
#ifdef _WIN32
|
||||
constexpr char green[] = "\x1b[1;32m";
|
||||
constexpr char red[] = "\x1b[1;31m";
|
||||
constexpr char yellow[] = "\x1b[1;33m";
|
||||
|
||||
|
||||
static void get_color(char level, int verbose, const char*& color_begin, const char*& color_end) {
|
||||
if (level == 'i') {
|
||||
if (verbose == 0) color_begin = "\x1b[1;32m"; else
|
||||
if (verbose < 10) color_begin = "\x1b[1;32m"; else
|
||||
if (verbose < 100) color_begin = "\x1b[1;32m"; else
|
||||
if (verbose < 1000) color_begin = "\x1b[1;32m";
|
||||
else color_begin = "\x1b[1;32m";
|
||||
} else if (level == 'w')
|
||||
color_begin = yellow;
|
||||
else if (level == 'e')
|
||||
color_begin = red;
|
||||
else // level == 'f'
|
||||
color_begin = red;
|
||||
color_end = "\x1b[m";
|
||||
}
|
||||
|
||||
#else
|
||||
constexpr char green[] = "\033[38;5;2m";
|
||||
constexpr char red[] = "\033[38;5;1m";
|
||||
constexpr char yellow[] = "\033[38;5;3m";
|
||||
|
|
@ -58,6 +81,8 @@ static void get_color(char level, int verbose, const char*& color_begin, const c
|
|||
color_end = "\033[m";
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
extern "C" void send_log(std::ostringstream&& out);
|
||||
extern "C" void flush_log();
|
||||
extern "C" void log_capture_start();
|
||||
|
|
|
|||
|
|
@ -6,10 +6,14 @@
|
|||
// ***************************************************************
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
#ifndef _WIN32
|
||||
#include <sys/prctl.h>
|
||||
#include <execinfo.h>
|
||||
#include <sys/wait.h>
|
||||
#else
|
||||
#include <windows.h>
|
||||
#endif
|
||||
#include <unistd.h>
|
||||
#include <iostream>
|
||||
#include "utils/tracer.h"
|
||||
|
||||
|
|
@ -24,6 +28,24 @@ DEFINE_FLAG_WITH_SETTER(int, gdb_attach, 0, "gdb attach self process.");
|
|||
|
||||
string _extra_gdb_cmd;
|
||||
|
||||
int system_popen(const char* cmd);
|
||||
|
||||
#ifdef _WIN32
|
||||
string get_cmds(const vector<const char*>& argv) {
|
||||
auto cmds = gdb_path;
|
||||
for (auto p : argv) {
|
||||
if (!p) continue;
|
||||
string cmd = p;
|
||||
cmds += " ";
|
||||
if (cmd.find(' ') != string::npos && cmd[0] != '"')
|
||||
cmds += '"' + cmd + '"';
|
||||
else
|
||||
cmds += cmd;
|
||||
}
|
||||
return cmds;
|
||||
}
|
||||
#endif
|
||||
|
||||
void setter_gdb_attach(int v) {
|
||||
if (v && gdb_path.size()) {
|
||||
static int gdb_attached = 0;
|
||||
|
|
@ -32,30 +54,71 @@ void setter_gdb_attach(int v) {
|
|||
// using gdb to print the stack trace
|
||||
char pid_buf[30];
|
||||
sprintf(pid_buf, "%d", getpid());
|
||||
char name_buf[512];
|
||||
name_buf[readlink("/proc/self/exe", name_buf, 511)]=0;
|
||||
int child_pid = fork();
|
||||
if (!child_pid) {
|
||||
LOGi << "gdb attach for" << name_buf << "pid=" >> pid_buf;
|
||||
|
||||
vector<const char*> argv{
|
||||
gdb_path.c_str(),
|
||||
"-ex", "catch throw"
|
||||
};
|
||||
if (auto n = extra_gdb_cmd.size()) {
|
||||
_extra_gdb_cmd = extra_gdb_cmd;
|
||||
_extra_gdb_cmd += '\0';
|
||||
argv.push_back("-ex");
|
||||
argv.push_back(&_extra_gdb_cmd[0]);
|
||||
for (uint i=0; i<n; i++) {
|
||||
if (_extra_gdb_cmd[i]==';') {
|
||||
argv.push_back("-ex");
|
||||
_extra_gdb_cmd[i] = '\0';
|
||||
argv.push_back(&_extra_gdb_cmd[i+1]);
|
||||
}
|
||||
vector<const char*> argv{
|
||||
gdb_path.c_str(),
|
||||
"-ex", "catch throw"
|
||||
};
|
||||
if (auto n = extra_gdb_cmd.size()) {
|
||||
_extra_gdb_cmd = extra_gdb_cmd;
|
||||
_extra_gdb_cmd += '\0';
|
||||
argv.push_back("-ex");
|
||||
argv.push_back(&_extra_gdb_cmd[0]);
|
||||
for (uint i=0; i<n; i++) {
|
||||
if (_extra_gdb_cmd[i]==';') {
|
||||
argv.push_back("-ex");
|
||||
_extra_gdb_cmd[i] = '\0';
|
||||
argv.push_back(&_extra_gdb_cmd[i+1]);
|
||||
}
|
||||
}
|
||||
argv.insert(argv.end(), {name_buf, pid_buf, NULL});
|
||||
}
|
||||
// argv.insert(argv.end(), {name_buf, pid_buf, NULL});
|
||||
argv.insert(argv.end(), {"-p", pid_buf, NULL});
|
||||
LOGi << "gdb attach for" << "pid=" >> pid_buf << argv;
|
||||
|
||||
#ifdef _WIN32
|
||||
// _spawnvp(_P_OVERLAY, gdb_path.c_str(), (char* const*)&argv[0]);
|
||||
// system_popen((gdb_path+" -p "+pid_buf).c_str());
|
||||
auto cmds = get_cmds(argv);
|
||||
|
||||
// Create the child process.
|
||||
PROCESS_INFORMATION piProcInfo;
|
||||
STARTUPINFO siStartInfo;
|
||||
BOOL bSuccess = false;
|
||||
// Set up members of the PROCESS_INFORMATION structure.
|
||||
ZeroMemory(&piProcInfo, sizeof(PROCESS_INFORMATION));
|
||||
|
||||
// Set up members of the STARTUPINFO structure.
|
||||
// This structure specifies the STDIN and STDOUT handles for redirection.
|
||||
ZeroMemory(&siStartInfo, sizeof(STARTUPINFO));
|
||||
siStartInfo.cb = sizeof(STARTUPINFO);
|
||||
// siStartInfo.hStdError = g_hChildStd_OUT_Wr;
|
||||
// siStartInfo.hStdOutput = g_hChildStd_OUT_Wr;
|
||||
siStartInfo.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
|
||||
siStartInfo.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE);
|
||||
siStartInfo.hStdError = GetStdHandle(STD_ERROR_HANDLE);
|
||||
siStartInfo.dwFlags |= STARTF_USESTDHANDLES;
|
||||
// Create the child process.
|
||||
bSuccess = CreateProcess(
|
||||
NULL,
|
||||
(char *)&cmds[0], // command line
|
||||
NULL, // process security attributes
|
||||
NULL, // primary thread security attributes
|
||||
true, // handles are inherited
|
||||
0, // creation flags
|
||||
NULL, // use parent's environment
|
||||
NULL, // use parent's current directory
|
||||
&siStartInfo, // STARTUPINFO pointer
|
||||
&piProcInfo); // receives PROCESS_INFORMATION
|
||||
|
||||
// If an error occurs, exit the application.
|
||||
if (!bSuccess)
|
||||
LOGf << "CreateProcess error, command:" << cmds;
|
||||
// sleep 5s, wait gdb attach
|
||||
sleep(5);
|
||||
#else
|
||||
int child_pid = fork();
|
||||
if (!child_pid) {
|
||||
auto ret = execvp(gdb_path.c_str(), (char* const*)&argv[0]);
|
||||
LOGf << "execvp failed return" << ret << gdb_path << extra_gdb_cmd;
|
||||
exit(1);
|
||||
|
|
@ -65,6 +128,7 @@ void setter_gdb_attach(int v) {
|
|||
// sleep 5s, wait gdb attach
|
||||
sleep(5);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -86,34 +150,34 @@ void print_trace() {
|
|||
sprintf(pid_buf, "%d", getpid());
|
||||
char st_buf[30];
|
||||
sprintf(st_buf, "set backtrace limit %d", trace_depth);
|
||||
char name_buf[512];
|
||||
name_buf[readlink("/proc/self/exe", name_buf, 511)]=0;
|
||||
int child_pid = fork();
|
||||
if (!child_pid) {
|
||||
std::cerr << "stack trace for " << name_buf << " pid=" << pid_buf << std::endl;
|
||||
|
||||
vector<const char*> argv{
|
||||
gdb_path.c_str(), "--batch", "-n",
|
||||
"-ex", "thread",
|
||||
"-ex", st_buf, // "set backtrace limit 10",
|
||||
"-ex", "bt",
|
||||
};
|
||||
if (has_pybt)
|
||||
argv.insert(argv.end(), {"-ex", "set backtrace limit 0", "-ex", "py-bt"});
|
||||
if (auto n = extra_gdb_cmd.size()) {
|
||||
_extra_gdb_cmd = extra_gdb_cmd;
|
||||
_extra_gdb_cmd += '\0';
|
||||
argv.push_back("-ex");
|
||||
argv.push_back(&_extra_gdb_cmd[0]);
|
||||
for (uint i=0; i<n; i++) {
|
||||
if (_extra_gdb_cmd[i]==';') {
|
||||
argv.push_back("-ex");
|
||||
_extra_gdb_cmd[i] = '\0';
|
||||
argv.push_back(&_extra_gdb_cmd[i+1]);
|
||||
}
|
||||
LOGi << "stack trace for pid=" << pid_buf;
|
||||
|
||||
vector<const char*> argv{
|
||||
gdb_path.c_str(), "--batch", "-n",
|
||||
"-ex", "thread",
|
||||
"-ex", st_buf, // "set backtrace limit 10",
|
||||
"-ex", "bt",
|
||||
};
|
||||
if (has_pybt)
|
||||
argv.insert(argv.end(), {"-ex", "set backtrace limit 0", "-ex", "py-bt"});
|
||||
if (auto n = extra_gdb_cmd.size()) {
|
||||
_extra_gdb_cmd = extra_gdb_cmd;
|
||||
_extra_gdb_cmd += '\0';
|
||||
argv.push_back("-ex");
|
||||
argv.push_back(&_extra_gdb_cmd[0]);
|
||||
for (uint i=0; i<n; i++) {
|
||||
if (_extra_gdb_cmd[i]==';') {
|
||||
argv.push_back("-ex");
|
||||
_extra_gdb_cmd[i] = '\0';
|
||||
argv.push_back(&_extra_gdb_cmd[i+1]);
|
||||
}
|
||||
}
|
||||
argv.insert(argv.end(), {name_buf, pid_buf, NULL});
|
||||
}
|
||||
argv.insert(argv.end(), {"-p", pid_buf, NULL});
|
||||
#ifndef _WIN32
|
||||
int child_pid = fork();
|
||||
if (!child_pid) {
|
||||
execvp(gdb_path.c_str(), (char* const*)&argv[0]);
|
||||
exit(0);
|
||||
} else {
|
||||
|
|
@ -121,7 +185,14 @@ void print_trace() {
|
|||
prctl(PR_SET_PTRACER, child_pid, 0, 0, 0);
|
||||
waitpid(child_pid,NULL,0);
|
||||
}
|
||||
} else {
|
||||
#else
|
||||
auto cmds = get_cmds(argv);
|
||||
LOGv << cmds;
|
||||
system_popen(cmds.c_str());
|
||||
#endif
|
||||
}
|
||||
#ifndef _WIN32
|
||||
else {
|
||||
void *trace[16];
|
||||
char **messages = (char **)NULL;
|
||||
int i, trace_size = 0;
|
||||
|
|
@ -149,6 +220,7 @@ void print_trace() {
|
|||
(void)ret;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
} // jittor
|
||||
} // jittor
|
||||
|
|
|
|||
|
|
@ -11,18 +11,7 @@ using namespace std;
|
|||
|
||||
void test_main();
|
||||
|
||||
void on_error() {
|
||||
throw std::exception();
|
||||
}
|
||||
|
||||
void expect_error(function<void()> func) {
|
||||
try {
|
||||
func();
|
||||
} catch (...) {
|
||||
return;
|
||||
}
|
||||
CHECK(0) << "Missing error";
|
||||
}
|
||||
void expect_error(function<void()> func);
|
||||
|
||||
int main() {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -175,5 +175,24 @@ class TestArray(unittest.TestCase):
|
|||
assert b.dtype == "float64"
|
||||
|
||||
|
||||
def test_all_type(self):
|
||||
with jt.flag_scope(auto_convert_64_to_32=0):
|
||||
types = [
|
||||
"bool",
|
||||
"int8", "uint8",
|
||||
"int16", "uint16",
|
||||
"int32", "uint32",
|
||||
"int64", "uint64",
|
||||
"float32", "float64",
|
||||
]
|
||||
for t in types:
|
||||
a = np.random.random(1000).astype(t)
|
||||
b = jt.array(a)
|
||||
assert str(b.dtype) == t
|
||||
c = b.numpy()
|
||||
assert str(c.dtype) == t
|
||||
np.testing.assert_allclose(a, c)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -20,7 +20,7 @@ import random
|
|||
pass_this_test = False
|
||||
msg = ""
|
||||
mid = 0
|
||||
if os.uname()[1] == "jittor-ce":
|
||||
if hasattr(os, "uname") and os.uname()[1] == "jittor-ce":
|
||||
mid = 1
|
||||
try:
|
||||
traindir = ["/data1/cjld/imagenet/train/","/home/cjld/imagenet/train/"][mid]
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import random
|
|||
pass_this_test = False
|
||||
msg = ""
|
||||
mid = 0
|
||||
if os.uname()[1] == "jittor-ce":
|
||||
if hasattr(os, "uname") and os.uname()[1] == "jittor-ce":
|
||||
mid = 1
|
||||
try:
|
||||
# check can we run this test
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from .test_core import expect_error
|
|||
import os
|
||||
|
||||
mid = 0
|
||||
if "jittor" in os.uname()[1]:
|
||||
if hasattr(os, "uname") and "jittor" in os.uname()[1]:
|
||||
mid = 1
|
||||
|
||||
class TestNanoString(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ except:
|
|||
skip_this_test = True
|
||||
|
||||
mid = 0
|
||||
if "jittor" in os.uname()[1]:
|
||||
if hasattr(os, "uname") and "jittor" in os.uname()[1]:
|
||||
mid = 1
|
||||
|
||||
def resize_and_crop(x, bbox, interpolation="nearest", out_size=[224,224]):
|
||||
|
|
|
|||
|
|
@ -6,15 +6,30 @@
|
|||
# ***************************************************************
|
||||
import unittest
|
||||
import jittor as jt
|
||||
import os
|
||||
import subprocess as sp
|
||||
import sys
|
||||
|
||||
class TestTracer(unittest.TestCase):
|
||||
def test_print_trace(self):
|
||||
jt.print_trace()
|
||||
|
||||
# force use addr2line
|
||||
jt.flags.gdb_path = ""
|
||||
with jt.flag_scope(gdb_path=""):
|
||||
jt.print_trace()
|
||||
if os.name != 'nt':
|
||||
# force use addr2line
|
||||
with jt.flag_scope(gdb_path=""):
|
||||
jt.print_trace()
|
||||
|
||||
def test_breakpoint(self):
|
||||
fname = os.path.join(jt.flags.cache_path, "test_breakpoint.py")
|
||||
with open(fname, 'w') as f:
|
||||
f.write("""
|
||||
import jittor as jt
|
||||
with jt.flag_scope(extra_gdb_cmd="c;q"):
|
||||
jt.flags.gdb_attach = 1
|
||||
""")
|
||||
out = sp.getoutput(sys.executable+' '+fname)
|
||||
print(out)
|
||||
assert "Attaching to" in out
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ def init(cc_path,s_path):
|
|||
|
||||
global file_idx
|
||||
file_idx=0
|
||||
file_name=cc_path.split("/")[-1]
|
||||
file_name=os.path.basename(cc_path)
|
||||
|
||||
idx=0
|
||||
for line in s_content:
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ def check(dirname, fname):
|
|||
src = src.replace(a, b)
|
||||
found = True
|
||||
if ac not in src and not found: return
|
||||
n = len(dirname.split('/'))-1
|
||||
n = len(dirname.split(os.path.sep))-1
|
||||
s = '.' + '/..' * n
|
||||
new_src = ""
|
||||
i = -1
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ for os_name in ['ubuntu', 'centos']:
|
|||
|
||||
obj_files = []
|
||||
for name in data_files:
|
||||
name = name.split("/")[-1]
|
||||
name = os.path.basename(name)
|
||||
fname = f"{obj_path}/{name}.o"
|
||||
assert os.path.isfile(fname), fname
|
||||
obj_files.append(fname)
|
||||
|
|
|
|||
|
|
@ -98,10 +98,12 @@ def simple_timer(name):
|
|||
|
||||
@contextlib.contextmanager
|
||||
def import_scope(flags):
|
||||
prev = sys.getdlopenflags()
|
||||
sys.setdlopenflags(flags)
|
||||
if os.name != 'nt':
|
||||
prev = sys.getdlopenflags()
|
||||
sys.setdlopenflags(flags)
|
||||
yield
|
||||
sys.setdlopenflags(prev)
|
||||
if os.name != 'nt':
|
||||
sys.setdlopenflags(prev)
|
||||
|
||||
def try_import_jit_utils_core(silent=None):
|
||||
global cc
|
||||
|
|
@ -126,7 +128,10 @@ def run_cmd(cmd, cwd=None, err_msg=None, print_error=True):
|
|||
r = sp.run(cmd, cwd=cwd, shell=True, stdout=sp.PIPE, stderr=sp.STDOUT)
|
||||
else:
|
||||
r = sp.run(cmd, shell=True, stdout=sp.PIPE, stderr=sp.STDOUT)
|
||||
s = r.stdout.decode('utf8')
|
||||
try:
|
||||
s = r.stdout.decode('utf8')
|
||||
except:
|
||||
s = r.stdout.decode('gbk')
|
||||
if r.returncode != 0:
|
||||
if print_error:
|
||||
sys.stderr.write(s)
|
||||
|
|
@ -156,6 +161,7 @@ def pool_cleanup():
|
|||
del p
|
||||
|
||||
def pool_initializer():
|
||||
try_import_jit_utils_core()
|
||||
cc.init_subprocess()
|
||||
|
||||
def run_cmds(cmds, cache_path, jittor_path, msg="run_cmds"):
|
||||
|
|
@ -163,12 +169,22 @@ def run_cmds(cmds, cache_path, jittor_path, msg="run_cmds"):
|
|||
bk = mp.current_process()._config.get('daemon')
|
||||
mp.current_process()._config['daemon'] = False
|
||||
if pool_size == 0:
|
||||
mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
|
||||
mem_bytes = get_total_mem()
|
||||
mem_gib = mem_bytes/(1024.**3)
|
||||
pool_size = min(16,max(int(mem_gib // 3), 1))
|
||||
LOG.i(f"Total mem: {mem_gib:.2f}GB, using {pool_size} procs for compiling.")
|
||||
if os.name == 'nt':
|
||||
# a hack way to by pass windows
|
||||
# multiprocess spawn init_main_from_path.
|
||||
# check spawn.py:get_preparation_data
|
||||
spec_bk = sys.modules['__main__'].__spec__
|
||||
tmp = lambda x:x
|
||||
tmp.name = '__main__'
|
||||
sys.modules['__main__'].__spec__ = tmp
|
||||
p = Pool(pool_size, initializer=pool_initializer)
|
||||
p.__enter__()
|
||||
if os.name == 'nt':
|
||||
sys.modules['__main__'].__spec__ = spec_bk
|
||||
import atexit
|
||||
atexit.register(pool_cleanup)
|
||||
cmds = [ [cmd, cache_path, jittor_path] for cmd in cmds ]
|
||||
|
|
@ -212,7 +228,7 @@ def find_cache_path():
|
|||
for c in " (){}": cache_name = cache_name.replace(c, "_")
|
||||
except:
|
||||
pass
|
||||
for name in cache_name.split("/"):
|
||||
for name in os.path.normpath(cache_name).split(os.path.sep):
|
||||
dirs.insert(-1, name)
|
||||
os.environ["cache_name"] = cache_name
|
||||
LOG.v("cache_name", cache_name)
|
||||
|
|
@ -275,6 +291,84 @@ def get_cc_type(cc_path):
|
|||
if "g++" in bname: return "g++"
|
||||
LOG.f(f"Unknown cc type: {bname}")
|
||||
|
||||
def get_py3_config_path():
|
||||
global _py3_config_path
|
||||
if _py3_config_path:
|
||||
return _py3_config_path
|
||||
|
||||
if os.name == 'nt':
|
||||
return None
|
||||
else:
|
||||
# Linux
|
||||
py3_config_paths = [
|
||||
os.path.dirname(sys.executable) + f"/python3.{sys.version_info.minor}-config",
|
||||
sys.executable + "-config",
|
||||
f"/usr/bin/python3.{sys.version_info.minor}-config",
|
||||
os.path.dirname(sys.executable) + "/python3-config",
|
||||
]
|
||||
if "python_config_path" in os.environ:
|
||||
py3_config_paths.insert(0, os.environ["python_config_path"])
|
||||
|
||||
for py3_config_path in py3_config_paths:
|
||||
if os.path.isfile(py3_config_path):
|
||||
break
|
||||
else:
|
||||
raise RuntimeError(f"python3.{sys.version_info.minor}-config "
|
||||
f"not found in {py3_config_paths}, please specify "
|
||||
f"enviroment variable 'python_config_path',"
|
||||
f" or apt install python3.{sys.version_info.minor}-dev")
|
||||
_py3_config_path = py3_config_path
|
||||
return py3_config_path
|
||||
|
||||
def get_py3_include_path():
|
||||
global _py3_include_path
|
||||
if _py3_include_path:
|
||||
return _py3_include_path
|
||||
|
||||
if os.name == 'nt':
|
||||
# Windows
|
||||
_py3_include_path = '-I"' + os.path.join(
|
||||
os.path.dirname(sys.executable),
|
||||
"include"
|
||||
) + '"'
|
||||
else:
|
||||
_py3_include_path = run_cmd(get_py3_config_path()+" --includes")
|
||||
return _py3_include_path
|
||||
|
||||
|
||||
def get_py3_extension_suffix():
|
||||
global _py3_extension_suffix
|
||||
if _py3_extension_suffix:
|
||||
return _py3_extension_suffix
|
||||
|
||||
if os.name == 'nt':
|
||||
# Windows
|
||||
_py3_extension_suffix = f".cp3{sys.version_info.minor}-win_amd64.pyd"
|
||||
else:
|
||||
_py3_extension_suffix = run_cmd(get_py3_config_path()+" --extension-suffix")
|
||||
return _py3_extension_suffix
|
||||
|
||||
def get_total_mem():
|
||||
if os.name == 'nt':
|
||||
from ctypes import Structure, c_int32, c_uint64, sizeof, byref, windll
|
||||
class MemoryStatusEx(Structure):
|
||||
_fields_ = [
|
||||
('length', c_int32),
|
||||
('memoryLoad', c_int32),
|
||||
('totalPhys', c_uint64),
|
||||
('availPhys', c_uint64),
|
||||
('totalPageFile', c_uint64),
|
||||
('availPageFile', c_uint64),
|
||||
('totalVirtual', c_uint64),
|
||||
('availVirtual', c_uint64),
|
||||
('availExtendedVirtual', c_uint64)]
|
||||
def __init__(self):
|
||||
self.length = sizeof(self)
|
||||
m = MemoryStatusEx()
|
||||
assert windll.kernel32.GlobalMemoryStatusEx(byref(m))
|
||||
return m.totalPhys
|
||||
else:
|
||||
return os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
|
||||
|
||||
is_in_ipynb = in_ipynb()
|
||||
cc = None
|
||||
|
|
@ -285,21 +379,14 @@ os.environ["cc_path"] = cc_path
|
|||
cc_type = get_cc_type(cc_path)
|
||||
cache_path = find_cache_path()
|
||||
|
||||
_py3_config_path = None
|
||||
_py3_include_path = None
|
||||
_py3_extension_suffix = None
|
||||
|
||||
py3_config_paths = [
|
||||
os.path.dirname(sys.executable) + f"/python3.{sys.version_info.minor}-config",
|
||||
sys.executable + "-config",
|
||||
f"/usr/bin/python3.{sys.version_info.minor}-config",
|
||||
os.path.dirname(sys.executable) + "/python3-config",
|
||||
]
|
||||
if "python_config_path" in os.environ:
|
||||
py3_config_paths.insert(0, os.environ["python_config_path"])
|
||||
|
||||
for py3_config_path in py3_config_paths:
|
||||
if os.path.isfile(py3_config_path):
|
||||
break
|
||||
else:
|
||||
raise RuntimeError(f"python3.{sys.version_info.minor}-config "
|
||||
f"not found in {py3_config_paths}, please specify "
|
||||
f"enviroment variable 'python_config_path',"
|
||||
f" or apt install python3.{sys.version_info.minor}-dev")
|
||||
if os.name == 'nt':
|
||||
os.RTLD_NOW = os.RTLD_GLOBAL = os.RTLD_DEEPBIND = 0
|
||||
path = os.path.dirname(cc_path).replace('/', '\\')
|
||||
sys.path.insert(0, path)
|
||||
os.environ["PATH"] = path+';'+os.environ["PATH"]
|
||||
if hasattr(os, "add_dll_directory"):
|
||||
os.add_dll_directory(path)
|
||||
|
|
|
|||
|
|
@ -19,11 +19,11 @@ if __name__ == "__main__":
|
|||
|
||||
s = ""
|
||||
# base should be something like python3.7m python3.8
|
||||
base = jittor_utils.run_cmd(jittor_utils.py3_config_path + " --includes").split()[0]
|
||||
base = jittor_utils.get_py3_include_path().split()[0]
|
||||
base = "python3" + base.split("python3")[-1]
|
||||
for arg in sys.argv[1:]:
|
||||
if arg == "--include-flags":
|
||||
s += jittor_utils.run_cmd(jittor_utils.py3_config_path + " --includes")
|
||||
s += jittor_utils.get_py3_include_path()
|
||||
s += " -I"+os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "jittor", "src"))
|
||||
s += " "
|
||||
elif arg == "--libs-flags":
|
||||
|
|
@ -31,6 +31,8 @@ if __name__ == "__main__":
|
|||
libpath = libbase + f"/lib{base}.so"
|
||||
assert os.path.isfile(libpath), f"lib not exist: {libpath}"
|
||||
s += f" -L{libbase} -l{base} -ldl "
|
||||
if os.name == 'nt'
|
||||
s = s.replace('-ldl', '')
|
||||
elif arg == "--cxx-flags":
|
||||
s += " --std=c++17 "
|
||||
elif arg == "--cxx-example":
|
||||
|
|
|
|||
|
|
@ -29,6 +29,8 @@ def get_cuda_driver():
|
|||
|
||||
def install_cuda():
|
||||
cuda_driver_version = get_cuda_driver()
|
||||
if not cuda_driver_version:
|
||||
return None
|
||||
LOG.i("cuda_driver_version: ", cuda_driver_version)
|
||||
|
||||
cuda_tgz = "cuda10.2_cudnn7_linux.tgz"
|
||||
|
|
|
|||
|
|
@ -1,4 +1,11 @@
|
|||
import fcntl
|
||||
try:
|
||||
import fcntl
|
||||
except ImportError:
|
||||
fcntl = None
|
||||
import win32file
|
||||
import pywintypes
|
||||
_OVERLAPPED = pywintypes.OVERLAPPED()
|
||||
|
||||
import os
|
||||
from jittor_utils import cache_path, LOG
|
||||
|
||||
|
|
@ -8,13 +15,21 @@ class Lock:
|
|||
LOG.v(f'OPEN LOCK path: {filename} PID: {os.getpid()}')
|
||||
self.is_locked = False
|
||||
|
||||
def lock(self):
|
||||
fcntl.flock(self.handle, fcntl.LOCK_EX)
|
||||
def lock(self):
|
||||
if fcntl:
|
||||
fcntl.flock(self.handle, fcntl.LOCK_EX)
|
||||
else:
|
||||
hfile = win32file._get_osfhandle(self.handle.fileno())
|
||||
win32file.LockFileEx(hfile, 2, 0, -0x10000, _OVERLAPPED)
|
||||
self.is_locked = True
|
||||
LOG.vv(f'LOCK PID: {os.getpid()}')
|
||||
|
||||
def unlock(self):
|
||||
fcntl.flock(self.handle, fcntl.LOCK_UN)
|
||||
def unlock(self):
|
||||
if fcntl:
|
||||
fcntl.flock(self.handle, fcntl.LOCK_UN)
|
||||
else:
|
||||
hfile = win32file._get_osfhandle(self.handle.fileno())
|
||||
win32file.UnlockFileEx(hfile, 0, -0x10000, _OVERLAPPED)
|
||||
self.is_locked = False
|
||||
LOG.vv(f'UNLOCK PID: {os.getpid()}')
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue